From 5f70c21ba4fd1ac59b6d423eb060f13251a31cdf Mon Sep 17 00:00:00 2001 From: Marshall Ward Date: Tue, 24 Nov 2015 22:32:52 +1100 Subject: [PATCH] Porting MOM5 to latest FMS (ulm) This is a large patch which updates MOM5 to use the latest FMS library. Based on the following FMS release: url: https://github.com/NOAA-GFDL/FMS.git commit: 35e491c497d4bb9b77bd6d8b9557badd53e56e19 Modifications to the `load_xgrid` subroutine of `xgrid.F90` (already present in the MOM5 source) have been integrated into this FMS release. Changes to FMS are not summarised here, only changes to MOM and support models (atmos, land, sis) are summarised below. Changes are as follows: - Updating the FMS (src/shared) library to the latest FMS release. - `write_version_number` updated in each module to support inputs - `version` and `tagname` variable names are now used consistently across modules - The `land_null` model now sets its `Current_domain` to support changes to `get_mosaic_xgrid` (which now requires an explicit domain) - FMS_compile.csh buildscript replaces the fixed filepaths with an autogenerated list using the `list_paths` utility The main purpose of this patch is to demonstrate that MOM5 still runs under the newer FMS release with only minor changes, and to start discussion of how to maintain MOM under future FMS releases. --- exp/FMS_compile.csh | 93 +- src/atmos_bgrid/driver/coupled/atmosphere.F90 | 2 +- .../driver/coupled/bgrid_physics.F90 | 2 +- src/atmos_bgrid/driver/shallow/atmosphere.F90 | 2 +- src/atmos_bgrid/driver/solo/atmosphere.F90 | 2 +- src/atmos_bgrid/model/bgrid_advection.F90 | 2 +- .../model/bgrid_conserve_energy.F90 | 4 +- src/atmos_bgrid/model/bgrid_core.F90 | 2 +- src/atmos_bgrid/model/bgrid_core_driver.F90 | 2 +- src/atmos_bgrid/model/bgrid_horiz_diff.F90 | 2 +- src/atmos_bgrid/model/bgrid_sponge.F90 | 2 +- src/atmos_bgrid/model/bgrid_vert_adjust.F90 | 2 +- src/atmos_bgrid/tools/bgrid_cold_start.F90 | 2 +- src/atmos_bgrid/tools/bgrid_diagnostics.F90 | 2 +- src/atmos_bgrid/tools/bgrid_horiz.F90 | 2 +- src/atmos_bgrid/tools/bgrid_integrals.F90 | 2 +- src/atmos_bgrid/tools/bgrid_masks.F90 | 2 +- src/atmos_bgrid/tools/bgrid_polar_filter.F90 | 2 +- src/atmos_bgrid/tools/bgrid_prog_var.F90 | 4 +- src/atmos_bgrid/tools/bgrid_vert.F90 | 2 +- src/atmos_coupled/atmos_model.F90 | 2 +- src/atmos_ebm/atmosphere.F90 | 2 +- src/atmos_ebm/ebm_diagnostics.F90 | 2 +- .../driver/coupled/atmosphere.F90 | 2 +- .../driver/coupled/fv_physics.F90 | 4 +- .../driver/solo/atmosphere.F90 | 2 +- src/atmos_fv_dynamics/model/fv_pack.F90 | 2 +- src/atmos_param/betts_miller/betts_miller.F90 | 2 +- src/atmos_param/betts_miller/bm_massflux.F90 | 2 +- src/atmos_param/betts_miller/bm_omp.F90 | 2 +- src/atmos_param/cg_drag/cg_drag.F90 | 2 +- .../cloud_generator/betaDistribution.F90 | 2 +- .../cloud_generator/cloud_generator.F90 | 2 +- src/atmos_param/cloud_obs/cloud_obs.F90 | 2 +- src/atmos_param/cloud_rad/cloud_rad.F90 | 2 +- src/atmos_param/cloud_zonal/cloud_zonal.F90 | 2 +- src/atmos_param/clouds/clouds.F90 | 2 +- src/atmos_param/cosp/cosp_diagnostics.F90 | 2 +- src/atmos_param/cosp/cosp_driver.F90 | 2 +- src/atmos_param/cosp/cosp_io.F90 | 6 +- src/atmos_param/cu_mo_trans/cu_mo_trans.F90 | 2 +- .../damping_driver/damping_driver.F90 | 2 +- src/atmos_param/diag_cloud/diag_cloud.F90 | 2 +- .../diag_cloud_rad/diag_cloud_rad.F90 | 2 +- .../diag_integral/diag_integral.F90 | 2 +- src/atmos_param/diffusivity/diffusivity.F90 | 2 +- src/atmos_param/donner_deep/fms_donner.F90 | 2 +- src/atmos_param/dry_adj/dry_adj.F90 | 2 +- src/atmos_param/edt/edt.F90 | 4 +- src/atmos_param/entrain/entrain.F90 | 4 +- src/atmos_param/fsrad/co2_data.F90 | 2 +- src/atmos_param/fsrad/co2int.F90 | 2 +- src/atmos_param/fsrad/fs_profile.F90 | 2 +- src/atmos_param/fsrad/fsrad.F90 | 2 +- src/atmos_param/fsrad/longwave.F90 | 2 +- src/atmos_param/fsrad/mcm_lw.F90 | 2 +- src/atmos_param/fsrad/mcm_sw_driver.F90 | 2 +- src/atmos_param/fsrad/mcm_swnew.F90 | 2 +- src/atmos_param/fsrad/rad_diag.F90 | 2 +- src/atmos_param/fsrad/rdparm.F90 | 2 +- src/atmos_param/fsrad/shortwave.F90 | 2 +- .../grey_radiation/grey_radiation.F90 | 2 +- src/atmos_param/lscale_cond/lscale_cond.F90 | 2 +- src/atmos_param/mg_drag/mg_drag.F90 | 2 +- src/atmos_param/moist_conv/moist_conv.F90 | 2 +- .../moist_processes/detr_ice_num.F90 | 2 +- .../moist_processes/moist_processes.F90 | 2 +- .../monin_obukhov/monin_obukhov.F90 | 2 +- src/atmos_param/my25_turb/my25_turb.F90 | 2 +- .../physics_driver/physics_driver.F90 | 2 +- .../radiation_driver/radiation_driver.F90 | 2 +- src/atmos_param/ras/ras.F90 | 2 +- src/atmos_param/rh_clouds/rh_clouds.F90 | 2 +- src/atmos_param/sea_esf_rad/aerosol.F90 | 2 +- .../sea_esf_rad/aerosolrad_package.F90 | 2 +- src/atmos_param/sea_esf_rad/bulkphys_rad.F90 | 2 +- src/atmos_param/sea_esf_rad/cloud_spec.F90 | 2 +- .../sea_esf_rad/cloudrad_diagnostics.F90 | 2 +- .../sea_esf_rad/cloudrad_package.F90 | 2 +- src/atmos_param/sea_esf_rad/diag_clouds_W.F90 | 2 +- .../sea_esf_rad/donner_deep_clouds_W.F90 | 2 +- src/atmos_param/sea_esf_rad/esfsw_driver.F90 | 2 +- .../sea_esf_rad/esfsw_parameters.F90 | 2 +- src/atmos_param/sea_esf_rad/gas_tf.F90 | 2 +- src/atmos_param/sea_esf_rad/isccp_clouds.F90 | 2 +- src/atmos_param/sea_esf_rad/lhsw_driver.F90 | 2 +- .../sea_esf_rad/longwave_clouds.F90 | 2 +- .../sea_esf_rad/longwave_driver.F90 | 2 +- .../sea_esf_rad/longwave_fluxes.F90 | 2 +- .../sea_esf_rad/longwave_params.F90 | 2 +- .../sea_esf_rad/longwave_tables.F90 | 2 +- .../sea_esf_rad/lw_gases_stdtf.F90 | 2 +- .../sea_esf_rad/mgrp_prscr_clds.F90 | 2 +- .../sea_esf_rad/microphys_cloud.F90 | 2 +- src/atmos_param/sea_esf_rad/microphys_rad.F90 | 2 +- src/atmos_param/sea_esf_rad/optical_path.F90 | 2 +- .../sea_esf_rad/original_fms_rad.F90 | 2 +- src/atmos_param/sea_esf_rad/ozone.F90 | 2 +- .../sea_esf_rad/rad_output_file.F90 | 2 +- src/atmos_param/sea_esf_rad/rad_utilities.F90 | 2 +- .../sea_esf_rad/radiation_diag.F90 | 2 +- .../sea_esf_rad/radiative_gases.F90 | 2 +- .../sea_esf_rad/rh_based_clouds.F90 | 2 +- src/atmos_param/sea_esf_rad/sea_esf_rad.F90 | 2 +- src/atmos_param/sea_esf_rad/sealw99.F90 | 2 +- .../sea_esf_rad/shortwave_driver.F90 | 2 +- .../sea_esf_rad/specified_clouds_W.F90 | 2 +- .../sea_esf_rad/standalone_clouds.F90 | 2 +- .../sea_esf_rad/strat_clouds_W.F90 | 2 +- src/atmos_param/sea_esf_rad/uw_clouds_W.F90 | 2 +- .../sea_esf_rad/zetac_clouds_W.F90 | 2 +- src/atmos_param/shallow_conv/shallow_conv.F90 | 2 +- src/atmos_param/shallow_cu/conv_closures.F90 | 2 +- src/atmos_param/shallow_cu/uw_conv.F90 | 2 +- .../stable_bl_turb/stable_bl_turb.F90 | 2 +- src/atmos_param/strat_cloud/aerosol_cloud.F90 | 2 +- src/atmos_param/strat_cloud/check_nan.F90 | 2 +- .../strat_cloud/cldwat2m_micro.F90 | 2 +- src/atmos_param/strat_cloud/gamma_mg.F90 | 2 +- src/atmos_param/strat_cloud/mg_const.F90 | 2 +- src/atmos_param/strat_cloud/microphysics.F90 | 2 +- .../strat_cloud/morrison_gettelman_microp.F90 | 2 +- src/atmos_param/strat_cloud/nc_cond.F90 | 2 +- src/atmos_param/strat_cloud/polysvp.F90 | 2 +- .../strat_cloud/rotstayn_klein_mp.F90 | 2 +- src/atmos_param/strat_cloud/simple_pdf.F90 | 2 +- src/atmos_param/strat_cloud/strat_cloud.F90 | 2 +- .../strat_cloud/strat_cloud_legacy.F90 | 2 +- .../strat_cloud/strat_cloud_utilities.F90 | 2 +- src/atmos_param/strat_cloud/strat_netcdf.F90 | 2 +- src/atmos_param/topo_drag/topo_drag.F90 | 2 +- src/atmos_param/vert_diff/vert_diff.F90 | 2 +- .../vert_diff_driver/vert_diff_driver.F90 | 2 +- .../vert_turb_driver/vert_turb_driver.F90 | 2 +- src/atmos_shared/atmos_nudge/atmos_nudge.F90 | 2 +- .../interpolator/interpolator.F90 | 2 +- .../tracer_driver/aer_ccn_act/aer_ccn_act.F90 | 2 +- .../aer_ccn_act/aerosol_params.F90 | 2 +- .../tracer_driver/aer_ccn_act/ice_nucl.F90 | 2 +- .../tracer_driver/atmos_age_tracer.F90 | 2 +- .../tracer_driver/atmos_carbon_aerosol.F90 | 4 +- src/atmos_shared/tracer_driver/atmos_ch3i.F90 | 2 +- src/atmos_shared/tracer_driver/atmos_co2.F90 | 4 +- .../tracer_driver/atmos_convection_tracer.F90 | 2 +- src/atmos_shared/tracer_driver/atmos_dust.F90 | 4 +- .../tracer_driver/atmos_radon.F90 | 2 +- .../tracer_driver/atmos_sea_salt.F90 | 2 +- src/atmos_shared/tracer_driver/atmos_soa.F90 | 4 +- .../tracer_driver/atmos_sulfate.F90 | 6 +- .../tracer_driver/atmos_sulfur_hex.F90 | 2 +- .../tracer_driver/atmos_tracer_driver.F90 | 2 +- .../tracer_driver/atmos_tracer_utilities.F90 | 2 +- .../stratchem/strat_chem_driver.F90 | 2 +- .../tracer_driver/tropchem/mo_fastjx.F90 | 2 +- .../tracer_driver/tropchem/mo_fphoto.F90 | 2 +- .../tropchem/tropchem_driver.F90 | 2 +- .../vert_advection/vert_advection.F90 | 2 +- .../driver/coupled/atmosphere.F90 | 2 +- .../driver/coupled/mcm_mca_lsc.F90 | 2 +- .../driver/coupled/mcm_moist_processes.F90 | 2 +- .../driver/coupled/spectral_physics.F90 | 2 +- src/atmos_spectral/driver/solo/atmosphere.F90 | 2 +- .../driver/solo/idealized_moist_phys.F90 | 2 +- .../driver/solo/mixed_layer.F90 | 2 +- .../init/ic_from_external_file.F90 | 2 +- src/atmos_spectral/init/jablonowski_2006.F90 | 2 +- src/atmos_spectral/init/polvani_2004.F90 | 2 +- src/atmos_spectral/init/polvani_2007.F90 | 2 +- .../init/spectral_init_cond.F90 | 2 +- .../init/spectral_initialize_fields.F90 | 2 +- .../init/topog_regularization.F90 | 2 +- src/atmos_spectral/init/vert_coordinate.F90 | 2 +- .../model/every_step_diagnostics.F90 | 2 +- src/atmos_spectral/model/fv_advection.F90 | 2 +- src/atmos_spectral/model/global_integral.F90 | 2 +- src/atmos_spectral/model/implicit.F90 | 2 +- src/atmos_spectral/model/leapfrog.F90 | 16 +- src/atmos_spectral/model/matrix_invert.F90 | 2 +- src/atmos_spectral/model/press_and_geopot.F90 | 2 +- src/atmos_spectral/model/spectral_damping.F90 | 2 +- .../model/spectral_dynamics.F90 | 4 +- src/atmos_spectral/model/water_borrowing.F90 | 2 +- .../tools/gauss_and_legendre.F90 | 4 +- src/atmos_spectral/tools/grid_fourier.F90 | 2 +- src/atmos_spectral/tools/spec_mpp.F90 | 2 +- src/atmos_spectral/tools/spherical.F90 | 2 +- .../tools/spherical_fourier.F90 | 2 +- src/atmos_spectral/tools/transforms.F90 | 2 +- src/coupler/coupler_main.F90 | 4 +- src/coupler/flux_exchange.F90 | 4 +- src/coupler/surface_flux.F90 | 2 +- src/ice_param/ice_albedo.F90 | 2 +- src/ice_param/ocean_albedo.F90 | 2 +- src/ice_param/ocean_rough.F90 | 2 +- src/ice_sis/ice_bergs.F90 | 2 +- src/ice_sis/ice_spec.F90 | 2 +- src/ice_sis/ice_type.F90 | 2 +- src/land_lad/land_model.F90 | 2 +- src/land_lad/land_types.F90 | 2 +- src/land_lad/numerics.F90 | 2 +- src/land_lad/soil/land_properties.F90 | 2 +- src/land_lad/soil/rivers.F90 | 2 +- src/land_lad/soil/soil.F90 | 2 +- src/land_lad/vegetation/vegetation.F90 | 2 +- src/land_lad2/canopy_air/canopy_air.F90 | 2 +- src/land_lad2/glacier/glac_tile.F90 | 2 +- src/land_lad2/glacier/glacier.F90 | 2 +- src/land_lad2/lake/lake.F90 | 2 +- src/land_lad2/lake/lake_tile.F90 | 2 +- src/land_lad2/land_data.F90 | 2 +- src/land_lad2/land_model.F90 | 2 +- src/land_lad2/river/river.F90 | 2 +- src/land_lad2/river/river_physics.F90 | 2 +- src/land_lad2/shared/land_debug.F90 | 2 +- src/land_lad2/shared/land_io.F90 | 2 +- src/land_lad2/shared/land_numerics.F90 | 2 +- src/land_lad2/shared/land_tile_diag.F90 | 2 +- src/land_lad2/snow/snow.F90 | 2 +- src/land_lad2/snow/snow_tile.F90 | 2 +- src/land_lad2/soil/soil.F90 | 2 +- src/land_lad2/soil/soil_tile.F90 | 2 +- src/land_lad2/soil/uptake.F90 | 2 +- src/land_lad2/topo_rough/topo_rough.F90 | 2 +- src/land_lad2/transitions/transitions.F90 | 2 +- src/land_lad2/vegetation/vegetation.F90 | 2 +- src/land_lad2/vegetation/vegn_data.F90 | 2 +- src/land_lad2/vegetation/vegn_dynamics.F90 | 2 +- src/land_lad2/vegetation/vegn_harvesting.F90 | 2 +- .../vegetation/vegn_photosynthesis.F90 | 2 +- src/land_lad2/vegetation/vegn_radiation.F90 | 2 +- .../vegetation/vegn_static_override.F90 | 2 +- src/land_null/land_model.F90 | 4 +- src/land_param/climap_albedo.F90 | 4 +- src/mom5/ocean_blobs/ocean_blob.F90 | 2 +- .../ocean_core/ocean_advection_velocity.F90 | 2 +- src/mom5/ocean_core/ocean_barotropic.F90 | 2 +- src/mom5/ocean_core/ocean_bbc.F90 | 2 +- src/mom5/ocean_core/ocean_coriolis.F90 | 2 +- src/mom5/ocean_core/ocean_density.F90 | 2 +- src/mom5/ocean_core/ocean_domains.F90 | 2 +- src/mom5/ocean_core/ocean_grids.F90 | 2 +- src/mom5/ocean_core/ocean_model.F90 | 2 +- src/mom5/ocean_core/ocean_obc.F90 | 2 +- src/mom5/ocean_core/ocean_pressure.F90 | 2 +- src/mom5/ocean_core/ocean_sbc.F90 | 2 +- src/mom5/ocean_core/ocean_thickness.F90 | 2 +- src/mom5/ocean_core/ocean_types.F90 | 2 +- src/mom5/ocean_core/ocean_velocity.F90 | 2 +- src/mom5/ocean_core/ocean_velocity_advect.F90 | 2 +- src/mom5/ocean_diag/ocean_adv_vel_diag.F90 | 2 +- src/mom5/ocean_diag/ocean_diagnostics.F90 | 2 +- src/mom5/ocean_diag/ocean_tracer_diag.F90 | 2 +- src/mom5/ocean_diag/ocean_velocity_diag.F90 | 2 +- .../lateral/ocean_bih_friction.F90 | 2 +- .../ocean_param/lateral/ocean_bih_tracer.F90 | 2 +- .../lateral/ocean_bihcgrid_friction.F90 | 2 +- .../lateral/ocean_bihcst_friction.F90 | 2 +- .../lateral/ocean_bihgen_friction.F90 | 2 +- .../lateral/ocean_lap_friction.F90 | 2 +- .../ocean_param/lateral/ocean_lap_tracer.F90 | 2 +- .../lateral/ocean_lapcgrid_friction.F90 | 2 +- .../lateral/ocean_lapcst_friction.F90 | 2 +- .../lateral/ocean_lapgen_friction.F90 | 2 +- .../lateral/ocean_mixdownslope.F90 | 2 +- .../lateral/ocean_sigma_transport.F90 | 2 +- .../lateral/ocean_submesoscale.F90 | 2 +- .../ocean_param/neutral/ocean_nphysics.F90 | 2 +- .../ocean_param/neutral/ocean_nphysicsA.F90 | 2 +- .../ocean_param/neutral/ocean_nphysicsB.F90 | 2 +- .../ocean_param/neutral/ocean_nphysicsC.F90 | 2 +- .../neutral/ocean_nphysics_new.F90 | 2 +- .../neutral/ocean_nphysics_util.F90 | 2 +- .../sources/ocean_increment_eta.F90 | 2 +- .../sources/ocean_increment_tracer.F90 | 2 +- .../sources/ocean_increment_velocity.F90 | 2 +- .../sources/ocean_momentum_source.F90 | 2 +- .../sources/ocean_overexchange.F90 | 2 +- .../ocean_param/sources/ocean_overflow.F90 | 2 +- .../sources/ocean_overflow_OFP.F90 | 2 +- .../ocean_param/sources/ocean_rivermix.F90 | 2 +- .../ocean_param/sources/ocean_riverspread.F90 | 2 +- .../ocean_param/sources/ocean_shortwave.F90 | 2 +- .../sources/ocean_shortwave_csiro.F90 | 2 +- .../sources/ocean_shortwave_gfdl.F90 | 2 +- .../sources/ocean_shortwave_jerlov.F90 | 2 +- .../ocean_param/sources/ocean_sponges_eta.F90 | 2 +- .../sources/ocean_sponges_tracer.F90 | 2 +- .../sources/ocean_sponges_velocity.F90 | 2 +- .../ocean_param/sources/ocean_xlandinsert.F90 | 2 +- .../ocean_param/sources/ocean_xlandmix.F90 | 2 +- .../ocean_param/vertical/ocean_convect.F90 | 2 +- .../ocean_param/vertical/ocean_form_drag.F90 | 2 +- .../ocean_param/vertical/ocean_vert_chen.F90 | 2 +- .../ocean_param/vertical/ocean_vert_const.F90 | 2 +- .../ocean_param/vertical/ocean_vert_gotm.F90 | 2 +- .../vertical/ocean_vert_kpp_mom4p0.F90 | 2 +- .../vertical/ocean_vert_kpp_mom4p1.F90 | 2 +- .../vertical/ocean_vert_kpp_test.F90 | 2 +- .../ocean_param/vertical/ocean_vert_mix.F90 | 2 +- .../ocean_param/vertical/ocean_vert_pp.F90 | 2 +- .../ocean_param/vertical/ocean_vert_tidal.F90 | 2 +- .../vertical/ocean_vert_tidal_test.F90 | 2 +- src/mom5/ocean_tracers/ocean_passive.F90 | 2 +- src/mom5/ocean_tracers/ocean_tempsalt.F90 | 2 +- .../ocean_tracers/ocean_tracer_advect.F90 | 2 +- src/mom5/ocean_wave/ocean_wave.F90 | 2 +- .../generic_tracers/generic_BLING.F90 | 2 +- .../generic_tracers/generic_ERGOM.F90 | 2 +- src/postprocessing/regrid/regrid.F90 | 2 +- .../generate_grids/atmos/atmos_grid.f90 | 2 +- .../atmos/atmos_grid_generator.f90 | 2 +- .../grid_transfer/grid_transfer.F90 | 2 +- .../generate_grids/ocean/compare_grid.f90 | 2 +- .../generate_grids/ocean/edit_grid.F90 | 2 +- .../generate_grids/ocean/hgrid.f90 | 2 +- .../ocean/ocean_grid_generator.f90 | 2 +- .../generate_grids/ocean/topog.f90 | 2 +- .../generate_grids/ocean/vgrid.f90 | 2 +- .../mom4_prep/idealized_bc/idealized_bc.f90 | 2 +- .../mom4_prep/idealized_ic/idealized_ic.f90 | 2 +- src/preprocessing/regrid/regrid.F90 | 2 +- src/preprocessing/regrid_2d/regrid_2d.f90 | 2 +- src/preprocessing/regrid_3d/regrid_3d.f90 | 2 +- .../river_regrid/river_regrid.f90 | 2 +- src/shared/README.md | 25 + src/shared/amip_interp/amip_interp.F90 | 365 +- src/shared/astronomy/astronomy.F90 | 17 +- src/shared/axis_utils/axis_utils.F90 | 69 +- src/shared/block_control/block_control.F90 | 90 + .../column_diagnostics/column_diagnostics.F90 | 6 +- src/shared/constants/constants.F90 | 7 +- src/shared/coupler/atmos_ocean_fluxes.F90 | 36 +- src/shared/coupler/coupler_types.F90 | 26 +- src/shared/coupler/ensemble_manager.F90 | 4 - src/shared/data_override/data_override.F90 | 40 +- src/shared/diag_manager/diag_axis.F90 | 70 +- src/shared/diag_manager/diag_data.F90 | 165 +- src/shared/diag_manager/diag_grid.F90 | 129 +- src/shared/diag_manager/diag_manager.F90 | 1721 +++++-- src/shared/diag_manager/diag_output.F90 | 160 +- src/shared/diag_manager/diag_table.F90 | 238 +- src/shared/diag_manager/diag_util.F90 | 675 ++- src/shared/drifters/cloud_interpolator.F90 | 14 +- src/shared/drifters/drifters.F90 | 7 +- src/shared/drifters/drifters_combine | 2 +- src/shared/drifters/drifters_comm.F90 | 4 +- src/shared/drifters/drifters_compute_k.h | 2 +- src/shared/drifters/drifters_core.F90 | 8 +- src/shared/drifters/drifters_input.F90 | 4 +- src/shared/drifters/drifters_io.F90 | 6 +- src/shared/drifters/drifters_push.h | 2 +- src/shared/drifters/drifters_set_field.h | 2 +- src/shared/drifters/readme.txt | 4 +- src/shared/exchange/stock_constants.F90 | 2 +- src/shared/exchange/xgrid.F90 | 424 +- src/shared/fft/fft.F90 | 9 +- src/shared/field_manager/field_manager.F90 | 299 +- src/shared/field_manager/fm_util.F90 | 44 +- src/shared/fms/fms.F90 | 65 +- src/shared/fms/fms_io.F90 | 4170 ++++++++++++----- src/shared/fms/read_data_2d.inc | 2 +- src/shared/fms/read_data_3d.inc | 2 +- src/shared/fms/read_data_4d.inc | 2 +- src/shared/fms/test_fms_io.F90 | 130 +- src/shared/horiz_interp/horiz_interp.F90 | 27 +- .../horiz_interp/horiz_interp_bicubic.F90 | 26 +- .../horiz_interp/horiz_interp_bilinear.F90 | 249 +- .../horiz_interp/horiz_interp_conserve.F90 | 23 +- .../horiz_interp/horiz_interp_spherical.F90 | 43 +- src/shared/horiz_interp/horiz_interp_type.F90 | 5 +- src/shared/horiz_interp/test_horiz_interp.F90 | 251 + src/shared/memutils/memutils.F90 | 2 +- src/shared/mosaic/gradient.F90 | 7 +- src/shared/mosaic/grid.F90 | 62 +- src/shared/mosaic/interp.h | 2 +- src/shared/mosaic/mosaic.F90 | 360 +- src/shared/mosaic/mosaic_util.h | 2 +- src/shared/mosaic/read_mosaic.c | 29 + src/shared/mosaic/read_mosaic.h | 3 + src/shared/mpp/include/group_update_pack.inc | 413 ++ .../mpp/include/group_update_unpack.inc | 91 + src/shared/mpp/include/mpp_alltoall_mpi.h | 6 +- src/shared/mpp/include/mpp_alltoall_nocomm.h | 3 +- src/shared/mpp/include/mpp_alltoall_sma.h | 3 +- src/shared/mpp/include/mpp_chksum.h | 13 +- src/shared/mpp/include/mpp_chksum_int.h | 50 +- src/shared/mpp/include/mpp_chksum_scalar.h | 9 +- src/shared/mpp/include/mpp_comm.inc | 62 +- src/shared/mpp/include/mpp_comm_mpi.inc | 6 +- src/shared/mpp/include/mpp_comm_nocomm.inc | 2 +- src/shared/mpp/include/mpp_comm_sma.inc | 6 +- src/shared/mpp/include/mpp_data_mpi.inc | 2 +- src/shared/mpp/include/mpp_data_nocomm.inc | 2 +- src/shared/mpp/include/mpp_data_sma.inc | 2 +- .../mpp/include/mpp_define_nest_domains.inc | 2 +- src/shared/mpp/include/mpp_do_get_boundary.h | 32 +- src/shared/mpp/include/mpp_do_update.h | 122 +- src/shared/mpp/include/mpp_do_updateV.h | 731 +-- .../mpp/include/mpp_do_updateV_nonblock.h | 760 +-- .../mpp/include/mpp_do_update_nonblock.h | 159 +- src/shared/mpp/include/mpp_domains_comm.inc | 2 +- src/shared/mpp/include/mpp_domains_define.inc | 1142 ++--- src/shared/mpp/include/mpp_domains_misc.inc | 82 + src/shared/mpp/include/mpp_domains_reduce.inc | 419 +- src/shared/mpp/include/mpp_domains_util.inc | 565 ++- src/shared/mpp/include/mpp_gather.h | 215 +- src/shared/mpp/include/mpp_get_boundary.h | 4 +- src/shared/mpp/include/mpp_global_sum.h | 26 +- src/shared/mpp/include/mpp_group_update.h | 973 ++++ src/shared/mpp/include/mpp_io_connect.inc | 45 +- src/shared/mpp/include/mpp_io_misc.inc | 8 +- src/shared/mpp/include/mpp_io_read.inc | 198 +- src/shared/mpp/include/mpp_io_util.inc | 144 +- src/shared/mpp/include/mpp_io_write.inc | 354 +- src/shared/mpp/include/mpp_read_2Ddecomp.h | 15 + src/shared/mpp/include/mpp_read_compressed.h | 93 + .../mpp/include/mpp_read_distributed_ascii.h | 29 + .../include/mpp_read_distributed_ascii.inc | 42 + src/shared/mpp/include/mpp_reduce_mpi.h | 4 +- src/shared/mpp/include/mpp_reduce_sma.h | 4 +- src/shared/mpp/include/mpp_scatter.h | 131 + src/shared/mpp/include/mpp_sum.inc | 2 +- src/shared/mpp/include/mpp_sum_mpi.h | 4 +- src/shared/mpp/include/mpp_sum_sma.h | 4 +- src/shared/mpp/include/mpp_transmit.inc | 2 +- src/shared/mpp/include/mpp_transmit_mpi.h | 22 +- src/shared/mpp/include/mpp_transmit_sma.h | 32 +- src/shared/mpp/include/mpp_update_domains2D.h | 96 +- .../include/mpp_update_domains2D_nonblock.h | 81 +- src/shared/mpp/include/mpp_util.inc | 36 +- src/shared/mpp/include/mpp_util_mpi.inc | 12 +- src/shared/mpp/include/mpp_util_nocomm.inc | 2 +- src/shared/mpp/include/mpp_util_sma.inc | 2 +- src/shared/mpp/include/mpp_write.h | 4 +- src/shared/mpp/include/mpp_write_compressed.h | 101 + .../mpp/include/mpp_write_unlimited_axis.h | 41 + src/shared/mpp/mpp.F90 | 53 +- src/shared/mpp/mpp_data.F90 | 2 +- src/shared/mpp/mpp_domains.F90 | 324 +- src/shared/mpp/mpp_efp.F90 | 665 +++ src/shared/mpp/mpp_io.F90 | 211 +- src/shared/mpp/mpp_parameter.F90 | 11 +- src/shared/mpp/mpp_pset.F90 | 2 +- src/shared/mpp/mpp_utilities.F90 | 18 +- src/shared/mpp/test_mpp.F90 | 254 +- src/shared/mpp/test_mpp_domains.F90 | 3137 ++++++------- src/shared/mpp/test_mpp_io.F90 | 6 +- src/shared/oda_tools/oda_core.F90 | 32 +- src/shared/oda_tools/oda_core_ecda.F90 | 220 +- src/shared/oda_tools/oda_types.F90 | 49 +- src/shared/sat_vapor_pres/sat_vapor_pres.F90 | 11 +- .../sat_vapor_pres/sat_vapor_pres_k.F90 | 26 +- src/shared/station_data/station_data.F90 | 4 +- src/shared/time_interp/time_interp.F90 | 13 +- .../time_interp/time_interp_external.F90 | 6 +- src/shared/time_manager/get_cal_time.F90 | 10 +- src/shared/time_manager/time_manager.F90 | 9 +- src/shared/topography/gaussian_topog.F90 | 9 +- src/shared/topography/topography.F90 | 9 +- src/shared/tracer_manager/tracer_manager.F90 | 15 +- src/shared/tridiagonal/tridiagonal.F90 | 7 +- 461 files changed, 15293 insertions(+), 8151 deletions(-) create mode 100644 src/shared/README.md create mode 100644 src/shared/block_control/block_control.F90 create mode 100644 src/shared/horiz_interp/test_horiz_interp.F90 create mode 100644 src/shared/mpp/include/group_update_pack.inc create mode 100644 src/shared/mpp/include/group_update_unpack.inc create mode 100644 src/shared/mpp/include/mpp_group_update.h create mode 100644 src/shared/mpp/include/mpp_read_compressed.h create mode 100644 src/shared/mpp/include/mpp_read_distributed_ascii.h create mode 100644 src/shared/mpp/include/mpp_read_distributed_ascii.inc create mode 100644 src/shared/mpp/include/mpp_scatter.h create mode 100644 src/shared/mpp/include/mpp_write_compressed.h create mode 100644 src/shared/mpp/include/mpp_write_unlimited_axis.h create mode 100644 src/shared/mpp/mpp_efp.F90 diff --git a/exp/FMS_compile.csh b/exp/FMS_compile.csh index 2f4582cee3..1e04fac8b3 100644 --- a/exp/FMS_compile.csh +++ b/exp/FMS_compile.csh @@ -3,97 +3,8 @@ set pathnames_shared = $code_dir/path_names_shared # path to file containing list of source paths -cat > $pathnames_shared < 0) diff --git a/src/atmos_param/entrain/entrain.F90 b/src/atmos_param/entrain/entrain.F90 index 4e3b6cd3d6..e39a28691d 100644 --- a/src/atmos_param/entrain/entrain.F90 +++ b/src/atmos_param/entrain/entrain.F90 @@ -371,7 +371,7 @@ subroutine entrain_init(lonb, latb, axes,time,idim,jdim,kdim) #endif if ( mpp_pe() == mpp_root_pe() ) then - call write_version_number() + call write_version_number(version, tagname) unit = stdlog() Write (unit,nml=entrain_nml) endif @@ -492,7 +492,7 @@ subroutine entrain_init(lonb, latb, axes,time,idim,jdim,kdim) threading='multi', form='formatted') do_print = .true. if ( mpp_pe() == mpp_root_pe() ) then - call write_version_number() + call write_version_number(version, tagname) Write (dpu ,nml=entrain_nml) endif endif ! (num_pts > 0) diff --git a/src/atmos_param/fsrad/co2_data.F90 b/src/atmos_param/fsrad/co2_data.F90 index a4a6750ce9..8292311730 100644 --- a/src/atmos_param/fsrad/co2_data.F90 +++ b/src/atmos_param/fsrad/co2_data.F90 @@ -808,7 +808,7 @@ Subroutine co2_data_init !------- write version number and namelist --------- if ( mpp_pe() == mpp_root_pe() ) then - call write_version_number() + call write_version_number(version, tagname) endif module_is_initialized = .true. diff --git a/src/atmos_param/fsrad/co2int.F90 b/src/atmos_param/fsrad/co2int.F90 index f5cc85a0cb..0704dfaa24 100644 --- a/src/atmos_param/fsrad/co2int.F90 +++ b/src/atmos_param/fsrad/co2int.F90 @@ -470,7 +470,7 @@ Subroutine co2int_init !------- write version number and namelist --------- if ( mpp_pe() == mpp_root_pe() ) then - call write_version_number() + call write_version_number(version, tagname) endif module_is_initialized = .true. diff --git a/src/atmos_param/fsrad/fs_profile.F90 b/src/atmos_param/fsrad/fs_profile.F90 index d23db0a0ee..da4b854e2f 100644 --- a/src/atmos_param/fsrad/fs_profile.F90 +++ b/src/atmos_param/fsrad/fs_profile.F90 @@ -291,7 +291,7 @@ Subroutine fs_profile_init !------- write version number and namelist --------- if ( mpp_pe() == mpp_root_pe() ) then - call write_version_number() + call write_version_number(version, tagname) endif module_is_initialized = .true. diff --git a/src/atmos_param/fsrad/fsrad.F90 b/src/atmos_param/fsrad/fsrad.F90 index db9577df43..dcd7a77bbc 100644 --- a/src/atmos_param/fsrad/fsrad.F90 +++ b/src/atmos_param/fsrad/fsrad.F90 @@ -176,7 +176,7 @@ Subroutine fsrad_init !------- write version number and namelist --------- if ( mpp_pe() == mpp_root_pe() ) then - call write_version_number() + call write_version_number(version, tagname) endif module_is_initialized = .true. diff --git a/src/atmos_param/fsrad/longwave.F90 b/src/atmos_param/fsrad/longwave.F90 index 3b18ba9a03..1a02f5db37 100644 --- a/src/atmos_param/fsrad/longwave.F90 +++ b/src/atmos_param/fsrad/longwave.F90 @@ -782,7 +782,7 @@ Subroutine longwave_init !------- write version number --------- if ( mpp_pe() == mpp_root_pe() ) then - call write_version_number() + call write_version_number(version, tagname) endif module_is_initialized = .true. diff --git a/src/atmos_param/fsrad/mcm_lw.F90 b/src/atmos_param/fsrad/mcm_lw.F90 index 223e76beab..7eafbafc60 100644 --- a/src/atmos_param/fsrad/mcm_lw.F90 +++ b/src/atmos_param/fsrad/mcm_lw.F90 @@ -177,7 +177,7 @@ subroutine mcm_lw_init(ix_in, jx_in, kx_in) !------- write version number and namelist --------- if ( mpp_pe() == mpp_root_pe() ) then - call write_version_number() + call write_version_number(version, tagname) endif module_is_initialized = .true. diff --git a/src/atmos_param/fsrad/mcm_sw_driver.F90 b/src/atmos_param/fsrad/mcm_sw_driver.F90 index 89dd66c287..154722568a 100644 --- a/src/atmos_param/fsrad/mcm_sw_driver.F90 +++ b/src/atmos_param/fsrad/mcm_sw_driver.F90 @@ -272,7 +272,7 @@ subroutine mcm_sw_driver_init(kx_in) !------- write version number and namelist --------- if ( mpp_pe() == mpp_root_pe() ) then - call write_version_number() + call write_version_number(version, tagname) endif module_is_initialized = .true. diff --git a/src/atmos_param/fsrad/mcm_swnew.F90 b/src/atmos_param/fsrad/mcm_swnew.F90 index e60a7e3b6a..74959302eb 100644 --- a/src/atmos_param/fsrad/mcm_swnew.F90 +++ b/src/atmos_param/fsrad/mcm_swnew.F90 @@ -486,7 +486,7 @@ subroutine mcm_swnew_init !------- write version number and namelist --------- if ( mpp_pe() == mpp_root_pe() ) then - call write_version_number() + call write_version_number(version, tagname) endif module_is_initialized = .true. diff --git a/src/atmos_param/fsrad/rad_diag.F90 b/src/atmos_param/fsrad/rad_diag.F90 index 638712fa66..3df0351273 100644 --- a/src/atmos_param/fsrad/rad_diag.F90 +++ b/src/atmos_param/fsrad/rad_diag.F90 @@ -88,7 +88,7 @@ subroutine RAD_DIAG_init !------- write version number and namelist --------- if ( mpp_pe() == mpp_root_pe() ) then - call write_version_number() + call write_version_number(version, tagname) endif module_is_initialized = .true. diff --git a/src/atmos_param/fsrad/rdparm.F90 b/src/atmos_param/fsrad/rdparm.F90 index cc09da30b4..05f169a04a 100644 --- a/src/atmos_param/fsrad/rdparm.F90 +++ b/src/atmos_param/fsrad/rdparm.F90 @@ -77,7 +77,7 @@ SUBROUTINE RDPARM_INIT (KDIM) !------- write version number and namelist --------- if ( mpp_pe() == mpp_root_pe() ) then - call write_version_number() + call write_version_number(version, tagname) endif module_is_initialized = .true. diff --git a/src/atmos_param/fsrad/shortwave.F90 b/src/atmos_param/fsrad/shortwave.F90 index bc158ef4ac..7e87cd9e23 100644 --- a/src/atmos_param/fsrad/shortwave.F90 +++ b/src/atmos_param/fsrad/shortwave.F90 @@ -1207,7 +1207,7 @@ SUBROUTINE SHORTWAVE_INIT !------- write version number and namelist --------- if ( mpp_pe() == mpp_root_pe() ) then - call write_version_number() + call write_version_number(version, tagname) endif module_is_initialized = .true. diff --git a/src/atmos_param/grey_radiation/grey_radiation.F90 b/src/atmos_param/grey_radiation/grey_radiation.F90 index f5d76459d5..5ce41c474e 100644 --- a/src/atmos_param/grey_radiation/grey_radiation.F90 +++ b/src/atmos_param/grey_radiation/grey_radiation.F90 @@ -141,7 +141,7 @@ subroutine grey_radiation_init(axes, Time) endif #endif -call write_version_number() +call write_version_number(version, tagname) if ( mpp_pe() == mpp_root_pe() ) then logunit = stdlog() write (logunit, nml=grey_radiation_nml) diff --git a/src/atmos_param/lscale_cond/lscale_cond.F90 b/src/atmos_param/lscale_cond/lscale_cond.F90 index a5bbe316aa..946409cb9f 100644 --- a/src/atmos_param/lscale_cond/lscale_cond.F90 +++ b/src/atmos_param/lscale_cond/lscale_cond.F90 @@ -265,7 +265,7 @@ subroutine lscale_cond_init () !---------- output namelist -------------------------------------------- if ( mpp_pe() == mpp_root_pe() ) then - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() write (logunit,nml=lscale_cond_nml) endif diff --git a/src/atmos_param/mg_drag/mg_drag.F90 b/src/atmos_param/mg_drag/mg_drag.F90 index a5cd44acdc..eb85e3c988 100644 --- a/src/atmos_param/mg_drag/mg_drag.F90 +++ b/src/atmos_param/mg_drag/mg_drag.F90 @@ -1005,7 +1005,7 @@ subroutine mg_drag_init( lonb, latb, hprime ) ! --- Output version !--------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if(mpp_pe() == mpp_root_pe()) write (logunit, nml=mg_drag_nml) diff --git a/src/atmos_param/moist_conv/moist_conv.F90 b/src/atmos_param/moist_conv/moist_conv.F90 index 0315641dad..472bd8388c 100644 --- a/src/atmos_param/moist_conv/moist_conv.F90 +++ b/src/atmos_param/moist_conv/moist_conv.F90 @@ -830,7 +830,7 @@ subroutine moist_conv_init (axes, Time, tracers_in_mca) !---------- output namelist -------------------------------------------- if ( mpp_pe() == mpp_root_pe() ) then - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() write (logunit,nml=moist_conv_nml) endif diff --git a/src/atmos_param/moist_processes/detr_ice_num.F90 b/src/atmos_param/moist_processes/detr_ice_num.F90 index d31f8c9727..43ab0b9bcd 100644 --- a/src/atmos_param/moist_processes/detr_ice_num.F90 +++ b/src/atmos_param/moist_processes/detr_ice_num.F90 @@ -73,7 +73,7 @@ SUBROUTINE detr_ice_num_init !--------- write version and namelist to standard log ------------ - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if ( mpp_pe() == mpp_root_pe() ) & write ( logunit, nml=detr_ice_num_nml ) diff --git a/src/atmos_param/moist_processes/moist_processes.F90 b/src/atmos_param/moist_processes/moist_processes.F90 index dcada305f8..471e6d9eac 100644 --- a/src/atmos_param/moist_processes/moist_processes.F90 +++ b/src/atmos_param/moist_processes/moist_processes.F90 @@ -3592,7 +3592,7 @@ subroutine moist_processes_init ( id, jd, kd, lonb, latb, lon, lat, phalf, pref, !--------- write version and namelist to standard log ------------ - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if ( mpp_pe() == mpp_root_pe() ) & write ( logunit, nml=moist_processes_nml ) diff --git a/src/atmos_param/monin_obukhov/monin_obukhov.F90 b/src/atmos_param/monin_obukhov/monin_obukhov.F90 index dfdbafa9eb..06f22fd496 100644 --- a/src/atmos_param/monin_obukhov/monin_obukhov.F90 +++ b/src/atmos_param/monin_obukhov/monin_obukhov.F90 @@ -111,7 +111,7 @@ subroutine monin_obukhov_init !---------- output namelist to log------------------------------------- if ( mpp_pe() == mpp_root_pe() ) then - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() write (logunit, nml=monin_obukhov_nml) endif diff --git a/src/atmos_param/my25_turb/my25_turb.F90 b/src/atmos_param/my25_turb/my25_turb.F90 index 6907f62295..de8a83137f 100644 --- a/src/atmos_param/my25_turb/my25_turb.F90 +++ b/src/atmos_param/my25_turb/my25_turb.F90 @@ -635,7 +635,7 @@ SUBROUTINE MY25_TURB_INIT( ix, jx, kx ) !--------------------------------------------------------------------- if ( mpp_pe() == mpp_root_pe() ) then - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() WRITE( logunit, nml = my25_turb_nml ) endif diff --git a/src/atmos_param/physics_driver/physics_driver.F90 b/src/atmos_param/physics_driver/physics_driver.F90 index 202e463b43..9670588a19 100644 --- a/src/atmos_param/physics_driver/physics_driver.F90 +++ b/src/atmos_param/physics_driver/physics_driver.F90 @@ -703,7 +703,7 @@ subroutine physics_driver_init (Time, lonb, latb, lon, lat, axes, pref, & !-------------------------------------------------------------------- ! write version number and namelist to log file. !-------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if (mpp_pe() == mpp_root_pe() ) & write(logunit, nml=physics_driver_nml) diff --git a/src/atmos_param/radiation_driver/radiation_driver.F90 b/src/atmos_param/radiation_driver/radiation_driver.F90 index 85673c00c1..570c073683 100644 --- a/src/atmos_param/radiation_driver/radiation_driver.F90 +++ b/src/atmos_param/radiation_driver/radiation_driver.F90 @@ -1212,7 +1212,7 @@ subroutine radiation_driver_init (lonb, latb, pref, axes, Time, & !--------------------------------------------------------------------- ! write version number and namelist to logfile. !--------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if (mpp_pe() == mpp_root_pe() ) & write (logunit, nml=radiation_driver_nml) diff --git a/src/atmos_param/ras/ras.F90 b/src/atmos_param/ras/ras.F90 index a92e5a8c98..d7ea7e1fb1 100644 --- a/src/atmos_param/ras/ras.F90 +++ b/src/atmos_param/ras/ras.F90 @@ -251,7 +251,7 @@ SUBROUTINE RAS_INIT( do_strat, do_liq_num, axes, Time, tracers_in_ras ) ! --- Write namelist !--------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() WRITE( logunit, nml = ras_nml ) diff --git a/src/atmos_param/rh_clouds/rh_clouds.F90 b/src/atmos_param/rh_clouds/rh_clouds.F90 index a0710d0e6b..077354c37e 100644 --- a/src/atmos_param/rh_clouds/rh_clouds.F90 +++ b/src/atmos_param/rh_clouds/rh_clouds.F90 @@ -180,7 +180,7 @@ subroutine rh_clouds_init (nlon, nlat, nlev) !---------- output namelist to log------------------------------------- if ( mpp_pe() == mpp_root_pe() ) then - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() write (logunit, nml=rh_clouds_nml) endif diff --git a/src/atmos_param/sea_esf_rad/aerosol.F90 b/src/atmos_param/sea_esf_rad/aerosol.F90 index ceca7f00e6..d35f4c945a 100644 --- a/src/atmos_param/sea_esf_rad/aerosol.F90 +++ b/src/atmos_param/sea_esf_rad/aerosol.F90 @@ -397,7 +397,7 @@ subroutine aerosol_init (lonb, latb, aerosol_names, & !--------------------------------------------------------------------- ! write version number and namelist to logfile. !--------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if (mpp_pe() == mpp_root_pe() ) & write (logunit, nml=aerosol_nml) diff --git a/src/atmos_param/sea_esf_rad/aerosolrad_package.F90 b/src/atmos_param/sea_esf_rad/aerosolrad_package.F90 index d3221f0b32..2c01d07d2e 100644 --- a/src/atmos_param/sea_esf_rad/aerosolrad_package.F90 +++ b/src/atmos_param/sea_esf_rad/aerosolrad_package.F90 @@ -645,7 +645,7 @@ subroutine aerosolrad_package_init (kmax, aerosol_names, lonb, latb) !--------------------------------------------------------------------- ! write version number and namelist to logfile. !--------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if (mpp_pe() == mpp_root_pe() ) & write (logunit, nml=aerosolrad_package_nml) diff --git a/src/atmos_param/sea_esf_rad/bulkphys_rad.F90 b/src/atmos_param/sea_esf_rad/bulkphys_rad.F90 index 8cbe6aaaa9..97e6faf76d 100644 --- a/src/atmos_param/sea_esf_rad/bulkphys_rad.F90 +++ b/src/atmos_param/sea_esf_rad/bulkphys_rad.F90 @@ -278,7 +278,7 @@ subroutine bulkphys_rad_init (min_cld_drop_rad_in, max_cld_drop_rad_in,& !--------------------------------------------------------------------- ! write namelist to logfile. !--------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if (mpp_pe() == mpp_root_pe() ) & write (logunit, nml=bulkphys_rad_nml) diff --git a/src/atmos_param/sea_esf_rad/cloud_spec.F90 b/src/atmos_param/sea_esf_rad/cloud_spec.F90 index 742e63b3bd..bf97dd400f 100644 --- a/src/atmos_param/sea_esf_rad/cloud_spec.F90 +++ b/src/atmos_param/sea_esf_rad/cloud_spec.F90 @@ -355,7 +355,7 @@ subroutine cloud_spec_init (pref, lonb, latb, axes, Time, & !---------------------------------------------------------------------- ! write version number and namelist to logfile. !--------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if (mpp_pe() == mpp_root_pe() ) & write (logunit, nml=cloud_spec_nml) diff --git a/src/atmos_param/sea_esf_rad/cloudrad_diagnostics.F90 b/src/atmos_param/sea_esf_rad/cloudrad_diagnostics.F90 index dd31ed5582..3b4f6a02ef 100644 --- a/src/atmos_param/sea_esf_rad/cloudrad_diagnostics.F90 +++ b/src/atmos_param/sea_esf_rad/cloudrad_diagnostics.F90 @@ -450,7 +450,7 @@ subroutine cloudrad_diagnostics_init (min_cld_drop_rad_in, & !--------------------------------------------------------------------- ! write namelist to logfile. !--------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if (mpp_pe() == mpp_root_pe() ) & write (logunit, nml=cloudrad_diagnostics_nml) diff --git a/src/atmos_param/sea_esf_rad/cloudrad_package.F90 b/src/atmos_param/sea_esf_rad/cloudrad_package.F90 index f76cfeeeec..8609452a4f 100644 --- a/src/atmos_param/sea_esf_rad/cloudrad_package.F90 +++ b/src/atmos_param/sea_esf_rad/cloudrad_package.F90 @@ -233,7 +233,7 @@ subroutine cloudrad_package_init (pref, lonb, latb, axes, Time, & !--------------------------------------------------------------------- ! write version number and namelist to logfile. !--------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if (mpp_pe() == mpp_root_pe() ) & write (logunit, nml=cloudrad_package_nml) diff --git a/src/atmos_param/sea_esf_rad/diag_clouds_W.F90 b/src/atmos_param/sea_esf_rad/diag_clouds_W.F90 index 60f8fb6179..02773300ed 100644 --- a/src/atmos_param/sea_esf_rad/diag_clouds_W.F90 +++ b/src/atmos_param/sea_esf_rad/diag_clouds_W.F90 @@ -142,7 +142,7 @@ subroutine diag_clouds_W_init (num_slingo_bands_out) #endif if ( mpp_pe() == mpp_root_pe() ) then - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() write (logunit,nml=diag_clouds_W_nml) endif diff --git a/src/atmos_param/sea_esf_rad/donner_deep_clouds_W.F90 b/src/atmos_param/sea_esf_rad/donner_deep_clouds_W.F90 index d14a1e2fd1..2bc56f0693 100644 --- a/src/atmos_param/sea_esf_rad/donner_deep_clouds_W.F90 +++ b/src/atmos_param/sea_esf_rad/donner_deep_clouds_W.F90 @@ -141,7 +141,7 @@ subroutine donner_deep_clouds_W_init (pref, lonb, latb, axes, Time) #endif if ( mpp_pe() == mpp_root_pe() ) then - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() write (logunit,nml=donner_deep_clouds_W_nml) endif diff --git a/src/atmos_param/sea_esf_rad/esfsw_driver.F90 b/src/atmos_param/sea_esf_rad/esfsw_driver.F90 index df0e48f2e3..60dfe6253e 100644 --- a/src/atmos_param/sea_esf_rad/esfsw_driver.F90 +++ b/src/atmos_param/sea_esf_rad/esfsw_driver.F90 @@ -416,7 +416,7 @@ subroutine esfsw_driver_init !--------------------------------------------------------------------- ! write version number and namelist to logfile. !--------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if (mpp_pe() == mpp_root_pe() ) & write (logunit, nml=esfsw_driver_nml) diff --git a/src/atmos_param/sea_esf_rad/esfsw_parameters.F90 b/src/atmos_param/sea_esf_rad/esfsw_parameters.F90 index 29135df08b..67cd0c4c86 100644 --- a/src/atmos_param/sea_esf_rad/esfsw_parameters.F90 +++ b/src/atmos_param/sea_esf_rad/esfsw_parameters.F90 @@ -207,7 +207,7 @@ subroutine esfsw_parameters_init ! write version number and namelist to logfile also write out ! some key parameters obtained from an input data file. !--------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if (mpp_pe() == mpp_root_pe() ) then write (logunit,9000) & diff --git a/src/atmos_param/sea_esf_rad/gas_tf.F90 b/src/atmos_param/sea_esf_rad/gas_tf.F90 index 2efd3b55f4..1e1e31ab76 100644 --- a/src/atmos_param/sea_esf_rad/gas_tf.F90 +++ b/src/atmos_param/sea_esf_rad/gas_tf.F90 @@ -442,7 +442,7 @@ subroutine gas_tf_init (pref) !--------------------------------------------------------------------- ! write version number and namelist to logfile. !--------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if (mpp_pe() == mpp_root_pe() ) & write (logunit, nml=gas_tf_nml) diff --git a/src/atmos_param/sea_esf_rad/isccp_clouds.F90 b/src/atmos_param/sea_esf_rad/isccp_clouds.F90 index 46bf755134..15b46879a1 100644 --- a/src/atmos_param/sea_esf_rad/isccp_clouds.F90 +++ b/src/atmos_param/sea_esf_rad/isccp_clouds.F90 @@ -308,7 +308,7 @@ subroutine isccp_clouds_init (axes, Time) !--------------------------------------------------------------------- ! write namelist to logfile. !--------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if (mpp_pe() == mpp_root_pe() ) & write (logunit, nml=isccp_clouds_nml) diff --git a/src/atmos_param/sea_esf_rad/lhsw_driver.F90 b/src/atmos_param/sea_esf_rad/lhsw_driver.F90 index 3826272c39..8829545d5f 100644 --- a/src/atmos_param/sea_esf_rad/lhsw_driver.F90 +++ b/src/atmos_param/sea_esf_rad/lhsw_driver.F90 @@ -210,7 +210,7 @@ subroutine lhsw_driver_init ( pref ) #endif if ( mpp_pe() == mpp_root_pe() ) then - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() write (logunit,nml=lhsw_driver_nml) endif diff --git a/src/atmos_param/sea_esf_rad/longwave_clouds.F90 b/src/atmos_param/sea_esf_rad/longwave_clouds.F90 index c9692f4c19..3a908eb263 100644 --- a/src/atmos_param/sea_esf_rad/longwave_clouds.F90 +++ b/src/atmos_param/sea_esf_rad/longwave_clouds.F90 @@ -162,7 +162,7 @@ subroutine longwave_clouds_init !--------------------------------------------------------------------- ! write version number and namelist to logfile. !--------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if (mpp_pe() == mpp_root_pe() ) & write (logunit, nml=longwave_clouds_nml) diff --git a/src/atmos_param/sea_esf_rad/longwave_driver.F90 b/src/atmos_param/sea_esf_rad/longwave_driver.F90 index fb57efe6c7..33b4ef1221 100644 --- a/src/atmos_param/sea_esf_rad/longwave_driver.F90 +++ b/src/atmos_param/sea_esf_rad/longwave_driver.F90 @@ -204,7 +204,7 @@ subroutine longwave_driver_init (latb, lonb, pref, Lw_tables) !--------------------------------------------------------------------- ! write version number and namelist to logfile. !--------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if (mpp_pe() == mpp_root_pe() ) & write (logunit, nml=longwave_driver_nml) diff --git a/src/atmos_param/sea_esf_rad/longwave_fluxes.F90 b/src/atmos_param/sea_esf_rad/longwave_fluxes.F90 index d06c260878..2bad7584f0 100644 --- a/src/atmos_param/sea_esf_rad/longwave_fluxes.F90 +++ b/src/atmos_param/sea_esf_rad/longwave_fluxes.F90 @@ -144,7 +144,7 @@ subroutine longwave_fluxes_init !--------------------------------------------------------------------- ! write version number and namelist to logfile. !--------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if (mpp_pe() == mpp_root_pe() ) & write (logunit, nml=longwave_fluxes_nml) diff --git a/src/atmos_param/sea_esf_rad/longwave_params.F90 b/src/atmos_param/sea_esf_rad/longwave_params.F90 index 9443148533..cefd2d8e05 100644 --- a/src/atmos_param/sea_esf_rad/longwave_params.F90 +++ b/src/atmos_param/sea_esf_rad/longwave_params.F90 @@ -166,7 +166,7 @@ subroutine longwave_params_init !--------------------------------------------------------------------- ! write version number and namelist to logfile. !--------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if (mpp_pe() == mpp_root_pe() ) then write (logunit, nml=longwave_params_nml) diff --git a/src/atmos_param/sea_esf_rad/longwave_tables.F90 b/src/atmos_param/sea_esf_rad/longwave_tables.F90 index 979ad6893f..af7c7848a1 100644 --- a/src/atmos_param/sea_esf_rad/longwave_tables.F90 +++ b/src/atmos_param/sea_esf_rad/longwave_tables.F90 @@ -295,7 +295,7 @@ subroutine longwave_tables_init (Lw_tables, tabsr, tab1, tab2, tab3, & !--------------------------------------------------------------------- ! write version number and namelist to logfile. !--------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if (mpp_pe() == mpp_root_pe() ) & write (logunit, nml=longwave_tables_nml) diff --git a/src/atmos_param/sea_esf_rad/lw_gases_stdtf.F90 b/src/atmos_param/sea_esf_rad/lw_gases_stdtf.F90 index 64dcf591af..bb38e5688e 100644 --- a/src/atmos_param/sea_esf_rad/lw_gases_stdtf.F90 +++ b/src/atmos_param/sea_esf_rad/lw_gases_stdtf.F90 @@ -403,7 +403,7 @@ subroutine lw_gases_stdtf_init ( pref) !--------------------------------------------------------------------- ! write version number and namelist to logfile. !--------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if (mpp_pe() == mpp_root_pe() ) & write (logunit, nml=lw_gases_stdtf_nml) diff --git a/src/atmos_param/sea_esf_rad/mgrp_prscr_clds.F90 b/src/atmos_param/sea_esf_rad/mgrp_prscr_clds.F90 index 0964c8c6f1..99561f7144 100644 --- a/src/atmos_param/sea_esf_rad/mgrp_prscr_clds.F90 +++ b/src/atmos_param/sea_esf_rad/mgrp_prscr_clds.F90 @@ -232,7 +232,7 @@ subroutine mgrp_prscr_clds_init ( pref, latb ) endif #endif - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if (mpp_pe() == mpp_root_pe() ) & write (logunit, nml = mgrp_prscr_clds_nml) diff --git a/src/atmos_param/sea_esf_rad/microphys_cloud.F90 b/src/atmos_param/sea_esf_rad/microphys_cloud.F90 index 5e670efd4d..fab5388e3e 100644 --- a/src/atmos_param/sea_esf_rad/microphys_cloud.F90 +++ b/src/atmos_param/sea_esf_rad/microphys_cloud.F90 @@ -238,7 +238,7 @@ subroutine microphys_cloud_init ! write version number and namelist to logfile. !------------------------------------------------------------------------ - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if (mpp_pe() == mpp_root_pe() ) & write (logunit, nml=microphys_cloud_nml) diff --git a/src/atmos_param/sea_esf_rad/microphys_rad.F90 b/src/atmos_param/sea_esf_rad/microphys_rad.F90 index ee3e034c3d..dba67ac5cc 100644 --- a/src/atmos_param/sea_esf_rad/microphys_rad.F90 +++ b/src/atmos_param/sea_esf_rad/microphys_rad.F90 @@ -544,7 +544,7 @@ subroutine microphys_rad_init (min_cld_drop_rad_in, max_cld_drop_rad_in, & !--------------------------------------------------------------------- ! write namelist and version number to logfile. !--------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if (mpp_pe() == mpp_root_pe() ) & write (logunit, nml=microphys_rad_nml) diff --git a/src/atmos_param/sea_esf_rad/optical_path.F90 b/src/atmos_param/sea_esf_rad/optical_path.F90 index b859a52dc5..e6c046c2fa 100644 --- a/src/atmos_param/sea_esf_rad/optical_path.F90 +++ b/src/atmos_param/sea_esf_rad/optical_path.F90 @@ -371,7 +371,7 @@ subroutine optical_path_init(pref) !--------------------------------------------------------------------- ! write version number and namelist to logfile. !--------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if (mpp_pe() == mpp_root_pe() ) & write (logunit, nml=optical_path_nml) diff --git a/src/atmos_param/sea_esf_rad/original_fms_rad.F90 b/src/atmos_param/sea_esf_rad/original_fms_rad.F90 index dab057d021..19fa1a13d4 100644 --- a/src/atmos_param/sea_esf_rad/original_fms_rad.F90 +++ b/src/atmos_param/sea_esf_rad/original_fms_rad.F90 @@ -324,7 +324,7 @@ subroutine original_fms_rad_init ( lonb, latb, pref, axes, Time , & ! write namelist to logfile. !--------------------------------------------------------------------- if ( mpp_pe() == mpp_root_pe() ) then - call write_version_number() + call write_version_number(version, tagname) write (unit, nml=original_fms_rad_nml) endif diff --git a/src/atmos_param/sea_esf_rad/ozone.F90 b/src/atmos_param/sea_esf_rad/ozone.F90 index 944eecdfab..03e32fa8c5 100644 --- a/src/atmos_param/sea_esf_rad/ozone.F90 +++ b/src/atmos_param/sea_esf_rad/ozone.F90 @@ -334,7 +334,7 @@ subroutine ozone_init (latb, lonb) !--------------------------------------------------------------------- ! write version number and namelist to logfile. !--------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if (mpp_pe() == mpp_root_pe() ) & write (logunit, nml=ozone_nml) diff --git a/src/atmos_param/sea_esf_rad/rad_output_file.F90 b/src/atmos_param/sea_esf_rad/rad_output_file.F90 index 2f63674dd9..af8eba8d18 100644 --- a/src/atmos_param/sea_esf_rad/rad_output_file.F90 +++ b/src/atmos_param/sea_esf_rad/rad_output_file.F90 @@ -301,7 +301,7 @@ subroutine rad_output_file_init (axes, Time, names, family_names) !--------------------------------------------------------------------- ! write version number and namelist to logfile. !--------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if (mpp_pe() == mpp_root_pe() ) & write (logunit, nml=rad_output_file_nml) diff --git a/src/atmos_param/sea_esf_rad/rad_utilities.F90 b/src/atmos_param/sea_esf_rad/rad_utilities.F90 index d27808f85a..6a09174b7e 100644 --- a/src/atmos_param/sea_esf_rad/rad_utilities.F90 +++ b/src/atmos_param/sea_esf_rad/rad_utilities.F90 @@ -1367,7 +1367,7 @@ subroutine rad_utilities_init !--------------------------------------------------------------------- ! write version number and namelist to logfile. !--------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if (mpp_pe() == mpp_root_pe() ) & write (logunit, nml=rad_utilities_nml) diff --git a/src/atmos_param/sea_esf_rad/radiation_diag.F90 b/src/atmos_param/sea_esf_rad/radiation_diag.F90 index 93e83c09db..b7a8715248 100644 --- a/src/atmos_param/sea_esf_rad/radiation_diag.F90 +++ b/src/atmos_param/sea_esf_rad/radiation_diag.F90 @@ -252,7 +252,7 @@ subroutine radiation_diag_init (latb, lonb, Lw_tables) !--------------------------------------------------------------------- ! write version number and namelist to logfile. !--------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if (mpp_pe() == mpp_root_pe() ) & write (logunit, nml=radiation_diag_nml) diff --git a/src/atmos_param/sea_esf_rad/radiative_gases.F90 b/src/atmos_param/sea_esf_rad/radiative_gases.F90 index 8baa0a37ce..4e22a15e13 100644 --- a/src/atmos_param/sea_esf_rad/radiative_gases.F90 +++ b/src/atmos_param/sea_esf_rad/radiative_gases.F90 @@ -536,7 +536,7 @@ subroutine radiative_gases_init (pref, latb, lonb) !--------------------------------------------------------------------- ! write version number and namelist to logfile. !--------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if (mpp_pe() == mpp_root_pe() ) & write (logunit, nml=radiative_gases_nml) diff --git a/src/atmos_param/sea_esf_rad/rh_based_clouds.F90 b/src/atmos_param/sea_esf_rad/rh_based_clouds.F90 index 0fc84f9a8b..709fe082b5 100644 --- a/src/atmos_param/sea_esf_rad/rh_based_clouds.F90 +++ b/src/atmos_param/sea_esf_rad/rh_based_clouds.F90 @@ -249,7 +249,7 @@ subroutine rh_based_clouds_init !---------------------------------------------------------------------- ! write version number and namelist to logfile. !--------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if (mpp_pe() == mpp_root_pe() ) & write (logunit, nml=rh_based_clouds_nml) diff --git a/src/atmos_param/sea_esf_rad/sea_esf_rad.F90 b/src/atmos_param/sea_esf_rad/sea_esf_rad.F90 index 1f2f8f3cd4..21f555992f 100644 --- a/src/atmos_param/sea_esf_rad/sea_esf_rad.F90 +++ b/src/atmos_param/sea_esf_rad/sea_esf_rad.F90 @@ -226,7 +226,7 @@ subroutine sea_esf_rad_init (lonb, latb, pref_r) !--------------------------------------------------------------------- ! write version number and namelist to logfile. !--------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if (mpp_pe() == mpp_root_pe() ) & write (logunit, nml=sea_esf_rad_nml) diff --git a/src/atmos_param/sea_esf_rad/sealw99.F90 b/src/atmos_param/sea_esf_rad/sealw99.F90 index dbd311b0a6..baedda2838 100644 --- a/src/atmos_param/sea_esf_rad/sealw99.F90 +++ b/src/atmos_param/sea_esf_rad/sealw99.F90 @@ -488,7 +488,7 @@ subroutine sealw99_init (latb, lonb, pref, Lw_tables) !--------------------------------------------------------------------- ! write version number and namelist to logfile. !--------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if (mpp_pe() == mpp_root_pe() ) & write (logunit, nml=sealw99_nml) diff --git a/src/atmos_param/sea_esf_rad/shortwave_driver.F90 b/src/atmos_param/sea_esf_rad/shortwave_driver.F90 index 0acda38c20..8ae80c80d0 100644 --- a/src/atmos_param/sea_esf_rad/shortwave_driver.F90 +++ b/src/atmos_param/sea_esf_rad/shortwave_driver.F90 @@ -207,7 +207,7 @@ subroutine shortwave_driver_init (latb, pref) !--------------------------------------------------------------------- ! write version number and namelist to logfile. !--------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if (mpp_pe() == mpp_root_pe() ) & write (logunit, nml=shortwave_driver_nml) diff --git a/src/atmos_param/sea_esf_rad/specified_clouds_W.F90 b/src/atmos_param/sea_esf_rad/specified_clouds_W.F90 index 039d4310d9..01e631bdfa 100644 --- a/src/atmos_param/sea_esf_rad/specified_clouds_W.F90 +++ b/src/atmos_param/sea_esf_rad/specified_clouds_W.F90 @@ -170,7 +170,7 @@ subroutine specified_clouds_W_init (lonb, latb) !------- write version number and namelist --------- if ( mpp_pe() == mpp_root_pe() ) then - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() write (logunit,nml=specified_clouds_W_nml) endif diff --git a/src/atmos_param/sea_esf_rad/standalone_clouds.F90 b/src/atmos_param/sea_esf_rad/standalone_clouds.F90 index b2b1944c4a..dc7469a313 100644 --- a/src/atmos_param/sea_esf_rad/standalone_clouds.F90 +++ b/src/atmos_param/sea_esf_rad/standalone_clouds.F90 @@ -272,7 +272,7 @@ subroutine standalone_clouds_init (pref, lonb, latb) !--------------------------------------------------------------------- ! write namelist to logfile. !--------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if (mpp_pe() == mpp_root_pe() ) & write (logunit, nml=standalone_clouds_nml) diff --git a/src/atmos_param/sea_esf_rad/strat_clouds_W.F90 b/src/atmos_param/sea_esf_rad/strat_clouds_W.F90 index df53e32a18..5a30139c1f 100644 --- a/src/atmos_param/sea_esf_rad/strat_clouds_W.F90 +++ b/src/atmos_param/sea_esf_rad/strat_clouds_W.F90 @@ -187,7 +187,7 @@ subroutine strat_clouds_W_init(latb, lonb) !---------------------------------------------------------------------- ! write version number and namelist to logfile. !--------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if (mpp_pe() == mpp_root_pe() ) & write (logunit, nml=strat_clouds_W_nml) diff --git a/src/atmos_param/sea_esf_rad/uw_clouds_W.F90 b/src/atmos_param/sea_esf_rad/uw_clouds_W.F90 index 09114bcb26..512e85c31a 100644 --- a/src/atmos_param/sea_esf_rad/uw_clouds_W.F90 +++ b/src/atmos_param/sea_esf_rad/uw_clouds_W.F90 @@ -152,7 +152,7 @@ subroutine uw_clouds_W_init (pref, lonb, latb, axes, Time) #endif if ( mpp_pe() == mpp_root_pe() ) then - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() write (logunit,nml=uw_clouds_W_nml) endif diff --git a/src/atmos_param/sea_esf_rad/zetac_clouds_W.F90 b/src/atmos_param/sea_esf_rad/zetac_clouds_W.F90 index 10eac377c4..7ccaacc0d2 100644 --- a/src/atmos_param/sea_esf_rad/zetac_clouds_W.F90 +++ b/src/atmos_param/sea_esf_rad/zetac_clouds_W.F90 @@ -160,7 +160,7 @@ subroutine zetac_clouds_W_init !---------------------------------------------------------------------- ! write version number and namelist to logfile. !--------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if (mpp_pe() == mpp_root_pe() ) & write (logunit, nml=zetac_clouds_W_nml) diff --git a/src/atmos_param/shallow_conv/shallow_conv.F90 b/src/atmos_param/shallow_conv/shallow_conv.F90 index 9f50d71437..5611243ec5 100644 --- a/src/atmos_param/shallow_conv/shallow_conv.F90 +++ b/src/atmos_param/shallow_conv/shallow_conv.F90 @@ -108,7 +108,7 @@ SUBROUTINE SHALLOW_CONV_INIT( kx ) !------- write version number and namelist --------- if ( mpp_pe() == mpp_root_pe() ) then - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() WRITE( logunit, nml = shallow_conv_nml ) endif diff --git a/src/atmos_param/shallow_cu/conv_closures.F90 b/src/atmos_param/shallow_cu/conv_closures.F90 index 6ce5356387..1e7b4d041e 100644 --- a/src/atmos_param/shallow_cu/conv_closures.F90 +++ b/src/atmos_param/shallow_cu/conv_closures.F90 @@ -426,7 +426,7 @@ subroutine conv_closures_init !--------------------------------------------------------------------- ! write version number and namelist to logfile. !--------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) !--------------------------------------------------------------------- ! mark the module as initialized. !--------------------------------------------------------------------- diff --git a/src/atmos_param/shallow_cu/uw_conv.F90 b/src/atmos_param/shallow_cu/uw_conv.F90 index d4a1985fef..60ee4d4ee6 100644 --- a/src/atmos_param/shallow_cu/uw_conv.F90 +++ b/src/atmos_param/shallow_cu/uw_conv.F90 @@ -336,7 +336,7 @@ SUBROUTINE UW_CONV_INIT(do_strat, axes, Time, kd, tracers_in_uw) !========Option for deep convection======================================= end if #endif - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() WRITE( logunit, nml = uw_closure_nml ) WRITE( logunit, nml = uw_conv_nml ) diff --git a/src/atmos_param/stable_bl_turb/stable_bl_turb.F90 b/src/atmos_param/stable_bl_turb/stable_bl_turb.F90 index da684668dc..c1caebf199 100644 --- a/src/atmos_param/stable_bl_turb/stable_bl_turb.F90 +++ b/src/atmos_param/stable_bl_turb/stable_bl_turb.F90 @@ -504,7 +504,7 @@ subroutine STABLE_BL_TURB_INIT ( axes, Time ) !--------------------------------------------------------------------- if ( mpp_pe() == mpp_root_pe() ) then - call write_version_number() + call write_version_number(version, tagname) unit = stdlog() WRITE( unit, nml = stable_bl_turb_nml ) endif diff --git a/src/atmos_param/strat_cloud/aerosol_cloud.F90 b/src/atmos_param/strat_cloud/aerosol_cloud.F90 index 34dbf03106..8b1ee7b62a 100644 --- a/src/atmos_param/strat_cloud/aerosol_cloud.F90 +++ b/src/atmos_param/strat_cloud/aerosol_cloud.F90 @@ -125,7 +125,7 @@ subroutine aerosol_cloud_init (Constants) !------------------------------------------------------------------------- ! write version and namelist to standard log. !------------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if (mpp_pe() == mpp_root_pe()) & write (logunit, nml=aerosol_cloud_nml) diff --git a/src/atmos_param/strat_cloud/check_nan.F90 b/src/atmos_param/strat_cloud/check_nan.F90 index 99686863fd..cc1b7cfb78 100644 --- a/src/atmos_param/strat_cloud/check_nan.F90 +++ b/src/atmos_param/strat_cloud/check_nan.F90 @@ -32,7 +32,7 @@ subroutine check_nan_init !------------------------------------------------------------------------ ! write version number to output file. !------------------------------------------------------------------------ - call write_version_number() + call write_version_number(version, tagname) !------------------------------------------------------------------------ ! declare module initialized. diff --git a/src/atmos_param/strat_cloud/cldwat2m_micro.F90 b/src/atmos_param/strat_cloud/cldwat2m_micro.F90 index b40a9cf8ae..e2388a496f 100644 --- a/src/atmos_param/strat_cloud/cldwat2m_micro.F90 +++ b/src/atmos_param/strat_cloud/cldwat2m_micro.F90 @@ -392,7 +392,7 @@ subroutine ini_micro !----------------------------------------------------------------------- ! write version and namelist to stdlog. !----------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if (mpp_pe() == mpp_root_pe()) & write (logunit, nml=cldwat2m_micro_nml) diff --git a/src/atmos_param/strat_cloud/gamma_mg.F90 b/src/atmos_param/strat_cloud/gamma_mg.F90 index 1e23c7a3e7..608d261acd 100644 --- a/src/atmos_param/strat_cloud/gamma_mg.F90 +++ b/src/atmos_param/strat_cloud/gamma_mg.F90 @@ -31,7 +31,7 @@ SUBROUTINE gamma_mg_init !----------------------------------------------------------------------- ! write version number to output file. !----------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) !----------------------------------------------------------------------- ! be sure needed modules are initialized. diff --git a/src/atmos_param/strat_cloud/mg_const.F90 b/src/atmos_param/strat_cloud/mg_const.F90 index f92a1c5281..e77af929ba 100644 --- a/src/atmos_param/strat_cloud/mg_const.F90 +++ b/src/atmos_param/strat_cloud/mg_const.F90 @@ -58,7 +58,7 @@ subroutine mg_const_init !------------------------------------------------------------------------- ! write version number to output file. !------------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) !------------------------------------------------------------------------- ! mark the module as initialized. diff --git a/src/atmos_param/strat_cloud/microphysics.F90 b/src/atmos_param/strat_cloud/microphysics.F90 index be0636d2c5..b91d9b8395 100644 --- a/src/atmos_param/strat_cloud/microphysics.F90 +++ b/src/atmos_param/strat_cloud/microphysics.F90 @@ -57,7 +57,7 @@ subroutine microphysics_init (Nml) !------------------------------------------------------------------------- ! write version number to output file. !------------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) !------------------------------------------------------------------------- ! make sure needed modules have been initialized. diff --git a/src/atmos_param/strat_cloud/morrison_gettelman_microp.F90 b/src/atmos_param/strat_cloud/morrison_gettelman_microp.F90 index 35ee347978..00d62b0bd3 100644 --- a/src/atmos_param/strat_cloud/morrison_gettelman_microp.F90 +++ b/src/atmos_param/strat_cloud/morrison_gettelman_microp.F90 @@ -272,7 +272,7 @@ SUBROUTINE morrison_gettelman_microp_init (do_pdf_clouds, qcvar_in) !----------------------------------------------------------------------- ! write version and namelist to stdlog. !----------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if (mpp_pe() == mpp_root_pe()) & write (logunit, nml=morrison_gettelman_microp_nml) diff --git a/src/atmos_param/strat_cloud/nc_cond.F90 b/src/atmos_param/strat_cloud/nc_cond.F90 index 86e0317c93..2970ec7441 100644 --- a/src/atmos_param/strat_cloud/nc_cond.F90 +++ b/src/atmos_param/strat_cloud/nc_cond.F90 @@ -102,7 +102,7 @@ SUBROUTINE nc_cond_init( do_pdf_clouds ) !------------------------------------------------------------------------- ! write version and namelist to standard log. !------------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if ( mpp_pe() == mpp_root_pe() ) & write ( logunit, nml=nc_cond_nml ) diff --git a/src/atmos_param/strat_cloud/polysvp.F90 b/src/atmos_param/strat_cloud/polysvp.F90 index 15f5af2356..79f95f80bf 100644 --- a/src/atmos_param/strat_cloud/polysvp.F90 +++ b/src/atmos_param/strat_cloud/polysvp.F90 @@ -54,7 +54,7 @@ SUBROUTINE polysvp_init !------------------------------------------------------------------------ ! write version number to output file. !------------------------------------------------------------------------ - call write_version_number() + call write_version_number(version, tagname) !------------------------------------------------------------------------ ! make sure needed modules have been initialized. diff --git a/src/atmos_param/strat_cloud/rotstayn_klein_mp.F90 b/src/atmos_param/strat_cloud/rotstayn_klein_mp.F90 index 9309a3cfbc..c055872cf6 100644 --- a/src/atmos_param/strat_cloud/rotstayn_klein_mp.F90 +++ b/src/atmos_param/strat_cloud/rotstayn_klein_mp.F90 @@ -110,7 +110,7 @@ SUBROUTINE rotstayn_klein_microp_init !------------------------------------------------------------------------- ! write version and namelist to standard log. !------------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if ( mpp_pe() == mpp_root_pe() ) & write (logunit, nml=rotstayn_klein_mp_nml) diff --git a/src/atmos_param/strat_cloud/simple_pdf.F90 b/src/atmos_param/strat_cloud/simple_pdf.F90 index 20c46f441c..e12978c4cb 100644 --- a/src/atmos_param/strat_cloud/simple_pdf.F90 +++ b/src/atmos_param/strat_cloud/simple_pdf.F90 @@ -40,7 +40,7 @@ SUBROUTINE simple_pdf_init !----------------------------------------------------------------------- ! write version number to output file. !----------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) !----------------------------------------------------------------------- ! make sure needed modules have been initialized. diff --git a/src/atmos_param/strat_cloud/strat_cloud.F90 b/src/atmos_param/strat_cloud/strat_cloud.F90 index fa5cddafc4..5e24c4a7d1 100644 --- a/src/atmos_param/strat_cloud/strat_cloud.F90 +++ b/src/atmos_param/strat_cloud/strat_cloud.F90 @@ -396,7 +396,7 @@ subroutine strat_cloud_init (axes, Time, idim, jdim, kdim, & !----------------------------------------------------------------------- ! write version and namelist to stdlog. !----------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if ( mpp_pe() == mpp_root_pe() ) & write (logunit, nml=strat_cloud_nml) diff --git a/src/atmos_param/strat_cloud/strat_cloud_legacy.F90 b/src/atmos_param/strat_cloud/strat_cloud_legacy.F90 index 8be3b2509b..a1242546d4 100644 --- a/src/atmos_param/strat_cloud/strat_cloud_legacy.F90 +++ b/src/atmos_param/strat_cloud/strat_cloud_legacy.F90 @@ -345,7 +345,7 @@ subroutine strat_cloud_legacy_init (do_pdf_clouds) !----------------------------------------------------------------------- ! write version number to output file. !----------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) !----------------------------------------------------------------------- ! make sure needed modules have been initialized. diff --git a/src/atmos_param/strat_cloud/strat_cloud_utilities.F90 b/src/atmos_param/strat_cloud/strat_cloud_utilities.F90 index 4e91a6f8ea..7a9f0ecd7f 100644 --- a/src/atmos_param/strat_cloud/strat_cloud_utilities.F90 +++ b/src/atmos_param/strat_cloud/strat_cloud_utilities.F90 @@ -474,7 +474,7 @@ subroutine strat_cloud_utilities_init !------------------------------------------------------------------------ ! write version number to output file. !------------------------------------------------------------------------ - call write_version_number() + call write_version_number(version, tagname) !------------------------------------------------------------------------ ! mark this module as initialized. diff --git a/src/atmos_param/strat_cloud/strat_netcdf.F90 b/src/atmos_param/strat_cloud/strat_netcdf.F90 index 9797d1c390..387717e935 100644 --- a/src/atmos_param/strat_cloud/strat_netcdf.F90 +++ b/src/atmos_param/strat_cloud/strat_netcdf.F90 @@ -52,7 +52,7 @@ subroutine strat_netcdf_init (axes, Time, diag_id, diag_pt, n_diag_4d, & !----------------------------------------------------------------------- ! write version info to standard log. !----------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) !------------------------------------------------------------------------ ! make sure needed modules have been initialized. diff --git a/src/atmos_param/topo_drag/topo_drag.F90 b/src/atmos_param/topo_drag/topo_drag.F90 index efdfd9f137..f76b7b5eb3 100644 --- a/src/atmos_param/topo_drag/topo_drag.F90 +++ b/src/atmos_param/topo_drag/topo_drag.F90 @@ -601,7 +601,7 @@ subroutine topo_drag_init (lonb, latb) ! write version number and namelist to logfile - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if (mpp_pe() == mpp_root_pe()) & write (logunit, nml=topo_drag_nml) diff --git a/src/atmos_param/vert_diff/vert_diff.F90 b/src/atmos_param/vert_diff/vert_diff.F90 index bac7a3c4f0..eb3452f59e 100644 --- a/src/atmos_param/vert_diff/vert_diff.F90 +++ b/src/atmos_param/vert_diff/vert_diff.F90 @@ -113,7 +113,7 @@ subroutine vert_diff_init (Tri_surf, idim, jdim, kdim, & character(len=128) :: scheme ! tracer diffusion scheme integer :: n, logunit - call write_version_number() + call write_version_number(version, tagname) ! get the number of prognostic tracers call get_number_tracers( MODEL_ATMOS, num_prog=ntprog) diff --git a/src/atmos_param/vert_diff_driver/vert_diff_driver.F90 b/src/atmos_param/vert_diff_driver/vert_diff_driver.F90 index 72bdf594d7..a8363ab5b2 100644 --- a/src/atmos_param/vert_diff_driver/vert_diff_driver.F90 +++ b/src/atmos_param/vert_diff_driver/vert_diff_driver.F90 @@ -411,7 +411,7 @@ subroutine vert_diff_driver_init ( Surf_diff, idim, jdim, kdim, & !--------- write version number and namelist ------------------ - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if(mpp_pe() == mpp_root_pe() ) write(logunit,nml=vert_diff_driver_nml) diff --git a/src/atmos_param/vert_turb_driver/vert_turb_driver.F90 b/src/atmos_param/vert_turb_driver/vert_turb_driver.F90 index b30917171f..cebd8d8571 100644 --- a/src/atmos_param/vert_turb_driver/vert_turb_driver.F90 +++ b/src/atmos_param/vert_turb_driver/vert_turb_driver.F90 @@ -643,7 +643,7 @@ subroutine vert_turb_driver_init (lonb, latb, id, jd, kd, axes, Time, & logunit = stdlog() if ( mpp_pe() == mpp_root_pe() ) then - call write_version_number() + call write_version_number(version, tagname) write (logunit,nml=vert_turb_driver_nml) endif diff --git a/src/atmos_shared/atmos_nudge/atmos_nudge.F90 b/src/atmos_shared/atmos_nudge/atmos_nudge.F90 index 53bf4c973f..73a0c50b87 100644 --- a/src/atmos_shared/atmos_nudge/atmos_nudge.F90 +++ b/src/atmos_shared/atmos_nudge/atmos_nudge.F90 @@ -314,7 +314,7 @@ subroutine atmos_nudge_init ( Time, axes, flag ) enddo 10 call close_file (unit) #endif - call write_version_number() + call write_version_number(version, tagname) logunit=stdlog() if (mpp_pe() == mpp_root_pe()) write (logunit, nml=atmos_nudge_nml) diff --git a/src/atmos_shared/interpolator/interpolator.F90 b/src/atmos_shared/interpolator/interpolator.F90 index 7efa88f232..c8283768dd 100644 --- a/src/atmos_shared/interpolator/interpolator.F90 +++ b/src/atmos_shared/interpolator/interpolator.F90 @@ -1026,7 +1026,7 @@ subroutine interpolator_init( clim_type, file_name, lonb_mod, latb_mod, & module_is_initialized = .true. -call write_version_number() +call write_version_number(version, tagname) end subroutine interpolator_init diff --git a/src/atmos_shared/tracer_driver/aer_ccn_act/aer_ccn_act.F90 b/src/atmos_shared/tracer_driver/aer_ccn_act/aer_ccn_act.F90 index 180a5fa493..58d6705769 100644 --- a/src/atmos_shared/tracer_driver/aer_ccn_act/aer_ccn_act.F90 +++ b/src/atmos_shared/tracer_driver/aer_ccn_act/aer_ccn_act.F90 @@ -202,7 +202,7 @@ subroutine aer_ccn_act_init () !--------------------------------------------------------------------- ! write version number and namelist to logfile. !-------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit=stdlog() if (mpp_pe() == mpp_root_pe() ) & write (logunit, nml=aer_ccn_act_nml) diff --git a/src/atmos_shared/tracer_driver/aer_ccn_act/aerosol_params.F90 b/src/atmos_shared/tracer_driver/aer_ccn_act/aerosol_params.F90 index ef4ba8b4d5..0c3194fbad 100644 --- a/src/atmos_shared/tracer_driver/aer_ccn_act/aerosol_params.F90 +++ b/src/atmos_shared/tracer_driver/aer_ccn_act/aerosol_params.F90 @@ -95,7 +95,7 @@ subroutine aerosol_params_init !------------------------------------------------------------------------- ! write version number to output file. !------------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) !------------------------------------------------------------------------- ! declare module to be initialized. diff --git a/src/atmos_shared/tracer_driver/aer_ccn_act/ice_nucl.F90 b/src/atmos_shared/tracer_driver/aer_ccn_act/ice_nucl.F90 index c5d820f056..f72980a4d9 100644 --- a/src/atmos_shared/tracer_driver/aer_ccn_act/ice_nucl.F90 +++ b/src/atmos_shared/tracer_driver/aer_ccn_act/ice_nucl.F90 @@ -141,7 +141,7 @@ SUBROUTINE ice_nucl_wpdf_init !--------- write version and namelist to standard log ------------ - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if ( mpp_pe() == mpp_root_pe() ) & write ( logunit, nml=ice_nucl_nml ) diff --git a/src/atmos_shared/tracer_driver/atmos_age_tracer.F90 b/src/atmos_shared/tracer_driver/atmos_age_tracer.F90 index 2807e5889c..058334444f 100644 --- a/src/atmos_shared/tracer_driver/atmos_age_tracer.F90 +++ b/src/atmos_shared/tracer_driver/atmos_age_tracer.F90 @@ -237,7 +237,7 @@ subroutine atmos_age_tracer_init( r, axes, Time, nage, & !---- write namelist ------------------ - call write_version_number() + call write_version_number(version, tagname) ! if ( mpp_pe() == mpp_root_pe() ) & ! write ( stdlog(), nml=atmos_age_tracer_nml ) diff --git a/src/atmos_shared/tracer_driver/atmos_carbon_aerosol.F90 b/src/atmos_shared/tracer_driver/atmos_carbon_aerosol.F90 index 34dea6e12f..0a56b5b5c8 100644 --- a/src/atmos_shared/tracer_driver/atmos_carbon_aerosol.F90 +++ b/src/atmos_shared/tracer_driver/atmos_carbon_aerosol.F90 @@ -990,7 +990,7 @@ subroutine atmos_carbon_aerosol_init (lonb, latb, axes, Time, mask) !--------------------------------------------------------------------- ! write version number and namelist to logfile. !--------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit=stdlog() if (mpp_pe() == mpp_root_pe() ) & write (logunit, nml=carbon_aerosol_nml) @@ -2237,7 +2237,7 @@ subroutine atmos_carbon_aerosol_init (lonb, latb, axes, Time, mask) endif - call write_version_number() + call write_version_number(version, tagname) module_is_initialized = .TRUE. !----------------------------------------------------------------------- diff --git a/src/atmos_shared/tracer_driver/atmos_ch3i.F90 b/src/atmos_shared/tracer_driver/atmos_ch3i.F90 index 232b170404..7d094b123e 100644 --- a/src/atmos_shared/tracer_driver/atmos_ch3i.F90 +++ b/src/atmos_shared/tracer_driver/atmos_ch3i.F90 @@ -100,7 +100,7 @@ subroutine atmos_ch3i_init( lonb_mod, latb_mod, axes, Time, mask ) !----------------------------------------------------------------------- ! ... write version number !----------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) !----------------------------------------------------------------------- ! ... read namelist diff --git a/src/atmos_shared/tracer_driver/atmos_co2.F90 b/src/atmos_shared/tracer_driver/atmos_co2.F90 index 56aaa67829..ec739a6a3a 100644 --- a/src/atmos_shared/tracer_driver/atmos_co2.F90 +++ b/src/atmos_shared/tracer_driver/atmos_co2.F90 @@ -577,7 +577,7 @@ subroutine atmos_co2_init (Time, axes) if (module_is_initialized) return - call write_version_number() + call write_version_number(version, tagname) !----------------------------------------------------------------------- ! read namelist. @@ -659,7 +659,7 @@ subroutine atmos_co2_init (Time, axes) write (logunit,*)' not using CO2 emissions: do_co2_emissions= ',do_co2_emissions endif -call write_version_number() +call write_version_number(version, tagname) module_is_initialized = .TRUE. diff --git a/src/atmos_shared/tracer_driver/atmos_convection_tracer.F90 b/src/atmos_shared/tracer_driver/atmos_convection_tracer.F90 index d20a4a4b86..937add5aad 100644 --- a/src/atmos_shared/tracer_driver/atmos_convection_tracer.F90 +++ b/src/atmos_shared/tracer_driver/atmos_convection_tracer.F90 @@ -225,7 +225,7 @@ subroutine atmos_convection_tracer_init (r, phalf, axes, Time, & !---- write namelist ------------------ - call write_version_number() + call write_version_number(version, tagname) logunit=stdlog() if ( mpp_pe() == mpp_root_pe() ) & write ( logunit, nml=atmos_convection_tracer_nml ) diff --git a/src/atmos_shared/tracer_driver/atmos_dust.F90 b/src/atmos_shared/tracer_driver/atmos_dust.F90 index 753b6f90bc..ce5eb0a746 100644 --- a/src/atmos_shared/tracer_driver/atmos_dust.F90 +++ b/src/atmos_shared/tracer_driver/atmos_dust.F90 @@ -245,7 +245,7 @@ subroutine atmos_dust_init (lonb, latb, axes, Time, mask) if (module_is_initialized) return - call write_version_number() + call write_version_number(version, tagname) !----------------------------------------------------------------------- ! read namelist. !----------------------------------------------------------------------- @@ -313,7 +313,7 @@ subroutine atmos_dust_init (lonb, latb, axes, Time, mask) vert_interp=(/INTERP_WEIGHTED_P/) ) - call write_version_number() + call write_version_number(version, tagname) module_is_initialized = .TRUE. diff --git a/src/atmos_shared/tracer_driver/atmos_radon.F90 b/src/atmos_shared/tracer_driver/atmos_radon.F90 index 7be9b3e79c..cc9aed9e4b 100644 --- a/src/atmos_shared/tracer_driver/atmos_radon.F90 +++ b/src/atmos_shared/tracer_driver/atmos_radon.F90 @@ -260,7 +260,7 @@ subroutine atmos_radon_init (r, axes, Time, nradon, mask) !---- write namelist ------------------ - call write_version_number() + call write_version_number(version, tagname) logunit=stdlog() if ( mpp_pe() == mpp_root_pe() ) & write ( logunit, nml=atmos_radon_nml ) diff --git a/src/atmos_shared/tracer_driver/atmos_sea_salt.F90 b/src/atmos_shared/tracer_driver/atmos_sea_salt.F90 index 1d8232769e..b7bb5e0bd0 100644 --- a/src/atmos_shared/tracer_driver/atmos_sea_salt.F90 +++ b/src/atmos_shared/tracer_driver/atmos_sea_salt.F90 @@ -375,7 +375,7 @@ subroutine atmos_sea_salt_init (lonb, latb, axes, Time, mask) #endif endif !--------- write version and namelist to standard log ------------ - call write_version_number() + call write_version_number(version, tagname) logunit=stdlog() if ( mpp_pe() == mpp_root_pe() ) & write ( logunit, nml=ssalt_nml ) diff --git a/src/atmos_shared/tracer_driver/atmos_soa.F90 b/src/atmos_shared/tracer_driver/atmos_soa.F90 index 8c1206dca7..0e73dad831 100644 --- a/src/atmos_shared/tracer_driver/atmos_soa.F90 +++ b/src/atmos_shared/tracer_driver/atmos_soa.F90 @@ -126,7 +126,7 @@ subroutine atmos_SOA_init ( lonb, latb, nlev, axes, Time, mask) !--------------------------------------------------------------------- ! write version number and namelist to logfile. !--------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit=stdlog() if (mpp_pe() == mpp_root_pe() ) & write (logunit, nml=secondary_organics_nml) @@ -171,7 +171,7 @@ subroutine atmos_SOA_init ( lonb, latb, nlev, axes, Time, mask) 'column SOA production by C4H10 + OH', & 'kg/m2/s') - call write_version_number() + call write_version_number(version, tagname) module_is_initialized = .TRUE. diff --git a/src/atmos_shared/tracer_driver/atmos_sulfate.F90 b/src/atmos_shared/tracer_driver/atmos_sulfate.F90 index 0a6c0d883a..1c3a6bc3b1 100644 --- a/src/atmos_shared/tracer_driver/atmos_sulfate.F90 +++ b/src/atmos_shared/tracer_driver/atmos_sulfate.F90 @@ -329,7 +329,7 @@ subroutine atmos_sulfate_init ( lonb, latb, nlev, axes, Time, mask) !---- write namelist ------------------ - call write_version_number() + call write_version_number(version, tagname) ! read namelist. !----------------------------------------------------------------------- @@ -350,7 +350,7 @@ subroutine atmos_sulfate_init ( lonb, latb, nlev, axes, Time, mask) !--------------------------------------------------------------------- ! write version number and namelist to logfile. !--------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit=stdlog() if (mpp_pe() == mpp_root_pe() ) & write (logunit, nml=simple_sulfate_nml) @@ -928,7 +928,7 @@ subroutine atmos_sulfate_init ( lonb, latb, nlev, axes, Time, mask) 'simpleH2O2 chemical production', & 'kgH2O2/m2/s') - call write_version_number() + call write_version_number(version, tagname) module_is_initialized = .TRUE. diff --git a/src/atmos_shared/tracer_driver/atmos_sulfur_hex.F90 b/src/atmos_shared/tracer_driver/atmos_sulfur_hex.F90 index 63f9981d2e..4758e1215b 100644 --- a/src/atmos_shared/tracer_driver/atmos_sulfur_hex.F90 +++ b/src/atmos_shared/tracer_driver/atmos_sulfur_hex.F90 @@ -281,7 +281,7 @@ subroutine atmos_sulfur_hex_init (lonb, latb, r, axes, Time, mask) !---- write namelist ------------------ - call write_version_number() + call write_version_number(version, tagname) n = get_tracer_index(MODEL_ATMOS,'sf6') if (n>0) then diff --git a/src/atmos_shared/tracer_driver/atmos_tracer_driver.F90 b/src/atmos_shared/tracer_driver/atmos_tracer_driver.F90 index c8e498ae7e..f43fdf64cd 100644 --- a/src/atmos_shared/tracer_driver/atmos_tracer_driver.F90 +++ b/src/atmos_shared/tracer_driver/atmos_tracer_driver.F90 @@ -1251,7 +1251,7 @@ subroutine atmos_tracer_driver_init (lonb, latb, r, axes, Time, phalf, mask) #endif endif !--------- write version and namelist to standard log ------------ - call write_version_number() + call write_version_number(version, tagname) logunit=stdlog() if ( mpp_pe() == mpp_root_pe() ) & write ( logunit, nml=atmos_tracer_driver_nml ) diff --git a/src/atmos_shared/tracer_driver/atmos_tracer_utilities.F90 b/src/atmos_shared/tracer_driver/atmos_tracer_utilities.F90 index 842be76438..d143ee14aa 100644 --- a/src/atmos_shared/tracer_driver/atmos_tracer_utilities.F90 +++ b/src/atmos_shared/tracer_driver/atmos_tracer_utilities.F90 @@ -373,7 +373,7 @@ subroutine atmos_tracer_utilities_init(lonb, latb, mass_axes, Time) 'm/s', missing_value=-999. ) - call write_version_number() + call write_version_number(version, tagname) if ( mpp_pe() == mpp_root_pe() ) then logunit=stdlog() diff --git a/src/atmos_shared/tracer_driver/stratchem/strat_chem_driver.F90 b/src/atmos_shared/tracer_driver/stratchem/strat_chem_driver.F90 index 9972333413..3f83b5e260 100644 --- a/src/atmos_shared/tracer_driver/stratchem/strat_chem_driver.F90 +++ b/src/atmos_shared/tracer_driver/stratchem/strat_chem_driver.F90 @@ -120,7 +120,7 @@ function strat_chem_driver_init() ! write version number and namelist to logfile. !--------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit=stdlog() if (mpp_pe() == mpp_root_pe()) write (logunit, nml=strat_chem_nml) diff --git a/src/atmos_shared/tracer_driver/tropchem/mo_fastjx.F90 b/src/atmos_shared/tracer_driver/tropchem/mo_fastjx.F90 index 85bf822e46..aad0d1a904 100644 --- a/src/atmos_shared/tracer_driver/tropchem/mo_fastjx.F90 +++ b/src/atmos_shared/tracer_driver/tropchem/mo_fastjx.F90 @@ -420,7 +420,7 @@ subroutine fastjx_init !---------------------------------------------------------------------- ! output version number and tagname to logfile. !---------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) ! ! Defaults & constants diff --git a/src/atmos_shared/tracer_driver/tropchem/mo_fphoto.F90 b/src/atmos_shared/tracer_driver/tropchem/mo_fphoto.F90 index 51653d5c02..a69aa6f93a 100644 --- a/src/atmos_shared/tracer_driver/tropchem/mo_fphoto.F90 +++ b/src/atmos_shared/tracer_driver/tropchem/mo_fphoto.F90 @@ -156,7 +156,7 @@ subroutine fprate_init(o3_column_top_in) !------------------------------------------------------------------------- ! write version number and tagname to stdlog. !------------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) indexer(TAB_NDX_JO2) = get_rxt_ndx( 'jo2' ) indexer(TAB_NDX_JO3) = get_rxt_ndx( 'jo3p' ) diff --git a/src/atmos_shared/tracer_driver/tropchem/tropchem_driver.F90 b/src/atmos_shared/tracer_driver/tropchem/tropchem_driver.F90 index c84f47d40d..a3f5def9f9 100644 --- a/src/atmos_shared/tracer_driver/tropchem/tropchem_driver.F90 +++ b/src/atmos_shared/tracer_driver/tropchem/tropchem_driver.F90 @@ -1379,7 +1379,7 @@ function tropchem_driver_init( r, mask, axes, Time, & !----------------------------------------------------------------------- ! ... write version number !----------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) !----------------------------------------------------------------------- ! ... read namelist diff --git a/src/atmos_shared/vert_advection/vert_advection.F90 b/src/atmos_shared/vert_advection/vert_advection.F90 index c33e000ab5..35288f100b 100644 --- a/src/atmos_shared/vert_advection/vert_advection.F90 +++ b/src/atmos_shared/vert_advection/vert_advection.F90 @@ -107,7 +107,7 @@ subroutine vert_advection_3d ( dt, w, dz, r, rdt, mask, scheme, form, flags ) integer :: kk if(.not.module_is_initialized) then - call write_version_number() + call write_version_number(version, tagname) module_is_initialized = .true. endif diff --git a/src/atmos_spectral/driver/coupled/atmosphere.F90 b/src/atmos_spectral/driver/coupled/atmosphere.F90 index ea098713e2..04bdcd8b76 100644 --- a/src/atmos_spectral/driver/coupled/atmosphere.F90 +++ b/src/atmos_spectral/driver/coupled/atmosphere.F90 @@ -105,7 +105,7 @@ subroutine atmosphere_init(Time_init, Time, Time_step_in, Surf_diff) enddo 20 call close_file (unit) -call write_version_number() +call write_version_number(version, tagname) if(mpp_pe() == mpp_root_pe()) write (stdlog(), nml=atmosphere_nml) !----------------------------------------------------------------------------------------- diff --git a/src/atmos_spectral/driver/coupled/mcm_mca_lsc.F90 b/src/atmos_spectral/driver/coupled/mcm_mca_lsc.F90 index d02fa99d18..8f5f4be450 100644 --- a/src/atmos_spectral/driver/coupled/mcm_mca_lsc.F90 +++ b/src/atmos_spectral/driver/coupled/mcm_mca_lsc.F90 @@ -150,7 +150,7 @@ subroutine init_mcm_mca_lsc enddo 20 call close_file (unit) - call write_version_number() + call write_version_number(version, tagname) if(mpp_pe() == mpp_root_pe()) write (stdlog(),nml=mcm_mca_lsc_nml) ! ix and kx stay fixed througout run diff --git a/src/atmos_spectral/driver/coupled/mcm_moist_processes.F90 b/src/atmos_spectral/driver/coupled/mcm_moist_processes.F90 index 0c52fbac7b..10d4f1f99d 100644 --- a/src/atmos_spectral/driver/coupled/mcm_moist_processes.F90 +++ b/src/atmos_spectral/driver/coupled/mcm_moist_processes.F90 @@ -389,7 +389,7 @@ subroutine mcm_moist_processes_init ( id, jd, kd, axes, Time ) !--------- write namelist ------------------ - call write_version_number() + call write_version_number(version, tagname) if(mpp_pe() == mpp_root_pe()) write (stdlog(), nml=mcm_moist_processes_nml) !------------ initialize various schemes ---------- diff --git a/src/atmos_spectral/driver/coupled/spectral_physics.F90 b/src/atmos_spectral/driver/coupled/spectral_physics.F90 index fc9583653a..93c14dc9cd 100644 --- a/src/atmos_spectral/driver/coupled/spectral_physics.F90 +++ b/src/atmos_spectral/driver/coupled/spectral_physics.F90 @@ -82,7 +82,7 @@ subroutine spectral_physics_init(Time, axes, Surf_diff, nhum_in, p_half, do_mcm_ if(module_is_initialized) return -call write_version_number() +call write_version_number(version, tagname) call fms_init call time_manager_init diff --git a/src/atmos_spectral/driver/solo/atmosphere.F90 b/src/atmos_spectral/driver/solo/atmosphere.F90 index 7a429cf1cc..5f66046cd8 100644 --- a/src/atmos_spectral/driver/solo/atmosphere.F90 +++ b/src/atmos_spectral/driver/solo/atmosphere.F90 @@ -112,7 +112,7 @@ subroutine atmosphere_init(Time_init, Time, Time_step_in) if(module_is_initialized) return -call write_version_number() +call write_version_number(version, tagname) Time_step = Time_step_in call get_time(Time_step, seconds, days) diff --git a/src/atmos_spectral/driver/solo/idealized_moist_phys.F90 b/src/atmos_spectral/driver/solo/idealized_moist_phys.F90 index 7b07ca0525..bca75d17e6 100644 --- a/src/atmos_spectral/driver/solo/idealized_moist_phys.F90 +++ b/src/atmos_spectral/driver/solo/idealized_moist_phys.F90 @@ -169,7 +169,7 @@ subroutine idealized_moist_phys_init(Time, Time_step_in, nhum, rad_lat_2d, rad_l if(module_is_initialized) return -call write_version_number() +call write_version_number(version, tagname) #ifdef INTERNAL_FILE_NML read (input_nml_file, nml=idealized_moist_phys_nml, iostat=io) diff --git a/src/atmos_spectral/driver/solo/mixed_layer.F90 b/src/atmos_spectral/driver/solo/mixed_layer.F90 index 00c925e8ee..301f1509c5 100644 --- a/src/atmos_spectral/driver/solo/mixed_layer.F90 +++ b/src/atmos_spectral/driver/solo/mixed_layer.F90 @@ -131,7 +131,7 @@ subroutine mixed_layer_init(is, ie, js, je, num_levels, t_surf, axes, Time) if(module_is_initialized) return -call write_version_number() +call write_version_number(version, tagname) unit = open_namelist_file () ierr=1 diff --git a/src/atmos_spectral/init/ic_from_external_file.F90 b/src/atmos_spectral/init/ic_from_external_file.F90 index 5bee90a46f..72e83b3a0f 100644 --- a/src/atmos_spectral/init/ic_from_external_file.F90 +++ b/src/atmos_spectral/init/ic_from_external_file.F90 @@ -72,7 +72,7 @@ subroutine ic_from_external_file(triang_trunc, tracer_attributes, vors, divs, ts enddo 20 call close_file (unit) #endif -call write_version_number() +call write_version_number(version, tagname) write (stdlog(), nml=ic_from_external_file_nml) !------------------------------------------------------------------------------- diff --git a/src/atmos_spectral/init/jablonowski_2006.F90 b/src/atmos_spectral/init/jablonowski_2006.F90 index bae9eb8119..51d3a16854 100644 --- a/src/atmos_spectral/init/jablonowski_2006.F90 +++ b/src/atmos_spectral/init/jablonowski_2006.F90 @@ -107,7 +107,7 @@ subroutine jablonowski_2006(sea_level_press, triang_trunc, & enddo 20 call close_file (unit) #endif -call write_version_number() +call write_version_number(version, tagname) if(mpp_pe() == mpp_root_pe()) write (stdlog(), nml=jablonowski_2006_nml) call compute_vert_coord(vert_coord_option, scale_heights, surf_res, exponent, p_press, p_sigma, sea_level_press, pk, bk) diff --git a/src/atmos_spectral/init/polvani_2004.F90 b/src/atmos_spectral/init/polvani_2004.F90 index 45028a9e57..66f4b08478 100644 --- a/src/atmos_spectral/init/polvani_2004.F90 +++ b/src/atmos_spectral/init/polvani_2004.F90 @@ -102,7 +102,7 @@ subroutine polvani_2004(sea_level_press, triang_trunc, vert_difference_option, p ierr = check_nml_error (io, 'polvani_2004_nml') enddo 20 call close_file (unit) -call write_version_number() +call write_version_number(version, tagname) if(mpp_pe() == mpp_root_pe()) write (stdlog(), nml=polvani_2004_nml) T_standard(1) = 288.15 diff --git a/src/atmos_spectral/init/polvani_2007.F90 b/src/atmos_spectral/init/polvani_2007.F90 index 3281890d50..6e30c7d52e 100644 --- a/src/atmos_spectral/init/polvani_2007.F90 +++ b/src/atmos_spectral/init/polvani_2007.F90 @@ -139,7 +139,7 @@ subroutine polvani_2007(triang_trunc, vert_difference_option, pk, bk, & enddo 20 call close_file (unit) #endif -call write_version_number() +call write_version_number(version, tagname) if(mpp_pe() == mpp_root_pe()) write (stdlog(), nml=polvani_2007_nml) call get_lon_max(lon_max) diff --git a/src/atmos_spectral/init/spectral_init_cond.F90 b/src/atmos_spectral/init/spectral_init_cond.F90 index 694c3a7718..9867f55359 100644 --- a/src/atmos_spectral/init/spectral_init_cond.F90 +++ b/src/atmos_spectral/init/spectral_init_cond.F90 @@ -92,7 +92,7 @@ subroutine spectral_init_cond(initial_state_option, tracer_attributes, reference enddo 20 call close_file (unit) #endif -call write_version_number() +call write_version_number(version, tagname) if(mpp_pe() == mpp_root_pe()) write (stdlog(), nml=spectral_init_cond_nml) call compute_vert_coord(vert_coord_option, scale_heights, surf_res, exponent, p_press, p_sigma, reference_sea_level_press, pk,bk) diff --git a/src/atmos_spectral/init/spectral_initialize_fields.F90 b/src/atmos_spectral/init/spectral_initialize_fields.F90 index 3c4f56bbd6..3809af95f0 100644 --- a/src/atmos_spectral/init/spectral_initialize_fields.F90 +++ b/src/atmos_spectral/init/spectral_initialize_fields.F90 @@ -42,7 +42,7 @@ subroutine spectral_initialize_fields(reference_sea_level_press, triang_trunc, i integer :: ms, me, ns, ne, is, ie, js, je, num_levels -call write_version_number() +call write_version_number(version, tagname) num_levels = size(ug,3) call get_grid_domain(is, ie, js, je) diff --git a/src/atmos_spectral/init/topog_regularization.F90 b/src/atmos_spectral/init/topog_regularization.F90 index ed3db70e1c..635f887c0a 100644 --- a/src/atmos_spectral/init/topog_regularization.F90 +++ b/src/atmos_spectral/init/topog_regularization.F90 @@ -275,7 +275,7 @@ subroutine topog_regularization_init(ocean_mask) real, allocatable, dimension(:,:,:) :: legendre_global, legendre logical, allocatable, dimension(:,:) :: ocean_mask_global -call write_version_number() +call write_version_number(version, tagname) if(.not.transforms_are_initialized()) then call error_mesg('topog_regularization_init','Transforms are not initialized',FATAL) diff --git a/src/atmos_spectral/init/vert_coordinate.F90 b/src/atmos_spectral/init/vert_coordinate.F90 index d8f90c637a..ca939ad411 100644 --- a/src/atmos_spectral/init/vert_coordinate.F90 +++ b/src/atmos_spectral/init/vert_coordinate.F90 @@ -77,7 +77,7 @@ subroutine compute_vert_coord (vert_coord_option, scale_heights, surf_res, expon character(len=16) :: chtmp2, chtmp3 if(.not.entry_to_logfile_done) then - call write_version_number() + call write_version_number(version, tagname) entry_to_logfile_done = .true. endif diff --git a/src/atmos_spectral/model/every_step_diagnostics.F90 b/src/atmos_spectral/model/every_step_diagnostics.F90 index e81236c94f..98c058fc5f 100644 --- a/src/atmos_spectral/model/every_step_diagnostics.F90 +++ b/src/atmos_spectral/model/every_step_diagnostics.F90 @@ -67,7 +67,7 @@ subroutine every_step_diagnostics_init(Time, lon_max, lat_max, num_levels_in, re real, dimension(num_levels_in) :: p_full, ln_p_full real, dimension(num_levels_in+1) :: p_half, ln_p_half -call write_version_number() +call write_version_number(version, tagname) call get_deg_lon(lon) call get_deg_lat(lat) diff --git a/src/atmos_spectral/model/fv_advection.F90 b/src/atmos_spectral/model/fv_advection.F90 index 1bc3e95997..bbad7cf530 100644 --- a/src/atmos_spectral/model/fv_advection.F90 +++ b/src/atmos_spectral/model/fv_advection.F90 @@ -47,7 +47,7 @@ subroutine fv_advection_init(nx_in, ny_in, yy_in, degrees_lon, advection_layout) if (module_is_initialized) return - call write_version_number() + call write_version_number(version, tagname) pe = mpp_pe() npes = mpp_npes() diff --git a/src/atmos_spectral/model/global_integral.F90 b/src/atmos_spectral/model/global_integral.F90 index 5cb3254f09..5a27af3fee 100644 --- a/src/atmos_spectral/model/global_integral.F90 +++ b/src/atmos_spectral/model/global_integral.F90 @@ -42,7 +42,7 @@ function mass_weighted_global_integral(field, surf_press) integer :: k, num_levels if(.not.entry_to_logfile_done) then - call write_version_number() + call write_version_number(version, tagname) entry_to_logfile_done=.true. endif diff --git a/src/atmos_spectral/model/implicit.F90 b/src/atmos_spectral/model/implicit.F90 index e9e046d478..0723dd7c36 100644 --- a/src/atmos_spectral/model/implicit.F90 +++ b/src/atmos_spectral/model/implicit.F90 @@ -75,7 +75,7 @@ subroutine implicit_init(pk_in, bk_in, ref_temperature_implicit_in, ref_surf_p_i if(module_is_initialized) return -call write_version_number() +call write_version_number(version, tagname) call get_spec_domain(ms, me, ns, ne) diff --git a/src/atmos_spectral/model/leapfrog.F90 b/src/atmos_spectral/model/leapfrog.F90 index 6b97ff6c22..af41595e5d 100644 --- a/src/atmos_spectral/model/leapfrog.F90 +++ b/src/atmos_spectral/model/leapfrog.F90 @@ -42,7 +42,7 @@ subroutine leapfrog_2level_A_3d_complex (a, dt_a, previous, current, future, del real, intent(in) :: delta_t, robert_coeff if(.not.entry_to_logfile_done) then - call write_version_number() + call write_version_number(version, tagname) entry_to_logfile_done = .true. endif @@ -66,7 +66,7 @@ subroutine leapfrog_2level_B_3d_complex (a, current, future, robert_coeff) real, intent(in) :: robert_coeff if(.not.entry_to_logfile_done) then - call write_version_number() + call write_version_number(version, tagname) entry_to_logfile_done = .true. endif @@ -85,7 +85,7 @@ subroutine leapfrog_2level_A_3d_real (a, dt_a, previous, current, future, delta_ real, intent(in) :: delta_t, robert_coeff if(.not.entry_to_logfile_done) then - call write_version_number() + call write_version_number(version, tagname) entry_to_logfile_done = .true. endif @@ -109,7 +109,7 @@ subroutine leapfrog_2level_B_3d_real (a, current, future, robert_coeff) real, intent(in) :: robert_coeff if(.not.entry_to_logfile_done) then - call write_version_number() + call write_version_number(version, tagname) entry_to_logfile_done = .true. endif @@ -132,7 +132,7 @@ subroutine leapfrog_2level_A_2d_complex (a, dt_a, previous, current, future, del complex, dimension(size(a,1),size(a,2),1) :: dt_a_3d if(.not.entry_to_logfile_done) then - call write_version_number() + call write_version_number(version, tagname) entry_to_logfile_done = .true. endif @@ -152,7 +152,7 @@ subroutine leapfrog_2level_B_2d_complex (a, current, future, robert_coeff) complex, dimension(size(a,1),size(a,2),1,size(a,3)) :: a_3d if(.not.entry_to_logfile_done) then - call write_version_number() + call write_version_number(version, tagname) entry_to_logfile_done = .true. endif @@ -171,7 +171,7 @@ subroutine leapfrog_3d_complex(a, dt_a, previous, current, future, delta_t, robe real, intent(in) :: delta_t, robert_coeff if(.not.entry_to_logfile_done) then - call write_version_number() + call write_version_number(version, tagname) entry_to_logfile_done = .true. endif @@ -200,7 +200,7 @@ subroutine leapfrog_2d_complex(a, dt_a, previous, current, future, delta_t, robe complex, dimension(size(a,1),size(a,2),1) :: dt_a_3d if(.not.entry_to_logfile_done) then - call write_version_number() + call write_version_number(version, tagname) entry_to_logfile_done = .true. endif diff --git a/src/atmos_spectral/model/matrix_invert.F90 b/src/atmos_spectral/model/matrix_invert.F90 index 3ddd5bdcb6..6380c0e06e 100644 --- a/src/atmos_spectral/model/matrix_invert.F90 +++ b/src/atmos_spectral/model/matrix_invert.F90 @@ -52,7 +52,7 @@ subroutine invert(matrix, det) ! PROGRAM IS TERMINATED. if(.not.entry_to_logfile_done) then - call write_version_number() + call write_version_number(version, tagname) entry_to_logfile_done = .true. endif diff --git a/src/atmos_spectral/model/press_and_geopot.F90 b/src/atmos_spectral/model/press_and_geopot.F90 index c1fbe43b33..c1ab36748c 100644 --- a/src/atmos_spectral/model/press_and_geopot.F90 +++ b/src/atmos_spectral/model/press_and_geopot.F90 @@ -71,7 +71,7 @@ subroutine press_and_geopot_init(pk_in, bk_in, use_virtual_temperature_in, vert_ if(module_is_initialized) return -call write_version_number() +call write_version_number(version, tagname) num_levels = size(pk_in,1)-1 diff --git a/src/atmos_spectral/model/spectral_damping.F90 b/src/atmos_spectral/model/spectral_damping.F90 index de67d5d402..812e322e5a 100644 --- a/src/atmos_spectral/model/spectral_damping.F90 +++ b/src/atmos_spectral/model/spectral_damping.F90 @@ -45,7 +45,7 @@ subroutine spectral_damping_init (damping_coeff, damping_order, damping_option, if(module_is_initialized) return -call write_version_number() +call write_version_number(version, tagname) num_levels = num_levels_in call get_spec_domain(ms, me, ns, ne) diff --git a/src/atmos_spectral/model/spectral_dynamics.F90 b/src/atmos_spectral/model/spectral_dynamics.F90 index de958e0ae8..826c48d2b9 100644 --- a/src/atmos_spectral/model/spectral_dynamics.F90 +++ b/src/atmos_spectral/model/spectral_dynamics.F90 @@ -227,9 +227,9 @@ subroutine spectral_dynamics_init(Time, Time_step_in, tracer_attributes, dry_mod 20 call close_file (unit) #endif -call write_version_number() +call write_version_number(version, tagname) if(mpp_pe() == mpp_root_pe()) write (stdlog(), nml=spectral_dynamics_nml) -call write_version_number() +call write_version_number(version, tagname) Time_step = Time_step_in Alarm_interval = set_time(print_interval(2), print_interval(1)) diff --git a/src/atmos_spectral/model/water_borrowing.F90 b/src/atmos_spectral/model/water_borrowing.F90 index b3dcb10b6b..ff0b12f2a1 100644 --- a/src/atmos_spectral/model/water_borrowing.F90 +++ b/src/atmos_spectral/model/water_borrowing.F90 @@ -28,7 +28,7 @@ subroutine water_borrowing(dt_qg, qg, current, p_half, delta_t) real, dimension(size(qg,1), size(qg,3)) :: dp if(.not.entry_to_logfile_done) then - call write_version_number() + call write_version_number(version, tagname) entry_to_logfile_done = .true. endif diff --git a/src/atmos_spectral/tools/gauss_and_legendre.F90 b/src/atmos_spectral/tools/gauss_and_legendre.F90 index 557b624ed9..db155d2c7b 100644 --- a/src/atmos_spectral/tools/gauss_and_legendre.F90 +++ b/src/atmos_spectral/tools/gauss_and_legendre.F90 @@ -38,7 +38,7 @@ subroutine compute_legendre(legendre, num_fourier, fourier_inc, & real, dimension(n_lat) :: cos_lat if(.not.entry_to_logfile_done) then - call write_version_number() + call write_version_number(version, tagname) entry_to_logfile_done=.true. endif @@ -105,7 +105,7 @@ subroutine compute_gaussian(sin_hem, wts_hem, n_hem) real :: pp, p1, p2, p3, z, z1 if(.not.entry_to_logfile_done) then - call write_version_number() + call write_version_number(version, tagname) entry_to_logfile_done=.true. endif diff --git a/src/atmos_spectral/tools/grid_fourier.F90 b/src/atmos_spectral/tools/grid_fourier.F90 index 10c3c5d7ad..113bae9b41 100644 --- a/src/atmos_spectral/tools/grid_fourier.F90 +++ b/src/atmos_spectral/tools/grid_fourier.F90 @@ -56,7 +56,7 @@ subroutine grid_fourier_init(num_lon_in, fourier_inc, check, longitude_origin) if(module_is_initialized) return -call write_version_number() +call write_version_number(version, tagname) if(mod(num_lon_in,2) .ne. 0) then write(chtmp,'(i8)') num_lon_in diff --git a/src/atmos_spectral/tools/spec_mpp.F90 b/src/atmos_spectral/tools/spec_mpp.F90 index 28698a0a0f..feaed9acfe 100644 --- a/src/atmos_spectral/tools/spec_mpp.F90 +++ b/src/atmos_spectral/tools/spec_mpp.F90 @@ -35,7 +35,7 @@ subroutine spec_mpp_init( num_fourier, num_spherical, num_lon, lat_max, grid_lay pe = mpp_pe() npes = mpp_npes() - call write_version_number() + call write_version_number(version, tagname) !grid domain: by default, 1D decomposition along Y layout = (/1,npes/) diff --git a/src/atmos_spectral/tools/spherical.F90 b/src/atmos_spectral/tools/spherical.F90 index c639969f72..caeebb889a 100644 --- a/src/atmos_spectral/tools/spherical.F90 +++ b/src/atmos_spectral/tools/spherical.F90 @@ -120,7 +120,7 @@ subroutine spherical_init(radius, num_fourier_in, fourier_inc_in, num_spherical_ if(module_is_initialized) return -call write_version_number() +call write_version_number(version, tagname) call get_spec_domain(ms, me, ns, ne) diff --git a/src/atmos_spectral/tools/spherical_fourier.F90 b/src/atmos_spectral/tools/spherical_fourier.F90 index 0b8d153f56..4c0506bbdb 100644 --- a/src/atmos_spectral/tools/spherical_fourier.F90 +++ b/src/atmos_spectral/tools/spherical_fourier.F90 @@ -116,7 +116,7 @@ subroutine spherical_fourier_init(radius, lat_max_in, num_fourier_in, & integer, intent(in) :: num_spherical_in logical, intent(in), optional :: south_to_north -call write_version_number() +call write_version_number(version, tagname) if(present(south_to_north)) then south_to_north_local = south_to_north diff --git a/src/atmos_spectral/tools/transforms.F90 b/src/atmos_spectral/tools/transforms.F90 index 3b7b74d65a..68defe3786 100644 --- a/src/atmos_spectral/tools/transforms.F90 +++ b/src/atmos_spectral/tools/transforms.F90 @@ -235,7 +235,7 @@ subroutine transforms_init(radius, & 20 call close_file (namelist_unit) #endif -call write_version_number() +call write_version_number(version, tagname) if(mpp_pe() == mpp_root_pe()) write (stdlog(), nml=transforms_nml) npes = mpp_npes() diff --git a/src/coupler/coupler_main.F90 b/src/coupler/coupler_main.F90 index bad5368bdf..71a4bfa27c 100644 --- a/src/coupler/coupler_main.F90 +++ b/src/coupler/coupler_main.F90 @@ -229,7 +229,7 @@ program coupler_main !----------------------------------------------------------------------- character(len=128) :: version = '$Id: coupler_main.F90,v 20.0 2013/12/13 23:27:07 fms Exp $' - character(len=128) :: tag = '$Name: tikal $' + character(len=128) :: tagname = '$Name: tikal $' !----------------------------------------------------------------------- !---- model defined-types ---- @@ -830,7 +830,7 @@ subroutine coupler_init endif !----- write version to logfile ------- - call write_version_number() + call write_version_number(version, tagname) !----- read namelist ------- diff --git a/src/coupler/flux_exchange.F90 b/src/coupler/flux_exchange.F90 index cc4ad638a4..1153c5ec92 100644 --- a/src/coupler/flux_exchange.F90 +++ b/src/coupler/flux_exchange.F90 @@ -262,7 +262,7 @@ module flux_exchange_mod !----------------------------------------------------------------------- character(len=128) :: version = '$Id: flux_exchange.F90,v 20.0 2013/12/13 23:27:41 fms Exp $' - character(len=128) :: tag = '$Name: tikal $' + character(len=128) :: tagname = '$Name: tikal $' !----------------------------------------------------------------------- !---- exchange grid maps ----- @@ -588,7 +588,7 @@ subroutine flux_exchange_init ( Time, Atm, Land, Ice, Ocean, Ocean_state,& #endif !----- write namelist to logfile ----- - call write_version_number() + call write_version_number(version, tagname) if( mpp_pe() == mpp_root_pe() )write( logunit, nml=flux_exchange_nml ) !----- find out number of atmospheric prognostic tracers and index of specific diff --git a/src/coupler/surface_flux.F90 b/src/coupler/surface_flux.F90 index 8f5e49a57c..1f78542001 100644 --- a/src/coupler/surface_flux.F90 +++ b/src/coupler/surface_flux.F90 @@ -736,7 +736,7 @@ subroutine surface_flux_init #endif ! write version number - call write_version_number() + call write_version_number(version, tagname) unit = stdlog() if ( mpp_pe() == mpp_root_pe() ) write (unit, nml=surface_flux_nml) diff --git a/src/ice_param/ice_albedo.F90 b/src/ice_param/ice_albedo.F90 index 1eb6d414c1..d0a457ba94 100644 --- a/src/ice_param/ice_albedo.F90 +++ b/src/ice_param/ice_albedo.F90 @@ -90,7 +90,7 @@ subroutine ice_albedo_init ( t_freeze ) !------- write version number and namelist --------- if ( mpp_pe() == mpp_root_pe() ) then - call write_version_number() + call write_version_number(version, tagname) unit = stdlog() write (unit, nml=ice_albedo_nml) endif diff --git a/src/ice_param/ocean_albedo.F90 b/src/ice_param/ocean_albedo.F90 index cda23782b0..6d1b059bae 100644 --- a/src/ice_param/ocean_albedo.F90 +++ b/src/ice_param/ocean_albedo.F90 @@ -423,7 +423,7 @@ subroutine ocean_albedo_init(ocean,lat) !------- write version number and namelist --------- if ( mpp_pe() == mpp_root_pe() ) then - call write_version_number() + call write_version_number(version, tagname) unit = stdlog() write (unit, nml=ocean_albedo_nml) endif diff --git a/src/ice_param/ocean_rough.F90 b/src/ice_param/ocean_rough.F90 index 225a325aef..4c7e3c16b1 100644 --- a/src/ice_param/ocean_rough.F90 +++ b/src/ice_param/ocean_rough.F90 @@ -229,7 +229,7 @@ subroutine ocean_rough_init !------- write version number and namelist --------- if ( mpp_pe() == mpp_root_pe() ) then - call write_version_number() + call write_version_number(version, tagname) unit = stdlog() write (unit,nml=ocean_rough_nml) write (unit,11) diff --git a/src/ice_sis/ice_bergs.F90 b/src/ice_sis/ice_bergs.F90 index 05365d811b..ccca225966 100644 --- a/src/ice_sis/ice_bergs.F90 +++ b/src/ice_sis/ice_bergs.F90 @@ -2579,7 +2579,7 @@ subroutine icebergs_init(bergs, & if (really_debug) debug=.true. ! One implies the other... ! Log version and parameters - call write_version_number() + call write_version_number(version, tagname) write (stdlogunit, icebergs_nml) if( reproduce_siena ) then diff --git a/src/ice_sis/ice_spec.F90 b/src/ice_sis/ice_spec.F90 index 78adccb53d..4dc2718e9f 100644 --- a/src/ice_sis/ice_spec.F90 +++ b/src/ice_sis/ice_spec.F90 @@ -73,7 +73,7 @@ subroutine get_sea_surface(Time, ts, cn, iceh) write (stdoutunit, ice_spec_nml) write (stdlogunit, ice_spec_nml) - call write_version_number() + call write_version_number(version, tagname) module_is_initialized = .true. endif diff --git a/src/ice_sis/ice_type.F90 b/src/ice_sis/ice_type.F90 index 340ffc74a4..3e0005e79d 100644 --- a/src/ice_sis/ice_type.F90 +++ b/src/ice_sis/ice_type.F90 @@ -373,7 +373,7 @@ subroutine ice_model_init (Ice, Time_Init, Time, Time_step_fast, Time_step_slow write (stdoutunit, ice_model_nml) write (stdlogunit, ice_model_nml) - call write_version_number() + call write_version_number(version, tagname) if (spec_ice) then slab_ice = .true. diff --git a/src/land_lad/land_model.F90 b/src/land_lad/land_model.F90 index 72b0abf737..963bad8d96 100644 --- a/src/land_lad/land_model.F90 +++ b/src/land_lad/land_model.F90 @@ -512,7 +512,7 @@ subroutine init_land_with_area & module_is_initialized = .TRUE. ! write the version and tagname to the logfile - call write_version_number() + call write_version_number(version, tagname) ! initialize tracers #ifdef LAND_BND_TRACERS diff --git a/src/land_lad/land_types.F90 b/src/land_lad/land_types.F90 index adb6adc2db..3406ba824e 100644 --- a/src/land_lad/land_types.F90 +++ b/src/land_lad/land_types.F90 @@ -334,7 +334,7 @@ subroutine land_types_init() ! module_is_initialized =.TRUE. - call write_version_number() + call write_version_number(version, tagname) end subroutine land_types_init ! diff --git a/src/land_lad/numerics.F90 b/src/land_lad/numerics.F90 index 91a86acdc1..ebfd62b9d5 100644 --- a/src/land_lad/numerics.F90 +++ b/src/land_lad/numerics.F90 @@ -85,7 +85,7 @@ module numerics_mod subroutine numerics_init() module_is_initialized =.TRUE. - call write_version_number() + call write_version_number(version, tagname) end subroutine numerics_init ! diff --git a/src/land_lad/soil/land_properties.F90 b/src/land_lad/soil/land_properties.F90 index 546190dba7..b02076244f 100644 --- a/src/land_lad/soil/land_properties.F90 +++ b/src/land_lad/soil/land_properties.F90 @@ -706,7 +706,7 @@ subroutine land_properties_init ( & call close_file (unit) endif - call write_version_number() + call write_version_number(version, tagname) if (mpp_pe() == mpp_root_pe()) then unit = stdlog() write (unit, nml=land_properties_nml) diff --git a/src/land_lad/soil/rivers.F90 b/src/land_lad/soil/rivers.F90 index e9d1ca0f42..5e8027a5cc 100644 --- a/src/land_lad/soil/rivers.F90 +++ b/src/land_lad/soil/rivers.F90 @@ -297,7 +297,7 @@ subroutine rivers_init & ! write the namelist to the log file if ( mpp_pe() == mpp_root_pe() ) then - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() write (logunit, nml=rivers_nml) endif diff --git a/src/land_lad/soil/soil.F90 b/src/land_lad/soil/soil.F90 index 2d2109e69c..74c64a1625 100644 --- a/src/land_lad/soil/soil.F90 +++ b/src/land_lad/soil/soil.F90 @@ -629,7 +629,7 @@ subroutine soil_init ( soil, gblon, gblat, garea, gfrac, & call get_restart_io_mode(do_netcdf_restart) ! write version information to a log file - call write_version_number() + call write_version_number(version, tagname) ! write the namelist to a log file if( mpp_pe()==0 ) then diff --git a/src/land_lad/vegetation/vegetation.F90 b/src/land_lad/vegetation/vegetation.F90 index 6586a4c5ab..133ec04441 100644 --- a/src/land_lad/vegetation/vegetation.F90 +++ b/src/land_lad/vegetation/vegetation.F90 @@ -247,7 +247,7 @@ subroutine vegetation_init & call get_restart_io_mode(do_netcdf_restart) ! write version and tag information to logfile - call write_version_number() + call write_version_number(version, tagname) ! write the namelist to a log file if( mpp_pe()==0 ) then unit = stdlog( ) diff --git a/src/land_lad2/canopy_air/canopy_air.F90 b/src/land_lad2/canopy_air/canopy_air.F90 index ffbd399e5d..e79a93fc25 100644 --- a/src/land_lad2/canopy_air/canopy_air.F90 +++ b/src/land_lad2/canopy_air/canopy_air.F90 @@ -96,7 +96,7 @@ subroutine read_cana_namelist() integer :: io ! i/o status for the namelist integer :: ierr ! error code, returned by i/o routines - call write_version_number() + call write_version_number(version, tagname) #ifdef INTERNAL_FILE_NML read (input_nml_file, nml=cana_nml, iostat=io) ierr = check_nml_error(io, 'cana_nml') diff --git a/src/land_lad2/glacier/glac_tile.F90 b/src/land_lad2/glacier/glac_tile.F90 index e7ba6b6018..b616c863aa 100644 --- a/src/land_lad2/glacier/glac_tile.F90 +++ b/src/land_lad2/glacier/glac_tile.F90 @@ -229,7 +229,7 @@ subroutine read_glac_data_namelist(glac_n_lev, glac_dz) integer :: i real :: z - call write_version_number() + call write_version_number(version, tagname) #ifdef INTERNAL_FILE_NML read (input_nml_file, nml=glac_data_nml, iostat=io) ierr = check_nml_error(io, 'glac_data_nml') diff --git a/src/land_lad2/glacier/glacier.F90 b/src/land_lad2/glacier/glacier.F90 index 2e3229d9e4..611cc9e255 100644 --- a/src/land_lad2/glacier/glacier.F90 +++ b/src/land_lad2/glacier/glacier.F90 @@ -101,7 +101,7 @@ subroutine read_glac_namelist() call read_glac_data_namelist(num_l, dz) - call write_version_number() + call write_version_number(version, tagname) #ifdef INTERNAL_FILE_NML read (input_nml_file, nml=glac_nml, iostat=io) ierr = check_nml_error(io, 'glac_nml') diff --git a/src/land_lad2/lake/lake.F90 b/src/land_lad2/lake/lake.F90 index 0fda4bfac5..4f632e4c33 100644 --- a/src/land_lad2/lake/lake.F90 +++ b/src/land_lad2/lake/lake.F90 @@ -132,7 +132,7 @@ subroutine read_lake_namelist() call read_lake_data_namelist(num_l) - call write_version_number() + call write_version_number(version, tagname) #ifdef INTERNAL_FILE_NML read (input_nml_file, nml=lake_nml, iostat=io) ierr = check_nml_error(io, 'lake_nml') diff --git a/src/land_lad2/lake/lake_tile.F90 b/src/land_lad2/lake/lake_tile.F90 index c2760dba07..3604109954 100644 --- a/src/land_lad2/lake/lake_tile.F90 +++ b/src/land_lad2/lake/lake_tile.F90 @@ -257,7 +257,7 @@ subroutine read_lake_data_namelist(lake_n_lev) integer :: i real :: z - call write_version_number() + call write_version_number(version, tagname) #ifdef INTERNAL_FILE_NML read (input_nml_file, nml=lake_data_nml, iostat=io) ierr = check_nml_error(io, 'lake_data_nml') diff --git a/src/land_lad2/land_data.F90 b/src/land_lad2/land_data.F90 index 94bbdb70be..d86c387f0a 100644 --- a/src/land_lad2/land_data.F90 +++ b/src/land_lad2/land_data.F90 @@ -207,7 +207,7 @@ subroutine land_data_init(layout, io_layout, time, dt_fast, dt_slow) integer :: io_id(1) ! write the version and tag name to the logfile - call write_version_number() + call write_version_number(version, tagname) ! define the processor layout information according to the global grid size call get_grid_ntiles('LND',ntiles) diff --git a/src/land_lad2/land_model.F90 b/src/land_lad2/land_model.F90 index f041c2143e..c08dd7b4f7 100644 --- a/src/land_lad2/land_model.F90 +++ b/src/land_lad2/land_model.F90 @@ -271,7 +271,7 @@ subroutine land_model_init & module_is_initialized = .TRUE. ! [1] print out version number - call write_version_number() + call write_version_number(version, tagname) ! initialize land model clocks landClock = mpp_clock_id('Land' ,CLOCK_FLAG_DEFAULT,CLOCK_COMPONENT) diff --git a/src/land_lad2/river/river.F90 b/src/land_lad2/river/river.F90 index fd95671ea6..d65009d192 100644 --- a/src/land_lad2/river/river.F90 +++ b/src/land_lad2/river/river.F90 @@ -221,7 +221,7 @@ subroutine river_init( land_lon, land_lat, time, dt_fast, land_domain, & #endif !--- write version and namelist info to logfile -------------------- - call write_version_number() + call write_version_number(version, tagname) unit=stdlog() write(unit, river_nml) diff --git a/src/land_lad2/river/river_physics.F90 b/src/land_lad2/river/river_physics.F90 index 82788b53fe..e1716ee81b 100644 --- a/src/land_lad2/river/river_physics.F90 +++ b/src/land_lad2/river/river_physics.F90 @@ -149,7 +149,7 @@ subroutine river_physics_init(River, domain, id_lon, id_lat ) #endif !--- write version and namelist info to logfile -------------------- - call write_version_number() + call write_version_number(version, tagname) unit=stdlog() write (unit, river_physics_nml) diff --git a/src/land_lad2/shared/land_debug.F90 b/src/land_lad2/shared/land_debug.F90 index 45b819c763..4cf1cb8d07 100644 --- a/src/land_lad2/shared/land_debug.F90 +++ b/src/land_lad2/shared/land_debug.F90 @@ -81,7 +81,7 @@ subroutine land_debug_init() integer :: unit, ierr, io, ntiles integer :: max_threads - call write_version_number() + call write_version_number(version, tagname) #ifdef INTERNAL_FILE_NML read (input_nml_file, nml=land_debug_nml, iostat=io) diff --git a/src/land_lad2/shared/land_io.F90 b/src/land_lad2/shared/land_io.F90 index b0511def5f..2fbdaed315 100644 --- a/src/land_lad2/shared/land_io.F90 +++ b/src/land_lad2/shared/land_io.F90 @@ -62,7 +62,7 @@ subroutine read_land_io_namelist() module_is_initialized = .TRUE. ! [1] print out version number - call write_version_number() + call write_version_number(version, tagname) #ifdef INTERNAL_FILE_NML read (input_nml_file, nml=land_io_nml, iostat=io) diff --git a/src/land_lad2/shared/land_numerics.F90 b/src/land_lad2/shared/land_numerics.F90 index ffeb19fe32..318d48b621 100644 --- a/src/land_lad2/shared/land_numerics.F90 +++ b/src/land_lad2/shared/land_numerics.F90 @@ -94,7 +94,7 @@ module land_numerics_mod subroutine numerics_init() module_is_initialized =.TRUE. - call write_version_number() + call write_version_number(version, tagname) end subroutine numerics_init diff --git a/src/land_lad2/shared/land_tile_diag.F90 b/src/land_lad2/shared/land_tile_diag.F90 index fe7d563eb5..95f063fd0e 100644 --- a/src/land_lad2/shared/land_tile_diag.F90 +++ b/src/land_lad2/shared/land_tile_diag.F90 @@ -98,7 +98,7 @@ subroutine tile_diag_init() if (module_is_initialized) return module_is_initialized = .true. - call write_version_number() + call write_version_number(version, tagname) ! initialize diag selectors call tile_selectors_init() diff --git a/src/land_lad2/snow/snow.F90 b/src/land_lad2/snow/snow.F90 index c33b0b402d..e3dd93f76d 100644 --- a/src/land_lad2/snow/snow.F90 +++ b/src/land_lad2/snow/snow.F90 @@ -105,7 +105,7 @@ subroutine read_snow_namelist() call read_snow_data_namelist(num_l,dz,mc_fict) - call write_version_number() + call write_version_number(version, tagname) #ifdef INTERNAL_FILE_NML read (input_nml_file, nml=snow_nml, iostat=io) ierr = check_nml_error(io, 'snow_nml') diff --git a/src/land_lad2/snow/snow_tile.F90 b/src/land_lad2/snow/snow_tile.F90 index fa76e0ce74..bd1f42ba1c 100644 --- a/src/land_lad2/snow/snow_tile.F90 +++ b/src/land_lad2/snow/snow_tile.F90 @@ -141,7 +141,7 @@ subroutine read_snow_data_namelist(snow_num_l, snow_dz, snow_mc_fict) integer :: io ! i/o status for the namelist integer :: ierr ! error code, returned by i/o routines - call write_version_number() + call write_version_number(version, tagname) #ifdef INTERNAL_FILE_NML read (input_nml_file, nml=snow_data_nml, iostat=io) ierr = check_nml_error(io, 'snow_data_nml') diff --git a/src/land_lad2/soil/soil.F90 b/src/land_lad2/soil/soil.F90 index bcd9051d45..00cbfe87fa 100644 --- a/src/land_lad2/soil/soil.F90 +++ b/src/land_lad2/soil/soil.F90 @@ -189,7 +189,7 @@ subroutine read_soil_namelist() call read_soil_data_namelist(num_l,dz,use_single_geo,gw_option) - call write_version_number() + call write_version_number(version, tagname) #ifdef INTERNAL_FILE_NML read (input_nml_file, nml=soil_nml, iostat=io) ierr = check_nml_error(io, 'soil_nml') diff --git a/src/land_lad2/soil/soil_tile.F90 b/src/land_lad2/soil/soil_tile.F90 index f1bb390024..ef069a9879 100644 --- a/src/land_lad2/soil/soil_tile.F90 +++ b/src/land_lad2/soil/soil_tile.F90 @@ -427,7 +427,7 @@ subroutine read_soil_data_namelist(soil_num_l, soil_dz, soil_single_geo, & integer :: i, rcode, ncid, varid, dimids(3) real :: z - call write_version_number() + call write_version_number(version, tagname) #ifdef INTERNAL_FILE_NML read (input_nml_file, nml=soil_data_nml, iostat=io) ierr = check_nml_error(io, 'soil_data_nml') diff --git a/src/land_lad2/soil/uptake.F90 b/src/land_lad2/soil/uptake.F90 index c1292bd3e0..f92dcd9105 100644 --- a/src/land_lad2/soil/uptake.F90 +++ b/src/land_lad2/soil/uptake.F90 @@ -51,7 +51,7 @@ subroutine uptake_init(num_l_in, dz_in, zfull_in) dz_in(:), & ! layer thickness zfull_in(:) ! layer centers - call write_version_number() + call write_version_number(version, tagname) module_is_initialized =.TRUE. num_l = num_l_in diff --git a/src/land_lad2/topo_rough/topo_rough.F90 b/src/land_lad2/topo_rough/topo_rough.F90 index 863b4491eb..4b4711ca10 100644 --- a/src/land_lad2/topo_rough/topo_rough.F90 +++ b/src/land_lad2/topo_rough/topo_rough.F90 @@ -105,7 +105,7 @@ subroutine topo_rough_init(time, lonb, latb, domain, id_lon,id_lat) logical :: used, got_stdev ! write the version and tagname to the logfile - call write_version_number() + call write_version_number(version, tagname) ! read and write (to logfile) namelist variables #ifdef INTERNAL_FILE_NML diff --git a/src/land_lad2/transitions/transitions.F90 b/src/land_lad2/transitions/transitions.F90 index 741a40ea29..9c89a32187 100644 --- a/src/land_lad2/transitions/transitions.F90 +++ b/src/land_lad2/transitions/transitions.F90 @@ -142,7 +142,7 @@ subroutine land_transitions_init(id_lon, id_lat) if(module_is_initialized) return call horiz_interp_init - call write_version_number() + call write_version_number(version, tagname) #ifdef INTERNAL_FILE_NML read (input_nml_file, nml=landuse_nml, iostat=io) diff --git a/src/land_lad2/vegetation/vegetation.F90 b/src/land_lad2/vegetation/vegetation.F90 index 2d19918979..9324469fac 100644 --- a/src/land_lad2/vegetation/vegetation.F90 +++ b/src/land_lad2/vegetation/vegetation.F90 @@ -184,7 +184,7 @@ subroutine read_vegn_namelist() call read_vegn_data_namelist() call read_static_vegn_namelist(use_static_veg) - call write_version_number() + call write_version_number(version, tagname) #ifdef INTERNAL_FILE_NML read (input_nml_file, nml=vegn_nml, iostat=io) ierr = check_nml_error(io, 'vegn_nml') diff --git a/src/land_lad2/vegetation/vegn_data.F90 b/src/land_lad2/vegetation/vegn_data.F90 index c4599d552a..0fe0e46436 100644 --- a/src/land_lad2/vegetation/vegn_data.F90 +++ b/src/land_lad2/vegetation/vegn_data.F90 @@ -442,7 +442,7 @@ subroutine read_vegn_data_namelist() type(table_printer_type) :: table - call write_version_number() + call write_version_number(version, tagname) #ifdef INTERNAL_FILE_NML read (input_nml_file, nml=vegn_data_nml, iostat=io) ierr = check_nml_error(io, 'vegn_data_nml') diff --git a/src/land_lad2/vegetation/vegn_dynamics.F90 b/src/land_lad2/vegetation/vegn_dynamics.F90 index 307e1f89ba..1050a8eafc 100644 --- a/src/land_lad2/vegetation/vegn_dynamics.F90 +++ b/src/land_lad2/vegetation/vegn_dynamics.F90 @@ -65,7 +65,7 @@ subroutine vegn_dynamics_init(id_lon, id_lat, time, delta_time, soil_decomp_opti real , intent(in) :: delta_time ! fast time step, s character(*) , intent(in) :: soil_decomp_option - call write_version_number() + call write_version_number(version, tagname) ! set up global variables dt_fast_yr = delta_time/seconds_per_year diff --git a/src/land_lad2/vegetation/vegn_harvesting.F90 b/src/land_lad2/vegetation/vegn_harvesting.F90 index d90ab2dbe5..a5893c32df 100644 --- a/src/land_lad2/vegetation/vegn_harvesting.F90 +++ b/src/land_lad2/vegetation/vegn_harvesting.F90 @@ -67,7 +67,7 @@ module vegn_harvesting_mod subroutine vegn_harvesting_init integer :: unit, ierr, io - call write_version_number() + call write_version_number(version, tagname) #ifdef INTERNAL_FILE_NML read (input_nml_file, nml=harvesting_nml, iostat=io) diff --git a/src/land_lad2/vegetation/vegn_photosynthesis.F90 b/src/land_lad2/vegetation/vegn_photosynthesis.F90 index 3d55aaf738..b6ad56e82d 100644 --- a/src/land_lad2/vegetation/vegn_photosynthesis.F90 +++ b/src/land_lad2/vegetation/vegn_photosynthesis.F90 @@ -40,7 +40,7 @@ module vegn_photosynthesis_mod subroutine vegn_photosynthesis_init(photosynthesis_to_use) character(*), intent(in) :: photosynthesis_to_use - call write_version_number() + call write_version_number(version, tagname) ! convert symbolic names of photosynthesis options into numeric IDs to ! speed up selection during run-time diff --git a/src/land_lad2/vegetation/vegn_radiation.F90 b/src/land_lad2/vegetation/vegn_radiation.F90 index cff75d3581..f66f69b421 100644 --- a/src/land_lad2/vegetation/vegn_radiation.F90 +++ b/src/land_lad2/vegetation/vegn_radiation.F90 @@ -58,7 +58,7 @@ subroutine vegn_radiation_init(rad_to_use,snow_rad_to_use) character(*), intent(in) :: rad_to_use character(*), intent(in) :: snow_rad_to_use - call write_version_number() + call write_version_number(version, tagname) ! convert symbolic names of radiation options into numeric IDs to ! speed up selection during run-time diff --git a/src/land_lad2/vegetation/vegn_static_override.F90 b/src/land_lad2/vegetation/vegn_static_override.F90 index 062094dc80..b1818ef668 100644 --- a/src/land_lad2/vegetation/vegn_static_override.F90 +++ b/src/land_lad2/vegetation/vegn_static_override.F90 @@ -95,7 +95,7 @@ subroutine read_static_vegn_namelist(static_veg_used) ! ---- local vars integer :: unit, ierr, io - call write_version_number() + call write_version_number(version, tagname) #ifdef INTERNAL_FILE_NML read (input_nml_file, nml=static_veg_nml, iostat=io) diff --git a/src/land_null/land_model.F90 b/src/land_null/land_model.F90 index 8a7a78c3ec..31aefeb43c 100644 --- a/src/land_null/land_model.F90 +++ b/src/land_null/land_model.F90 @@ -53,6 +53,7 @@ module land_model_mod ! This is the null version use fms_mod, only : write_version_number, error_mesg, FATAL, mpp_npes, stdout use fms_mod, only : open_namelist_file, check_nml_error, file_exist, close_file +use fms_mod, only : set_domain use fms_io_mod, only : parse_mask_table use grid_mod, only : get_grid_ntiles, get_grid_size, define_cube_mosaic @@ -197,7 +198,7 @@ subroutine land_model_init (cplr2land, land2cplr, time_init, time, dt_fast, dt_s stdoutunit = stdout() - call write_version_number() + call write_version_number(version, tagname) ! define the processor layout information according to the global grid size call get_grid_ntiles('LND',ntiles) @@ -231,6 +232,7 @@ subroutine land_model_init (cplr2land, land2cplr, time_init, time, dt_fast, dt_s endif land2cplr%domain = domain + call set_domain(domain) npes_per_tile = mpp_npes()/ntiles face = (mpp_pe()-mpp_root_pe())/npes_per_tile + 1 diff --git a/src/land_param/climap_albedo.F90 b/src/land_param/climap_albedo.F90 index cf4306f827..00d62bff15 100644 --- a/src/land_param/climap_albedo.F90 +++ b/src/land_param/climap_albedo.F90 @@ -52,7 +52,7 @@ subroutine get_climap_albedo (lonb, latb, n, albedo) !------- write version number and namelist --------- if ( mpp_pe() == mpp_root_pe() ) then - call write_version_number() + call write_version_number(version, tagname) endif do_init = .false. @@ -141,7 +141,7 @@ subroutine get_climap_albedo_mcm (lonb, latb, nlong, nlatg, is, ie, js, je, n, & !--- write version id to logfile --- if (do_init) then if (mpp_pe() == mpp_root_pe()) & - call write_version_number() + call write_version_number(version, tagname) do_init = .false. diff --git a/src/mom5/ocean_blobs/ocean_blob.F90 b/src/mom5/ocean_blobs/ocean_blob.F90 index c5d4f730e9..f76b519354 100644 --- a/src/mom5/ocean_blobs/ocean_blob.F90 +++ b/src/mom5/ocean_blobs/ocean_blob.F90 @@ -683,7 +683,7 @@ subroutine ocean_blob_init (Grid, Domain, Time, T_prog, Dens, Thickness, & '==>Note: Using the Lagrangian buoyancy blobs scheme.' Ocean_options%lagrangian_blobs = 'Did use Lagrangian blobs.' - call write_version_number() + call write_version_number(version, tagname) ! Register diagnostic fields allocate(id_tend_blob(num_prog_tracers)) diff --git a/src/mom5/ocean_core/ocean_advection_velocity.F90 b/src/mom5/ocean_core/ocean_advection_velocity.F90 index 611213adbc..b7f0ec8b58 100644 --- a/src/mom5/ocean_core/ocean_advection_velocity.F90 +++ b/src/mom5/ocean_core/ocean_advection_velocity.F90 @@ -292,7 +292,7 @@ subroutine ocean_advection_velocity_init(Grid, Domain, Time, Time_steps, Thickne '==>Error from ocean_advection_velocity_mod: module already initialized.') endif - call write_version_number() + call write_version_number(version, tagname) module_is_initialized = .TRUE. diff --git a/src/mom5/ocean_core/ocean_barotropic.F90 b/src/mom5/ocean_core/ocean_barotropic.F90 index ff9f9e5d44..97564b40ee 100644 --- a/src/mom5/ocean_core/ocean_barotropic.F90 +++ b/src/mom5/ocean_core/ocean_barotropic.F90 @@ -987,7 +987,7 @@ subroutine ocean_barotropic_init(Grid, Domain, Time, Time_steps, Ocean_options, if (diag_step == 0) diag_step = 1 - call write_version_number() + call write_version_number(version, tagname) #ifdef INTERNAL_FILE_NML read (input_nml_file, nml=ocean_barotropic_nml, iostat=io_status) diff --git a/src/mom5/ocean_core/ocean_bbc.F90 b/src/mom5/ocean_core/ocean_bbc.F90 index e645ba81fd..c9a50cca9c 100644 --- a/src/mom5/ocean_core/ocean_bbc.F90 +++ b/src/mom5/ocean_core/ocean_bbc.F90 @@ -274,7 +274,7 @@ subroutine ocean_bbc_init(Grid, Domain, Time, T_prog, Velocity, & module_is_initialized = .TRUE. -call write_version_number() +call write_version_number(version, tagname) #ifdef INTERNAL_FILE_NML read (input_nml_file, nml=ocean_bbc_nml, iostat=io_status) diff --git a/src/mom5/ocean_core/ocean_coriolis.F90 b/src/mom5/ocean_core/ocean_coriolis.F90 index 3cde775f20..59def2138c 100644 --- a/src/mom5/ocean_core/ocean_coriolis.F90 +++ b/src/mom5/ocean_core/ocean_coriolis.F90 @@ -148,7 +148,7 @@ subroutine ocean_coriolis_init(Grid, Domain, Time, Time_steps, Ocean_options, ho module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) ! provide for namelist over-ride of defaults #ifdef INTERNAL_FILE_NML diff --git a/src/mom5/ocean_core/ocean_density.F90 b/src/mom5/ocean_core/ocean_density.F90 index 1eed623d3e..e4c238c423 100644 --- a/src/mom5/ocean_core/ocean_density.F90 +++ b/src/mom5/ocean_core/ocean_density.F90 @@ -800,7 +800,7 @@ subroutine ocean_density_init (Grid, Domain, Time, Time_steps, Thickness, T_prog if (T_prog(n)%longname == 'Preformed Salinity') salt_variable = PREFORMED_SALT enddo - call write_version_number() + call write_version_number(version, tagname) ! provide for namelist override of defaults #ifdef INTERNAL_FILE_NML diff --git a/src/mom5/ocean_core/ocean_domains.F90 b/src/mom5/ocean_core/ocean_domains.F90 index 73a5e78b61..cc0d6e98aa 100644 --- a/src/mom5/ocean_core/ocean_domains.F90 +++ b/src/mom5/ocean_core/ocean_domains.F90 @@ -94,7 +94,7 @@ subroutine ocean_domain_init() integer :: ierr,ioun,io_status,stdoutunit,stdlogunit stdoutunit=stdout();stdlogunit=stdlog() - call write_version_number() + call write_version_number(version, tagname) ! provide for namelist over-ride #ifdef INTERNAL_FILE_NML diff --git a/src/mom5/ocean_core/ocean_grids.F90 b/src/mom5/ocean_core/ocean_grids.F90 index 476fc019f7..13552d07b7 100644 --- a/src/mom5/ocean_core/ocean_grids.F90 +++ b/src/mom5/ocean_core/ocean_grids.F90 @@ -201,7 +201,7 @@ subroutine ocean_grids_init(ver_coordinate, ver_coordinate_class, hor_grid, debu vert_coordinate_class = ver_coordinate_class horz_grid = hor_grid - call write_version_number() + call write_version_number(version, tagname) if (PRESENT(debug)) debug_this_module = debug diff --git a/src/mom5/ocean_core/ocean_model.F90 b/src/mom5/ocean_core/ocean_model.F90 index fbbd441a3a..cbc9db7af2 100644 --- a/src/mom5/ocean_core/ocean_model.F90 +++ b/src/mom5/ocean_core/ocean_model.F90 @@ -771,7 +771,7 @@ subroutine ocean_model_init(Ocean, Ocean_state, Time_init, Time_in) call mpp_clock_begin(id_init) - call write_version_number() + call write_version_number(version, tagname) write(stdoutunit,'(/54x,a/)') '======== STARTING MOM INITIALIZATION ========' diff --git a/src/mom5/ocean_core/ocean_obc.F90 b/src/mom5/ocean_core/ocean_obc.F90 index 3ac3ff8cc7..f7f10b90a8 100644 --- a/src/mom5/ocean_core/ocean_obc.F90 +++ b/src/mom5/ocean_core/ocean_obc.F90 @@ -732,7 +732,7 @@ subroutine ocean_obc_init(have_obc, Time_steps, Domain, Grid, Ocean_options, ver write (stdlogunit, ocean_obc_nml) !--- write out version information --------------------------------- - call write_version_number() + call write_version_number(version, tagname) !--- if there is no open boundary, just return Obc%nobc = nobc diff --git a/src/mom5/ocean_core/ocean_pressure.F90 b/src/mom5/ocean_core/ocean_pressure.F90 index a23dd56198..74a86092ec 100644 --- a/src/mom5/ocean_core/ocean_pressure.F90 +++ b/src/mom5/ocean_core/ocean_pressure.F90 @@ -198,7 +198,7 @@ subroutine ocean_pressure_init(Grid, Domain, Time, ver_coordinate, & module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) vert_coordinate = ver_coordinate vert_coordinate_class = ver_coordinate_class diff --git a/src/mom5/ocean_core/ocean_sbc.F90 b/src/mom5/ocean_core/ocean_sbc.F90 index 2718845626..21ba6d1c4b 100644 --- a/src/mom5/ocean_core/ocean_sbc.F90 +++ b/src/mom5/ocean_core/ocean_sbc.F90 @@ -896,7 +896,7 @@ subroutine ocean_sbc_init(Grid, Domain, Time, T_prog, T_diag, & dtime = dtime_t horz_grid = hor_grid - call write_version_number() + call write_version_number(version, tagname) #ifdef INTERNAL_FILE_NML read (input_nml_file, nml=ocean_sbc_nml, iostat=io_status) diff --git a/src/mom5/ocean_core/ocean_thickness.F90 b/src/mom5/ocean_core/ocean_thickness.F90 index 49a0882541..50539d3901 100644 --- a/src/mom5/ocean_core/ocean_thickness.F90 +++ b/src/mom5/ocean_core/ocean_thickness.F90 @@ -391,7 +391,7 @@ subroutine ocean_thickness_init (Time, Time_steps, Domain, Grid, Ext_mode, Thic module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) ! provide for namelist over-ride of defaults #ifdef INTERNAL_FILE_NML diff --git a/src/mom5/ocean_core/ocean_types.F90 b/src/mom5/ocean_core/ocean_types.F90 index 5215f138b4..c22b8f8fc4 100644 --- a/src/mom5/ocean_core/ocean_types.F90 +++ b/src/mom5/ocean_core/ocean_types.F90 @@ -1354,7 +1354,7 @@ subroutine ocean_types_init() endif module_is_initialized = .true. - call write_version_number() + call write_version_number(version, tagname) return diff --git a/src/mom5/ocean_core/ocean_velocity.F90 b/src/mom5/ocean_core/ocean_velocity.F90 index 1e0cb3b48f..6dc6445bc1 100644 --- a/src/mom5/ocean_core/ocean_velocity.F90 +++ b/src/mom5/ocean_core/ocean_velocity.F90 @@ -346,7 +346,7 @@ subroutine ocean_velocity_init (Grid, Domain, Time, Time_steps, Ocean_options, & module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) have_obc = obc tendency = Time_steps%tendency diff --git a/src/mom5/ocean_core/ocean_velocity_advect.F90 b/src/mom5/ocean_core/ocean_velocity_advect.F90 index d8cb74ffca..09a2ca37b3 100644 --- a/src/mom5/ocean_core/ocean_velocity_advect.F90 +++ b/src/mom5/ocean_core/ocean_velocity_advect.F90 @@ -175,7 +175,7 @@ subroutine ocean_velocity_advect_init(Grid, Domain, Time, obc, hor_grid, debug) module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) Grd => Grid Dom => Domain diff --git a/src/mom5/ocean_diag/ocean_adv_vel_diag.F90 b/src/mom5/ocean_diag/ocean_adv_vel_diag.F90 index e179190772..5893b17739 100644 --- a/src/mom5/ocean_diag/ocean_adv_vel_diag.F90 +++ b/src/mom5/ocean_diag/ocean_adv_vel_diag.F90 @@ -175,7 +175,7 @@ subroutine ocean_adv_vel_diag_init(Grid, Domain, Time, Time_steps, T_prog, Dens, module_is_initialized = .TRUE. -call write_version_number() +call write_version_number(version, tagname) #ifdef INTERNAL_FILE_NML read (input_nml_file, nml=ocean_adv_vel_diag_nml, iostat=io_status) diff --git a/src/mom5/ocean_diag/ocean_diagnostics.F90 b/src/mom5/ocean_diag/ocean_diagnostics.F90 index 70d327e345..53e771e4c4 100644 --- a/src/mom5/ocean_diag/ocean_diagnostics.F90 +++ b/src/mom5/ocean_diag/ocean_diagnostics.F90 @@ -88,7 +88,7 @@ subroutine ocean_diag_init(Grid, Domain, Time, Time_steps, Thickness, T_prog, T_ module_is_initialized = .TRUE. -call write_version_number() +call write_version_number(version, tagname) #ifndef MOM_STATIC_ARRAYS call get_local_indices(Domain, isd, ied, jsd, jed, isc, iec, jsc, jec) diff --git a/src/mom5/ocean_diag/ocean_tracer_diag.F90 b/src/mom5/ocean_diag/ocean_tracer_diag.F90 index 6f1ebf0eb5..adb875167c 100644 --- a/src/mom5/ocean_diag/ocean_tracer_diag.F90 +++ b/src/mom5/ocean_diag/ocean_tracer_diag.F90 @@ -427,7 +427,7 @@ subroutine ocean_tracer_diag_init(Grid, Domain, Time, Time_steps, Thickness, T_p num_prog_tracers = size(T_prog,1) num_diag_tracers = size(T_diag,1) - call write_version_number() + call write_version_number(version, tagname) #ifdef INTERNAL_FILE_NML read (input_nml_file, nml=ocean_tracer_diag_nml, iostat=io_status) diff --git a/src/mom5/ocean_diag/ocean_velocity_diag.F90 b/src/mom5/ocean_diag/ocean_velocity_diag.F90 index 299bd9245e..a3a9416033 100644 --- a/src/mom5/ocean_diag/ocean_velocity_diag.F90 +++ b/src/mom5/ocean_diag/ocean_velocity_diag.F90 @@ -258,7 +258,7 @@ subroutine ocean_velocity_diag_init(Grid, Domain, Time, Time_steps, hor_grid) module_is_initialized = .TRUE. -call write_version_number() +call write_version_number(version, tagname) #ifdef INTERNAL_FILE_NML read (input_nml_file, nml=ocean_velocity_diag_nml, iostat=io_status) diff --git a/src/mom5/ocean_param/lateral/ocean_bih_friction.F90 b/src/mom5/ocean_param/lateral/ocean_bih_friction.F90 index ff37d6ea62..c703c8ed76 100644 --- a/src/mom5/ocean_param/lateral/ocean_bih_friction.F90 +++ b/src/mom5/ocean_param/lateral/ocean_bih_friction.F90 @@ -160,7 +160,7 @@ subroutine ocean_bih_friction_init(Grid, Domain, Time, Ocean_options, dtime, obc module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) ! provide for namelist over-ride of defaults #ifdef INTERNAL_FILE_NML diff --git a/src/mom5/ocean_param/lateral/ocean_bih_tracer.F90 b/src/mom5/ocean_param/lateral/ocean_bih_tracer.F90 index de407e1451..9cf1819571 100644 --- a/src/mom5/ocean_param/lateral/ocean_bih_tracer.F90 +++ b/src/mom5/ocean_param/lateral/ocean_bih_tracer.F90 @@ -170,7 +170,7 @@ subroutine ocean_bih_tracer_init(Grid, Domain, Time, T_prog, Ocean_options, dtim num_prog_tracers = size(T_prog(:)) - call write_version_number() + call write_version_number(version, tagname) ! provide for namelist over-ride of defaults #ifdef INTERNAL_FILE_NML diff --git a/src/mom5/ocean_param/lateral/ocean_bihcgrid_friction.F90 b/src/mom5/ocean_param/lateral/ocean_bihcgrid_friction.F90 index bac0f43579..0e322eff38 100644 --- a/src/mom5/ocean_param/lateral/ocean_bihcgrid_friction.F90 +++ b/src/mom5/ocean_param/lateral/ocean_bihcgrid_friction.F90 @@ -398,7 +398,7 @@ subroutine ocean_bihcgrid_friction_init(Grid, Domain, Time, Ocean_options, d_tim module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) ! provide for namelist over-ride of defaults diff --git a/src/mom5/ocean_param/lateral/ocean_bihcst_friction.F90 b/src/mom5/ocean_param/lateral/ocean_bihcst_friction.F90 index 35236d5535..4b1bc7a8c6 100644 --- a/src/mom5/ocean_param/lateral/ocean_bihcst_friction.F90 +++ b/src/mom5/ocean_param/lateral/ocean_bihcst_friction.F90 @@ -192,7 +192,7 @@ subroutine ocean_bihcst_friction_init(Grid, Domain, Time, Ocean_options, d_time, module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) ! provide for namelist over-ride of defaults #ifdef INTERNAL_FILE_NML diff --git a/src/mom5/ocean_param/lateral/ocean_bihgen_friction.F90 b/src/mom5/ocean_param/lateral/ocean_bihgen_friction.F90 index b136873d6f..68801031ad 100644 --- a/src/mom5/ocean_param/lateral/ocean_bihgen_friction.F90 +++ b/src/mom5/ocean_param/lateral/ocean_bihgen_friction.F90 @@ -476,7 +476,7 @@ subroutine ocean_bihgen_friction_init(Grid, Domain, Time, Ocean_options, d_time, module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) ! provide for namelist over-ride of defaults #ifdef INTERNAL_FILE_NML diff --git a/src/mom5/ocean_param/lateral/ocean_lap_friction.F90 b/src/mom5/ocean_param/lateral/ocean_lap_friction.F90 index 2eba63d3d4..ee18aa778c 100644 --- a/src/mom5/ocean_param/lateral/ocean_lap_friction.F90 +++ b/src/mom5/ocean_param/lateral/ocean_lap_friction.F90 @@ -158,7 +158,7 @@ subroutine ocean_lap_friction_init(Grid, Domain, Time, Ocean_options, dtime, obc module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) ! provide for namelist over-ride of defaults #ifdef INTERNAL_FILE_NML diff --git a/src/mom5/ocean_param/lateral/ocean_lap_tracer.F90 b/src/mom5/ocean_param/lateral/ocean_lap_tracer.F90 index d69a371504..3fdd95bffb 100644 --- a/src/mom5/ocean_param/lateral/ocean_lap_tracer.F90 +++ b/src/mom5/ocean_param/lateral/ocean_lap_tracer.F90 @@ -190,7 +190,7 @@ subroutine ocean_lap_tracer_init(Grid, Domain, Time, T_prog, Ocean_options, dtim num_prog_tracers = size(T_prog(:)) - call write_version_number() + call write_version_number(version, tagname) ! provide for namelist over-ride of defaults #ifdef INTERNAL_FILE_NML diff --git a/src/mom5/ocean_param/lateral/ocean_lapcgrid_friction.F90 b/src/mom5/ocean_param/lateral/ocean_lapcgrid_friction.F90 index 159f142e8d..d444b31df3 100644 --- a/src/mom5/ocean_param/lateral/ocean_lapcgrid_friction.F90 +++ b/src/mom5/ocean_param/lateral/ocean_lapcgrid_friction.F90 @@ -410,7 +410,7 @@ subroutine ocean_lapcgrid_friction_init(Grid, Domain, Time, Ocean_options, d_tim module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) #ifndef MOM_STATIC_ARRAYS call get_local_indices(Domain, isd, ied, jsd, jed, isc, iec, jsc, jec) diff --git a/src/mom5/ocean_param/lateral/ocean_lapcst_friction.F90 b/src/mom5/ocean_param/lateral/ocean_lapcst_friction.F90 index 295ad93b36..6634dbeffc 100644 --- a/src/mom5/ocean_param/lateral/ocean_lapcst_friction.F90 +++ b/src/mom5/ocean_param/lateral/ocean_lapcst_friction.F90 @@ -186,7 +186,7 @@ subroutine ocean_lapcst_friction_init(Grid, Domain, Time, Ocean_options, d_time, module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) ! provide for namelist over-ride of defaults #ifdef INTERNAL_FILE_NML diff --git a/src/mom5/ocean_param/lateral/ocean_lapgen_friction.F90 b/src/mom5/ocean_param/lateral/ocean_lapgen_friction.F90 index 4314bc8abd..bf09e3bbc6 100644 --- a/src/mom5/ocean_param/lateral/ocean_lapgen_friction.F90 +++ b/src/mom5/ocean_param/lateral/ocean_lapgen_friction.F90 @@ -511,7 +511,7 @@ subroutine ocean_lapgen_friction_init(Grid, Domain, Time, Ocean_options, d_time, module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) #ifndef MOM_STATIC_ARRAYS call get_local_indices(Domain, isd, ied, jsd, jed, isc, iec, jsc, jec) diff --git a/src/mom5/ocean_param/lateral/ocean_mixdownslope.F90 b/src/mom5/ocean_param/lateral/ocean_mixdownslope.F90 index 74b2e660d1..282d473491 100644 --- a/src/mom5/ocean_param/lateral/ocean_mixdownslope.F90 +++ b/src/mom5/ocean_param/lateral/ocean_mixdownslope.F90 @@ -267,7 +267,7 @@ subroutine ocean_mixdownslope_init(Grid, Domain, Time, Dens, T_prog, Ocean_optio module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) #ifndef MOM_STATIC_ARRAYS call get_local_indices(Domain,isd,ied,jsd,jed,isc,iec,jsc,jec) nk = Grid%nk diff --git a/src/mom5/ocean_param/lateral/ocean_sigma_transport.F90 b/src/mom5/ocean_param/lateral/ocean_sigma_transport.F90 index 610926269e..1f6affe206 100644 --- a/src/mom5/ocean_param/lateral/ocean_sigma_transport.F90 +++ b/src/mom5/ocean_param/lateral/ocean_sigma_transport.F90 @@ -432,7 +432,7 @@ subroutine ocean_sigma_transport_init(Grid, Domain, Time, Dens, T_prog, Ocean_op module_is_initialized = .TRUE. vert_coordinate = ver_coordinate - call write_version_number() + call write_version_number(version, tagname) ! provide for namelist over-ride of defaults #ifdef INTERNAL_FILE_NML diff --git a/src/mom5/ocean_param/lateral/ocean_submesoscale.F90 b/src/mom5/ocean_param/lateral/ocean_submesoscale.F90 index dea51a0071..7590fa7dec 100644 --- a/src/mom5/ocean_param/lateral/ocean_submesoscale.F90 +++ b/src/mom5/ocean_param/lateral/ocean_submesoscale.F90 @@ -586,7 +586,7 @@ subroutine ocean_submesoscale_init(Grid, Domain, Time, Dens, T_prog, Ocean_optio module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) #ifdef INTERNAL_FILE_NML read (input_nml_file, nml=ocean_submesoscale_nml, iostat=io_status) diff --git a/src/mom5/ocean_param/neutral/ocean_nphysics.F90 b/src/mom5/ocean_param/neutral/ocean_nphysics.F90 index 1d9661cfba..b114b45209 100644 --- a/src/mom5/ocean_param/neutral/ocean_nphysics.F90 +++ b/src/mom5/ocean_param/neutral/ocean_nphysics.F90 @@ -154,7 +154,7 @@ subroutine ocean_nphysics_init(Grid, Domain, Time, Time_steps, Thickness, Dens, module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) Dom => Domain Grd => Grid diff --git a/src/mom5/ocean_param/neutral/ocean_nphysicsA.F90 b/src/mom5/ocean_param/neutral/ocean_nphysicsA.F90 index ea3ec84b89..cb17870d57 100644 --- a/src/mom5/ocean_param/neutral/ocean_nphysicsA.F90 +++ b/src/mom5/ocean_param/neutral/ocean_nphysicsA.F90 @@ -563,7 +563,7 @@ subroutine ocean_nphysicsA_init(Grid, Domain, Time, Time_steps, Thickness, Dens, module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) num_prog_tracers = size(T_prog(:)) dtime = Time_steps%dtime_t diff --git a/src/mom5/ocean_param/neutral/ocean_nphysicsB.F90 b/src/mom5/ocean_param/neutral/ocean_nphysicsB.F90 index 44f9c79e9b..35c447a1e4 100644 --- a/src/mom5/ocean_param/neutral/ocean_nphysicsB.F90 +++ b/src/mom5/ocean_param/neutral/ocean_nphysicsB.F90 @@ -550,7 +550,7 @@ subroutine ocean_nphysicsB_init(Grid, Domain, Time, Time_steps, Thickness, Dens, module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) num_prog_tracers = size(T_prog(:)) dtime = Time_steps%dtime_t diff --git a/src/mom5/ocean_param/neutral/ocean_nphysicsC.F90 b/src/mom5/ocean_param/neutral/ocean_nphysicsC.F90 index e54f75dcb2..e946e45c48 100644 --- a/src/mom5/ocean_param/neutral/ocean_nphysicsC.F90 +++ b/src/mom5/ocean_param/neutral/ocean_nphysicsC.F90 @@ -692,7 +692,7 @@ subroutine ocean_nphysicsC_init(Grid, Domain, Time, Time_steps, Thickness, Dens, module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) num_prog_tracers = size(T_prog(:)) dtime = Time_steps%dtime_t diff --git a/src/mom5/ocean_param/neutral/ocean_nphysics_new.F90 b/src/mom5/ocean_param/neutral/ocean_nphysics_new.F90 index 20f69e461a..b72b4d6d5c 100644 --- a/src/mom5/ocean_param/neutral/ocean_nphysics_new.F90 +++ b/src/mom5/ocean_param/neutral/ocean_nphysics_new.F90 @@ -217,7 +217,7 @@ subroutine ocean_nphysics_new_init(Grid, Domain, Time, Time_steps, Thickness, & module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) ! provide for namelist over-ride of default values #ifdef INTERNAL_FILE_NML diff --git a/src/mom5/ocean_param/neutral/ocean_nphysics_util.F90 b/src/mom5/ocean_param/neutral/ocean_nphysics_util.F90 index 9f6e3f7745..6fa30707fd 100644 --- a/src/mom5/ocean_param/neutral/ocean_nphysics_util.F90 +++ b/src/mom5/ocean_param/neutral/ocean_nphysics_util.F90 @@ -938,7 +938,7 @@ subroutine ocean_nphysics_util_init(Grid, Domain, Time, Time_steps, Dens, T_prog module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) Dom => Domain Grd => Grid diff --git a/src/mom5/ocean_param/sources/ocean_increment_eta.F90 b/src/mom5/ocean_param/sources/ocean_increment_eta.F90 index f4d91bc99e..9a210bc502 100644 --- a/src/mom5/ocean_param/sources/ocean_increment_eta.F90 +++ b/src/mom5/ocean_param/sources/ocean_increment_eta.F90 @@ -147,7 +147,7 @@ subroutine ocean_increment_eta_init(Grid, Domain, Time) module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) ! provide for namelist over-ride of default values #ifdef INTERNAL_FILE_NML diff --git a/src/mom5/ocean_param/sources/ocean_increment_tracer.F90 b/src/mom5/ocean_param/sources/ocean_increment_tracer.F90 index dffd77b76b..e364a50941 100644 --- a/src/mom5/ocean_param/sources/ocean_increment_tracer.F90 +++ b/src/mom5/ocean_param/sources/ocean_increment_tracer.F90 @@ -154,7 +154,7 @@ subroutine ocean_increment_tracer_init(Grid, Domain, Time, T_prog) allocate( Increment(num_prog_tracers) ) - call write_version_number() + call write_version_number(version, tagname) ! provide for namelist over-ride of default values #ifdef INTERNAL_FILE_NML diff --git a/src/mom5/ocean_param/sources/ocean_increment_velocity.F90 b/src/mom5/ocean_param/sources/ocean_increment_velocity.F90 index 04764a27ea..e503f27981 100644 --- a/src/mom5/ocean_param/sources/ocean_increment_velocity.F90 +++ b/src/mom5/ocean_param/sources/ocean_increment_velocity.F90 @@ -151,7 +151,7 @@ subroutine ocean_increment_velocity_init(Grid, Domain, Time) allocate( Increment(2) ) - call write_version_number() + call write_version_number(version, tagname) ! provide for namelist over-ride of default values #ifdef INTERNAL_FILE_NML diff --git a/src/mom5/ocean_param/sources/ocean_momentum_source.F90 b/src/mom5/ocean_param/sources/ocean_momentum_source.F90 index a43e48cc5b..97074e21a4 100644 --- a/src/mom5/ocean_param/sources/ocean_momentum_source.F90 +++ b/src/mom5/ocean_param/sources/ocean_momentum_source.F90 @@ -147,7 +147,7 @@ subroutine ocean_momentum_source_init(Grid, Domain, Time, Ocean_options, debug) module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) call get_local_indices(Domain,isd,ied,jsd,jed,isc,iec,jsc,jec) nk = Grid%nk diff --git a/src/mom5/ocean_param/sources/ocean_overexchange.F90 b/src/mom5/ocean_param/sources/ocean_overexchange.F90 index 591c21d042..0f908d9713 100644 --- a/src/mom5/ocean_param/sources/ocean_overexchange.F90 +++ b/src/mom5/ocean_param/sources/ocean_overexchange.F90 @@ -277,7 +277,7 @@ subroutine ocean_overexchange_init(Grid, Domain, Time, Dens, T_prog, Ocean_optio module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) #ifndef MOM_STATIC_ARRAYS call get_local_indices(Domain,isd,ied,jsd,jed,isc,iec,jsc,jec) nk = Grid%nk diff --git a/src/mom5/ocean_param/sources/ocean_overflow.F90 b/src/mom5/ocean_param/sources/ocean_overflow.F90 index f1f2d08398..0e2bc693f0 100644 --- a/src/mom5/ocean_param/sources/ocean_overflow.F90 +++ b/src/mom5/ocean_param/sources/ocean_overflow.F90 @@ -233,7 +233,7 @@ subroutine ocean_overflow_init(Grid, Domain, Time, Dens, T_prog, Ocean_options, module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) #ifndef MOM_STATIC_ARRAYS call get_local_indices(Domain,isd,ied,jsd,jed,isc,iec,jsc,jec) nk = Grid%nk diff --git a/src/mom5/ocean_param/sources/ocean_overflow_OFP.F90 b/src/mom5/ocean_param/sources/ocean_overflow_OFP.F90 index dd5257b4ce..75f207eb5d 100644 --- a/src/mom5/ocean_param/sources/ocean_overflow_OFP.F90 +++ b/src/mom5/ocean_param/sources/ocean_overflow_OFP.F90 @@ -528,7 +528,7 @@ subroutine ocean_overflow_OFP_init(Grid, Domain, Time, Dens, T_prog, Ocean_optio module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) #ifndef MOM_STATIC_ARRAYS call get_local_indices(Domain,isd,ied,jsd,jed,isc,iec,jsc,jec) nk = Grid%nk diff --git a/src/mom5/ocean_param/sources/ocean_rivermix.F90 b/src/mom5/ocean_param/sources/ocean_rivermix.F90 index 232067b5ba..16c8064d47 100644 --- a/src/mom5/ocean_param/sources/ocean_rivermix.F90 +++ b/src/mom5/ocean_param/sources/ocean_rivermix.F90 @@ -401,7 +401,7 @@ subroutine ocean_rivermix_init(Grid, Domain, Time, Time_steps, Dens, & module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) call get_local_indices(Domain,isd,ied,jsd,jed,isc,iec,jsc,jec) nk = Grid%nk diff --git a/src/mom5/ocean_param/sources/ocean_riverspread.F90 b/src/mom5/ocean_param/sources/ocean_riverspread.F90 index aee085476b..2ec89676c4 100644 --- a/src/mom5/ocean_param/sources/ocean_riverspread.F90 +++ b/src/mom5/ocean_param/sources/ocean_riverspread.F90 @@ -203,7 +203,7 @@ subroutine ocean_riverspread_init(Grid, Domain, Ocean_options, dtime_t) module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) call get_local_indices(Domain,isd,ied,jsd,jed,isc,iec,jsc,jec) nk = Grid%nk diff --git a/src/mom5/ocean_param/sources/ocean_shortwave.F90 b/src/mom5/ocean_param/sources/ocean_shortwave.F90 index 41403421af..cd1256e74c 100644 --- a/src/mom5/ocean_param/sources/ocean_shortwave.F90 +++ b/src/mom5/ocean_param/sources/ocean_shortwave.F90 @@ -135,7 +135,7 @@ subroutine ocean_shortwave_init(Grid, Domain, Time, Dens, vert_coordinate, Ocean module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) #ifdef INTERNAL_FILE_NML read (input_nml_file, nml=ocean_shortwave_nml, iostat=io_status) diff --git a/src/mom5/ocean_param/sources/ocean_shortwave_csiro.F90 b/src/mom5/ocean_param/sources/ocean_shortwave_csiro.F90 index c759602bab..43f4072734 100644 --- a/src/mom5/ocean_param/sources/ocean_shortwave_csiro.F90 +++ b/src/mom5/ocean_param/sources/ocean_shortwave_csiro.F90 @@ -180,7 +180,7 @@ subroutine ocean_shortwave_csiro_init(Grid, Domain, Time, Ocean_options) module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) #ifdef INTERNAL_FILE_NML read (input_nml_file, nml=ocean_shortwave_csiro_nml, iostat=io_status) diff --git a/src/mom5/ocean_param/sources/ocean_shortwave_gfdl.F90 b/src/mom5/ocean_param/sources/ocean_shortwave_gfdl.F90 index 3b913f3453..bcbaea1659 100644 --- a/src/mom5/ocean_param/sources/ocean_shortwave_gfdl.F90 +++ b/src/mom5/ocean_param/sources/ocean_shortwave_gfdl.F90 @@ -283,7 +283,7 @@ subroutine ocean_shortwave_gfdl_init(Grid, Domain, Time, ver_coordinate, Ocean_o module_is_initialized = .TRUE. vert_coordinate = ver_coordinate - call write_version_number() + call write_version_number(version, tagname) #ifdef INTERNAL_FILE_NML read (input_nml_file, nml=ocean_shortwave_gfdl_nml, iostat=io_status) diff --git a/src/mom5/ocean_param/sources/ocean_shortwave_jerlov.F90 b/src/mom5/ocean_param/sources/ocean_shortwave_jerlov.F90 index 9bcc2f1cb3..7a33177f9f 100644 --- a/src/mom5/ocean_param/sources/ocean_shortwave_jerlov.F90 +++ b/src/mom5/ocean_param/sources/ocean_shortwave_jerlov.F90 @@ -257,7 +257,7 @@ subroutine ocean_shortwave_jerlov_init(Grid, Domain, ver_coordinate, Ocean_optio module_is_initialized = .TRUE. vert_coordinate = ver_coordinate - call write_version_number() + call write_version_number(version, tagname) #ifdef INTERNAL_FILE_NML read (input_nml_file, nml=ocean_shortwave_jerlov_nml, iostat=io_status) diff --git a/src/mom5/ocean_param/sources/ocean_sponges_eta.F90 b/src/mom5/ocean_param/sources/ocean_sponges_eta.F90 index 0e9189cd90..ad01f5881e 100644 --- a/src/mom5/ocean_param/sources/ocean_sponges_eta.F90 +++ b/src/mom5/ocean_param/sources/ocean_sponges_eta.F90 @@ -152,7 +152,7 @@ subroutine ocean_sponges_eta_init(Grid, Domain, Time, dtime, Ocean_options) allocate( Sponge(1) ) - call write_version_number() + call write_version_number(version, tagname) ! provide for namelist over-ride of default values #ifdef INTERNAL_FILE_NML diff --git a/src/mom5/ocean_param/sources/ocean_sponges_tracer.F90 b/src/mom5/ocean_param/sources/ocean_sponges_tracer.F90 index 7d6fa8929c..bf98f924c5 100644 --- a/src/mom5/ocean_param/sources/ocean_sponges_tracer.F90 +++ b/src/mom5/ocean_param/sources/ocean_sponges_tracer.F90 @@ -183,7 +183,7 @@ subroutine ocean_sponges_tracer_init(Grid, Domain, Time, T_prog, dtime, Ocean_op allocate( Sponge(num_prog_tracers) ) - call write_version_number() + call write_version_number(version, tagname) ! provide for namelist over-ride of default values #ifdef INTERNAL_FILE_NML diff --git a/src/mom5/ocean_param/sources/ocean_sponges_velocity.F90 b/src/mom5/ocean_param/sources/ocean_sponges_velocity.F90 index 96ff7a1168..3b344ee693 100644 --- a/src/mom5/ocean_param/sources/ocean_sponges_velocity.F90 +++ b/src/mom5/ocean_param/sources/ocean_sponges_velocity.F90 @@ -162,7 +162,7 @@ subroutine ocean_sponges_velocity_init(Grid, Domain, Time, dtime, Ocean_options) allocate( Sponge_u(1) ) allocate( Sponge_v(1) ) - call write_version_number() + call write_version_number(version, tagname) ! provide for namelist over-ride of default values #ifdef INTERNAL_FILE_NML diff --git a/src/mom5/ocean_param/sources/ocean_xlandinsert.F90 b/src/mom5/ocean_param/sources/ocean_xlandinsert.F90 index 8fcf2c7640..0b2376ffeb 100644 --- a/src/mom5/ocean_param/sources/ocean_xlandinsert.F90 +++ b/src/mom5/ocean_param/sources/ocean_xlandinsert.F90 @@ -261,7 +261,7 @@ subroutine ocean_xlandinsert_init(Grid, Domain, Time, Dens, T_prog, Ocean_option module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) call get_local_indices(Domain,isd,ied,jsd,jed,isc,iec,jsc,jec) call get_global_indices(Domain,isg,ieg,jsg,jeg) diff --git a/src/mom5/ocean_param/sources/ocean_xlandmix.F90 b/src/mom5/ocean_param/sources/ocean_xlandmix.F90 index 31e14cc6c4..f64e999853 100644 --- a/src/mom5/ocean_param/sources/ocean_xlandmix.F90 +++ b/src/mom5/ocean_param/sources/ocean_xlandmix.F90 @@ -295,7 +295,7 @@ subroutine ocean_xlandmix_init(Grid, Domain, Time, Dens, T_prog, Ocean_options, module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) call get_local_indices(Domain,isd,ied,jsd,jed,isc,iec,jsc,jec) call get_global_indices(Domain,isg,ieg,jsg,jeg) diff --git a/src/mom5/ocean_param/vertical/ocean_convect.F90 b/src/mom5/ocean_param/vertical/ocean_convect.F90 index fa952e52bc..3575cd86aa 100644 --- a/src/mom5/ocean_param/vertical/ocean_convect.F90 +++ b/src/mom5/ocean_param/vertical/ocean_convect.F90 @@ -220,7 +220,7 @@ subroutine ocean_convect_init (Grid, Domain, Time, Dens, T_prog, Ocean_options, module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) num_prog_tracers = size(T_prog(:)) diff --git a/src/mom5/ocean_param/vertical/ocean_form_drag.F90 b/src/mom5/ocean_param/vertical/ocean_form_drag.F90 index b55fce6bb1..9108e94b7e 100644 --- a/src/mom5/ocean_param/vertical/ocean_form_drag.F90 +++ b/src/mom5/ocean_param/vertical/ocean_form_drag.F90 @@ -348,7 +348,7 @@ subroutine ocean_form_drag_init(Grid, Domain, Time, Time_steps, Ocean_options, d module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) call get_local_indices(Domain,isd,ied,jsd,jed,isc,iec,jsc,jec) nk = Grid%nk diff --git a/src/mom5/ocean_param/vertical/ocean_vert_chen.F90 b/src/mom5/ocean_param/vertical/ocean_vert_chen.F90 index 42db6fdde3..aecbca1d7a 100644 --- a/src/mom5/ocean_param/vertical/ocean_vert_chen.F90 +++ b/src/mom5/ocean_param/vertical/ocean_vert_chen.F90 @@ -252,7 +252,7 @@ subroutine ocean_vert_chen_init (Grid, Domain, Time, Time_steps, T_prog, hor_gri endif module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) ! provide for namelist over-ride of defaults #ifdef INTERNAL_FILE_NML diff --git a/src/mom5/ocean_param/vertical/ocean_vert_const.F90 b/src/mom5/ocean_param/vertical/ocean_vert_const.F90 index 2db0c82d8c..d2455115db 100644 --- a/src/mom5/ocean_param/vertical/ocean_vert_const.F90 +++ b/src/mom5/ocean_param/vertical/ocean_vert_const.F90 @@ -120,7 +120,7 @@ subroutine ocean_vert_const_init (Grid, Domain, Time, Time_steps, T_prog) module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) ! provide for namelist over-ride of default values #ifdef INTERNAL_FILE_NML diff --git a/src/mom5/ocean_param/vertical/ocean_vert_gotm.F90 b/src/mom5/ocean_param/vertical/ocean_vert_gotm.F90 index 346cc9c654..376b3a14e2 100644 --- a/src/mom5/ocean_param/vertical/ocean_vert_gotm.F90 +++ b/src/mom5/ocean_param/vertical/ocean_vert_gotm.F90 @@ -333,7 +333,7 @@ subroutine ocean_vert_gotm_init (Grid, Domain, Time, Time_steps, T_prog, obc, de have_obc = obc module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) ! provide for namelist over-ride of defaults #ifdef INTERNAL_FILE_NML diff --git a/src/mom5/ocean_param/vertical/ocean_vert_kpp_mom4p0.F90 b/src/mom5/ocean_param/vertical/ocean_vert_kpp_mom4p0.F90 index 43d4bec95b..efcd684326 100644 --- a/src/mom5/ocean_param/vertical/ocean_vert_kpp_mom4p0.F90 +++ b/src/mom5/ocean_param/vertical/ocean_vert_kpp_mom4p0.F90 @@ -538,7 +538,7 @@ subroutine ocean_vert_kpp_mom4p0_init (Grid, Domain, Time, Time_steps, Dens, T_p rho_cp = rho0*cp_ocean inv_rho_cp = 1.0/rho_cp - call write_version_number() + call write_version_number(version, tagname) ! provide for namelist over-ride of defaults #ifdef INTERNAL_FILE_NML diff --git a/src/mom5/ocean_param/vertical/ocean_vert_kpp_mom4p1.F90 b/src/mom5/ocean_param/vertical/ocean_vert_kpp_mom4p1.F90 index 6188cb1113..cd8fa6ebc1 100644 --- a/src/mom5/ocean_param/vertical/ocean_vert_kpp_mom4p1.F90 +++ b/src/mom5/ocean_param/vertical/ocean_vert_kpp_mom4p1.F90 @@ -562,7 +562,7 @@ subroutine ocean_vert_kpp_mom4p1_init (Grid, Domain, Time, Time_steps, Dens, T_p rho_cp = rho0*cp_ocean inv_rho_cp = 1.0/rho_cp - call write_version_number() + call write_version_number(version, tagname) ! provide for namelist over-ride of defaults #ifdef INTERNAL_FILE_NML diff --git a/src/mom5/ocean_param/vertical/ocean_vert_kpp_test.F90 b/src/mom5/ocean_param/vertical/ocean_vert_kpp_test.F90 index a17ced3f7c..7f15b6b7d6 100644 --- a/src/mom5/ocean_param/vertical/ocean_vert_kpp_test.F90 +++ b/src/mom5/ocean_param/vertical/ocean_vert_kpp_test.F90 @@ -571,7 +571,7 @@ subroutine ocean_vert_kpp_test_init (Grid, Domain, Time, Time_steps, Dens, T_pro inv_rho_cp = 1.0/rho_cp horz_grid = hor_grid - call write_version_number() + call write_version_number(version, tagname) ! provide for namelist over-ride of defaults #ifdef INTERNAL_FILE_NML diff --git a/src/mom5/ocean_param/vertical/ocean_vert_mix.F90 b/src/mom5/ocean_param/vertical/ocean_vert_mix.F90 index 93393ae0ed..073e83bbfe 100644 --- a/src/mom5/ocean_param/vertical/ocean_vert_mix.F90 +++ b/src/mom5/ocean_param/vertical/ocean_vert_mix.F90 @@ -728,7 +728,7 @@ subroutine ocean_vert_mix_init (Grid, Domain, Time, Dens, Velocity, Time_steps, if (trim(T_prog(n)%name) == 'salt') index_salt = n enddo - call write_version_number() + call write_version_number(version, tagname) #ifdef INTERNAL_FILE_NML read (input_nml_file, nml=ocean_vert_mix_nml, iostat=io_status) diff --git a/src/mom5/ocean_param/vertical/ocean_vert_pp.F90 b/src/mom5/ocean_param/vertical/ocean_vert_pp.F90 index 7db995d77d..ff95b6e98e 100644 --- a/src/mom5/ocean_param/vertical/ocean_vert_pp.F90 +++ b/src/mom5/ocean_param/vertical/ocean_vert_pp.F90 @@ -184,7 +184,7 @@ subroutine ocean_vert_pp_init (Grid, Domain, Time, Time_steps, T_prog, hor_grid) module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) ! provide for namelist over-ride of defaults #ifdef INTERNAL_FILE_NML diff --git a/src/mom5/ocean_param/vertical/ocean_vert_tidal.F90 b/src/mom5/ocean_param/vertical/ocean_vert_tidal.F90 index 166263fe20..e7518bdefe 100644 --- a/src/mom5/ocean_param/vertical/ocean_vert_tidal.F90 +++ b/src/mom5/ocean_param/vertical/ocean_vert_tidal.F90 @@ -543,7 +543,7 @@ subroutine ocean_vert_tidal_init(Grid, Domain, Time, T_prog, Velocity, Ocean_opt module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) #ifdef INTERNAL_FILE_NML read (input_nml_file, nml=ocean_vert_tidal_nml, iostat=io_status) diff --git a/src/mom5/ocean_param/vertical/ocean_vert_tidal_test.F90 b/src/mom5/ocean_param/vertical/ocean_vert_tidal_test.F90 index 52340cef20..c1281e997e 100644 --- a/src/mom5/ocean_param/vertical/ocean_vert_tidal_test.F90 +++ b/src/mom5/ocean_param/vertical/ocean_vert_tidal_test.F90 @@ -561,7 +561,7 @@ subroutine ocean_vert_tidal_test_init(Grid, Domain, Time, T_prog, Velocity, Ocea module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) #ifdef INTERNAL_FILE_NML read (input_nml_file, nml=ocean_vert_tidal_test_nml, iostat=io_status) diff --git a/src/mom5/ocean_tracers/ocean_passive.F90 b/src/mom5/ocean_tracers/ocean_passive.F90 index 8f2f7a04d4..62ecf1e009 100644 --- a/src/mom5/ocean_tracers/ocean_passive.F90 +++ b/src/mom5/ocean_tracers/ocean_passive.F90 @@ -433,7 +433,7 @@ subroutine ocean_passive_init(Domain, Grid, Ocean_options, debug) Dom => Domain Grd => Grid - call write_version_number() + call write_version_number(version, tagname) ! provide for namelist override of defaults #ifdef INTERNAL_FILE_NML diff --git a/src/mom5/ocean_tracers/ocean_tempsalt.F90 b/src/mom5/ocean_tracers/ocean_tempsalt.F90 index af5833c518..f7da00443b 100644 --- a/src/mom5/ocean_tracers/ocean_tempsalt.F90 +++ b/src/mom5/ocean_tracers/ocean_tempsalt.F90 @@ -326,7 +326,7 @@ subroutine ocean_tempsalt_init(Domain, Grid, Ocean_options, itemp, isalt, debug) Dom => Domain Grd => Grid - call write_version_number() + call write_version_number(version, tagname) ! provide for namelist override of defaults #ifdef INTERNAL_FILE_NML diff --git a/src/mom5/ocean_tracers/ocean_tracer_advect.F90 b/src/mom5/ocean_tracers/ocean_tracer_advect.F90 index 9367578c73..43b22b6ff5 100644 --- a/src/mom5/ocean_tracers/ocean_tracer_advect.F90 +++ b/src/mom5/ocean_tracers/ocean_tracer_advect.F90 @@ -524,7 +524,7 @@ subroutine ocean_tracer_advect_init (Grid, Domain, Time, Dens, T_prog, obc, debu have_obc = obc - call write_version_number() + call write_version_number(version, tagname) ! provide for namelist over-ride of defaults #ifdef INTERNAL_FILE_NML diff --git a/src/mom5/ocean_wave/ocean_wave.F90 b/src/mom5/ocean_wave/ocean_wave.F90 index b3a63b66dd..1d9999d6e8 100644 --- a/src/mom5/ocean_wave/ocean_wave.F90 +++ b/src/mom5/ocean_wave/ocean_wave.F90 @@ -167,7 +167,7 @@ subroutine ocean_wave_init(Grid, Domain, Waves, Time, Time_steps, Ocean_options, call mpp_clock_begin(id_init) - call write_version_number() + call write_version_number(version, tagname) #ifdef INTERNAL_FILE_NML read (input_nml_file, ocean_wave_nml, iostat=io_status) diff --git a/src/ocean_shared/generic_tracers/generic_BLING.F90 b/src/ocean_shared/generic_tracers/generic_BLING.F90 index e5cf9d3e5e..b572d9e883 100644 --- a/src/ocean_shared/generic_tracers/generic_BLING.F90 +++ b/src/ocean_shared/generic_tracers/generic_BLING.F90 @@ -606,7 +606,7 @@ end subroutine generic_BLING_register subroutine generic_BLING_init(tracer_list) type(g_tracer_type), pointer :: tracer_list - call write_version_number() + call write_version_number(version, tagname) !Specify and initialize all parameters used by this package call user_add_params diff --git a/src/ocean_shared/generic_tracers/generic_ERGOM.F90 b/src/ocean_shared/generic_tracers/generic_ERGOM.F90 index 6c5d6ed0e9..29fa9a058c 100644 --- a/src/ocean_shared/generic_tracers/generic_ERGOM.F90 +++ b/src/ocean_shared/generic_tracers/generic_ERGOM.F90 @@ -853,7 +853,7 @@ subroutine generic_ERGOM_register(tracer_list) stdoutunit=stdout();stdlogunit=stdlog() - call write_version_number() + call write_version_number(version, tagname) ! provide for namelist over-ride of defaults #ifdef INTERNAL_FILE_NML diff --git a/src/postprocessing/regrid/regrid.F90 b/src/postprocessing/regrid/regrid.F90 index d2b4176232..50f70a20ac 100644 --- a/src/postprocessing/regrid/regrid.F90 +++ b/src/postprocessing/regrid/regrid.F90 @@ -226,7 +226,7 @@ subroutine regrid_init call close_file (unit) !--- write out version information --------------------------------- - call write_version_number() + call write_version_number(version, tagname) if(num_flds == 0) call error_handler('regrid: nml num_fiels = 0 should be a positive number') if(num_flds .gt. max_flds) call error_handler('regrid: nml num_fiels is greater than maximum'// & diff --git a/src/preprocessing/generate_grids/atmos/atmos_grid.f90 b/src/preprocessing/generate_grids/atmos/atmos_grid.f90 index e42b44ed32..e7ed71a957 100644 --- a/src/preprocessing/generate_grids/atmos/atmos_grid.f90 +++ b/src/preprocessing/generate_grids/atmos/atmos_grid.f90 @@ -181,7 +181,7 @@ subroutine atmos_grid_init endif !--- write version info and namelist to logfile ---------------------- - call write_version_number() + call write_version_number(version, tagname) write (stdout(), nml=atmos_grid_nml) module_is_initialized = .true. diff --git a/src/preprocessing/generate_grids/atmos/atmos_grid_generator.f90 b/src/preprocessing/generate_grids/atmos/atmos_grid_generator.f90 index a16082bfa5..29a20f0d06 100644 --- a/src/preprocessing/generate_grids/atmos/atmos_grid_generator.f90 +++ b/src/preprocessing/generate_grids/atmos/atmos_grid_generator.f90 @@ -71,7 +71,7 @@ program grid_generator endif !--- write version info and namelist to logfile ---------------------- write (stdout(), nml=atmos_grid_generator_nml) - call write_version_number() + call write_version_number(version, tagname) !--- generate data --------------------------------------------------- call atmos_grid_init diff --git a/src/preprocessing/generate_grids/grid_transfer/grid_transfer.F90 b/src/preprocessing/generate_grids/grid_transfer/grid_transfer.F90 index 5a1f6d83d1..228616e562 100644 --- a/src/preprocessing/generate_grids/grid_transfer/grid_transfer.F90 +++ b/src/preprocessing/generate_grids/grid_transfer/grid_transfer.F90 @@ -107,7 +107,7 @@ subroutine grid_transfer_init endif !--- write version information - call write_version_number() + call write_version_number(version, tagname) end subroutine grid_transfer_init diff --git a/src/preprocessing/generate_grids/ocean/compare_grid.f90 b/src/preprocessing/generate_grids/ocean/compare_grid.f90 index f2c3215480..741e06fd6d 100644 --- a/src/preprocessing/generate_grids/ocean/compare_grid.f90 +++ b/src/preprocessing/generate_grids/ocean/compare_grid.f90 @@ -125,7 +125,7 @@ subroutine compare_grid_init endif !--- write out version information --------------------------------- - call write_version_number() + call write_version_number(version, tagname) end subroutine compare_grid_init diff --git a/src/preprocessing/generate_grids/ocean/edit_grid.F90 b/src/preprocessing/generate_grids/ocean/edit_grid.F90 index 62e8495881..641c87f028 100644 --- a/src/preprocessing/generate_grids/ocean/edit_grid.F90 +++ b/src/preprocessing/generate_grids/ocean/edit_grid.F90 @@ -198,7 +198,7 @@ subroutine edit_grid_init endif !--- write out version information --------------------------------- - call write_version_number() + call write_version_number(version, tagname) end subroutine edit_grid_init diff --git a/src/preprocessing/generate_grids/ocean/hgrid.f90 b/src/preprocessing/generate_grids/ocean/hgrid.f90 index 475d3f853c..0c65aafcf8 100644 --- a/src/preprocessing/generate_grids/ocean/hgrid.f90 +++ b/src/preprocessing/generate_grids/ocean/hgrid.f90 @@ -358,7 +358,7 @@ subroutine hgrid_init !--- write version info and namelist to logfile ---------------------- - call write_version_number() + call write_version_number(version, tagname) write (stdout(), nml=hgrid_nml) module_is_initialized = .true. diff --git a/src/preprocessing/generate_grids/ocean/ocean_grid_generator.f90 b/src/preprocessing/generate_grids/ocean/ocean_grid_generator.f90 index b9c2074302..675ac6cb1d 100644 --- a/src/preprocessing/generate_grids/ocean/ocean_grid_generator.f90 +++ b/src/preprocessing/generate_grids/ocean/ocean_grid_generator.f90 @@ -95,7 +95,7 @@ program ocean_grid_generator call mpp_error(FATAL,'ocean_grid_generator: file input.nml does not exist') endif - call write_version_number() + call write_version_number(version, tagname) if (mpp_pe() == mpp_root_pe()) write(stdout(), nml=ocean_grid_generator_nml) !--- generate data --------------------------------------------------- diff --git a/src/preprocessing/generate_grids/ocean/topog.f90 b/src/preprocessing/generate_grids/ocean/topog.f90 index 8810dc32fb..f71ef1c708 100644 --- a/src/preprocessing/generate_grids/ocean/topog.f90 +++ b/src/preprocessing/generate_grids/ocean/topog.f90 @@ -357,7 +357,7 @@ subroutine topog_init(Topog, Hgrid) !--- write version info and namelist to logfile ---------------------- - call write_version_number() + call write_version_number(version, tagname) write (stdout(), nml=topog_nml) write (stdout(), nml=obc_nml) diff --git a/src/preprocessing/generate_grids/ocean/vgrid.f90 b/src/preprocessing/generate_grids/ocean/vgrid.f90 index 88f3135b0c..0ac44d8b8a 100644 --- a/src/preprocessing/generate_grids/ocean/vgrid.f90 +++ b/src/preprocessing/generate_grids/ocean/vgrid.f90 @@ -183,7 +183,7 @@ subroutine vgrid_init !--- write version info and namelist to logfile ---------------------- - call write_version_number() + call write_version_number(version, tagname) if (mpp_pe() == mpp_root_pe()) then write (stdout(), nml=vgrid_nml) endif diff --git a/src/preprocessing/mom4_prep/idealized_bc/idealized_bc.f90 b/src/preprocessing/mom4_prep/idealized_bc/idealized_bc.f90 index 808969f45d..662694133f 100644 --- a/src/preprocessing/mom4_prep/idealized_bc/idealized_bc.f90 +++ b/src/preprocessing/mom4_prep/idealized_bc/idealized_bc.f90 @@ -167,7 +167,7 @@ subroutine idealized_bc_init ' generate_wind_bc, generate_water_bc should be true') !--- write out version information and namelist option --------------- - call write_version_number() + call write_version_number(version, tagname) if (mpp_pe() == mpp_root_pe()) then write (stdout(), nml= idealized_bc_nml) endif diff --git a/src/preprocessing/mom4_prep/idealized_ic/idealized_ic.f90 b/src/preprocessing/mom4_prep/idealized_ic/idealized_ic.f90 index e43e5b30bd..1c44194f01 100644 --- a/src/preprocessing/mom4_prep/idealized_ic/idealized_ic.f90 +++ b/src/preprocessing/mom4_prep/idealized_ic/idealized_ic.f90 @@ -424,7 +424,7 @@ subroutine idealized_ic_init 'idealized_ic_mod: nml "generate_temp_ic"="generate_salt_ic"="generate_salt_ic"=false. At least one should be true') !--- write out version information and namelist option --------------- - call write_version_number() + call write_version_number(version, tagname) if (mpp_pe() == mpp_root_pe()) then write (stdout(), nml= idealized_ic_nml) endif diff --git a/src/preprocessing/regrid/regrid.F90 b/src/preprocessing/regrid/regrid.F90 index aebeccf274..7961cedb02 100644 --- a/src/preprocessing/regrid/regrid.F90 +++ b/src/preprocessing/regrid/regrid.F90 @@ -223,7 +223,7 @@ subroutine regrid_init call close_file (unit) !--- write out version information --------------------------------- - call write_version_number() + call write_version_number(version, tagname) if(num_flds == 0) call error_handler('regrid: nml num_fiels = 0 should be a positive number') if(num_flds .gt. max_flds) call error_handler('regrid: nml num_fiels is greater than maximum'// & diff --git a/src/preprocessing/regrid_2d/regrid_2d.f90 b/src/preprocessing/regrid_2d/regrid_2d.f90 index 0bd4aabfe6..74d497338f 100644 --- a/src/preprocessing/regrid_2d/regrid_2d.f90 +++ b/src/preprocessing/regrid_2d/regrid_2d.f90 @@ -220,7 +220,7 @@ subroutine regrid_2d_init enddo !--- write version information - call write_version_number() + call write_version_number(version, tagname) end subroutine regrid_2d_init diff --git a/src/preprocessing/regrid_3d/regrid_3d.f90 b/src/preprocessing/regrid_3d/regrid_3d.f90 index 8422b97728..a98c751974 100644 --- a/src/preprocessing/regrid_3d/regrid_3d.f90 +++ b/src/preprocessing/regrid_3d/regrid_3d.f90 @@ -237,7 +237,7 @@ subroutine regrid_3d_init enddo !--- write version information - call write_version_number() + call write_version_number(version, tagname) end subroutine regrid_3d_init diff --git a/src/preprocessing/river_regrid/river_regrid.f90 b/src/preprocessing/river_regrid/river_regrid.f90 index 1aed800baa..ea2e0db370 100644 --- a/src/preprocessing/river_regrid/river_regrid.f90 +++ b/src/preprocessing/river_regrid/river_regrid.f90 @@ -164,7 +164,7 @@ subroutine river_regrid_init D2R = PI/180.0 !--- write out version information --------------------------------- - call write_version_number() + call write_version_number(version, tagname) !--- read the river data from river_input_file call read_river_src_data( ) diff --git a/src/shared/README.md b/src/shared/README.md new file mode 100644 index 0000000000..5d4b8aa8b3 --- /dev/null +++ b/src/shared/README.md @@ -0,0 +1,25 @@ +# Flexible Modeling System (FMS) + +The Flexible Modeling System (FMS) is a software framework for supporting the +efficient development, construction, execution, and scientific interpretation +of atmospheric, oceanic, and climate system models. + +More information is available on the [GFDL FMS page](http://www.gfdl.noaa.gov/fms). + +# Disclaimer + +The United States Department of Commerce (DOC) GitHub project code is provided +on an 'as is' basis and the user assumes responsibility for its use. DOC has +relinquished control of the information and no longer has responsibility to +protect the integrity, confidentiality, or availability of the information. Any +claims against the Department of Commerce stemming from the use of its GitHub +project will be governed by all applicable Federal law. Any reference to +specific commercial products, processes, or services by service mark, +trademark, manufacturer, or otherwise, does not constitute or imply their +endorsement, recommendation or favoring by the Department of Commerce. The +Department of Commerce seal and logo, or the seal and logo of a DOC bureau, +shall not be used in any manner to imply endorsement of any commercial product +or activity by DOC or the United States Government. + +This project code is made available through GitHub but is managed by NOAA-GFDL +at https://gitlab.gfdl.noaa.gov. diff --git a/src/shared/amip_interp/amip_interp.F90 b/src/shared/amip_interp/amip_interp.F90 index 5f7276ab8b..83c5ccc272 100644 --- a/src/shared/amip_interp/amip_interp.F90 +++ b/src/shared/amip_interp/amip_interp.F90 @@ -2,10 +2,11 @@ module amip_interp_mod -! +! ! Bruce Wyman ! +! ! ! Provides observed SST and ice mask data sets that have been @@ -62,6 +63,12 @@ module amip_interp_mod use time_manager_mod, only: time_type, operator(+), operator(>), & get_date, set_time, set_date +! add by JHC +use get_cal_time_mod, only: get_cal_time +use mpp_io_mod, only : mpp_open, mpp_read, MPP_RDONLY, MPP_NETCDF, & + MPP_MULTI, MPP_SINGLE, mpp_close, mpp_get_times +! end add by JHC + use horiz_interp_mod, only: horiz_interp_init, horiz_interp, & horiz_interp_new, horiz_interp_del, & horiz_interp_type, assignment(=) @@ -71,7 +78,7 @@ module amip_interp_mod open_namelist_file, open_ieee32_file, & mpp_pe, close_file, lowercase, mpp_root_pe, & NOTE, mpp_error, fms_error_handler -use fms_io_mod, only: read_data +use fms_io_mod, only: read_data, field_size ! add by JHC use constants_mod, only: TFREEZE, pi use platform_mod, only: R4_KIND, I2_KIND use mpp_mod, only: input_nml_file @@ -100,11 +107,13 @@ module amip_interp_mod ! ---- version number ----- -character(len=128) :: version = '$Id: amip_interp.F90,v 19.0 2012/01/06 21:54:21 fms Exp $' -character(len=128) :: tagname = '$Name: tikal $' +character(len=128) :: version = '$Id$' +character(len=128) :: tagname = '$Name$' real, allocatable:: temp1(:,:), temp2(:,:) - +! add by JHC + real, allocatable, dimension(:,:) :: tempamip +! end add by JHC !----------------------------------------------------------------------- !------ private defined data type -------- @@ -215,7 +224,7 @@ module amip_interp_mod ! type amip_interp_type private - type (horiz_interp_type) :: Hintrp + type (horiz_interp_type) :: Hintrp, Hintrp2 ! add by JHC real, pointer :: data1(:,:) =>NULL(), & data2(:,:) =>NULL() type (date_type) :: Date1, Date2 @@ -315,6 +324,8 @@ module amip_interp_mod ! 'reynolds_eof' ! 'reynolds_oi' ! 'hurrell' + ! add by JHC: + ! 'daily', when "use_daily=.T." character(len=16) :: date_out_of_range = 'fail' ! use 'fail' ! 'initclimo' @@ -337,6 +348,12 @@ module amip_interp_mod !global temperature perturbation used for sensitivity experiments real :: sst_pert = 0. +! add by JHC + character(len=6) :: sst_pert_type = 'fixed' ! use 'random' or 'fixed' + logical :: do_sst_pert = .false. + logical :: use_daily = .false. ! if '.true.', give 'data_set = 'daily'' +! end add by JHC + ! SJL: During nudging: use_ncep_sst = .T.; no_anom_sst = .T. ! during forecast: use_ncep_sst = .T.; no_anom_sst = .F. ! For seasonal forecast: use_ncep_ice = .F. @@ -349,7 +366,11 @@ module amip_interp_mod namelist /amip_interp_nml/ use_ncep_sst, no_anom_sst, use_ncep_ice, tice_crit, & interp_oi_sst, data_set, date_out_of_range, & use_zonal, teq, tdif, tann, tlag, amip_date, & - sst_pert, verbose, i_sst, j_sst, forecast_mode + ! add by JHC + sst_pert, sst_pert_type, do_sst_pert, & + use_daily, & + ! end add by JHC + verbose, i_sst, j_sst, forecast_mode ! @@ -364,7 +385,9 @@ module amip_interp_mod ! ! -subroutine get_amip_sst (Time, Interp, sst, err_msg) +! modified by JHC +subroutine get_amip_sst (Time, Interp, sst, err_msg, lon_model, lat_model) +!subroutine get_amip_sst (Time, Interp, sst, err_msg) type (time_type), intent(in) :: Time type (amip_interp_type), intent(inout) :: Interp @@ -380,6 +403,22 @@ subroutine get_amip_sst (Time, Interp, sst, err_msg) type(time_type) :: Amip_Time integer :: tod(3),dum +! add by JHC + real, intent(in), dimension(:,:), optional :: lon_model, lat_model + real :: pert + integer :: i, j, mobs_sst, nobs_sst + integer :: jhctod(6) + type (time_type) :: Udate + character(len=4) :: yyyy + integer :: nrecords, ierr, k, yr, mo, dy + integer :: siz(4) + integer, dimension(:), allocatable :: ryr, rmo, rdy + character(len=30) :: time_unit + real, dimension(:), allocatable :: timeval + character(len=maxc) :: ncfilename +! end add by JHC + + if(present(err_msg)) err_msg = '' if(.not.Interp%I_am_initialized) then if(fms_error_handler('get_amip_sst','The amip_interp_type variable is not initialized',err_msg)) return @@ -397,108 +436,243 @@ subroutine get_amip_sst (Time, Interp, sst, err_msg) Amip_Time = Time endif - if ( .not. allocated(temp1) ) allocate (temp1(mobs,nobs)) - if ( .not. allocated(temp2) ) allocate (temp2(mobs,nobs)) - -if (use_zonal) then - call zonal_sst (Amip_Time, sice, temp1) - call horiz_interp ( Interp%Hintrp, temp1, sst ) -else +! add by JHC +if ( .not.use_daily ) then +! end add by JHC + + if ( .not. allocated(temp1) ) allocate (temp1(mobs,nobs)) + if ( .not. allocated(temp2) ) allocate (temp2(mobs,nobs)) + + if (use_zonal) then + call zonal_sst (Amip_Time, sice, temp1) + call horiz_interp ( Interp%Hintrp, temp1, sst ) + else !----------------------------------------------------------------------- !---------- get new observed sea surface temperature ------------------- ! ---- time interpolation for months ----- - call time_interp (Amip_Time, fmonth, year1, year2, month1, month2) + call time_interp (Amip_Time, fmonth, year1, year2, month1, month2) ! ---- force climatology ---- - if (Interp % use_climo) then - year1=0; year2=0 - endif - if (Interp % use_annual) then - year1=0; year2=0 - month1=0; month2=0 - endif + if (Interp % use_climo) then + year1=0; year2=0 + endif + if (Interp % use_annual) then + year1=0; year2=0 + month1=0; month2=0 + endif ! --------------------------- - Date1 = date_type( year1, month1, 0 ) - Date2 = date_type( year2, month2, 0 ) + Date1 = date_type( year1, month1, 0 ) + Date2 = date_type( year2, month2, 0 ) ! -- open/rewind file -- - unit = -1 + unit = -1 !----------------------------------------------------------------------- - if (Date1 /= Interp % Date1) then + if (Date1 /= Interp % Date1) then ! ---- use Date2 for Date1 ---- - if (Date1 == Interp % Date2) then - Interp % Date1 = Interp % Date2 - Interp % data1 = Interp % data2 - temp1(:,:) = temp2(:,:) ! SJL BUG fix: June 24, 2011 - else - call read_record ('sst', Date1, Udate1, temp1) - if ( use_ncep_sst .and. (.not. no_anom_sst) ) then - temp1(:,:) = temp1(:,:) + sst_anom(:,:) - endif - call horiz_interp ( Interp%Hintrp, temp1, Interp%data1 ) - call clip_data ('sst', Interp%data1) - Interp % Date1 = Date1 - endif - endif + if (Date1 == Interp % Date2) then + Interp % Date1 = Interp % Date2 + Interp % data1 = Interp % data2 + temp1(:,:) = temp2(:,:) ! SJL BUG fix: June 24, 2011 + else + call read_record ('sst', Date1, Udate1, temp1) + if ( use_ncep_sst .and. (.not. no_anom_sst) ) then + temp1(:,:) = temp1(:,:) + sst_anom(:,:) + endif + call horiz_interp ( Interp%Hintrp, temp1, Interp%data1 ) + call clip_data ('sst', Interp%data1) + Interp % Date1 = Date1 + endif + endif !----------------------------------------------------------------------- - if (Date2 /= Interp % Date2) then - call read_record ('sst', Date2, Udate2, temp2) - if ( use_ncep_sst .and. (.not. no_anom_sst) ) then - temp2(:,:) = temp2(:,:) + sst_anom(:,:) - endif - call horiz_interp ( Interp%Hintrp, temp2, Interp%data2 ) - call clip_data ('sst', Interp%data2) - Interp % Date2 = Date2 - endif + if (Date2 /= Interp % Date2) then + call read_record ('sst', Date2, Udate2, temp2) + if ( use_ncep_sst .and. (.not. no_anom_sst) ) then + temp2(:,:) = temp2(:,:) + sst_anom(:,:) + endif + call horiz_interp ( Interp%Hintrp, temp2, Interp%data2 ) + call clip_data ('sst', Interp%data2) + Interp % Date2 = Date2 + endif ! ---- if the unit was opened, close it and print dates ---- - if (unit /= -1) then - call close_file (unit) - if (verbose > 0 .and. mpp_pe() == 0) & - call print_dates (Amip_Time, & - Interp % Date1, Udate1, & - Interp % Date2, Udate2, fmonth) - endif + if (unit /= -1) then + call close_file (unit) + if (verbose > 0 .and. mpp_pe() == 0) & + call print_dates (Amip_Time, & + Interp % Date1, Udate1, & + Interp % Date2, Udate2, fmonth) + endif !----------------------------------------------------------------------- !---------- time interpolation (between months) of sst's --------------- !----------------------------------------------------------------------- - sst = Interp % data1 + fmonth * (Interp % data2 - Interp % data1) + sst = Interp % data1 + fmonth * (Interp % data2 - Interp % data1) !------------------------------------------------------------------------------- ! SJL mods for NWP and TCSF --- ! Nudging runs: (Note: NCEP SST updated only every 6-hr) ! Compute SST anomaly from global SST datasets for subsequent forecast runs !------------------------------------------------------------------------------- - if ( use_ncep_sst .and. no_anom_sst ) then - sst_anom(:,:) = sst_ncep(:,:) - (temp1(:,:) + fmonth*(temp2(:,:) - temp1(:,:)) ) - call horiz_interp ( Interp%Hintrp, sst_ncep, sst ) - call clip_data ('sst', sst) + if ( use_ncep_sst .and. no_anom_sst ) then + sst_anom(:,:) = sst_ncep(:,:) - (temp1(:,:) + fmonth*(temp2(:,:) - temp1(:,:)) ) + call horiz_interp ( Interp%Hintrp, sst_ncep, sst ) + call clip_data ('sst', sst) + endif + +!!! DEBUG CODE +! call get_date(Amip_Time,jhctod(1),jhctod(2),jhctod(3),jhctod(4),jhctod(5),jhctod(6)) +! if (mpp_pe() == 0) then +! write (*,200) 'JHC: use_daily = F, AMIP_Time: ',jhctod(1),jhctod(2),jhctod(3),jhctod(4),jhctod(5),jhctod(6) +! write (*,300) 'JHC: use_daily = F, interped SST: ', sst(1,1),sst(5,5),sst(10,10) +! endif +!!! END DEBUG CODE + + endif +! add by JHC +else + call get_date(Amip_Time,jhctod(1),jhctod(2),jhctod(3),jhctod(4),jhctod(5),jhctod(6)) + if (mpp_pe() == mpp_root_pe()) write(*,200) 'amip_interp_mod: use_daily = T, Amip_Time = ',jhctod(1),jhctod(2),jhctod(3),jhctod(4),jhctod(5),jhctod(6) + + yr = jhctod(1); mo = jhctod(2); dy = jhctod(3) + + write (yyyy,'(i4)') jhctod(1) -endif + file_name_sst = 'INPUT/' // 'sst.day.mean.'//yyyy//'.v2.nc' + ncfilename = trim(file_name_sst) + time_unit = 'days since 1978-01-01 00:00:00' -! add on non-zero sea surface temperature perturbation (namelist option) -! this perturbation may be useful in accessing model sensitivities + mobs_sst = 1440; nobs_sst = 720 + + call set_sst_grid_edges_daily(mobs_sst, nobs_sst) + call horiz_interp_new ( Interp%Hintrp2, lon_bnd, lat_bnd, & + lon_model, lat_model, interp_method="bilinear" ) - if ( abs(sst_pert) > 0.0001 ) then - sst = sst + sst_pert - endif + if ( (.NOT. file_exist(ncfilename)) ) call mpp_error ('amip_interp_mod', & + 'cannot find daily SST input data file: '//trim(ncfilename), NOTE) + + if (file_exist(ncfilename)) then + if (mpp_pe() == mpp_root_pe()) call mpp_error ('amip_interp_mod', & + 'Reading NetCDF formatted daily SST from: '//trim(ncfilename), NOTE) + + call field_size(ncfilename, 'TIME', siz) + nrecords = siz (1) + if (nrecords < 1) call mpp_error('amip_interp_mod', & + 'Invalid number of SST records in daily SST data file: '//trim(ncfilename), FATAL) + allocate(timeval(nrecords), ryr(nrecords), rmo(nrecords), rdy(nrecords)) + + call mpp_open( unit, ncfilename, MPP_RDONLY, MPP_NETCDF, MPP_MULTI, MPP_SINGLE ) + call mpp_get_times(unit, timeval) + call mpp_close(unit) + +!!! DEBUG CODE +! if (mpp_pe() == 0) then +! print *, 'JHC: nrecords = ', nrecords +! print *, 'JHC: TIME = ', timeval +! endif +!!! END DEBUG CODE + + ierr = 1 + do k = 1, nrecords + + Udate = get_cal_time (timeval(k), time_unit, 'julian') + call get_date(Udate,jhctod(1),jhctod(2),jhctod(3),jhctod(4),jhctod(5),jhctod(6)) + ryr(k) = jhctod(1); rmo(k) = jhctod(2); rdy(k) = jhctod(3) + + if ( yr == ryr(k) .and. mo == rmo(k) .and. dy == rdy (k) ) ierr = 0 + if (ierr==0) exit + + enddo +!!! DEBUG CODE + if (mpp_pe() == 0) then + print *, 'JHC: k =', k + print *, 'JHC: ryr(k) rmo(k) rdy(k)',ryr(k), rmo(k), rdy(k) + print *, 'JHC: yr mo dy ',yr, mo, dy + endif +!!! END DEBUG CODE + if (ierr .ne. 0) call mpp_error('amip_interp_mod', & + 'Model time is out of range not in SST data: '//trim(ncfilename), FATAL) + endif ! if(file_exist(ncfilename)) + + + !---- read NETCDF data ---- + if ( .not. allocated(tempamip) ) allocate (tempamip(mobs_sst,nobs_sst)) + + if (file_exist(ncfilename)) then + call read_data(ncfilename, 'SST', tempamip, timelevel=k, no_domain=.true.) + tempamip = tempamip + TFREEZE + +!!! DEBUG CODE +! if (mpp_pe() == 0) then +! print*, 'JHC: TFREEZE = ', TFREEZE +! print*, lbound(sst) +! print*, ubound(sst) +! print*, lbound(tempamip) +! print*, ubound(tempamip) +! write(*,300) 'JHC: tempamip : ', tempamip(100,100), tempamip(200,200), tempamip(300,300) +! endif +!!! END DEBUG CODE + + call horiz_interp ( Interp%Hintrp2, tempamip, sst ) + call clip_data ('sst', sst) + + endif + +!!! DEBUG CODE +! if (mpp_pe() == 400) then +! write(*,300)'JHC: use_daily = T, daily SST: ', sst(1,1),sst(5,5),sst(10,10) +! print *,'JHC: use_daily = T, daily SST: ', sst +! endif +!!! END DEBUG CODE + +200 format(a35, 6(i5,1x)) +300 format(a35, 3(f7.3,2x)) + +endif +! end add by JHC + +! add by JHC: add on non-zero sea surface temperature perturbation (namelist option) +! This perturbation may be useful in accessing model sensitivities + + if ( do_sst_pert ) then + + if ( trim(sst_pert_type) == 'fixed' ) then + sst = sst + sst_pert + else if ( trim(sst_pert_type) == 'random' ) then + call random_seed() +!!! DEBUG CODE +! if (mpp_pe() == 0) then +! print*, 'mobs = ', mobs +! print*, 'nobs = ', nobs +! print*, lbound(sst) +! print*, ubound(sst) +! endif +!!! END DEBUG CODE + do i = 1, size(sst,1) + do j = 1, size(sst,2) + call random_number(pert) + sst (i,j) = sst (i,j) + sst_pert*((pert-0.5)*2) + end do + end do + endif + + endif +! end add by JHC + !----------------------------------------------------------------------- end subroutine get_amip_sst - !####################################################################### ! ! @@ -771,7 +945,7 @@ subroutine amip_interp_init() #endif ! ----- write namelist/version info ----- - call write_version_number() + call write_version_number (version, tagname) unit = stdlog ( ) if (mpp_pe() == 0) then @@ -819,7 +993,17 @@ subroutine amip_interp_init() tice_crit_k = 271.38 if (mpp_pe() == 0) & call error_mesg ('amip_interp_init', 'using HURRELL sst', NOTE) - Date_end = date_type( 2001, 12, 0 ) + Date_end = date_type( 2011, 8, 16 ) ! updated by JHC +! add by JHC + else if (lowercase(trim(data_set)) == 'daily') then + file_name_sst = 'INPUT/' // 'hurrell_sst.data' + file_name_ice = 'INPUT/' // 'hurrell_ice.data' + mobs = 360; nobs = 180 + call set_sst_grid_edges_oi + if (mpp_pe() == 0) & + call error_mesg ('amip_interp_init', 'using AVHRR daily sst', NOTE) + Date_end = date_type( 2011, 8, 16 ) +! end add by JHC else if (lowercase(trim(data_set)) == 'reynolds_eof') then file_name_sst = 'INPUT/' // 'reynolds_sst.data' file_name_ice = 'INPUT/' // 'reynolds_sst.data' @@ -937,12 +1121,15 @@ subroutine set_sst_grid_edges_amip1 end subroutine set_sst_grid_edges_amip1 !####################################################################### - subroutine set_sst_grid_edges_oi integer :: i, j real :: hpie, dlon, dlat, wb, sb +! add by JHC + if(allocated(lon_bnd)) deallocate(lon_bnd) + if(allocated(lat_bnd)) deallocate(lat_bnd) +! end add by JHC allocate ( lon_bnd(mobs+1), lat_bnd(nobs+1) ) ! ---- compute grid edges (do only once) ----- @@ -963,6 +1150,37 @@ subroutine set_sst_grid_edges_oi enddo end subroutine set_sst_grid_edges_oi +!####################################################################### +! add by JHC + subroutine set_sst_grid_edges_daily(mobs_sst, nobs_sst) + + integer :: i, j, mobs_sst, nobs_sst + real :: hpie, dlon, dlat, wb, sb + + if(allocated(lon_bnd)) deallocate(lon_bnd) + if(allocated(lat_bnd)) deallocate(lat_bnd) + allocate ( lon_bnd(mobs_sst+1), lat_bnd(nobs_sst+1) ) + +! ---- compute grid edges (do only once) ----- + + hpie = 0.5*pi + + dlon = 4.*hpie/float(mobs_sst); wb = 0.0 + lon_bnd(1) = wb + do i = 2, mobs_sst+1 + lon_bnd(i) = wb + dlon * float(i-1) + enddo + lon_bnd(mobs_sst+1) = lon_bnd(1) + 4.*hpie + + dlat = 2.*hpie/float(nobs_sst); sb = -hpie + lat_bnd(1) = sb; lat_bnd(nobs_sst+1) = hpie + do j = 2, nobs_sst + lat_bnd(j) = sb + dlat * float(j-1) + enddo + + end subroutine set_sst_grid_edges_daily +! end add by JHC +!####################################################################### subroutine a2a_bilinear(nx, ny, dat1, n1, n2, dat2) @@ -1191,7 +1409,8 @@ subroutine read_record (type, Date, Adate, dat) else if(type(1:3) == 'ice') then ncfilename = trim(file_name_ice)//'.nc' if (lowercase(trim(data_set)) == 'amip2' .or. & - lowercase(trim(data_set)) == 'hurrell') ncfieldname = 'ice' + lowercase(trim(data_set)) == 'hurrell' .or. & + lowercase(trim(data_set)) == 'daily') ncfieldname = 'ice' ! modified by JHC endif !---- make sure IEEE format file is open ---- diff --git a/src/shared/astronomy/astronomy.F90 b/src/shared/astronomy/astronomy.F90 index 19bf449615..09f2593028 100644 --- a/src/shared/astronomy/astronomy.F90 +++ b/src/shared/astronomy/astronomy.F90 @@ -1,9 +1,10 @@ module astronomy_mod -! +! ! fil ! ! ! +! ! ! astronomy_mod provides astronomical variables for use ! by other modules within fms. the only currently used interface is @@ -45,8 +46,8 @@ module astronomy_mod !--------------------------------------------------------------------- !----------- version number for this module -------------------------- -character(len=128) :: version = '$Id: astronomy.F90,v 20.0 2013/12/14 00:18:17 fms Exp $' -character(len=128) :: tagname = '$Name: tikal $' +character(len=128) :: version = '$Id$' +character(len=128) :: tagname = '$Name$' !--------------------------------------------------------------------- @@ -287,7 +288,7 @@ subroutine astronomy_init (latb, lonb) !--------------------------------------------------------------------- ! write version number and namelist to logfile. !--------------------------------------------------------------------- - call write_version_number() + call write_version_number (version, tagname) if (mpp_pe() == mpp_root_pe() ) then unit = stdlog() write (unit, nml=astronomy_nml) @@ -2988,9 +2989,9 @@ subroutine astronomy_end !---------------------------------------------------------------------- ! check if the module has been initialized. !---------------------------------------------------------------------- - if (.not. module_is_initialized) & - call error_mesg ( 'astronomy_mod', & - ' module has not been initialized', FATAL) + if (.not. module_is_initialized) return +! call error_mesg ( 'astronomy_mod', & +! ' module has not been initialized', FATAL) !---------------------------------------------------------------------- ! deallocate module variables. @@ -3152,7 +3153,7 @@ function r_inv_squared (ang) ! its square (r_inv_squared) to the calling routine. !-------------------------------------------------------------------- rad_per = per*deg_to_rad - r = (1 - ecc**2)/(1. + ecc*cos(ang - rad_per)) + r = (1. - ecc**2)/(1. + ecc*cos(ang - rad_per)) r_inv_squared = r**(-2) diff --git a/src/shared/axis_utils/axis_utils.F90 b/src/shared/axis_utils/axis_utils.F90 index 59f3b42db6..1947234315 100644 --- a/src/shared/axis_utils/axis_utils.F90 +++ b/src/shared/axis_utils/axis_utils.F90 @@ -1,8 +1,8 @@ module axis_utils_mod ! - !M.J. Harrison + !M.J. Harrison ! - !Bruce Wyman + !Bruce Wyman ! ! @@ -27,9 +27,9 @@ module axis_utils_mod use mpp_io_mod, only: axistype, atttype, default_axis, default_att, & mpp_get_atts, mpp_get_axis_data, mpp_modify_meta, & mpp_get_att_name, mpp_get_att_type, mpp_get_att_char, & - mpp_get_att_length + mpp_get_att_length, mpp_get_axis_bounds use mpp_mod, only: mpp_error, FATAL, stdout - use fms_mod, only: lowercase, string_array_index + use fms_mod, only: lowercase, string_array_index, fms_error_handler implicit none @@ -43,8 +43,8 @@ module axis_utils_mod integer, parameter :: maxatts = 100 real, parameter :: epsln= 1.e-10 real, parameter :: fp5 = 0.5, f360 = 360.0 - character(len=256) :: version = '$Id: axis_utils.F90,v 20.0 2013/12/14 00:18:21 fms Exp $' - character(len=256) :: tagname = '$Name: tikal $' + character(len=256) :: version = '$Id$' + character(len=256) :: tagname = '$Name$' interface interp_1d module procedure interp_1d_1d @@ -125,52 +125,39 @@ subroutine get_axis_cart(axis, cart) end subroutine get_axis_cart - subroutine get_axis_bounds(axis,axis_bound,axes) + subroutine get_axis_bounds(axis,axis_bound,axes,bnd_name,err_msg) type(axistype), intent(in) :: axis type(axistype), intent(inout) :: axis_bound type(axistype), intent(in), dimension(:) :: axes + character(len=*), intent(out), optional :: bnd_name, err_msg - type(atttype), dimension(:), allocatable :: att real, dimension(:), allocatable :: data, tmp integer :: i, len - character(len=128) :: bounds_name = 'none', name, units + character(len=128) :: name, units character(len=256) :: longname character(len=1) :: cartesian + logical :: bounds_found + if(present(err_msg)) then + err_msg = '' + endif axis_bound = default_axis - allocate(att(maxatts)) - att = default_att - call mpp_get_atts(axis,atts=att) - - do i=1,maxatts - if (mpp_get_att_type(att(i)) == NF_CHAR) then - ! if (str_contains(att(i)%name,'bounds') .or. str_contains(att(i)%name,'edge')) then - if (string_array_index('bounds',(/mpp_get_att_name(att(i))/)) .or. & - string_array_index('edge',(/mpp_get_att_name(att(i))/))) then - bounds_name = mpp_get_att_char(att(i)) - endif - endif - enddo + call mpp_get_atts(axis,units=units,longname=longname,& + cartesian=cartesian, len=len) + if(len .LE. 0) return + allocate(data(len+1)) - if (trim(bounds_name) /= 'none') then - do i=1,size(axes(:)) - call mpp_get_atts(axes(i),name=name) - if (lowercase(trim(name)) == lowercase(trim(bounds_name))) then - axis_bound = axes(i) - endif - enddo - call mpp_get_atts(axis_bound,len=len) - if (len < 1) call mpp_error(FATAL,'error locating boundary axis for '//bounds_name) - else - call mpp_get_atts(axis,name=name,units=units,longname=longname,& - cartesian=cartesian,len=len) + bounds_found = mpp_get_axis_bounds(axis, data, name=name) + longname = trim(longname)//' bounds' + + if(.not.bounds_found .and. len>1 ) then + ! The following calculation can not be done for len=1 + call mpp_get_atts(axis,name=name) name = trim(name)//'_bounds' - longname = trim(longname)//' bounds' allocate(tmp(len)) call mpp_get_axis_data(axis,tmp) - allocate(data(len+1)) do i=2,len data(i)= tmp(i-1)+fp5*(tmp(i)-tmp(i-1)) enddo @@ -180,11 +167,13 @@ subroutine get_axis_bounds(axis,axis_bound,axes) if (data(1) == 0.0) then if (abs(data(len+1)-360.) > epsln) data(len+1)=360.0 endif + endif + if(bounds_found .OR. len>1) then call mpp_modify_meta(axis_bound,name=name,units=units,longname=& - longname,cartesian=cartesian,data=data) - deallocate(tmp) - deallocate(data) + longname,cartesian=cartesian,data=data) endif + if(allocated(tmp)) deallocate(tmp) + deallocate(data) return end subroutine get_axis_bounds @@ -621,7 +610,7 @@ subroutine interp_1d_cubic_spline(grid1, grid2, data1, data2, yp1, ypn) h = grid1(khi)-grid1(klo) a = (grid1(khi) - grid2(k))/h b = (grid2(k) - grid1(klo))/h - data2(k) = a*data1(klo) + b*data1(khi)+ ((a**3-a)*y2(klo) + (b**3-b)*y2(khi))*(h**2)/6 + data2(k) = a*data1(klo) + b*data1(khi)+ ((a**3-a)*y2(klo) + (b**3-b)*y2(khi))*(h**2)/6. enddo end subroutine interp_1d_cubic_spline diff --git a/src/shared/block_control/block_control.F90 b/src/shared/block_control/block_control.F90 new file mode 100644 index 0000000000..518f92f354 --- /dev/null +++ b/src/shared/block_control/block_control.F90 @@ -0,0 +1,90 @@ +module block_control_mod +#include + +use mpp_mod, only: mpp_error, NOTE, FATAL +use mpp_domains_mod, only: mpp_compute_extent + + public block_control_type + type block_control_type + integer :: nx_block, ny_block + integer :: nblks + integer :: isc, iec, jsc, jec + integer :: npz + integer, dimension(:), _ALLOCATABLE :: ibs _NULL, & + ibe _NULL, & + jbs _NULL, & + jbe _NULL + end type block_control_type + +public :: define_blocks + +contains + +!---------------------------------------------------------------------- +! set up "blocks" used for OpenMP threading of column-based +! calculations using rad_n[x/y]xblock from coupler_nml +!--------------------------------------------------------------------- + subroutine define_blocks (component, Block, isc, iec, jsc, jec, kpts, & + nx_block, ny_block, message) + character(len=*), intent(in) :: component + type(block_control_type), intent(inout) :: Block + integer, intent(in) :: isc, iec, jsc, jec, kpts + integer, intent(in) :: nx_block, ny_block + logical, intent(inout) :: message +!--- local variables + integer :: blocks + integer, dimension(nx_block) :: i1, i2 + integer, dimension(ny_block) :: j1, j2 + character(len=132) :: text + integer :: i, j, nblks + + if (message) then + if ((mod(iec-isc+1,nx_block) .ne. 0) .or. (mod(jec-jsc+1,ny_block) .ne. 0)) then + write( text,'(a,a,2i4,a,2i4,a)' ) trim(component),'define_blocks: domain (',& + (iec-isc+1), (jec-jsc+1),') is not an even divisor with definition (',& + nx_block, ny_block,') - blocks will not be uniform' + call mpp_error( NOTE, trim(text) ) + endif + message = .false. + endif + +!--- set up blocks + if (iec-isc+1 .lt. nx_block) & + call mpp_error(FATAL, 'block_control: number of '//trim(component)//' nxblocks .gt. & + &number of elements in MPI-domain size') + if (jec-jsc+1 .lt. ny_block) & + call mpp_error(FATAL, 'block_control: number of '//trim(component)//' nyblocks .gt. & + &number of elements in MPI-domain size') + call mpp_compute_extent(isc,iec,nx_block,i1,i2) + call mpp_compute_extent(jsc,jec,ny_block,j1,j2) + + nblks = nx_block*ny_block + Block%isc = isc + Block%iec = iec + Block%jsc = jsc + Block%jec = jec + Block%npz = kpts + Block%nx_block = nx_block + Block%ny_block = ny_block + Block%nblks = nblks + + if (.not._ALLOCATED(Block%ibs)) & + allocate (Block%ibs(nblks), & + Block%ibe(nblks), & + Block%jbs(nblks), & + Block%jbe(nblks)) + + blocks=0 + do j = 1, ny_block + do i = 1, nx_block + blocks = blocks + 1 + Block%ibs(blocks) = i1(i) + Block%jbs(blocks) = j1(j) + Block%ibe(blocks) = i2(i) + Block%jbe(blocks) = j2(j) + enddo + enddo + + end subroutine define_blocks + +end module block_control_mod diff --git a/src/shared/column_diagnostics/column_diagnostics.F90 b/src/shared/column_diagnostics/column_diagnostics.F90 index 69937d910d..8e115901b1 100644 --- a/src/shared/column_diagnostics/column_diagnostics.F90 +++ b/src/shared/column_diagnostics/column_diagnostics.F90 @@ -34,8 +34,8 @@ module column_diagnostics_mod !----------- ****** VERSION NUMBER ******* --------------------------- -character(len=128) :: version = '$Id: column_diagnostics.F90,v 20.0 2013/12/14 00:18:24 fms Exp $' -character(len=128) :: tag = '$Name: tikal $' +character(len=128) :: version = '$Id$' +character(len=128) :: tag = '$Name$' @@ -145,7 +145,7 @@ subroutine column_diagnostics_init !--------------------------------------------------------------------- ! write version number and namelist to logfile. !--------------------------------------------------------------------- - call write_version_number() + call write_version_number (version, tag) if (mpp_pe() == mpp_root_pe()) then unit = stdlog() write (unit, nml=column_diagnostics_nml) diff --git a/src/shared/constants/constants.F90 b/src/shared/constants/constants.F90 index 7f02965fb6..539fbac2e0 100644 --- a/src/shared/constants/constants.F90 +++ b/src/shared/constants/constants.F90 @@ -1,10 +1,11 @@ module constants_mod -! +! ! Bruce Wyman ! +! ! ! Defines useful constants for Earth. @@ -18,8 +19,8 @@ module constants_mod implicit none private -character(len=128) :: version='$Id: constants.F90,v 17.0 2009/07/21 03:18:26 fms Exp $' -character(len=128) :: tagname='$Name: tikal $' +character(len=128) :: version='$Id$' +character(len=128) :: tagname='$Name$' !dummy variable to use in HUGE initializations real :: realnumber diff --git a/src/shared/coupler/atmos_ocean_fluxes.F90 b/src/shared/coupler/atmos_ocean_fluxes.F90 index 35c2053af2..fb6024eb5e 100644 --- a/src/shared/coupler/atmos_ocean_fluxes.F90 +++ b/src/shared/coupler/atmos_ocean_fluxes.F90 @@ -19,10 +19,10 @@ !----------------------------------------------------------------------- ! ! -! Richard D. Slater +! Richard D. Slater ! ! -! John P. Dunne +! John P. Dunne ! ! ! @@ -188,8 +188,8 @@ module atmos_ocean_fluxes_mod !{ !---------------------------------------------------------------------- ! -character(len=128) :: version = '$Id: atmos_ocean_fluxes.F90,v 18.0 2010/03/02 23:55:03 fms Exp $' -character(len=128) :: tagname = '$Name: tikal $' +character(len=128) :: version = '$Id$' +character(len=128) :: tagname = '$Name$' ! !----------------------------------------------------------------------- @@ -408,12 +408,12 @@ function aof_set_coupler_flux(name, flux_type, implementation, atm_tr_index, par endif else if (implementation .eq. implementation_test) then - long_err_msg = 'Undefined flux_type/implementation (flux_type given from field_table): ' - long_err_msg = long_err_msg // trim(flux_type_test) // '/implementation/' // trim(implementation_test) + long_err_msg = 'Undefined flux_type/implementation (flux_type given from field_table): ' + long_err_msg = long_err_msg // trim(flux_type_test) // '/implementation/' // trim(implementation_test) call mpp_error(FATAL, trim(error_header) // long_err_msg) else - long_err_msg = ' Undefined flux_type/implementation (both given from field_table): ' - long_err_msg = long_err_msg // trim(flux_type_test) // '/implementation/' // trim(implementation_test) + long_err_msg = ' Undefined flux_type/implementation (both given from field_table): ' + long_err_msg = long_err_msg // trim(flux_type_test) // '/implementation/' // trim(implementation_test) call mpp_error(FATAL, trim(error_header) // long_err_msg) endif endif @@ -968,10 +968,6 @@ subroutine atmos_ocean_fluxes_calc(gas_fields_atm, gas_fields_ice, & character(len=64), parameter :: sub_name = 'atmos_ocean_fluxes_calc' character(len=256), parameter :: error_header = & '==>Error from ' // trim(mod_name) // '(' // trim(sub_name) // '):' -character(len=256), parameter :: warn_header = & - '==>Warning from ' // trim(mod_name) // '(' // trim(sub_name) // '):' -character(len=256), parameter :: note_header = & - '==>Note from ' // trim(mod_name) // '(' // trim(sub_name) // '):' ! !----------------------------------------------------------------------- @@ -1037,14 +1033,14 @@ subroutine atmos_ocean_fluxes_calc(gas_fields_atm, gas_fields_ice, & if (gas_fluxes%bc(n)%implementation .eq. 'ocmip2') then !}{ do i = 1, length !{ - if (seawater(i) == 1) then !{ + if (seawater(i) == 1.) then !{ gas_fluxes%bc(n)%field(ind_kw)%values(i) = gas_fluxes%bc(n)%param(1) * gas_fields_atm%bc(n)%field(ind_u10)%values(i)**2 cair(i) = & gas_fields_ice%bc(n)%field(ind_alpha)%values(i) * & gas_fields_atm%bc(n)%field(ind_pCair)%values(i) * & gas_fields_atm%bc(n)%field(ind_psurf)%values(i) * gas_fluxes%bc(n)%param(2) gas_fluxes%bc(n)%field(ind_flux)%values(i) = gas_fluxes%bc(n)%field(ind_kw)%values(i) * & - sqrt(660 / (gas_fields_ice%bc(n)%field(ind_sc_no)%values(i) + epsln)) * & + sqrt(660. / (gas_fields_ice%bc(n)%field(ind_sc_no)%values(i) + epsln)) * & (gas_fields_ice%bc(n)%field(ind_csurf)%values(i) - cair(i)) gas_fluxes%bc(n)%field(ind_deltap)%values(i) = (gas_fields_ice%bc(n)%field(ind_csurf)%values(i) - cair(i)) / & (gas_fields_ice%bc(n)%field(ind_alpha)%values(i) * permeg + epsln) @@ -1077,7 +1073,7 @@ subroutine atmos_ocean_fluxes_calc(gas_fields_atm, gas_fields_ice, & if (gas_fluxes%bc(n)%implementation .eq. 'ocmip2_data') then !{ do i = 1, length !{ - if (seawater(i) == 1) then !{ + if (seawater(i) == 1.) then !{ kw(i) = gas_fluxes%bc(n)%param(1) * gas_fields_atm%bc(n)%field(ind_u10)%values(i) cair(i) = & gas_fields_ice%bc(n)%field(ind_alpha)%values(i) * & @@ -1095,7 +1091,7 @@ subroutine atmos_ocean_fluxes_calc(gas_fields_atm, gas_fields_ice, & elseif (gas_fluxes%bc(n)%implementation .eq. 'ocmip2') then !}{ do i = 1, length !{ - if (seawater(i) == 1) then !{ + if (seawater(i) == 1.) then !{ kw(i) = gas_fluxes%bc(n)%param(1) * gas_fields_atm%bc(n)%field(ind_u10)%values(i)**2 cair(i) = & gas_fields_ice%bc(n)%field(ind_alpha)%values(i) * & @@ -1113,7 +1109,7 @@ subroutine atmos_ocean_fluxes_calc(gas_fields_atm, gas_fields_ice, & elseif (gas_fluxes%bc(n)%implementation .eq. 'linear') then !}{ do i = 1, length !{ - if (seawater(i) == 1) then !{ + if (seawater(i) == 1.) then !{ kw(i) = gas_fluxes%bc(n)%param(1) * max(0.0, gas_fields_atm%bc(n)%field(ind_u10)%values(i) - gas_fluxes%bc(n)%param(2)) cair(i) = & gas_fields_ice%bc(n)%field(ind_alpha)%values(i) * & @@ -1147,7 +1143,7 @@ subroutine atmos_ocean_fluxes_calc(gas_fields_atm, gas_fields_ice, & if (gas_fluxes%bc(n)%implementation .eq. 'dry') then !{ do i = 1, length !{ - if (seawater(i) == 1) then !{ + if (seawater(i) == 1.) then !{ gas_fluxes%bc(n)%field(ind_flux)%values(i) = & gas_fields_atm%bc(n)%field(ind_deposition)%values(i) / gas_fluxes%bc(n)%param(1) else !}{ @@ -1158,7 +1154,7 @@ subroutine atmos_ocean_fluxes_calc(gas_fields_atm, gas_fields_ice, & elseif (gas_fluxes%bc(n)%implementation .eq. 'wet') then !}{ do i = 1, length !{ - if (seawater(i) == 1) then !{ + if (seawater(i) == 1.) then !{ gas_fluxes%bc(n)%field(ind_flux)%values(i) = & gas_fields_atm%bc(n)%field(ind_deposition)%values(i) / gas_fluxes%bc(n)%param(1) else !}{ @@ -1186,7 +1182,7 @@ subroutine atmos_ocean_fluxes_calc(gas_fields_atm, gas_fields_ice, & if (gas_fluxes%bc(n)%implementation .eq. 'river') then !{ do i = 1, length !{ - if (seawater(i) == 1) then !{ + if (seawater(i) == 1.) then !{ gas_fluxes%bc(n)%field(ind_flux)%values(i) = & gas_fields_atm%bc(n)%field(ind_deposition)%values(i) / gas_fluxes%bc(n)%param(1) else !}{ diff --git a/src/shared/coupler/coupler_types.F90 b/src/shared/coupler/coupler_types.F90 index 8cf23970df..713407ba34 100644 --- a/src/shared/coupler/coupler_types.F90 +++ b/src/shared/coupler/coupler_types.F90 @@ -19,11 +19,11 @@ module coupler_types_mod !{ ! or see: http://www.gnu.org/licenses/gpl.html !----------------------------------------------------------------------- ! -! +! ! Richard D. Slater ! ! -! +! ! John Dunne ! ! @@ -171,8 +171,8 @@ module coupler_types_mod !{ implicit none ! !----------------------------------------------------------------------- - character(len=128) :: version = '$Id: coupler_types.F90,v 18.0 2010/03/02 23:55:06 fms Exp $' - character(len=128) :: tag = '$Name: tikal $' + character(len=128) :: version = '$Id$' + character(len=128) :: tag = '$Name$' !----------------------------------------------------------------------- real, parameter :: bound_tol = 1e-7 @@ -253,7 +253,7 @@ module coupler_types_mod !{ logical :: use_atm_pressure logical :: use_10m_wind_speed logical :: pass_through_ice - real :: mol_wt = 0.0 + real :: mol_wt = 0.0 end type coupler_3d_field_type type, public :: coupler_3d_bc_type !{ @@ -289,7 +289,7 @@ module coupler_types_mod !{ logical :: use_atm_pressure logical :: use_10m_wind_speed logical :: pass_through_ice - real :: mol_wt = 0.0 + real :: mol_wt = 0.0 end type coupler_2d_field_type type, public :: coupler_2d_bc_type !{ @@ -325,7 +325,7 @@ module coupler_types_mod !{ logical :: use_atm_pressure logical :: use_10m_wind_speed logical :: pass_through_ice - real :: mol_wt = 0.0 + real :: mol_wt = 0.0 end type coupler_1d_field_type type, public :: coupler_1d_bc_type !{ @@ -439,10 +439,6 @@ subroutine coupler_types_init character(len=64), parameter :: sub_name = 'coupler_types_init' character(len=256), parameter :: error_header = & '==>Error from ' // trim(mod_name) // '(' // trim(sub_name) // '):' -character(len=256), parameter :: warn_header = & - '==>Warning from ' // trim(mod_name) // '(' // trim(sub_name) // '):' -character(len=256), parameter :: note_header = & - '==>Note from ' // trim(mod_name) // '(' // trim(sub_name) // '):' ! !----------------------------------------------------------------------- @@ -976,10 +972,6 @@ subroutine coupler_type_copy_1d_2d(var_in, var_out, is, ie, js, je, & character(len=64), parameter :: sub_name = 'coupler_type_copy_1d_2d' character(len=256), parameter :: error_header = & '==>Error from ' // trim(mod_name) // '(' // trim(sub_name) // '):' -character(len=256), parameter :: warn_header = & - '==>Warning from ' // trim(mod_name) // '(' // trim(sub_name) // '):' -character(len=256), parameter :: note_header = & - '==>Note from ' // trim(mod_name) // '(' // trim(sub_name) // '):' ! !----------------------------------------------------------------------- @@ -1156,10 +1148,6 @@ subroutine coupler_type_copy_1d_3d(var_in, var_out, is, ie, js, je, kd, & character(len=64), parameter :: sub_name = 'coupler_type_copy_1d_3d' character(len=256), parameter :: error_header = & '==>Error from ' // trim(mod_name) // '(' // trim(sub_name) // '):' -character(len=256), parameter :: warn_header = & - '==>Warning from ' // trim(mod_name) // '(' // trim(sub_name) // '):' -character(len=256), parameter :: note_header = & - '==>Note from ' // trim(mod_name) // '(' // trim(sub_name) // '):' ! !----------------------------------------------------------------------- diff --git a/src/shared/coupler/ensemble_manager.F90 b/src/shared/coupler/ensemble_manager.F90 index 394ceed49f..cf67973fab 100644 --- a/src/shared/coupler/ensemble_manager.F90 +++ b/src/shared/coupler/ensemble_manager.F90 @@ -6,7 +6,6 @@ module ensemble_manager_mod use mpp_mod, only : mpp_pe, mpp_declare_pelist use mpp_mod, only : input_nml_file use fms_io_mod, only : set_filename_appendix - use diag_manager_mod, only : set_diag_filename_appendix IMPLICIT NONE @@ -358,9 +357,6 @@ subroutine ensemble_pelist_setup(concurrent, atmos_npes, ocean_npes, land_npes, write( text,'(a,i2.2)' ) 'ens_', ensemble_id !Append ensemble_id to the restart filenames call set_filename_appendix(trim(text)) - !Append ensemble_id to the diag_out filenames - write( text,'(a,i2.2)' ) '.ens_', ensemble_id - call set_diag_filename_appendix(trim(text)) endif end subroutine ensemble_pelist_setup diff --git a/src/shared/data_override/data_override.F90 b/src/shared/data_override/data_override.F90 index 684a211027..10e4d1145c 100644 --- a/src/shared/data_override/data_override.F90 +++ b/src/shared/data_override/data_override.F90 @@ -24,15 +24,15 @@ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! module data_override_mod ! -! +! ! G.T. Nong ! ! -! +! ! M.J. Harrison ! ! -! +! ! M. Winton ! @@ -80,8 +80,8 @@ module data_override_mod implicit none private -character(len=128) :: version = '$Id: data_override.F90,v 20.0 2013/12/14 00:18:34 fms Exp $' -character(len=128) :: tagname = '$Name: tikal $' +character(len=128) :: version = '$Id$' +character(len=128) :: tagname = '$Name$' type data_type character(len=3) :: gridname @@ -224,7 +224,7 @@ subroutine data_override_init(Atm_domain_in, Ocean_domain_in, Ice_domain_in, Lan radian_to_deg = 180./PI deg_to_radian = PI/180. - call write_version_number() + call write_version_number (version, tagname) ! Initialize user-provided data table default_table%gridname = 'none' @@ -371,27 +371,27 @@ subroutine data_override_init(Atm_domain_in, Ocean_domain_in, Ice_domain_in, Lan if(file_open) call mpp_error(FATAL, trim(grid_file)//' already opened') if(field_exist(grid_file, "x_T" ) .OR. field_exist(grid_file, "geolon_t" ) ) then - if (atm_on) then + if (atm_on .and. .not. allocated(lon_local_atm) ) then call mpp_get_compute_domain( atm_domain,is,ie,js,je) allocate(lon_local_atm(is:ie,js:je), lat_local_atm(is:ie,js:je)) call get_grid_version_1(grid_file, 'atm', atm_domain, is, ie, js, je, lon_local_atm, lat_local_atm, & min_glo_lon_atm, max_glo_lon_atm ) endif - if (ocn_on) then + if (ocn_on .and. .not. allocated(lon_local_ocn) ) then call mpp_get_compute_domain( ocn_domain,is,ie,js,je) allocate(lon_local_ocn(is:ie,js:je), lat_local_ocn(is:ie,js:je)) call get_grid_version_1(grid_file, 'ocn', ocn_domain, is, ie, js, je, lon_local_ocn, lat_local_ocn, & min_glo_lon_ocn, max_glo_lon_ocn ) endif - if (lnd_on) then + if (lnd_on .and. .not. allocated(lon_local_lnd) ) then call mpp_get_compute_domain( lnd_domain,is,ie,js,je) allocate(lon_local_lnd(is:ie,js:je), lat_local_lnd(is:ie,js:je)) call get_grid_version_1(grid_file, 'lnd', lnd_domain, is, ie, js, je, lon_local_lnd, lat_local_lnd, & min_glo_lon_lnd, max_glo_lon_lnd ) endif - if (ice_on) then + if (ice_on .and. .not. allocated(lon_local_ice) ) then call mpp_get_compute_domain( ice_domain,is,ie,js,je) allocate(lon_local_ice(is:ie,js:je), lat_local_ice(is:ie,js:je)) call get_grid_version_1(grid_file, 'ice', ice_domain, is, ie, js, je, lon_local_ice, lat_local_ice, & @@ -406,28 +406,28 @@ subroutine data_override_init(Atm_domain_in, Ocean_domain_in, Ice_domain_in, Lan if(count .NE. 1) call mpp_error(FATAL, 'data_override_mod: the grid file is a solo mosaic, ' // & 'one and only one of atm_on, lnd_on or ice_on/ocn_on should be true') endif - if (atm_on) then + if (atm_on .and. .not. allocated(lon_local_atm) ) then call mpp_get_compute_domain(atm_domain,is,ie,js,je) allocate(lon_local_atm(is:ie,js:je), lat_local_atm(is:ie,js:je)) call get_grid_version_2(grid_file, 'atm', atm_domain, is, ie, js, je, lon_local_atm, lat_local_atm, & min_glo_lon_atm, max_glo_lon_atm ) endif - if (ocn_on) then + if (ocn_on .and. .not. allocated(lon_local_ocn) ) then call mpp_get_compute_domain( ocn_domain,is,ie,js,je) allocate(lon_local_ocn(is:ie,js:je), lat_local_ocn(is:ie,js:je)) call get_grid_version_2(grid_file, 'ocn', ocn_domain, is, ie, js, je, lon_local_ocn, lat_local_ocn, & min_glo_lon_ocn, max_glo_lon_ocn ) endif - if (lnd_on) then + if (lnd_on .and. .not. allocated(lon_local_lnd) ) then call mpp_get_compute_domain( lnd_domain,is,ie,js,je) allocate(lon_local_lnd(is:ie,js:je), lat_local_lnd(is:ie,js:je)) call get_grid_version_2(grid_file, 'lnd', lnd_domain, is, ie, js, je, lon_local_lnd, lat_local_lnd, & min_glo_lon_lnd, max_glo_lon_lnd ) endif - if (ice_on) then + if (ice_on .and. .not. allocated(lon_local_ice) ) then call mpp_get_compute_domain( ice_domain,is,ie,js,je) allocate(lon_local_ice(is:ie,js:je), lat_local_ice(is:ie,js:je)) call get_grid_version_2(grid_file, 'ocn', ice_domain, is, ie, js, je, lon_local_ice, lat_local_ice, & @@ -911,6 +911,9 @@ subroutine data_override_3d(gridname,fieldname_code,data,time,override,data_inde horz_interp=override_array(curr_position)%horz_interp(window_id), & is_in=is_in,ie_in=ie_in,js_in=js_in,je_in=je_in,window_id=window_id) data(:,:,1) = data(:,:,1)*factor + do i = 2, size(data,3) + data(:,:,i) = data(:,:,1) + enddo else allocate(mask_out(size(data,1), size(data,2),1)) mask_out = .false. @@ -921,11 +924,13 @@ subroutine data_override_3d(gridname,fieldname_code,data,time,override,data_inde where(mask_out(:,:,1)) data(:,:,1) = data(:,:,1)*factor end where + do i = 2, size(data,3) + where(mask_out(:,:,1)) + data(:,:,i) = data(:,:,1) + end where + enddo deallocate(mask_out) endif - do i = 2, size(data,3) - data(:,:,i) = data(:,:,1) - enddo else if( data_table(index1)%region_type == NO_REGION ) then call time_interp_external(id_time,time,data,verbose=.false., & @@ -1382,6 +1387,7 @@ program test call mpp_define_domains( (/1,nlon,1,nlat/), layout, Domain, name='test_data_override') call data_override_init(Ice_domain_in=Domain, Ocean_domain_in=Domain) + call data_override_init(Ice_domain_in=Domain, Ocean_domain_in=Domain) call mpp_get_compute_domain(Domain, is, ie, js, je) call get_grid diff --git a/src/shared/diag_manager/diag_axis.F90 b/src/shared/diag_manager/diag_axis.F90 index 036b606653..d7fa28a5af 100644 --- a/src/shared/diag_manager/diag_axis.F90 +++ b/src/shared/diag_manager/diag_axis.F90 @@ -1,15 +1,15 @@ MODULE diag_axis_mod - ! + ! ! Seth Underwood ! - ! diag_axis_mod is an integral part - ! of diag_manager_mod. It helps to create axis IDs - ! that are used in register_diag_field. + ! diag_axis_mod is an integral part + ! of diag_manager_mod. It helps to create axis IDs + ! that are used in register_diag_field. ! ! Users first create axis ID by calling - ! diag_axis_init, then use this axis ID in + ! diag_axis_init, then use this axis ID in ! register_diag_field. ! @@ -33,9 +33,9 @@ MODULE diag_axis_mod ! Module variables ! Parameters CHARACTER(len=128), PARAMETER :: version =& - & '$Id: diag_axis.F90,v 20.0 2013/12/14 00:18:37 fms Exp $' + & '$Id$' CHARACTER(len=128), PARAMETER :: tagname =& - & '$Name: tikal $' + & '$Name$' ! counter of number of axes defined INTEGER, DIMENSION(:), ALLOCATABLE :: num_subaxes @@ -95,7 +95,7 @@ INTEGER FUNCTION diag_axis_init(name, DATA, units, cart_name, long_name, directi CHARACTER(len=*), INTENT(in) :: name REAL, DIMENSION(:), INTENT(in) :: DATA CHARACTER(len=*), INTENT(in) :: units - CHARACTER(len=*), INTENT(in) :: cart_name + CHARACTER(len=*), INTENT(in) :: cart_name CHARACTER(len=*), INTENT(in), OPTIONAL :: long_name, set_name INTEGER, INTENT(in), OPTIONAL :: direction, edges TYPE(domain1d), INTENT(in), OPTIONAL :: Domain @@ -110,7 +110,7 @@ INTEGER FUNCTION diag_axis_init(name, DATA, units, cart_name, long_name, directi CHARACTER(len=128) :: emsg IF ( .NOT.module_is_initialized ) THEN - call write_version_number() + CALL write_version_number( version, tagname ) ENDIF IF ( PRESENT(tile_count)) THEN @@ -118,7 +118,7 @@ INTEGER FUNCTION diag_axis_init(name, DATA, units, cart_name, long_name, directi ELSE tile = 1 END IF - + ! Allocate the axes IF (.NOT. ALLOCATED(Axis_sets)) ALLOCATE(Axis_sets(max_num_axis_sets)) IF (.NOT. ALLOCATED(Axes)) ALLOCATE(Axes(max_axes)) @@ -174,7 +174,7 @@ INTEGER FUNCTION diag_axis_init(name, DATA, units, cart_name, long_name, directi END IF END IF END DO - + !---- register axis ---- num_def_axes = num_def_axes + 1 ! max_axes exceeded, increase it via diag_manager_nml @@ -189,13 +189,13 @@ INTEGER FUNCTION diag_axis_init(name, DATA, units, cart_name, long_name, directi & TRIM(uppercase(cart_name)) == 'T' .OR.& & TRIM(uppercase(cart_name)) == 'N' ) THEN Axes(diag_axis_init)%cart_name = TRIM(uppercase(cart_name)) - ELSE + ELSE ! Invalid cart_name name. CALL error_mesg('diag_axis_mod::diag_axis_init', 'Invalid cart_name name.', FATAL) END IF !---- allocate storage for coordinate values of axis ---- - IF ( Axes(diag_axis_init)%cart_name == 'T' ) THEN + IF ( Axes(diag_axis_init)%cart_name == 'T' ) THEN axlen = 0 ELSE axlen = SIZE(DATA(:)) @@ -205,7 +205,7 @@ INTEGER FUNCTION diag_axis_init(name, DATA, units, cart_name, long_name, directi ! Initialize Axes(diag_axis_init) Axes(diag_axis_init)%name = TRIM(name) Axes(diag_axis_init)%data = DATA(1:axlen) - Axes(diag_axis_init)%units = units + Axes(diag_axis_init)%units = units Axes(diag_axis_init)%length = axlen Axes(diag_axis_init)%set = set ! start and end are used in subaxes information only @@ -225,7 +225,7 @@ INTEGER FUNCTION diag_axis_init(name, DATA, units, cart_name, long_name, directi ELSE Axes(diag_axis_init)%aux = 'none' END IF - + !---- axis direction (-1, 0, or +1) ---- IF ( PRESENT(direction) )THEN IF ( ABS(direction) /= 1 .AND. direction /= 0 )& @@ -257,11 +257,11 @@ INTEGER FUNCTION diag_axis_init(name, DATA, units, cart_name, long_name, directi IF ( Axes(diag_axis_init)%cart_name == 'X' ) Axes(diag_axis_init)%Domain = domain_x IF ( Axes(diag_axis_init)%cart_name == 'Y' ) Axes(diag_axis_init)%Domain = domain_y ELSE IF ( PRESENT(Domain)) THEN - !---- domain1d type ---- + !---- domain1d type ---- Axes(diag_axis_init)%Domain2 = null_domain2d ! needed since not 2-D domain Axes(diag_axis_init)%Domain = Domain ELSE - Axes(diag_axis_init)%Domain2 = null_domain2d + Axes(diag_axis_init)%Domain2 = null_domain2d Axes(diag_axis_init)%Domain = null_domain1d END IF @@ -271,7 +271,7 @@ INTEGER FUNCTION diag_axis_init(name, DATA, units, cart_name, long_name, directi CALL mpp_get_compute_domain(Axes(diag_axis_init)%Domain, isc, iec) CALL mpp_get_global_domain(Axes(diag_axis_init)%Domain, isg, ieg) IF ( Axes(diag_axis_init)%length == ieg - isg + 2 ) THEN - Axes(diag_axis_init)%shift = 1 + Axes(diag_axis_init)%shift = 1 END IF END IF @@ -283,7 +283,7 @@ INTEGER FUNCTION diag_axis_init(name, DATA, units, cart_name, long_name, directi IF ( Axes(edges)%cart_name /= Axes(diag_axis_init)%cart_name) ierr=1 IF ( Axes(edges)%length /= Axes(diag_axis_init)%length+1 ) ierr=ierr+2 IF ( Axes(edges)%set /= Axes(diag_axis_init)%set ) ierr=ierr+4 - IF ( ierr > 0 ) THEN + IF ( ierr > 0 ) THEN ! Edges axis does not match axis (code ). WRITE (emsg,'("Edges axis does not match axis (code ",I1,").")') ierr CALL error_mesg('diag_axis_mod::diag_axis_init', emsg, FATAL) @@ -312,7 +312,7 @@ END FUNCTION diag_axis_init ! ! Given the ID of a parent axis, create a subaxis and fill it with data, ! and return the ID of the corresponding subaxis. - ! + ! ! The subaxis is defined on the parent axis from start_indx ! to end_indx. ! @@ -326,13 +326,13 @@ INTEGER FUNCTION diag_subaxes_init(axis, subdata, start_indx, end_indx, domain_2 INTEGER, INTENT(in) :: axis REAL, DIMENSION(:), INTENT(in) :: subdata INTEGER, INTENT(in) :: start_indx - INTEGER, INTENT(in) :: end_indx + INTEGER, INTENT(in) :: end_indx TYPE(domain2d), INTENT(in), OPTIONAL :: domain_2d INTEGER :: i, nsub_axis, direction INTEGER :: xbegin, xend, ybegin, yend INTEGER :: ad_xbegin, ad_xend, ad_ybegin, ad_yend - CHARACTER(len=128) :: name, nsub_name + CHARACTER(len=128) :: name, nsub_name CHARACTER(len=128) :: units CHARACTER(len=128) :: cart_name CHARACTER(len=128) :: long_name @@ -377,14 +377,14 @@ INTEGER FUNCTION diag_subaxes_init(axis, subdata, start_indx, end_indx, domain_2 Axes(axis)%end(nsub_axis) = end_indx if ( hasDomain ) Axes(axis)%subaxis_domain2(nsub_axis) = domain_2d END IF - + ! Create new name for the subaxis from name of parent axis - ! If subaxis already exists, get the index and return + ! If subaxis already exists, get the index and return IF(subaxis_set) THEN IF ( Axes(axis)%set > 0 ) THEN - diag_subaxes_init = get_axis_num(name, set_name=TRIM(Axis_sets(Axes(axis)%set))) + diag_subaxes_init = get_axis_num(name, set_name=TRIM(Axis_sets(Axes(axis)%set))) ELSE - diag_subaxes_init = get_axis_num(name) + diag_subaxes_init = get_axis_num(name) END IF ELSE ! get a new index for subaxis @@ -406,7 +406,7 @@ INTEGER FUNCTION diag_subaxes_init(axis, subdata, start_indx, end_indx, domain_2 END IF END FUNCTION diag_subaxes_init ! - + ! ! ! Return information about the axis with index ID @@ -452,7 +452,7 @@ SUBROUTINE get_diag_axis(id, name, units, long_name, cart_name,& direction = Axes(id)%direction edges = Axes(id)%edges Domain = Axes(id)%Domain - IF ( Axes(id)%length > SIZE(DATA(:)) ) THEN + IF ( Axes(id)%length > SIZE(DATA(:)) ) THEN ! array data is too small. CALL error_mesg('diag_axis_mod::get_diag_axis', 'array data is too small', FATAL) ELSE @@ -499,7 +499,7 @@ SUBROUTINE get_diag_axis_data(id, DATA) REAL, DIMENSION(:), INTENT(out) :: DATA CALL valid_id_check(id, 'get_diag_axis_data') - IF (Axes(id)%length > SIZE(DATA(:))) THEN + IF (Axes(id)%length > SIZE(DATA(:))) THEN ! array data is too small CALL error_mesg('diag_axis_mod::get_diag_axis_data', 'array data is too small', FATAL) ELSE @@ -564,7 +564,7 @@ END SUBROUTINE get_diag_axis_domain_name INTEGER FUNCTION get_axis_length(id) INTEGER, INTENT(in) :: id - INTEGER :: length + INTEGER :: length CALL valid_id_check(id, 'get_axis_length') IF ( Axes(id)%Domain .NE. null_domain1d ) THEN @@ -634,7 +634,7 @@ INTEGER FUNCTION get_tile_count(ids) INTEGER :: i, id, flag - IF ( SIZE(ids(:)) < 1 ) THEN + IF ( SIZE(ids(:)) < 1 ) THEN ! input argument has incorrect size. CALL error_mesg('diag_axis_mod::get_tile_count', 'input argument has incorrect size', FATAL) END IF @@ -695,7 +695,7 @@ TYPE(domain2d) FUNCTION get_domain2d(ids) INTEGER :: i, id, flag - IF ( SIZE(ids(:)) < 1 ) THEN + IF ( SIZE(ids(:)) < 1 ) THEN ! input argument has incorrect size. CALL error_mesg('diag_axis_mod::get_domain2d', 'input argument has incorrect size', FATAL) END IF @@ -729,14 +729,14 @@ END FUNCTION get_domain2d ! ! X shift value. ! Y shift value. - SUBROUTINE get_axes_shift(ids, ishift, jshift) + SUBROUTINE get_axes_shift(ids, ishift, jshift) INTEGER, DIMENSION(:), INTENT(in) :: ids INTEGER, INTENT(out) :: ishift, jshift INTEGER :: i, id !-- get the value of the shift. - ishift = 0 + ishift = 0 jshift = 0 DO i = 1, SIZE(ids(:)) id = ids(i) @@ -823,7 +823,7 @@ END FUNCTION get_axis_set_num ! SUBROUTINE valid_id_check(id, routine_name) ! ! - ! Check to see if the given axis id is a valid id. If the axis id is invalid, + ! Check to see if the given axis id is a valid id. If the axis id is invalid, ! call a FATAL error. If the ID is valid, just return. ! ! Axis id to check for validity diff --git a/src/shared/diag_manager/diag_data.F90 b/src/shared/diag_manager/diag_data.F90 index d3d9a4e29c..cad175ad9d 100644 --- a/src/shared/diag_manager/diag_data.F90 +++ b/src/shared/diag_manager/diag_data.F90 @@ -1,16 +1,16 @@ #include MODULE diag_data_mod - ! + ! ! Seth Underwood ! - + ! ! Type descriptions and global variables for the diag_manager modules. ! ! - ! Notation: + ! Notation: !
!
input field
!
The data structure describing the field as @@ -29,7 +29,7 @@ MODULE diag_data_mod ! ! Each input field associated with one or several output fields via array of ! indices output_fields; each output field points to the single "parent" input - ! field with the input_field index, and to the output file with the output_file + ! field with the input_field index, and to the output file with the output_file ! index ! @@ -46,7 +46,6 @@ MODULE diag_data_mod PUBLIC - ! ! ! Maximum number of fields per file. @@ -66,7 +65,15 @@ MODULE diag_data_mod ! ! ! - + ! + ! Value used in the region specification of the diag_table to indicate to use the full axis instead of a sub-axis + ! + ! + ! Alternate value used in the region specification of the diag_table to indicate to use the full axis instead of a sub-axis + ! + ! + ! Return value for a diag_field that isn't found in the diag_table + ! ! Specify storage limits for fixed size tables used for pointers, etc. INTEGER, PARAMETER :: MAX_FIELDS_PER_FILE = 300 !< Maximum number of fields per file. INTEGER, PARAMETER :: DIAG_OTHER = 0 @@ -79,7 +86,10 @@ MODULE diag_data_mod INTEGER, PARAMETER :: DIAG_SECONDS = 1, DIAG_MINUTES = 2, DIAG_HOURS = 3 INTEGER, PARAMETER :: DIAG_DAYS = 4, DIAG_MONTHS = 5, DIAG_YEARS = 6 INTEGER, PARAMETER :: MAX_SUBAXES = 10 + INTEGER, PARAMETER :: GLO_REG_VAL = -999 + INTEGER, PARAMETER :: GLO_REG_VAL_ALT = -1 REAL, PARAMETER :: CMOR_MISSING_VALUE = 1.0e20 !< CMOR standard missing value + INTEGER, PARAMETER :: DIAG_FIELD_NOT_FOUND = -1 ! ! @@ -101,12 +111,12 @@ MODULE diag_data_mod ! ID returned from diag_subaxes_init of 3 subaces. ! TYPE diag_grid - REAL, DIMENSION(3) :: start, END ! start and end coordinates (lat,lon,depth) of local domain to output + REAL, DIMENSION(3) :: start, END ! start and end coordinates (lat,lon,depth) of local domain to output INTEGER, DIMENSION(3) :: l_start_indx, l_end_indx ! start and end indices at each LOCAL PE INTEGER, DIMENSION(3) :: subaxes ! id returned from diag_subaxes_init of 3 subaxes END TYPE diag_grid ! - + ! ! ! Diagnostic field type @@ -134,6 +144,38 @@ MODULE diag_data_mod END TYPE diag_fieldtype ! + ! + ! + ! Attribute type for diagnostic fields + ! + ! + ! Data type of attribute values (NF_INT, NF_FLOAT, NF_CHAR) + ! + ! + ! Number of values in attribute, or if a character string then + ! length of the string. + ! + ! + ! Name of the attribute + ! + ! + ! Character string to hold character value of attribute + ! + ! + ! REAL array to hold value of REAL attributes. + ! + ! + ! INTEGER array to hold value of INTEGER attributes. + ! + type :: diag_atttype + INTEGER :: type + INTEGER :: len + CHARACTER(len=128) :: name + CHARACTER(len=1280) :: catt + REAL, _ALLOCATABLE, DIMENSION(:) :: fatt _NULL + INTEGER, _ALLOCATABLE, DIMENSION(:) :: iatt _NULL + end type diag_atttype + ! ! ! ! Define the region for field output. @@ -159,7 +201,7 @@ MODULE diag_data_mod REAL :: zend END TYPE coord_type ! - + ! ! ! Type to define the diagnostic files that will be written as defined by the diagnostic table. @@ -222,6 +264,12 @@ MODULE diag_data_mod ! ! ! + ! + ! Array to hold user definable attributes + ! + ! + ! Number of defined attibutes + ! TYPE file_type CHARACTER(len=128) :: name !< Name of the output file. CHARACTER(len=128) :: long_name @@ -245,9 +293,11 @@ MODULE diag_data_mod TYPE(time_type) :: start_time !< Time file opened. TYPE(time_type) :: close_time !< Time file closed. File does not allow data after close time TYPE(diag_fieldtype):: f_avg_start, f_avg_end, f_avg_nitems, f_bounds + TYPE(diag_atttype), _ALLOCATABLE, dimension(:) :: attributes _NULL + INTEGER :: num_attributes END TYPE file_type - ! - + ! + ! ! ! Type to hold the input field description @@ -290,23 +340,36 @@ MODULE diag_data_mod ! ! ! + ! + ! + ! + ! The current level of OpenMP nesting + ! ! ! ! ! + ! + ! + ! + ! Indicates if the mask_ignore_warning has been issued for this input + ! field. Once .TRUE. the warning message is suppressed on all subsequent + ! send_data calls. + ! TYPE input_field_type CHARACTER(len=128) :: module_name, field_name, long_name, units, standard_name CHARACTER(len=64) :: interp_method INTEGER, DIMENSION(3) :: axes - INTEGER :: num_axes + INTEGER :: num_axes LOGICAL :: missing_value_present, range_present REAL :: missing_value REAL, DIMENSION(2) :: range - INTEGER, _ALLOCATABLE, dimension(:) :: output_fields + INTEGER, _ALLOCATABLE, dimension(:) :: output_fields _NULL INTEGER :: num_output_fields INTEGER, DIMENSION(3) :: size LOGICAL :: static, register, mask_variant, local INTEGER :: numthreads + INTEGER :: active_omp_level INTEGER :: tile_count TYPE(coord_type) :: local_coord TYPE(time_type) :: time @@ -337,11 +400,17 @@ MODULE diag_data_mod ! ! .TRUE. if the output field is averaged over time interval. ! + ! + ! .TRUE. if the output field is the rms. In this case, time_average will also be true. + ! ! - ! .TRUE. if any of time_min, time_max, or time_average is true + ! .TRUE. if any of time_min, time_max, time_rms, or time_average is true ! ! ! + ! + ! Power to use When calculating the mean_pow(n) + ! ! ! Time method field from the input file ! @@ -403,29 +472,37 @@ MODULE diag_data_mod ! ! ! + ! + ! Array to hold user definable attributes + ! + ! + ! Number of defined attibutes + ! TYPE output_field_type INTEGER :: input_field ! index of the corresponding input field in the table INTEGER :: output_file ! index of the output file in the table CHARACTER(len=128) :: output_name LOGICAL :: time_average ! true if the output field is averaged over time interval + LOGICAL :: time_rms ! true if the output field is the rms. If true, then time_average is also LOGICAL :: static LOGICAL :: time_max ! true if the output field is maximum over time interval LOGICAL :: time_min ! true if the output field is minimum over time interval - LOGICAL :: time_ops ! true if any of time_min, time_max, or time_average is true + LOGICAL :: time_ops ! true if any of time_min, time_max, time_rms or time_average is true INTEGER :: pack - CHARACTER(len=50) :: time_method ! time method field from the input file - ! coordianes of the buffer and counter are (x, y, z, time-of-day) + INTEGER :: pow_value !< Power value to use for mean_pow(n) calculations + CHARACTER(len=50) :: time_method ! time method field from the input file + ! coordinates of the buffer and counter are (x, y, z, time-of-day) REAL, _ALLOCATABLE, DIMENSION(:,:,:,:) :: buffer _NULL REAL, _ALLOCATABLE, DIMENSION(:,:,:,:) :: counter _NULL - ! the following two counters are used in time-averaging for some - ! combination of the field options. Their size is the length of the + ! the following two counters are used in time-averaging for some + ! combination of the field options. Their size is the length of the ! diurnal axis; the counters must be tracked separately for each of - ! the diurnal interval, becaus the number of time slices accumulated + ! the diurnal interval, because the number of time slices accumulated ! in each can be different, depending on time step and the number of ! diurnal samples. REAL, _ALLOCATABLE, DIMENSION(:) :: count_0d INTEGER, _ALLOCATABLE, dimension(:) :: num_elements - + TYPE(time_type) :: last_output, next_output, next_next_output TYPE(diag_fieldtype) :: f_type INTEGER, DIMENSION(4) :: axes @@ -436,6 +513,8 @@ MODULE diag_data_mod LOGICAL :: reduced_k_range INTEGER :: imin, imax, jmin, jmax, kmin, kmax TYPE(time_type) :: Time_of_prev_field_data + TYPE(diag_atttype), _ALLOCATABLE, dimension(:) :: attributes _NULL + INTEGER :: num_attributes END TYPE output_field_type ! @@ -508,12 +587,12 @@ MODULE diag_data_mod CHARACTER(len=128) :: tile_name='N/A' END TYPE diag_global_att_type ! - + ! Private CHARACTER Arrays for the CVS version and tagname. CHARACTER(len=128),PRIVATE :: version =& - & '$Id: diag_data.F90,v 20.0 2013/12/14 00:18:41 fms Exp $' + & '$Id$' CHARACTER(len=128),PRIVATE :: tagname =& - & '$Name: tikal $' + & '$Name$' ! ! @@ -543,7 +622,7 @@ MODULE diag_data_mod ! ! Maximum number of input fields. Increase via the diag_manager_nml namelist. ! - ! + ! ! Maximum number of output_fields per input_field. ! ! @@ -556,20 +635,35 @@ MODULE diag_data_mod ! ! Indicates if we should overwrite the MISSING_VALUE to use the CMOR missing value. ! - ! + ! ! Issue warnings if the output field has values outside the given ! range for a variable. ! - ! + ! ! Cause a fatal error if the output field has a value outside the ! given range for a variable. ! + ! + ! Maximum number of user definable attributes per field. + ! + ! + ! Maximum number of user definable global attributes per file. + ! + ! + ! Indicates if the file start date will be prepended to the file name. .TRUE. is + ! only supported if the diag_manager_init routine is called with the optional time_init parameter. + ! This was usually done by FRE after the model run. + ! + ! + ! Will determine which value to use when checking a regional output if the region is the full axis or a sub-axis. + ! The values are defined as GLO_REG_VAL (-999) and GLO_REG_VAL_ALT (-1) in diag_data_mod. + ! LOGICAL :: append_pelist_name = .FALSE. LOGICAL :: mix_snapshot_average_fields =.FALSE. INTEGER :: max_files = 31 !< Maximum number of output files allowed. Increase via diag_manager_nml. INTEGER :: max_output_fields = 300 !< Maximum number of output fields. Increase via diag_manager_nml. INTEGER :: max_input_fields = 300 !< Maximum number of input fields. Increase via diag_manager_nml. - INTEGER :: MAX_OUT_PER_IN_FIELD = 150 !< Maximum number of output_fields per input_field. Increase via diag_manager_nml. + INTEGER :: max_out_per_in_field = 150 !< Maximum number of output_fields per input_field. Increase via diag_manager_nml. INTEGER :: max_axes = 60 !< Maximum number of independent axes. LOGICAL :: do_diag_field_log = .FALSE. LOGICAL :: write_bytes_in_file = .FALSE. @@ -579,7 +673,11 @@ MODULE diag_data_mod LOGICAL :: use_cmor = .FALSE. LOGICAL :: issue_oor_warnings = .TRUE. LOGICAL :: oor_warnings_fatal = .FALSE. + LOGICAL :: region_out_use_alt_value = .TRUE. + INTEGER :: max_field_attributes = 2 + INTEGER :: max_file_attributes = 2 + LOGICAL :: prepend_date = .TRUE. ! ! ! Fill value used. Value will be NF90_FILL_REAL if using the @@ -588,7 +686,7 @@ MODULE diag_data_mod #ifdef use_netCDF REAL :: FILL_VALUE = NF_FILL_REAL ! from file /usr/local/include/netcdf.inc #else - REAL :: FILL_VALUE = 9.9692099683868690e+36 + REAL :: FILL_VALUE = 9.9692099683868690e+36 #endif INTEGER :: pack_size = 1 ! 1 for double and 2 for float @@ -601,6 +699,10 @@ MODULE diag_data_mod REAL :: MAX_VALUE, MIN_VALUE ! + ! + ! Time diag_manager_init called. If init_time not included in + ! diag_manager_init call, then same as base_time + ! ! ! ! @@ -609,6 +711,7 @@ MODULE diag_data_mod ! ! ! + TYPE(time_type) :: diag_init_time TYPE(time_type) :: base_time INTEGER :: base_year, base_month, base_day, base_hour, base_minute, base_second CHARACTER(len = 256):: global_descriptor @@ -628,7 +731,6 @@ MODULE diag_data_mod ! ! - ! ! TYPE(time_type) :: time_zero LOGICAL :: first_send_data_call = .TRUE. @@ -636,8 +738,7 @@ MODULE diag_data_mod INTEGER :: diag_log_unit CHARACTER(len=10), DIMENSION(6) :: time_unit_list = (/'seconds ', 'minutes ',& & 'hours ', 'days ', 'months ', 'years '/) - CHARACTER(len=32), SAVE :: filename_appendix = '' CHARACTER(len=32) :: pelist_name INTEGER :: oor_warning = WARNING - + END MODULE diag_data_mod diff --git a/src/shared/diag_manager/diag_grid.F90 b/src/shared/diag_manager/diag_grid.F90 index f254b48c96..8c09ae722c 100644 --- a/src/shared/diag_manager/diag_grid.F90 +++ b/src/shared/diag_manager/diag_grid.F90 @@ -1,9 +1,10 @@ #include MODULE diag_grid_mod - ! + ! ! Seth Underwood ! + ! ! ! diag_grid_mod is a set of procedures to work with the ! model's global grid to allow regional output. @@ -30,7 +31,7 @@ MODULE diag_grid_mod !
  • Single point region in Cubed Sphere
    • !
  • Single tile regions in the cubed sphere
    • ! - !
    + ! !
    !     @@ -60,9 +61,9 @@ MODULE diag_grid_mod ! Parameters CHARACTER(len=128), PARAMETER :: version =& - & '$Id: diag_grid.F90,v 20.0 2013/12/14 00:18:45 fms Exp $' + & '$Id$' CHARACTER(len=128), PARAMETER :: tagname =& - & '$Name: tikal $' + & '$Name$' ! Derived data types ! @@ -177,7 +178,7 @@ MODULE diag_grid_mod ! ! ! Send the global grid to the diag_manager_mod for - ! regional output. + ! regional output. ! ! ! - ! Send data over to output fields. + ! Send data over to output fields. ! ! ! send_data is overloaded for fields having zero dimension @@ -243,11 +266,11 @@ MODULE diag_manager_mod ! By default, a field will be written out entirely in its global grid. ! Users can also specify regions in which the field will be output. The ! region is specified in diag-table just before the end of output_field - ! replacing "none". + ! replacing "none". ! ! For example, by default: ! - ! "ocean_mod","Vorticity","vorticity","file1","all",.false.,"none",2 + ! "ocean_mod","Vorticity","vorticity","file1","all",.false.,"none",2 ! ! for regional output: ! @@ -266,16 +289,16 @@ MODULE diag_manager_mod ! diag_manager_mod using the diag_grid_init ! subroutine. - ! - ! NOTE: When using regional output the files containing regional - ! outputs should be different from files containing global (default) output. - ! It is a FATAL error to have one file containing both regional and global - ! results. For maximum flexibility and independence from PE counts one file + ! + ! NOTE: When using regional output the files containing regional + ! outputs should be different from files containing global (default) output. + ! It is a FATAL error to have one file containing both regional and global + ! results. For maximum flexibility and independence from PE counts one file ! should contain just one region. - ! + ! ! Time averaging is supported in regional output. - ! - ! Physical fields (written in "physics windows" of atmospheric code) are + ! + ! Physical fields (written in "physics windows" of atmospheric code) are ! fully supported for regional outputs. ! ! NOTE: Most fields are defined in the data domain but use the @@ -317,7 +340,7 @@ MODULE diag_manager_mod ! ! ! Return field index for subsequent calls to @@ -344,8 +367,10 @@ MODULE diag_manager_mod ! ! ! - ! + ! ! + ! + ! INTERFACE register_diag_field MODULE PROCEDURE register_diag_field_scalar MODULE PROCEDURE register_diag_field_array @@ -354,13 +379,13 @@ MODULE diag_manager_mod ! ! - ! Send tile-averaged data over to output fields. + ! Send tile-averaged data over to output fields. ! ! ! - ! send_tile_averaged_data is overloaded for 3D and 4D arrays. + ! send_tile_averaged_data is overloaded for 3D and 4D arrays. ! diag_field_id corresponds to the ID returned by previous call ! to register_diag_field. Logical masks can be used to mask out ! undefined and/or unused values. Note that the dimension of output field @@ -378,6 +403,34 @@ MODULE diag_manager_mod END INTERFACE ! + ! + ! + ! Add a attribute to the output field + ! + ! + ! + ! Add an arbitrary attribute and value to the output variable. Any number + ! of attributes can be added to a given field. All attribute addition must + ! be done before first send_data call. + ! + ! If a real or integer attribute is already defined, a FATAL error will be called. + ! If a character attribute is already defined, then it will be prepended to the + ! existing attribute value. + ! + ! + ! + ! + INTERFACE diag_field_add_attribute + MODULE PROCEDURE diag_field_add_attribute_scalar_r + MODULE PROCEDURE diag_field_add_attribute_scalar_i + MODULE PROCEDURE diag_field_add_attribute_scalar_c + MODULE PROCEDURE diag_field_add_attribute_r1d + MODULE PROCEDURE diag_field_add_attribute_i1d + END INTERFACE diag_field_add_attribute + ! + CONTAINS ! @@ -392,9 +445,12 @@ MODULE diag_manager_mod ! ! ! + ! + ! ! INTEGER FUNCTION register_diag_field_scalar(module_name, field_name, init_time, & - & long_name, units, missing_value, range, standard_name, do_not_log, err_msg) + & long_name, units, missing_value, range, standard_name, do_not_log, err_msg,& + & area, volume) CHARACTER(len=*), INTENT(in) :: module_name, field_name TYPE(time_type), OPTIONAL, INTENT(in) :: init_time CHARACTER(len=*), OPTIONAL, INTENT(in) :: long_name, units, standard_name @@ -402,11 +458,13 @@ INTEGER FUNCTION register_diag_field_scalar(module_name, field_name, init_time, REAL, DIMENSION(2), OPTIONAL, INTENT(in) :: RANGE LOGICAL, OPTIONAL, INTENT(in) :: do_not_log ! if TRUE, field information is not logged CHARACTER(len=*), OPTIONAL, INTENT(out):: err_msg - + INTEGER, OPTIONAL, INTENT(in) :: area, volume + IF ( PRESENT(init_time) ) THEN register_diag_field_scalar = register_diag_field_array(module_name, field_name,& & (/null_axis_id/), init_time,long_name, units, missing_value, range, & - & standard_name=standard_name, do_not_log=do_not_log, err_msg=err_msg) + & standard_name=standard_name, do_not_log=do_not_log, err_msg=err_msg,& + & area=area, volume=volume) ELSE IF ( PRESENT(err_msg) ) err_msg = '' register_diag_field_scalar = register_static_field(module_name, field_name,& @@ -428,12 +486,14 @@ END FUNCTION register_diag_field_scalar ! ! ! - ! ! ! + ! diag_field_id containing the cell area field + ! diag_field_id containing the cell volume field + ! INTEGER FUNCTION register_diag_field_array(module_name, field_name, axes, init_time, & & long_name, units, missing_value, range, mask_variant, standard_name, verbose,& - & do_not_log, err_msg, interp_method, tile_count) + & do_not_log, err_msg, interp_method, tile_count, area, volume) CHARACTER(len=*), INTENT(in) :: module_name, field_name INTEGER, INTENT(in) :: axes(:) TYPE(time_type), INTENT(in) :: init_time @@ -444,11 +504,14 @@ INTEGER FUNCTION register_diag_field_array(module_name, field_name, axes, init_t CHARACTER(len=*), OPTIONAL, INTENT(out):: err_msg CHARACTER(len=*), OPTIONAL, INTENT(in) :: interp_method INTEGER, OPTIONAL, INTENT(in) :: tile_count + INTEGER, OPTIONAL, INTENT(in) :: area, volume INTEGER :: field, j, ind, file_num, freq + INTEGER :: i, cm_ind, cm_file_num INTEGER :: output_units INTEGER :: stdout_unit LOGICAL :: mask_variant1, verbose1 + LOGICAL :: cm_found CHARACTER(len=128) :: msg ! get stdout unit number @@ -460,20 +523,20 @@ INTEGER FUNCTION register_diag_field_array(module_name, field_name, axes, init_t mask_variant1 = .FALSE. END IF - IF ( PRESENT(verbose) ) THEN + IF ( PRESENT(verbose) ) THEN verbose1 = verbose - ELSE + ELSE verbose1 = .FALSE. END IF IF ( PRESENT(err_msg) ) err_msg = '' - + ! Call register static, then set static back to false register_diag_field_array = register_static_field(module_name, field_name, axes,& & long_name, units, missing_value, range, mask_variant1, standard_name=standard_name,& & DYNAMIC=.TRUE., do_not_log=do_not_log, interp_method=interp_method, tile_count=tile_count) - IF ( .NOT.first_send_data_call ) THEN + IF ( .NOT.first_send_data_call ) THEN ! ! module/output_field / registered AFTER first ! send_data call, TOO LATE @@ -481,23 +544,53 @@ INTEGER FUNCTION register_diag_field_array(module_name, field_name, axes, init_t IF ( mpp_pe() == mpp_root_pe() ) & & CALL error_mesg ('diag_manager_mod::register_diag_field', 'module/output_field '& &//TRIM(module_name)//'/'// TRIM(field_name)//& - &' registered AFTER first send_data call, TOO LATE', WARNING) + &' registered AFTER first send_data call, TOO LATE', WARNING) END IF IF ( register_diag_field_array < 0 ) THEN ! ! module/output_field / NOT found in diag_table ! - IF ( debug_diag_manager .OR. verbose1 ) THEN + IF ( debug_diag_manager .OR. verbose1 ) THEN IF ( mpp_pe() == mpp_root_pe() ) & & CALL error_mesg ('diag_manager_mod::register_diag_field', 'module/output_field '& &//TRIM(module_name)//'/'// TRIM(field_name)//' NOT found in diag_table',& - & WARNING) + & WARNING) END IF - ELSE + ELSE input_fields(register_diag_field_array)%static = .FALSE. field = register_diag_field_array - IF ( PRESENT(standard_name) ) input_fields(field)%standard_name = standard_name + + + ! Verify that area and volume do not point to the same variable + IF ( PRESENT(volume).AND.PRESENT(area) ) THEN + IF ( area.EQ.volume ) THEN + CALL error_mesg ('diag_manager_mod::register_diag_field', 'module/output_field '& + &//TRIM(module_name)//'/'// TRIM(field_name)//' AREA and VOLUME CANNOT be the same variable.& + & Contact the developers.',& + & FATAL) + END IF + END IF + + ! Check for the existence of the area/volume field(s) + IF ( PRESENT(area) ) THEN + IF ( area < 0 ) THEN + CALL error_mesg ('diag_manager_mod::register_diag_field', 'module/output_field '& + &//TRIM(module_name)//'/'// TRIM(field_name)//' AREA measures field NOT found in diag_table.& + & Contact the model liaison.',& + & FATAL) + END IF + END IF + IF ( PRESENT(volume) ) THEN + IF ( volume < 0 ) THEN + CALL error_mesg ('diag_manager_mod::register_diag_field', 'module/output_field '& + &//TRIM(module_name)//'/'// TRIM(field_name)//' VOLUME measures field NOT found in diag_table.& + & Contact the model liaison.',& + & FATAL) + END IF + END IF + + IF ( PRESENT(standard_name) ) input_fields(field)%standard_name = standard_name DO j = 1, input_fields(field)%num_output_fields ind = input_fields(field)%output_fields(j) @@ -508,7 +601,7 @@ INTEGER FUNCTION register_diag_field_array(module_name, field_name, axes, init_t file_num = output_fields(ind)%output_file IF ( file_num == max_files ) CYCLE IF ( output_fields(ind)%local_output ) THEN - IF ( output_fields(ind)%need_compute) THEN + IF ( output_fields(ind)%need_compute) THEN files(file_num)%local = .TRUE. END IF END IF @@ -542,6 +635,15 @@ INTEGER FUNCTION register_diag_field_array(module_name, field_name, axes, init_t WRITE(stdout_unit,* ) 'module/output_field '//TRIM(module_name)//'/'//TRIM(field_name)// & & ' will be output in region:'//TRIM(msg) END IF + + ! Set the cell_measures attribute in the out file + CALL init_field_cell_measures(output_fields(ind), area=area, volume=volume, err_msg=err_msg) + IF ( LEN_TRIM(err_msg).GT.0 ) THEN + CALL error_mesg ('diag_manager_mod::register_diag_field',& + & TRIM(err_msg)//' for module/field '//TRIM(module_name)//'/'//TRIM(field_name),& + & FATAL) + END IF + END DO END IF END FUNCTION register_diag_field_array @@ -572,9 +674,11 @@ END FUNCTION register_diag_field_array ! ! ! + ! Field ID for the area field associated with this field + ! Field ID for the volume field associated with this field INTEGER FUNCTION register_static_field(module_name, field_name, axes, long_name, units,& & missing_value, range, mask_variant, standard_name, DYNAMIC, do_not_log, interp_method,& - & tile_count) + & tile_count, area, volume) CHARACTER(len=*), INTENT(in) :: module_name, field_name INTEGER, DIMENSION(:), INTENT(in) :: axes CHARACTER(len=*), OPTIONAL, INTENT(in) :: long_name, units, standard_name @@ -584,7 +688,7 @@ INTEGER FUNCTION register_static_field(module_name, field_name, axes, long_name, LOGICAL, OPTIONAL, INTENT(in) :: DYNAMIC LOGICAL, OPTIONAL, INTENT(in) :: do_not_log ! if TRUE, field information is not logged CHARACTER(len=*), OPTIONAL, INTENT(in) :: interp_method - INTEGER, OPTIONAL, INTENT(in) :: tile_count + INTEGER, OPTIONAL, INTENT(in) :: tile_count, area, volume REAL :: missing_value_use INTEGER :: field, num_axes, j, out_num, k @@ -594,33 +698,33 @@ INTEGER FUNCTION register_static_field(module_name, field_name, axes, long_name, CHARACTER(len=128) :: msg ! Fatal error if the module has not been initialized. - IF ( .NOT.module_is_initialized ) THEN + IF ( .NOT.module_is_initialized ) THEN ! diag_manager has NOT been initialized CALL error_mesg ('diag_manager_mod::register_static_field', 'diag_manager has NOT been initialized', FATAL) END IF ! Check if OPTIONAL parameters were passed in. IF ( PRESENT(missing_value) ) THEN - IF ( use_cmor ) THEN + IF ( use_cmor ) THEN missing_value_use = CMOR_MISSING_VALUE ELSE missing_value_use = missing_value END IF END IF - - IF ( PRESENT(mask_variant) ) THEN + + IF ( PRESENT(mask_variant) ) THEN mask_variant1 = mask_variant - ELSE + ELSE mask_variant1 = .FALSE. END IF - + IF ( PRESENT(DYNAMIC) ) THEN dynamic1 = DYNAMIC ELSE dynamic1 = .FALSE. END IF - IF ( PRESENT(tile_count) ) THEN + IF ( PRESENT(tile_count) ) THEN tile = tile_count ELSE tile = 1 @@ -629,7 +733,7 @@ INTEGER FUNCTION register_static_field(module_name, field_name, axes, long_name, IF ( PRESENT(do_not_log) ) THEN allow_log = .NOT.do_not_log ELSE - allow_log = .TRUE. + allow_log = .TRUE. END IF ! Namelist do_diag_field_log is by default false. Thus to log the @@ -655,7 +759,7 @@ INTEGER FUNCTION register_static_field(module_name, field_name, axes, long_name, ! CALL error_mesg ('diag_manager_mod::register_diag_field', 'module/output_field '//trim(module_name)//'/'//& & TRIM(field_name)//' is not registered for tile_count = 1, should not register for tile_count > 1',& - & FATAL) + & FATAL) END IF CALL init_input_field(module_name, field_name, tile) @@ -672,7 +776,7 @@ INTEGER FUNCTION register_static_field(module_name, field_name, axes, long_name, & files(file_num)%name,output_fields(out_num)%time_method, output_fields(out_num)%pack, tile) END IF END DO - field = register_static_field + field = register_static_field END IF ! Store information for this input field into input field table @@ -687,7 +791,35 @@ INTEGER FUNCTION register_static_field(module_name, field_name, axes, long_name, ! not register twice ! CALL error_mesg ('diag_manager_mod::register_diag_field', 'module/output_field '//trim(module_name)//'/'//& - & TRIM(field_name)//' ALREADY registered, should not register twice', FATAL) + & TRIM(field_name)//' ALREADY registered, should not register twice', FATAL) + END IF + + ! Verify that area and volume do not point to the same variable + IF ( PRESENT(volume).AND.PRESENT(area) ) THEN + IF ( area.EQ.volume ) THEN + CALL error_mesg ('diag_manager_mod::register_static_field', 'module/output_field '& + &//TRIM(module_name)//'/'// TRIM(field_name)//' AREA and VOLUME CANNOT be the same variable.& + & Contact the developers.',& + & FATAL) + END IF + END IF + + ! Check for the existence of the area/volume field(s) + IF ( PRESENT(area) ) THEN + IF ( area < 0 ) THEN + CALL error_mesg ('diag_manager_mod::register_static_field', 'module/output_field '& + &//TRIM(module_name)//'/'// TRIM(field_name)//' AREA measures field NOT found in diag_table.& + & Contact the model liaison.n',& + & FATAL) + END IF + END IF + IF ( PRESENT(volume) ) THEN + IF ( volume < 0 ) THEN + CALL error_mesg ('diag_manager_mod::register_static_field', 'module/output_field '& + &//TRIM(module_name)//'/'// TRIM(field_name)//' VOLUME measures field NOT found in diag_table& + & Contact the model liaison.',& + & FATAL) + END IF END IF ! Set flag that this field was registered @@ -703,22 +835,22 @@ INTEGER FUNCTION register_static_field(module_name, field_name, axes, long_name, ELSE input_fields(field)%long_name = input_fields(field)%field_name END IF - - IF ( PRESENT(standard_name) ) input_fields(field)%standard_name = standard_name - + + IF ( PRESENT(standard_name) ) input_fields(field)%standard_name = standard_name + IF ( PRESENT(units) ) THEN input_fields(field)%units = TRIM(units) ELSE input_fields(field)%units = 'none' END IF - + IF ( PRESENT(missing_value) ) THEN input_fields(field)%missing_value = missing_value_use input_fields(field)%missing_value_present = .TRUE. ELSE input_fields(field)%missing_value_present = .FALSE. END IF - + IF ( PRESENT(range) ) THEN input_fields(field)%range = range ! don't use the range if it is not a valid range @@ -729,7 +861,7 @@ INTEGER FUNCTION register_static_field(module_name, field_name, axes, long_name, END IF IF ( PRESENT(interp_method) ) THEN - IF ( TRIM(interp_method) .NE. 'conserve_order1' ) THEN + IF ( TRIM(interp_method) .NE. 'conserve_order1' ) THEN ! ! when registering module/output_field / then optional ! argument interp_method = , but it should be "conserve_order1" @@ -740,7 +872,7 @@ INTEGER FUNCTION register_static_field(module_name, field_name, axes, long_name, & ', but it should be "conserve_order1"', FATAL) END IF input_fields(field)%interp_method = TRIM(interp_method) - ELSE + ELSE input_fields(field)%interp_method = '' END IF @@ -748,7 +880,7 @@ INTEGER FUNCTION register_static_field(module_name, field_name, axes, long_name, num_axes = SIZE(axes(:)) ! num_axes should be <= 3. input_fields(field)%axes(1:num_axes) = axes input_fields(field)%num_axes = num_axes - + siz = 1 DO j = 1, num_axes IF ( axes(j) .LE. 0 ) THEN @@ -756,7 +888,7 @@ INTEGER FUNCTION register_static_field(module_name, field_name, axes, long_name, ! module/output_field / has non-positive axis_id ! CALL error_mesg ('diag_manager_mod::register_diag_field', 'module/output_field '//trim(module_name)//'/'//& - & TRIM(field_name)//' has non-positive axis_id', FATAL) + & TRIM(field_name)//' has non-positive axis_id', FATAL) END IF siz(j) = get_axis_length(axes(j)) END DO @@ -772,7 +904,7 @@ INTEGER FUNCTION register_static_field(module_name, field_name, axes, long_name, ! Need to loop through all output_fields associated and allocate their buffers DO j = 1, input_fields(field)%num_output_fields out_num = input_fields(field)%output_fields(j) - ! Range is required when pack >= 4 + ! Range is required when pack >= 4 IF ( output_fields(out_num)%pack>=4 .AND. .NOT.input_fields(field)%range_present ) THEN IF(mpp_pe() .EQ. mpp_root_pe()) THEN ! @@ -788,10 +920,10 @@ INTEGER FUNCTION register_static_field(module_name, field_name, axes, long_name, ! if local_output (size of output_fields does NOT equal size of input_fields) IF ( output_fields(out_num)%reduced_k_range ) THEN CALL get_subfield_vert_size(axes, out_num) - + local_start(3) = output_fields(out_num)%output_grid%l_start_indx(3) local_end(3) = output_fields(out_num)%output_grid%l_end_indx(3) - local_siz(3) = local_end(3) - local_start(3) +1 + local_siz(3) = local_end(3) - local_start(3) +1 ALLOCATE(output_fields(out_num)%buffer(siz(1), siz(2), local_siz(3),& & output_fields(out_num)%n_diurnal_samples)) @@ -816,7 +948,7 @@ INTEGER FUNCTION register_static_field(module_name, field_name, axes, long_name, DO k = 1, num_axes local_start(k) = output_fields(out_num)%output_grid%l_start_indx(k) local_end(k) = output_fields(out_num)%output_grid%l_end_indx(k) - local_siz(k) = local_end(k) - local_start(k) +1 + local_siz(k) = local_end(k) - local_start(k) +1 END DO ALLOCATE(output_fields(out_num)%buffer(local_siz(1), local_siz(2), local_siz(3),& & output_fields(out_num)%n_diurnal_samples)) @@ -832,7 +964,7 @@ INTEGER FUNCTION register_static_field(module_name, field_name, axes, long_name, files(output_fields(out_num)%output_file)%local = .true. END IF ELSE ! the field is output globally - ! size of output_fields equal size of input_fields + ! size of output_fields equal size of input_fields ALLOCATE(output_fields(out_num)%buffer(siz(1), siz(2), siz(3),& & output_fields(out_num)%n_diurnal_samples)) IF(output_fields(out_num)%time_max) THEN @@ -844,7 +976,7 @@ INTEGER FUNCTION register_static_field(module_name, field_name, axes, long_name, END IF output_fields(out_num)%total_elements = siz(1)*siz(2)*siz(3) END IF - + ! Reset to false in register_field if this is not static output_fields(out_num)%static = .TRUE. ! check if time average is true for static field @@ -871,7 +1003,7 @@ INTEGER FUNCTION register_static_field(module_name, field_name, axes, long_name, ! this should be changed later to take into account time-of-day axis output_fields(out_num)%num_axes = input_fields(field)%num_axes ! Axes are copied from input_fields if output globally or from subaxes if output locally - IF ( .NOT.output_fields(out_num)%local_output ) THEN + IF ( .NOT.output_fields(out_num)%local_output ) THEN output_fields(out_num)%axes(1:input_fields(field)%num_axes) =& & input_fields(field)%axes(1:input_fields(field)%num_axes) ELSE @@ -879,7 +1011,7 @@ INTEGER FUNCTION register_static_field(module_name, field_name, axes, long_name, & output_fields(out_num)%output_grid%subaxes(1:input_fields(field)%num_axes) END IF - ! if necessary, initialize the diurnal time axis and append its index in the + ! if necessary, initialize the diurnal time axis and append its index in the ! output field axes array IF ( output_fields(out_num)%n_diurnal_samples > 1 ) THEN output_fields(out_num)%axes(output_fields(out_num)%num_axes+1) =& @@ -887,12 +1019,20 @@ INTEGER FUNCTION register_static_field(module_name, field_name, axes, long_name, output_fields(out_num)%num_axes = output_fields(out_num)%num_axes+1 END IF - IF ( output_fields(out_num)%reduced_k_range ) THEN + IF ( output_fields(out_num)%reduced_k_range ) THEN output_fields(out_num)%axes(3) = output_fields(out_num)%output_grid%subaxes(3) END IF ! Initialize a time variable used in an error check output_fields(out_num)%Time_of_prev_field_data = Time_zero + + ! Set the cell_measures attribute in the out file + CALL init_field_cell_measures(output_fields(out_num), area=area, volume=volume, err_msg=msg) + IF ( LEN_TRIM(msg).GT.0 ) THEN + CALL error_mesg ('diag_manager_mod::register_static_field',& + & TRIM(msg)//' for module/field '//TRIM(module_name)//'/'//TRIM(field_name),& + & FATAL) + END IF END DO IF ( input_fields(field)%mask_variant ) THEN @@ -908,6 +1048,205 @@ INTEGER FUNCTION register_static_field(module_name, field_name, axes, long_name, END FUNCTION register_static_field ! + ! + ! + ! Return the diagnostic field ID of a given variable. + ! + ! + ! + ! get_diag_field_id will return the ID returned during the register_diag_field call. If + ! the variable is not in the diag_table, then the value "DIAG_FIELD_NOT_FOUND" will be + ! returned. + ! + ! Module name that registered the variable + ! Variable name + INTEGER FUNCTION get_diag_field_id(module_name, field_name) + CHARACTER(len=*), INTENT(in) :: module_name, field_name + + ! find_input_field will return DIAG_FIELD_NOT_FOUND if the field is not + ! included in the diag_table + get_diag_field_id = find_input_field(module_name, field_name, tile_count=1) + END FUNCTION get_diag_field_id + ! + + ! + ! + ! Finds the corresponding related output field and file + ! + ! + ! + ! Finds the corresponding related output field and file for a given input field + ! + ! input field ID to find the corresponding + ! Output field that field must correspond to + ! output_field index of related output field + ! file index of the out_field_id output field + LOGICAL FUNCTION get_related_field(field, rel_field, out_field_id, out_file_id) + INTEGER, INTENT(in) :: field + TYPE(output_field_type), INTENT(in) :: rel_field + INTEGER, INTENT(out) :: out_field_id, out_file_id + + INTEGER :: i, cm_ind, cm_file_num + INTEGER :: rel_file + + ! Output file index of field to compare to + rel_file = rel_field%output_file + + ! Default return values + out_field_id = -1 + out_file_id = -1 + get_related_field = .FALSE. + + ! First check if any fields are in the same file as rel_field + DO i = 1, input_fields(field)%num_output_fields + cm_ind = input_fields(field)%output_fields(i) + cm_file_num = output_fields(cm_ind)%output_file + + IF ( cm_file_num.EQ.rel_file.AND.& + & (( (output_fields(cm_ind)%time_ops.EQV.rel_field%time_ops) .AND.& + & (output_fields(cm_ind)%next_output.EQ.rel_field%next_output) .AND.& + & (output_fields(cm_ind)%last_output.EQ.rel_field%last_output) ).OR.& + & (output_fields(cm_ind)%static.OR.rel_field%static) ) ) THEN + get_related_field = .TRUE. + out_field_id = cm_ind + out_file_id = cm_file_num + EXIT + END IF + END DO + + ! Now look for the field in a different file + IF ( .NOT.get_related_field ) THEN + DO i = 1, input_fields(field)%num_output_fields + cm_ind = input_fields(field)%output_fields(i) + cm_file_num = output_fields(cm_ind)%output_file + + ! If time_method, freq, output_units, next_output, and last_output the same, or + ! the output_field is static then valid for cell_measures +!!$ For now, only static fields can be in an external file +!!$ IF ( ( (files(cm_file_num)%output_freq.EQ.files(rel_file)%output_freq) .AND.& +!!$ & (files(cm_file_num)%output_units.EQ.files(rel_file)%output_units) .AND.& +!!$ & (output_fields(cm_ind)%time_ops.EQV.rel_field%time_ops) .AND.& +!!$ & (output_fields(cm_ind)%next_output.EQ.rel_field%next_output) .AND.& +!!$ & (output_fields(cm_ind)%last_output.EQ.rel_field%last_output) ).OR.& +!!$ & ( output_fields(cm_ind)%static.OR.rel_field%static ) ) THEN + IF ( output_fields(cm_ind)%static.OR.rel_field%static ) THEN + get_related_field = .TRUE. + out_field_id = cm_ind + out_file_id = cm_file_num + EXIT + END IF + END DO + END IF + END FUNCTION get_related_field + ! + + ! + ! + ! If needed, add cell_measures and associated_file attribute to out field/file + ! + ! + ! + ! If needed, add cell_measures and associated_file attribute to out field/file + ! + ! Output field that needs the cell_measures + ! Field ID for area + ! Field ID for volume + ! + SUBROUTINE init_field_cell_measures(output_field, area, volume, err_msg) + TYPE(output_field_type), INTENT(inout) :: output_field + INTEGER, INTENT(in), OPTIONAL :: area, volume + CHARACTER(len=*), INTENT(out), OPTIONAL :: err_msg + + INTEGER :: cm_ind, cm_file_num, file_num + INTEGER :: year, month, day, hour, minute, second + CHARACTER(len=25) :: date_prefix + + IF ( PRESENT(err_msg) ) THEN + err_msg = '' + END IF + + ! Verify that area/volume are defined (.gt.0 + IF ( PRESENT(area) ) THEN + IF ( area.LE.0 ) THEN + IF ( fms_error_handler('diag_manager_mod::init_field_cell_measure',& + & 'AREA field not in diag_table for field '//TRIM(input_fields(output_field%input_field)%module_name)//& + & '/'//TRIM(input_fields(output_field%input_field)%field_name), err_msg) ) RETURN + END IF + END IF + + IF ( PRESENT(volume) ) THEN + IF ( volume.LE.0 ) THEN + IF ( fms_error_handler('diag_manager_mod::init_field_cell_measure',& + & 'VOLUME field not in diag_table for field '//TRIM(input_fields(output_field%input_field)%module_name)//& + & '/'//TRIM(input_fields(output_field%input_field)%field_name), err_msg) ) RETURN + END IF + END IF + + ! Get the file number that the output_field will be written to + file_num = output_field%output_file + + ! Create the date_string + IF ( prepend_date ) THEN + call get_date(diag_init_time, year, month, day, hour, minute, second) + write (date_prefix, '(1I20.4, 2I2.2,".")') year, month, day + date_prefix=adjustl(date_prefix) + ELSE + date_prefix='' + END IF + + ! Take care of the cell_measures attribute + IF ( PRESENT(area) ) THEN + IF ( get_related_field(area, output_field, cm_ind, cm_file_num) ) THEN + CALL prepend_attribute(output_field, 'cell_measures',& + & 'area: '//TRIM(output_fields(cm_ind)%output_name)) + IF ( cm_file_num.NE.file_num ) THEN + ! Not in the same file, set the global attribute associated_files + ! Should look like :associated_files = " output_name: output_file_name " ; + ! Need to append *.nc as files()%name does not include this. + CALL prepend_attribute(files(file_num), 'associated_files',& + & TRIM(output_fields(cm_ind)%output_name)//': '//& + & TRIM(date_prefix)//TRIM(files(cm_file_num)%name)//'.nc') + END IF + ELSE + IF ( fms_error_handler('diag_manager_mod::init_field_cell_measures',& + & 'AREA measures field "'//TRIM(input_fields(area)%module_name)//'/'//& + & TRIM(input_fields(area)%field_name)//& + & '" NOT in diag_table with correct output frequency for field '//& + & TRIM(input_fields(output_field%input_field)%module_name)//& + & '/'//TRIM(input_fields(output_field%input_field)%field_name), err_msg) ) RETURN + END IF + END IF + + + IF ( PRESENT(volume) ) THEN + IF ( get_related_field(volume, output_field, cm_ind, cm_file_num) ) THEN + CALL prepend_attribute(output_field, 'cell_measures',& + & 'volume: '//TRIM(output_fields(cm_ind)%output_name)) + IF ( cm_file_num.NE.file_num ) THEN + ! Not in the same file, set the global attribute associated_files + ! Should look like :associated_files = " output_name: output_file_name " ; + CALL prepend_attribute(files(file_num), 'associated_files',& + & TRIM(output_fields(cm_ind)%output_name)//': '//& + & TRIM(date_prefix)//TRIM(files(cm_file_num)%name)//'.nc') + END IF + ELSE + IF ( fms_error_handler('diag_manager_mod::init_field_cell_measures',& + & 'VOLUME measures field "'//TRIM(input_fields(volume)%module_name)//'/'//& + & TRIM(input_fields(volume)%field_name)//& + & '" NOT in diag_table with correct output frequency for field '//& + & TRIM(input_fields(output_field%input_field)%module_name)//& + & '/'//TRIM(input_fields(output_field%input_field)%field_name), err_msg) ) RETURN + END IF + END IF + END SUBROUTINE init_field_cell_measures + ! + ! ! ! @@ -978,7 +1317,7 @@ LOGICAL FUNCTION send_data_1d(diag_field_id, field, time, is_in, mask, rmask, ie field_out(:, 1, 1) = field ! Default values for mask - IF ( PRESENT(mask) ) THEN + IF ( PRESENT(mask) ) THEN mask_out(:, 1, 1) = mask ELSE mask_out = .TRUE. @@ -1042,12 +1381,12 @@ LOGICAL FUNCTION send_data_2d(diag_field_id, field, time, is_in, js_in, & field_out(:, :, 1) = field ! Default values for mask - IF ( PRESENT(mask) ) THEN + IF ( PRESENT(mask) ) THEN mask_out(:, :, 1) = mask - ELSE + ELSE mask_out = .TRUE. END IF - + IF ( PRESENT(rmask) ) WHERE ( rmask < 0.5 ) mask_out(:, :, 1) = .FALSE. IF ( PRESENT(mask) .OR. PRESENT(rmask) ) THEN send_data_2d = send_data_3d(diag_field_id, field_out, time, is_in=is_in, js_in=js_in, ks_in=1, mask=mask_out,& @@ -1079,13 +1418,14 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & REAL, DIMENSION(:,:,:), INTENT(in) :: field REAL, INTENT(in), OPTIONAL :: weight TYPE (time_type), INTENT(in), OPTIONAL :: time - INTEGER, INTENT(in), OPTIONAL :: is_in, js_in, ks_in,ie_in,je_in, ke_in + INTEGER, INTENT(in), OPTIONAL :: is_in, js_in, ks_in,ie_in,je_in, ke_in LOGICAL, DIMENSION(:,:,:), INTENT(in), OPTIONAL :: mask REAL, DIMENSION(:,:,:), INTENT(in), OPTIONAL :: rmask CHARACTER(len=*), INTENT(out), OPTIONAL :: err_msg REAL :: weight1 REAL :: missvalue + INTEGER :: pow_value INTEGER :: ksr, ker INTEGER :: i, out_num, file_num, n1, n2, n3, number_of_outputs, ii,f1,f2,f3,f4 INTEGER :: freq, units, is, js, ks, ie, je, ke, i1, j1,k1, j, k @@ -1095,12 +1435,14 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & INTEGER :: day,second,tick ! components of the current date INTEGER :: status INTEGER :: numthreads + INTEGER :: active_omp_level #if defined(_OPENMP) INTEGER :: omp_get_num_threads !< OMP function + INTEGER :: omp_get_level !< OMP function #endif LOGICAL :: average, phys_window, need_compute LOGICAL :: reduced_k_range, local_output - LOGICAL :: time_max, time_min + LOGICAL :: time_max, time_min, time_rms LOGICAL :: missvalue_present LOGICAL, ALLOCATABLE, DIMENSION(:,:,:) :: oor_mask CHARACTER(len=256) :: err_msg_local @@ -1120,6 +1462,14 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & IF ( fms_error_handler('diag_manager_mod::send_data_3d', 'diag_manager NOT initialized', err_msg) ) RETURN END IF err_msg_local = '' + ! The following lines are commented out as they have not been included in the code prior to now, + ! and there are a lot of send_data calls before register_diag_field calls. A method to do this safely + ! needs to be developed. + ! + ! Set first_send_data_call to .FALSE. on first non-static field. +!!$ IF ( .NOT.input_fields(diag_field_id)%static .AND. first_send_data_call ) THEN +!!$ first_send_data_call = .FALSE. +!!$ END IF ! oor_mask is only used for checking out of range values. ALLOCATE(oor_mask(SIZE(field,1),SIZE(field,2),SIZE(field,3)), STAT=status) @@ -1129,9 +1479,9 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & IF ( fms_error_handler('diag_manager_mod::send_data_3d', err_msg_local, err_msg) ) RETURN END IF - IF ( PRESENT(mask) ) THEN + IF ( PRESENT(mask) ) THEN oor_mask = mask - ELSE + ELSE oor_mask = .TRUE. END IF IF ( PRESENT(rmask) ) WHERE ( rmask < 0.5 ) oor_mask = .FALSE. @@ -1149,14 +1499,14 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & ! of presence/absence of is,ie,js,je. The checks below should catch improper combinations. IF ( PRESENT(ie_in) ) THEN IF ( .NOT.PRESENT(is_in) ) THEN - IF ( fms_error_handler('diag_manager_mod::send_data_3d', 'ie_in present without is_in', err_msg) ) THEN + IF ( fms_error_handler('diag_manager_mod::send_data_3d', 'ie_in present without is_in', err_msg) ) THEN DEALLOCATE(oor_mask) RETURN END IF END IF IF ( PRESENT(js_in) .AND. .NOT.PRESENT(je_in) ) THEN IF ( fms_error_handler('diag_manager_modsend_data_3d',& - & 'is_in and ie_in present, but js_in present without je_in', err_msg) ) THEN + & 'is_in and ie_in present, but js_in present without je_in', err_msg) ) THEN DEALLOCATE(oor_mask) RETURN END IF @@ -1164,14 +1514,14 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & END IF IF ( PRESENT(je_in) ) THEN IF ( .NOT.PRESENT(js_in) ) THEN - IF ( fms_error_handler('diag_manager_mod::send_data_3d', 'je_in present without js_in', err_msg) ) THEN + IF ( fms_error_handler('diag_manager_mod::send_data_3d', 'je_in present without js_in', err_msg) ) THEN DEALLOCATE(oor_mask) RETURN END IF END IF IF ( PRESENT(is_in) .AND. .NOT.PRESENT(ie_in) ) THEN IF ( fms_error_handler('diag_manager_mod::send_data_3d',& - & 'js_in and je_in present, but is_in present without ie_in', err_msg)) THEN + & 'js_in and je_in present, but is_in present without ie_in', err_msg)) THEN DEALLOCATE(oor_mask) RETURN END IF @@ -1196,14 +1546,14 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & IF ( PRESENT(ke_in) ) ke = ke_in twohi = n1-(ie-is+1) IF ( MOD(twohi,2) /= 0 ) THEN - IF ( fms_error_handler('diag_manager_mod::send_data_3d', 'non-symmetric halos in first dimension', err_msg) ) THEN + IF ( fms_error_handler('diag_manager_mod::send_data_3d', 'non-symmetric halos in first dimension', err_msg) ) THEN DEALLOCATE(oor_mask) RETURN END IF END IF twohj = n2-(je-js+1) IF ( MOD(twohj,2) /= 0 ) THEN - IF ( fms_error_handler('diag_manager_mod::send_data_3d', 'non-symmetric halos in second dimension', err_msg) ) THEN + IF ( fms_error_handler('diag_manager_mod::send_data_3d', 'non-symmetric halos in second dimension', err_msg) ) THEN DEALLOCATE(oor_mask) RETURN END IF @@ -1227,12 +1577,12 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & f4=n2-hj ! weight is for time averaging where each time level may has a different weight - IF ( PRESENT(weight) ) THEN + IF ( PRESENT(weight) ) THEN weight1 = weight ELSE weight1 = 1. END IF - + ! Is there a missing_value? missvalue_present = input_fields(diag_field_id)%missing_value_present IF ( missvalue_present ) missvalue = input_fields(diag_field_id)%missing_value @@ -1240,10 +1590,13 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & number_of_outputs = input_fields(diag_field_id)%num_output_fields !$OMP CRITICAL input_fields(diag_field_id)%numthreads = 1 + active_omp_level=0 #if defined(_OPENMP) input_fields(diag_field_id)%numthreads = omp_get_num_threads() + input_fields(diag_field_id)%active_omp_level = omp_get_level() #endif numthreads = input_fields(diag_field_id)%numthreads + active_omp_level = input_fields(diag_field_id)%active_omp_level !$OMP END CRITICAL if(present(time)) input_fields(diag_field_id)%time = time @@ -1318,16 +1671,21 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & units = files(file_num)%output_units ! Is this output field being time averaged? average = output_fields(out_num)%time_average + ! Is this output field the rms? + ! If so, then average is also .TRUE. + time_rms = output_fields(out_num)%time_rms + ! Power value for rms or pow(x) calculations + pow_value = output_fields(out_num)%pow_value ! Looking for max and min value of this field over the sampling interval? time_max = output_fields(out_num)%time_max - time_min = output_fields(out_num)%time_min + time_min = output_fields(out_num)%time_min IF ( output_fields(out_num)%total_elements > SIZE(field(f1:f2,f3:f4,ks:ke)) ) THEN output_fields(out_num)%phys_window = .TRUE. ELSE output_fields(out_num)%phys_window = .FALSE. END IF phys_window = output_fields(out_num)%phys_window - IF ( need_compute ) THEN + IF ( need_compute ) THEN l_start = output_fields(out_num)%output_grid%l_start_indx l_end = output_fields(out_num)%output_grid%l_end_indx END IF @@ -1339,7 +1697,7 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & CALL get_time(time,second,day,tick) ! current date sample = set_time(second,0,tick)/dt + 1 END IF - + ! Get the vertical layer start and end index. IF ( reduced_k_range ) THEN l_start(3) = output_fields(out_num)%output_grid%l_start_indx(3) @@ -1358,19 +1716,19 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & & TRIM(input_fields(diag_field_id)%module_name),& & TRIM(output_fields(out_num)%output_name) IF ( fms_error_handler('diag_manager_mod::send_data_3d', 'module/output_field '//TRIM(error_string)//& - & ', time must be present when output frequency = EVERY_TIME', err_msg)) THEN + & ', time must be present when output frequency = EVERY_TIME', err_msg)) THEN DEALLOCATE(oor_mask) RETURN END IF END IF END IF - END IF + END IF IF ( .NOT.output_fields(out_num)%static .AND. .NOT.PRESENT(time) ) THEN WRITE (error_string,'(a,"/",a)')& & TRIM(input_fields(diag_field_id)%module_name), & & TRIM(output_fields(out_num)%output_name) IF ( fms_error_handler('diag_manager_mod::send_data_3d', 'module/output_field '//TRIM(error_string)//& - & ', time must be present for nonstatic field', err_msg)) THEN + & ', time must be present for nonstatic field', err_msg)) THEN DEALLOCATE(oor_mask) RETURN END IF @@ -1378,18 +1736,18 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & ! Is it time to output for this field; CAREFUL ABOUT > vs >= HERE !--- The fields send out within openmp parallel region will be written out in - !--- diag_send_complete. - IF ( numthreads == 1) then + !--- diag_send_complete. + IF ( (numthreads == 1) .AND. (active_omp_level.LE.1) ) then IF ( .NOT.output_fields(out_num)%static .AND. freq /= END_OF_RUN ) THEN IF ( time > output_fields(out_num)%next_output ) THEN ! A non-static field that has skipped a time level is an error IF ( time > output_fields(out_num)%next_next_output .AND. freq > 0 ) THEN - IF ( mpp_pe() .EQ. mpp_root_pe() ) THEN + IF ( mpp_pe() .EQ. mpp_root_pe() ) THEN WRITE (error_string,'(a,"/",a)')& & TRIM(input_fields(diag_field_id)%module_name), & & TRIM(output_fields(out_num)%output_name) IF ( fms_error_handler('diag_manager_mod::send_data_3d', 'module/output_field '//TRIM(error_string)//& - & ' is skipped one time level in output data', err_msg)) THEN + & ' is skipped one time level in output data', err_msg)) THEN DEALLOCATE(oor_mask) RETURN END IF @@ -1400,7 +1758,7 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & IF(status == -1) THEN IF ( mpp_pe() .EQ. mpp_root_pe() ) THEN IF(fms_error_handler('diag_manager_mod::send_data_3d','module/output_field '//TRIM(error_string)//& - & ', write EMPTY buffer', err_msg)) THEN + & ', write EMPTY buffer', err_msg)) THEN DEALLOCATE(oor_mask) RETURN END IF @@ -1408,57 +1766,63 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & END IF END IF !time > output_fields(out_num)%next_output END IF !.not.output_fields(out_num)%static .and. freq /= END_OF_RUN - ! Finished output of previously buffered data, now deal with buffering new data + ! Finished output of previously buffered data, now deal with buffering new data END IF IF ( .NOT.output_fields(out_num)%static .AND. .NOT.need_compute .AND. debug_diag_manager ) THEN CALL check_bounds_are_exact_dynamic(out_num, diag_field_id, Time, err_msg=err_msg_local) IF ( err_msg_local /= '' ) THEN - IF ( fms_error_handler('diag_manager_mod::send_data_3d', err_msg_local, err_msg) ) THEN + IF ( fms_error_handler('diag_manager_mod::send_data_3d', err_msg_local, err_msg) ) THEN DEALLOCATE(oor_mask) RETURN END IF END IF END IF - + ! Take care of submitted field data IF ( average ) THEN IF ( input_fields(diag_field_id)%mask_variant ) THEN IF ( need_compute ) THEN WRITE (error_string,'(a,"/",a)') & & TRIM(input_fields(diag_field_id)%module_name), & - & TRIM(output_fields(out_num)%output_name) + & TRIM(output_fields(out_num)%output_name) IF ( fms_error_handler('diag_manager_mod::send_data_3d', 'module/output_field '//TRIM(error_string)//& - & ', regional output NOT supported with mask_variant', err_msg)) THEN + & ', regional output NOT supported with mask_variant', err_msg)) THEN DEALLOCATE(oor_mask) RETURN END IF END IF ! Should reduced_k_range data be supported with the mask_variant option ????? - ! If not, error message should be produced and the reduced_k_range loop below eliminated + ! If not, error message should be produced and the reduced_k_range loop below eliminated IF ( PRESENT(mask) ) THEN - IF ( missvalue_present ) THEN + IF ( missvalue_present ) THEN IF ( debug_diag_manager ) THEN CALL update_bounds(out_num, is-hi, ie-hi, js-hj, je-hj, ks, ke) CALL check_out_of_bounds(out_num, diag_field_id, err_msg=err_msg_local) IF ( err_msg_local /= '' ) THEN - IF ( fms_error_handler('diag_manager_mod::send_data_3d', err_msg_local, err_msg) ) THEN + IF ( fms_error_handler('diag_manager_mod::send_data_3d', err_msg_local, err_msg) ) THEN DEALLOCATE(oor_mask) RETURN END IF END IF END IF IF( numthreads>1 .AND. phys_window ) then - IF ( reduced_k_range ) THEN + IF ( reduced_k_range ) THEN DO k= ksr, ker - k1= k - ksr + 1 + k1= k - ksr + 1 DO j=js, je DO i=is, ie IF ( mask(i-is+1+hi, j-js+1+hj, k) ) THEN - output_fields(out_num)%buffer(i-hi,j-hj,k1,sample) =& - & output_fields(out_num)%buffer(i-hi,j-hj,k1,sample) +& - & field(i-is+1+hi, j-js+1+hj, k) * weight1 + IF ( pow_value /= 1 ) THEN + output_fields(out_num)%buffer(i-hi,j-hj,k1,sample) =& + & output_fields(out_num)%buffer(i-hi,j-hj,k1,sample) +& + & (field(i-is+1+hi, j-js+1+hj, k) * weight1)**(pow_value) + ELSE + output_fields(out_num)%buffer(i-hi,j-hj,k1,sample) =& + & output_fields(out_num)%buffer(i-hi,j-hj,k1,sample) +& + & field(i-is+1+hi, j-js+1+hj, k) * weight1 + END IF output_fields(out_num)%counter(i-hi,j-hj,k1,sample) =& & output_fields(out_num)%counter(i-hi,j-hj,k1,sample) + weight1 END IF @@ -1466,13 +1830,19 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & END DO END DO ELSE - DO k=ks, ke + DO k=ks, ke DO j=js, je DO i=is, ie IF ( mask(i-is+1+hi, j-js+1+hj, k) ) THEN - output_fields(out_num)%buffer(i-hi,j-hj,k,sample) =& - & output_fields(out_num)%buffer(i-hi,j-hj,k,sample) +& - & field(i-is+1+hi,j-js+1+hj,k)*weight1 + IF ( pow_value /= 1 ) THEN + output_fields(out_num)%buffer(i-hi,j-hj,k,sample) =& + & output_fields(out_num)%buffer(i-hi,j-hj,k,sample) +& + & (field(i-is+1+hi,j-js+1+hj,k)*weight1)**(pow_value) + ELSE + output_fields(out_num)%buffer(i-hi,j-hj,k,sample) =& + & output_fields(out_num)%buffer(i-hi,j-hj,k,sample) +& + & field(i-is+1+hi,j-js+1+hj,k)*weight1 + END IF output_fields(out_num)%counter(i-hi,j-hj,k,sample) =& &output_fields(out_num)%counter(i-hi,j-hj,k,sample) + weight1 END IF @@ -1482,15 +1852,21 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & END IF ELSE !$OMP CRITICAL - IF ( reduced_k_range ) THEN + IF ( reduced_k_range ) THEN DO k= ksr, ker - k1= k - ksr + 1 + k1= k - ksr + 1 DO j=js, je DO i=is, ie IF ( mask(i-is+1+hi, j-js+1+hj, k) ) THEN - output_fields(out_num)%buffer(i-hi,j-hj,k1,sample) =& - & output_fields(out_num)%buffer(i-hi,j-hj,k1,sample) +& - & field(i-is+1+hi, j-js+1+hj, k) * weight1 + IF ( pow_value /= 1 ) THEN + output_fields(out_num)%buffer(i-hi,j-hj,k1,sample) =& + & output_fields(out_num)%buffer(i-hi,j-hj,k1,sample) +& + & (field(i-is+1+hi, j-js+1+hj, k) * weight1)**(pow_value) + ELSE + output_fields(out_num)%buffer(i-hi,j-hj,k1,sample) =& + & output_fields(out_num)%buffer(i-hi,j-hj,k1,sample) +& + & field(i-is+1+hi, j-js+1+hj, k) * weight1 + END IF output_fields(out_num)%counter(i-hi,j-hj,k1,sample) =& & output_fields(out_num)%counter(i-hi,j-hj,k1,sample) + weight1 END IF @@ -1498,13 +1874,19 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & END DO END DO ELSE - DO k=ks, ke + DO k=ks, ke DO j=js, je DO i=is, ie IF ( mask(i-is+1+hi, j-js+1+hj, k) ) THEN - output_fields(out_num)%buffer(i-hi,j-hj,k,sample) =& - & output_fields(out_num)%buffer(i-hi,j-hj,k,sample) +& - & field(i-is+1+hi,j-js+1+hj,k)*weight1 + IF ( pow_value /= 1 ) THEN + output_fields(out_num)%buffer(i-hi,j-hj,k,sample) =& + & output_fields(out_num)%buffer(i-hi,j-hj,k,sample) +& + & (field(i-is+1+hi,j-js+1+hj,k)*weight1)**(pow_value) + ELSE + output_fields(out_num)%buffer(i-hi,j-hj,k,sample) =& + & output_fields(out_num)%buffer(i-hi,j-hj,k,sample) +& + & field(i-is+1+hi,j-js+1+hj,k)*weight1 + END IF output_fields(out_num)%counter(i-hi,j-hj,k,sample) =& &output_fields(out_num)%counter(i-hi,j-hj,k,sample) + weight1 END IF @@ -1519,7 +1901,7 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & & TRIM(input_fields(diag_field_id)%module_name), & & TRIM(output_fields(out_num)%output_name) IF ( fms_error_handler('diag_manager_mod::send_data_3d', 'module/output_field '//TRIM(error_string)//& - & ', variable mask but no missing value defined', err_msg)) THEN + & ', variable mask but no missing value defined', err_msg)) THEN DEALLOCATE(oor_mask) RETURN END IF @@ -1529,7 +1911,7 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & & TRIM(input_fields(diag_field_id)%module_name), & & TRIM(output_fields(out_num)%output_name) IF(fms_error_handler('diag_manager_mod::send_data_3d','module/output_field '//TRIM(error_string)//& - & ', variable mask but no mask given', err_msg)) THEN + & ', variable mask but no mask given', err_msg)) THEN DEALLOCATE(oor_mask) RETURN END IF @@ -1541,17 +1923,23 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & IF (numthreads>1 .AND. phys_window) then DO k = l_start(3), l_end(3) k1 = k-l_start(3)+1 - DO j = js, je + DO j = js, je DO i = is, ie IF ( l_start(1)+hi <= i .AND. i <= l_end(1)+hi .AND. l_start(2)+hj <= j .AND. j <= l_end(2)+hj ) THEN - i1 = i-l_start(1)-hi+1 + i1 = i-l_start(1)-hi+1 j1= j-l_start(2)-hj+1 IF ( mask(i-is+1+hi, j-js+1+hj, k) ) THEN - output_fields(out_num)%buffer(i1,j1,k1,sample) =& - & output_fields(out_num)%buffer(i1,j1,k1,sample) +& - & field(i-is+1+hi,j-js+1+hj,k) * weight1 + IF ( pow_value /= 1 ) THEN + output_fields(out_num)%buffer(i1,j1,k1,sample) =& + & output_fields(out_num)%buffer(i1,j1,k1,sample) +& + & (field(i-is+1+hi,j-js+1+hj,k) * weight1)**(pow_value) + ELSE + output_fields(out_num)%buffer(i1,j1,k1,sample) =& + & output_fields(out_num)%buffer(i1,j1,k1,sample) +& + & field(i-is+1+hi,j-js+1+hj,k) * weight1 + END IF ELSE - output_fields(out_num)%buffer(i1,j1,k1,sample) = missvalue + output_fields(out_num)%buffer(i1,j1,k1,sample) = missvalue END IF END IF END DO @@ -1561,17 +1949,23 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & !$OMP CRITICAL DO k = l_start(3), l_end(3) k1 = k-l_start(3)+1 - DO j = js, je + DO j = js, je DO i = is, ie IF ( l_start(1)+hi <= i .AND. i <= l_end(1)+hi .AND. l_start(2)+hj <= j .AND. j <= l_end(2)+hj ) THEN - i1 = i-l_start(1)-hi+1 + i1 = i-l_start(1)-hi+1 j1= j-l_start(2)-hj+1 IF ( mask(i-is+1+hi, j-js+1+hj, k) ) THEN - output_fields(out_num)%buffer(i1,j1,k1,sample) =& - & output_fields(out_num)%buffer(i1,j1,k1,sample) +& - & field(i-is+1+hi,j-js+1+hj,k) * weight1 + IF ( pow_value /= 1 ) THEN + output_fields(out_num)%buffer(i1,j1,k1,sample) =& + & output_fields(out_num)%buffer(i1,j1,k1,sample) +& + & (field(i-is+1+hi,j-js+1+hj,k) * weight1)**(pow_value) + ELSE + output_fields(out_num)%buffer(i1,j1,k1,sample) =& + & output_fields(out_num)%buffer(i1,j1,k1,sample) +& + & field(i-is+1+hi,j-js+1+hj,k) * weight1 + END IF ELSE - output_fields(out_num)%buffer(i1,j1,k1,sample) = missvalue + output_fields(out_num)%buffer(i1,j1,k1,sample) = missvalue END IF END IF END DO @@ -1580,7 +1974,7 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & !$OMP END CRITICAL ENDIF !$OMP CRITICAL - DO j = js, je + DO j = js, je DO i = is, ie IF ( l_start(1)+hi <= i .AND. i <= l_end(1)+hi .AND. l_start(2)+hj <= j .AND. j <= l_end(2)+hj ) THEN output_fields(out_num)%num_elements(sample) = & @@ -1589,16 +1983,22 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & END DO END DO !$OMP END CRITICAL - ELSE IF ( reduced_k_range ) THEN + ELSE IF ( reduced_k_range ) THEN IF (numthreads>1 .AND. phys_window) then DO k=ksr, ker - k1 = k - ksr + 1 + k1 = k - ksr + 1 DO j=js, je DO i=is, ie IF ( mask(i-is+1+hi,j-js+1+hj,k) ) THEN - output_fields(out_num)%buffer(i-hi,j-hj,k1,sample) =& - & output_fields(out_num)%buffer(i-hi,j-hj,k1,sample) +& - & field(i-is+1+hi,j-js+1+hj,k) * weight1 + IF ( pow_value /= 1 ) THEN + output_fields(out_num)%buffer(i-hi,j-hj,k1,sample) =& + & output_fields(out_num)%buffer(i-hi,j-hj,k1,sample) +& + & (field(i-is+1+hi,j-js+1+hj,k) * weight1)**(pow_value) + ELSE + output_fields(out_num)%buffer(i-hi,j-hj,k1,sample) =& + & output_fields(out_num)%buffer(i-hi,j-hj,k1,sample) +& + & field(i-is+1+hi,j-js+1+hj,k) * weight1 + END IF ELSE output_fields(out_num)%buffer(i-hi,j-hj,k1,sample)= missvalue END IF @@ -1608,13 +2008,19 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & ELSE !$OMP CRITICAL DO k=ksr, ker - k1 = k - ksr + 1 + k1 = k - ksr + 1 DO j=js, je DO i=is, ie IF ( mask(i-is+1+hi,j-js+1+hj,k) ) THEN - output_fields(out_num)%buffer(i-hi,j-hj,k1,sample) =& - & output_fields(out_num)%buffer(i-hi,j-hj,k1,sample) +& - & field(i-is+1+hi,j-js+1+hj,k) * weight1 + IF ( pow_value /= 1 ) THEN + output_fields(out_num)%buffer(i-hi,j-hj,k1,sample) =& + & output_fields(out_num)%buffer(i-hi,j-hj,k1,sample) +& + & (field(i-is+1+hi,j-js+1+hj,k) * weight1)**(pow_value) + ELSE + output_fields(out_num)%buffer(i-hi,j-hj,k1,sample) =& + & output_fields(out_num)%buffer(i-hi,j-hj,k1,sample) +& + & field(i-is+1+hi,j-js+1+hj,k) * weight1 + END IF ELSE output_fields(out_num)%buffer(i-hi,j-hj,k1,sample)= missvalue END IF @@ -1628,7 +2034,7 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & CALL update_bounds(out_num, is-hi, ie-hi, js-hj, je-hj, ks, ke) CALL check_out_of_bounds(out_num, diag_field_id, err_msg=err_msg_local) IF ( err_msg_local /= '' ) THEN - IF ( fms_error_handler('diag_manager_mod::send_data_3d', err_msg_local, err_msg) ) THEN + IF ( fms_error_handler('diag_manager_mod::send_data_3d', err_msg_local, err_msg) ) THEN DEALLOCATE(oor_mask) RETURN END IF @@ -1639,9 +2045,15 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & DO j=js, je DO i=is, ie IF ( mask(i-is+1+hi,j-js+1+hj,k) ) THEN - output_fields(out_num)%buffer(i-hi,j-hj,k,sample) =& - & output_fields(out_num)%buffer(i-hi,j-hj,k,sample) +& - & field(i-is+1+hi,j-js+1+hj,k) * weight1 + IF ( pow_value /= 1 ) THEN + output_fields(out_num)%buffer(i-hi,j-hj,k,sample) =& + & output_fields(out_num)%buffer(i-hi,j-hj,k,sample) +& + & (field(i-is+1+hi,j-js+1+hj,k) * weight1)**(pow_value) + ELSE + output_fields(out_num)%buffer(i-hi,j-hj,k,sample) =& + & output_fields(out_num)%buffer(i-hi,j-hj,k,sample) +& + & field(i-is+1+hi,j-js+1+hj,k) * weight1 + END IF ELSE output_fields(out_num)%buffer(i-hi,j-hj,k,sample)= missvalue END IF @@ -1654,16 +2066,22 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & DO j=js, je DO i=is, ie IF ( mask(i-is+1+hi,j-js+1+hj,k) ) THEN - output_fields(out_num)%buffer(i-hi,j-hj,k,sample) =& - & output_fields(out_num)%buffer(i-hi,j-hj,k,sample) +& - & field(i-is+1+hi,j-js+1+hj,k) * weight1 + IF ( pow_value /= 1 ) THEN + output_fields(out_num)%buffer(i-hi,j-hj,k,sample) =& + & output_fields(out_num)%buffer(i-hi,j-hj,k,sample) +& + & (field(i-is+1+hi,j-js+1+hj,k) * weight1)**(pow_value) + ELSE + output_fields(out_num)%buffer(i-hi,j-hj,k,sample) =& + & output_fields(out_num)%buffer(i-hi,j-hj,k,sample) +& + & field(i-is+1+hi,j-js+1+hj,k) * weight1 + END IF ELSE output_fields(out_num)%buffer(i-hi,j-hj,k,sample)= missvalue END IF END DO END DO - END DO -!$OMP END CRITICAL + END DO +!$OMP END CRITICAL END IF END IF !$OMP CRITICAL @@ -1675,7 +2093,7 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & IF ( ANY(mask(f1:f2,f3:f4,ks:ke)) ) output_fields(out_num)%count_0d(sample) =& & output_fields(out_num)%count_0d(sample)+weight1 END IF -!$OMP END CRITICAL +!$OMP END CRITICAL ELSE ! missing value NOT present IF ( (.NOT.ALL(mask(f1:f2,f3:f4,ks:ke)) .AND. mpp_pe() .EQ. mpp_root_pe()).AND.& @@ -1690,34 +2108,44 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & & trim(input_fields(diag_field_id)%module_name), WARNING) input_fields(diag_field_id)%issued_mask_ignore_warning = .TRUE. END IF - IF ( need_compute ) THEN + IF ( need_compute ) THEN IF (numthreads>1 .AND. phys_window) then - DO j = js, je + DO j = js, je DO i = is, ie IF ( l_start(1)+hi <= i .AND. i <= l_end(1)+hi .AND. l_start(2)+hj <= j .AND. j <= l_end(2)+hj ) THEN - i1 = i-l_start(1)-hi+1 + i1 = i-l_start(1)-hi+1 j1 = j-l_start(2)-hj+1 - output_fields(out_num)%buffer(i1,j1,:,sample)= output_fields(out_num)%buffer(i1,j1,:,sample)+ & - & field(i-is+1+hi,j-js+1+hj,l_start(3):l_end(3))*weight1 + IF ( pow_value /= 1 ) THEN + output_fields(out_num)%buffer(i1,j1,:,sample)= output_fields(out_num)%buffer(i1,j1,:,sample)+ & + & (field(i-is+1+hi,j-js+1+hj,l_start(3):l_end(3))*weight1)**(pow_value) + ELSE + output_fields(out_num)%buffer(i1,j1,:,sample)= output_fields(out_num)%buffer(i1,j1,:,sample)+ & + & field(i-is+1+hi,j-js+1+hj,l_start(3):l_end(3))*weight1 + END IF END IF END DO END DO ELSE !$OMP CRITICAL - DO j = js, je + DO j = js, je DO i = is, ie IF ( l_start(1)+hi <= i .AND. i <= l_end(1)+hi .AND. l_start(2)+hj <= j .AND. j <= l_end(2)+hj ) THEN - i1 = i-l_start(1)-hi+1 + i1 = i-l_start(1)-hi+1 j1 = j-l_start(2)-hj+1 - output_fields(out_num)%buffer(i1,j1,:,sample)= output_fields(out_num)%buffer(i1,j1,:,sample)+ & - & field(i-is+1+hi,j-js+1+hj,l_start(3):l_end(3))*weight1 + IF ( pow_value /= 1 ) THEN + output_fields(out_num)%buffer(i1,j1,:,sample)= output_fields(out_num)%buffer(i1,j1,:,sample)+ & + & (field(i-is+1+hi,j-js+1+hj,l_start(3):l_end(3))*weight1)**(pow_value) + ELSE + output_fields(out_num)%buffer(i1,j1,:,sample)= output_fields(out_num)%buffer(i1,j1,:,sample)+ & + & field(i-is+1+hi,j-js+1+hj,l_start(3):l_end(3))*weight1 + END IF END IF END DO END DO !$OMP END CRITICAL END IF !$OMP CRITICAL - DO j = js, je + DO j = js, je DO i = is, ie IF ( l_start(1)+hi <= i .AND. i <= l_end(1)+hi .AND. l_start(2)+hj <= j .AND. j <= l_end(2)+hj ) THEN output_fields(out_num)%num_elements(sample)=& @@ -1727,20 +2155,32 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & END DO END DO !$OMP END CRITICAL - ELSE IF ( reduced_k_range ) THEN + ELSE IF ( reduced_k_range ) THEN IF (numthreads>1 .AND. phys_window) then ksr= l_start(3) ker= l_end(3) - output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,:,sample) =& - & output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,:,sample)& - & + field(f1:f2,f3:f4,ksr:ker)*weight1 - ELSE -!$OMP CRITICAL + IF ( pow_value /= 1 ) THEN + output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,:,sample) =& + & output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,:,sample) +& + & (field(f1:f2,f3:f4,ksr:ker)*weight1)**(pow_value) + ELSE + output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,:,sample) =& + & output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,:,sample) +& + & field(f1:f2,f3:f4,ksr:ker)*weight1 + END IF + ELSE +!$OMP CRITICAL ksr= l_start(3) ker= l_end(3) - output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,:,sample) =& - & output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,:,sample)& - & + field(f1:f2,f3:f4,ksr:ker)*weight1 + IF ( pow_value /= 1 ) THEN + output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,:,sample) =& + & output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,:,sample) +& + & (field(f1:f2,f3:f4,ksr:ker)*weight1)**(pow_value) + ELSE + output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,:,sample) =& + & output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,:,sample) +& + & field(f1:f2,f3:f4,ksr:ker)*weight1 + END IF !$OMP END CRITICAL END IF ELSE @@ -1748,44 +2188,62 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & CALL update_bounds(out_num, is-hi, ie-hi, js-hj, je-hj, ks, ke) CALL check_out_of_bounds(out_num, diag_field_id, err_msg=err_msg_local) IF ( err_msg_local /= '') THEN - IF ( fms_error_handler('diag_manager_mod::send_data_3d', err_msg_local, err_msg) ) THEN + IF ( fms_error_handler('diag_manager_mod::send_data_3d', err_msg_local, err_msg) ) THEN DEALLOCATE(oor_mask) RETURN END IF END IF END IF IF (numthreads>1 .AND. phys_window) then - output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,ks:ke,sample) =& - & output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,ks:ke,sample)& - & + field(f1:f2,f3:f4,ks:ke)*weight1 + IF ( pow_value /= 1 ) THEN + output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,ks:ke,sample) =& + & output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,ks:ke,sample) +& + & (field(f1:f2,f3:f4,ks:ke)*weight1)**(pow_value) + ELSE + output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,ks:ke,sample) =& + & output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,ks:ke,sample) +& + & field(f1:f2,f3:f4,ks:ke)*weight1 + END IF ELSE -!$OMP CRITICAL - output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,ks:ke,sample) =& - & output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,ks:ke,sample)& - & + field(f1:f2,f3:f4,ks:ke)*weight1 -!$OMP END CRITICAL +!$OMP CRITICAL + IF ( pow_value /= 1 ) THEN + output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,ks:ke,sample) =& + & output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,ks:ke,sample) +& + & (field(f1:f2,f3:f4,ks:ke)*weight1)**(pow_value) + ELSE + output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,ks:ke,sample) =& + & output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,ks:ke,sample) +& + & field(f1:f2,f3:f4,ks:ke)*weight1 + END IF +!$OMP END CRITICAL END IF END IF -!$OMP CRITICAL +!$OMP CRITICAL IF ( .NOT.phys_window ) output_fields(out_num)%count_0d(sample) =& & output_fields(out_num)%count_0d(sample) + weight1 -!$OMP END CRITICAL +!$OMP END CRITICAL END IF ELSE ! mask NOT present IF ( missvalue_present ) THEN - IF ( need_compute ) THEN + IF ( need_compute ) THEN if( numthreads>1 .AND. phys_window ) then DO k = l_start(3), l_end(3) - k1 = k - l_start(3) + 1 - DO j = js, je + k1 = k - l_start(3) + 1 + DO j = js, je DO i = is, ie IF ( l_start(1)+hi <= i .AND. i <= l_end(1)+hi .AND. l_start(2)+hj <= j .AND. j <= l_end(2)+hj) THEN - i1 = i-l_start(1)-hi+1 - j1= j-l_start(2)-hj+1 + i1 = i-l_start(1)-hi+1 + j1= j-l_start(2)-hj+1 IF ( field(i-is+1+hi,j-js+1+hj,k) /= missvalue ) THEN - output_fields(out_num)%buffer(i1,j1,k1,sample) =& - & output_fields(out_num)%buffer(i1,j1,k1,sample) +& - & field(i-is+1+hi,j-js+1+hj,k) * weight1 + IF ( pow_value /= 1 ) THEN + output_fields(out_num)%buffer(i1,j1,k1,sample) =& + & output_fields(out_num)%buffer(i1,j1,k1,sample) +& + & (field(i-is+1+hi,j-js+1+hj,k) * weight1)**(pow_value) + ELSE + output_fields(out_num)%buffer(i1,j1,k1,sample) =& + & output_fields(out_num)%buffer(i1,j1,k1,sample) +& + & field(i-is+1+hi,j-js+1+hj,k) * weight1 + END IF ELSE output_fields(out_num)%buffer(i1,j1,k1,sample) = missvalue END IF @@ -1796,16 +2254,22 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & ELSE !$OMP CRITICAL DO k = l_start(3), l_end(3) - k1 = k - l_start(3) + 1 - DO j = js, je + k1 = k - l_start(3) + 1 + DO j = js, je DO i = is, ie IF ( l_start(1)+hi <= i .AND. i <= l_end(1)+hi .AND. l_start(2)+hj <= j .AND. j <= l_end(2)+hj) THEN - i1 = i-l_start(1)-hi+1 - j1= j-l_start(2)-hj+1 + i1 = i-l_start(1)-hi+1 + j1= j-l_start(2)-hj+1 IF ( field(i-is+1+hi,j-js+1+hj,k) /= missvalue ) THEN - output_fields(out_num)%buffer(i1,j1,k1,sample) =& - & output_fields(out_num)%buffer(i1,j1,k1,sample) +& - & field(i-is+1+hi,j-js+1+hj,k) * weight1 + IF ( pow_value /= 1 ) THEN + output_fields(out_num)%buffer(i1,j1,k1,sample) =& + & output_fields(out_num)%buffer(i1,j1,k1,sample) +& + & (field(i-is+1+hi,j-js+1+hj,k) * weight1)**(pow_value) + ELSE + output_fields(out_num)%buffer(i1,j1,k1,sample) =& + & output_fields(out_num)%buffer(i1,j1,k1,sample) +& + & field(i-is+1+hi,j-js+1+hj,k) * weight1 + END IF ELSE output_fields(out_num)%buffer(i1,j1,k1,sample) = missvalue END IF @@ -1816,7 +2280,7 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & !$OMP END CRITICAL END IF !$OMP CRITICAL - DO j = js, je + DO j = js, je DO i = is, ie IF ( l_start(1)+hi <= i .AND. i <= l_end(1)+hi .AND. l_start(2)+hj <= j .AND. j <= l_end(2)+hj) THEN output_fields(out_num)%num_elements(sample) =& @@ -1825,14 +2289,11 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & END DO END DO IF ( .NOT.phys_window ) THEN - !rab if(any(field(l_start(1)+hi:l_end(1)+hi,l_start(2)+hj:l_end(2)+hj,l_start(3):l_end(3)) /= & - !rab & missvalue)) & - !rab & output_fields(out_num)%count_0d = output_fields(out_num)%count_0d + weight1 outer0: DO k = l_start(3), l_end(3) DO j=l_start(2)+hj, l_end(2)+hj DO i=l_start(1)+hi, l_end(1)+hi IF ( field(i,j,k) /= missvalue ) THEN - output_fields(out_num)%count_0d(sample) = output_fields(out_num)%count_0d(sample) + weight1 + output_fields(out_num)%count_0d(sample) = output_fields(out_num)%count_0d(sample) + weight1 EXIT outer0 END IF END DO @@ -1840,7 +2301,7 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & END DO outer0 END IF !$OMP END CRITICAL - ELSE IF ( reduced_k_range ) THEN + ELSE IF ( reduced_k_range ) THEN if( numthreads>1 .AND. phys_window ) then ksr= l_start(3) ker= l_end(3) @@ -1849,9 +2310,15 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & DO j=js, je DO i=is, ie IF ( field(i-is+1+hi,j-js+1+hj,k) /= missvalue ) THEN - output_fields(out_num)%buffer(i-hi,j-hj,k1,sample) =& - & output_fields(out_num)%buffer(i-hi,j-hj,k1,sample) +& - & field(i-is+1+hi,j-js+1+hj,k) * weight1 + IF ( pow_value /= 1 ) THEN + output_fields(out_num)%buffer(i-hi,j-hj,k1,sample) =& + & output_fields(out_num)%buffer(i-hi,j-hj,k1,sample) +& + & (field(i-is+1+hi,j-js+1+hj,k) * weight1)**(pow_value) + ELSE + output_fields(out_num)%buffer(i-hi,j-hj,k1,sample) =& + & output_fields(out_num)%buffer(i-hi,j-hj,k1,sample) +& + & field(i-is+1+hi,j-js+1+hj,k) * weight1 + END IF ELSE output_fields(out_num)%buffer(i-hi,j-hj,k1,sample) = missvalue END IF @@ -1867,9 +2334,15 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & DO j=js, je DO i=is, ie IF ( field(i-is+1+hi,j-js+1+hj,k) /= missvalue ) THEN - output_fields(out_num)%buffer(i-hi,j-hj,k1,sample) =& - & output_fields(out_num)%buffer(i-hi,j-hj,k1,sample) +& - & field(i-is+1+hi,j-js+1+hj,k) * weight1 + IF ( pow_value /= 1 ) THEN + output_fields(out_num)%buffer(i-hi,j-hj,k1,sample) =& + & output_fields(out_num)%buffer(i-hi,j-hj,k1,sample) +& + & (field(i-is+1+hi,j-js+1+hj,k) * weight1)**(pow_value) + ELSE + output_fields(out_num)%buffer(i-hi,j-hj,k1,sample) =& + & output_fields(out_num)%buffer(i-hi,j-hj,k1,sample) +& + & field(i-is+1+hi,j-js+1+hj,k) * weight1 + END IF ELSE output_fields(out_num)%buffer(i-hi,j-hj,k1,sample) = missvalue END IF @@ -1878,16 +2351,13 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & END DO !$OMP END CRITICAL END IF - !rab - !rab if(any(field(f1:f2,f3:f4,ks:ke) /= missvalue)) & - !rab & output_fields(out_num)%count_0d = output_fields(out_num)%count_0d + weight1 !$OMP CRITICAL outer3: DO k = ksr, ker k1=k-ksr+1 - DO j=f3, f4 - DO i=f1, f2 + DO j=f3, f4 + DO i=f1, f2 IF ( field(i,j,k) /= missvalue ) THEN - output_fields(out_num)%count_0d = output_fields(out_num)%count_0d + weight1 + output_fields(out_num)%count_0d = output_fields(out_num)%count_0d + weight1 EXIT outer3 END IF END DO @@ -1899,7 +2369,7 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & CALL update_bounds(out_num, is-hi, ie-hi, js-hj, je-hj, ks, ke) CALL check_out_of_bounds(out_num, diag_field_id, err_msg=err_msg_local) IF ( err_msg_local /= '' ) THEN - IF ( fms_error_handler('diag_manager_mod::send_data_3d', err_msg_local, err_msg) ) THEN + IF ( fms_error_handler('diag_manager_mod::send_data_3d', err_msg_local, err_msg) ) THEN DEALLOCATE(oor_mask) RETURN END IF @@ -1910,9 +2380,15 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & DO j=js, je DO i=is, ie IF ( field(i-is+1+hi,j-js+1+hj,k) /= missvalue ) THEN - output_fields(out_num)%buffer(i-hi,j-hj,k,sample) =& - & output_fields(out_num)%buffer(i-hi,j-hj,k,sample) +& - & field(i-is+1+hi,j-js+1+hj,k) * weight1 + IF ( pow_value /= 1 ) THEN + output_fields(out_num)%buffer(i-hi,j-hj,k,sample) =& + & output_fields(out_num)%buffer(i-hi,j-hj,k,sample) +& + & (field(i-is+1+hi,j-js+1+hj,k) * weight1)**(pow_value) + ELSE + output_fields(out_num)%buffer(i-hi,j-hj,k,sample) =& + & output_fields(out_num)%buffer(i-hi,j-hj,k,sample) +& + & field(i-is+1+hi,j-js+1+hj,k) * weight1 + END IF ELSE output_fields(out_num)%buffer(i-hi,j-hj,k,sample) = missvalue END IF @@ -1925,9 +2401,15 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & DO j=js, je DO i=is, ie IF ( field(i-is+1+hi,j-js+1+hj,k) /= missvalue ) THEN - output_fields(out_num)%buffer(i-hi,j-hj,k,sample) =& - & output_fields(out_num)%buffer(i-hi,j-hj,k,sample) +& - & field(i-is+1+hi,j-js+1+hj,k) * weight1 + IF ( pow_value /= 1 ) THEN + output_fields(out_num)%buffer(i-hi,j-hj,k,sample) =& + & output_fields(out_num)%buffer(i-hi,j-hj,k,sample) +& + & (field(i-is+1+hi,j-js+1+hj,k) * weight1)**(pow_value) + ELSE + output_fields(out_num)%buffer(i-hi,j-hj,k,sample) =& + & output_fields(out_num)%buffer(i-hi,j-hj,k,sample) +& + & field(i-is+1+hi,j-js+1+hj,k) * weight1 + END IF ELSE output_fields(out_num)%buffer(i-hi,j-hj,k,sample) = missvalue END IF @@ -1937,11 +2419,11 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & !$OMP END CRITICAL END IF !$OMP CRITICAL - outer1: DO k=ks, ke - DO j=f3, f4 - DO i=f1, f2 + outer1: DO k=ks, ke + DO j=f3, f4 + DO i=f1, f2 IF ( field(i,j,k) /= missvalue ) THEN - output_fields(out_num)%count_0d(sample) = output_fields(out_num)%count_0d(sample) + weight1 + output_fields(out_num)%count_0d(sample) = output_fields(out_num)%count_0d(sample) + weight1 EXIT outer1 END IF END DO @@ -1952,25 +2434,35 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & ELSE ! no missing value defined, No mask IF ( need_compute ) THEN IF( numthreads > 1 .AND. phys_window ) then - DO j = js, je - DO i = is, ie + DO j = js, je + DO i = is, ie IF ( l_start(1)+hi <= i .AND. i <= l_end(1)+hi .AND. l_start(2)+hj <= j .AND. j <= l_end(2)+hj ) THEN - i1 = i-l_start(1)-hi+1 + i1 = i-l_start(1)-hi+1 j1= j-l_start(2)-hj+1 - output_fields(out_num)%buffer(i1,j1,:,sample)= output_fields(out_num)%buffer(i1,j1,:,sample) +& - & field(i-is+1+hi,j-js+1+hj,l_start(3):l_end(3))*weight1 + IF ( pow_value /= 1 ) THEN + output_fields(out_num)%buffer(i1,j1,:,sample)= output_fields(out_num)%buffer(i1,j1,:,sample) +& + & (field(i-is+1+hi,j-js+1+hj,l_start(3):l_end(3))*weight1)**(pow_value) + ELSE + output_fields(out_num)%buffer(i1,j1,:,sample)= output_fields(out_num)%buffer(i1,j1,:,sample) +& + & field(i-is+1+hi,j-js+1+hj,l_start(3):l_end(3))*weight1 + END IF END IF END DO END DO ELSE !$OMP CRITICAL - DO j = js, je - DO i = is, ie + DO j = js, je + DO i = is, ie IF ( l_start(1)+hi <= i .AND. i <= l_end(1)+hi .AND. l_start(2)+hj <= j .AND. j <= l_end(2)+hj ) THEN - i1 = i-l_start(1)-hi+1 + i1 = i-l_start(1)-hi+1 j1= j-l_start(2)-hj+1 - output_fields(out_num)%buffer(i1,j1,:,sample)= output_fields(out_num)%buffer(i1,j1,:,sample) +& - & field(i-is+1+hi,j-js+1+hj,l_start(3):l_end(3))*weight1 + IF ( pow_value /= 1 ) THEN + output_fields(out_num)%buffer(i1,j1,:,sample)= output_fields(out_num)%buffer(i1,j1,:,sample) +& + & (field(i-is+1+hi,j-js+1+hj,l_start(3):l_end(3))*weight1)**(pow_value) + ELSE + output_fields(out_num)%buffer(i1,j1,:,sample)= output_fields(out_num)%buffer(i1,j1,:,sample) +& + & field(i-is+1+hi,j-js+1+hj,l_start(3):l_end(3))*weight1 + END IF END IF END DO END DO @@ -1978,8 +2470,8 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & END IF !$OMP CRITICAL - DO j = js, je - DO i = is, ie + DO j = js, je + DO i = is, ie IF ( l_start(1)+hi <= i .AND. i <= l_end(1)+hi .AND. l_start(2)+hj <= j .AND. j <= l_end(2)+hj ) THEN output_fields(out_num)%num_elements(sample) =& & output_fields(out_num)%num_elements(sample)+l_end(3)-l_start(3)+1 @@ -1987,41 +2479,65 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & END DO END DO !$OMP END CRITICAL - ! Accumulate time average - ELSE IF ( reduced_k_range ) THEN + ! Accumulate time average + ELSE IF ( reduced_k_range ) THEN ksr= l_start(3) ker= l_end(3) IF( numthreads > 1 .AND. phys_window ) then - output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,:,sample) =& - & output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,:,sample) + & - & field(f1:f2,f3:f4,ksr:ker)*weight1 + IF ( pow_value /= 1 ) THEN + output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,:,sample) =& + & output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,:,sample) + & + & (field(f1:f2,f3:f4,ksr:ker)*weight1)**(pow_value) + ELSE + output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,:,sample) =& + & output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,:,sample) + & + & field(f1:f2,f3:f4,ksr:ker)*weight1 + END IF ELSE !$OMP CRITICAL - output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,:,sample) =& - & output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,:,sample) + & - & field(f1:f2,f3:f4,ksr:ker)*weight1 + IF ( pow_value /= 1 ) THEN + output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,:,sample) =& + & output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,:,sample) + & + & (field(f1:f2,f3:f4,ksr:ker)*weight1)**(pow_value) + ELSE + output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,:,sample) =& + & output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,:,sample) + & + & field(f1:f2,f3:f4,ksr:ker)*weight1 + END IF !$OMP END CRITICAL END IF - ELSE + ELSE IF ( debug_diag_manager ) THEN CALL update_bounds(out_num, is-hi, ie-hi, js-hj, je-hj, ks, ke) CALL check_out_of_bounds(out_num, diag_field_id, err_msg=err_msg_local) IF ( err_msg_local /= '' ) THEN - IF ( fms_error_handler('diag_manager_mod::send_data_3d', err_msg_local, err_msg) ) THEN + IF ( fms_error_handler('diag_manager_mod::send_data_3d', err_msg_local, err_msg) ) THEN DEALLOCATE(oor_mask) RETURN END IF END IF END IF IF( numthreads > 1 .AND. phys_window ) then - output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,ks:ke,sample) =& - & output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,ks:ke,sample) +& - & field(f1:f2,f3:f4,ks:ke)*weight1 + IF ( pow_value /= 1 ) THEN + output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,ks:ke,sample) =& + & output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,ks:ke,sample) +& + & (field(f1:f2,f3:f4,ks:ke)*weight1)**(pow_value) + ELSE + output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,ks:ke,sample) =& + & output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,ks:ke,sample) +& + & field(f1:f2,f3:f4,ks:ke)*weight1 + END IF ELSE !$OMP CRITICAL - output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,ks:ke,sample) =& - & output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,ks:ke,sample) +& - & field(f1:f2,f3:f4,ks:ke)*weight1 + IF ( pow_value /= 1 ) THEN + output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,ks:ke,sample) =& + & output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,ks:ke,sample) +& + & (field(f1:f2,f3:f4,ks:ke)*weight1)**(pow_value) + ELSE + output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,ks:ke,sample) =& + & output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,ks:ke,sample) +& + & field(f1:f2,f3:f4,ks:ke)*weight1 + END IF !$OMP END CRITICAL END IF END IF @@ -2049,7 +2565,7 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & DO j = js, je DO i = is, ie IF ( l_start(1)+hi <= i .AND. i <= l_end(1)+hi .AND. l_start(2)+hj <= j .AND. j <= l_end(2)+hj ) THEN - i1 = i-l_start(1)-hi+1 + i1 = i-l_start(1)-hi+1 j1= j-l_start(2)-hj+1 IF ( mask(i-is+1+hi,j-js+1+hj,k) .AND.& & field(i-is+1+hi,j-js+1+hj,k)>output_fields(out_num)%buffer(i1,j1,k1,sample)) THEN @@ -2059,8 +2575,8 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & END DO END DO END DO - ! Maximum time value with masking - ELSE IF ( reduced_k_range ) THEN + ! Maximum time value with masking + ELSE IF ( reduced_k_range ) THEN ksr = l_start(3) ker = l_end(3) WHERE ( mask(f1:f2,f3:f4,ksr:ker) .AND. & @@ -2071,7 +2587,7 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & CALL update_bounds(out_num, is-hi, ie-hi, js-hj, je-hj, ks, ke) CALL check_out_of_bounds(out_num, diag_field_id, err_msg=err_msg_local) IF ( err_msg_local /= '' ) THEN - IF ( fms_error_handler('diag_manager_mod::send_data_3d', err_msg_local, err_msg) ) THEN + IF ( fms_error_handler('diag_manager_mod::send_data_3d', err_msg_local, err_msg) ) THEN DEALLOCATE(oor_mask) RETURN END IF @@ -2088,7 +2604,7 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & DO j = js, je DO i = is, ie IF(l_start(1)+hi <= i .AND. i <= l_end(1)+hi .AND. l_start(2)+hj <= j .AND. j <= l_end(2)+hj ) THEN - i1 = i-l_start(1)-hi+1 + i1 = i-l_start(1)-hi+1 j1 = j-l_start(2)-hj+1 IF ( field(i-is+1+hi,j-js+1+hj,k) > output_fields(out_num)%buffer(i1,j1,k1,sample) ) THEN output_fields(out_num)%buffer(i1,j1,k1,sample) = field(i-is+1+hi,j-js+1+hj,k) @@ -2097,8 +2613,8 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & END DO END DO END DO - ! Maximum time value - ELSE IF ( reduced_k_range ) THEN + ! Maximum time value + ELSE IF ( reduced_k_range ) THEN ksr = l_start(3) ker = l_end(3) WHERE ( field(f1:f2,f3:f4,ksr:ker) > output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,:,sample) ) & @@ -2108,7 +2624,7 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & CALL update_bounds(out_num, is-hi, ie-hi, js-hj, je-hj, ks, ke) CALL check_out_of_bounds(out_num, diag_field_id, err_msg=err_msg_local) IF ( err_msg_local /= '' ) THEN - IF ( fms_error_handler('diag_manager_mod::send_data_3d', err_msg_local, err_msg) ) THEN + IF ( fms_error_handler('diag_manager_mod::send_data_3d', err_msg_local, err_msg) ) THEN DEALLOCATE(oor_mask) RETURN END IF @@ -2127,7 +2643,7 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & DO j = js, je DO i = is, ie IF ( l_start(1)+hi <= i .AND. i <= l_end(1)+hi .AND. l_start(2)+hj <= j .AND. j <= l_end(2)+hj ) THEN - i1 = i-l_start(1)-hi+1 + i1 = i-l_start(1)-hi+1 j1 = j-l_start(2)-hj+1 IF ( mask(i-is+1+hi,j-js+1+hj,k) .AND.& & field(i-is+1+hi,j-js+1+hj,k) < output_fields(out_num)%buffer(i1,j1,k1,sample) ) THEN @@ -2137,8 +2653,8 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & END DO END DO END DO - ! Minimum time value with masking - ELSE IF ( reduced_k_range ) THEN + ! Minimum time value with masking + ELSE IF ( reduced_k_range ) THEN ksr= l_start(3) ker= l_end(3) WHERE ( mask(f1:f2,f3:f4,ksr:ker) .AND.& @@ -2149,7 +2665,7 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & CALL update_bounds(out_num, is-hi, ie-hi, js-hj, je-hj, ks, ke) CALL check_out_of_bounds(out_num, diag_field_id, err_msg=err_msg_local) IF ( err_msg_local /= '' ) THEN - IF ( fms_error_handler('diag_manager_mod::send_data_3d', err_msg_local, err_msg) ) THEN + IF ( fms_error_handler('diag_manager_mod::send_data_3d', err_msg_local, err_msg) ) THEN DEALLOCATE(oor_mask) RETURN END IF @@ -2157,7 +2673,7 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & END IF WHERE ( mask(f1:f2,f3:f4,ks:ke) .AND.& & field(f1:f2,f3:f4,ks:ke) < output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,ks:ke,sample) ) & - & output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,ks:ke,sample) = field(f1:f2,f3:f4,ks:ke) + & output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,ks:ke,sample) = field(f1:f2,f3:f4,ks:ke) END IF ELSE IF ( need_compute ) THEN @@ -2166,7 +2682,7 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & DO j = js, je DO i = is, ie IF ( l_start(1)+hi <=i.AND.i<=l_end(1)+hi.AND.l_start(2)+hj<=j.AND.j<=l_end(2)+hj) THEN - i1 = i-l_start(1)-hi+1 + i1 = i-l_start(1)-hi+1 j1= j-l_start(2)-hj+1 IF ( field(i-is+1+hi,j-js+1+hj,k) < output_fields(out_num)%buffer(i1,j1,k1,sample) ) THEN output_fields(out_num)%buffer(i1,j1,k1,sample) = field(i-is+1+hi,j-js+1+hj,k) @@ -2175,8 +2691,8 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & END DO END DO END DO - ! Minimum time value - ELSE IF ( reduced_k_range ) THEN + ! Minimum time value + ELSE IF ( reduced_k_range ) THEN ksr= l_start(3) ker= l_end(3) WHERE ( field(f1:f2,f3:f4,ksr:ker) < output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,:,sample) ) & @@ -2186,7 +2702,7 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & CALL update_bounds(out_num, is-hi, ie-hi, js-hj, je-hj, ks, ke) CALL check_out_of_bounds(out_num, diag_field_id, err_msg=err_msg_local) IF ( err_msg_local /= '' ) THEN - IF ( fms_error_handler('diag_manager_mod::send_data_3d', err_msg_local, err_msg) ) THEN + IF ( fms_error_handler('diag_manager_mod::send_data_3d', err_msg_local, err_msg) ) THEN DEALLOCATE(oor_mask) RETURN END IF @@ -2200,17 +2716,17 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & ELSE ! ( not average, not min, max) output_fields(out_num)%count_0d(sample) = 1 IF ( need_compute ) THEN - DO j = js, je - DO i = is, ie + DO j = js, je + DO i = is, ie IF ( l_start(1)+hi <= i .AND. i <= l_end(1)+hi .AND. l_start(2)+hj <= j .AND. j <= l_end(2)+hj ) THEN - i1 = i-l_start(1)-hi+1 + i1 = i-l_start(1)-hi+1 j1 = j-l_start(2)-hj+1 output_fields(out_num)%buffer(i1,j1,:,sample) = field(i-is+1+hi,j-js+1+hj,l_start(3):l_end(3)) END IF END DO END DO ! instantaneous output - ELSE IF ( reduced_k_range ) THEN + ELSE IF ( reduced_k_range ) THEN ksr = l_start(3) ker = l_end(3) output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,:,sample) = field(f1:f2,f3:f4,ksr:ker) @@ -2219,7 +2735,7 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & CALL update_bounds(out_num, is-hi, ie-hi, js-hj, je-hj, ks, ke) CALL check_out_of_bounds(out_num, diag_field_id, err_msg=err_msg_local) IF ( err_msg_local /= '' ) THEN - IF ( fms_error_handler('diag_manager_mod::send_data_3d', err_msg_local, err_msg) ) THEN + IF ( fms_error_handler('diag_manager_mod::send_data_3d', err_msg_local, err_msg) ) THEN DEALLOCATE(oor_mask) RETURN END IF @@ -2227,26 +2743,26 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & END IF output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,ks:ke,sample) = field(f1:f2,f3:f4,ks:ke) END IF - + IF ( PRESENT(mask) .AND. missvalue_present ) THEN IF ( need_compute ) THEN DO k = l_start(3), l_end(3) k1 = k - l_start(3) + 1 DO j = js, je - DO i = is, ie + DO i = is, ie IF ( l_start(1)+hi <= i .AND. i <= l_end(1)+hi .AND. l_start(2)+hj <= j .AND. j <= l_end(2)+hj ) THEN - i1 = i-l_start(1)-hi+1 - j1 = j-l_start(2)-hj+1 + i1 = i-l_start(1)-hi+1 + j1 = j-l_start(2)-hj+1 IF ( .NOT.mask(i-is+1+hi,j-js+1+hj,k) )& - & output_fields(out_num)%buffer(i1,j1,k1,sample) = missvalue + & output_fields(out_num)%buffer(i1,j1,k1,sample) = missvalue END IF END DO END DO END DO - ELSE IF ( reduced_k_range ) THEN + ELSE IF ( reduced_k_range ) THEN ksr= l_start(3) ker= l_end(3) - DO k=ksr, ker + DO k=ksr, ker k1= k - ksr + 1 DO j=js, je DO i=is, ie @@ -2271,33 +2787,33 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & IF ( output_fields(out_num)%static .AND. .NOT.need_compute .AND. debug_diag_manager ) THEN CALL check_bounds_are_exact_static(out_num, diag_field_id, err_msg=err_msg_local) IF ( err_msg_local /= '' ) THEN - IF ( fms_error_handler('diag_manager_mod::send_data_3d', err_msg_local, err_msg)) THEN + IF ( fms_error_handler('diag_manager_mod::send_data_3d', err_msg_local, err_msg)) THEN DEALLOCATE(oor_mask) RETURN END IF END IF END IF - - ! If rmask and missing value present, then insert missing value + + ! If rmask and missing value present, then insert missing value IF ( PRESENT(rmask) .AND. missvalue_present ) THEN IF ( need_compute ) THEN DO k = l_start(3), l_end(3) k1 = k - l_start(3) + 1 - DO j = js, je - DO i = is, ie + DO j = js, je + DO i = is, ie IF ( l_start(1)+hi <= i .AND. i <= l_end(1)+hi .AND. l_start(2)+hj <= j .AND. j <= l_end(2)+hj ) THEN - i1 = i-l_start(1)-hi+1 + i1 = i-l_start(1)-hi+1 j1 = j-l_start(2)-hj+1 IF ( rmask(i-is+1+hi,j-js+1+hj,k) < 0.5 ) & - & output_fields(out_num)%buffer(i1,j1,k1,sample) = missvalue + & output_fields(out_num)%buffer(i1,j1,k1,sample) = missvalue END IF END DO END DO END DO - ELSE IF ( reduced_k_range ) THEN + ELSE IF ( reduced_k_range ) THEN ksr= l_start(3) ker= l_end(3) - DO k= ksr, ker + DO k= ksr, ker k1 = k - ksr + 1 DO j=js, je DO i=is, ie @@ -2319,7 +2835,7 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & END IF END DO num_out_fields - + DEALLOCATE(oor_mask) END FUNCTION send_data_3d ! @@ -2331,7 +2847,7 @@ END FUNCTION send_data_3d ! ! LOGICAL FUNCTION send_tile_averaged_data2d ( id, field, area, time, mask ) - INTEGER, INTENT(in) :: id ! id od the diagnostic field + INTEGER, INTENT(in) :: id ! id od the diagnostic field REAL, INTENT(in) :: field(:,:,:) ! field to average and send REAL, INTENT(in) :: area (:,:,:) ! area of tiles (== averaging weights), arbitrary units TYPE(time_type), INTENT(in) :: time ! current time @@ -2389,10 +2905,10 @@ END FUNCTION send_tile_averaged_data3d ! (lon, lat) result of averaging SUBROUTINE average_tiles(diag_field_id, x, area, mask, out) INTEGER, INTENT(in) :: diag_field_id - REAL, DIMENSION(:,:,:), INTENT(in) :: x + REAL, DIMENSION(:,:,:), INTENT(in) :: x REAL, DIMENSION(:,:,:), INTENT(in) :: area LOGICAL, DIMENSION(:,:,:), INTENT(in) :: mask - REAL, DIMENSION(:,:), INTENT(out) :: out + REAL, DIMENSION(:,:), INTENT(out) :: out INTEGER :: it ! iterator over tile number REAL, DIMENSION(SIZE(x,1),SIZE(x,2)) :: s ! area accumulator @@ -2401,22 +2917,22 @@ SUBROUTINE average_tiles(diag_field_id, x, area, mask, out) ! Initialize local_missing_value IF ( input_fields(diag_field_id)%missing_value_present ) THEN local_missing_value = input_fields(diag_field_id)%missing_value - ELSE + ELSE local_missing_value = 0.0 END IF - + ! Initialize s and out to zero. s(:,:) = 0.0 out(:,:) = 0.0 - + DO it = 1, SIZE(area,3) - WHERE ( mask(:,:,it) ) + WHERE ( mask(:,:,it) ) out(:,:) = out(:,:) + x(:,:,it)*area(:,:,it) s(:,:) = s(:,:) + area(:,:,it) END WHERE END DO - WHERE ( s(:,:) > 0 ) + WHERE ( s(:,:) > 0 ) out(:,:) = out(:,:)/s(:,:) ELSEWHERE out(:,:) = local_missing_value @@ -2432,14 +2948,14 @@ INTEGER FUNCTION writing_field(out_num, at_diag_end, error_string, time) TYPE(time_type) :: middle_time LOGICAL :: time_max, time_min, reduced_k_range, missvalue_present - LOGICAL :: average, need_compute, phys_window + LOGICAL :: average, time_rms, need_compute, phys_window INTEGER :: in_num, file_num, freq, units INTEGER :: b1,b2,b3,b4 ! size of buffer along x,y,z,and diurnal axes INTEGER :: i, j, k, m REAL :: missvalue, num writing_field = 0 - + need_compute = output_fields(out_num)%need_compute in_num = output_fields(out_num)%input_field @@ -2451,9 +2967,12 @@ INTEGER FUNCTION writing_field(out_num, at_diag_end, error_string, time) phys_window = output_fields(out_num)%phys_window ! Is this output field being time averaged? average = output_fields(out_num)%time_average + ! Are we taking the rms of the field? + ! If so, then average is also .TRUE. + time_rms = output_fields(out_num)%time_rms ! Looking for max and min value of this field over the sampling interval? time_max = output_fields(out_num)%time_max - time_min = output_fields(out_num)%time_min + time_min = output_fields(out_num)%time_min file_num = output_fields(out_num)%output_file freq = files(file_num)%output_freq units = files(file_num)%output_units @@ -2461,17 +2980,19 @@ INTEGER FUNCTION writing_field(out_num, at_diag_end, error_string, time) ! If average get size: Average intervals are last_output, next_output IF ( average ) THEN b1=SIZE(output_fields(out_num)%buffer,1) - b2=SIZE(output_fields(out_num)%buffer,2) + b2=SIZE(output_fields(out_num)%buffer,2) b3=SIZE(output_fields(out_num)%buffer,3) b4=SIZE(output_fields(out_num)%buffer,4) - IF ( input_fields(in_num)%mask_variant ) THEN - DO m=1, b4 + IF ( input_fields(in_num)%mask_variant ) THEN + DO m=1, b4 DO k=1, b3 DO j=1, b2 - DO i=1, b1 + DO i=1, b1 IF ( output_fields(out_num)%counter(i,j,k,m) > 0. )THEN output_fields(out_num)%buffer(i,j,k,m) = & & output_fields(out_num)%buffer(i,j,k,m)/output_fields(out_num)%counter(i,j,k,m) + IF ( time_rms ) output_fields(out_num)%buffer(i,j,k,m) = & + SQRT(output_fields(out_num)%buffer(i,j,k,m)) ELSE output_fields(out_num)%buffer(i,j,k,m) = missvalue END IF @@ -2495,13 +3016,18 @@ INTEGER FUNCTION writing_field(out_num, at_diag_end, error_string, time) DO k=1, b3 DO j=1, b2 DO i=1, b1 - IF ( output_fields(out_num)%buffer(i,j,k,m) /= missvalue ) & - & output_fields(out_num)%buffer(i,j,k,m) = output_fields(out_num)%buffer(i,j,k,m)/num + IF ( output_fields(out_num)%buffer(i,j,k,m) /= missvalue ) THEN + output_fields(out_num)%buffer(i,j,k,m) = output_fields(out_num)%buffer(i,j,k,m)/num + IF ( time_rms ) output_fields(out_num)%buffer(i,j,k,m) =& + & SQRT(output_fields(out_num)%buffer(i,j,k,m)) + END IF END DO END DO END DO ELSE output_fields(out_num)%buffer(:,:,:,m) = output_fields(out_num)%buffer(:,:,:,m)/num + IF ( time_rms ) output_fields(out_num)%buffer(:,:,:,m) =& + & SQRT(output_fields(out_num)%buffer(:,:,:,m)) END IF ELSE IF ( .NOT. at_diag_end ) THEN IF ( missvalue_present ) THEN @@ -2518,7 +3044,7 @@ INTEGER FUNCTION writing_field(out_num, at_diag_end, error_string, time) END IF ! mask_variant ELSE IF ( time_min .OR. time_max ) THEN IF ( missvalue_present ) THEN - WHERE ( ABS(output_fields(out_num)%buffer) == MIN_VALUE ) + WHERE ( ABS(output_fields(out_num)%buffer) == MIN_VALUE ) output_fields(out_num)%buffer = missvalue END WHERE END IF ! if missvalue is NOT present buffer retains max_value or min_value @@ -2550,7 +3076,7 @@ INTEGER FUNCTION writing_field(out_num, at_diag_end, error_string, time) END IF output_fields(out_num)%count_0d(:) = 0.0 output_fields(out_num)%num_elements(:) = 0 - IF ( time_max ) THEN + IF ( time_max ) THEN output_fields(out_num)%buffer = MAX_VALUE ELSE IF ( time_min ) THEN output_fields(out_num)%buffer = MIN_VALUE @@ -2573,7 +3099,7 @@ END SUBROUTINE diag_manager_set_time_end SUBROUTINE diag_send_complete(time_step, err_msg) TYPE (time_type), INTENT(in) :: time_step character(len=*), INTENT(out), optional :: err_msg - + type(time_type) :: next_time, time integer :: file, j, out_num, in_num, freq, status logical :: local_output, need_compute @@ -2593,7 +3119,7 @@ SUBROUTINE diag_send_complete(time_step, err_msg) out_num = files(file)%fields(j) !this is position of output_field in array output_fields in_num = output_fields(out_num)%input_field - IF ( input_fields(in_num)%numthreads == 1 ) CYCLE + IF ( (input_fields(in_num)%numthreads == 1) .AND. (input_fields(in_num)%active_omp_level.LE.1) ) CYCLE IF ( output_fields(out_num)%static .OR. freq == END_OF_RUN ) CYCLE time = input_fields(in_num)%time IF ( time >= time_end ) CYCLE @@ -2609,7 +3135,7 @@ SUBROUTINE diag_send_complete(time_step, err_msg) IF ( next_time > output_fields(out_num)%next_output ) THEN ! A non-static field that has skipped a time level is an error IF ( next_time > output_fields(out_num)%next_next_output .AND. freq > 0 ) THEN - IF ( mpp_pe() .EQ. mpp_root_pe() ) THEN + IF ( mpp_pe() .EQ. mpp_root_pe() ) THEN WRITE (error_string,'(a,"/",a)')& & TRIM(input_fields(in_num)%module_name), & & TRIM(output_fields(out_num)%output_name) @@ -2654,7 +3180,7 @@ SUBROUTINE diag_manager_end(time) CALL mpp_close (diag_log_unit) END IF DO file = 1, num_files - CALL closing_file(file, time) + CALL closing_file(file, time) END DO END SUBROUTINE diag_manager_end ! @@ -2694,16 +3220,16 @@ SUBROUTINE closing_file(file, time) ! skip all PEs not participating in outputting this field IF ( local_output .AND. (.NOT. need_compute) ) CYCLE - ! skip fields that were not registered or non-static + ! skip fields that were not registered or non-static input_num = output_fields(i)%input_field IF ( input_fields(input_num)%static ) CYCLE - IF ( .NOT.input_fields(input_num)%register ) CYCLE + IF ( .NOT.input_fields(input_num)%register ) CYCLE freq = files(file)%output_freq IF ( freq /= END_OF_RUN .AND. files(file)%file_unit < 0 & & .AND. ALL(output_fields(i)%num_elements(:) == 0)& & .AND. ALL(output_fields(i)%count_0d(:) == 0) ) CYCLE ! Is it time to output for this field; CAREFUL ABOUT >= vs > HERE - ! For end should be >= because no more data is coming + ! For end should be >= because no more data is coming IF ( time >= output_fields(i)%next_output .OR. freq == END_OF_RUN ) THEN IF ( time >= output_fields(i)%next_next_output .AND. freq > 0 ) THEN WRITE (message,'(a,"/",a)') TRIM(input_fields(input_num)%module_name), & @@ -2712,7 +3238,7 @@ SUBROUTINE closing_file(file, time) ! / skip one time ! level, maybe send_data never called ! - IF ( mpp_pe() .EQ. mpp_root_pe() ) & + IF ( mpp_pe() .EQ. mpp_root_pe() ) & & CALL error_mesg('diag_manager_mod::closing_file', 'module/output_field ' //& & TRIM(message)//', skip one time level, maybe send_data never called', WARNING) ELSE @@ -2727,7 +3253,7 @@ SUBROUTINE closing_file(file, time) & TRIM(output_fields(i)%output_name)//' NOT available,'//& & ' check if output interval > runlength. Netcdf fill_values are written', NOTE) output_fields(i)%buffer = FILL_VALUE - CALL diag_data_out(file, i, output_fields(i)%buffer, time, .TRUE.) + CALL diag_data_out(file, i, output_fields(i)%buffer, time, .TRUE.) END IF END DO ! Now it's time to output static fields @@ -2754,9 +3280,11 @@ END SUBROUTINE closing_file ! Open and read diag_table. Select fields and files for diagnostic output. ! ! + ! Model time diag_manager initialized ! - SUBROUTINE diag_manager_init(diag_model_subset, err_msg) + SUBROUTINE diag_manager_init(diag_model_subset, time_init, err_msg) INTEGER, OPTIONAL, INTENT(IN) :: diag_model_subset + INTEGER, DIMENSION(6), OPTIONAL, INTENT(IN) :: time_init CHARACTER(len=*), INTENT(out), OPTIONAL :: err_msg CHARACTER(len=*), PARAMETER :: SEP = '|' @@ -2776,7 +3304,8 @@ SUBROUTINE diag_manager_init(diag_model_subset, err_msg) NAMELIST /diag_manager_nml/ append_pelist_name, mix_snapshot_average_fields, max_output_fields, & & max_input_fields, max_axes, do_diag_field_log, write_bytes_in_file, debug_diag_manager,& & max_num_axis_sets, max_files, use_cmor, issue_oor_warnings,& - & oor_warnings_fatal, max_out_per_in_field, conserve_water + & oor_warnings_fatal, max_out_per_in_field, conserve_water, region_out_use_alt_value, max_field_attributes,& + & max_file_attributes, prepend_date ! If the module was already initialized do nothing IF ( module_is_initialized ) RETURN @@ -2799,7 +3328,7 @@ SUBROUTINE diag_manager_init(diag_model_subset, err_msg) stdout_unit = stdout() ! version number to logfile - call write_version_number() + CALL write_version_number(version, tagname) Time_zero = set_time(0,0) !--- initialize time_end to time_zero @@ -2826,7 +3355,7 @@ SUBROUTINE diag_manager_init(diag_model_subset, err_msg) END IF #endif ! Check the status of reading the diag_manager_nml - + IF ( check_nml_error(IOSTAT=mystat, NML_NAME='DIAG_MANAGER_NML') < 0 ) THEN IF ( mpp_pe() == mpp_root_pe() ) THEN CALL error_mesg('diag_manager_mod::diag_manager_init', 'DIAG_MANAGER_NML not found in input.nml. Using defaults.',& @@ -2834,12 +3363,12 @@ SUBROUTINE diag_manager_init(diag_model_subset, err_msg) END IF END IF - IF ( mpp_pe() == mpp_root_pe() ) THEN + IF ( mpp_pe() == mpp_root_pe() ) THEN WRITE (stdlog_unit, diag_manager_nml) END IF ! Issue note about using the CMOR missing value. - IF ( use_cmor ) THEN + IF ( use_cmor ) THEN err_msg_local = '' WRITE (err_msg_local,'(ES8.1E2)') CMOR_MISSING_VALUE CALL error_mesg('diag_manager_mod::diag_manager_init', 'Using CMOR missing value ('//TRIM(err_msg_local)//').', NOTE) @@ -2856,7 +3385,7 @@ SUBROUTINE diag_manager_init(diag_model_subset, err_msg) END IF IF ( mix_snapshot_average_fields ) THEN - IF ( mpp_pe() == mpp_root_pe() ) THEN + IF ( mpp_pe() == mpp_root_pe() ) THEN CALL error_mesg('diag_manager_mod::diag_manager_init', 'Setting diag_manager_nml variable '//& & 'mix_snapshot_average_fields = .TRUE. will cause ERRORS in the time coordinates '//& & 'of all time averaged fields. Strongly recommend setting mix_snapshot_average_fields '//& @@ -2865,24 +3394,37 @@ SUBROUTINE diag_manager_init(diag_model_subset, err_msg) END IF ALLOCATE(output_fields(max_output_fields)) ALLOCATE(input_fields(max_input_fields)) - do j = 1, max_input_fields - allocate(input_fields(j)%output_fields(MAX_OUT_PER_IN_FIELD)) - enddo + DO j = 1, max_input_fields + ALLOCATE(input_fields(j)%output_fields(MAX_OUT_PER_IN_FIELD)) + END DO ALLOCATE(files(max_files)) ALLOCATE(pelist(mpp_npes())) CALL mpp_get_current_pelist(pelist, pelist_name) + ! set the diag_init_time if time_init present. Otherwise, set it to base_time + IF ( PRESENT(time_init) ) THEN + diag_init_time = set_date(time_init(1), time_init(2), time_init(3), time_init(4),& + & time_init(5), time_init(6)) + ELSE + diag_init_time = base_time + IF ( prepend_date .EQV. .TRUE. ) THEN + CALL error_mesg('diag_manager_mod::diag_manager_init',& + & 'prepend_date only supported when diag_manager_init is called with time_init present.', NOTE) + prepend_date = .FALSE. + END IF + END IF + CALL parse_diag_table(DIAG_SUBSET=diag_subset_output, ISTAT=mystat, ERR_MSG=err_msg_local) IF ( mystat /= 0 ) THEN IF ( fms_error_handler('diag_manager_mod::diag_manager_init',& & 'Error parsing diag_table. '//TRIM(err_msg_local), err_msg) ) RETURN END IF - + !initialize files%bytes_written to zero files(:)%bytes_written = 0 ! open diag field log file - IF ( do_diag_field_log ) THEN + IF ( do_diag_field_log.AND.mpp_pe().EQ.mpp_root_pe() ) THEN CALL mpp_open(diag_log_unit, 'diag_field_log.out', nohdrs=.TRUE.) WRITE (diag_log_unit,'(777a)') & & 'Module', SEP, 'Field', SEP, 'Long Name', SEP,& @@ -2901,7 +3443,7 @@ END SUBROUTINE diag_manager_init ! ! - ! Return base time for diagnostics. + ! Return base time for diagnostics. ! ! ! ! Writes axis data to file. This subroutine is to be called once per file - ! after all write_meta_data calls, and before the first + ! after all write_meta_data calls, and before the first ! diag_field_out call. ! ! Output file unit number SUBROUTINE done_meta_data(file_unit) - INTEGER, INTENT(in) :: file_unit + INTEGER, INTENT(in) :: file_unit INTEGER :: i @@ -657,14 +748,14 @@ END SUBROUTINE get_diag_global_att ! ! SUBROUTINE set_diag_global_att(component, gridType, tileName) - CHARACTER(len=*),INTENT(in) :: component, gridType, tileName + CHARACTER(len=*),INTENT(in) :: component, gridType, tileName ! The following two lines are set to remove compile time warnings ! about 'only used once'. CHARACTER(len=64) :: component_tmp component_tmp = component ! Don't know how to set these for specific component - ! Want to be able to say + ! Want to be able to say ! if(output_file has component) then diag_global_att%grid_type = gridType diag_global_att%tile_name = tileName @@ -673,4 +764,3 @@ END SUBROUTINE set_diag_global_att !     END MODULE diag_output_mod - diff --git a/src/shared/diag_manager/diag_table.F90 b/src/shared/diag_manager/diag_table.F90 index 7e8c0414a5..e3f9bc5b46 100644 --- a/src/shared/diag_manager/diag_table.F90 +++ b/src/shared/diag_manager/diag_table.F90 @@ -1,7 +1,8 @@ MODULE diag_table_mod - ! + ! ! Seth Underwood ! + ! ! ! diag_table_mod is a set of subroutines use to parse out the data from a diag_table. This module ! will also setup the arrays required to store the information by counting the number of input fields, output files, and @@ -124,9 +125,8 @@ MODULE diag_table_mod ! ! !
  • - ! Field Section: Field lines contain 8 fields. Field lines can be intermixed with file lines, but the file must - ! be defined before any fields that are to be written to the file. Fields line can contain fields that are not written - ! to any files. The file name for these fields is null. + ! Field Section: Field lines contain 8 fields. Field lines can be intermixed with file lines. Fields line can contain + ! fields that are not written to any files. The file name for these fields is null. ! ! Field lines have the following format:
    ! 
    @@ -156,6 +156,8 @@ MODULE diag_table_mod
   !             
    Average from the last time written to the current time.
    
   !             
    .FALSE., none
    
   !             
    No reduction performed.  Write current time step value only.
    
+  !             
    rms
     
    Calculate the root mean square from the last time written to the current time.
    
+  !             
    pow##
     
    Calculate the mean of the power ## from the last time written to the current time.
    
   !             
    min
     
    Minimum value from last write to current time.
    
   !             
    max
     
    Maximum value from last write to current time.
    
   !             
    diurnal##
     
    ## diurnal averages
    
@@ -214,27 +216,27 @@ MODULE diag_table_mod
   USE fms_mod, ONLY: fms_error_handler, error_mesg, file_exist, stdlog, mpp_pe, mpp_root_pe, FATAL, WARNING, lowercase, close_file
   USE time_manager_mod, ONLY: get_calendar_type, NO_CALENDAR, set_date, set_time, month_name, time_type
   USE constants_mod, ONLY: SECONDS_PER_HOUR, SECONDS_PER_MINUTE
-  
+
   USE diag_data_mod, ONLY: global_descriptor, base_time, base_year, base_month, base_day, base_hour, base_minute, base_second,&
-       & DIAG_OTHER, DIAG_OCEAN, DIAG_ALL, coord_type, append_pelist_name, pelist_name, filename_appendix
+       & DIAG_OTHER, DIAG_OCEAN, DIAG_ALL, coord_type, append_pelist_name, pelist_name
   USE diag_util_mod, ONLY: init_file, check_duplicate_output_fields, init_input_field, init_output_field
 
   IMPLICIT NONE
 
   PRIVATE
   PUBLIC :: parse_diag_table
-  
+
   TYPE field_description_type
      CHARACTER(len=128) :: module_name, field_name, output_name, file_name
      CHARACTER(len=50) :: time_sampling
-     CHARACTER(len=50) :: time_method   
+     CHARACTER(len=50) :: time_method
      CHARACTER(len=50) :: spatial_ops
      TYPE(coord_type) :: regional_coords
      INTEGER :: pack
   END TYPE field_description_type
 
   TYPE file_description_type
-     INTEGER :: output_freq 
+     INTEGER :: output_freq
      INTEGER :: file_format
      INTEGER :: new_file_freq
      INTEGER :: file_duration
@@ -256,7 +258,7 @@ MODULE diag_table_mod
   CHARACTER(len=*), PARAMETER :: UNALLOWED_ALL = UNALLOWED_QTE//","
 
 CONTAINS
-  
+
   ! 
   !   
   !     Parse the diag_table in preparation for diagnostic output.
@@ -344,6 +346,7 @@ SUBROUTINE parse_diag_table(diag_subset, istat, err_msg)
     INTEGER :: commentStart !< Index location of first '#' on line
     INTEGER :: diag_subset_output !< local value of diag_subset
     INTEGER :: nfields, nfiles !< Number of fields and files.  Not used yet.
+    INTEGER :: npass !< number of passes done while parsing the diag_table (1 for files, 2 for fields)
     INTEGER, TARGET :: mystat !< variable to hold return status of function/subroutine calls.
     INTEGER, POINTER :: pstat !< pointer that points to istat if preset, otherwise, points to mystat.
 
@@ -367,17 +370,17 @@ SUBROUTINE parse_diag_table(diag_subset, istat, err_msg)
 
     IF ( PRESENT(diag_subset) ) THEN
        diag_subset_output = diag_subset
-    ELSE 
+    ELSE
        diag_subset_output = DIAG_ALL
     END IF
-    
+
     ! get the stdlog unit number
     stdlog_unit = stdlog()
     num_lines = get_ascii_file_num_lines('diag_table', DT_LINE_LENGTH)
     allocate(diag_table(num_lines))
 
     call read_ascii_file('diag_table', DT_LINE_LENGTH, diag_table)
-    
+
     ! Read in the global file labeling string
     READ (UNIT=diag_table(1), FMT=*, IOSTAT=mystat) global_descriptor
     IF ( mystat /= 0 ) THEN
@@ -385,14 +388,14 @@ SUBROUTINE parse_diag_table(diag_subset, istat, err_msg)
        IF ( fms_error_handler('diag_table_mod::parse_diag_table', 'Error reading the global descriptor from the diagnostic table.',&
             & err_msg) ) RETURN
     END IF
-    
+
     ! Read in the base date
     READ (UNIT=diag_table(2), FMT=*, IOSTAT=mystat) base_year, base_month, base_day, base_hour, base_minute, base_second
     IF ( mystat /= 0 ) THEN
        pstat = mystat
        IF ( fms_error_handler('diag_manager_init', 'Error reading the base date from the diagnostic table.', err_msg) ) RETURN
     END IF
-    
+
     ! Set up the time type for base time
     IF ( get_calendar_type() /= NO_CALENDAR ) THEN
        IF ( base_year==0 .OR. base_month==0 .OR. base_day==0 ) THEN
@@ -416,88 +419,94 @@ SUBROUTINE parse_diag_table(diag_subset, istat, err_msg)
 
     nfiles=0
     nfields=0
-    parser: DO line_num=3, num_lines
-       ! Read in the entire line from the file.
-       ! If there is a read error, give a warning, and
-       ! cycle the parser loop.
-       READ (diag_table(line_num), FMT='(A)', IOSTAT=mystat) record_line
-       ! Increase line counter, and put in string for use in warning/error messages.
-       WRITE (line_number, '(I5)') line_num
-
-       IF ( mystat > 0 ) THEN
-          IF ( mpp_pe() == mpp_root_pe() ) &
-               & CALL error_mesg("diag_table_mod::parse_diag_table",&
-               & "Problem reading the diag_table (line:" //line_number//").", FATAL)
-          CYCLE parser
-       ELSE IF ( mystat < 0 ) THEN
-          EXIT parser
-       END IF
-       
-       ! How long is the read in string?
-       record_len = LEN_TRIM(record_line)
-
-       ! ignore blank lines and  lines with comments only (comment marker '#')
-       commentStart = INDEX(record_line,'#')
-       IF ( commentStart .NE. 0 ) record_line = record_line(1:commentStart-1)
-       IF ( LEN_TRIM(record_line) == 0 .OR. record_len == 0 ) CYCLE parser
-
-       init: IF ( is_a_file(TRIM(record_line)) ) THEN
-          temp_file = parse_file_line(LINE=record_line, ISTAT=mystat, ERR_MSG=local_err_msg)
-          
-          IF ( mystat > 0 ) THEN 
-             CALL error_mesg("diag_table_mod::parse_diag_table",&
-                  & TRIM(local_err_msg)//" (line:" //TRIM(line_number)//").", FATAL)
-          ELSE IF ( mystat < 0 ) THEN
-             IF ( mpp_pe() == mpp_root_pe() )&
+    pass: DO npass = 1, 2
+       parser: DO line_num=3, num_lines
+          ! Read in the entire line from the file.
+          ! If there is a read error, give a warning, and
+          ! cycle the parser loop.
+          READ (diag_table(line_num), FMT='(A)', IOSTAT=mystat) record_line
+          ! Increase line counter, and put in string for use in warning/error messages.
+          WRITE (line_number, '(I5)') line_num
+
+          IF ( mystat > 0 ) THEN
+             IF ( mpp_pe() == mpp_root_pe() ) &
                   & CALL error_mesg("diag_table_mod::parse_diag_table",&
-                  & TRIM(local_err_msg)//" (line: "//TRIM(line_number)//").", WARNING)
-             CYCLE parser
-          ELSE IF ( (diag_subset_output == DIAG_OTHER .AND. VERIFY('ocean', lowercase(temp_file%file_name)) == 0).OR.&
-               &    (diag_subset_output == DIAG_OCEAN .AND. VERIFY('ocean', lowercase(temp_file%file_name)) /= 0) ) THEN
+                  & "Problem reading the diag_table (line:" //line_number//").", FATAL)
              CYCLE parser
-          ELSE IF ( temp_file%new_file_freq > 0 ) THEN ! Call the init_file subroutine.  The '1' is for the tile_count
-             CALL init_file(temp_file%file_name, temp_file%output_freq, temp_file%iOutput_freq_units, temp_file%file_format,&
-                  & temp_file%iTime_units, temp_file%long_name, 1, temp_file%new_file_freq, temp_file%iNew_file_freq_units,&
-                  & temp_file%start_time, temp_file%file_duration, temp_file%iFile_duration_units)
-          ELSE
-             CALL init_file(temp_file%file_name, temp_file%output_freq, temp_file%iOutput_freq_units, temp_file%file_format,&
-                  & temp_file%iTime_units, temp_file%long_name, 1)
-          END IF
-          
-          ! Increment number of files
-          nfiles = nfiles + 1
-       ELSE ! We have a field.
-          temp_field = parse_field_line(LINE=record_line, ISTAT=mystat, ERR_MSG=local_err_msg)
-
-          ! Check for errors, then initialize the input and output field
-          IF (  mystat > 0 ) THEN
-             CALL error_mesg("diag_table_mod::parse_diag_table",&
-                  & TRIM(local_err_msg)//" (line: "//TRIM(line_number)//").",FATAL)
           ELSE IF ( mystat < 0 ) THEN
-             IF ( mpp_pe() == mpp_root_pe() )&
-                  & CALL error_mesg("diag_table_mod::Parse_diag_table",&
-                  & TRIM(local_err_msg)//" (line: "//TRIM(line_number)//").",WARNING)
-             CYCLE parser
-          ELSE IF ( (diag_subset_output == DIAG_OTHER .AND. VERIFY('ocean', lowercase(temp_field%file_name)) == 0).OR.&
-               &    (diag_subset_output == DIAG_OCEAN .AND. VERIFY('ocean', lowercase(temp_field%file_name)) /= 0) ) THEN 
-             CYCLE parser
-          ELSE IF ( lowercase(TRIM(temp_field%spatial_ops)) == 'none' ) THEN
-             CALL init_input_field(temp_field%module_name, temp_field%field_name, 1)
-             CALL init_output_field(temp_field%module_name, temp_field%field_name, temp_field%output_name, temp_field%file_name,&
-                  & temp_field%time_method, temp_field%pack, 1)
-          ELSE 
-             CALL init_input_field(temp_field%module_name, temp_field%field_name, 1)
-             CALL init_output_field(temp_field%module_name, temp_field%field_name, temp_field%output_name, temp_field%file_name,&
-                  & temp_field%time_method, temp_field%pack, 1, temp_field%regional_coords)
+             EXIT parser
           END IF
 
-          ! Increment number of fields
-          nfields = nfields + 1
-       END IF init
-    END DO parser
+          ! How long is the read in string?
+          record_len = LEN_TRIM(record_line)
+
+          ! ignore blank lines and  lines with comments only (comment marker '#')
+          commentStart = INDEX(record_line,'#')
+          IF ( commentStart .NE. 0 ) record_line = record_line(1:commentStart-1)
+          IF ( LEN_TRIM(record_line) == 0 .OR. record_len == 0 ) CYCLE parser
+
+          init: IF ( npass == 1 ) THEN ! Checking for files only
+             IF ( is_a_file(TRIM(record_line)) ) THEN
+                temp_file = parse_file_line(LINE=record_line, ISTAT=mystat, ERR_MSG=local_err_msg)
+
+                IF ( mystat > 0 ) THEN
+                   CALL error_mesg("diag_table_mod::parse_diag_table",&
+                        & TRIM(local_err_msg)//" (line:" //TRIM(line_number)//").", FATAL)
+                ELSE IF ( mystat < 0 ) THEN
+                   IF ( mpp_pe() == mpp_root_pe() )&
+                        & CALL error_mesg("diag_table_mod::parse_diag_table",&
+                        & TRIM(local_err_msg)//" (line: "//TRIM(line_number)//").", WARNING)
+                   CYCLE parser
+                ELSE IF ( (diag_subset_output == DIAG_OTHER .AND. VERIFY('ocean', lowercase(temp_file%file_name)) == 0).OR.&
+                     &    (diag_subset_output == DIAG_OCEAN .AND. VERIFY('ocean', lowercase(temp_file%file_name)) /= 0) ) THEN
+                   CYCLE parser
+                ELSE IF ( temp_file%new_file_freq > 0 ) THEN ! Call the init_file subroutine.  The '1' is for the tile_count
+                   CALL init_file(temp_file%file_name, temp_file%output_freq, temp_file%iOutput_freq_units, temp_file%file_format,&
+                        & temp_file%iTime_units, temp_file%long_name, 1, temp_file%new_file_freq, temp_file%iNew_file_freq_units,&
+                        & temp_file%start_time, temp_file%file_duration, temp_file%iFile_duration_units)
+                ELSE
+                   CALL init_file(temp_file%file_name, temp_file%output_freq, temp_file%iOutput_freq_units, temp_file%file_format,&
+                        & temp_file%iTime_units, temp_file%long_name, 1)
+                END IF
+
+                ! Increment number of files
+                nfiles = nfiles + 1
+             END IF
+          ELSE ! Looking for fields
+             IF ( .NOT.is_a_file(TRIM(record_line)) ) THEN
+                temp_field = parse_field_line(LINE=record_line, ISTAT=mystat, ERR_MSG=local_err_msg)
+
+                ! Check for errors, then initialize the input and output field
+                IF (  mystat > 0 ) THEN
+                   CALL error_mesg("diag_table_mod::parse_diag_table",&
+                        & TRIM(local_err_msg)//" (line: "//TRIM(line_number)//").",FATAL)
+                ELSE IF ( mystat < 0 ) THEN
+                   IF ( mpp_pe() == mpp_root_pe() )&
+                        & CALL error_mesg("diag_table_mod::Parse_diag_table",&
+                        & TRIM(local_err_msg)//" (line: "//TRIM(line_number)//").",WARNING)
+                   CYCLE parser
+                ELSE IF ( (diag_subset_output == DIAG_OTHER .AND. VERIFY('ocean', lowercase(temp_field%file_name)) == 0).OR.&
+                     &    (diag_subset_output == DIAG_OCEAN .AND. VERIFY('ocean', lowercase(temp_field%file_name)) /= 0) ) THEN
+                   CYCLE parser
+                ELSE IF ( lowercase(TRIM(temp_field%spatial_ops)) == 'none' ) THEN
+                   CALL init_input_field(temp_field%module_name, temp_field%field_name, 1)
+                   CALL init_output_field(temp_field%module_name, temp_field%field_name, temp_field%output_name, temp_field%file_name,&
+                        & temp_field%time_method, temp_field%pack, 1)
+                ELSE
+                   CALL init_input_field(temp_field%module_name, temp_field%field_name, 1)
+                   CALL init_output_field(temp_field%module_name, temp_field%field_name, temp_field%output_name, temp_field%file_name,&
+                        & temp_field%time_method, temp_field%pack, 1, temp_field%regional_coords)
+                END IF
+
+                ! Increment number of fields
+                nfields = nfields + 1
+             END IF
+          END IF init
+       END DO parser
+    END DO pass
 
     ! Close the diag_table file.
-    deallocate(diag_table)
+    DEALLOCATE(diag_table)
 
     ! check duplicate output_fields in the diag_table
     CALL check_duplicate_output_fields(ERR_MSG=local_err_msg)
@@ -537,15 +546,15 @@ SUBROUTINE open_diag_table(iunit, iostat, err_msg)
 
     IF ( PRESENT(iostat) ) THEN
        pstat => iostat
-    ELSE 
+    ELSE
        pstat => mystat
     END IF
-    
+
     IF ( .NOT.file_exist('diag_table') ) THEN
        pstat = 1
        IF ( fms_error_handler('diag_table_mod::open_diag_table',&
             & 'diag_table file does not exist.', err_msg) ) RETURN
-    ELSE 
+    ELSE
        pstat = 0
     END IF
 
@@ -667,7 +676,7 @@ TYPE(file_description_type) FUNCTION parse_file_line(line, istat, err_msg)
             & 'Unallowed character in file_duration_units in the diag_table.', err_msg) ) RETURN
     END IF
 
-            
+
     ! Fix the file name
     parse_file_line%file_name = fix_file_name(TRIM(parse_file_line%file_name))
 
@@ -677,7 +686,7 @@ TYPE(file_description_type) FUNCTION parse_file_line(line, istat, err_msg)
        IF ( fms_error_handler('diag_table_mod::parse_file_line', 'Invalid file format for file description in the diag_table.',&
             & err_msg) ) RETURN
     END IF
-    
+
     ! check for known units
     parse_file_line%iTime_units = find_unit_ivalue(parse_file_line%time_units)
     parse_file_line%iOutput_freq_units = find_unit_ivalue(parse_file_line%output_freq_units)
@@ -714,7 +723,7 @@ TYPE(file_description_type) FUNCTION parse_file_line(line, istat, err_msg)
     new_file_freq_present: IF ( parse_file_line%new_file_freq > 0 ) THEN ! New file frequency present.
        IF ( LEN_TRIM(parse_file_line%start_time_s) > 0 ) THEN ! start time present
           READ (parse_file_line%start_time_s, FMT=*, IOSTAT=mystat) year, month, day, hour, minute, second
-          IF ( mystat /= 0 ) THEN 
+          IF ( mystat /= 0 ) THEN
              pstat = 1
              IF ( fms_error_handler('diag_table_mod::parse_file_line',&
                   & 'Invalid start time in the file description in diag_table.', err_msg) ) RETURN
@@ -788,37 +797,37 @@ TYPE(field_description_type) FUNCTION parse_field_line(line, istat, err_msg)
     IF ( SCAN(parse_field_line%module_name, UNALLOWED_ALL) > 0 ) THEN
        pstat = 1
        IF ( fms_error_handler('diag_table_mod::parse_field_line',&
-            & 'Unallowed Unallowed character in module_name in the diag_table.', err_msg) ) RETURN
+            & 'Unallowed character in module_name in the diag_table.', err_msg) ) RETURN
     END IF
     IF ( SCAN(parse_field_line%field_name, UNALLOWED_ALL) > 0 ) THEN
        pstat = 1
        IF ( fms_error_handler('diag_table_mod::parse_field_line',&
-            & 'Unallowed Unallowed character in field_name in the diag_table.', err_msg) ) RETURN
+            & 'Unallowed character in field_name in the diag_table.', err_msg) ) RETURN
     END IF
     IF ( SCAN(parse_field_line%output_name, UNALLOWED_ALL) > 0 ) THEN
        pstat = 1
        IF ( fms_error_handler('diag_table_mod::parse_field_line',&
-            & 'Unallowed Unallowed character in output_name in the diag_table.', err_msg) ) RETURN
+            & 'Unallowed character in output_name in the diag_table.', err_msg) ) RETURN
     END IF
     IF ( SCAN(parse_field_line%file_name, UNALLOWED_ALL) > 0 ) THEN
        pstat = 1
        IF ( fms_error_handler('diag_table_mod::parse_field_line',&
-            & 'Unallowed Unallowed character in file_name in the diag_table.', err_msg) ) RETURN
+            & 'Unallowed character in file_name in the diag_table.', err_msg) ) RETURN
     END IF
     IF ( SCAN(parse_field_line%time_sampling, UNALLOWED_ALL) > 0 ) THEN
        pstat = 1
        IF ( fms_error_handler('diag_table_mod::parse_field_line',&
-            & 'Unallowed Unallowed character in time_sampling in the diag_table.', err_msg) ) RETURN
+            & 'Unallowed character in time_sampling in the diag_table.', err_msg) ) RETURN
     END IF
     IF ( SCAN(parse_field_line%time_method, UNALLOWED_ALL) > 0 ) THEN
        pstat = 1
        IF ( fms_error_handler('diag_table_mod::parse_field_line',&
-            & 'Unallowed Unallowed character in time_method in the diag_table.', err_msg) ) RETURN
+            & 'Unallowed character in time_method in the diag_table.', err_msg) ) RETURN
     END IF
     IF ( SCAN(parse_field_line%spatial_ops, UNALLOWED_QTE) > 0 ) THEN
        pstat = 1
        IF ( fms_error_handler('diag_table_mod::parse_field_line',&
-            & 'Unallowed Unallowed character in spatial_ops in the diag_table.', err_msg) ) RETURN
+            & 'Unallowed character in spatial_ops in the diag_table.', err_msg) ) RETURN
     END IF
 
     ! Fix the file name
@@ -830,7 +839,7 @@ TYPE(field_description_type) FUNCTION parse_field_line(line, istat, err_msg)
        IF ( fms_error_handler('diag_table_mod::parse_field_line',&
             & 'Packing is out of range for the field description in diag_table.', err_msg) ) RETURN
     END IF
-    
+
     IF ( lowercase(TRIM(parse_field_line%spatial_ops)) /= 'none' ) THEN
        READ (parse_field_line%spatial_ops, FMT=*, IOSTAT=mystat) parse_field_line%regional_coords
        IF ( mystat /= 0 ) THEN
@@ -859,7 +868,7 @@ PURE LOGICAL FUNCTION is_a_file(line)
     CHARACTER(len=*), INTENT(in) :: line
 
     CHARACTER(len=5) :: first
-    INTEGER :: second 
+    INTEGER :: second
     INTEGER :: mystat !< IO status from read
 
 #if defined __PATHSCALE__ || defined _CRAYFTN
@@ -893,9 +902,8 @@ END FUNCTION is_a_file
   !     PURE CHARACTER(len=128) FUNCTION fix_file_name(file_name_string)
   !   
   !   
-  !     Removes any trailing '.nc' and appends to the file name additional information
-  !     depending on if we are running an ensemble, or requesting append_pelist_name.
-  !     
+  !     Removes any trailing '.nc' and appends (if requested) append_pelist_name.
+  !
   !     Presently, the ensemble appendix will override the append_pelist_name variable.
   !   
   !   String containing the file name from the diag_table.
@@ -909,19 +917,16 @@ PURE CHARACTER(len=128) FUNCTION fix_file_name(file_name_string)
     file_name_len = LEN_TRIM(file_name_string)
 
     ! Remove trailing '.nc' from the file_name, and append suffixes
-    IF ( file_name_len > 2 ) THEN 
+    IF ( file_name_len > 2 ) THEN
        IF ( file_name_string(file_name_len-2:file_name_len) == '.nc' ) THEN
           fix_file_name = file_name_string(1:file_name_len-3)
           file_name_len = file_name_len - 3
        END IF
     END IF
-       
-    ! If using ensembles, then append the ensemble information
-    ! Or add the optional suffix based on the pe list name if the
+
+    ! Add the optional suffix based on the pe list name if the
     ! append_pelist_name == .TRUE.
-    IF ( LEN_TRIM(filename_appendix) > 0 ) THEN 
-       fix_file_name(file_name_len+1:) = TRIM(filename_appendix)    
-    ELSE IF ( append_pelist_name ) THEN
+    IF ( append_pelist_name ) THEN
        fix_file_name(file_name_len+1:) = TRIM(pelist_name)
     END IF
   END FUNCTION fix_file_name
@@ -961,7 +966,7 @@ PURE INTEGER FUNCTION find_unit_ivalue(unit_string)
        find_unit_ivalue = 3
     CASE ('days')
        find_unit_ivalue = 4
-    CASE ('months') 
+    CASE ('months')
        find_unit_ivalue = 5
     CASE ('years')
        find_unit_ivalue = 6
@@ -990,4 +995,3 @@ END SUBROUTINE initialize_output_arrays
   ! 
   ! 
 END MODULE diag_table_mod
-
diff --git a/src/shared/diag_manager/diag_util.F90 b/src/shared/diag_manager/diag_util.F90
index a471d618bf..67372e533f 100644
--- a/src/shared/diag_manager/diag_util.F90
+++ b/src/shared/diag_manager/diag_util.F90
@@ -1,5 +1,7 @@
+#include 
+
 MODULE diag_util_mod
-  ! 
+  ! 
   !   Seth Underwood
   ! 
   ! 
@@ -15,7 +17,7 @@ MODULE diag_util_mod
   ! 
   !   
   !     Make an interface check_bounds_are_exact for the subroutines check_bounds_are_exact_static and
-  !     check_bounds_are_exact_dynamic. 
+  !     check_bounds_are_exact_dynamic.
   !     
       !       INTERFACE check_bounds_are_exact
   !         MODULE PROCEDURE check_bounds_are_exact_static
@@ -31,7 +33,9 @@ MODULE diag_util_mod
        & base_second, num_files, max_files, max_fields_per_file, max_out_per_in_field,&
        & max_input_fields,num_input_fields, max_output_fields, num_output_fields, coord_type,&
        & mix_snapshot_average_fields, global_descriptor, CMOR_MISSING_VALUE, use_cmor, pack_size,&
-       & debug_diag_manager, conserve_water
+       & debug_diag_manager, conserve_water, output_field_type, max_field_attributes, max_file_attributes,&
+       & file_type, prepend_date, region_out_use_alt_value, GLO_REG_VAL, GLO_REG_VAL_ALT,&
+       & DIAG_FIELD_NOT_FOUND, diag_init_time
   USE diag_axis_mod, ONLY  : get_diag_axis_data, get_axis_global_length, get_diag_axis_cart,&
        & get_domain1d, get_domain2d, diag_subaxes_init, diag_axis_init, get_diag_axis, get_axis_aux,&
        & get_axes_shift, get_diag_axis_name, get_diag_axis_domain_name
@@ -48,19 +52,67 @@ MODULE diag_util_mod
        & OPERATOR(<), OPERATOR(>=), OPERATOR(<=)
   USE mpp_io_mod, ONLY : mpp_close
   USE mpp_mod, ONLY : mpp_npes
+  USE fms_io_mod, ONLY : get_instance_filename
   USE constants_mod, ONLY : SECONDS_PER_DAY, SECONDS_PER_HOUR, SECONDS_PER_MINUTE
 
+#ifdef use_netCDF
+  USE netcdf, ONLY: NF90_CHAR
+#endif
+
   IMPLICIT NONE
   PRIVATE
   PUBLIC get_subfield_size, log_diag_field_info, update_bounds, check_out_of_bounds,&
        & check_bounds_are_exact_dynamic, check_bounds_are_exact_static, init_file, diag_time_inc,&
        & find_input_field, init_input_field, init_output_field, diag_data_out, write_static,&
-       & check_duplicate_output_fields, get_date_dif, get_subfield_vert_size, sync_file_times
+       & check_duplicate_output_fields, get_date_dif, get_subfield_vert_size, sync_file_times,&
+       & prepend_attribute, attribute_init
+
+  ! 
+  !   
+  !     prepend a value to a string attribute in the output field or output file
+  !   
+  !   
+  !   
+  !     Prepend a character string to a character attribute for a give field, or to a global attribute
+  !     in a give file.
+  !   
+  !   
+  !   
+  !   
+  !   
+  INTERFACE prepend_attribute
+     MODULE PROCEDURE prepend_attribute_field
+     MODULE PROCEDURE prepend_attribute_file
+  END INTERFACE prepend_attribute
+  ! 
+
+  ! 
+  !   
+  !     Allocates the atttype in out_file
+  !   
+  !   
+  !   
+  !     Allocates memory in out_file for the attributes.  Will FATAL if err_msg is not included
+  !     in the subroutine call.
+  !   
+  !   output field to allocate memory for attribute
+  !   output file to allocate memory for attribute
+  !   Error message, passed back to calling function
+  INTERFACE attribute_init
+     MODULE PROCEDURE attribute_init_field
+     MODULE PROCEDURE attribute_init_file
+  END INTERFACE attribute_init
+  ! 
 
   CHARACTER(len=128),PRIVATE  :: version =&
-       & '$Id: diag_util.F90,v 20.0 2013/12/14 00:18:58 fms Exp $'
+       & '$Id$'
   CHARACTER(len=128),PRIVATE  :: tagname =&
-       & '$Name: tikal $'
+       & '$Name$'
 
 CONTAINS
 
@@ -72,7 +124,7 @@ MODULE diag_util_mod
   !     SUBROUTINE get_subfield_size(axes, outnum)
   !   
   !   
-  !     Get the size, start and end indices for output_fields(outnum), then  
+  !     Get the size, start and end indices for output_fields(outnum), then
   !     fill in output_fields(outnum)%output_grid%(start_indx, end_indx)
   !   
   !   Axes of the input_field.
@@ -83,23 +135,31 @@ SUBROUTINE get_subfield_size(axes, outnum)
 
     REAL, ALLOCATABLE   :: global_lat(:), global_lon(:), global_depth(:)
     INTEGER :: global_axis_size
-    INTEGER :: i,xbegin,xend,ybegin,yend,xbegin_l,xend_l,ybegin_l,yend_l 
+    INTEGER :: i,xbegin,xend,ybegin,yend,xbegin_l,xend_l,ybegin_l,yend_l
     CHARACTER(len=1) :: cart
     TYPE(domain2d) :: Domain2, Domain2_new
     TYPE(domain1d) :: Domain1, Domain1x, Domain1y
     REAL :: start(3), end(3) ! start and end coordinates in 3 axes
-    INTEGER :: gstart_indx(3), gend_indx(3) ! global start and end indices of output domain in 3 axes 
+    INTEGER :: gstart_indx(3), gend_indx(3) ! global start and end indices of output domain in 3 axes
     REAL, ALLOCATABLE :: subaxis_x(:), subaxis_y(:), subaxis_z(:) !containing local coordinates in x,y,z axes
     CHARACTER(len=128) :: msg
     INTEGER :: ishift, jshift
+    INTEGER :: grv !< Value used to determine if the region defined in the diag_table is for the whole axis, or a sub-axis
     CHARACTER(len=128), DIMENSION(2) :: axis_domain_name
 
     !initilization for local output
     ! initially out of (lat/lon/depth) range
     start = -1.e10
-    end = -1.e10 
+    end = -1.e10
     gstart_indx = -1
-    gend_indx=-1
+    gend_indx = -1
+
+    ! get the value to compare to determine if writing full axis data
+    IF ( region_out_use_alt_value ) THEN
+       grv = GLO_REG_VAL_ALT
+    ELSE
+       grv = GLO_REG_VAL
+    END IF
 
     ! get axis data (lat, lon, depth) and indices
     start = output_fields(outnum)%output_grid%start
@@ -121,23 +181,23 @@ SUBROUTINE get_subfield_size(axes, outnum)
                   & 'wrong order of axes, X should come first',FATAL)
              ALLOCATE(global_lon(global_axis_size))
              CALL get_diag_axis_data(axes(i),global_lon)
-             IF( INT( start(i)*END(i) ) == 1 ) THEN 
+             IF( INT(start(i)) == grv .AND. INT(end(i)) == grv ) THEN
                 gstart_indx(i) = 1
                 gend_indx(i) = global_axis_size
                 output_fields(outnum)%output_grid%subaxes(i) = axes(i)
-             ELSE 
+             ELSE
                 gstart_indx(i) = get_index(start(i),global_lon)
                 gend_indx(i) = get_index(END(i),global_lon)
              END IF
              ALLOCATE(subaxis_x(gstart_indx(i):gend_indx(i)))
-             subaxis_x=global_lon(gstart_indx(i):gend_indx(i))   
+             subaxis_x=global_lon(gstart_indx(i):gend_indx(i))
           CASE ('Y')
              ! wrong order of axes, Y should come second.
              IF( i.NE.2 ) CALL error_mesg('diag_util_mod::get_subfield_size',&
                   & 'wrong order of axes, Y should come second',FATAL)
              ALLOCATE(global_lat(global_axis_size))
              CALL get_diag_axis_data(axes(i),global_lat)
-             IF( INT( start(i)*END(i) ) == 1 ) THEN 
+             IF( INT(start(i)) == grv .AND. INT(END(i)) == grv ) THEN
                 gstart_indx(i) = 1
                 gend_indx(i) = global_axis_size
                 output_fields(outnum)%output_grid%subaxes(i) = axes(i)
@@ -149,9 +209,9 @@ SUBROUTINE get_subfield_size(axes, outnum)
              subaxis_y=global_lat(gstart_indx(i):gend_indx(i))
           CASE ('Z')
              ! wrong values in vertical axis of region
-             IF ( start(i)*END(i)<0 ) CALL error_mesg('diag_util_mod::get_subfield_size',&
+             IF ( start(i)*END(i)<0. ) CALL error_mesg('diag_util_mod::get_subfield_size',&
                   & 'wrong values in vertical axis of region',FATAL)
-             IF ( start(i)>=0 .AND. END(i)>0 ) THEN 
+             IF ( start(i)>=0. .AND. END(i)>0. ) THEN
                 ALLOCATE(global_depth(global_axis_size))
                 CALL get_diag_axis_data(axes(i),global_depth)
                 gstart_indx(i) = get_index(start(i),global_depth)
@@ -176,7 +236,7 @@ SUBROUTINE get_subfield_size(axes, outnum)
        END DO
 
        DO i = 1, SIZE(axes(:))
-          IF( gstart_indx(i) == -1 .OR. gend_indx(i) == -1 ) THEN
+          IF ( gstart_indx(i) == -1 .OR. gend_indx(i) == -1 ) THEN
              ! 
              !   can not find gstart_indx/gend_indx for ,
              !   check region bounds for axis .
@@ -219,9 +279,9 @@ SUBROUTINE get_subfield_size(axes, outnum)
           ! 
           !   wrong values in vertical axis of region
           ! 
-          IF ( start(3)*END(3)<0 ) CALL error_mesg('diag_util_mod::get_subfield_size',&
+          IF ( start(3)*END(3)<0. ) CALL error_mesg('diag_util_mod::get_subfield_size',&
                & 'wrong values in vertical axis of region',FATAL)
-          IF ( start(3)>=0 .AND. END(3)>0 ) THEN 
+          IF ( start(3)>=0. .AND. END(3)>0. ) THEN
              ALLOCATE(global_depth(global_axis_size))
              CALL get_diag_axis_data(axes(3),global_depth)
              gstart_indx(3) = get_index(start(3),global_depth)
@@ -229,7 +289,7 @@ SUBROUTINE get_subfield_size(axes, outnum)
              gend_indx(3) = get_index(END(3),global_depth)
              IF( start(3) >= MAXVAL(global_depth) ) gstart_indx(3)= global_axis_size
              IF( END(3)   >= MAXVAL(global_depth) ) gend_indx(3)  = global_axis_size
-             
+
              ALLOCATE(subaxis_z(gstart_indx(3):gend_indx(3)))
              subaxis_z=global_depth(gstart_indx(3):gend_indx(3))
              output_fields(outnum)%output_grid%subaxes(3) =&
@@ -242,19 +302,19 @@ SUBROUTINE get_subfield_size(axes, outnum)
           END IF
        END IF
     END IF
-    
+
     ! get domain and compute_domain(xbegin,xend,ybegin,yend)
-    xbegin=-1
-    xend=-1
-    ybegin=-1
-    yend=-1
+    xbegin = -1
+    xend = -1
+    ybegin = -1
+    yend = -1
 
     Domain2 = get_domain2d(axes)
     IF ( Domain2 .NE. NULL_DOMAIN2D ) THEN
-       CALL mpp_get_compute_domain(Domain2,xbegin,xend,ybegin,yend)
+       CALL mpp_get_compute_domain(Domain2, xbegin, xend, ybegin, yend)
        CALL mpp_get_domain_components(Domain2, Domain1x, Domain1y)
     ELSE
-       DO i = 1, MIN(SIZE(axes(:)),2)    
+       DO i = 1, MIN(SIZE(axes(:)),2)
           Domain1 = get_domain1d(axes(i))
           IF ( Domain1 .NE. NULL_DOMAIN1D ) THEN
              CALL get_diag_axis_cart(axes(i),cart)
@@ -278,17 +338,17 @@ SUBROUTINE get_subfield_size(axes, outnum)
     xend = xend+ishift
     yend = yend+jshift
 
-    IF ( xbegin== -1 .OR. xend==-1 .OR. ybegin==-1 .OR. yend==-1 ) THEN
+    IF ( xbegin == -1 .OR. xend == -1 .OR. ybegin == -1 .OR. yend == -1 ) THEN
        ! wrong compute domain indices
-       CALL error_mesg('diag_util_mod::get_subfield_size', 'wrong compute domain indices',FATAL)  
+       CALL error_mesg('diag_util_mod::get_subfield_size', 'wrong compute domain indices',FATAL)
     END IF
-      
+
     ! get the area containing BOTH compute domain AND local output area
-    IF(gstart_indx(1)> xend .OR. xbegin > gend_indx(1)) THEN
+    IF( gstart_indx(1) > xend .OR. xbegin > gend_indx(1) ) THEN
        output_fields(outnum)%output_grid%l_start_indx(1) = -1
        output_fields(outnum)%output_grid%l_end_indx(1) = -1
        output_fields(outnum)%need_compute = .FALSE. ! not involved
-    ELSEIF (gstart_indx(2)> yend .OR. ybegin > gend_indx(2)) THEN
+    ELSEIF ( gstart_indx(2) > yend .OR. ybegin > gend_indx(2) ) THEN
        output_fields(outnum)%output_grid%l_start_indx(2) = -1
        output_fields(outnum)%output_grid%l_end_indx(2) = -1
        output_fields(outnum)%need_compute = .FALSE. ! not involved
@@ -314,20 +374,20 @@ SUBROUTINE get_subfield_size(axes, outnum)
        output_fields(outnum)%output_grid%subaxes(2) =&
             & diag_subaxes_init(axes(2),subaxis_y, gstart_indx(2),gend_indx(2),Domain2_new)
        DO i = 1, SIZE(axes(:))
-          IF(output_fields(outnum)%output_grid%subaxes(i) == -1) THEN  
+          IF ( output_fields(outnum)%output_grid%subaxes(i) == -1 ) THEN
              ! 
              !    error at i = 
              ! 
              WRITE(msg,'(a,"/",I4)') 'at i = ',i
              CALL error_mesg('diag_util_mod::get_subfield_size '//TRIM(output_fields(outnum)%output_name),&
-                  'error '//TRIM(msg), FATAL)   
+                  'error '//TRIM(msg), FATAL)
           END IF
        END DO
 
        ! local start index should start from 1
-       output_fields(outnum)%output_grid%l_start_indx(1) = MAX(xbegin, gstart_indx(1)) - xbegin + 1   
+       output_fields(outnum)%output_grid%l_start_indx(1) = MAX(xbegin, gstart_indx(1)) - xbegin + 1
        output_fields(outnum)%output_grid%l_start_indx(2) = MAX(ybegin, gstart_indx(2)) - ybegin + 1
-       output_fields(outnum)%output_grid%l_end_indx(1) = MIN(xend, gend_indx(1)) - xbegin + 1 
+       output_fields(outnum)%output_grid%l_end_indx(1) = MIN(xend, gend_indx(1)) - xbegin + 1
        output_fields(outnum)%output_grid%l_end_indx(2) = MIN(yend, gend_indx(2)) - ybegin + 1
        IF ( SIZE(axes(:))>2 ) THEN
           output_fields(outnum)%output_grid%l_start_indx(3) = gstart_indx(3)
@@ -339,10 +399,9 @@ SUBROUTINE get_subfield_size(axes, outnum)
     END IF
     IF ( ALLOCATED(subaxis_x) ) DEALLOCATE(subaxis_x, global_lon)
     IF ( ALLOCATED(subaxis_y) ) DEALLOCATE(subaxis_y, global_lat)
-
   END SUBROUTINE get_subfield_size
   ! 
-  
+
   ! 
   !   
   !     Get size, start and end indices for output fields.
@@ -364,21 +423,21 @@ SUBROUTINE get_subfield_vert_size(axes, outnum)
     REAL, ALLOCATABLE, DIMENSION(:) :: global_depth
     REAL, ALLOCATABLE, DIMENSION(:) :: subaxis_z !containing local coordinates in x,y,z axes
     INTEGER :: i, global_axis_size
-    INTEGER, DIMENSION(3) :: gstart_indx, gend_indx ! global start and end indices of output domain in 3 axes 
+    INTEGER, DIMENSION(3) :: gstart_indx, gend_indx ! global start and end indices of output domain in 3 axes
     CHARACTER(len=1) :: cart
     CHARACTER(len=128) :: msg
 
     !initilization for local output
     start = -1.e10
     end = -1.e10 ! initially out of (lat/lon/depth) range
-    gstart_indx = -1 
+    gstart_indx = -1
     gend_indx=-1
 
     ! get axis data (lat, lon, depth) and indices
     start= output_fields(outnum)%output_grid%start
     end = output_fields(outnum)%output_grid%end
 
-    DO i = 1, SIZE(axes(:))   
+    DO i = 1, SIZE(axes(:))
        global_axis_size = get_axis_global_length(axes(i))
        output_fields(outnum)%output_grid%subaxes(i) = -1
        CALL get_diag_axis_cart(axes(i), cart)
@@ -399,9 +458,9 @@ SUBROUTINE get_subfield_vert_size(axes, outnum)
           output_fields(outnum)%output_grid%subaxes(i) = axes(i)
        CASE ('Z')
           ! wrong values in vertical axis of region
-          IF( start(i)*END(i) < 0 ) CALL error_mesg('diag_util_mod::get_subfield_vert_size',&
+          IF( start(i)*END(i) < 0. ) CALL error_mesg('diag_util_mod::get_subfield_vert_size',&
                & 'wrong values in vertical axis of region',FATAL)
-          IF( start(i) >= 0 .AND. END(i) > 0 ) THEN 
+          IF( start(i) >= 0. .AND. END(i) > 0. ) THEN
              ALLOCATE(global_depth(global_axis_size))
              CALL get_diag_axis_data(axes(i),global_depth)
              gstart_indx(i) = get_index(start(i),global_depth)
@@ -431,7 +490,7 @@ SUBROUTINE get_subfield_vert_size(axes, outnum)
     END DO
 
     DO i = 1,SIZE(axes(:))
-       IF ( gstart_indx(i)== -1 .OR. gend_indx(i)== -1 ) THEN
+       IF ( gstart_indx(i) == -1 .OR. gend_indx(i) == -1 ) THEN
           ! 
           !   can not find gstart_indx/gend_indx for 
           !   check region bounds for axis
@@ -456,7 +515,7 @@ SUBROUTINE get_subfield_vert_size(axes, outnum)
     END IF
   END SUBROUTINE get_subfield_vert_size
   ! 
-  
+
   ! 
   ! 
   !   
@@ -483,13 +542,13 @@ INTEGER FUNCTION get_index(number, array)
     DO i = 2, n-1
        IF( (array(i-1)array(i+1)) .OR. (array(i-1)>array(i).AND.array(i)array NOT monotonously ordered
-          CALL error_mesg('diag_util_mod::get_index', 'array NOT monotonously ordered',FATAL) 
+          CALL error_mesg('diag_util_mod::get_index', 'array NOT monotonously ordered',FATAL)
        END IF
     END DO
     get_index = -1
     found = .FALSE.
-    ! search in increasing array 
-    DO i = 1, n-1                
+    ! search in increasing array
+    DO i = 1, n-1
        IF ( (array(i)<=number).AND.(array(i+1)>= number) ) THEN
           IF( number - array(i) <= array(i+1) - number ) THEN
              get_index = i
@@ -506,7 +565,7 @@ INTEGER FUNCTION get_index(number, array)
        DO i = 1, n-1
           IF ( (array(i)>=number).AND.(array(i+1)<= number) ) THEN
              IF ( array(i)-number <= number-array(i+1) ) THEN
-                get_index = i 
+                get_index = i
                 found = .TRUE.
              ELSE
                 get_index = i+1
@@ -518,25 +577,28 @@ INTEGER FUNCTION get_index(number, array)
     END IF
     ! if still not found, is it less than the first element
     ! or greater than last element? (Increasing Array)
+    ! But it must be within 2x the axis spacing
+    ! i.e. array(1)-(array(3)-array(1)).LT.number .AND. or 2*array(1)-array(3).LT.number
     IF ( .NOT. found ) THEN
-       IF ( array(1).GT.number ) THEN
+       IF ( 2*array(1)-array(3).LT.number .AND. number.LT.array(1) ) THEN
           get_index = 1
           found = .TRUE.
-       ELSE IF ( array(n).LT.number ) THEN
+       ELSE IF ( array(n).LT.number .AND. number.LT.2*array(n)-array(n-2) ) THEN
           get_index = n
           found = .TRUE.
        ELSE
           found = .FALSE.
        END IF
     END IF
-   
-   ! if still not found, is it greater than the first element
-   ! or less than the last element? (Decreasing Array)
+
+    ! if still not found, is it greater than the first element
+    ! or less than the last element? (Decreasing Array)
+    ! But it must be within 2x the axis spacing (see above)
     IF ( .NOT. found ) THEN
-       IF ( array(1).LT.number ) THEN
+       IF ( 2*array(1)-array(3).GT.number .AND. number.GT.array(1) ) THEN
           get_index = 1
           found = .TRUE.
-       ELSE IF ( array(n).GT.number ) THEN
+       ELSE IF ( array(n).GT.number .AND. number.GT.2*array(n)-array(n-2) ) THEN
           get_index = n
           found = .TRUE.
        ELSE
@@ -600,16 +662,16 @@ SUBROUTINE log_diag_field_info(module_name, field_name, axes, long_name, units,&
 
     IF ( PRESENT(long_name) ) THEN
        lname  = TRIM(long_name)
-    ELSE 
+    ELSE
        lname  = ''
     END IF
-    
+
     IF ( PRESENT(units) ) THEN
        lunits = TRIM(units)
     ELSE
        lunits = ''
     END IF
- 
+
     WRITE (numaxis,'(i1)') SIZE(axes)
 
     IF (PRESENT(missing_value)) THEN
@@ -675,7 +737,7 @@ END SUBROUTINE log_diag_field_info
   !   Upper k bound.
   SUBROUTINE update_bounds(out_num, lower_i, upper_i, lower_j, upper_j, lower_k, upper_k)
     INTEGER, INTENT(in) :: out_num, lower_i, upper_i, lower_j, upper_j, lower_k, upper_k
-    
+
     output_fields(out_num)%imin = MIN(output_fields(out_num)%imin, lower_i)
     output_fields(out_num)%imax = MAX(output_fields(out_num)%imax, upper_i)
     output_fields(out_num)%jmin = MIN(output_fields(out_num)%jmin, lower_j)
@@ -890,7 +952,7 @@ SUBROUTINE check_bounds_are_exact_static(out_num, diag_field_id, err_msg)
     output_fields(out_num)%jmin = VERY_LARGE_AXIS_LENGTH
     output_fields(out_num)%kmax = 0
     output_fields(out_num)%kmin = VERY_LARGE_AXIS_LENGTH
-    
+
   END SUBROUTINE check_bounds_are_exact_static
   ! 
 
@@ -944,32 +1006,32 @@ SUBROUTINE init_file(name, output_freq, output_units, FORMAT, time_units, long_n
             & in the namelist diag_manager_nml.', FATAL)
     END IF
 
-    IF ( PRESENT(new_file_freq) ) THEN 
+    IF ( PRESENT(new_file_freq) ) THEN
        new_file_freq1 = new_file_freq
-    ELSE 
+    ELSE
        new_file_freq1 = VERY_LARGE_FILE_FREQ
     END IF
-    
-    IF ( PRESENT(new_file_freq_units) ) THEN 
-       new_file_freq_units1 = new_file_freq_units 
+
+    IF ( PRESENT(new_file_freq_units) ) THEN
+       new_file_freq_units1 = new_file_freq_units
     ELSE IF ( get_calendar_type() == NO_CALENDAR ) THEN
        new_file_freq_units1 = DIAG_DAYS
-    ELSE 
+    ELSE
        new_file_freq_units1 = DIAG_YEARS
     END IF
-    
+
     IF ( PRESENT(file_duration) ) THEN
-       file_duration1 = file_duration 
+       file_duration1 = file_duration
     ELSE
        file_duration1 = new_file_freq1
     END IF
-    
-    IF ( PRESENT(file_duration_units) ) THEN 
+
+    IF ( PRESENT(file_duration_units) ) THEN
        file_duration_units1 = file_duration_units
-    ELSE 
+    ELSE
        file_duration_units1 = new_file_freq_units1
     END IF
-    
+
     files(num_files)%tile_count = tile_count
     files(num_files)%name = TRIM(name)
     files(num_files)%output_freq = output_freq
@@ -985,7 +1047,7 @@ SUBROUTINE init_file(name, output_freq, output_units, FORMAT, time_units, long_n
     files(num_files)%new_file_freq_units = new_file_freq_units1
     files(num_files)%duration = file_duration1
     files(num_files)%duration_units = file_duration_units1
-    IF ( PRESENT(start_time) ) THEN 
+    IF ( PRESENT(start_time) ) THEN
        files(num_files)%start_time = start_time
     ELSE
        files(num_files)%start_time = base_time
@@ -1000,7 +1062,7 @@ SUBROUTINE init_file(name, output_freq, output_units, FORMAT, time_units, long_n
        CALL error_mesg('diag_util_mod::init_file', 'close time GREATER than next_open time, check file duration,&
             & file frequency in '//files(num_files)%name, FATAL)
     END IF
-    
+
     ! add time_axis_id and time_bounds_id here
     WRITE(time_units_str, 11) TRIM(time_unit_list(files(num_files)%time_units)), base_year,&
          & base_month, base_day, base_hour, base_minute, base_second
@@ -1132,7 +1194,7 @@ TYPE(time_type) FUNCTION diag_time_inc(time, output_freq, output_units, err_msg)
        ELSE
           diag_time_inc = increment_date(time, output_freq, 0, 0, 0, 0, 0, err_msg=error_message_local)
        END IF
-    ELSE 
+    ELSE
        error_message_local = 'illegal output units'
     END IF
 
@@ -1192,11 +1254,11 @@ INTEGER FUNCTION find_input_field(module_name, field_name, tile_count)
 
     INTEGER :: i
 
-    find_input_field = -1 ! Default return value if not found.
+    find_input_field = DIAG_FIELD_NOT_FOUND ! Default return value if not found.
     DO i = 1, num_input_fields
        IF(tile_count == input_fields(i)%tile_count .AND.&
             & TRIM(input_fields(i)%module_name) == TRIM(module_name) .AND.&
-            & lowercase(TRIM(input_fields(i)%field_name)) == lowercase(TRIM(field_name))) THEN 
+            & lowercase(TRIM(input_fields(i)%field_name)) == lowercase(TRIM(field_name))) THEN
           find_input_field = i
           RETURN
        END IF
@@ -1243,6 +1305,7 @@ SUBROUTINE init_input_field(module_name, field_name, tile_count)
     input_fields(num_input_fields)%standard_name = 'none'
     input_fields(num_input_fields)%tile_count = tile_count
     input_fields(num_input_fields)%numthreads = 1
+    input_fields(num_input_fields)%active_omp_level = 0
     input_fields(num_input_fields)%time = time_zero
   END SUBROUTINE init_input_field
   ! 
@@ -1277,9 +1340,20 @@ SUBROUTINE init_output_field(module_name, field_name, output_name, output_file,&
     INTEGER :: out_num, in_num, file_num, file_num_tile1
     INTEGER :: num_fields, i, method_selected, l1
     INTEGER :: ioerror
+    REAL :: pow_value
+    INTEGER :: grv !< Value used to determine if the region defined in the diag_table is for the whole axis, or a sub-axis
     CHARACTER(len=128) :: error_msg
     CHARACTER(len=50) :: t_method
 
+    ! Value to use to determine if a region is to be output on the full axis, or sub-axis
+    ! get the value to compare to determine if writing full axis data
+    IF ( region_out_use_alt_value ) THEN
+       grv = GLO_REG_VAL_ALT
+    ELSE
+       grv = GLO_REG_VAL
+    END IF
+
+
     ! Get a number for this output field
     num_output_fields = num_output_fields + 1
     IF ( num_output_fields > max_output_fields ) THEN
@@ -1375,6 +1449,7 @@ SUBROUTINE init_output_field(module_name, field_name, output_name, output_file,&
     ! Enter the other data for this output field
     output_fields(out_num)%output_name = TRIM(output_name)
     output_fields(out_num)%pack = pack
+    output_fields(out_num)%pow_value = 1
     output_fields(out_num)%num_axes = 0
     output_fields(out_num)%total_elements = 0
     output_fields(out_num)%region_elements = 0
@@ -1391,8 +1466,9 @@ SUBROUTINE init_output_field(module_name, field_name, output_name, output_file,&
     ! Initialize all time method to false
     method_selected = 0
     output_fields(out_num)%time_average = .FALSE.
+    output_fields(out_num)%time_rms = .FALSE.
     output_fields(out_num)%time_min = .FALSE.
-    output_fields(out_num)%time_max = .FALSE. 
+    output_fields(out_num)%time_max = .FALSE.
     output_fields(out_num)%time_ops = .FALSE.
     output_fields(out_num)%written_once = .FALSE.
 
@@ -1421,12 +1497,32 @@ SUBROUTINE init_output_field(module_name, field_name, output_name, output_file,&
        output_fields(out_num)%time_average = .TRUE.
        method_selected = method_selected+1
        t_method='mean'
+    ELSEIF ( INDEX(t_method,'pow') == 1 ) THEN
+       ! Get the integer number from the t_method
+       READ (UNIT=t_method(4:LEN_TRIM(t_method)), FMT=*, IOSTAT=ioerror) pow_value
+       IF ( ioerror /= 0 .OR. output_fields(out_num)%pow_value < 1  .OR. FLOOR(pow_value) /= CEILING(pow_value) ) THEN
+          ! 
+          !   Invalid power number in time operation "".  Must be a positive integer.
+          ! 
+          CALL error_mesg('diag_util_mod::init_output_field',&
+               & 'Invalid power number in time operation "'//TRIM(t_method)//'".  Must be a positive integer', FATAL)
+       END IF
+       output_fields(out_num)%pow_value = INT(pow_value)
+       output_fields(out_num)%time_average = .TRUE.
+       method_selected = method_selected+1
+       t_method = 'mean_pow('//t_method(4:LEN_TRIM(t_method))//')'
     ELSE
        SELECT CASE(TRIM(t_method))
        CASE ( '.true.', 'mean', 'average', 'avg' )
           output_fields(out_num)%time_average = .TRUE.
           method_selected = method_selected+1
           t_method = 'mean'
+       CASE ( 'rms' )
+          output_fields(out_num)%time_average = .TRUE.
+          output_fields(out_num)%time_rms = .TRUE.
+          output_fields(out_num)%pow_value = 2.0
+          method_selected = method_selected+1
+          t_method = 'root_mean_square'
        CASE ( '.false.', 'none', 'point' )
           output_fields(out_num)%time_average = .FALSE.
           method_selected = method_selected+1
@@ -1437,17 +1533,17 @@ SUBROUTINE init_output_field(module_name, field_name, output_name, output_file,&
           IF ( output_fields(out_num)%output_name(l1-2:l1) /= 'max' ) &
                output_fields(out_num)%output_name = TRIM(output_name)//'_max'
           method_selected = method_selected+1
-          t_method = 'max'        
+          t_method = 'max'
        CASE ( 'minimum', 'min' )
           output_fields(out_num)%time_min = .TRUE.
           l1 = LEN_TRIM(output_fields(out_num)%output_name)
           IF ( output_fields(out_num)%output_name(l1-2:l1) /= 'min' )&
                & output_fields(out_num)%output_name = TRIM(output_name)//'_min'
           method_selected = method_selected+1
-          t_method = 'min'        
+          t_method = 'min'
        END SELECT
     END IF
-    
+
     ! reconcile logical flags
     output_fields(out_num)%time_ops = output_fields(out_num)%time_min.OR.output_fields(out_num)%time_max&
          & .OR.output_fields(out_num)%time_average
@@ -1457,8 +1553,8 @@ SUBROUTINE init_output_field(module_name, field_name, output_name, output_file,&
     IF ( PRESENT(local_coord) ) THEN
        input_fields(in_num)%local = .TRUE.
        input_fields(in_num)%local_coord = local_coord
-       IF ( INT(local_coord%xbegin * local_coord%xbegin) == 1 .AND.&
-            & INT(local_coord%ybegin * local_coord%ybegin) ==1 ) THEN
+       IF ( INT(local_coord%xbegin) ==  grv .AND. INT(local_coord%xend) == grv .AND.&
+            & INT(local_coord%ybegin) == grv .AND. INT(local_coord%yend) == grv ) THEN
           output_fields(out_num)%local_output = .FALSE.
           output_fields(out_num)%need_compute = .FALSE.
           output_fields(out_num)%reduced_k_range = .TRUE.
@@ -1493,16 +1589,17 @@ SUBROUTINE init_output_field(module_name, field_name, output_name, output_file,&
 
     output_fields(out_num)%time_method = TRIM(t_method)
 
-    ! allocate counters: NOTE that for simplicity we always allocate them, even 
-    ! if they are superceeded by 4D "counter" array. This isn't most memory 
+    ! allocate counters: NOTE that for simplicity we always allocate them, even
+    ! if they are superceeded by 4D "counter" array. This isn't most memory
     ! efficient, approach, but probably tolerable since they are so small anyway
     ALLOCATE(output_fields(out_num)%count_0d(output_fields(out_num)%n_diurnal_samples))
     ALLOCATE(output_fields(out_num)%num_elements(output_fields(out_num)%n_diurnal_samples))
     output_fields(out_num)%count_0d(:) = 0
     output_fields(out_num)%num_elements(:) = 0
+    output_fields(out_num)%num_attributes = 0
   END SUBROUTINE init_output_field
   ! 
-  
+
   ! 
   ! 
   !   
@@ -1519,28 +1616,30 @@ END SUBROUTINE init_output_field
   SUBROUTINE opening_file(file, time)
     ! WARNING: Assumes that all data structures are fully initialized
     INTEGER, INTENT(in) :: file
-    TYPE(time_type), INTENT(in) :: time  
+    TYPE(time_type), INTENT(in) :: time
 
     REAL, DIMENSION(2) :: DATA
-    INTEGER :: j, field_num, input_field_num, num_axes, k 
+    INTEGER :: j, field_num, input_field_num, num_axes, k
     INTEGER :: field_num1
     INTEGER :: position
     INTEGER :: dir, edges
     INTEGER :: ntileMe
+    INTEGER :: year, month, day, hour, minute, second
     INTEGER, ALLOCATABLE :: tile_id(:)
     INTEGER, DIMENSION(1) :: time_axis_id, time_bounds_id
     ! size of this axes array must be at least max num. of
     ! axes per field + 2; the last two elements are for time
     ! and time bounds dimensions
-    INTEGER, DIMENSION(6) :: axes 
+    INTEGER, DIMENSION(6) :: axes
     LOGICAL :: time_ops, aux_present, match_aux_name
     LOGICAL :: all_scalar_or_1d
     CHARACTER(len=7) :: prefix
-    CHARACTER (len = 7) :: avg_name = 'average'
+    CHARACTER(len=7) :: avg_name = 'average'
     CHARACTER(len=128) :: time_units, timeb_units, avg, error_string, filename, aux_name,fieldname
     CHARACTER(len=128) :: suffix, base_name
     CHARACTER(len=32) :: time_name, timeb_name,time_longname, timeb_longname, cart_name
     CHARACTER(len=256) :: fname
+    CHARACTER(len=24) :: start_date
     TYPE(domain1d) :: domain
     TYPE(domain2d) :: domain2
 
@@ -1549,7 +1648,7 @@ SUBROUTINE opening_file(file, time)
     ! it's unlikely that a file starts with word "rregion", need to check anyway.
     IF ( LEN(files(file)%name) >=7 .AND. .NOT.files(file)%local ) THEN
        prefix = files(file)%name(1:7)
-       IF ( lowercase(prefix) == 'rregion' ) THEN 
+       IF ( lowercase(prefix) == 'rregion' ) THEN
           ! 
           !   file name should not start with word "rregion"
           ! 
@@ -1557,7 +1656,7 @@ SUBROUTINE opening_file(file, time)
                & 'file name should not start with word "rregion"', WARNING)
        END IF
     END IF
-    
+
     ! Here is where time_units string must be set up; time since base date
     WRITE (time_units, 11) TRIM(time_unit_list(files(file)%time_units)), base_year,&
          & base_month, base_day, base_hour, base_minute, base_second
@@ -1572,18 +1671,23 @@ SUBROUTINE opening_file(file, time)
           !   filename  does not contain % for time stamp string
           ! 
           CALL error_mesg('diag_util_mod::opening_file',&
-               & 'file name '//TRIM(files(file)%name)//' does not contain % for time stamp string', FATAL) 
+               & 'file name '//TRIM(files(file)%name)//' does not contain % for time stamp string', FATAL)
        END IF
        suffix = get_time_string(files(file)%name, time)
     ELSE
        suffix = ' '
     END IF
+
+    ! Add ensemble ID to filename
+    fname=base_name
+    call get_instance_filename(fname, base_name)
+
     ! Add CVS tag as prefix of filename  (currently not implemented)
     !  i1 = INDEX(tagname,':') + 2
     !  i2 = len_trim(tagname) - 2
     !  if(i2 <=i1)  call error_mesg('diag_util opening_file','error in CVS tagname index',FATAL)
     !  prefix2 = tagname(i1:i2)//'_'
-    IF ( files(file)%local ) THEN      
+    IF ( files(file)%local ) THEN
        ! prepend "rregion" to all local files for post processing, the prefix will be removed in postprocessing
        filename = 'rregion'//TRIM(base_name)//TRIM(suffix)
     ELSE
@@ -1591,6 +1695,14 @@ SUBROUTINE opening_file(file, time)
        filename = TRIM(base_name)//TRIM(suffix)
     END IF
 
+    ! prepend the file start date if prepend_date == .TRUE.
+    IF ( prepend_date ) THEN
+       call get_date(diag_init_time, year, month, day, hour, minute, second)
+       write (start_date, '(1I20.4, 2I2.2)') year, month, day
+
+       filename = TRIM(adjustl(start_date))//'.'//TRIM(filename)
+    END IF
+
     ! Loop through all fields with this file to output axes
     ! JWD: This is a klooge; need something more robust
     domain2 = NULL_DOMAIN2D
@@ -1624,9 +1736,15 @@ SUBROUTINE opening_file(file, time)
        END IF
     END IF
 
-    CALL diag_output_init(filename, files(file)%format, global_descriptor,&
-         & files(file)%file_unit, all_scalar_or_1d, domain2) 
-    files(file)%bytes_written = 0 
+    IF ( _ALLOCATED(files(file)%attributes) ) THEN
+       CALL diag_output_init(filename, files(file)%format, global_descriptor,&
+            & files(file)%file_unit, all_scalar_or_1d, domain2,&
+            & attributes=files(file)%attributes(1:files(file)%num_attributes))
+    ELSE
+       CALL diag_output_init(filename, files(file)%format, global_descriptor,&
+            & files(file)%file_unit, all_scalar_or_1d, domain2)
+    END IF
+    files(file)%bytes_written = 0
     ! Does this file contain time_average fields?
     time_ops = .FALSE.
     DO j = 1, files(file)%num_fields
@@ -1649,7 +1767,7 @@ SUBROUTINE opening_file(file, time)
              !   NOT registered
              ! 
              CALL error_mesg('diag_util_mod::opening_file',&
-                  & 'module/field_name ('//TRIM(error_string)//') NOT registered', WARNING)  
+                  & 'module/field_name ('//TRIM(error_string)//') NOT registered', WARNING)
           END IF
           CYCLE
        END IF
@@ -1684,7 +1802,7 @@ SUBROUTINE opening_file(file, time)
        CALL write_axis_meta_data(files(file)%file_unit, axes(1:num_axes + 1), time_ops)
        IF ( time_ops ) THEN
           axes(num_axes + 2) = files(file)%time_bounds_id
-          CALL write_axis_meta_data(files(file)%file_unit, axes(1:num_axes + 2))     
+          CALL write_axis_meta_data(files(file)%file_unit, axes(1:num_axes + 2))
        END IF
     END DO
 
@@ -1721,7 +1839,7 @@ SUBROUTINE opening_file(file, time)
        ! check if any field has the same name as aux_name
        IF ( aux_present .AND. .NOT.match_aux_name ) THEN
           fieldname = output_fields(field_num)%output_name
-          IF ( INDEX(aux_name, TRIM(fieldname)) > 0 ) match_aux_name = .TRUE.   
+          IF ( INDEX(aux_name, TRIM(fieldname)) > 0 ) match_aux_name = .TRUE.
        END IF
 
        ! Put the time axis in the axis field
@@ -1750,7 +1868,9 @@ SUBROUTINE opening_file(file, time)
                   & input_fields(input_field_num)%missing_value, avg_name = avg,&
                   & time_method=output_fields(field_num)%time_method,&
                   & standard_name = input_fields(input_field_num)%standard_name,&
-                  & interp_method = input_fields(input_field_num)%interp_method)
+                  & interp_method = input_fields(input_field_num)%interp_method,&
+                  & attributes=output_fields(field_num)%attributes,&
+                  & num_attributes=output_fields(field_num)%num_attributes)
           ELSE
              output_fields(field_num)%f_type = write_field_meta_data(files(file)%file_unit,&
                   & output_fields(field_num)%output_name, axes(1:num_axes),&
@@ -1759,7 +1879,9 @@ SUBROUTINE opening_file(file, time)
                   & input_fields(input_field_num)%range, output_fields(field_num)%pack,&
                   & input_fields(input_field_num)%missing_value, avg_name = avg,&
                   & time_method=output_fields(field_num)%time_method,&
-                  & standard_name = input_fields(input_field_num)%standard_name)
+                  & standard_name = input_fields(input_field_num)%standard_name,&
+                  & attributes=output_fields(field_num)%attributes,&
+                  & num_attributes=output_fields(field_num)%num_attributes)
           END IF
           ! NEED TO TAKE CARE OF TIME AVERAGING INFO TOO BOTH CASES
        ELSE
@@ -1772,7 +1894,9 @@ SUBROUTINE opening_file(file, time)
                   & avg_name = avg,&
                   & time_method=output_fields(field_num)%time_method,&
                   & standard_name = input_fields(input_field_num)%standard_name,&
-                  & interp_method = input_fields(input_field_num)%interp_method)
+                  & interp_method = input_fields(input_field_num)%interp_method,&
+                  & attributes=output_fields(field_num)%attributes,&
+                  & num_attributes=output_fields(field_num)%num_attributes)
           ELSE
              output_fields(field_num)%f_type = write_field_meta_data(files(file)%file_unit,&
                   & output_fields(field_num)%output_name, axes(1:num_axes),&
@@ -1781,7 +1905,9 @@ SUBROUTINE opening_file(file, time)
                   & input_fields(input_field_num)%range, output_fields(field_num)%pack,&
                   & avg_name = avg,&
                   & time_method=output_fields(field_num)%time_method,&
-                  & standard_name = input_fields(input_field_num)%standard_name)
+                  & standard_name = input_fields(input_field_num)%standard_name,&
+                  & attributes=output_fields(field_num)%attributes,&
+                  & num_attributes=output_fields(field_num)%num_attributes)
           END IF
        END IF
     END DO
@@ -1798,7 +1924,7 @@ SUBROUTINE opening_file(file, time)
             & "End time for average period", pack=pack_size)
        files(file)%f_avg_nitems = write_field_meta_data(files(file)%file_unit,&
             & avg_name // '_DT', time_axis_id,&
-            & TRIM(time_unit_list(files(file)%time_units)),& 
+            & TRIM(time_unit_list(files(file)%time_units)),&
             & "Length of average period", pack=pack_size)
     END IF
 
@@ -1808,11 +1934,11 @@ SUBROUTINE opening_file(file, time)
        CALL get_diag_axis( time_axis_id(1), time_name, time_units, time_longname,&
             & cart_name, dir, edges, Domain, DATA)
        CALL get_diag_axis( time_bounds_id(1), timeb_name, timeb_units, timeb_longname,&
-            & cart_name, dir, edges, Domain, DATA)     
+            & cart_name, dir, edges, Domain, DATA)
        files(file)%f_bounds =  write_field_meta_data(files(file)%file_unit,&
             & TRIM(time_name)//'_bounds', (/time_bounds_id,time_axis_id/),&
             & TRIM(time_unit_list(files(file)%time_units)),&
-            & TRIM(time_name)//' axis boundaries', pack=pack_size)      
+            & TRIM(time_name)//' axis boundaries', pack=pack_size)
     END IF
     ! Let lower levels know that all meta data has been sent
     CALL done_meta_data(files(file)%file_unit)
@@ -1827,7 +1953,7 @@ SUBROUTINE opening_file(file, time)
   END SUBROUTINE opening_file
   ! 
   ! 
-  
+
   ! 
   ! 
   !   
@@ -1863,31 +1989,31 @@ CHARACTER(len=128) FUNCTION get_time_string(filename, current_time)
     len = LEN_TRIM(filename)
     first_percent = INDEX(filename, '%')
     filetail = filename(first_percent:len)
-    ! compute year string 
+    ! compute year string
     position = INDEX(filetail, 'yr')
     IF ( position > 0 ) THEN
        width = filetail(position-1:position-1)
        yr1_s = yr1
        format(7:9) = width//'.'//width
-       WRITE(yr, format) yr1_s   
+       WRITE(yr, format) yr1_s
        yr2 = 0
-    ELSE  
+    ELSE
        yr = ' '
        yr2 = yr1 - 1
     END IF
-    ! compute month string 
+    ! compute month string
     position = INDEX(filetail, 'mo')
-    IF ( position > 0 ) THEN   
+    IF ( position > 0 ) THEN
        width = filetail(position-1:position-1)
-       mo1_s = yr2*12 + mo1  
+       mo1_s = yr2*12 + mo1
        format(7:9) = width//'.'//width
        WRITE(mo, format) mo1_s
     ELSE
        mo = ' '
     END IF
-    ! compute day string        
+    ! compute day string
     IF ( LEN_TRIM(mo) > 0 ) THEN ! month present
-       dy1_s = dy1 
+       dy1_s = dy1
        dy2 = dy1_s - 1
     ELSE IF ( LEN_TRIM(yr) >0 )  THEN ! no month, year present
        ! compute julian day
@@ -1905,11 +2031,11 @@ CHARACTER(len=128) FUNCTION get_time_string(filename, current_time)
        dy2 = dy1_s - 1
     ELSE ! no month, no year
        CALL get_time(current_time, abs_sec, abs_day)
-       dy1_s = abs_day  
-       dy2 = dy1_s 
+       dy1_s = abs_day
+       dy2 = dy1_s
     END IF
     position = INDEX(filetail, 'dy')
-    IF ( position > 0 ) THEN 
+    IF ( position > 0 ) THEN
        width = filetail(position-1:position-1)
        FORMAT(7:9) = width//'.'//width
        WRITE(dy, FORMAT) dy1_s
@@ -2064,7 +2190,7 @@ SUBROUTINE diag_data_out(file, field, dat, time, final_call_in, static_write_in)
          & (SIZE(dat,1)*SIZE(dat,2)*SIZE(dat,3))*(8/output_fields(field)%pack)
     IF ( .NOT.output_fields(field)%written_once ) output_fields(field)%written_once = .TRUE.
     ! *** inserted this line because start_dif < 0 for static fields ***
-    IF ( .NOT.output_fields(field)%static ) THEN 
+    IF ( .NOT.output_fields(field)%static ) THEN
        start_dif = get_date_dif(output_fields(field)%last_output, base_time,files(file)%time_units)
        IF ( .NOT.mix_snapshot_average_fields ) THEN
           end_dif = get_date_dif(output_fields(field)%next_output, base_time, files(file)%time_units)
@@ -2089,7 +2215,7 @@ SUBROUTINE diag_data_out(file, field, dat, time, final_call_in, static_write_in)
              CALL diag_field_out(files(file)%file_unit, files(file)%f_avg_nitems, dt_time(1:1,:,:,:), dif)
 
              ! Include boundary variable for CF compliance
-             CALL diag_field_out(files(file)%file_unit, files(file)%f_bounds, time_data(1:2,:,:,:), dif)         
+             CALL diag_field_out(files(file)%file_unit, files(file)%f_bounds, time_data(1:2,:,:,:), dif)
              EXIT
           END IF
        END IF
@@ -2131,7 +2257,7 @@ SUBROUTINE check_and_open(file, time, do_write)
     TYPE(time_type), INTENT(in) :: time
     LOGICAL, INTENT(out) :: do_write
 
-    IF ( time >= files(file)%start_time ) THEN 
+    IF ( time >= files(file)%start_time ) THEN
        IF ( files(file)%file_unit < 0 ) THEN ! need to open a new file
           CALL opening_file(file, time)
           do_write = .TRUE.
@@ -2139,16 +2265,16 @@ SUBROUTINE check_and_open(file, time, do_write)
           do_write = .TRUE.
           IF ( time > files(file)%close_time .AND. time < files(file)%next_open ) THEN
              do_write = .FALSE. ! file still open but receives NO MORE data
-          ELSE IF ( time > files(file)%next_open ) THEN ! need to close current file and open a new one 
+          ELSE IF ( time > files(file)%next_open ) THEN ! need to close current file and open a new one
              CALL write_static(file)  ! write all static fields and close this file
-             CALL opening_file(file, time)        
+             CALL opening_file(file, time)
              files(file)%start_time = files(file)%next_open
              files(file)%close_time =&
-                  & diag_time_inc(files(file)%start_time,files(file)%duration, files(file)%duration_units)  
+                  & diag_time_inc(files(file)%start_time,files(file)%duration, files(file)%duration_units)
              files(file)%next_open =&
                   & diag_time_inc(files(file)%next_open, files(file)%new_file_freq,&
                   & files(file)%new_file_freq_units)
-             IF ( files(file)%close_time > files(file)%next_open ) THEN 
+             IF ( files(file)%close_time > files(file)%next_open ) THEN
                 ! 
                 !    has close time GREATER than next_open time,
                 !   check file duration and frequency
@@ -2191,7 +2317,7 @@ SUBROUTINE write_static(file)
        IF ( .NOT.output_fields(i)%static ) CYCLE
        CALL diag_data_out(file, i, output_fields(i)%buffer, files(file)%last_flush, .TRUE., .TRUE.)
     END DO
-    ! Close up this file   
+    ! Close up this file
     CALL mpp_close(files(file)%file_unit)
     files(file)%file_unit = -1
   END SUBROUTINE write_static
@@ -2218,7 +2344,7 @@ SUBROUTINE check_duplicate_output_fields(err_msg)
 
     IF ( PRESENT(err_msg) ) err_msg=''
     ! Do the checking when more than 1 output_fileds present
-    IF ( num_output_fields <= 1 ) RETURN 
+    IF ( num_output_fields <= 1 ) RETURN
     err_msg_local = ''
 
     i_loop: DO i = 1, num_output_fields-1
@@ -2238,4 +2364,293 @@ SUBROUTINE check_duplicate_output_fields(err_msg)
     END IF
   END SUBROUTINE check_duplicate_output_fields
   ! 
+
+  ! 
+  !   
+  !     Allocates the atttype in out_field
+  !   
+  !   
+  !   
+  !     Allocates memory in out_field for the attributes.  Will FATAL if err_msg is not included
+  !     in the subroutine call.
+  !   
+  !   output field to allocate memory for attribute
+  !   Error message, passed back to calling function
+  SUBROUTINE attribute_init_field(out_field, err_msg)
+    TYPE(output_field_type), INTENT(inout) :: out_field
+    CHARACTER(LEN=*), INTENT(out), OPTIONAL :: err_msg
+
+    INTEGER :: istat
+
+    ! Need to initialize err_msg if present
+    IF ( PRESENT(err_msg) ) err_msg = ''
+
+    ! Allocate memory for the attributes
+    IF ( .NOT._ALLOCATED(out_field%attributes) ) THEN
+       ALLOCATE(out_field%attributes(max_field_attributes), STAT=istat)
+       IF ( istat.NE.0 ) THEN
+          ! 
+          !   Unable to allocate memory for attribute  to module/input_field /
+          ! 
+          IF ( fms_error_handler('diag_util_mod::attribute_init_field',&
+               & 'Unable to allocate memory for attributes', err_msg) ) THEN
+             RETURN
+          END IF
+       ELSE
+          ! Set equal to 0.  It will be increased when attributes added
+          out_field%num_attributes = 0
+       END IF
+    END IF
+  END SUBROUTINE attribute_init_field
+  ! 
+
+  ! 
+  !   
+  !     Prepends the attribute value to an already existing attribute.  If the
+  !     attribute isn't yet defined, then creates a new attribute
+  !   
+  !   
+  !   
+  !     Checks if the attribute already exists in the out_field.  If it exists,
+  !     then prepend the prepend_value, otherwise, create the attribute
+  !     with the prepend_value. err_msg indicates no duplicates found.
+  !   
+  !   output field that will get the attribute
+  !   Name of the attribute
+  !   Value to prepend
+  !   Error message, passed back to calling routine
+  SUBROUTINE prepend_attribute_field(out_field, att_name, prepend_value, err_msg)
+    TYPE(output_field_type), INTENT(inout) :: out_field
+    CHARACTER(len=*), INTENT(in) :: att_name, prepend_value
+    CHARACTER(len=*), INTENT(out) , OPTIONAL :: err_msg
+
+    INTEGER :: length, i, this_attribute
+    CHARACTER(len=512) :: err_msg_local
+
+    ! Initialize string characters
+    err_msg_local=''
+    IF ( PRESENT(err_msg) ) err_msg = ''
+
+    ! Make sure the attributes for this out field have been initialized
+    CALL attribute_init_field(out_field, err_msg_local)
+    IF ( TRIM(err_msg_local) .NE. '' ) THEN
+       IF ( fms_error_handler('diag_util_mod::prepend_attribute_field', TRIM(err_msg_local), err_msg) ) THEN
+          RETURN
+       END IF
+    END IF
+
+    ! Find if attribute exists
+    this_attribute = 0
+    DO i=1, out_field%num_attributes
+       IF ( TRIM(out_field%attributes(i)%name) .EQ. TRIM(att_name) ) THEN
+          this_attribute = i
+          EXIT
+       END IF
+    END DO
+
+    IF ( this_attribute > 0 ) THEN
+       IF ( out_field%attributes(this_attribute)%type .NE. NF90_CHAR ) THEN
+          ! 
+          !   Attribute  is not a character attribute.
+          ! 
+          IF ( fms_error_handler('diag_util_mod::prepend_attribute_field', &
+               & 'Attribute "'//TRIM(att_name)//'" is not a character attribute.',&
+               & err_msg) ) THEN
+             RETURN
+          END IF
+       END IF
+    ELSE
+       ! Defining a new attribute
+       ! Increase the number of field attributes
+       this_attribute = out_field%num_attributes + 1
+       IF ( this_attribute .GT. max_field_attributes ) THEN
+          ! 
+          !   Number of attributes exceeds max_field_attributes for attribute .
+          !   Increase diag_manager_nml:max_field_attributes.
+          ! 
+          IF ( fms_error_handler('diag_util_mod::prepend_attribute_field',&
+               & 'Number of attributes exceeds max_field_attributes for attribute "'&
+               & //TRIM(att_name)//'".  Increase diag_manager_nml:max_field_attributes.',&
+               & err_msg) ) THEN
+             RETURN
+          END IF
+       ELSE
+          out_field%num_attributes = this_attribute
+          ! Set name and type
+          out_field%attributes(this_attribute)%name = att_name
+          out_field%attributes(this_attribute)%type = NF90_CHAR
+          ! Initialize catt to a blank string, as len_trim doesn't always work on an uninitialized string
+          out_field%attributes(this_attribute)%catt = ''
+       END IF
+    END IF
+
+    ! Check if string is already included, and return if found
+    IF ( INDEX(TRIM(out_field%attributes(this_attribute)%catt), TRIM(prepend_value)).EQ.0 ) THEN
+       ! Check if new string length goes beyond the length of catt
+       length = LEN_TRIM(TRIM(prepend_value)//" "//TRIM(out_field%attributes(this_attribute)%catt))
+       IF ( length.GT.LEN(out_field%attributes(this_attribute)%catt) ) THEN
+          ! 
+          !   Prepend length for attribute  is longer than allowed.
+          ! 
+          IF ( fms_error_handler('diag_util_mod::prepend_attribute_field',&
+               & 'Prepend length for attribute "'//TRIM(att_name)//'" is longer than allowed.',&
+               & err_msg) ) THEN
+             RETURN
+          END IF
+       END IF
+       ! Set fields
+       out_field%attributes(this_attribute)%catt =&
+            & TRIM(prepend_value)//' '//TRIM(out_field%attributes(this_attribute)%catt)
+       out_field%attributes(this_attribute)%len = length
+    END IF
+  END SUBROUTINE prepend_attribute_field
+  ! 
+
+  ! 
+  !   
+  !     Allocates the atttype in out_file
+  !   
+  !   
+  !   
+  !     Allocates memory in out_file for the attributes.  Will FATAL if err_msg is not included
+  !     in the subroutine call.
+  !   
+  !   output file to allocate memory for attribute
+  !   Error message, passed back to calling function
+  SUBROUTINE attribute_init_file(out_file, err_msg)
+    TYPE(file_type), INTENT(inout) :: out_file
+    CHARACTER(LEN=*), INTENT(out), OPTIONAL :: err_msg
+
+    INTEGER :: istat
+
+    ! Initialize err_msg
+    IF ( PRESENT(err_msg) ) err_msg = ''
+
+    ! Allocate memory for the attributes
+    IF ( .NOT._ALLOCATED(out_file%attributes) ) THEN
+       ALLOCATE(out_file%attributes(max_field_attributes), STAT=istat)
+       IF ( istat.NE.0 ) THEN
+          ! 
+          !   Unable to allocate memory for file attributes
+          ! 
+          IF ( fms_error_handler('diag_util_mod::attribute_init_file', 'Unable to allocate memory for file attributes', err_msg) ) THEN
+             RETURN
+          END IF
+       ELSE
+          ! Set equal to 0.  It will be increased when attributes added
+          out_file%num_attributes = 0
+       END IF
+    END IF
+  END SUBROUTINE attribute_init_file
+  ! 
+
+  ! 
+  !   
+  !     Prepends the attribute value to an already existing attribute.  If the
+  !     attribute isn't yet defined, then creates a new attribute
+  !   
+  !   
+  !   
+  !     Checks if the attribute already exists in the out_file.  If it exists,
+  !     then prepend the prepend_value, otherwise, create the attribute
+  !     with the prepend_value. err_msg indicates no duplicates found.
+  !   
+  !   output file that will get the attribute
+  !   Name of the attribute
+  !   Value to prepend
+  !   Error message, passed back to calling routine
+  SUBROUTINE prepend_attribute_file(out_file, att_name, prepend_value, err_msg)
+    TYPE(file_type), INTENT(inout) :: out_file
+    CHARACTER(len=*), INTENT(in) :: att_name, prepend_value
+    CHARACTER(len=*), INTENT(out) , OPTIONAL :: err_msg
+
+    INTEGER :: length, i, this_attribute
+    CHARACTER(len=512) :: err_msg_local
+
+    ! Initialize string variables
+    err_msg_local = ''
+    IF ( PRESENT(err_msg) ) err_msg = ''
+
+    ! Make sure the attributes for this out file have been initialized
+    CALL attribute_init_file(out_file, err_msg_local)
+    IF ( TRIM(err_msg_local) .NE. '' ) THEN
+       IF ( fms_error_handler('diag_util_mod::prepend_attribute_file', TRIM(err_msg_local), err_msg) ) THEN
+          RETURN
+       END IF
+    END IF
+
+    ! Find if attribute exists
+    this_attribute = 0
+    DO i=1, out_file%num_attributes
+       IF ( TRIM(out_file%attributes(i)%name) .EQ. TRIM(att_name) ) THEN
+          this_attribute = i
+          EXIT
+       END IF
+    END DO
+
+    IF ( this_attribute > 0 ) THEN
+       IF ( out_file%attributes(this_attribute)%type .NE. NF90_CHAR ) THEN
+          ! 
+          !   Attribute  is not a character attribute.
+          ! 
+          IF ( fms_error_handler('diag_util_mod::prepend_attribute_file',&
+               & 'Attribute "'//TRIM(att_name)//'" is not a character attribute.',&
+               & err_msg) ) THEN
+             RETURN
+          END IF
+       END IF
+    ELSE
+       ! Defining a new attribute
+       ! Increase the number of file attributes
+       this_attribute = out_file%num_attributes + 1
+       IF ( this_attribute .GT. max_file_attributes ) THEN
+          ! 
+          !   Number of attributes exceeds max_file_attributes for attribute .
+          !   Increase diag_manager_nml:max_file_attributes.
+          ! 
+          IF ( fms_error_handler('diag_util_mod::prepend_attribute_file',&
+               & 'Number of attributes exceeds max_file_attributes for attribute "'&
+               &//TRIM(att_name)//'".  Increase diag_manager_nml:max_file_attributes.',&
+               & err_msg) ) THEN
+             RETURN
+          END IF
+       ELSE
+          out_file%num_attributes = this_attribute
+          ! Set name and type
+          out_file%attributes(this_attribute)%name = att_name
+          out_file%attributes(this_attribute)%type = NF90_CHAR
+          ! Initialize catt to a blank string, as len_trim doesn't always work on an uninitialized string
+          out_file%attributes(this_attribute)%catt = ''
+       END IF
+    END IF
+
+    ! Only add string only if not already defined
+    IF ( INDEX(TRIM(out_file%attributes(this_attribute)%catt), TRIM(prepend_value)).EQ.0 ) THEN
+       ! Check if new string length goes beyond the length of catt
+       length = LEN_TRIM(TRIM(prepend_value)//" "//TRIM(out_file%attributes(this_attribute)%catt))
+       IF ( length.GT.LEN(out_file%attributes(this_attribute)%catt) ) THEN
+          ! 
+          !   Prepend length for attribute  is longer than allowed.
+          ! 
+          IF ( fms_error_handler('diag_util_mod::prepend_attribute_file',&
+               & 'Prepend length for attribute "'//TRIM(att_name)//'" is longer than allowed.',&
+               & err_msg) ) THEN
+             RETURN
+          END IF
+       END IF
+       ! Set files
+       out_file%attributes(this_attribute)%catt =&
+            & TRIM(prepend_value)//' '//TRIM(out_file%attributes(this_attribute)%catt)
+       out_file%attributes(this_attribute)%len = length
+    END IF
+  END SUBROUTINE prepend_attribute_file
+  ! 
 END MODULE diag_util_mod
diff --git a/src/shared/drifters/cloud_interpolator.F90 b/src/shared/drifters/cloud_interpolator.F90
index 82a0428635..05f8cf74c8 100644
--- a/src/shared/drifters/cloud_interpolator.F90
+++ b/src/shared/drifters/cloud_interpolator.F90
@@ -13,8 +13,8 @@ MODULE cloud_interpolator_mod
   public :: cld_ntrp_expand_index, cld_ntrp_contract_indices
 #endif
 
-character(128), parameter :: version = '$Id: cloud_interpolator.F90,v 14.0 2007/03/15 22:38:35 fms Exp $'
-real, parameter           :: tol = 10*epsilon(1.)
+character(128), parameter :: version = '$Id$'
+real, parameter           :: tol = 10.0*epsilon(1.)
 
 CONTAINS
 
@@ -77,7 +77,7 @@ _PURE subroutine cld_ntrp_linear_cell_interp(fvals, ts, f, ier)
     real    basis
 
     ier = 0
-    f   = 0
+    f   = 0.
     nd   = size(ts)
     if(size(fvals) /= 2**nd) then
        ier = 1
@@ -85,10 +85,10 @@ _PURE subroutine cld_ntrp_linear_cell_interp(fvals, ts, f, ier)
     endif
     
     do Ic = 0, 2**nd - 1
-       basis = 1
+       basis = 1.
        call cld_ntrp_expand_index(Ic, ie, iflag)
        do j = 1, nd
-          basis = basis * (  (1-ie(j))*(1.0-ts(j)) + ie(j)*ts(j) )
+          basis = basis * (  (1.0-real(ie(j)))*(1.0-ts(j)) + real(ie(j))*ts(j) )
        end do
        f = f + fvals(Ic)*basis
     end do
@@ -133,7 +133,7 @@ _PURE subroutine cld_ntrp_locate_cell(axis, x, index, ier)
        return
     endif
 
-    index = floor((n-1)*(x - axis_1)/(axis_n-axis_1)) + 1
+    index = floor(real(n-1)*(x - axis_1)/(axis_n-axis_1)) + 1
     index  = min(n-1, index)
     index1 = index+1
 
@@ -221,7 +221,7 @@ _PURE subroutine cld_ntrp_get_cell_values(nsizes, fnodes, indices, fvals, ier)
     integer id, nt, nd, flat_index, Ic, iflag
     integer, dimension(size(nsizes)) :: cell_indices, node_indices
     ier = 0
-    fvals = 0
+    fvals = 0.
 
     nd = size(nsizes)
     if(nd /= size(indices)) then
diff --git a/src/shared/drifters/drifters.F90 b/src/shared/drifters/drifters.F90
index bc40760037..a5b1168c76 100644
--- a/src/shared/drifters/drifters.F90
+++ b/src/shared/drifters/drifters.F90
@@ -1,17 +1,18 @@
 !FDOC_TAG_GFDL fdoc.pl generated xml skeleton
-! $Id: drifters.F90,v 20.0 2013/12/14 00:19:02 fms Exp $
+! $Id$
 
 #include 
 #include "fms_switches.h"
 #define _FLATTEN(A) reshape((A), (/size((A))/) )
 
 module drifters_mod
-! 
+! 
 !   Alexander Pletzer
 ! 
 ! 
 !   
 ! 
+! 
 ! 
 !   
 ! 
@@ -109,7 +110,7 @@ module drifters_mod
   public :: drifters_print_checksums, drifters_save, drifters_write_restart, drifters_distribute
 
   integer, parameter, private :: MAX_STR_LEN = 128
-  character(len=MAX_STR_LEN), parameter, private :: version = '$Id: drifters.F90,v 20.0 2013/12/14 00:19:02 fms Exp $'
+  character(len=MAX_STR_LEN), parameter, private :: version = '$Id$'
   real :: DRFT_EMPTY_ARRAY(0)
 
   type drifters_type
diff --git a/src/shared/drifters/drifters_combine b/src/shared/drifters/drifters_combine
index 297effcc1b..f2ef5e6e0f 100755
--- a/src/shared/drifters/drifters_combine
+++ b/src/shared/drifters/drifters_combine
@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 
-VERSION = "$Id: drifters_combine,v 13.0 2006/03/28 21:38:16 fms Exp $"
+VERSION = "$Id$"
 
 from Scientific.IO.NetCDF import NetCDFFile
 import sys
diff --git a/src/shared/drifters/drifters_comm.F90 b/src/shared/drifters/drifters_comm.F90
index c60dec4c02..0a7d674e2d 100644
--- a/src/shared/drifters/drifters_comm.F90
+++ b/src/shared/drifters/drifters_comm.F90
@@ -1,7 +1,7 @@
 #include 
 #include "fms_switches.h"
 
-! $Id: drifters_comm.F90,v 20.0 2013/12/14 00:19:06 fms Exp $
+! $Id$
 
 module drifters_comm_mod
 
@@ -834,7 +834,7 @@ program main
      
      if(drfts%np > 0) then 
         do i=1,drfts%np
-           print '(a,i2,a,i3,a,i3,a, i3, a,2f10.6)', 'PE: ',pe, ' it=', it, ' np=', drfts%np, ' ip=', i, &
+           print '(a,i6,a,i3,a,i3,a, i3, a,2f10.6)', 'PE: ',pe, ' it=', it, ' np=', drfts%np, ' ip=', i, &
                 & ' x,y=', drfts%positions(1,i), drfts%positions(2,i)
         enddo
      endif
diff --git a/src/shared/drifters/drifters_compute_k.h b/src/shared/drifters/drifters_compute_k.h
index 084426cb79..90842efb54 100644
--- a/src/shared/drifters/drifters_compute_k.h
+++ b/src/shared/drifters/drifters_compute_k.h
@@ -1,5 +1,5 @@
 ! -*-f90-*-
-! $Id: drifters_compute_k.h,v 13.0 2006/03/28 21:38:20 fms Exp $
+! $Id$
 
 subroutine drifters_compute_k_XXX(self, positions, u, v, &
 #if _DIMS >= 3
diff --git a/src/shared/drifters/drifters_core.F90 b/src/shared/drifters/drifters_core.F90
index 4c3ac36594..d59699f2a9 100644
--- a/src/shared/drifters/drifters_core.F90
+++ b/src/shared/drifters/drifters_core.F90
@@ -1,4 +1,4 @@
-! $Id: drifters_core.F90,v 14.0 2007/03/15 22:38:50 fms Exp $
+! $Id$
 !
 ! nf95 -r8 -g -I ~/regression/ia64/23-Jun-2005/CM2.1U_Control-1990_E1.k32pe/include/ -D_TEST_DRIFTERS -D_F95 quicksort.F90 drifters_core.F90
 
@@ -16,7 +16,7 @@ module drifters_core_mod
 
   ! Globals
   integer, parameter, private   :: MAX_STR_LEN = 128
-  character(MAX_STR_LEN), parameter, private :: version = '$Id: drifters_core.F90,v 14.0 2007/03/15 22:38:50 fms Exp $'
+  character(MAX_STR_LEN), parameter, private :: version = '$Id$'
 
   type drifters_core_type
      ! Be sure to update drifters_core_new, drifters_core_del and drifters_core_copy_new
@@ -49,7 +49,7 @@ subroutine drifters_core_new(self, nd, npdim, ermesg)
 
     allocate(self%positions(nd, npdim), stat=iflag)
     if(iflag/=0) ier = ier + 1
-    self%positions   = 0
+    self%positions   = 0.
 
     allocate(self%ids(npdim), stat=iflag)
     if(iflag/=0) ier = ier + 1
@@ -129,7 +129,7 @@ subroutine drifters_core_resize(self, npdim, ermesg)
 
     allocate(self%positions(self%nd, npdim), stat=iflag)
     allocate(self%ids(npdim), stat=iflag)
-    self%positions = 0
+    self%positions = 0.0
     ! default id numbers
     self%ids       = (/ (i, i=1,npdim) /)
     self%positions(:, 1:self%np) = positions
diff --git a/src/shared/drifters/drifters_input.F90 b/src/shared/drifters/drifters_input.F90
index 1868d55b67..ea3e2a73bf 100644
--- a/src/shared/drifters/drifters_input.F90
+++ b/src/shared/drifters/drifters_input.F90
@@ -1,4 +1,4 @@
-! $Id: drifters_input.F90,v 14.0 2007/03/15 22:38:53 fms Exp $
+! $Id$
 
 #include 
 
@@ -11,7 +11,7 @@ module drifters_input_mod
 
   ! Globals
   integer, parameter, private   :: MAX_STR_LEN = 128
-  character(MAX_STR_LEN), parameter, private :: version = '$Id: drifters_input.F90,v 14.0 2007/03/15 22:38:53 fms Exp $'
+  character(MAX_STR_LEN), parameter, private :: version = '$Id$'
   character, parameter, private :: SEPARATOR = ' '
 
   type drifters_input_type
diff --git a/src/shared/drifters/drifters_io.F90 b/src/shared/drifters/drifters_io.F90
index f120bdf52c..7002a04366 100644
--- a/src/shared/drifters/drifters_io.F90
+++ b/src/shared/drifters/drifters_io.F90
@@ -1,4 +1,4 @@
-! $Id: drifters_io.F90,v 14.0 2007/03/15 22:38:56 fms Exp $
+! $Id$
 
 !!#include 
 
@@ -13,9 +13,9 @@ module drifters_io_mod
   ! Globals
   integer, parameter, private   :: MAX_STR_LEN = 128
   character(MAX_STR_LEN), parameter, private :: &
-       & version = '$Id: drifters_io.F90,v 14.0 2007/03/15 22:38:56 fms Exp $'
+       & version = '$Id$'
 
-  real :: drfts_eps_t = 10*epsilon(1.)
+  real :: drfts_eps_t = 10.*epsilon(1.)
   
 
   type drifters_io_type
diff --git a/src/shared/drifters/drifters_push.h b/src/shared/drifters/drifters_push.h
index ddd2f0e031..d7b51c36a7 100644
--- a/src/shared/drifters/drifters_push.h
+++ b/src/shared/drifters/drifters_push.h
@@ -1,5 +1,5 @@
 ! -*-f90-*-
-! $Id: drifters_push.h,v 13.0 2006/03/28 21:38:35 fms Exp $
+! $Id$
 !============================================================================
 subroutine drifters_push_XXX(self, u, v, &
 #if _DIMS >= 3
diff --git a/src/shared/drifters/drifters_set_field.h b/src/shared/drifters/drifters_set_field.h
index 4c79c9e202..2e388d8e5b 100644
--- a/src/shared/drifters/drifters_set_field.h
+++ b/src/shared/drifters/drifters_set_field.h
@@ -1,5 +1,5 @@
 ! -*-f90-*-
-! $Id: drifters_set_field.h,v 13.0 2006/03/28 21:38:37 fms Exp $
+! $Id$
 
 subroutine drifters_set_field_XXX(self, index_field, x, y, &
 #if _DIMS >= 3
diff --git a/src/shared/drifters/readme.txt b/src/shared/drifters/readme.txt
index d4e3484e72..b51dca7824 100644
--- a/src/shared/drifters/readme.txt
+++ b/src/shared/drifters/readme.txt
@@ -1,9 +1,9 @@
 Drifters
 ========
 
-$Id: readme.txt,v 13.0 2006/03/28 21:38:43 fms Exp $
+$Id$
 
-Alexander.Pletzer
+Alexander.Pletzer@noaa.gov
 
 Overview:
 ---------
diff --git a/src/shared/exchange/stock_constants.F90 b/src/shared/exchange/stock_constants.F90
index 972c8affbf..50b63889b2 100644
--- a/src/shared/exchange/stock_constants.F90
+++ b/src/shared/exchange/stock_constants.F90
@@ -7,7 +7,7 @@ module stock_constants_mod
 
   implicit none
 
-  character(len=128), parameter :: version = '$Id: stock_constants.F90,v 17.0 2009/07/21 03:19:07 fms Exp $'
+  character(len=128), parameter :: version = '$Id$'
 
 
   integer,public,    parameter                :: NELEMS=3
diff --git a/src/shared/exchange/xgrid.F90 b/src/shared/exchange/xgrid.F90
index c1c0f6ae5a..fa1da52499 100644
--- a/src/shared/exchange/xgrid.F90
+++ b/src/shared/exchange/xgrid.F90
@@ -38,18 +38,19 @@
 !             PE.  For the make_exchange_reproduce option, a special side 1 get
 !             is used.  This get communicates individual exchange cells.  The
 !             cells are summed in the order they appear in the grid spec. file.
-!                                    Michael Winton (Michael.Winton) Oct 2001
+!                                    Michael Winton (Michael.Winton@noaa.gov) Oct 2001
 !
 !~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 module xgrid_mod
 
-! 
+! 
 !   Michael Winton
 ! 
-! 
+! 
 !   Zhi Liang
 ! 
 
+! 
 
 ! 
 !    xgrid_mod implements exchange grids for coupled models running on
@@ -117,8 +118,8 @@ module xgrid_mod
                            mpp_clock_begin, mpp_clock_end, MPP_CLOCK_SYNC,    &
                            COMM_TAG_1, COMM_TAG_2, COMM_TAG_3, COMM_TAG_4,    &
                            COMM_TAG_5, COMM_TAG_6, COMM_TAG_7, COMM_TAG_8,    &
-			   COMM_TAG_9, COMM_TAG_10
-use mpp_mod,         only: input_nml_file, mpp_set_current_pelist, mpp_sum, mpp_sync        
+                           COMM_TAG_9, COMM_TAG_10
+use mpp_mod,         only: input_nml_file, mpp_set_current_pelist, mpp_sum, mpp_sync
 use mpp_domains_mod, only: mpp_get_compute_domain, mpp_get_compute_domains, &
                            Domain2d, mpp_global_sum, mpp_update_domains,    &
                            mpp_modify_domain, mpp_get_data_domain, XUPDATE, &
@@ -175,21 +176,16 @@ module xgrid_mod
 !     information. The purpose of this namelist is to improve performance of setup_xmap when running
 !     on highr processor count and solve receiving size mismatch issue on high processor count.
 !     Try to set nsubset = mpp_npes/MPI_rank_per_node.
-!     This parameter is not used when do_alltoall and do_alltoallv are enabled.
-!   
-!   
-!    Use the MPI_Alltoall collective in place of fixed-size point-to-point
-!    operations during the initial exchange grid calculation.  This parameter
-!    removes any issues due to large numbers of messages (such as hangs) and
-!    shows improved performance at higher PE counts.  When enabled, the nsubset
-!    parameter is ignored.
+!
+!     This parameter is not used when `collective_setup` is enabled.
 !   
-!   
-!    Use the MPI_Alltoallv collective in place of variable-size point-to-point
-!    operations during the initial exchange grid calculation.  This parameter
-!    removes any issues due to large numbers of messages (such as hangs) and
-!    shows improved performance at higher PE counts.  When enabled, the nsubset
-!    parameter is ignored.
+!   
+!    Use the MPI_Alltoall collectives in place of point-to-point operations
+!    during the xgrid setup.  This parameter can resolve potential issues
+!    related to large numbers of messages, such as model hangs.  It also shows
+!    improved performance at higher PE counts.
+!
+!    When enabled, the `nsubset` parameter is ignored.
 !   
 logical :: make_exchange_reproduce = .false. ! exactly same on different # PEs
 logical :: xgrid_log = .false. 
@@ -198,10 +194,10 @@ module xgrid_mod
 logical :: xgrid_clocks_on = .false.
 logical :: monotonic_exchange = .false.
 integer :: nsubset = 0 ! 0 means mpp_npes()
-logical :: do_alltoall = .true.
-logical :: do_alltoallv = .true.
-namelist /xgrid_nml/ make_exchange_reproduce, interp_method, debug_stocks, xgrid_log, xgrid_clocks_on, &
-    monotonic_exchange, nsubset, do_alltoall, do_alltoallv
+logical :: collective_setup = .true.
+namelist /xgrid_nml/ &
+    make_exchange_reproduce, interp_method, debug_stocks, xgrid_log, &
+    xgrid_clocks_on, monotonic_exchange, nsubset, collective_setup
 ! 
 logical :: init = .true.
 integer :: remapping_method
@@ -285,6 +281,7 @@ module xgrid_mod
 
 type xcell_type
   integer :: i1, j1, i2, j2 ! indices of cell in model arrays on both sides
+  integer :: recv_pos       ! position in the receive buffer.
   integer :: pe             ! other side pe that has this cell
   integer :: tile           ! tile index of side 1 mosaic.
   real    :: area           ! geographic area of exchange cell
@@ -363,11 +360,13 @@ module xgrid_mod
    integer          :: count
    integer          :: pe
    integer          :: buffer_pos
-   integer, pointer :: i(:)  =>NULL()
-   integer, pointer :: j(:)  =>NULL()
-   integer, pointer :: tile(:)  =>NULL()
-   real,    pointer :: di(:) =>NULL()
-   real,    pointer :: dj(:) =>NULL()
+   integer, _ALLOCATABLE :: i(:) _NULL
+   integer, _ALLOCATABLE :: j(:) _NULL
+   integer, _ALLOCATABLE :: g(:) _NULL
+   integer, _ALLOCATABLE :: xLoc(:) _NULL
+   integer, _ALLOCATABLE :: tile(:) _NULL
+   real,    _ALLOCATABLE :: di(:) _NULL
+   real,    _ALLOCATABLE :: dj(:) _NULL
 end type overlap_type
 
 type comm_type
@@ -415,11 +414,12 @@ module xgrid_mod
   integer, pointer, dimension(:) :: ind_put1 =>NULL() ! indx for side1 put and side 2get.
   type(comm_type), pointer       :: put1 =>NULL()      ! for put_1_to_xgrid
   type(comm_type), pointer       :: get1 =>NULL()      ! for get_1_from_xgrid
+  type(comm_type), pointer       :: get1_repro =>NULL()! for get_1_from_xgrid_repro
 end type xmap_type
 
 !-----------------------------------------------------------------------
- character(len=128) :: version = '$Id: xgrid.F90,v 20.0 2013/12/14 00:19:20 fms Exp $'
- character(len=128) :: tagname = '$Name: tikal $'
+ character(len=128) :: version = '$Id$'
+ character(len=128) :: tagname = '$Name$'
 
  real, parameter                              :: EPS = 1.0e-10
  real, parameter                              :: LARGE_NUMBER = 1.e20
@@ -427,6 +427,7 @@ module xgrid_mod
  integer :: id_put_1_to_xgrid_order_1 = 0
  integer :: id_put_1_to_xgrid_order_2 = 0
  integer :: id_get_1_from_xgrid = 0
+ integer :: id_get_1_from_xgrid_repro = 0
  integer :: id_get_2_from_xgrid = 0
  integer :: id_put_2_to_xgrid = 0
  integer :: id_setup_xmap = 0
@@ -503,7 +504,7 @@ subroutine xgrid_init(remap_method)
 #endif
 
 !--------- write version number and namelist ------------------
-  call write_version_number()
+  call write_version_number (version, tagname)
 
   unit = stdlog ( )
   out_unit = stdout()
@@ -535,6 +536,7 @@ subroutine xgrid_init(remap_method)
      id_put_1_to_xgrid_order_1 = mpp_clock_id("put_1_to_xgrid_order_1", flags=MPP_CLOCK_SYNC)
      id_put_1_to_xgrid_order_2 = mpp_clock_id("put_1_to_xgrid_order_2", flags=MPP_CLOCK_SYNC)
      id_get_1_from_xgrid       = mpp_clock_id("get_1_from_xgrid", flags=MPP_CLOCK_SYNC) 
+     id_get_1_from_xgrid_repro = mpp_clock_id("get_1_from_xgrid_repro", flags=MPP_CLOCK_SYNC)
      id_get_2_from_xgrid       = mpp_clock_id("get_2_from_xgrid", flags=MPP_CLOCK_SYNC) 
      id_put_2_to_xgrid         = mpp_clock_id("put_2_to_xgrid", flags=MPP_CLOCK_SYNC) 
      id_setup_xmap             = mpp_clock_id("setup_xmap", flags=MPP_CLOCK_SYNC) 
@@ -729,7 +731,7 @@ subroutine load_xgrid (xmap, grid, grid_file, grid1_id, grid_id, tile1, tile2, u
         ! check the units of "xgrid_area 
         call get_var_att_value(grid_file, "xgrid_area", "units", attvalue)
         if( trim(attvalue) == 'm2' ) then
-           garea = 4*PI*RADIUS*RADIUS;
+           garea = 4.0*PI*RADIUS*RADIUS;
            area_tmp = tmp(:,1)/garea
         else if( trim(attvalue) == 'none' ) then
            area_tmp = tmp(:,1)
@@ -865,9 +867,9 @@ subroutine load_xgrid (xmap, grid, grid_file, grid1_id, grid_id, tile1, tile2, u
 
      !--- send the size of the data on side 1 to be sent over.
 
-     if (do_alltoall) then
-        call mpp_alltoall(nsend1, nrecv1)
-        call mpp_alltoall(nsend2, nrecv2)
+     if (collective_setup) then
+        call mpp_alltoall(nsend1, 1, nrecv1, 1)
+        call mpp_alltoall(nsend2, 1, nrecv2, 1)
      else
         do n = 0, npes-1
            p = mod(mypos+npes-n, npes)
@@ -910,7 +912,7 @@ subroutine load_xgrid (xmap, grid, grid_file, grid1_id, grid_id, tile1, tile2, u
      nxgrid2 = sum(nrecv2)
      if(nxgrid1>0 .OR. nxgrid2>0) allocate(recv_buffer(nxgrid1*nset1+nxgrid2*nset2))
 
-     if (do_alltoallv) then
+     if (collective_setup) then
         ! Construct the send and receive counters
         send_cnt(:) = nset1 * nsend1(:) + nset2 * nsend2(:)
         recv_cnt(:) = nset1 * nrecv1(:) + nset2 * nrecv2(:)
@@ -1042,7 +1044,7 @@ subroutine load_xgrid (xmap, grid, grid_file, grid1_id, grid_id, tile1, tile2, u
         deallocate(x_local)
      else
         allocate( grid%x( grid%size ) )
-        grid%x%di = 0; grid%x%dj = 0
+        grid%x%di = 0.0; grid%x%dj = 0.0
      end if
   end if
 
@@ -1059,7 +1061,7 @@ subroutine load_xgrid (xmap, grid, grid_file, grid1_id, grid_id, tile1, tile2, u
            if(scale_exist) then
               grid%x(ll)%scale = scale(l)
            else
-              grid%x(ll)%scale = 1
+              grid%x(ll)%scale = 1.0
            endif
            if(use_higher_order) then
               grid%x(ll)%di  = di(l)
@@ -1193,7 +1195,7 @@ subroutine load_xgrid (xmap, grid, grid_file, grid1_id, grid_id, tile1, tile2, u
         deallocate(x_local)
      else
         allocate( grid%x_repro( grid%size_repro ) )
-        grid%x_repro%di = 0; grid%x_repro%dj = 0
+        grid%x_repro%di = 0.0; grid%x_repro%dj = 0.0
      end if
      do l=1,nxgrid1
         if (in_box(i1_side1(l),j1_side1(l), grid1%is_me,grid1%ie_me, grid1%js_me,grid1%je_me) ) then
@@ -1374,7 +1376,7 @@ subroutine get_area_elements(file, name, domain, data)
      call error_mesg('xgrid_mod', 'no field named '//trim(name)//' in grid file '//trim(file)// &
                      ' Will set data to negative values...', NOTE)
      ! area elements no present in grid_spec file, set to negative values....
-     data = -1
+     data = -1.0
   endif    
 
 end subroutine get_area_elements
@@ -1833,6 +1835,11 @@ subroutine setup_xmap(xmap, grid_ids, grid_domains, grid_file, atm_grid)
 
   call set_comm_put1(xmap)
 
+  if(make_exchange_reproduce) then
+    allocate(xmap%get1_repro)
+    call set_comm_get1_repro(xmap)
+  endif
+
   call mpp_clock_end(id_set_comm)
 
   call mpp_clock_begin(id_regen)
@@ -1841,12 +1848,12 @@ subroutine setup_xmap(xmap, grid_ids, grid_domains, grid_file, atm_grid)
 
   call mpp_clock_begin(id_conservation_check)
 
-  xxx = conservation_check(grid1%area*0+1.0, grid1%id, xmap)
+  xxx = conservation_check(grid1%area*0.0+1.0, grid1%id, xmap)
   write(out_unit,* )"Checked data is array of constant 1"
   write(out_unit,* )grid1%id,'(',xmap%grids(:)%id,')=', xxx 
 
   do g=2,size(xmap%grids(:))
-     xxx = conservation_check(xmap%grids(g)%frac_area*0+1.0, xmap%grids(g)%id, xmap )
+     xxx = conservation_check(xmap%grids(g)%frac_area*0.0+1.0, xmap%grids(g)%id, xmap )
      write( out_unit,* )xmap%grids(g)%id,'(',xmap%grids(:)%id,')=', xxx 
   enddo
   ! create an random number 2d array
@@ -1933,7 +1940,7 @@ function get_nest_contact(mosaic_file, tile_nest_out, tile_parent_out, is_nest_o
     t2 = tile2(n);
     ! For nesting, the contact index of one tile must match its global domain 
     if( (nx(t1) .NE. nx1_contact .OR. ny(t1) .NE. ny1_contact ) .AND. &
-	(nx(t2) .NE. nx2_contact .OR. ny(t2) .NE. ny2_contact ) ) cycle
+        (nx(t2) .NE. nx2_contact .OR. ny(t2) .NE. ny2_contact ) ) cycle
     if(nx1_contact == nx2_contact .AND. ny1_contact == ny2_contact) then
       call error_mesg('xgrid_mod', 'There is no refinement for the overlapping region', FATAL)
     endif
@@ -1976,7 +1983,102 @@ function get_nest_contact(mosaic_file, tile_nest_out, tile_parent_out, is_nest_o
   
 end function get_nest_contact
 
+!#######################################################################
+subroutine set_comm_get1_repro(xmap)
+  type (xmap_type), intent(inout) :: xmap
+  integer, dimension(xmap%npes) :: pe_ind, cnt
+  integer, dimension(0:xmap%npes-1) :: send_ind, recv_ind, pl
+  integer :: npes, nsend, nrecv, mypos
+  integer :: m, p, pos, n, g, l
+  type(comm_type), pointer, save :: comm => NULL()
+
+  comm => xmap%get1_repro
+  npes = xmap%npes
+
+  nrecv = 0
+  mypos = mpp_pe() - mpp_root_pe()
+  do m=0,npes-1 
+    p = mod(mypos+npes-m, npes)
+    if( xmap%recv_count_repro(p) > 0 ) then
+      nrecv = nrecv + 1
+      pe_ind(nrecv) = p
+    endif
+  enddo
+
+  comm%nrecv = nrecv
+  if( nrecv > 0 ) then
+    allocate(comm%recv(nrecv))
+    pos = 0
+    do n = 1, nrecv
+      p = pe_ind(n)
+      comm%recv(n)%count = xmap%recv_count_repro(p)
+      comm%recv(n)%pe = p + xmap%root_pe
+      comm%recv(n)%buffer_pos = pos 
+      pos = pos + comm%recv(n)%count
+    enddo
+  endif
+
+
+  ! send information
+  nsend = 0
+  mypos = mpp_pe() - mpp_root_pe()
+  do m=0,xmap%npes-1 
+    p = mod(mypos+m, npes)
+    if( xmap%send_count_repro(p) > 0 ) then
+      nsend = nsend + 1
+      pe_ind(nsend) = p
+      send_ind(p) = nsend
+    endif
+  enddo
+
+  comm%nsend = nsend
+  if( nsend > 0 ) then
+     allocate(comm%send(nsend))
+     pos = 0
+     cnt(:) = 0
+     do n = 1, nsend
+        p = pe_ind(n)
+        comm%send(n)%count = xmap%send_count_repro(p)
+        comm%send(n)%pe = p + xmap%root_pe 
+        comm%send(n)%buffer_pos = pos
+        pos = pos + comm%send(n)%count
+        allocate(comm%send(n)%i(comm%send(n)%count))
+        allocate(comm%send(n)%j(comm%send(n)%count))
+        allocate(comm%send(n)%g(comm%send(n)%count))
+        allocate(comm%send(n)%xLoc(comm%send(n)%count))
+     enddo
+
+     do g=2,size(xmap%grids(:))
+        do l=1,xmap%grids(g)%size ! index into this side 2 grid's patterns
+           p = xmap%grids(g)%x(l)%pe-xmap%root_pe 
+           n = send_ind(p)
+           cnt(n) = cnt(n) + 1
+           pos = cnt(n)
+           comm%send(n)%i(pos) = xmap%grids(g)%x(l)%i2
+           comm%send(n)%j(pos) = xmap%grids(g)%x(l)%j2
+           comm%send(n)%g(pos) = g
+        enddo
+     enddo
+     !--- make sure the count is correct
+     do n = 1, nsend
+        if( comm%send(n)%count .NE. cnt(n) ) call error_mesg('xgrid_mod', &
+             'comm%send(n)%count .NE. cnt(n)', FATAL)
+     enddo
+   endif
+
+   !--- set up the recv_pos for unpack the data.
+   pl(:) = 1
+   do g=2,size(xmap%grids(:))
+      do l=1,xmap%grids(g)%size_repro ! index into side1 grid's patterns
+         p = xmap%grids(g)%x_repro(l)%pe-xmap%root_pe
+         xmap%grids(g)%x_repro(l)%recv_pos = pl(p)
+         pl(p) = pl(p) + 1
+      end do
+   end do
+
+
 
+end subroutine set_comm_get1_repro
 
 !#######################################################################
 subroutine set_comm_get1(xmap)
@@ -2114,37 +2216,10 @@ subroutine set_comm_get1(xmap)
      enddo
   endif
 
-
   mypos = mpp_pe()-mpp_root_pe()
-!  do n = 0, npes-1
-!     p = mod(mypos+npes-n, npes)
-!     call mpp_recv(recv_size(p), glen=1, from_pe=pelist(p), block=.false., tag=COMM_TAG_3)
-!  enddo
-
-  !--- send data
-!  do n = 0, npes-1
-!     p = mod(mypos+n, npes)
-!     call mpp_send(send_size(p), plen=1, to_pe=pelist(p), tag=COMM_TAG_3)
-!  enddo
-
-!  call mpp_sync_self(check=EVENT_RECV)
-!  call mpp_sync_self()
-
-!  call mpp_sync()
-  
-!  do p = 0, npes-1
-        
-!     if(recv_size(p) .NE. xmap%your2my1_size(p)) then
-!        print*, "My = ", mpp_pe(), p, recv_size(p), xmap%your2my1_size(p)
-!          call error_mesg("xgrid_mod", &
-!             "recv_size(p) .NE. xmap%your2my1_size(p)", FATAL)
-!     endif
-!  end do
  
+  ! send/recv for get_1_from_xgrid_recv
   recv_size(:) = xmap%your2my1_size(:)
-
-
-  !--- set up for send for get_1_from_xgrid, also is recv for put_1_to_xgrid
   nsend = count( send_size> 0)  
   comm%nsend = nsend
   if(nsend>0) then
@@ -2230,7 +2305,7 @@ subroutine set_comm_get1(xmap)
      do p = 0, npes-1
         recv_buffer_pos(p) = buffer_pos
         buffer_pos = buffer_pos +  recv_size(p)
-     enddo      
+     enddo
      pos = 0
      buffer_pos = 0        
      do m=0,npes-1
@@ -2274,8 +2349,8 @@ subroutine set_comm_get1(xmap)
         endif
      enddo
   endif
-
   call mpp_sync_self()
+
   if(allocated(send_buf) ) deallocate(send_buf)
   if(allocated(recv_buf) ) deallocate(recv_buf)
   if(allocated(pelist)   ) deallocate(pelist)
@@ -2583,6 +2658,9 @@ subroutine regen(xmap)
   integer              :: tile1
   integer              :: ll
   logical              :: overlap_with_nest
+  integer              :: cnt(xmap%get1%nsend)
+  integer              :: i,j,n,xloc,pos,nsend,m,npes, mypos
+  integer              :: send_ind(0:xmap%npes-1)
 
   max_size = 0
 
@@ -2692,6 +2770,34 @@ subroutine regen(xmap)
      xmap%grids(g)%last_get = xmap%size_get2
   end do
 
+  !---set up information for get_1_from_xgrid_repro
+  if(make_exchange_reproduce .AND. xmap%get1_repro%nsend >0) then
+     xloc = 0
+     nsend = 0
+     npes = xmap%npes
+     mypos = mpp_pe() - mpp_root_pe()
+     cnt(:) = 0
+     do m=0,npes-1 
+        p = mod(mypos+m, npes)
+        if( xmap%send_count_repro(p) > 0 ) then
+          nsend = nsend + 1
+          send_ind(p) = nsend
+        endif
+     enddo
+     do g=2,size(xmap%grids(:))
+        do l=1,xmap%grids(g)%size ! index into this side 2 grid's patterns
+           i = xmap%grids(g)%x(l)%i2
+           j = xmap%grids(g)%x(l)%j2
+           p = xmap%grids(g)%x(l)%pe-xmap%root_pe 
+           n = send_ind(p) 
+           cnt(n) = cnt(n) + 1
+           pos = cnt(n) 
+           xmap%get1_repro%send(n)%xLoc(pos) = xloc
+           xloc = xloc + count(xmap%grids(g)%frac_area(i,j,:)/=0.0)
+        enddo
+     enddo
+  endif
+
 end subroutine regen
 
 !#######################################################################
@@ -3173,6 +3279,8 @@ subroutine put_1_to_xgrid_order_1(d_addrs, x_addrs, xmap, isize, jsize, xsize, l
      end do
   else
      start_pos = 0
+!$OMP parallel do default(none) shared(lsize,x_addrs,comm,recv_buffer,xmap) &
+!$OMP                          private(ptr_x,count,ibegin,istart,iend,pos,unpack_buffer)
      do l = 1, lsize
         ptr_x = x_addrs(l)
         do p = 1, comm%nrecv
@@ -3222,7 +3330,7 @@ subroutine put_1_to_xgrid_order_2(d_addrs, x_addrs, xmap, isize, jsize, xsize, l
   real                            :: recv_buffer(xmap%put1%recvsize*lsize*3)
   real                            :: send_buffer(xmap%put1%sendsize*lsize*3)
   real                            :: unpack_buffer(xmap%put1%recvsize*3)
-
+  logical                         :: on_west_edge, on_east_edge, on_south_edge, on_north_edge
   real, dimension(isize, jsize)   :: d
   real, dimension(xsize)          :: x
   pointer(ptr_d, d)
@@ -3236,26 +3344,34 @@ subroutine put_1_to_xgrid_order_2(d_addrs, x_addrs, xmap, isize, jsize, xsize, l
   isd = grid1%isd_me
   jsd = grid1%jsd_me
 
-  tmp = LARGE_NUMBER
+!$OMP parallel do default(none) shared(lsize,tmp,d_addrs,isize,jsize) private(ptr_d)
   do l = 1, lsize
+     tmp(:,:,l) = LARGE_NUMBER
      ptr_d = d_addrs(l)
      tmp(1:isize,1:jsize,l) = d(:,:)
   enddo
 
   if(grid1%is_latlon) then
      call mpp_update_domains(tmp,grid1%domain_with_halo)
+!$OMP parallel do default(none) shared(lsize,tmp,grid1,is,ie,js,je,isd,jsd,tmpx,tmpy)
      do l = 1, lsize
         tmpy(:,:,l) = grad_merid_latlon(tmp(:,:,l), grid1%lat, is, ie, js, je, isd, jsd)
         tmpx(:,:,l) = grad_zonal_latlon(tmp(:,:,l), grid1%lon, grid1%lat, is, ie, js, je, isd, jsd)
      enddo
   else
      call mpp_update_domains(tmp,grid1%domain)
+     on_west_edge  = (is==1)
+     on_east_edge  = (ie==grid1%im)
+     on_south_edge = (js==1)
+     on_north_edge = (je==grid1%jm)
+!$OMP parallel do default(none) shared(lsize,tmp,grid1,tmpx,tmpy, &
+!$OMP                                  on_west_edge,on_east_edge,on_south_edge,on_north_edge)
      do l = 1, lsize
-        call gradient_cubic(tmp(:,:,l), xmap%grids(1)%box%dx, xmap%grids(1)%box%dy, xmap%grids(1)%box%area, &
-                            xmap%grids(1)%box%edge_w, xmap%grids(1)%box%edge_e, xmap%grids(1)%box%edge_s,   &
-                            xmap%grids(1)%box%edge_n, xmap%grids(1)%box%en1, xmap%grids(1)%box%en2,         &
-                            xmap%grids(1)%box%vlon, xmap%grids(1)%box%vlat, tmpx(:,:,l), tmpy(:,:,l),       &
-                            is==1, ie==grid1%im, js==1, je==grid1%jm)
+        call gradient_cubic(tmp(:,:,l), grid1%box%dx, grid1%box%dy, grid1%box%area,   &
+                            grid1%box%edge_w, grid1%box%edge_e, grid1%box%edge_s,     &
+                            grid1%box%edge_n, grid1%box%en1, grid1%box%en2,           &
+                            grid1%box%vlon, grid1%box%vlat, tmpx(:,:,l), tmpy(:,:,l), &
+                            on_west_edge, on_east_edge, on_south_edge, on_north_edge)
      enddo
   end if     
 
@@ -3275,6 +3391,7 @@ subroutine put_1_to_xgrid_order_2(d_addrs, x_addrs, xmap, isize, jsize, xsize, l
 
   !--- compute d_bar_max and d_bar_min.
   if(monotonic_exchange) then
+!$OMP parallel do default(none) shared(lsize,isize,jsize,d_bar_max,d_bar_min,d_max,d_min,tmp)
      do l = 1, lsize
         do j = 1, jsize
            do i = 1, isize
@@ -3391,11 +3508,14 @@ subroutine put_1_to_xgrid_order_2(d_addrs, x_addrs, xmap, isize, jsize, xsize, l
   else
      if( lsize == 1) then
         ptr_x = x_addrs(1)
+!$OMP parallel do default(none) shared(xmap,recv_buffer,ptr_x) private(pos)
         do l=1,xmap%size_put1
            pos = xmap%x1_put(l)%pos
            x(l) = recv_buffer(3*pos-2) + recv_buffer(3*pos-1)*xmap%x1_put(l)%dj + recv_buffer(3*pos)*xmap%x1_put(l)%di
         end do
      else
+!$OMP parallel do default(none) shared(lsize,comm,xmap,recv_buffer,x_addrs) &
+!$OMP                          private(ptr_x,pos,ibegin,istart,iend,count,unpack_buffer)
         do l = 1, lsize
            ptr_x = x_addrs(l)
            pos = 0
@@ -3454,7 +3574,6 @@ subroutine get_1_from_xgrid(d_addrs, x_addrs, xmap, isize, jsize, xsize, lsize)
   comm => xmap%get1
   grid1 => xmap%grids(1)
 
-
   do p = 1, comm%nrecv
      recv => comm%recv(p)
      msgsize = recv%count*lsize
@@ -3463,6 +3582,7 @@ subroutine get_1_from_xgrid(d_addrs, x_addrs, xmap, isize, jsize, xsize, lsize)
   enddo
 
   dg = 0.0;
+!$OMP parallel do default(none) shared(lsize,xmap,dg,x_addrs) private(dgp,ptr_x)
   do l = 1, lsize
      ptr_x = x_addrs(l)
      do i=1,xmap%size
@@ -3471,6 +3591,7 @@ subroutine get_1_from_xgrid(d_addrs, x_addrs, xmap, isize, jsize, xsize, lsize)
      enddo
   enddo
 
+
   !--- send the data
   buffer_pos = 0
   istart = 1
@@ -3496,16 +3617,19 @@ subroutine get_1_from_xgrid(d_addrs, x_addrs, xmap, isize, jsize, xsize, lsize)
   !--- unpack the buffer
   do l = 1, lsize
      ptr_d = d_addrs(l)
-     d = 0
+     d = 0.0
   enddo
   !--- To bitwise reproduce old results, first copy the data onto its own pe.
 
   do p = 1, comm%nrecv
      recv => comm%recv(p)
      count = recv%count
-     pos = recv%buffer_pos*lsize
+     buffer_pos = recv%buffer_pos*lsize
      if( recv%pe == xmap%me ) then
+!$OMP parallel do default(none) shared(lsize,recv,recv_buffer,buffer_pos,d_addrs,count) &
+!$OMP                          private(ptr_d,i,j,pos)
         do l = 1, lsize
+           pos = buffer_pos + (l-1)*count
            ptr_d = d_addrs(l)
            do n = 1,count
               i = recv%i(n)
@@ -3525,8 +3649,11 @@ subroutine get_1_from_xgrid(d_addrs, x_addrs, xmap, isize, jsize, xsize, lsize)
      if( recv%pe == xmap%me ) then
         cycle
      endif
-     pos = recv%buffer_pos*lsize
+     buffer_pos = recv%buffer_pos*lsize
+!$OMP parallel do default(none) shared(lsize,recv,recv_buffer,buffer_pos,d_addrs) &
+!$OMP                          private(ptr_d,i,j,pos)
      do l = 1, lsize
+        pos = buffer_pos + (l-1)*recv%count
         ptr_d = d_addrs(l)
         do n = 1, recv%count
            i = recv%i(n)
@@ -3540,6 +3667,7 @@ subroutine get_1_from_xgrid(d_addrs, x_addrs, xmap, isize, jsize, xsize, lsize)
   !
   ! normalize with side 1 grid cell areas
   !
+!$OMP parallel do default(none) shared(lsize,d_addrs,grid1) private(ptr_d)
   do l = 1, lsize
      ptr_d = d_addrs(l)
      d = d * grid1%area_inv
@@ -3557,66 +3685,64 @@ subroutine get_1_from_xgrid_repro(d_addrs, x_addrs, xmap, xsize, lsize)
   type (xmap_type),              intent(inout) :: xmap
   integer,                          intent(in) :: xsize, lsize
 
-  real,    dimension(:), allocatable :: x_psum
-  integer, dimension(:), allocatable :: pe_psum
-  integer                            :: l1, l2, l3, g, i, j, k, p, l
-  integer,  dimension(0:xmap%npes-1) :: pl
-  type (grid_type),    pointer, save :: grid =>NULL()
+  integer                            :: g, i, j, k, p, l, n, l2, m, l3
+  integer                            :: msgsize, buffer_pos, pos
+  type (grid_type), pointer, save :: grid =>NULL()
+  type(comm_type),  pointer, save :: comm => NULL()
+  type(overlap_type), pointer, save  :: send => NULL()
+  type(overlap_type), pointer, save  :: recv => NULL()
+    integer,  dimension(0:xmap%npes-1) :: pl, ml
   real                               :: recv_buffer(xmap%recv_count_repro_tot*lsize)
   real                               :: send_buffer(xmap%send_count_repro_tot*lsize)
-  real                               :: unpack_buffer(xmap%get1%recvsize)
   real                               :: d(xmap%grids(1)%is_me:xmap%grids(1)%ie_me, &
                                           xmap%grids(1)%js_me:xmap%grids(1)%je_me)
   real, dimension(xsize)             :: x
   pointer(ptr_d, d)
   pointer(ptr_x, x)
 
+  call mpp_clock_begin(id_get_1_from_xgrid_repro)
+  comm => xmap%get1_repro
   !--- pre-post receiving
-  l2 = 0;
-  do p=0,xmap%npes-1
-    l1 = l2 + 1
-    l2 = l2 + xmap%recv_count_repro(p)*lsize
-    if (xmap%recv_count_repro(p)>0) then ! can receive from myself
-      ! Force use of "scalar", integer pointer mpp interface.
-      call mpp_recv(recv_buffer(l1), glen=l2-l1+1, from_pe=p+xmap%root_pe, block=.false., tag=COMM_TAG_10);
-      pl(p) = l1
-    end if
-  end do
+  do p = 1, comm%nrecv
+     recv => comm%recv(p)
+     msgsize = recv%count*lsize
+     buffer_pos = recv%buffer_pos*lsize
+     call mpp_recv(recv_buffer(buffer_pos+1), glen=msgsize, from_pe = recv%pe, block=.false., tag=COMM_TAG_10)
+     n = recv%pe -xmap%root_pe
+     pl(n) = buffer_pos 
+     ml(n) = recv%count
+  enddo
 
-  allocate ( x_psum  (xmap%send_count_repro_tot*lsize) )
-  allocate ( pe_psum (xmap%send_count_repro_tot*lsize) )
-  x_psum = 0.0
-  l1 = 0 ! index into partition summed exchange grid variable
-  do l = 1, lsize
-     l2 = 0 ! index into exchange grid variable
-     ptr_x = x_addrs(l)
-     do g=2,size(xmap%grids(:))
-        do l3=1,xmap%grids(g)%size ! index into this side 2 grid's patterns
-           l1 = l1 + 1
-           do k=1,xmap%grids(g)%km
-              i = xmap%grids(g)%x(l3)%i2
-              j = xmap%grids(g)%x(l3)%j2
-              if (xmap%grids(g)%frac_area(i,j,k)/=0.0) then
-                 l2 = l2 + 1
-                 x_psum (l1) = x_psum(l1) + xmap%x1(l2)%area * x(l2)
-                 pe_psum(l1) = xmap%grids(g)%x(l3)%pe
-              end if
-           end do
-        end do
-     end do
+  !pack the data
+  send_buffer(:) = 0.0
+!$OMP parallel do default(none) shared(lsize,x_addrs,comm,xmap,send_buffer) &
+!$OMP                          private(ptr_x,i,j,g,l2,pos,send)
+  do p = 1, comm%nsend
+     pos = comm%send(p)%buffer_pos*lsize
+     send => comm%send(p)
+     do l = 1,lsize
+        ptr_x = x_addrs(l)
+        do n = 1, send%count
+           i = send%i(n)
+           j = send%j(n)
+           g = send%g(n)
+           l2 = send%xloc(n)
+           pos = pos + 1
+           do k =1, xmap%grids(g)%km
+             if(xmap%grids(g)%frac_area(i,j,k)/=0.0) then
+              l2 = l2+1
+              send_buffer(pos) = send_buffer(pos) + xmap%x1(l2)%area *x(l2)
+             endif
+           enddo
+         enddo
+      enddo
+   enddo
+
+  do p =1, comm%nsend
+     buffer_pos = comm%send(p)%buffer_pos*lsize
+     msgsize = comm%send(p)%count*lsize
+     call mpp_send(send_buffer(buffer_pos+1), plen=msgsize, to_pe=comm%send(p)%pe, tag=COMM_TAG_10)
   enddo
-  l2 = 0;
-
-  do p=0,xmap%npes-1
-    l1 = l2 + 1
-    l2 = l2 + xmap%send_count_repro(p)*lsize
-    if (xmap%send_count_repro(p)>0) then ! can send to myself
-      send_buffer(l1:l2) = pack(x_psum, pe_psum==p+xmap%root_pe)
-      ! Force use of "scalar", integer pointer mpp interface.
-      call mpp_send(send_buffer(l1), plen=l2-l1+1, to_pe=p+xmap%root_pe, tag=COMM_TAG_10);
-    end if
-  end do
-  deallocate ( x_psum, pe_psum)
 
   do l = 1, lsize
      ptr_d = d_addrs(l)
@@ -3625,8 +3751,9 @@ subroutine get_1_from_xgrid_repro(d_addrs, x_addrs, xmap, xsize, lsize)
 
   call mpp_sync_self(check=EVENT_RECV)
 
-
-  do l = 1, lsize  
+!$OMP parallel do default(none) shared(lsize,d_addrs,xmap,recv_buffer,pl,ml) &
+!$OMP                          private(ptr_d,grid,i,j,p,pos)
+  do l = 1, lsize
      ptr_d = d_addrs(l)
      do g=2,size(xmap%grids(:))
         grid => xmap%grids(g)
@@ -3634,8 +3761,8 @@ subroutine get_1_from_xgrid_repro(d_addrs, x_addrs, xmap, xsize, lsize)
            i = grid%x_repro(l3)%i1
            j = grid%x_repro(l3)%j1
            p = grid%x_repro(l3)%pe-xmap%root_pe
-           d(i,j) = d(i,j) + recv_buffer(pl(p))
-           pl(p) = pl(p) + 1
+           pos = pl(p) + (l-1)*ml(p) + grid%x_repro(l3)%recv_pos
+           d(i,j) = d(i,j) + recv_buffer(pos)
         end do
      end do
      ! normalize with side 1 grid cell areas
@@ -3644,6 +3771,8 @@ subroutine get_1_from_xgrid_repro(d_addrs, x_addrs, xmap, xsize, lsize)
 
   call mpp_sync_self()       
 
+  call mpp_clock_end(id_get_1_from_xgrid_repro)
+
 end subroutine get_1_from_xgrid_repro
 
 !#######################################################################
@@ -3736,7 +3865,7 @@ function conservation_check_side2(d, grid_id, xmap,remap_method) ! this one for
       endif
       call put_to_xgrid(d, grid_id, x_over, xmap)  ! put from this side 2
     else
-      call put_to_xgrid(0 * grid2%frac_area, grid2%id, x_over, xmap) ! zero rest
+      call put_to_xgrid(0.0 * grid2%frac_area, grid2%id, x_over, xmap) ! zero rest
     end if
   end do
 
@@ -3912,7 +4041,7 @@ subroutine stock_move_3d(from, to, grid_index, data, xmap, &
      return
   endif
 
-     from_dq = delta_t * 4*PI*radius**2 * sum( sum(xmap%grids(grid_index)%area * &
+     from_dq = delta_t * 4.0*PI*radius**2 * sum( sum(xmap%grids(grid_index)%area * &
           & sum(xmap%grids(grid_index)%frac_area * data, DIM=3), DIM=1))
      to_dq = from_dq
 
@@ -3923,8 +4052,8 @@ subroutine stock_move_3d(from, to, grid_index, data, xmap, &
   if(present(verbose).and.debug_stocks) then
      call mpp_sum(from_dq)
      call mpp_sum(to_dq)
-     from_dq = from_dq/(4*PI*radius**2)
-     to_dq   = to_dq  /(4*PI*radius**2)
+     from_dq = from_dq/(4.0*PI*radius**2)
+     to_dq   = to_dq  /(4.0*PI*radius**2)
      if(mpp_pe()==mpp_root_pe()) then
         write(stocks_file,'(a,es19.12,a,es19.12,a)') verbose, from_dq,' [*/m^2]'
      endif
@@ -3966,7 +4095,7 @@ subroutine stock_move_2d(from, to, grid_index, data, xmap, &
   if( .not. present(grid_index) .or. grid_index==1 ) then
 
      ! only makes sense if grid_index == 1
-     from_dq = delta_t * 4*PI*radius**2 * sum(sum(xmap%grids(1)%area * data, DIM=1))
+     from_dq = delta_t * 4.0*PI*radius**2 * sum(sum(xmap%grids(1)%area * data, DIM=1))
      to_dq = from_dq
 
   else
@@ -3983,8 +4112,8 @@ subroutine stock_move_2d(from, to, grid_index, data, xmap, &
   if(debug_stocks) then
      call mpp_sum(from_dq)
      call mpp_sum(to_dq)
-     from_dq = from_dq/(4*PI*radius**2)
-     to_dq   = to_dq  /(4*PI*radius**2)
+     from_dq = from_dq/(4.0*PI*radius**2)
+     to_dq   = to_dq  /(4.0*PI*radius**2)
      if(mpp_pe()==mpp_root_pe()) then
         write(stocks_file,'(a,es19.12,a,es19.12,a)') verbose, from_dq,' [*/m^2]'
      endif
@@ -4007,14 +4136,14 @@ subroutine stock_integrate_2d(data, xmap, delta_t, radius, res, ier)
   integer, intent(out)            :: ier
 
   ier = 0
-  res = 0
+  res = 0.0
 
   if(.not. associated(xmap%grids) ) then
      ier = 6
      return
   endif
 
-  res = delta_t * 4*PI*radius**2 * sum(sum(xmap%grids(1)%area * data, DIM=1))
+  res = delta_t * 4.0*PI*radius**2 * sum(sum(xmap%grids(1)%area * data, DIM=1))
 
 end subroutine stock_integrate_2d
 !#######################################################################
@@ -4065,7 +4194,7 @@ subroutine stock_print(stck, Time, comp_name, index, ref_value, radius, pelist)
 
   if(mpp_pe() == mpp_root_pe()) then
      ! normalize to 1 earth m^2
-     planet_area = 4*PI*radius**2
+     planet_area = 4.0*PI*radius**2
      f_value       = f_value     / planet_area
      c_value       = c_value     / planet_area
 
@@ -4170,6 +4299,7 @@ end module xgrid_mod
 ! 
 
 !      
+!      A  guide to grid coupling in FMS.
 !   
 !   
 !      A simple xgrid  example. 
diff --git a/src/shared/fft/fft.F90 b/src/shared/fft/fft.F90
index 529d3cddc1..7e8f51eab6 100644
--- a/src/shared/fft/fft.F90
+++ b/src/shared/fft/fft.F90
@@ -1,10 +1,11 @@
 
 module fft_mod
 
-! 
+! 
 !   Bruce Wyman
 ! 
 
+! 
 
 ! 
 !     Performs simultaneous fast Fourier transforms (FFTs) between
@@ -180,8 +181,8 @@ module fft_mod
 logical :: module_is_initialized=.false.
 
 !  cvs version and tag name
-character(len=128) :: version = '$Id: fft.F90,v 13.0 2006/03/28 21:38:54 fms Exp $'
-character(len=128) :: tagname = '$Name: tikal $'
+character(len=128) :: version = '$Id$'
+character(len=128) :: tagname = '$Name$'
 
 !-----------------------------------------------------------------------
 !
@@ -846,7 +847,7 @@ subroutine fft_init (n)
 
 !  write version and tag name to log file
    if (do_log) then
-      call write_version_number()
+      call write_version_number (version, tagname)
       do_log = .false.
    endif
 
diff --git a/src/shared/field_manager/field_manager.F90 b/src/shared/field_manager/field_manager.F90
index 4da6ab1293..dd80961e34 100644
--- a/src/shared/field_manager/field_manager.F90
+++ b/src/shared/field_manager/field_manager.F90
@@ -1,26 +1,27 @@
 module field_manager_mod
 #ifndef MAXFIELDS_ 
-#define MAXFIELDS_ 250
+#define MAXFIELDS_ 150
 #endif
 
 #ifndef MAXFIELDMETHODS_
-#define MAXFIELDMETHODS_ 250
+#define MAXFIELDMETHODS_ 150
 #endif
 
 !
-!  William Cooke
+!  William Cooke
 ! 
 ! 
-!  Richard D. Slater
+!  Richard D. Slater
 ! 
 !
-!  Matthew Harrison
+!  Matthew Harrison
 ! 
 !
-!  John P. Dunne
+!  John P. Dunne
 ! 
 !
 ! 
 
 ! 
 
@@ -181,8 +182,8 @@ module field_manager_mod
 private
 
 
-character(len=128) :: version = '$Id: field_manager.F90,v 20.0 2013/12/14 00:19:26 fms Exp $'
-character(len=128) :: tagname = '$Name: tikal $'
+character(len=128) :: version = '$Id$'
+character(len=128) :: tagname = '$Name$'
 logical            :: module_is_initialized  = .false.
 
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
@@ -613,6 +614,16 @@ subroutine field_manager_init(nfields, table_name)
    if(present(nfields)) nfields = num_fields
    return
 endif
+
+#ifdef PRESERVE_UNIT_CASE
+! 
+!   The case of the units in the field_table is preserved.  This option is
+!   still experimental.  It is possible other model components expect the units
+!   to be lowercase.  Please notify the developers if any issues are discovered.
+! 
+call mpp_error(NOTE,trim(note_header)//"Preserving the unit's case is experimental.")
+#endif
+
 num_fields = 0
 call initialize
 
@@ -641,7 +652,7 @@ subroutine field_manager_init(nfields, table_name)
 
 call mpp_open(iunit,file=trim(tbl_name), form=MPP_ASCII, action=MPP_RDONLY)
 !write_version_number should precede all writes to stdlog from field_manager
-call write_version_number()
+call write_version_number (version, tagname)
 log_unit = stdlog()
 do while (.TRUE.)
    read(iunit,'(a)',end=89,err=99) record
@@ -803,8 +814,22 @@ subroutine field_manager_init(nfields, table_name)
         case(4)
 ! If there is no control string then the last string can be omitted and there are only 4 '"' in the record.
           read(record,*,end=99,err=99) text_method_short
-          fields(num_fields)%methods(m)%method_type = lowercase(trim(text_method_short%method_type))
-          fields(num_fields)%methods(m)%method_name = lowercase(trim(text_method_short%method_name))
+          fields(num_fields)%methods(m)%method_type =&
+               & lowercase(trim(text_method_short%method_type))
+#ifdef PRESERVE_UNIT_CASE
+
+          if ( trim(fields(num_fields)%methods(m)%method_type) == 'units' ) then
+             ! Do not lowercase if units
+             fields(num_fields)%methods(m)%method_name =&
+                  & trim(text_method_short%method_name)
+          else
+             fields(num_fields)%methods(m)%method_name =&
+                  & lowercase(trim(text_method_short%method_name))
+          end if
+#else
+          fields(num_fields)%methods(m)%method_name =&
+               & lowercase(trim(text_method_short%method_name))
+#endif
           fields(num_fields)%methods(m)%method_control = " "
 
           type_str    = text_method_short%method_type
@@ -1272,16 +1297,12 @@ subroutine field_manager_end
 !        local parameters
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 character(len=17), parameter :: sub_name     = 'field_manager_end'
-character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
-                                               '(' // trim(sub_name) // '): '
-character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
-                                               '(' // trim(sub_name) // '): '
 character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
                                                '(' // trim(sub_name) // '): '
 
 integer :: unit 
 
-call write_version_number()
+call write_version_number (version, tagname)
 if ( mpp_pe() == mpp_root_pe() ) then
    unit = stdlog()
    write (unit,'(/,(a))') trim(note_header), 'Exiting field_manager, have a nice day ...'
@@ -1314,13 +1335,6 @@ subroutine strip_front_blanks(name)
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 !        local parameters
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-character(len=18), parameter :: sub_name     = 'strip_front_blanks'
-character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
-                                               '(' // trim(sub_name) // '): '
-character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
-                                               '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
 
 integer :: i, j
 
@@ -1372,13 +1386,6 @@ function find_field_index_old(model, field_name)
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 !        local parameters
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-character(len=16), parameter :: sub_name     = 'find_field_index'
-character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
-                                               '(' // trim(sub_name) // '): '
-character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
-                                               '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
 integer :: i
 
 find_field_index_old = NO_FIELD
@@ -1407,13 +1414,6 @@ function find_field_index_new(field_name)
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 !        local parameters
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-character(len=16), parameter :: sub_name     = 'find_field_index'
-character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
-                                               '(' // trim(sub_name) // '): '
-character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
-                                               '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
 integer :: i
 
 find_field_index_new = NO_FIELD
@@ -1469,10 +1469,6 @@ subroutine get_field_info(n,fld_type,fld_name,model,num_methods)
 character(len=14), parameter :: sub_name     = 'get_field_info'
 character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
                                                '(' // trim(sub_name) // '): '
-character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
-                                               '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
 
 !   
 !     The field index is invalid because it is less than 1 or greater than the 
@@ -1523,10 +1519,6 @@ subroutine get_field_method(n,m,method)
 character(len=16), parameter :: sub_name     = 'get_field_method'
 character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
                                                '(' // trim(sub_name) // '): '
-character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
-                                               '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
 
 !   
 !     The field index is invalid because it is less than 1 or greater than the 
@@ -1577,10 +1569,6 @@ subroutine get_field_methods(n,methods)
 character(len=17), parameter :: sub_name     = 'get_field_methods'
 character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
                                                '(' // trim(sub_name) // '): '
-character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
-                                               '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
 
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 !        local variables
@@ -1764,8 +1752,6 @@ function  create_field(parent_p, name)                        &
                                                '(' // trim(sub_name) // '): '
 character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
                                                '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
 integer                      :: ier
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 !        local variables
@@ -1902,8 +1888,6 @@ recursive function dump_list(list_p, recursive, depth)                &
 integer,                  parameter :: max_depth    = 128
 character(len=max_depth), parameter :: blank        = '    '
 character(len=9),  parameter :: sub_name     = 'dump_list'
-character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
-                                               '(' // trim(sub_name) // '): '
 character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
                                                '(' // trim(sub_name) // '): '
 character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
@@ -2154,16 +2138,6 @@ subroutine find_base(name, path, base)  !{
 character(len=*), intent(out) :: path
 character(len=*), intent(out) :: base
 
-!+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-!        local parameters
-!+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-character(len=9),  parameter :: sub_name     = 'find_base'
-character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
-                                               '(' // trim(sub_name) // '): '
-character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
-                                               '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 !        local variables
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
@@ -2267,16 +2241,6 @@ function find_field(name, this_list_p)                                &
 character(len=*), intent(in) :: name
 type (field_def), pointer    :: this_list_p
 
-!+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-!        local parameters
-!+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-character(len=10), parameter :: sub_name     = 'find_field'
-character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
-                                               '(' // trim(sub_name) // '): '
-character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
-                                               '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 !        local variables
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
@@ -2358,16 +2322,6 @@ subroutine find_head(name, head, rest)  !{
 character(len=*), intent(out) :: head
 character(len=*), intent(out) :: rest
 
-!+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-!        local parameters
-!+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-character(len=9),  parameter :: sub_name     = 'find_head'
-character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
-                                               '(' // trim(sub_name) // '): '
-character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
-                                               '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 !        local variables
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
@@ -2464,8 +2418,6 @@ function find_list(path, relative_p, create)                    &
 !        local parameters
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 character(len=9),  parameter :: sub_name     = 'find_list'
-character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
-                                               '(' // trim(sub_name) // '): '
 character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
                                                '(' // trim(sub_name) // '): '
 character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
@@ -2620,16 +2572,6 @@ function fm_change_list(name)                                        &
 !
 character(len=*), intent(in)  :: name
 
-!+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-!        local parameters
-!+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-character(len=14), parameter :: sub_name     = 'fm_change_list'
-character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
-                                               '(' // trim(sub_name) // '): '
-character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
-                                               '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 !        local variables
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
@@ -2700,12 +2642,8 @@ function  fm_change_root(name)                                        &
 !        local parameters
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 character(len=14), parameter :: sub_name     = 'fm_change_root'
-character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
-                                               '(' // trim(sub_name) // '): '
 character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
                                                '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 !        local variables
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
@@ -2821,12 +2759,8 @@ function  fm_dump_list(name, recursive)                        &
 !        local parameters
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 character(len=12), parameter :: sub_name     = 'fm_dump_list'
-character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
-                                               '(' // trim(sub_name) // '): '
 character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
                                                '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 !        local variables
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
@@ -2920,16 +2854,6 @@ function fm_exists(name)                                                &
 !
 character(len=*), intent(in) :: name
 
-!+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-!        local parameters
-!+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-character(len=9),  parameter :: sub_name     = 'fm_exists'
-character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
-                                               '(' // trim(sub_name) // '): '
-character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
-                                               '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 !        local variables
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
@@ -2989,12 +2913,8 @@ function  fm_get_index(name)                        &
 !        local parameters
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 character(len=12), parameter :: sub_name     = 'fm_get_index'
-character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
-                                               '(' // trim(sub_name) // '): '
 character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
                                                '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 !        local variables
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
@@ -3070,16 +2990,6 @@ function  fm_get_current_list()                                        &
 !        arguments
 !
 
-!+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-!        local parameters
-!+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-character(len=19), parameter :: sub_name     = 'fm_get_current_list'
-character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
-                                               '(' // trim(sub_name) // '): '
-character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
-                                               '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 !        local variables
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
@@ -3171,12 +3081,8 @@ function  fm_get_length(name)                        &
 !        local parameters
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 character(len=13), parameter :: sub_name     = 'fm_get_length'
-character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
-                                               '(' // trim(sub_name) // '): '
 character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
                                                '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 !        local variables
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
@@ -3268,12 +3174,8 @@ function  fm_get_type(name)                        &
 !        local parameters
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 character(len=11), parameter :: sub_name     = 'fm_get_type'
-character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
-                                               '(' // trim(sub_name) // '): '
 character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
                                                '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 !        local variables
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
@@ -3370,12 +3272,8 @@ function  fm_get_value_integer(name, value, index)                 &
 !        local parameters
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 character(len=20), parameter :: sub_name     = 'fm_get_value_integer'
-character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
-                                               '(' // trim(sub_name) // '): '
 character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
                                                '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 !        local variables
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
@@ -3497,12 +3395,8 @@ function  fm_get_value_logical(name, value, index)                 &
 !        local parameters
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 character(len=20), parameter :: sub_name     = 'fm_get_value_logical'
-character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
-                                               '(' // trim(sub_name) // '): '
 character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
                                                '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 !        local variables
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
@@ -3627,12 +3521,8 @@ function  fm_get_value_real(name, value, index)                 &
 !        local parameters
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 character(len=17), parameter :: sub_name     = 'fm_get_value_real'
-character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
-                                               '(' // trim(sub_name) // '): '
 character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
                                                '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 !        local variables
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
@@ -3760,12 +3650,8 @@ function  fm_get_value_string(name, value, index)                 &
 !        local parameters
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 character(len=19), parameter :: sub_name     = 'fm_get_value_string'
-character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
-                                               '(' // trim(sub_name) // '): '
 character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
                                                '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 !        local variables
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
@@ -3922,8 +3808,6 @@ function fm_intersection(lists, dim)                        &
                                                '(' // trim(sub_name) // '): '
 character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
                                                '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 !        local variables
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
@@ -4130,12 +4014,8 @@ function  fm_loop_over_list(list, name, field_type, index)        &
 !        local parameters
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 character(len=17), parameter :: sub_name     = 'fm_loop_over_list'
-character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
-                                               '(' // trim(sub_name) // '): '
 character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
                                                '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 !        local variables
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
@@ -4271,12 +4151,8 @@ function  fm_new_list(name, create, keep)                        &
 !        local parameters
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 character(len=11), parameter :: sub_name     = 'fm_new_list'
-character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
-                                               '(' // trim(sub_name) // '): '
 character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
                                                '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 !        local variables
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
@@ -4427,12 +4303,8 @@ function  fm_new_value_integer(name, value, create, index, append)     &
 !        local parameters
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 character(len=20), parameter :: sub_name     = 'fm_new_value_integer'
-character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
-                                               '(' // trim(sub_name) // '): '
 character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
                                                '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 !        local variables
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
@@ -4660,12 +4532,8 @@ function  fm_new_value_logical(name, value, create, index, append) &
 !        local parameters
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 character(len=20), parameter :: sub_name     = 'fm_new_value_logical'
-character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
-                                               '(' // trim(sub_name) // '): '
 character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
                                                '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 !        local variables
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
@@ -4899,12 +4767,8 @@ function  fm_new_value_real(name, value, create, index, append) &
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 
 character(len=17), parameter :: sub_name     = 'fm_new_value_real'
-character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
-                                               '(' // trim(sub_name) // '): '
 character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
                                                '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 !        local variables
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
@@ -5126,12 +4990,8 @@ function  fm_new_value_string(name, value, create, index, append) &
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 
 character(len=19), parameter :: sub_name     = 'fm_new_value_string'
-character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
-                                               '(' // trim(sub_name) // '): '
 character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
                                                '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 !        local variables
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
@@ -5367,17 +5227,6 @@ subroutine  fm_reset_loop
 !        arguments
 !
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-!        local parameters
-!+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-
-character(len=13), parameter :: sub_name     = 'fm_reset_loop'
-character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
-                                               '(' // trim(sub_name) // '): '
-character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
-                                               '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
-!+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 !        local variables
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 
@@ -5420,17 +5269,6 @@ subroutine  fm_return_root  !{
 !        arguments
 !
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-!        local parameters
-!+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-
-character(len=14), parameter :: sub_name     = 'fm_return_root'
-character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
-                                               '(' // trim(sub_name) // '): '
-character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
-                                               '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
-!+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 !        local variables
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 !
@@ -5495,16 +5333,6 @@ function get_field(name, this_list_p)                                        &
 character(len=*), intent(in)     :: name
 type (field_def), pointer        :: this_list_p
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-!        local parameters
-!+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-character(len=9),  parameter :: sub_name     = 'get_field'
-character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
-                                               '(' // trim(sub_name) // '): '
-character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
-                                               '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
-!+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 !        local variables
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 character(len=fm_path_name_len)  :: path
@@ -5574,16 +5402,6 @@ function fm_modify_name(oldname, newname)
 character(len=*), intent(in)     :: oldname
 character(len=*), intent(in)     :: newname
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-!        local parameters
-!+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-character(len=14), parameter :: sub_name     = 'fm_modify_name'
-character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
-                                               '(' // trim(sub_name) // '): '
-character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
-                                               '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
-!+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 !        local variables
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 character(len=fm_path_name_len)  :: path
@@ -5643,16 +5461,6 @@ subroutine initialize  !{
 !        arguments
 !
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-!        local parameters
-!+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-character(len=10), parameter :: sub_name     = 'initialize'
-character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
-                                               '(' // trim(sub_name) // '): '
-character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
-                                               '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
-!+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 !        local variables
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 integer :: ier
@@ -5744,12 +5552,8 @@ function  make_list(this_list_p, name)                        &
 !        local parameters
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 character(len=9),  parameter :: sub_name     = 'make_list'
-character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
-                                               '(' // trim(sub_name) // '): '
 character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
                                                '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 !        local variables
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
@@ -5851,12 +5655,8 @@ function fm_query_method(name, method_name, method_control)                &
 !        local parameters
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 character(len=15), parameter :: sub_name     = 'fm_query_method'
-character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
-                                               '(' // trim(sub_name) // '): '
 character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
                                                '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 !        local variables
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
@@ -5973,14 +5773,8 @@ recursive function query_method(list_p, recursive, name, method_name, method_con
 !        local parameters
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 character(len=12), parameter :: sub_name     = 'query_method'
-character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
-                                               '(' // trim(sub_name) // '): '
 character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
                                                '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
-integer,                  parameter :: max_depth = 64
-character(len=max_depth), parameter :: blank = '    '
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 !        local variables
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
@@ -6128,8 +5922,6 @@ function fm_copy_list(list_name, suffix, create ) &
                                                '(' // trim(sub_name) // '): '
 character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
                                                '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 !        local variables
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
@@ -6287,12 +6079,8 @@ function fm_find_methods(list_name, methods, control ) &
 !        local parameters
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 character(len=15), parameter :: sub_name     = 'fm_find_methods'
-character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
-                                               '(' // trim(sub_name) // '): '
 character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
                                                '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 !        local variables
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
@@ -6403,17 +6191,8 @@ recursive function find_method(list_p, recursive, num_meth, method, control)   &
 !        local parameters
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 character(len=11), parameter :: sub_name     = 'find_method'
-character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
-                                               '(' // trim(sub_name) // '): '
 character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
                                                '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
-!+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-!        local parameters
-!+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-integer, parameter                          :: max_depth = 64
-character(len=max_depth), parameter         :: blank = '    '
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 !        local variables
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
@@ -6558,10 +6337,6 @@ subroutine  fm_set_verbosity(verbosity)  !{
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 
 character(len=16), parameter :: sub_name     = 'fm_set_verbosity'
-character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
-                                               '(' // trim(sub_name) // '): '
-character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
-                                               '(' // trim(sub_name) // '): '
 character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
                                                '(' // trim(sub_name) // '): '
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
diff --git a/src/shared/field_manager/fm_util.F90 b/src/shared/field_manager/fm_util.F90
index 578c1dff6b..5bc1cb1824 100644
--- a/src/shared/field_manager/fm_util.F90
+++ b/src/shared/field_manager/fm_util.F90
@@ -1,9 +1,9 @@
 module fm_util_mod  !{
 ! 
-! Richard D. Slater
+! Richard D. Slater
 !
 !
-! John P. Dunne
+! John P. Dunne
 !
 !
 !
@@ -87,8 +87,8 @@ module fm_util_mod  !{
 character(len=fm_path_name_len) :: save_current_list
 character(len=fm_path_name_len) :: save_path
 character(len=fm_path_name_len) :: save_name
-character(len=128) :: version = '$Id: fm_util.F90,v 17.0 2009/07/21 03:19:16 fms Exp $'
-character(len=128) :: tagname = '$Name: tikal $'
+character(len=128) :: version = '$Id$'
+character(len=128) :: tagname = '$Name$'
 
 !
 !        Interface definitions for overloaded routines
@@ -146,12 +146,6 @@ subroutine fm_util_set_caller(caller)  !{
 
 character(len=*), intent(in)          :: caller
 
-!
-!       Local parameters
-!
-
-character(len=48), parameter  :: sub_name = 'fm_util_set_caller'
-
 !
 !       Local variables
 !
@@ -196,12 +190,6 @@ subroutine fm_util_reset_caller  !{
 !       arguments
 !
 
-!
-!       Local parameters
-!
-
-character(len=48), parameter  :: sub_name = 'fm_util_reset_caller'
-
 !
 !       Local variables
 !
@@ -239,12 +227,6 @@ subroutine fm_util_set_good_name_list(good_name_list)  !{
 
 character(len=*), intent(in)          :: good_name_list
 
-!
-!       Local parameters
-!
-
-character(len=48), parameter  :: sub_name = 'fm_util_set_good_name_list'
-
 !
 !       Local variables
 !
@@ -285,12 +267,6 @@ subroutine fm_util_reset_good_name_list  !{
 !       arguments
 !
 
-!
-!       Local parameters
-!
-
-character(len=48), parameter  :: sub_name = 'fm_util_reset_good_name_list'
-
 !
 !       Local variables
 !
@@ -328,12 +304,6 @@ subroutine fm_util_set_no_overwrite(no_overwrite)  !{
 
 logical, intent(in)          :: no_overwrite
 
-!
-!       Local parameters
-!
-
-character(len=48), parameter  :: sub_name = 'fm_util_set_no_overwrite'
-
 !
 !       Local variables
 !
@@ -374,12 +344,6 @@ subroutine fm_util_reset_no_overwrite  !{
 !       arguments
 !
 
-!
-!       Local parameters
-!
-
-character(len=48), parameter  :: sub_name = 'fm_util_reset_no_overwrite'
-
 !
 !       Local variables
 !
diff --git a/src/shared/fms/fms.F90 b/src/shared/fms/fms.F90
index f0146bd219..6f46b00d13 100644
--- a/src/shared/fms/fms.F90
+++ b/src/shared/fms/fms.F90
@@ -1,10 +1,11 @@
 
 module fms_mod
 
-! 
+! 
 !   Bruce Wyman
 ! 
 
+! 
 
 ! 
 !   The fms module provides routines that are commonly used
@@ -21,10 +22,12 @@ module fms_mod
 !     These include namelist files, restart files, and 32-bit IEEE
 !     data files. There also is a matching interface to close the files.
 !     If other file types are needed the mpp_open and mpp_close
+!     interfaces in module mpp_io must be used.
    
 !    3. Read and write distributed data to simple native unformatted files.
 !     This type of file (called a restart file) is used to checkpoint
 !     model integrations for a subsequent restart of the run.
    
 !    4. For convenience there are several routines published from
+!     the mpp module. These are routines for getting processor
 !     numbers, commonly used I/O unit numbers, error handling, and timing sections of code.
 ! 
 
@@ -134,14 +137,15 @@ module fms_mod
                        MPP_SINGLE, MPP_MULTI, MPP_DELETE, mpp_io_exit, &
                        fieldtype, mpp_get_atts, mpp_get_info, mpp_get_fields
 
-use fms_io_mod, only : read_data, write_data, fms_io_init, fms_io_exit, field_size, &
+use fms_io_mod, only : fms_io_init, fms_io_exit, field_size, &
+                       read_data, write_data, read_compressed, read_distributed, &
                        open_namelist_file, open_restart_file, open_ieee32_file, close_file, &
                        set_domain, get_domain_decomp, nullify_domain, &
                        open_file, open_direct_file, string, get_mosaic_tile_grid, &
                        get_mosaic_tile_file, get_global_att_value, file_exist, field_exist
 
 use memutils_mod, only: print_memuse_stats, memutils_init
-use version_mod, only: MOM_VERSION
+use constants_mod, only: constants_version=>version, constants_tagname=>tagname !pjp: PI not computed
 
 
 implicit none
@@ -156,7 +160,7 @@ module fms_mod
           open_file, open_direct_file
 
 ! routines for reading/writing distributed data
-public :: set_domain, read_data, write_data
+public :: set_domain, read_data, write_data, read_compressed, read_distributed
 public :: get_domain_decomp, field_size, nullify_domain
 public :: get_global_att_value
 
@@ -221,6 +225,7 @@ module fms_mod
 !     module: mpp_clock_id, mpp_clock_begin, and mpp_clock_end.
 !     The fms module makes these routines public.
 !     A list of timed code sections will be printed to STDOUT.
+!     See the MPP
 !     module for more details.
 !       
 !   
@@ -230,6 +235,7 @@ module fms_mod
 !     DETAILED also turns on detailed message-passing performance diagnosis.
 !     Both SYNC and DETAILED will  work correctly on innermost clock nest
 !     and distort outer clocks, and possibly the overall code time.
+!     See the MPP
 !     module for more details.
 !   
 !   
@@ -274,6 +280,12 @@ module fms_mod
    logical, private :: do_nml_error_init = .true.
    private  nml_error_init
 
+
+!  ---- version number -----
+
+  character(len=128) :: version = '$Id$'
+  character(len=128) :: tagname = '$Name$'
+
   logical :: module_is_initialized = .FALSE.
 
 
@@ -401,7 +413,7 @@ subroutine fms_init (localcomm )
 
 !--- write version info and namelist to logfile ---
 
-    call write_version_number()
+    call write_version_number (version, tagname)
     if (mpp_pe() == mpp_root_pe()) then
       unit = stdlog()
       write (unit, nml=fms_nml)
@@ -411,6 +423,8 @@ subroutine fms_init (localcomm )
     call memutils_init( print_memory_usage )
     call print_memuse_stats('fms_init')
 
+    call write_version_number (constants_version,constants_tagname)
+
 end subroutine fms_init
 ! 
 
@@ -748,41 +762,60 @@ END SUBROUTINE nml_error_init
 ! 
 
 !   
-!     Prints to the log file (or a specified unit) the version (git hash)
+!     Prints to the log file (or a specified unit) the (cvs) version id string and
+!     (cvs) tag name.
 !   
 !   
-!     Prints to the log file (stdlog) or a specified unit the version (git hash)
+!     Prints to the log file (stdlog) or a specified unit the (cvs) version id string
+!      and (cvs) tag name.
 !   
 !   
+
+!   
+!    string that contains routine name and version number.
+!   
+!   
+!    The tag/name string, this is usually the Name string
+!    returned by CVS when checking out the code.
+!   
 !   
 !    The Fortran unit number of an open formatted file. If this unit number 
 !    is not supplied the log file unit number is used (stdlog). 
 !   
 ! prints module version number to the log file of specified unit number
 
- subroutine write_version_number (unit)
+ subroutine write_version_number (version, tag, unit)
 
+!   in:  version = string that contains routine name and version number
+!
 !   optional in:
-!        unit    = alternate unit number to direct output
+!        tag = cvs tag name that code was checked out with
+!        unit    = alternate unit number to direct output  
 !                  (default: unit=stdlog)
 
-   integer,          intent(in), optional :: unit
+   character(len=*), intent(in) :: version
+   character(len=*), intent(in), optional :: tag 
+   integer,          intent(in), optional :: unit 
 
-   integer :: logunit
+   integer :: logunit 
 
-   if (.not.module_is_initialized) call fms_init ()
+   if (.not.module_is_initialized) call fms_init ( )
 
      logunit = stdlog()
      if (present(unit)) then
          logunit = unit
-     else
+     else    
        ! only allow stdlog messages on root pe
          if ( mpp_pe() /= mpp_root_pe() ) return
-     endif
+     endif   
 
-     write (logunit,'(/,80("="),/(a))') trim(MOM_VERSION)
+     if (present(tag)) then
+         write (logunit,'(/,80("="),/(a))') trim(version), trim(tag)
+     else    
+         write (logunit,'(/,80("="),/(a))') trim(version)
+     endif   
 
  end subroutine write_version_number
 ! 
diff --git a/src/shared/fms/fms_io.F90 b/src/shared/fms/fms_io.F90
index cee4b0f52c..d73787e3c3 100644
--- a/src/shared/fms/fms_io.F90
+++ b/src/shared/fms/fms_io.F90
@@ -4,19 +4,19 @@ module fms_io_mod
 
 !
 !
-! 
+! 
 ! Zhi Liang
 ! 
 
-! 
-! M.J. Harrison 
+! 
+! M.J. Harrison
 ! 
 !
-! 
-! M.J. Harrison 
+! 
+! M.J. Harrison
 ! 
 
-! 
+! 
 ! B. Wyman
 ! 
 
@@ -36,27 +36,14 @@ module fms_io_mod
 ! fms_netcdf_override
 ! fms_netcdf_restart
 !
-! because default values of both flags are .true., the default behavior of the entire model is 
+! because default values of both flags are .true., the default behavior of the entire model is
 ! to use netCDF IO mode. To turn off netCDF restart, simply set fms_netcdf_restart to .false.
 !
-! Fei.Liu
-! 05222006
-! Read distributed files in NetCDF is available. Details can be found in read_data_3d_new
-! 
    -!threading_read='multi', threading_write='multi', fileset_write='multi' (default)
-!threading_read='multi', threading_write='single', fileset_write='single'
-! 
    
 !
-!   
+! 
 ! 
 ! threading_read can be 'single' or 'multi'
 ! 
-! 
-! threading_write can be 'single' or 'multi'
-! 
-! 
-! fileset_write can be 'single' or 'multi'
-! 
 ! 
 !   .true. : fms_netcdf_restart overrides individual do_netcdf_restart value (default behavior)
 !   .false.: individual module settings has a precedence over the global setting, therefore fms_netcdf_restart is ignored
@@ -68,18 +55,18 @@ module fms_io_mod
 !   module setting takes over.
 ! 
 ! 
-!   .true. : time_stamp will be added to the restart file name as a prefix when 
-!            optional argument time_stamp is passed into routine save_restart. 
+!   .true. : time_stamp will be added to the restart file name as a prefix when
+!            optional argument time_stamp is passed into routine save_restart.
 !   .false.: time_stmp will not be added to the restart file name even though
 !            time_stamp is passed into save_restart.
 !    default is true.
 ! 
 ! 
 !    set print_chksum (default is false) to true to print out chksum of fields that are
-!    read and written through save_restart/restore_state. The chksum is accross all the 
-!    processors, so there will be only one chksum even there are multiple-tiles in the 
+!    read and written through save_restart/restore_state. The chksum is accross all the
+!    processors, so there will be only one chksum even there are multiple-tiles in the
 !    grid. For the multiple case, the filename appeared in the message will contain
-!    tile1 because the message is print out from root pe and on root pe the tile id is tile1. 
+!    tile1 because the message is print out from root pe and on root pe the tile id is tile1.
 ! 
 ! 
 !    set debug_mask_list (default is false) to true to print out mask_list reading from mask_table.
@@ -88,7 +75,7 @@ module fms_io_mod
 !    Set checksum_required (default is true) to true to compare checksums stored in the attribute of a
 !    field against the checksum after reading in the data. This check mitigates the possibility of data
 !    that gets corrupted on write or read from being used in a n ongoing fashion. The checksum is across
-!    all the  processors, so there will be only one checksum even if there are multiple-tiles in the 
+!    all the  processors, so there will be only one checksum even if there are multiple-tiles in the
 !    grid. For the decomposed file case, the filename appearing in the message will contain tile1
 !    because the message is printed out from the root pe and on root pe the tile id is tile1.
 !
@@ -99,20 +86,26 @@ module fms_io_mod
 
 use mpp_io_mod,      only: mpp_open, mpp_close, mpp_io_init, mpp_io_exit, mpp_read, mpp_write
 use mpp_io_mod,      only: mpp_write_meta, mpp_get_info, mpp_get_atts, mpp_get_fields
+use mpp_io_mod,      only: mpp_read_compressed, mpp_write_compressed, mpp_def_dim
+use mpp_io_mod,      only: mpp_write_unlimited_axis, mpp_read_distributed_ascii
 use mpp_io_mod,      only: mpp_get_axes, mpp_get_axis_data, mpp_get_att_char, mpp_get_att_name
-use mpp_io_mod,      only: mpp_get_att_real_scalar, mpp_attribute_exist
+use mpp_io_mod,      only: mpp_get_att_real_scalar, mpp_attribute_exist, mpp_is_dist_ioroot
 use mpp_io_mod,      only: fieldtype, axistype, atttype, default_field, default_axis, default_att
 use mpp_io_mod,      only: MPP_NETCDF, MPP_ASCII, MPP_MULTI, MPP_SINGLE, MPP_OVERWR, MPP_RDONLY
 use mpp_io_mod,      only: MPP_IEEE32, MPP_NATIVE, MPP_DELETE, MPP_APPEND, MPP_SEQUENTIAL, MPP_DIRECT
 use mpp_io_mod,      only: MAX_FILE_SIZE, mpp_get_att_value
-use mpp_domains_mod, only: domain2d, domain1d, NULL_DOMAIN1D, NULL_DOMAIN2D, operator( .EQ. ), CENTER
+use mpp_io_mod,      only: mpp_get_dimension_length
+use mpp_domains_mod, only: domain2d, domain1d, NULL_DOMAIN1D, NULL_DOMAIN2D, operator( .EQ. )
+use mpp_domains_mod, only: CENTER, EAST, WEST, NORTH, SOUTH, CORNER
 use mpp_domains_mod, only: mpp_get_domain_components, mpp_get_compute_domain, mpp_get_data_domain
 use mpp_domains_mod, only: mpp_get_domain_shift, mpp_get_global_domain, mpp_global_field, mpp_domain_is_tile_root_pe
-use mpp_domains_mod, only: mpp_get_ntile_count, mpp_get_current_ntile, mpp_get_tile_id, mpp_mosaic_defined
-use mpp_domains_mod, only: mpp_get_io_domain
+use mpp_domains_mod, only: mpp_get_ntile_count, mpp_get_current_ntile, mpp_get_tile_id
+use mpp_domains_mod, only: mpp_get_pelist, mpp_get_io_domain, mpp_get_domain_npes
 use mpp_mod,         only: mpp_error, FATAL, NOTE, WARNING, mpp_pe, mpp_root_pe, mpp_npes, stdlog, stdout
 use mpp_mod,         only: mpp_broadcast, ALL_PES, mpp_chksum, mpp_get_current_pelist, mpp_npes, lowercase
 use mpp_mod,         only: input_nml_file, mpp_get_current_pelist_name, uppercase
+use mpp_mod,         only: mpp_gather, mpp_scatter, mpp_send, mpp_recv, mpp_sync_self, COMM_TAG_1, EVENT_RECV
+use mpp_mod,	     only: MPP_FILL_DOUBLE,MPP_FILL_INT
 
 use platform_mod, only: r8_kind
 
@@ -129,29 +122,77 @@ module fms_io_mod
 integer, parameter, private :: MAX_TIME_LEVEL_WRITE = 20
 integer, parameter          :: max_axis_size=10000
 
+! Index postions for axes in restart_file_type
+! This is done so the user may define the axes
+! in any order but a check can be performed
+! to ensure no registration of duplicate axis
+integer, parameter, private :: XIDX=1
+integer, parameter, private :: YIDX=2
+integer, parameter, private :: CIDX=3
+integer, parameter, private :: ZIDX=4
+integer, parameter, private :: HIDX=5
+integer, parameter, private :: TIDX=6
+integer, parameter, private :: UIDX=7
+integer, parameter, private :: NIDX=7
+
+type meta_type
+  type(meta_type), pointer :: prev=>null(), next=>null()
+!!$ Gfortran on gaea does not yet support deferred length character strings
+!!$  character(len=:),allocatable  :: name
+  character(len=256)   :: name
+  real,    allocatable :: rval(:)
+  integer, allocatable :: ival(:)
+!!$ Gfortran on gaea does not yet support deferred length character strings
+!!$  character(len=:), allocatable :: cval
+  character(len=256)   :: cval
+end type meta_type
+
+type ax_type
+   private
+   character(len=128) :: name = ''
+   character(len=128) :: units = ''
+   character(len=128) :: longname = ''
+   character(len=8)   :: cartesian = ''
+   character(len=256) :: compressed = ''
+   character(len=128) :: dimlen_name = ''
+   character(len=128) :: dimlen_lname = ''
+   character(len=128) :: calendar = ''
+   integer            :: sense              !Orientation of z axis definition
+   integer            :: dimlen             !max dim of elements across global domain
+   real               :: min             !valid min for real axis data
+   integer            :: imin            !valid min for integer axis data
+   integer,allocatable :: idx(:)         !compressed io-domain index vector
+   integer,allocatable :: nelems(:)      !num elements for each rank in io domain
+   real, pointer      :: data(:) =>NULL()    !real axis values (not used if time axis)
+   type(domain2d),pointer :: domain =>NULL() ! domain associated with compressed axis
+end type ax_type
+
 type var_type
    private
-   character(len=128)                     :: name
-   character(len=128)                     :: longname
-   character(len=128)                     :: units
-   real, dimension(:,:,:,:), _ALLOCATABLE :: buffer _NULL 
-   logical                                :: domain_present
-   logical                                :: write_on_this_pe
-   integer                                :: domain_idx
-   logical                                :: is_dimvar
+   character(len=128)                     :: name = ''
+   character(len=128)                     :: longname = ''
+   character(len=128)                     :: units = ''
+   real, dimension(:,:,:,:), _ALLOCATABLE :: buffer _NULL
+   logical                                :: domain_present = .FALSE.
+   integer                                :: domain_idx = -1
+   logical                                :: is_dimvar = .FALSE.
+   logical                                :: read_only = .FALSE.
    type(fieldtype)                        :: field
    type(axistype)                         :: axis
    integer                                :: position
    integer                                :: ndim
-   integer                                :: siz(4)      ! X/Y/Z/T extent of fields (data domain 
+   integer                                :: siz(5)      ! X/Y/Z/T/A extent of fields (data domain
                                                          ! size for distributed writes;global size for reads)
-   integer                                :: gsiz(4)     ! global X/Y/Z/T extent of fields 
-   integer                                :: csiz(4)     ! actual data size in the file
-   integer                                :: id_axes(3)  ! store index for x/y/z axistype.
+   integer                                :: gsiz(4)     ! global X/Y/Z/A extent of fields
+   integer                                :: id_axes(4)  ! store index for x/y/z/a axistype.
    logical                                :: initialized ! indicate if the field is read or not in routine save_state.
    logical                                :: mandatory   ! indicate if the field is mandatory to be when restart.
    integer                                :: is, ie, js, je  ! index of the data in compute domain
    real                                   :: default_data 
+   character(len=8)                       :: compressed_axis !< If on a compressed axis, which axis
+   integer, dimension(:), allocatable     :: pelist
+   integer                                :: ishift, jshift ! can be used to shift indices when no_domain=T
+   integer                                :: x_halo, y_halo ! can be used to indicate halo size when no_domain=T
 end type var_type
 
 type Ptr0Dr
@@ -170,6 +211,10 @@ module fms_io_mod
    real, dimension(:,:,:), pointer :: p => NULL()
 end type Ptr3Dr
 
+type Ptr4Dr
+   real, dimension(:,:,:,:), pointer :: p => NULL()
+end type Ptr4Dr
+
 type Ptr0Di
    integer,                   pointer :: p => NULL()
 end type Ptr0Di
@@ -188,16 +233,24 @@ module fms_io_mod
 
 type restart_file_type
    private
-   integer                                  :: unit ! mpp_io unit for netcdf file
-   character(len=128)                       :: name
-   integer                                  :: nvar, natt, max_ntime
-   logical                                  :: is_root_pe
-   integer                                  :: tile_count
+   integer                                  :: unit = -1 ! mpp_io unit for netcdf file
+   character(len=128)                       :: name = ''
+   integer                                  :: register_id = 0
+   integer                                  :: nvar = 0
+   integer                                  :: natt = 0
+   integer                                  :: max_ntime = 0
+   logical                                  :: is_root_pe = .FALSE.
+   logical                                  :: is_compressed = .FALSE.
+   logical                                  :: unlimited_axis = .FALSE.
+   integer                                  :: tile_count = 1
+   type(ax_type),  allocatable              :: axes(:)  ! Currently define X,Y,Compressed, unlimited and maybe Z
+   type(meta_type),                pointer  :: first =>NULL() ! pointer to first additional global metadata element
    type(var_type), dimension(:),   pointer  :: var  => NULL()
    type(Ptr0Dr),   dimension(:,:), pointer  :: p0dr => NULL()
    type(Ptr1Dr),   dimension(:,:), pointer  :: p1dr => NULL()
    type(Ptr2Dr),   dimension(:,:), pointer  :: p2dr => NULL()
    type(Ptr3Dr),   dimension(:,:), pointer  :: p3dr => NULL()
+   type(Ptr4Dr),   dimension(:,:), pointer  :: p4dr => NULL()
    type(Ptr0Di),   dimension(:,:), pointer  :: p0di => NULL()
    type(Ptr1Di),   dimension(:,:), pointer  :: p1di => NULL()
    type(Ptr2Di),   dimension(:,:), pointer  :: p2di => NULL()
@@ -222,6 +275,24 @@ module fms_io_mod
    module procedure read_data_2d_region
 end interface
 
+interface read_distributed
+   module procedure read_distributed_r1D
+   module procedure read_distributed_r3D
+   module procedure read_distributed_r5D
+   module procedure read_distributed_i1D
+   module procedure read_distributed_iscalar
+   module procedure read_distributed_a1D
+end interface
+
+! Only need read compressed att; write is handled in with
+! mpp_io calls in save_compressed_restart
+interface read_compressed
+   module procedure read_compressed_i1d
+   module procedure read_compressed_i2d
+   module procedure read_compressed_1d
+   module procedure read_compressed_2d
+end interface read_compressed
+
 interface write_data
    module procedure write_data_3d_new
    module procedure write_data_2d_new
@@ -243,6 +314,7 @@ module fms_io_mod
    module procedure register_restart_field_r1d
    module procedure register_restart_field_r2d
    module procedure register_restart_field_r3d
+   module procedure register_restart_field_r4d
    module procedure register_restart_field_i0d
    module procedure register_restart_field_i1d
    module procedure register_restart_field_i2d
@@ -255,6 +327,14 @@ module fms_io_mod
    module procedure register_restart_field_i1d_2level
    module procedure register_restart_field_i2d_2level
    module procedure register_restart_field_i3d_2level
+   module procedure register_restart_region_r2d
+   module procedure register_restart_region_r3d
+end interface
+
+interface register_restart_axis
+   module procedure register_restart_axis_r1d
+   module procedure register_restart_axis_i1d
+   module procedure register_restart_axis_unlimited
 end interface
 
 interface reset_field_pointer
@@ -262,6 +342,7 @@ module fms_io_mod
    module procedure reset_field_pointer_r1d
    module procedure reset_field_pointer_r2d
    module procedure reset_field_pointer_r3d
+   module procedure reset_field_pointer_r4d
    module procedure reset_field_pointer_i0d
    module procedure reset_field_pointer_i1d
    module procedure reset_field_pointer_i2d
@@ -286,6 +367,7 @@ module fms_io_mod
    module procedure query_initialized_name
    module procedure query_initialized_r2d
    module procedure query_initialized_r3d
+   module procedure query_initialized_r4d
 end interface
 
 interface set_initialized
@@ -293,6 +375,7 @@ module fms_io_mod
    module procedure set_initialized_name
    module procedure set_initialized_r2d
    module procedure set_initialized_r3d
+   module procedure set_initialized_r4d
 end interface
 
 interface get_global_att_value
@@ -304,17 +387,20 @@ module fms_io_mod
   module procedure get_var_att_value_text
 end interface
 
+interface parse_mask_table
+  module procedure parse_mask_table_2d
+  module procedure parse_mask_table_3d
+end interface
+
 integer :: num_files_r = 0 ! number of currently opened files for reading
 integer :: num_files_w = 0 ! number of currently opened files for writing
 integer :: num_domains = 0 ! number of domains in array_domain
-integer :: num_registered_files ! mumber of files registered by calling register_restart_file
+integer :: num_registered_files = 0 ! mumber of files registered by calling register_restart_file
 
-integer :: thread_r, thread_w, fset_w, form
+integer :: thread_r, form
 logical :: module_is_initialized = .FALSE.
 
-character(len=32) :: pelist_name
-character(len=7)  :: pe_name
-character(len=128):: error_msg  
+character(len=128):: error_msg
 logical           :: great_circle_algorithm=.FALSE.
 
 !------ private data, pointer to current 2d domain ------
@@ -324,24 +410,27 @@ module fms_io_mod
 integer, private :: is,ie,js,je      ! compute domain
 integer, private :: isd,ied,jsd,jed  ! data domain
 integer, private :: isg,ieg,jsg,jeg  ! global domain
-character(len=128),      dimension(:), allocatable         :: registered_file ! file names registered through register_restart_file 
+character(len=128),      dimension(:), allocatable         :: registered_file ! file names registered through register_restart_file
 type(restart_file_type), dimension(:), allocatable         :: files_read  ! store files that are read through read_data
 type(restart_file_type), dimension(:), allocatable, target :: files_write ! store files that are written through write_data
-type(domain2d), dimension(max_domains), save       :: array_domain
+type(domain2d), dimension(max_domains), target, save  :: array_domain
 type(domain1d), dimension(max_domains), save       :: domain_x, domain_y
-public  :: read_data, write_data, fms_io_init, fms_io_exit, field_size
-public  :: open_namelist_file, open_restart_file, open_ieee32_file, close_file 
+public  :: read_data, read_compressed, write_data, read_distributed
+public  :: fms_io_init, fms_io_exit, field_size, get_field_size
+public  :: open_namelist_file, open_restart_file, open_ieee32_file, close_file
 public  :: set_domain, nullify_domain, get_domain_decomp, return_domain
 public  :: open_file, open_direct_file
 public  :: get_restart_io_mode, get_tile_string, string
-public  :: get_mosaic_tile_grid, get_mosaic_tile_file
+public  :: get_mosaic_tile_grid, get_mosaic_tile_file, get_file_name
 public  :: get_global_att_value, get_var_att_value
 public  :: file_exist, field_exist
-public  :: register_restart_field, save_restart, restore_state
-public  :: restart_file_type, query_initialized, set_initialized
+public  :: register_restart_field, register_restart_axis, save_restart, restore_state
+public  :: set_meta_global
+public  :: save_restart_border, restore_state_border
+public  :: restart_file_type, query_initialized, set_initialized, free_restart_type
 public  :: reset_field_name, reset_field_pointer
 private :: lookup_field_r, lookup_axis, unique_axes
-
+public  :: dimension_size
 public  :: set_filename_appendix, get_instance_filename
 public  :: parse_mask_table
 public  :: get_great_circle_algorithm
@@ -357,29 +446,27 @@ module fms_io_mod
 logical           :: fms_netcdf_override = .true.
 logical           :: fms_netcdf_restart  = .true.
 character(len=32) :: threading_read      = 'multi'
-character(len=32) :: threading_write     = 'multi'
-character(len=32) :: fileset_write       = 'multi'
-character(len=32) :: format              = 'netcdf' 
+character(len=32) :: format              = 'netcdf'
 logical           :: read_all_pe         = .TRUE.
 character(len=64) :: iospec_ieee32       = '-N ieee_32'
 integer           :: max_files_w         = 40
 integer           :: max_files_r         = 40
+integer           :: dr_set_size         = 10
 logical           :: read_data_bug       = .false.
 logical           :: time_stamp_restart  = .true.
 logical           :: print_chksum        = .false.
 logical           :: show_open_namelist_file_warning = .false.
-logical           :: debug_mask_list     = .false. 
-logical           :: checksum_required   = .true. 
+logical           :: debug_mask_list     = .false.
+logical           :: checksum_required   = .true.
   namelist /fms_io_nml/ fms_netcdf_override, fms_netcdf_restart, &
-       threading_read, threading_write, &
-       fileset_write, format, read_all_pe, iospec_ieee32,max_files_w,max_files_r, &
+       threading_read, format, read_all_pe, iospec_ieee32,max_files_w,max_files_r, &
        read_data_bug, time_stamp_restart, print_chksum, show_open_namelist_file_warning, &
-       debug_mask_list, checksum_required 
+       debug_mask_list, checksum_required, dr_set_size
 
 integer            :: pack_size  ! = 1 for double = 2 for float
 
-character(len=128) :: version = '$Id: fms_io.F90,v 20.0 2013/12/14 00:20:08 fms Exp $'
-character(len=128) :: tagname = '$Name: tikal $'
+character(len=128) :: version = '$Id$'
+character(len=128) :: tagname = '$Name$'
 
 contains
 
@@ -392,9 +479,9 @@ module fms_io_mod
 ! fms_netcdf_override
 ! fms_netcdf_restart
 !
-! because default values of both flags are .true., the default behavior of the entire model is 
+! because default values of both flags are .true., the default behavior of the entire model is
 ! to use netCDF IO mode. To turn off netCDF restart, simply set fms_netcdf_restart to .false.
-! 
+!
 ! 
 ! 
 subroutine fms_io_init()
-    
+
   integer                            :: i, unit, io_status, logunit
   integer, allocatable, dimension(:) :: pelist
   real(DOUBLE_KIND)                  :: doubledata = 0
@@ -457,7 +544,7 @@ subroutine fms_io_init()
   pack_size = size(transfer(doubledata, realarray))
   if( pack_size .NE. 1 .AND. pack_size .NE. 2) call mpp_error(FATAL,'=>fms_io_init: pack_size should be 1 or 2')
 
-  select case (threading_read) 
+  select case (threading_read)
   case ('multi')
      thread_r = MPP_MULTI
   case ('single')
@@ -467,24 +554,6 @@ subroutine fms_io_init()
   end select
 ! take namelist options if present
 
-  select case (fileset_write) 
-  case ('multi')
-     fset_w = MPP_MULTI
-  case ('single')
-     fset_w = MPP_SINGLE
-  case default
-     call mpp_error(FATAL,'fms_io_init: fileset_write should be multi/single but you chose'//trim(fileset_write))
-  end select
-
-  select case (threading_write) 
-  case ('multi')
-     thread_w = MPP_MULTI
-  case ('single')
-     thread_w = MPP_SINGLE
-  case default
-     call mpp_error(FATAL,'fms_io_init: threading_write should be multi/single but you chose'//trim(threading_write))
-  end select
-
   select case(format)
   case ('netcdf')
      form=MPP_NETCDF
@@ -496,15 +565,6 @@ subroutine fms_io_init()
   allocate(files_write(max_files_w),files_read(max_files_r))
   allocate(registered_file(max_files_w))
 
-  allocate(pelist(mpp_npes()))        
-  call mpp_get_current_pelist(pelist,pelist_name)
-  if(mpp_npes()>10000) then
-     write(pe_name,'(a,i6.6)' )'.', mpp_pe()    
-  else
-     write(pe_name,'(a,i4.4)' )'.', mpp_pe()    
-  endif
-  deallocate(pelist)
-
   do i = 1, max_domains
      array_domain(i) = NULL_DOMAIN2D
   enddo
@@ -513,7 +573,7 @@ subroutine fms_io_init()
 
   !This is set here instead of at the end of the routine to prevent the read_data call below from stopping the model
   module_is_initialized = .TRUE.
-  
+
   !--- read INPUT/grid_spec.nc to decide the value of great_circle_algorithm
   !--- great_circle_algorithm could be true only for mosaic grid.
   great_circle_algorithm = .false.
@@ -545,7 +605,7 @@ subroutine fms_io_init()
 
   if(great_circle_algorithm .AND. (mpp_pe() == mpp_root_pe()) ) then
      call mpp_error(NOTE,"fms_io_mod: great_circle algorithm will be used in the model run")
-  endif  
+  endif
 
 end subroutine fms_io_init
 
@@ -563,7 +623,7 @@ subroutine fms_io_exit()
     integer                             :: num_x_axes, num_y_axes, num_z_axes
     integer                             :: unit
     real, dimension(max_axis_size)      :: axisdata
-    real                                :: tlev  
+    real                                :: tlev
     integer,        dimension(max_axes) :: id_x_axes, siz_x_axes
     integer,        dimension(max_axes) :: id_y_axes, siz_y_axes
     integer,        dimension(max_axes) :: id_z_axes, siz_z_axes
@@ -574,6 +634,8 @@ subroutine fms_io_exit()
     character(len=256)                  :: filename
     character(len=10)                   :: axisname
     logical                             :: domain_present
+    logical                             :: write_on_this_pe
+    type(domain2d), pointer :: io_domain =>NULL()
 
     if( .NOT.module_is_initialized )return !make sure it's only called once per PE
 
@@ -604,26 +666,36 @@ subroutine fms_io_exit()
        num_z_axes = unique_axes(files_write(i), 3, id_z_axes, siz_z_axes          )
 
        if( domain_present ) then
-          call mpp_open(unit,trim(filename),action=MPP_OVERWR,form=form,threading=thread_w,&
-               fileset=fset_w, is_root_pe=files_write(i)%is_root_pe, domain=array_domain(files_write(i)%var(j)%domain_idx))
+          call mpp_open(unit,trim(filename),action=MPP_OVERWR,form=form, &
+               is_root_pe=files_write(i)%is_root_pe, domain=array_domain(files_write(i)%var(j)%domain_idx))
        else  ! global data
           call mpp_open(unit,trim(filename),action=MPP_OVERWR,form=form,threading=MPP_SINGLE,&
                fileset=MPP_SINGLE, is_root_pe=files_write(i)%is_root_pe)
        end if
 
+       write_on_this_pe = .false.
+       if(domain_present) then
+          io_domain => mpp_get_io_domain(array_domain(files_write(i)%var(j)%domain_idx))
+          if(associated(io_domain)) then
+             if(mpp_domain_is_tile_root_pe(io_domain)) write_on_this_pe = .true.
+          endif
+       endif
+       !--- always write out from root pe
+       if( files_write(i)%is_root_pe ) write_on_this_pe = .true.
+
        do j = 1, num_x_axes
          if (j < 10) then
              write(axisname,'(a,i1)') 'xaxis_',j
           else
              write(axisname,'(a,i2)') 'xaxis_',j
-          endif          
+          endif
           if(id_x_axes(j) > 0) then
              call mpp_write_meta(unit,x_axes(j),axisname,'none',axisname, &
                   data=axisdata(1:siz_x_axes(j)),domain=domain_x(id_x_axes(j)),cartesian='X')
           else
              call mpp_write_meta(unit,x_axes(j),axisname,'none',axisname, &
                   data=axisdata(1:siz_x_axes(j)),cartesian='X')
-          endif             
+          endif
        end do
 
        do j = 1, num_y_axes
@@ -631,14 +703,14 @@ subroutine fms_io_exit()
              write(axisname,'(a,i1)') 'yaxis_',j
           else
              write(axisname,'(a,i2)') 'yaxis_',j
-          endif          
+          endif
           if(id_y_axes(j) > 0) then
              call mpp_write_meta(unit,y_axes(j),axisname,'none',axisname, &
                   data=axisdata(1:siz_y_axes(j)),domain=domain_y(id_y_axes(j)),cartesian='Y')
           else
              call mpp_write_meta(unit,y_axes(j),axisname,'none',axisname, &
                   data=axisdata(1:siz_y_axes(j)),cartesian='Y')
-          endif             
+          endif
        end do
 
        do j = 1, num_z_axes
@@ -646,7 +718,7 @@ subroutine fms_io_exit()
              write(axisname,'(a,i1)') 'zaxis_',j
           else
              write(axisname,'(a,i2)') 'zaxis_',j
-          endif          
+          endif
           call mpp_write_meta(unit,z_axes(j),axisname,'none',axisname, &
                data=axisdata(1:siz_z_axes(j)),cartesian='Z')
        end do
@@ -692,8 +764,7 @@ subroutine fms_io_exit()
              if(cur_var%domain_present) then
                 call mpp_write(unit, cur_var%field,array_domain(cur_var%domain_idx), cur_var%buffer(:,:,:,kk), tlev, &
                                default_data=cur_var%default_data)
-             else if (thread_w == MPP_MULTI .or. cur_var%write_on_this_pe .OR. &
-                      (files_write(i)%is_root_pe.and.thread_w == MPP_SINGLE)) then
+             else if (write_on_this_pe) then
                 call mpp_write(unit, cur_var%field, cur_var%buffer(:,:,:,kk), tlev)
              end if
           enddo ! end j loop
@@ -701,7 +772,7 @@ subroutine fms_io_exit()
        call mpp_close(unit)
     enddo ! end i loop
 
-    !--- release the memory 
+    !--- release the memory
 
     do i = 1,  num_files_w
        do j = 1, files_write(i)%nvar
@@ -712,7 +783,7 @@ subroutine fms_io_exit()
   cur_var=>NULL()
   module_is_initialized = .false.
   num_files_w = 0
-  num_files_r = 0    
+  num_files_r = 0
 
 end subroutine fms_io_exit
 !.....................................................................
@@ -720,7 +791,7 @@ end subroutine fms_io_exit
 
 ! 
     !
-    ! This subroutine performs writing "fieldname" to file "filename". All values of "fieldname" 
+    ! This subroutine performs writing "fieldname" to file "filename". All values of "fieldname"
     ! will be written to a temporary file. The final NETCDF file will be created only at a later step
     ! when the user calls fms_io_exit. Therefore, make sure that fms_io_exit is called after all
     ! fields have been written by this subroutine.
@@ -744,14 +815,14 @@ end subroutine fms_io_exit
 subroutine write_data_i3d_new(filename, fieldname, data, domain,                  &
                               no_domain, position, tile_count, data_default)
 
-  character(len=*), intent(in) :: filename, fieldname 
+  character(len=*), intent(in) :: filename, fieldname
   integer, dimension(:,:,:), intent(in) :: data
   type(domain2d), intent(in), optional :: domain
   logical, intent(in), optional :: no_domain
   integer, intent(in), optional :: position, tile_count, data_default
   real :: default_data
 
-  default_data = 0
+  default_data = TRANSFER(MPP_FILL_INT,default_data)
   if(present(data_default)) default_data = real(data_default)
 
   call write_data_3d_new(filename, fieldname, real(data), domain,  &
@@ -761,14 +832,14 @@ end subroutine write_data_i3d_new
 subroutine write_data_i2d_new(filename, fieldname, data, domain, &
                               no_domain, position, tile_count, data_default)
 
-  character(len=*), intent(in) :: filename, fieldname 
+  character(len=*), intent(in) :: filename, fieldname
   integer, dimension(:,:), intent(in) :: data
   type(domain2d), intent(in), optional :: domain
   logical, intent(in), optional :: no_domain
   integer, intent(in), optional :: position, tile_count, data_default
   real :: default_data
 
-  default_data = 0
+  default_data = TRANSFER(MPP_FILL_INT,default_data)
   if(present(data_default)) default_data = real(data_default)
   call write_data_2d_new(filename, fieldname, real(data), domain, &
                          no_domain, position, tile_count, data_default=default_data)
@@ -778,13 +849,13 @@ end subroutine write_data_i2d_new
 subroutine write_data_i1d_new(filename, fieldname, data, domain, &
                               no_domain, tile_count, data_default)
   type(domain2d), intent(in), optional :: domain
-  character(len=*), intent(in) :: filename, fieldname 
+  character(len=*), intent(in) :: filename, fieldname
   integer, dimension(:), intent(in) :: data
   logical, intent(in), optional :: no_domain
   integer, intent(in), optional :: tile_count, data_default
   real :: default_data
 
-  default_data = 0
+  default_data = TRANSFER(MPP_FILL_INT,default_data)
   if(present(data_default)) default_data = real(data_default)
   call write_data_1d_new(filename, fieldname, real(data), domain, &
                          no_domain, tile_count, data_default=default_data)
@@ -793,13 +864,13 @@ end subroutine write_data_i1d_new
 subroutine write_data_iscalar_new(filename, fieldname, data, domain, &
                                   no_domain, tile_count, data_default)
   type(domain2d), intent(in), optional :: domain
-  character(len=*), intent(in) :: filename, fieldname 
+  character(len=*), intent(in) :: filename, fieldname
   integer, intent(in) :: data
   logical, intent(in), optional :: no_domain
   integer, intent(in), optional :: tile_count, data_default
   real :: default_data
 
-  default_data = 0
+  default_data = TRANSFER(MPP_FILL_INT,default_data)
   if(present(data_default)) default_data = real(data_default)
   call write_data_scalar_new(filename, fieldname, real(data), domain, &
                              no_domain, tile_count, data_default=default_data)
@@ -809,11 +880,11 @@ end subroutine write_data_iscalar_new
 subroutine write_data_3d_new(filename, fieldname, data, domain, no_domain, scalar_or_1d, &
                              position, tile_count, data_default)
 
-  character(len=*),         intent(in)         :: filename, fieldname 
+  character(len=*),         intent(in)         :: filename, fieldname
   real, dimension(:,:,:),   intent(in)         :: data
   type(domain2d), optional, intent(in), target :: domain
   real,           optional, intent(in)         :: data_default
-  logical,        optional, intent(in)         :: no_domain   
+  logical,        optional, intent(in)         :: no_domain
   logical,        optional, intent(in)         :: scalar_or_1d
   integer,        optional, intent(in)         :: position, tile_count
 
@@ -832,16 +903,15 @@ subroutine write_data_3d_new(filename, fieldname, data, domain, no_domain, scala
   type(domain2d), pointer, save   :: d_ptr   =>NULL()
   type(var_type), pointer, save   :: cur_var =>NULL()
   type(restart_file_type), pointer, save :: cur_file =>NULL()
-  type(domain2d), pointer,save            :: io_domain=>NULL()
 
 ! Initialize files to default values
-  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(write_data_3d_new): need to call fms_io_init')  
+  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(write_data_3d_new): need to call fms_io_init')
 
 
   if(PRESENT(data_default))then
      default_data=data_default
   else
-     default_data=0.
+     default_data=MPP_FILL_DOUBLE
   endif
 
   if(present(tile_count) .AND. .not. present(domain)) call mpp_error(FATAL, &
@@ -857,7 +927,7 @@ subroutine write_data_3d_new(filename, fieldname, data, domain, no_domain, scala
 
   if(is_no_domain) then
      if(PRESENT(domain)) &
-       call mpp_error(FATAL, 'fms_io(write_data_3d_new): no_domain cannot be .true. when optional argument domain is present.')  
+       call mpp_error(FATAL, 'fms_io(write_data_3d_new): no_domain cannot be .true. when optional argument domain is present.')
   else if(PRESENT(domain))then
      d_ptr => domain
   else if (ASSOCIATED(Current_domain)) then
@@ -877,7 +947,7 @@ subroutine write_data_3d_new(filename, fieldname, data, domain, no_domain, scala
   !Append a string to the file name
   append_string=''
 
-  !If the filename_appendix  is set override the passed argument. 
+  !If the filename_appendix  is set override the passed argument.
   if(len_trim(filename_appendix) > 0)  then
      append_pelist = .true.
      append_string = filename_appendix
@@ -900,14 +970,14 @@ subroutine write_data_3d_new(filename, fieldname, data, domain, no_domain, scala
      endif
   enddo
 
-  if (index_file < 0) then 
+  if (index_file < 0) then
      if(num_files_w == max_files_w) &  ! need to have bigger max_files_w
-          call mpp_error(FATAL,'fms_io(write_data_3d_new): max_files_w exceeded, increase it via fms_io_nml')    
+          call mpp_error(FATAL,'fms_io(write_data_3d_new): max_files_w exceeded, increase it via fms_io_nml')
      ! record the file name in array files_write
      num_files_w=num_files_w + 1
      index_file = num_files_w
      cur_file => files_write(index_file)
-     cur_file%name = trim(fname)         
+     cur_file%name = trim(fname)
      cur_file%tile_count=1
      if(present(tile_count)) cur_file%tile_count = tile_count
      if(ASSOCIATED(d_ptr))then
@@ -922,7 +992,7 @@ subroutine write_data_3d_new(filename, fieldname, data, domain, no_domain, scala
      do i = 1, max_fields
         cur_file%var(i)%name           = 'none'
         cur_file%var(i)%domain_present = .false.
-        cur_file%var(i)%write_on_this_pe = .false.
+        cur_file%var(i)%read_only      = .false.
         cur_file%var(i)%domain_idx     = -1
         cur_file%var(i)%is_dimvar      = .false.
         cur_file%var(i)%position       = CENTER
@@ -938,7 +1008,7 @@ subroutine write_data_3d_new(filename, fieldname, data, domain, no_domain, scala
      if(trim(cur_file%var(i)%name) == trim(fieldname)) then
         index_field = i
         exit
-     end if 
+     end if
   end do
 
   if(index_field > 0) then
@@ -949,10 +1019,10 @@ subroutine write_data_3d_new(filename, fieldname, data, domain, no_domain, scala
      if( cur_var%siz(4) > MAX_TIME_LEVEL_WRITE ) call mpp_error(FATAL, 'fms_io(write_data_3d_new): ' // &
           'the time level of field '//trim(cur_var%name)//' in file '//trim(cur_file%name)// &
           ' is greater than MAX_TIME_LEVEL_WRITE(=20), increase MAX_TIME_LEVEL_WRITE or check your code')
-  else 
+  else
      cur_file%nvar = cur_file%nvar +1
      if(cur_file%nvar>max_fields) then
-        write(error_msg,'(I3,"/",I3)') cur_file%nvar, max_fields 
+        write(error_msg,'(I3,"/",I3)') cur_file%nvar, max_fields
         call  mpp_error(FATAL,'fms_io(write_data_3d_new): max_fields exceeded, needs increasing, nvar/max_fields=' &
              //trim(error_msg))
      endif
@@ -967,14 +1037,7 @@ subroutine write_data_3d_new(filename, fieldname, data, domain, no_domain, scala
      cur_var%default_data = default_data
      cur_var%ndim = 3
      if(present(position)) cur_var%position = position
-     
-     if(ASSOCIATED(d_ptr)) then
-        io_domain => mpp_get_io_domain(d_ptr)
-        if(associated(io_domain)) then
-           if(mpp_domain_is_tile_root_pe(io_domain)) cur_var%write_on_this_pe = .true.
-        endif
-     endif
-     
+
      if(ASSOCIATED(d_ptr) .AND. .NOT. is_scalar_or_1d)then
         cur_var%domain_present = .true.
         domain_idx = lookup_domain(d_ptr)
@@ -1012,9 +1075,9 @@ subroutine write_data_3d_new(filename, fieldname, data, domain, no_domain, scala
      endif
   end if
 
-  ! copy the data to the buffer 
-  ! if the time level is greater than the size(cur_var%buffer,4), 
-  ! need to increase the buffer size 
+  ! copy the data to the buffer
+  ! if the time level is greater than the size(cur_var%buffer,4),
+  ! need to increase the buffer size
 
   if(cur_var%siz(4) == 1) then
      allocate(cur_var%buffer(cur_var%siz(1), cur_var%siz(2), cur_var%siz(3), cur_var%siz(4)) )
@@ -1023,18 +1086,318 @@ subroutine write_data_3d_new(filename, fieldname, data, domain, no_domain, scala
      tmp_buffer = cur_var%buffer
      deallocate(cur_var%buffer)
      allocate(cur_var%buffer(cur_var%siz(1), cur_var%siz(2), cur_var%siz(3), cur_var%siz(4)) )
-     cur_var%buffer(:,:,:,1:size(tmp_buffer,4)) = tmp_buffer 
+     cur_var%buffer(:,:,:,1:size(tmp_buffer,4)) = tmp_buffer
      deallocate(tmp_buffer)
   endif
 
-  cur_var%buffer(:,:,:,cur_var%siz(4)) = data ! copy current data to buffer for future write out 
+  cur_var%buffer(:,:,:,cur_var%siz(4)) = data ! copy current data to buffer for future write out
 
   d_ptr =>NULL()
   cur_var =>NULL()
   cur_file =>NULL()
 
 end subroutine write_data_3d_new
-!   
+! 
+
+!-------------------------------------------------------------------------------
+!
+!   This routine will register an integer restart file axis
+!
+!-------------------------------------------------------------------------------
+subroutine register_restart_axis_r1d(fileObj,filename,fieldname,data,cartesian,units,longname,sense,min,calendar)
+  type(restart_file_type),    intent(inout)      :: fileObj
+  character(len=*),           intent(in)         :: filename, fieldname
+  real,                       intent(in), target :: data(:)
+  character(len=*),           intent(in)         :: cartesian
+  character(len=*), optional, intent(in)         :: units, longname
+  integer,          optional, intent(in)         :: sense
+  real,             optional, intent(in)         :: min !valid min for real axis data
+  character(len=*), optional, intent(in)         :: calendar
+
+  integer :: idx
+
+
+  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(register_restart_axis_r1d): need to call fms_io_init')
+
+  select case(trim(cartesian))
+    case('X')
+      idx = XIDX
+    case('Y')
+      idx = YIDX
+    case('Z')
+      idx = ZIDX
+    case('T')
+      idx = TIDX
+    case default
+      call mpp_error(FATAL,'fms_io(register_restart_axis_r1d): Axis must be one of X,Y or Z ' // &
+                                                           'but has value '//trim(cartesian))
+  end select
+  if(.not. ALLOCATED(fileObj%axes)) allocate(fileObj%axes(NIDX))
+  if(ASSOCIATED(fileObj%axes(idx)%data)) &
+       call mpp_error(FATAL,'fms_io(register_restart_axis_r1d): '//trim(cartesian)//' axis has already been defined')
+  fileObj%name = filename
+  fileObj%axes(idx)%name = fieldname
+  fileObj%axes(idx)%data =>data
+  fileObj%axes(idx)%cartesian = cartesian
+  fileObj%axes(idx)%dimlen = -1   ! This is not a compressed axis
+  if(PRESENT(units)) fileObj%axes(idx)%units = units
+  if(PRESENT(longname)) fileObj%axes(idx)%longname = longname
+  if(PRESENT(min)) fileObj%axes(idx)%min = min
+  if(idx == TIDX) then
+     if(PRESENT(calendar)) fileObj%axes(idx)%calendar = trim(calendar)
+  endif
+  if(PRESENT(sense)) then
+     if(idx /= ZIDX) call mpp_error(FATAL,'fms_io(register_restart_axis_r1d): Only the Z axis may define sense; ' // &
+                                    'Axis = '//trim(cartesian))
+     if(abs(sense) /= 1) call mpp_error(FATAL,'fms_io(register_restart_axis_r1d): Value of sense must be +/- 1')
+     fileObj%axes(idx)%sense = sense
+  endif
+end subroutine register_restart_axis_r1d
+
+!-------------------------------------------------------------------------------
+!
+!   This routine will register the compressed index restart file axis
+!
+!-------------------------------------------------------------------------------
+subroutine register_restart_axis_i1d(fileObj,filename,fieldname,data,compressed, &
+                                     compressed_axis,dimlen,dimlen_name,dimlen_lname,units,longname,imin)
+  type(restart_file_type),    intent(inout)      :: fileObj
+  character(len=*),           intent(in)         :: filename, fieldname
+  integer,                    intent(in)         :: data(:)
+  character(len=*),           intent(in)         :: compressed
+  character(len=*),           intent(in)         :: compressed_axis !< which compressed axis (C or H)
+  integer,                    intent(in)         :: dimlen
+  character(len=*), optional, intent(in)         :: dimlen_name, dimlen_lname !< dimlen axis name and longname
+  character(len=*), optional, intent(in)         :: units, longname
+  integer,          optional, intent(in)         :: imin !valid min for integer axis data
+
+  integer :: ssize,rsize,npes
+  integer :: idx
+  integer, allocatable :: pelist(:)
+  type(domain2d), pointer :: io_domain=>NULL()
+
+
+  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(register_restart_axis_i1d): need to call fms_io_init')
+
+  select case(trim(compressed_axis))
+  case('C')
+     idx = CIDX
+  case('H')
+     idx = HIDX
+  case default
+     call mpp_error(FATAL,'fms_io(register_restart_axis_r1d): Axis must be one of C or H ' // &
+          'but has value '//trim(compressed_axis))
+  end select
+  
+  if(.not. ALLOCATED(fileObj%axes)) allocate(fileObj%axes(NIDX))
+  if(ALLOCATED(fileObj%axes(idx)%idx)) &
+                 call mpp_error(FATAL,'fms_io(register_restart_axis_i1d): Compressed axis ' //&
+                 trim(compressed_axis) // ' has already been defined')
+  fileObj%name = filename
+  fileObj%is_compressed = .true.
+  fileObj%unlimited_axis = .false.
+  fileObj%axes(idx)%name = fieldname
+  if(ASSOCIATED(current_domain)) then
+     fileObj%axes(idx)%domain =>current_domain
+     io_domain =>mpp_get_io_domain(current_domain)
+     if(.not. ASSOCIATED(io_domain)) &
+                 call mpp_error(FATAL,'fms_io(register_restart_axis_i1d): The io domain must be defined')
+     npes = mpp_get_domain_npes(io_domain)
+     allocate(fileObj%axes(idx)%nelems(npes)); fileObj%axes(idx)%nelems = 0
+     allocate(pelist(npes))
+     call mpp_get_pelist(io_domain,pelist)
+     ssize = size(data)
+     call mpp_gather((/ssize/),fileObj%axes(idx)%nelems,pelist)
+     rsize = sum(fileObj%axes(idx)%nelems)
+     allocate( fileObj%axes(idx)%idx(rsize) )
+  !  Note that the gatherV implied here is asymmetric; only root needs to know the vector of recv sizes
+     call mpp_gather(data,ssize,fileObj%axes(idx)%idx,fileObj%axes(idx)%nelems,pelist)
+     deallocate(pelist); io_domain=>NULL()
+  else
+     call mpp_error(FATAL,'fms_io(register_restart_axis_i1d): The domain must be defined through set_domain')
+  endif
+  fileObj%axes(idx)%compressed = compressed
+  fileObj%axes(idx)%dimlen = dimlen
+  if(PRESENT(dimlen_name)) fileObj%axes(idx)%dimlen_name = dimlen_name
+  if(PRESENT(dimlen_lname)) fileObj%axes(idx)%dimlen_lname = dimlen_lname
+  if(PRESENT(units)) fileObj%axes(idx)%units = units
+  if(PRESENT(longname)) fileObj%axes(idx)%longname = longname
+  if(PRESENT(imin)) fileObj%axes(idx)%imin = imin
+end subroutine register_restart_axis_i1d
+
+!-------------------------------------------------------------------------------
+
+subroutine register_restart_axis_unlimited(fileObj,filename,fieldname,nelem,units,longname)
+  type(restart_file_type),    intent(inout)      :: fileObj
+  character(len=*),           intent(in)         :: filename, fieldname
+  integer                                        :: nelem  ! Number of elements on rank
+  character(len=*), optional, intent(in)         :: units, longname
+
+  integer :: idx,npes
+  integer, allocatable :: pelist(:)
+  type(domain2d), pointer :: io_domain=>NULL()
+
+
+  if(.not.module_is_initialized) &
+                call mpp_error(FATAL,'fms_io(register_restart_axis_unlimited): need to call fms_io_init')
+  idx = UIDX
+
+  if(.not. ALLOCATED(fileObj%axes)) allocate(fileObj%axes(NIDX))
+  if(ALLOCATED(fileObj%axes(idx)%idx)) &
+               call mpp_error(FATAL,'fms_io(register_restart_axis_unlimited): Unlimited axis has already been defined')
+  fileObj%name = filename
+  fileObj%is_compressed = .false.
+  fileObj%unlimited_axis = .true.
+  fileObj%axes(idx)%name = fieldname
+  if(ASSOCIATED(current_domain)) then
+     fileObj%axes(idx)%domain =>current_domain
+     io_domain =>mpp_get_io_domain(current_domain)
+     if(.not. ASSOCIATED(io_domain)) &
+                 call mpp_error(FATAL,'fms_io(register_restart_axis_i1d): The io domain must be defined')
+     npes = mpp_get_domain_npes(io_domain)
+     allocate(fileObj%axes(idx)%nelems(npes)); fileObj%axes(idx)%nelems = 0
+     allocate(pelist(npes))
+     call mpp_get_pelist(io_domain,pelist)
+     call mpp_gather((/nelem/),fileObj%axes(idx)%nelems,pelist)
+     deallocate(pelist); io_domain=>NULL()
+  else
+     call mpp_error(FATAL,'fms_io(register_restart_axis_unlimited): The domain must be defined through set_domain')
+  endif
+  if(PRESENT(units)) fileObj%axes(idx)%units = units
+  if(PRESENT(longname)) fileObj%axes(idx)%longname = longname
+end subroutine register_restart_axis_unlimited
+
+!
+!   This routine is the destructor for the file object
+!
+!-------------------------------------------------------------------------------
+subroutine free_restart_type(fileObj)
+  type(restart_file_type), intent(inout)      :: fileObj
+  type(meta_type),pointer                :: this
+  type(meta_type),pointer                :: this_p
+  integer :: id, n
+
+  !--- remove file name from registered_file
+  id = fileObj%register_id
+  if( id > num_registered_files .OR. id < 1 ) then
+     print*, " register_id = ", id, " and num_registered_files = ", num_registered_files
+     call mpp_error(FATAL, &
+        'fms_io(free_restart_type): fileObj%register_id should be between 1 and num_registered_files')
+  endif
+  if( trim(fileObj%name) .NE. trim(registered_file(id)) ) &
+     call mpp_error(FATAL, 'fms_io(free_restart_type): fileObj%name .NE. registered_file(id)')
+  do n = id+1, num_registered_files
+     registered_file(n-1) = trim(registered_file(n))
+  enddo
+  registered_file(num_registered_files) = ''
+  num_registered_files = num_registered_files - 1
+
+  fileObj%register_id = 0
+  fileObj%unit = -1
+  fileObj%name = ''
+  fileObj%nvar = -1
+  fileObj%natt = -1
+  fileObj%max_ntime = -1
+  fileObj%tile_count = -1
+  if(ALLOCATED(fileObj%axes)) deallocate(fileObj%axes)
+  if(ASSOCIATED(fileObj%var)) deallocate(fileObj%var)
+  if(ASSOCIATED(fileObj%p0dr)) deallocate(fileObj%p0dr)
+  if(ASSOCIATED(fileObj%p1dr)) deallocate(fileObj%p1dr)
+  if(ASSOCIATED(fileObj%p2dr)) deallocate(fileObj%p2dr)
+  if(ASSOCIATED(fileObj%p3dr)) deallocate(fileObj%p3dr)
+  if(ASSOCIATED(fileObj%p0di)) deallocate(fileObj%p0di)
+  if(ASSOCIATED(fileObj%p1di)) deallocate(fileObj%p1di)
+  if(ASSOCIATED(fileObj%p2di)) deallocate(fileObj%p2di)
+  if(ASSOCIATED(fileObj%p3di)) deallocate(fileObj%p3di)
+  if(ASSOCIATED(fileObj%first)) then
+     this =>fileObj%first
+     do while(associated(this%next))
+        this =>this%next  ! Find the last element
+     enddo
+     do while(associated(this))  ! Deallocate from the last element to the first
+       this_p =>this%prev
+!!$ Gfortran on gaea does not yet support deferred length character strings
+!!$       deallocate(this%name)
+       this%name=''  ! Remove this line when Gfortran supports deferred length character strings
+       if(allocated(this%rval)) deallocate(this%rval)
+       if(allocated(this%ival)) deallocate(this%ival)
+!!$ Gfortran on gaea does not yet support deferred length character strings
+!!$       if(allocated(this%cval)) deallocate(this%cval)
+       this%cval=''  ! Remove this line when Gfortran supports deferred length character strings
+       deallocate(this)
+       this =>this_p
+     enddo
+     fileObj%first =>NULL()
+  endif
+end subroutine free_restart_type
+
+!-------------------------------------------------------------------------------
+!
+!   The routine sets up a list of global metadata expressions for save_restart
+!
+!-------------------------------------------------------------------------------
+subroutine set_meta_global(fileObj, name, rval, ival, cval)
+  type(restart_file_type), intent(inout) :: fileObj
+  character(len=*), intent(in)           :: name
+  real,             intent(in), optional :: rval(:)
+  integer,          intent(in), optional :: ival(:)
+  character(len=*), intent(in), optional :: cval
+  type(meta_type),pointer                :: this
+  type(meta_type),pointer                :: this_n
+
+  this =>fileObj%first
+  if(associated(this))then
+     do while(associated(this%next))
+        this =>this%next
+     enddo
+     allocate(this_n); this%next =>this_n; this_n%prev =>this; this =>this_n
+  else
+     allocate(this)
+     fileObj%first =>this
+  endif
+
+! Per mpp_write_meta_global, only one type of data can be associated with the metadata
+!!$ Gfortran on gaea does not yet support deferred length character strings
+!!$  allocate(character(len(name)) :: this%name); this%name = name
+  this%name = name  ! Remove this line when Gfortran supports deferred length character stings
+  if(present(rval))then
+     allocate(this%rval(size(rval))); this%rval=rval
+  elseif(present(ival))then
+     allocate(this%ival(size(ival))); this%ival=ival
+  elseif(present(cval))then
+!!$ Gfortran on gaea does not yet support deferred length character strings
+!!$     allocate(character(len(cval)) :: this%cval); this%cval = cval
+     this%cval=cval  ! Remove this line when Gfortran supports deferred length character stings
+  endif
+end subroutine set_meta_global
+
+
+!-------------------------------------------------------------------------------
+!
+!   The routine writes the global metadata
+!
+!-------------------------------------------------------------------------------
+subroutine write_meta_global(unit,fileObj)
+  integer,                 intent(in) :: unit
+  type(restart_file_type), intent(in) :: fileObj
+  type(meta_type), pointer            :: this
+
+  this =>fileObj%first
+  do while(associated(this))
+     if(allocated(this%rval))then
+        call mpp_write_meta(unit,this%name,rval=this%rval)
+     elseif(allocated(this%ival))then
+        call mpp_write_meta(unit,this%name,ival=this%ival)
+!!$ Gfortran on gaea does not yet support deferred length character strings
+!!$     elseif(allocated(this%cval))then
+     elseif(len_trim(this%cval).GT.0)then  ! Remove this line when Gfortran supports deferred length character stings
+        call mpp_write_meta(unit,this%name,cval=this%cval)
+     else
+        call mpp_write_meta(unit,this%name)
+     endif
+     this =>this%next
+  enddo
+end subroutine write_meta_global
 
 !-------------------------------------------------------------------------------
 !
@@ -1042,28 +1405,29 @@ end subroutine write_data_3d_new
 !
 !-------------------------------------------------------------------------------
 function register_restart_field_r0d(fileObj, filename, fieldname, data, domain, mandatory, &
-                                    no_domain, position, tile_count, data_default, longname, units)
+                                    no_domain, position, tile_count, data_default, longname, units, read_only)
   type(restart_file_type),    intent(inout)      :: fileObj
   character(len=*),           intent(in)         :: filename, fieldname
-  real,                       intent(in), target :: data 
+  real,                       intent(in), target :: data
   type(domain2d),   optional, intent(in), target :: domain
   logical,          optional, intent(in)         :: no_domain
   real,             optional, intent(in)         :: data_default
   logical,          optional, intent(in)         :: mandatory
   integer,          optional, intent(in)         :: position, tile_count
   character(len=*), optional, intent(in)         :: longname, units
+  logical,          optional, intent(in)         :: read_only
   integer                                        :: index_field
   integer                                        :: register_restart_field_r0d
 
-  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(register_restart_field_r0d): need to call fms_io_init')  
+  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(register_restart_field_r0d): need to call fms_io_init')
   call setup_one_field(fileObj, filename, fieldname, (/1, 1, 1, 1/), index_field, domain, mandatory, &
                        no_domain, scalar_or_1d=.true., position=position, tile_count=tile_count, &
-                       data_default=data_default, longname=longname, units=units)
+                       data_default=data_default, longname=longname, units=units, read_only=read_only)
   fileObj%p0dr(fileObj%var(index_field)%siz(4), index_field)%p => data
   fileObj%var(index_field)%ndim = 0
-  register_restart_field_r0d = index_field  
+  register_restart_field_r0d = index_field
 
-  return    
+  return
 
 end function register_restart_field_r0d
 
@@ -1073,30 +1437,32 @@ end function register_restart_field_r0d
 !
 !-------------------------------------------------------------------------------
 function register_restart_field_r1d(fileObj, filename, fieldname, data, domain, mandatory, &
-                             no_domain, position, tile_count, data_default, longname, units)
+                             no_domain, position, tile_count, data_default, longname, units, compressed_axis, read_only)
   type(restart_file_type), intent(inout)         :: fileObj
   character(len=*),           intent(in)         :: filename, fieldname
-  real, dimension(:),         intent(in), target :: data 
+  real, dimension(:),         intent(in), target :: data
   type(domain2d),   optional, intent(in), target :: domain
   logical,          optional, intent(in)         :: no_domain
   real,             optional, intent(in)         :: data_default
   integer,          optional, intent(in)         :: position, tile_count
   logical,          optional, intent(in)         :: mandatory
-  character(len=*), optional, intent(in)         :: longname, units
+  character(len=*), optional, intent(in)         :: longname, units, compressed_axis
+  logical,          optional, intent(in)         :: read_only
   integer                                        :: index_field
   integer                                        :: register_restart_field_r1d
 
-  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(register_restart_field_r1d): need to call fms_io_init')  
+  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(register_restart_field_r1d): need to call fms_io_init')
   call setup_one_field(fileObj, filename, fieldname, (/size(data,1), 1, 1, 1/), index_field, domain, mandatory, &
                        no_domain, scalar_or_1d=.true., position=position, tile_count=tile_count, &
-                       data_default=data_default, longname=longname, units=units )
+                       data_default=data_default, longname=longname, units=units, compressed_axis=compressed_axis, &
+                       read_only=read_only )
 
   fileObj%p1dr(fileObj%var(index_field)%siz(4), index_field)%p => data
   fileObj%var(index_field)%ndim = 1
-  register_restart_field_r1d = index_field  
+  register_restart_field_r1d = index_field
+
+  return
 
-  return    
-  
 end function register_restart_field_r1d
 
 !-------------------------------------------------------------------------------
@@ -1104,30 +1470,35 @@ end function register_restart_field_r1d
 !   The routine will register a 2-D real restart file field with one time level
 !
 !-------------------------------------------------------------------------------
-function register_restart_field_r2d(fileObj, filename, fieldname, data, domain, mandatory, &
-                                    no_domain, position, tile_count, data_default, longname, units)
+function register_restart_field_r2d(fileObj, filename, fieldname, data, domain, mandatory, no_domain, &
+                                    compressed, position, tile_count, data_default, longname, units, &
+                                    compressed_axis, read_only)
   type(restart_file_type), intent(inout)         :: fileObj
   character(len=*),           intent(in)         :: filename, fieldname
-  real,     dimension(:,:),   intent(in), target :: data 
+  real,     dimension(:,:),   intent(in), target :: data
   type(domain2d),   optional, intent(in), target :: domain
   real,             optional, intent(in)         :: data_default
-  logical,          optional, intent(in)         :: no_domain   
+  logical,          optional, intent(in)         :: no_domain
+  logical,          optional, intent(in)         :: compressed
   integer,          optional, intent(in)         :: position, tile_count
   logical,          optional, intent(in)         :: mandatory
-  character(len=*), optional, intent(in)         :: longname, units
-
+  character(len=*), optional, intent(in)         :: longname, units, compressed_axis
+  logical,          optional, intent(in)         :: read_only
+  logical                                        :: is_compressed
   integer                                        :: index_field
   integer                                        :: register_restart_field_r2d
 
-  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(register_restart_field_r2d): need to call fms_io_init')  
+  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(register_restart_field_r2d): need to call fms_io_init')
+  is_compressed = .false.
+  if(present(compressed)) is_compressed=compressed
   call setup_one_field(fileObj, filename, fieldname, (/size(data,1), size(data,2), 1, 1/), &
-                       index_field, domain, mandatory, no_domain, .false., &
-                       position, tile_count, data_default, longname, units)
+                       index_field, domain, mandatory, no_domain, is_compressed, &
+                       position, tile_count, data_default, longname, units, compressed_axis, read_only=read_only)
   fileObj%p2dr(fileObj%var(index_field)%siz(4), index_field)%p => data
   fileObj%var(index_field)%ndim = 2
-  register_restart_field_r2d = index_field    
+  register_restart_field_r2d = index_field
 
-  return    
+  return
 
 end function register_restart_field_r2d
 
@@ -1138,60 +1509,103 @@ end function register_restart_field_r2d
 !
 !-------------------------------------------------------------------------------
 function register_restart_field_r3d(fileObj, filename, fieldname, data, domain, mandatory, &
-                             no_domain, position, tile_count, data_default, longname, units)
+                             no_domain, position, tile_count, data_default, longname, units, read_only)
   type(restart_file_type), intent(inout)         :: fileObj
   character(len=*),           intent(in)         :: filename, fieldname
-  real,     dimension(:,:,:), intent(in), target :: data 
+  real,     dimension(:,:,:), intent(in), target :: data
   type(domain2d),   optional, intent(in), target :: domain
   real,             optional, intent(in)         :: data_default
-  logical,          optional, intent(in)         :: no_domain   
+  logical,          optional, intent(in)         :: no_domain
   integer,          optional, intent(in)         :: position, tile_count
   logical,          optional, intent(in)         :: mandatory
   character(len=*), optional, intent(in)         :: longname, units
+  logical,          optional, intent(in)         :: read_only
   integer                                        :: index_field
   integer                                        :: register_restart_field_r3d
 
-  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(register_restart_field_r3d): need to call fms_io_init')  
+  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(register_restart_field_r3d): need to call fms_io_init')
   call setup_one_field(fileObj, filename, fieldname, (/size(data,1), size(data,2), size(data,3), 1/), &
                        index_field, domain, mandatory, no_domain, .false., &
-                       position, tile_count, data_default, longname, units)
+                       position, tile_count, data_default, longname, units, read_only=read_only)
   fileObj%p3dr(fileObj%var(index_field)%siz(4), index_field)%p => data
   fileObj%var(index_field)%ndim = 3
-  register_restart_field_r3d = index_field   
+  register_restart_field_r3d = index_field
 
-  return    
+  return
 
 end function register_restart_field_r3d
 
+
+!-------------------------------------------------------------------------------
+!
+!   The routine will register a 4-D real restart file field with one time level
+!
+!-------------------------------------------------------------------------------
+function register_restart_field_r4d(fileObj, filename, fieldname, data, domain, mandatory, &
+                             no_domain, position, tile_count, data_default, longname, units, read_only)
+  type(restart_file_type),   intent(inout)         :: fileObj
+  character(len=*),             intent(in)         :: filename, fieldname
+  real,     dimension(:,:,:,:), intent(in), target :: data
+  type(domain2d),   optional,   intent(in), target :: domain
+  real,             optional,   intent(in)         :: data_default
+  logical,          optional,   intent(in)         :: no_domain
+  integer,          optional,   intent(in)         :: position, tile_count
+  logical,          optional,   intent(in)         :: mandatory
+  character(len=*), optional,   intent(in)         :: longname, units
+  logical,          optional,   intent(in)         :: read_only
+  integer                                          :: index_field
+  integer                                          :: register_restart_field_r4d
+
+  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(register_restart_field_r4d): need to call fms_io_init')
+  call setup_one_field(fileObj, filename, fieldname, (/size(data,1), size(data,2), size(data,3), 1, size(data,4)/), &
+                       index_field, domain, mandatory, no_domain, .false., &
+                       position, tile_count, data_default, longname, units, read_only=read_only)
+  fileObj%p4dr(fileObj%var(index_field)%siz(4), index_field)%p => data
+  fileObj%var(index_field)%ndim = 4
+  register_restart_field_r4d = index_field
+
+  return
+
+end function register_restart_field_r4d
+
+
 !-------------------------------------------------------------------------------
 !
 !   The routine will register a scalar integer restart file field with one time level
 !
 !-------------------------------------------------------------------------------
 function register_restart_field_i0d(fileObj, filename, fieldname, data, domain, mandatory, &
-                             no_domain, position, tile_count, data_default, longname, units)
+                             no_domain, position, tile_count, data_default, longname, units, read_only)
   type(restart_file_type), intent(inout)         :: fileObj
   character(len=*),           intent(in)         :: filename, fieldname
-  integer,                    intent(in), target :: data 
+  integer,                    intent(in), target :: data
   type(domain2d),   optional, intent(in), target :: domain
-  real,             optional, intent(in)         :: data_default
+  integer,             optional, intent(in)      :: data_default
   integer,          optional, intent(in)         :: position, tile_count
   logical,          optional, intent(in)         :: mandatory
-  logical,          optional, intent(in)         :: no_domain  
+  logical,          optional, intent(in)         :: no_domain
   character(len=*), optional, intent(in)         :: longname, units
+  logical,          optional, intent(in)         :: read_only
   integer                                        :: index_field
   integer                                        :: register_restart_field_i0d
+  real                                           :: data_default_r
+
+  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(register_restart_field_i0d): need to call fms_io_init')
+
+  if (KIND(data_default)/=KIND(data)) call mpp_error(FATAL,'fms_io(register_restart_field_i0d): data_default and data different KIND()')
+  data_default_r = TRANSFER(MPP_FILL_INT,data_default_r) 
+  if (present(data_default)) data_default_r = TRANSFER(data_default ,data_default_r)
 
-  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(register_restart_field_i0d): need to call fms_io_init')  
   call setup_one_field(fileObj, filename, fieldname, (/1, 1, 1, 1/), index_field, domain, &
                        mandatory, no_domain=no_domain, scalar_or_1d=.true., position=position, tile_count=tile_count, &
-                       data_default=data_default, longname=longname, units=units)
+                          data_default=data_default_r, longname=longname, units=units, read_only=read_only)
+  
   fileObj%p0di(fileObj%var(index_field)%siz(4), index_field)%p => data
   fileObj%var(index_field)%ndim = 0
   register_restart_field_i0d = index_field
-  
+
   return
-   
+
 end function register_restart_field_i0d
 
 !-------------------------------------------------------------------------------
@@ -1200,29 +1614,38 @@ end function register_restart_field_i0d
 !
 !-------------------------------------------------------------------------------
 function register_restart_field_i1d(fileObj, filename, fieldname, data, domain, mandatory, &
-                             no_domain, position, tile_count, data_default, longname, units)
+                             no_domain, position, tile_count, data_default, longname, units, &
+                             compressed_axis, read_only)
   type(restart_file_type), intent(inout)         :: fileObj
   character(len=*),           intent(in)         :: filename, fieldname
-  integer, dimension(:),      intent(in), target :: data 
+  integer, dimension(:),      intent(in), target :: data
   type(domain2d),   optional, intent(in), target :: domain
-  real,             optional, intent(in)         :: data_default
+  integer,          optional, intent(in)         :: data_default
   integer,          optional, intent(in)         :: position, tile_count
   logical,          optional, intent(in)         :: mandatory
   logical,          optional, intent(in)         :: no_domain
-  character(len=*), optional, intent(in)         :: longname, units
+  character(len=*), optional, intent(in)         :: longname, units, compressed_axis
+  logical,          optional, intent(in)         :: read_only
   integer                                        :: index_field
   integer                                        :: register_restart_field_i1d
+  real                                           :: data_default_r
+
+  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(register_restart_field_i1d): need to call fms_io_init')
+
+  if (KIND(data_default)/=KIND(data)) call mpp_error(FATAL,'fms_io(register_restart_field_i1d): data_default and data different KIND()')
+  data_default_r = TRANSFER(MPP_FILL_INT,data_default_r) 
+  if (present(data_default)) data_default_r = TRANSFER(data_default ,data_default_r)
 
-  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(register_restart_field_i1d): need to call fms_io_init')  
   call setup_one_field(fileObj, filename, fieldname, (/size(data,1), 1, 1, 1/), index_field, domain, &
                        mandatory, no_domain=no_domain, scalar_or_1d=.true., position=position, tile_count=tile_count, &
-                       data_default=data_default, longname=longname, units=units)
+                       data_default=data_default_r, longname=longname, units=units, compressed_axis=compressed_axis, &
+                       read_only=read_only)
   fileObj%p1di(fileObj%var(index_field)%siz(4), index_field)%p => data
   fileObj%var(index_field)%ndim = 1
   register_restart_field_i1d = index_field
-  
+
   return
-  
+
 end function register_restart_field_i1d
 
 
@@ -1231,29 +1654,42 @@ end function register_restart_field_i1d
 !   The routine will register a 2-D real restart file field with one time level
 !
 !-------------------------------------------------------------------------------
-function register_restart_field_i2d(fileObj, filename, fieldname, data, domain, mandatory, &
-                             no_domain, position, tile_count, data_default, longname, units)
+function register_restart_field_i2d(fileObj, filename, fieldname, data, domain, mandatory, no_domain, &
+                             compressed, position, tile_count, data_default, longname, units, &
+                             compressed_axis, read_only)
   type(restart_file_type), intent(inout)         :: fileObj
   character(len=*),           intent(in)         :: filename, fieldname
-  integer,  dimension(:,:),   intent(in), target :: data 
+  integer,  dimension(:,:),   intent(in), target :: data
   type(domain2d),   optional, intent(in), target :: domain
-  real,             optional, intent(in)         :: data_default
-  logical,          optional, intent(in)         :: no_domain   
+  integer,          optional, intent(in)         :: data_default
+  logical,          optional, intent(in)         :: no_domain
+  logical,          optional, intent(in)         :: compressed
   integer,          optional, intent(in)         :: position, tile_count
   logical,          optional, intent(in)         :: mandatory
-  character(len=*), optional, intent(in)         :: longname, units
+  character(len=*), optional, intent(in)         :: longname, units, compressed_axis
+  logical,          optional, intent(in)         :: read_only
+  logical                                        :: is_compressed
   integer                                        :: index_field
   integer                                        :: register_restart_field_i2d
+  real                                           :: data_default_r
+
+  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(register_restart_field_i2d): need to call fms_io_init')
+  is_compressed = .false.
+  if(present(compressed)) is_compressed=compressed
 
-  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(register_restart_field_i2d): need to call fms_io_init')  
+  if (KIND(data_default)/=KIND(data)) call mpp_error(FATAL,'fms_io(register_restart_field_i2d): data_default and data different KIND()')
+  data_default_r = TRANSFER(MPP_FILL_INT,data_default_r) 
+  if (present(data_default)) data_default_r = TRANSFER(data_default ,data_default_r)
+  
   call setup_one_field(fileObj, filename, fieldname, (/size(data,1), size(data,2), 1, 1/), &
-                       index_field, domain, mandatory, no_domain, .false., &
-                       position, tile_count, data_default, longname, units)
+                       index_field, domain, mandatory, no_domain, is_compressed, &
+                       position, tile_count, data_default_r, longname, units, compressed_axis, &
+                       read_only=read_only)
   fileObj%p2di(fileObj%var(index_field)%siz(4), index_field)%p => data
   fileObj%var(index_field)%ndim = 2
   register_restart_field_i2d = index_field
-  
-  return 
+
+  return
 
 end function register_restart_field_i2d
 
@@ -1263,28 +1699,35 @@ end function register_restart_field_i2d
 !
 !-------------------------------------------------------------------------------
 function register_restart_field_i3d(fileObj, filename, fieldname, data, domain, mandatory, &
-                             no_domain, position, tile_count, data_default, longname, units)
+                             no_domain, position, tile_count, data_default, longname, units, read_only)
   type(restart_file_type), intent(inout)         :: fileObj
   character(len=*),           intent(in)         :: filename, fieldname
-  integer,  dimension(:,:,:), intent(in), target :: data 
+  integer,  dimension(:,:,:), intent(in), target :: data
   type(domain2d),   optional, intent(in), target :: domain
-  real,             optional, intent(in)         :: data_default
-  logical,          optional, intent(in)         :: no_domain   
+  integer,             optional, intent(in)         :: data_default
+  logical,          optional, intent(in)         :: no_domain
   integer,          optional, intent(in)         :: position, tile_count
   logical,          optional, intent(in)         :: mandatory
   character(len=*), optional, intent(in)         :: longname, units
+  logical,          optional, intent(in)         :: read_only
   integer                                        :: index_field
   integer                                        :: register_restart_field_i3d
+  real                                           :: data_default_r
+
+  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(register_restart_field_i3d): need to call fms_io_init')
 
-  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(register_restart_field_i3d): need to call fms_io_init')  
+  if (KIND(data_default)/=KIND(data)) call mpp_error(FATAL,'fms_io(register_restart_field_i3d): data_default and data different KIND()')
+  data_default_r = TRANSFER(MPP_FILL_INT,data_default_r) 
+  if (present(data_default)) data_default_r = TRANSFER(data_default ,data_default_r)
+  
   call setup_one_field(fileObj, filename, fieldname, (/size(data,1), size(data,2), size(data,3), 1/), &
                        index_field, domain, mandatory, no_domain, .false., &
-                       position, tile_count, data_default, longname, units)
+                       position, tile_count, data_default_r, longname, units, read_only=read_only)
   fileObj%p3di(fileObj%var(index_field)%siz(4), index_field)%p => data
   fileObj%var(index_field)%ndim = 3
   register_restart_field_i3d = index_field
-  
-  return  
+
+  return
 
 end function register_restart_field_i3d
 
@@ -1294,30 +1737,31 @@ end function register_restart_field_i3d
 !
 !-------------------------------------------------------------------------------
 function register_restart_field_r0d_2level(fileObj, filename, fieldname, data1, data2, domain, mandatory, &
-                             no_domain, position, tile_count, data_default, longname, units)
+                             no_domain, position, tile_count, data_default, longname, units, read_only)
   type(restart_file_type), intent(inout)         :: fileObj
   character(len=*),           intent(in)         :: filename, fieldname
-  real,                       intent(in), target :: data1, data2 
+  real,                       intent(in), target :: data1, data2
   type(domain2d),   optional, intent(in), target :: domain
   real,             optional, intent(in)         :: data_default
   integer,          optional, intent(in)         :: position, tile_count
   logical,          optional, intent(in)         :: mandatory
   logical,          optional, intent(in)         :: no_domain
   character(len=*), optional, intent(in)         :: longname, units
+  logical,          optional, intent(in)         :: read_only
   integer                                        :: index_field
   integer                                        :: register_restart_field_r0d_2level
 
   if(.not.module_is_initialized) call mpp_error(FATAL, &
-      'fms_io(register_restart_field_r0d_2level): need to call fms_io_init')  
+      'fms_io(register_restart_field_r0d_2level): need to call fms_io_init')
   call setup_one_field(fileObj, filename, fieldname, (/1, 1, 1, 2/), index_field, domain, &
                        mandatory, no_domain=no_domain, scalar_or_1d=.true., position=position, tile_count=tile_count, &
-                       data_default=data_default, longname=longname, units=units)
+                       data_default=data_default, longname=longname, units=units, read_only=read_only)
   fileObj%p0dr(1, index_field)%p => data1
   fileObj%p0dr(2, index_field)%p => data2
   fileObj%var(index_field)%ndim = 0
   register_restart_field_r0d_2level = index_field
-  
-  return  
+
+  return
 
 end function register_restart_field_r0d_2level
 
@@ -1327,30 +1771,31 @@ end function register_restart_field_r0d_2level
 !
 !-------------------------------------------------------------------------------
 function register_restart_field_r1d_2level(fileObj, filename, fieldname, data1, data2, domain, mandatory, &
-                             no_domain, position, tile_count, data_default, longname, units)
+                             no_domain, position, tile_count, data_default, longname, units, read_only)
   type(restart_file_type), intent(inout)         :: fileObj
   character(len=*),           intent(in)         :: filename, fieldname
-  real,     dimension(:),     intent(in), target :: data1, data2 
+  real,     dimension(:),     intent(in), target :: data1, data2
   type(domain2d),   optional, intent(in), target :: domain
   real,             optional, intent(in)         :: data_default
   integer,          optional, intent(in)         :: position, tile_count
   logical,          optional, intent(in)         :: mandatory
   logical,          optional, intent(in)         :: no_domain
   character(len=*), optional, intent(in)         :: longname, units
+  logical,          optional, intent(in)         :: read_only
   integer                                        :: index_field
   integer                                        :: register_restart_field_r1d_2level
 
   if(.not.module_is_initialized) call mpp_error(FATAL, &
-      'fms_io(register_restart_field_r1d_2level): need to call fms_io_init')  
+      'fms_io(register_restart_field_r1d_2level): need to call fms_io_init')
   call setup_one_field(fileObj, filename, fieldname, (/size(data1,1), 1, 1, 2/), index_field, domain, &
                        mandatory, no_domain=no_domain, scalar_or_1d=.true., position=position, tile_count=tile_count, &
-                       data_default=data_default, longname=longname, units=units)
+                       data_default=data_default, longname=longname, units=units, read_only=read_only)
   fileObj%p1dr(1, index_field)%p => data1
   fileObj%p1dr(2, index_field)%p => data2
   fileObj%var(index_field)%ndim = 1
   register_restart_field_r1d_2level = index_field
-  
-  return    
+
+  return
 
 end function register_restart_field_r1d_2level
 
@@ -1360,30 +1805,31 @@ end function register_restart_field_r1d_2level
 !
 !-------------------------------------------------------------------------------
 function register_restart_field_r2d_2level(fileObj, filename, fieldname, data1, data2, domain, mandatory, &
-                             no_domain, position, tile_count, data_default, longname, units)
+                             no_domain, position, tile_count, data_default, longname, units, read_only)
   type(restart_file_type), intent(inout)         :: fileObj
   character(len=*),           intent(in)         :: filename, fieldname
-  real,     dimension(:,:),   intent(in), target :: data1, data2 
+  real,     dimension(:,:),   intent(in), target :: data1, data2
   type(domain2d),   optional, intent(in), target :: domain
   real,             optional, intent(in)         :: data_default
-  logical,          optional, intent(in)         :: no_domain   
+  logical,          optional, intent(in)         :: no_domain
   integer,          optional, intent(in)         :: position, tile_count
   logical,          optional, intent(in)         :: mandatory
   character(len=*), optional, intent(in)         :: longname, units
+  logical,          optional, intent(in)         :: read_only
   integer                                        :: index_field
   integer                                        :: register_restart_field_r2d_2level
 
   if(.not.module_is_initialized) call mpp_error(FATAL, &
-      'fms_io(register_restart_field_r2d_2level): need to call fms_io_init')  
+      'fms_io(register_restart_field_r2d_2level): need to call fms_io_init')
   call setup_one_field(fileObj, filename, fieldname, (/size(data1,1), size(data1,2), 1, 2/), &
                        index_field, domain, mandatory, no_domain, .false., &
-                       position, tile_count, data_default, longname, units)
+                       position, tile_count, data_default, longname, units, read_only=read_only)
   fileObj%p2dr(1, index_field)%p => data1
   fileObj%p2dr(2, index_field)%p => data2
   fileObj%var(index_field)%ndim = 2
   register_restart_field_r2d_2level = index_field
-  
-  return    
+
+  return
 
 end function register_restart_field_r2d_2level
 
@@ -1393,30 +1839,31 @@ end function register_restart_field_r2d_2level
 !
 !-------------------------------------------------------------------------------
 function register_restart_field_r3d_2level(fileObj, filename, fieldname, data1, data2, domain, mandatory, &
-                             no_domain, position, tile_count, data_default, longname, units)
+                             no_domain, position, tile_count, data_default, longname, units, read_only)
   type(restart_file_type), intent(inout)         :: fileObj
   character(len=*),           intent(in)         :: filename, fieldname
-  real,     dimension(:,:,:), intent(in), target :: data1, data2 
+  real,     dimension(:,:,:), intent(in), target :: data1, data2
   type(domain2d),   optional, intent(in), target :: domain
   real,             optional, intent(in)         :: data_default
-  logical,          optional, intent(in)         :: no_domain   
+  logical,          optional, intent(in)         :: no_domain
   integer,          optional, intent(in)         :: position, tile_count
   logical,          optional, intent(in)         :: mandatory
   character(len=*), optional, intent(in)         :: longname, units
+  logical,          optional, intent(in)         :: read_only
   integer                                        :: index_field
   integer                                        :: register_restart_field_r3d_2level
 
   if(.not.module_is_initialized) call mpp_error(FATAL, &
-      'fms_io(register_restart_field_r3d_2level): need to call fms_io_init')  
+      'fms_io(register_restart_field_r3d_2level): need to call fms_io_init')
   call setup_one_field(fileObj, filename, fieldname, (/size(data1,1), size(data1,2), size(data1,3), 2/), &
                        index_field, domain, mandatory, no_domain, .false., &
-                       position, tile_count, data_default, longname, units)
+                       position, tile_count, data_default, longname, units, read_only=read_only)
   fileObj%p3dr(1, index_field)%p => data1
   fileObj%p3dr(2, index_field)%p => data2
   fileObj%var(index_field)%ndim = 3
   register_restart_field_r3d_2level = index_field
-  
-  return    
+
+  return
 
 end function register_restart_field_r3d_2level
 
@@ -1426,31 +1873,32 @@ end function register_restart_field_r3d_2level
 !
 !-------------------------------------------------------------------------------
 function register_restart_field_i0d_2level(fileObj, filename, fieldname, data1, data2, domain, mandatory, &
-                             no_domain, position, tile_count, data_default, longname, units)
+                             no_domain, position, tile_count, data_default, longname, units, read_only)
   type(restart_file_type), intent(inout)         :: fileObj
   character(len=*),           intent(in)         :: filename, fieldname
-  integer,                    intent(in), target :: data1, data2 
+  integer,                    intent(in), target :: data1, data2
   type(domain2d),   optional, intent(in), target :: domain
   real,             optional, intent(in)         :: data_default
   integer,          optional, intent(in)         :: position, tile_count
   logical,          optional, intent(in)         :: mandatory
   logical,          optional, intent(in)         :: no_domain
   character(len=*), optional, intent(in)         :: longname, units
+  logical,          optional, intent(in)         :: read_only
   integer                                        :: index_field
   integer                                        :: register_restart_field_i0d_2level
 
   if(.not.module_is_initialized) call mpp_error(FATAL, &
-      'fms_io(register_restart_field_i0d_2level): need to call fms_io_init')  
+      'fms_io(register_restart_field_i0d_2level): need to call fms_io_init')
   call setup_one_field(fileObj, filename, fieldname, (/1, 1, 1, 2/), index_field, domain, &
                        mandatory, no_domain=no_domain, scalar_or_1d=.true., position=position, tile_count=tile_count, &
-                       data_default=data_default, longname=longname, units=units)
+                       data_default=data_default, longname=longname, units=units, read_only=read_only)
   fileObj%p0di(1, index_field)%p => data1
   fileObj%p0di(2, index_field)%p => data2
   fileObj%var(index_field)%ndim = 0
   register_restart_field_i0d_2level = index_field
-  
-  return   
- 
+
+  return
+
 end function register_restart_field_i0d_2level
 
 !-------------------------------------------------------------------------------
@@ -1459,31 +1907,32 @@ end function register_restart_field_i0d_2level
 !
 !-------------------------------------------------------------------------------
 function register_restart_field_i1d_2level(fileObj, filename, fieldname, data1, data2, domain, mandatory, &
-                             no_domain, position, tile_count, data_default, longname, units)
+                             no_domain, position, tile_count, data_default, longname, units, read_only)
   type(restart_file_type), intent(inout)         :: fileObj
   character(len=*),           intent(in)         :: filename, fieldname
-  integer,  dimension(:),     intent(in), target :: data1, data2 
+  integer,  dimension(:),     intent(in), target :: data1, data2
   type(domain2d),   optional, intent(in), target :: domain
   real,             optional, intent(in)         :: data_default
   integer,          optional, intent(in)         :: position, tile_count
   logical,          optional, intent(in)         :: mandatory
   logical,          optional, intent(in)         :: no_domain
   character(len=*), optional, intent(in)         :: longname, units
+  logical,          optional, intent(in)         :: read_only
   integer                                        :: index_field
   integer                                        :: register_restart_field_i1d_2level
 
   if(.not.module_is_initialized) call mpp_error(FATAL, &
-      'fms_io(register_restart_field_i1d_2level): need to call fms_io_init')  
+      'fms_io(register_restart_field_i1d_2level): need to call fms_io_init')
   call setup_one_field(fileObj, filename, fieldname, (/size(data1,1), 1, 1, 2/), index_field, domain, &
                        mandatory, no_domain=no_domain, scalar_or_1d=.true., position=position, tile_count=tile_count, &
-                       data_default=data_default, longname=longname, units=units)
+                       data_default=data_default, longname=longname, units=units, read_only=read_only)
   fileObj%p1di(1, index_field)%p => data1
   fileObj%p1di(2, index_field)%p => data2
   fileObj%var(index_field)%ndim = 1
   register_restart_field_i1d_2level = index_field
-  
-  return  
-  
+
+  return
+
 end function register_restart_field_i1d_2level
 
 !-------------------------------------------------------------------------------
@@ -1492,30 +1941,31 @@ end function register_restart_field_i1d_2level
 !
 !-------------------------------------------------------------------------------
 function register_restart_field_i2d_2level(fileObj, filename, fieldname, data1, data2, domain, mandatory, &
-                             no_domain, position, tile_count, data_default, longname, units)
+                             no_domain, position, tile_count, data_default, longname, units, read_only)
   type(restart_file_type), intent(inout)         :: fileObj
   character(len=*),           intent(in)         :: filename, fieldname
-  integer,  dimension(:,:),   intent(in), target :: data1, data2 
+  integer,  dimension(:,:),   intent(in), target :: data1, data2
   type(domain2d),   optional, intent(in), target :: domain
   real,             optional, intent(in)         :: data_default
-  logical,          optional, intent(in)         :: no_domain   
+  logical,          optional, intent(in)         :: no_domain
   integer,          optional, intent(in)         :: position, tile_count
   logical,          optional, intent(in)         :: mandatory
   character(len=*), optional, intent(in)         :: longname, units
+  logical,          optional, intent(in)         :: read_only
   integer                                        :: index_field
   integer                                        :: register_restart_field_i2d_2level
 
   if(.not.module_is_initialized) call mpp_error(FATAL, &
-      'fms_io(register_restart_field_i2d_2level): need to call fms_io_init')  
+      'fms_io(register_restart_field_i2d_2level): need to call fms_io_init')
   call setup_one_field(fileObj, filename, fieldname, (/size(data1,1), size(data1,2), 1, 2/), &
                        index_field, domain, mandatory, no_domain, .false., &
-                       position, tile_count, data_default, longname, units)
+                       position, tile_count, data_default, longname, units, read_only=read_only)
   fileObj%p2di(1, index_field)%p => data1
   fileObj%p2di(2, index_field)%p => data2
   fileObj%var(index_field)%ndim = 2
   register_restart_field_i2d_2level = index_field
-  
-  return    
+
+  return
 
 end function register_restart_field_i2d_2level
 
@@ -1525,423 +1975,1422 @@ end function register_restart_field_i2d_2level
 !
 !-------------------------------------------------------------------------------
 function register_restart_field_i3d_2level(fileObj, filename, fieldname, data1, data2, domain, mandatory, &
-                             no_domain, position, tile_count, data_default, longname, units)
+                             no_domain, position, tile_count, data_default, longname, units, read_only)
   type(restart_file_type), intent(inout)         :: fileObj
   character(len=*),           intent(in)         :: filename, fieldname
-  integer,  dimension(:,:,:), intent(in), target :: data1, data2 
+  integer,  dimension(:,:,:), intent(in), target :: data1, data2
   type(domain2d),   optional, intent(in), target :: domain
   real,             optional, intent(in)         :: data_default
-  logical,          optional, intent(in)         :: no_domain   
+  logical,          optional, intent(in)         :: no_domain
   integer,          optional, intent(in)         :: position, tile_count
   logical,          optional, intent(in)         :: mandatory
   character(len=*), optional, intent(in)         :: longname, units
+  logical,          optional, intent(in)         :: read_only
   integer                                        :: index_field
   integer                                        :: register_restart_field_i3d_2level
 
   if(.not.module_is_initialized) call mpp_error(FATAL, &
-      'fms_io(register_restart_field_i3d_2level): need to call fms_io_init')  
+      'fms_io(register_restart_field_i3d_2level): need to call fms_io_init')
   call setup_one_field(fileObj, filename, fieldname, (/size(data1,1), size(data1,2), size(data1,3), 2/), &
                        index_field, domain, mandatory, no_domain, .false., &
-                       position, tile_count, data_default, longname, units)
+                       position, tile_count, data_default, longname, units, read_only=read_only)
   fileObj%p3di(1, index_field)%p => data1
   fileObj%p3di(2, index_field)%p => data2
   fileObj%var(index_field)%ndim = 3
   register_restart_field_i3d_2level = index_field
-  
-  return    
+
+  return
 
 end function register_restart_field_i3d_2level
 
+!-------------------------------------------------------------------------------
+!
+!   The routine will register a 2-D real for a generic region defined
+!   by the global_size variable.
+!
+!-------------------------------------------------------------------------------
+function register_restart_region_r2d (fileObj, filename, fieldname, data, indices, global_size, &
+                                      pelist, is_root_pe, longname, units, position, &
+                                      x_halo, y_halo, ishift, jshift, read_only)
+  type(restart_file_type), intent(inout)         :: fileObj
+  character(len=*),           intent(in)         :: filename, fieldname
+  real,       dimension(:,:), intent(in), target :: data
+  integer,      dimension(:), intent(in)         :: indices, global_size, pelist
+  logical,                    intent(in)         :: is_root_pe
+  character(len=*), optional, intent(in)         :: longname, units
+  integer,          optional, intent(in)         :: position, x_halo, y_halo, ishift, jshift
+  logical,          optional, intent(in)         :: read_only
+  integer :: index_field, l_position
+  integer :: register_restart_region_r2d
+
+  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(register_restart_region_r2d): need to call fms_io_init')
+  if ((is_root_pe) .and. (.not.ANY(mpp_pe().eq.pelist))) call mpp_error(FATAL, &
+                    'fms_io(register_restart_region_r2d) designated root_pe is not a member of pelist')
+  l_position = CENTER
+  if (present(position)) l_position = position
+  call setup_one_field(fileObj, filename, fieldname, (/size(data,1), size(data,2), 1, 1/), &
+                       index_field, no_domain=.true., position=l_position, longname=longname, units=units, &
+                       read_only=read_only)
+  fileObj%p2dr(fileObj%var(index_field)%siz(4), index_field)%p => data
+  fileObj%var(index_field)%ndim = 2
+  fileObj%var(index_field)%is = indices(1)
+  fileObj%var(index_field)%ie = indices(2)
+  fileObj%var(index_field)%js = indices(3)
+  fileObj%var(index_field)%je = indices(4)
+  fileObj%var(index_field)%gsiz(1) = global_size(1)
+  fileObj%var(index_field)%gsiz(2) = global_size(2)
+  fileObj%is_root_pe = is_root_pe
+  fileObj%var(index_field)%x_halo = 0
+  fileObj%var(index_field)%y_halo = 0
+  fileObj%var(index_field)%ishift = 0
+  fileObj%var(index_field)%jshift = 0
+  if (present(x_halo)) fileObj%var(index_field)%x_halo = x_halo
+  if (present(y_halo)) fileObj%var(index_field)%y_halo = y_halo
+  if (present(ishift)) fileObj%var(index_field)%ishift = ishift
+  if (present(jshift)) fileObj%var(index_field)%jshift = jshift
+  if (allocated(fileObj%var(index_field)%pelist)) deallocate(fileObj%var(index_field)%pelist)
+  if (allocated(fileObj%var(index_field)%pelist)) deallocate(fileObj%var(index_field)%pelist)
+  allocate(fileObj%var(index_field)%pelist(size(pelist)))
+  fileObj%var(index_field)%pelist = pelist
+  register_restart_region_r2d = index_field
+
+  return
+end function register_restart_region_r2d
+
+!-------------------------------------------------------------------------------
+!
+!   The routine will register a 3-D real for a generic region defined
+!   by the global_size variable.
+!
+!-------------------------------------------------------------------------------
+function register_restart_region_r3d (fileObj, filename, fieldname, data, indices, global_size, &
+                                      pelist, is_root_pe, longname, units, position, &
+                                      x_halo, y_halo, ishift, jshift, read_only)
+  type(restart_file_type), intent(inout)         :: fileObj
+  character(len=*),           intent(in)         :: filename, fieldname
+  real,     dimension(:,:,:), intent(in), target :: data
+  integer,      dimension(:), intent(in)         :: indices, global_size, pelist
+  logical,                    intent(in)         :: is_root_pe
+  character(len=*), optional, intent(in)         :: longname, units
+  logical,          optional, intent(in)         :: read_only
+  integer,          optional, intent(in)         :: position, x_halo, y_halo, ishift, jshift
+  integer :: index_field, l_position
+  integer :: register_restart_region_r3d
+
+  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(register_restart_region_r3d): need to call fms_io_init')
+  if ((is_root_pe) .and. (.not.ANY(mpp_pe().eq.pelist))) call mpp_error(FATAL, &
+                    'fms_io(register_restart_region_r3d) designated root_pe is not a member of pelist')
+  l_position = CENTER
+  if (present(position)) l_position = position
+  call setup_one_field(fileObj, filename, fieldname, (/size(data,1), size(data,2), size(data,3), 1/), &
+                       index_field, no_domain=.true., position=l_position, longname=longname, units=units, &
+                       read_only=read_only)
+  fileObj%p3dr(fileObj%var(index_field)%siz(4), index_field)%p => data
+  fileObj%var(index_field)%ndim = 3
+  fileObj%var(index_field)%is = indices(1)
+  fileObj%var(index_field)%ie = indices(2)
+  fileObj%var(index_field)%js = indices(3)
+  fileObj%var(index_field)%je = indices(4)
+  fileObj%var(index_field)%gsiz(1) = global_size(1)
+  fileObj%var(index_field)%gsiz(2) = global_size(2)
+  fileObj%var(index_field)%gsiz(3) = global_size(3)
+  fileObj%is_root_pe = is_root_pe
+  fileObj%var(index_field)%x_halo = 0
+  fileObj%var(index_field)%y_halo = 0
+  fileObj%var(index_field)%ishift = 0
+  fileObj%var(index_field)%jshift = 0
+  if (present(x_halo)) fileObj%var(index_field)%x_halo = x_halo
+  if (present(y_halo)) fileObj%var(index_field)%y_halo = y_halo
+  if (present(ishift)) fileObj%var(index_field)%ishift = ishift
+  if (present(jshift)) fileObj%var(index_field)%jshift = jshift
+  if (allocated(fileObj%var(index_field)%pelist)) deallocate(fileObj%var(index_field)%pelist)
+  allocate(fileObj%var(index_field)%pelist(size(pelist)))
+  fileObj%var(index_field)%pelist = pelist
+  register_restart_region_r3d = index_field
+
+  return
+end function register_restart_region_r3d
+
 !-------------------------------------------------------------------------------
 !
 !  saves all registered variables to restart files. Those variables are set 
 !  through register_restart_field
 !
 !-------------------------------------------------------------------------------
-
-subroutine save_restart(fileObj, time_stamp, directory )
+subroutine save_restart(fileObj, time_stamp, directory, append, time_level)
   type(restart_file_type), intent(inout) :: fileObj
   character(len=*), intent(in), optional :: directory
   character(len=*), intent(in), optional :: time_stamp
-  ! Arguments: 
+  ! Arguments:
   !  (in)      directory  - The directory where the restart file goes.
   !  (in)      time_stamp - character format of the time of this restart file.
+  logical, intent(in), optional :: append
+  real,    intent(in), optional :: time_level
   character(len=256) :: dir
-  character(len=256) :: restartpath          ! The restart file path (dir/file).
-  character(len=258) :: restartpath_var      ! The restart file path for the current variable
   character(len=80)  :: restartname          ! The restart file name (no dir).
-  character(len=8)   :: suffix               ! A suffix (like _2) that is appended to the name of files after the first.
-  integer            :: var_sz, size_in_file ! The size in bytes of each variable and of the variables already in a file.
-  integer            :: start_var, next_var  ! The starting variables of the current and next files.
-  integer            :: unit                 ! The mpp unit of the open file.
-  real, dimension(max_axis_size)      :: axisdata
-  integer,        dimension(max_axes) :: id_x_axes, siz_x_axes
-  integer,        dimension(max_axes) :: id_y_axes, siz_y_axes
-  integer,        dimension(max_axes) :: id_z_axes, siz_z_axes
-  integer,        dimension(max_axes) :: x_axes_indx, y_axes_indx, z_axes_indx
-  type(axistype), dimension(max_axes) :: x_axes, y_axes, z_axes
-  type(axistype)                      :: t_axes            
-  integer                             :: num_var_axes
-  type(axistype), dimension(4)        :: var_axes
-  type(var_type), pointer, save       :: cur_var=>NULL()
-  integer                             :: num_x_axes, num_y_axes, num_z_axes
-  integer                             :: naxes_x, naxes_y, naxes_z
-  integer                             :: nfiles, i, j, k, l, siz, ind_dom
-  logical                             :: domain_present
-  real                                :: tlev  
-  character(len=10)                   :: axisname  
-  integer                             :: meta_size
-  type(domain2d)                      :: domain
+  character(len=336) :: restartpath          ! The restart file path (dir/file).
 
-  real, allocatable, dimension(:,:,:) :: r3d
-  real, allocatable, dimension(:,:)   :: r2d, global_r2d
-  real, allocatable, dimension(:)     :: r1d  
-  real                                :: r0d
-  integer(LONG_KIND), allocatable, dimension(:)    :: check_val
-  character(len=256)                  :: checksum_char
-integer :: isc, iec, jsc, jec
-integer :: isg, ieg, jsg, jeg
-integer :: ishift, jshift, iadd, jadd
+  ! This approach is taken rather than interface overloading in order to preserve
+  ! use of the register_restart_field infrastructure
 
   if (.not.associated(fileObj%var)) call mpp_error(FATAL, "fms_io(save_restart): " // &
       "restart_file_type data must be initialized by calling register_restart_field before using it")
 
-  do i=1,max_axis_size
-     axisdata(i) = i
-  enddo
-
   dir = "RESTART"
   if(present(directory)) dir = directory
 
   restartname = fileObj%name
-  nfiles = 0
   if(time_stamp_restart) then
      if (PRESENT(time_stamp)) then
+        if(len_trim(restartname)+len_trim(time_stamp) > 79) call mpp_error(FATAL, "fms_io(save_restart): " // &
+          "Length of restart file name + time_stamp is greater than allowed character length of 79")
         restartname = trim(time_stamp)//"."//trim(restartname)
      endif
   end if
   if(len_trim(dir) > 0) then
+     if(len_trim(dir)+len_trim(restartname) > 335) call mpp_error(FATAL, "fms_io(save_restart): " // &
+       "Length of full restart path + file name is greater than allowed character length of 355")
      restartpath = trim(dir)//"/"// trim(restartname)
   else
      restartpath = trim(restartname)
   end if
-  !--- check if any field in this file present domain.
-  domain_present = .false.
-  do j = 1, fileObj%nvar
-     if (fileObj%var(j)%domain_present) then
-        domain_present = .true.
-        ind_dom = j
-        exit
-     end if
+
+  if(fileObj%is_compressed .AND. ALLOCATED(fileObj%axes)) then
+     ! fileObj%axes must also be allocated if the file contains compressed axes
+     ! But will this always be true in the future?
+     call save_compressed_restart(fileObj,restartpath,append,time_level)
+  elseif(fileObj%unlimited_axis .AND. ALLOCATED(fileObj%axes)) then
+     call save_unlimited_axis_restart(fileObj,restartpath)
+  else
+     call save_default_restart(fileObj,restartpath)
+  endif
+
+  if(print_chksum) call write_chksum(fileObj, MPP_OVERWR)
+end subroutine save_restart
+
+!---- return true if all fields in fileObj is read only
+function all_field_read_only(fileObj)
+  type(restart_file_type), intent(in) :: fileObj
+  logical                             :: all_field_read_only
+  integer :: j
+
+  all_field_read_only = .TRUE.
+  do j = 1, fileObj%nvar
+     if( .not. fileObj%var(j)%read_only) then
+        all_field_read_only = .FALSE.
+        exit
+     endif
+  enddo
+
+  return
+
+end function all_field_read_only
+
+!-------------------------------------------------------------------------------
+!
+!  saves all registered variables to restart files. Those variables are set
+!  through register_restart_field
+!
+!-------------------------------------------------------------------------------
+
+subroutine save_compressed_restart(fileObj,restartpath,append,time_level)
+  type(restart_file_type), intent(inout),target :: fileObj
+  character(len=336)                     :: restartpath ! The restart file path (dir/file).
+
+  ! Optional arguments:
+
+  ! If neither append or time_level is present:
+  !   routine writes both meta data and field data.
+
+  ! If append is present and append=.true.:
+  !   Only field data is written.
+  !   The field data is appended to a new time level.
+  !   time_level must also be present and it must be >= 0.0
+  !   The value of time_level is written as a new value of the time axis data.
+
+  ! If time_level is present and time_level < 0.0:
+  !   A new file is opened and only the meta data is written.
+
+  ! If append is present and append=.false.:
+  !   Behaves the same was as if it were not present. That is, meta data is
+  !   written and whether or not field data is written is determined time_level.
+
+  logical, intent(in), optional :: append
+  real,    intent(in), optional :: time_level
+
+  integer            :: unit                 ! The mpp unit of the open file.
+  type(axistype)                      :: x_axis, y_axis, z_axis
+  type(axistype)                      :: t_axis, c_axis, h_axis  ! time & sparse compressed vector axes
+  type(axistype)                      :: comp_axis
+  logical                             :: naxis_z=.false.
+  type(axistype), dimension(4)        :: var_axes
+  type(var_type), pointer, save       :: cur_var=>NULL()
+  integer                             :: i, j, k, l, num_var_axes, cpack, idx, mpp_action
+  real                                :: tlev
+  real, allocatable, dimension(:,:)   :: r2d
+  real, allocatable, dimension(:)     :: r1d
+  real                                :: r0d
+  integer(LONG_KIND), allocatable, dimension(:)    :: check_val
+  character(len=256)                  :: checksum_char
+  logical                             :: domain_present, write_meta_data, write_field_data
+  logical                             :: c_axis_defined, h_axis_defined
+  type(domain2d), pointer :: domain =>NULL()
+  type(ax_type),  pointer :: axis   =>NULL()
+
+  !-- no need to proceed if all the variables are read only.
+  if( all_field_read_only(fileObj) ) return
+
+  if (.not.ALLOCATED(fileObj%axes(CIDX)%idx) .and. .not.ALLOCATED(fileObj%axes(HIDX)%idx) ) then
+     call mpp_error(FATAL, "fms_io(save_compressed_restart): A compressed axis has "// &
+          "not been defined for file "//trim(fileObj%name))
+  else if (ALLOCATED(fileObj%axes(CIDX)%idx)) then
+     domain =>fileObj%axes(CIDX)%domain
+  else
+     domain =>fileObj%axes(HIDX)%domain
+  endif
+
+  if(present(append)) then
+    if(append .and. .not.present(time_level)) then
+      call mpp_error(FATAL, 'fms_io(save_compressed_restart): time_level must be present when append=.true.'// &
+                    ' for file '//trim(fileObj%name))
+    endif
+  endif
+
+  mpp_action = MPP_OVERWR
+  write_meta_data  = .true.
+  if(present(append)) then
+    if(append) then
+      mpp_action = MPP_APPEND
+      write_meta_data = .false. ! Assuming meta data is already written when routine is called to append to field data.
+      if(time_level < 0.0) then
+        call mpp_error(FATAL, 'fms_io(save_compressed_restart): time_level cannot be negative when append is .true.'// &
+                      ' for file '//trim(fileObj%name))
+      endif
+    endif
+  endif
+
+  write_field_data = .true.
+  if(present(time_level)) then
+    write_field_data = time_level >= 0.0 ! Using negative value of time_level as a flag that there is no valid field data to write.
+  endif
+
+  call mpp_open(unit,trim(restartpath),action=mpp_action,form=form, &
+          is_root_pe=fileObj%is_root_pe, domain=domain)
+
+  if(write_meta_data) then
+    ! User has defined axes and these are assumed to be unique
+    ! Unfortunately it has proven difficult to write a generalized form because
+    ! of the variations possible across all of the axes
+    ! Currently support only 1 user defined axis of each type
+    ! In fact, this config is specifically designed to support the land model
+    ! sparse, compressed tile data
+    axis => fileobj%axes(XIDX)
+    if(.not. ASSOCIATED(axis)) call mpp_error(FATAL, "fms_io(save_compressed_restart): "// &
+                " The X axis has not been defined for "// &
+                " file "//trim(fileObj%name) )
+    call mpp_write_meta(unit,x_axis,axis%name,axis%units,axis%longname,data=axis%data,cartesian='X')
+
+    axis => fileobj%axes(YIDX)
+    if(.not. ASSOCIATED(axis)) call mpp_error(FATAL, "fms_io(save_compressed_restart): "// &
+                " The Y axis has not been defined for "// &
+                " file "//trim(fileObj%name) )
+    call mpp_write_meta(unit,y_axis,axis%name,axis%units,axis%longname,data=axis%data,cartesian='Y')
+
+    axis => fileobj%axes(ZIDX)
+    naxis_z = .false.
+    if(ASSOCIATED(axis%data))then
+       call mpp_write_meta(unit,z_axis,axis%name,axis%units,axis%longname, &
+            data=axis%data,cartesian='Z')
+       naxis_z = .true.
+    endif
+
+    ! The compressed axis
+    axis => fileObj%axes(CIDX)
+    if(ALLOCATED(axis%idx)) then
+       call mpp_def_dim(unit,trim(axis%dimlen_name),axis%dimlen,trim(axis%dimlen_lname), (/(i,i=1,axis%dimlen)/))
+       call mpp_write_meta(unit,c_axis,axis%name,axis%units,axis%longname, &
+                           data=axis%idx,compressed=axis%compressed,min=axis%imin)
+       c_axis_defined = .TRUE.
+    else
+       c_axis_defined = .FALSE.
+    endif
+
+    axis => fileObj%axes(HIDX)
+    if (ALLOCATED(axis%idx)) then
+       call mpp_def_dim(unit,trim(axis%dimlen_name),axis%dimlen,trim(axis%dimlen_lname), (/(i,i=1,axis%dimlen)/))
+       call mpp_write_meta(unit,h_axis,axis%name,axis%units,axis%longname, &
+                         data=axis%idx,compressed=axis%compressed,min=axis%imin)
+       h_axis_defined = .TRUE.
+    else
+       h_axis_defined = .FALSE.
+    endif
+
+    ! write out time axis
+    axis => fileobj%axes(TIDX)
+    if(ASSOCIATED(axis%data))then
+       call mpp_write_meta(unit,t_axis, axis%name, units=axis%units, longname=axis%longname, cartesian='T', calendar=axis%calendar)
+    else
+       call mpp_write_meta(unit,t_axis, 'Time','time level','Time',cartesian='T')
+    endif
+
+    ! write metadata for fields
+    do j = 1,fileObj%nvar
+       cur_var => fileObj%var(j)
+       if(cur_var%read_only) cycle
+       if(cur_var%siz(4) > 1 .AND. cur_var%siz(4) .NE. fileObj%max_ntime ) call mpp_error(FATAL, &
+        "fms_io(save_restart): "//trim(cur_var%name)//" in file "//trim(fileObj%name)// &
+        " has more than one time level, but number of time level is not equal to max_ntime")
+
+       select case (trim(cur_var%compressed_axis))
+       case ('C')
+          comp_axis = c_axis
+       case ('H')
+          comp_axis = h_axis
+       case default
+          if (ALLOCATED(fileObj%axes(CIDX)%idx)) then
+             comp_axis = c_axis
+          else
+             comp_axis = h_axis
+          endif
+       end select
+
+       if(cur_var%ndim == 0) then
+          num_var_axes = 1
+          var_axes(1) = t_axis
+       elseif(cur_var%ndim == 1) then
+          num_var_axes = 1
+          var_axes(1) = comp_axis
+          if(cur_var%siz(4) == fileObj%max_ntime) then
+             num_var_axes = 2
+             var_axes(2) = t_axis
+          endif
+       elseif(cur_var%ndim == 2) then
+          num_var_axes = 2
+          var_axes(1) = comp_axis
+          var_axes(2) = z_axis
+          if(cur_var%siz(4) == fileObj%max_ntime) then
+             num_var_axes = 3
+             var_axes(3) = t_axis
+          endif
+       else
+        call mpp_error(FATAL, "fms_io(save_compressed_restart): "//trim(cur_var%name)//" in file "// &
+           trim(fileObj%name)//" has more than two dimension (not including time level)")        
+       endif
+
+       cpack = pack_size  ! Default size of real
+        allocate(check_val(max(1,cur_var%siz(4))))
+        do k = 1, cur_var%siz(4)
+           if ( Associated(fileObj%p0dr(k,j)%p) ) then
+              check_val(k) = mpp_chksum(fileObj%p0dr(k,j)%p, (/mpp_pe()/), mask_val=cur_var%default_data)
+           else if ( Associated(fileObj%p1dr(k,j)%p) ) then
+              check_val(k) = mpp_chksum(fileObj%p1dr(k,j)%p(:), mask_val=cur_var%default_data)
+           else if ( Associated(fileObj%p2dr(k,j)%p) ) then
+              check_val(k) = mpp_chksum(fileObj%p2dr(k,j)%p(:,:), mask_val=cur_var%default_data)
+           else if ( Associated(fileObj%p3dr(k,j)%p) ) then
+              call mpp_error(FATAL, "fms_io(save_compressed_restart): real 3D restart fields are not currently supported"// &
+                   trim(cur_var%name)//" of file "//trim(fileObj%name) )
+           else if ( Associated(fileObj%p0di(k,j)%p) ) then
+              check_val(k) = fileObj%p0di(k,j)%p
+              cpack = 0  ! Write data as integer*4
+           else if ( Associated(fileObj%p1di(k,j)%p) ) then
+              check_val(k) = mpp_chksum(fileObj%p1di(k,j)%p(:), mask_val=cur_var%default_data)
+              cpack = 0  ! Write data as integer*4
+           else if ( Associated(fileObj%p2di(k,j)%p) ) then
+              check_val(k) = mpp_chksum(fileObj%p2di(k,j)%p(:,:), mask_val=cur_var%default_data)
+              cpack = 0  ! Write data as integer*4
+           else if ( Associated(fileObj%p3di(k,j)%p) ) then
+              call mpp_error(FATAL, "fms_io(save_compressed_restart): integer 3D restart fields are not currently supported"// &
+                   trim(cur_var%name)//" of file "//trim(fileObj%name) )
+           else
+              call mpp_error(FATAL, "fms_io(save_restart): There is no pointer associated with the data of field "// &
+                   trim(cur_var%name)//" of file "//trim(fileObj%name) )
+           end if
+        enddo
+! The chksum could not reproduce when running on different processor count. So commenting out now.
+! Also the chksum of compressed data is not read.
+       if(write_field_data) then ! Write checksums only if valid field data exists
+          call mpp_write_meta(unit,cur_var%field, var_axes(1:num_var_axes), cur_var%name, &
+                cur_var%units,cur_var%longname,pack=cpack,checksum=check_val,fill=cur_var%default_data)
+       else
+          call mpp_write_meta(unit,cur_var%field, var_axes(1:num_var_axes), cur_var%name, &
+                 cur_var%units,cur_var%longname,pack=cpack,fill=cur_var%default_data)
+       endif
+        deallocate(check_val)
+    enddo
+
+    ! write values for ndim of spatial and compressed axes
+    call mpp_write(unit,x_axis)
+    call mpp_write(unit,y_axis)
+    if (c_axis_defined) call mpp_write(unit,c_axis)
+    if (h_axis_defined) call mpp_write(unit,h_axis)
+    if(naxis_z) call mpp_write(unit,z_axis)
+
+  endif ! End of section to write meta data. Write meta data only if not appending.
+
+  if(write_field_data) then
+    ! write data of each field
+    do k = 1, fileObj%max_ntime
+       if(present(time_level)) then
+          tlev = time_level
+       else
+          tlev = k
+       endif
+       do j=1,fileObj%nvar
+          cur_var => fileObj%var(j)
+          if(cur_var%read_only) cycle
+
+          select case (trim(cur_var%compressed_axis))
+          case ('C')
+             idx = CIDX
+          case ('H')
+             idx = HIDX
+          case default
+             if (ALLOCATED(fileObj%axes(CIDX)%idx)) then
+                idx = CIDX
+             else
+                idx = HIDX
+             endif
+          end select
+
+          ! If some fields only have one time level, we do not need to write the second level, just keep
+          ! the data missing.
+          if(k <= cur_var%siz(4)) then
+             if ( Associated(fileObj%p0dr(k,j)%p) ) then
+                call mpp_write(unit, cur_var%field, fileObj%p0dr(k,j)%p, tlev)
+             elseif ( Associated(fileObj%p1dr(k,j)%p) ) then
+                call mpp_write_compressed(unit, cur_var%field, domain, fileObj%p1dr(k,j)%p, &
+                     fileObj%axes(idx)%nelems(:), tstamp=tlev, default_data=cur_var%default_data)
+             elseif ( Associated(fileObj%p2dr(k,j)%p) ) then
+                call mpp_write_compressed(unit, cur_var%field, domain, fileObj%p2dr(k,j)%p, &
+                     fileObj%axes(idx)%nelems(:), tstamp=tlev, default_data=cur_var%default_data)
+             elseif ( Associated(fileObj%p0di(k,j)%p) ) then
+                r0d =  fileObj%p0di(k,j)%p
+                call mpp_write(unit, cur_var%field, r0d, tlev)
+             elseif ( Associated(fileObj%p1di(k,j)%p) ) then
+                allocate(r1d(cur_var%siz(1)) )
+                r1d = fileObj%p1di(k,j)%p
+                call mpp_write_compressed(unit, cur_var%field, domain, r1d, &
+                     fileObj%axes(idx)%nelems(:), tstamp=tlev, default_data=cur_var%default_data)
+                deallocate(r1d)
+             elseif ( Associated(fileObj%p2di(k,j)%p) ) then
+                allocate(r2d(cur_var%siz(1), cur_var%siz(2)) )
+                r2d = fileObj%p2di(k,j)%p
+                call mpp_write_compressed(unit, cur_var%field, domain, r2d, &
+                     fileObj%axes(idx)%nelems(:), tstamp=tlev, default_data=cur_var%default_data)
+                deallocate(r2d)
+             else
+                call mpp_error(FATAL, "fms_io(save_restart): There is no pointer associated with the data of field "// &
+                       trim(cur_var%name)//" of file "//trim(fileObj%name) )
+             endif
+          endif
+       enddo ! end j loop
+    enddo ! end k loop
+    cur_var =>NULL()
+  endif
+  call mpp_close(unit)
+end subroutine save_compressed_restart
+
+!-------------------------------------------------------------------------------
+!
+!  saves all registered variables to restart files. Those variables are set
+!  through register_restart_field
+!
+!-------------------------------------------------------------------------------
+
+subroutine save_unlimited_axis_restart(fileObj,restartpath)
+  type(restart_file_type), intent(inout),target :: fileObj
+  character(len=336)                     :: restartpath ! The restart file path (dir/file).
+
+  integer            :: unit                 ! The mpp unit of the open file.
+  type(axistype)                      :: u_axis
+  type(axistype), dimension(4)        :: var_axes
+  type(var_type), pointer, save       :: cur_var=>NULL()
+  integer                             :: i, j, k, l, num_var_axes, cpack, idx
+  real, allocatable, dimension(:)     :: r1d
+  integer(LONG_KIND)                  :: check_val
+  character(len=256)                  :: checksum_char
+  type(domain2d), pointer :: domain =>NULL()
+  type(ax_type),  pointer :: axis   =>NULL()
+
+
+  if ( .NOT.fileObj%unlimited_axis ) then
+     call mpp_error(FATAL, "fms_io(save_unlimited_axis_restart): An unlimited axis has "// &
+          "not been defined for file "//trim(fileObj%name))
+  endif
+  domain =>fileObj%axes(UIDX)%domain
+
+  call mpp_open(unit,trim(restartpath),action=MPP_OVERWR,form=form, &
+                is_root_pe=fileObj%is_root_pe, domain=domain)
+
+  ! Set unlimited axis
+  axis => fileobj%axes(UIDX)
+  call mpp_write_meta(unit,u_axis,axis%name,data=sum(axis%nelems(:)),unlimited=.true.)
+  call write_meta_global(unit,fileObj)  ! Write any additional global metadata
+  call mpp_write(unit,u_axis)
+
+  ! write metadata for fields
+  do j = 1,fileObj%nvar
+     cur_var => fileObj%var(j)
+     if(cur_var%siz(4) > 1) call mpp_error(FATAL, &
+      "fms_io(save_restart): "//trim(cur_var%name)//" in file "//trim(fileObj%name)// &
+      " has more than one time level. Only single time level is currrently supported")
+
+     if(cur_var%ndim == 1) then
+        num_var_axes = 1
+        var_axes(1) = u_axis
+        else
+        call mpp_error(FATAL, 'fms_io(save_unlimited_axis_restart): Only vectors are currently supported')
+     endif
+
+     cpack = pack_size  ! Default size of real
+     if ( Associated(fileObj%p1dr(1,j)%p) ) then
+        check_val = mpp_chksum(fileObj%p1dr(1,j)%p(:))
+     else if ( Associated(fileObj%p1di(1,j)%p) ) then
+           ! Fill values are -HUGE(i4) which don't behave as desired for checksum algorithm
+        check_val = mpp_chksum(INT(fileObj%p1di(1,j)%p(:),8))
+           cpack = 0  ! Write data as integer*4
+        else
+        call mpp_error(FATAL, "fms_io(save_unlimited_axis_restart): There is no pointer associated with the record data of field "// &
+                trim(cur_var%name)//" of file "//trim(fileObj%name) )
+        end if
+     call mpp_write_meta(unit,cur_var%field, var_axes(1:num_var_axes), cur_var%name, &
+              cur_var%units,cur_var%longname,pack=cpack,checksum=(/check_val/))
+  enddo ! end j loop
+
+  ! write data of each field
+     do j=1,fileObj%nvar
+        cur_var => fileObj%var(j)
+     if ( Associated(fileObj%p1dr(1,j)%p) ) then
+        call mpp_write_unlimited_axis(unit,cur_var%field,domain,fileObj%p1dr(1,j)%p,fileObj%axes(UIDX)%nelems(:))
+     elseif ( Associated(fileObj%p1di(1,j)%p) ) then
+              allocate(r1d(cur_var%siz(1)) )
+        r1d = fileObj%p1di(1,j)%p
+        call mpp_write_unlimited_axis(unit,cur_var%field,domain,r1d,fileObj%axes(UIDX)%nelems(:))
+              deallocate(r1d)
+           else
+              call mpp_error(FATAL, "fms_io(save_restart): There is no pointer associated with the data of field "// &
+                     trim(cur_var%name)//" of file "//trim(fileObj%name) )
+           endif
+     enddo ! end j loop
+  call mpp_close(unit)
+  cur_var =>NULL()
+end subroutine save_unlimited_axis_restart
+
+!-------------------------------------------------------------------------------
+!
+!  saves all registered variables to restart files. Those variables are set
+!  through register_restart_field
+!
+!-------------------------------------------------------------------------------
+
+subroutine save_default_restart(fileObj,restartpath)
+  type(restart_file_type), intent(inout) :: fileObj
+  character(len=336)                     :: restartpath ! The restart file path (dir/file).
+
+  character(len=8)   :: suffix               ! A suffix (like _2) that is appended to the name of files after the first.
+  integer            :: var_sz, size_in_file ! The size in bytes of each variable and of the variables already in a file.
+  integer            :: unit                 ! The mpp unit of the open file.
+  real, dimension(max_axis_size)      :: axisdata
+  integer,        dimension(max_axes) :: id_x_axes, siz_x_axes
+  integer,        dimension(max_axes) :: id_y_axes, siz_y_axes
+  integer,        dimension(max_axes) :: id_z_axes, siz_z_axes
+  integer,        dimension(max_axes) :: id_a_axes, siz_a_axes
+  integer,        dimension(max_axes) :: x_axes_indx, y_axes_indx, z_axes_indx, a_axes_indx
+  type(axistype), dimension(max_axes) :: x_axes, y_axes, z_axes, a_axes
+  type(axistype)                      :: t_axes
+  integer                             :: num_var_axes
+  type(axistype), dimension(5)        :: var_axes
+  type(var_type), pointer, save       :: cur_var=>NULL()
+  integer                             :: num_x_axes, num_y_axes, num_z_axes, num_a_axes
+  integer                             :: naxes_x, naxes_y, naxes_z, naxes_a
+  integer                             :: i, j, k, l, siz, ind_dom
+  logical                             :: domain_present
+  real                                :: tlev
+  character(len=10)                   :: axisname
+  integer                             :: meta_size
+  type(domain2d)                      :: domain
+
+  real, allocatable, dimension(:,:,:) :: r3d
+  real, allocatable, dimension(:,:)   :: r2d
+  real, allocatable, dimension(:)     :: r1d
+  real                                :: r0d
+  integer(LONG_KIND), allocatable, dimension(:)    :: check_val
+  character(len=256)                  :: checksum_char
+  integer :: isc, iec, jsc, jec
+  integer :: isg, ieg, jsg, jeg
+  integer :: ishift, jshift, iadd, jadd
+  logical :: write_on_this_pe
+  type(domain2d), pointer :: io_domain =>NULL()
+
+  if (.not.associated(fileObj%var)) call mpp_error(FATAL, "fms_io(save_restart): " // &
+      "restart_file_type data must be initialized by calling register_restart_field before using it")
+
+  !-- no need to proceed if all the variables are read only.
+  if( all_field_read_only(fileObj) ) return
+
+  do i=1,max_axis_size
+     axisdata(i) = i
+  enddo
+
+  !--- check if any field in this file present domain.
+  domain_present = .false.
+  do j = 1, fileObj%nvar
+     if (fileObj%var(j)%domain_present) then
+        domain_present = .true.
+        ind_dom = j
+        exit
+     end if
   end do
   num_x_axes = unique_axes(fileObj, 1, id_x_axes, siz_x_axes, domain_x)
   num_y_axes = unique_axes(fileObj, 2, id_y_axes, siz_y_axes, domain_y)
   num_z_axes = unique_axes(fileObj, 3, id_z_axes, siz_z_axes          )
-  next_var = 1
-  meta_size = 0
+  num_a_axes = unique_axes(fileObj, 4, id_a_axes, siz_a_axes          )
+
+  write_on_this_pe = .false.
+  if(domain_present) then
+     io_domain => mpp_get_io_domain(array_domain(fileObj%var(ind_dom)%domain_idx))
+     if(associated(io_domain)) then
+       if(mpp_domain_is_tile_root_pe(io_domain)) write_on_this_pe = .true.
+     endif
+  endif
+  !--- always write out from root pe
+  if( fileObj%is_root_pe ) write_on_this_pe = .true.
+  
+  if( domain_present ) then
+     call mpp_open(unit,trim(restartpath),action=MPP_OVERWR,form=form,&
+          is_root_pe=fileObj%is_root_pe, domain=array_domain(fileObj%var(ind_dom)%domain_idx) )
+  else  ! global data
+     call mpp_open(unit,trim(restartpath),action=MPP_OVERWR,form=form,threading=MPP_SINGLE,&
+          fileset=MPP_SINGLE, is_root_pe=fileObj%is_root_pe)
+  end if
+
+  naxes_x = 0
+  x_axes_indx = 0
+  y_axes_indx = 0
+  z_axes_indx = 0
+  a_axes_indx = 0
+
+  ! write_out x_axes
   do j = 1, num_x_axes
-     meta_size = meta_size + siz_x_axes(j)
+     ! make sure this axis is used by some variable
+     do l=1,fileObj%nvar
+        if(fileObj%var(l)%read_only) cycle
+        if( fileObj%var(l)%id_axes(1) == j ) exit
+     end do
+     if( l > fileObj%nvar ) cycle
+     naxes_x = naxes_x + 1
+     x_axes_indx(naxes_x) = j
+     if (naxes_x < 10) then
+        write(axisname,'(a,i1)') 'xaxis_',naxes_x
+     else
+        write(axisname,'(a,i2)') 'xaxis_',naxes_x
+     endif
+     if(id_x_axes(j) > 0) then
+        call mpp_write_meta(unit,x_axes(j),axisname,'none',axisname, &
+             data=axisdata(1:siz_x_axes(j)),domain=domain_x(id_x_axes(j)),cartesian='X')
+     else
+        call mpp_write_meta(unit,x_axes(j),axisname,'none',axisname, &
+             data=axisdata(1:siz_x_axes(j)),cartesian='X')
+     endif
   end do
+
+  ! write out y_axes
+  naxes_y = 0
   do j = 1, num_y_axes
-     meta_size = meta_size + siz_y_axes(j)
+     ! make sure this axis is used by some variable
+     do l=1,fileObj%nvar
+        if(fileObj%var(l)%read_only) cycle
+        if( fileObj%var(l)%id_axes(2) == j ) exit
+     end do
+     if( l > fileObj%nvar ) cycle
+     naxes_y = naxes_y + 1
+     y_axes_indx(naxes_y) = j
+     if (naxes_y < 10) then
+        write(axisname,'(a,i1)') 'yaxis_',naxes_y
+     else
+        write(axisname,'(a,i2)') 'yaxis_',naxes_y
+     endif
+     if(id_y_axes(j) > 0) then
+        call mpp_write_meta(unit,y_axes(j),axisname,'none',axisname, &
+             data=axisdata(1:siz_y_axes(j)),domain=domain_y(id_y_axes(j)),cartesian='Y')
+     else
+        call mpp_write_meta(unit,y_axes(j),axisname,'none',axisname, &
+             data=axisdata(1:siz_y_axes(j)),cartesian='Y')
+     endif
   end do
+
+  ! write out z_axes
+  naxes_z = 0
   do j = 1, num_z_axes
-     meta_size = meta_size + siz_z_axes(j)
+     ! make sure this axis is used by some variable
+     do l=1,fileObj%nvar
+        if(fileObj%var(l)%read_only) cycle
+        if( fileObj%var(l)%id_axes(3) == j ) exit
+     end do
+     if( l > fileObj%nvar ) cycle
+     naxes_z = naxes_z + 1
+     z_axes_indx(naxes_z) = j
+     if (naxes_z < 10) then
+        write(axisname,'(a,i1)') 'zaxis_',naxes_z
+     else
+        write(axisname,'(a,i2)') 'zaxis_',naxes_z
+     endif
+     call mpp_write_meta(unit,z_axes(j),axisname,'none',axisname, &
+          data=axisdata(1:siz_z_axes(j)),cartesian='Z')
+  end do
+
+  ! write out a_axes
+  naxes_a = 0
+  do j = 1, num_a_axes
+     ! make sure this axis is used by some variable
+     do l=1,fileObj%nvar
+        if(fileObj%var(l)%read_only) cycle
+        if( fileObj%var(l)%id_axes(4) == j ) exit
+     end do
+     if( l > fileObj%nvar ) cycle
+     naxes_a = naxes_a + 1
+     a_axes_indx(naxes_a) = j
+     if (naxes_a < 10) then
+        write(axisname,'(a,i1)') 'aaxis_',naxes_a
+     else
+        write(axisname,'(a,i2)') 'aaxis_',naxes_a
+     endif
+     call mpp_write_meta(unit,a_axes(j),axisname,'none',axisname, &
+          data=axisdata(1:siz_a_axes(j)),cartesian='N')
   end do
-  meta_size = 8*(meta_size*2+1000)
 
-  do while (next_var <= fileObj%nvar )
-     start_var = next_var
-     size_in_file = meta_size
-     restartpath_var = restartpath
-     do j=start_var,fileObj%nvar
-        cur_var => fileObj%var(j)
-        var_sz = 8*cur_var%csiz(1)*cur_var%csiz(2)*cur_var%csiz(3)
-        if ((j==start_var) .OR. (size_in_file < MAX_FILE_SIZE-var_sz)) then
-           size_in_file = size_in_file + var_sz
-        else 
-           exit
-        endif
-     enddo
-     next_var = j
-     ! For distribute write, normally will not over the limit. 
-     if( nfiles > 0 ) then
-        if(fset_w == MPP_MULTI .AND. domain_present) call mpp_error(FATAL, "fms_io_mod(save_restart): "// &
-             "For distribute write(fileset_write='multi'), the file size should not be very large and need to be split")
-        if (nfiles < 10) then
-           write(suffix,'("_",I1)') nfiles
-        else if(nfiles < 100) then
-           write(suffix,'("_",I2)') nfiles
-        else
-           call mpp_error(FATAL, "fms_io(save_restart): num_files should be less than 100")
-        endif
-        !--- remove .nc from restartpath and attach suffix.
-        siz = len_trim(restartpath)
-        if(restartpath(siz-2:siz) == ".nc") then
-           restartpath_var = restartpath(1:siz-3)//trim(suffix)
-        else      
-           restartpath_var = trim(restartpath) // trim(suffix)
-        end if
-     end if
-     if( domain_present ) then
-        call mpp_open(unit,trim(restartpath_var),action=MPP_OVERWR,form=form,threading=thread_w,&
-             fileset=fset_w, is_root_pe=fileObj%is_root_pe, domain=array_domain(fileObj%var(ind_dom)%domain_idx) )
-     else  ! global data
-        call mpp_open(unit,trim(restartpath_var),action=MPP_OVERWR,form=form,threading=MPP_SINGLE,&
-             fileset=MPP_SINGLE, is_root_pe=fileObj%is_root_pe)
-     end if
+  ! write out time axis
+  call mpp_write_meta(unit,t_axes,&
+       'Time','time level','Time',cartesian='T')
+  ! write metadata for fields
+  do j = 1,fileObj%nvar
+     cur_var => fileObj%var(j)
+     if(cur_var%read_only) cycle
+     if(cur_var%siz(4) > 1 .AND. cur_var%siz(4) .NE. fileObj%max_ntime ) call mpp_error(FATAL, &
+      "fms_io(save_restart): "//trim(cur_var%name)//" in file "//trim(fileObj%name)// &
+      " has more than one time level, but number of time level is not equal to max_ntime")
+
+     if(cur_var%ndim == 0) then
+        num_var_axes = 1
+        var_axes(1) = t_axes
+     else if(cur_var%ndim == 1) then
+        num_var_axes = 1
+        var_axes(1) = x_axes(cur_var%id_axes(1))
+        if(cur_var%siz(4) == fileObj%max_ntime) then
+           num_var_axes = 2
+           var_axes(2) = t_axes
+        end if
+     else if(cur_var%ndim == 2) then
+        num_var_axes = 2
+        var_axes(1) = x_axes(cur_var%id_axes(1))
+        var_axes(2) = y_axes(cur_var%id_axes(2))
+        if(cur_var%siz(4) == fileObj%max_ntime) then
+           num_var_axes = 3
+           var_axes(3) = t_axes
+        end if
+     else if(cur_var%ndim == 3) then
+        num_var_axes = 3
+        var_axes(1) = x_axes(cur_var%id_axes(1))
+        var_axes(2) = y_axes(cur_var%id_axes(2))
+        var_axes(3) = z_axes(cur_var%id_axes(3))
+        if(cur_var%siz(4) == fileObj%max_ntime) then
+           num_var_axes = 4
+           var_axes(4) = t_axes
+        end if
+     else if(cur_var%ndim == 4) then
+        num_var_axes = 4
+        var_axes(1) = x_axes(cur_var%id_axes(1))
+        var_axes(2) = y_axes(cur_var%id_axes(2))
+        var_axes(3) = z_axes(cur_var%id_axes(3))
+        var_axes(4) = a_axes(cur_var%id_axes(4))
+        if(cur_var%siz(4) == fileObj%max_ntime) then
+           num_var_axes = 5
+           var_axes(5) = t_axes
+        end if
+     end if
+
+     if ( cur_var%domain_idx > 0) then
+       call mpp_get_compute_domain(array_domain(cur_var%domain_idx), isc, iec, jsc, jec)
+       call mpp_get_global_domain(array_domain(cur_var%domain_idx), isg, ieg, jsg, jeg)
+       call mpp_get_domain_shift(array_domain(cur_var%domain_idx), ishift, jshift, cur_var%position)
+     else if (ASSOCIATED(Current_domain)) then
+       call mpp_get_compute_domain(Current_domain, isc, iec, jsc, jec)
+       call mpp_get_global_domain(Current_domain, isg, ieg, jsg, jeg)
+       call mpp_get_domain_shift(Current_domain, ishift, jshift, cur_var%position)
+     else
+       iec = cur_var%ie
+       isc = cur_var%is
+       ieg = cur_var%ie
+       jec = cur_var%je
+       jsc = cur_var%js
+       jeg = cur_var%je
+       ishift = 0
+       jshift = 0
+     endif
+!     call return_domain(domain)
+     iadd = iec-isc ! Size of the i-dimension on this processor (-1 as it is an increment)
+     jadd = jec-jsc ! Size of the j-dimension on this processor
+     if(iec == ieg) iadd = iadd + ishift
+     if(jec == jeg) jadd = jadd + jshift
+
+     allocate(check_val(max(1,cur_var%siz(4))))
+     do k = 1, cur_var%siz(4)
+        if ( Associated(fileObj%p0dr(k,j)%p) ) then
+           check_val(k) = mpp_chksum(fileObj%p0dr(k,j)%p, (/mpp_pe()/) )
+        else if ( Associated(fileObj%p1dr(k,j)%p) ) then
+           check_val(k) = mpp_chksum(fileObj%p1dr(k,j)%p, (/mpp_pe()/) )
+        else if ( Associated(fileObj%p2dr(k,j)%p) ) then
+           check_val(k) = mpp_chksum(fileObj%p2dr(k,j)%p(cur_var%is:cur_var%is+iadd, cur_var%js:cur_var%js+jadd) )
+        else if ( Associated(fileObj%p3dr(k,j)%p) ) then
+           check_val(k) = mpp_chksum(fileObj%p3dr(k,j)%p(cur_var%is:cur_var%is+iadd, cur_var%js:cur_var%js+jadd, :) )
+        else if ( Associated(fileObj%p4dr(k,j)%p) ) then
+           check_val(k) = mpp_chksum(fileObj%p4dr(k,j)%p(cur_var%is:cur_var%is+iadd, cur_var%js:cur_var%js+jadd, :, :) )
+        else if ( Associated(fileObj%p0di(k,j)%p) ) then
+           check_val(k) = fileObj%p0di(k,j)%p
+        else if ( Associated(fileObj%p1di(k,j)%p) ) then
+           check_val(k) = mpp_chksum(fileObj%p1di(k,j)%p, (/mpp_pe()/) )
+        else if ( Associated(fileObj%p2di(k,j)%p) ) then
+           check_val(k) = mpp_chksum(fileObj%p2di(k,j)%p(cur_var%is:cur_var%is+iadd, cur_var%js:cur_var%js+jadd) )
+        else if ( Associated(fileObj%p3di(k,j)%p) ) then
+           check_val(k) = mpp_chksum(fileObj%p3di(k,j)%p(cur_var%is:cur_var%is+iadd, cur_var%js:cur_var%js+jadd, :))
+        else
+           call mpp_error(FATAL, "fms_io(save_restart): There is no pointer associated with the data of  field "// &
+                trim(cur_var%name)//" of file "//trim(fileObj%name) )
+        end if
+     enddo
+     call mpp_write_meta(unit,cur_var%field, var_axes(1:num_var_axes), cur_var%name, &
+              cur_var%units,cur_var%longname,pack=pack_size,checksum=check_val)
+     deallocate(check_val)
+  enddo
+
+  ! write values for ndim of spatial axes
+  do j = 1, naxes_x
+     call mpp_write(unit,x_axes(x_axes_indx(j)))
+  enddo
+  do j = 1, naxes_y
+     call mpp_write(unit,y_axes(y_axes_indx(j)))
+  enddo
+  do j = 1, naxes_z
+     call mpp_write(unit,z_axes(z_axes_indx(j)))
+  enddo
+
+  do j = 1, naxes_a
+     call mpp_write(unit,a_axes(a_axes_indx(j)))
+  enddo
+
+  ! write data of each field
+  do k = 1, fileObj%max_ntime
+     do j=1,fileObj%nvar
+        cur_var => fileObj%var(j)
+        if(cur_var%read_only) cycle
+        tlev=k
+        ! If some fields only have one time level, we do not need to write the second level, just keep
+        ! the data missing.
+        if(k <= cur_var%siz(4)) then
+           if(cur_var%domain_present) then  ! one 2-D or 3-D case possible present domain
+              if( Associated(fileObj%p2dr(k,j)%p) ) then
+                 call mpp_write(unit, cur_var%field, array_domain(cur_var%domain_idx), fileObj%p2dr(k,j)%p, tlev, &
+                                default_data=cur_var%default_data)
+              else if( Associated(fileObj%p3dr(k,j)%p) ) then
+                 call mpp_write(unit, cur_var%field, array_domain(cur_var%domain_idx), fileObj%p3dr(k,j)%p, tlev, &
+                                default_data=cur_var%default_data)
+              else if( Associated(fileObj%p4dr(k,j)%p) ) then
+                 call mpp_write(unit, cur_var%field, array_domain(cur_var%domain_idx), fileObj%p4dr(k,j)%p, tlev, &
+                                default_data=cur_var%default_data)
+              else if( Associated(fileObj%p2di(k,j)%p) ) then
+                 allocate(r2d(cur_var%siz(1), cur_var%siz(2)) )
+                 r2d = fileObj%p2di(k,j)%p
+                 call mpp_write(unit, cur_var%field, array_domain(cur_var%domain_idx), r2d, tlev, &
+                                default_data=cur_var%default_data)
+                 deallocate(r2d)
+              else if( Associated(fileObj%p3di(k,j)%p) ) then
+                 allocate(r3d(cur_var%siz(1), cur_var%siz(2), cur_var%siz(3)) )
+                 r3d = fileObj%p3di(k,j)%p
+                 call mpp_write(unit, cur_var%field, array_domain(cur_var%domain_idx), r3d, tlev, &
+                                default_data=cur_var%default_data)
+                 deallocate(r3d)
+              else
+                 call mpp_error(FATAL, "fms_io(save_restart): domain is present, "// &
+                      "field "//trim(cur_var%name)//" of file "//trim(fileObj%name)// &
+                      ", but none of p2dr, p3dr, p2di and p3di is associated")
+              end if
+           else if (write_on_this_pe) then
+              if ( Associated(fileObj%p0dr(k,j)%p) ) then
+                 call mpp_write(unit, cur_var%field, fileObj%p0dr(k,j)%p, tlev)
+              else if ( Associated(fileObj%p1dr(k,j)%p) ) then
+                 call mpp_write(unit, cur_var%field, fileObj%p1dr(k,j)%p, tlev)
+              else if ( Associated(fileObj%p2dr(k,j)%p) ) then
+                 call mpp_write(unit, cur_var%field, fileObj%p2dr(k,j)%p, tlev)
+              else if ( Associated(fileObj%p3dr(k,j)%p) ) then
+                 call mpp_write(unit, cur_var%field, fileObj%p3dr(k,j)%p, tlev)
+              else if ( Associated(fileObj%p4dr(k,j)%p) ) then
+                 call mpp_write(unit, cur_var%field, fileObj%p4dr(k,j)%p, tlev)
+              else if ( Associated(fileObj%p0di(k,j)%p) ) then
+                 r0d =  fileObj%p0di(k,j)%p
+                 call mpp_write(unit, cur_var%field, r0d,                  tlev)
+              else if ( Associated(fileObj%p1di(k,j)%p) ) then
+                 allocate(r1d(cur_var%siz(1)) )
+                 r1d = fileObj%p1di(k,j)%p
+                 call mpp_write(unit, cur_var%field, r1d,                  tlev)
+                 deallocate(r1d)
+              else if ( Associated(fileObj%p2di(k,j)%p) ) then
+                 allocate(r2d(cur_var%siz(1), cur_var%siz(2)) )
+                 r2d = fileObj%p2di(k,j)%p
+                 call mpp_write(unit, cur_var%field, r2d,                  tlev)
+                 deallocate(r2d)
+              else if ( Associated(fileObj%p3di(k,j)%p) ) then
+                 allocate(r3d(cur_var%siz(1), cur_var%siz(2), cur_var%siz(3)) )
+                 r3d = fileObj%p3di(k,j)%p
+                 call mpp_write(unit, cur_var%field, r3d,                  tlev)
+                 deallocate(r3d)
+              else
+                 call mpp_error(FATAL, "fms_io(save_restart): There is no pointer associated with the data of  field "// &
+                      trim(cur_var%name)//" of file "//trim(fileObj%name) )
+              end if
+           end if
+        end if
+     enddo ! end j loop
+  enddo ! end k loop
+  call mpp_close(unit)
+  cur_var =>NULL()
+end subroutine save_default_restart
+!-------------------------------------------------------------------------------
+!
+!  saves all registered border/halo variables to restart files. Those variables 
+!  are set through register_restart_field (region option)
+!
+!-------------------------------------------------------------------------------
+subroutine save_restart_border (fileObj, time_stamp, directory)
+  type(restart_file_type), intent(inout) :: fileObj
+  character(len=*),        intent(in), optional :: directory
+  character(len=*),        intent(in), optional :: time_stamp
+
+  character(len=256) :: dir
+  character(len=256) :: restartpath          ! The restart file path (dir/file).
+  character(len=80)  :: restartname          ! The restart file name (no dir).
+!rab  integer            :: start_var, next_var  ! The starting variables of the current and next files.
+  integer            :: unit                 ! The mpp unit of the open file.
+  real, dimension(max_axis_size)      :: axisdata
+  integer,        dimension(max_axes) :: id_x_axes, siz_x_axes
+  integer,        dimension(max_axes) :: id_y_axes, siz_y_axes
+  integer,        dimension(max_axes) :: id_z_axes, siz_z_axes
+  integer,        dimension(max_axes) :: x_axes_indx, y_axes_indx, z_axes_indx
+  type(axistype), dimension(max_axes) :: x_axes, y_axes, z_axes
+  type(axistype)                      :: t_axes
+  integer                             :: num_var_axes
+  type(axistype), dimension(4)        :: var_axes
+  type(var_type), pointer, save       :: cur_var=>NULL()
+  integer                             :: num_x_axes, num_y_axes, num_z_axes
+  integer                             :: naxes_x, naxes_y, naxes_z
+  integer                             :: i, j, k, l
+  integer                             :: isc, iec, jsc, jec
+  integer                             :: is, ie, js, je
+  integer                             :: i_add, i1, i2
+  integer                             :: j_add, j1, j2
+  integer                             :: i_glob, j_glob, k_glob
+  real                                :: tlev
+  character(len=10)                   :: axisname
+
+  real, allocatable, dimension(:,:)   :: r2d
+  real, allocatable, dimension(:,:,:) :: r3d
+  integer(LONG_KIND), allocatable, dimension(:)    :: check_val
+
+  !-- no need to proceed if all the variables are read only.
+  if( all_field_read_only(fileObj) ) return
+
+  do i=1,max_axis_size
+   axisdata(i) = i
+  enddo
+
+  dir = "RESTART"
+  if(present(directory)) dir = directory
+
+  restartname = fileObj%name
+  if (time_stamp_restart) then
+    if (PRESENT(time_stamp)) then
+      restartname = trim(time_stamp)//"."//trim(restartname)
+    endif
+  end if
+  if (len_trim(dir) > 0) then
+    restartpath = trim(dir)//"/"// trim(restartname)
+  else
+    restartpath = trim(restartname)
+  end if
+
+  num_x_axes = unique_axes(fileObj, 1, id_x_axes, siz_x_axes)
+  num_y_axes = unique_axes(fileObj, 2, id_y_axes, siz_y_axes)
+  num_z_axes = unique_axes(fileObj, 3, id_z_axes, siz_z_axes)
+
+  call mpp_open(unit,trim(restartpath),action=MPP_OVERWR,form=MPP_NETCDF,threading=MPP_SINGLE,&
+                fileset=MPP_SINGLE, is_root_pe=fileObj%is_root_pe)
+
+! write out axes
+  naxes_x = 0
+  x_axes_indx = 0
+  y_axes_indx = 0
+  z_axes_indx = 0
+
+! write out x_axes metadata
+  do j = 1, num_x_axes
+    ! make sure this axis is used by some variable
+    do l=1, fileObj%nvar
+      if(fileObj%var(l)%read_only) cycle
+      if (fileObj%var(l)%id_axes(1) == j) exit
+    end do
+    if( l > fileObj%nvar ) cycle  
+    naxes_x = naxes_x + 1
+    x_axes_indx(naxes_x) = j
+    if (naxes_x < 10) then
+      write(axisname,'(a,i1)') 'xaxis_',naxes_x
+    else
+      write(axisname,'(a,i2)') 'xaxis_',naxes_x
+    endif
+    call mpp_write_meta(unit,x_axes(j),axisname,'none',axisname, &
+                        data=axisdata(1:siz_x_axes(j)),cartesian='X')
+  end do
+
+! write out y_axes metadata
+  naxes_y = 0
+  do j = 1, num_y_axes
+    ! make sure this axis is used by some variable
+    do l=1, fileObj%nvar
+      if(fileObj%var(l)%read_only) cycle
+      if (fileObj%var(l)%id_axes(2) == j) exit
+    end do
+    if( l > fileObj%nvar ) cycle
+    naxes_y = naxes_y + 1
+    y_axes_indx(naxes_y) = j
+    if (naxes_y < 10) then
+      write(axisname,'(a,i1)') 'yaxis_',naxes_y
+    else
+      write(axisname,'(a,i2)') 'yaxis_',naxes_y
+    endif
+    call mpp_write_meta(unit,y_axes(j),axisname,'none',axisname, &
+                        data=axisdata(1:siz_y_axes(j)),cartesian='Y')
+  end do
+
+! write out z_axes metadata
+  naxes_z = 0
+  do j = 1, num_z_axes
+    ! make sure this axis is used by some variable
+    do l=1, fileObj%nvar
+      if(fileObj%var(l)%read_only) cycle
+      if (fileObj%var(l)%id_axes(3) == j) exit
+    end do
+    if( l > fileObj%nvar ) cycle
+    naxes_z = naxes_z + 1
+    z_axes_indx(naxes_z) = j
+    if (naxes_z < 10) then
+      write(axisname,'(a,i1)') 'zaxis_',naxes_z
+    else
+      write(axisname,'(a,i2)') 'zaxis_',naxes_z
+    endif
+    call mpp_write_meta(unit,z_axes(j),axisname,'none',axisname, &
+                        data=axisdata(1:siz_z_axes(j)),cartesian='Z')
+  end do
+
+! write out time axis
+  call mpp_write_meta(unit,t_axes,'Time','time level', &
+                      'Time',cartesian='T')
+
+! write metadata for fields
+  do j = 1, fileObj%nvar
+    cur_var => fileObj%var(j)
+    if(cur_var%read_only) cycle
+    if ((cur_var%siz(4) > 1) .AND. (cur_var%siz(4).NE.fileObj%max_ntime)) call mpp_error(FATAL, &
+     "fms_io(save_restart_border): "//trim(cur_var%name)//" in file "//trim(fileObj%name)// &
+     " has more than one time level, but number of time level is not equal to max_ntime")
+
+    if (cur_var%ndim == 2) then
+      num_var_axes = 2
+      var_axes(1) = x_axes(cur_var%id_axes(1))
+      var_axes(2) = y_axes(cur_var%id_axes(2))
+      if(cur_var%siz(4) == fileObj%max_ntime) then
+        num_var_axes = 3
+        var_axes(3) = t_axes
+      end if
+    else if (cur_var%ndim == 3) then
+      num_var_axes = 3
+      var_axes(1) = x_axes(cur_var%id_axes(1))
+      var_axes(2) = y_axes(cur_var%id_axes(2))
+      var_axes(3) = z_axes(cur_var%id_axes(3))
+      if(cur_var%siz(4) == fileObj%max_ntime) then
+        num_var_axes = 4
+        var_axes(4) = t_axes
+      end if
+    else
+      call mpp_error(FATAL, "fms_io(save_restart_border): "//trim(cur_var%name)//" in file "// &
+         trim(fileObj%name)//" has more than three dimension (not including time level)")      
+    end if
+   
+! cycle the loop for pes not a member of the current pelist
+    if (.not.ANY(mpp_pe().eq.cur_var%pelist(:))) cycle
+
+! IN ORDER TO GET CHECKSUM INFO, PERFORM THE GATHER AS IF YOU WILL BE DOING THE WRITE
+! BUT INSTEAD CHECKSUM THE RESULTING TEMPORARY ARRAY
+    allocate(check_val(max(1,cur_var%siz(4))))
+    do k = 1, cur_var%siz(4)
+! cycle the loop for pes not a member of the current pelist
+      if (.not.ANY(mpp_pe().eq.cur_var%pelist(:))) cycle
+      isc = cur_var%is
+      iec = cur_var%ie
+      jsc = cur_var%js
+      jec = cur_var%je
+! set up indices for local array segment pointer (pointer is 1-based)
+      i1 = 1 + cur_var%x_halo
+      i2 = i1 + (iec-isc)
+      j1 = 1 + cur_var%y_halo
+      j2 = j1 + (jec-jsc)
+! set up index shifts for global array r*d (1-based, but potentially needs offsets: i_add, j_add)
+      i_add = cur_var%ishift
+      j_add = cur_var%jshift
+! If some fields only have one time level, we do not need to write the second level, just keep
+! the data missing.
+      if(k <= cur_var%siz(4)) then
+        if ( Associated(fileObj%p2dr(k,j)%p) ) then
+          i_glob = cur_var%gsiz(1)
+          j_glob = cur_var%gsiz(2)
+          if (fileObj%is_root_pe) allocate(r2d(i_glob, j_glob))
+          call mpp_gather(isc+i_add, iec+i_add, jsc+j_add, jec+j_add, cur_var%pelist, & 
+                          fileObj%p2dr(k,j)%p(i1:i2,j1:j2), & 
+                          r2d, fileObj%is_root_pe)
+          check_val(k) = mpp_chksum(r2d, (/mpp_pe()/))
+          if (allocated(r2d)) deallocate(r2d)
+        else if ( Associated(fileObj%p3dr(k,j)%p) ) then
+          i_glob = cur_var%gsiz(1)
+          j_glob = cur_var%gsiz(2)
+          k_glob = cur_var%gsiz(3)
+          if (fileObj%is_root_pe) allocate(r3d(i_glob, j_glob, k_glob))
+          call mpp_gather(isc+i_add, iec+i_add, jsc+j_add, jec+j_add, k_glob, cur_var%pelist, &
+                          fileObj%p3dr(k,j)%p(i1:i2,j1:j2,:), r3d, fileObj%is_root_pe)
+          check_val(k) = mpp_chksum(r3d, (/mpp_pe()/))
+          if (allocated(r3d)) deallocate(r3d)
+        else
+          call mpp_error(FATAL, "fms_io(save_restart_border): no pointer associated with data of field "// &
+               trim(cur_var%name)//" in file "//trim(fileObj%name) )
+        end if
+      end if
+    enddo ! end k loop
+    call mpp_write_meta(unit,cur_var%field, var_axes(1:num_var_axes), cur_var%name, &
+                        cur_var%units,cur_var%longname,pack=pack_size,checksum=check_val)
+    if (allocated(check_val)) deallocate(check_val)
+  enddo
+
+! write values for ndim of spatial axes
+  do j = 1, naxes_x
+     call mpp_write(unit,x_axes(x_axes_indx(j)))
+  enddo
+  do j = 1, naxes_y
+     call mpp_write(unit,y_axes(y_axes_indx(j)))
+  enddo
+  do j = 1, naxes_z
+     call mpp_write(unit,z_axes(z_axes_indx(j)))
+  enddo
+
+! write data of each field
+  do k = 1, fileObj%max_ntime
+    tlev=k
+    do j=1, fileObj%nvar
+      cur_var => fileObj%var(j)
+      if(cur_var%read_only) cycle
+! cycle the loop for pes not a member of the current pelist
+      if (.not.ANY(mpp_pe().eq.cur_var%pelist(:))) cycle
+      isc = cur_var%is
+      iec = cur_var%ie
+      jsc = cur_var%js
+      jec = cur_var%je
+! set up indices for local array segment pointer (pointer is 1-based)
+      i1 = 1 + cur_var%x_halo
+      i2 = i1 + (iec-isc)
+      j1 = 1 + cur_var%y_halo
+      j2 = j1 + (jec-jsc)
+! set up index shifts for global array r*d (1-based, but potentially needs offsets: i_add, j_add)
+      i_add = cur_var%ishift
+      j_add = cur_var%jshift
+! If some fields only have one time level, we do not need to write the second level, just keep
+! the data missing.
+      if(k <= cur_var%siz(4)) then
+        if (Associated(fileObj%p2dr(k,j)%p)) then
+          i_glob = cur_var%gsiz(1)
+          j_glob = cur_var%gsiz(2)
+          if (fileObj%is_root_pe) allocate(r2d(i_glob, j_glob))
+          call mpp_gather(isc+i_add, iec+i_add, jsc+j_add, jec+j_add, cur_var%pelist, & 
+                          fileObj%p2dr(k,j)%p(i1:i2,j1:j2),   r2d, fileObj%is_root_pe)
+          call mpp_write(unit, cur_var%field, r2d, tlev)
+          if (allocated(r2d)) deallocate(r2d)
+        else if (Associated(fileObj%p3dr(k,j)%p)) then
+          i_glob = cur_var%gsiz(1)
+          j_glob = cur_var%gsiz(2)
+          k_glob = cur_var%gsiz(3)
+          if (fileObj%is_root_pe) allocate(r3d(i_glob, j_glob, k_glob))
+          call mpp_gather(isc+i_add, iec+i_add, jsc+j_add, jec+j_add, k_glob, cur_var%pelist, &
+                          fileObj%p3dr(k,j)%p(i1:i2,j1:j2,:), r3d, fileObj%is_root_pe)
+          call mpp_write(unit, cur_var%field, r3d, tlev)
+          if (allocated(r3d)) deallocate(r3d)
+        else
+          call mpp_error(FATAL, "fms_io(save_restart_border): no pointer associated with data of field "// &
+               trim(cur_var%name)//" in file "//trim(fileObj%name) )
+        end if
+      end if
+    enddo ! end j loop
+  enddo ! end k loop
+  call mpp_close(unit)
+
+  cur_var =>NULL()
+
+  if(print_chksum) call write_chksum(fileObj, MPP_OVERWR)
+  return
+
+end subroutine save_restart_border
+
+
+!-------------------------------------------------------------------------------
+!
+!  restores all registered border/halo variables to restart files. Those 
+!  variables are set through register_restart_field (region option)
+!
+!-------------------------------------------------------------------------------
+subroutine restore_state_border(fileObj, directory)
+  type(restart_file_type), intent(inout)       :: fileObj
+  character(len=*),      intent(in), optional  :: directory
+! Arguments: 
+!  (in)      directory - The directory where the restart or save
+!                        files should be found. The default is 'INPUT'
+  character(len=128) :: dir    
+  character(len=256) :: restartpath ! The restart file path (dir/file). 
+  character(len=200) :: filepath    ! The path (dir/file) to the file being opened.
+  character(len=80)  :: varname     ! A variable's name.
+  character(len=256) :: mesg        ! Message to be constructed for checksum error.
+  type(var_type), pointer, save       :: cur_var=>NULL()
+  integer                             :: ndim, nvar, natt, ntime, tlev, siz
+  type(fieldtype), allocatable        :: fields(:)
+  logical                             :: fexist
+  integer                             :: j, n, l, k, unit
+  real, allocatable, dimension(:,:,:) :: r3d
+  real, allocatable, dimension(:,:)   :: r2d
+  integer                             :: isc, iec, jsc, jec 
+  logical                             :: check_exist
+  integer                             :: i1, i2, j1, j2
+  integer                             :: ishift, jshift, i_add, j_add
+  integer                             :: i_glob, j_glob, k_glob
+  integer(LONG_KIND), dimension(3)    :: checksum_file
+  integer(LONG_KIND)                  :: checksum_data
+  logical                             :: is_there_a_checksum
 
-     ! write_out x_axes
-     naxes_x = 0
-     x_axes_indx = 0
-     y_axes_indx = 0
-     z_axes_indx = 0
+  if (.not.associated(fileObj%var)) call mpp_error(FATAL, "fms_io(restore_state_border): " // &
+      "restart_file_type data must be initialized by calling register_restart_field before using it")
 
-     do j = 1, num_x_axes
-        ! make sure this axis is used by some variable 
-        do l=start_var,next_var-1
-           if( fileObj%var(l)%id_axes(1) == j ) exit
-        end do
-        if(l == next_var) cycle  
-        naxes_x = naxes_x + 1
-        x_axes_indx(naxes_x) = j
-        if (naxes_x < 10) then
-           write(axisname,'(a,i1)') 'xaxis_',naxes_x
-        else
-           write(axisname,'(a,i2)') 'xaxis_',naxes_x
-        endif
-        if(id_x_axes(j) > 0) then
-           call mpp_write_meta(unit,x_axes(j),axisname,'none',axisname, &
-                data=axisdata(1:siz_x_axes(j)),domain=domain_x(id_x_axes(j)),cartesian='X')
-        else
-           call mpp_write_meta(unit,x_axes(j),axisname,'none',axisname, &
-                data=axisdata(1:siz_x_axes(j)),cartesian='X')
-        endif
-     end do
+  dir = 'INPUT'
+  if(present(directory)) dir = directory
 
-     ! write out y_axes
-     naxes_y = 0
-     do j = 1, num_y_axes
-        ! make sure this axis is used by some variable 
-        do l=start_var,next_var-1
-           if( fileObj%var(l)%id_axes(2) == j ) exit
-        end do
-        if(l == next_var) cycle  
-        naxes_y = naxes_y + 1
-        y_axes_indx(naxes_y) = j
-        if (naxes_y < 10) then
-           write(axisname,'(a,i1)') 'yaxis_',naxes_y
-        else
-           write(axisname,'(a,i2)') 'yaxis_',naxes_y
-        endif
-        if(id_y_axes(j) > 0) then
-           call mpp_write_meta(unit,y_axes(j),axisname,'none',axisname, &
-                data=axisdata(1:siz_y_axes(j)),domain=domain_y(id_y_axes(j)),cartesian='Y')
-        else
-           call mpp_write_meta(unit,y_axes(j),axisname,'none',axisname, &
-                data=axisdata(1:siz_y_axes(j)),cartesian='Y')
-        endif
-     end do
+  if(len_trim(dir) > 0) then
+     restartpath = trim(dir)//"/"// trim(fileObj%name)
+  else
+     restartpath = trim(fileObj%name)
+  end if
 
-     ! write out z_axes
-     naxes_z = 0
-     do j = 1, num_z_axes
-        ! make sure this axis is used by some variable 
-        do l=start_var,next_var-1
-           if( fileObj%var(l)%id_axes(3) == j ) exit
-        end do
-        if(l == next_var) cycle  
-        naxes_z = naxes_z + 1
-        z_axes_indx(naxes_z) = j
-        if (naxes_z < 10) then
-           write(axisname,'(a,i1)') 'zaxis_',naxes_z
-        else
-           write(axisname,'(a,i2)') 'zaxis_',naxes_z
-        endif
-        call mpp_write_meta(unit,z_axes(j),axisname,'none',axisname, &
-             data=axisdata(1:siz_z_axes(j)),cartesian='Z')
-     end do
+!--- first open the restart files
+!--- NOTE: For distributed restart file, we are assuming there is only one file exist.
 
-     ! write out time axis  
-     call mpp_write_meta(unit,t_axes,&
-          'Time','time level','Time',cartesian='T')
-     ! write metadata for fields
-     do j = start_var,next_var-1
-        cur_var => fileObj%var(j)
-        if(cur_var%siz(4) > 1 .AND. cur_var%siz(4) .NE. fileObj%max_ntime ) call mpp_error(FATAL, &
-         "fms_io(save_restart): "//trim(cur_var%name)//" in file "//trim(fileObj%name)// &
-         " has more than one time level, but number of time level is not equal to max_ntime")
-
-        if(cur_var%ndim == 0) then
-           num_var_axes = 1
-           var_axes(1) = t_axes
-        else if(cur_var%ndim == 1) then
-           num_var_axes = 1
-           var_axes(1) = x_axes(cur_var%id_axes(1))
-           if(cur_var%siz(4) == fileObj%max_ntime) then
-              num_var_axes = 2
-              var_axes(2) = t_axes
-           end if
-        else if(cur_var%ndim == 2) then
-           num_var_axes = 2
-           var_axes(1) = x_axes(cur_var%id_axes(1))
-           var_axes(2) = y_axes(cur_var%id_axes(2))
-           if(cur_var%siz(4) == fileObj%max_ntime) then
-              num_var_axes = 3
-              var_axes(3) = t_axes
-           end if
-        else if(cur_var%ndim == 3) then
-           num_var_axes = 3
-           var_axes(1) = x_axes(cur_var%id_axes(1))
-           var_axes(2) = y_axes(cur_var%id_axes(2))
-           var_axes(3) = z_axes(cur_var%id_axes(3))
-           if(cur_var%siz(4) == fileObj%max_ntime) then
-              num_var_axes = 4
-              var_axes(4) = t_axes
-           end if
-        end if
-        
-        if ( cur_var%domain_idx > 0) then
-          call mpp_get_compute_domain(array_domain(cur_var%domain_idx), isc, iec, jsc, jec)
-          call mpp_get_global_domain(array_domain(cur_var%domain_idx), isg, ieg, jsg, jeg)
-          call mpp_get_domain_shift(array_domain(cur_var%domain_idx), ishift, jshift, cur_var%position)
-        else if (ASSOCIATED(Current_domain)) then
-          call mpp_get_compute_domain(Current_domain, isc, iec, jsc, jec)
-          call mpp_get_global_domain(Current_domain, isg, ieg, jsg, jeg)
-          call mpp_get_domain_shift(Current_domain, ishift, jshift, cur_var%position)
-       else
-          iec = cur_var%ie
-          isc = cur_var%is
-          ieg = cur_var%ie
-          jec = cur_var%je
-          jsc = cur_var%js
-          jeg = cur_var%je
-          ishift = 0
-          jshift = 0
-        endif
-!        call return_domain(domain)
-        iadd = iec-isc ! Size of the i-dimension on this processor (-1 as it is an increment)
-        jadd = jec-jsc ! Size of the j-dimension on this processor
-        if(iec == ieg) iadd = iadd + ishift
-        if(jec == jeg) jadd = jadd + jshift
+  inquire (file=trim(restartpath), exist=fexist)
+  if (.not.fexist) call mpp_error(FATAL, "fms_io(restore_state_border): unable to find any restart &
+                                 &files specified by "//trim(restartpath))
+  call mpp_open(unit,trim(restartpath),action=MPP_RDONLY,form=MPP_NETCDF,threading=MPP_SINGLE,&
+                fileset=MPP_SINGLE, is_root_pe=fileObj%is_root_pe)
 
+! Read each variable from the first file in which it is found.
+  call mpp_get_info(unit, ndim, nvar, natt, ntime)
 
-        allocate(check_val(max(1,cur_var%siz(4))))
-        do k = 1, cur_var%siz(4)
-           if ( Associated(fileObj%p0dr(k,j)%p) ) then
-              check_val(k) = mpp_chksum(fileObj%p0dr(k,j)%p, (/mpp_pe()/) )
-           else if ( Associated(fileObj%p1dr(k,j)%p) ) then
-              check_val(k) = mpp_chksum(fileObj%p1dr(k,j)%p, (/mpp_pe()/) )
-           else if ( Associated(fileObj%p2dr(k,j)%p) ) then
-              check_val(k) = mpp_chksum(fileObj%p2dr(k,j)%p(cur_var%is:cur_var%is+iadd, cur_var%js:cur_var%js+jadd) )
-           else if ( Associated(fileObj%p3dr(k,j)%p) ) then
-              check_val(k) = mpp_chksum(fileObj%p3dr(k,j)%p(cur_var%is:cur_var%is+iadd, cur_var%js:cur_var%js+jadd, :) )
-           else if ( Associated(fileObj%p0di(k,j)%p) ) then
-              check_val(k) = fileObj%p0di(k,j)%p
-           else if ( Associated(fileObj%p1di(k,j)%p) ) then
-              check_val(k) = mpp_chksum(fileObj%p1di(k,j)%p, (/mpp_pe()/) )
-           else if ( Associated(fileObj%p2di(k,j)%p) ) then
-              check_val(k) = mpp_chksum(fileObj%p2di(k,j)%p(cur_var%is:cur_var%is+iadd, cur_var%js:cur_var%js+jadd) )
-           else if ( Associated(fileObj%p3di(k,j)%p) ) then
-              check_val(k) = mpp_chksum(fileObj%p3di(k,j)%p(cur_var%is:cur_var%is+iadd, cur_var%js:cur_var%js+jadd, :))
-           else
-              call mpp_error(FATAL, "fms_io(save_restart): There is no pointer associated with the data of  field "// &
-                   trim(cur_var%name)//" of file "//trim(fileObj%name) )
-           end if
-        enddo
-        call mpp_write_meta(unit,cur_var%field, var_axes(1:num_var_axes), cur_var%name, &
-                 cur_var%units,cur_var%longname,pack=pack_size,checksum=check_val)
-        deallocate(check_val)
-     enddo
+  allocate(fields(nvar))
+  call mpp_get_fields(unit,fields(1:nvar))
 
-     ! write values for ndim of spatial axes
-     do j = 1, naxes_x
-        call mpp_write(unit,x_axes(x_axes_indx(j)))
-     enddo
-     do j = 1, naxes_y
-        call mpp_write(unit,y_axes(y_axes_indx(j)))
-     enddo
-     do j = 1, naxes_z
-        call mpp_write(unit,z_axes(z_axes_indx(j)))
-     enddo
+  do j=1,fileObj%nvar
+    cur_var => fileObj%var(j)
+! cycle the loop for pes not a member of the current pelist
+    if (.not.ANY(mpp_pe().eq.cur_var%pelist(:))) cycle
+    isc = cur_var%is
+    iec = cur_var%ie
+    jsc = cur_var%js
+    jec = cur_var%je
+! set up indices for local array segment pointer (pointer is 1-based)
+    i1 = 1 + cur_var%x_halo
+    i2 = i1 + (iec-isc)
+    j1 = 1 + cur_var%y_halo
+    j2 = j1 + (jec-jsc)
+! set up index shifts for global array r*d (1-based, but potentially needs offsets: i_add, j_add)
+    i_add = cur_var%ishift
+    j_add = cur_var%jshift
+    do l=1, nvar
+      call mpp_get_atts(fields(l),name=varname)
+      if (lowercase(trim(varname)) == lowercase(trim(cur_var%name))) then
+        cur_var%initialized = .true.
+        check_exist = mpp_attribute_exist(fields(l),"checksum")
+        checksum_file = 0
+        is_there_a_checksum = .false.
+        if ( check_exist  ) then
+          call mpp_get_atts(fields(l),checksum=checksum_file)
+          is_there_a_checksum = .true.
+        endif
+        if (.NOT. checksum_required) is_there_a_checksum = .false. ! Do not need to do data checksumming.
 
-     ! write data of each field
-     do k = 1, fileObj%max_ntime
-        do j=start_var,next_var-1
-           cur_var => fileObj%var(j)
-           tlev=k
-           ! If some fields only have one time level, we do not need to write the second level, just keep
-           ! the data missing.
-           if(k <= cur_var%siz(4)) then
-              if(cur_var%domain_present) then  ! one 2-D or 3-D case possible present domain
-                 if( Associated(fileObj%p2dr(k,j)%p) ) then
-                    call mpp_write(unit, cur_var%field, array_domain(cur_var%domain_idx), fileObj%p2dr(k,j)%p, tlev, &
-                                   default_data=cur_var%default_data)
-                 else if( Associated(fileObj%p3dr(k,j)%p) ) then
-                    call mpp_write(unit, cur_var%field, array_domain(cur_var%domain_idx), fileObj%p3dr(k,j)%p, tlev, &
-                                   default_data=cur_var%default_data)
-                 else if( Associated(fileObj%p2di(k,j)%p) ) then
-                    allocate(r2d(cur_var%siz(1), cur_var%siz(2)) )
-                    r2d = fileObj%p2di(k,j)%p
-                    call mpp_write(unit, cur_var%field, array_domain(cur_var%domain_idx), r2d, tlev, &
-                                   default_data=cur_var%default_data)
-                    deallocate(r2d)
-                 else if( Associated(fileObj%p3di(k,j)%p) ) then
-                    allocate(r3d(cur_var%siz(1), cur_var%siz(2), cur_var%siz(3)) )
-                    r3d = fileObj%p3di(k,j)%p
-                    call mpp_write(unit, cur_var%field, array_domain(cur_var%domain_idx), r3d, tlev, &
-                                   default_data=cur_var%default_data)
-                    deallocate(r3d)
-                 else
-                    call mpp_error(FATAL, "fms_io(save_restart): domain is present and thread_w  "// &
-                         "is MPP_MULTI, field "//trim(cur_var%name)//" of file "//trim(fileObj%name)// &
-                         ", but none of p2dr, p3dr, p2di and p3di is associated") 
-                 end if
-              else if (thread_w == MPP_MULTI .or. cur_var%write_on_this_pe .or. &
-                       (fileObj%is_root_pe.and.thread_w == MPP_SINGLE)) then     
-                 if ( Associated(fileObj%p0dr(k,j)%p) ) then
-                    call mpp_write(unit, cur_var%field, fileObj%p0dr(k,j)%p, tlev)
-                 else if ( Associated(fileObj%p1dr(k,j)%p) ) then
-                    call mpp_write(unit, cur_var%field, fileObj%p1dr(k,j)%p, tlev)
-                 else if ( Associated(fileObj%p2dr(k,j)%p) ) then
-                    call mpp_write(unit, cur_var%field, fileObj%p2dr(k,j)%p, tlev)
-                 else if ( Associated(fileObj%p3dr(k,j)%p) ) then
-                    call mpp_write(unit, cur_var%field, fileObj%p3dr(k,j)%p, tlev)
-                 else if ( Associated(fileObj%p0di(k,j)%p) ) then
-                    r0d =  fileObj%p0di(k,j)%p
-                    call mpp_write(unit, cur_var%field, r0d,                  tlev)
-                 else if ( Associated(fileObj%p1di(k,j)%p) ) then
-                    allocate(r1d(cur_var%siz(1)) )
-                    r1d = fileObj%p1di(k,j)%p
-                    call mpp_write(unit, cur_var%field, r1d,                  tlev)
-                    deallocate(r1d)
-                 else if ( Associated(fileObj%p2di(k,j)%p) ) then
-                    allocate(r2d(cur_var%siz(1), cur_var%siz(2)) )
-                    r2d = fileObj%p2di(k,j)%p
-                    call mpp_write(unit, cur_var%field, r2d,                  tlev)
-                    deallocate(global_r2d, r2d)
-                 else if ( Associated(fileObj%p3di(k,j)%p) ) then
-                    allocate(r3d(cur_var%siz(1), cur_var%siz(2), cur_var%siz(3)) )
-                    r3d = fileObj%p3di(k,j)%p
-                    call mpp_write(unit, cur_var%field, r3d,                  tlev)
-                    deallocate(r3d)
-                 else
-                    call mpp_error(FATAL, "fms_io(save_restart): There is no pointer associated with the data of  field "// &
-                         trim(cur_var%name)//" of file "//trim(fileObj%name) )
-                 end if
-              end if
-           end if
-        enddo ! end j loop
-     enddo ! end k loop
-     call mpp_close(unit)
-     nfiles = nfiles+1
+        do k = 1, cur_var%siz(4)
+          tlev = k
+! read the field and scatter it to the rest of the pelist
+          if (Associated(fileObj%p2dr(k,j)%p)) then
+            i_glob = cur_var%gsiz(1)
+            j_glob = cur_var%gsiz(2)
+            if (fileObj%is_root_pe) allocate(r2d(i_glob, j_glob))
+            call mpp_read(unit, fields(l), r2d, tlev)
+            call mpp_scatter(isc+i_add, iec+i_add, jsc+j_add, jec+j_add, cur_var%pelist, &
+                             fileObj%p2dr(k,j)%p(i1:i2,j1:j2), r2d, fileObj%is_root_pe)
+            if ((fileObj%is_root_pe) .and. (is_there_a_checksum)) checksum_data = mpp_chksum(r2d, (/mpp_pe()/) )
+            if (allocated(r2d)) deallocate(r2d)
+          else if (Associated(fileObj%p3dr(k,j)%p)) then
+            i_glob = cur_var%gsiz(1)
+            j_glob = cur_var%gsiz(2)
+            k_glob = cur_var%gsiz(3)
+            if (fileObj%is_root_pe) allocate(r3d(i_glob, j_glob, k_glob))
+            call mpp_read(unit, fields(l), r3d, tlev)
+            call mpp_scatter(isc+i_add, iec+i_add, jsc+j_add, jec+j_add, k_glob, cur_var%pelist, &
+                             fileObj%p3dr(k,j)%p(i1:i2,j1:j2,:), r3d, fileObj%is_root_pe)
+            if ((fileObj%is_root_pe) .and. (is_there_a_checksum)) checksum_data = mpp_chksum(r3d, (/mpp_pe()/) )
+            if (allocated(r3d)) deallocate(r3d)
+          else
+            call mpp_error(FATAL, "fms_io(retore_state_border): no pointer associated with data of field "// &
+                  trim(cur_var%name)//" in file "//trim(fileObj%name) )
+          end if
+          if ((fileObj%is_root_pe) .and. (is_there_a_checksum) .and. (checksum_file(k)/=checksum_data)) then
+            write (mesg,'(a,Z16,a,Z16,a)') "Checksum of input field "// uppercase(trim(varname))//" ", checksum_data,&
+                         " does not match value ", checksum_file(k), " stored in "//uppercase(trim(fileObj%name)//"." )
+            call mpp_error(FATAL, "fms_io(restore_state_border): "//trim(mesg) )
+          endif
+        end do
+        exit ! Start search for next restart variable.
+      endif
+    enddo
   enddo
 
+  deallocate(fields)
+
+  call close_file(unit)
+
   cur_var =>NULL()
 
-  if(print_chksum) call write_chksum(fileObj, MPP_OVERWR)
+  ! check whether all fields have been found
+  do j = 1, fileObj%nvar
+    if (.not.ANY(mpp_pe().eq.fileObj%var(j)%pelist(:))) cycle
+    if (.NOT. fileObj%var(j)%initialized) then
+      if (fileObj%var(j)%mandatory) then
+        call mpp_error(FATAL, "fms_io(restore_state_border): unable to find mandatory variable "// &
+                       trim(fileObj%var(j)%name)//" in restart file "//trim(fileObj%name) )
+      end if
+    end if
+  end do
 
-end subroutine save_restart
+  if(print_chksum) call write_chksum(fileObj, MPP_RDONLY )
+  return
+
+end subroutine restore_state_border
 
 !-------------------------------------------------------------------------------
 !    This subroutine will calculate chksum and print out chksum information.
@@ -1991,7 +3440,7 @@ subroutine write_chksum(fileObj, action)
      if(iec == ieg) iadd = iadd + ishift
      if(jec == jeg) jadd = jadd + jshift
 
-     if(action == MPP_OVERWR .OR. (action == MPP_RDONLY .AND. cur_var%initialized) ) then 
+     if(action == MPP_OVERWR .OR. (action == MPP_RDONLY .AND. cur_var%initialized) ) then
         do k = 1, cur_var%siz(4)
            if ( Associated(fileObj%p0dr(k,j)%p) ) then
               data_chksum = mpp_chksum(fileObj%p0dr(k,j)%p, (/mpp_pe()/) )
@@ -2001,6 +3450,8 @@ subroutine write_chksum(fileObj, action)
               data_chksum = mpp_chksum(fileObj%p2dr(k,j)%p(cur_var%is:cur_var%is+iadd,cur_var%js:cur_var%js+jadd) )
            else if ( Associated(fileObj%p3dr(k,j)%p) ) then
               data_chksum = mpp_chksum(fileObj%p3dr(k,j)%p(cur_var%is:cur_var%is+iadd,cur_var%js:cur_var%js+jadd, :) )
+           else if ( Associated(fileObj%p4dr(k,j)%p) ) then
+              data_chksum = mpp_chksum(fileObj%p4dr(k,j)%p(cur_var%is:cur_var%is+iadd,cur_var%js:cur_var%js+jadd, :, :) )
            else if ( Associated(fileObj%p0di(k,j)%p) ) then
               data_chksum = fileObj%p0di(k,j)%p
            else if ( Associated(fileObj%p1di(k,j)%p) ) then
@@ -2015,7 +3466,7 @@ subroutine write_chksum(fileObj, action)
            end if
            outunit = stdout()
            write(outunit,'(a, I1, a, Z16)')'fms_io('//trim(routine_name)//'): At time level = ', k, ', chksum for "'// &
-                trim(cur_var%name)// '" of "'// trim(fileObj%name)// '" = ', data_chksum    
+                trim(cur_var%name)// '" of "'// trim(fileObj%name)// '" = ', data_chksum
 
         enddo
      endif
@@ -2034,12 +3485,12 @@ subroutine restore_state_all(fileObj, directory)
   type(restart_file_type), intent(inout)       :: fileObj
   character(len=*),      intent(in), optional  :: directory
 
-! Arguments: 
+! Arguments:
 !  (in)      directory - The directory where the restart or save
 !                        files should be found. The default is 'INPUT'
 
-  character(len=128) :: dir    
-  character(len=256) :: restartpath ! The restart file path (dir/file). 
+  character(len=128) :: dir
+  character(len=256) :: restartpath ! The restart file path (dir/file).
   character(len=200) :: filepath    ! The path (dir/file) to the file being opened.
   character(len=8)   :: suffix      ! A suffix (like "_2") that is added to any
                                     ! additional restart files.
@@ -2059,10 +3510,11 @@ subroutine restore_state_all(fileObj, directory)
   integer                             :: tile_id(1)
   real, allocatable, dimension(:,:,:) :: r3d
   real, allocatable, dimension(:,:)   :: r2d
-  real, allocatable, dimension(:)     :: r1d  
+  real, allocatable, dimension(:)     :: r1d
   real                                :: r0d
   type(domain2d), pointer, save       :: io_domain=>NULL()
-  integer                             :: isc, iec, jsc, jec, check_exist
+  integer                             :: isc, iec, jsc, jec
+  logical                             :: check_exist
   integer                             :: isg, ieg, jsg, jeg
   integer                             :: ishift, jshift, iadd, jadd
   integer(LONG_KIND), dimension(3)    :: checksum_file
@@ -2094,9 +3546,8 @@ subroutine restore_state_all(fileObj, directory)
 
   !--- first open all the restart files
   !--- NOTE: For distributed restart file, we are assuming there is only one file exist.
-
-  inquire (file=trim(restartpath)//trim(pe_name), exist=fexist)
-  if(.NOT. fexist .and. domain_present) then
+  fexist = .FALSE.
+  if(domain_present) then
      io_domain => mpp_get_io_domain(array_domain(domain_idx))
      if(associated(io_domain)) then
         tile_id = mpp_get_tile_id(io_domain)
@@ -2111,13 +3562,9 @@ subroutine restore_state_all(fileObj, directory)
   endif
   if(fexist) then
      nfile = 1
-     if(domain_present) then
-        call mpp_open(unit(nfile), trim(restartpath), form=form,action=MPP_RDONLY,threading=thread_r, &
-             fileset=MPP_MULTI, domain=array_domain(domain_idx) )
-     else
-        call mpp_open(unit(nfile), trim(restartpath), form=form,action=MPP_RDONLY,threading=thread_r, &
-             fileset=MPP_MULTI)
-     endif
+     !--- domain_present is true
+     call mpp_open(unit(nfile), trim(restartpath), form=form,action=MPP_RDONLY, &
+           threading=MPP_MULTI, domain=array_domain(domain_idx) )
   else
      do while(.true.)
         if (num_restart < 10) then
@@ -2129,19 +3576,19 @@ subroutine restore_state_all(fileObj, directory)
            siz = len_trim(restartpath)
            if(restartpath(siz-2:siz) == ".nc") then
               filepath = restartpath(1:siz-3)//trim(suffix)
-           else      
+           else
               filepath = trim(restartpath) // trim(suffix)
            end if
         else
            filepath = trim(restartpath)
         end if
-        inquire (file=trim(filepath), exist=fexist) 
+        inquire (file=trim(filepath), exist=fexist)
         if(.not. fexist) inquire(file=trim(filepath)//".nc", exist=fexist)
         if(fexist) then
            nfile = nfile + 1
            if(nfile > max_split_file) call mpp_error(FATAL, &
                 "fms_io(restore_state_all): nfile is larger than max_split_file, increase max_split_file")
-           call mpp_open(unit(nfile), trim(filepath), form=form,action=MPP_RDONLY,threading=thread_r, &
+           call mpp_open(unit(nfile), trim(filepath), form=form,action=MPP_RDONLY,threading=MPP_MULTI, &
                 fileset=MPP_SINGLE)
         else
            exit
@@ -2201,7 +3648,7 @@ subroutine restore_state_all(fileObj, directory)
               check_exist = mpp_attribute_exist(fields(l),"checksum")
               checksum_file = 0
               is_there_a_checksum = .false.
-              if ( check_exist > 0 ) then
+              if ( check_exist ) then
                 call mpp_get_atts(fields(l),checksum=checksum_file)
                 is_there_a_checksum = .true.
               endif
@@ -2209,7 +3656,7 @@ subroutine restore_state_all(fileObj, directory)
 
               do k = 1, cur_var%siz(4)
                  tlev = k
-                 if(domain_present) then 
+                 if(domain_present) then
                     if( Associated(fileObj%p0dr(k,j)%p) ) then
                        call mpp_read(unit(n), fields(l), fileObj%p0dr(k,j)%p, tlev)
                        if ( is_there_a_checksum ) checksum_data = mpp_chksum(fileObj%p0dr(k,j)%p, (/mpp_pe()/) )
@@ -2224,6 +3671,10 @@ subroutine restore_state_all(fileObj, directory)
                        call mpp_read(unit(n), fields(l), array_domain(domain_idx), fileObj%p3dr(k,j)%p, tlev)
                        if ( is_there_a_checksum ) &
                          checksum_data = mpp_chksum(fileObj%p3dr(k,j)%p(cur_var%is:cur_var%is+iadd,cur_var%js:cur_var%js+jadd, :) )
+                    else if( Associated(fileObj%p4dr(k,j)%p) ) then
+                       call mpp_read(unit(n), fields(l), array_domain(domain_idx), fileObj%p4dr(k,j)%p, tlev)
+                       if ( is_there_a_checksum ) &
+                         checksum_data = mpp_chksum(fileObj%p4dr(k,j)%p(cur_var%is:cur_var%is+iadd,cur_var%js:cur_var%js+jadd,:,:))
                     else if( Associated(fileObj%p0di(k,j)%p) ) then
                        call mpp_read(unit(n), fields(l), r0d, tlev)
                        fileObj%p0di(k,j)%p = r0d
@@ -2246,13 +3697,13 @@ subroutine restore_state_all(fileObj, directory)
                        allocate(r3d(cur_var%siz(1), cur_var%siz(2), cur_var%siz(3)) )
                        r3d = 0
                        call mpp_read(unit(n), fields(l), array_domain(domain_idx), r3d, tlev)
-                       fileObj%p3di(k,j)%p(isc:iec,jsc:jec,:) = r3d(isc:iec,jsc:jec,:) 
+                       fileObj%p3di(k,j)%p(isc:iec,jsc:jec,:) = r3d(isc:iec,jsc:jec,:)
                        if ( is_there_a_checksum ) &
                          checksum_data = mpp_chksum(fileObj%p3di(k,j)%p(cur_var%is:cur_var%is+iadd,cur_var%js:cur_var%js+jadd, :))
                        deallocate(r3d)
                     else
                        call mpp_error(FATAL, "fms_io(restore_state_all): domain is present for the field "//trim(varname)// &
-                            " of file "//trim(fileObj%name)//", but none of p2dr, p3dr, p2di and p3di is associated") 
+                            " of file "//trim(fileObj%name)//", but none of p2dr, p3dr, p2di and p3di is associated")
                     end if
                  else
                     if( Associated(fileObj%p0dr(k,j)%p) ) then
@@ -2269,20 +3720,24 @@ subroutine restore_state_all(fileObj, directory)
                        call mpp_read(unit(n), fields(l), fileObj%p3dr(k,j)%p, tlev)
                        if ( is_there_a_checksum ) &
                          checksum_data = mpp_chksum(fileObj%p3dr(k,j)%p(cur_var%is:cur_var%is+iadd,cur_var%js:cur_var%js+jadd, :) )
+                    else if( Associated(fileObj%p4dr(k,j)%p) ) then
+                       call mpp_read(unit(n), fields(l), fileObj%p4dr(k,j)%p, tlev)
+                       if ( is_there_a_checksum ) &
+                         checksum_data = mpp_chksum(fileObj%p4dr(k,j)%p(cur_var%is:cur_var%is+iadd,cur_var%js:cur_var%js+jadd,:,:))
                     else if( Associated(fileObj%p0di(k,j)%p) ) then
                        call mpp_read(unit(n), fields(l), r0d, tlev)
                        fileObj%p0di(k,j)%p = r0d
                        if ( is_there_a_checksum ) checksum_data = fileObj%p0di(k,j)%p
                     else if( Associated(fileObj%p1di(k,j)%p) ) then
                        allocate(r1d(cur_var%siz(1)) )
-                       call mpp_read(unit(n), fields(l), r1d, tlev)                
+                       call mpp_read(unit(n), fields(l), r1d, tlev)
                        fileObj%p1di(k,j)%p = r1d
                        if ( is_there_a_checksum ) checksum_data = mpp_chksum(fileObj%p1di(k,j)%p, (/mpp_pe()/) )
                        deallocate(r1d)
                     else if( Associated(fileObj%p2di(k,j)%p) ) then
                        allocate(r2d(cur_var%siz(1), cur_var%siz(2)) )
                        r2d = 0
-                       call mpp_read(unit(n), fields(l), r2d, tlev)                
+                       call mpp_read(unit(n), fields(l), r2d, tlev)
                        fileObj%p2di(k,j)%p = r2d
                        if ( is_there_a_checksum ) &
                          checksum_data = mpp_chksum(fileObj%p2di(k,j)%p(cur_var%is:cur_var%is+iadd,cur_var%js:cur_var%js+jadd) )
@@ -2290,7 +3745,7 @@ subroutine restore_state_all(fileObj, directory)
                     else if( Associated(fileObj%p3di(k,j)%p) ) then
                        allocate(r3d(cur_var%siz(1), cur_var%siz(2), cur_var%siz(3)) )
                        r3d = 0
-                       call mpp_read(unit(n), fields(l), r3d, tlev)                
+                       call mpp_read(unit(n), fields(l), r3d, tlev)
                        fileObj%p3di(k,j)%p = r3d
                        if ( is_there_a_checksum ) &
                          checksum_data = mpp_chksum(fileObj%p3di(k,j)%p(cur_var%is:cur_var%is+iadd,cur_var%js:cur_var%js+jadd, :))
@@ -2346,12 +3801,12 @@ subroutine restore_state_one_field(fileObj, id_field, directory)
   integer,                 intent(in)          :: id_field
   character(len=*),      intent(in), optional  :: directory
 
-! Arguments: 
+! Arguments:
 !  (in)      directory - The directory where the restart or save
 !                        files should be found. The default is 'INPUT'
 
-  character(len=128) :: dir    
-  character(len=256) :: restartpath ! The restart file path (dir/file). 
+  character(len=128) :: dir
+  character(len=256) :: restartpath ! The restart file path (dir/file).
   character(len=200) :: filepath    ! The path (dir/file) to the file being opened.
   character(len=8)   :: suffix      ! A suffix (like "_2") that is added to any
                                     ! additional restart files.
@@ -2371,10 +3826,11 @@ subroutine restore_state_one_field(fileObj, id_field, directory)
   integer                             :: j, n, l, k, missing_fields, domain_idx
   real, allocatable, dimension(:,:,:) :: r3d
   real, allocatable, dimension(:,:)   :: r2d
-  real, allocatable, dimension(:)     :: r1d  
+  real, allocatable, dimension(:)     :: r1d
   real                                :: r0d
   type(domain2d), pointer, save       :: io_domain=>NULL()
-  integer                             :: isc, iec, jsc, jec, check_exist
+  integer                             :: isc, iec, jsc, jec
+  logical                             :: check_exist
   integer                             :: isg, ieg, jsg, jeg
   integer                             :: ishift, jshift, iadd, jadd
   integer(LONG_KIND), dimension(3)    :: checksum_file ! There should be no more than 3 timelevels in a restart file.
@@ -2422,8 +3878,8 @@ subroutine restore_state_one_field(fileObj, id_field, directory)
   end if
   !--- first open all the restart files
   !--- NOTE: For distributed restart file, we are assuming there is only one file exist.
-  inquire (file=trim(restartpath)//trim(pe_name), exist=fexist)     
-  if(.NOT. fexist .and. domain_present) then
+  fexist = .FALSE.
+  if(domain_present) then
      io_domain => mpp_get_io_domain(array_domain(domain_idx))
      if(associated(io_domain)) then
         tile_id = mpp_get_tile_id(io_domain)
@@ -2439,13 +3895,9 @@ subroutine restore_state_one_field(fileObj, id_field, directory)
 
   if(fexist) then
      nfile = 1
-     if(domain_present) then
-        call mpp_open(unit(nfile), trim(restartpath), form=form,action=MPP_RDONLY,threading=thread_r, &
-             fileset=MPP_MULTI, domain=array_domain(domain_idx) )
-     else
-        call mpp_open(unit(nfile), trim(restartpath), form=form,action=MPP_RDONLY,threading=thread_r, &
-             fileset=MPP_MULTI)
-     endif
+     !--- domain_present is true here.
+     call mpp_open(unit(nfile), trim(restartpath), form=form,action=MPP_RDONLY, &
+             threading=MPP_MULTI, domain=array_domain(domain_idx) )
   else
      do while(.true.)
         if (num_restart < 10) then
@@ -2457,19 +3909,19 @@ subroutine restore_state_one_field(fileObj, id_field, directory)
            siz = len_trim(restartpath)
            if(restartpath(siz-2:siz) == ".nc") then
               filepath = restartpath(1:siz-3)//trim(suffix)
-           else      
+           else
               filepath = trim(restartpath) // trim(suffix)
            end if
         else
            filepath = trim(restartpath)
         end if
-        inquire (file=trim(filepath), exist=fexist) 
+        inquire (file=trim(filepath), exist=fexist)
         if(.not. fexist) inquire(file=trim(filepath)//".nc", exist=fexist)
         if(fexist) then
            nfile = nfile + 1
            if(nfile > max_split_file) call mpp_error(FATAL, &
                 "fms_io(restore_state_one_field): nfile is larger than max_split_file, increase max_split_file")
-           call mpp_open(unit(nfile), trim(filepath), form=form,action=MPP_RDONLY,threading=thread_r, &
+           call mpp_open(unit(nfile), trim(filepath), form=form,action=MPP_RDONLY,threading=MPP_MULTI, &
                 fileset=MPP_SINGLE)
         else
            exit
@@ -2497,7 +3949,7 @@ subroutine restore_state_one_field(fileObj, id_field, directory)
            check_exist = mpp_attribute_exist(fields(l),"checksum")
            checksum_file = 0
            is_there_a_checksum = .false.
-           if ( check_exist > 0 ) then
+           if ( check_exist ) then
              call mpp_get_atts(fields(l),checksum=checksum_file)
              is_there_a_checksum = .true.
            endif
@@ -2508,12 +3960,12 @@ subroutine restore_state_one_field(fileObj, id_field, directory)
            jec = cur_var%je
            do k = 1, cur_var%siz(4)
               tlev = k
-              if(domain_present) then        
+              if(domain_present) then
                  if( Associated(fileObj%p0dr(k,j)%p) ) then
                     call mpp_read(unit(n), fields(l), fileObj%p0dr(k,j)%p, tlev)
                     if ( is_there_a_checksum ) checksum_data = mpp_chksum(fileObj%p0dr(k,j)%p, (/mpp_pe()/) )
                  else if( Associated(fileObj%p1dr(k,j)%p) ) then
-                    call mpp_read(unit(n), fields(l), fileObj%p1dr(k,j)%p, tlev)       
+                    call mpp_read(unit(n), fields(l), fileObj%p1dr(k,j)%p, tlev)
                     if ( is_there_a_checksum ) checksum_data = mpp_chksum(fileObj%p1dr(k,j)%p, (/mpp_pe()/) )
                  else if( Associated(fileObj%p2dr(k,j)%p) ) then
                     call mpp_read(unit(n), fields(l), array_domain(domain_idx), fileObj%p2dr(k,j)%p, tlev)
@@ -2523,6 +3975,10 @@ subroutine restore_state_one_field(fileObj, id_field, directory)
                     call mpp_read(unit(n), fields(l), array_domain(domain_idx), fileObj%p3dr(k,j)%p, tlev)
                     if ( is_there_a_checksum ) checksum_data =&
                          & mpp_chksum(fileObj%p3dr(k,j)%p(cur_var%is:cur_var%is+iadd,cur_var%js:cur_var%js+jadd, :) )
+                 else if( Associated(fileObj%p4dr(k,j)%p) ) then
+                    call mpp_read(unit(n), fields(l), array_domain(domain_idx), fileObj%p4dr(k,j)%p, tlev)
+                    if ( is_there_a_checksum ) checksum_data =&
+                         & mpp_chksum(fileObj%p4dr(k,j)%p(cur_var%is:cur_var%is+iadd,cur_var%js:cur_var%js+jadd, :,:) )
                  else if( Associated(fileObj%p0di(k,j)%p) ) then
                     call mpp_read(unit(n), fields(l), r0d, tlev)
                     fileObj%p0di(k,j)%p = r0d
@@ -2537,7 +3993,7 @@ subroutine restore_state_one_field(fileObj, id_field, directory)
                     allocate(r2d(cur_var%siz(1), cur_var%siz(2)) )
                     r2d = 0
                     call mpp_read(unit(n), fields(l), array_domain(domain_idx), r2d, tlev)
-                    fileObj%p2di(k,j)%p(isc:iec,jsc:jec) = r2d(isc:iec,jsc:jec) 
+                    fileObj%p2di(k,j)%p(isc:iec,jsc:jec) = r2d(isc:iec,jsc:jec)
                     if ( is_there_a_checksum ) checksum_data =&
                          & mpp_chksum(fileObj%p2di(k,j)%p(cur_var%is:cur_var%is+iadd,cur_var%js:cur_var%js+jadd) )
                     deallocate(r2d)
@@ -2545,13 +4001,13 @@ subroutine restore_state_one_field(fileObj, id_field, directory)
                     allocate(r3d(cur_var%siz(1), cur_var%siz(2), cur_var%siz(3)) )
                     r3d = 0
                     call mpp_read(unit(n), fields(l), array_domain(domain_idx), r3d, tlev)
-                    fileObj%p3di(k,j)%p(isc:iec,jsc:jec,:) = r3d(isc:iec,jsc:jec,:) 
+                    fileObj%p3di(k,j)%p(isc:iec,jsc:jec,:) = r3d(isc:iec,jsc:jec,:)
                     if ( is_there_a_checksum ) checksum_data =&
                          & mpp_chksum(fileObj%p3di(k,j)%p(cur_var%is:cur_var%is+iadd,cur_var%js:cur_var%js+jadd, :))
                     deallocate(r3d)
                  else
                     call mpp_error(FATAL, "fms_io(restore_state_one_field): domain is present for the field "//trim(varname)// &
-                         " of file "//trim(fileObj%name)//", but none of p2dr, p3dr, p2di and p3di is associated") 
+                         " of file "//trim(fileObj%name)//", but none of p2dr, p3dr, p2di and p3di is associated")
                  end if
               else
                  if( Associated(fileObj%p0dr(k,j)%p) ) then
@@ -2568,20 +4024,24 @@ subroutine restore_state_one_field(fileObj, id_field, directory)
                     call mpp_read(unit(n), fields(l), fileObj%p3dr(k,j)%p, tlev)
                     if ( is_there_a_checksum ) checksum_data =&
                          & mpp_chksum(fileObj%p3dr(k,j)%p(cur_var%is:cur_var%is+iadd,cur_var%js:cur_var%js+jadd, :) )
+                 else if( Associated(fileObj%p4dr(k,j)%p) ) then
+                    call mpp_read(unit(n), fields(l), fileObj%p4dr(k,j)%p, tlev)
+                    if ( is_there_a_checksum ) checksum_data =&
+                         & mpp_chksum(fileObj%p4dr(k,j)%p(cur_var%is:cur_var%is+iadd,cur_var%js:cur_var%js+jadd, :, :) )
                  else if( Associated(fileObj%p0di(k,j)%p) ) then
                     call mpp_read(unit(n), fields(l), r0d, tlev)
                     fileObj%p0di(k,j)%p = r0d
                     if ( is_there_a_checksum ) checksum_data = fileObj%p0di(k,j)%p
                  else if( Associated(fileObj%p1di(k,j)%p) ) then
                     allocate(r1d(cur_var%siz(1)) )
-                    call mpp_read(unit(n), fields(l), r1d, tlev)                
+                    call mpp_read(unit(n), fields(l), r1d, tlev)
                     fileObj%p1di(k,j)%p = r1d
                     if ( is_there_a_checksum ) checksum_data = fileObj%p0di(k,j)%p
                     deallocate(r1d)
                  else if( Associated(fileObj%p2di(k,j)%p) ) then
                     allocate(r2d(cur_var%siz(1), cur_var%siz(2)) )
                     r2d = 0
-                    call mpp_read(unit(n), fields(l), r2d, tlev)                
+                    call mpp_read(unit(n), fields(l), r2d, tlev)
                     fileObj%p2di(k,j)%p = r2d
                     if ( is_there_a_checksum ) checksum_data =&
                          & mpp_chksum(fileObj%p2di(k,j)%p(cur_var%is:cur_var%is+iadd,cur_var%js:cur_var%js+jadd) )
@@ -2589,7 +4049,7 @@ subroutine restore_state_one_field(fileObj, id_field, directory)
                  else if( Associated(fileObj%p3di(k,j)%p) ) then
                     allocate(r3d(cur_var%siz(1), cur_var%siz(2), cur_var%siz(3)) )
                     r3d = 0
-                    call mpp_read(unit(n), fields(l), r3d, tlev)                
+                    call mpp_read(unit(n), fields(l), r3d, tlev)
                     fileObj%p3di(k,j)%p = r3d
                     if ( is_there_a_checksum ) checksum_data =&
                          & mpp_chksum(fileObj%p3di(k,j)%p(cur_var%is:cur_var%is+iadd,cur_var%js:cur_var%js+jadd, :))
@@ -2630,22 +4090,24 @@ end subroutine restore_state_one_field
 
 !-------------------------------------------------------------------------------
 !
-!     This routine will setup one entry to be written out 
+!     This routine will setup one entry to be written out
 !
 !-------------------------------------------------------------------------------
 subroutine setup_one_field(fileObj, filename, fieldname, field_siz, index_field,  domain, mandatory, &
-                           no_domain, scalar_or_1d, position, tile_count, data_default, longname, units)
+                           no_domain, scalar_or_1d, position, tile_count, data_default, longname, units, &
+                           compressed_axis, read_only)
   type(restart_file_type), intent(inout)         :: fileObj
-  character(len=*),         intent(in)           :: filename, fieldname 
+  character(len=*),         intent(in)           :: filename, fieldname
   integer, dimension(:),    intent(in)           :: field_siz
   integer,                  intent(out)          :: index_field
   type(domain2d), optional, intent(in), target   :: domain
   real,           optional, intent(in)           :: data_default
   logical,        optional, intent(in)           :: no_domain
-  logical,        optional, intent(in)           :: scalar_or_1d   
+  logical,        optional, intent(in)           :: scalar_or_1d
   integer,        optional, intent(in)           :: position, tile_count
   logical,          optional, intent(in)         :: mandatory
-  character(len=*), optional, intent(in)         :: longname, units
+  character(len=*), optional, intent(in)         :: longname, units, compressed_axis
+  logical,        optional, intent(in)           :: read_only  !The variable will not be written to restart file.
 
   !--- local variables
   integer                         :: i, domain_idx
@@ -2659,17 +4121,16 @@ subroutine setup_one_field(fileObj, filename, fieldname, field_siz, index_field,
   character(len=256)              :: fname, filename2, append_string
   type(domain2d), pointer, save   :: d_ptr   =>NULL()
   type(var_type), pointer, save   :: cur_var =>NULL()
-  type(domain2d), pointer, save   :: io_domain =>NULL()
-  integer                         :: length
+  integer                         :: length, n_field_siz
 
-  if(ANY(field_siz < 1)) then
-     call mpp_error(FATAL, "fms_io(setup_one_field): each entry of field_size should be a positive integer")
+  if(ANY(field_siz < 0)) then
+     call mpp_error(FATAL, "fms_io(setup_one_field): each entry of field_size should be a non-negative integer")
   end if
 
   if(PRESENT(data_default))then
      default_data=data_default
   else
-     default_data=0.
+     default_data = MPP_FILL_DOUBLE
   endif
 
   if(present(tile_count) .AND. .not. present(domain)) call mpp_error(FATAL, &
@@ -2702,7 +4163,7 @@ subroutine setup_one_field(fileObj, filename, fieldname, field_siz, index_field,
 
   !Append a string to the file name
   append_string=''
-  !If the filename_appendix  is set override the passed argument. 
+  !If the filename_appendix  is set override the passed argument.
   if(len_trim(filename_appendix) > 0)   append_string = filename_appendix
 
   if(len_trim(append_string) > 0) filename2 = trim(filename2)//'.'//trim(append_string)
@@ -2722,18 +4183,25 @@ subroutine setup_one_field(fileObj, filename, fieldname, field_siz, index_field,
      allocate(fileObj%p1dr(MAX_TIME_LEVEL_REGISTER, max_fields))
      allocate(fileObj%p2dr(MAX_TIME_LEVEL_REGISTER, max_fields))
      allocate(fileObj%p3dr(MAX_TIME_LEVEL_REGISTER, max_fields))
+     allocate(fileObj%p4dr(MAX_TIME_LEVEL_REGISTER, max_fields))
      allocate(fileObj%p0di(MAX_TIME_LEVEL_REGISTER, max_fields))
      allocate(fileObj%p1di(MAX_TIME_LEVEL_REGISTER, max_fields))
      allocate(fileObj%p2di(MAX_TIME_LEVEL_REGISTER, max_fields))
      allocate(fileObj%p3di(MAX_TIME_LEVEL_REGISTER, max_fields))
      !--- make sure fname is not used in other restart_file_type object.
      do i = 1, num_registered_files
-        if(trim(fname) == trim(registered_file(i)) ) call mpp_error(FATAL, &
-          'fms_io(setup_one_field): '//trim(fname)//' is already registered with other restart_file_type data')
+        if(trim(fname) == trim(registered_file(i)) ) then
+           call mpp_error(NOTE, &
+             'fms_io(setup_one_field): '//trim(fname)//' is already registered with other restart_file_type data')
+           exit
+        endif
      end do
      num_registered_files = num_registered_files + 1
+     if( num_registered_files > max_files_w ) call mpp_error(WARNING, &
+         'fms_io(setup_one_field): num_registered_files > max_files_w, increase fms_io_nml max_files_w')
      registered_file(num_registered_files) = trim(fname)
-     fileObj%name = trim(fname)         
+     fileObj%register_id = num_registered_files
+     fileObj%name = trim(fname)
      fileObj%tile_count=1
      if(present(tile_count)) fileObj%tile_count = tile_count
      if(ASSOCIATED(d_ptr))then
@@ -2747,17 +4215,18 @@ subroutine setup_one_field(fileObj, filename, fieldname, field_siz, index_field,
      do i = 1, max_fields
         fileObj%var(i)%name           = 'none'
         fileObj%var(i)%domain_present = .false.
-        fileObj%var(i)%write_on_this_pe = .false.
         fileObj%var(i)%domain_idx     = -1
         fileObj%var(i)%is_dimvar      = .false.
         fileObj%var(i)%position       = CENTER
         fileObj%var(i)%siz(:)         = 0
         fileObj%var(i)%gsiz(:)        = 0
         fileObj%var(i)%id_axes(:)     = -1
-        fileObj%var(i)%longname       = "";
-        fileObj%var(i)%units          = "none";
+        fileObj%var(i)%longname       = '';
+        fileObj%var(i)%units          = 'none';
         fileObj%var(i)%mandatory      = .true.
         fileObj%var(i)%initialized    = .false.
+        fileObj%var(i)%compressed_axis = ''
+        fileObj%var(i)%read_only      = .false.
      end do
   endif
 
@@ -2767,7 +4236,7 @@ subroutine setup_one_field(fileObj, filename, fieldname, field_siz, index_field,
      if(trim(fileObj%var(i)%name) == trim(fieldname)) then
         index_field = i
         exit
-     end if 
+     end if
   end do
 
   if(index_field > 0) then
@@ -2778,25 +4247,27 @@ subroutine setup_one_field(fileObj, filename, fieldname, field_siz, index_field,
 
      cur_var%siz(4) =  cur_var%siz(4) + field_siz(4)
      if(fileObj%max_ntime < cur_var%siz(4) ) fileObj%max_ntime = cur_var%siz(4)
-     ! the time level should be no larger than MAX_TIME_LEVEL_REGISTER ( = 2) 
+     ! the time level should be no larger than MAX_TIME_LEVEL_REGISTER ( = 2)
      if( cur_var%siz(4) > MAX_TIME_LEVEL_REGISTER ) call mpp_error(FATAL, 'fms_io(setup_one_field): ' // &
           'the time level of field '//trim(cur_var%name)//' in file '//trim(fileObj%name)// &
           ' is greater than MAX_TIME_LEVEL_REGISTER(=2), increase MAX_TIME_LEVEL_REGISTER or check your code')
-  else 
+  else
      fileObj%nvar = fileObj%nvar +1
      if(fileObj%nvar>max_fields) then
-        write(error_msg,'(I3,"/",I3)') fileObj%nvar, max_fields 
+        write(error_msg,'(I3,"/",I3)') fileObj%nvar, max_fields
         call  mpp_error(FATAL,'fms_io(setup_one_field): max_fields exceeded, needs increasing, nvar/max_fields=' &
              //trim(error_msg))
      endif
      index_field =  fileObj%nvar
      cur_var   => fileObj%var(index_field)
-     cur_var%siz(:)  = field_siz(:)
+     n_field_siz = size(field_siz(:))
+     cur_var%siz(1:n_field_siz)  = field_siz(1:n_field_siz)
      cur_var%gsiz(3) = field_siz(3)
-     cur_var%csiz(3) = field_siz(3)
+     if(n_field_siz == 5) cur_var%gsiz(4) = field_siz(5)
      cur_var%name = fieldname
      cur_var%default_data = default_data
      if(present(mandatory)) cur_var%mandatory = mandatory
+     if(present(read_only)) cur_var%read_only = read_only
      if(present(longname)) then
         cur_var%longname = longname
      else
@@ -2804,15 +4275,10 @@ subroutine setup_one_field(fileObj, filename, fieldname, field_siz, index_field,
      end if
      if(present(units))    cur_var%units    = units
      if(present(position)) cur_var%position = position
-     cur_var%is = 1; cur_var%ie =  cur_var%siz(1) 
+     if(present(compressed_axis)) cur_var%compressed_axis = compressed_axis
+     cur_var%is = 1; cur_var%ie =  cur_var%siz(1)
      cur_var%js = 1; cur_var%je =  cur_var%siz(2)
-     if(ASSOCIATED(d_ptr)) then
-        io_domain => mpp_get_io_domain(d_ptr)
-        if(associated(io_domain)) then
-           if(mpp_domain_is_tile_root_pe(io_domain)) cur_var%write_on_this_pe = .true.
-        endif
-     endif
-     
+
      if(ASSOCIATED(d_ptr) .AND. .NOT. is_scalar_or_1d ) then
         cur_var%domain_present = .true.
         domain_idx = lookup_domain(d_ptr)
@@ -2847,23 +4313,9 @@ subroutine setup_one_field(fileObj, filename, fieldname, field_siz, index_field,
         cur_var%je   = cur_var%js + cysize - 1;
         cur_var%gsiz(1)   = gxsize
         cur_var%gsiz(2)   = gysize
-        io_domain => mpp_get_io_domain(array_domain(domain_idx))
-        if(associated(io_domain)) then
-           call mpp_get_global_domain( io_domain, xsize=cxsize, ysize=cysize,tile_count=tile_count)
-           cur_var%csiz(1)   = cxsize
-           cur_var%csiz(2)   = cysize
-        else if(thread_w == MPP_MULTI) then
-           call mpp_get_compute_domain(array_domain(domain_idx), xsize=cxsize,ysize=cysize,tile_count=tile_count)
-           cur_var%csiz(1)   = cxsize
-           cur_var%csiz(2)   = cysize
-        else
-           cur_var%csiz(1)   = cur_var%gsiz(1)
-           cur_var%csiz(2)   = cur_var%gsiz(2)
-        end if
      else
         cur_var%domain_present=.false.
         cur_var%gsiz(1:2) = field_siz(1:2)
-        cur_var%csiz(1:2) = field_siz(1:2)
      endif
   end if
 
@@ -2877,14 +4329,14 @@ end subroutine setup_one_field
 subroutine write_data_2d_new(filename, fieldname, data, domain,    &
                              no_domain, position,tile_count, data_default)
 
-  character(len=*), intent(in)                 :: filename, fieldname 
+  character(len=*), intent(in)                 :: filename, fieldname
   real, dimension(:,:), intent(in)             :: data
   real, dimension(size(data,1),size(data,2),1) :: data_3d
   real, intent(in), optional                   :: data_default
   type(domain2d), intent(in), optional         :: domain
   logical, intent(in), optional                :: no_domain
   integer, intent(in), optional                :: position, tile_count
- 
+
   if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(write_data_2d_new):need to call fms_io_init first')
   data_3d(:,:,1) = data(:,:)
 
@@ -2896,19 +4348,19 @@ end subroutine write_data_2d_new
 ! ........................................................
 subroutine write_data_1d_new(filename, fieldname, data,domain, &
                              no_domain, tile_count, data_default)
-  
+
   type(domain2d), intent(in), optional   :: domain
-  character(len=*), intent(in)           :: filename, fieldname 
+  character(len=*), intent(in)           :: filename, fieldname
   real, dimension(:), intent(in)         :: data
   real, dimension(size(data(:)),1,1)     :: data_3d
   real, intent(in), optional             :: data_default
   logical, intent(in), optional          :: no_domain
   integer, intent(in), optional          :: tile_count
-  
-  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(write_data_1d_new): module not initialized')  
-  data_3d(:,1,1) = data(:)  
+
+  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(write_data_1d_new): module not initialized')
+  data_3d(:,1,1) = data(:)
   call write_data_3d_new(filename, fieldname, data_3d,domain,   &
-                         no_domain=no_domain, scalar_or_1d=.true., tile_count=tile_count, data_default=data_default)  
+                         no_domain=no_domain, scalar_or_1d=.true., tile_count=tile_count, data_default=data_default)
 end subroutine write_data_1d_new
 
 ! ..........................................................
@@ -2916,14 +4368,14 @@ subroutine write_data_scalar_new(filename, fieldname, data, domain, &
                                  no_domain, tile_count, data_default)
 
   type(domain2d), intent(in), optional   :: domain
-  character(len=*), intent(in)           :: filename, fieldname 
+  character(len=*), intent(in)           :: filename, fieldname
   real, intent(in)                       :: data
   real, dimension(1,1,1)                 :: data_3d
   real, intent(in), optional             :: data_default
   logical, intent(in), optional          :: no_domain
   integer, intent(in), optional          :: tile_count
-    
-  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(write_data_scalar_new):  module not initialized: '//fieldname)  
+
+  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(write_data_scalar_new):  module not initialized: '//fieldname)
 
   data_3d(1,1,1) = data
   call write_data_3d_new(filename, fieldname, data_3d,domain, &
@@ -2944,7 +4396,7 @@ function lookup_field_r(nfile,fieldname)
   do j = 1, files_read(nfile)%nvar
      if (trim(files_read(nfile)%var(j)%name) == trim(fieldname)) then
         lookup_field_r = j
-        exit 
+        exit
      endif
   enddo
   return
@@ -2969,7 +4421,7 @@ end function lookup_domain
 !.........................................................
 function lookup_axis(axis_sizes,siz,domains,dom)
 
-! Given axis size (global), this function returns the axis id  
+! Given axis size (global), this function returns the axis id
 
   integer, intent(in)      :: axis_sizes(:), siz
   type(domain1d), optional :: domains(:)
@@ -2982,7 +4434,7 @@ function lookup_axis(axis_sizes,siz,domains,dom)
   do j=1,size(axis_sizes(:))
      if (siz == axis_sizes(j)) then
         if (PRESENT(domains)) then
-           if (dom .EQ. domains(j)) then 
+           if (dom .EQ. domains(j)) then
               lookup_axis = j
               exit
            endif
@@ -2995,44 +4447,106 @@ function lookup_axis(axis_sizes,siz,domains,dom)
   if (lookup_axis == -1) call mpp_error(FATAL,'fms_io(lookup_axis): could not find axis in set of axes')
 end function lookup_axis
 !.....................................................................
-! 
+! 
+!
+! Given filename and fieldname, this subroutine returns the size of field
+!
+!   
+!   
+!    File name
+!   
+!   
+!    Field  name
+!   
+!   
+!    siz must be a dimension(4) array to retrieve the size of the field
+!   
+!   
+!    if this flag is present, field_size will not abort if
+!    called for a non-existent field.
+!    Instead it will return T or F depending on
+!    whether or not the field was found.
+!   
+subroutine field_size(filename, fieldname, siz, field_found, domain, no_domain )
+
+  character(len=*), intent(in)                 :: filename, fieldname
+  integer,       intent(inout)                 :: siz(:)
+  logical,       intent(out), optional         :: field_found
+  type(domain2d), intent(in), optional, target :: domain
+  logical,       intent(in),  optional         :: no_domain
+
+  integer                              :: nfile, unit
+  logical                              :: found, found_file
+  character(len=256)                   :: actual_file
+  logical                              :: read_dist, io_domain_exist, is_no_domain
+
+  if (size(siz(:)) < 4) call mpp_error(FATAL,'fms_io(field_size): size array must be >=4 to receive field size of ' &
+       //trim(fieldname)//' in file '// trim(filename))
+
+  is_no_domain = .false.
+  if(present(no_domain)) is_no_domain = no_domain
+
+!--- first need to get the filename, when is_no_domain is true, only check file without tile
+!--- if is_no_domain is false, first check no_domain=.false., then check no_domain = .true.
+  found_file = get_file_name(filename, actual_file, read_dist, io_domain_exist, no_domain=is_no_domain, &
+                             domain=domain)
+  !--- when is_no_domain is true and file is not found, send out error message.
+  if(is_no_domain .AND. .NOT. found_file) call mpp_error(FATAL, &
+         'fms_io_mod(field_size): file '//trim(filename)//' and corresponding distributed file are not found')
+  found = .false.
+  if(found_file) then
+     call get_file_unit(actual_file, unit, nfile, read_dist, io_domain_exist, domain=domain)
+     call get_size(unit,fieldname,siz,found)
+  endif
+
+  if(.not.found .AND. .not. is_no_domain) then
+    found_file =  get_file_name(filename, actual_file, read_dist, io_domain_exist, no_domain=.true.)
+    if(found_file) then
+      call get_file_unit(actual_file, unit, nfile, read_dist, io_domain_exist, domain=domain)
+      call get_size(unit,fieldname,siz,found)
+    endif
+  endif
+
+! If field_found is present we assume that it is being checked on exit.
+! If not present and the field was not found, exit with a FATAL error.
+  if( PRESENT(field_found) )then
+     field_found = found
+  else if (.not. found )then
+     call mpp_error(FATAL, 'fms_io(field_size): field '//trim(fieldname)//' NOT found in file '//trim(actual_file))
+  end if
+
+  return
+end subroutine field_size
+! 
+
+!.....................................................................
+! 
 !
-! Given filename and fieldname, this subroutine returns the size of field
+! Given filename and dimension name, this function returns the size of field
 !
 !   
 !   
 !    File name
 !   
-!   
+!   
 !    Field  name
 !   
-!   
-!    siz must be a dimension(4) array to retrieve the size of the field
-!   
-!   
-!    if this flag is present, field_size will not abort if
-!    called for a non-existent field.
-!    Instead it will return T or F depending on
-!    whether or not the field was found.
-!   
-subroutine field_size(filename, fieldname, siz, field_found, domain, no_domain )
+function dimension_size(filename, dimname, domain, no_domain )
 
-  character(len=*), intent(in)                 :: filename, fieldname
-  integer,       intent(inout)                 :: siz(:)
-  logical,       intent(out), optional         :: field_found  
+  character(len=*), intent(in)                 :: filename, dimname
   type(domain2d), intent(in), optional, target :: domain
   logical,       intent(in),  optional         :: no_domain
+  integer                                      :: dimension_size
 
   integer                              :: nfile, unit
   logical                              :: found, found_file
   character(len=256)                   :: actual_file
   logical                              :: read_dist, io_domain_exist, is_no_domain
 
-  if (size(siz(:)) < 4) call mpp_error(FATAL,'fms_io(field_size): size array must be >=4 to receive field size of ' &
-       //trim(fieldname)//' in file '// trim(filename))
-
   is_no_domain = .false.
   if(present(no_domain)) is_no_domain = no_domain
 
@@ -3042,31 +4556,112 @@ subroutine field_size(filename, fieldname, siz, field_found, domain, no_domain )
                              domain=domain)
   !--- when is_no_domain is true and file is not found, send out error message.
   if(is_no_domain .AND. .NOT. found_file) call mpp_error(FATAL, &
-         'fms_io_mod(field_size): file '//trim(filename)//' and corresponding distributed file are not found')
+         'fms_io_mod(dimesion_size): file '//trim(filename)//' and corresponding distributed file are not found')
   found = .false.
   if(found_file) then
      call get_file_unit(actual_file, unit, nfile, read_dist, io_domain_exist, domain=domain)
-     call get_size(unit,fieldname,siz,found)    
+     dimension_size = mpp_get_dimension_length(unit, dimname, found)
   endif
 
   if(.not.found .AND. .not. is_no_domain) then
     found_file =  get_file_name(filename, actual_file, read_dist, io_domain_exist, no_domain=.true.)
     if(found_file) then
       call get_file_unit(actual_file, unit, nfile, read_dist, io_domain_exist, domain=domain)
-      call get_size(unit,fieldname,siz,found)
+      dimension_size = mpp_get_dimension_length(unit, dimname, found)
     endif
   endif
 
+  if(.not. found) call mpp_error(FATAL, &
+         'fms_io_mod(dimesion_size): failed at inquiring size of dimesion '//trim(dimname)//' from file '//trim(filename))
+
+  return
+end function dimension_size
+! 
+
+
+!.....................................................................
+! 
+!
+! Given filename and fieldname, this subroutine returns the size of field
+! This is the io subset interface to field_size
+!
+!   
+!   
+!    File name
+!   
+!   
+!    Field  name
+!   
+!   
+!    siz must be a dimension(4) array to retrieve the size of the field
+!   
+!   
+!    if this flag is present, field_size will not abort if
+!    called for a non-existent field.
+!    Instead it will return T or F depending on
+!    whether or not the field was found.
+!   
+subroutine get_field_size(filename, fieldname, siz, field_found, domain, no_domain)
+
+  character(len=*), intent(in)                 :: filename, fieldname
+  integer,       intent(inout)                 :: siz(:)
+  logical,       intent(out), optional         :: field_found
+  type(domain2d), intent(in), optional, target :: domain
+  logical,       intent(in),  optional         :: no_domain
+
+  integer :: npes, p
+  integer, allocatable :: pelist(:)
+  logical :: found
+  type(domain2d), pointer :: domain_in =>NULL()
+  type(domain2d), pointer :: io_domain =>NULL()
+
+
+  if(PRESENT(domain)) then
+     domain_in =>domain
+  elseif(ASSOCIATED(current_domain)) then
+     domain_in =>current_domain
+  else
+     call mpp_error(FATAL,'fms_io(get_field_size): The domain must be defined')
+  endif
+
+  io_domain =>mpp_get_io_domain(domain)
+  if(.not. ASSOCIATED(io_domain)) call mpp_error(FATAL,'fms_io(get_field_size): The io domain must be defined')
+
+  npes = mpp_get_domain_npes(io_domain)
+  allocate(pelist(npes))
+  call mpp_get_pelist(io_domain,pelist)
+
+  if(mpp_pe() == pelist(1)) then
+     call field_size(filename, fieldname, siz, found, domain, no_domain)
+     if(.not. found) siz(:) = -1
+  endif
+  !--- z1l replace mpp_broadcast with mpp_send/mpp_recv to avoid hang in calling MPI_COMM_CREATE
+  !---     because size(pelist) might be different for different rank.
+  !--- prepost receive
+  if( mpp_pe() == pelist(1) ) then
+     do p = 2, npes
+        call mpp_send(siz(1), plen=size(siz(:)), to_pe=pelist(p), tag=COMM_TAG_1)
+     enddo
+     call mpp_sync_self()
+  else
+     call mpp_recv(siz(1), glen=size(siz(:)), from_pe=pelist(1), block=.false., tag=COMM_TAG_1)
+     call mpp_sync_self(check=EVENT_RECV)
+  endif
+
+  found = .true.
+  if(siz(1) == -1) found=.false.
+
 ! If field_found is present we assume that it is being checked on exit.
 ! If not present and the field was not found, exit with a FATAL error.
   if( PRESENT(field_found) )then
      field_found = found
   else if (.not. found )then
-     call mpp_error(FATAL, 'fms_io(field_size): field '//trim(fieldname)//' NOT found in file '//trim(actual_file))
-  end if
-
-  return
-end subroutine field_size
+  ! Force the error to be trapped with the correct file name
+     if(mpp_pe() == pelist(1)) call field_size(filename, fieldname, siz, domain=domain, no_domain=no_domain)
+  endif
+end subroutine get_field_size
 ! 
 
 subroutine get_size(unit, fieldname, siz, found)
@@ -3083,7 +4678,7 @@ subroutine get_size(unit, fieldname, siz, found)
      found = .false.
      call mpp_get_info(unit,ndim,nvar,natt,ntime)
      if (nvar > max_fields) then
-        write(error_msg,'(I3,"/",I3)') nvar,max_fields 
+        write(error_msg,'(I3,"/",I3)') nvar,max_fields
         call  mpp_error(FATAL,'fms_io(field_size): max_fields too small, needs increasing, nvar/max_fields=' &
              //trim(error_msg))!//' in file '//trim(filename))
      endif
@@ -3157,63 +4752,60 @@ end subroutine get_size
 !   
 !=====================================================================================
 subroutine read_data_i3d_new(filename,fieldname,data,domain,timelevel, &
-                             no_domain,position, tile_count, is_compressed)
+                                        no_domain,position, tile_count)
   character(len=*),           intent(in)   :: filename, fieldname
-  integer, dimension(:,:,:), intent(inout) :: data ! 3 dimensional data    
+  integer, dimension(:,:,:), intent(inout) :: data ! 3 dimensional data
   type(domain2d), intent(in),   optional   :: domain
   integer, intent(in),          optional   :: timelevel
   logical, intent(in),          optional   :: no_domain
   integer, intent(in) ,         optional   :: position, tile_count
-  logical, intent(in),          optional   :: is_compressed
 
   real, dimension(size(data,1),size(data,2),size(data,3)) :: r_data
   r_data = 0
   call read_data_3d_new(filename,fieldname,r_data,domain,timelevel, &
-                        no_domain, .false., position, tile_count, is_compressed)
+                        no_domain, .false., position, tile_count)
   data = CEILING(r_data)
 end subroutine read_data_i3d_new
 
 subroutine read_data_i2d_new(filename,fieldname,data,domain,timelevel, &
-                             no_domain,position, tile_count, is_compressed)
+                             no_domain,position, tile_count)
   character(len=*),         intent(in)   :: filename, fieldname
-  integer, dimension(:,:), intent(inout) :: data ! 2 dimensional data    
+  integer, dimension(:,:), intent(inout) :: data ! 2 dimensional data
   type(domain2d), intent(in), optional   :: domain
   integer, intent(in),        optional   :: timelevel
   logical, intent(in),        optional   :: no_domain
   integer, intent(in) ,       optional   :: position, tile_count
-  logical, intent(in),        optional   :: is_compressed
   real, dimension(size(data,1),size(data,2)) :: r_data
 
   r_data = 0
   call read_data_2d_new(filename,fieldname,r_data,domain,timelevel, &
-                        no_domain, position, tile_count, is_compressed)
+                        no_domain, position, tile_count)
   data = CEILING(r_data)
 end subroutine read_data_i2d_new
 !.....................................................................
 subroutine read_data_i1d_new(filename,fieldname,data,domain,timelevel, &
-                             no_domain, tile_count, is_compressed)
+                             no_domain, tile_count)
   character(len=*), intent(in)           :: filename, fieldname
-  integer, dimension(:), intent(inout)   :: data ! 1 dimensional data    
+  integer, dimension(:), intent(inout)   :: data ! 1 dimensional data
   type(domain2d), intent(in), optional   :: domain
   integer, intent(in) , optional         :: timelevel
   logical, intent(in), optional          :: no_domain
   integer, intent(in), optional          :: tile_count
-  logical, intent(in), optional          :: is_compressed
 
   real, dimension(size(data,1))        :: r_data
 
   call read_data_1d_new(filename,fieldname,r_data,domain,timelevel, &
-                        no_domain, tile_count, is_compressed)
+                        no_domain, tile_count)
   data = CEILING(r_data)
 end subroutine read_data_i1d_new
 !.....................................................................
 subroutine read_data_iscalar_new(filename,fieldname,data,domain,timelevel, &
                                  no_domain, tile_count)
   character(len=*), intent(in)           :: filename, fieldname
-  integer, intent(inout)                 :: data     
+  integer, intent(inout)                 :: data
   type(domain2d), intent(in), optional   :: domain
   integer, intent(in) , optional         :: timelevel
-  logical, intent(in), optional          :: no_domain 
+  logical, intent(in), optional          :: no_domain
   integer, intent(in), optional          :: tile_count
 
   real                                 :: r_data
@@ -3223,15 +4815,14 @@ subroutine read_data_iscalar_new(filename,fieldname,data,domain,timelevel, &
 end subroutine read_data_iscalar_new
 !=====================================================================================
 subroutine read_data_3d_new(filename,fieldname,data,domain,timelevel, &
-      no_domain, scalar_or_1d, position, tile_count, is_compressed)
+                            no_domain, scalar_or_1d, position, tile_count)
   character(len=*),                  intent(in) :: filename, fieldname
-  real, dimension(:,:,:),         intent(inout) :: data ! 3 dimensional data    
+  real, dimension(:,:,:),         intent(inout) :: data ! 3 dimensional data
   type(domain2d), target, optional,  intent(in) :: domain
   integer,                optional,  intent(in) :: timelevel
-  logical,                optional,  intent(in) :: no_domain 
+  logical,                optional,  intent(in) :: no_domain
   logical,                optional,  intent(in) :: scalar_or_1d
   integer,                optional,  intent(in) :: position, tile_count
-  logical,                optional,  intent(in) :: is_compressed
 
   character(len=256)            :: fname
   integer                       :: unit, siz_in(4)
@@ -3244,7 +4835,6 @@ subroutine read_data_3d_new(filename,fieldname,data,domain,timelevel, &
   integer                       :: ishift, jshift
   logical                       :: is_scalar_or_1d = .false.
   logical                       :: is_no_domain = .false.
-  logical                       :: compressed 
   logical                       :: read_dist, io_domain_exist, found_file
   type(domain2d), pointer, save :: d_ptr =>NULL()
   type(domain2d), pointer, save :: io_domain =>NULL()
@@ -3252,19 +4842,16 @@ subroutine read_data_3d_new(filename,fieldname,data,domain,timelevel, &
 
 ! read disttributed files is used when reading restart files that are NOT mppnccombined. In this
 ! case PE 0 will read file_res.nc.0000, PE 1 will read file_res.nc.0001 and so forth.
-! 
-! namelist to be used with read_dist_files: threading_read=multi,
-! threading_write=multi, fileset_write=multi.
-
+!
 ! Initialize files to default values
-  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(read_data_3d_new):  module not initialized')  
+  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(read_data_3d_new):  module not initialized')
   is_no_domain = .false.
   if (PRESENT(no_domain)) THEN
      if(PRESENT(domain) .AND. no_domain) &
        call mpp_error(FATAL, 'fms_io(read_data_3d_new): no_domain cannot be .true. when optional argument domain is present.')
      is_no_domain = no_domain
-  endif  
- 
+  endif
+
   if(PRESENT(domain))then
      d_ptr => domain
   elseif (ASSOCIATED(Current_domain) .AND. .NOT. is_no_domain ) then
@@ -3274,18 +4861,15 @@ subroutine read_data_3d_new(filename,fieldname,data,domain,timelevel, &
   is_scalar_or_1d = .false.
   if(present(scalar_or_1d)) is_scalar_or_1d = scalar_or_1d
 
-  compressed = .false.
-  if(present(is_compressed)) compressed = is_compressed
-
   if(.not. PRESENT(domain) .and. .not. ASSOCIATED(Current_domain) ) is_no_domain = .true.
 
   found_file = get_file_name(filename, fname, read_dist, io_domain_exist, is_no_domain, domain,  tile_count)
   if(.not.found_file) call mpp_error(FATAL, 'fms_io_mod(read_data_3d_new): file ' //trim(filename)// &
-          '(with the consideration of tile number) and corresponding distributed file are not found')  
+          '(with the consideration of tile number) and corresponding distributed file are not found')
   call get_file_unit(fname, unit, file_index, read_dist, io_domain_exist, domain=domain)
 
   siz_in(3) = size(data,3)
-  if(is_no_domain .or. .NOT. associated(d_ptr) .or. is_scalar_or_1d .or. compressed) then
+  if(is_no_domain .or. .NOT. associated(d_ptr) .or. is_scalar_or_1d) then
      gxsize = size(data,1)
      gysize = size(data,2)
   else if(read_dist) then
@@ -3306,48 +4890,45 @@ subroutine read_data_3d_new(filename,fieldname,data,domain,timelevel, &
        call mpp_error(FATAL,'fms_io(read_data_3d_new): data should be on either computer domain '//&
                             'or data domain when domain is present. '//&
                             'shape(data)=',shape(data),'  cxsize,cysize,dxsize,dysize=',(/cxsize,cysize,dxsize,dysize/))
-     end if 
+     end if
   endif
 
   if (PRESENT(timelevel)) then
      tlev = timelevel
   else
      tlev = 1
-  endif  
-
-  if ((thread_r == MPP_MULTI).or.(mpp_pe()==mpp_root_pe())) then
-     call get_field_id(unit, file_index, fieldname, index_field, is_no_domain, .false. )
-     siz_in = files_read(file_index)%var(index_field)%siz
-     if(files_read(file_index)%var(index_field)%is_dimvar ) then
-        if (.not. read_dist) then
-           if (siz_in(1) /= gxsize) &
-                call mpp_error(FATAL,'fms_io(read_data_3d_new), field '//trim(fieldname)// &
-                ' in file '//trim(filename)//' field size mismatch 2')
-        endif
-     else
-        if (siz_in(1) /= gxsize .or. siz_in(2) /= gysize .or. siz_in(3) /= size(data,3)) then
-           PRINT *, gxsize, gysize, size(data, 3), siz_in(1), siz_in(2), siz_in(3)
-           call mpp_error(FATAL,'fms_io(read_data_3d_new), field '//trim(fieldname)// &
-                ' in file '//trim(filename)//': field size mismatch 1')
-        endif
-     end if
-     if ( tlev < 1 .or. files_read(file_index)%max_ntime < tlev)  then
-        write(error_msg,'(I5,"/",I5)') tlev, files_read(file_index)%max_ntime
-        call mpp_error(FATAL,'fms_io(read_data_3d_new): time level out of range, time level/max_time_level=' &
-             //trim(error_msg)//' in field/file: '//trim(fieldname)//'/'//trim(filename))
+  endif
+
+  call get_field_id(unit, file_index, fieldname, index_field, is_no_domain, .false. )
+  siz_in(1:4) = files_read(file_index)%var(index_field)%siz(1:4)
+  if(files_read(file_index)%var(index_field)%is_dimvar ) then
+     if (.not. read_dist) then
+        if (siz_in(1) /= gxsize) &
+             call mpp_error(FATAL,'fms_io(read_data_3d_new), field '//trim(fieldname)// &
+             ' in file '//trim(filename)//' field size mismatch 2')
+     endif
+  else
+     if (siz_in(1) /= gxsize .or. siz_in(2) /= gysize .or. siz_in(3) /= size(data,3)) then
+        PRINT *, gxsize, gysize, size(data, 3), siz_in(1), siz_in(2), siz_in(3)
+        call mpp_error(FATAL,'fms_io(read_data_3d_new), field '//trim(fieldname)// &
+             ' in file '//trim(filename)//': field size mismatch 1')
      endif
+  end if
+  if ( tlev < 1 .or. files_read(file_index)%max_ntime < tlev)  then
+     write(error_msg,'(I5,"/",I5)') tlev, files_read(file_index)%max_ntime
+     call mpp_error(FATAL,'fms_io(read_data_3d_new): time level out of range, time level/max_time_level=' &
+          //trim(error_msg)//' in field/file: '//trim(fieldname)//'/'//trim(filename))
+  endif
 
-     
-     if(is_no_domain .OR. is_scalar_or_1d .or. compressed) then
-        if (files_read(file_index)%var(index_field)%is_dimvar) then
-           call mpp_get_axis_data(files_read(file_index)%var(index_field)%axis,data(:,1,1))
-        else
-           call mpp_read(unit,files_read(file_index)%var(index_field)%field,data(:,:,:),tlev)
-        endif
-     else 
-        call mpp_read(unit,files_read(file_index)%var(index_field)%field,d_ptr,data,tlev,tile_count)
+  if(is_no_domain .OR. is_scalar_or_1d) then
+     if (files_read(file_index)%var(index_field)%is_dimvar) then
+        call mpp_get_axis_data(files_read(file_index)%var(index_field)%axis,data(:,1,1))
+     else
+        call mpp_read(unit,files_read(file_index)%var(index_field)%field,data(:,:,:),tlev)
      endif
-  endif  
+  else
+     call mpp_read(unit,files_read(file_index)%var(index_field)%field,d_ptr,data,tlev,tile_count)
+  endif
 
   d_ptr =>NULL()
 
@@ -3355,16 +4936,185 @@ subroutine read_data_3d_new(filename,fieldname,data,domain,timelevel, &
 end subroutine read_data_3d_new
 
 
+!=====================================================================================
+subroutine read_compressed_i1d(filename,fieldname,data,domain,timelevel,start,nread,threading)
+  character(len=*), intent(in)           :: filename, fieldname
+  integer, dimension(:), intent(inout)   :: data ! 1 dimensional data
+  type(domain2d), intent(in), optional   :: domain
+  integer, intent(in) , optional         :: timelevel
+  integer, intent(in) , optional         :: start(:), nread(:)
+  integer, intent(in) , optional         :: threading
+  real, dimension(size(data))        :: r_data
+
+  r_data = 0.0
+  call read_compressed_1d(filename,fieldname,r_data,domain,timelevel,start,nread,threading)
+  data = CEILING(r_data)
+end subroutine read_compressed_i1d
+!.....................................................................
+subroutine read_compressed_i2d(filename,fieldname,data,domain,timelevel,start,nread,threading)
+  character(len=*),         intent(in)   :: filename, fieldname
+  integer, dimension(:,:), intent(inout) :: data ! 2 dimensional data
+  type(domain2d), intent(in), optional   :: domain
+  integer, intent(in),        optional   :: timelevel
+  integer, intent(in) , optional         :: start(:), nread(:)
+  integer, intent(in) , optional         :: threading
+  real, dimension(size(data,1),size(data,2)) :: r_data
+
+  r_data = 0.0
+  call read_compressed_2d(filename,fieldname,r_data,domain,timelevel,start,nread,threading)
+  data = CEILING(r_data)
+end subroutine read_compressed_i2d
+!.....................................................................
+subroutine read_compressed_1d(filename,fieldname,data,domain,timelevel,start,nread,threading)
+  character(len=*), intent(in)           :: filename, fieldname
+  real, dimension(:), intent(inout)      :: data     !1 dimensional data
+  real, dimension(size(data,1),1)        :: data_2d
+  type(domain2d), intent(in), optional   :: domain
+  integer, intent(in) , optional         :: timelevel
+  integer, intent(in) , optional         :: start(:), nread(:)
+  integer, intent(in) , optional         :: threading
+#ifdef use_CRI_pointers
+  pointer( p, data_2d )
+  p = LOC(data)
+#endif
+  call read_compressed_2d(filename,fieldname,data_2d,domain,timelevel,start,nread,threading)
+end subroutine read_compressed_1d
+!.....................................................................
+subroutine read_compressed_2d(filename,fieldname,data,domain,timelevel,start,nread,threading)
+  character(len=*), intent(in)           :: filename, fieldname
+  real, dimension(:,:), intent(inout)    :: data     !2 dimensional data
+  type(domain2d), target, optional, intent(in) :: domain
+  integer, intent(in) , optional         :: timelevel
+  integer, intent(in) , optional         :: start(:), nread(:)
+  integer, intent(in) , optional         :: threading
+
+  character(len=256)            :: fname
+  integer                       :: unit, siz_in(4)
+  integer                       :: file_index  ! index of the opened file in array files
+  integer                       :: index_field ! position of the fieldname in the list of variables
+  logical                       :: read_dist, io_domain_exist, found_file
+  type(domain2d), pointer, save :: d_ptr =>NULL()
+  type(domain2d), pointer, save :: io_domain =>NULL()
+
+! Initialize files to default values
+  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(read_compressed_2d):  module not initialized')
+
+  if(PRESENT(domain))then
+     d_ptr => domain
+  elseif (ASSOCIATED(Current_domain)) then
+     d_ptr => Current_domain
+  else
+     call mpp_error(FATAL,'fms_io(read_compressed_2d): Domain must be an argument or set by set_domain()')
+  endif
+
+  found_file = get_file_name(filename, fname, read_dist, io_domain_exist, domain=d_ptr)
+  if(.not. found_file) then
+     found_file = get_file_name(filename, fname, read_dist, io_domain_exist, no_domain=.true. )
+  endif
+  if(.not.found_file) call mpp_error(FATAL, 'fms_io_mod(read_compressed_2d): file ' //trim(filename)// &
+          '(with the consideration of tile number) and corresponding distributed file are not found')
+  call get_file_unit(fname, unit, file_index, read_dist, io_domain_exist, domain=d_ptr)
+  call get_field_id(unit, file_index, fieldname, index_field, .false., .false. )
+
+  if (files_read(file_index)%var(index_field)%is_dimvar) then
+     call mpp_get_axis_data(files_read(file_index)%var(index_field)%axis,data(:,1))
+  else
+     call mpp_read_compressed(unit,files_read(file_index)%var(index_field)%field,d_ptr,data,timelevel,start,nread,threading)
+  endif
+  d_ptr =>NULL()
+end subroutine read_compressed_2d
+
+!.....................................................................
+subroutine read_distributed_a1D(unit,fmt,iostat,data)
+  integer, intent(in)               :: unit
+  character(*), intent(in)          :: fmt
+  integer, intent(out)              :: iostat
+  character(len=*), dimension(:), intent(inout) :: data
+
+  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(read_distributed_a1D):  module not initialized')
+  call mpp_read_distributed_ascii(unit,fmt,dr_set_size,data,iostat)
+end subroutine read_distributed_a1D
+
+!.....................................................................
+subroutine read_distributed_i1D(unit,fmt,iostat,data)
+  integer, intent(in)               :: unit
+  character(*), intent(in)          :: fmt
+  integer, intent(out)              :: iostat
+  integer, dimension(:), intent(inout) :: data
+
+  integer, allocatable :: pelist(:)
+  integer              :: i,lsize
+  logical              :: is_ioroot=.false.
+
+  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(read_distributed_i1D):  module not initialized')
+  call mpp_read_distributed_ascii(unit,fmt,dr_set_size,data,iostat)
+end subroutine read_distributed_i1D
+
+!.....................................................................
+subroutine read_distributed_iscalar(unit,fmt,iostat,data)
+  integer, intent(in)               :: unit
+  character(*), intent(in)          :: fmt
+  integer, intent(out)              :: iostat
+  integer, intent(inout) :: data
+
+  integer                           :: idata(1)
+  pointer(ptr,idata)
+
+  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(read_distributed_iscalar):  module not initialized')
+  ptr = LOC(data)
+  call read_distributed(unit,fmt,iostat,idata)
+end subroutine read_distributed_iscalar
+
+!.....................................................................
+subroutine read_distributed_r3D(unit,fmt,iostat,data)
+  integer, intent(in)               :: unit
+  character(*), intent(in)          :: fmt
+  integer, intent(out)              :: iostat
+  real, dimension(:,:,:), intent(inout) :: data
+
+  real :: data1D(size(data))
+  pointer(ptr,data1D)
+
+  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(read_distributed_r5D):  module not initialized')
+  ptr = LOC(data)
+  call read_distributed(unit,fmt,iostat,data1D)
+end subroutine read_distributed_r3D
+
+!.....................................................................
+subroutine read_distributed_r5D(unit,fmt,iostat,data)
+  integer, intent(in)               :: unit
+  character(*), intent(in)          :: fmt
+  integer, intent(out)              :: iostat
+  real, dimension(:,:,:,:,:), intent(inout) :: data
+
+  real :: data1D(size(data))
+  pointer(ptr,data1D)
+
+  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(read_distributed_r5D):  module not initialized')
+  ptr = LOC(data)
+  call read_distributed(unit,fmt,iostat,data1D)
+end subroutine read_distributed_r5D
+
+!.....................................................................
+subroutine read_distributed_r1D(unit,fmt,iostat,data)
+  integer, intent(in)               :: unit
+  character(*), intent(in)          :: fmt
+  integer, intent(out)              :: iostat
+  real, dimension(:), intent(inout) :: data
+
+  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(read_distributed_r1D):  module not initialized')
+  call mpp_read_distributed_ascii(unit,fmt,dr_set_size,data,iostat)
+end subroutine read_distributed_r1D
+
 !=====================================================================================
 subroutine read_data_2d_region(filename,fieldname,data,start,nread,domain, &
-                                 no_domain, tile_count, is_compressed)
+                                 no_domain, tile_count)
   character(len=*),                  intent(in) :: filename, fieldname
-  real, dimension(:,:),           intent(inout) :: data ! 3 dimensional data    
+  real, dimension(:,:),           intent(inout) :: data ! 3 dimensional data
   integer, dimension(:),             intent(in) :: start, nread
   type(domain2d), target,  optional, intent(in) :: domain
   logical,                 optional, intent(in) :: no_domain
   integer,                 optional, intent(in) :: tile_count
-  logical,                 optional, intent(in) :: is_compressed
   character(len=256)            :: fname
   integer                       :: unit, siz_in(4)
   integer                       :: file_index  ! index of the opened file in array files
@@ -3375,10 +5125,10 @@ subroutine read_data_2d_region(filename,fieldname,data,start,nread,domain, &
 
 
 ! Initialize files to default values
-  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(read_data_2d_region):  module not initialized')  
+  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(read_data_2d_region):  module not initialized')
   is_no_domain = .false.
   if (PRESENT(no_domain)) is_no_domain = no_domain
- 
+
   if(PRESENT(domain))then
      d_ptr => domain
   elseif (ASSOCIATED(Current_domain) .AND. .NOT. is_no_domain ) then
@@ -3389,18 +5139,16 @@ subroutine read_data_2d_region(filename,fieldname,data,start,nread,domain, &
 
   found_file = get_file_name(filename, fname, read_dist, io_domain_exist, is_no_domain, domain,  tile_count)
   if(.not.found_file) call mpp_error(FATAL, 'fms_io_mod(read_data_2d_region): file ' //trim(filename)// &
-          '(with the consideration of tile number) and corresponding distributed file are not found')  
+          '(with the consideration of tile number) and corresponding distributed file are not found')
   call get_file_unit(fname, unit, file_index, read_dist, io_domain_exist, domain=domain)
 
 
-  if ((thread_r == MPP_MULTI).or.(mpp_pe()==mpp_root_pe())) then
-     call get_field_id(unit, file_index, fieldname, index_field, is_no_domain, .false. )
-     siz_in = files_read(file_index)%var(index_field)%siz
-     if(files_read(file_index)%var(index_field)%is_dimvar) then
-        call mpp_error(FATAL, 'fms_io_mod(read_data_2d_region): the field should not be a dimension variable')
-     endif
-     call mpp_read(unit,files_read(file_index)%var(index_field)%field,data,start, nread)
-  endif  
+  call get_field_id(unit, file_index, fieldname, index_field, is_no_domain, .false. )
+  siz_in(1:4) = files_read(file_index)%var(index_field)%siz(1:4)
+  if(files_read(file_index)%var(index_field)%is_dimvar) then
+     call mpp_error(FATAL, 'fms_io_mod(read_data_2d_region): the field should not be a dimension variable')
+  endif
+  call mpp_read(unit,files_read(file_index)%var(index_field)%field,data,start, nread)
 
   d_ptr =>NULL()
 
@@ -3419,102 +5167,95 @@ subroutine read_data_text(filename,fieldname,data,level)
   character(len=256)             :: fname
 
 ! Initialize files to default values
-  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(read_data_text):  module not initialized')  
+  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(read_data_text):  module not initialized')
 
   file_opened=.false.
   if (PRESENT(level)) then
      lev = level
   else
      lev = 1
-  endif  
+  endif
 
   found_file = get_file_name(filename, fname, read_dist, io_domain_exist, no_domain=.true. )
  if(.not.found_file) call mpp_error(FATAL, 'fms_io_mod(read_data_text): file ' //trim(filename)// &
-          '(with the consideration of tile number) and corresponding distributed file are not found')  
+          '(with the consideration of tile number) and corresponding distributed file are not found')
   call get_file_unit(fname, unit, file_index, read_dist, io_domain_exist )
 
-! Get info of this file and field   
-  if ((thread_r == MPP_MULTI).or.(mpp_pe()==mpp_root_pe())) then
-     call get_field_id(unit, file_index, fieldname, index_field, .true., .true. )     
+! Get info of this file and field
+  call get_field_id(unit, file_index, fieldname, index_field, .true., .true. )
 
-     if ( lev < 1 .or. lev > files_read(file_index)%var(index_field)%siz(1) )  then
-        write(error_msg,'(I5,"/",I5)') lev, files_read(file_index)%var(index_field)%siz(1)
-        call mpp_error(FATAL,'fms_io(read_data_text): text level out of range, level/max_level=' &
-             //trim(error_msg)//' in field/file: '//trim(fieldname)//'/'//trim(filename))
-     endif
+  if ( lev < 1 .or. lev > files_read(file_index)%var(index_field)%siz(1) )  then
+     write(error_msg,'(I5,"/",I5)') lev, files_read(file_index)%var(index_field)%siz(1)
+     call mpp_error(FATAL,'fms_io(read_data_text): text level out of range, level/max_level=' &
+          //trim(error_msg)//' in field/file: '//trim(fieldname)//'/'//trim(filename))
+  endif
 
-     call mpp_read(unit,files_read(file_index)%var(index_field)%field,data, level=level)
-  endif  
+  call mpp_read(unit,files_read(file_index)%var(index_field)%field,data, level=level)
   return
 end subroutine read_data_text
-!.............................................................. 
+!..............................................................
 ! 
 subroutine read_data_2d_new(filename,fieldname,data,domain,timelevel,&
-      no_domain,position,tile_count, is_compressed)
+                            no_domain,position,tile_count)
   character(len=*), intent(in)                 :: filename, fieldname
-  real, dimension(:,:), intent(inout)          :: data     !2 dimensional data 
+  real, dimension(:,:), intent(inout)          :: data     !2 dimensional data
   real, dimension(size(data,1),size(data,2),1) :: data_3d
   type(domain2d), intent(in), optional         :: domain
   integer, intent(in) , optional               :: timelevel
   logical, intent(in), optional                :: no_domain
   integer, intent(in) , optional               :: position, tile_count
-  logical, intent(in), optional                :: is_compressed
+
 
   integer                                      :: isc,iec,jsc,jec,isd,ied,jsd,jed
   integer                                      :: isg,ieg,jsg,jeg
   integer                                      :: xsize_c,ysize_c,xsize_d,ysize_d
   integer                                      :: xsize_g,ysize_g, ishift, jshift
-  logical                                      :: compressed
 
 !#ifdef use_CRI_pointers
 !  pointer( p, data_3d )
 !  p = LOC(data)
 !#endif
 
-  compressed= .false.
-  if (PRESENT(is_compressed)) compressed=is_compressed
-
   call read_data_3d_new(filename,fieldname,data_3d,domain,timelevel,&
-        no_domain,.false., position,tile_count,is_compressed=compressed)
-  
+                        no_domain,.false., position,tile_count)
+
   if(PRESENT(domain)) then
      call mpp_get_global_domain( domain,isg,ieg,jsg,jeg,xsize=xsize_g,ysize=ysize_g, tile_count=tile_count, position=position)
      call mpp_get_compute_domain( domain,isc,iec,jsc,jec,xsize=xsize_c,ysize=ysize_c, tile_count=tile_count, position=position)
      call mpp_get_data_domain( domain,isd,ied,jsd,jed,xsize=xsize_d,ysize=ysize_d, tile_count=tile_count, position=position)
      call mpp_get_domain_shift  (domain, ishift, jshift, position)
-     if(((size(data,1)==xsize_c) .and. (size(data,2)==ysize_c)) .or. compressed) then !on_comp_domain
+     if(((size(data,1)==xsize_c) .and. (size(data,2)==ysize_c))) then !on_comp_domain
         data(:,:) = data_3d(:,:,1)
      else if((size(data,1)==xsize_d) .and. (size(data,2)==ysize_d)) then !on_data_domain
         data(isc-isd+1:iec-isd+1,jsc-jsd+1:jec-jsd+1) = data_3d(isc-isd+1:iec-isd+1,jsc-jsd+1:jec-jsd+1,1)
      else if((size(data,1)==xsize_g) .and. (size(data,2)==ysize_g)) then !on_global_domain
         data(:,:) = data_3d(:,:,1)
-     else 
+     else
         call mpp_error(FATAL,'error in read_data_2d_new, field '//trim(fieldname)// &
                       ' in file '//trim(filename)//' data must be in compute or data domain')
      endif
-  else     
+  else
      data(:,:) = data_3d(:,:,1)
   endif
 
 end subroutine read_data_2d_new
 !.....................................................................
 subroutine read_data_1d_new(filename,fieldname,data,domain,timelevel,&
-      no_domain, tile_count, is_compressed)
+                                               no_domain, tile_count)
   character(len=*), intent(in)           :: filename, fieldname
-  real, dimension(:), intent(inout)      :: data     !1 dimensional data 
+  real, dimension(:), intent(inout)      :: data     !1 dimensional data
   real, dimension(size(data,1),1,1)      :: data_3d
   type(domain2d), intent(in), optional   :: domain
   integer, intent(in) , optional         :: timelevel
   logical, intent(in), optional          :: no_domain
   integer, intent(in), optional          :: tile_count
-  logical, intent(in), optional          :: is_compressed
 #ifdef use_CRI_pointers
   pointer( p, data_3d )
   p = LOC(data)
 #endif
 
   call read_data_3d_new(filename,fieldname,data_3d,domain,timelevel,&
-        no_domain=no_domain, scalar_or_1d=.true., tile_count=tile_count, is_compressed=is_compressed)
+        no_domain=no_domain, scalar_or_1d=.true., tile_count=tile_count)
 
 end subroutine read_data_1d_new
 !.....................................................................
@@ -3524,7 +5265,7 @@ subroutine read_data_scalar_new(filename,fieldname,data,domain,timelevel,&
 
 ! this subroutine is for reading a single number
   character(len=*), intent(in)           :: filename, fieldname
-  real, intent(inout)                    :: data     !zero dimension data 
+  real, intent(inout)                    :: data     !zero dimension data
   real, dimension(1,1,1)                 :: data_3d
   type(domain2d), intent(in), optional   :: domain
   integer, intent(in) , optional         :: timelevel
@@ -3554,13 +5295,14 @@ function unique_axes(file, index, id_axes, siz_axes, dom)
   type(var_type), pointer, save :: cur_var => NULL()
   integer :: i,j
   logical :: found
-  
+
   unique_axes=0
 
-  if(index <0 .OR. index > 3) call mpp_error(FATAL,"unique_axes(fms_io_mod): index should be 1, 2 or 3")
+  if(index <0 .OR. index > 4) call mpp_error(FATAL,"unique_axes(fms_io_mod): index should be 1, 2, 3 or 4")
 
   do i = 1, file%nvar
      cur_var => file%var(i)
+     if(cur_var%read_only) cycle
      if(cur_var%ndim < index) cycle
      found = .false.
      do j = 1, unique_axes
@@ -3579,7 +5321,7 @@ function unique_axes(file, index, id_axes, siz_axes, dom)
               found = .true.
               exit
            end if
-        end if  
+        end if
      end do
      if(found) then
         cur_var%id_axes(index) = j
@@ -3603,10 +5345,10 @@ function unique_axes(file, index, id_axes, siz_axes, dom)
         endif
         cur_var%id_axes(index) = unique_axes
      end if
-  end do       
+  end do
 
   cur_var => NULL()
-  
+
   return
 
 end function unique_axes
@@ -3619,7 +5361,7 @@ end function unique_axes
   !
   ! reading can be done either by all PEs (default) or by only the root PE
   ! this is controlled by namelist variable "read_all_pe".
-  
+
   ! By default, array data is expected to be declared in data domain and no_halo
   !is NOT needed, however IF data is decalared in COMPUTE domain then optional NO_HALO should be .true.
 
@@ -3629,12 +5371,12 @@ subroutine read_data_2d ( unit, data, end)
 
   integer, intent(in)                        :: unit
   real,    intent(out), dimension(isd:,jsd:) :: data
-  logical, intent(out), optional             :: end  
+  logical, intent(out), optional             :: end
   real, dimension(isg:ieg,jsg:jeg)           :: gdata
   integer                                    :: len
   logical                                    :: no_halo
 
-  include "read_data_2d.inc"  
+  include "read_data_2d.inc"
 end subroutine read_data_2d
 
 !#######################################################################
@@ -3643,7 +5385,7 @@ subroutine read_ldata_2d ( unit, data, end)
 
   integer, intent(in)                        :: unit
   logical, intent(out), dimension(isd:,jsd:) :: data
-  logical, intent(out), optional             :: end  
+  logical, intent(out), optional             :: end
   logical, dimension(isg:ieg,jsg:jeg)        :: gdata
   integer                                    :: len
   logical                                    :: no_halo
@@ -3668,7 +5410,7 @@ end subroutine read_idata_2d
 
 #ifdef OVERLOAD_C8
 subroutine read_cdata_2d ( unit, data, end)
-  
+
   integer, intent(in)                           :: unit
   complex,    intent(out), dimension(isd:,jsd:) :: data
   logical, intent(out), optional                :: end
@@ -3686,7 +5428,7 @@ subroutine read_data_3d ( unit, data, end)
 
   integer, intent(in)                           :: unit
   real,    intent(out), dimension(isd:,jsd:,:)  :: data
-  logical, intent(out), optional                :: end  
+  logical, intent(out), optional                :: end
   real, dimension(isg:ieg,jsg:jeg,size(data,3)) :: gdata
   integer                                       :: len
   logical                                       :: no_halo
@@ -3706,7 +5448,7 @@ subroutine read_cdata_3d ( unit, data, end)
   integer                                          :: len
   logical                                          :: no_halo
 
-  include "read_data_3d.inc"  
+  include "read_data_3d.inc"
 end subroutine read_cdata_3d
 #endif
 
@@ -3721,8 +5463,8 @@ subroutine read_data_4d ( unit, data, end)
   integer                                                    :: len
   logical                                                    :: no_halo
 ! WARNING: memory usage with this routine could be costly
-   
-  include "read_data_4d.inc"  
+
+  include "read_data_4d.inc"
 end subroutine read_data_4d
 
 !#######################################################################
@@ -3737,7 +5479,7 @@ subroutine read_cdata_4d ( unit, data, end)
   integer                                                       :: len
   logical                                                       :: no_halo
 ! WARNING: memory usage with this routine could be costly
- 
+
   include "read_data_4d.inc"
 end subroutine read_cdata_4d
 #endif
@@ -3749,7 +5491,7 @@ end subroutine read_cdata_4d
 !#######################################################################
 subroutine write_data_2d ( unit, data )
   integer, intent(in)                       :: unit
-  real,    intent(in), dimension(isd:,jsd:) :: data  
+  real,    intent(in), dimension(isd:,jsd:) :: data
   real, dimension(isg:ieg,jsg:jeg) :: gdata
 
   include "write_data.inc"
@@ -3796,7 +5538,7 @@ subroutine write_data_3d ( unit, data )
   integer, intent(in) :: unit
   real,    intent(in), dimension(isd:,jsd:,:) :: data
   real, dimension(isg:ieg,jsg:jeg,size(data,3)) :: gdata
-    
+
   include "write_data.inc"
 end subroutine write_data_3d
 
@@ -3860,7 +5602,7 @@ end subroutine write_cdata_4d
 
 subroutine read_eof (end_found)
   logical, intent(out), optional :: end_found
-  
+
   if (present(end_found))then
      end_found = .true.
   else
@@ -3893,14 +5635,14 @@ subroutine reset_field_name(fileObj, id_field, name)
 
   fileObj%var(id_field)%name = trim(name)
 
-end subroutine reset_field_name 
+end subroutine reset_field_name
 
 !#######################################################################
 
 subroutine reset_field_pointer_r0d(fileObj, id_field, data)
   type(restart_file_type), intent(inout)      :: fileObj
   integer,                 intent(in)         :: id_field
-  real,                    intent(in), target :: data   
+  real,                    intent(in), target :: data
 
   if (.not.associated(fileObj%var)) call mpp_error(FATAL, "fms_io(reset_field_pointer_r0d): " // &
       "restart_file_type data must be initialized by calling register_restart_field before using it")
@@ -3921,7 +5663,7 @@ end subroutine reset_field_pointer_r0d
 subroutine reset_field_pointer_r1d(fileObj, id_field, data)
   type(restart_file_type), intent(inout)      :: fileObj
   integer,                 intent(in)         :: id_field
-  real, dimension(:),      intent(in), target :: data   
+  real, dimension(:),      intent(in), target :: data
 
   if (.not.associated(fileObj%var)) call mpp_error(FATAL, "fms_io(reset_field_pointer_r1d): " // &
       "restart_file_type data must be initialized by calling register_restart_field before using it")
@@ -3942,7 +5684,7 @@ end subroutine reset_field_pointer_r1d
 subroutine reset_field_pointer_r2d(fileObj, id_field, data)
   type(restart_file_type), intent(inout)      :: fileObj
   integer,                 intent(in)         :: id_field
-  real, dimension(:,:),    intent(in), target :: data   
+  real, dimension(:,:),    intent(in), target :: data
 
   if (.not.associated(fileObj%var)) call mpp_error(FATAL, "fms_io(reset_field_pointer_r2d): " // &
       "restart_file_type data must be initialized by calling register_restart_field before using it")
@@ -3963,7 +5705,7 @@ end subroutine reset_field_pointer_r2d
 subroutine reset_field_pointer_r3d(fileObj, id_field, data)
   type(restart_file_type), intent(inout)      :: fileObj
   integer,                 intent(in)         :: id_field
-  real, dimension(:,:,:),  intent(in), target :: data   
+  real, dimension(:,:,:),  intent(in), target :: data
 
   if (.not.associated(fileObj%var)) call mpp_error(FATAL, "fms_io(reset_field_pointer_r3d): " // &
       "restart_file_type data must be initialized by calling register_restart_field before using it")
@@ -3979,12 +5721,34 @@ subroutine reset_field_pointer_r3d(fileObj, id_field, data)
 
 end subroutine reset_field_pointer_r3d
 
+!#######################################################################
+
+subroutine reset_field_pointer_r4d(fileObj, id_field, data)
+  type(restart_file_type),   intent(inout)      :: fileObj
+  integer,                   intent(in)         :: id_field
+  real, dimension(:,:,:,:),  intent(in), target :: data
+
+  if (.not.associated(fileObj%var)) call mpp_error(FATAL, "fms_io(reset_field_pointer_r4d): " // &
+      "restart_file_type data must be initialized by calling register_restart_field before using it")
+
+  if(id_field < 0 .OR. id_field > fileObj%nvar) call mpp_error(FATAL, &
+         "fms_io(reset_field_pointer_r4d): id_field should be positive integer and "// &
+         "no larger than number of fields in the file "//trim(fileObj%name) )
+  if(fileObj%var(id_field)%siz(4) .NE. 1) call mpp_error(FATAL, &
+         "fms_io(reset_field_pointer_r4d): one-level reset_field_pointer is called, but "//&
+         "field "//trim(fileObj%var(id_field)%name)//" of file "//trim(fileObj%name)//" is not one level" )
+
+  fileObj%p4dr(1, id_field)%p => data
+
+end subroutine reset_field_pointer_r4d
+
+
 !#######################################################################
 
 subroutine reset_field_pointer_i0d(fileObj, id_field, data)
   type(restart_file_type), intent(inout)      :: fileObj
   integer,                 intent(in)         :: id_field
-  integer,                 intent(in), target :: data   
+  integer,                 intent(in), target :: data
 
   if (.not.associated(fileObj%var)) call mpp_error(FATAL, "fms_io(reset_field_pointer_i0d): " // &
       "restart_file_type data must be initialized by calling register_restart_field before using it")
@@ -4005,7 +5769,7 @@ end subroutine reset_field_pointer_i0d
 subroutine reset_field_pointer_i1d(fileObj, id_field, data)
   type(restart_file_type), intent(inout)      :: fileObj
   integer,                 intent(in)         :: id_field
-  integer, dimension(:),   intent(in), target :: data   
+  integer, dimension(:),   intent(in), target :: data
 
   if (.not.associated(fileObj%var)) call mpp_error(FATAL, "fms_io(reset_field_pointer_i1d): " // &
       "restart_file_type data must be initialized by calling register_restart_field before using it")
@@ -4026,7 +5790,7 @@ end subroutine reset_field_pointer_i1d
 subroutine reset_field_pointer_i2d(fileObj, id_field, data)
   type(restart_file_type), intent(inout)      :: fileObj
   integer,                 intent(in)         :: id_field
-  integer, dimension(:,:), intent(in), target :: data   
+  integer, dimension(:,:), intent(in), target :: data
 
   if (.not.associated(fileObj%var)) call mpp_error(FATAL, "fms_io(reset_field_pointer_i2d): " // &
       "restart_file_type data must be initialized by calling register_restart_field before using it")
@@ -4047,7 +5811,7 @@ end subroutine reset_field_pointer_i2d
 subroutine reset_field_pointer_i3d(fileObj, id_field, data)
   type(restart_file_type),   intent(inout)      :: fileObj
   integer,                   intent(in)         :: id_field
-  integer, dimension(:,:,:), intent(in), target :: data   
+  integer, dimension(:,:,:), intent(in), target :: data
 
   if (.not.associated(fileObj%var)) call mpp_error(FATAL, "fms_io(reset_field_pointer_i3d): " // &
       "restart_file_type data must be initialized by calling register_restart_field before using it")
@@ -4068,7 +5832,7 @@ end subroutine reset_field_pointer_i3d
 subroutine reset_field_pointer_r0d_2level(fileObj, id_field, data1, data2)
   type(restart_file_type), intent(inout)      :: fileObj
   integer,                 intent(in)         :: id_field
-  real,                    intent(in), target :: data1, data2   
+  real,                    intent(in), target :: data1, data2
 
   if (.not.associated(fileObj%var)) call mpp_error(FATAL, "fms_io(reset_field_pointer_r0d_2level): " // &
       "restart_file_type data must be initialized by calling register_restart_field before using it")
@@ -4090,7 +5854,7 @@ end subroutine reset_field_pointer_r0d_2level
 subroutine reset_field_pointer_r1d_2level(fileObj, id_field, data1, data2)
   type(restart_file_type), intent(inout)      :: fileObj
   integer,                 intent(in)         :: id_field
-  real, dimension(:),      intent(in), target :: data1, data2   
+  real, dimension(:),      intent(in), target :: data1, data2
 
   if (.not.associated(fileObj%var)) call mpp_error(FATAL, "fms_io(reset_field_pointer_r1d_2level): " // &
       "restart_file_type data must be initialized by calling register_restart_field before using it")
@@ -4112,7 +5876,7 @@ end subroutine reset_field_pointer_r1d_2level
 subroutine reset_field_pointer_r2d_2level(fileObj, id_field, data1, data2)
   type(restart_file_type), intent(inout)      :: fileObj
   integer,                 intent(in)         :: id_field
-  real, dimension(:,:),    intent(in), target :: data1, data2   
+  real, dimension(:,:),    intent(in), target :: data1, data2
 
   if (.not.associated(fileObj%var)) call mpp_error(FATAL, "fms_io(reset_field_pointer_r2d_2level): " // &
       "restart_file_type data must be initialized by calling register_restart_field before using it")
@@ -4134,7 +5898,7 @@ end subroutine reset_field_pointer_r2d_2level
 subroutine reset_field_pointer_r3d_2level(fileObj, id_field, data1, data2)
   type(restart_file_type), intent(inout)      :: fileObj
   integer,                 intent(in)         :: id_field
-  real, dimension(:,:,:),  intent(in), target :: data1, data2   
+  real, dimension(:,:,:),  intent(in), target :: data1, data2
 
   if (.not.associated(fileObj%var)) call mpp_error(FATAL, "fms_io(reset_field_pointer_r3d_2level): " // &
       "restart_file_type data must be initialized by calling register_restart_field before using it")
@@ -4156,7 +5920,7 @@ end subroutine reset_field_pointer_r3d_2level
 subroutine reset_field_pointer_i0d_2level(fileObj, id_field, data1, data2)
   type(restart_file_type), intent(inout)      :: fileObj
   integer,                 intent(in)         :: id_field
-  integer,                 intent(in), target :: data1, data2   
+  integer,                 intent(in), target :: data1, data2
 
   if (.not.associated(fileObj%var)) call mpp_error(FATAL, "fms_io(reset_field_pointer_i0d_2level): " // &
       "restart_file_type data must be initialized by calling register_restart_field before using it")
@@ -4178,7 +5942,7 @@ end subroutine reset_field_pointer_i0d_2level
 subroutine reset_field_pointer_i1d_2level(fileObj, id_field, data1, data2)
   type(restart_file_type), intent(inout)      :: fileObj
   integer,                 intent(in)         :: id_field
-  integer, dimension(:),   intent(in), target :: data1, data2   
+  integer, dimension(:),   intent(in), target :: data1, data2
 
   if (.not.associated(fileObj%var)) call mpp_error(FATAL, "fms_io(reset_field_pointer_i1d_2level): " // &
       "restart_file_type data must be initialized by calling register_restart_field before using it")
@@ -4200,7 +5964,7 @@ end subroutine reset_field_pointer_i1d_2level
 subroutine reset_field_pointer_i2d_2level(fileObj, id_field, data1, data2)
   type(restart_file_type), intent(inout)      :: fileObj
   integer,                 intent(in)         :: id_field
-  integer, dimension(:,:), intent(in), target :: data1, data2   
+  integer, dimension(:,:), intent(in), target :: data1, data2
 
   if (.not.associated(fileObj%var)) call mpp_error(FATAL, "fms_io(reset_field_pointer_i2d_2level): " // &
       "restart_file_type data must be initialized by calling register_restart_field before using it")
@@ -4222,7 +5986,7 @@ end subroutine reset_field_pointer_i2d_2level
 subroutine reset_field_pointer_i3d_2level(fileObj, id_field, data1, data2)
   type(restart_file_type),   intent(inout)      :: fileObj
   integer,                   intent(in)         :: id_field
-  integer, dimension(:,:,:), intent(in), target :: data1, data2   
+  integer, dimension(:,:,:), intent(in), target :: data1, data2
 
   if (.not.associated(fileObj%var)) call mpp_error(FATAL, "fms_io(reset_field_pointer_i3d_2level): " // &
       "restart_file_type data must be initialized by calling register_restart_field before using it")
@@ -4241,7 +6005,7 @@ end subroutine reset_field_pointer_i3d_2level
 
 !#########################################################################
 !   This function returns .true. if the field referred to by id has
-! initialized from a restart file, and .false. otherwise. 
+! initialized from a restart file, and .false. otherwise.
 !
 ! Arguments: id - A integer that is the index of the field in fileObj.
 !  (in)  fileObj - The control structure returned by a previous call to
@@ -4266,7 +6030,7 @@ end function query_initialized_id
 
 !#########################################################################
 !   This function returns .true. if the field referred to by name has
-! initialized from a restart file, and .false. otherwise. 
+! initialized from a restart file, and .false. otherwise.
 !
 ! Arguments: name - A pointer to the field that is being queried.
 !  (in)  fileObj - The control structure returned by a previous call to
@@ -4309,7 +6073,7 @@ end function query_initialized_name
 !  (in)      CS - The control structure returned by a previous call to
 !                 restart_init.
 function query_initialized_r2d(fileObj, f_ptr, name)
-  type(restart_file_type),   intent(inout) :: fileObj 
+  type(restart_file_type),   intent(inout) :: fileObj
   real, dimension(:,:), target, intent(in) :: f_ptr
   character(len=*),             intent(in) :: name
 
@@ -4331,14 +6095,57 @@ function query_initialized_r2d(fileObj, f_ptr, name)
   if (m>fileObj%nvar) then
      if (mpp_pe() == mpp_root_pe() ) call mpp_error(NOTE, "fms_io(query_initialized_r2d): Unable to find "// &
           trim(name)//" queried by pointer, "//"probably because of the suspect comparison of pointers by ASSOCIATED.")
-     query_initialized_r2d = query_initialized_name(fileObj, name)
-     if (mpp_pe() == mpp_root_pe() .AND. query_initialized_r2d) call mpp_error(NOTE, &
-          "fms_io(query_initialized_r2d): "//trim(name)// " initialization confirmed by name.")
+     query_initialized_r2d = query_initialized_name(fileObj, name)
+     if (mpp_pe() == mpp_root_pe() .AND. query_initialized_r2d) call mpp_error(NOTE, &
+          "fms_io(query_initialized_r2d): "//trim(name)// " initialization confirmed by name.")
+  endif
+
+  return
+
+end function query_initialized_r2d
+
+!#########################################################################
+!   This function returns 1 if the field pointed to by f_ptr has
+! initialized from a restart file, and 0 otherwise.  If f_ptr is
+! NULL, it tests whether the entire restart file has been success-
+! fully read.
+!
+! Arguments: f_ptr - A pointer to the field that is being queried.
+!  (in)      name - The name of the field that is being queried.
+!  (in)      CS - The control structure returned by a previous call to
+!                 restart_init.
+function query_initialized_r3d(fileObj, f_ptr, name)
+  type(restart_file_type),     intent(inout) :: fileObj
+  real, dimension(:,:,:), target, intent(in) :: f_ptr
+  character(len=*),               intent(in) :: name
+
+  logical :: query_initialized_r3d
+  integer :: m
+
+  if (.not.associated(fileObj%var)) call mpp_error(FATAL, "fms_io(query_initialized_r3d): " // &
+      "restart_file_type data must be initialized by calling register_restart_field before using it")
+
+  query_initialized_r3d = .false.
+  do m=1, fileObj%nvar
+     if (ASSOCIATED(fileObj%p3dr(1,m)%p,f_ptr)) then
+        if (fileObj%var(m)%initialized) query_initialized_r3d = .true.
+        exit
+     endif
+  enddo
+  ! Assume that you are going to initialize it now, so set flag to initialized if
+  ! queried again.
+  if (m>fileObj%nvar) then
+     if (mpp_pe() == mpp_root_pe() ) call mpp_error(NOTE, "fms_io(query_initialized_r3d): Unable to find "// &
+          trim(name)//" queried by pointer, "//"probably because of the suspect comparison of pointers by ASSOCIATED.")
+     query_initialized_r3d = query_initialized_name(fileObj, name)
+     if (mpp_pe() == mpp_root_pe() .AND. query_initialized_r3d) call mpp_error(NOTE, &
+          "fms_io(query_initialized_r3d): "//trim(name)// " initialization confirmed by name.")
   endif
 
   return
 
-end function query_initialized_r2d
+end function query_initialized_r3d
+
 
 !#########################################################################
 !   This function returns 1 if the field pointed to by f_ptr has
@@ -4350,41 +6157,40 @@ end function query_initialized_r2d
 !  (in)      name - The name of the field that is being queried.
 !  (in)      CS - The control structure returned by a previous call to
 !                 restart_init.
-function query_initialized_r3d(fileObj, f_ptr, name)
-  type(restart_file_type),     intent(inout) :: fileObj 
-  real, dimension(:,:,:), target, intent(in) :: f_ptr
-  character(len=*),               intent(in) :: name
+function query_initialized_r4d(fileObj, f_ptr, name)
+  type(restart_file_type),       intent(inout) :: fileObj
+  real, dimension(:,:,:,:), target, intent(in) :: f_ptr
+  character(len=*),                 intent(in) :: name
 
-  logical :: query_initialized_r3d
+  logical :: query_initialized_r4d
   integer :: m
 
-  if (.not.associated(fileObj%var)) call mpp_error(FATAL, "fms_io(query_initialized_r3d): " // &
+  if (.not.associated(fileObj%var)) call mpp_error(FATAL, "fms_io(query_initialized_r4d): " // &
       "restart_file_type data must be initialized by calling register_restart_field before using it")
 
-  query_initialized_r3d = .false.
+  query_initialized_r4d = .false.
   do m=1, fileObj%nvar
-     if (ASSOCIATED(fileObj%p3dr(1,m)%p,f_ptr)) then
-        if (fileObj%var(m)%initialized) query_initialized_r3d = .true.
+     if (ASSOCIATED(fileObj%p4dr(1,m)%p,f_ptr)) then
+        if (fileObj%var(m)%initialized) query_initialized_r4d = .true.
         exit
      endif
   enddo
   ! Assume that you are going to initialize it now, so set flag to initialized if
   ! queried again.
   if (m>fileObj%nvar) then
-     if (mpp_pe() == mpp_root_pe() ) call mpp_error(NOTE, "fms_io(query_initialized_r3d): Unable to find "// &
+     if (mpp_pe() == mpp_root_pe() ) call mpp_error(NOTE, "fms_io(query_initialized_r4d): Unable to find "// &
           trim(name)//" queried by pointer, "//"probably because of the suspect comparison of pointers by ASSOCIATED.")
-     query_initialized_r3d = query_initialized_name(fileObj, name)
-     if (mpp_pe() == mpp_root_pe() .AND. query_initialized_r3d) call mpp_error(NOTE, &
-          "fms_io(query_initialized_r3d): "//trim(name)// " initialization confirmed by name.")
+     query_initialized_r4d = query_initialized_name(fileObj, name)
+     if (mpp_pe() == mpp_root_pe() .AND. query_initialized_r4d) call mpp_error(NOTE, &
+          "fms_io(query_initialized_r4d): "//trim(name)// " initialization confirmed by name.")
   endif
 
   return
 
-end function query_initialized_r3d
-
+end function query_initialized_r4d
 
 !#########################################################################
-!   This function sets that a variable has been initialized for future queries. 
+!   This function sets that a variable has been initialized for future queries.
 !
 ! Arguments: name - A pointer to the field whose initialization status is being set.
 !  (in)  fileObj - The control structure returned by a previous call to
@@ -4412,7 +6218,7 @@ subroutine set_initialized_id(fileObj, id, is_set)
 end subroutine set_initialized_id
 
 !#########################################################################
-!   This function sets that a variable has been initialized for future queries. 
+!   This function sets that a variable has been initialized for future queries.
 !
 ! Arguments: name - A pointer to the field whose initialization status is being set.
 !  (in)  fileObj - The control structure returned by a previous call to
@@ -4446,7 +6252,7 @@ subroutine set_initialized_name(fileObj, name, is_set)
 end subroutine set_initialized_name
 
 !#########################################################################
-!   This function sets that a variable has been initialized for future queries. 
+!   This function sets that a variable has been initialized for future queries.
 !
 ! Arguments: name - A pointer to the field whose initialization status is being set.
 !  (in)  fileObj - The control structure returned by a previous call to
@@ -4493,7 +6299,7 @@ subroutine set_initialized_r2d(fileObj, f_ptr, name, is_set)
 end subroutine set_initialized_r2d
 
 !#########################################################################
-!   This function sets that a variable has been initialized for future queries. 
+!   This function sets that a variable has been initialized for future queries.
 !
 ! Arguments: name - A pointer to the field whose initialization status is being set.
 !  (in)  fileObj - The control structure returned by a previous call to
@@ -4540,13 +6346,60 @@ subroutine set_initialized_r3d(fileObj, f_ptr, name, is_set)
 end subroutine set_initialized_r3d
 
 
+!#########################################################################
+!   This function sets that a variable has been initialized for future queries.
+!
+! Arguments: name - A pointer to the field whose initialization status is being set.
+!  (in)  fileObj - The control structure returned by a previous call to
+!                  register_restart_field
+subroutine set_initialized_r4d(fileObj, f_ptr, name, is_set)
+  type(restart_file_type),       intent(inout) :: fileObj
+  real, dimension(:,:,:,:), target, intent(in) :: f_ptr
+  character(len=*),                 intent(in) :: name
+  logical,          optional,       intent(in) :: is_set
+  logical :: set_val
+  integer :: m
+
+  set_val = .true.
+  if (present(is_set)) set_val = is_set
+
+  if (.not.associated(fileObj%var)) call mpp_error(FATAL, "fms_io(set_initialized_r4d): " // &
+      "restart_file_type data must be initialized by calling set_restart_field before using it")
+
+  do m=1, fileObj%nvar
+     if (ASSOCIATED(fileObj%p4dr(1,m)%p,f_ptr)) then
+        fileObj%var(m)%initialized = set_val
+        return
+     endif
+  enddo
+
+  if (m>fileObj%nvar .AND. mpp_pe() == mpp_root_pe() ) then
+    call mpp_error(NOTE,"fms_io(set_initialized_r4d): Unable to find "// &
+          trim(name)//" queried by pointer, "//"probably because of the suspect comparison of pointers by ASSOCIATED"//&
+                        " when attempting to set initialization.")
+  end if
+
+  do m=1,fileObj%nvar
+    if (trim(name) == fileObj%var(m)%name) then
+      fileObj%var(m)%initialized = set_val
+      return
+    endif
+  enddo
+
+  if (m>fileObj%nvar .AND. mpp_pe() == mpp_root_pe() ) then
+    call mpp_error(NOTE,"fms_io(set_initialized_r4d): Unknown restart variable "//name// &
+                        " attempted to set initialization.")
+  end if
+
+end subroutine set_initialized_r4d
+
 !#######################################################################
 !#######################################################################
 !
 ! routines for opening specific types of files:
 !
-!                       form        action 
-! open_namelist_file  MPP_ASCII   MPP_RDONLY  
+!                       form        action
+! open_namelist_file  MPP_ASCII   MPP_RDONLY
 ! open restart_file   MPP_NATIVE
 ! open_ieee32_file    MPP_IEEE32
 !
@@ -4581,14 +6434,14 @@ function open_namelist_file (file) result (unit)
   if(show_open_namelist_file_warning) call mpp_error(WARNING, "fms_io_mod: open_namelist_file should not be called when INTERNAL_FILE_NML is defined")
 #endif
 
-  if (.not.module_is_initialized) call fms_io_init ( )    
+  if (.not.module_is_initialized) call fms_io_init ( )
   if (present(file)) then
      call mpp_open ( unit, file, form=MPP_ASCII, action=MPP_RDONLY, &
           access=MPP_SEQUENTIAL, threading=MPP_SINGLE )
   else
 !  the following code is necessary for using alternate namelist files (nests, stretched grids, etc)
      pelist_name = mpp_get_current_pelist_name()
-     if ( file_exist('input_'//trim(pelist_name)//'.nml') ) then
+     if ( file_exist('input_'//trim(pelist_name)//'.nml', no_domain=.true.) ) then
         filename='input_'//trim(pelist_name)//'.nml'
      else
         filename='input.nml'
@@ -4600,7 +6453,7 @@ end function open_namelist_file
 !     
 
 ! 
-!    
+!   
 ! Opens single restart file for reading by all PEs or
 ! writing by root PE only
 ! the file has native format and no mpp header records.
@@ -4616,7 +6469,7 @@ end function open_namelist_file
 ! 
 function open_restart_file (file, action) result (unit)
   character(len=*), intent(in) :: file, action
-  integer :: unit  
+  integer :: unit
   integer :: mpp_action
 
   if (.not.module_is_initialized) call fms_io_init ( )
@@ -4631,7 +6484,7 @@ function open_restart_file (file, action) result (unit)
   case default
      call mpp_error(FATAL,'fms_io(open_restart_file): action should be either read or write in file'//trim(file))
   end select
-  
+
   call mpp_open ( unit, file, form=MPP_NATIVE, action=mpp_action, &
        access=MPP_SEQUENTIAL, threading=MPP_SINGLE, nohdrs=.true. )
 
@@ -4640,7 +6493,7 @@ end function open_restart_file
 
 
 ! 
-!    
+!   
 ! Opens single direct access file for reading by all PEs or
 ! writing by root PE only
 ! the file has native format and no mpp header records.
@@ -4673,8 +6526,8 @@ end function open_direct_file
 ! 
 
 ! 
-!     
-! Opens single 32-bit ieee file for reading by all PEs or 
+!   
+! Opens single 32-bit ieee file for reading by all PEs or
 ! writing by root PE only (writing is not recommended)
 ! the file has no mpp header records.
 !   
@@ -4689,7 +6542,7 @@ end function open_direct_file
 ! 
 function open_ieee32_file (file, action) result (unit)
   character(len=*), intent(in) :: file, action
-  integer :: unit  
+  integer :: unit
   integer :: mpp_action
 
   if (.not.module_is_initialized) call fms_io_init ( )
@@ -4703,7 +6556,7 @@ function open_ieee32_file (file, action) result (unit)
   case default
      call mpp_error (FATAL,'fms_io(open_ieee32_file): action should be either read or write in file'//trim(file))
   end select
-  
+
   if (iospec_ieee32(1:1) == ' ') then
      call mpp_open ( unit, file, form=MPP_IEEE32, action=mpp_action, &
           access=MPP_SEQUENTIAL, threading=MPP_SINGLE,    &
@@ -4729,12 +6582,20 @@ end function open_ieee32_file
 ! action to be performed: can be 'delete'
 ! 
 
-subroutine close_file (unit, status)
+subroutine close_file (unit, status, dist)
   integer,          intent(in)           :: unit
   character(len=*), intent(in), optional :: status
-  
+  logical,          intent(in), optional :: dist
+
   if (.not.module_is_initialized) call fms_io_init ( )
-  if (unit == stdlog()) return    
+  if(PRESENT(dist))then
+    ! If distributed, return if not I/O root
+    if(dist)then
+      if(.not. mpp_is_dist_ioroot(dr_set_size)) return
+    endif
+  endif
+
+  if (unit == stdlog()) return
   if (present(status)) then
      if (lowercase(trim(status)) == 'delete') then
         call mpp_close (unit, action=MPP_DELETE)
@@ -4748,7 +6609,7 @@ end subroutine close_file
 ! 
 
 !#######################################################################
-  
+
 
 ! 
 !   
@@ -4760,7 +6621,7 @@ end subroutine close_file
 ! this Domain2
 ! 
 subroutine set_domain (Domain2)
-    
+
   type(domain2D), intent(in), target :: Domain2
 
   if (.NOT.module_is_initialized) call fms_io_init ( )
@@ -4768,9 +6629,9 @@ subroutine set_domain (Domain2)
 !  --- set_domain must be called before a read_data or write_data ---
   if (associated(Current_domain)) nullify (Current_domain)
   Current_domain => Domain2
-  
+
   !  --- module indexing to shorten read/write routines ---
-  
+
   call mpp_get_compute_domain (Current_domain,is ,ie ,js ,je )
   call mpp_get_data_domain    (Current_domain,isd,ied,jsd,jed)
   call mpp_get_global_domain  (Current_domain,isg,ieg,jsg,jeg)
@@ -4796,7 +6657,7 @@ end subroutine nullify_domain
 
 ! 
 !   
-! This routine is the reverse of set_domain above. This routine is called when 
+! This routine is the reverse of set_domain above. This routine is called when
 ! users want to retrieve the domain2d that is used in fms_io_mod
 !   
 ! 
@@ -4808,7 +6669,7 @@ subroutine return_domain(domain2)
   if (associated(Current_domain)) then
      domain2 = Current_domain
   else
-     domain2 = NULL_DOMAIN2D       
+     domain2 = NULL_DOMAIN2D
   endif
 end subroutine return_domain
 ! 
@@ -4832,7 +6693,7 @@ end subroutine return_domain
 subroutine get_domain_decomp ( x, y )
 
   integer, intent(out), dimension(4) :: x, y
-  
+
   if (mpp_pe() == mpp_root_pe())  call mpp_error(NOTE, &
        'subroutine get_domain_decomp will be removed with the next release')
   x = (/ isg, ieg, is, ie /)
@@ -4841,7 +6702,7 @@ subroutine get_domain_decomp ( x, y )
 end subroutine get_domain_decomp
 ! 
 
-subroutine get_axis_cart(axis, cart)      
+subroutine get_axis_cart(axis, cart)
 
   type(axistype), intent(in) :: axis
   character(len=1), intent(out) :: cart
@@ -4862,9 +6723,9 @@ subroutine get_axis_cart(axis, cart)
   lon_units = (/'degrees_e   ', 'degrees_east'/)
   lat_units = (/'degrees_n    ', 'degrees_north'/)
   z_units = (/'cm ','m  ','pa ','hpa'/)
-  t_units = (/'sec', 'min','hou','day'/)  
+  t_units = (/'sec', 'min','hou','day'/)
   call mpp_get_atts(axis,cartesian=axis_cart)
-  cart = 'N'  
+  cart = 'N'
   if (axis_cart == 'x' ) cart = 'X'
   if (axis_cart == 'y' ) cart = 'Y'
   if (axis_cart == 'z' ) cart = 'Z'
@@ -4902,22 +6763,21 @@ subroutine get_axis_cart(axis, cart)
         if (name(1:3) == trim(t_units(i))) cart = 'T'
      enddo
   end if
-  
+
   return
 end subroutine get_axis_cart
 
-
 ! The following function is here as a last resort.
-! This is copied from what was utilities_mod in order that redundant code 
+! This is copied from what was utilities_mod in order that redundant code
 ! could be deleted.
 
- function open_file ( file, form, action, access, threading, recl ) &
-             result ( unit )
+function open_file(file, form, action, access, threading, recl, dist) result(unit)
 
- character(len=*), intent(in) :: file 
+ character(len=*), intent(in) :: file
  character(len=*), intent(in), optional :: form, action, access, threading
- integer         , intent(in), optional :: recl 
- integer  :: unit 
+ integer         , intent(in), optional :: recl
+ logical         , intent(in), optional :: dist  ! Distributed open?
+ integer  :: unit
 
  character(len=32) :: form_local, action_local, access_local, thread_local
  character(len=32) :: action_ieee32
@@ -4925,9 +6785,23 @@ function open_file ( file, form, action, access, threading, recl ) &
  integer :: mpp_format, mpp_action, mpp_access, mpp_thread
 !-----------------------------------------------------------------------
 
-   if ( .not. module_is_initialized ) then
-        call fms_io_init ( )
-!        do_init = .false.
+   if ( .not. module_is_initialized ) call fms_io_init ( )
+
+   if (present(action)) then    ! must be present
+      action_local = action
+   else
+      call mpp_error (FATAL, 'open_file in fms_mod : argument action not present')
+   endif
+
+   unit = 0  ! Initialize return value. Note that mpp_open will call mpi_abort on error
+   if(PRESENT(dist))then
+     if(lowercase(trim(action_local)) /= 'read') &
+       call mpp_error(FATAL,'open_file in fms_mod: distributed'//lowercase(trim(action_local))// &
+                              ' not currently supported')
+     ! If distributed, return if not I/O root
+     if(dist) then
+       if(.not. mpp_is_dist_ioroot(dr_set_size)) return
+     endif
    endif
 
 !   ---- return stdlog if this is the logfile ----
@@ -4937,7 +6811,7 @@ function open_file ( file, form, action, access, threading, recl ) &
        return
     endif
 
-!   ---- is this file open and connected to a unit ?? ---- 
+!   ---- is this file open and connected to a unit ?? ----
 
    inquire (file=trim(file), opened=open, number=unit)
 
@@ -4947,7 +6821,7 @@ function open_file ( file, form, action, access, threading, recl ) &
    if ( open .and. unit >= 0 ) then
       call mpp_error (FATAL, 'open_file in fms_mod : '// &
                        'file '//trim(file)//' is already open')
-   endif   
+   endif
 
 !  --- defaults ---
 
@@ -4957,13 +6831,6 @@ function open_file ( file, form, action, access, threading, recl ) &
    no_headers   = .true.
    do_ieee32    = .false.
 
-   if (present(action)) then    ! must be present
-      action_local = action
-   else
-      call mpp_error (FATAL, 'open_file in fms_mod : argument action not present')
-   endif
-
-
 !   --- file format ---
 
     select case (lowercase(trim(form_local)))
@@ -5028,7 +6895,7 @@ function open_file ( file, form, action, access, threading, recl ) &
     if ( .not.do_ieee32 ) then
        call mpp_open ( unit, file, form=mpp_format, action=mpp_action, &
                        access=mpp_access, threading=mpp_thread,        &
-                       nohdrs=no_headers, recl=recl )
+                       fileset=MPP_SINGLE,nohdrs=no_headers, recl=recl )
     else
      ! special open for ieee32 file
      ! fms_mod has iospec value
@@ -5112,22 +6979,13 @@ subroutine get_mosaic_tile_file(file_in, file_out, is_no_domain, domain, tile_co
     character(len=*), intent(out)                  :: file_out
     logical,          intent(in)                   :: is_no_domain
     type(domain2D),   intent(in), optional, target :: domain
-    integer,          intent(in), optional         :: tile_count 
+    integer,          intent(in), optional         :: tile_count
     character(len=256)                             :: basefile, tilename
     integer                                        :: lens, ntiles, ntileMe, tile, my_tile_id
     integer, dimension(:), allocatable             :: tile_id
     type(domain2d), pointer, save                  :: d_ptr =>NULL()
     logical                                        :: domain_exist
 
-    !--- deal with the situation that the file is alreday in the full name.
-    lens = len_trim(file_in)
-    if(lens > 8) then
-       if(file_in(lens-7:lens) == '.nc'//trim(pe_name) ) then
-         file_out = file_in
-         return
-        endif
-    endif
-
     if(index(file_in, '.nc', back=.true.)==0) then
        basefile = trim(file_in)
     else
@@ -5137,39 +6995,37 @@ subroutine get_mosaic_tile_file(file_in, file_out, is_no_domain, domain, tile_co
        basefile = file_in(1:lens-3)
     end if
 
-    if(mpp_mosaic_defined())then
-       !--- get the tile name
-       ntiles = 1
-       my_tile_id = 1
-       domain_exist = .false.
-       if(PRESENT(domain))then
-          domain_exist = .true.
-          ntiles = mpp_get_ntile_count(domain)
-          d_ptr => domain
-       elseif (ASSOCIATED(Current_domain) .AND. .NOT. is_no_domain ) then
-          domain_exist = .true.
-          ntiles = mpp_get_ntile_count(Current_domain)
-          d_ptr => Current_domain
-       endif
-       
-       if(domain_exist) then
-          ntileMe = mpp_get_current_ntile(d_ptr)
-          allocate(tile_id(ntileMe))
-          tile_id = mpp_get_tile_id(d_ptr)
-          tile = 1
-          if(present(tile_count)) tile = tile_count
-          my_tile_id = tile_id(tile)
-       endif
+    !--- get the tile name
+    ntiles = 1
+    my_tile_id = 1
+    domain_exist = .false.
+    if(PRESENT(domain))then
+       domain_exist = .true.
+       ntiles = mpp_get_ntile_count(domain)
+       d_ptr => domain
+    elseif (ASSOCIATED(Current_domain) .AND. .NOT. is_no_domain ) then
+       domain_exist = .true.
+       ntiles = mpp_get_ntile_count(Current_domain)
+       d_ptr => Current_domain
+    endif
 
-       if(ntiles > 1 .or. my_tile_id > 1 )then
-          tilename = 'tile'//string(my_tile_id)
-          if(index(basefile,'.'//trim(tilename),back=.true.) == 0)then
-             basefile = trim(basefile)//'.'//trim(tilename);
-          end if
-       end if
-       if(allocated(tile_id)) deallocate(tile_id)
+    if(domain_exist) then
+       ntileMe = mpp_get_current_ntile(d_ptr)
+       allocate(tile_id(ntileMe))
+       tile_id = mpp_get_tile_id(d_ptr)
+       tile = 1
+       if(present(tile_count)) tile = tile_count
+       my_tile_id = tile_id(tile)
     endif
 
+    if(ntiles > 1 .or. my_tile_id > 1 )then
+       tilename = 'tile'//string(my_tile_id)
+       if(index(basefile,'.'//trim(tilename),back=.true.) == 0)then
+          basefile = trim(basefile)//'.'//trim(tilename);
+       end if
+    end if
+    if(allocated(tile_id)) deallocate(tile_id)
+
     file_out = trim(basefile)//'.nc'
 
     d_ptr =>NULL()
@@ -5177,11 +7033,11 @@ subroutine get_mosaic_tile_file(file_in, file_out, is_no_domain, domain, tile_co
   end subroutine get_mosaic_tile_file
 
   !#############################################################################
-  subroutine get_mosaic_tile_grid(grid_file, mosaic_file, domain, tile_count) 
+  subroutine get_mosaic_tile_grid(grid_file, mosaic_file, domain, tile_count)
     character(len=*), intent(out)          :: grid_file
     character(len=*), intent(in)           :: mosaic_file
     type(domain2D),   intent(in)           :: domain
-    integer,          intent(in), optional :: tile_count 
+    integer,          intent(in), optional :: tile_count
     integer                                :: tile, ntileMe
     integer, dimension(:), allocatable     :: tile_id
 
@@ -5189,7 +7045,7 @@ subroutine get_mosaic_tile_grid(grid_file, mosaic_file, domain, tile_count)
     if(present(tile_count)) tile = tile_count
     ntileMe = mpp_get_current_ntile(domain)
     allocate(tile_id(ntileMe))
-    tile_id = mpp_get_tile_id(domain)     
+    tile_id = mpp_get_tile_id(domain)
     call read_data(mosaic_file, "gridfiles", grid_file, level=tile_id(tile) )
     grid_file = 'INPUT/'//trim(grid_file)
     deallocate(tile_id)
@@ -5206,7 +7062,7 @@ subroutine get_var_att_value_text(file, varname, attname, attvalue)
     call mpp_open(unit,trim(file),MPP_RDONLY,MPP_NETCDF,threading=MPP_MULTI,fileset=MPP_SINGLE)
     call mpp_get_att_value(unit, varname, attname, attvalue)
     call mpp_close(unit)
- 
+
     return
 
   end subroutine get_var_att_value_text
@@ -5278,7 +7134,7 @@ function get_file_name(orig_file, actual_file, read_dist, io_domain_exist, no_do
     logical,                         intent(out) :: io_domain_exist
     logical,                optional, intent(in) :: no_domain
     type(domain2D), target, optional, intent(in) :: domain
-    integer,                optional, intent(in) :: tile_count  
+    integer,                optional, intent(in) :: tile_count
     logical                                      :: get_file_name
 
     type(domain2d), pointer, save :: d_ptr, io_domain
@@ -5289,11 +7145,6 @@ function get_file_name(orig_file, actual_file, read_dist, io_domain_exist, no_do
     is_no_domain=.false.
     if(PRESENT(no_domain)) is_no_domain = no_domain
 
-    if(present(domain)) then
-       d_ptr => domain
-    elseif (ASSOCIATED(Current_domain) .AND. .NOT. is_no_domain ) then
-       d_ptr => Current_domain
-    endif
 
     fexist          = .false.
     read_dist       = .false.
@@ -5308,9 +7159,16 @@ function get_file_name(orig_file, actual_file, read_dist, io_domain_exist, no_do
           get_file_name = .true.
           return
        endif
-    endif 
-    
-    !JWD:  This is likely a temporary fix. Since fms_io needs to know tile_count, 
+    endif
+
+    if(present(domain)) then
+       d_ptr => domain
+    elseif (ASSOCIATED(Current_domain) .AND. .NOT. is_no_domain ) then
+       d_ptr => Current_domain
+    endif
+
+
+    !JWD:  This is likely a temporary fix. Since fms_io needs to know tile_count,
     !JWD:  I just don't see how the physics can remain "tile neutral"
     call get_mosaic_tile_file(orig_file, actual_file, is_no_domain, domain, tile_count)
 
@@ -5318,7 +7176,7 @@ function get_file_name(orig_file, actual_file, read_dist, io_domain_exist, no_do
     if(ASSOCIATED(d_ptr)) then
        io_domain => mpp_get_io_domain(d_ptr)
        if(associated(io_domain)) then
-          tile_id = mpp_get_tile_id(io_domain)       
+          tile_id = mpp_get_tile_id(io_domain)
           write(fname, '(a,i4.4)' ) trim(actual_file)//'.', tile_id(1)
           inquire (file=trim(fname), exist=fexist)
           if(.not. fexist) then
@@ -5326,11 +7184,9 @@ function get_file_name(orig_file, actual_file, read_dist, io_domain_exist, no_do
              inquire (file=trim(fname), exist=fexist)
           endif
           if(fexist) io_domain_exist = .true.
-       endif   
+       endif
        io_domain=>NULL()
-    endif 
-
-    if(.not. fexist) inquire (file=trim(actual_file)//trim(pe_name), exist=fexist)
+    endif
 
     if(fexist) then
        read_dist = .true.
@@ -5344,59 +7200,62 @@ function get_file_name(orig_file, actual_file, read_dist, io_domain_exist, no_do
        d_ptr => NULL()
        get_file_name = .true.
        return
-    endif    
+    endif
 
     !Perhaps the file has an ensemble instance appendix
-    call get_instance_filename(orig_file, actual_file)
-    if(index(orig_file, '.nc', back=.true.) == 0) then
-       inquire (file=trim(actual_file), exist=fexist)
+    if(len_trim(filename_appendix) > 0) then
+       call get_instance_filename(orig_file, actual_file)
+       if(index(orig_file, '.nc', back=.true.) == 0) then
+          inquire (file=trim(actual_file), exist=fexist)
+          if(fexist) then
+             d_ptr => NULL()
+             get_file_name = .true.
+             return
+          endif
+       endif
+
+       call get_mosaic_tile_file(actual_file, actual_file, is_no_domain, domain, tile_count)
+       !--- check if the file is group redistribution.
+       if(ASSOCIATED(d_ptr)) then
+          io_domain => mpp_get_io_domain(d_ptr)
+          if(associated(io_domain)) then
+             tile_id = mpp_get_tile_id(io_domain)
+             if(mpp_npes()>10000) then
+                write(fname, '(a,i6.6)' ) trim(actual_file)//'.', tile_id(1)
+             else
+                write(fname, '(a,i4.4)' ) trim(actual_file)//'.', tile_id(1)
+             endif
+             inquire (file=trim(fname), exist=fexist)
+             if(fexist) io_domain_exist = .true.
+          endif
+          io_domain=>NULL()
+       endif
+
        if(fexist) then
+          read_dist = .true.
+          d_ptr => NULL()
           get_file_name = .true.
           return
        endif
-    endif 
 
-    call get_mosaic_tile_file(actual_file, actual_file, is_no_domain, domain, tile_count)
-    !--- check if the file is group redistribution.
-    if(ASSOCIATED(d_ptr)) then
-       io_domain => mpp_get_io_domain(d_ptr)
-       if(associated(io_domain)) then
-          tile_id = mpp_get_tile_id(io_domain)       
-          if(mpp_npes()>10000) then
-             write(fname, '(a,i6.6)' ) trim(actual_file)//'.', tile_id(1)
-          else
-             write(fname, '(a,i4.4)' ) trim(actual_file)//'.', tile_id(1)
-          endif
-          inquire (file=trim(fname), exist=fexist)
-          if(fexist) io_domain_exist = .true.
-       endif   
-       io_domain=>NULL()
-    endif 
+       inquire (file=trim(actual_file), exist=fexist)
 
-    if(.not. fexist) inquire (file=trim(actual_file)//trim(pe_name), exist=fexist)  
-    if(fexist) then
-       read_dist = .true. 
-       d_ptr => NULL()
-       get_file_name = .true.
-       return
-    endif    
-    inquire (file=trim(actual_file), exist=fexist)
-    
-    if(fexist) then
-       d_ptr => NULL()
-       get_file_name = .true.
-       return
+       if(fexist) then
+          d_ptr => NULL()
+          get_file_name = .true.
+          return
+       endif
     endif
 
   end function get_file_name
-  
+
 
   !#############################################################################
   subroutine get_file_unit(filename, unit, index_file, read_dist, io_domain_exist, domain )
     character(len=*),         intent(in) :: filename
-    integer,                 intent(out) :: unit, index_file 
+    integer,                 intent(out) :: unit, index_file
     logical,                  intent(in) :: read_dist, io_domain_exist
-    type(domain2d), optional, intent(in) :: domain    
+    type(domain2d), optional, intent(in) :: domain
 
     logical  :: file_opened
     integer  :: i
@@ -5407,19 +7266,15 @@ subroutine get_file_unit(filename, unit, index_file, read_dist, io_domain_exist,
        if (files_read(i)%name == trim(filename))  then
           index_file = i
           unit = files_read(index_file)%unit
-          return 
+          return
        endif
     enddo
 
     ! need to open the file now
-    ! Increase num_files_r and set file_type 
+    ! Increase num_files_r and set file_type
     if(num_files_r == max_files_r) &  ! need to have bigger max_files_r
          call mpp_error(FATAL,'fms_io(get_file_unit): max_files_r exceeded, increase it via fms_io_nml')
-    num_files_r=num_files_r + 1 
-    if (read_dist .and. thread_r == MPP_SINGLE) then
-       call mpp_error(FATAL,'fms_io(get_file_unit): single-threaded read from distributed fileset not allowed' &
-            //'change threading_read to MULTI')
-    endif
+    num_files_r=num_files_r + 1
     if(read_dist) then
        if(io_domain_exist) then
           if(present(domain)) then
@@ -5433,18 +7288,18 @@ subroutine get_file_unit(filename, unit, index_file, read_dist, io_domain_exist,
                    'either domain is present or current_domain is associated')
           endif
        else
-          call mpp_open(unit,trim(filename),form=form,action=MPP_RDONLY,threading=thread_r, &
+          call mpp_open(unit,trim(filename),form=form,action=MPP_RDONLY,threading=MPP_MULTI, &
             fileset=MPP_MULTI)
        endif
     else
-       call mpp_open(unit,trim(filename),form=form,action=MPP_RDONLY,threading=thread_r, &
+       call mpp_open(unit,trim(filename),form=form,action=MPP_RDONLY,threading=MPP_MULTI, &
             fileset=MPP_SINGLE)
     end if
     files_read(num_files_r)%name = trim(filename)
     allocate(files_read(num_files_r)%var (max_fields) )
     files_read(num_files_r)%nvar = 0
     index_file = num_files_r
-    files_read(index_file)%unit = unit   
+    files_read(index_file)%unit = unit
 
   end subroutine get_file_unit
 
@@ -5471,10 +7326,10 @@ subroutine get_field_id(unit, index_file, fieldname, index_field, is_no_domain,
        endif
     enddo
 
-    !--- fieldname is not read, so need to get fieldname from file 
+    !--- fieldname is not read, so need to get fieldname from file
     files_read(index_file)%nvar = files_read(index_file)%nvar + 1
     if(files_read(index_file)%nvar > max_fields) then
-       write(error_msg,'(I3,"/",I3)') files_read(index_file)%nvar, max_fields 
+       write(error_msg,'(I3,"/",I3)') files_read(index_file)%nvar, max_fields
        call  mpp_error(FATAL,'fms_io(get_field_id): max_fields exceeded, needs increasing, nvar/max_fields=' &
             //trim(error_msg))
     endif
@@ -5485,13 +7340,15 @@ subroutine get_field_id(unit, index_file, fieldname, index_field, is_no_domain,
        call mpp_error(FATAL,'fms_io(get_field_id): max_fields too small needs increasing,nvar/max_fields=' &
             //trim(error_msg)//'in file'//trim(files_read(index_file)%name))
     endif
-    call mpp_get_fields(unit, fields(1:nvar))     
+    call mpp_get_fields(unit, fields(1:nvar))
     siz_in = 1
     index_field = files_read(index_file)%nvar
     files_read(index_file)%var(index_field)%is_dimvar = .false.
 
     do i=1, nvar
        call mpp_get_atts(fields(i),name=name,ndim=var_dim,siz=siz_in)
+       if(var_dim .GT. 4) call mpp_error(FATAL, 'fms_io(get_field_id): number of dimension of field '// &
+                trim(name)//' in file '//trim(files_read(index_file)%name)//' should not be greater than 4')
        if (lowercase(trim(name)) == lowercase(trim(fieldname))) then ! found the variable
           if(var_dim .lt.3) then
              do j=var_dim+1,3
@@ -5500,8 +7357,8 @@ subroutine get_field_id(unit, index_file, fieldname, index_field, is_no_domain,
           endif
           files_read(index_file)%var(index_field)%name    = fieldname
           files_read(index_file)%var(index_field)%field   = fields(i)
-          files_read(index_file)%var(index_field)%siz(:)  = siz_in
-          files_read(index_file)%var(index_field)%gsiz(:) = siz_in
+          files_read(index_file)%var(index_field)%siz(1:4)  = siz_in(1:4)
+          files_read(index_file)%var(index_field)%gsiz(1:3) = siz_in(1:3)
           return
        endif
     enddo
@@ -5509,21 +7366,21 @@ subroutine get_field_id(unit, index_file, fieldname, index_field, is_no_domain,
     !--- the fieldname may be a dimension variable.
     if( .not. is_not_dim) then
        if (ndim > max_axes) then
-          write(error_msg,'(I3,"/",I3)') ndim, max_axes 
+          write(error_msg,'(I3,"/",I3)') ndim, max_axes
           call  mpp_error(FATAL,'fms_io(get_field_id): max_axes exceeded, needs increasing, ndim/max_fields=' &
-               //trim(error_msg)//' in file '//trim(files_read(index_file)%name))             
+               //trim(error_msg)//' in file '//trim(files_read(index_file)%name))
        endif
        call mpp_get_axes(unit, axes(1:ndim))
        do i=1,ndim
-          call mpp_get_atts(axes(i), name=name, len = siz_in(1)) 
+          call mpp_get_atts(axes(i), name=name, len = siz_in(1))
           if (lowercase(trim(name)) == lowercase(trim(fieldname))) then
 !             if(.not. is_no_domain) call mpp_error(FATAL, &
 !                  'fms_io(get_field_id): the field is a dimension variable, no_domain should be true.')
              files_read(index_file)%var(index_field)%is_dimvar = .true.
              files_read(index_file)%var(index_field)%name      = fieldname
              files_read(index_file)%var(index_field)%axis      = axes(i)
-             files_read(index_file)%var(index_field)%siz(:)    = siz_in
-             files_read(index_file)%var(index_field)%gsiz(:)   = siz_in
+             files_read(index_file)%var(index_field)%siz(1:4)    = siz_in(1:4)
+             files_read(index_file)%var(index_field)%gsiz(1:3)   = siz_in(1:3)
              return
           endif
        enddo
@@ -5556,7 +7413,7 @@ end subroutine get_field_id
 !     A file name (or path name) that is checked for existence.
 !   
 !   
-!     This function returns a logical result.  If file_name exists the result 
+!     This function returns a logical result.  If file_name exists the result
 !     is true, otherwise false is returned.
 !     If the length of character string "file_name" is zero or the first
 !     character is blank, then the returned value will be false.
@@ -5564,14 +7421,14 @@ end subroutine get_field_id
 !     routine open_file.
 !   
 !   
-!     Before calling write_data you must first call set_domain with domain2d data 
+!     Before calling write_data you must first call set_domain with domain2d data
 !     type associated with the distributed data you are writing.
 !   
 
  function file_exist (file_name, domain, no_domain)
   character(len=*), intent(in)         :: file_name
   type(domain2d), intent(in), optional :: domain
-  logical,        intent(iN), optional :: no_domain  
+  logical,        intent(iN), optional :: no_domain
 
   logical                              :: file_exist, is_no_domain
   character(len=256)                   :: fname
@@ -5582,10 +7439,10 @@ function file_exist (file_name, domain, no_domain)
    !--- to deal with mosaic file, in this case, the file is assumed to be in netcdf format
    file_exist = get_file_name(file_name, fname, read_dist, io_domain_exist, no_domain=is_no_domain, domain=domain)
    if(is_no_domain) return
-   if(.not.file_exist) file_exist=get_file_name(file_name, fname, read_dist, io_domain_exist, no_domain=.true.)   
+   if(.not.file_exist) file_exist=get_file_name(file_name, fname, read_dist, io_domain_exist, no_domain=.true.)
 
    return
- 
+
  end function file_exist
 ! 
 
@@ -5594,10 +7451,10 @@ end function file_exist
 ! 
 
 !   
-!     check if a given field name exists in a given file name. 
+!     check if a given field name exists in a given file name.
 !   
 !   
-!     check if a given field name exists in a given file name. 
+!     check if a given field name exists in a given file name.
 !     If the field_name string has zero length or the
 !     first character is blank return a false result.
 !     if the file file_name don't exist, return a false result.
@@ -5613,7 +7470,7 @@ end function file_exist
 !     A field name that is checked for existence.
 !   
 !   
-!     This function returns a logical result.  If field exists in the 
+!     This function returns a logical result.  If field exists in the
 !     file file_name, the result is true, otherwise false is returned.
 !     If the length of character string "field_name" is zero or the first
 !     character is blank, then the returned value will be false.
@@ -5637,7 +7494,7 @@ function field_exist (file_name, field_name, domain, no_domain)
    if (field_name(1:1) == ' ')    return
 
    is_no_domain = .false.
-   if(present(no_domain)) is_no_domain = no_domain   
+   if(present(no_domain)) is_no_domain = no_domain
 
    file_exist=get_file_name(file_name, fname, read_dist, io_domain_exist, no_domain=is_no_domain, domain=domain)
    if(file_exist) then
@@ -5673,17 +7530,25 @@ end function field_exist
 
 subroutine set_filename_appendix(string_in)
   character(len=*) , intent(in) :: string_in
-  filename_appendix = trim(string_in)
+
+  integer :: index_num
+
+  ! Check if string has already been added
+  index_num = index(filename_appendix, string_in)
+  if ( index_num .le. 0 ) then
+     filename_appendix = trim(filename_appendix)//trim(string_in)
+  end if
+
 end subroutine set_filename_appendix
 
 subroutine get_instance_filename(name_in,name_out)
   character(len=*)  , intent(in)  :: name_in
   character(len=*), intent(inout) :: name_out
   integer :: length
-  
+
   length = len_trim(name_in)
   name_out = name_in(1:length)
-  
+
   if(len_trim(filename_appendix) > 0) then
      if(name_in(length-2:length) == '.nc') then
         name_out = name_in(1:length-3)//'.'//trim(filename_appendix)//'.nc'
@@ -5691,12 +7556,11 @@ subroutine get_instance_filename(name_in,name_out)
         name_out = name_in(1:length)  //'.'//trim(filename_appendix)
      end if
   end if
-  
-end subroutine get_instance_filename
 
+end subroutine get_instance_filename
 
 !#######################################################################
-subroutine parse_mask_table(mask_table, maskmap, modelname)
+subroutine parse_mask_table_2d(mask_table, maskmap, modelname)
 
   character(len=*), intent(in) :: mask_table
   logical,         intent(out) :: maskmap(:,:)
@@ -5713,21 +7577,21 @@ subroutine parse_mask_table(mask_table, maskmap, modelname)
      call mpp_open(unit, mask_table, action=MPP_RDONLY)
      read(unit, FMT=*, IOSTAT=mystat) nmask
      if( mystat /= 0 ) call mpp_error(FATAL, &
-          "fms_io(parse_mask_table): Error reading nmask from file " //trim(mask_table))
+          "fms_io(parse_mask_table_2d): Error reading nmask from file " //trim(mask_table))
      write(stdoutunit,*)"parse_mask_table: Number of domain regions masked in ", trim(modelname), " = ", nmask
      if( nmask > 0 ) then
         !--- read layout from mask_table and confirm it matches the shape of maskmap
         read(unit, FMT=*, IOSTAT=mystat) layout
         if( mystat /= 0 ) call mpp_error(FATAL, &
-             "fms_io(parse_mask_talbe): Error reading layout from file " //trim(mask_table))
+             "fms_io(parse_mask_talbe_2d): Error reading layout from file " //trim(mask_table))
         if( (layout(1) .NE. size(maskmap,1)) .OR. (layout(2) .NE. size(maskmap,2)) )then
            write(stdoutunit,*)"layout=", layout, ", size(maskmap) = ", size(maskmap,1), size(maskmap,2)
-           call mpp_error(FATAL, "fms_io(parse_mask_table): layout in file "//trim(mask_table)// &
+           call mpp_error(FATAL, "fms_io(parse_mask_table_2d): layout in file "//trim(mask_table)// &
                   "does not match size of maskmap for "//trim(modelname))
         endif
         !--- make sure mpp_npes() == layout(1)*layout(2) - nmask
         if( mpp_npes() .NE. layout(1)*layout(2) - nmask ) call mpp_error(FATAL, &
-           "fms_io(parse_mask_table): mpp_npes() .NE. layout(1)*layout(2) - nmask for "//trim(modelname))
+           "fms_io(parse_mask_table_2d): mpp_npes() .NE. layout(1)*layout(2) - nmask for "//trim(modelname))
       endif
    endif
 
@@ -5748,31 +7612,121 @@ subroutine parse_mask_table(mask_table, maskmap, modelname)
         if (record(1:10) == '          ') cycle
         n = n + 1
         if( n > nmask ) then
-           call mpp_error(FATAL, "fms_io(parse_mask_table): number of mask_list entry "// &
-                "is greater than nmask in file "//trim(mask_table) ) 
+           call mpp_error(FATAL, "fms_io(parse_mask_table_2d): number of mask_list entry "// &
+                "is greater than nmask in file "//trim(mask_table) )
         endif
-        read(record,*,err=888) mask_list(n,1), mask_list(n,2) 
+        read(record,*,err=888) mask_list(n,1), mask_list(n,2)
      enddo
-888  call mpp_error(FATAL, "fms_io(parse_mask_table):  Error in reading mask_list from file "//trim(mask_table))    
+888  call mpp_error(FATAL, "fms_io(parse_mask_table_2d):  Error in reading mask_list from file "//trim(mask_table))
 
 999  continue
      !--- make sure the number of entry for mask_list is nmask
      if( n .NE. nmask) call mpp_error(FATAL, &
-        "fms_io(parse_mask_table): number of mask_list entry does not match nmask in file "//trim(mask_table)) 
+        "fms_io(parse_mask_table_2d): number of mask_list entry does not match nmask in file "//trim(mask_table))
      call mpp_close(unit)
   endif
-  
+
   call mpp_broadcast(mask_list, 2*nmask, mpp_root_pe())
   do n = 1, nmask
      if(debug_mask_list) then
-       write(stdoutunit,*) "==>NOTE from parse_mask_table: ", trim(modelname), " mask_list = ", mask_list(n,1), mask_list(n,2)
+       write(stdoutunit,*) "==>NOTE from parse_mask_table_2d: ", trim(modelname), " mask_list = ", mask_list(n,1), mask_list(n,2)
      endif
      maskmap(mask_list(n,1),mask_list(n,2)) = .false.
   enddo
 
   deallocate(mask_list)
 
-end subroutine parse_mask_table
+end subroutine parse_mask_table_2d
+
+
+!#######################################################################
+subroutine parse_mask_table_3d(mask_table, maskmap, modelname)
+
+  character(len=*), intent(in) :: mask_table
+  logical,         intent(out) :: maskmap(:,:,:)
+  character(len=*), intent(in) :: modelname
+  integer                      :: nmask, layout(2)
+  integer, allocatable         :: mask_list(:,:)
+  integer                      :: unit, mystat, n, stdoutunit, ntiles
+  character(len=128)           :: record
+
+  maskmap = .true.
+  nmask = 0
+  stdoutunit = stdout()
+  if( mpp_pe() == mpp_root_pe() ) then
+     call mpp_open(unit, mask_table, action=MPP_RDONLY)
+     read(unit, FMT=*, IOSTAT=mystat) nmask
+     if( mystat /= 0 ) call mpp_error(FATAL, &
+          "fms_io(parse_mask_table_3d): Error reading nmask from file " //trim(mask_table))
+     write(stdoutunit,*)"parse_mask_table: Number of domain regions masked in ", trim(modelname), " = ", nmask
+     if( nmask > 0 ) then
+        !--- read layout from mask_table and confirm it matches the shape of maskmap
+        read(unit, FMT=*, IOSTAT=mystat) layout(1), layout(2), ntiles
+        if( mystat /= 0 ) call mpp_error(FATAL, &
+             "fms_io(parse_mask_talbe_3d): Error reading layout from file " //trim(mask_table))
+        if( (layout(1) .NE. size(maskmap,1)) .OR. (layout(2) .NE. size(maskmap,2)) )then
+           write(stdoutunit,*)"layout=", layout, ", size(maskmap) = ", size(maskmap,1), size(maskmap,2)
+           call mpp_error(FATAL, "fms_io(parse_mask_table_3d): layout in file "//trim(mask_table)// &
+                  "does not match size of maskmap for "//trim(modelname))
+        endif
+        if( ntiles .NE. size(maskmap,3) ) then
+           write(stdoutunit,*)"ntiles=", ntiles, ", size(maskmap,3) = ", size(maskmap,3)
+           call mpp_error(FATAL, "fms_io(parse_mask_table_3d): ntiles in file "//trim(mask_table)// &
+                  "does not match size of maskmap for "//trim(modelname))
+        endif
+        !--- make sure mpp_npes() == layout(1)*layout(2) - nmask
+        if( mpp_npes() .NE. layout(1)*layout(2)*ntiles - nmask ) then
+           print*, "layout=", layout, nmask, mpp_npes()
+           call mpp_error(FATAL, &
+              "fms_io(parse_mask_table_3d): mpp_npes() .NE. layout(1)*layout(2) - nmask for "//trim(modelname))
+        endif
+      endif
+   endif
+
+   call mpp_broadcast(nmask, mpp_root_pe())
+
+   if(nmask==0) then
+      if( mpp_pe() == mpp_root_pe() ) call mpp_close(unit)
+      return
+   endif
+
+   allocate(mask_list(nmask,3))
+
+   if( mpp_pe() == mpp_root_pe() ) then
+     n = 0
+     do while( .true. )
+        read(unit,'(a)',end=999) record
+        if (record(1:1) == '#') cycle
+        if (record(1:10) == '          ') cycle
+        n = n + 1
+        if( n > nmask ) then
+           call mpp_error(FATAL, "fms_io(parse_mask_table_3d): number of mask_list entry "// &
+                "is greater than nmask in file "//trim(mask_table) )
+        endif
+        read(record,*,err=888) mask_list(n,1), mask_list(n,2), mask_list(n,3)
+     enddo
+888  call mpp_error(FATAL, "fms_io(parse_mask_table_3d):  Error in reading mask_list from file "//trim(mask_table))
+
+999  continue
+     !--- make sure the number of entry for mask_list is nmask
+     if( n .NE. nmask) call mpp_error(FATAL, &
+        "fms_io(parse_mask_table_3d): number of mask_list entry does not match nmask in file "//trim(mask_table))
+     call mpp_close(unit)
+  endif
+
+  call mpp_broadcast(mask_list, 3*nmask, mpp_root_pe())
+  do n = 1, nmask
+     if(debug_mask_list) then
+       write(stdoutunit,*) "==>NOTE from parse_mask_table_3d: ", trim(modelname), " mask_list = ", &
+                           mask_list(n,1), mask_list(n,2), mask_list(n,3)
+     endif
+     maskmap(mask_list(n,1),mask_list(n,2),mask_list(n,3)) = .false.
+  enddo
+
+  deallocate(mask_list)
+
+end subroutine parse_mask_table_3d
+
 
 function get_great_circle_algorithm()
    logical :: get_great_circle_algorithm
@@ -5786,5 +7740,3 @@ end function get_great_circle_algorithm
 
 
 end module fms_io_mod
-
-
diff --git a/src/shared/fms/read_data_2d.inc b/src/shared/fms/read_data_2d.inc
index 5a00047840..6970f62b6b 100644
--- a/src/shared/fms/read_data_2d.inc
+++ b/src/shared/fms/read_data_2d.inc
@@ -1,5 +1,5 @@
 ! -*-f90-*-
-! $Id: read_data_2d.inc,v 13.0 2006/03/28 21:39:18 fms Exp $
+! $Id$
 
 
    if((size(data,1)== ied-isd+1).and.(size(data,2)==jed-jsd+1)) then
diff --git a/src/shared/fms/read_data_3d.inc b/src/shared/fms/read_data_3d.inc
index 04ab5f8a60..f2dc5f4a80 100644
--- a/src/shared/fms/read_data_3d.inc
+++ b/src/shared/fms/read_data_3d.inc
@@ -1,5 +1,5 @@
 ! -*-f90-*-
-! $Id: read_data_3d.inc,v 13.0 2006/03/28 21:39:21 fms Exp $
+! $Id$
 
    if((size(data,1)== ied-isd+1).and.(size(data,2)==jed-jsd+1)) then
      no_halo = .false.
diff --git a/src/shared/fms/read_data_4d.inc b/src/shared/fms/read_data_4d.inc
index f0c3419afc..d3160a367f 100644
--- a/src/shared/fms/read_data_4d.inc
+++ b/src/shared/fms/read_data_4d.inc
@@ -1,5 +1,5 @@
 ! -*-f90-*-
-! $Id: read_data_4d.inc,v 13.0 2006/03/28 21:39:24 fms Exp $
+! $Id$
 
    if((size(data,1)== ied-isd+1).and.(size(data,2)==jed-jsd+1)) then
      no_halo = .false.
diff --git a/src/shared/fms/test_fms_io.F90 b/src/shared/fms/test_fms_io.F90
index 2c617fc542..7529e07b51 100644
--- a/src/shared/fms/test_fms_io.F90
+++ b/src/shared/fms/test_fms_io.F90
@@ -22,6 +22,7 @@ program fms_io_test
  integer :: sizey_latlon_grid = 90
  integer :: size_cubic_grid = 48
  integer :: nz = 10, nt = 2, halo = 1
+ integer :: nl = 5
  integer :: stackmax =4000000
  integer :: num_step = 4 ! number of time steps to run, this is used for intermediate run.
                          ! set num_step = 0 for no intermediate run.
@@ -31,22 +32,25 @@ program fms_io_test
  integer :: layout_latlon(2) = (/0,0/)  
  integer :: io_layout(2) = (/0,0/) ! set ndivs_x and ndivs_y to divide each tile into io_layout(1)*io_layout(2)
                                    ! group and write out data from the root pe of each group.
+ logical :: read_only = .False.
 
  namelist /test_fms_io_nml/ sizex_latlon_grid, sizey_latlon_grid, size_cubic_grid, &
-                            nz, nt, halo, num_step, stackmax, do_write, layout_cubic, layout_latlon, io_layout
+                            nz, nt, halo, num_step, stackmax, do_write, layout_cubic, layout_latlon, io_layout, &
+                            read_only, nl
 
  integer           :: unit, io_status, step
  character(len=20) :: time_stamp
 
  type data_storage_type
-    real,    allocatable, dimension(:,:,:,:) :: data1_r3d, data2_r3d, data1_r3d_read, data2_r3d_read
-    real,    allocatable, dimension(:,:,:)   :: data1_r2d, data2_r2d, data1_r2d_read, data2_r2d_read
-    real,    allocatable, dimension(:,:)     :: data1_r1d, data2_r1d, data1_r1d_read, data2_r1d_read
-    real,    allocatable, dimension(:)       :: data1_r0d, data2_r0d, data1_r0d_read, data2_r0d_read
-    integer, allocatable, dimension(:,:,:,:) :: data1_i3d, data2_i3d, data1_i3d_read, data2_i3d_read
-    integer, allocatable, dimension(:,:,:)   :: data1_i2d, data2_i2d, data1_i2d_read, data2_i2d_read
-    integer, allocatable, dimension(:,:)     :: data1_i1d, data2_i1d, data1_i1d_read, data2_i1d_read
-    integer, allocatable, dimension(:)       :: data1_i0d, data2_i0d, data1_i0d_read, data2_i0d_read
+    real,    allocatable, dimension(:,:,:,:,:) :: data1_r4d, data2_r4d, data1_r4d_read, data2_r4d_read
+    real,    allocatable, dimension(:,:,:,:)   :: data1_r3d, data2_r3d, data1_r3d_read, data2_r3d_read
+    real,    allocatable, dimension(:,:,:)     :: data1_r2d, data2_r2d, data1_r2d_read, data2_r2d_read
+    real,    allocatable, dimension(:,:)       :: data1_r1d, data2_r1d, data1_r1d_read, data2_r1d_read
+    real,    allocatable, dimension(:)         :: data1_r0d, data2_r0d, data1_r0d_read, data2_r0d_read
+    integer, allocatable, dimension(:,:,:,:)   :: data1_i3d, data2_i3d, data1_i3d_read, data2_i3d_read
+    integer, allocatable, dimension(:,:,:)     :: data1_i2d, data2_i2d, data1_i2d_read, data2_i2d_read
+    integer, allocatable, dimension(:,:)       :: data1_i1d, data2_i1d, data1_i1d_read, data2_i1d_read
+    integer, allocatable, dimension(:)         :: data1_i0d, data2_i0d, data1_i0d_read, data2_i0d_read
  end type data_storage_type
  
  type(data_storage_type), save :: latlon_data
@@ -99,11 +103,11 @@ program fms_io_test
 
  if(file_exist('INPUT/'//trim(file_latlon), domain_latlon)) then
     call restore_state(restart_latlon)
-    call compare_restart("latlon_grid save_restore", latlon_data)
+    call compare_restart("latlon_grid save_restore", latlon_data, .true.)
  end if
  if(file_exist('INPUT/'//trim(file_cubic), domain_cubic) ) then
     call restore_state(restart_cubic)
-    call compare_restart("cubic_grid save_restore", cubic_data)
+    call compare_restart("cubic_grid save_restore", cubic_data, .true. )
  end if
  
  !---copy data
@@ -149,7 +153,7 @@ subroutine setup_test_restart(restart_data, type, ntiles, storage, file, layout_
     integer, dimension(1)                    :: tile1, tile2
     integer, dimension(1)                    :: istart1, iend1, jstart1, jend1
     integer, dimension(1)                    :: istart2, iend2, jstart2, jend2
-    integer                                  :: i, j, k, nx, ny
+    integer                                  :: i, j, k, nx, ny, l
     integer                                  :: isc, iec, jsc, jec
     integer                                  :: isd, ied, jsd, jed
     integer                                  :: id_restart
@@ -205,6 +209,10 @@ subroutine setup_test_restart(restart_data, type, ntiles, storage, file, layout_
 
     allocate(storage%data1_r3d(isd:ied, jsd:jed, nz, nt), storage%data1_r3d_read(isd:ied, jsd:jed, nz, nt) )
     allocate(storage%data2_r3d(isd:ied, jsd:jed, nz, nt), storage%data2_r3d_read(isd:ied, jsd:jed, nz, nt) )
+    allocate(storage%data1_r4d(isd:ied, jsd:jed, nz, nl, nt) )
+    allocate(storage%data1_r4d_read(isd:ied, jsd:jed, nz, nl, nt) )
+    allocate(storage%data2_r4d(isd:ied, jsd:jed, nz, nl, nt) )
+    allocate(storage%data2_r4d_read(isd:ied, jsd:jed, nz, nl, nt) )
     allocate(storage%data1_i3d(isd:ied, jsd:jed, nz, nt), storage%data1_i3d_read(isd:ied, jsd:jed, nz, nt) )
     allocate(storage%data2_i3d(isd:ied, jsd:jed, nz, nt), storage%data2_i3d_read(isd:ied, jsd:jed, nz, nt) )
     allocate(storage%data1_r2d(isd:ied, jsd:jed,     nt), storage%data1_r2d_read(isd:ied, jsd:jed,     nt) )
@@ -228,11 +236,22 @@ subroutine setup_test_restart(restart_data, type, ntiles, storage, file, layout_
     storage%data1_i1d = 0; storage%data1_i1d_read = 0; storage%data2_i1d = 0; storage%data2_i1d_read = 0
     storage%data1_r0d = 0; storage%data1_r0d_read = 0; storage%data2_r0d = 0; storage%data2_r0d_read = 0
     storage%data1_i0d = 0; storage%data1_i0d_read = 0; storage%data2_i0d = 0; storage%data2_i0d_read = 0
+    storage%data1_r4d = 0; storage%data1_r4d_read = 0; storage%data2_r4d = 0; storage%data2_r4d_read = 0
     do n = 1, nt
        storage%data1_r0d(n) =  tile + n*1e-3
        storage%data2_r0d(n) = -tile - n*1e-3
        storage%data1_i0d(n) =  tile*1e3 + n
        storage%data2_i0d(n) = -tile*1e3 - n
+       do l = 1, nl
+          do k = 1, nz
+             do j = jsc, jec
+                do i = isc, iec
+                   storage%data1_r4d(i,j,k,l,n) =  l*1e9 + tile*1e6 + n*1e3 + k + i*1e-3 + j*1e-6;
+                   storage%data2_r4d(i,j,k,l,n) =  -l*1e9 - tile*1e6 - n*1e3 + k - i*1e-3 - j*1e-6;
+                enddo
+             enddo
+          enddo
+       enddo
        do k = 1, nz
           storage%data1_r1d(k,n) =   tile*1e3 + n + k*1e-3
           storage%data2_r1d(k,n) =  -tile*1e3 - n - k*1e-3
@@ -276,7 +295,7 @@ subroutine setup_test_restart(restart_data, type, ntiles, storage, file, layout_
           call read_data(file_r, "data1_i0d", storage%data1_i0d_read(      n), domain, timelevel = n )
           call read_data(file_r, "data2_i0d", storage%data2_i0d_read(      n), domain, timelevel = n )
        end do
-       call compare_restart(type//" read_write", storage)
+       call compare_restart(type//" read_write", storage, .false.)
     end if
 
 
@@ -305,13 +324,20 @@ subroutine setup_test_restart(restart_data, type, ntiles, storage, file, layout_
     !--- test register_restart_field, save_restart, restore_state
 
     id_restart = register_restart_field(restart_data, file, "data1_r3d", storage%data1_r3d_read(:,:,:,1), &
-                                domain, longname="first data_r3d",units="none")
+                                domain, longname="first data_r3d",units="none", read_only=read_only)
     id_restart = register_restart_field(restart_data, file, "data1_r3d", storage%data1_r3d_read(:,:,:,2), &
-                                domain, longname="first data_r3d",units="none")
+                                domain, longname="first data_r3d",units="none", read_only=read_only)
     id_restart = register_restart_field(restart_data, file, "data2_r3d", storage%data2_r3d_read(:,:,:,1), &
                                 storage%data2_r3d_read(:,:,:,2), &
                                 domain, longname="second data_i3d", units="none")
 
+    id_restart = register_restart_field(restart_data, file, "data1_r4d", storage%data1_r4d_read(:,:,:,:,1), &
+                                domain, longname="first data_r4d",units="none")
+    id_restart = register_restart_field(restart_data, file, "data1_r4d", storage%data1_r4d_read(:,:,:,:,2), &
+                                domain, longname="first data_r4d",units="none")
+    id_restart = register_restart_field(restart_data, file, "data2_r4d", storage%data2_r4d_read(:,:,:,:,1), &
+                                domain, longname="second data_r4d",units="none")
+
     id_restart = register_restart_field(restart_data, file, "data1_i3d", storage%data1_i3d_read(:,:,:,1), &
                                 domain, longname="first data_i3d",units="none")
     id_restart = register_restart_field(restart_data, file, "data1_i3d", storage%data1_i3d_read(:,:,:,2), &
@@ -321,20 +347,20 @@ subroutine setup_test_restart(restart_data, type, ntiles, storage, file, layout_
                                 domain, longname="second data_i3d", units="none")
 
     id_restart = register_restart_field(restart_data, file, "data1_r2d", storage%data1_r2d_read(:,:,  1), &
-                                domain, longname="first data_r2d",units="none")
+                                domain, longname="first data_r2d",units="none", read_only=read_only)
     id_restart = register_restart_field(restart_data, file, "data1_r2d", storage%data1_r2d_read(:,:,  2), &
-                                domain, longname="first data_r2d",units="none")
+                                domain, longname="first data_r2d",units="none", read_only=read_only)
     id_restart = register_restart_field(restart_data, file, "data2_r2d", storage%data2_r2d_read(:,:,  1), &
                                 storage%data2_r2d_read(:,:,2), &
                                 domain, longname="second data_i2d", units="none")
 
     id_restart = register_restart_field(restart_data, file, "data1_i2d", storage%data1_i2d_read(:,:,  1), &
-                                domain, longname="first data_i2d",units="none")
+                                domain, longname="first data_i2d",units="none", read_only=read_only)
     id_restart = register_restart_field(restart_data, file, "data1_i2d", storage%data1_i2d_read(:,:,  2), &
-                                domain, longname="first data_i2d",units="none")
+                                domain, longname="first data_i2d",units="none", read_only=read_only)
     id_restart = register_restart_field(restart_data, file, "data2_i2d", storage%data2_i2d_read(:,:,  1), &
                                 storage%data2_i2d_read(:,:,2), &
-                                domain, longname="second data_i2d", units="none")
+                                domain, longname="second data_i2d", units="none", read_only=read_only)
 
     id_restart = register_restart_field(restart_data, file, "data1_r1d", storage%data1_r1d_read(:,    1), &
                                 domain, longname="first data_r1d",units="none")
@@ -345,18 +371,18 @@ subroutine setup_test_restart(restart_data, type, ntiles, storage, file, layout_
                                 domain, longname="second data_i1d", units="none")
 
     id_restart = register_restart_field(restart_data, file, "data1_i1d", storage%data1_i1d_read(:,    1), &
-                                domain, longname="first data_i1d",units="none")
+                                domain, longname="first data_i1d",units="none", read_only=read_only)
     id_restart = register_restart_field(restart_data, file, "data1_i1d", storage%data1_i1d_read(:,    2), &
-                                domain, longname="first data_i1d",units="none")
+                                domain, longname="first data_i1d",units="none", read_only=read_only)
     id_restart = register_restart_field(restart_data, file, "data2_i1d", storage%data2_i1d_read(:,    1), &
                                 storage%data2_i1d_read(:,  2), &
                                 domain, longname="second data_i1d", units="none")
 
 
     id_restart = register_restart_field(restart_data, file, "data1_r0d", storage%data1_r0d_read(      1), &
-                                domain, longname="first data_r0d",units="none")
+                                domain, longname="first data_r0d",units="none", read_only=read_only)
     id_restart = register_restart_field(restart_data, file, "data1_r0d", storage%data1_r0d_read(      2), &
-                                domain, longname="first data_r0d",units="none")
+                                domain, longname="first data_r0d",units="none", read_only=read_only)
     id_restart = register_restart_field(restart_data, file, "data2_r0d", storage%data2_r0d_read(      1), &
                                 storage%data2_r0d_read(    2), &
                                 domain, longname="second data_i0d", units="none")
@@ -371,10 +397,15 @@ subroutine setup_test_restart(restart_data, type, ntiles, storage, file, layout_
 
   end subroutine setup_test_restart
 
-  subroutine compare_restart(type, storage)
+  subroutine compare_restart(type, storage, compare_r4d)
     character(len=*), intent(in)             :: type
     type(data_storage_type), intent(inout)   :: storage
+    logical,                 intent(in   )   :: compare_r4d
 
+       if(compare_r4d) then   
+          call compare_data_r5d(storage%data1_r4d, storage%data1_r4d_read, type//" data1_r4d")
+          call compare_data_r5d(storage%data2_r4d(:,:,:,:,1:1), storage%data2_r4d_read(:,:,:,:,1:1), type//" data2_r4d")
+       endif
        call compare_data_r4d(storage%data1_r3d, storage%data1_r3d_read, type//" data1_r3d")
        call compare_data_r4d(storage%data2_r3d, storage%data2_r3d_read, type//" data2_r3d")
        call compare_data_i4d(storage%data1_i3d, storage%data1_i3d_read, type//" data1_i3d")
@@ -405,11 +436,13 @@ subroutine release_storage_memory(storage)
     deallocate(storage%data1_i1d, storage%data2_i1d, storage%data1_i1d_read, storage%data2_i1d_read)
     deallocate(storage%data1_r0d, storage%data2_r0d, storage%data1_r0d_read, storage%data2_r0d_read)
     deallocate(storage%data1_i0d, storage%data2_i0d, storage%data1_i0d_read, storage%data2_i0d_read)
+    deallocate(storage%data1_r4d, storage%data2_r4d, storage%data1_r4d_read, storage%data2_r4d_read)
 
   end subroutine release_storage_memory
 
   subroutine copy_restart_data(storage)
     type(data_storage_type), intent(inout)   :: storage
+    integer :: n, l
 
     storage%data1_r3d_read = storage%data1_r3d; storage%data2_r3d_read = storage%data2_r3d
     storage%data1_i3d_read = storage%data1_i3d; storage%data2_i3d_read = storage%data2_i3d
@@ -419,11 +452,52 @@ subroutine copy_restart_data(storage)
     storage%data1_i1d_read = storage%data1_i1d; storage%data2_i1d_read = storage%data2_i1d
     storage%data1_r0d_read = storage%data1_r0d; storage%data2_r0d_read = storage%data2_r0d
     storage%data1_i0d_read = storage%data1_i0d; storage%data2_i0d_read = storage%data2_i0d
+    storage%data1_r4d_read = storage%data1_r4d; storage%data2_r4d_read = storage%data2_r4d
 
     return
 
   end subroutine copy_restart_data
 
+  subroutine compare_data_r5d( a, b, string )
+    real, intent(in), dimension(:,:,:,:,:) :: a, b
+    character(len=*), intent(in)         :: string
+    integer(LONG_KIND)                   :: sum1, sum2
+    integer                              :: i, j, k, l, n
+    integer, parameter                   :: stdunit = 6
+
+    if(size(a,1) .ne. size(b,1) .or. size(a,2) .ne. size(b,2) .or. size(a,3) .ne. size(b,3) &
+      .or. size(a,4) .ne. size(b,4) .or. size(a,5) .ne. size(b,5) ) &
+         call mpp_error(FATAL,'compare_data_r5d: size of a and b does not match')
+    do n = 1, size(a,5)
+       do l = 1, size(a,4)
+          do k = 1, size(a,3)
+             do j = 1, size(a,2)
+                do i = 1, size(a,1)
+                   if(a(i,j,k,l,n) .ne. b(i,j,k,l,n)) then
+                      write(stdunit,'(a,i3,a,i3,a,i3,a,i3,a,i3,a,i3,a,f18.6,a,f18.6)')" at pe ", mpp_pe(), &
+                           ", at point (",i,", ", j, ", ", k, ", ", l, ", ", n, "), a = ", a(i,j,k,l,n), ",&
+                           b = ", b(i,j,k,l,n)
+                      call mpp_error(FATAL, trim(string)//': point by point comparison are not OK.')
+                   endif
+                enddo
+             enddo
+          enddo
+       enddo
+    enddo
+    sum1 = mpp_chksum( a, (/mpp_pe()/) )
+    sum2 = mpp_chksum( b, (/mpp_pe()/) )
+
+    if( sum1.EQ.sum2 )then
+       if( mpp_pe() .EQ. mpp_root_pe() )call mpp_error( NOTE, trim(string)//': OK.' )
+       !--- in some case, even though checksum agree, the two arrays 
+       !    actually are different, like comparing (1.1,-1.2) with (-1.1,1.2)
+       !--- hence we need to check the value point by point.
+    else
+       call mpp_error( FATAL, trim(string)//': chksums are not OK.' )
+    end if
+  end subroutine compare_data_r5d
+
+
   subroutine compare_data_r4d( a, b, string )
     real, intent(in), dimension(:,:,:,:) :: a, b
     character(len=*), intent(in)         :: string
@@ -439,7 +513,7 @@ subroutine compare_data_r4d( a, b, string )
           do j = 1, size(a,2)
              do i = 1, size(a,1)
                 if(a(i,j,k,l) .ne. b(i,j,k,l)) then
-                   write(stdunit,'(a,i3,a,i3,a,i3,a,i3,a,i3,a,f18.9,a,f18.9)')" at pe ", mpp_pe(), &
+                   write(stdunit,'(a,i3,a,i3,a,i3,a,i3,a,i3,a,f18.6,a,f18.6)')" at pe ", mpp_pe(), &
                         ", at point (",i,", ", j, ", ", k, ", ", l, "), a = ", a(i,j,k,l), ", b = ", b(i,j,k,l)
                    call mpp_error(FATAL, trim(string)//': point by point comparison are not OK.')
                 endif
@@ -533,7 +607,7 @@ subroutine compare_data_r2d( a, b, string )
     do l = 1, size(a,2)
        do i = 1, size(a,1)
           if(a(i,l) .ne. b(i,l)) then
-             write(stdunit,'(a,i3,a,i3,a,i3,a,f16.9,a,f16.9)')" at pe ", mpp_pe(), &
+             write(stdunit,'(a,i3,a,i3,a,i3,a,f16.6,a,f16.6)')" at pe ", mpp_pe(), &
                   ", at point (",i, ", ", l, "), a = ", a(i,l), ", b = ", b(i,l)
              call mpp_error(FATAL, trim(string)//': point by point comparison are not OK.')
           endif
@@ -576,7 +650,7 @@ subroutine compare_data_r1d( a, b, string )
 
     do l = 1, size(a(:))
        if(a(l) .ne. b(l)) then
-          write(stdunit,'(a,i3,a,i3,a,f16.9,a,f16.9)')" at pe ", mpp_pe(), &
+          write(stdunit,'(a,i3,a,i3,a,f16.6,a,f16.6)')" at pe ", mpp_pe(), &
                ", at point (",l, "), a = ", a(l), ", b = ", b(l)
           call mpp_error(FATAL, trim(string)//': point by point comparison are not OK.')
        endif
diff --git a/src/shared/horiz_interp/horiz_interp.F90 b/src/shared/horiz_interp/horiz_interp.F90
index 38357a8159..9deee50af3 100644
--- a/src/shared/horiz_interp/horiz_interp.F90
+++ b/src/shared/horiz_interp/horiz_interp.F90
@@ -24,9 +24,10 @@
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 module horiz_interp_mod
 
-!  Zhi Liang 
-!  Bruce Wyman 
+!  Zhi Liang 
+!  Bruce Wyman 
 
+! 
 
 ! 
 !   Performs spatial interpolation between grids.
@@ -235,8 +236,8 @@ module horiz_interp_mod
  namelist /horiz_interp_nml/ reproduce_siena
 
 !-----------------------------------------------------------------------
- character(len=128) :: version = '$Id: horiz_interp.F90,v 20.0 2013/12/14 00:20:17 fms Exp $'
- character(len=128) :: tagname = '$Name: tikal $'
+ character(len=128) :: version = '$Id$'
+ character(len=128) :: tagname = '$Name$'
  logical            :: module_is_initialized = .FALSE.
 !-----------------------------------------------------------------------
 
@@ -255,7 +256,7 @@ subroutine horiz_interp_init
   integer :: unit, ierr, io
 
   if(module_is_initialized) return
-  call write_version_number()
+  call write_version_number (version, tagname)
 
 #ifdef INTERNAL_FILE_NML
   read (input_nml_file, horiz_interp_nml, iostat=io)
@@ -277,13 +278,9 @@ subroutine horiz_interp_init
   endif
 
   if( reproduce_siena ) then
-     if( mpp_pe() == mpp_root_pe() ) then
-        call mpp_error(WARNING, "horiz_interp_mod: You have overridden the default value of reproduce_siena " // &
-                             "and set it to .true. in horiz_interp_nml. This is a temporary workaround to " // &
-                             "allow for consistency in continuing experiments.  Please use the default " //&
-                             "value (.false.) as this option will be removed in a future release. ")
-     endif
-     call set_reproduce_siena_true( )
+     call mpp_error(FATAL, "horiz_interp_mod: You have overridden the default value of reproduce_siena " // &
+                           "and set it to .true. in horiz_interp_nml. This is a temporary workaround to " // &
+                           "allow for consistency in continuing experiments. Please remove this namelist " )
   endif
 
   call horiz_interp_conserve_init
@@ -720,7 +717,8 @@ end subroutine horiz_interp_new_1d_dst
 
 ! 
  subroutine horiz_interp_base_2d ( Interp, data_in, data_out, verbose, &
-                                   mask_in, mask_out, missing_value, missing_permit, err_msg )
+                                   mask_in, mask_out, missing_value, missing_permit, &
+                                   err_msg, new_missing_handle )
 !
 !-----------------------------------------------------------------------
    type (horiz_interp_type), intent(in) :: Interp
@@ -732,6 +730,7 @@ subroutine horiz_interp_base_2d ( Interp, data_in, data_out, verbose, &
       real, intent(in),                   optional :: missing_value
       integer, intent(in),                optional :: missing_permit
    character(len=*), intent(out),         optional :: err_msg
+      logical, intent(in),                optional :: new_missing_handle
 !-----------------------------------------------------------------------
    if(present(err_msg)) err_msg = ''
    if(.not.Interp%I_am_initialized) then
@@ -743,7 +742,7 @@ subroutine horiz_interp_base_2d ( Interp, data_in, data_out, verbose, &
       call horiz_interp_conserve(Interp,data_in, data_out, verbose, mask_in, mask_out)
    case(BILINEAR)
       call horiz_interp_bilinear(Interp,data_in, data_out, verbose, mask_in, mask_out, &
-                             missing_value, missing_permit )
+                             missing_value, missing_permit, new_missing_handle )
    case(BICUBIC)
       call horiz_interp_bicubic(Interp,data_in, data_out, verbose, mask_in, mask_out, &
                              missing_value, missing_permit )
diff --git a/src/shared/horiz_interp/horiz_interp_bicubic.F90 b/src/shared/horiz_interp/horiz_interp_bicubic.F90
index b2a738f269..7afbbfc315 100644
--- a/src/shared/horiz_interp/horiz_interp_bicubic.F90
+++ b/src/shared/horiz_interp/horiz_interp_bicubic.F90
@@ -41,8 +41,8 @@ module horiz_interp_bicubic_mod
     module procedure horiz_interp_bicubic_new_1d_s
   end interface
 
-   character(len=128) :: version="$Id: horiz_interp_bicubic.F90,v 19.0 2012/01/06 21:57:52 fms Exp $"
-   character(len=128) :: tagname = '$Name: tikal $'
+   character(len=128) :: version="$Id$"
+   character(len=128) :: tagname = '$Name$'
    logical            :: module_is_initialized = .FALSE.
    integer            :: verbose_bicubic = 0
    
@@ -84,7 +84,7 @@ module horiz_interp_bicubic_mod
   subroutine horiz_interp_bicubic_init
 
      if(module_is_initialized) return
-     call write_version_number()
+     call write_version_number (version, tagname)
      module_is_initialized = .true.
      tpi = 2.0*PI
 
@@ -545,13 +545,17 @@ subroutine bcucof(y,y1,y2,y12,d1,d2,c)
       integer i,j,k,l
       real d1d2,xx,cl(16),wt(16,16),x(16)
       save wt
-      data wt/1,0,-3,2,4*0,-3,0,9,-6,2,0,-6,4,8*0,3,0,-9,6,-2,0,6,-4,10* &
-       0,9,-6,2*0,-6,4,2*0,3,-2,6*0,-9,6,2*0,6,-4,4*0,1,0,-3,2,-2,0,6,-4, &
-       1,0,-3,2,8*0,-1,0,3,-2,1,0,-3,2,10*0,-3,2,2*0,3,-2,6*0,3,-2,2*0,   &
-      -6,4,2*0,3,-2,0,1,-2,1,5*0,-3,6,-3,0,2,-4,2,9*0,3,-6,3,0,-2,4,-2,  &
-       10*0,-3,3,2*0,2,-2,2*0,-1,1,6*0,3,-3,2*0,-2,2,5*0,1,-2,1,0,-2,4,   &
-      -2,0,1,-2,1,9*0,-1,2,-1,0,1,-2,1,10*0,1,-1,2*0,-1,1,6*0,-1,1,2*0,  &
-       2,-2,2*0,-1,1/
+      data wt/1., 0., -3., 2., 4*0., -3., 0., 9., -6., 2., 0., -6., 4., 8*0., &
+              3., 0., -9., 6., -2., 0., 6., -4., 10*0., 9., -6., 2*0., -6., &
+              4., 2*0., 3., -2., 6*0., -9., 6., 2*0., 6., -4., 4*0., 1., 0., &
+              -3., 2., -2., 0., 6., -4., 1., 0., -3., 2., 8*0., -1., 0., 3., &
+              -2., 1., 0., -3., 2., 10*0., -3., 2., 2*0., 3., -2., 6*0., 3., &
+              -2., 2*0., -6., 4., 2*0., 3., -2., 0., 1., -2., 1., 5*0., -3., &
+              6., -3., 0., 2., -4., 2., 9*0., 3., -6., 3., 0., -2., 4., -2., &
+              10*0., -3., 3., 2*0., 2., -2., 2*0., -1., 1., 6*0., 3., -3., &
+              2*0., -2., 2., 5*0., 1., -2., 1., 0., -2., 4., -2., 0., 1., -2., &
+              1., 9*0., -1., 2., -1., 0., 1., -2., 1., 10*0., 1., -1., 2*0., &
+              -1., 1., 6*0., -1., 1., 2*0., 2., -2., 2*0., -1., 1./
 
       d1d2=d1*d2
       do i=1,4
@@ -647,7 +651,7 @@ subroutine fill_xy(fi, ics, ice, jcs, jce, mask, maxpass)
                  jnu = min(j+1,jce)
                  do js=jnl,jnu
                    do is=inl,inr
-                     if (work_old(is,js) .ne. blank .and. mask(is,js).ne.0) then
+                     if (work_old(is,js) .ne. blank .and. mask(is,js).ne.0.) then
                        tavr = tavr + work_old(is,js)
                        iavr = iavr+1
                      endif
diff --git a/src/shared/horiz_interp/horiz_interp_bilinear.F90 b/src/shared/horiz_interp/horiz_interp_bilinear.F90
index 006eab302e..a632cde5e5 100644
--- a/src/shared/horiz_interp/horiz_interp_bilinear.F90
+++ b/src/shared/horiz_interp/horiz_interp_bilinear.F90
@@ -1,7 +1,8 @@
 module horiz_interp_bilinear_mod
 
-  !  Zhi Liang 
+  !  Zhi Liang 
 
+  ! 
 
   ! 
   !   Performs spatial interpolation between grids using bilinear interpolation
@@ -38,8 +39,8 @@ module horiz_interp_bilinear_mod
   integer, parameter :: DUMMY = -999
 
   !-----------------------------------------------------------------------
-  character(len=128) :: version = '$Id: horiz_interp_bilinear.F90,v 20.0 2013/12/14 00:20:22 fms Exp $'
-  character(len=128) :: tagname = '$Name: tikal $'
+  character(len=128) :: version = '$Id$'
+  character(len=128) :: tagname = '$Name$'
   logical            :: module_is_initialized = .FALSE.
 
 contains
@@ -56,7 +57,7 @@ module horiz_interp_bilinear_mod
   subroutine horiz_interp_bilinear_init
 
     if(module_is_initialized) return
-    call write_version_number()
+    call write_version_number (version, tagname)
     module_is_initialized = .true.
 
   end subroutine horiz_interp_bilinear_init
@@ -395,7 +396,7 @@ subroutine horiz_interp_bilinear_new_2d ( Interp, lon_in, lat_in, lon_out, lat_o
           a  = b2*c1-b1*c2
           b  = a1*b2-a2*b1+c1*d2-c2*d1+c2*lon-c1*lat
           c  = a2*lon-a1*lat+a1*d2-a2*d1
-          quadra = b*b-4*a*c
+          quadra = b*b-4.*a*c
           if(abs(quadra) < epsln) quadra = 0.0
           if(quadra < 0.0) call mpp_error(FATAL, &
                "horiz_interp_bilinear_mod: No solution existed for this quadratic equation")
@@ -440,7 +441,7 @@ subroutine horiz_interp_bilinear_new_2d ( Interp, lon_in, lat_in, lon_out, lat_o
              if (x > 1.0) x = 1.0
              if (y > 1.0) y = 1.0
            endif 
-           if( x>1 .or. x<0 .or. y>1 .or. y < 0) call mpp_error(FATAL, &
+           if( x>1. .or. x<0. .or. y>1. .or. y < 0.) call mpp_error(FATAL, &
                 "horiz_interp_bilinear_mod: weight should be between 0 and 1")
            Interp % wti(m,n,1)=1.0-x; Interp % wti(m,n,2)=x   
            Interp % wtj(m,n,1)=1.0-y; Interp % wtj(m,n,2)=y          
@@ -957,7 +958,7 @@ end function intersect
   !   
 
   subroutine horiz_interp_bilinear ( Interp, data_in, data_out, verbose, mask_in,mask_out, &
-       missing_value, missing_permit)
+       missing_value, missing_permit, new_handle_missing )
     !-----------------------------------------------------------------------
     type (horiz_interp_type), intent(in)        :: Interp
     real, intent(in),  dimension(:,:)           :: data_in
@@ -967,6 +968,7 @@ subroutine horiz_interp_bilinear ( Interp, data_in, data_out, verbose, mask_in,m
     real, intent(out), dimension(:,:), optional :: mask_out
     real, intent(in),                  optional :: missing_value
     integer, intent(in),               optional :: missing_permit
+    logical, intent(in),               optional :: new_handle_missing
     !-----------------------------------------------------------------------
     integer :: nlon_in, nlat_in, nlon_out, nlat_out, n, m,         &
          is, ie, js, je, iverbose, max_missing, num_missing, &
@@ -974,6 +976,8 @@ subroutine horiz_interp_bilinear ( Interp, data_in, data_out, verbose, mask_in,m
     real    :: dwtsum, wtsum, min_in, max_in, avg_in, &
          min_out, max_out, avg_out, wtw, wte, wts, wtn
     real    :: mask(size(data_in,1), size(data_in,2) )
+    logical :: set_to_missing, is_missing(4), new_handler
+    real    :: f1, f2, f3, f4, middle, w, s
 
     num_missing = 0
 
@@ -998,6 +1002,12 @@ subroutine horiz_interp_bilinear ( Interp, data_in, data_out, verbose, mask_in,m
        max_missing = 0
     endif
 
+    if(present(new_handle_missing)) then
+       new_handler = new_handle_missing
+    else
+       new_handler = .false.
+    endif
+
     if(max_missing .gt. 3 .or. max_missing .lt. 0) call mpp_error(FATAL, &
          'horiz_interp_bilinear_mod: missing_permit should be between 0 and 3')
 
@@ -1007,60 +1017,217 @@ subroutine horiz_interp_bilinear ( Interp, data_in, data_out, verbose, mask_in,m
     if (size(data_out,1) /= nlon_out .or. size(data_out,2) /= nlat_out) &
          call mpp_error(FATAL,'horiz_interp_bilinear_mod: size of output array incorrect')
 
-    do n = 1, nlat_out
-       do m = 1, nlon_out
-          is = Interp % i_lon (m,n,1); ie = Interp % i_lon (m,n,2)
-          js = Interp % j_lat (m,n,1); je = Interp % j_lat (m,n,2)
-          wtw = Interp % wti   (m,n,1)
-          wte = Interp % wti   (m,n,2)
-          wts = Interp % wtj   (m,n,1)
-          wtn = Interp % wtj   (m,n,2)
-
-          if(present(missing_value) ) then
+    if(new_handler) then
+       if( .not. present(missing_value) )  call mpp_error(FATAL, &
+            "horiz_interp_bilinear_mod: misisng_value must be present when new_handle_missing is .true.")
+       if( present(mask_in) ) call mpp_error(FATAL, &
+            "horiz_interp_bilinear_mod: mask_in should not be present when new_handle_missing is .true.")
+       do n = 1, nlat_out
+          do m = 1, nlon_out
+             is = Interp % i_lon (m,n,1); ie = Interp % i_lon (m,n,2)
+             js = Interp % j_lat (m,n,1); je = Interp % j_lat (m,n,2)
+             wtw = Interp % wti   (m,n,1)
+             wte = Interp % wti   (m,n,2)
+             wts = Interp % wtj   (m,n,1)
+             wtn = Interp % wtj   (m,n,2)
+
+             is_missing = .false.
              num_missing = 0
+             set_to_missing = .false.
              if(data_in(is,js) == missing_value) then
                 num_missing = num_missing+1
-                mask(is,js) = 0.0
+                is_missing(1) = .true.
+                if(wtw .GE. 0.5 .AND. wts .GE. 0.5) set_to_missing = .true.
              endif
+
              if(data_in(ie,js) == missing_value) then
                 num_missing = num_missing+1
-                mask(ie,js) = 0.0
+                is_missing(2) = .true.
+                if(wte .GE. 0.5 .AND. wts .GE. 0.5) set_to_missing = .true.
              endif
              if(data_in(ie,je) == missing_value) then
                 num_missing = num_missing+1
-                mask(ie,je) = 0.0
+                is_missing(3) = .true.
+                if(wte .GE. 0.5 .AND. wtn .GE. 0.5) set_to_missing = .true.
              endif
              if(data_in(is,je) == missing_value) then
                 num_missing = num_missing+1
-                mask(is,je) = 0.0
+                is_missing(4) = .true.
+                if(wtw .GE. 0.5 .AND. wtn .GE. 0.5) set_to_missing = .true.
              endif
-          endif
 
-          dwtsum = data_in(is,js)*mask(is,js)*wtw*wts &
-               + data_in(ie,js)*mask(ie,js)*wte*wts &
-               + data_in(ie,je)*mask(ie,je)*wte*wtn &
-               + data_in(is,je)*mask(is,je)*wtw*wtn 
-          wtsum  = mask(is,js)*wtw*wts + mask(ie,js)*wte*wts  &
-               + mask(ie,je)*wte*wtn + mask(is,je)*wtw*wtn
+             if( num_missing == 4 .OR. set_to_missing ) then
+                data_out(m,n) = missing_value
+                if(present(mask_out)) mask_out(m,n) = 0.0
+                 cycle
+             else if(num_missing == 0) then
+                f1 = data_in(is,js)
+                f2 = data_in(ie,js)
+                f3 = data_in(ie,je)
+                f4 = data_in(is,je)
+                w = wtw
+                s = wts
+             else if(num_missing == 3) then  !--- three missing value 
+                if(.not. is_missing(1) ) then
+                   data_out(m,n) = data_in(is,js)
+                else if(.not. is_missing(2) ) then
+                   data_out(m,n) = data_in(ie,js)
+                else if(.not. is_missing(3) ) then
+                   data_out(m,n) = data_in(ie,je)
+                else if(.not. is_missing(4) ) then
+                   data_out(m,n) = data_in(is,je)
+                endif
+                if(present(mask_out) ) mask_out(m,n) = 1.0
+                cycle
+             else   !--- one or two missing value
+                if( num_missing == 1) then
+                   if( is_missing(1) .OR. is_missing(3) ) then
+                      middle = 0.5*(data_in(ie,js)+data_in(is,je))
+                   else
+                      middle = 0.5*(data_in(is,js)+data_in(ie,je))
+                   endif
+                else ! num_missing = 2
+                   if( is_missing(1) .AND. is_missing(2) ) then
+                      middle = 0.5*(data_in(ie,je)+data_in(is,je))
+                   else if( is_missing(1) .AND. is_missing(3) ) then
+                      middle = 0.5*(data_in(ie,js)+data_in(is,je))
+                   else if( is_missing(1) .AND. is_missing(4) ) then
+                      middle = 0.5*(data_in(ie,js)+data_in(ie,je))
+                   else if( is_missing(2) .AND. is_missing(3) ) then
+                      middle = 0.5*(data_in(is,js)+data_in(is,je))
+                   else if( is_missing(2) .AND. is_missing(4) ) then
+                      middle = 0.5*(data_in(is,js)+data_in(ie,je))
+                   else if( is_missing(3) .AND. is_missing(4) ) then
+                      middle = 0.5*(data_in(is,js)+data_in(ie,js))
+                   endif
+                endif
 
-          if(.not. present(mask_in) .and. .not. present(missing_value)) wtsum = 1.0
+                if( wtw .GE. 0.5 .AND. wts .GE. 0.5 ) then  ! zone 1
+                   w = 2.0*(wtw-0.5)
+                   s = 2.0*(wts-0.5)
+                   f1 = data_in(is,js)
+                   if(is_missing(2)) then
+                      f2 = f1
+                   else
+                      f2 = 0.5*(data_in(is,js)+data_in(ie,js))
+                   endif
+                   f3 = middle
+                   if(is_missing(4)) then
+                      f4 = f1
+                   else
+                      f4 = 0.5*(data_in(is,js)+data_in(is,je))
+                   endif
+                else if( wte .GE. 0.5 .AND. wts .GE. 0.5 ) then  ! zone 2
+                   w = 2.0*(1.0-wte)
+                   s = 2.0*(wts-0.5)
+                   f2 = data_in(ie,js)
+                   if(is_missing(1)) then
+                      f1 = f2
+                   else
+                      f1 = 0.5*(data_in(is,js)+data_in(ie,js))
+                   endif
+                   f4 = middle
+                   if(is_missing(3)) then
+                      f3 = f2
+                   else
+                      f3 = 0.5*(data_in(ie,js)+data_in(ie,je))
+                   endif
+                else if( wte .GE. 0.5 .AND. wtn .GE. 0.5 ) then  ! zone 3
+                   w = 2.0*(1.0-wte)
+                   s = 2.0*(1.0-wtn)
+                   f3 = data_in(ie,je)
+                   if(is_missing(2)) then
+                      f2 = f3
+                   else
+                      f2 = 0.5*(data_in(ie,js)+data_in(ie,je))
+                   endif
+                   f1 = middle
+                   if(is_missing(4)) then
+                      f4 = f3
+                   else
+                      f4 = 0.5*(data_in(ie,je)+data_in(is,je))
+                   endif
+                else if( wtw .GE. 0.5 .AND. wtn .GE. 0.5 ) then  ! zone 4
+                   w = 2.0*(wtw-0.5)
+                   s = 2.0*(1.0-wtn)
+                   f4 = data_in(is,je)
+                   if(is_missing(1)) then
+                      f1 = f4
+                   else
+                      f1 = 0.5*(data_in(is,js)+data_in(is,je))
+                   endif
+                   f2 = middle
+                   if(is_missing(3)) then
+                      f3 = f4
+                   else
+                      f3 = 0.5*(data_in(ie,je)+data_in(is,je))
+                   endif
+                else 
+                   call mpp_error(FATAL, &
+                      "horiz_interp_bilinear_mod: the point should be in one of the four zone")
+                endif
+             endif
+
+             data_out(m,n) = f3 + (f4-f3)*w + (f2-f3)*s + ((f1-f2)+(f3-f4))*w*s 
+            if(present(mask_out)) mask_out(m,n) = 1.0
+          enddo
+       enddo       
+    else
+       do n = 1, nlat_out
+          do m = 1, nlon_out
+             is = Interp % i_lon (m,n,1); ie = Interp % i_lon (m,n,2)
+             js = Interp % j_lat (m,n,1); je = Interp % j_lat (m,n,2)
+             wtw = Interp % wti   (m,n,1)
+             wte = Interp % wti   (m,n,2)
+             wts = Interp % wtj   (m,n,1)
+             wtn = Interp % wtj   (m,n,2)
+
+             if(present(missing_value) ) then
+                num_missing = 0
+                if(data_in(is,js) == missing_value) then
+                   num_missing = num_missing+1
+                   mask(is,js) = 0.0
+                endif
+                if(data_in(ie,js) == missing_value) then
+                   num_missing = num_missing+1
+                   mask(ie,js) = 0.0
+                endif
+                if(data_in(ie,je) == missing_value) then
+                   num_missing = num_missing+1
+                   mask(ie,je) = 0.0
+                endif
+                if(data_in(is,je) == missing_value) then
+                   num_missing = num_missing+1
+                   mask(is,je) = 0.0
+                endif
+             endif
+
+             dwtsum = data_in(is,js)*mask(is,js)*wtw*wts &
+                  + data_in(ie,js)*mask(ie,js)*wte*wts &
+                  + data_in(ie,je)*mask(ie,je)*wte*wtn &
+                  + data_in(is,je)*mask(is,je)*wtw*wtn 
+             wtsum  = mask(is,js)*wtw*wts + mask(ie,js)*wte*wts  &
+                  + mask(ie,je)*wte*wtn + mask(is,je)*wtw*wtn
+
+             if(.not. present(mask_in) .and. .not. present(missing_value)) wtsum = 1.0
 
-          if(num_missing .gt. max_missing ) then
-             data_out(m,n) = missing_value
-             if(present(mask_out)) mask_out(m,n) = 0.0
-          else if(wtsum .lt. epsln) then 
-             if(present(missing_value)) then
+             if(num_missing .gt. max_missing ) then
                 data_out(m,n) = missing_value
+                if(present(mask_out)) mask_out(m,n) = 0.0
+             else if(wtsum .lt. epsln) then 
+                if(present(missing_value)) then
+                   data_out(m,n) = missing_value
+                else
+                   data_out(m,n) = 0.0
+                endif
+                if(present(mask_out)) mask_out(m,n) = 0.0      
              else
-                data_out(m,n) = 0.0
+                data_out(m,n) = dwtsum/wtsum
+                if(present(mask_out)) mask_out(m,n) = wtsum
              endif
-             if(present(mask_out)) mask_out(m,n) = 0.0      
-          else
-             data_out(m,n) = dwtsum/wtsum
-             if(present(mask_out)) mask_out(m,n) = wtsum
-          endif
+          enddo
        enddo
-    enddo
+    endif
     !***********************************************************************
     ! compute statistics: minimum, maximum, and mean
     !-----------------------------------------------------------------------
diff --git a/src/shared/horiz_interp/horiz_interp_conserve.F90 b/src/shared/horiz_interp/horiz_interp_conserve.F90
index c705798499..a58d372cac 100644
--- a/src/shared/horiz_interp/horiz_interp_conserve.F90
+++ b/src/shared/horiz_interp/horiz_interp_conserve.F90
@@ -1,8 +1,9 @@
 module horiz_interp_conserve_mod
 
-  !  Bruce Wyman 
-  !  Zhi Liang 
+  !  Bruce Wyman 
+  !  Zhi Liang 
 
+  ! 
 
   ! 
   !   Performs spatial interpolation between grids using conservative interpolation
@@ -90,8 +91,8 @@ module horiz_interp_conserve_mod
 
   integer :: pe, root_pe
   !-----------------------------------------------------------------------
-  character(len=128) :: version = '$Id: horiz_interp_conserve.F90,v 20.0 2013/12/14 00:20:25 fms Exp $'
-  character(len=128) :: tagname = '$Name: tikal $'
+  character(len=128) :: version = '$Id$'
+  character(len=128) :: tagname = '$Name$'
   logical            :: module_is_initialized = .FALSE.
 
   logical         :: great_circle_algorithm = .false.
@@ -110,7 +111,7 @@ module horiz_interp_conserve_mod
   subroutine horiz_interp_conserve_init
 
     if(module_is_initialized) return
-    call write_version_number()
+    call write_version_number (version, tagname)
 
     great_circle_algorithm = get_great_circle_algorithm()
 
@@ -350,7 +351,7 @@ subroutine horiz_interp_conserve_new_1dx2d ( Interp, lon_in, lat_in, lon_out, la
     nlon_in  = size(lon_in(:)) - 1;  nlat_in  = size(lat_in(:)) - 1
     nlon_out = size(lon_out,1) - 1;  nlat_out = size(lon_out,2) - 1
 
-    mask_src = 1
+    mask_src = 1.
     if(present(mask_in)) then
        if( (size(mask_in,1) .NE. nlon_in) .OR.  (size(mask_in,2) .NE. nlat_in)) call mpp_error(FATAL, &
          'horiz_interp_conserve_mod: size mismatch between mask_in and lon_in/lat_in')
@@ -388,7 +389,7 @@ subroutine horiz_interp_conserve_new_1dx2d ( Interp, lon_in, lat_in, lon_out, la
     Interp%j_dst = j_dst(1:nxgrid)+1
 
     ! sum over exchange grid area to get destination grid area
-    dst_area = 0
+    dst_area = 0.
     do i = 1, nxgrid
        dst_area(Interp%i_dst(i), Interp%j_dst(i)) = dst_area(Interp%i_dst(i), Interp%j_dst(i)) + xgrid_area(i)       
     end do    
@@ -440,7 +441,7 @@ subroutine horiz_interp_conserve_new_2dx1d ( Interp, lon_in, lat_in, lon_out, la
     nlon_in  = size(lon_in,1)   - 1;  nlat_in  = size(lon_in,2)   - 1
     nlon_out = size(lon_out(:)) - 1;  nlat_out = size(lat_out(:)) - 1
 
-    mask_src = 1
+    mask_src = 1.
     if(present(mask_in)) then
        if( (size(mask_in,1) .NE. nlon_in) .OR.  (size(mask_in,2) .NE. nlat_in)) call mpp_error(FATAL, &
          'horiz_interp_conserve_mod: size mismatch between mask_in and lon_in/lat_in')
@@ -478,7 +479,7 @@ subroutine horiz_interp_conserve_new_2dx1d ( Interp, lon_in, lat_in, lon_out, la
     Interp%j_dst = j_dst(1:nxgrid)+1
 
     ! sum over exchange grid area to get destination grid area
-    dst_area = 0
+    dst_area = 0.
     do i = 1, nxgrid
        dst_area(Interp%i_dst(i), Interp%j_dst(i)) = dst_area(Interp%i_dst(i), Interp%j_dst(i)) + xgrid_area(i)       
     end do    
@@ -531,7 +532,7 @@ subroutine horiz_interp_conserve_new_2dx2d ( Interp, lon_in, lat_in, lon_out, la
     nlon_in  = size(lon_in,1)  - 1;  nlat_in  = size(lon_in,2)  - 1
     nlon_out = size(lon_out,1) - 1;  nlat_out = size(lon_out,2) - 1
 
-    mask_src = 1
+    mask_src = 1.
     if(present(mask_in)) then
        if( (size(mask_in,1) .NE. nlon_in) .OR.  (size(mask_in,2) .NE. nlat_in)) call mpp_error(FATAL, &
          'horiz_interp_conserve_mod: size mismatch between mask_in and lon_in/lat_in')
@@ -563,7 +564,7 @@ subroutine horiz_interp_conserve_new_2dx2d ( Interp, lon_in, lat_in, lon_out, la
     Interp%j_dst = j_dst(1:nxgrid)+1
 
     ! sum over exchange grid area to get destination grid area
-    dst_area = 0
+    dst_area = 0.
     do i = 1, nxgrid
        dst_area(Interp%i_dst(i), Interp%j_dst(i)) = dst_area(Interp%i_dst(i), Interp%j_dst(i)) + xgrid_area(i)       
     end do    
diff --git a/src/shared/horiz_interp/horiz_interp_spherical.F90 b/src/shared/horiz_interp/horiz_interp_spherical.F90
index 63e62a54e3..00b3c26491 100644
--- a/src/shared/horiz_interp/horiz_interp_spherical.F90
+++ b/src/shared/horiz_interp/horiz_interp_spherical.F90
@@ -1,8 +1,9 @@
 module horiz_interp_spherical_mod
 
-  !  Matthew Harrison 
-  !  Zhi Liang 
+  !  Matthew Harrison 
+  !  Zhi Liang 
 
+  ! 
 
   ! 
   !   Performs spatial interpolation between grids using inverse-distance-weighted scheme.
@@ -59,8 +60,8 @@ module horiz_interp_spherical_mod
   namelist /horiz_interp_spherical_nml/ search_method
 
   !-----------------------------------------------------------------------
-  character(len=128) :: version = '$Id: horiz_interp_spherical.F90,v 20.0 2013/12/14 00:20:28 fms Exp $'
-  character(len=128) :: tagname = '$Name: tikal $'
+  character(len=128) :: version = '$Id$'
+  character(len=128) :: tagname = '$Name$'
   logical            :: module_is_initialized = .FALSE.
 
 contains
@@ -79,7 +80,7 @@ subroutine horiz_interp_spherical_init
 
 
     if(module_is_initialized) return
-    call write_version_number()
+    call write_version_number (version, tagname)
 #ifdef INTERNAL_FILE_NML
       read (input_nml_file, horiz_interp_spherical_nml, iostat=io)
       ierr = check_nml_error(io,'horiz_interp_spherical_nml') 
@@ -171,10 +172,10 @@ subroutine horiz_interp_spherical_new(Interp, lon_in,lat_in,lon_out,lat_out, &
     logical :: src_is_modulo
     real    :: min_theta_dst, max_theta_dst, min_phi_dst, max_phi_dst
     real    :: min_theta_src, max_theta_src, min_phi_src, max_phi_src 
-    integer, dimension(:),        allocatable        :: ilon, jlat
-    integer, dimension(:,:,:),    allocatable        :: map_src_add
-    integer, dimension(:,:),      allocatable        :: num_found
-    real, dimension(:,:,:),       allocatable        :: map_src_dist
+    integer :: map_src_add(size(lon_out,1),size(lon_out,2),max_neighbors)
+    real    :: map_src_dist(size(lon_out,1),size(lon_out,2),max_neighbors)
+    integer :: num_found(size(lon_out,1),size(lon_out,2))
+    integer :: ilon(max_neighbors), jlat(max_neighbors)
     real, dimension(size(lon_out,1),size(lon_out,2)) :: theta_dst, phi_dst
     real, dimension(size(lon_in,1)*size(lon_in,2))   :: theta_src, phi_src
 
@@ -247,16 +248,19 @@ subroutine horiz_interp_spherical_new(Interp, lon_in,lat_in,lon_out,lat_out, &
        if (min_theta_dst < min_theta_src) print *, '=> WARNING : longitude of dest grid exceeds src'
        if (max_theta_dst > max_theta_src) print *, '=> WARNING : longitude of dest grid exceeds src'
     endif
-    allocate(map_src_add(map_dst_xsize,map_dst_ysize,max_neighbors),    &
-         map_src_dist(map_dst_xsize,map_dst_ysize,max_neighbors),   &
-         num_found(map_dst_xsize,map_dst_ysize),                    &
-         ilon(max_neighbors),jlat(max_neighbors)  )
 
     ! allocate memory to data type
-    allocate(Interp%i_lon(map_dst_xsize,map_dst_ysize,max_neighbors), &
-         Interp%j_lat(map_dst_xsize,map_dst_ysize,max_neighbors),     &
-         Interp%src_dist(map_dst_xsize,map_dst_ysize,max_neighbors),  &
-         Interp%num_found(map_dst_xsize,map_dst_ysize)       )
+    if(ASSOCIATED(Interp%i_lon)) then
+       if(size(Interp%i_lon,1) .NE. map_dst_xsize .OR.     &
+          size(Interp%i_lon,2) .NE. map_dst_ysize )  call mpp_error(FATAL, &
+          'horiz_interp_spherical_mod: size(Interp%i_lon(:),1) .NE. map_dst_xsize .OR. '// &
+          'size(Interp%i_lon(:),2) .NE. map_dst_ysize')
+    else
+       allocate(Interp%i_lon(map_dst_xsize,map_dst_ysize,max_neighbors), &
+            Interp%j_lat(map_dst_xsize,map_dst_ysize,max_neighbors),     &
+            Interp%src_dist(map_dst_xsize,map_dst_ysize,max_neighbors),  &
+            Interp%num_found(map_dst_xsize,map_dst_ysize)       )
+    endif
 
     map_src_add         = 0
     map_src_dist        = large
@@ -300,7 +304,6 @@ subroutine horiz_interp_spherical_new(Interp, lon_in,lat_in,lon_out,lat_out, &
     Interp%nlon_src = map_src_xsize; Interp%nlat_src = map_src_ysize
     Interp%nlon_dst = map_dst_xsize; Interp%nlat_dst = map_dst_ysize
 
-    deallocate(map_src_add, map_src_dist, ilon, jlat)
     return
 
   end subroutine horiz_interp_spherical_new
@@ -944,8 +947,8 @@ function spherical_distance(theta1,phi1,theta2,phi2)
     endif
   
     dot = cos(phi1)*cos(phi2)*cos(theta1-theta2) + sin(phi1)*sin(phi2)
-    if(dot > 1 ) dot = 1.
-    if(dot < -1) dot = -1
+    if(dot > 1. ) dot = 1.
+    if(dot < -1.) dot = -1.
     spherical_distance = acos(dot)
 
     return
diff --git a/src/shared/horiz_interp/horiz_interp_type.F90 b/src/shared/horiz_interp/horiz_interp_type.F90
index 5f8080e4d0..7f2a41dbf9 100644
--- a/src/shared/horiz_interp/horiz_interp_type.F90
+++ b/src/shared/horiz_interp/horiz_interp_type.F90
@@ -23,8 +23,9 @@
 !!                                                                   !!
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 module horiz_interp_type_mod
-!  Zhi Liang 
+!  Zhi Liang 
 
+! 
 
 ! 
 !     define derived data type that contains indices and weights used for subsequent 
@@ -146,7 +147,7 @@ subroutine stats ( dat, low, high, avg, miss, missing_value, mask )
          miss = miss + buffer_int(1)
          npts = npts + buffer_int(2)
       enddo         
-      if(npts == 0) then
+      if(npts == 0.) then
          print*, 'Warning: no points is valid'
       else
          avg = dsum/real(npts)
diff --git a/src/shared/horiz_interp/test_horiz_interp.F90 b/src/shared/horiz_interp/test_horiz_interp.F90
new file mode 100644
index 0000000000..9efa898ed4
--- /dev/null
+++ b/src/shared/horiz_interp/test_horiz_interp.F90
@@ -0,0 +1,251 @@
+#ifdef TEST_HORIZ_INTERP
+!z1l: currently only test bilinear interpolation.
+
+program test
+
+  use mpp_mod,          only : mpp_error, mpp_pe,  mpp_npes, mpp_root_pe
+  use mpp_mod,          only : FATAL, stdout, stdlog
+  use mpp_io_mod,       only : mpp_open, mpp_close, mpp_read
+  use mpp_io_mod,       only : axistype, fieldtype
+  use mpp_io_mod,       only : mpp_get_info, mpp_get_fields, mpp_get_times
+  use mpp_io_mod,       only : mpp_get_axes, mpp_get_axis_data, mpp_get_atts
+  use mpp_io_mod,       only : MPP_RDONLY, MPP_NETCDF, MPP_MULTI, MPP_SINGLE, MPP_OVERWR
+  use mpp_io_mod,       only : mpp_get_att_name, mpp_get_att_char, mpp_get_att_type, mpp_get_att_real
+  use mpp_io_mod,       only : mpp_write_meta, axistype, fieldtype, mpp_write, mpp_close 
+  use mpp_domains_mod,  only : mpp_update_domains, mpp_define_domains, domain1d
+  use mpp_domains_mod,  only : domain2d, mpp_define_layout, mpp_get_compute_domain
+  use mpp_domains_mod,  only : mpp_get_domain_components
+  use horiz_interp_mod, only : horiz_interp_new, horiz_interp, horiz_interp_end, horiz_interp_type
+  use horiz_interp_mod, only : horiz_interp_init
+  use axis_utils_mod,   only : get_axis_cart
+  use fms_io_mod,       only : read_data, write_data
+  use fms_io_mod,       only : field_size, fms_io_exit
+  use fms_mod,          only : fms_init, fms_end, open_namelist_file, close_file, file_exist, field_exist
+  use fms_mod,          only : check_nml_error, write_version_number, lowercase
+  use constants_mod,    only : constants_init, PI
+  use horiz_interp_mod, only : horiz_interp_new, horiz_interp, horiz_interp_end, horiz_interp_type
+
+implicit none
+
+  character(len=256) :: src_file = ""
+  character(len=256) :: dst_grid = "INPUT/grid_spec.nc"
+  character(len=256) :: field_name = ""
+  character(len=256) :: dst_file = "output.nc"
+  logical            :: new_missing_handle = .false.
+  character(len=256) :: interp_method = "bilinear"
+
+  real, allocatable, dimension(:)            :: x_src, y_src
+  real, allocatable, dimension(:,:)          :: x_dst, y_dst
+  type(axistype), allocatable, dimension(:)  :: axes
+  type(fieldtype), allocatable, dimension(:) :: fields
+  type(domain2d)                             :: Domain
+  integer :: unit, ierr, io, src_unit, src_field_index, nk
+  integer :: nx_src, ny_src, nx_dst, ny_dst, is, ie, js, je
+  real    :: missing_value
+  integer :: n, ntimes
+
+  namelist /test_horiz_interp_nml/ src_file, field_name, dst_file, dst_grid, new_missing_handle, &
+           interp_method
+
+  call fms_init
+  call horiz_interp_init
+  call constants_init
+
+#ifdef INTERNAL_FILE_NML
+  read (input_nml_file, test_horiz_interp_nml, iostat=io)
+  ierr = check_nml_error(io, 'test_horiz_interp_nml')
+#else
+  if (file_exist('input.nml')) then
+     unit = open_namelist_file ( )
+     ierr=1
+     do while (ierr /= 0)
+        read(unit, nml=test_horiz_interp_nml, iostat=io, end=10)
+        ierr = check_nml_error(io, 'test_horiz_interp_nml')
+     enddo
+10   call close_file (unit)
+  endif
+#endif
+
+
+  if( .not. file_exist(src_file) ) call mpp_error(FATAL, &
+       "test_horiz_interp: src_file = "//trim(src_file)//" does not exist")
+  if( .not. field_exist(src_file, field_name) ) call mpp_error(FATAL, &
+       "test_horiz_interp: field_name = "//trim(field_name)//" does not exist in file "//trim(src_file) )
+
+
+  ! reading the grid information and missing value from src_file
+  call mpp_open(src_unit, trim(src_file), &
+         action=MPP_RDONLY, form=MPP_NETCDF, threading=MPP_MULTI, fileset=MPP_SINGLE)
+  call read_src_file()
+
+  ! reading the dst_grid file
+  call read_dst_grid()
+
+  !--- currently only test for first time level. The following will read the input data,
+  !--- do the remapping and write out data
+  call process_data()
+  
+  call mpp_close(src_unit)
+
+  call fms_io_exit
+
+  call fms_end()
+
+contains
+
+
+  ! read the input data, do the remapping and write out data
+  subroutine process_data()
+     real, allocatable, dimension(:,:,:) :: src_data, dst_data
+     type(horiz_interp_type)             :: Interp
+     type(axistype)     :: xaxis, yaxis, zaxis, taxis
+     type(domain1D)     :: xdom, ydom
+     type(fieldtype)    :: field
+     real    :: D2R
+     integer :: k, i
+
+     call mpp_get_domain_components( domain, xdom, ydom )
+     !--- set up meta data
+     call mpp_open(unit,dst_file,action=MPP_OVERWR,form=MPP_NETCDF,threading=MPP_MULTI, fileset=MPP_MULTI)
+     call mpp_write_meta( unit, xaxis, 'lon', 'none', 'X index', 'X', domain=xdom, data=(/(i-1.,i=1,nx_dst)/) )
+     call mpp_write_meta( unit, yaxis, 'lat', 'none', 'Y index', 'Y', domain=ydom, data=(/(i-1.,i=1,ny_dst)/) )
+     call mpp_write_meta( unit, zaxis, 'level', 'none', 'Z index', 'Z', data=(/(i-1.,i=1,nk)/) )
+     call mpp_write_meta( unit, taxis, 'time', 'none', 'T index', 'T' )
+     call mpp_write_meta( unit, field, (/xaxis, yaxis, zaxis, taxis/), field_name, 'none', 'none')
+     call mpp_write( unit, xaxis )
+     call mpp_write( unit, yaxis )
+     call mpp_write( unit, zaxis )
+
+
+     D2R = PI/180.
+
+     allocate(src_data(nx_src,ny_src,nk))
+     allocate(dst_data(is:ie,js:je,nk))
+
+     call horiz_interp_new(Interp, x_src*D2R, y_src*D2R, x_dst*D2R, y_dst*D2R, &
+          interp_method = trim(interp_method), grid_at_center = .true. )    
+
+     do n = 1, ntimes
+        call mpp_read(src_unit,fields(src_field_index),src_data, tindex=n)
+
+        do k = 1, nk
+           call horiz_interp(interp, src_data(:,:,k), dst_data(:,:,k), &
+                missing_value=missing_value, new_missing_handle=new_missing_handle)
+        enddo
+        call mpp_write(unit, field, domain, dst_data, n*1.0)
+     enddo
+
+     call mpp_close(unit)
+     deallocate(src_data, dst_data)
+  
+  end subroutine process_data
+
+
+  subroutine read_dst_grid
+    integer :: layout(2), start(4), nread(4)
+    character(len=256) :: tile_file
+    integer :: i, j, siz(4)
+    real, allocatable :: tmp(:,:)
+
+    if (field_exist(dst_grid, "gridfiles" )) then
+       call read_data(dst_grid, "gridfiles", tile_file)
+       tile_file = 'INPUT/'//trim(tile_file)
+    else
+       call mpp_error(FATAL, "test_horiz_interp: field gridfiles does not exist in file "//trim(dst_grid) )
+    endif
+
+    call field_size(tile_file, "x", siz)
+    nx_dst = (siz(1)-1)/2
+    ny_dst = (siz(2)-1)/2
+
+    if(layout(1)*layout(2) .NE. mpp_npes() ) then
+       call mpp_define_layout( (/1,nx_dst,1,ny_dst/), mpp_npes(), layout )
+    end if
+
+    call mpp_define_domains( (/1,nx_dst,1,ny_dst/), layout, Domain, name='test_data_override')
+
+    call mpp_get_compute_domain(Domain, is, ie, js, je)
+
+    allocate(tmp(2*is-1:2*ie+1,2*js-1:2*je+1))
+    allocate(x_dst(is:ie,js:je), y_dst(is:ie,js:je))
+ 
+    start = 1; nread = 1
+    start(1) = 2*is-1; nread(1) = 2*(ie-is+1)+1
+    start(2) = 2*js-1; nread(2) = 2*(je-js+1)+1
+
+    call read_data(tile_file, "x", tmp, start, nread, domain) 
+    do j = js, je
+       do i = is, ie
+          x_dst(i,j) = tmp(2*i,2*j)
+       enddo
+    enddo
+    call read_data(tile_file, "y", tmp, start, nread, domain)   
+    do j = js, je
+       do i = is, ie
+          y_dst(i,j) = tmp(2*i,2*j)
+       enddo
+    enddo
+
+    deallocate(tmp)
+
+  end subroutine read_dst_grid
+
+  subroutine read_src_file
+
+    integer                                    :: ndim, nvar, natt, n
+    integer                                    :: nt, i, j, k, jj, len1
+    character(len=1)                           :: cart
+    character(len=32)                          :: name
+
+    call mpp_get_info(src_unit, ndim, nvar, natt, ntimes)
+    
+    allocate(fields(nvar))
+    call mpp_get_fields(src_unit, fields)
+    src_field_index = 0
+    do i=1,nvar
+       call mpp_get_atts(fields(i),name=name)
+       if (lowercase(trim(field_name)) == lowercase(trim(name))) then
+             src_field_index = i
+       endif
+    enddo
+    if(src_field_index == 0) call mpp_error(FATAL, 'test_horiz_interp: field '&
+            //trim(field_name)//' is not in the file '//trim(src_file) )
+    !--- get the src grid
+    call mpp_get_atts(fields(src_field_index),ndim=ndim)
+    allocate(axes(ndim))
+    call mpp_get_atts(fields(src_field_index),axes=axes)
+    nx_src=0; ny_src=0; nk=1
+    do j=1,ndim
+       call mpp_get_atts(axes(j),len=len1)
+       call get_axis_cart(axes(j),cart)
+       select case (cart)
+       case ('X')
+          nx_src = len1
+          allocate(x_src(nx_src))
+          call mpp_get_axis_data(axes(j),x_src)
+       case('Y')
+          ny_src = len1
+          allocate(y_src(ny_src))
+          call mpp_get_axis_data(axes(j),y_src)
+       case('Z')
+          nk = len1
+       end select
+    enddo
+    if(nx_src==0) call mpp_error(FATAL,'test_horiz_interp: file ' &
+         //trim(src_file)//' does not contain axis with cartesian attributes = "X" ')
+    if(ny_src==0) call mpp_error(FATAL,'test_horiz_interp: file '&
+         //trim(src_file)//' does not contain axis with cartesian attributes = "Y" ')
+
+    !--- get the missing value
+    call mpp_get_atts(fields(src_field_index),missing=missing_value)
+
+  end subroutine read_src_file
+
+
+end program test
+
+#else
+module null_test_horiz_interp
+end module  
+
+#endif 
diff --git a/src/shared/memutils/memutils.F90 b/src/shared/memutils/memutils.F90
index 1f8ab610bb..80771b9f56 100644
--- a/src/shared/memutils/memutils.F90
+++ b/src/shared/memutils/memutils.F90
@@ -1,5 +1,5 @@
 module memutils_mod
-!Author: Balaji (V.Balaji)
+!Author: Balaji (V.Balaji@noaa.gov)
 !Various operations for memory management
 !these currently include efficient methods for memory-to-memory copy
 !including strided data and arbitrary gather-scatter vectors
diff --git a/src/shared/mosaic/gradient.F90 b/src/shared/mosaic/gradient.F90
index 446a089d41..d41af3035f 100644
--- a/src/shared/mosaic/gradient.F90
+++ b/src/shared/mosaic/gradient.F90
@@ -1,8 +1,9 @@
 module gradient_mod
-! 
+! 
 !   Zhi Liang
 ! 
 
+! 
 
 ! 
 !    gradient_mod implements some utility routines to calculate gradient.
@@ -23,8 +24,8 @@ module gradient_mod
 public :: gradient_cubic
 public :: calc_cubic_grid_info
 
-character(len=128) :: version = '$Id: gradient.F90,v 16.0 2008/07/30 22:46:00 fms Exp $'
-character(len=128) :: tagname = '$Name: tikal $'
+character(len=128) :: version = '$Id$'
+character(len=128) :: tagname = '$Name$'
 
 contains
 
diff --git a/src/shared/mosaic/grid.F90 b/src/shared/mosaic/grid.F90
index d9b604b6fa..262efd14a4 100644
--- a/src/shared/mosaic/grid.F90
+++ b/src/shared/mosaic/grid.F90
@@ -48,8 +48,8 @@ module grid_mod
 ! ==== module constants ======================================================
 character(len=*), parameter :: &
      module_name = 'grid_mod', &
-     version     = '$Id: grid.F90,v 20.0 2013/12/14 00:20:48 fms Exp $', &
-     tagname     = '$Name: tikal $'
+     version     = '$Id$', &
+     tagname     = '$Name$'
 
 character(len=*), parameter :: &
      grid_dir  = 'INPUT/',     &      ! root directory for all grid files
@@ -62,6 +62,9 @@ module grid_mod
      VERSION_1 = 1,   &
      VERSION_2 = 2
 
+integer, parameter :: BUFSIZE = 1048576  ! This is used to control memory usage in get_grid_comp_area
+                                         ! We may change this to a namelist variable is needed.
+
 ! ==== module variables ======================================================
 integer :: grid_version = -1
 logical :: great_circle_algorithm = .FALSE.
@@ -194,7 +197,7 @@ subroutine get_grid_cell_area(component, tile, cellarea, domain)
              FATAL)
      end select
      ! convert area to m2
-     cellarea = cellarea*4*PI*radius**2
+     cellarea = cellarea*4.*PI*radius**2
   case(VERSION_2)
      if (present(domain)) then
         call mpp_get_compute_domain(domain,xsize=nlon,ysize=nlat)
@@ -244,6 +247,7 @@ subroutine get_grid_comp_area(component,tile,area,domain)
   integer :: num_nest_tile, ntiles
   logical :: is_nest
   integer :: found_xgrid_files ! how many xgrid files we actually found in the grid spec
+  integer :: ibegin, iend, bsize, l
 
   select case (get_grid_version())
   case(VERSION_0,VERSION_1)
@@ -315,7 +319,7 @@ subroutine get_grid_comp_area(component,tile,area,domain)
            endif
         end if
      end do
-     area(:,:) = 0
+     area(:,:) = 0.
      if(field_exist(grid_file,xgrid_name)) then
         ! get the number of the exchange-grid files
         call field_size(grid_file,xgrid_name,siz)
@@ -343,15 +347,26 @@ subroutine get_grid_comp_area(component,tile,area,domain)
 
            ! finally read the exchange grid
            nxgrid = get_mosaic_xgrid_size(grid_dir//xgrid_file)
-           allocate(i1(nxgrid), j1(nxgrid), i2(nxgrid), j2(nxgrid), xgrid_area(nxgrid))
-           call get_mosaic_xgrid(grid_dir//xgrid_file, i1, j1, i2, j2, xgrid_area)
-           ! and sum the exchange grid areas
-           do m = 1, nxgrid
-              i = i2(m); j = j2(m)
-              if (iie) cycle
-              if (jje) cycle
-              area(i+i0,j+j0) = area(i+i0,j+j0) + xgrid_area(m)
-           end do
+           if(nxgrid < BUFSIZE) then
+              allocate(i1(nxgrid), j1(nxgrid), i2(nxgrid), j2(nxgrid), xgrid_area(nxgrid))
+           else
+              allocate(i1(BUFSIZE), j1(BUFSIZE), i2(BUFSIZE), j2(BUFSIZE), xgrid_area(BUFSIZE))
+           endif
+           ibegin = 1
+           do l = 1,nxgrid,BUFSIZE
+              bsize = min(BUFSIZE, nxgrid-l+1)
+              iend = ibegin + bsize - 1
+              call get_mosaic_xgrid(grid_dir//xgrid_file, i1(1:bsize), j1(1:bsize), i2(1:bsize), j2(1:bsize), &
+                                    xgrid_area(1:bsize), ibegin, iend)
+              ! and sum the exchange grid areas
+              do m = 1, bsize
+                 i = i2(m); j = j2(m)
+                 if (iie) cycle
+                 if (jje) cycle
+                 area(i+i0,j+j0) = area(i+i0,j+j0) + xgrid_area(m)
+              end do
+              ibegin = iend + 1
+           enddo
            deallocate(i1, j1, i2, j2, xgrid_area)
         enddo
         if (found_xgrid_files == 0) &
@@ -362,7 +377,7 @@ subroutine get_grid_comp_area(component,tile,area,domain)
      deallocate(nest_tile_name)
   end select ! version
   ! convert area to m2
-  area = area*4*PI*radius**2
+  area = area*4.*PI*radius**2
 end subroutine
 
 ! ============================================================================
@@ -446,7 +461,7 @@ subroutine get_grid_cell_vertices_1D(component, tile, glonb, glatb)
 
      start = 1; nread = 1
      nread(2) = 2*nlat+1
-     call read_data(filename2, "x", tmp, start, nread, no_domain=.TRUE.)
+     call read_data(filename2, "y", tmp, start, nread, no_domain=.TRUE.)
      glatb(1:nlat+1) = tmp(1,1:2*nlat+1:2)
      deallocate(tmp)
   end select
@@ -821,11 +836,12 @@ end subroutine get_grid_cell_centers_2D
 ! domain for current processor
 ! ============================================================================
 ! this subroutine probably does not belong in the grid_mod 
-subroutine define_cube_mosaic ( component, domain, layout, halo )
+subroutine define_cube_mosaic ( component, domain, layout, halo, maskmap )
   character(len=*) , intent(in)    :: component
   type(domain2d)   , intent(inout) :: domain
   integer          , intent(in)    :: layout(2)
   integer, optional, intent(in)    :: halo 
+  logical, optional, intent(in)    :: maskmap(:,:,:)
 
   ! ---- local constants
   
@@ -835,7 +851,7 @@ subroutine define_cube_mosaic ( component, domain, layout, halo )
   integer :: ntiles     ! number of tiles
   integer :: ncontacts  ! number of contacts between mosaic tiles
   integer :: n
-  integer :: ng         ! halo size
+  integer :: ng, pe_pos, npes         ! halo size
   integer, allocatable :: nlon(:), nlat(:), global_indices(:,:)
   integer, allocatable :: pe_start(:), pe_end(:), layout_2d(:,:)
   integer, allocatable :: tile1(:),tile2(:)
@@ -849,11 +865,18 @@ subroutine define_cube_mosaic ( component, domain, layout, halo )
   allocate(layout_2d(2,ntiles))
   call get_grid_size(component,nlon,nlat)
 
+  pe_pos = mpp_root_pe()
   do n = 1, ntiles
      global_indices(:,n) = (/ 1, nlon(n), 1, nlat(n) /)
      layout_2d     (:,n) = layout
-     pe_start        (n) = mpp_root_pe() + (n-1)*layout(1)*layout(2)
-     pe_end          (n) = mpp_root_pe() +     n*layout(1)*layout(2) - 1
+     if(present(maskmap)) then
+        npes = count(maskmap(:,:,n))
+     else
+        npes = layout(1)*layout(2)
+     endif
+     pe_start(n) = pe_pos
+     pe_end  (n) = pe_pos + npes - 1
+     pe_pos      = pe_end(n) + 1
   enddo
 
   varname=trim(lowercase(component))//'_mosaic_file'
@@ -877,6 +900,7 @@ subroutine define_cube_mosaic ( component, domain, layout, halo )
        is2, ie2, js2, je2, &
        pe_start=pe_start, pe_end=pe_end, symmetry=.true.,  &
        shalo = ng, nhalo = ng, whalo = ng, ehalo = ng,     &
+       maskmap = maskmap,                                  &
        name = trim(component)//'Cubic-Sphere Grid' )
 
   deallocate(nlon,nlat,global_indices,pe_start,pe_end,layout_2d)
diff --git a/src/shared/mosaic/interp.h b/src/shared/mosaic/interp.h
index d6b0543d78..d704bee065 100644
--- a/src/shared/mosaic/interp.h
+++ b/src/shared/mosaic/interp.h
@@ -8,7 +8,7 @@
 /*********************************************************************
                      interp.h
    This header files contains defition of some interpolation routine  (1-D or 2-D).
-   contact: Zhi.Liang
+   contact: Zhi.Liang@noaa.gov
 *********************************************************************/
 void cubic_spline_sp(int size1, int size2, const double *grid1, const double *grid2, const double *data1,
                   double *data2 );
diff --git a/src/shared/mosaic/mosaic.F90 b/src/shared/mosaic/mosaic.F90
index 8645be6847..794c8219a0 100644
--- a/src/shared/mosaic/mosaic.F90
+++ b/src/shared/mosaic/mosaic.F90
@@ -1,9 +1,10 @@
 module mosaic_mod
 
-! 
+! 
 !   Zhi Liang
 ! 
 
+! 
 
 ! 
 !    mosaic_mod implements some utility routines to read mosaic information.
@@ -16,11 +17,22 @@ module mosaic_mod
 !    grid information. Each routine will call a C-version routine to get these information.
 ! 
 
-use mpp_mod, only : mpp_error, FATAL
+use mpp_mod,    only : mpp_error, FATAL, mpp_pe, mpp_root_pe
+use mpp_io_mod, only : MPP_MULTI
+use fms_io_mod, only : dimension_size, field_exist, read_data, read_compressed
+use constants_mod, only : PI, RADIUS
 
 implicit none
 private
 
+character(len=*), parameter :: &
+     grid_dir  = 'INPUT/'      ! root directory for all grid files
+
+integer, parameter :: &
+     MAX_NAME = 256,  & ! max length of the variable names
+     MAX_FILE = 1024, & ! max length of the file names
+     X_REFINE = 2,    & ! supergrid size/model grid size in x-direction
+     Y_REFINE = 2       ! supergrid size/model grid size in y-direction
 
 ! --- public interface
 
@@ -37,8 +49,8 @@ module mosaic_mod
 
 logical :: module_is_initialized = .true.
 ! version information varaible
- character(len=128) :: version = '$Id: mosaic.F90,v 20.0 2013/12/14 00:22:25 fms Exp $'
- character(len=128) :: tagname = '$Name: tikal $'
+ character(len=128) :: version = '$Id$'
+ character(len=128) :: tagname = '$Name$'
 
 contains
 
@@ -61,7 +73,7 @@ subroutine mosaic_init()
   module_is_initialized = .TRUE.
 
 !--------- write version number and namelist ------------------
-!  call write_version_number()
+!  call write_version_number (version, tagname)
 
 end subroutine mosaic_init
 ! 
@@ -83,17 +95,8 @@ end subroutine mosaic_init
   function get_mosaic_xgrid_size(xgrid_file)
     character(len=*), intent(in)          :: xgrid_file
     integer                               :: get_mosaic_xgrid_size
-    character(len=len_trim(xgrid_file)+1) :: xfile    
-    integer                               :: read_mosaic_xgrid_size
-    integer                               :: strlen
-
-    !---- transfer to C-stype string
-    strlen = len_trim(xgrid_file)
-    xfile(1:strlen) = xgrid_file(1:strlen)
-    strlen = strlen+1
-    xfile(strlen:strlen) = CHAR(0)
 
-    get_mosaic_xgrid_size = read_mosaic_xgrid_size(xfile)
+    get_mosaic_xgrid_size = dimension_size(xgrid_file, "ncells", no_domain=.TRUE.)
 
     return   
 
@@ -125,46 +128,53 @@ end function get_mosaic_xgrid_size
 !   
 !     area of the exchange grid. The area is scaled to represent unit earth area.
 !   
-  subroutine get_mosaic_xgrid(xgrid_file, i1, j1, i2, j2, area, di, dj, istart, iend)
+  subroutine get_mosaic_xgrid(xgrid_file, i1, j1, i2, j2, area, ibegin, iend)
     character(len=*), intent(in) :: xgrid_file
     integer,       intent(inout) :: i1(:), j1(:), i2(:), j2(:)
     real,          intent(inout) :: area(:)
-    real, optional,intent(inout) :: di(:), dj(:)
-    integer, optional,intent(in) :: istart, iend 
-
-    character(len=len_trim(xgrid_file)+1) :: xfile
-    integer :: n, strlen, nxgrid
-
-    !---- transfer to C-stype string
-    strlen = len_trim(xgrid_file)
-    xfile(1:strlen) = xgrid_file(1:strlen)
-    strlen = strlen+1
-    xfile(strlen:strlen) = CHAR(0)
-
-    !--- order 2 xgrid will be implemented later 
-    nxgrid = size(i1(:))
-
-    if(PRESENT(di)) then
-       if( PRESENT(istart) .OR. PRESENT(iend) ) &
-          call mpp_error(FATAL, "mosaic_mod: istart and iend should not present when di is present, contact developer")
-       if(.NOT. PRESENT(dj) ) call mpp_error(FATAL, "mosaic_mod: when di is present, dj should be present")
-       call read_mosaic_xgrid_order2(xfile, i1, j1, i2, j2, area, di, dj)
+    integer, optional, intent(in) :: ibegin, iend
+
+    integer                            :: start(4), nread(4), istart
+    real,    dimension(2, size(i1(:))) :: tile1_cell, tile2_cell
+    integer                            :: nxgrid, n
+    real                               :: garea
+    real                               :: get_global_area;
+
+    garea = get_global_area();
+
+    ! When start and nread present, make sure nread(1) is the same as the size of the data
+    if(present(ibegin) .and. present(iend)) then
+       istart = ibegin
+       nxgrid = iend - ibegin + 1
+       if(nxgrid .NE. size(i1(:))) call mpp_error(FATAL, "get_mosaic_xgrid: nxgrid .NE. size(i1(:))")
+       if(nxgrid .NE. size(j1(:))) call mpp_error(FATAL, "get_mosaic_xgrid: nxgrid .NE. size(j1(:))")
+       if(nxgrid .NE. size(i2(:))) call mpp_error(FATAL, "get_mosaic_xgrid: nxgrid .NE. size(i2(:))")
+       if(nxgrid .NE. size(j2(:))) call mpp_error(FATAL, "get_mosaic_xgrid: nxgrid .NE. size(j2(:))")
+       if(nxgrid .NE. size(area(:))) call mpp_error(FATAL, "get_mosaic_xgrid: nxgrid .NE. size(area(:))")
     else
-       if( PRESENT(istart) .AND. PRESENT(iend) ) then
-          if( iend-istart+1 .NE. nxgrid) call mpp_error(FATAL, "mosaic_mod: iend-istart+1 must equal size(i1)")
-          call read_mosaic_xgrid_order1_region(xfile, i1, j1, i2, j2, area, istart-1, iend-1) ! convert to c-index
-       else
-          call read_mosaic_xgrid_order1(xfile, i1, j1, i2, j2, area)
-       endif
-    end if
-
-    ! in C, programming, the starting index is 0, so need add 1 to the index.
-    do n = 1, nxgrid
-       i1(n) = i1(n) + 1
-       j1(n) = j1(n) + 1
-       i2(n) = i2(n) + 1
-       j2(n) = j2(n) + 1
+       istart = 1
+       nxgrid = size(i1(:))
+    endif
+
+    start  = 1; nread = 1
+    start(1) = istart; nread(1) = nxgrid
+    call read_compressed(xgrid_file, 'xgrid_area', area, start=start, nread=nread, threading=MPP_MULTI)
+    start = 1; nread = 1
+    nread(1) = 2
+    start(2) = istart; nread(2) = nxgrid
+    call read_compressed(xgrid_file, 'tile1_cell', tile1_cell, start=start, nread=nread, threading=MPP_MULTI)
+    call read_compressed(xgrid_file, 'tile2_cell', tile2_cell, start=start, nread=nread, threading=MPP_MULTI)
+
+     do n = 1, nxgrid
+       i1(n) = tile1_cell(1,n) 
+       j1(n) = tile1_cell(2,n)
+       i2(n) = tile2_cell(1,n) 
+       j2(n) = tile2_cell(2,n)
+       area(n) = area(n)/garea
     end do
+    
+    return
+
   end subroutine get_mosaic_xgrid
 ! 
 
@@ -186,17 +196,9 @@ function get_mosaic_ntiles(mosaic_file)
     character(len=*), intent(in) :: mosaic_file
     integer                      :: get_mosaic_ntiles 
 
-    character(len=len_trim(mosaic_file)+1) :: mfile    
-    integer                                :: strlen
-    integer                                :: read_mosaic_ntiles
+    get_mosaic_ntiles = dimension_size(mosaic_file, "ntiles")
 
-    !---- transfer to C-stype string
-    strlen = len_trim(mosaic_file)
-    mfile(1:strlen) = mosaic_file(1:strlen)
-    strlen = strlen+1
-    mfile(strlen:strlen) = CHAR(0)
-
-    get_mosaic_ntiles = read_mosaic_ntiles(mfile)
+    return
 
   end function get_mosaic_ntiles
 ! 
@@ -223,13 +225,13 @@ function get_mosaic_ncontacts( mosaic_file)
     integer                                :: strlen
     integer                                :: read_mosaic_ncontacts
 
-    !---- transfer to C-stype string
-    strlen = len_trim(mosaic_file)
-    mfile(1:strlen) = mosaic_file(1:strlen)
-    strlen = strlen+1
-    mfile(strlen:strlen) = CHAR(0)
+    if(field_exist(mosaic_file, "contacts") ) then
+      get_mosaic_ncontacts = dimension_size(mosaic_file, "ncontact", no_domain=.TRUE.)
+    else
+      get_mosaic_ncontacts = 0
+    endif
 
-    get_mosaic_ncontacts = read_mosaic_ncontacts(mfile)
+    return
 
   end function get_mosaic_ncontacts
 ! 
@@ -259,16 +261,25 @@ subroutine get_mosaic_grid_sizes( mosaic_file, nx, ny)
     character(len=*),         intent(in) :: mosaic_file
     integer, dimension(:), intent(inout) :: nx, ny
 
-    character(len=len_trim(mosaic_file)+1) :: mfile    
-    integer                                :: strlen
-
-    !---- transfer to C-stype string
-    strlen = len_trim(mosaic_file)
-    mfile(1:strlen) = mosaic_file(1:strlen)
-    strlen = strlen+1
-    mfile(strlen:strlen) = CHAR(0)
-
-    call read_mosaic_grid_sizes(mfile, nx, ny)
+    character(len=MAX_FILE) :: gridfile
+    integer                 :: ntiles, n    
+
+    ntiles = get_mosaic_ntiles(mosaic_file)
+    if(ntiles .NE. size(nx(:)) .OR. ntiles .NE. size(ny(:)) ) then
+      call mpp_error(FATAL, "get_mosaic_grid_sizes: size of nx/ny does not equal to ntiles")
+    endif
+    do n = 1, ntiles
+      call read_data(mosaic_file, 'gridfiles', gridfile, level=n)
+      gridfile = grid_dir//trim(gridfile)
+      nx(n) = dimension_size(gridfile, "nx")
+      ny(n) = dimension_size(gridfile, "ny")
+      if(mod(nx(n),x_refine) .NE. 0) call mpp_error(FATAL, "get_mosaic_grid_sizes: nx is not divided by x_refine");
+      if(mod(ny(n),y_refine) .NE. 0) call mpp_error(FATAL, "get_mosaic_grid_sizes: ny is not divided by y_refine");
+      nx(n) = nx(n)/x_refine;
+      ny(n) = ny(n)/y_refine;
+    enddo
+
+    return
 
   end subroutine get_mosaic_grid_sizes
 ! 
@@ -324,30 +335,141 @@ subroutine get_mosaic_contact( mosaic_file, tile1, tile2, istart1, iend1, jstart
     integer, dimension(:), intent(inout) :: tile1, tile2
     integer, dimension(:), intent(inout) :: istart1, iend1, jstart1, jend1
     integer, dimension(:), intent(inout) :: istart2, iend2, jstart2, jend2
-    character(len=len_trim(mosaic_file)+1) :: mfile    
-    integer                                :: strlen
-
-    !---- transfer to C-stype string
-    strlen = len_trim(mosaic_file)
-    mfile(1:strlen) = mosaic_file(1:strlen)
-    strlen = strlen+1
-    mfile(strlen:strlen) = CHAR(0)
-
-    call read_mosaic_contact(mfile, tile1, tile2, istart1, iend1, jstart1, jend1, &
-                            istart2, iend2, jstart2, jend2)
-    !--- transfer C-index to Fortran-index.
-    istart1 = istart1 + 1
-    iend1   = iend1   + 1
-    jstart1 = jstart1 + 1
-    jend1   = jend1   + 1
-    istart2 = istart2 + 1
-    iend2   = iend2   + 1
-    jstart2 = jstart2 + 1
-    jend2   = jend2   + 1
+    character(len=MAX_NAME), allocatable :: gridtiles(:)
+    character(len=MAX_NAME)              :: contacts
+    character(len=MAX_NAME)              :: strlist(8)
+    integer :: ntiles, n, m, ncontacts, nstr, ios
+    integer :: i1_type, j1_type, i2_type, j2_type
+    logical :: found
+
+    ntiles = get_mosaic_ntiles(mosaic_file)
+    allocate(gridtiles(ntiles))    
+    do n = 1, ntiles
+      call read_data(mosaic_file, 'gridtiles', gridtiles(n), level=n)
+    enddo
+
+    ncontacts = get_mosaic_ncontacts(mosaic_file)
+
+    do n = 1, ncontacts
+      call read_data(mosaic_file, "contacts", contacts, level=n)
+      nstr = parse_string(contacts, ":", strlist)      
+      if(nstr .NE. 4) call mpp_error(FATAL, &
+         "mosaic_mod(get_mosaic_contact): number of elements in contact seperated by :/:: should be 4")
+      found = .false.
+      do m = 1, ntiles
+        if(trim(gridtiles(m)) == trim(strlist(2)) ) then !found the tile name 
+          found = .true.
+          tile1(n) = m
+          exit
+        endif
+      enddo
+    
+      if(.not.found) call mpp_error(FATAL, &
+         "mosaic_mod(get_mosaic_contact):the first tile name specified in contact is not found in tile list")
+
+      found = .false.
+      do m = 1, ntiles
+        if(trim(gridtiles(m)) == trim(strlist(4)) ) then !found the tile name 
+          found = .true.
+          tile2(n) = m
+          exit
+        endif
+      enddo
+    
+      if(.not.found) call mpp_error(FATAL, &
+         "mosaic_mod(get_mosaic_contact):the second tile name specified in contact is not found in tile list")
+
+      call read_data(mosaic_file, "contact_index", contacts, level=n)
+      nstr = parse_string(contacts, ":,", strlist)
+      if(nstr .NE. 8) then
+        if(mpp_pe()==mpp_root_pe()) then 
+          print*, "nstr is ", nstr
+          print*, "contacts is ", contacts
+          do m = 1, nstr
+            print*, "strlist is ", trim(strlist(m))
+          enddo
+        endif
+        call mpp_error(FATAL, &
+               "mosaic_mod(get_mosaic_contact): number of elements in contact_index seperated by :/, should be 8")
+      endif
+      read(strlist(1), *, iostat=ios) istart1(n)
+      if(ios .NE. 0) call mpp_error(FATAL, &
+         "mosaic_mod(get_mosaic_contact): Error in reading istart1")
+      read(strlist(2), *, iostat=ios) iend1(n)
+      if(ios .NE. 0) call mpp_error(FATAL, &
+         "mosaic_mod(get_mosaic_contact): Error in reading iend1")
+      read(strlist(3), *, iostat=ios) jstart1(n)
+      if(ios .NE. 0) call mpp_error(FATAL, &
+         "mosaic_mod(get_mosaic_contact): Error in reading jstart1")
+      read(strlist(4), *, iostat=ios) jend1(n)
+      if(ios .NE. 0) call mpp_error(FATAL, &
+         "mosaic_mod(get_mosaic_contact): Error in reading jend1")
+      read(strlist(5), *, iostat=ios) istart2(n)
+      if(ios .NE. 0) call mpp_error(FATAL, &
+         "mosaic_mod(get_mosaic_contact): Error in reading istart2")
+      read(strlist(6), *, iostat=ios) iend2(n)
+      if(ios .NE. 0) call mpp_error(FATAL, &
+         "mosaic_mod(get_mosaic_contact): Error in reading iend2")
+      read(strlist(7), *, iostat=ios) jstart2(n)
+      if(ios .NE. 0) call mpp_error(FATAL, &
+         "mosaic_mod(get_mosaic_contact): Error in reading jstart2")
+      read(strlist(8), *, iostat=ios) jend2(n)
+      if(ios .NE. 0) call mpp_error(FATAL, &
+         "mosaic_mod(get_mosaic_contact): Error in reading jend2")
+
+      i1_type = transfer_to_model_index(istart1(n), iend1(n), x_refine)
+      j1_type = transfer_to_model_index(jstart1(n), jend1(n), y_refine)
+      i2_type = transfer_to_model_index(istart2(n), iend2(n), x_refine)
+      j2_type = transfer_to_model_index(jstart2(n), jend2(n), y_refine)
+
+      if( i1_type == 0 .AND. j1_type == 0 ) call mpp_error(FATAL, &
+         "mosaic_mod(get_mosaic_contact): istart1==iend1 and jstart1==jend1")
+      if( i2_type == 0 .AND. j2_type == 0 ) call mpp_error(FATAL, &
+         "mosaic_mod(get_mosaic_contact): istart2==iend2 and jstart2==jend2")
+      if( i1_type + j1_type .NE. i2_type + j2_type ) call mpp_error(FATAL, &
+         "mosaic_mod(get_mosaic_contact): It is not a line or overlap contact")
+
+   enddo
+
+      deallocate(gridtiles)
 
   end subroutine get_mosaic_contact
 ! 
 
+
+function transfer_to_model_index(istart, iend, refine_ratio)
+   integer, intent(inout) :: istart, iend
+   integer                :: refine_ratio
+   integer                :: transfer_to_model_index
+   integer                :: istart_in, iend_in
+
+   istart_in = istart
+   iend_in = iend
+
+   if( istart_in == iend_in ) then
+      transfer_to_model_index = 0
+      istart = (istart_in + 1)/refine_ratio
+      iend   = istart
+   else 
+      transfer_to_model_index = 1
+      if( iend_in > istart_in ) then
+        istart = istart_in + 1
+        iend   = iend_in
+      else 
+        istart = istart_in
+        iend   = iend_in + 1
+      endif
+      if( mod(istart, refine_ratio) .NE. 0 .OR. mod(iend,refine_ratio) .NE. 0) call mpp_error(FATAL, &
+         "mosaic_mod(transfer_to_model_index): mismatch between refine_ratio and istart/iend")
+      istart = istart/refine_ratio
+      iend = iend/refine_ratio
+
+   endif
+
+   return
+
+end function transfer_to_model_index
+
   !###############################################################################
   ! 
   !   
@@ -453,6 +575,52 @@ function is_inside_polygon(lon1, lat1, lon2, lat2 )
 
   end function is_inside_polygon
 
+  function parse_string(string, set, value)
+  character(len=*),  intent(in) :: string
+  character(len=*),  intent(in) :: set
+  character(len=*), intent(out) :: value(:)
+  integer                       :: parse_string
+  integer :: nelem, length, first, last
+
+     nelem = size(value(:))
+     length = len_trim(string)
+
+     first = 1; last = 0
+     parse_string = 0
+
+     do while(first .LE. length)
+       parse_string = parse_string + 1
+       if(parse_string>nelem) then
+         call mpp_error(FATAL, "mosaic_mod(parse_string) : number of element is greater than size(value(:))")
+       endif
+       last = first - 1 + scan(string(first:length), set)
+       if(last == first-1 ) then  ! not found, end of string
+         value(parse_string) = string(first:length)
+         exit
+       else
+         if(last <= first) then
+           call mpp_error(FATAL, "mosaic_mod(parse_string) : last <= first")
+         endif
+         value(parse_string) = string(first:(last-1))
+         first = last + 1
+         ! scan to make sure the next is not the character in the set
+         do while (first == last+1)
+           last = first - 1 + scan(string(first:length), set)
+           if(last == first) then
+             first = first+1
+           else
+             exit
+           endif
+         end do
+       endif
+     enddo
+
+     return
+
+  end function parse_string
+
+
+     
 end module mosaic_mod
 
 
diff --git a/src/shared/mosaic/mosaic_util.h b/src/shared/mosaic/mosaic_util.h
index 0641640981..c7c0cefc42 100644
--- a/src/shared/mosaic/mosaic_util.h
+++ b/src/shared/mosaic/mosaic_util.h
@@ -2,7 +2,7 @@
                       mosaic_util.h
     This header file provide some utilities routine that will be used in many tools.
     
-    contact: Zhi.Liang
+    contact: Zhi.Liang@noaa.gov
 ***********************************************************************/
 #ifndef MOSAIC_UTIL_H_
 #define MOSAIC_UTIL_H_
diff --git a/src/shared/mosaic/read_mosaic.c b/src/shared/mosaic/read_mosaic.c
index 27ed0b2e4a..460f2c42c4 100644
--- a/src/shared/mosaic/read_mosaic.c
+++ b/src/shared/mosaic/read_mosaic.c
@@ -375,6 +375,35 @@ int read_mosaic_xgrid_size( const char *xgrid_file )
   return ncells;
 }
 
+#ifdef OVERLOAD_R4
+float get_global_area(void) 
+{
+  float garea;
+#else
+double get_global_area(void) 
+{
+  double garea;
+#endif
+  garea = 4*M_PI*RADIUS*RADIUS;
+
+  return garea;
+};
+
+#ifndef __AIX
+#ifdef OVERLOAD_R4
+float get_global_area_(void)
+{
+  float garea;
+#else
+double get_global_area_(void)
+{
+  double garea;
+#endif
+  garea = 4*M_PI*RADIUS*RADIUS;
+
+  return garea;
+};
+#endif
 
 
 /****************************************************************************/
diff --git a/src/shared/mosaic/read_mosaic.h b/src/shared/mosaic/read_mosaic.h
index b6256d2dcc..cd56d15d1c 100644
--- a/src/shared/mosaic/read_mosaic.h
+++ b/src/shared/mosaic/read_mosaic.h
@@ -7,11 +7,13 @@ void read_mosaic_xgrid_order1(const char *xgrid_file, int *i1, int *j1, int *i2,
 void read_mosaic_xgrid_order1_region(const char *xgrid_file, int *i1, int *j1, int *i2, int *j2, float *area, int *isc, int *iec );
 void read_mosaic_xgrid_order2(const char *xgrid_file, int *i1, int *j1, int *i2, int *j2, 
                               float *area, float *di, float *dj );
+float get_global_area(void);
 #else
 void read_mosaic_xgrid_order1(const char *xgrid_file, int *i1, int *j1, int *i2, int *j2, double *area );
 void read_mosaic_xgrid_order1_region(const char *xgrid_file, int *i1, int *j1, int *i2, int *j2, double *area, int *isc, int *iec );
 void read_mosaic_xgrid_order2(const char *xgrid_file, int *i1, int *j1, int *i2, int *j2, 
                               double *area, double *di, double *dj );
+double get_global_area(void);
 #endif
 int read_mosaic_ntiles(const char *mosaic_file);
 int read_mosaic_ncontacts(const char *mosaic_file);
@@ -20,4 +22,5 @@ void read_mosaic_contact(const char *mosaic_file, int *tile1, int *tile2, int *i
 			 int *jstart1, int *jend1, int *istart2, int *iend2, int *jstart2, int *jend2);
 void read_mosaic_grid_data(const char *mosaic_file, const char *name, int nx, int ny,
                            double *data, int level, int ioff, int joff);
+
 #endif
diff --git a/src/shared/mpp/include/group_update_pack.inc b/src/shared/mpp/include/group_update_pack.inc
new file mode 100644
index 0000000000..db5f44ec59
--- /dev/null
+++ b/src/shared/mpp/include/group_update_pack.inc
@@ -0,0 +1,413 @@
+if( group%k_loop_inside ) then
+!$OMP parallel do default(none) shared(npack,group,ptr,nvector,ksize,buffer_start_pos) &
+!$OMP                          private(buffer_pos,pos,m,is,ie,js,je,rotation, &
+!$OMP                                  ptr_field, ptr_fieldx, ptr_fieldy,n,k)
+  do n = 1, npack
+     buffer_pos = group%pack_buffer_pos(n) + buffer_start_pos
+     pos  = buffer_pos
+     is = group%pack_is(n); ie = group%pack_ie(n)
+     js = group%pack_js(n); je = group%pack_je(n)
+     rotation = group%pack_rotation(n)
+     if( group%pack_type(n) == FIELD_S ) then
+        select case( rotation )
+        case(ZERO)
+           do l=1, group%nscalar  ! loop over number of fields
+              ptr_field = group%addrs_s(l)
+              do k = 1, ksize
+                 do j = js, je
+                    do i = is, ie
+                       pos = pos + 1
+                       buffer(pos) = field(i,j,k)
+                    end do
+                 end do
+              enddo
+           enddo
+        case( MINUS_NINETY )
+           do l=1,group%nscalar  ! loop over number of fields
+              ptr_field = group%addrs_s(l)
+              do k = 1, ksize
+                 do i = is, ie
+                    do j = je, js, -1
+                       pos = pos + 1
+                       buffer(pos) = field(i,j,k)
+                    end do
+                 end do
+              end do
+           end do
+        case( NINETY )
+           do l=1,group%nscalar  ! loop over number of fields
+              ptr_field = group%addrs_s(l)
+              do k = 1, ksize
+                 do i = ie, is, -1
+                    do j = js, je
+                       pos = pos + 1
+                       buffer(pos) = field(i,j,k)
+                    end do
+                 end do
+              end do
+           end do
+        case( ONE_HUNDRED_EIGHTY )
+           do l=1,group%nscalar  ! loop over number of fields
+              ptr_field = group%addrs_s(l)
+              do k = 1, ksize
+                 do j = je, js, -1
+                    do i = ie, is, -1
+                       pos = pos + 1
+                       buffer(pos) = field(i,j,k)
+                    end do
+                 end do
+              end do
+           end do
+        end select
+     else if( group%pack_type(n) == FIELD_X  ) then
+        select case( rotation )
+        case(ZERO)
+           do l=1, nvector  ! loop over number of fields
+              ptr_fieldx = group%addrs_x(l)
+              do k = 1, ksize
+                 do j = js, je
+                    do i = is, ie
+                       pos = pos + 1
+                       buffer(pos) = fieldx(i,j,k)
+                    end do
+                 end do
+              end do
+           end do
+        case( MINUS_NINETY )
+           if( BTEST(group%flags_v,SCALAR_BIT) ) then
+              do l=1,nvector  ! loop over number of fields
+                 ptr_fieldy = group%addrs_y(l)
+                 do k = 1, ksize
+                    do i = is, ie
+                       do j = je, js, -1
+                          pos = pos + 1
+                          buffer(pos) = fieldy(i,j,k)
+                       end do
+                    end do
+                 end do
+              end do
+           else
+              do l=1,nvector  ! loop over number of fields
+                 ptr_fieldy = group%addrs_y(l)
+                 do k = 1, ksize
+                    do i = is, ie
+                       do j = je, js, -1
+                          pos = pos + 1
+                          buffer(pos) = -fieldy(i,j,k)
+                       end do
+                    end do
+                 end do
+              end do
+           end if
+        case( NINETY )
+           do l=1, nvector  ! loop over number of fields
+              ptr_fieldy = group%addrs_y(l)
+              do k = 1, ksize
+                 do i = ie, is, -1
+                    do j = js, je
+                       pos = pos + 1
+                       buffer(pos) = fieldy(i,j,k)
+                    end do
+                 end do
+              end do
+           end do
+        case( ONE_HUNDRED_EIGHTY )
+           if( BTEST(group%flags_v,SCALAR_BIT) ) then
+              do l=1,nvector  ! loop over number of fields
+                 ptr_fieldx = group%addrs_x(l)
+                 do k = 1, ksize
+                    do j = je, js, -1
+                       do i = ie, is, -1
+                          pos = pos + 1
+                          buffer(pos) =  fieldx(i,j,k)
+                       end do
+                    end do
+                 end do
+              end do
+           else
+              do l=1,nvector  ! loop over number of fields
+                 ptr_fieldx = group%addrs_x(l)
+                 do k = 1, ksize
+                    do j = je, js, -1
+                       do i = ie, is, -1
+                          pos = pos + 1
+                          buffer(pos) =  -fieldx(i,j,k)
+                       end do
+                    end do
+                 end do
+              end do
+           end if
+        end select ! select case( rotation(n) )
+     else if( group%pack_type(n) == FIELD_Y ) then
+        select case( rotation )
+        case(ZERO)
+           do l=1, nvector  ! loop over number of fields
+              ptr_fieldy = group%addrs_y(l)
+              do k = 1, ksize
+                 do j = js, je
+                    do i = is, ie
+                       pos = pos + 1
+                       buffer(pos) = fieldy(i,j,k)
+                    end do
+                 end do
+              end do
+           end do
+        case( MINUS_NINETY )
+           do l=1,nvector  ! loop over number of fields
+              ptr_fieldx = group%addrs_x(l)
+              do k = 1, ksize
+                 do i = is, ie
+                    do j = je, js, -1
+                       pos = pos + 1
+                       buffer(pos) = fieldx(i,j,k)
+                    end do
+                 end do
+              end do
+           end do
+        case( NINETY )
+           if( BTEST(group%flags_v,SCALAR_BIT) ) then
+              do l=1, nvector  ! loop over number of fields
+                 ptr_fieldx = group%addrs_x(l)
+                 do k = 1, ksize
+                    do i = ie, is, -1
+                       do j = js, je
+                          pos = pos + 1
+                          buffer(pos) = fieldx(i,j,k)
+                       end do
+                    end do
+                 end do
+              end do
+           else
+              do l=1,nvector  ! loop over number of fields
+                 ptr_fieldx = group%addrs_x(l)
+                 do k = 1, ksize
+                    do i = ie, is, -1
+                       do j = js, je
+                          pos = pos + 1
+                          buffer(pos) = -fieldx(i,j,k)
+                       end do
+                    end do
+                 end do
+              end do
+           end if
+        case( ONE_HUNDRED_EIGHTY )
+           if( BTEST(group%flags_v,SCALAR_BIT) ) then
+              do l=1,nvector  ! loop over number of fields
+                 ptr_fieldy = group%addrs_y(l)
+                 do k = 1, ksize
+                    do j = je, js, -1
+                       do i = ie, is, -1
+                          pos = pos + 1
+                          buffer(pos) =  fieldy(i,j,k)
+                       end do
+                    end do
+                 end do
+              end do
+           else
+              do l=1,nvector  ! loop over number of fields
+                 ptr_fieldy = group%addrs_y(l)
+                 do k = 1, ksize
+                    do j = je, js, -1
+                       do i = ie, is, -1
+                          pos = pos + 1
+                          buffer(pos) =  -fieldy(i,j,k)
+                       end do
+                    end do
+                 end do
+              end do
+           end if
+        end select ! select case( rotation(n) )
+     endif
+  enddo
+else
+!$OMP parallel do default(none) shared(npack,group,ptr,nvector,ksize,buffer_start_pos) &
+!$OMP                          private(buffer_pos,pos,m,is,ie,js,je,rotation, &
+!$OMP                                  ptr_field, ptr_fieldx, ptr_fieldy,n,k)
+  do nk = 1, npack*ksize
+     n = (nk-1)/ksize + 1
+     k = mod((nk-1), ksize) + 1
+     buffer_pos = group%pack_buffer_pos(n) + buffer_start_pos
+     pos  = buffer_pos + (k-1)*group%pack_size(n)
+     is = group%pack_is(n); ie = group%pack_ie(n)
+     js = group%pack_js(n); je = group%pack_je(n)
+     rotation = group%pack_rotation(n)
+     if( group%pack_type(n) == FIELD_S ) then
+        select case( rotation )
+        case(ZERO)
+           do l=1, group%nscalar  ! loop over number of fields
+              ptr_field = group%addrs_s(l)
+              do j = js, je
+                 do i = is, ie
+                    pos = pos + 1
+                    buffer(pos) = field(i,j,k)
+                 end do
+              end do
+           enddo
+        case( MINUS_NINETY )
+           do l=1,group%nscalar  ! loop over number of fields
+              ptr_field = group%addrs_s(l)
+              do i = is, ie
+                 do j = je, js, -1
+                    pos = pos + 1
+                    buffer(pos) = field(i,j,k)
+                 end do
+              end do
+           end do
+        case( NINETY )
+           do l=1,group%nscalar  ! loop over number of fields
+              ptr_field = group%addrs_s(l)
+              do i = ie, is, -1
+                 do j = js, je
+                    pos = pos + 1
+                    buffer(pos) = field(i,j,k)
+                 end do
+              end do
+           end do
+        case( ONE_HUNDRED_EIGHTY )
+           do l=1,group%nscalar  ! loop over number of fields
+              ptr_field = group%addrs_s(l)
+              do j = je, js, -1
+                 do i = ie, is, -1
+                    pos = pos + 1
+                    buffer(pos) = field(i,j,k)
+                 end do
+              end do
+           end do
+        end select
+     else if( group%pack_type(n) == FIELD_X  ) then
+        select case( rotation )
+        case(ZERO)
+           do l=1, nvector  ! loop over number of fields
+              ptr_fieldx = group%addrs_x(l)
+              do j = js, je
+                 do i = is, ie
+                    pos = pos + 1
+                    buffer(pos) = fieldx(i,j,k)
+                 end do
+              end do
+           end do
+        case( MINUS_NINETY )
+           if( BTEST(group%flags_v,SCALAR_BIT) ) then
+              do l=1,nvector  ! loop over number of fields
+                 ptr_fieldy = group%addrs_y(l)
+                 do i = is, ie
+                    do j = je, js, -1
+                       pos = pos + 1
+                       buffer(pos) = fieldy(i,j,k)
+                    end do
+                 end do
+              end do
+           else
+              do l=1,nvector  ! loop over number of fields
+                 ptr_fieldy = group%addrs_y(l)
+                 do i = is, ie
+                    do j = je, js, -1
+                       pos = pos + 1
+                       buffer(pos) = -fieldy(i,j,k)
+                    end do
+                 end do
+              end do
+           end if
+        case( NINETY )
+           do l=1, nvector  ! loop over number of fields
+              ptr_fieldy = group%addrs_y(l)
+              do i = ie, is, -1
+                 do j = js, je
+                    pos = pos + 1
+                    buffer(pos) = fieldy(i,j,k)
+                 end do
+              end do
+           end do
+        case( ONE_HUNDRED_EIGHTY )
+           if( BTEST(group%flags_v,SCALAR_BIT) ) then
+              do l=1,nvector  ! loop over number of fields
+                 ptr_fieldx = group%addrs_x(l)
+                 do j = je, js, -1
+                    do i = ie, is, -1
+                       pos = pos + 1
+                       buffer(pos) =  fieldx(i,j,k)
+                    end do
+                 end do
+              end do
+           else
+              do l=1,nvector  ! loop over number of fields
+                 ptr_fieldx = group%addrs_x(l)
+                 do j = je, js, -1
+                    do i = ie, is, -1
+                       pos = pos + 1
+                       buffer(pos) =  -fieldx(i,j,k)
+                    end do
+                 end do
+              end do
+           end if
+        end select ! select case( rotation(n) )
+     else if( group%pack_type(n) == FIELD_Y ) then
+        select case( rotation )
+        case(ZERO)
+           do l=1, nvector  ! loop over number of fields
+              ptr_fieldy = group%addrs_y(l)
+              do j = js, je
+                 do i = is, ie
+                    pos = pos + 1
+                    buffer(pos) = fieldy(i,j,k)
+                 end do
+              end do
+           end do
+        case( MINUS_NINETY )
+           do l=1,nvector  ! loop over number of fields
+              ptr_fieldx = group%addrs_x(l)
+              do i = is, ie
+                 do j = je, js, -1
+                    pos = pos + 1
+                    buffer(pos) = fieldx(i,j,k)
+                 end do
+              end do
+           end do
+        case( NINETY )
+           if( BTEST(group%flags_v,SCALAR_BIT) ) then
+              do l=1, nvector  ! loop over number of fields
+                 ptr_fieldx = group%addrs_x(l)
+                 do i = ie, is, -1
+                    do j = js, je
+                       pos = pos + 1
+                       buffer(pos) = fieldx(i,j,k)
+                    end do
+                 end do
+              end do
+           else
+              do l=1,nvector  ! loop over number of fields
+                 ptr_fieldx = group%addrs_x(l)
+                 do i = ie, is, -1
+                    do j = js, je
+                       pos = pos + 1
+                       buffer(pos) = -fieldx(i,j,k)
+                    end do
+                 end do
+              end do
+           end if
+        case( ONE_HUNDRED_EIGHTY )
+           if( BTEST(group%flags_v,SCALAR_BIT) ) then
+              do l=1,nvector  ! loop over number of fields
+                 ptr_fieldy = group%addrs_y(l)
+                 do j = je, js, -1
+                    do i = ie, is, -1
+                       pos = pos + 1
+                       buffer(pos) =  fieldy(i,j,k)
+                    end do
+                 end do
+              end do
+           else
+              do l=1,nvector  ! loop over number of fields
+                 ptr_fieldy = group%addrs_y(l)
+                 do j = je, js, -1
+                    do i = ie, is, -1
+                       pos = pos + 1
+                       buffer(pos) =  -fieldy(i,j,k)
+                    end do
+                 end do
+              end do
+           end if
+        end select ! select case( rotation(n) )
+     endif
+  enddo
+endif
diff --git a/src/shared/mpp/include/group_update_unpack.inc b/src/shared/mpp/include/group_update_unpack.inc
new file mode 100644
index 0000000000..bf6f9f00c5
--- /dev/null
+++ b/src/shared/mpp/include/group_update_unpack.inc
@@ -0,0 +1,91 @@
+if( group%k_loop_inside ) then
+!$OMP parallel do default(none) shared(nunpack,group,nscalar,ptr,nvector,ksize,buffer_start_pos) &
+!$OMP                          private(buffer_pos,pos,m,is, ie, js, je,rotation, &
+!$OMP                                  ptr_field, ptr_fieldx, ptr_fieldy, n,k )
+  do n = nunpack, 1, -1
+     buffer_pos = group%unpack_buffer_pos(n) + buffer_start_pos
+     pos = buffer_pos 
+     is = group%unpack_is(n); ie = group%unpack_ie(n)
+     js = group%unpack_js(n); je = group%unpack_je(n)
+     if( group%unpack_type(n) == FIELD_S ) then
+        do l=1,nscalar  ! loop over number of fields
+           ptr_field = group%addrs_s(l)
+           do k = 1, ksize
+              do j = js, je
+                 do i = is, ie
+                    pos = pos + 1
+                    field(i,j,k) = buffer(pos)
+                 end do
+              end do
+           end do
+        end do
+     else if( group%unpack_type(n) == FIELD_X ) then
+        do l=1,nvector  ! loop over number of fields
+           ptr_fieldx = group%addrs_x(l)
+           do k = 1, ksize
+              do j = js, je
+                 do i = is, ie
+                    pos = pos + 1
+                    fieldx(i,j,k) = buffer(pos)
+                 end do
+              end do
+           end do
+        end do
+     else if( group%unpack_type(n) == FIELD_Y ) then
+        do l=1,nvector  ! loop over number of fields
+           ptr_fieldy = group%addrs_y(l)
+           do k = 1, ksize
+              do j = js, je
+                 do i = is, ie
+                    pos = pos + 1
+                    fieldy(i,j,k) = buffer(pos)
+                 end do
+              end do
+           end do
+        end do
+     endif
+  enddo
+else
+!$OMP parallel do default(none) shared(nunpack,group,nscalar,ptr,nvector,ksize,buffer_start_pos) &
+!$OMP                          private(buffer_pos,pos,m,is, ie, js, je,rotation, &
+!$OMP                                  ptr_field, ptr_fieldx, ptr_fieldy,n,k)
+  do nk = nunpack*ksize, 1, -1
+     n = (nk-1)/ksize + 1
+     k = mod((nk-1), ksize) + 1
+     buffer_pos = group%unpack_buffer_pos(n) + buffer_start_pos
+     pos = buffer_pos + (k-1)*group%unpack_size(n)
+     is = group%unpack_is(n); ie = group%unpack_ie(n)
+     js = group%unpack_js(n); je = group%unpack_je(n)
+     if( group%unpack_type(n) == FIELD_S ) then
+        do l=1,nscalar  ! loop over number of fields
+           ptr_field = group%addrs_s(l)
+           do j = js, je
+              do i = is, ie
+                 pos = pos + 1
+                 field(i,j,k) = buffer(pos)
+              end do
+           end do
+        end do
+     else if( group%unpack_type(n) == FIELD_X ) then
+        do l=1,nvector  ! loop over number of fields
+           ptr_fieldx = group%addrs_x(l)
+           do j = js, je
+              do i = is, ie
+                 pos = pos + 1
+                 fieldx(i,j,k) = buffer(pos)
+              end do
+           end do
+        end do
+     else if( group%unpack_type(n) == FIELD_Y ) then
+        do l=1,nvector  ! loop over number of fields
+           ptr_fieldy = group%addrs_y(l)
+           do j = js, je
+              do i = is, ie
+                 pos = pos + 1
+                 fieldy(i,j,k) = buffer(pos)
+              end do
+           end do
+        end do
+     endif
+  enddo
+endif
diff --git a/src/shared/mpp/include/mpp_alltoall_mpi.h b/src/shared/mpp/include/mpp_alltoall_mpi.h
index bd129ea7bc..a0949650ec 100644
--- a/src/shared/mpp/include/mpp_alltoall_mpi.h
+++ b/src/shared/mpp/include/mpp_alltoall_mpi.h
@@ -1,7 +1,9 @@
-subroutine MPP_ALLTOALL_(sbuf, rbuf, pelist)
+subroutine MPP_ALLTOALL_(sbuf, scount, rbuf, rcount, pelist)
 
     MPP_TYPE_, intent(in) :: sbuf(:)
     MPP_TYPE_, intent(inout) :: rbuf(:)
+    integer,   intent(in) :: scount, rcount
+   
 
     integer, intent(in), optional :: pelist(0:)
     integer :: n
@@ -17,7 +19,7 @@ subroutine MPP_ALLTOALL_(sbuf, rbuf, pelist)
     if (verbose) call mpp_error(NOTE, 'MPP_ALLTOALL_: using MPI_Alltoall...')
 
     ! TODO: Message lengths greater than 1
-    call MPI_Alltoall(sbuf, 1, MPI_TYPE_, rbuf, 1, MPI_TYPE_, &
+    call MPI_Alltoall(sbuf, scount, MPI_TYPE_, rbuf, rcount, MPI_TYPE_, &
                       peset(n)%id, error)
 
     if (current_clock .NE. 0) &
diff --git a/src/shared/mpp/include/mpp_alltoall_nocomm.h b/src/shared/mpp/include/mpp_alltoall_nocomm.h
index 328cee07e9..70118a3d35 100644
--- a/src/shared/mpp/include/mpp_alltoall_nocomm.h
+++ b/src/shared/mpp/include/mpp_alltoall_nocomm.h
@@ -1,7 +1,8 @@
-subroutine MPP_ALLTOALL_(sbuf, rbuf, pelist)
+subroutine MPP_ALLTOALL_(sbuf, scount, rbuf, rcount, pelist)
 
     MPP_TYPE_, dimension(:), intent(in) :: sbuf
     MPP_TYPE_, dimension(:), intent(inout) :: rbuf
+    integer,   intent(in) :: scount, rcount
 
     integer, intent(in), optional :: pelist(0:)
 
diff --git a/src/shared/mpp/include/mpp_alltoall_sma.h b/src/shared/mpp/include/mpp_alltoall_sma.h
index 5a8c1e6c88..afaece1459 100644
--- a/src/shared/mpp/include/mpp_alltoall_sma.h
+++ b/src/shared/mpp/include/mpp_alltoall_sma.h
@@ -1,7 +1,8 @@
-subroutine MPP_ALLTOALL_(sbuf, rbuf, pelist)
+subroutine MPP_ALLTOALL_(sbuf, scount, rbuf, rcount, pelist)
 
     MPP_TYPE_, dimension(:), intent(in) :: sbuf
     MPP_TYPE_, dimension(:), intent(inout) :: rbuf
+    integer,   intent(in) :: scount, rcount
 
     integer, intent(in), optional :: pelist(0:)
 
diff --git a/src/shared/mpp/include/mpp_chksum.h b/src/shared/mpp/include/mpp_chksum.h
index 372924d9c6..ffa7086a3d 100644
--- a/src/shared/mpp/include/mpp_chksum.h
+++ b/src/shared/mpp/include/mpp_chksum.h
@@ -1,11 +1,20 @@
-    function MPP_CHKSUM_( var, pelist )
+function MPP_CHKSUM_( var, pelist , mask_val)
 !mold is a dummy array to be used by TRANSFER()
 !must be same TYPE as result
 !result is LONG_KIND, which will actually be int ifdef no_8byte_integers
-      integer(LONG_KIND) :: MPP_CHKSUM_, mold(1)
+  !optional mask_val is masked away in checksum_int.h function via PACK()
+  integer(LONG_KIND) :: MPP_CHKSUM_
+  integer(LONG_KIND) :: mold(1)
       MPP_TYPE_, intent(in) :: var MPP_RANK_
       integer, intent(in), optional :: pelist(:)
+  MPP_TYPE_, intent(in),optional :: mask_val
 
+  if ( PRESENT(mask_val) ) then
+     MPP_CHKSUM_ = mpp_chksum( TRANSFER(var,mold), pelist, &
+          mask_val= TRANSFER(mask_val,mold(1) ) )
+  else
       MPP_CHKSUM_ = mpp_chksum( TRANSFER(var,mold), pelist )
+  end if
+  
       return
     end function MPP_CHKSUM_
diff --git a/src/shared/mpp/include/mpp_chksum_int.h b/src/shared/mpp/include/mpp_chksum_int.h
index bce804fd00..2d2109d09e 100644
--- a/src/shared/mpp/include/mpp_chksum_int.h
+++ b/src/shared/mpp/include/mpp_chksum_int.h
@@ -1,8 +1,52 @@
-    function MPP_CHKSUM_INT_( var, pelist )
-      MPP_TYPE_ :: MPP_CHKSUM_INT_
+function MPP_CHKSUM_INT_( var, pelist, mask_val )
+  integer(LONG_KIND) :: MPP_CHKSUM_INT_
       MPP_TYPE_, intent(in) :: var MPP_RANK_
       integer, optional :: pelist(:)
-      MPP_CHKSUM_INT_ = sum(var)
+  MPP_TYPE_, intent(in), optional :: mask_val
+  
+  if ( PRESENT(mask_val) ) then
+     !PACK on var/=mask_val ignores values in var
+     !equiv to setting those values=0, but on sparse arrays
+     !pack should return much smaller array to sum
+     MPP_CHKSUM_INT_ = sum( INT( PACK(var,var/=mask_val),LONG_KIND) )
+  else
+     MPP_CHKSUM_INT_ = sum(INT(var,LONG_KIND))
+  end if
+
       call mpp_sum( MPP_CHKSUM_INT_, pelist )
       return
+
     end function MPP_CHKSUM_INT_
+
+
+!Handles real mask for easier implimentation
+! until exists full integer vartypes...
+function MPP_CHKSUM_INT_RMASK_( var, pelist, mask_val )
+  integer(LONG_KIND) :: MPP_CHKSUM_INT_RMASK_
+  MPP_TYPE_, intent(in) :: var MPP_RANK_
+  integer, optional :: pelist(:)
+  real, intent(in) :: mask_val
+  integer(KIND(var))::tmpVarP
+  integer(KIND(mask_val)) :: tmpFullP,tmpZeroMasked
+
+
+  if ( KIND(mask_val) == KIND(var)) then
+     !same numBytes
+     !cast to MPP_TYPE_
+     tmpVarP = TRANSFER(mask_val , tmpVarP)
+  else if (KIND(mask_val) /=  KIND(var) ) then
+     !check if still safe to duck type to lower precision
+     tmpFullP = TRANSFER(mask_val,tmpFullP ) ! transfer bits to int mold of same numBytes
+     tmpZeroMasked = IBITS(tmpFullP,0,BIT_SIZE(tmpVarP))
+     if (tmpFullP /= tmpZeroMasked ) then
+        !as an int, mask_val is actually /using/ higher bits than var, and so not a valid mask...
+        call mpp_error(FATAL, "mpp_chksum_int.h was called with real mask_val, and mask_val can not be safely cast to int type of var (nonzero high bits).")
+     end if
+     tmpVarP = INT(tmpFullP, KIND(var) ) ! attempt cast of mask_val as int down to the int precision of var, could do transfer...
+  end if
+
+  MPP_CHKSUM_INT_RMASK_ = mpp_chksum(var,pelist,mask_val=tmpVarP)
+
+  return
+
+end function MPP_CHKSUM_INT_RMASK_
diff --git a/src/shared/mpp/include/mpp_chksum_scalar.h b/src/shared/mpp/include/mpp_chksum_scalar.h
index be992534b5..1cad513250 100644
--- a/src/shared/mpp/include/mpp_chksum_scalar.h
+++ b/src/shared/mpp/include/mpp_chksum_scalar.h
@@ -1,14 +1,21 @@
-    function MPP_CHKSUM_( var, pelist )
+function MPP_CHKSUM_( var, pelist, mask_val )
 !mold is a dummy array to be used by TRANSFER()
 !must be same TYPE as result
 !result is LONG_KIND, which will actually be int ifdef no_8byte_integers
+  !mold and mask_val must be same numBytes, otherwise undefined behavior
       integer(LONG_KIND) :: MPP_CHKSUM_
       MPP_TYPE_, intent(in) :: var
       integer, intent(in), optional :: pelist(:)
       integer(LONG_KIND) :: mold(1)
+  MPP_TYPE_, intent(in), optional :: mask_val
       pointer( p, mold )
 
       p = LOC(var)
+
+  if ( PRESENT(mask_val) ) then
+     MPP_CHKSUM_ = mpp_chksum( mold, pelist, TRANSFER(mask_val, mold(1)) )
+  else
       MPP_CHKSUM_ = mpp_chksum( mold, pelist )
+  end if
       return
     end function MPP_CHKSUM_
diff --git a/src/shared/mpp/include/mpp_comm.inc b/src/shared/mpp/include/mpp_comm.inc
index aff06aa91d..95a3d2a601 100644
--- a/src/shared/mpp/include/mpp_comm.inc
+++ b/src/shared/mpp/include/mpp_comm.inc
@@ -1,7 +1,6 @@
 
 ! -*-f90-*-
-! $Id: mpp_comm.inc,v 19.0 2012/01/06 22:00:37 fms Exp $
-
+! $Id$
 
 #ifdef use_libSMA
 #include 
@@ -14,6 +13,8 @@
 #ifndef no_8byte_integers
 #undef MPP_CHKSUM_INT_
 #define MPP_CHKSUM_INT_ mpp_chksum_i8_1d
+#undef MPP_CHKSUM_INT_RMASK_
+#define MPP_CHKSUM_INT_RMASK_ mpp_chksum_i8_1d_rmask
 #undef MPP_TYPE_
 #define MPP_TYPE_ integer(LONG_KIND)
 #undef MPP_RANK_
@@ -22,6 +23,8 @@
 
 #undef MPP_CHKSUM_INT_
 #define MPP_CHKSUM_INT_ mpp_chksum_i8_2d
+#undef MPP_CHKSUM_INT_RMASK_
+#define MPP_CHKSUM_INT_RMASK_ mpp_chksum_i8_2d_rmask
 #undef MPP_TYPE_
 #define MPP_TYPE_ integer(LONG_KIND)
 #undef MPP_RANK_
@@ -30,6 +33,8 @@
 
 #undef MPP_CHKSUM_INT_
 #define MPP_CHKSUM_INT_ mpp_chksum_i8_3d
+#undef MPP_CHKSUM_INT_RMASK_
+#define MPP_CHKSUM_INT_RMASK_ mpp_chksum_i8_3d_rmask
 #undef MPP_TYPE_
 #define MPP_TYPE_ integer(LONG_KIND)
 #undef MPP_RANK_
@@ -38,6 +43,8 @@
 
 #undef MPP_CHKSUM_INT_
 #define MPP_CHKSUM_INT_ mpp_chksum_i8_4d
+#undef MPP_CHKSUM_INT_RMASK_
+#define MPP_CHKSUM_INT_RMASK_ mpp_chksum_i8_4d_rmask
 #undef MPP_TYPE_
 #define MPP_TYPE_ integer(LONG_KIND)
 #undef MPP_RANK_
@@ -46,6 +53,8 @@
 
 #undef MPP_CHKSUM_INT_
 #define MPP_CHKSUM_INT_ mpp_chksum_i8_5d
+#undef MPP_CHKSUM_INT_RMASK_
+#define MPP_CHKSUM_INT_RMASK_ mpp_chksum_i8_5d_rmask
 #undef MPP_TYPE_
 #define MPP_TYPE_ integer(LONG_KIND)
 #undef MPP_RANK_
@@ -55,6 +64,8 @@
 
 #undef MPP_CHKSUM_INT_
 #define MPP_CHKSUM_INT_ mpp_chksum_i4_1d
+#undef MPP_CHKSUM_INT_RMASK_
+#define MPP_CHKSUM_INT_RMASK_ mpp_chksum_i4_1d_rmask
 #undef MPP_TYPE_
 #define MPP_TYPE_ integer(INT_KIND)
 #undef MPP_RANK_
@@ -63,6 +74,8 @@
 
 #undef MPP_CHKSUM_INT_
 #define MPP_CHKSUM_INT_ mpp_chksum_i4_2d
+#undef MPP_CHKSUM_INT_RMASK_
+#define MPP_CHKSUM_INT_RMASK_ mpp_chksum_i4_2d_rmask
 #undef MPP_TYPE_
 #define MPP_TYPE_ integer(INT_KIND)
 #undef MPP_RANK_
@@ -71,6 +84,8 @@
 
 #undef MPP_CHKSUM_INT_
 #define MPP_CHKSUM_INT_ mpp_chksum_i4_3d
+#undef MPP_CHKSUM_INT_RMASK_
+#define MPP_CHKSUM_INT_RMASK_ mpp_chksum_i4_3d_rmask
 #undef MPP_TYPE_
 #define MPP_TYPE_ integer(INT_KIND)
 #undef MPP_RANK_
@@ -79,6 +94,8 @@
 
 #undef MPP_CHKSUM_INT_
 #define MPP_CHKSUM_INT_ mpp_chksum_i4_4d
+#undef MPP_CHKSUM_INT_RMASK_
+#define MPP_CHKSUM_INT_RMASK_ mpp_chksum_i4_4d_rmask
 #undef MPP_TYPE_
 #define MPP_TYPE_ integer(INT_KIND)
 #undef MPP_RANK_
@@ -87,6 +104,8 @@
 
 #undef MPP_CHKSUM_INT_
 #define MPP_CHKSUM_INT_ mpp_chksum_i4_5d
+#undef MPP_CHKSUM_INT_RMASK_
+#define MPP_CHKSUM_INT_RMASK_ mpp_chksum_i4_5d_rmask
 #undef MPP_TYPE_
 #define MPP_TYPE_ integer(INT_KIND)
 #undef MPP_RANK_
@@ -293,23 +312,62 @@
 
 !#################################################
 #undef MPP_GATHER_1D_
+#undef MPP_GATHER_1DV_
 #undef MPP_TYPE_
 #define MPP_TYPE_ integer(INT_KIND)
 #define MPP_GATHER_1D_ mpp_gather_int4_1d
 #define MPP_GATHER_1DV_ mpp_gather_int4_1dv
+#undef  MPP_GATHER_PELIST_2D_
+#undef  MPP_GATHER_PELIST_3D_
+#define MPP_GATHER_PELIST_2D_ mpp_gather_pelist_int4_2d
+#define MPP_GATHER_PELIST_3D_ mpp_gather_pelist_int4_3d
 #include 
 
 #undef MPP_GATHER_1D_
+#undef MPP_GATHER_1DV_
 #undef MPP_TYPE_
 #define MPP_TYPE_ real(FLOAT_KIND)
 #define MPP_GATHER_1D_ mpp_gather_real4_1d
 #define MPP_GATHER_1DV_ mpp_gather_real4_1dv
+#undef  MPP_GATHER_PELIST_2D_
+#undef  MPP_GATHER_PELIST_3D_
+#define MPP_GATHER_PELIST_2D_ mpp_gather_pelist_real4_2d
+#define MPP_GATHER_PELIST_3D_ mpp_gather_pelist_real4_3d
 #include 
 
 #undef MPP_GATHER_1D_
+#undef MPP_GATHER_1DV_
 #undef MPP_TYPE_
 #define MPP_TYPE_ real(DOUBLE_KIND)
 #define MPP_GATHER_1D_ mpp_gather_real8_1d
 #define MPP_GATHER_1DV_ mpp_gather_real8_1dv
+#undef  MPP_GATHER_PELIST_2D_
+#undef  MPP_GATHER_PELIST_3D_
+#define MPP_GATHER_PELIST_2D_ mpp_gather_pelist_real8_2d
+#define MPP_GATHER_PELIST_3D_ mpp_gather_pelist_real8_3d
 #include 
 
+!#################################################
+#undef  MPP_SCATTER_PELIST_2D_
+#undef  MPP_SCATTER_PELIST_3D_
+#undef MPP_TYPE_
+#define MPP_TYPE_ integer(INT_KIND)
+#define MPP_SCATTER_PELIST_2D_ mpp_scatter_pelist_int4_2d
+#define MPP_SCATTER_PELIST_3D_ mpp_scatter_pelist_int4_3d
+#include 
+
+#undef  MPP_SCATTER_PELIST_2D_
+#undef  MPP_SCATTER_PELIST_3D_
+#undef MPP_TYPE_
+#define MPP_TYPE_ real(FLOAT_KIND)
+#define MPP_SCATTER_PELIST_2D_ mpp_scatter_pelist_real4_2d
+#define MPP_SCATTER_PELIST_3D_ mpp_scatter_pelist_real4_3d
+#include 
+
+#undef  MPP_SCATTER_PELIST_2D_
+#undef  MPP_SCATTER_PELIST_3D_
+#undef MPP_TYPE_
+#define MPP_TYPE_ real(DOUBLE_KIND)
+#define MPP_SCATTER_PELIST_2D_ mpp_scatter_pelist_real8_2d
+#define MPP_SCATTER_PELIST_3D_ mpp_scatter_pelist_real8_3d
+#include 
diff --git a/src/shared/mpp/include/mpp_comm_mpi.inc b/src/shared/mpp/include/mpp_comm_mpi.inc
index 3fcd991453..a8a878497f 100644
--- a/src/shared/mpp/include/mpp_comm_mpi.inc
+++ b/src/shared/mpp/include/mpp_comm_mpi.inc
@@ -1,6 +1,6 @@
 
 ! -*-f90-*-
-! $Id: mpp_comm_mpi.inc,v 20.0 2013/12/14 00:24:07 fms Exp $
+! $Id$
 
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
@@ -392,7 +392,7 @@ end subroutine mpp_gsm_free
       if( .NOT.ANY(from_pe.EQ.peset(current_peset_num)%list) ) &
            call mpp_error( FATAL, 'mpp_broadcast_text: broadcasting from invalid PE.' )
 
-      if( current_clock.NE.0 )call SYSTEM_CLOCK(start_tick)
+      if( debug .and. (current_clock.NE.0) )call SYSTEM_CLOCK(start_tick)
  ! find the rank of from_pe in the pelist.     
       do i = 1, mpp_npes()
          if(peset(n)%list(i) == from_pe) then
@@ -402,7 +402,7 @@ end subroutine mpp_gsm_free
       enddo
       lptr = LOC (data)
       if( mpp_npes().GT.1 ) call MPI_BCAST( data, length*size(data(:)), MPI_CHARACTER, from_rank, peset(n)%id, error )
-      if( current_clock.NE.0 )call increment_current_clock( EVENT_BROADCAST, length )
+      if( debug .and. (current_clock.NE.0) )call increment_current_clock( EVENT_BROADCAST, length )
       return
     end subroutine mpp_broadcast_char
 
diff --git a/src/shared/mpp/include/mpp_comm_nocomm.inc b/src/shared/mpp/include/mpp_comm_nocomm.inc
index 268a26e7f7..7aa2cee176 100644
--- a/src/shared/mpp/include/mpp_comm_nocomm.inc
+++ b/src/shared/mpp/include/mpp_comm_nocomm.inc
@@ -1,6 +1,6 @@
 
 ! -*-f90-*-
-! $Id: mpp_comm_nocomm.inc,v 20.0 2013/12/14 00:24:09 fms Exp $
+! $Id$
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 !                                                                             !
diff --git a/src/shared/mpp/include/mpp_comm_sma.inc b/src/shared/mpp/include/mpp_comm_sma.inc
index f47b299d79..5f152e3a22 100644
--- a/src/shared/mpp/include/mpp_comm_sma.inc
+++ b/src/shared/mpp/include/mpp_comm_sma.inc
@@ -1,6 +1,6 @@
 
 ! -*-f90-*-
-! $Id: mpp_comm_sma.inc,v 20.0 2013/12/14 00:24:11 fms Exp $
+! $Id$
 
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
@@ -339,7 +339,7 @@ end subroutine mpp_malloc
       if( .NOT.ANY(from_pe.EQ.peset(current_peset_num)%list) ) &
            call mpp_error( FATAL, 'mpp_broadcast_text: broadcasting from invalid PE.' )
 
-      if( current_clock.NE.0 )call SYSTEM_CLOCK(start_tick)
+      if( debug .and. (current_clock.NE.0) )call SYSTEM_CLOCK(start_tick)
 
       ptr = LOC(mpp_stack)
       words = size(bdata(:))*size(transfer(bdata(1),word))
@@ -365,7 +365,7 @@ end subroutine mpp_malloc
           end do
       end if
 
-      if( current_clock.NE.0 )call increment_current_clock( EVENT_BROADCAST, length )
+      if( debug .and. (current_clock.NE.0) )call increment_current_clock( EVENT_BROADCAST, length )
       return
     end subroutine mpp_broadcast_char
 
diff --git a/src/shared/mpp/include/mpp_data_mpi.inc b/src/shared/mpp/include/mpp_data_mpi.inc
index 09053cbce0..0351dbb1dc 100644
--- a/src/shared/mpp/include/mpp_data_mpi.inc
+++ b/src/shared/mpp/include/mpp_data_mpi.inc
@@ -1,6 +1,6 @@
 
 ! -*-f90-*-
-! $Id: mpp_data_mpi.inc,v 19.0 2012/01/06 22:00:45 fms Exp $
+! $Id$
 
 !----------------------------------------------------------------!
 ! The following data is used in mpp_mod and its components       !
diff --git a/src/shared/mpp/include/mpp_data_nocomm.inc b/src/shared/mpp/include/mpp_data_nocomm.inc
index 5417b286af..702edcc0b6 100644
--- a/src/shared/mpp/include/mpp_data_nocomm.inc
+++ b/src/shared/mpp/include/mpp_data_nocomm.inc
@@ -1,6 +1,6 @@
 
 ! -*-f90-*-
-! $Id: mpp_data_nocomm.inc,v 19.0 2012/01/06 22:01:17 fms Exp $
+! $Id$
 
 !----------------------------------------------------------------!
 ! The following data is used in mpp_mod and its components       !
diff --git a/src/shared/mpp/include/mpp_data_sma.inc b/src/shared/mpp/include/mpp_data_sma.inc
index 4c1f28a29e..29afd40361 100644
--- a/src/shared/mpp/include/mpp_data_sma.inc
+++ b/src/shared/mpp/include/mpp_data_sma.inc
@@ -1,6 +1,6 @@
 
 ! -*-f90-*-
-! $Id: mpp_data_sma.inc,v 19.0 2012/01/06 22:01:19 fms Exp $
+! $Id$
 
 !----------------------------------------------------------------!
 ! The following data is used in mpp_mod and its components       !
diff --git a/src/shared/mpp/include/mpp_define_nest_domains.inc b/src/shared/mpp/include/mpp_define_nest_domains.inc
index 8b88199551..77c38a8095 100644
--- a/src/shared/mpp/include/mpp_define_nest_domains.inc
+++ b/src/shared/mpp/include/mpp_define_nest_domains.inc
@@ -1,5 +1,5 @@
 ! -*-f90-*-
-! $Id: mpp_define_nest_domains.inc,v 20.0 2013/12/14 00:24:16 fms Exp $
+! $Id$
 
 
 !#############################################################################
diff --git a/src/shared/mpp/include/mpp_do_get_boundary.h b/src/shared/mpp/include/mpp_do_get_boundary.h
index 3cd3aa3045..908bc12831 100644
--- a/src/shared/mpp/include/mpp_do_get_boundary.h
+++ b/src/shared/mpp/include/mpp_do_get_boundary.h
@@ -309,7 +309,7 @@ end subroutine MPP_DO_GET_BOUNDARY_3D_
 
 
 subroutine MPP_DO_GET_BOUNDARY_3D_V_(f_addrsx, f_addrsy, domain, boundx, boundy, b_addrsx, b_addrsy, &
-                                        bsizex, bsizey, ke, d_type, flags)
+                                        bsizex, bsizey, ke, d_type, flags, gridtype)
   type(domain2D),     intent(in)  :: domain
   type(overlapSpec),  intent(in)  :: boundx, boundy
   integer(LONG_KIND), intent(in)  :: f_addrsx(:,:), f_addrsy(:,:)
@@ -317,6 +317,7 @@ subroutine MPP_DO_GET_BOUNDARY_3D_V_(f_addrsx, f_addrsy, domain, boundx, boundy,
   integer,            intent(in)  :: bsizex(:), bsizey(:), ke
   MPP_TYPE_, intent(in)           :: d_type  ! creates unique interface
   integer, intent(in)             :: flags
+  integer, intent(in)             :: gridtype
 
   MPP_TYPE_ :: fieldx(boundx%xbegin:boundx%xend, boundx%ybegin:boundx%yend,ke)
   MPP_TYPE_ :: fieldy(boundy%xbegin:boundy%xend, boundy%ybegin:boundy%yend,ke)
@@ -342,7 +343,7 @@ subroutine MPP_DO_GET_BOUNDARY_3D_V_(f_addrsx, f_addrsy, domain, boundx, boundy,
   integer                 :: rank_x, rank_y, cur_rank, ind_x, ind_y
   integer                 :: nsend_x, nsend_y, nrecv_x, nrecv_y, num
   character(len=8)        :: text
-  integer                 :: outunit
+  integer                 :: outunit, shift, midpoint
 
   MPP_TYPE_ :: buffer(size(mpp_domains_stack(:)))
   pointer( ptr, buffer )
@@ -989,6 +990,33 @@ subroutine MPP_DO_GET_BOUNDARY_3D_V_(f_addrsx, f_addrsy, domain, boundx, boundy,
      cur_rank = min(rank_x, rank_y)
   end do
 
+  !--- domain always is symmetry
+  shift = 1
+  tMe = 1
+  if( BTEST(domain%fold,NORTH) .AND. (.NOT.BTEST(flags,SCALAR_BIT)) )then
+     j = domain%y(1)%global%end+shift
+     if( domain%y(1)%data%begin.LE.j .AND. j.LE.domain%y(1)%data%end+shift )then !fold is within domain
+        !poles set to 0: BGRID only
+        if( gridtype.EQ.BGRID_NE )then
+           midpoint = (domain%x(1)%global%begin+domain%x(1)%global%end-1+shift)/2
+           j  = domain%y(1)%global%end+shift - domain%y(1)%compute%begin + 1
+           is = domain%x(1)%global%begin; ie = domain%x(1)%global%end+shift
+           do i = is ,ie, midpoint
+              if( domain%x(1)%compute%begin == i )then
+                 do l=1,l_size
+                    ptr_wbufferx = b_addrsx(3, l, tMe)     
+                    ptr_wbuffery = b_addrsy(3, l, tMe)     
+                    do k = 1,ke
+                       wbufferx(j,k) = 0
+                       wbuffery(j,k) = 0
+                    end do
+                 end do
+              end if
+           end do
+        endif
+     endif
+  endif
+
   call mpp_sync_self( )
 
 
diff --git a/src/shared/mpp/include/mpp_do_update.h b/src/shared/mpp/include/mpp_do_update.h
index 9b74e67068..48a58348e4 100644
--- a/src/shared/mpp/include/mpp_do_update.h
+++ b/src/shared/mpp/include/mpp_do_update.h
@@ -1,19 +1,15 @@
 ! -*-f90-*- 
-    subroutine MPP_DO_UPDATE_3D_( f_addrs, domain, update, d_type, ke, b_addrs, b_size, flags)
+    subroutine MPP_DO_UPDATE_3D_( f_addrs, domain, update, d_type, ke, flags)
 !updates data domain of 3D field whose computational domains have been computed
       integer(LONG_KIND),         intent(in) :: f_addrs(:,:)
       type(domain2D),             intent(in) :: domain
       type(overlapSpec),          intent(in) :: update
       MPP_TYPE_,                  intent(in) :: d_type  ! creates unique interface
       integer,                    intent(in) :: ke
-      integer(LONG_KIND),         intent(in) :: b_addrs(:,:)
-      integer,                    intent(in) :: b_size
       integer, optional,          intent(in) :: flags
 
       MPP_TYPE_ :: field(update%xbegin:update%xend, update%ybegin:update%yend,ke)
-      MPP_TYPE_ :: fillbuffer(b_size)
       pointer(ptr_field, field)
-      pointer(ptr_buffer, fillbuffer)
       integer                     :: update_flags
       type(overlap_type), pointer :: overPtr => NULL()      
       character(len=8)            :: text
@@ -174,7 +170,8 @@
                tMe = overPtr%tileMe(n)
                is = overPtr%is(n); ie = overPtr%ie(n)
                js = overPtr%js(n); je = overPtr%je(n)
-               if( overptr%is_refined(n) ) then
+               select case( overPtr%rotation(n) )
+               case(ZERO)
                   do l=1,l_size  ! loop over number of fields
                      ptr_field = f_addrs(l, tMe)
                      do k = 1,ke  
@@ -186,58 +183,43 @@
                         end do
                      end do
                   end do
-               else
-                  select case( overPtr%rotation(n) )
-                  case(ZERO)
-                     do l=1,l_size  ! loop over number of fields
-                        ptr_field = f_addrs(l, tMe)
-                        do k = 1,ke  
-                           do j = js, je
-                              do i = is, ie
-                                 pos = pos + 1
-                                 buffer(pos) = field(i,j,k)
-                              end do
+               case( MINUS_NINETY ) 
+                  do l=1,l_size  ! loop over number of fields
+                     ptr_field = f_addrs(l, tMe)
+                     do k = 1,ke  
+                        do i = is, ie
+                           do j = je, js, -1
+                              pos = pos + 1
+                              buffer(pos) = field(i,j,k)
                            end do
                         end do
                      end do
-                  case( MINUS_NINETY ) 
-                     do l=1,l_size  ! loop over number of fields
-                        ptr_field = f_addrs(l, tMe)
-                        do k = 1,ke  
-                           do i = is, ie
-                              do j = je, js, -1
-                                 pos = pos + 1
-                                 buffer(pos) = field(i,j,k)
-                              end do
+                  end do
+               case( NINETY ) 
+                  do l=1,l_size  ! loop over number of fields
+                     ptr_field = f_addrs(l, tMe)
+                     do k = 1,ke  
+                        do i = ie, is, -1
+                           do j = js, je
+                              pos = pos + 1
+                              buffer(pos) = field(i,j,k)
                            end do
                         end do
                      end do
-                  case( NINETY ) 
-                     do l=1,l_size  ! loop over number of fields
-                        ptr_field = f_addrs(l, tMe)
-                        do k = 1,ke  
+                  end do
+               case( ONE_HUNDRED_EIGHTY ) 
+                  do l=1,l_size  ! loop over number of fields
+                     ptr_field = f_addrs(l, tMe)
+                     do k = 1,ke  
+                        do j = je, js, -1
                            do i = ie, is, -1
-                              do j = js, je
-                                 pos = pos + 1
-                                 buffer(pos) = field(i,j,k)
-                              end do
-                           end do
-                        end do
-                     end do
-                  case( ONE_HUNDRED_EIGHTY ) 
-                     do l=1,l_size  ! loop over number of fields
-                        ptr_field = f_addrs(l, tMe)
-                        do k = 1,ke  
-                           do j = je, js, -1
-                              do i = ie, is, -1
-                                 pos = pos + 1
-                                 buffer(pos) = field(i,j,k)
-                              end do
+                              pos = pos + 1
+                              buffer(pos) = field(i,j,k)
                            end do
                         end do
                      end do
-                  end select
-               end if
+                  end do
+               end select
             endif
          end do ! do n = 1, overPtr%count
          send_msgsize(m) = pos-buffer_pos
@@ -278,44 +260,18 @@
                msgsize = (ie-is+1)*(je-js+1)*ke*l_size
                pos = buffer_pos - msgsize
                buffer_pos = pos
-               if(OverPtr%is_refined(n)) then
-                  index = overPtr%index(n)
-                  is1 = update%rSpec(tMe)%isNbr(index); ie1 = update%rSpec(tMe)%ieNbr(index)
-                  js1 = update%rSpec(tMe)%jsNbr(index); je1 = update%rSpec(tMe)%jeNbr(index)
-                  ni = ie1 - is1 + 1
-                  nj = je1 - js1 + 1
-                  total = ni*nj
-                  start = (update%rSpec(tMe)%start(index)-1)*ke
-                  if(start+total*ke>b_size ) call mpp_error(FATAL, &
-                       "MPP_DO_UPDATE: b_size is less than the size of the data to be filled.")
-                  msgsize = ie - is + 1
-                  do l=1, l_size  ! loop over number of fields
-                     ptr_buffer = b_addrs(l, tMe)
-                     start1 = start + (js-js1)*ni + is - is1
-                     do k = 1, ke
-                        start2 = start1
-                        do j = js, je
-                           fillbuffer(start2+1:start2+msgsize) = buffer(pos+1:pos+msgsize)
-                           start2 = start2 + ni
-                           pos   = pos + msgsize
+               do l=1,l_size  ! loop over number of fields
+                  ptr_field = f_addrs(l, tMe)
+                  do k = 1,ke
+                     do j = js, je
+                        do i = is, ie
+                           pos = pos + 1
+                           field(i,j,k) = buffer(pos)
                         end do
-                        start1 = start1 + total
                      end do
                   end do
-               else
-                  do l=1,l_size  ! loop over number of fields
-                     ptr_field = f_addrs(l, tMe)
-                     do k = 1,ke
-                        do j = js, je
-                           do i = is, ie
-                              pos = pos + 1
-                              field(i,j,k) = buffer(pos)
-                           end do
-                        end do
-                     end do
-                  end do
-               endif
-            end if
+               end do
+            endif
          end do ! do n = 1, overPtr%count
       end do
       call mpp_clock_end(unpk_clock)
diff --git a/src/shared/mpp/include/mpp_do_updateV.h b/src/shared/mpp/include/mpp_do_updateV.h
index 648843b553..90c4e41d13 100644
--- a/src/shared/mpp/include/mpp_do_updateV.h
+++ b/src/shared/mpp/include/mpp_do_updateV.h
@@ -1,37 +1,30 @@
 ! -*-f90-*- 
     subroutine MPP_DO_UPDATE_3D_V_(f_addrsx,f_addrsy, domain, update_x, update_y, &
-                                   d_type, ke, b_addrsx, b_addrsy, b_sizex, b_sizey, gridtype, flags)
+                                   d_type, ke, gridtype, flags)
 !updates data domain of 3D field whose computational domains have been computed
       integer(LONG_KIND),  intent(in)        :: f_addrsx(:,:), f_addrsy(:,:)
       type(domain2d),      intent(in)        :: domain
       type(overlapSpec),   intent(in)        :: update_x, update_y
       integer,             intent(in)        :: ke
       MPP_TYPE_, intent(in)                  :: d_type  ! creates unique interface
-      integer(LONG_KIND), intent(in)         :: b_addrsx(:,:), b_addrsy(:,:)
-      integer,            intent(in)         :: b_sizex, b_sizey
       integer, intent(in)                    :: gridtype
       integer, intent(in),          optional :: flags
 
       MPP_TYPE_ :: fieldx(update_x%xbegin:update_x%xend, update_x%ybegin:update_x%yend,ke)
       MPP_TYPE_ :: fieldy(update_y%xbegin:update_y%xend, update_y%ybegin:update_y%yend,ke)
-      MPP_TYPE_ :: bufferx(b_sizex)
-      MPP_TYPE_ :: buffery(b_sizey)
       pointer(ptr_fieldx, fieldx)
       pointer(ptr_fieldy, fieldy)
-      pointer(ptr_bufferx, bufferx)
-      pointer(ptr_buffery, buffery)
 
       integer :: update_flags
       integer :: l_size, l, i, j, k, is, ie, js, je, n, m
       integer :: pos, nlist, msgsize
       integer :: to_pe, from_pe, midpoint
       integer :: tMe, dir
-      integer :: index, is1, ie1, js1, je1, ni, nj, total, start1, start, start2
 
       integer :: send_start_pos, nsend
       integer :: send_msgsize(2*MAXLIST)
       integer :: send_pe(2*MAXLIST)
-      integer, allocatable :: msg1(:), msg2(:)
+      integer,    allocatable :: msg1(:), msg2(:)
       logical :: send(8), recv(8), update_edge_only
       MPP_TYPE_ :: buffer(size(mpp_domains_stack(:)))
       pointer(ptr,buffer )
@@ -322,221 +315,112 @@
 
                      is = update_x%send(ind_x)%is(n); ie = update_x%send(ind_x)%ie(n)
                      js = update_x%send(ind_x)%js(n); je = update_x%send(ind_x)%je(n)
-                     if( update_x%send(ind_x)%is_refined(n) ) then
-                        select case( update_x%send(ind_x)%rotation(n) )
-                        case(ZERO)
+                     select case( update_x%send(ind_x)%rotation(n) )
+                     case(ZERO)
+                        do l=1,l_size  ! loop over number of fields
+                           ptr_fieldx = f_addrsx(l,tMe)
+                           ptr_fieldy = f_addrsy(l,tMe)
+                           do k = 1,ke
+                              do j = js, je
+                                 do i = is, ie
+                                    pos = pos + 2
+                                    buffer(pos-1) = fieldx(i,j,k)
+                                    buffer(pos)   = fieldy(i,j,k)
+                                 end do
+                              end do
+                           end do
+                        end do
+                     case( MINUS_NINETY ) 
+                        if( BTEST(update_flags,SCALAR_BIT) ) then
                            do l=1,l_size  ! loop over number of fields
                               ptr_fieldx = f_addrsx(l,tMe)
                               ptr_fieldy = f_addrsy(l,tMe)
                               do k = 1,ke
-                                 do j = js, je
-                                    do i = is, ie
+                                 do i = is, ie
+                                    do j = je, js, -1
                                        pos = pos + 2
-                                       buffer(pos-1) = fieldx(i,j,k)
-                                       buffer(pos)   = fieldy(i,j,k)
+                                       buffer(pos-1) = fieldy(i,j,k)
+                                       buffer(pos)   = fieldx(i,j,k)
                                     end do
                                  end do
                               end do
                            end do
-                        case(MINUS_NINETY)
-                           if( BTEST(update_flags,SCALAR_BIT) ) then
-                              do l=1,l_size  ! loop over number of fields
-                                 ptr_fieldx = f_addrsx(l,tMe)
-                                 ptr_fieldy = f_addrsy(l,tMe)
-                                 do k = 1,ke
-                                    do j = js, je
-                                       do i = is, ie
-                                          pos = pos + 2
-                                          buffer(pos-1) =  fieldy(i,j,k)
-                                          buffer(pos)   =  fieldx(i,j,k)
-                                       end do
-                                    end do
-                                 end do
-                              end do
-                           else
-                              do l=1,l_size  ! loop over number of fields
-                                 ptr_fieldx = f_addrsx(l,tMe)
-                                 ptr_fieldy = f_addrsy(l,tMe)
-                                 do k = 1,ke
-                                    do j = js, je
-                                       do i = is, ie
-                                          pos = pos + 2
-                                          buffer(pos-1) = -fieldy(i,j,k)
-                                          buffer(pos)   =  fieldx(i,j,k)
-                                       end do
-                                    end do
-                                 end do
-                              end do
-                           end if
-                        case(NINETY)
-                           if( BTEST(update_flags,SCALAR_BIT) ) then
-                              do l=1,l_size  ! loop over number of fields
-                                 ptr_fieldx = f_addrsx(l,tMe)
-                                 ptr_fieldy = f_addrsy(l,tMe)
-                                 do k = 1,ke
-                                    do j = js, je
-                                       do i = is, ie
-                                          pos = pos + 2
-                                          buffer(pos-1) =  fieldy(i,j,k)
-                                          buffer(pos)   =  fieldx(i,j,k)
-                                       end do
-                                    end do
-                                 end do
-                              end do
-                           else
-                              do l=1,l_size  ! loop over number of fields
-                                 ptr_fieldx = f_addrsx(l,tMe)
-                                 ptr_fieldy = f_addrsy(l,tMe)
-                                 do k = 1,ke
-                                    do j = js, je
-                                       do i = is, ie
-                                          pos = pos + 2
-                                          buffer(pos-1) =  fieldy(i,j,k)
-                                          buffer(pos)   = -fieldx(i,j,k)
-                                       end do
-                                    end do
-                                 end do
-                              end do
-                           end if
-                        case(ONE_HUNDRED_EIGHTY)
-                           if( BTEST(update_flags,SCALAR_BIT) ) then
-                              do l=1,l_size  ! loop over number of fields
-                                 ptr_fieldx = f_addrsx(l,tMe)
-                                 ptr_fieldy = f_addrsy(l,tMe)
-                                 do k = 1,ke
-                                    do j = js, je
-                                       do i = is, ie
-                                          pos = pos + 2
-                                          buffer(pos-1) = fieldx(i,j,k)
-                                          buffer(pos)   = fieldy(i,j,k)
-                                       end do
+                        else
+                           do l=1,l_size  ! loop over number of fields
+                              ptr_fieldx = f_addrsx(l,tMe)
+                              ptr_fieldy = f_addrsy(l,tMe)
+                              do k = 1,ke
+                                 do i = is, ie
+                                    do j = je, js, -1
+                                       pos = pos + 2
+                                       buffer(pos-1) = -fieldy(i,j,k)
+                                       buffer(pos)   =  fieldx(i,j,k)
                                     end do
                                  end do
                               end do
-                           else
-                              do l=1,l_size  ! loop over number of fields
-                                 ptr_fieldx = f_addrsx(l,tMe)
-                                 ptr_fieldy = f_addrsy(l,tMe)
-                                 do k = 1,ke
+                           end do
+                        end if
+                     case( NINETY )
+                        if( BTEST(update_flags,SCALAR_BIT) ) then
+                           do l=1,l_size  ! loop over number of fields
+                              ptr_fieldx = f_addrsx(l,tMe)
+                              ptr_fieldy = f_addrsy(l,tMe)
+                              do k = 1,ke
+                                 do i = ie, is, -1
                                     do j = js, je
-                                       do i = is, ie
-                                          pos = pos + 2
-                                          buffer(pos-1) = -fieldx(i,j,k)
-                                          buffer(pos)   = -fieldy(i,j,k)
-                                       end do
+                                       pos = pos + 2
+                                       buffer(pos-1) = fieldy(i,j,k)
+                                       buffer(pos)   = fieldx(i,j,k)
                                     end do
                                  end do
                               end do
-                           end if
-                        end select  ! select case( rotation(n) )
-                     else   ! if( is_refined(n) )
-                        select case( update_x%send(ind_x)%rotation(n) )
-                        case(ZERO)
+                           end do
+                        else
                            do l=1,l_size  ! loop over number of fields
                               ptr_fieldx = f_addrsx(l,tMe)
                               ptr_fieldy = f_addrsy(l,tMe)
                               do k = 1,ke
-                                 do j = js, je
-                                    do i = is, ie
+                                 do i = ie, is, -1
+                                    do j = js, je
                                        pos = pos + 2
-                                       buffer(pos-1) = fieldx(i,j,k)
-                                       buffer(pos)   = fieldy(i,j,k)
+                                       buffer(pos-1) = fieldy(i,j,k)
+                                       buffer(pos)   = -fieldx(i,j,k)
                                     end do
                                  end do
                               end do
                            end do
-                        case( MINUS_NINETY ) 
-                           if( BTEST(update_flags,SCALAR_BIT) ) then
-                              do l=1,l_size  ! loop over number of fields
-                                 ptr_fieldx = f_addrsx(l,tMe)
-                                 ptr_fieldy = f_addrsy(l,tMe)
-                                 do k = 1,ke
-                                    do i = is, ie
-                                       do j = je, js, -1
-                                          pos = pos + 2
-                                          buffer(pos-1) = fieldy(i,j,k)
-                                          buffer(pos)   = fieldx(i,j,k)
-                                       end do
-                                    end do
-                                 end do
-                              end do
-                           else
-                              do l=1,l_size  ! loop over number of fields
-                                 ptr_fieldx = f_addrsx(l,tMe)
-                                 ptr_fieldy = f_addrsy(l,tMe)
-                                 do k = 1,ke
-                                    do i = is, ie
-                                       do j = je, js, -1
-                                          pos = pos + 2
-                                          buffer(pos-1) = -fieldy(i,j,k)
-                                          buffer(pos)   =  fieldx(i,j,k)
-                                       end do
-                                    end do
-                                 end do
-                              end do
-                           end if
-                        case( NINETY )
-                           if( BTEST(update_flags,SCALAR_BIT) ) then
-                              do l=1,l_size  ! loop over number of fields
-                                 ptr_fieldx = f_addrsx(l,tMe)
-                                 ptr_fieldy = f_addrsy(l,tMe)
-                                 do k = 1,ke
+                        end if
+                     case( ONE_HUNDRED_EIGHTY )
+                        if( BTEST(update_flags,SCALAR_BIT) ) then
+                           do l=1,l_size  ! loop over number of fields
+                              ptr_fieldx = f_addrsx(l,tMe)
+                              ptr_fieldy = f_addrsy(l,tMe)
+                              do k = 1,ke
+                                 do j = je, js, -1
                                     do i = ie, is, -1
-                                       do j = js, je
-                                          pos = pos + 2
-                                          buffer(pos-1) = fieldy(i,j,k)
-                                          buffer(pos)   = fieldx(i,j,k)
-                                       end do
+                                       pos = pos + 2
+                                       buffer(pos-1) =  fieldx(i,j,k)
+                                       buffer(pos)   =  fieldy(i,j,k)
                                     end do
                                  end do
                               end do
-                           else
-                              do l=1,l_size  ! loop over number of fields
-                                 ptr_fieldx = f_addrsx(l,tMe)
-                                 ptr_fieldy = f_addrsy(l,tMe)
-                                 do k = 1,ke
+                           end do
+                        else
+                           do l=1,l_size  ! loop over number of fields
+                              ptr_fieldx = f_addrsx(l,tMe)
+                              ptr_fieldy = f_addrsy(l,tMe)
+                              do k = 1,ke
+                                 do j = je, js, -1
                                     do i = ie, is, -1
-                                       do j = js, je
-                                          pos = pos + 2
-                                          buffer(pos-1) = fieldy(i,j,k)
-                                          buffer(pos)   = -fieldx(i,j,k)
-                                       end do
-                                    end do
-                                 end do
-                              end do
-                           end if
-                        case( ONE_HUNDRED_EIGHTY )
-                           if( BTEST(update_flags,SCALAR_BIT) ) then
-                              do l=1,l_size  ! loop over number of fields
-                                 ptr_fieldx = f_addrsx(l,tMe)
-                                 ptr_fieldy = f_addrsy(l,tMe)
-                                 do k = 1,ke
-                                    do j = je, js, -1
-                                       do i = ie, is, -1
-                                          pos = pos + 2
-                                          buffer(pos-1) =  fieldx(i,j,k)
-                                          buffer(pos)   =  fieldy(i,j,k)
-                                       end do
-                                    end do
-                                 end do
-                              end do
-                           else
-                              do l=1,l_size  ! loop over number of fields
-                                 ptr_fieldx = f_addrsx(l,tMe)
-                                 ptr_fieldy = f_addrsy(l,tMe)
-                                 do k = 1,ke
-                                    do j = je, js, -1
-                                       do i = ie, is, -1
-                                          pos = pos + 2
-                                          buffer(pos-1) =  -fieldx(i,j,k)
-                                          buffer(pos)   =  -fieldy(i,j,k)
-                                       end do
+                                       pos = pos + 2
+                                       buffer(pos-1) =  -fieldx(i,j,k)
+                                       buffer(pos)   =  -fieldy(i,j,k)
                                     end do
                                  end do
                               end do
-                           end if
-                        end select ! select case( rotation(n) )
-                     end if ! if( is_refined(n) )
+                           end do
+                        end if
+                     end select ! select case( rotation(n) )
                   end if ! if( send(dir) ) 
                end do ! do n = 1, update_x%send(ind_x)%count
                ind_x = ind_x+1
@@ -558,162 +442,90 @@
                      tMe = update_x%send(ind_x)%tileMe(n)
                      is = update_x%send(ind_x)%is(n); ie = update_x%send(ind_x)%ie(n)
                      js = update_x%send(ind_x)%js(n); je = update_x%send(ind_x)%je(n)
-                     if( update_x%send(ind_x)%is_refined(n) ) then
-                        select case( update_x%send(ind_x)%rotation(n) )
-                        case(ZERO)
-                           do l=1,l_size  ! loop over number of fields
-                              ptr_fieldx = f_addrsx(l, tMe)
-                              ptr_fieldy = f_addrsy(l, tMe)
-                              do k = 1,ke
-                                 do j = js, je
-                                    do i = is, ie
-                                       pos = pos + 1
-                                       buffer(pos) = fieldx(i,j,k)
-                                    end do
+                     select case( update_x%send(ind_x)%rotation(n) )
+                     case(ZERO)
+                        do l=1,l_size  ! loop over number of fields
+                           ptr_fieldx = f_addrsx(l, tMe)
+                           ptr_fieldy = f_addrsy(l, tMe)
+                           do k = 1,ke
+                              do j = js, je
+                                 do i = is, ie
+                                    pos = pos + 1
+                                    buffer(pos) = fieldx(i,j,k)
                                  end do
                               end do
                            end do
-                        case(MINUS_NINETY)
-                           if( BTEST(update_flags,SCALAR_BIT) ) then
-                              do l=1,l_size  ! loop over number of fields
-                                 ptr_fieldx = f_addrsx(l, tMe)
-                                 ptr_fieldy = f_addrsy(l, tMe)
-                                 do k = 1,ke
-                                    do j = js, je
-                                       do i = is, ie
-                                          pos = pos + 1
-                                          buffer(pos) = fieldy(i,j,k)
-                                       end do
-                                    end do
-                                 end do
-                              end do
-                           else
-                              do l=1,l_size  ! loop over number of fields
-                                 ptr_fieldx = f_addrsx(l, tMe)
-                                 ptr_fieldy = f_addrsy(l, tMe)
-                                 do k = 1,ke
-                                    do j = js, je
-                                       do i = is, ie
-                                          pos = pos + 1
-                                          buffer(pos) = -fieldy(i,j,k)
-                                       end do
-                                    end do
-                                 end do
-                              end do
-                           end if
-                        case(NINETY)
+                        end do
+                     case(MINUS_NINETY)
+                        if( BTEST(update_flags,SCALAR_BIT) ) then
                            do l=1,l_size  ! loop over number of fields
                               ptr_fieldx = f_addrsx(l, tMe)
                               ptr_fieldy = f_addrsy(l, tMe)
-                              do k = 1, ke
-                                 do j = js, je
-                                    do i = is, ie
+                              do k = 1,ke
+                                 do i = is, ie
+                                    do j = je, js, -1
                                        pos = pos + 1
                                        buffer(pos) = fieldy(i,j,k)
                                     end do
                                  end do
                               end do
                            end do
-                        case(ONE_HUNDRED_EIGHTY)
+                        else
                            do l=1,l_size  ! loop over number of fields
                               ptr_fieldx = f_addrsx(l, tMe)
                               ptr_fieldy = f_addrsy(l, tMe)
                               do k = 1,ke
-                                 do j = js, je
-                                    do i = is, ie
+                                 do i = is, ie
+                                    do j = je, js, -1
                                        pos = pos + 1
-                                       buffer(pos) = fieldx(i,j,k)
+                                       buffer(pos) = -fieldy(i,j,k)
                                     end do
                                  end do
                               end do
                            end do
-                        end select
-                     else
-                        select case( update_x%send(ind_x)%rotation(n) )
-                        case(ZERO)
+                        end if
+                     case(NINETY)
+                        do l=1,l_size  ! loop over number of fields
+                           ptr_fieldx = f_addrsx(l, tMe)
+                           ptr_fieldy = f_addrsy(l, tMe)
+                           do k = 1, ke
+                              do i = ie, is, -1
+                                 do j = js, je
+                                    pos = pos + 1
+                                    buffer(pos) = fieldy(i,j,k)
+                                 end do
+                              end do
+                           end do
+                        end do
+                     case(ONE_HUNDRED_EIGHTY)
+                        if( BTEST(update_flags,SCALAR_BIT) ) then
                            do l=1,l_size  ! loop over number of fields
                               ptr_fieldx = f_addrsx(l, tMe)
                               ptr_fieldy = f_addrsy(l, tMe)
                               do k = 1,ke
-                                 do j = js, je
-                                    do i = is, ie
+                                 do j = je, js, -1
+                                    do i = ie, is, -1
                                        pos = pos + 1
                                        buffer(pos) = fieldx(i,j,k)
                                     end do
                                  end do
                               end do
                            end do
-                        case(MINUS_NINETY)
-                           if( BTEST(update_flags,SCALAR_BIT) ) then
-                              do l=1,l_size  ! loop over number of fields
-                                 ptr_fieldx = f_addrsx(l, tMe)
-                                 ptr_fieldy = f_addrsy(l, tMe)
-                                 do k = 1,ke
-                                    do i = is, ie
-                                       do j = je, js, -1
-                                          pos = pos + 1
-                                          buffer(pos) = fieldy(i,j,k)
-                                       end do
-                                    end do
-                                 end do
-                              end do
-                           else
-                              do l=1,l_size  ! loop over number of fields
-                                 ptr_fieldx = f_addrsx(l, tMe)
-                                 ptr_fieldy = f_addrsy(l, tMe)
-                                 do k = 1,ke
-                                    do i = is, ie
-                                       do j = je, js, -1
-                                          pos = pos + 1
-                                          buffer(pos) = -fieldy(i,j,k)
-                                       end do
-                                    end do
-                                 end do
-                              end do
-                           end if
-                        case(NINETY)
+                        else
                            do l=1,l_size  ! loop over number of fields
                               ptr_fieldx = f_addrsx(l, tMe)
                               ptr_fieldy = f_addrsy(l, tMe)
-                              do k = 1, ke
-                                 do i = ie, is, -1
-                                    do j = js, je
+                              do k = 1,ke
+                                 do j = je, js, -1
+                                    do i = ie, is, -1
                                        pos = pos + 1
-                                       buffer(pos) = fieldy(i,j,k)
+                                       buffer(pos) = -fieldx(i,j,k)
                                     end do
                                  end do
                               end do
                            end do
-                        case(ONE_HUNDRED_EIGHTY)
-                           if( BTEST(update_flags,SCALAR_BIT) ) then
-                              do l=1,l_size  ! loop over number of fields
-                                 ptr_fieldx = f_addrsx(l, tMe)
-                                 ptr_fieldy = f_addrsy(l, tMe)
-                                 do k = 1,ke
-                                    do j = je, js, -1
-                                       do i = ie, is, -1
-                                          pos = pos + 1
-                                          buffer(pos) = fieldx(i,j,k)
-                                       end do
-                                    end do
-                                 end do
-                              end do
-                           else
-                              do l=1,l_size  ! loop over number of fields
-                                 ptr_fieldx = f_addrsx(l, tMe)
-                                 ptr_fieldy = f_addrsy(l, tMe)
-                                 do k = 1,ke
-                                    do j = je, js, -1
-                                       do i = ie, is, -1
-                                          pos = pos + 1
-                                          buffer(pos) = -fieldx(i,j,k)
-                                       end do
-                                    end do
-                                 end do
-                              end do
-                           end if
-                        end select
-                     end if
+                        end if
+                     end select
                   end if
                end do
                ind_x = ind_x+1
@@ -732,162 +544,90 @@
                      tMe = update_y%send(ind_y)%tileMe(n)
                      is = update_y%send(ind_y)%is(n); ie = update_y%send(ind_y)%ie(n)
                      js = update_y%send(ind_y)%js(n); je = update_y%send(ind_y)%je(n)
-                     if( update_y%send(ind_y)%is_refined(n) ) then
-                        select case( update_y%send(ind_y)%rotation(n) )
-                        case(ZERO)
-                           do l=1,l_size  ! loop over number of fields
-                              ptr_fieldx = f_addrsx(l, tMe)
-                              ptr_fieldy = f_addrsy(l, tMe)
-                              do k = 1,ke
-                                 do j = js, je
-                                    do i = is, ie
-                                       pos = pos + 1
-                                       buffer(pos) = fieldy(i,j,k)
-                                    end do
+                     select case( update_y%send(ind_y)%rotation(n) )
+                     case(ZERO)
+                        do l=1,l_size  ! loop over number of fields
+                           ptr_fieldx = f_addrsx(l, tMe)
+                           ptr_fieldy = f_addrsy(l, tMe)
+                           do k = 1,ke
+                              do j = js, je
+                                 do i = is, ie
+                                    pos = pos + 1
+                                    buffer(pos) = fieldy(i,j,k)
                                  end do
                               end do
                            end do
-                        case(MINUS_NINETY)
+                        end do
+                     case(MINUS_NINETY)
+                        do l=1,l_size  ! loop over number of fields
+                           ptr_fieldx = f_addrsx(l, tMe)
+                           ptr_fieldy = f_addrsy(l, tMe)
+                           do k = 1,ke
+                              do i = is, ie
+                                 do j = je, js, -1
+                                    pos = pos + 1
+                                    buffer(pos) = fieldx(i,j,k)
+                                 end do
+                              end do
+                           end do
+                        end do
+                     case(NINETY)
+                        if( BTEST(update_flags,SCALAR_BIT) ) then
                            do l=1,l_size  ! loop over number of fields
                               ptr_fieldx = f_addrsx(l, tMe)
                               ptr_fieldy = f_addrsy(l, tMe)
                               do k = 1,ke
-                                 do j = js, je
-                                    do i = is, ie
+                                 do i = ie, is, -1
+                                    do j = js, je
                                        pos = pos + 1
                                        buffer(pos) = fieldx(i,j,k)
                                     end do
                                  end do
                               end do
                            end do
-                        case(NINETY)
-                           if( BTEST(update_flags,SCALAR_BIT) ) then
-                              do l=1,l_size  ! loop over number of fields
-                                 ptr_fieldx = f_addrsx(l, tMe)
-                                 ptr_fieldy = f_addrsy(l, tMe)
-                                 do k = 1,ke
-                                    do j = js, je
-                                       do i = is, ie
-                                          pos = pos + 1
-                                          buffer(pos) = fieldx(i,j,k)
-                                       end do
-                                    end do
-                                 end do
-                              end do
-                           else
-                              do l=1,l_size  ! loop over number of fields
-                                 ptr_fieldx = f_addrsx(l, tMe)
-                                 ptr_fieldy = f_addrsy(l, tMe)
-                                 do k = 1,ke
-                                    do j = js, je
-                                       do i = is, ie
-                                          pos = pos + 1
-                                          buffer(pos) = -fieldx(i,j,k)
-                                       end do
-                                    end do
-                                 end do
-                              end do
-                           end if
-                        case(ONE_HUNDRED_EIGHTY)
+                        else
                            do l=1,l_size  ! loop over number of fields
                               ptr_fieldx = f_addrsx(l, tMe)
                               ptr_fieldy = f_addrsy(l, tMe)
                               do k = 1,ke
-                                 do j = js, je
-                                    do i = is, ie
+                                 do i = ie, is, -1
+                                    do j = js, je
                                        pos = pos + 1
-                                       buffer(pos) = fieldy(i,j,k)
+                                       buffer(pos) = -fieldx(i,j,k)
                                     end do
                                  end do
                               end do
                            end do
-                        end select
-                     else
-                        select case( update_y%send(ind_y)%rotation(n) )
-                        case(ZERO)
+                        end if
+                     case(ONE_HUNDRED_EIGHTY)
+                        if( BTEST(update_flags,SCALAR_BIT) ) then
                            do l=1,l_size  ! loop over number of fields
                               ptr_fieldx = f_addrsx(l, tMe)
                               ptr_fieldy = f_addrsy(l, tMe)
                               do k = 1,ke
-                                 do j = js, je
-                                    do i = is, ie
+                                 do j = je, js, -1
+                                    do i = ie, is, -1
                                        pos = pos + 1
                                        buffer(pos) = fieldy(i,j,k)
                                     end do
                                  end do
                               end do
                            end do
-                        case(MINUS_NINETY)
+                        else
                            do l=1,l_size  ! loop over number of fields
                               ptr_fieldx = f_addrsx(l, tMe)
                               ptr_fieldy = f_addrsy(l, tMe)
                               do k = 1,ke
-                                 do i = is, ie
-                                    do j = je, js, -1
+                                 do j = je, js, -1
+                                    do i = ie, is, -1
                                        pos = pos + 1
-                                       buffer(pos) = fieldx(i,j,k)
+                                       buffer(pos) = -fieldy(i,j,k)
                                     end do
                                  end do
                               end do
                            end do
-                        case(NINETY)
-                           if( BTEST(update_flags,SCALAR_BIT) ) then
-                              do l=1,l_size  ! loop over number of fields
-                                 ptr_fieldx = f_addrsx(l, tMe)
-                                 ptr_fieldy = f_addrsy(l, tMe)
-                                 do k = 1,ke
-                                    do i = ie, is, -1
-                                       do j = js, je
-                                          pos = pos + 1
-                                          buffer(pos) = fieldx(i,j,k)
-                                       end do
-                                    end do
-                                 end do
-                              end do
-                           else
-                              do l=1,l_size  ! loop over number of fields
-                                 ptr_fieldx = f_addrsx(l, tMe)
-                                 ptr_fieldy = f_addrsy(l, tMe)
-                                 do k = 1,ke
-                                    do i = ie, is, -1
-                                       do j = js, je
-                                          pos = pos + 1
-                                          buffer(pos) = -fieldx(i,j,k)
-                                       end do
-                                    end do
-                                 end do
-                              end do
-                           end if
-                        case(ONE_HUNDRED_EIGHTY)
-                           if( BTEST(update_flags,SCALAR_BIT) ) then
-                              do l=1,l_size  ! loop over number of fields
-                                 ptr_fieldx = f_addrsx(l, tMe)
-                                 ptr_fieldy = f_addrsy(l, tMe)
-                                 do k = 1,ke
-                                    do j = je, js, -1
-                                       do i = ie, is, -1
-                                          pos = pos + 1
-                                          buffer(pos) = fieldy(i,j,k)
-                                       end do
-                                    end do
-                                 end do
-                              end do
-                           else
-                              do l=1,l_size  ! loop over number of fields
-                                 ptr_fieldx = f_addrsx(l, tMe)
-                                 ptr_fieldy = f_addrsy(l, tMe)
-                                 do k = 1,ke
-                                    do j = je, js, -1
-                                       do i = ie, is, -1
-                                          pos = pos + 1
-                                          buffer(pos) = -fieldy(i,j,k)
-                                       end do
-                                    end do
-                                 end do
-                              end do
-                           end if
-                        end select
-                     end if
+                        end if
+                     end select
                   endif
                enddo
                ind_y = ind_y+1
@@ -941,50 +681,19 @@
                      msgsize = (ie-is+1)*(je-js+1)*ke*2*l_size
                      pos = buffer_pos - msgsize
                      buffer_pos = pos
-                     if(update_x%recv(ind_x)%is_refined(n)) then
-                        index = update_x%recv(ind_x)%index(n)
-                        is1 = update_x%rSpec(tMe)%isNbr(index); ie1 = update_x%rSpec(tMe)%ieNbr(index)
-                        js1 = update_x%rSpec(tMe)%jsNbr(index); je1 = update_x%rSpec(tMe)%jeNbr(index)
-                        ni = ie1 - is1 + 1
-                        nj = je1 - js1 + 1
-                        total = ni*nj
-                        start = (update_x%rSpec(tMe)%start(index)-1)*ke
-
-                        if(start+total*ke>b_sizex ) call mpp_error(FATAL, &
-                             "MPP_DO_UPDATE_V: b_size is less than the size of the data to be filled.")
-                        msgsize = ie - is + 1
-                        do l=1, l_size  ! loop over number of fields
-                           ptr_bufferx = b_addrsx(l, tMe)
-                           ptr_buffery = b_addrsy(l, tMe)
-                           start1 = start + (js-js1)*ni + is - is1 
-                           do k = 1, ke
-                              start2 = start1
-                              do j = js, je
-                                 do i = start2+1, start2+msgsize
-                                    pos = pos + 2
-                                    bufferx(i) = buffer(pos-1)
-                                    buffery(i) = buffer(pos)
-                                 end do
-                                 start2 = start2 + ni
-                              end do
-                              start1 = start1 + total
-                           end do
-                        end do
-                     else
-                        do l=1, l_size  ! loop over number of fields
-                           ptr_fieldx = f_addrsx(l, tMe)
-                           ptr_fieldy = f_addrsy(l, tMe)
-                           do k = 1,ke
-                              do j = js, je
-                                 do i = is, ie
-                                    pos = pos + 2
-                                    fieldx(i,j,k) = buffer(pos-1)
-                                    fieldy(i,j,k) = buffer(pos)
-                                 end do
+                     do l=1, l_size  ! loop over number of fields
+                        ptr_fieldx = f_addrsx(l, tMe)
+                        ptr_fieldy = f_addrsy(l, tMe)
+                        do k = 1,ke
+                           do j = js, je
+                              do i = is, ie
+                                 pos = pos + 2
+                                 fieldx(i,j,k) = buffer(pos-1)
+                                 fieldy(i,j,k) = buffer(pos)
                               end do
                            end do
                         end do
-                     end if
+                     end do
                   end if ! end if( recv(dir) )
                end do  ! do dir=8,1,-1 
                ind_x = ind_x-1
@@ -1008,47 +717,18 @@
                      msgsize = (ie-is+1)*(je-js+1)*ke*l_size
                      pos = buffer_pos - msgsize
                      buffer_pos = pos
-                     if(update_y%recv(ind_y)%is_refined(n)) then
-                        index = update_y%recv(ind_y)%index(n)
-                        is1 = update_y%rSpec(tMe)%isNbr(index); ie1 = update_y%rSpec(tMe)%ieNbr(index)
-                        js1 = update_y%rSpec(tMe)%jsNbr(index); je1 = update_y%rSpec(tMe)%jeNbr(index)
-                        ni = ie1 - is1 + 1
-                        nj = je1 - js1 + 1
-                        total = ni*nj
-                        start = (update_y%rSpec(tMe)%start(index)-1)*ke
-                        if(start+total*ke>b_sizex ) call mpp_error(FATAL, &
-                             "MPP_DO_UPDATE_V: b_size is less than the size of the data to be filled.")
-                        msgsize = ie - is + 1
-                        do l=1,l_size  ! loop over number of fields
-                           ptr_bufferx = b_addrsx(l, tMe)
-                           ptr_buffery = b_addrsy(l, tMe)  
-                           start1 = start + (js-js1)*ni + is - is1 
-                           do k = 1, ke
-                              start2 = start1
-                              do j = js, je
-                                 do i = start2+1, start2+msgsize
-                                    pos = pos + 1
-                                    buffery(i) = buffer(pos)
-                                 end do
-                                 start2 = start2 + ni
-                              end do
-                              start1 = start1 + total
-                           end do
-                        end do
-                     else
-                        do l=1,l_size  ! loop over number of fields
-                           ptr_fieldx = f_addrsx(l, tMe)
-                           ptr_fieldy = f_addrsy(l, tMe)
-                           do k = 1,ke
-                              do j = js, je
-                                 do i = is, ie
-                                    pos = pos + 1
-                                    fieldy(i,j,k) = buffer(pos)
-                                 end do
+                     do l=1,l_size  ! loop over number of fields
+                        ptr_fieldx = f_addrsx(l, tMe)
+                        ptr_fieldy = f_addrsy(l, tMe)
+                        do k = 1,ke
+                           do j = js, je
+                              do i = is, ie
+                                 pos = pos + 1
+                                 fieldy(i,j,k) = buffer(pos)
                               end do
                            end do
                         end do
-                     end if
+                     end do
                   end if
                end do
                ind_y = ind_y-1
@@ -1069,47 +749,18 @@
                      msgsize = (ie-is+1)*(je-js+1)*ke*l_size
                      pos = buffer_pos - msgsize
                      buffer_pos = pos
-                     if(update_x%recv(ind_x)%is_refined(n)) then
-                        index = update_x%recv(ind_x)%index(n)
-                        is1 = update_x%rSpec(tMe)%isNbr(index); ie1 = update_x%rSpec(tMe)%ieNbr(index)
-                        js1 = update_x%rSpec(tMe)%jsNbr(index); je1 = update_x%rSpec(tMe)%jeNbr(index)
-                        ni = ie1 - is1 + 1
-                        nj = je1 - js1 + 1
-                        total = ni*nj
-                        start = (update_x%rSpec(tMe)%start(index)-1)*ke
-                        if(start+total*ke>b_sizex ) call mpp_error(FATAL, &
-                             "MPP_DO_UPDATE_V: b_size is less than the size of the data to be filled.")
-                        msgsize = ie - is + 1
-                        do l=1,l_size  ! loop over number of fields
-                           ptr_bufferx = b_addrsx(l, tMe)
-                           ptr_buffery = b_addrsy(l, tMe)  
-                           start1 = start + (js-js1)*ni + is - is1 
-                           do k = 1, ke
-                              start2 = start1
-                              do j = js, je
-                                 do i = start2+1, start2+msgsize
-                                    pos = pos + 1
-                                    bufferx(i) = buffer(pos)
-                                 end do
-                                 start2 = start2 + ni
-                              end do
-                              start1 = start1 + total
-                           end do
-                        end do
-                     else
-                        do l=1,l_size  ! loop over number of fields
-                           ptr_fieldx = f_addrsx(l, tMe)
-                           ptr_fieldy = f_addrsy(l, tMe)
-                           do k = 1,ke
-                              do j = js, je
-                                 do i = is, ie
-                                    pos = pos + 1
-                                    fieldx(i,j,k) = buffer(pos)
-                                 end do
+                     do l=1,l_size  ! loop over number of fields
+                        ptr_fieldx = f_addrsx(l, tMe)
+                        ptr_fieldy = f_addrsy(l, tMe)
+                        do k = 1,ke
+                           do j = js, je
+                              do i = is, ie
+                                 pos = pos + 1
+                                 fieldx(i,j,k) = buffer(pos)
                               end do
                            end do
                         end do
-                     end if
+                     end do
                   end if
                end do
                ind_x = ind_x-1
diff --git a/src/shared/mpp/include/mpp_do_updateV_nonblock.h b/src/shared/mpp/include/mpp_do_updateV_nonblock.h
index 2a3f2703fe..be2b12e300 100644
--- a/src/shared/mpp/include/mpp_do_updateV_nonblock.h
+++ b/src/shared/mpp/include/mpp_do_updateV_nonblock.h
@@ -23,6 +23,7 @@ subroutine MPP_START_DO_UPDATE_3D_V_(id_update, f_addrsx, f_addrsy, domain, upda
   integer            :: ind_x, ind_y
   integer            :: nsend, nrecv, sendsize, recvsize
   integer            :: request
+  integer            :: send_msgsize(update_x%nsend+update_y%nsend)
   integer            :: ind_send_x(update_x%nsend+update_y%nsend), ind_send_y(update_x%nsend+update_y%nsend)
   integer            :: ind_recv_x(update_x%nrecv+update_y%nrecv), ind_recv_y(update_x%nrecv+update_y%nrecv)
   integer            :: from_pe_list(update_x%nrecv+update_y%nrecv), to_pe_list(update_x%nsend+update_y%nsend)
@@ -126,7 +127,7 @@ subroutine MPP_START_DO_UPDATE_3D_V_(id_update, f_addrsx, f_addrsy, domain, upda
         end do
      endif
      if( msgsize.GT.0 )then
-        msgsize = msgsize*ke_sum
+     msgsize = msgsize*ke_sum
         sendsize = sendsize + msgsize
         nonblock_data(id_update)%buffer_pos_send(m) = buffer_pos
         buffer_pos = buffer_pos + msgsize
@@ -157,35 +158,32 @@ subroutine MPP_START_DO_UPDATE_3D_V_(id_update, f_addrsx, f_addrsy, domain, upda
 
   !--- recv
   call mpp_clock_begin(recv_clock_nonblock)
-  !$OMP parallel do schedule(dynamic) default(shared) private(msgsize,ind_x,ind_y,from_pe,dir,buffer_pos,request)  
   do m = 1, nrecv
      msgsize = nonblock_data(id_update)%size_recv(m)
      from_pe = from_pe_list(m)
      if( msgsize .GT. 0 )then
         buffer_pos = nonblock_data(id_update)%buffer_pos_recv(m)
-!$OMP CRITICAL
         call mpp_recv( buffer(buffer_pos+1), glen=msgsize, from_pe=from_pe, block=.false., &
                    tag=id_update, request=request)
-!$OMP END CRITICAL
         nonblock_data(id_update)%request_recv(m) = request
 #ifdef use_libMPI
         nonblock_data(id_update)%type_recv(m) = MPI_TYPE_
 #endif
      end if
   end do
-!$OMP end parallel do
+
   call mpp_clock_end(recv_clock_nonblock)
 
   !--- send
 
   call mpp_clock_begin(send_pack_clock_nonblock)
 
-!$OMP parallel do schedule(dynamic) default(shared) private(ind_x,ind_y,to_pe,buffer_pos,pos,dir,tMe, &
-!$OMP          is,ie,js,je,ptr_fieldx,ptr_fieldy,msgsize,request)
+!$OMP parallel do schedule(dynamic) default(shared) private(ind_x,ind_y,buffer_pos,pos,dir,tMe, &
+!$OMP          is,ie,js,je,ptr_fieldx,ptr_fieldy)
   do m = 1, nsend
+     send_msgsize(m) = 0
      ind_x = ind_send_x(m)
      ind_y = ind_send_y(m)
-     to_pe = to_pe_list(m)
      buffer_pos = nonblock_data(id_update)%buffer_pos_send(m)
      pos = buffer_pos
      
@@ -198,221 +196,112 @@ subroutine MPP_START_DO_UPDATE_3D_V_(id_update, f_addrsx, f_addrsy, domain, upda
                  tMe = update_x%send(ind_x)%tileMe(n)
                  is = update_x%send(ind_x)%is(n); ie = update_x%send(ind_x)%ie(n)
                  js = update_x%send(ind_x)%js(n); je = update_x%send(ind_x)%je(n)
-                 if( update_x%send(ind_x)%is_refined(n) ) then
-                    select case( update_x%send(ind_x)%rotation(n) )
-                    case(ZERO)
+                 select case( update_x%send(ind_x)%rotation(n) )
+                 case(ZERO)
+                    do l=1,l_size  ! loop over number of fields
+                       ptr_fieldx = f_addrsx(l,tMe)
+                       ptr_fieldy = f_addrsy(l,tMe)
+                       do k = 1,ke_list(l,tMe)
+                          do j = js, je
+                             do i = is, ie
+                                pos = pos + 2
+                                buffer(pos-1) = fieldx(i,j,k)
+                                buffer(pos)   = fieldy(i,j,k)
+                             end do
+                          end do
+                       end do
+                    end do
+                 case( MINUS_NINETY ) 
+                    if( BTEST(flags,SCALAR_BIT) ) then
                        do l=1,l_size  ! loop over number of fields
                           ptr_fieldx = f_addrsx(l,tMe)
                           ptr_fieldy = f_addrsy(l,tMe)
                           do k = 1,ke_list(l,tMe)
-                             do j = js, je
-                                do i = is, ie
+                             do i = is, ie
+                                do j = je, js, -1
                                    pos = pos + 2
-                                   buffer(pos-1) = fieldx(i,j,k)
-                                   buffer(pos)   = fieldy(i,j,k)
+                                   buffer(pos-1) = fieldy(i,j,k)
+                                   buffer(pos)   = fieldx(i,j,k)
                                 end do
                              end do
                           end do
                        end do
-                    case(MINUS_NINETY)
-                       if( BTEST(flags,SCALAR_BIT) ) then
-                          do l=1,l_size  ! loop over number of fields
-                             ptr_fieldx = f_addrsx(l,tMe)
-                             ptr_fieldy = f_addrsy(l,tMe)
-                             do k = 1,ke_list(l,tMe)
-                                do j = js, je
-                                   do i = is, ie
-                                      pos = pos + 2
-                                      buffer(pos-1) =  fieldy(i,j,k)
-                                      buffer(pos)   =  fieldx(i,j,k)
-                                   end do
-                                end do
-                             end do
-                          end do
-                       else
-                          do l=1,l_size  ! loop over number of fields
-                             ptr_fieldx = f_addrsx(l,tMe)
-                             ptr_fieldy = f_addrsy(l,tMe)
-                             do k = 1,ke_list(l,tMe)
-                                do j = js, je
-                                   do i = is, ie
-                                      pos = pos + 2
-                                      buffer(pos-1) = -fieldy(i,j,k)
-                                      buffer(pos)   =  fieldx(i,j,k)
-                                   end do
-                                end do
-                             end do
-                          end do
-                       end if
-                    case(NINETY)
-                       if( BTEST(flags,SCALAR_BIT) ) then
-                          do l=1,l_size  ! loop over number of fields
-                             ptr_fieldx = f_addrsx(l,tMe)
-                             ptr_fieldy = f_addrsy(l,tMe)
-                             do k = 1,ke_list(l,tMe)
-                                do j = js, je
-                                   do i = is, ie
-                                      pos = pos + 2
-                                      buffer(pos-1) =  fieldy(i,j,k)
-                                      buffer(pos)   =  fieldx(i,j,k)
-                                   end do
-                                end do
-                             end do
-                          end do
-                       else
-                          do l=1,l_size  ! loop over number of fields
-                             ptr_fieldx = f_addrsx(l,tMe)
-                             ptr_fieldy = f_addrsy(l,tMe)
-                             do k = 1,ke_list(l,tMe)
-                                do j = js, je
-                                   do i = is, ie
-                                      pos = pos + 2
-                                      buffer(pos-1) =  fieldy(i,j,k)
-                                      buffer(pos)   = -fieldx(i,j,k)
-                                   end do
-                                end do
-                             end do
-                          end do
-                       end if
-                    case(ONE_HUNDRED_EIGHTY)
-                       if( BTEST(flags,SCALAR_BIT) ) then
-                          do l=1,l_size  ! loop over number of fields
-                             ptr_fieldx = f_addrsx(l,tMe)
-                             ptr_fieldy = f_addrsy(l,tMe)
-                             do k = 1,ke_list(l,tMe)
-                                do j = js, je
-                                   do i = is, ie
-                                      pos = pos + 2
-                                      buffer(pos-1) = fieldx(i,j,k)
-                                      buffer(pos)   = fieldy(i,j,k)
-                                   end do
+                    else
+                       do l=1,l_size  ! loop over number of fields
+                          ptr_fieldx = f_addrsx(l,tMe)
+                          ptr_fieldy = f_addrsy(l,tMe)
+                          do k = 1,ke_list(l,tMe)
+                             do i = is, ie
+                                do j = je, js, -1
+                                   pos = pos + 2
+                                   buffer(pos-1) = -fieldy(i,j,k)
+                                   buffer(pos)   =  fieldx(i,j,k)
                                 end do
                              end do
                           end do
-                       else
-                          do l=1,l_size  ! loop over number of fields
-                             ptr_fieldx = f_addrsx(l,tMe)
-                             ptr_fieldy = f_addrsy(l,tMe)
-                             do k = 1,ke_list(l,tMe)
+                       end do
+                    end if
+                 case( NINETY )
+                    if( BTEST(flags,SCALAR_BIT) ) then
+                       do l=1,l_size  ! loop over number of fields
+                          ptr_fieldx = f_addrsx(l,tMe)
+                          ptr_fieldy = f_addrsy(l,tMe)
+                          do k = 1,ke_list(l,tMe)
+                             do i = ie, is, -1
                                 do j = js, je
-                                   do i = is, ie
-                                      pos = pos + 2
-                                      buffer(pos-1) = -fieldx(i,j,k)
-                                      buffer(pos)   = -fieldy(i,j,k)
-                                   end do
+                                   pos = pos + 2
+                                   buffer(pos-1) = fieldy(i,j,k)
+                                   buffer(pos)   = fieldx(i,j,k)
                                 end do
                              end do
                           end do
-                       end if
-                    end select  ! select case( rotation(n) )
-                 else   ! if( is_refined(n) )
-                    select case( update_x%send(ind_x)%rotation(n) )
-                    case(ZERO)
+                       end do
+                    else
                        do l=1,l_size  ! loop over number of fields
                           ptr_fieldx = f_addrsx(l,tMe)
                           ptr_fieldy = f_addrsy(l,tMe)
                           do k = 1,ke_list(l,tMe)
-                             do j = js, je
-                                do i = is, ie
+                             do i = ie, is, -1
+                                do j = js, je
                                    pos = pos + 2
-                                   buffer(pos-1) = fieldx(i,j,k)
-                                   buffer(pos)   = fieldy(i,j,k)
+                                   buffer(pos-1) = fieldy(i,j,k)
+                                   buffer(pos)   = -fieldx(i,j,k)
                                 end do
                              end do
                           end do
                        end do
-                    case( MINUS_NINETY ) 
-                       if( BTEST(flags,SCALAR_BIT) ) then
-                          do l=1,l_size  ! loop over number of fields
-                             ptr_fieldx = f_addrsx(l,tMe)
-                             ptr_fieldy = f_addrsy(l,tMe)
-                             do k = 1,ke_list(l,tMe)
-                                do i = is, ie
-                                   do j = je, js, -1
-                                      pos = pos + 2
-                                      buffer(pos-1) = fieldy(i,j,k)
-                                      buffer(pos)   = fieldx(i,j,k)
-                                   end do
-                                end do
-                             end do
-                          end do
-                       else
-                          do l=1,l_size  ! loop over number of fields
-                             ptr_fieldx = f_addrsx(l,tMe)
-                             ptr_fieldy = f_addrsy(l,tMe)
-                             do k = 1,ke_list(l,tMe)
-                                do i = is, ie
-                                   do j = je, js, -1
-                                      pos = pos + 2
-                                      buffer(pos-1) = -fieldy(i,j,k)
-                                      buffer(pos)   =  fieldx(i,j,k)
-                                   end do
-                                end do
-                             end do
-                          end do
-                       end if
-                    case( NINETY )
-                       if( BTEST(flags,SCALAR_BIT) ) then
-                          do l=1,l_size  ! loop over number of fields
-                             ptr_fieldx = f_addrsx(l,tMe)
-                             ptr_fieldy = f_addrsy(l,tMe)
-                             do k = 1,ke_list(l,tMe)
+                    end if
+                 case( ONE_HUNDRED_EIGHTY )
+                    if( BTEST(flags,SCALAR_BIT) ) then
+                       do l=1,l_size  ! loop over number of fields
+                          ptr_fieldx = f_addrsx(l,tMe)
+                          ptr_fieldy = f_addrsy(l,tMe)
+                          do k = 1,ke_list(l,tMe)
+                             do j = je, js, -1
                                 do i = ie, is, -1
-                                   do j = js, je
-                                      pos = pos + 2
-                                      buffer(pos-1) = fieldy(i,j,k)
-                                      buffer(pos)   = fieldx(i,j,k)
-                                   end do
+                                   pos = pos + 2
+                                   buffer(pos-1) =  fieldx(i,j,k)
+                                   buffer(pos)   =  fieldy(i,j,k)
                                 end do
                              end do
                           end do
-                       else
-                          do l=1,l_size  ! loop over number of fields
-                             ptr_fieldx = f_addrsx(l,tMe)
-                             ptr_fieldy = f_addrsy(l,tMe)
-                             do k = 1,ke_list(l,tMe)
+                       end do
+                    else
+                       do l=1,l_size  ! loop over number of fields
+                          ptr_fieldx = f_addrsx(l,tMe)
+                          ptr_fieldy = f_addrsy(l,tMe)
+                          do k = 1,ke_list(l,tMe)
+                             do j = je, js, -1
                                 do i = ie, is, -1
-                                   do j = js, je
-                                      pos = pos + 2
-                                      buffer(pos-1) = fieldy(i,j,k)
-                                      buffer(pos)   = -fieldx(i,j,k)
-                                   end do
-                                end do
-                             end do
-                          end do
-                       end if
-                    case( ONE_HUNDRED_EIGHTY )
-                       if( BTEST(flags,SCALAR_BIT) ) then
-                          do l=1,l_size  ! loop over number of fields
-                             ptr_fieldx = f_addrsx(l,tMe)
-                             ptr_fieldy = f_addrsy(l,tMe)
-                             do k = 1,ke_list(l,tMe)
-                                do j = je, js, -1
-                                   do i = ie, is, -1
-                                      pos = pos + 2
-                                      buffer(pos-1) =  fieldx(i,j,k)
-                                      buffer(pos)   =  fieldy(i,j,k)
-                                   end do
-                                end do
-                             end do
-                          end do
-                       else
-                          do l=1,l_size  ! loop over number of fields
-                             ptr_fieldx = f_addrsx(l,tMe)
-                             ptr_fieldy = f_addrsy(l,tMe)
-                             do k = 1,ke_list(l,tMe)
-                                do j = je, js, -1
-                                   do i = ie, is, -1
-                                      pos = pos + 2
-                                      buffer(pos-1) =  -fieldx(i,j,k)
-                                      buffer(pos)   =  -fieldy(i,j,k)
-                                   end do
+                                   pos = pos + 2
+                                   buffer(pos-1) =  -fieldx(i,j,k)
+                                   buffer(pos)   =  -fieldy(i,j,k)
                                 end do
                              end do
                           end do
-                       end if
-                    end select ! select case( rotation(n) )
-                 end if ! if( is_refined(n) )
+                       end do
+                    end if
+                 end select ! select case( rotation(n) )
               end if ! if( send(dir) ) 
            end do ! do n = 1, update_x%send(ind_x)%count
         endif
@@ -424,162 +313,90 @@ subroutine MPP_START_DO_UPDATE_3D_V_(id_update, f_addrsx, f_addrsy, domain, upda
                  tMe = update_x%send(ind_x)%tileMe(n)
                  is = update_x%send(ind_x)%is(n); ie = update_x%send(ind_x)%ie(n)
                  js = update_x%send(ind_x)%js(n); je = update_x%send(ind_x)%je(n)
-                 if( update_x%send(ind_x)%is_refined(n) ) then
-                    select case( update_x%send(ind_x)%rotation(n) )
-                    case(ZERO)
-                       do l=1,l_size  ! loop over number of fields
-                          ptr_fieldx = f_addrsx(l,tMe)
-                          ptr_fieldy = f_addrsy(l,tMe)
-                          do k = 1,ke_list(l,tMe)
-                             do j = js, je
-                                do i = is, ie
-                                   pos = pos + 1
-                                   buffer(pos) = fieldx(i,j,k)
-                                end do
+                 select case( update_x%send(ind_x)%rotation(n) )
+                 case(ZERO)
+                    do l=1,l_size  ! loop over number of fields
+                       ptr_fieldx = f_addrsx(l,tMe)
+                       ptr_fieldy = f_addrsy(l,tMe)
+                       do k = 1,ke_list(l,tMe)
+                          do j = js, je
+                             do i = is, ie
+                                pos = pos + 1
+                                buffer(pos) = fieldx(i,j,k)
                              end do
                           end do
                        end do
-                    case(MINUS_NINETY)
-                       if( BTEST(flags,SCALAR_BIT) ) then
-                          do l=1,l_size  ! loop over number of fields
-                             ptr_fieldx = f_addrsx(l,tMe)
-                             ptr_fieldy = f_addrsy(l,tMe)
-                             do k = 1,ke_list(l,tMe)
-                                do j = js, je
-                                   do i = is, ie
-                                      pos = pos + 1
-                                      buffer(pos) = fieldy(i,j,k)
-                                   end do
-                                end do
-                             end do
-                          end do
-                       else
-                          do l=1,l_size  ! loop over number of fields
-                             ptr_fieldx = f_addrsx(l,tMe)
-                             ptr_fieldy = f_addrsy(l,tMe)
-                             do k = 1,ke_list(l,tMe)
-                                do j = js, je
-                                   do i = is, ie
-                                      pos = pos + 1
-                                      buffer(pos) = -fieldy(i,j,k)
-                                   end do
-                                end do
-                             end do
-                          end do
-                       end if
-                    case(NINETY)
+                    end do
+                 case(MINUS_NINETY)
+                    if( BTEST(flags,SCALAR_BIT) ) then
                        do l=1,l_size  ! loop over number of fields
                           ptr_fieldx = f_addrsx(l,tMe)
                           ptr_fieldy = f_addrsy(l,tMe)
-                          do k = 1, ke_list(l,tMe)
-                             do j = js, je
-                                do i = is, ie
+                          do k = 1,ke_list(l,tMe)
+                             do i = is, ie
+                                do j = je, js, -1
                                    pos = pos + 1
                                    buffer(pos) = fieldy(i,j,k)
                                 end do
                              end do
                           end do
                        end do
-                    case(ONE_HUNDRED_EIGHTY)
+                    else
                        do l=1,l_size  ! loop over number of fields
                           ptr_fieldx = f_addrsx(l,tMe)
                           ptr_fieldy = f_addrsy(l,tMe)
                           do k = 1,ke_list(l,tMe)
-                             do j = js, je
-                                do i = is, ie
+                             do i = is, ie
+                                do j = je, js, -1
                                    pos = pos + 1
-                                   buffer(pos) = fieldx(i,j,k)
+                                   buffer(pos) = -fieldy(i,j,k)
                                 end do
                              end do
                           end do
                        end do
-                    end select
-                 else
-                    select case( update_x%send(ind_x)%rotation(n) )
-                    case(ZERO)
+                    end if
+                 case(NINETY)
+                    do l=1,l_size  ! loop over number of fields
+                       ptr_fieldx = f_addrsx(l,tMe)
+                       ptr_fieldy = f_addrsy(l,tMe)
+                       do k = 1, ke_list(l,tMe)
+                          do i = ie, is, -1
+                             do j = js, je
+                                pos = pos + 1
+                                buffer(pos) = fieldy(i,j,k)
+                             end do
+                          end do
+                       end do
+                    end do
+                 case(ONE_HUNDRED_EIGHTY)
+                    if( BTEST(flags,SCALAR_BIT) ) then
                        do l=1,l_size  ! loop over number of fields
                           ptr_fieldx = f_addrsx(l,tMe)
                           ptr_fieldy = f_addrsy(l,tMe)
                           do k = 1,ke_list(l,tMe)
-                             do j = js, je
-                                do i = is, ie
+                             do j = je, js, -1
+                                do i = ie, is, -1
                                    pos = pos + 1
                                    buffer(pos) = fieldx(i,j,k)
                                 end do
                              end do
                           end do
                        end do
-                    case(MINUS_NINETY)
-                       if( BTEST(flags,SCALAR_BIT) ) then
-                          do l=1,l_size  ! loop over number of fields
-                             ptr_fieldx = f_addrsx(l,tMe)
-                             ptr_fieldy = f_addrsy(l,tMe)
-                             do k = 1,ke_list(l,tMe)
-                                do i = is, ie
-                                   do j = je, js, -1
-                                      pos = pos + 1
-                                      buffer(pos) = fieldy(i,j,k)
-                                   end do
-                                end do
-                             end do
-                          end do
-                       else
-                          do l=1,l_size  ! loop over number of fields
-                             ptr_fieldx = f_addrsx(l,tMe)
-                             ptr_fieldy = f_addrsy(l,tMe)
-                             do k = 1,ke_list(l,tMe)
-                                do i = is, ie
-                                   do j = je, js, -1
-                                      pos = pos + 1
-                                      buffer(pos) = -fieldy(i,j,k)
-                                   end do
-                                end do
-                             end do
-                          end do
-                       end if
-                    case(NINETY)
+                    else
                        do l=1,l_size  ! loop over number of fields
                           ptr_fieldx = f_addrsx(l,tMe)
                           ptr_fieldy = f_addrsy(l,tMe)
-                          do k = 1, ke_list(l,tMe)
-                             do i = ie, is, -1
-                                do j = js, je
+                          do k = 1,ke_list(l,tMe)
+                             do j = je, js, -1
+                                do i = ie, is, -1
                                    pos = pos + 1
-                                   buffer(pos) = fieldy(i,j,k)
+                                   buffer(pos) = -fieldx(i,j,k)
                                 end do
                              end do
                           end do
                        end do
-                    case(ONE_HUNDRED_EIGHTY)
-                       if( BTEST(flags,SCALAR_BIT) ) then
-                          do l=1,l_size  ! loop over number of fields
-                             ptr_fieldx = f_addrsx(l,tMe)
-                             ptr_fieldy = f_addrsy(l,tMe)
-                             do k = 1,ke_list(l,tMe)
-                                do j = je, js, -1
-                                   do i = ie, is, -1
-                                      pos = pos + 1
-                                      buffer(pos) = fieldx(i,j,k)
-                                   end do
-                                end do
-                             end do
-                          end do
-                       else
-                          do l=1,l_size  ! loop over number of fields
-                             ptr_fieldx = f_addrsx(l,tMe)
-                             ptr_fieldy = f_addrsy(l,tMe)
-                             do k = 1,ke_list(l,tMe)
-                                do j = je, js, -1
-                                   do i = ie, is, -1
-                                      pos = pos + 1
-                                      buffer(pos) = -fieldx(i,j,k)
-                                   end do
-                                end do
-                             end do
-                          end do
-                       end if
-                    end select
-                 end if
+                    end if
+                 end select
               end if
            end do
         endif
@@ -590,177 +407,107 @@ subroutine MPP_START_DO_UPDATE_3D_V_(id_update, f_addrsx, f_addrsy, domain, upda
                  tMe = update_y%send(ind_y)%tileMe(n)
                  is = update_y%send(ind_y)%is(n); ie = update_y%send(ind_y)%ie(n)
                  js = update_y%send(ind_y)%js(n); je = update_y%send(ind_y)%je(n)
-                 if( update_y%send(ind_y)%is_refined(n) ) then
-                    select case( update_y%send(ind_y)%rotation(n) )
-                    case(ZERO)
-                       do l=1,l_size  ! loop over number of fields
-                          ptr_fieldx = f_addrsx(l,tMe)
-                          ptr_fieldy = f_addrsy(l,tMe)
-                          do k = 1,ke_list(l,tMe)
-                             do j = js, je
-                                do i = is, ie
-                                   pos = pos + 1
-                                   buffer(pos) = fieldy(i,j,k)
-                                end do
+                 select case( update_y%send(ind_y)%rotation(n) )
+                 case(ZERO)
+                    do l=1,l_size  ! loop over number of fields
+                       ptr_fieldx = f_addrsx(l,tMe)
+                       ptr_fieldy = f_addrsy(l,tMe)
+                       do k = 1,ke_list(l,tMe)
+                          do j = js, je
+                             do i = is, ie
+                                pos = pos + 1
+                                buffer(pos) = fieldy(i,j,k)
+                             end do
+                          end do
+                       end do
+                    end do
+                 case(MINUS_NINETY)
+                    do l=1,l_size  ! loop over number of fields
+                       ptr_fieldx = f_addrsx(l,tMe)
+                       ptr_fieldy = f_addrsy(l,tMe)
+                       do k = 1,ke_list(l,tMe)
+                          do i = is, ie
+                             do j = je, js, -1
+                                pos = pos + 1
+                                buffer(pos) = fieldx(i,j,k)
                              end do
                           end do
-                       enddo
-                    case(MINUS_NINETY)
+                       end do
+                    end do
+                 case(NINETY)
+                    if( BTEST(flags,SCALAR_BIT) ) then
                        do l=1,l_size  ! loop over number of fields
                           ptr_fieldx = f_addrsx(l,tMe)
                           ptr_fieldy = f_addrsy(l,tMe)
                           do k = 1,ke_list(l,tMe)
-                             do j = js, je
-                                do i = is, ie
+                             do i = ie, is, -1
+                                do j = js, je
                                    pos = pos + 1
                                    buffer(pos) = fieldx(i,j,k)
                                 end do
                              end do
                           end do
                        end do
-                    case(NINETY)
-                       if( BTEST(flags,SCALAR_BIT) ) then
-                          do l=1,l_size  ! loop over number of fields
-                             ptr_fieldx = f_addrsx(l,tMe)
-                             ptr_fieldy = f_addrsy(l,tMe)
-                             do k = 1,ke_list(l,tMe)
-                                do j = js, je
-                                   do i = is, ie
-                                      pos = pos + 1
-                                      buffer(pos) = fieldx(i,j,k)
-                                   end do
-                                end do
-                             end do
-                          end do
-                       else
-                          do l=1,l_size  ! loop over number of fields
-                             ptr_fieldx = f_addrsx(l,tMe)
-                             ptr_fieldy = f_addrsy(l,tMe)
-                             do k = 1,ke_list(l,tMe)
-                                do j = js, je
-                                   do i = is, ie
-                                      pos = pos + 1
-                                      buffer(pos) = -fieldx(i,j,k)
-                                   end do
-                                end do
-                             end do
-                          end do
-                       end if
-                    case(ONE_HUNDRED_EIGHTY)
+                    else
                        do l=1,l_size  ! loop over number of fields
                           ptr_fieldx = f_addrsx(l,tMe)
                           ptr_fieldy = f_addrsy(l,tMe)
                           do k = 1,ke_list(l,tMe)
-                             do j = js, je
-                                do i = is, ie
+                             do i = ie, is, -1
+                                do j = js, je
                                    pos = pos + 1
-                                   buffer(pos) = fieldy(i,j,k)
+                                   buffer(pos) = -fieldx(i,j,k)
                                 end do
                              end do
                           end do
                        end do
-                    end select
-                 else
-                    select case( update_y%send(ind_y)%rotation(n) )
-                    case(ZERO)
+                    end if
+                 case(ONE_HUNDRED_EIGHTY)
+                    if( BTEST(flags,SCALAR_BIT) ) then
                        do l=1,l_size  ! loop over number of fields
                           ptr_fieldx = f_addrsx(l,tMe)
                           ptr_fieldy = f_addrsy(l,tMe)
                           do k = 1,ke_list(l,tMe)
-                             do j = js, je
-                                do i = is, ie
+                             do j = je, js, -1
+                                do i = ie, is, -1
                                    pos = pos + 1
                                    buffer(pos) = fieldy(i,j,k)
                                 end do
                              end do
                           end do
                        end do
-                    case(MINUS_NINETY)
+                    else
                        do l=1,l_size  ! loop over number of fields
                           ptr_fieldx = f_addrsx(l,tMe)
                           ptr_fieldy = f_addrsy(l,tMe)
                           do k = 1,ke_list(l,tMe)
-                             do i = is, ie
-                                do j = je, js, -1
+                             do j = je, js, -1
+                                do i = ie, is, -1
                                    pos = pos + 1
-                                   buffer(pos) = fieldx(i,j,k)
+                                   buffer(pos) = -fieldy(i,j,k)
                                 end do
                              end do
                           end do
                        end do
-                    case(NINETY)
-                       if( BTEST(flags,SCALAR_BIT) ) then
-                          do l=1,l_size  ! loop over number of fields
-                             ptr_fieldx = f_addrsx(l,tMe)
-                             ptr_fieldy = f_addrsy(l,tMe)
-                             do k = 1,ke_list(l,tMe)
-                                do i = ie, is, -1
-                                   do j = js, je
-                                      pos = pos + 1
-                                      buffer(pos) = fieldx(i,j,k)
-                                   end do
-                                end do
-                             end do
-                          end do
-                       else
-                          do l=1,l_size  ! loop over number of fields
-                             ptr_fieldx = f_addrsx(l,tMe)
-                             ptr_fieldy = f_addrsy(l,tMe)
-                             do k = 1,ke_list(l,tMe)
-                                do i = ie, is, -1
-                                   do j = js, je
-                                      pos = pos + 1
-                                      buffer(pos) = -fieldx(i,j,k)
-                                   end do
-                                end do
-                             end do
-                          end do
-                       end if
-                    case(ONE_HUNDRED_EIGHTY)
-                       if( BTEST(flags,SCALAR_BIT) ) then
-                          do l=1,l_size  ! loop over number of fields
-                             ptr_fieldx = f_addrsx(l,tMe)
-                             ptr_fieldy = f_addrsy(l,tMe)
-                             do k = 1,ke_list(l,tMe)
-                                do j = je, js, -1
-                                   do i = ie, is, -1
-                                      pos = pos + 1
-                                      buffer(pos) = fieldy(i,j,k)
-                                   end do
-                                end do
-                             end do
-                          end do
-                       else
-                          do l=1,l_size  ! loop over number of fields
-                             ptr_fieldx = f_addrsx(l,tMe)
-                             ptr_fieldy = f_addrsy(l,tMe)
-                             do k = 1,ke_list(l,tMe)
-                                do j = je, js, -1
-                                   do i = ie, is, -1
-                                      pos = pos + 1
-                                      buffer(pos) = -fieldy(i,j,k)
-                                   end do
-                                end do
-                             end do
-                          end do
-                       end if
-                    end select
-                 end if
+                    end if
+                 end select
               endif
            enddo
         endif
      end select
-
-     msgsize = pos - buffer_pos
+     send_msgsize(m) = pos - buffer_pos
+  enddo
+!$OMP end parallel do
+  do m = 1, nsend
+     msgsize = send_msgsize(m)
+     to_pe = to_pe_list(m)
+     buffer_pos = nonblock_data(id_update)%buffer_pos_send(m)
      if( msgsize .GT.0 )then
-!$OMP CRITICAL
         call mpp_send( buffer(buffer_pos+1), plen=msgsize, to_pe=to_pe, &
                        tag=id_update, request=request )
-!$OMP END CRITICAL
         nonblock_data(id_update)%request_send(m) = request
      end if
   end do
-!$OMP end parallel do
 
   call mpp_clock_end(send_pack_clock_nonblock)
 
@@ -769,8 +516,7 @@ end subroutine MPP_START_DO_UPDATE_3D_V_
 
 !###############################################################################
 subroutine MPP_COMPLETE_DO_UPDATE_3D_V_(id_update, f_addrsx, f_addrsy, domain, update_x, update_y,     &
-                                        d_type, ke_max, ke_list, b_addrsx, b_addrsy, b_sizex, b_sizey, &
-                                        gridtype, flags) 
+                                        d_type, ke_max, ke_list, gridtype, flags) 
   integer,             intent(in) :: id_update
   integer(LONG_KIND),  intent(in) :: f_addrsx(:,:), f_addrsy(:,:)
   type(domain2d),      intent(in) :: domain
@@ -778,8 +524,6 @@ subroutine MPP_COMPLETE_DO_UPDATE_3D_V_(id_update, f_addrsx, f_addrsy, domain, u
   integer,             intent(in) :: ke_max
   integer,             intent(in) :: ke_list(:,:)
   MPP_TYPE_,           intent(in) :: d_type  ! creates unique interface
-  integer(LONG_KIND),  intent(in) :: b_addrsx(:,:), b_addrsy(:,:)
-  integer,             intent(in) :: b_sizex, b_sizey
   integer,             intent(in) :: gridtype
   integer,             intent(in) :: flags
 
@@ -787,13 +531,8 @@ subroutine MPP_COMPLETE_DO_UPDATE_3D_V_(id_update, f_addrsx, f_addrsy, domain, u
   !--- local variables
   MPP_TYPE_ :: fieldx(update_x%xbegin:update_x%xend, update_x%ybegin:update_x%yend,ke_max)
   MPP_TYPE_ :: fieldy(update_y%xbegin:update_y%xend, update_y%ybegin:update_y%yend,ke_max)
-  MPP_TYPE_ :: bufferx(b_sizex)
-  MPP_TYPE_ :: buffery(b_sizey)
   pointer(ptr_fieldx, fieldx)
   pointer(ptr_fieldy, fieldy)
-  pointer(ptr_bufferx, bufferx)
-  pointer(ptr_buffery, buffery)
-
 
   MPP_TYPE_ :: recv_buffer(size(mpp_domains_stack_nonblock(:)))
   pointer( ptr, recv_buffer )
@@ -801,7 +540,6 @@ subroutine MPP_COMPLETE_DO_UPDATE_3D_V_(id_update, f_addrsx, f_addrsy, domain, u
   integer :: i, j, k, l, is, ie, js, je, n, ke_sum, l_size, m
   integer :: pos, nlist, msgsize, tile, buffer_pos
   integer :: ind_x, ind_y, nrecv, nsend
-  integer :: index, is1, ie1, js1, je1, ni, nj, total, start1, start, start2
   integer :: ind_recv_x(update_x%nrecv+update_y%nrecv), ind_recv_y(update_x%nrecv+update_y%nrecv)
   integer :: start_pos_recv(update_x%nrecv+update_y%nrecv)
   integer :: from_pe_list(update_x%nrecv+update_y%nrecv)
@@ -853,7 +591,7 @@ subroutine MPP_COMPLETE_DO_UPDATE_3D_V_(id_update, f_addrsx, f_addrsy, domain, u
 
   call mpp_clock_begin(unpk_clock_nonblock)
 !$OMP parallel do schedule(dynamic) default(shared) private(ind_x,ind_y,buffer_pos,pos,dir,tMe,is,ie,js,je, &
-!$OMP          msgsize,index,is1,ie1,js1,je1,ni,nj,total,start,ptr_bufferx,ptr_buffery,start1,start2,ptr_fieldx,ptr_fieldy)
+!$OMP          msgsize,ptr_fieldx,ptr_fieldy)
   do m = nrecv,1,-1
      ind_x = ind_recv_x(m)
      ind_y = ind_recv_y(m)
@@ -871,51 +609,19 @@ subroutine MPP_COMPLETE_DO_UPDATE_3D_V_(id_update, f_addrsx, f_addrsy, domain, u
                  msgsize = (ie-is+1)*(je-js+1)*ke_sum*2
                  pos = buffer_pos - msgsize
                  buffer_pos = pos
-                 if(update_x%recv(ind_x)%is_refined(n)) then
-                    index = update_x%recv(ind_x)%index(n)
-                    is1 = update_x%rSpec(tMe)%isNbr(index); ie1 = update_x%rSpec(tMe)%ieNbr(index)
-                    js1 = update_x%rSpec(tMe)%jsNbr(index); je1 = update_x%rSpec(tMe)%jeNbr(index)
-                    ni = ie1 - is1 + 1
-                    nj = je1 - js1 + 1
-                    total = ni*nj
-                    start = (update_x%rSpec(tMe)%start(index)-1)*ke_max
-
-                    if(start+total*ke_max>size(bufferx) .or. start+total*ke_max>size(buffery) ) call mpp_error(FATAL, &
-                         "MPP_COMPLETE_DO_UPDATE_3D_V: size of bufferx or buffery is less than the size of the data to be filled.")
-                    msgsize = ie - is + 1
-                    do l=1, l_size  ! loop over number of fields
-                       ptr_bufferx = b_addrsx(l, tMe)
-                       ptr_buffery = b_addrsy(l, tMe)
-                       if(l==1) start = (update_x%rSpec(tMe)%start(index)-1)*ke_list(l,tMe)
-                       start1 = start + (js-js1)*ni + is - is1 
-                       do k = 1, ke_list(l,tMe)
-                          start2 = start1
-                          do j = js, je
-                             do i = start2+1, start2+msgsize
-                                pos = pos + 2
-                                bufferx(i) = recv_buffer(pos-1)
-                                buffery(i) = recv_buffer(pos)
-                             end do
-                             start2 = start2 + ni
+                 do l=1, l_size  ! loop over number of fields
+                    ptr_fieldx = f_addrsx(l, tMe)
+                    ptr_fieldy = f_addrsy(l, tMe)
+                    do k = 1,ke_list(l,tMe)
+                       do j = js, je
+                          do i = is, ie
+                             pos = pos + 2
+                             fieldx(i,j,k) = recv_buffer(pos-1)
+                             fieldy(i,j,k) = recv_buffer(pos)
                           end do
-                          start1 = start1 + total
                        end do
                     enddo
-                 else
-                    do l=1, l_size  ! loop over number of fields
-                       ptr_fieldx = f_addrsx(l, tMe)
-                       ptr_fieldy = f_addrsy(l, tMe)
-                       do k = 1,ke_list(l,tMe)
-                          do j = js, je
-                             do i = is, ie
-                                pos = pos + 2
-                                fieldx(i,j,k) = recv_buffer(pos-1)
-                                fieldy(i,j,k) = recv_buffer(pos)
-                             end do
-                          end do
-                       enddo
-                    end do
-                 end if
+                 end do
               end if ! end if( recv(dir) )
            end do  ! do dir=8,1,-1 
         endif
@@ -931,47 +637,17 @@ subroutine MPP_COMPLETE_DO_UPDATE_3D_V_(id_update, f_addrsx, f_addrsy, domain, u
                  msgsize = (ie-is+1)*(je-js+1)*ke_sum
                  pos = buffer_pos - msgsize
                  buffer_pos = pos
-                 if(update_y%recv(ind_y)%is_refined(n)) then
-                    index = update_y%recv(ind_y)%index(n)
-                    is1 = update_y%rSpec(tMe)%isNbr(index); ie1 = update_y%rSpec(tMe)%ieNbr(index)
-                    js1 = update_y%rSpec(tMe)%jsNbr(index); je1 = update_y%rSpec(tMe)%jeNbr(index)
-                    ni = ie1 - is1 + 1
-                    nj = je1 - js1 + 1
-                    total = ni*nj
-                    start = (update_y%rSpec(tMe)%start(index)-1)*ke_max
-                    if(start+total*ke_max>size(buffery) ) call mpp_error(FATAL, &
-                      "MPP_COMPLETE_DO_UPDATE_3D_V: size of buffery is less than the size of the data to be filled.")
-
-                    msgsize = ie - is + 1
-                    do l = 1, l_size
-                       ptr_buffery = b_addrsy(l, tMe)
-                       if(l==1) start = (update_y%rSpec(tMe)%start(index)-1)*ke_list(l,tMe)
-                       start1 = start + (js-js1)*ni + is - is1 
-                       do k = 1, ke_list(l,tMe)
-                          start2 = start1
-                          do j = js, je
-                             do i = start2+1, start2+msgsize
-                                pos = pos + 1
-                                buffery(i) = recv_buffer(pos)
-                             end do
-                             start2 = start2 + ni
+                 do l=1, l_size  ! loop over number of fields
+                    ptr_fieldy = f_addrsy(l, tMe)
+                    do k = 1,ke_list(l,tMe)
+                       do j = js, je
+                          do i = is, ie
+                             pos = pos + 1
+                             fieldy(i,j,k) = recv_buffer(pos)
                           end do
-                          start1 = start1 + total
                        end do
-                    enddo
-                 else
-                    do l=1, l_size  ! loop over number of fields
-                       ptr_fieldy = f_addrsy(l, tMe)
-                       do k = 1,ke_list(l,tMe)
-                          do j = js, je
-                             do i = is, ie
-                                pos = pos + 1
-                                fieldy(i,j,k) = recv_buffer(pos)
-                             end do
-                          end do
-                       end do 
                     end do
-                 end if
+                 end do
               end if
            end do
         endif
@@ -985,47 +661,17 @@ subroutine MPP_COMPLETE_DO_UPDATE_3D_V_(id_update, f_addrsx, f_addrsy, domain, u
                  msgsize = (ie-is+1)*(je-js+1)*ke_sum
                  pos = buffer_pos - msgsize
                  buffer_pos = pos
-                 if(update_x%recv(ind_x)%is_refined(n)) then
-                    index = update_x%recv(ind_x)%index(n)
-                    is1 = update_x%rSpec(tMe)%isNbr(index); ie1 = update_x%rSpec(tMe)%ieNbr(index)
-                    js1 = update_x%rSpec(tMe)%jsNbr(index); je1 = update_x%rSpec(tMe)%jeNbr(index)
-                    ni = ie1 - is1 + 1
-                    nj = je1 - js1 + 1
-                    total = ni*nj
-                    start = (update_x%rSpec(tMe)%start(index)-1)*ke_max
-                    if(start+total*ke_max>size(bufferx) ) call mpp_error(FATAL, &
-                      "MPP_COMPLETE_DO_UPDATE_3D_V: size of bufferx is less than the size of the data to be filled.")
-
-                    msgsize = ie - is + 1
-                    do l=1, l_size  ! loop over number of fields
-                       ptr_bufferx = b_addrsx(l, tMe)
-                       if(l==1) start = (update_x%rSpec(tMe)%start(index)-1)*ke_list(l,tMe)
-                       start1 = start + (js-js1)*ni + is - is1 
-                       do k = 1, ke_list(l,tMe)
-                          start2 = start1
-                          do j = js, je
-                             do i = start2+1, start2+msgsize
-                                pos = pos + 1
-                                bufferx(i) = recv_buffer(pos)
-                             end do
-                             start2 = start2 + ni
-                          end do
-                          start1 = start1 + total
-                       end do
-                    end do
-                 else
-                    do l=1, l_size  ! loop over number of fields
-                       ptr_fieldx = f_addrsx(l, tMe)
-                       do k = 1,ke_list(l,tMe)
-                          do j = js, je
-                             do i = is, ie
-                                pos = pos + 1
-                                fieldx(i,j,k) = recv_buffer(pos)
-                             end do
+                 do l=1, l_size  ! loop over number of fields
+                    ptr_fieldx = f_addrsx(l, tMe)
+                    do k = 1,ke_list(l,tMe)
+                       do j = js, je
+                          do i = is, ie
+                             pos = pos + 1
+                             fieldx(i,j,k) = recv_buffer(pos)
                           end do
                        end do
                     end do
-                 end if
+                 end do
               end if
            end do
         endif
diff --git a/src/shared/mpp/include/mpp_do_update_nonblock.h b/src/shared/mpp/include/mpp_do_update_nonblock.h
index c800c45416..4d0a553108 100644
--- a/src/shared/mpp/include/mpp_do_update_nonblock.h
+++ b/src/shared/mpp/include/mpp_do_update_nonblock.h
@@ -18,6 +18,7 @@ subroutine MPP_START_DO_UPDATE_3D_(id_update, f_addrs, domain, update, d_type, k
   logical                     :: send(8), recv(8), update_edge_only
   integer                     :: l_size, ke_sum, my_id_update
   integer                     :: request
+  integer                     :: send_msgsize(MAXLIST)
   character(len=128)          :: text
   MPP_TYPE_                   :: buffer(size(mpp_domains_stack_nonblock(:)))
   MPP_TYPE_                   :: field(update%xbegin:update%xend, update%ybegin:update%yend,ke_max)
@@ -123,16 +124,13 @@ subroutine MPP_START_DO_UPDATE_3D_(id_update, f_addrs, domain, update, d_type, k
 
   ! pre-postrecv
   call mpp_clock_begin(recv_clock_nonblock)
-!$OMP parallel do schedule(dynamic) default(shared) private(dir,from_pe,buffer_pos, request, msgsize)
   do m = 1, update%nrecv
      msgsize = nonblock_data(id_update)%size_recv(m)
      if( msgsize.GT.0 )then
         from_pe =  update%recv(m)%pe
         buffer_pos = nonblock_data(id_update)%buffer_pos_recv(m)
-!$OMP CRITICAL
         call mpp_recv( buffer(buffer_pos+1), glen=msgsize, from_pe=from_pe, block=.FALSE., &
              tag=id_update, request=request)
-!$OMP END CRITICAL
         nonblock_data(id_update)%request_recv(m) = request
 
 #ifdef use_libMPI
@@ -140,7 +138,6 @@ subroutine MPP_START_DO_UPDATE_3D_(id_update, f_addrs, domain, update, d_type, k
 #endif
      end if
   end do ! end do m = 1, update%nrecv  
-!$OMP end parallel do
 
   call mpp_clock_end(recv_clock_nonblock)
 
@@ -149,8 +146,9 @@ subroutine MPP_START_DO_UPDATE_3D_(id_update, f_addrs, domain, update, d_type, k
 !$OMP parallel do schedule(dynamic) default(shared) private(buffer_pos,pos,dir,tMe,is,ie,js,je,ptr_field,to_pe, &
 !$OMP msgsize,request)
   do m = 1, update%nsend
+     send_msgsize(m) = 0
      if( update%send(m)%count == 0 )cycle
-  
+     
      buffer_pos = nonblock_data(id_update)%buffer_pos_send(m)
      pos = buffer_pos
 
@@ -160,10 +158,11 @@ subroutine MPP_START_DO_UPDATE_3D_(id_update, f_addrs, domain, update, d_type, k
            tMe = update%send(m)%tileMe(n)
            is = update%send(m)%is(n); ie = update%send(m)%ie(n)
            js = update%send(m)%js(n); je = update%send(m)%je(n)
-           if( update%send(m)%is_refined(n) ) then
+           select case( update%send(m)%rotation(n) )
+           case(ZERO)
               do l=1,l_size  ! loop over number of fields
                  ptr_field = f_addrs(l, tMe)
-                 do k = 1,ke_list(l,tMe)
+                 do k = 1,ke_list(l,tMe)  
                     do j = js, je
                        do i = is, ie
                           pos = pos + 1
@@ -172,75 +171,60 @@ subroutine MPP_START_DO_UPDATE_3D_(id_update, f_addrs, domain, update, d_type, k
                     end do
                  end do
               enddo
-           else
-              select case( update%send(m)%rotation(n) )
-              case(ZERO)
-                 do l=1,l_size  ! loop over number of fields
-                    ptr_field = f_addrs(l, tMe)
-                    do k = 1,ke_list(l,tMe)  
-                       do j = js, je
-                          do i = is, ie
-                             pos = pos + 1
-                             buffer(pos) = field(i,j,k)
-                          end do
-                       end do
-                    end do
-                 enddo
-              case( MINUS_NINETY ) 
-                 do l=1,l_size  ! loop over number of fields
-                    ptr_field = f_addrs(l, tMe)
-                    do k = 1,ke_list(l,tMe)  
-                       do i = is, ie
-                          do j = je, js, -1
-                             pos = pos + 1
-                             buffer(pos) = field(i,j,k)
-                          end do
+           case( MINUS_NINETY ) 
+              do l=1,l_size  ! loop over number of fields
+                 ptr_field = f_addrs(l, tMe)
+                 do k = 1,ke_list(l,tMe)  
+                    do i = is, ie
+                       do j = je, js, -1
+                          pos = pos + 1
+                          buffer(pos) = field(i,j,k)
                        end do
                     end do
                  end do
-              case( NINETY ) 
-                 do l=1,l_size  ! loop over number of fields
-                    ptr_field = f_addrs(l, tMe)
+              end do
+           case( NINETY ) 
+              do l=1,l_size  ! loop over number of fields
+                 ptr_field = f_addrs(l, tMe)
 
-                    do k = 1,ke_list(l,tMe)  
-                       do i = ie, is, -1
-                          do j = js, je
-                             pos = pos + 1
-                             buffer(pos) = field(i,j,k)
-                          end do
+                 do k = 1,ke_list(l,tMe)  
+                    do i = ie, is, -1
+                       do j = js, je
+                          pos = pos + 1
+                          buffer(pos) = field(i,j,k)
                        end do
                     end do
                  end do
-              case( ONE_HUNDRED_EIGHTY ) 
-                 do l=1,l_size  ! loop over number of fields
-                    ptr_field = f_addrs(l, tMe)
-
-                    do k = 1,ke_list(l,tMe)  
-                       do j = je, js, -1
-                          do i = ie, is, -1
-                             pos = pos + 1
-                             buffer(pos) = field(i,j,k)
-                          end do
+              end do
+           case( ONE_HUNDRED_EIGHTY ) 
+              do l=1,l_size  ! loop over number of fields
+                 ptr_field = f_addrs(l, tMe)
+                 do k = 1,ke_list(l,tMe)  
+                    do j = je, js, -1
+                       do i = ie, is, -1
+                          pos = pos + 1
+                          buffer(pos) = field(i,j,k)
                        end do
                     end do
                  end do
-              end select
-           end if
-        endif
-     end do ! do n = 1, update%send(m)%count
-
+              end do
+           end select
+       endif
+    end do ! do n = 1, update%send(m)%count
+    send_msgsize(m) = pos - buffer_pos
+ enddo
+ !$OMP end parallel do
 
-     msgsize = pos - buffer_pos
+  do m = 1, update%nsend
+    msgsize = send_msgsize(m)
      if( msgsize .GT.0 )then
+        buffer_pos = nonblock_data(id_update)%buffer_pos_send(m)
         to_pe = update%send(m)%pe
-!$OMP CRITICAL
-        call mpp_send( buffer(buffer_pos+1), plen= msgsize, to_pe=to_pe, &
+        call mpp_send( buffer(buffer_pos+1), plen=msgsize , to_pe=to_pe, &
                        tag=id_update, request=request)
-!$OMP END CRITICAL
         nonblock_data(id_update)%request_send(m) = request
      end if
   end do ! end do ist = 0,nlist-1
-!$OMP end parallel do
 
   call mpp_clock_end(send_pack_clock_nonblock)
 
@@ -251,8 +235,7 @@ end subroutine MPP_START_DO_UPDATE_3D_
 
 !###############################################################################
 
-subroutine MPP_COMPLETE_DO_UPDATE_3D_(id_update, f_addrs, domain, update, d_type, ke_max, ke_list, &
-                                      b_addrs, b_size, flags) 
+subroutine MPP_COMPLETE_DO_UPDATE_3D_(id_update, f_addrs, domain, update, d_type, ke_max, ke_list, flags) 
   integer,             intent(in) :: id_update
   integer(LONG_KIND),  intent(in) :: f_addrs(:,:)
   type(domain2d),      intent(in) :: domain
@@ -260,26 +243,19 @@ subroutine MPP_COMPLETE_DO_UPDATE_3D_(id_update, f_addrs, domain, update, d_type
   integer,             intent(in) :: ke_max
   integer,             intent(in) :: ke_list(:,:)
   MPP_TYPE_,           intent(in) :: d_type  ! creates unique interface
-  integer(LONG_KIND),  intent(in) :: b_addrs(:,:)
-  integer,             intent(in) :: b_size
   integer,             intent(in) :: flags
 
   !--- local variables
-  integer                     :: i, j, k, m, n, l, dir, count, tMe
+  integer                     :: i, j, k, m, n, l, dir, count, tMe, tNbr
   integer                     :: buffer_pos, msgsize, from_pe, pos
   integer                     :: is, ie, js, je
-  integer                     :: start, start1, start2, index
-  integer                     :: is1, ie1, js1, je1, ni, nj, total
   logical                     :: send(8), recv(8), update_edge_only
   integer                     :: l_size, ke_sum, sendsize, recvsize
   character(len=128)          :: text
   MPP_TYPE_                   :: recv_buffer(size(mpp_domains_stack_nonblock(:)))
   MPP_TYPE_                   :: field(update%xbegin:update%xend, update%ybegin:update%yend,ke_max)
-  MPP_TYPE_                   :: buffer(b_size)
   pointer( ptr, recv_buffer )
   pointer(ptr_field, field)
-  pointer(ptr_buffer, buffer) 
-
 
   update_edge_only = BTEST(flags, EDGEONLY)
   recv(1) = BTEST(flags,EAST)
@@ -323,7 +299,7 @@ subroutine MPP_COMPLETE_DO_UPDATE_3D_(id_update, f_addrs, domain, update, d_type
   !--unpack the data
   call mpp_clock_begin(unpk_clock_nonblock)
 !$OMP parallel do schedule(dynamic) default(shared) private(dir,buffer_pos,pos,tMe,is,ie,js,je,msgsize, &
-!$OMP          ptr_field, index,is1,ie1,js1,je1,total,start,ptr_buffer,start1,start2)
+!$OMP          ptr_field)
   do m = update%nrecv, 1, -1
      if( update%recv(m)%count == 0 )cycle
      buffer_pos = nonblock_data(id_update)%buffer_pos_recv(m) + nonblock_data(id_update)%size_recv(m)
@@ -338,44 +314,17 @@ subroutine MPP_COMPLETE_DO_UPDATE_3D_(id_update, f_addrs, domain, update, d_type
            msgsize = (ie-is+1)*(je-js+1)*ke_sum
            pos = buffer_pos - msgsize
            buffer_pos = pos
-           if(update%recv(m)%is_refined(n)) then
-              index = update%recv(m)%index(n)
-              is1 = update%rSpec(tMe)%isNbr(index); ie1 = update%rSpec(tMe)%ieNbr(index)
-              js1 = update%rSpec(tMe)%jsNbr(index); je1 = update%rSpec(tMe)%jeNbr(index)
-              ni = ie1 - is1 + 1
-              nj = je1 - js1 + 1
-              total = ni*nj
-              start = (update%rSpec(tMe)%start(index)-1)*ke_max
-              if(start+total*ke_max>size(buffer) ) call mpp_error(FATAL, &
-                   "MPP_COMPETE_UPDATE_DOMAINS: size of buffer is less than the size of the data to be filled.")
-              msgsize = ie - is + 1
-              do l=1, l_size  ! loop over number of fields
-                 ptr_buffer = b_addrs(l, tMe)
-                 if(l==1) start = (update%rSpec(tMe)%start(index)-1)*ke_list(l,tMe)
-                 start1 = start + (js-js1)*ni + is - is1
-                 do k = 1, ke_list(l,tMe)
-                    start2 = start1
-                    do j = js, je
-                       buffer(start2+1:start2+msgsize) = recv_buffer(pos+1:pos+msgsize)
-                       start2 = start2 + ni
-                       pos   = pos + msgsize
-                    end do
-                    start1 = start1 + total
-                 end do
-              enddo
-           else
-              do l=1, l_size  ! loop over number of fields
-                 ptr_field = f_addrs(l, tMe)
-                 do k = 1,ke_list(l,tMe)
-                    do j = js, je
-                       do i = is, ie
-                          pos = pos + 1
-                          field(i,j,k) = recv_buffer(pos)
-                       end do
+           do l=1, l_size  ! loop over number of fields
+              ptr_field = f_addrs(l, tMe)
+              do k = 1,ke_list(l,tMe)
+                 do j = js, je
+                    do i = is, ie
+                       pos = pos + 1
+                       field(i,j,k) = recv_buffer(pos)
                     end do
                  end do
               end do
-           endif
+           end do
         end if
      end do ! do n = 1, update%recv(m)%count
   end do
diff --git a/src/shared/mpp/include/mpp_domains_comm.inc b/src/shared/mpp/include/mpp_domains_comm.inc
index 9b2a3842fc..928d85e5ce 100644
--- a/src/shared/mpp/include/mpp_domains_comm.inc
+++ b/src/shared/mpp/include/mpp_domains_comm.inc
@@ -1,6 +1,6 @@
 
 ! -*-f90-*-
-! $Id: mpp_domains_comm.inc,v 20.0 2013/12/14 00:24:40 fms Exp $
+! $Id$
 
 
     function mpp_redistribute_init_comm(domain_in,l_addrs_in, domain_out,l_addrs_out, &
diff --git a/src/shared/mpp/include/mpp_domains_define.inc b/src/shared/mpp/include/mpp_domains_define.inc
index 3510ef8134..a97e743e54 100644
--- a/src/shared/mpp/include/mpp_domains_define.inc
+++ b/src/shared/mpp/include/mpp_domains_define.inc
@@ -1,6 +1,6 @@
 
 ! -*-f90-*-
-! $Id: mpp_domains_define.inc,v 20.0 2013/12/14 00:24:42 fms Exp $
+! $Id$
 
 
   ! 
@@ -140,6 +140,32 @@
 
   end subroutine mpp_define_mosaic_pelist
 
+  !-- The following implementation is different from mpp_compute_extents 
+  !-- The last block might have most points 
+  subroutine mpp_compute_block_extent(isg,ieg,ndivs,ibegin,iend)
+    integer,                 intent(in) :: isg, ieg, ndivs
+    integer, dimension(:), intent(out)  :: ibegin, iend
+
+    integer :: ndiv, imax, ndmax
+    integer :: is, ie, n
+
+    ie = ieg
+    do ndiv=ndivs,1,-1
+      !domain is sized by dividing remaining points by remaining domains
+      is = ie - CEILING( REAL(ie-isg+1)/ndiv ) + 1
+      ibegin(ndiv) = is
+      iend(ndiv) = ie
+      
+      if( ie.LT.is )call mpp_error( FATAL,  &
+          'MPP_DEFINE_DOMAINS(mpp_compute_block_extent): domain extents must be positive definite.' )
+      if( ndiv.EQ.1 .AND. ibegin(ndiv) .NE. isg ) &
+          call mpp_error( FATAL, 'mpp_compute_block_extent: domain extents do not span space completely.' )
+      ie = is - 1
+     end do
+
+  end subroutine mpp_compute_block_extent
+
+
   !#####################################################################
   subroutine mpp_compute_extent(isg,ieg,ndivs,ibegin,iend, extent )
     integer,                 intent(in)          :: isg, ieg, ndivs
@@ -491,7 +517,7 @@
     ndivx = size(domain%pearray,1)
     ndivy = size(domain%pearray,2)
     allocate(posarray(0:ndivx-1, 0:ndivy-1))
-    n = 0
+    n = domain%tile_root_pe - mpp_root_pe()
     posarray = -1
     do j = 0,ndivy-1
        do i = 0,ndivx-1
@@ -622,8 +648,8 @@
     type(overlapSpec), pointer :: update=>NULL()
     character(len=1)     :: position
     integer              :: msgsize, l, p, is, ie, js, je, from_pe
-    integer, allocatable :: msg1(:), msg2(:), msg3(:)
     integer              :: outunit
+    logical              :: send(8), recv(8)
 
     outunit = stdout()
     if( .NOT.module_is_initialized )call mpp_error( FATAL, 'MPP_DEFINE_DOMAINS2D: You must first call mpp_domains_init.' )
@@ -957,7 +983,7 @@
        !set up domain%list
        if( mpp_pe().EQ.pes(0) .AND. PRESENT(name) )then
           logunit = stdlog()
-          write( logunit, '(/a,i3,a,i3)' )trim(name)//' domain decomposition: ', ndivx, ' X', ndivy
+          write( logunit, '(/a,i5,a,i5)' )trim(name)//' domain decomposition: ', ndivx, ' X', ndivy
           write( logunit, '(3x,a)' )'pe,   is,  ie,  js,  je,    isd, ied, jsd, jed'
        end if
     end if   !     if( ANY(pes == mpp_pe()) ) 
@@ -1004,7 +1030,7 @@
 
 
        !--- when ncontacts is nonzero, set_check_overlap will be called in mpp_define
-       if(domain%symmetry .AND. domain%ncontacts == 0 ) then
+       if(domain%symmetry .AND. (domain%ncontacts == 0 .OR. domain%ntiles == 1) ) then
           call set_check_overlap( domain, CORNER )
           call set_check_overlap( domain, EAST   )
           call set_check_overlap( domain, NORTH  )
@@ -1022,66 +1048,16 @@
     !--- check the send and recv size are matching.
     !--- or ntiles>1 mosaic, 
     !--- the check will be done in mpp_define_mosaic
-    if(debug_message_passing .and. domain%ntiles == 1 ) then
-       allocate(msg1(0:nlist-1), msg2(0:nlist-1), msg3(0:nlist-1) )
-       do p = 1, 4
-          msg1 = 0
-          msg2 = 0
-          select case(p)
-          case(1)
-             update=>domain%update_T
-             position = "T"
-          case(2)
-             update=>domain%update_C
-             position = "C"
-          case(3)
-             update=>domain%update_E
-             position = "E"
-          case(4)
-             update=>domain%update_N
-             position = "N"
-          end select
-          do m = 1, update%nrecv
-             msgsize = 0
-             do n = 1, update%recv(m)%count
-                is = update%recv(m)%is(n); ie = update%recv(m)%ie(n)
-                js = update%recv(m)%js(n); je = update%recv(m)%je(n)
-                msgsize = msgsize + (ie-is+1)*(je-js+1)
-             end do
-             from_pe = update%recv(m)%pe
-             l = from_pe-mpp_root_pe()
-             call mpp_recv( msg1(l), glen=1, from_pe=from_pe, block=.FALSE., tag=COMM_TAG_1)
-             msg2(l) = msgsize
-          enddo
-
-          do m = 1, update%nsend
-             msgsize = 0
-             do n = 1, update%send(m)%count
-                is = update%send(m)%is(n); ie = update%send(m)%ie(n)
-                js = update%send(m)%js(n); je = update%send(m)%je(n)
-                msgsize = msgsize + (ie-is+1)*(je-js+1)
-             end do
-             l = update%send(m)%pe-mpp_root_pe()
-             msg3(l) = msgsize
-             call mpp_send( msg3(l), plen=1, to_pe=update%send(m)%pe, tag=COMM_TAG_1)
-          enddo
-          call mpp_sync_self(check=EVENT_RECV)
-
-          do m = 0, nlist-1
-             if(msg1(m) .NE. msg2(m)) then
-                print*, "My pe = ", mpp_pe(), ",domain name =", trim(domain%name), ",at position=",position,",from pe=", &
-                     domain%list(m)%pe, ":send size = ", msg1(m), ", recv size = ", msg2(m)
-                call mpp_error(FATAL, "mpp_define_domains2D: mismatch on send and recv size")
-             endif
-          enddo
-          call mpp_sync_self()
-       enddo
-       write(outunit,*)"NOTE from mpp_define_domains2D: message sizes are matched between send and recv for domain " &
-            //trim(domain%name)
-       deallocate(msg1, msg2, msg3)
-       update=>NULL()
+    if(debug_message_passing .and. (domain%ncontacts == 0 .OR. domain%ntiles == 1) ) then
+       send = .true.
+       recv = .true.
+       call check_message_size(domain, domain%update_T, send, recv, 'T')
+       call check_message_size(domain, domain%update_E, send, recv, 'E')
+       call check_message_size(domain, domain%update_C, send, recv, 'C')
+       call check_message_size(domain, domain%update_N, send, recv, 'N')
     endif
 
+
     !print out decomposition, this didn't consider maskmap.
     if( mpp_pe() .EQ. pes(0) .AND. PRESENT(name) )then
        write(*,*) trim(name)//' domain decomposition'
@@ -1095,8 +1071,70 @@
     deallocate( pes, pesall)
 
 
-    return
-  end subroutine mpp_define_domains2D
+  return
+end subroutine mpp_define_domains2D
+
+
+!#####################################################################
+subroutine check_message_size(domain, update, send, recv, position)
+  type(domain2d),       intent(in) :: domain 
+  type(overlapSpec),    intent(in) :: update
+  logical,              intent(in) :: send(:)
+  logical,              intent(in) :: recv(:)
+  character,            intent(in) :: position     
+
+  integer, dimension(0:size(domain%list(:))-1) :: msg1, msg2, msg3
+  integer :: m, n, l, dir, is, ie, js, je, from_pe, msgsize
+  integer :: nlist
+
+  nlist = size(domain%list(:))     
+
+
+  msg1 = 0
+  msg2 = 0
+  do m = 1, update%nrecv
+     msgsize = 0
+     do n = 1, update%recv(m)%count
+        dir = update%recv(m)%dir(n)
+        if( recv(dir) ) then
+           is = update%recv(m)%is(n); ie = update%recv(m)%ie(n)
+           js = update%recv(m)%js(n); je = update%recv(m)%je(n)
+           msgsize = msgsize + (ie-is+1)*(je-js+1)
+        endif
+     end do
+     from_pe = update%recv(m)%pe
+     l = from_pe-mpp_root_pe()
+     call mpp_recv( msg1(l), glen=1, from_pe=from_pe, block=.FALSE., tag=COMM_TAG_1)
+     msg2(l) = msgsize
+  enddo
+
+  do m = 1, update%nsend
+     msgsize = 0
+     do n = 1, update%send(m)%count
+        dir = update%send(m)%dir(n)
+        if(send(dir))then
+           is = update%send(m)%is(n); ie = update%send(m)%ie(n)
+           js = update%send(m)%js(n); je = update%send(m)%je(n)
+           msgsize = msgsize + (ie-is+1)*(je-js+1)
+        endif
+     end do
+     l = update%send(m)%pe-mpp_root_pe()
+     msg3(l) = msgsize
+     call mpp_send( msg3(l), plen=1, to_pe=update%send(m)%pe, tag=COMM_TAG_1)
+  enddo
+  call mpp_sync_self(check=EVENT_RECV)
+
+  do m = 0, nlist-1
+     if(msg1(m) .NE. msg2(m)) then
+        print*, "My pe = ", mpp_pe(), ",domain name =", trim(domain%name), ",at position=",position,",from pe=", &
+             domain%list(m)%pe, ":send size = ", msg1(m), ", recv size = ", msg2(m)
+        call mpp_error(FATAL, "mpp_define_domains2D: mismatch on send and recv size")
+     endif
+  enddo
+  call mpp_sync_self()
+
+
+end subroutine check_message_size
 
   !#####################################################################
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
@@ -1152,6 +1190,7 @@
     integer              :: msgsize, l, p, is, ie, js, je, from_pe
     integer, allocatable :: msg1(:), msg2(:), msg3(:)
     integer              :: outunit
+    logical              :: send(8), recv(8)
 
     outunit = stdout()
     mosaic_defined = .true.
@@ -1449,61 +1488,20 @@
     !--- currently only check T and C-cell. For ntiles>1 mosaic, 
     !--- the check will be done in mpp_define_mosaic
     if(debug_message_passing) then
-       allocate(msg1(0:nlist-1), msg2(0:nlist-1), msg3(0:nlist-1) )
-       do p = 1, 2
-          msg1 = 0
-          msg2 = 0
-          select case(p)
-          case(1)
-             update=>domain%update_T
-             position = "T"
-          case(2)
-             update=>domain%update_C
-             position = "C"
-          end select
-          do m = 1, update%nrecv
-             msgsize = 0
-             do n = 1, update%recv(m)%count
-                is = update%recv(m)%is(n); ie = update%recv(m)%ie(n)
-                js = update%recv(m)%js(n); je = update%recv(m)%je(n)
-                msgsize = msgsize + (ie-is+1)*(je-js+1)
-             end do
-             from_pe = update%recv(m)%pe
-             l = from_pe-mpp_root_pe()
-             call mpp_recv( msg1(l), glen=1, from_pe=from_pe, block=.FALSE., tag=COMM_TAG_2)
-             msg2(l) = msgsize
-          enddo
-
-          do m = 1, update%nsend
-             msgsize = 0
-             do n = 1, update%send(m)%count
-                is = update%send(m)%is(n); ie = update%send(m)%ie(n)
-                js = update%send(m)%js(n); je = update%send(m)%je(n)
-                msgsize = msgsize + (ie-is+1)*(je-js+1)
-             end do
-             msg3(m) = msgsize
-             call mpp_send( msg3(m), plen=1, to_pe=update%send(m)%pe, tag=COMM_TAG_2)
-          enddo
-          call mpp_sync_self(check=EVENT_RECV)
-
-          do m = 0, nlist-1
-             if(msg1(m) .NE. msg2(m)) then
-                print*, "My pe = ", mpp_pe(), ",domain name =", trim(domain%name), ",at position=",position,",from pe=", &
-                     domain%list(m)%pe, ":send size = ", msg1(m), ", recv size = ", msg2(m)
-                call mpp_error(FATAL, "mpp_define_mosaic: mismatch on send and recv size")
-             endif
-          enddo
-          call mpp_sync_self()
-       enddo
-       write(outunit,*)"NOTE from mpp_define_mosaic: message sizes are matched between send and recv for domain " &
-            //trim(domain%name)//"(T and C-cell)"
-       deallocate(msg1, msg2)
-       update=>NULL()
+       send = .true.
+       recv = .true.
+       call check_message_size(domain, domain%update_T, send, recv, 'T')
+       call check_message_size(domain, domain%update_C, send, recv, 'C')
+       call check_message_size(domain, domain%update_E, send, recv, 'E')
+       call check_message_size(domain, domain%update_N, send, recv, 'N')
     endif
 
+
     !--- release memory
     deallocate(align1, align2, is1, ie1, js1, je1, is2, ie2, js2, je2  )
     deallocate(isgList, iegList, jsgList, jegList, refine1, refine2 )
+
+
   end subroutine mpp_define_mosaic
 
 !#####################################################################
@@ -5144,7 +5142,6 @@
     rotate = ZERO
     if(present(rotation)) rotate = rotation
     overlap_out%rotation  (count) = overlap_in%rotation(index)
-    overlap_out%is_refined(count) = overlap_in%is_refined(index)
     overlap_out%dir       (count) = dir
     overlap_out%tileMe    (count) = overlap_in%tileMe(index)   
     overlap_out%tileNbr   (count) = overlap_in%tileNbr(index)   
@@ -5493,12 +5490,8 @@
                    case ( WEST, EAST )
                       ioff = isd - is2Send(n) 
                       joff = jsd - js2Send(n)
-                      joff = get_refine_count( joff, refineSend(n), .true., dir )
-                      nyd  = get_refine_count( nyd, refineSend(n), .false., dir  )
                    case ( SOUTH, NORTH )
                       ioff = isd - is2Send(n) 
-                      ioff = get_refine_count( ioff, refineSend(n), .true., dir  )
-                      nxd  = get_refine_count( nxd, refineSend(n), .false., dir  )
                       joff = jsd - js2Send(n)
                    end select
 
@@ -5518,8 +5511,8 @@
                    js = max(jsc1,jsc2); je = min(jec1,jec2)
                    if(ie.GE.is .AND. je.GE.js )then
                       if(.not. associated(overlapSend(m)%tileMe)) call allocate_update_overlap(overlapSend(m), MAXOVERLAP)
-                      call insert_mosaic_update_overlap(overlapSend(m), domain%list(m)%pe, tMe, tNbr, &
-                                                        is, ie, js, je, dir, rotateSend(n), .true., refineSend(n) .NE. 1 )
+                    call insert_overlap_type(overlapSend(m), domain%list(m)%pe, tMe, tNbr, &
+                         is, ie, js, je, dir, rotateSend(n), .true. )
                    endif
                 end do ! end do dir = 1, 8
              end do ! end do tNbr = 1, ntileNbr
@@ -5549,12 +5542,8 @@
                       ioff = ie2Recv(n) - iec
                    endif
                    joff = jsc - js2Recv(n)
-                   joff = get_refine_count( joff, 1/refineRecv(n), .true., dir  )
-                   nyc  = get_refine_count( nyc, 1/refineRecv(n), .false., dir  )
                 case ( NORTH, SOUTH )
                    ioff = isc - is2Recv(n) 
-                   ioff = get_refine_count( ioff, 1/refineRecv(n), .true., dir  )
-                   nxc  = get_refine_count( nxc, 1/refineRecv(n), .false., dir  )
                    if(align2Recv(n) == SOUTH) then
                       joff = jsc - js2Recv(n)
                    else
@@ -5625,48 +5614,9 @@
                    js = max(jsd1,jsd2); je = min(jed1,jed2)
                    if(ie.GE.is .AND. je.GE.js )then
                       if(.not. associated(overlapRecv(m)%tileMe)) call allocate_update_overlap(overlapRecv(m), MAXOVERLAP)
-                      call insert_mosaic_update_overlap(overlapRecv(m), domain%list(m)%pe, tMe, tNbr, &
-                                                        is, ie, js, je, dir, rotateRecv(n), .true., refineSend(n) .NE. 1 )
+                    call insert_overlap_type(overlapRecv(m), domain%list(m)%pe, tMe, tNbr, &
+                         is, ie, js, je, dir, rotateRecv(n), .true.)
                       count                              = overlapRecv(m)%count
-                      overlapRecv(m)%isMe        (count) = is
-                      overlapRecv(m)%ieMe        (count) = ie
-                      overlapRecv(m)%jsMe        (count) = js
-                      overlapRecv(m)%jeMe        (count) = je
-                      !--- get the index on the pe sending data when there is refinement
-                      if( overlapRecv(m)%is_refined(count) ) then
-                         select case ( align1Recv(n) )
-                         case ( WEST, EAST )
-                            isoff = 0;               ieoff = ie - is + 1
-                            jsoff = js - js1Recv(n); jeoff = je - js1Recv(n) + 1
-                            jsoff = get_refine_count( jsoff, refineRecv(n), .true., dir  )
-                            jeoff = get_refine_count( jeoff, refineRecv(n), .false., dir  ) - 1
-                         case ( NORTH, SOUTH )
-                            isoff = is - is1Recv(n); ieoff = ie - is1Recv(n) + 1
-                            jsoff = 0;               jeoff = je - js
-                            isoff = get_refine_count( isoff, refineRecv(n), .true., dir  )
-                            ieoff = get_refine_count( ieoff, refineRecv(n), .false., dir  ) - 1
-                         end select
-
-                         !--- get the index in neighbor pe.
-                         select case ( rotateRecv(n) )
-                         case ( ZERO )
-                            isd2 = is2Recv(n) + isoff; ied2 = is2Recv(n) + ieoff
-                            jsd2 = js2Recv(n) + jsoff; jed2 = js2Recv(n) + jeoff
-                         case ( NINETY )                      ! N -> W or S -> E
-                            isd2 = ie2Recv(n) - jeoff; ied2 = ie2Recv(n) - jsoff
-                            jsd2 = js2Recv(n);         jed2 = js2Recv(n) + ieoff - isoff
-                         case ( MINUS_NINETY )                ! W -> N or E -> S
-                            isd2 = is2Recv(n);         ied2 = is2Recv(n) + jeoff - jsoff
-                            jsd2 = je2Recv(n) - ieoff; jed2 = je2Recv(n) - isoff
-                         end select
-                         isd2= max(isd2,isc); ied2 = min(ied2, iec); jsd2= max(jsd2,jsc); jed2 = min(jed2, jec)
-                         if( isd2 > ied2 .OR. jsd2 > jed2 ) call mpp_error(FATAL, &
-                              "mpp_domains_define.inc:  the neighbor index is out of range")
-                         overlapRecv(m)%is(count) = isd2
-                         overlapRecv(m)%ie(count) = ied2
-                         overlapRecv(m)%js(count) = jsd2
-                         overlapRecv(m)%je(count) = jed2
-                      end if
                    endif
                 end do ! end do dir = 1, 8
              end do ! end do tNbr = 1, ntileNbr
@@ -5732,10 +5682,10 @@
                          if(overlapSend(m)%tileMe(l) .NE. tMe) cycle
                          if(overlapSend(m)%tileNbr(l) .NE. tNbr) cycle
                          if(overlapSend(m)%dir(l) .NE. dirlist(n) ) cycle
-                         call insert_mosaic_update_overlap(domain%update_T%send(nsend2), overlapSend(m)%pe,                     &
+                       call insert_overlap_type(domain%update_T%send(nsend2), overlapSend(m)%pe,                     &
                               overlapSend(m)%tileMe(l), overlapSend(m)%tileNbr(l), overlapSend(m)%is(l), overlapSend(m)%ie(l),  &
                               overlapSend(m)%js(l), overlapSend(m)%je(l), overlapSend(m)%dir(l), overlapSend(m)%rotation(l),    &
-                              overlapSend(m)%from_contact(l),  overlapSend(m)%is_refined(l) )                   
+                            overlapSend(m)%from_contact(l)  )                   
                       end do
                    end do
                 end do
@@ -5798,16 +5748,11 @@
                          if(overlapRecv(m)%tileMe(l) .NE. tMe) cycle
                          if(overlapRecv(m)%tileNbr(l) .NE. tNbr) cycle
                          if(overlapRecv(m)%dir(l) .NE. dirlist(n) ) cycle
-                         call insert_mosaic_update_overlap(domain%update_T%recv(nrecv2), overlapRecv(m)%pe, &
+                       call insert_overlap_type(domain%update_T%recv(nrecv2), overlapRecv(m)%pe, &
                               overlapRecv(m)%tileMe(l), overlapRecv(m)%tileNbr(l), overlapRecv(m)%is(l), overlapRecv(m)%ie(l),  &
                               overlapRecv(m)%js(l), overlapRecv(m)%je(l), overlapRecv(m)%dir(l), overlapRecv(m)%rotation(l),    &
-                              overlapRecv(m)%from_contact(l),  overlaprecv(m)%is_refined(l) )                   
+                            overlapRecv(m)%from_contact(l))                   
                          count = domain%update_T%recv(nrecv2)%count
-                         domain%update_T%recv(nrecv2)%isMe        (count) = overlapRecv(m)%isMe        (l)
-                         domain%update_T%recv(nrecv2)%ieMe        (count) = overlapRecv(m)%ieMe        (l)
-                         domain%update_T%recv(nrecv2)%jsMe        (count) = overlapRecv(m)%jsMe        (l)
-                         domain%update_T%recv(nrecv2)%jeMe        (count) = overlapRecv(m)%jeMe        (l)
-                         domain%update_T%recv(nrecv2)%index       (count) = overlapRecv(m)%index       (l)
                       end do
                    end do
                 end do
@@ -5840,170 +5785,11 @@
        deallocate(nCont(n)%is2, nCont(n)%ie2, nCont(n)%js2, nCont(n)%je2 )
     end do
 
-    !--- define the refined overlapping and group the overlapping 
-    !--- between same tiles( in different direction or different pe ). 
-    !--- set up the index position in the return buffer.
-    call define_refine_overlap( domain, CENTER )  
-
     domain%initialized = .true.
 
 
   end subroutine define_contact_point
 
-  !--- currently assume no grid is divided over two pes.
-  !--- we may remove this restriction in the future. 
-  function get_refine_count(num , refine, use_floor, dir )
-    integer, intent(in) :: num
-    real,    intent(in) :: refine
-    logical, intent(in) :: use_floor
-    integer, intent(in) :: dir
-    integer             :: get_refine_count
-    real, parameter     :: EPSLN = 1.e-10
-    real                :: rval
-
-    rval = num*refine
-
-    if(mod(dir,2) == 1) then ! WEST, EAST, SOUTH, NORTH
-       get_refine_count = nint(rval)
-       if(abs(rval-get_refine_count) > EPSLN) call mpp_error(FATAL, &
-            "get_refine_count: The two adjacent tiles with refinement does not align well in E/W/S/N direction")
-    else
-       get_refine_count = floor(rval)
-       if(use_floor) then
-          if( rval - get_refine_count > 1 - EPSLN ) get_refine_count = get_refine_count + 1 ! Deal with truncation error
-       else
-          if(rval - get_refine_count  > EPSLN) get_refine_count = get_refine_count + 1 ! Deal with truncation error
-       end if
-    end if
-
-
-    return
-
-  end function get_refine_count
-
-  !####################################################################################
-  !--- The following will define the refined overlapping and group the overlapping 
-  !--- between same tiles( in different direction or different pe ). 
-  !--- set up the index position in the return buffer.
-  subroutine define_refine_overlap(domain, position)
-    type(domain2d), intent(inout) :: domain
-    integer,        intent(in)    :: position
-
-    logical                        :: new_overlap
-    integer,           parameter   :: MAXOVERLAP = 100
-    integer, dimension(MAXOVERLAP) :: isMe, ieMe, jsMe, jeMe, isNbr, ieNbr, jsNbr, jeNbr
-    integer, dimension(MAXOVERLAP) :: rotation, tileList, dirList
-    integer                        :: count, tMe, ntileMe, nlist, pos
-    integer                        :: m, n, l, dir, istart, iend, dirdiff
-    type(overlap_type), pointer    :: OverPtr => NULL()
-    type(overlapSpec), pointer     :: update => NULL()
-
-    select case(position)
-    case (CENTER)
-       update => domain%update_T
-    case (CORNER)
-       update => domain%update_C
-    case (EAST)
-       update => domain%update_E
-    case (NORTH)
-       update => domain%update_N
-    case default
-       call mpp_error(FATAL, &
-         "mpp_domains_define.inc(define_refine_overlap): position should be CENTER, CORNER, EAST or NORTH")
-    end select
-
-    ntileMe = size(domain%x(:))
-    nlist = size(domain%list(:))    
-
-    allocate(update%rSpec(ntileMe) )
-
-    do tMe = 1, ntileMe
-       count = 0
-       do m = 1, update%nrecv
-          overPtr => update%recv(m)
-          if( overptr%count == 0)cycle
-          pos = overPtr%pe - mpp_root_pe()
-          do l = 1, overPtr%count
-             if(OverPtr%is_refined(l) .AND. overPtr%tileMe(l) == tMe ) then
-                dir = overPtr%dir(l)
-                new_overlap = .true.
-                do n = 1, count
-                   if( tileList(n) == domain%list(pos)%tile_id(overPtr%tileNbr(l)) ) then ! same tile
-                      dirdiff = dir - dirList(n)   ! dir always no less than dirList(n).
-                      if( (dir == 8 .AND. dirList(n) == 1) .OR. (dir == 1 .AND. dirList(n) == 8) ) dirdiff = 1
-                      dirdiff = abs(dirdiff)                      
-                      if( dirdiff .LE. 1 ) then    ! same direction
-                         if( mod(dirList(n),2) == 0 ) then
-                            if(mod(dir,2) == 0 ) call mpp_error(FATAL, &
-                                 "mpp_get_mosaic_refine_overlap: corner overlap should be unique.")
-                            dirList(n) = dir
-                         end if
-                         if( dirList(n) == 1 .OR. dirList(n) == 5 ) then ! EAST, or WEST
-                            if( isMe(n) .NE. overPtr%isMe(l) .OR. ieMe(n) .NE. overPtr%ieMe(l) ) call mpp_error(FATAL, &
-                                 "mpp_get_mosaic_refine_overlap: mismatch on my index in the overlapping 1" )
-                            jeMe(n) = max(jeMe(n),  overPtr%jeMe(l) )
-                            jsMe(n) = min(jsMe(n),  overPtr%jsMe(l) )
-                         else if(dirList(n) == 3 .OR. dirList(n) == 7 ) then ! SOUTH or NORTH
-                            if( jsMe(n) .NE. overPtr%jsMe(l) .OR. jeMe(n) .NE. overPtr%jeMe(l) ) call mpp_error(FATAL, &
-                                 "mpp_get_mosaic_refine_overlap: mismatch on my index in the overlapping 2" )
-                            ieMe(n) = max(ieMe(n),  overPtr%ieMe(l) )
-                            isMe(n) = min(isMe(n),  overPtr%isMe(l) )
-                         end if
-                         isNbr(n) = min(isNbr(n), overPtr%is(l))
-                         ieNbr(n) = max(ieNbr(n), overPtr%ie(l))
-                         jsNbr(n) = min(jsNbr(n), overPtr%js(l))
-                         jeNbr(n) = max(jeNbr(n), overPtr%je(l))
-                         OverPtr%index(l) = n
-                         new_overlap = .false.
-                         exit
-                      end if
-                   end if
-                end do
-
-                if(new_overlap) then
-                   count = count + 1
-                   if(count > MAXOVERLAP) call mpp_error(FATAL,  &
-                        "mpp_get_mosaic_refine_overlap: number of overlapping is larger than MAXOVERLAP")
-                   tileList(count) = domain%list(pos)%tile_id(overPtr%tileNbr(l))
-                   dirList(count)  = dir
-                   rotation(count) = overPtr%rotation(l)
-                   OverPtr%index(l)= count
-                   isMe(count) = overPtr%isMe(l);    ieMe(count) = overPtr%ieMe(l)
-                   jsMe(count) = overPtr%jsMe(l);    jeMe(count) = overPtr%jeMe(l)
-                   isNbr(count) = overPtr%is(l);     ieNbr(count) = overPtr%ie(l)
-                   jsNbr(count) = overPtr%js(l);     jeNbr(count) = overPtr%je(l)
-                end if
-             end if
-          end do
-       end do
-       if(count > 0) then
-          allocate(update%rSpec(tMe)%isMe(count),  update%rSpec(tMe)%ieMe(count) )
-          allocate(update%rSpec(tMe)%jsMe(count),  update%rSpec(tMe)%jeMe(count) )
-          allocate(update%rSpec(tMe)%isNbr(count), update%rSpec(tMe)%ieNbr(count) )
-          allocate(update%rSpec(tMe)%jsNbr(count), update%rSpec(tMe)%jeNbr(count) )
-          allocate(update%rSpec(tMe)%start(count), update%rSpec(tMe)%end  (count) )
-          allocate(update%rSpec(tMe)%dir  (count), update%rSpec(tMe)%rotation(count) )
-          update%rSpec(tMe)%isMe = isMe(1:count); update%rSpec(tMe)%isNbr = isNbr(1:count)
-          update%rSpec(tMe)%ieMe = ieMe(1:count); update%rSpec(tMe)%ieNbr = ieNbr(1:count)
-          update%rSpec(tMe)%jsMe = jsMe(1:count); update%rSpec(tMe)%jsNbr = jsNbr(1:count)
-          update%rSpec(tMe)%jeMe = jeMe(1:count); update%rSpec(tMe)%jeNbr = jeNbr(1:count)
-          update%rSpec(tMe)%count = count
-          update%rSpec(tMe)%dir   = dirList(1:count)
-          update%rSpec(tMe)%rotation = rotation(1:count)
-          update%rSpec(tMe)%total = sum( (update%rSpec(tMe)%ieNbr-update%rSpec(tMe)%isNbr+1)* &
-               (update%rSpec(tMe)%jeNbr-update%rSpec(tMe)%jsNbr+1)  )
-          iend = 0
-          do n = 1, count
-             istart = iend + 1
-             iend = istart + (ieNbr(n) - isNbr(n) + 1)*(jeNbr(n) - jsNbr(n) + 1) - 1
-             update%rSpec(tMe)%start(n) = istart
-             update%rSpec(tMe)%end(n)   = iend
-          enddo
-       end if
-  end do
-
-end subroutine define_refine_overlap
-
 !##############################################################################
 !--- always fill the contact according to index order.
 subroutine fill_contact(Contact, tile, is1, ie1, js1, je1, is2, ie2, js2, je2, align1, align2, refine1, refine2 )
@@ -6171,10 +5957,9 @@ subroutine set_contact_point(domain, position)
                  isoff1 = jshift; ieoff1 = jshift; jsoff1 = 0;      jeoff1 = 0
               end select
            end select
-           call insert_mosaic_update_overlap(overlap, PtrIn%pe, PtrIn%tileMe(n), PtrIn%tileNbr(n), &
+           call insert_overlap_type(overlap, PtrIn%pe, PtrIn%tileMe(n), PtrIn%tileNbr(n), &
                                              Ptrin%is(n) + isoff1, Ptrin%ie(n) + ieoff1, Ptrin%js(n) + jsoff1, &
-                                             Ptrin%je(n) + jeoff1, PtrIn%dir(n), PtrIn%rotation(n), PtrIn%from_contact(n), &
-                                             PtrIn%is_refined(n) )
+                Ptrin%je(n) + jeoff1, PtrIn%dir(n), PtrIn%rotation(n), PtrIn%from_contact(n))
         end if
      end do ! do n = 1, prtIn%count
      if(overlap%count > 0) then
@@ -6245,51 +6030,10 @@ subroutine set_contact_point(domain, position)
            case ( 8 ) ! NE
               isoff1 = ishift; ieoff1 = ishift; jsoff1 = jshift; jeoff1 = jshift
            end select
-           call insert_mosaic_update_overlap(overlap, PtrIn%pe, PtrIn%tileMe(n), PtrIn%tileNbr(n), &
+           call insert_overlap_type(overlap, PtrIn%pe, PtrIn%tileMe(n), PtrIn%tileNbr(n), &
                                              Ptrin%is(n) + isoff1, Ptrin%ie(n) + ieoff1, Ptrin%js(n) + jsoff1, &
-                                             Ptrin%je(n) + jeoff1, PtrIn%dir(n), PtrIn%rotation(n), PtrIn%from_contact(n), &
-                                             PtrIn%is_refined(n) )
+                Ptrin%je(n) + jeoff1, PtrIn%dir(n), PtrIn%rotation(n), PtrIn%from_contact(n))
            count                     = overlap%count
-           overlap%isMe      (count) = Ptrin%isMe(n) + isoff1
-           overlap%ieMe      (count) = Ptrin%ieMe(n) + ieoff1
-           overlap%jsMe      (count) = Ptrin%jsMe(n) + jsoff1
-           overlap%jeMe      (count) = Ptrin%jeMe(n) + jeoff1
-           !--- figure out neighbor offset when there is refinemnet
-           if( PtrIn%is_refined(n) ) then
-              isoff2 = isoff1; ieoff2 = ieoff1; jsoff2 = jsoff1; jeoff2 = jeoff1
-              if( ptrIn%rotation(n) == NINETY) then
-                 select case ( dir )                                       
-                 case ( 1, 5 ) !S -> E, N -> W
-                    isoff2 = jsoff1; ieoff2 = jeoff1; jsoff2 = isoff1; jeoff2 = ieoff1
-                 case ( 2 ) 
-                    isoff2 = jshift; ieoff2 = jshift; jsoff2 = ishift; jeoff2 = ishift
-                 case ( 4 ) 
-                    isoff2 = jshift; ieoff2 = jshift; jsoff2 = 0;      jeoff2 = 0
-                 case ( 6 ) 
-                    isoff2 = 0;      ieoff2 = 0;      jsoff2 = 0;      jeoff2 = 0
-                 case ( 8 ) 
-                    isoff2 = 0;      ieoff2 = 0;      jsoff2 = ishift; jeoff2 = ishift
-                 end select
-              else if( ptrIn%rotation(n) == MINUS_NINETY) then
-                 select case ( dir )                                       
-                 case ( 3, 7 ) !E -> S, W -> N
-                    isoff2 = jsoff1; ieoff2 = jeoff1; jsoff2 = isoff1; jeoff2 = ieoff1
-                 case ( 2 )
-                    isoff2 = 0;      ieoff2 = 0;      jsoff2 = 0;      jeoff2 = 0 
-                 case ( 4 ) 
-                    isoff2 = 0;      ieoff2 = 0;      jsoff2 = ishift; jeoff2 = ishift
-                 case ( 6 ) 
-                    isoff2 = jshift; ieoff2 = jshift; jsoff2 = ishift; jeoff2 = ishift
-                 case ( 8 ) 
-                    isoff2 = jshift; ieoff2 = jshift; jsoff2 = 0;      jeoff2 = 0
-                 end select
-              end if
-              !--- calculate overlapping
-              overlap%is(count) = Ptrin%is(n) + isoff2
-              overlap%ie(count) = Ptrin%ie(n) + ieoff2
-              overlap%js(count) = Ptrin%js(n) + jsoff2
-              overlap%je(count) = Ptrin%je(n) + jeoff2
-           end if
         end if
      end do ! do n = 1, ptrIn%count
      if(overlap%count > 0) then
@@ -6330,13 +6074,6 @@ subroutine set_contact_point(domain, position)
           "mpp_domains_define.inc(set_contact_point): pos should equal to nrecv")
   endif
 
-
-
-  !--- define the refined overlapping and group the overlapping 
-  !--- between same tiles( in different direction or different pe ). 
-  !--- set up the index position in the return buffer.
-  call define_refine_overlap( domain, position )  
-
   call deallocate_overlap_type(overlap)
 
 end subroutine set_contact_point
@@ -6379,7 +6116,6 @@ nsend = 0
 maxsize = 0
 do m = 1, update%nsend
     do n = 1, update%send(m)%count
-       if( update%send(m)%is_refined(n) ) cycle 
        if( update%send(m)%rotation(n) == ONE_HUNDRED_EIGHTY ) cycle
        if( ( (position == EAST .OR. position == CORNER) .AND. update%send(m)%dir(n) == 1 ) .OR. &
             ( (position == NORTH .OR. position == CORNER) .AND. update%send(m)%dir(n) == 7 ) ) then
@@ -6401,7 +6137,6 @@ nlist = size(domain%list(:))
 pos = 0
 do m = 1, update%nsend
  do n = 1, update%send(m)%count
-    if( update%send(m)%is_refined(n) ) cycle
     if( update%send(m)%rotation(n) == ONE_HUNDRED_EIGHTY ) cycle
     ! comparing east direction on currently pe
     if( (position == EAST .OR. position == CORNER) .AND. update%send(m)%dir(n) == 1 ) then  
@@ -6455,7 +6190,6 @@ nrecv = 0
 maxsize = 0
 do m = 1, update%nrecv
     do n = 1, update%recv(m)%count
-       if( update%recv(m)%is_refined(n) ) cycle 
        if( update%recv(m)%rotation(n) == ONE_HUNDRED_EIGHTY ) cycle
        if( ( (position == EAST .OR. position == CORNER) .AND. update%recv(m)%dir(n) == 1 ) .OR. &
             ( (position == NORTH .OR. position == CORNER) .AND. update%recv(m)%dir(n) == 7 ) ) then
@@ -6476,7 +6210,6 @@ endif
 pos = 0
 do m = 1, update%nrecv
  do n = 1, update%recv(m)%count
-    if( update%recv(m)%is_refined(n) ) cycle
     if( update%recv(m)%rotation(n) == ONE_HUNDRED_EIGHTY ) cycle
     if( (position == EAST .OR. position == CORNER) .AND. update%recv(m)%dir(n) == 1 ) then  
        is = update%recv(m)%is(n) - 1
@@ -6515,23 +6248,38 @@ subroutine set_bound_overlap( domain, position )
   integer,        intent(in)    :: position
   integer                       :: m, n, l, count, dr, tMe, i
   integer, parameter            :: MAXCOUNT = 100
-  integer, dimension(MAXCOUNT)  :: dir, rotation, is, ie, js, je, isMe, ieMe, jsMe, jeMe, tileMe
+  integer, dimension(MAXCOUNT)  :: dir, rotation, is, ie, js, je, tileMe, index
   integer, dimension(size(domain%x(:)), 4)           :: nrecvl
-  integer, dimension(size(domain%x(:)), 4, MAXCOUNT) :: isl, iel, jsl, jel, islMe, ielMe, jslMe, jelMe
+  integer, dimension(size(domain%x(:)), 4, MAXCOUNT) :: isl, iel, jsl, jel
   type(overlap_type),  pointer   :: overlap => NULL()
   type(overlapSpec),   pointer   :: update => NULL()
   type(overlapSpec),   pointer   :: bound => NULL()
   integer                        :: nlist_send, nlist_recv, ishift, jshift
   integer                        :: ism, iem, jsm, jem, nsend, nrecv
   integer                        :: isg, ieg, jsg, jeg, nlist, list
-  integer                        :: isc1, iec1, jsc1, jec1
-  integer                        :: isc2, iec2, jsc2, jec2
+!  integer                        :: isc1, iec1, jsc1, jec1
+!  integer                        :: isc2, iec2, jsc2, jec2
   integer                        :: isd, ied, jsd, jed
+  integer                        :: npes_x, npes_y, ipos, jpos, inbr, jnbr
+  integer                        :: isc, iec, jsc, jec, my_pe
+  integer                        :: pe_south1, pe_south2, pe_west0, pe_west1, pe_west2
+  integer                        :: is_south1, ie_south1, js_south1, je_south1
+  integer                        :: is_south2, ie_south2, js_south2, je_south2
+  integer                        :: is_west0, ie_west0, js_west0, je_west0
+  integer                        :: is_west1, ie_west1, js_west1, je_west1
+  integer                        :: is_west2, ie_west2, js_west2, je_west2
+  logical                        :: x_cyclic, y_cyclic, folded_north
+
+  is_south1=0; ie_south1=0; js_south1=0; je_south1=0
+  is_south2=0; ie_south2=0; js_south2=0; je_south2=0
+  is_west0=0; ie_west0=0; js_west0=0; je_west0=0
+  is_west1=0; ie_west1=0; js_west1=0; je_west1=0
+  is_west2=0; ie_west2=0; js_west2=0; je_west2=0
 
 
   if( position == CENTER .OR. .NOT. domain%symmetry ) return
   call mpp_get_domain_shift(domain, ishift, jshift, position)
-  call mpp_get_global_domain(domain, isg, ieg, jsg, jeg, position=position)
+  call mpp_get_global_domain(domain, isg, ieg, jsg, jeg)
   call mpp_get_memory_domain ( domain, ism, iem, jsm, jem )
 
   select case(position)
@@ -6551,8 +6299,8 @@ subroutine set_bound_overlap( domain, position )
   bound%xbegin = ism; bound%xend = iem + ishift
   bound%ybegin = jsm; bound%yend = jem + jshift
 
-  nlist_send = update%nsend
-  nlist_recv = update%nrecv
+  nlist_send = max(update%nsend,4)
+  nlist_recv = max(update%nrecv,4)
   bound%nsend = nlist_send
   bound%nrecv = nlist_recv
   if(nlist_send >0) then
@@ -6566,58 +6314,147 @@ subroutine set_bound_overlap( domain, position )
   !--- loop over the list of domains to find the boundary overlap for send
   nlist = size(domain%list(:))
 
-  nsend = 0
-  !--- will computing overlap for tripolar grid.
-  if( BTEST(domain%fold,NORTH) ) then
-     !--- currently only set up for west and south boundary.
-    !---south boundary
+  npes_x = size(domain%x(1)%list(:))
+  npes_y = size(domain%y(1)%list(:))
+  x_cyclic = domain%x(1)%cyclic
+  y_cyclic = domain%y(1)%cyclic
+  folded_north = BTEST(domain%fold,NORTH)
+  ipos = domain%x(1)%pos    
+  jpos = domain%y(1)%pos
+  isc  = domain%x(1)%compute%begin; iec = domain%x(1)%compute%end
+  jsc  = domain%y(1)%compute%begin; jec = domain%y(1)%compute%end
+
+  nsend = 0 
+  if(domain%ntiles == 1) then  ! use neighbor processor to configure send and recv
+     ! currently only set up for west and south boundary
+
+     ! south boundary for send
+     pe_south1 = NULL_PE; pe_south2 = NULL_PE
      if( position == NORTH .OR. position == CORNER ) then
-        isc1 = domain%x(1)%compute%begin;      iec1 = domain%x(1)%compute%end +ishift
-        jsc1 = domain%y(1)%compute%end+jshift; jec1 = jsc1
+        inbr = ipos; jnbr = jpos + 1
+        if( jnbr == npes_y .AND. y_cyclic) jnbr = 0
+        if( inbr .GE. 0 .AND. inbr .LT. npes_x .AND. jnbr .GE. 0 .AND. jnbr .LT. npes_y ) then 
+           pe_south1 = domain%pearray(inbr,jnbr)
+           is_south1 = isc + ishift; ie_south1 = iec+ishift
+           js_south1 = jec + jshift; je_south1 = js_south1
+        endif         
      endif
-     !--- west boundary
-     if( position == EAST .OR. position == CORNER ) then
-        isc2 = domain%x(1)%compute%end+ishift; iec2 = isc2
-        jsc2 = domain%y(1)%compute%begin;      jec2 = domain%y(1)%compute%end + jshift
+     !--- send to the southwest processor when position is NORTH
+     if( position == CORNER ) then
+        inbr = ipos + 1; jnbr = jpos + 1
+        if( inbr == npes_x .AND. x_cyclic) inbr = 0
+        if( jnbr == npes_y .AND. y_cyclic) jnbr = 0
+        if( inbr .GE. 0 .AND. inbr .LT. npes_x .AND. jnbr .GE. 0 .AND. jnbr .LT. npes_y ) then 
+           pe_south2 = domain%pearray(inbr,jnbr)
+           is_south2 = iec + ishift; ie_south2 = is_south2
+           js_south2 = jec + jshift; je_south2 = js_south2
+        endif         
      endif
 
-     do list = 0,nlist-1
-        m = mod( domain%pos+list, nlist )
-        count = 0
-
-        !--- south boundary
-        if( position == NORTH .OR. position == CORNER ) then
-           isd = domain%list(m)%x(1)%compute%begin; ied = domain%list(m)%x(1)%compute%end + ishift
-           jsd = domain%list(m)%y(1)%compute%begin; jed = jsd
-           if( isc1 == isd .AND. iec1 == ied .AND. jsc1 == jsd) then
-              count = count + 1          
-              is(count) = isc1; ie(count) = iec1
-              js(count) = jsc1; je(count) = jec1
-              dir(count) = 2
+     !---west boundary for send
+     pe_west0 = NULL_PE; pe_west1 = NULL_PE; pe_west2 = NULL_PE
+     if( position == EAST ) then
+        inbr = ipos+1; jnbr = jpos
+        if( inbr == npes_x .AND. x_cyclic) inbr = 0
+        if( inbr .GE. 0 .AND. inbr .LT. npes_x .AND. jnbr .GE. 0 .AND. jnbr .LT. npes_y ) then 
+           pe_west1 = domain%pearray(inbr,jnbr)
+           is_west1 = iec + ishift; ie_west1 = is_west1
+           js_west1 = jsc + jshift; je_west1 = jec + jshift
+        endif         
+     else if ( position == CORNER ) then  ! possible split into two parts.
+        !--- on the fold.
+        if( folded_north .AND. jec == jeg .AND. ipos .LT. (npes_x-1)/2 ) then
+           inbr = npes_x - ipos - 1; jnbr = jpos
+           if( inbr .GE. 0 .AND. inbr .LT. npes_x .AND. jnbr .GE. 0 .AND. jnbr .LT. npes_y ) then 
+               pe_west0 = domain%pearray(inbr,jnbr)
+               is_west0 = iec+ishift; ie_west0 = is_west0
+               js_west0 = jec+jshift; je_west0 = js_west0
            endif
         endif
 
-        !--- west boundary
-        if( position == EAST .OR. position == CORNER ) then
-           isd = domain%list(m)%x(1)%compute%begin; ied = isd
-           jsd = domain%list(m)%y(1)%compute%begin; jed = domain%list(m)%y(1)%compute%end + jshift
-           !--- cyclic in x-direction is assumed for folded-north
-           if(isd == isg) then
-              isd = ieg; ied = ieg
-           endif
-           if( isc2 == isd .AND. jsc2 == jsd .AND. jec2 == jed ) then
-              count = count + 1          
-              is(count) = isc2; ie(count) = iec2
-              js(count) = jsc2; je(count) = jec2
-              dir(count) = 3
+        if( folded_north .AND. jec == jeg .AND. ipos .GE. npes_x/2 .AND. ipos .LT. (npes_x-1) ) then
+           inbr = ipos+1; jnbr = jpos
+           if( inbr == npes_x .AND. x_cyclic) inbr = 0
+           if( inbr .GE. 0 .AND. inbr .LT. npes_x .AND. jnbr .GE. 0 .AND. jnbr .LT. npes_y ) then 
+              pe_west1 = domain%pearray(inbr,jnbr)
+              is_west1 = iec + ishift; ie_west1 = is_west1
+              js_west1 = jsc + jshift; je_west1 = jec
            endif
+        else
+           inbr = ipos+1; jnbr = jpos
+           if( inbr == npes_x .AND. x_cyclic) inbr = 0
+           if( inbr .GE. 0 .AND. inbr .LT. npes_x .AND. jnbr .GE. 0 .AND. jnbr .LT. npes_y ) then 
+              pe_west1 = domain%pearray(inbr,jnbr)
+              is_west1 = iec + ishift; ie_west1 = is_west1
+              js_west1 = jsc + jshift; je_west1 = jec + jshift
+           endif         
         endif
+     endif 
+     !--- send to the southwest processor when position is NORTH
+     if( position == CORNER ) then
+        inbr = ipos + 1; jnbr = jpos + 1
+        if( inbr == npes_x .AND. x_cyclic) inbr = 0
+        if( jnbr == npes_y .AND. y_cyclic) jnbr = 0
+        if( inbr .GE. 0 .AND. inbr .LT. npes_x .AND. jnbr .GE. 0 .AND. jnbr .LT. npes_y ) then 
+           pe_west2 = domain%pearray(inbr,jnbr)
+           is_west2 = iec + ishift; ie_west2 = is_west2
+           js_west2 = jec + jshift; je_west2 = js_west2
+        endif         
+     endif
+
+!write(1000+mpp_pe(),*)"send south 1", pe_south1, is_south1, ie_south1, js_south1, je_south1
+!write(1000+mpp_pe(),*)"send south 2", pe_south2, is_south2, ie_south2, js_south2, je_south2
+!write(1000+mpp_pe(),*)"send west 0", pe_west0, is_west0, ie_west0, js_west0, je_west0
+!write(1000+mpp_pe(),*)"send west 1", pe_west1, is_west1, ie_west1, js_west1, je_west1
+!write(1000+mpp_pe(),*)"send west 2", pe_west2, is_west2, ie_west2, js_west2, je_west2
+
+
+     do list = 0,nlist-1
+        m = mod( domain%pos+list, nlist )
+        count = 0
+        my_pe = domain%list(m)%pe
+        if(my_pe == pe_south1) then
+           count = count + 1
+           is(count) = is_south1; ie(count) = ie_south1
+           js(count) = js_south1; je(count) = je_south1
+           dir(count) = 2
+           rotation(count) = ZERO
+        endif          
+        if(my_pe == pe_south2) then
+           count = count + 1
+           is(count) = is_south2; ie(count) = ie_south2
+           js(count) = js_south2; je(count) = je_south2
+           dir(count) = 2
+           rotation(count) = ZERO
+        endif          
+
+        if(my_pe == pe_west0) then
+           count = count + 1
+           is(count) = is_west0; ie(count) = ie_west0
+           js(count) = js_west0; je(count) = je_west0
+           dir(count) = 3
+           rotation(count) = ONE_HUNDRED_EIGHTY
+        endif          
+        if(my_pe == pe_west1) then
+           count = count + 1
+           is(count) = is_west1; ie(count) = ie_west1
+           js(count) = js_west1; je(count) = je_west1
+           dir(count) = 3
+           rotation(count) = ZERO
+        endif          
+        if(my_pe == pe_west2) then
+           count = count + 1
+           is(count) = is_west2; ie(count) = ie_west2
+           js(count) = js_west2; je(count) = je_west2
+           dir(count) = 3
+           rotation(count) = ZERO
+        endif          
 
         if(count >0) then
            nsend = nsend + 1
-           if(nsend > nlist_send) call mpp_error(FATAL, "set_bound_overlap: nsend > nlist_send")
+           if(nsend > nlist_send)  call mpp_error(FATAL, "set_bound_overlap: nsend > nlist_send")
            bound%send(nsend)%count = count
-           bound%send(nsend)%pe    = domain%list(m)%pe
+           bound%send(nsend)%pe    = my_pe
            allocate(bound%send(nsend)%is(count),  bound%send(nsend)%ie(count) )
            allocate(bound%send(nsend)%js(count),  bound%send(nsend)%je(count) )
            allocate(bound%send(nsend)%dir(count), bound%send(nsend)%rotation(count) )
@@ -6628,12 +6465,16 @@ subroutine set_bound_overlap( domain, position )
            bound%send(nsend)%je(:)       = je(1:count)
            bound%send(nsend)%dir(:)      = dir(1:count)
            bound%send(nsend)%tileMe(:)   = 1
-           bound%send(nsend)%rotation(:) = ZERO           
+           bound%send(nsend)%rotation(:) = rotation(1:count)     
+!write(1000+mpp_pe(),*) "send:", count, my_pe
+!do i = 1, count
+!   write(1000+mpp_pe(),*) "send index:", is(i), ie(i), js(i), je(i), dir(i), rotation(i)
+!enddo
         endif
      enddo
   else 
      !--- The following did not consider wide halo case.
-     do m = 1, nlist_send
+     do m = 1, update%nsend
         overlap => update%send(m)
         if( overlap%count == 0 ) cycle
         count = 0
@@ -6738,98 +6579,162 @@ subroutine set_bound_overlap( domain, position )
   nrecv       = 0
 
   !--- will computing overlap for tripolar grid.
-  if( BTEST(domain%fold,NORTH) ) then
-     tMe = 1
-     !--- currently only set up for west and south boundary.
-     !---south boundary
+  if( domain%ntiles == 1 ) then
+     ! currently only set up for west and south boundary
+
+     ! south boundary for recv
+     pe_south1 = NULL_PE; pe_south2 = NULL_PE
      if( position == NORTH .OR. position == CORNER ) then
-        isc1 = domain%x(1)%compute%begin; iec1 = domain%x(1)%compute%end +ishift
-        jsc1 = domain%y(1)%compute%begin; jec1 = jsc1
-     endif
-     !--- west boundary
-     if( position == EAST .OR. position == CORNER ) then
-        isc2 = domain%x(1)%compute%begin; iec2 = isc2
-        jsc2 = domain%y(1)%compute%begin; jec2 = domain%y(1)%compute%end + jshift
+        inbr = ipos; jnbr = jpos - 1
+        if( jnbr == -1 .AND. y_cyclic) jnbr = npes_y-1
+        if( inbr .GE. 0 .AND. inbr .LT. npes_x .AND. jnbr .GE. 0 .AND. jnbr .LT. npes_y ) then 
+           pe_south1 = domain%pearray(inbr,jnbr)
+           is_south1 = isc + ishift; ie_south1 = iec+ishift
+           js_south1 = jsc;          je_south1 = js_south1
+        endif         
      endif
 
-     do list = 0,nlist-1
-        m = mod( domain%pos+nlist-list, nlist )
-        count = 0
+     !--- south boudary for recv:  the southwest point when position is NORTH
+     if( position == CORNER ) then
+        inbr = ipos - 1; jnbr = jpos - 1
+        if( inbr == -1 .AND. x_cyclic) inbr = npes_x-1
+        if( jnbr == -1 .AND. y_cyclic) jnbr = npes_y-1
+        if( inbr .GE. 0 .AND. inbr .LT. npes_x .AND. jnbr .GE. 0 .AND. jnbr .LT. npes_y ) then 
+           pe_south2 = domain%pearray(inbr,jnbr)
+           is_south2 = isc; ie_south2 = is_south2
+           js_south2 = jsc; je_south2 = js_south2
+        endif         
+     endif
 
-        !--- south boundary
-        if( position == NORTH .OR. position == CORNER ) then
-           isd = domain%list(m)%x(1)%compute%begin;      ied = domain%list(m)%x(1)%compute%end + ishift
-           jsd = domain%list(m)%y(1)%compute%end+jshift; jed = jsd
-           if( isc1 == isd .AND. iec1 == ied .AND. jsc1 == jsd) then
-              count = count + 1          
-              is(count) = isc1; ie(count) = iec1
-              js(count) = jsc1; je(count) = jec1
-              dir(count) = 2
-              nrecvl(tMe, 2) = nrecvl(tMe,2) + 1
-              islMe(tMe,2,nrecvl(tMe, 2)) = is(count)
-              ielMe(tMe,2,nrecvl(tMe, 2)) = ie(count)
-              jslMe(tMe,2,nrecvl(tMe, 2)) = js(count)
-              jelMe(tMe,2,nrecvl(tMe, 2)) = je(count)
-              isl  (tMe,2,nrecvl(tMe, 2)) = is(count)
-              iel  (tMe,2,nrecvl(tMe, 2)) = ie(count)
-              jsl  (tMe,2,nrecvl(tMe, 2)) = js(count)
-              jel  (tMe,2,nrecvl(tMe, 2)) = je(count)
 
+     !---west boundary for recv
+     pe_west0 = NULL_PE; pe_west1 = NULL_PE; pe_west2 = NULL_PE
+     if( position == EAST ) then
+        inbr = ipos-1; jnbr = jpos
+        if( inbr == -1 .AND. x_cyclic) inbr = npes_x-1
+        if( inbr .GE. 0 .AND. inbr .LT. npes_x .AND. jnbr .GE. 0 .AND. jnbr .LT. npes_y ) then 
+           pe_west1 = domain%pearray(inbr,jnbr)
+           is_west1 = isc;          ie_west1 = is_west1
+           js_west1 = jsc + jshift; je_west1 = jec + jshift
+        endif         
+     else if ( position == CORNER ) then  ! possible split into two parts.
+        !--- on the fold.
+        if( folded_north .AND. jec == jeg .AND. ipos .GT. npes_x/2 ) then
+           inbr = npes_x - ipos - 1; jnbr = jpos
+           if( inbr .GE. 0 .AND. inbr .LT. npes_x .AND. jnbr .GE. 0 .AND. jnbr .LT. npes_y ) then 
+               pe_west0 = domain%pearray(inbr,jnbr)
+               is_west0 = isc;        ie_west0 = is_west0
+               js_west0 = jec+jshift; je_west0 = js_west0
            endif
-        endif
-
-        !--- west boundary
-        if( position == EAST .OR. position == CORNER ) then
-           isd = domain%list(m)%x(1)%compute%end+ishift; ied = isd
-           jsd = domain%list(m)%y(1)%compute%begin;      jed = domain%list(m)%y(1)%compute%end + jshift
-           !--- cyclic in x-direction is assumed for folded-north
-           if(ied == ieg) then
-              isd = isg; ied = isg
+           inbr = ipos-1; jnbr = jpos
+           if( inbr == -1 .AND. x_cyclic) inbr = npes_x-1
+           if( inbr .GE. 0 .AND. inbr .LT. npes_x .AND. jnbr .GE. 0 .AND. jnbr .LT. npes_y ) then 
+              pe_west1 = domain%pearray(inbr,jnbr)
+              is_west1 = isc;          ie_west1 = is_west1
+              js_west1 = jsc + jshift; je_west1 = jec
            endif
-           if( isc2 == isd .AND. jsc2 == jsd .AND. jec2 == jed ) then
-              count = count + 1          
-              is(count) = isc2; ie(count) = iec2
-              js(count) = jsc2; je(count) = jec2
-              dir(count) = 3
-              nrecvl(tMe, 3) = nrecvl(tMe,3) + 1
-              islMe(tMe,3,nrecvl(tMe, 3)) = is(count)
-              ielMe(tMe,3,nrecvl(tMe, 3)) = ie(count)
-              jslMe(tMe,3,nrecvl(tMe, 3)) = js(count)
-              jelMe(tMe,3,nrecvl(tMe, 3)) = je(count)
-              isl  (tMe,3,nrecvl(tMe, 3)) = is(count)
-              iel  (tMe,3,nrecvl(tMe, 3)) = ie(count)
-              jsl  (tMe,3,nrecvl(tMe, 3)) = js(count)
-              jel  (tMe,3,nrecvl(tMe, 3)) = je(count)
+        else
+           inbr = ipos-1; jnbr = jpos
+           if( inbr == -1 .AND. x_cyclic) inbr = npes_x-1
+           if( inbr .GE. 0 .AND. inbr .LT. npes_x .AND. jnbr .GE. 0 .AND. jnbr .LT. npes_y ) then 
+              pe_west1 = domain%pearray(inbr,jnbr)
+              is_west1 = isc;          ie_west1 = is_west1
+              js_west1 = jsc + jshift; je_west1 = jec+jshift
            endif
         endif
+     endif 
+
+     !--- west boundary for recv: the southwest point when position is CORNER
+     if( position == CORNER ) then
+        inbr = ipos - 1; jnbr = jpos - 1
+        if( inbr == -1 .AND. x_cyclic) inbr = npes_x - 1
+        if( jnbr == -1 .AND. y_cyclic) jnbr = npes_y - 1
+        if( inbr .GE. 0 .AND. inbr .LT. npes_x .AND. jnbr .GE. 0 .AND. jnbr .LT. npes_y ) then 
+           pe_west2 = domain%pearray(inbr,jnbr)
+           is_west2 = isc; ie_west2 = is_west2
+           js_west2 = jsc; je_west2 = js_west2
+        endif         
+     endif
+
+!write(1000+mpp_pe(),*)"recv south 1", pe_south1, is_south1, ie_south1, js_south1, je_south1
+!write(1000+mpp_pe(),*)"recv south 2", pe_south2, is_south2, ie_south2, js_south2, je_south2
+!write(1000+mpp_pe(),*)"recv west 0", pe_west0, is_west0, ie_west0, js_west0, je_west0
+!write(1000+mpp_pe(),*)"recv west 1", pe_west1, is_west1, ie_west1, js_west1, je_west1
+!write(1000+mpp_pe(),*)"recv west 2", pe_west2, is_west2, ie_west2, js_west2, je_west2
+     
+     tMe = 1
+     do list = 0,nlist-1
+        m = mod( domain%pos+nlist-list, nlist )
+        count = 0
+        my_pe = domain%list(m)%pe
+        if(my_pe == pe_south1) then
+           count = count + 1
+           is(count) = is_south1; ie(count) = ie_south1
+           js(count) = js_south1; je(count) = je_south1
+           dir(count) = 2
+           rotation(count) = ZERO
+           index(count) = 1 + ishift
+        endif          
+        if(my_pe == pe_south2) then
+           count = count + 1
+           is(count) = is_south2; ie(count) = ie_south2
+           js(count) = js_south2; je(count) = je_south2
+           dir(count) = 2
+           rotation(count) = ZERO
+           index(count) = 1
+        endif          
+        if(my_pe == pe_west0) then
+           count = count + 1
+           is(count) = is_west0; ie(count) = ie_west0
+           js(count) = js_west0; je(count) = je_west0
+           dir(count) = 3
+           rotation(count) = ONE_HUNDRED_EIGHTY
+           index(count) = jec-jsc+1+jshift
+        endif          
+        if(my_pe == pe_west1) then
+           count = count + 1
+           is(count) = is_west1; ie(count) = ie_west1
+           js(count) = js_west1; je(count) = je_west1
+           dir(count) = 3
+           rotation(count) = ZERO
+           index(count) = 1 + jshift
+        endif          
+        if(my_pe == pe_west2) then
+           count = count + 1
+           is(count) = is_west2; ie(count) = ie_west2
+           js(count) = js_west2; je(count) = je_west2
+           dir(count) = 3
+           rotation(count) = ZERO
+           index(count) = 1
+        endif       
 
         if(count >0) then
            nrecv = nrecv + 1
            if(nrecv > nlist_recv) call mpp_error(FATAL, "set_bound_overlap: nrecv > nlist_recv")
            bound%recv(nrecv)%count = count
-           bound%recv(nrecv)%pe    = domain%list(m)%pe
-           allocate(bound%recv(nrecv)%isMe(count),   bound%recv(nrecv)%ieMe(count) )
-           allocate(bound%recv(nrecv)%jsMe(count),   bound%recv(nrecv)%jeMe(count) )
+           bound%recv(nrecv)%pe    = my_pe
            allocate(bound%recv(nrecv)%is(count),     bound%recv(nrecv)%ie(count) )
            allocate(bound%recv(nrecv)%js(count),     bound%recv(nrecv)%je(count) )
            allocate(bound%recv(nrecv)%dir(count),    bound%recv(nrecv)%index(count)  )
            allocate(bound%recv(nrecv)%tileMe(count), bound%recv(nrecv)%rotation(count) )
+           
            bound%recv(nrecv)%is(:)       = is(1:count)
            bound%recv(nrecv)%ie(:)       = ie(1:count)
            bound%recv(nrecv)%js(:)       = js(1:count)
            bound%recv(nrecv)%je(:)       = je(1:count)
-           bound%recv(nrecv)%isMe(:)     = is(1:count)
-           bound%recv(nrecv)%ieMe(:)     = ie(1:count)
-           bound%recv(nrecv)%jsMe(:)     = js(1:count)
-           bound%recv(nrecv)%jeMe(:)     = je(1:count)
-
            bound%recv(nrecv)%dir(:)      = dir(1:count)
            bound%recv(nrecv)%tileMe(:)   = 1
-           bound%recv(nrecv)%rotation(:) = ZERO           
+           bound%recv(nrecv)%rotation(:) = rotation(1:count)    
+           bound%recv(nrecv)%index(:)        = index(1:count)
+!write(1000+mpp_pe(),*) "recv:", count, my_pe
+!do i = 1, count
+!   write(1000+mpp_pe(),*) "recv index:", is(i), ie(i), js(i), je(i), dir(i), rotation(i)
+!enddo
+
         endif
      enddo
   else 
-     do m = 1, nlist_recv
+     do m = 1, update%nrecv
         overlap => update%recv(m)
         if( overlap%count == 0 ) cycle
         count = 0
@@ -6840,36 +6745,13 @@ subroutine set_bound_overlap( domain, position )
               count=count+1
               dir(count) = 1
               rotation(count) = overlap%rotation(n)
-              isMe(count)     = overlap%isMe(n) - 1
-              ieMe(count) = isMe(count)
-              jsMe(count)     = overlap%jsMe(n)
-              jeMe(count)     = overlap%jeMe(n)
               tileMe(count)   = overlap%tileMe(n)
-              if( overlap%is_refined(n)) then
-                 select case( rotation(count) ) 
-                 case( ZERO ) ! W -> E
-                    is(count) = overlap%is(n) - 1
-                    ie(count) = is(count)
-                    js(count) = overlap%js(n)
-                    je(count) = overlap%je(n)
-                 case( NINETY ) ! S -> E
-                    is(count) = overlap%is(n)
-                    ie(count) = overlap%ie(n)
-                    js(count) = overlap%js(n) - 1
-                    je(count) = js(count)
-                 end select
-              else
-                 is(count) = overlap%is(n) - 1
-                 ie(count) = is(count)
-                 js(count) = overlap%js(n)
-                 je(count) = overlap%je(n)
-              end if
+              is(count) = overlap%is(n) - 1
+              ie(count) = is(count)
+              js(count) = overlap%js(n)
+              je(count) = overlap%je(n)
               tMe                        = tileMe(count)
               nrecvl(tMe, 1) = nrecvl(tMe,1) + 1
-              islMe(tMe,1,nrecvl(tMe, 1)) = isMe(count)
-              ielMe(tMe,1,nrecvl(tMe, 1)) = ieMe(count)
-              jslMe(tMe,1,nrecvl(tMe, 1)) = jsMe(count)
-              jelMe(tMe,1,nrecvl(tMe, 1)) = jeMe(count)
               isl  (tMe,1,nrecvl(tMe, 1)) = is  (count)
               iel  (tMe,1,nrecvl(tMe, 1)) = ie  (count)
               jsl  (tMe,1,nrecvl(tMe, 1)) = js  (count)
@@ -6880,36 +6762,13 @@ subroutine set_bound_overlap( domain, position )
               count=count+1
               dir(count) = 2
               rotation(count) = overlap%rotation(n)
-              isMe(count)     = overlap%isMe(n)
-              ieMe(count)     = overlap%ieMe(n)
-              jsMe(count)     = overlap%jeMe(n) + 1
-              jeMe(count) = jsMe(count)
               tileMe(count)   = overlap%tileMe(n)
-              if( overlap%is_refined(n)) then
-                 select case( rotation(count) ) 
-                 case( ZERO ) ! N->S
-                    is(count) = overlap%is(n)
-                    ie(count) = overlap%ie(n)
-                    js(count) = overlap%je(n) + 1
-                    je(count) = js(count)
-                 case( MINUS_NINETY ) ! E->S
-                    is(count) = overlap%ie(n) + 1
-                    ie(count) = is(count)
-                    js(count) = overlap%js(n)
-                    je(count) = overlap%je(n)
-                 end select
-              else
-                 is(count) = overlap%is(n)
-                 ie(count) = overlap%ie(n)
-                 js(count) = overlap%je(n) + 1
-                 je(count) = js(count)
-              end if
+              is(count) = overlap%is(n)
+              ie(count) = overlap%ie(n)
+              js(count) = overlap%je(n) + 1
+              je(count) = js(count)
               tMe                        = tileMe(count)
               nrecvl(tMe, 2) = nrecvl(tMe,2) + 1
-              islMe(tMe,2,nrecvl(tMe, 2)) = isMe(count)
-              ielMe(tMe,2,nrecvl(tMe, 2)) = ieMe(count)
-              jslMe(tMe,2,nrecvl(tMe, 2)) = jsMe(count)
-              jelMe(tMe,2,nrecvl(tMe, 2)) = jeMe(count)
               isl  (tMe,2,nrecvl(tMe, 2)) = is  (count)
               iel  (tMe,2,nrecvl(tMe, 2)) = ie  (count)
               jsl  (tMe,2,nrecvl(tMe, 2)) = js  (count)
@@ -6920,36 +6779,13 @@ subroutine set_bound_overlap( domain, position )
               count=count+1
               dir(count) = 3
               rotation(count) = overlap%rotation(n)
-              isMe(count)     = overlap%ieMe(n) + 1 
-              ieMe(count) = isMe(count)
-              jsMe(count)     = overlap%jsMe(n)
-              jeMe(count)     = overlap%jeMe(n)
               tileMe(count)   = overlap%tileMe(n)
-              if( overlap%is_refined(n)) then
-                 select case( rotation(count) ) 
-                 case( ZERO ) ! E->W
-                    is(count) = overlap%ie(n) + 1
-                    ie(count) = is(count)
-                    js(count) = overlap%js(n)
-                    je(count) = overlap%je(n)
-                 case( NINETY ) ! S -> E
-                    is(count) = overlap%is(n)
-                    ie(count) = overlap%ie(n)
-                    js(count) = overlap%je(n) + 1
-                    je(count) = js(count)
-                 end select
-              else
-                 is(count) = overlap%ie(n) + 1
-                 ie(count) = is(count)
-                 js(count) = overlap%js(n)
-                 je(count) = overlap%je(n)
-              end if
+              is(count) = overlap%ie(n) + 1
+              ie(count) = is(count)
+              js(count) = overlap%js(n)
+              je(count) = overlap%je(n)
               tMe                        = tileMe(count)
               nrecvl(tMe, 3) = nrecvl(tMe,3) + 1
-              islMe(tMe,3,nrecvl(tMe, 3)) = isMe(count)
-              ielMe(tMe,3,nrecvl(tMe, 3)) = ieMe(count)
-              jslMe(tMe,3,nrecvl(tMe, 3)) = jsMe(count)
-              jelMe(tMe,3,nrecvl(tMe, 3)) = jeMe(count)
               isl  (tMe,3,nrecvl(tMe, 3)) = is  (count)
               iel  (tMe,3,nrecvl(tMe, 3)) = ie  (count)
               jsl  (tMe,3,nrecvl(tMe, 3)) = js  (count)
@@ -6960,36 +6796,13 @@ subroutine set_bound_overlap( domain, position )
               count=count+1
               dir(count) = 4
               rotation(count) = overlap%rotation(n)
-              isMe(count)     = overlap%isMe(n)
-              ieMe(count)     = overlap%ieMe(n)
-              jsMe(count)     = overlap%jsMe(n) - 1
-              jeMe(count) = jsMe(count)
               tileMe(count) = overlap%tileMe(n)
-              if( overlap%is_refined(n)) then
-                 select case( rotation(count) ) 
-                 case( ZERO ) ! S->N
-                    is(count) = overlap%is(n)
-                    ie(count) = overlap%ie(n)
-                    js(count) = overlap%js(n) - 1
-                    je(count) = js(count)
-                 case( MINUS_NINETY ) ! W->N
-                    is(count) = overlap%is(n) - 1
-                    ie(count) = is(count)
-                    js(count) = overlap%js(n)
-                    je(count) = overlap%je(n)
-                 end select
-              else
-                 is(count) = overlap%is(n)
-                 ie(count) = overlap%ie(n)
-                 js(count) = overlap%js(n) - 1
-                 je(count) = js(count)
-              end if
+              is(count) = overlap%is(n)
+              ie(count) = overlap%ie(n)
+              js(count) = overlap%js(n) - 1
+              je(count) = js(count)
               tMe                        = tileMe(count)
               nrecvl(tMe, 4) = nrecvl(tMe,4) + 1
-              islMe(tMe,4,nrecvl(tMe, 4)) = isMe(count)
-              ielMe(tMe,4,nrecvl(tMe, 4)) = ieMe(count)
-              jslMe(tMe,4,nrecvl(tMe, 4)) = jsMe(count)
-              jelMe(tMe,4,nrecvl(tMe, 4)) = jeMe(count)
               isl  (tMe,4,nrecvl(tMe, 4)) = is  (count)
               iel  (tMe,4,nrecvl(tMe, 4)) = ie  (count)
               jsl  (tMe,4,nrecvl(tMe, 4)) = js  (count)
@@ -7000,16 +6813,10 @@ subroutine set_bound_overlap( domain, position )
            nrecv = nrecv + 1
            bound%recv(nrecv)%count = count
            bound%recv(nrecv)%pe    = overlap%pe
-           allocate(bound%recv(nrecv)%isMe(count),   bound%recv(nrecv)%ieMe(count) )
-           allocate(bound%recv(nrecv)%jsMe(count),   bound%recv(nrecv)%jeMe(count) )
            allocate(bound%recv(nrecv)%is(count),     bound%recv(nrecv)%ie(count) )
            allocate(bound%recv(nrecv)%js(count),     bound%recv(nrecv)%je(count) )
            allocate(bound%recv(nrecv)%dir(count),    bound%recv(nrecv)%index(count)  )
            allocate(bound%recv(nrecv)%tileMe(count), bound%recv(nrecv)%rotation(count) )
-           bound%recv(nrecv)%isMe(:)     = isMe(1:count)
-           bound%recv(nrecv)%ieMe(:)     = ieMe(1:count)
-           bound%recv(nrecv)%jsMe(:)     = jsMe(1:count)
-           bound%recv(nrecv)%jeMe(:)     = jeMe(1:count)
            bound%recv(nrecv)%is(:)       = is(1:count)
            bound%recv(nrecv)%ie(:)       = ie(1:count)
            bound%recv(nrecv)%js(:)       = js(1:count)
@@ -7019,33 +6826,38 @@ subroutine set_bound_overlap( domain, position )
            bound%recv(nrecv)%rotation(:) = rotation(1:count)
         end if
      end do ! end  do list = 0, nlist
-  endif
-  bound%nrecv = nrecv
-
-  !--- find the boundary index for each contact within the east boundary
-  do m = 1, nrecv
-     do n = 1, bound%recv(m)%count
-        tMe = bound%recv(m)%tileMe(n)
-        dr = bound%recv(m)%dir(n)
-        bound%recv(m)%index(n) = 1
-        do l = 1, nrecvl(tMe,dr)
-           if(dr == 1 .OR. dr == 3) then  ! EAST, WEST
-              if( bound%recv(m)%jsMe(n) > jslMe(tMe, dr, l) ) then
-                 bound%recv(m)%index(n) = bound%recv(m)%index(n)                  + &
-                      max(abs(jel(tMe, dr, l)-jsl(tMe, dr, l)), &
-                      abs(iel(tMe, dr, l)-isl(tMe, dr, l))) + 1 - jshift
-              end if
-           else                             ! South, North
-              if( bound%recv(m)%isMe(n) > islMe(tMe, dr, l) ) then
-                 bound%recv(m)%index(n) = bound%recv(m)%index(n)                  + &
-                      max(abs(jel(tMe, dr, l)-jsl(tMe, dr, l)), &
-                      abs(iel(tMe, dr, l)-isl(tMe, dr, l))) + 1 - ishift
+     !--- find the boundary index for each contact within the east boundary
+     do m = 1, nrecv
+        do n = 1, bound%recv(m)%count
+           tMe = bound%recv(m)%tileMe(n)
+           dr = bound%recv(m)%dir(n)
+           bound%recv(m)%index(n) = 1
+           do l = 1, nrecvl(tMe,dr)
+              if(dr == 1 .OR. dr == 3) then  ! EAST, WEST
+                 if( bound%recv(m)%js(n) > jsl(tMe, dr, l) ) then
+                    if( bound%recv(m)%rotation(n) == ONE_HUNDRED_EIGHTY ) then
+                       bound%recv(m)%index(n) = bound%recv(m)%index(n)                  + &
+                            max(abs(jel(tMe, dr, l)-jsl(tMe, dr, l))+1, &
+                            abs(iel(tMe, dr, l)-isl(tMe, dr, l))+1)
+                    else
+                       bound%recv(m)%index(n) = bound%recv(m)%index(n)                  + &
+                            max(abs(jel(tMe, dr, l)-jsl(tMe, dr, l)), &
+                            abs(iel(tMe, dr, l)-isl(tMe, dr, l))) + 1 - jshift
+                    endif
+                 end if
+              else                             ! South, North
+                 if( bound%recv(m)%is(n) > isl(tMe, dr, l) ) then
+                    bound%recv(m)%index(n) = bound%recv(m)%index(n)                  + &
+                         max(abs(jel(tMe, dr, l)-jsl(tMe, dr, l)), &
+                         abs(iel(tMe, dr, l)-isl(tMe, dr, l))) + 1 - ishift
+                 end if
               end if
-           end if
+           end do
         end do
      end do
-  end do
 
+  endif
+  bound%nrecv = nrecv
 
 
 end subroutine set_bound_overlap
@@ -7908,15 +7720,10 @@ subroutine allocate_update_overlap( overlap, count)
   allocate(overlap%tileMe      (count), overlap%tileNbr (count) )
   allocate(overlap%is          (count), overlap%ie      (count) )
   allocate(overlap%js          (count), overlap%je      (count) )
-  allocate(overlap%isMe        (count), overlap%ieMe    (count) )
-  allocate(overlap%jsMe        (count), overlap%jeMe    (count) )
   allocate(overlap%dir         (count), overlap%rotation(count) )
-  allocate(overlap%is_refined  (count), overlap%index   (count) )
   allocate(overlap%from_contact(count), overlap%msgsize (count) )
-  overlap%is_refined   = .FALSE.
   overlap%rotation     =  ZERO
   overlap%from_contact = .FALSE.
-  overlap%index        = -1   
 
 end subroutine allocate_update_overlap
 
@@ -7949,7 +7756,7 @@ end subroutine allocate_update_overlap
     endif
 
     if(is_overlapped) then
-       if( overlap%pe == NULL_PE ) then
+     if( overlap%count == 0 ) then
           overlap%pe = pe
        else
           if(overlap%pe .NE. pe) call mpp_error(FATAL,  &
@@ -7976,25 +7783,25 @@ end subroutine allocate_update_overlap
   end subroutine insert_update_overlap
 
   !#####################################################################################
-  subroutine insert_mosaic_update_overlap(overlap, pe, tileMe, tileNbr, is, ie, js, je, dir, &
-                                          rotation, from_contact, refined)
+subroutine insert_overlap_type(overlap, pe, tileMe, tileNbr, is, ie, js, je, dir, &
+     rotation, from_contact)
     type(overlap_type), intent(inout) :: overlap
     integer,            intent(in   ) :: tileMe, tileNbr, pe
     integer,            intent(in   ) :: is, ie, js, je
     integer,            intent(in   ) :: dir, rotation
-    logical,            intent(in   ) :: from_contact, refined
+  logical,            intent(in   ) :: from_contact
     integer                           :: count
 
-    if( overlap%pe == NULL_PE ) then
+  if( overlap%count == 0 ) then
        overlap%pe = pe
     else
        if(overlap%pe .NE. pe) call mpp_error(FATAL,  &
-            "mpp_domains_define.inc(insert_mosaic_update_overlap): mismatch on pe")
+          "mpp_domains_define.inc(insert_overlap_type): mismatch on pe")
     endif
     overlap%count = overlap%count+1
     count = overlap%count
     if(count > MAXOVERLAP) call mpp_error(FATAL, &
-         "mpp_domains_define.inc(insert_mosaic_update_overlap): number of overlap is greater than MAXOVERLAP, increase MAXOVERLAP")
+       "mpp_domains_define.inc(insert_overlap_type): number of overlap is greater than MAXOVERLAP, increase MAXOVERLAP")
     overlap%tileMe      (count) = tileMe
     overlap%tileNbr     (count) = tileNbr
     overlap%is          (count) = is
@@ -8004,10 +7811,10 @@ end subroutine allocate_update_overlap
     overlap%dir         (count) = dir
     overlap%rotation    (count) = rotation
     overlap%from_contact(count) = from_contact
-    overlap%is_refined  (count) = refined
     overlap%msgsize     (count) = (ie-is+1)*(je-js+1)
 
-  end subroutine insert_mosaic_update_overlap
+end subroutine insert_overlap_type
+
 
 !#######################################################################
 subroutine deallocate_overlap_type( overlap)
@@ -8025,14 +7832,8 @@ subroutine deallocate_overlap_type( overlap)
   if(ASSOCIATED(overlap%ie)) deallocate(overlap%ie)
   if(ASSOCIATED(overlap%js)) deallocate(overlap%js)
   if(ASSOCIATED(overlap%je)) deallocate(overlap%je)
-  if(ASSOCIATED(overlap%isMe)) deallocate(overlap%isMe)
-  if(ASSOCIATED(overlap%ieMe)) deallocate(overlap%ieMe)
-  if(ASSOCIATED(overlap%jsMe)) deallocate(overlap%jsMe)
-  if(ASSOCIATED(overlap%jeMe)) deallocate(overlap%jeMe)
   if(ASSOCIATED(overlap%dir)) deallocate(overlap%dir)
   if(ASSOCIATED(overlap%rotation)) deallocate(overlap%rotation)
-  if(ASSOCIATED(overlap%is_refined)) deallocate(overlap%is_refined)
-  if(ASSOCIATED(overlap%index)) deallocate(overlap%index)
   if(ASSOCIATED(overlap%from_contact)) deallocate(overlap%from_contact)
   if(ASSOCIATED(overlap%msgsize)) deallocate(overlap%msgsize)
   overlap%count = 0
@@ -8091,14 +7892,8 @@ subroutine add_update_overlap( overlap_out, overlap_in)
      overlap%ie          (1:count_out) = overlap_out%ie          (1:count_out)
      overlap%js          (1:count_out) = overlap_out%js          (1:count_out)
      overlap%je          (1:count_out) = overlap_out%je          (1:count_out)
-     overlap%isMe        (1:count_out) = overlap_out%isMe        (1:count_out)
-     overlap%ieMe        (1:count_out) = overlap_out%ieMe        (1:count_out)
-     overlap%jsMe        (1:count_out) = overlap_out%jsMe        (1:count_out)
-     overlap%jeMe        (1:count_out) = overlap_out%jeMe        (1:count_out)
      overlap%dir         (1:count_out) = overlap_out%dir         (1:count_out)
      overlap%rotation    (1:count_out) = overlap_out%rotation    (1:count_out)
-     overlap%is_refined  (1:count_out) = overlap_out%is_refined  (1:count_out)
-     overlap%index       (1:count_out) = overlap_out%index       (1:count_out)
      overlap%from_contact(1:count_out) = overlap_out%from_contact(1:count_out)
      call deallocate_overlap_type(overlap_out)
      call allocate_update_overlap(overlap_out, count)
@@ -8108,13 +7903,8 @@ subroutine add_update_overlap( overlap_out, overlap_in)
      overlap_out%ie          (1:count_out) = overlap%ie          (1:count_out)
      overlap_out%js          (1:count_out) = overlap%js          (1:count_out)
      overlap_out%je          (1:count_out) = overlap%je          (1:count_out)
-     overlap_out%isMe        (1:count_out) = overlap%isMe        (1:count_out)
-     overlap_out%ieMe        (1:count_out) = overlap%ieMe        (1:count_out)
-     overlap_out%jsMe        (1:count_out) = overlap%jsMe        (1:count_out)
-     overlap_out%jeMe        (1:count_out) = overlap%jeMe        (1:count_out)
      overlap_out%dir         (1:count_out) = overlap%dir         (1:count_out)
      overlap_out%rotation    (1:count_out) = overlap%rotation    (1:count_out)
-     overlap_out%is_refined  (1:count_out) = overlap%is_refined  (1:count_out)
      overlap_out%index       (1:count_out) = overlap%index       (1:count_out)
      overlap_out%from_contact(1:count_out) = overlap%from_contact(1:count_out)        
      overlap_out%msgsize     (1:count_out) = overlap%msgsize     (1:count_out)
@@ -8127,14 +7917,8 @@ subroutine add_update_overlap( overlap_out, overlap_in)
   overlap_out%ie          (count_out+1:count) = overlap_in%ie          (1:count_in)
   overlap_out%js          (count_out+1:count) = overlap_in%js          (1:count_in)
   overlap_out%je          (count_out+1:count) = overlap_in%je          (1:count_in)
-  overlap_out%isMe        (count_out+1:count) = overlap_in%isMe        (1:count_in)
-  overlap_out%ieMe        (count_out+1:count) = overlap_in%ieMe        (1:count_in)
-  overlap_out%jsMe        (count_out+1:count) = overlap_in%jsMe        (1:count_in)
-  overlap_out%jeMe        (count_out+1:count) = overlap_in%jeMe        (1:count_in)
   overlap_out%dir         (count_out+1:count) = overlap_in%dir         (1:count_in)
   overlap_out%rotation    (count_out+1:count) = overlap_in%rotation    (1:count_in)
-  overlap_out%is_refined  (count_out+1:count) = overlap_in%is_refined  (1:count_in)
-  overlap_out%index       (count_out+1:count) = overlap_in%index       (1:count_in)
   overlap_out%from_contact(count_out+1:count) = overlap_in%from_contact(1:count_in)
 
   do n = count_out+1, count
diff --git a/src/shared/mpp/include/mpp_domains_misc.inc b/src/shared/mpp/include/mpp_domains_misc.inc
index 77267e3131..5e3121919d 100644
--- a/src/shared/mpp/include/mpp_domains_misc.inc
+++ b/src/shared/mpp/include/mpp_domains_misc.inc
@@ -117,6 +117,16 @@
           nest_recv_clock     = mpp_clock_id( 'nest recv' )
           nest_unpk_clock     = mpp_clock_id( 'nest unpk' )
           nest_wait_clock     = mpp_clock_id( 'nest wait' )
+          group_pack_clock     = mpp_clock_id( 'group pack' )
+          group_send_clock     = mpp_clock_id( 'group send' )
+          group_recv_clock     = mpp_clock_id( 'group recv' )
+          group_unpk_clock     = mpp_clock_id( 'group unpk' )
+          group_wait_clock     = mpp_clock_id( 'group wait' )
+          nonblock_group_pack_clock     = mpp_clock_id( 'nonblock group pack' )
+          nonblock_group_send_clock     = mpp_clock_id( 'nonblock group send' )
+          nonblock_group_recv_clock     = mpp_clock_id( 'nonblock group recv' )
+          nonblock_group_unpk_clock     = mpp_clock_id( 'nonblock group unpk' )
+          nonblock_group_wait_clock     = mpp_clock_id( 'nonblock group wait' )
       end if
       return
     end subroutine mpp_domains_init
@@ -1731,3 +1741,75 @@ end subroutine init_nonblock_type
 #define MPP_DO_GET_BOUNDARY_3D_V_ mpp_do_get_boundary_r4_3dv
 #include 
 #endif
+
+#undef MPP_TYPE_
+#define MPP_TYPE_ real(DOUBLE_KIND)
+#undef MPI_TYPE_
+#define MPI_TYPE_ MPI_REAL8
+#undef MPP_CREATE_GROUP_UPDATE_2D_
+#define MPP_CREATE_GROUP_UPDATE_2D_ mpp_create_group_update_r8_2d
+#undef MPP_CREATE_GROUP_UPDATE_3D_
+#define MPP_CREATE_GROUP_UPDATE_3D_ mpp_create_group_update_r8_3d
+#undef MPP_CREATE_GROUP_UPDATE_4D_
+#define MPP_CREATE_GROUP_UPDATE_4D_ mpp_create_group_update_r8_4d
+#undef MPP_CREATE_GROUP_UPDATE_2D_V_
+#define MPP_CREATE_GROUP_UPDATE_2D_V_ mpp_create_group_update_r8_2dv
+#undef MPP_CREATE_GROUP_UPDATE_3D_V_
+#define MPP_CREATE_GROUP_UPDATE_3D_V_ mpp_create_group_update_r8_3dv
+#undef MPP_CREATE_GROUP_UPDATE_4D_V_
+#define MPP_CREATE_GROUP_UPDATE_4D_V_ mpp_create_group_update_r8_4dv
+#undef MPP_DO_GROUP_UPDATE_
+#define MPP_DO_GROUP_UPDATE_ mpp_do_group_update_r8
+#undef MPP_START_GROUP_UPDATE_
+#define MPP_START_GROUP_UPDATE_ mpp_start_group_update_r8
+#undef MPP_COMPLETE_GROUP_UPDATE_
+#define MPP_COMPLETE_GROUP_UPDATE_ mpp_complete_group_update_r8
+#undef MPP_RESET_GROUP_UPDATE_FIELD_2D_
+#define MPP_RESET_GROUP_UPDATE_FIELD_2D_ mpp_reset_group_update_field_r8_2d
+#undef MPP_RESET_GROUP_UPDATE_FIELD_3D_
+#define MPP_RESET_GROUP_UPDATE_FIELD_3D_ mpp_reset_group_update_field_r8_3d
+#undef MPP_RESET_GROUP_UPDATE_FIELD_4D_
+#define MPP_RESET_GROUP_UPDATE_FIELD_4D_ mpp_reset_group_update_field_r8_4d
+#undef MPP_RESET_GROUP_UPDATE_FIELD_2D_V_
+#define MPP_RESET_GROUP_UPDATE_FIELD_2D_V_ mpp_reset_group_update_field_r8_2dv
+#undef MPP_RESET_GROUP_UPDATE_FIELD_3D_V_
+#define MPP_RESET_GROUP_UPDATE_FIELD_3D_V_ mpp_reset_group_update_field_r8_3dv
+#undef MPP_RESET_GROUP_UPDATE_FIELD_4D_V_
+#define MPP_RESET_GROUP_UPDATE_FIELD_4D_V_ mpp_reset_group_update_field_r8_4dv
+#include 
+
+#undef MPP_TYPE_
+#define MPP_TYPE_ real(FLOAT_KIND)
+#undef MPI_TYPE_
+#define MPI_TYPE_ MPI_REAL4
+#undef MPP_CREATE_GROUP_UPDATE_2D_
+#define MPP_CREATE_GROUP_UPDATE_2D_ mpp_create_group_update_r4_2d
+#undef MPP_CREATE_GROUP_UPDATE_3D_
+#define MPP_CREATE_GROUP_UPDATE_3D_ mpp_create_group_update_r4_3d
+#undef MPP_CREATE_GROUP_UPDATE_4D_
+#define MPP_CREATE_GROUP_UPDATE_4D_ mpp_create_group_update_r4_4d
+#undef MPP_CREATE_GROUP_UPDATE_2D_V_
+#define MPP_CREATE_GROUP_UPDATE_2D_V_ mpp_create_group_update_r4_2dv
+#undef MPP_CREATE_GROUP_UPDATE_3D_V_
+#define MPP_CREATE_GROUP_UPDATE_3D_V_ mpp_create_group_update_r4_3dv
+#undef MPP_CREATE_GROUP_UPDATE_4D_V_
+#define MPP_CREATE_GROUP_UPDATE_4D_V_ mpp_create_group_update_r4_4dv
+#undef MPP_DO_GROUP_UPDATE_
+#define MPP_DO_GROUP_UPDATE_ mpp_do_group_update_r4
+#undef MPP_START_GROUP_UPDATE_
+#define MPP_START_GROUP_UPDATE_ mpp_start_group_update_r4
+#undef MPP_COMPLETE_GROUP_UPDATE_
+#define MPP_COMPLETE_GROUP_UPDATE_ mpp_complete_group_update_r4
+#undef MPP_RESET_GROUP_UPDATE_FIELD_2D_
+#define MPP_RESET_GROUP_UPDATE_FIELD_2D_ mpp_reset_group_update_field_r4_2d
+#undef MPP_RESET_GROUP_UPDATE_FIELD_3D_
+#define MPP_RESET_GROUP_UPDATE_FIELD_3D_ mpp_reset_group_update_field_r4_3d
+#undef MPP_RESET_GROUP_UPDATE_FIELD_4D_
+#define MPP_RESET_GROUP_UPDATE_FIELD_4D_ mpp_reset_group_update_field_r4_4d
+#undef MPP_RESET_GROUP_UPDATE_FIELD_2D_V_
+#define MPP_RESET_GROUP_UPDATE_FIELD_2D_V_ mpp_reset_group_update_field_r4_2dv
+#undef MPP_RESET_GROUP_UPDATE_FIELD_3D_V_
+#define MPP_RESET_GROUP_UPDATE_FIELD_3D_V_ mpp_reset_group_update_field_r4_3dv
+#undef MPP_RESET_GROUP_UPDATE_FIELD_4D_V_
+#define MPP_RESET_GROUP_UPDATE_FIELD_4D_V_ mpp_reset_group_update_field_r4_4dv
+#include 
diff --git a/src/shared/mpp/include/mpp_domains_reduce.inc b/src/shared/mpp/include/mpp_domains_reduce.inc
index f268075764..673a9b1a99 100644
--- a/src/shared/mpp/include/mpp_domains_reduce.inc
+++ b/src/shared/mpp/include/mpp_domains_reduce.inc
@@ -1,6 +1,6 @@
 
 ! -*-f90-*-
-! $Id: mpp_domains_reduce.inc,v 20.0 2013/12/14 00:24:46 fms Exp $
+! $Id$
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 !                                                                             !
@@ -161,7 +161,7 @@
 !                   MPP_GLOBAL_SUM: global sum of field                       !
 !                                                                             !
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-
+#define DO_EFP_SUM_
 #undef MPP_GLOBAL_SUM_
 #define MPP_GLOBAL_SUM_ mpp_global_sum_r8_2d
 #undef MPP_EXTRA_INDICES_
@@ -194,40 +194,6 @@
 #define MPP_TYPE_ real(DOUBLE_KIND)
 #include 
 
-#ifdef OVERLOAD_C8
-#undef MPP_GLOBAL_SUM_
-#define MPP_GLOBAL_SUM_ mpp_global_sum_c8_2d
-#undef MPP_EXTRA_INDICES_
-#define MPP_EXTRA_INDICES_
-#undef MPP_TYPE_
-#define MPP_TYPE_ complex(DOUBLE_KIND)
-#include 
-
-#undef MPP_GLOBAL_SUM_
-#define MPP_GLOBAL_SUM_ mpp_global_sum_c8_3d
-#undef MPP_EXTRA_INDICES_
-#define MPP_EXTRA_INDICES_ ,:
-#undef MPP_TYPE_
-#define MPP_TYPE_ complex(DOUBLE_KIND)
-#include 
-
-#undef MPP_GLOBAL_SUM_
-#define MPP_GLOBAL_SUM_ mpp_global_sum_c8_4d
-#undef MPP_EXTRA_INDICES_
-#define MPP_EXTRA_INDICES_ ,:,:
-#undef MPP_TYPE_
-#define MPP_TYPE_ complex(DOUBLE_KIND)
-#include 
-
-#undef MPP_GLOBAL_SUM_
-#define MPP_GLOBAL_SUM_ mpp_global_sum_c8_5d
-#undef MPP_EXTRA_INDICES_
-#define MPP_EXTRA_INDICES_ ,:,:,:
-#undef MPP_TYPE_
-#define MPP_TYPE_ complex(DOUBLE_KIND)
-#include 
-#endif
-
 #ifdef OVERLOAD_R4
 #undef MPP_GLOBAL_SUM_
 #define MPP_GLOBAL_SUM_ mpp_global_sum_r4_2d
@@ -262,40 +228,7 @@
 #include 
 #endif
 
-#ifdef OVERLOAD_C4
-#undef MPP_GLOBAL_SUM_
-#define MPP_GLOBAL_SUM_ mpp_global_sum_c4_2d
-#undef MPP_EXTRA_INDICES_
-#define MPP_EXTRA_INDICES_
-#undef MPP_TYPE_
-#define MPP_TYPE_ complex(FLOAT_KIND)
-#include 
-
-#undef MPP_GLOBAL_SUM_
-#define MPP_GLOBAL_SUM_ mpp_global_sum_c4_3d
-#undef MPP_EXTRA_INDICES_
-#define MPP_EXTRA_INDICES_ ,:
-#undef MPP_TYPE_
-#define MPP_TYPE_ complex(FLOAT_KIND)
-#include 
-
-#undef MPP_GLOBAL_SUM_
-#define MPP_GLOBAL_SUM_ mpp_global_sum_c4_4d
-#undef MPP_EXTRA_INDICES_
-#define MPP_EXTRA_INDICES_ ,:,:
-#undef MPP_TYPE_
-#define MPP_TYPE_ complex(FLOAT_KIND)
-#include 
-
-#undef MPP_GLOBAL_SUM_
-#define MPP_GLOBAL_SUM_ mpp_global_sum_c4_5d
-#undef MPP_EXTRA_INDICES_
-#define MPP_EXTRA_INDICES_ ,:,:,:
-#undef MPP_TYPE_
-#define MPP_TYPE_ complex(FLOAT_KIND)
-#include 
-#endif
-
+#undef DO_EFP_SUM_
 #ifndef no_8byte_integers
 #undef MPP_GLOBAL_SUM_
 #define MPP_GLOBAL_SUM_ mpp_global_sum_i8_2d
@@ -361,7 +294,8 @@
 #undef MPP_TYPE_
 #define MPP_TYPE_ integer(INT_KIND)
 #include 
-    
+
+
 !gag
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 !                                                                             !
@@ -401,40 +335,6 @@
 #define MPP_TYPE_ real(DOUBLE_KIND)
 #include 
 
-#ifdef OVERLOAD_C8
-#undef MPP_GLOBAL_SUM_TL_
-#define MPP_GLOBAL_SUM_TL_ mpp_global_sum_tl_c8_2d
-#undef MPP_EXTRA_INDICES_
-#define MPP_EXTRA_INDICES_
-#undef MPP_TYPE_
-#define MPP_TYPE_ complex(DOUBLE_KIND)
-#include 
-
-#undef MPP_GLOBAL_SUM_TL_
-#define MPP_GLOBAL_SUM_TL_ mpp_global_sum_tl_c8_3d
-#undef MPP_EXTRA_INDICES_
-#define MPP_EXTRA_INDICES_ ,:
-#undef MPP_TYPE_
-#define MPP_TYPE_ complex(DOUBLE_KIND)
-#include 
-
-#undef MPP_GLOBAL_SUM_TL_
-#define MPP_GLOBAL_SUM_TL_ mpp_global_sum_tl_c8_4d
-#undef MPP_EXTRA_INDICES_
-#define MPP_EXTRA_INDICES_ ,:,:
-#undef MPP_TYPE_
-#define MPP_TYPE_ complex(DOUBLE_KIND)
-#include 
-
-#undef MPP_GLOBAL_SUM_TL_
-#define MPP_GLOBAL_SUM_TL_ mpp_global_sum_tl_c8_5d
-#undef MPP_EXTRA_INDICES_
-#define MPP_EXTRA_INDICES_ ,:,:,:
-#undef MPP_TYPE_
-#define MPP_TYPE_ complex(DOUBLE_KIND)
-#include 
-#endif
-
 #ifdef OVERLOAD_R4
 #undef MPP_GLOBAL_SUM_TL_
 #define MPP_GLOBAL_SUM_TL_ mpp_global_sum_tl_r4_2d
@@ -469,315 +369,6 @@
 #include 
 #endif
 
-#ifdef OVERLOAD_C4
-#undef MPP_GLOBAL_SUM_TL_
-#define MPP_GLOBAL_SUM_TL_ mpp_global_sum_tl_c4_2d
-#undef MPP_EXTRA_INDICES_
-#define MPP_EXTRA_INDICES_
-#undef MPP_TYPE_
-#define MPP_TYPE_ complex(FLOAT_KIND)
-#include 
-
-#undef MPP_GLOBAL_SUM_TL_
-#define MPP_GLOBAL_SUM_TL_ mpp_global_sum_tl_c4_3d
-#undef MPP_EXTRA_INDICES_
-#define MPP_EXTRA_INDICES_ ,:
-#undef MPP_TYPE_
-#define MPP_TYPE_ complex(FLOAT_KIND)
-#include 
-
-#undef MPP_GLOBAL_SUM_TL_
-#define MPP_GLOBAL_SUM_TL_ mpp_global_sum_tl_c4_4d
-#undef MPP_EXTRA_INDICES_
-#define MPP_EXTRA_INDICES_ ,:,:
-#undef MPP_TYPE_
-#define MPP_TYPE_ complex(FLOAT_KIND)
-#include 
-
-#undef MPP_GLOBAL_SUM_TL_
-#define MPP_GLOBAL_SUM_TL_ mpp_global_sum_tl_c4_5d
-#undef MPP_EXTRA_INDICES_
-#define MPP_EXTRA_INDICES_ ,:,:,:
-#undef MPP_TYPE_
-#define MPP_TYPE_ complex(FLOAT_KIND)
-#include 
-#endif
-
-#ifndef no_8byte_integers
-#undef MPP_GLOBAL_SUM_TL_
-#define MPP_GLOBAL_SUM_TL_ mpp_global_sum_tl_i8_2d
-#undef MPP_EXTRA_INDICES_
-#define MPP_EXTRA_INDICES_
-#undef MPP_TYPE_
-#define MPP_TYPE_ integer(LONG_KIND)
-#include 
-
-#undef MPP_GLOBAL_SUM_TL_
-#define MPP_GLOBAL_SUM_TL_ mpp_global_sum_tl_i8_3d
-#undef MPP_EXTRA_INDICES_
-#define MPP_EXTRA_INDICES_ ,:
-#undef MPP_TYPE_
-#define MPP_TYPE_ integer(LONG_KIND)
-#include 
-
-#undef MPP_GLOBAL_SUM_TL_
-#define MPP_GLOBAL_SUM_TL_ mpp_global_sum_tl_i8_4d
-#undef MPP_EXTRA_INDICES_
-#define MPP_EXTRA_INDICES_ ,:,:
-#undef MPP_TYPE_
-#define MPP_TYPE_ integer(LONG_KIND)
-#include 
-
-#undef MPP_GLOBAL_SUM_TL_
-#define MPP_GLOBAL_SUM_TL_ mpp_global_sum_tl_i8_5d
-#undef MPP_EXTRA_INDICES_
-#define MPP_EXTRA_INDICES_ ,:,:,:
-#undef MPP_TYPE_
-#define MPP_TYPE_ integer(LONG_KIND)
-#include 
-#endif
-
-#undef MPP_GLOBAL_SUM_TL_
-#define MPP_GLOBAL_SUM_TL_ mpp_global_sum_tl_i4_2d
-#undef MPP_EXTRA_INDICES_
-#define MPP_EXTRA_INDICES_
-#undef MPP_TYPE_
-#define MPP_TYPE_ integer(INT_KIND)
-#include 
-
-#undef MPP_GLOBAL_SUM_TL_
-#define MPP_GLOBAL_SUM_TL_ mpp_global_sum_tl_i4_3d
-#undef MPP_EXTRA_INDICES_
-#define MPP_EXTRA_INDICES_ ,:
-#undef MPP_TYPE_
-#define MPP_TYPE_ integer(INT_KIND)
-#include 
-
-#undef MPP_GLOBAL_SUM_TL_
-#define MPP_GLOBAL_SUM_TL_ mpp_global_sum_tl_i4_4d
-#undef MPP_EXTRA_INDICES_
-#define MPP_EXTRA_INDICES_ ,:,:
-#undef MPP_TYPE_
-#define MPP_TYPE_ integer(INT_KIND)
-#include 
-
-#undef MPP_GLOBAL_SUM_TL_
-#define MPP_GLOBAL_SUM_TL_ mpp_global_sum_tl_i4_5d
-#undef MPP_EXTRA_INDICES_
-#define MPP_EXTRA_INDICES_ ,:,:,:
-#undef MPP_TYPE_
-#define MPP_TYPE_ integer(INT_KIND)
-#include 
-!gag
-
-!bnc
-!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-!                                                                             !
-!         MPP_GLOBAL_SUM_AD: global adjoint sum of field                      !
-!                                                                             !
-!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-
-!!$#undef MPP_GLOBAL_SUM_AD_
-!!$#define MPP_GLOBAL_SUM_AD_ mpp_global_sum_ad_r8_2d
-!!$#undef MPP_EXTRA_INDICES_
-!!$#define MPP_EXTRA_INDICES_
-!!$#undef MPP_TYPE_
-!!$#define MPP_TYPE_ real(DOUBLE_KIND)
-!!$#include 
-!!$
-!!$#undef MPP_GLOBAL_SUM_AD_
-!!$#define MPP_GLOBAL_SUM_AD_ mpp_global_sum_ad_r8_3d
-!!$#undef MPP_EXTRA_INDICES_
-!!$#define MPP_EXTRA_INDICES_ ,:
-!!$#undef MPP_TYPE_
-!!$#define MPP_TYPE_ real(DOUBLE_KIND)
-!!$#include 
-!!$
-!!$#undef MPP_GLOBAL_SUM_AD_
-!!$#define MPP_GLOBAL_SUM_AD_ mpp_global_sum_ad_r8_4d
-!!$#undef MPP_EXTRA_INDICES_
-!!$#define MPP_EXTRA_INDICES_ ,:,:
-!!$#undef MPP_TYPE_
-!!$#define MPP_TYPE_ real(DOUBLE_KIND)
-!!$#include 
-!!$
-!!$#undef MPP_GLOBAL_SUM_AD_
-!!$#define MPP_GLOBAL_SUM_AD_ mpp_global_sum_ad_r8_5d
-!!$#undef MPP_EXTRA_INDICES_
-!!$#define MPP_EXTRA_INDICES_ ,:,:,:
-!!$#undef MPP_TYPE_
-!!$#define MPP_TYPE_ real(DOUBLE_KIND)
-!!$#include 
-!!$
-!!$#ifdef OVERLOAD_C8
-!!$#undef MPP_GLOBAL_SUM_AD_
-!!$#define MPP_GLOBAL_SUM_AD_ mpp_global_sum_ad_c8_2d
-!!$#undef MPP_EXTRA_INDICES_
-!!$#define MPP_EXTRA_INDICES_
-!!$#undef MPP_TYPE_
-!!$#define MPP_TYPE_ complex(DOUBLE_KIND)
-!!$#include 
-!!$
-!!$#undef MPP_GLOBAL_SUM_AD_
-!!$#define MPP_GLOBAL_SUM_AD_ mpp_global_sum_ad_c8_3d
-!!$#undef MPP_EXTRA_INDICES_
-!!$#define MPP_EXTRA_INDICES_ ,:
-!!$#undef MPP_TYPE_
-!!$#define MPP_TYPE_ complex(DOUBLE_KIND)
-!!$#include 
-!!$
-!!$#undef MPP_GLOBAL_SUM_AD_
-!!$#define MPP_GLOBAL_SUM_AD_ mpp_global_sum_ad_c8_4d
-!!$#undef MPP_EXTRA_INDICES_
-!!$#define MPP_EXTRA_INDICES_ ,:,:
-!!$#undef MPP_TYPE_
-!!$#define MPP_TYPE_ complex(DOUBLE_KIND)
-!!$#include 
-!!$
-!!$#undef MPP_GLOBAL_SUM_AD_
-!!$#define MPP_GLOBAL_SUM_AD_ mpp_global_sum_ad_c8_5d
-!!$#undef MPP_EXTRA_INDICES_
-!!$#define MPP_EXTRA_INDICES_ ,:,:,:
-!!$#undef MPP_TYPE_
-!!$#define MPP_TYPE_ complex(DOUBLE_KIND)
-!!$#include 
-!!$#endif
-!!$
-!!$#ifdef OVERLOAD_R4
-!!$#undef MPP_GLOBAL_SUM_AD_
-!!$#define MPP_GLOBAL_SUM_AD_ mpp_global_sum_ad_r4_2d
-!!$#undef MPP_EXTRA_INDICES_
-!!$#define MPP_EXTRA_INDICES_
-!!$#undef MPP_TYPE_
-!!$#define MPP_TYPE_ real(FLOAT_KIND)
-!!$#include 
-!!$
-!!$#undef MPP_GLOBAL_SUM_AD_
-!!$#define MPP_GLOBAL_SUM_AD_ mpp_global_sum_ad_r4_3d
-!!$#undef MPP_EXTRA_INDICES_
-!!$#define MPP_EXTRA_INDICES_ ,:
-!!$#undef MPP_TYPE_
-!!$#define MPP_TYPE_ real(FLOAT_KIND)
-!!$#include 
-!!$
-!!$#undef MPP_GLOBAL_SUM_AD_
-!!$#define MPP_GLOBAL_SUM_AD_ mpp_global_sum_ad_r4_4d
-!!$#undef MPP_EXTRA_INDICES_
-!!$#define MPP_EXTRA_INDICES_ ,:,:
-!!$#undef MPP_TYPE_
-!!$#define MPP_TYPE_ real(FLOAT_KIND)
-!!$#include 
-!!$
-!!$#undef MPP_GLOBAL_SUM_AD_
-!!$#define MPP_GLOBAL_SUM_AD_ mpp_global_sum_ad_r4_5d
-!!$#undef MPP_EXTRA_INDICES_
-!!$#define MPP_EXTRA_INDICES_ ,:,:,:
-!!$#undef MPP_TYPE_
-!!$#define MPP_TYPE_ real(FLOAT_KIND)
-!!$#include 
-!!$#endif
-!!$
-!!$#ifdef OVERLOAD_C4
-!!$#undef MPP_GLOBAL_SUM_AD_
-!!$#define MPP_GLOBAL_SUM_AD_ mpp_global_sum_ad_c4_2d
-!!$#undef MPP_EXTRA_INDICES_
-!!$#define MPP_EXTRA_INDICES_
-!!$#undef MPP_TYPE_
-!!$#define MPP_TYPE_ complex(FLOAT_KIND)
-!!$#include 
-!!$
-!!$#undef MPP_GLOBAL_SUM_AD_
-!!$#define MPP_GLOBAL_SUM_AD_ mpp_global_sum_ad_c4_3d
-!!$#undef MPP_EXTRA_INDICES_
-!!$#define MPP_EXTRA_INDICES_ ,:
-!!$#undef MPP_TYPE_
-!!$#define MPP_TYPE_ complex(FLOAT_KIND)
-!!$#include 
-!!$
-!!$#undef MPP_GLOBAL_SUM_AD_
-!!$#define MPP_GLOBAL_SUM_AD_ mpp_global_sum_ad_c4_4d
-!!$#undef MPP_EXTRA_INDICES_
-!!$#define MPP_EXTRA_INDICES_ ,:,:
-!!$#undef MPP_TYPE_
-!!$#define MPP_TYPE_ complex(FLOAT_KIND)
-!!$#include 
-!!$
-!!$#undef MPP_GLOBAL_SUM_AD_
-!!$#define MPP_GLOBAL_SUM_AD_ mpp_global_sum_ad_c4_5d
-!!$#undef MPP_EXTRA_INDICES_
-!!$#define MPP_EXTRA_INDICES_ ,:,:,:
-!!$#undef MPP_TYPE_
-!!$#define MPP_TYPE_ complex(FLOAT_KIND)
-!!$#include 
-!!$#endif
-!!$
-!!$#ifndef no_8byte_integers
-!!$#undef MPP_GLOBAL_SUM_AD_
-!!$#define MPP_GLOBAL_SUM_AD_ mpp_global_sum_ad_i8_2d
-!!$#undef MPP_EXTRA_INDICES_
-!!$#define MPP_EXTRA_INDICES_
-!!$#undef MPP_TYPE_
-!!$#define MPP_TYPE_ integer(LONG_KIND)
-!!$#include 
-!!$
-!!$#undef MPP_GLOBAL_SUM_AD_
-!!$#define MPP_GLOBAL_SUM_AD_ mpp_global_sum_ad_i8_3d
-!!$#undef MPP_EXTRA_INDICES_
-!!$#define MPP_EXTRA_INDICES_ ,:
-!!$#undef MPP_TYPE_
-!!$#define MPP_TYPE_ integer(LONG_KIND)
-!!$#include 
-!!$
-!!$#undef MPP_GLOBAL_SUM_AD_
-!!$#define MPP_GLOBAL_SUM_AD_ mpp_global_sum_ad_i8_4d
-!!$#undef MPP_EXTRA_INDICES_
-!!$#define MPP_EXTRA_INDICES_ ,:,:
-!!$#undef MPP_TYPE_
-!!$#define MPP_TYPE_ integer(LONG_KIND)
-!!$#include 
-!!$
-!!$#undef MPP_GLOBAL_SUM_AD_
-!!$#define MPP_GLOBAL_SUM_AD_ mpp_global_sum_ad_i8_5d
-!!$#undef MPP_EXTRA_INDICES_
-!!$#define MPP_EXTRA_INDICES_ ,:,:,:
-!!$#undef MPP_TYPE_
-!!$#define MPP_TYPE_ integer(LONG_KIND)
-!!$#include 
-!!$#endif
-!!$
-!!$#undef MPP_GLOBAL_SUM_AD_
-!!$#define MPP_GLOBAL_SUM_AD_ mpp_global_sum_ad_i4_2d
-!!$#undef MPP_EXTRA_INDICES_
-!!$#define MPP_EXTRA_INDICES_
-!!$#undef MPP_TYPE_
-!!$#define MPP_TYPE_ integer(INT_KIND)
-!!$#include 
-!!$
-!!$#undef MPP_GLOBAL_SUM_AD_
-!!$#define MPP_GLOBAL_SUM_AD_ mpp_global_sum_ad_i4_3d
-!!$#undef MPP_EXTRA_INDICES_
-!!$#define MPP_EXTRA_INDICES_ ,:
-!!$#undef MPP_TYPE_
-!!$#define MPP_TYPE_ integer(INT_KIND)
-!!$#include 
-!!$
-!!$#undef MPP_GLOBAL_SUM_AD_
-!!$#define MPP_GLOBAL_SUM_AD_ mpp_global_sum_ad_i4_4d
-!!$#undef MPP_EXTRA_INDICES_
-!!$#define MPP_EXTRA_INDICES_ ,:,:
-!!$#undef MPP_TYPE_
-!!$#define MPP_TYPE_ integer(INT_KIND)
-!!$#include 
-!!$
-!!$#undef MPP_GLOBAL_SUM_AD_
-!!$#define MPP_GLOBAL_SUM_AD_ mpp_global_sum_ad_i4_5d
-!!$#undef MPP_EXTRA_INDICES_
-!!$#define MPP_EXTRA_INDICES_ ,:,:,:
-!!$#undef MPP_TYPE_
-!!$#define MPP_TYPE_ integer(INT_KIND)
-!!$#include 
-!!$!bnc
-
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 !                                                                             !
 !              MPP_GLOBAL_FIELD: get global field from domain field           !
diff --git a/src/shared/mpp/include/mpp_domains_util.inc b/src/shared/mpp/include/mpp_domains_util.inc
index 4588ea3baf..30f3daad01 100644
--- a/src/shared/mpp/include/mpp_domains_util.inc
+++ b/src/shared/mpp/include/mpp_domains_util.inc
@@ -1,6 +1,6 @@
 
 ! -*-f90-*-
-! $Id: mpp_domains_util.inc,v 20.0 2013/12/14 00:24:48 fms Exp $
+! $Id$
 
   ! 
   !  
@@ -1019,76 +1019,6 @@ end subroutine mpp_get_domain_shift
 
   end function mpp_domain_is_tile_root_pe
 
-  !#######################################################################
-  !--- the order of overlapping will be consistent with the unpack done in update_domains.
-  function mpp_get_refine_overlap_number(domain, position, whalo, ehalo, shalo, nhalo, tile_count)
-     type(domain2d), intent(inout)        :: domain
-     integer,        intent(in), optional :: position                   ! position of the cell, CENTER/EAST/NORTH/CORNER
-     integer,        intent(in), optional :: whalo, ehalo, shalo, nhalo ! halo size to be updated
-     integer,        intent(in), optional :: tile_count                ! tile number on current pe.
-     integer                              :: mpp_get_refine_overlap_number
-     type(overlapSpec), pointer           :: update => NULL()
-     integer                              :: tMe, whalosz, ehalosz, shalosz, nhalosz, pos
-
-     tMe = 1
-     if(present(tile_count)) tMe = tile_count
-     pos = CENTER;           if(present(position)) pos = position
-     whalosz = domain%whalo; if(present(whalo)) whalosz = whalo
-     ehalosz = domain%ehalo; if(present(ehalo)) ehalosz = ehalo
-     shalosz = domain%shalo; if(present(shalo)) shalosz = shalo
-     nhalosz = domain%nhalo; if(present(nhalo)) nhalosz = nhalo
-     update => search_update_overlap( domain, whalosz, ehalosz, shalosz, nhalosz, pos )
-     mpp_get_refine_overlap_number = update%rSpec(tMe)%count
-     update => NULL()
-
-     return    
- 
-  end function mpp_get_refine_overlap_number
-
-  !#######################################################################
-  ! return overlap information for refined contact.
-  subroutine mpp_get_mosaic_refine_overlap(domain, isMe, ieMe, jsMe, jeMe, isNbr, ieNbr, jsNbr, jeNbr, &
-                                           dir, rotation, position, whalo, ehalo, shalo, nhalo, tile_count)
-     type(domain2d), intent(inout)        :: domain
-     integer, dimension(:), intent(inout) :: isMe,  ieMe,  jsMe,  jeMe  ! index on current tile
-     integer, dimension(:), intent(inout) :: isNbr, ieNbr, jsNbr, jeNbr ! index on neighbor tile
-     integer, dimension(:), intent(inout) :: dir                        ! direction
-     integer, dimension(:), intent(inout) :: rotation                   ! rotation angle
-     integer, optional,     intent(in)    :: position                   ! position of the cell, CENTER/EAST/NORTH/CORNER
-     integer, optional,     intent(in)    :: whalo, ehalo, shalo, nhalo ! halo size to be updated
-     integer, optional,     intent(in)    :: tile_count                ! tile number on current pe.
-     type(overlapSpec), pointer           :: update => NULL()
-     integer                              :: tMe, count, whalosz, ehalosz, shalosz, nhalosz, pos
-
-     tMe = 1
-     if(present(tile_count)) tMe = tile_count
-     pos = CENTER;           if(present(position)) pos = position
-     whalosz = domain%whalo; if(present(whalo)) whalosz = whalo
-     ehalosz = domain%ehalo; if(present(ehalo)) ehalosz = ehalo
-     shalosz = domain%shalo; if(present(shalo)) shalosz = shalo
-     nhalosz = domain%nhalo; if(present(nhalo)) nhalosz = nhalo
-
-     update => search_update_overlap( domain, whalosz, ehalosz, shalosz, nhalosz, pos )
-     count = update%rSpec(tMe)%count
-     if(count>0) then
-        if(size(isMe(:)) < count) call mpp_error(FATAL, &
-           "mpp_domains_util.inc(mpp_get_mosaic_refine_overlap): size of isMe is less than the number of overlap")
-        isMe(1:count)      = update%rSpec(tMe)%isMe
-        ieMe(1:count)      = update%rSpec(tMe)%ieMe
-        jsMe(1:count)      = update%rSpec(tMe)%jsMe
-        jeMe(1:count)      = update%rSpec(tMe)%jeMe
-        isNbr(1:count)     = update%rSpec(tMe)%isNbr
-        ieNbr(1:count)     = update%rSpec(tMe)%ieNbr
-        jsNbr(1:count)     = update%rSpec(tMe)%jsNbr
-        jeNbr(1:count)     = update%rSpec(tMe)%jeNbr
-        dir  (1:count)     = update%rSpec(tMe)%dir
-        rotation(1:count)  = update%rSpec(tMe)%rotation
-     endif
-
-     return
-
-  end subroutine mpp_get_mosaic_refine_overlap
-
   !#########################################################################
   ! return number of processors used on current tile.
   function mpp_get_tile_npes(domain)
@@ -1613,6 +1543,22 @@ end subroutine mpp_get_tile_compute_domains
 
   end function get_rank_unpack
 
+  function get_mesgsize(overlap, do_dir)
+   type(overlap_type), intent(in) :: overlap
+   logical,            intent(in) :: do_dir(:)
+   integer                       :: get_mesgsize
+   integer                       :: n, dir
+
+   get_mesgsize = 0
+   do n = 1, overlap%count
+      dir = overlap%dir(n)
+      if(do_dir(dir)) then
+         get_mesgsize = get_mesgsize + overlap%msgsize(n)
+      end if
+   end do
+
+  end function get_mesgsize
+
   !#############################################################################
   subroutine mpp_set_domain_symmetry(domain, symmetry)
     type(domain2D), intent(inout) :: domain
@@ -1675,3 +1621,480 @@ end subroutine mpp_get_tile_compute_domains
      return
 
   end subroutine mpp_copy_domain2D
+
+  !######################################################################
+  subroutine set_group_update(group, domain)
+     type(mpp_group_update_type), intent(inout) :: group  
+     type(domain2D),              intent(inout) :: domain
+     integer   :: nscalar, nvector, nlist
+     integer   :: nsend, nrecv, nsend_old, nrecv_old
+     integer   :: nsend_s, nsend_x, nsend_y
+     integer   :: nrecv_s, nrecv_x, nrecv_y
+     integer   :: update_buffer_pos, tot_recv_size, tot_send_size
+     integer   :: msgsize_s, msgsize_x, msgsize_y, msgsize
+     logical   :: recv_s(8), send_s(8), recv_v(8), send_v(8)
+     integer   :: ntot, n, l, m, ksize
+     integer   :: i_s, i_x, i_y, rank_s, rank_x, rank_y, rank
+     integer   :: ind_s(3*MAXOVERLAP)
+     integer   :: ind_x(3*MAXOVERLAP)
+     integer   :: ind_y(3*MAXOVERLAP)
+     integer   :: pelist(3*MAXOVERLAP), to_pe_list(3*MAXOVERLAP)
+     integer   :: buffer_pos_recv(3*MAXOVERLAP), buffer_pos_send(3*MAXOVERLAP)
+     integer   :: recv_size(3*MAXOVERLAP), send_size(3*MAXOVERLAP)
+     integer   :: position_x, position_y, npack, nunpack, dir
+     integer   :: pack_buffer_pos, unpack_buffer_pos
+     integer   :: omp_get_num_threads, nthreads
+     character(len=8)            :: text
+     type(overlap_type), pointer :: overPtr => NULL()
+     type(overlapSpec), pointer  :: update_s => NULL()
+     type(overlapSpec), pointer  :: update_x => NULL()
+     type(overlapSpec), pointer  :: update_y => NULL()
+
+     nscalar = group%nscalar
+     nvector = group%nvector
+
+     !--- get the overlap data type
+     select case(group%gridtype)
+     case (AGRID)
+        position_x = CENTER
+        position_y = CENTER
+     case (BGRID_NE, BGRID_SW)
+        position_x = CORNER
+        position_y = CORNER
+     case (CGRID_NE, CGRID_SW)
+        position_x = EAST
+        position_y = NORTH
+     case (DGRID_NE, DGRID_SW)
+        position_x = NORTH
+        position_y = EAST
+     case default
+        call mpp_error(FATAL, "set_group_update: invalid value of gridtype")
+     end select
+     if(nscalar>0) then
+        update_s => search_update_overlap(domain, group%whalo_s, group%ehalo_s, &
+             group%shalo_s, group%nhalo_s, group%position)
+     endif
+     if(nvector>0) then
+        update_x => search_update_overlap(domain, group%whalo_v, group%ehalo_v, &
+             group%shalo_v, group%nhalo_v, position_x)
+        update_y => search_update_overlap(domain, group%whalo_v, group%ehalo_v, &
+             group%shalo_v, group%nhalo_v, position_y)
+     endif
+
+     if(nscalar > 0) then
+        recv_s = group%recv_s
+        send_s = recv_s
+     endif
+     if(nvector > 0) then
+        recv_v = group%recv_v
+        send_v = recv_v
+     end if
+     nlist   = size(domain%list(:))
+     group%initialized = .true.
+     nsend_s = 0; nsend_x = 0; nsend_y = 0
+     nrecv_s = 0; nrecv_x = 0; nrecv_y = 0
+
+     if(nscalar > 0) then
+         !--- This check could not be done because of memory domain
+!        if( group%isize_s .NE. (group%ie_s-group%is_s+1) .OR. group%jsize_s .NE. (group%je_s-group%js_s+1))  &
+!           call mpp_error(FATAL, "set_group_update: mismatch of size of the field and domain memory domain")
+        nsend_s = update_s%nsend
+        nrecv_s = update_s%nrecv
+     endif
+
+     !--- ksize_s must equal ksize_v
+     if(nvector > 0 .AND. nscalar > 0) then
+        if(group%ksize_s .NE. group%ksize_v) then
+           call mpp_error(FATAL, "set_group_update: ksize_s and ksize_v are not equal")
+        endif
+        ksize = group%ksize_s
+     else if (nscalar > 0) then
+        ksize = group%ksize_s
+     else if (nvector > 0) then
+        ksize = group%ksize_v
+     else
+        call mpp_error(FATAL, "set_group_update: nscalar and nvector are all 0")
+     endif
+
+    nthreads = 1
+!$OMP PARALLEL
+!$   nthreads = omp_get_num_threads()
+!$OMP END PARALLEL
+     if( nthreads > nthread_control_loop ) then
+        group%k_loop_inside = .FALSE.
+     else
+        group%k_loop_inside = .TRUE.
+     endif
+
+     if(nvector > 0) then
+        !--- This check could not be done because of memory domain
+!        if( group%isize_x .NE. (group%ie_x-group%is_x+1) .OR. group%jsize_x .NE. (group%je_x-group%js_x+1))  &
+!           call mpp_error(FATAL, "set_group_update: mismatch of size of the fieldx and domain memory domain")
+!        if( group%isize_y .NE. (group%ie_y-group%is_y+1) .OR. group%jsize_y .NE. (group%je_y-group%js_y+1))  &
+!           call mpp_error(FATAL, "set_group_update: mismatch of size of the fieldy and domain memory domain")
+        nsend_x = update_x%nsend
+        nrecv_x = update_x%nrecv
+        nsend_y = update_y%nsend
+        nrecv_y = update_y%nrecv
+     endif
+
+     !figure out message size for each processor.
+     ntot = nrecv_s + nrecv_x + nrecv_y
+     if(ntot > 3*MAXOVERLAP) call mpp_error(FATAL, "set_group_update: ntot is greater than 3*MAXOVERLAP")
+     n = 1
+     i_s = 1
+     i_x = 1
+     i_y = 1
+     ind_s = -1
+     ind_x = -1
+     ind_y = -1
+     nrecv = 0
+     do while(n<=ntot)
+        if( i_s <= nrecv_s ) then
+           rank_s = update_s%recv(i_s)%pe-domain%pe
+           if(rank_s .LE. 0) rank_s = rank_s + nlist
+        else
+           rank_s = -1
+        endif
+        if( i_x <= nrecv_x ) then
+           rank_x = update_x%recv(i_x)%pe-domain%pe
+           if(rank_x .LE. 0) rank_x = rank_x + nlist
+        else
+           rank_x = -1
+        endif
+        if( i_y <= nrecv_y ) then
+           rank_y = update_y%recv(i_y)%pe-domain%pe
+           if(rank_y .LE. 0) rank_y = rank_y + nlist
+        else
+           rank_y = -1
+        endif
+        nrecv = nrecv + 1   
+        rank = maxval((/rank_s, rank_x, rank_y/))
+        if(rank == rank_s) then
+           n = n + 1
+           ind_s(nrecv) = i_s
+           pelist(nrecv) = update_s%recv(i_s)%pe
+           i_s = i_s + 1
+        endif
+        if(rank == rank_x) then
+           n = n + 1
+           ind_x(nrecv) = i_x
+           pelist(nrecv) = update_x%recv(i_x)%pe
+           i_x = i_x + 1
+        endif
+        if(rank == rank_y) then
+           n = n + 1
+           ind_y(nrecv) = i_y
+           pelist(nrecv) = update_y%recv(i_y)%pe
+           i_y = i_y + 1
+        endif
+     enddo
+
+     nrecv_old = nrecv
+     nrecv     = 0
+     update_buffer_pos = 0
+     tot_recv_size = 0
+
+     !--- setup for recv
+     do l = 1, nrecv_old
+        msgsize_s = 0
+        msgsize_x = 0
+        msgsize_y = 0
+        m = ind_s(l)
+        if(m>0) msgsize_s = get_mesgsize(update_s%recv(m), recv_s)*ksize*nscalar
+        m = ind_x(l)
+        if(m>0) msgsize_x = get_mesgsize(update_x%recv(m), recv_v)*ksize*nvector
+        m = ind_y(l)
+        if(m>0) msgsize_y = get_mesgsize(update_y%recv(m), recv_v)*ksize*nvector
+        msgsize = msgsize_s + msgsize_x + msgsize_y
+        if( msgsize.GT.0 )then
+           tot_recv_size = tot_recv_size + msgsize
+           nrecv = nrecv + 1
+           if(nrecv > MAXOVERLAP) then
+              call mpp_error(FATAL, "set_group_update: nrecv is greater than MAXOVERLAP, increase MAXOVERLAP")
+           endif
+           group%from_pe(nrecv) = pelist(l)
+           group%recv_size(nrecv) = msgsize
+           group%buffer_pos_recv(nrecv) = update_buffer_pos
+           update_buffer_pos = update_buffer_pos + msgsize
+        end if
+     end do
+     group%nrecv = nrecv
+
+     !--- setup for unpack
+     nunpack = 0
+     unpack_buffer_pos = 0
+     do l = 1, nrecv_old
+        m = ind_s(l)
+        if(m>0) then
+           overptr => update_s%recv(m)
+           do n = 1, overptr%count
+              dir = overptr%dir(n)
+              if(recv_s(dir)) then
+                 nunpack = nunpack + 1
+                 if(nunpack > MAXOVERLAP) call mpp_error(FATAL,  &
+                   "set_group_update: nunpack is greater than MAXOVERLAP, increase MAXOVERLAP 1")
+                 group%unpack_type(nunpack) = FIELD_S
+                 group%unpack_buffer_pos(nunpack) = unpack_buffer_pos
+                 group%unpack_rotation(nunpack) = overptr%rotation(n) 
+                 group%unpack_is(nunpack) = overptr%is(n)
+                 group%unpack_ie(nunpack) = overptr%ie(n)
+                 group%unpack_js(nunpack) = overptr%js(n)
+                 group%unpack_je(nunpack) = overptr%je(n) 
+                 group%unpack_size(nunpack) = overptr%msgsize(n)*nscalar
+                 unpack_buffer_pos = unpack_buffer_pos + group%unpack_size(nunpack)*ksize
+              end if
+           end do   
+        end if
+
+        m = ind_x(l)
+        if(m>0) then
+           overptr => update_x%recv(m)
+           do n = 1, overptr%count
+              dir = overptr%dir(n)
+              if(recv_v(dir)) then
+                 nunpack = nunpack + 1
+                 if(nunpack > MAXOVERLAP) call mpp_error(FATAL,  &
+                   "set_group_update: nunpack is greater than MAXOVERLAP, increase MAXOVERLAP 2")
+                 group%unpack_type(nunpack) = FIELD_X
+                 group%unpack_buffer_pos(nunpack) = unpack_buffer_pos
+                 group%unpack_rotation(nunpack) = overptr%rotation(n) 
+                 group%unpack_is(nunpack) = overptr%is(n)
+                 group%unpack_ie(nunpack) = overptr%ie(n)
+                 group%unpack_js(nunpack) = overptr%js(n)
+                 group%unpack_je(nunpack) = overptr%je(n) 
+                 group%unpack_size(nunpack) = overptr%msgsize(n)*nvector
+                 unpack_buffer_pos = unpack_buffer_pos + group%unpack_size(nunpack)*ksize
+              end if
+           end do   
+        end if
+
+        m = ind_y(l)
+        if(m>0) then
+           overptr => update_y%recv(m)
+           do n = 1, overptr%count
+              dir = overptr%dir(n)
+              if(recv_v(dir)) then
+                 nunpack = nunpack + 1
+                 if(nunpack > MAXOVERLAP) call mpp_error(FATAL,  &
+                   "set_group_update: nunpack is greater than MAXOVERLAP, increase MAXOVERLAP 3")
+                 group%unpack_type(nunpack) = FIELD_Y
+                 group%unpack_buffer_pos(nunpack) = unpack_buffer_pos
+                 group%unpack_rotation(nunpack) = overptr%rotation(n) 
+                 group%unpack_is(nunpack) = overptr%is(n)
+                 group%unpack_ie(nunpack) = overptr%ie(n)
+                 group%unpack_js(nunpack) = overptr%js(n)
+                 group%unpack_je(nunpack) = overptr%je(n) 
+                 group%unpack_size(nunpack) = overptr%msgsize(n)*nvector
+                 unpack_buffer_pos = unpack_buffer_pos + group%unpack_size(nunpack)*ksize
+              end if
+           end do   
+        end if
+     end do
+     group%nunpack = nunpack
+
+     if(update_buffer_pos .NE. unpack_buffer_pos ) call mpp_error(FATAL,  &
+                   "set_group_update: update_buffer_pos .NE. unpack_buffer_pos")
+
+     !figure out message size for each processor.
+     ntot = nsend_s + nsend_x + nsend_y
+     n = 1
+     i_s = 1
+     i_x = 1
+     i_y = 1
+     ind_s = -1
+     ind_x = -1
+     ind_y = -1
+     nsend = 0
+     do while(n<=ntot)
+        if( i_s <= nsend_s ) then
+           rank_s = update_s%send(i_s)%pe-domain%pe
+           if(rank_s .LT. 0) rank_s = rank_s + nlist
+        else
+           rank_s = nlist+1
+        endif
+        if( i_x <= nsend_x ) then
+           rank_x = update_x%send(i_x)%pe-domain%pe
+           if(rank_x .LT. 0) rank_x = rank_x + nlist
+        else
+           rank_x = nlist+1
+        endif
+        if( i_y <= nsend_y ) then
+           rank_y = update_y%send(i_y)%pe-domain%pe
+           if(rank_y .LT. 0) rank_y = rank_y + nlist
+        else
+           rank_y = nlist+1
+        endif
+        nsend = nsend + 1   
+        rank = minval((/rank_s, rank_x, rank_y/))
+        if(rank == rank_s) then
+           n = n + 1
+           ind_s(nsend) = i_s
+           pelist(nsend) = update_s%send(i_s)%pe
+           i_s = i_s + 1
+        endif
+        if(rank == rank_x) then
+           n = n + 1
+           ind_x(nsend) = i_x
+           pelist(nsend) = update_x%send(i_x)%pe
+           i_x = i_x + 1
+        endif
+        if(rank == rank_y) then
+           n = n + 1
+           ind_y(nsend) = i_y
+           pelist(nsend) = update_y%send(i_y)%pe
+           i_y = i_y + 1
+        endif
+     enddo
+
+     nsend_old = nsend
+     nsend     = 0
+     tot_send_size = 0
+     do l = 1, nsend_old
+        msgsize_s = 0
+        msgsize_x = 0
+        msgsize_y = 0
+        m = ind_s(l)
+        if(m>0) msgsize_s = get_mesgsize(update_s%send(m), send_s)*ksize*nscalar
+        m = ind_x(l)
+        if(m>0) msgsize_x = get_mesgsize(update_x%send(m), send_v)*ksize*nvector
+        m = ind_y(l)
+        if(m>0) msgsize_y = get_mesgsize(update_y%send(m), send_v)*ksize*nvector
+        msgsize = msgsize_s + msgsize_x + msgsize_y
+        if( msgsize.GT.0 )then
+           tot_send_size = tot_send_size + msgsize
+           nsend = nsend + 1
+           if(nsend > MAXOVERLAP) then
+              call mpp_error(FATAL, "set_group_update: nsend is greater than MAXOVERLAP, increase MAXOVERLAP")
+           endif
+           send_size(nsend) = msgsize
+           group%to_pe(nsend) = pelist(l)
+           group%buffer_pos_send(nsend) = update_buffer_pos
+           group%send_size(nsend) = msgsize
+           update_buffer_pos = update_buffer_pos + msgsize
+        end if
+     end do
+     group%nsend = nsend
+
+     !--- setup for pack
+     npack = 0
+     pack_buffer_pos = unpack_buffer_pos
+     do l = 1, nsend_old
+        m = ind_s(l)
+        if(m>0) then
+           overptr => update_s%send(m)
+           do n = 1, overptr%count
+              dir = overptr%dir(n)
+              if(send_s(dir)) then
+                 npack = npack + 1
+                 if(npack > MAXOVERLAP) call mpp_error(FATAL,  &
+                   "set_group_update: npack is greater than MAXOVERLAP, increase MAXOVERLAP 1")
+                 group%pack_type(npack) = FIELD_S
+                 group%pack_buffer_pos(npack) = pack_buffer_pos
+                 group%pack_rotation(npack) = overptr%rotation(n) 
+                 group%pack_is(npack) = overptr%is(n)
+                 group%pack_ie(npack) = overptr%ie(n)
+                 group%pack_js(npack) = overptr%js(n)
+                 group%pack_je(npack) = overptr%je(n) 
+                 group%pack_size(npack) = overptr%msgsize(n)*nscalar
+                 pack_buffer_pos = pack_buffer_pos + group%pack_size(npack)*ksize
+              end if
+           end do   
+        end if
+
+        m = ind_x(l)
+        if(m>0) then
+           overptr => update_x%send(m)
+           do n = 1, overptr%count
+              dir = overptr%dir(n)
+              if(send_v(dir)) then
+                 npack = npack + 1
+                 if(npack > MAXOVERLAP) call mpp_error(FATAL,  &
+                   "set_group_update: npack is greater than MAXOVERLAP, increase MAXOVERLAP 2")
+                 group%pack_type(npack) = FIELD_X
+                 group%pack_buffer_pos(npack) = pack_buffer_pos
+                 group%pack_rotation(npack) = overptr%rotation(n) 
+                 group%pack_is(npack) = overptr%is(n)
+                 group%pack_ie(npack) = overptr%ie(n)
+                 group%pack_js(npack) = overptr%js(n)
+                 group%pack_je(npack) = overptr%je(n) 
+                 group%pack_size(npack) = overptr%msgsize(n)*nvector
+                 pack_buffer_pos = pack_buffer_pos + group%pack_size(npack)*ksize
+              end if
+           end do   
+        end if
+
+        m = ind_y(l)
+        if(m>0) then
+           overptr => update_y%send(m)
+           do n = 1, overptr%count
+              dir = overptr%dir(n)
+              if(send_v(dir)) then
+                 npack = npack + 1
+                 if(npack > MAXOVERLAP) call mpp_error(FATAL,  &
+                   "set_group_update: npack is greater than MAXOVERLAP, increase MAXOVERLAP 3")
+                 group%pack_type(npack) = FIELD_Y
+                 group%pack_buffer_pos(npack) = pack_buffer_pos
+                 group%pack_rotation(npack) = overptr%rotation(n) 
+                 group%pack_is(npack) = overptr%is(n)
+                 group%pack_ie(npack) = overptr%ie(n)
+                 group%pack_js(npack) = overptr%js(n)
+                 group%pack_je(npack) = overptr%je(n) 
+                 group%pack_size(npack) = overptr%msgsize(n)*nvector
+                 pack_buffer_pos = pack_buffer_pos + group%pack_size(npack)*ksize
+              end if
+           end do   
+        end if
+     end do
+     group%npack = npack
+     if(update_buffer_pos .NE. pack_buffer_pos ) call mpp_error(FATAL,  &
+                   "set_group_update: update_buffer_pos .NE. pack_buffer_pos")
+
+     !--- make sure the buffer is large enough
+     mpp_domains_stack_hwm = max( mpp_domains_stack_hwm, tot_recv_size+tot_send_size )
+
+     if( mpp_domains_stack_hwm.GT.mpp_domains_stack_size )then
+        write( text,'(i8)' )mpp_domains_stack_hwm
+        call mpp_error( FATAL, 'set_group_update: mpp_domains_stack overflow, '// &
+                     'call mpp_domains_set_stack_size('//trim(text)//') from all PEs.' )
+     end if
+
+     group%tot_msgsize = tot_recv_size+tot_send_size
+
+end subroutine set_group_update
+
+
+!######################################################################
+  subroutine mpp_clear_group_update(group)
+     type(mpp_group_update_type), intent(inout) :: group
+
+     group%nscalar = 0
+     group%nvector = 0
+     group%nsend   = 0
+     group%nrecv   = 0
+     group%npack   = 0
+     group%nunpack = 0
+     group%initialized = .false.
+
+  end subroutine mpp_clear_group_update
+
+!#####################################################################
+  function mpp_group_update_initialized(group)
+     type(mpp_group_update_type), intent(in) :: group
+     logical :: mpp_group_update_initialized
+
+     mpp_group_update_initialized = group%initialized
+
+  end function mpp_group_update_initialized
+
+!#####################################################################
+  function mpp_group_update_is_set(group)
+     type(mpp_group_update_type), intent(in) :: group
+     logical :: mpp_group_update_is_set
+
+     mpp_group_update_is_set = (group%nscalar > 0 .OR. group%nvector > 0)
+
+  end function mpp_group_update_is_set
+
+
+
diff --git a/src/shared/mpp/include/mpp_gather.h b/src/shared/mpp/include/mpp_gather.h
index 9a199ab198..af2b0b2052 100644
--- a/src/shared/mpp/include/mpp_gather.h
+++ b/src/shared/mpp/include/mpp_gather.h
@@ -1,51 +1,222 @@
-subroutine MPP_GATHER_1D_(sbuf, rbuf)
+subroutine MPP_GATHER_1D_(sbuf, rbuf,pelist)
+! JWD: Did not create mpp_gather_2d because have no requirement for it
+! JWD: See mpp_gather_2dv below
    MPP_TYPE_, dimension(:),    intent(in) :: sbuf
    MPP_TYPE_, dimension(:), intent(inout) :: rbuf
-   integer :: cnt, l, nproc
+   integer,   dimension(:),    intent(in), optional :: pelist(:)
+
+   integer :: cnt, l, nproc, op_root
+   integer, allocatable :: pelist2(:)
+
+
+!  If pelist is provided, the first position must be
+!  the operation root
+   if(PRESENT(pelist))then
+      nproc = size(pelist)
+      allocate(pelist2(nproc))
+      pelist2 = pelist
+   else
+      nproc = mpp_npes()
+      allocate(pelist2(nproc))
+      pelist2 = (/ (l, l=root_pe, nproc-1+root_pe) /)
+   endif
+   op_root = pelist2(1)
 
-   nproc = mpp_npes()
    cnt = size(sbuf(:))
-   if(size(rbuf(:)) .NE. cnt*nproc) call mpp_error(FATAL, &
-          "MPP_GATHER_1D_: size(rbuf) should equal to npes*size(sbuf) ")
+   if(size(rbuf(:)) < cnt*nproc) call mpp_error(FATAL, &
+          "MPP_GATHER_1D_: size(rbuf) must be at least npes*size(sbuf) ")
 
    !--- pre-post receiving
-   if(pe == root_pe ) then
+   if(pe == op_root) then
       rbuf(1:cnt) = sbuf
-      do l = 1, nproc-1
-         call mpp_recv(rbuf(l*cnt+1), glen=cnt, from_pe=root_pe+l, block=.FALSE., tag=COMM_TAG_1 )
+      do l = 2, nproc
+         call mpp_recv(rbuf((l-1)*cnt+1), glen=cnt, from_pe=pelist2(l), block=.FALSE., tag=COMM_TAG_1 )
       enddo
    else
-      call mpp_send(sbuf(1), plen=cnt, to_pe=root_pe, tag=COMM_TAG_1)
+      call mpp_send(sbuf(1), plen=cnt, to_pe=op_root, tag=COMM_TAG_1)
    endif
 
    call mpp_sync_self(check=EVENT_RECV)
    call mpp_sync_self()
-
+   deallocate(pelist2)
 end subroutine MPP_GATHER_1D_
 
-
-subroutine MPP_GATHER_1DV_(sbuf, ssize, rbuf, rsize)
+subroutine MPP_GATHER_1DV_(sbuf, ssize, rbuf, rsize, pelist)
    MPP_TYPE_, dimension(:),    intent(in) :: sbuf
    MPP_TYPE_, dimension(:), intent(inout) :: rbuf
    integer,                    intent(in) :: ssize
-   integer,   dimension(:),    intent(in) :: rsize 
-   integer :: cnt, l, nproc, pos
+   integer,   dimension(:),    intent(in) :: rsize
+   integer,   dimension(:),    intent(in), optional :: pelist(:)
+
+   integer :: cnt, l, nproc, pos, op_root
+   integer, allocatable :: pelist2(:)
+
+!  If pelist is provided, the first position must be
+!  the operation root
+   if(PRESENT(pelist))then
+      nproc = size(pelist)
+      allocate(pelist2(nproc)) 
+      pelist2 = pelist
+   else
+      nproc = mpp_npes()
+      allocate(pelist2(nproc))        
+      pelist2 = (/ (l, l=0+root_pe, nproc-1+root_pe) /)
+   endif
+   op_root = pelist2(1)
 
-   nproc = mpp_npes()
 
    !--- pre-post receiving
-   if(pe == root_pe ) then
-      rbuf(1:ssize) = sbuf
+   if(pe == op_root) then
+      rbuf(1:ssize) = sbuf(:)
       pos = ssize
-      do l = 1, nproc-1
-         call mpp_recv(rbuf(pos+1), glen=rsize(l+1), from_pe=root_pe+l, block=.FALSE., tag=COMM_TAG_2 )
-         pos = pos + rsize(l+1)
+      do l = 2, nproc
+         if(rsize(l) >0) then
+            call mpp_recv(rbuf(pos+1), glen=rsize(l), from_pe=pelist2(l), block=.FALSE., tag=COMM_TAG_2 )
+            pos = pos + rsize(l)
+         endif
       enddo
    else
-      call mpp_send(sbuf(1), plen=ssize, to_pe=root_pe, tag=COMM_TAG_2)
+      if(ssize>0) call mpp_send(sbuf(1), plen=ssize, to_pe=op_root, tag=COMM_TAG_2)
    endif
 
    call mpp_sync_self(check=EVENT_RECV)
    call mpp_sync_self()
-
+   deallocate(pelist2)
 end subroutine MPP_GATHER_1DV_
+
+
+subroutine MPP_GATHER_PELIST_2D_(is, ie, js, je, pelist, array_seg, data, is_root_pe, &
+                                 ishift, jshift)
+   integer,                           intent(in)    :: is, ie, js, je
+   integer,   dimension(:),           intent(in)    :: pelist
+   MPP_TYPE_, dimension(is:ie,js:je), intent(in)    :: array_seg
+   MPP_TYPE_, dimension(:,:),         intent(inout) :: data
+   logical,                           intent(in)    :: is_root_pe
+   integer,   optional,               intent(in)    :: ishift, jshift
+
+   MPP_TYPE_ ::  arr3D(size(array_seg,1),size(array_seg,2),1)
+   MPP_TYPE_ :: data3D(size(     data,1),size(     data,2),1)
+   pointer( aptr,  arr3D )
+   pointer( dptr, data3D )
+   aptr = LOC(array_seg)
+   dptr = LOC(     data)
+
+   call mpp_gather(is, ie, js, je, 1, pelist, arr3D, data3D, is_root_pe, &
+                   ishift, jshift)
+   return
+
+end subroutine MPP_GATHER_PELIST_2D_
+
+
+subroutine MPP_GATHER_PELIST_3D_(is, ie, js, je, nk, pelist, array_seg, data, is_root_pe, &
+                                 ishift, jshift)
+   integer,                                intent(in)    :: is, ie, js, je, nk
+   integer,   dimension(:),                intent(in)    :: pelist
+   MPP_TYPE_, dimension(is:ie,js:je,1:nk), intent(in)    :: array_seg
+   MPP_TYPE_, dimension(:,:,:),            intent(inout) :: data
+   logical,                                intent(in)    :: is_root_pe
+   integer,   optional,                    intent(in)    :: ishift, jshift
+
+   integer :: i, msgsize, root_pe, root_pe_test
+   integer :: i1, i2, j1, j2, ioff, joff
+   integer :: my_ind(4), gind(4,size(pelist))
+   type array3D
+     MPP_TYPE_, dimension(:,:,:), allocatable :: data
+   endtype array3D
+   type(array3d), dimension(size(pelist)) :: temp
+
+   if (.not.ANY(mpp_pe().eq.pelist(:))) return
+
+   if (is_root_pe) then
+     root_pe = mpp_pe()
+     root_pe_test = 999
+     if (.not.ANY(pelist(:).eq.root_pe)) call mpp_error(FATAL, &
+                "fms_io(mpp_gather_pelist): root_pe not a member of pelist")
+   else
+     root_pe = 0
+     root_pe_test = -999
+   endif
+! need this check in case MPI-rank 0 is a member of the pelist
+   call mpp_max(root_pe_test, pelist)
+   if (root_pe_test.lt.0) call mpp_error(FATAL, &
+                "fms_io(mpp_gather_pelist): root_pe not specified or not a member of the pelist")
+! need to make sure only one root_pe has been specified
+   call mpp_sum(root_pe, pelist)
+   if ((is_root_pe) .and. (mpp_pe().ne.root_pe)) call mpp_error(FATAL, &
+                "fms_io(mpp_gather_pelist): too many root_pes specified")
+
+
+   ioff=0
+   joff=0
+   if (present(ishift)) ioff=ishift
+   if (present(jshift)) joff=jshift
+
+   my_ind(1) = is
+   my_ind(2) = ie
+   my_ind(3) = js
+   my_ind(4) = je
+
+! gather indices into global index on root_pe
+   if (is_root_pe) then
+     do i = 1, size(pelist)
+! root_pe data copy - no send to self
+       if (pelist(i).eq.root_pe) then
+         gind(:,i) = my_ind(:)
+       else
+         call mpp_recv(gind(:,i:i), 4, pelist(i), .FALSE., COMM_TAG_1)
+       endif
+     enddo
+     call mpp_sync_self(check=EVENT_RECV)
+     gind(1,:)=gind(1,:)+ioff
+     gind(2,:)=gind(2,:)+ioff
+     gind(3,:)=gind(3,:)+joff
+     gind(4,:)=gind(4,:)+joff
+! check indices to make sure they are within the range of "data"
+     if ((minval(gind).lt.1) .OR. (maxval(gind(1:2,:)).gt.size(data,1)) .OR. (maxval(gind(3:4,:)).gt.size(data,2))) &
+         call mpp_error(FATAL,"fms_io(mpp_gather_pelist): specified indices (with shift) are outside of the & 
+                        &range of the receiving array")
+   else
+! non root_pe's send indices to root_pe
+     call mpp_send(my_ind(:), 4, root_pe, COMM_TAG_1)
+     call mpp_sync_self(check=EVENT_SEND)
+   endif
+
+!  gather segments into data based on indices
+   if (is_root_pe) then
+     do i = 1, size(pelist)
+       if (pelist(i).ne.root_pe) then    ! no send to self
+         i1 = gind(1,i)
+         i2 = gind(2,i)
+         j1 = gind(3,i)
+         j2 = gind(4,i)
+         msgsize = (i2-i1+1)*(j2-j1+1)*nk
+         allocate(temp(i)%data(i1:i2,j1:j2,1:nk))
+         call mpp_recv(temp(i)%data(i1:i2,j1:j2,1:nk), msgsize, pelist(i), .FALSE., COMM_TAG_2)
+       endif
+     enddo
+     call mpp_sync_self(check=EVENT_RECV)
+!  unbuffer/copy the data into the return array
+     do i = 1, size(pelist)
+       if (pelist(i).eq.root_pe) then
+!        data copy - no send to self
+         data(is+ioff:ie+ioff,js+joff:je+joff,1:nk) = array_seg(is:ie,js:je,1:nk)
+       else
+         i1 = gind(1,i)
+         i2 = gind(2,i)
+         j1 = gind(3,i)
+         j2 = gind(4,i)
+         data(i1:i2,j1:j2,1:nk)=temp(i)%data(i1:i2,j1:j2,1:nk)
+         deallocate(temp(i)%data)
+       endif
+     enddo
+   else
+!    non root_pe's send data to root_pe
+     msgsize = (my_ind(2)-my_ind(1)+1) * (my_ind(4)-my_ind(3)+1) * nk
+     call mpp_send(array_seg, msgsize, root_pe, COMM_TAG_2)
+     call mpp_sync_self(check=EVENT_SEND)
+   endif
+
+   call mpp_sync_self()
+   return
+
+end subroutine MPP_GATHER_PELIST_3D_
diff --git a/src/shared/mpp/include/mpp_get_boundary.h b/src/shared/mpp/include/mpp_get_boundary.h
index 60be0228d9..02d2d7fae6 100644
--- a/src/shared/mpp/include/mpp_get_boundary.h
+++ b/src/shared/mpp/include/mpp_get_boundary.h
@@ -497,7 +497,7 @@ subroutine MPP_GET_BOUNDARY_2D_V_(fieldx, fieldy, domain, ebufferx, sbufferx, wb
      if(ASSOCIATED(boundx) ) then
         call mpp_do_get_boundary(f_addrsx(1:l_size,1:ntile), f_addrsy(1:l_size,1:ntile), domain, boundx, boundy, &
              b_addrsx(:,1:l_size,1:ntile), b_addrsy(:,1:l_size,1:ntile), bsizex, &
-             bsizey, ksize, d_type, update_flags)
+             bsizey, ksize, d_type, update_flags, grid_offset_type)
      endif
      l_size=0; f_addrsx=-9999; f_addrsy=-9999; bsizex=0; bsizey=0; 
      b_addrsx=-9999; b_addrsy=-9999; isize=0;  jsize=0;  ksize=0
@@ -721,7 +721,7 @@ subroutine MPP_GET_BOUNDARY_3D_V_(fieldx, fieldy, domain, ebufferx, sbufferx, wb
      if(ASSOCIATED(boundx) ) then
         call mpp_do_get_boundary(f_addrsx(1:l_size,1:ntile), f_addrsy(1:l_size,1:ntile), domain, boundx, boundy, &
              b_addrsx(:,1:l_size,1:ntile), b_addrsy(:,1:l_size,1:ntile), bsizex, &
-             bsizey, ksize, d_type, update_flags)
+             bsizey, ksize, d_type, update_flags, grid_offset_type)
      endif
      l_size=0; f_addrsx=-9999; f_addrsy=-9999; bsizex=0; bsizey=0; 
      b_addrsx=-9999; b_addrsy=-9999; isize=0;  jsize=0;  ksize=0
diff --git a/src/shared/mpp/include/mpp_global_sum.h b/src/shared/mpp/include/mpp_global_sum.h
index c1b9954fa2..7684fe50b8 100644
--- a/src/shared/mpp/include/mpp_global_sum.h
+++ b/src/shared/mpp/include/mpp_global_sum.h
@@ -1,10 +1,11 @@
-  function MPP_GLOBAL_SUM_( domain, field, flags, position, tile_count )
+  function MPP_GLOBAL_SUM_( domain, field, flags, position, tile_count, overflow_check)
     MPP_TYPE_ :: MPP_GLOBAL_SUM_
     type(domain2D), intent(in) :: domain
     MPP_TYPE_, intent(in) :: field(:,: MPP_EXTRA_INDICES_ )
     integer, intent(in), optional :: flags
     integer, intent(in), optional :: position
     integer, intent(in), optional :: tile_count
+    logical, intent(in), optional :: overflow_check
 
     MPP_TYPE_, dimension(:,:),       allocatable :: field2D
     MPP_TYPE_, dimension(:,:),       allocatable :: global2D
@@ -99,11 +100,32 @@
              MPP_GLOBAL_SUM_ = sum(gsum(1:domain%ntiles))
           end if
        end if
+    else if ( global_flag == BITWISE_EFP_SUM )then
+#ifdef DO_EFP_SUM_
+       !this is bitwise across different PE counts using EFP sum
+       if( ntile > 1 ) then 
+          call mpp_error( FATAL, 'MPP_GLOBAL_SUM_: multiple tile per pe is not supported for BITWISE_EFP_SUM')
+       endif
+       allocate( field2D (isc:iec,jsc:jec) )
+       do j = jsc, jec
+          do i = isc, iec
+             field2D(i,j) = sum( field(i+ioff:i+ioff,j+joff:j+joff MPP_EXTRA_INDICES_) )
+          end do
+       end do
+       !--- using efp sum.
+       if(efp_sum_overflow_check) then
+          MPP_GLOBAL_SUM_ = mpp_reproducing_sum(field2D, overflow_check=.true.)
+       else
+          MPP_GLOBAL_SUM_ = mpp_reproducing_sum(field2D, overflow_check=overflow_check)
+       endif
+#else
+        call mpp_error( FATAL, 'MPP_GLOBAL_SUM_: BITWISE_EFP_SUM is only implemented for real number, contact developer') 
+#endif
     else  !this is not bitwise-exact across different PE counts
        ioffset = domain%x(tile)%loffset*ishift; joffset = domain%y(tile)%loffset*jshift
        mygsum(tile) = sum( field(is+ioff:ie+ioff+ioffset, js+joff:je+joff+joffset MPP_EXTRA_INDICES_) )
        if(tile == ntile) then
-          MPP_GLOBAL_SUM_ = sum(mygsum)
+          MPP_GLOBAL_SUM_ = sum(mygsum(1:ntile))
           call mpp_sum( MPP_GLOBAL_SUM_, domain%list(:)%pe )
        end if
     end if
diff --git a/src/shared/mpp/include/mpp_group_update.h b/src/shared/mpp/include/mpp_group_update.h
new file mode 100644
index 0000000000..df4518e608
--- /dev/null
+++ b/src/shared/mpp/include/mpp_group_update.h
@@ -0,0 +1,973 @@
+! -*-f90-*-
+subroutine MPP_CREATE_GROUP_UPDATE_2D_(group, field, domain, flags, position, &
+     whalo, ehalo, shalo, nhalo)
+  type(mpp_group_update_type), intent(inout) :: group
+  MPP_TYPE_,            intent(inout)        :: field(:,:)
+  type(domain2D),       intent(inout)        :: domain  
+  integer,              intent(in), optional :: flags
+  integer,              intent(in), optional :: position
+  integer,              intent(in), optional :: whalo, ehalo, shalo, nhalo
+
+  MPP_TYPE_ :: field3D(size(field,1),size(field,2),1)
+  pointer( ptr, field3D )
+  ptr = LOC(field)
+
+  call mpp_create_group_update(group, field3D, domain, flags, position, whalo, ehalo, shalo, nhalo)
+
+  return
+
+end subroutine MPP_CREATE_GROUP_UPDATE_2D_
+
+subroutine MPP_CREATE_GROUP_UPDATE_3D_(group, field, domain, flags, position, whalo, ehalo, shalo, nhalo)
+  type(mpp_group_update_type), intent(inout) :: group
+  MPP_TYPE_,        intent(inout)        :: field(:,:,:)
+  type(domain2D),   intent(inout)        :: domain  
+  integer,          intent(in), optional :: flags
+  integer,          intent(in), optional :: position
+  integer,          intent(in), optional :: whalo, ehalo, shalo, nhalo ! specify halo region to be updated.
+
+  integer          :: update_position, update_whalo, update_ehalo, update_shalo, update_nhalo
+  integer          :: update_flags, isize, jsize, ksize
+  integer          :: nscalar
+  character(len=3) :: text
+  logical          :: set_mismatch, update_edge_only
+  logical          :: recv(8)
+
+  if(group%initialized) then
+      call mpp_error(FATAL, "MPP_CREATE_GROUP_UPDATE_3D: group is already initialized")
+  endif
+
+  if(present(whalo)) then
+     update_whalo = whalo
+     if(abs(update_whalo) > domain%whalo ) call mpp_error(FATAL, "MPP_CREATE_GROUP_UPDATE: "// &
+          "optional argument whalo should not be larger than the whalo when define domain.")
+  else
+     update_whalo = domain%whalo
+  end if
+  if(present(ehalo)) then
+     update_ehalo = ehalo
+     if(abs(update_ehalo) > domain%ehalo ) call mpp_error(FATAL, "MPP_CREATE_GROUP_UPDATE: "// &
+          "optional argument ehalo should not be larger than the ehalo when define domain.")
+  else
+     update_ehalo = domain%ehalo
+  end if
+  if(present(shalo)) then
+     update_shalo = shalo
+     if(abs(update_shalo) > domain%shalo ) call mpp_error(FATAL, "MPP_CREATE_GROUP_UPDATE: "// &
+          "optional argument shalo should not be larger than the shalo when define domain.")
+  else
+     update_shalo = domain%shalo
+  end if
+  if(present(nhalo)) then
+     update_nhalo = nhalo
+     if(abs(update_nhalo) > domain%nhalo ) call mpp_error(FATAL, "MPP_CREATE_GROUP_UPDATE: "// &
+          "optional argument nhalo should not be larger than the nhalo when define domain.")
+  else
+     update_nhalo = domain%nhalo
+  end if
+  update_position = CENTER
+  !--- when there is NINETY or MINUS_NINETY rotation for some contact, the salar data can not be on E or N-cell,
+  if(present(position)) then
+     update_position = position 
+     if(domain%rotated_ninety .AND. ( position == EAST .OR. position == NORTH ) )  &
+          call mpp_error(FATAL, 'MPP_CREATE_GROUP_UPDATE_3D: hen there is NINETY or MINUS_NINETY rotation, ' // &
+          'can not use scalar version update_domain for data on E or N-cell' )
+  end if
+
+  if( domain%max_ntile_pe > 1 ) then
+     call mpp_error(FATAL,'MPP_CREATE_GROUP_UPDATE: do not support multiple tile per processor')
+  endif
+
+  update_flags = XUPDATE+YUPDATE 
+  if(present(flags)) update_flags = flags
+
+  group%nscalar = group%nscalar + 1
+  nscalar = group%nscalar
+  if( nscalar > MAX_DOMAIN_FIELDS)then
+     write( text,'(i2)' ) MAX_DOMAIN_FIELDS
+     call mpp_error(FATAL,'MPP_CREATE_GROUP_UPDATE: MAX_DOMAIN_FIELDS='//text//' exceeded for group update.' )
+  endif
+
+  isize = size(field,1); jsize=size(field,2); ksize = size(field,3)
+
+  group%addrs_s(nscalar) = LOC(field)
+  if( group%nscalar == 1 ) then
+     group%flags_s  = update_flags
+     group%whalo_s  = update_whalo
+     group%ehalo_s  = update_ehalo
+     group%shalo_s  = update_shalo
+     group%nhalo_s  = update_nhalo
+     group%position = update_position
+     group%isize_s  = isize
+     group%jsize_s  = jsize
+     group%ksize_s  = ksize
+     call mpp_get_memory_domain(domain, group%is_s, group%ie_s, group%js_s, group%je_s, position=position)
+
+     update_edge_only = BTEST(update_flags, EDGEONLY)
+     recv(1) = BTEST(update_flags,EAST)
+     recv(3) = BTEST(update_flags,SOUTH)
+     recv(5) = BTEST(update_flags,WEST)
+     recv(7) = BTEST(update_flags,NORTH)
+     if( update_edge_only ) then
+        recv(2) = .false.
+        recv(4) = .false.
+        recv(6) = .false.
+        recv(8) = .false.
+        if( .NOT. (recv(1) .OR. recv(3) .OR. recv(5) .OR. recv(7)) ) then
+           recv(1) = .true.
+           recv(3) = .true.
+           recv(5) = .true.
+           recv(7) = .true.
+        endif
+     else
+        recv(2) = recv(1) .AND. recv(3)
+        recv(4) = recv(3) .AND. recv(5)
+        recv(6) = recv(5) .AND. recv(7)
+        recv(8) = recv(7) .AND. recv(1)
+     endif
+     group%recv_s = recv
+  else
+     set_mismatch = .false.
+     set_mismatch = set_mismatch .OR. (group%flags_s  .NE. update_flags)
+     set_mismatch = set_mismatch .OR. (group%whalo_s  .NE. update_whalo)
+     set_mismatch = set_mismatch .OR. (group%ehalo_s  .NE. update_ehalo)
+     set_mismatch = set_mismatch .OR. (group%shalo_s  .NE. update_shalo)
+     set_mismatch = set_mismatch .OR. (group%nhalo_s  .NE. update_nhalo)
+     set_mismatch = set_mismatch .OR. (group%position .NE. update_position)
+     set_mismatch = set_mismatch .OR. (group%isize_s  .NE. isize)
+     set_mismatch = set_mismatch .OR. (group%jsize_s  .NE. jsize)
+     set_mismatch = set_mismatch .OR. (group%ksize_s  .NE. ksize)
+
+     if(set_mismatch)then
+        write( text,'(i2)' ) nscalar
+        call mpp_error(FATAL,'MPP_CREATE_GROUP_UPDATE_3D: Incompatible field at count '//text//' for group update.' )
+     endif
+  endif
+
+  return
+
+end subroutine MPP_CREATE_GROUP_UPDATE_3D_
+
+
+subroutine MPP_CREATE_GROUP_UPDATE_4D_(group, field, domain, flags, position, &
+     whalo, ehalo, shalo, nhalo)
+  type(mpp_group_update_type), intent(inout) :: group
+  MPP_TYPE_,            intent(inout)        :: field(:,:,:,:)
+  type(domain2D),       intent(inout)        :: domain  
+  integer,              intent(in), optional :: flags
+  integer,              intent(in), optional :: position
+  integer,              intent(in), optional :: whalo, ehalo, shalo, nhalo
+
+  MPP_TYPE_ :: field3D(size(field,1),size(field,2),size(field,3)*size(field,4))
+  pointer( ptr, field3D )
+  ptr = LOC(field)
+
+  call mpp_create_group_update(group, field3D, domain, flags, position, whalo, ehalo, shalo, nhalo)
+
+  return
+
+end subroutine MPP_CREATE_GROUP_UPDATE_4D_
+
+subroutine MPP_CREATE_GROUP_UPDATE_2D_V_( group, fieldx, fieldy, domain, flags, gridtype, &
+                                          whalo, ehalo, shalo, nhalo)
+
+  type(mpp_group_update_type), intent(inout) :: group
+  MPP_TYPE_,        intent(inout)            :: fieldx(:,:), fieldy(:,:)
+  type(domain2D),   intent(inout)            :: domain
+  integer,          intent(in),     optional :: flags, gridtype
+  integer,          intent(in),     optional :: whalo, ehalo, shalo, nhalo
+  MPP_TYPE_ :: field3Dx(size(fieldx,1),size(fieldx,2),1)
+  MPP_TYPE_ :: field3Dy(size(fieldy,1),size(fieldy,2),1)
+  pointer( ptrx, field3Dx )
+  pointer( ptry, field3Dy )
+  ptrx = LOC(fieldx)
+  ptry = LOC(fieldy)
+
+  call  mpp_create_group_update(group, field3Dx, field3Dy, domain, flags, gridtype, &
+                                          whalo, ehalo, shalo, nhalo)
+
+  return
+
+end subroutine MPP_CREATE_GROUP_UPDATE_2D_V_
+
+
+
+subroutine MPP_CREATE_GROUP_UPDATE_3D_V_( group, fieldx, fieldy, domain, flags, gridtype, &
+                                          whalo, ehalo, shalo, nhalo)
+  type(mpp_group_update_type), intent(inout) :: group
+  MPP_TYPE_,        intent(inout)        :: fieldx(:,:,:), fieldy(:,:,:)
+  type(domain2D),   intent(inout)        :: domain
+  integer,          intent(in), optional :: flags, gridtype
+  integer,          intent(in), optional :: whalo, ehalo, shalo, nhalo
+
+  integer          :: update_whalo, update_ehalo, update_shalo, update_nhalo
+  integer          :: update_flags, isize_x, jsize_x, ksize_x, isize_y, jsize_y, ksize_y
+  integer          :: nvector, update_gridtype, position_x, position_y
+  character(len=3) :: text
+  logical          :: set_mismatch, update_edge_only
+  logical          :: recv(8)
+
+
+  if(group%initialized) then
+     call mpp_error(FATAL, "MPP_CREATE_GROUP_UPDATE_V: group is already initialized")
+  endif
+
+  if(present(whalo)) then
+     update_whalo = whalo
+     if(abs(update_whalo) > domain%whalo ) call mpp_error(FATAL, "MPP_CREATE_GROUP_UPDATE_V: "// &
+          "optional argument whalo should not be larger than the whalo when define domain.")
+  else
+     update_whalo = domain%whalo
+  end if
+  if(present(ehalo)) then
+     update_ehalo = ehalo
+     if(abs(update_ehalo) > domain%ehalo ) call mpp_error(FATAL, "MPP_CREATE_GROUP_UPDATE_V: "// &
+          "optional argument ehalo should not be larger than the ehalo when define domain.")
+  else
+     update_ehalo = domain%ehalo
+  end if
+  if(present(shalo)) then
+     update_shalo = shalo
+     if(abs(update_shalo) > domain%shalo ) call mpp_error(FATAL, "MPP_CREATE_GROUP_UPDATE_V: "// &
+          "optional argument shalo should not be larger than the shalo when define domain.")
+  else
+     update_shalo = domain%shalo
+  end if
+  if(present(nhalo)) then
+     update_nhalo = nhalo
+     if(abs(update_nhalo) > domain%nhalo ) call mpp_error(FATAL, "MPP_CREATE_GROUP_UPDATE_V: "// &
+          "optional argument nhalo should not be larger than the nhalo when define domain.")
+  else
+     update_nhalo = domain%nhalo
+  end if
+
+  update_gridtype = AGRID
+  if(PRESENT(gridtype)) update_gridtype = gridtype
+
+  if( domain%max_ntile_pe > 1 ) then
+     call mpp_error(FATAL,'MPP_CREATE_GROUP_UPDATE_V: do not support multiple tile per processor')
+  endif
+
+  update_flags = XUPDATE+YUPDATE   !default
+  if( PRESENT(flags) )update_flags = flags
+
+  group%nvector = group%nvector + 1
+  nvector = group%nvector
+  if( nvector > MAX_DOMAIN_FIELDS)then
+     write( text,'(i2)' ) MAX_DOMAIN_FIELDS
+     call mpp_error(FATAL,'MPP_CREATE_GROUP_UPDATE_V: MAX_DOMAIN_FIELDS='//text//' exceeded for group update.' )
+  endif
+
+  isize_x = size(fieldx,1); jsize_x = size(fieldx,2); ksize_x = size(fieldx,3) 
+  isize_y = size(fieldy,1); jsize_y = size(fieldy,2); ksize_y = size(fieldy,3) 
+  
+  if(ksize_x .NE. ksize_y) call mpp_error(FATAL,  &
+           'MPP_CREATE_GROUP_UPDATE_V: mismatch of ksize between fieldx and fieldy')
+
+  group%addrs_x(nvector) = LOC(fieldx)
+  group%addrs_y(nvector) = LOC(fieldy)
+
+  if( group%nvector == 1 ) then
+     group%flags_v  = update_flags
+     group%whalo_v  = update_whalo
+     group%ehalo_v  = update_ehalo
+     group%shalo_v  = update_shalo
+     group%nhalo_v  = update_nhalo
+     group%gridtype = update_gridtype
+     group%isize_x  = isize_x
+     group%jsize_x  = jsize_x
+     group%isize_y  = isize_y
+     group%jsize_y  = jsize_y
+     group%ksize_v  = ksize_x
+     update_edge_only = BTEST(update_flags, EDGEONLY)
+     recv(1) = BTEST(update_flags,EAST)
+     recv(3) = BTEST(update_flags,SOUTH)
+     recv(5) = BTEST(update_flags,WEST)
+     recv(7) = BTEST(update_flags,NORTH)
+     if( update_edge_only ) then
+        recv(2) = .false.
+        recv(4) = .false.
+        recv(6) = .false.
+        recv(8) = .false.
+        if( .NOT. (recv(1) .OR. recv(3) .OR. recv(5) .OR. recv(7)) ) then
+           recv(1) = .true.
+           recv(3) = .true.
+           recv(5) = .true.
+           recv(7) = .true.
+        endif
+     else
+        recv(2) = recv(1) .AND. recv(3)
+        recv(4) = recv(3) .AND. recv(5)
+        recv(6) = recv(5) .AND. recv(7)
+        recv(8) = recv(7) .AND. recv(1)
+     endif
+     group%recv_v = recv
+     select case(group%gridtype)
+     case (AGRID)
+        position_x = CENTER
+        position_y = CENTER
+     case (BGRID_NE, BGRID_SW)
+        position_x = CORNER
+        position_y = CORNER
+     case (CGRID_NE, CGRID_SW)
+        position_x = EAST
+        position_y = NORTH
+     case (DGRID_NE, DGRID_SW)
+        position_x = NORTH
+        position_y = EAST
+     case default
+        call mpp_error(FATAL, "mpp_CREATE_GROUP_UPDATE_V: invalid value of gridtype")
+     end select
+
+     call mpp_get_memory_domain(domain, group%is_x, group%ie_x, group%js_x, group%je_x, position=position_x)
+     call mpp_get_memory_domain(domain, group%is_y, group%ie_y, group%js_y, group%je_y, position=position_y)
+  else
+     set_mismatch = .false.
+     set_mismatch = set_mismatch .OR. (group%flags_v  .NE. update_flags)
+     set_mismatch = set_mismatch .OR. (group%whalo_v  .NE. update_whalo)
+     set_mismatch = set_mismatch .OR. (group%ehalo_v  .NE. update_ehalo)
+     set_mismatch = set_mismatch .OR. (group%shalo_v  .NE. update_shalo)
+     set_mismatch = set_mismatch .OR. (group%nhalo_v  .NE. update_nhalo)
+     set_mismatch = set_mismatch .OR. (group%gridtype .NE. update_gridtype)
+     set_mismatch = set_mismatch .OR. (group%isize_x  .NE. isize_x)
+     set_mismatch = set_mismatch .OR. (group%jsize_x  .NE. jsize_x)
+     set_mismatch = set_mismatch .OR. (group%isize_y  .NE. isize_y)
+     set_mismatch = set_mismatch .OR. (group%jsize_y  .NE. jsize_y)
+     set_mismatch = set_mismatch .OR. (group%ksize_v  .NE. ksize_x)
+
+     if(set_mismatch)then
+        write( text,'(i2)' ) nvector
+        call mpp_error(FATAL,'MPP_CREATE_GROUP_UPDATE_V: Incompatible field at count '//text//' for group update.' )
+     endif
+  endif
+
+  return
+
+end subroutine MPP_CREATE_GROUP_UPDATE_3D_V_
+
+subroutine MPP_CREATE_GROUP_UPDATE_4D_V_( group, fieldx, fieldy, domain, flags, gridtype, &
+                                          whalo, ehalo, shalo, nhalo)
+
+  type(mpp_group_update_type), intent(inout) :: group
+  MPP_TYPE_,        intent(inout)            :: fieldx(:,:,:,:), fieldy(:,:,:,:)
+  type(domain2D),   intent(inout)            :: domain
+  integer,          intent(in),     optional :: flags, gridtype
+  integer,          intent(in),     optional :: whalo, ehalo, shalo, nhalo
+  MPP_TYPE_ :: field3Dx(size(fieldx,1),size(fieldx,2),size(fieldx,3)*size(fieldx,4))
+  MPP_TYPE_ :: field3Dy(size(fieldy,1),size(fieldy,2),size(fieldy,3)*size(fieldy,4))
+  pointer( ptrx, field3Dx )
+  pointer( ptry, field3Dy )
+  ptrx = LOC(fieldx)
+  ptry = LOC(fieldy)
+
+  call  mpp_create_group_update(group, field3Dx, field3Dy, domain, flags, gridtype, &
+                                          whalo, ehalo, shalo, nhalo)
+
+  return
+
+end subroutine MPP_CREATE_GROUP_UPDATE_4D_V_
+
+
+subroutine MPP_DO_GROUP_UPDATE_(group, domain, d_type)
+  type(mpp_group_update_type), intent(inout) :: group
+  type(domain2D),              intent(inout) :: domain  
+  MPP_TYPE_,                   intent(in)    :: d_type
+
+  integer   :: nscalar, nvector, nlist
+  logical   :: recv_v(8)
+  integer   :: nsend, nrecv, flags_v
+  integer   :: msgsize
+  integer   :: from_pe, to_pe, buffer_pos, pos
+  integer   :: ksize, is, ie, js, je
+  integer   :: n, l, m, i, j, k, buffer_start_pos, nk
+  integer   :: shift, gridtype, midpoint
+  integer   :: npack, nunpack, rotation
+  character(len=8)            :: text
+
+  MPP_TYPE_ :: buffer(mpp_domains_stack_size)
+  MPP_TYPE_ :: field (group%is_s:group%ie_s,group%js_s:group%je_s, group%ksize_s)
+  MPP_TYPE_ :: fieldx(group%is_x:group%ie_x,group%js_x:group%je_x, group%ksize_v)
+  MPP_TYPE_ :: fieldy(group%is_y:group%ie_y,group%js_y:group%je_y, group%ksize_v)
+  pointer(ptr, buffer )
+  pointer(ptr_field, field)
+  pointer(ptr_fieldx, fieldx)
+  pointer(ptr_fieldy, fieldy)
+
+  nscalar = group%nscalar
+  nvector = group%nvector
+  nlist   = size(domain%list(:))
+  gridtype = group%gridtype
+
+  !--- ksize_s must equal ksize_v 
+  if(nvector > 0 .AND. nscalar > 0) then
+     if(group%ksize_s .NE. group%ksize_v) then
+        call mpp_error(FATAL, "MPP_DO_GROUP_UPDATE: ksize_s and ksize_v are not equal")
+     endif
+     ksize = group%ksize_s
+  else if (nscalar > 0) then
+     ksize = group%ksize_s
+  else if (nvector > 0) then
+     ksize = group%ksize_v
+  else
+     call mpp_error(FATAL, "MPP_DO_GROUP_UPDATE: nscalar and nvector are all 0")
+  endif
+  if(nvector > 0) recv_v = group%recv_v
+
+  ptr = LOC(mpp_domains_stack)
+
+  !--- set reset_index_s and reset_index_v to 0 
+  group%reset_index_s = 0
+  group%reset_index_v = 0
+
+  if(.not. group%initialized) call set_group_update(group,domain)
+
+  nrecv = group%nrecv
+  nsend = group%nsend
+
+  !---pre-post receive.
+  call mpp_clock_begin(group_recv_clock)
+  do m = 1, nrecv
+     msgsize = group%recv_size(m)
+     from_pe = group%from_pe(m)
+     if( msgsize .GT. 0 )then
+        buffer_pos = group%buffer_pos_recv(m)
+        call mpp_recv( buffer(buffer_pos+1), glen=msgsize, from_pe=from_pe, block=.false., &
+             tag=COMM_TAG_1)
+     end if
+  end do
+
+ !pack the data
+  call mpp_clock_end(group_recv_clock)
+
+  flags_v = group%flags_v
+  npack = group%npack
+
+  call mpp_clock_begin(group_pack_clock)
+  !pack the data
+  buffer_start_pos = 0
+#include 
+  call mpp_clock_end(group_pack_clock)
+
+  call mpp_clock_begin(group_send_clock)
+  do n = 1, nsend  
+     msgsize = group%send_size(n)
+     if( msgsize .GT. 0 )then
+        buffer_pos = group%buffer_pos_send(n)
+        to_pe = group%to_pe(n)
+        call mpp_send( buffer(buffer_pos+1), plen=msgsize, to_pe=to_pe, tag=COMM_TAG_1)
+     endif
+  enddo
+  call mpp_clock_end(group_send_clock)
+
+  if(nrecv>0) then
+     call mpp_clock_begin(group_wait_clock)
+     call mpp_sync_self(check=EVENT_RECV)
+     call mpp_clock_end(group_wait_clock)
+  endif
+
+  !---unpack the buffer
+  nunpack = group%nunpack
+  call mpp_clock_begin(group_unpk_clock)
+#include 
+  call mpp_clock_end(group_unpk_clock)
+
+  ! ---northern boundary fold
+  shift = 0
+  if(domain%symmetry) shift = 1
+  if( nvector >0 .AND. BTEST(domain%fold,NORTH) .AND. (.NOT.BTEST(flags_v,SCALAR_BIT)) )then
+     j = domain%y(1)%global%end+shift
+     if( domain%y(1)%data%begin.LE.j .AND. j.LE.domain%y(1)%data%end+shift )then !fold is within domain
+        !poles set to 0: BGRID only
+        if( gridtype.EQ.BGRID_NE )then
+           midpoint = (domain%x(1)%global%begin+domain%x(1)%global%end-1+shift)/2
+           j  = domain%y(1)%global%end+shift
+           is = domain%x(1)%global%begin; ie = domain%x(1)%global%end+shift
+           if( .NOT. domain%symmetry ) is = is - 1
+           do i = is ,ie, midpoint
+              if( domain%x(1)%data%begin.LE.i .AND. i.LE. domain%x(1)%data%end+shift )then
+                 do l=1,nvector
+                    ptr_fieldx = group%addrs_x(l)
+                    ptr_fieldy = group%addrs_y(l)
+                    do k = 1,ksize
+                       fieldx(i,j,k) = 0.
+                       fieldy(i,j,k) = 0.
+                    end do
+                 end do
+              end if
+           end do
+        endif
+        ! the following code code block correct an error where the data in your halo coming from
+        ! other half may have the wrong sign
+        !off west edge, when update north or west direction
+        j = domain%y(1)%global%end+shift
+        if ( recv_v(7) .OR. recv_v(5) ) then
+           select case(gridtype)
+           case(BGRID_NE)
+              if(domain%symmetry) then
+                 is = domain%x(1)%global%begin
+              else
+                 is = domain%x(1)%global%begin - 1
+              end if
+              if( is.GT.domain%x(1)%data%begin )then
+
+                 if( 2*is-domain%x(1)%data%begin.GT.domain%x(1)%data%end+shift ) &
+                      call mpp_error( FATAL, 'MPP_DO_UPDATE_V: folded-north BGRID_NE west edge ubound error.' )
+                 do l=1,nvector
+                    ptr_fieldx = group%addrs_x(l)
+                    ptr_fieldy = group%addrs_y(l)
+                    do k = 1,ksize
+                       do i = domain%x(1)%data%begin,is-1
+                          fieldx(i,j,k) = fieldx(2*is-i,j,k)
+                          fieldy(i,j,k) = fieldy(2*is-i,j,k)
+                       end do
+                    end do
+                 end do
+              end if
+           case(CGRID_NE)
+              is = domain%x(1)%global%begin
+              if( is.GT.domain%x(1)%data%begin )then
+                 if( 2*is-domain%x(1)%data%begin-1.GT.domain%x(1)%data%end ) &
+                      call mpp_error( FATAL, 'MPP_DO_UPDATE_V: folded-north CGRID_NE west edge ubound error.' )
+                 do l=1,nvector
+                    ptr_fieldy = group%addrs_y(l)
+                    do k = 1,ksize
+                       do i = domain%x(1)%data%begin,is-1
+                          fieldy(i,j,k) = fieldy(2*is-i-1,j,k)
+                       end do
+                    end do
+                 end do
+              end if
+           end select
+        end if
+        !off east edge
+        is = domain%x(1)%global%end
+        if(domain%x(1)%cyclic .AND. is.LT.domain%x(1)%data%end )then
+           ie = domain%x(1)%data%end
+           is = is + 1
+           select case(gridtype)
+           case(BGRID_NE)
+              is = is + shift
+              ie = ie + shift
+              do l=1,nvector
+                 ptr_fieldx = group%addrs_x(l)
+                 ptr_fieldy = group%addrs_y(l)
+                 do k = 1,ksize
+                    do i = is,ie
+                       fieldx(i,j,k) = -fieldx(i,j,k)
+                       fieldy(i,j,k) = -fieldy(i,j,k)
+                    end do
+                 end do
+              end do
+           case(CGRID_NE)
+              do l=1,nvector
+                 ptr_fieldy = group%addrs_y(l)
+                 do k = 1,ksize
+                    do i = is, ie
+                       fieldy(i,j,k) = -fieldy(i,j,k)
+                    end do
+                 end do
+              end do
+           end select
+        end if
+     end if
+  else if( BTEST(domain%fold,SOUTH) .OR. BTEST(domain%fold,WEST) .OR. BTEST(domain%fold,EAST) ) then
+     call mpp_error(FATAL, "MPP_DO_GROUP_UPDATE: this interface does not support folded_south, " // &
+          "folded_west of folded_east, contact developer")
+  endif
+
+  if(nsend>0) then
+     call mpp_clock_begin(group_wait_clock)
+     call mpp_sync_self( )
+     call mpp_clock_end(group_wait_clock)
+  endif
+
+end subroutine MPP_DO_GROUP_UPDATE_
+
+
+subroutine MPP_START_GROUP_UPDATE_(group, domain, d_type, reuse_buffer)
+  type(mpp_group_update_type), intent(inout) :: group
+  type(domain2D),              intent(inout) :: domain  
+  MPP_TYPE_,                   intent(in)    :: d_type
+  logical,  optional,          intent(in)    :: reuse_buffer
+
+  integer   :: nscalar, nvector
+  integer   :: nsend, nrecv, flags_v
+  integer   :: msgsize, npack, rotation
+  integer   :: from_pe, to_pe, buffer_pos, pos
+  integer   :: ksize, is, ie, js, je
+  integer   :: n, l, m, i, j, k, buffer_start_pos, nk
+  logical   :: reuse_buf_pos
+  character(len=8)            :: text
+
+  MPP_TYPE_ :: buffer(size(mpp_domains_stack_nonblock(:)))
+  MPP_TYPE_ :: field (group%is_s:group%ie_s,group%js_s:group%je_s, group%ksize_s)
+  MPP_TYPE_ :: fieldx(group%is_x:group%ie_x,group%js_x:group%je_x, group%ksize_v)
+  MPP_TYPE_ :: fieldy(group%is_y:group%ie_y,group%js_y:group%je_y, group%ksize_v)
+  pointer( ptr, buffer )
+  pointer(ptr_field, field)
+  pointer(ptr_fieldx, fieldx)
+  pointer(ptr_fieldy, fieldy)
+
+  nscalar = group%nscalar
+  nvector = group%nvector
+
+  if(nscalar>0) then
+     ksize = group%ksize_s
+  else
+     ksize = group%ksize_v
+  endif
+
+  !--- set reset_index_s and reset_index_v to 0 
+  group%reset_index_s = 0
+  group%reset_index_v = 0
+
+  reuse_buf_pos = .FALSE.
+  if (PRESENT(reuse_buffer)) reuse_buf_pos = reuse_buffer
+
+  if (.not. group%initialized) then
+    call set_group_update(group,domain)
+  endif
+
+  if (.not. reuse_buf_pos) then
+     group%buffer_start_pos = nonblock_group_buffer_pos
+     nonblock_group_buffer_pos = nonblock_group_buffer_pos + group%tot_msgsize
+     mpp_domains_stack_hwm = nonblock_group_buffer_pos + 1    
+     if( mpp_domains_stack_hwm .GT. mpp_domains_stack_size )then
+        write( text,'(i8)' )mpp_domains_stack_hwm
+        call mpp_error( FATAL, 'set_group_update: mpp_domains_stack overflow, '// &
+                     'call mpp_domains_set_stack_size('//trim(text)//') from all PEs.' )
+     end if
+
+  else if( group%buffer_start_pos < 0 ) then
+     call mpp_error(FATAL, "MPP_START_GROUP_UPDATE: group%buffer_start_pos is not set")
+  endif
+
+  nrecv = group%nrecv
+  nsend = group%nsend
+
+  ptr = LOC(mpp_domains_stack_nonblock)
+
+  ! Make sure it is not in the middle of the old version of non-blocking halo update.
+  if(num_update>0) call mpp_error(FATAL, "MPP_START_GROUP_UPDATE: can not be called in the middle of "// &
+              "mpp_start_update_domains/mpp_complete_update_domains call")
+
+  num_nonblock_group_update = num_nonblock_group_update + 1
+
+  !---pre-post receive.
+  call mpp_clock_begin(nonblock_group_recv_clock)
+  do m = 1, nrecv
+     msgsize = group%recv_size(m)
+     from_pe = group%from_pe(m)
+     if( msgsize .GT. 0 )then
+        buffer_pos = group%buffer_pos_recv(m) + group%buffer_start_pos
+        call mpp_recv( buffer(buffer_pos+1), glen=msgsize, from_pe=from_pe, block=.false., &
+             tag=COMM_TAG_1, request=group%request_recv(m))
+#ifdef use_libMPI
+        group%type_recv(m) = MPI_TYPE_
+#endif
+     end if
+  end do
+  call mpp_clock_end(nonblock_group_recv_clock)
+
+  flags_v = group%flags_v
+
+  !pack the data
+  call mpp_clock_begin(nonblock_group_pack_clock)
+  npack = group%npack
+  buffer_start_pos = group%buffer_start_pos
+#include 
+  call mpp_clock_end(nonblock_group_pack_clock)
+
+  call mpp_clock_begin(nonblock_group_send_clock)
+  do n = 1, nsend  
+     msgsize = group%send_size(n)
+     if( msgsize .GT. 0 )then
+        buffer_pos = group%buffer_pos_send(n) + group%buffer_start_pos
+        to_pe = group%to_pe(n)
+        call mpp_send( buffer(buffer_pos+1), plen=msgsize, to_pe=to_pe, tag=COMM_TAG_1, &
+                       request=group%request_send(n))
+     endif
+  enddo
+  call mpp_clock_end(nonblock_group_send_clock)
+
+end subroutine MPP_START_GROUP_UPDATE_
+
+subroutine MPP_COMPLETE_GROUP_UPDATE_(group, domain, d_type)
+  type(mpp_group_update_type), intent(inout) :: group
+  type(domain2D),              intent(inout) :: domain  
+  MPP_TYPE_,                   intent(in)    :: d_type
+
+  integer   :: nsend, nrecv, nscalar, nvector
+  integer   :: k, buffer_pos, msgsize, pos, m, n, l
+  integer   :: is, ie, js, je, dir, ksize, i, j
+  integer   :: shift, gridtype, midpoint, flags_v
+  integer   :: nunpack, rotation, buffer_start_pos, nk
+  logical   :: recv_v(8)
+  MPP_TYPE_ :: buffer(size(mpp_domains_stack_nonblock(:)))
+  MPP_TYPE_ :: field (group%is_s:group%ie_s,group%js_s:group%je_s, group%ksize_s)
+  MPP_TYPE_ :: fieldx(group%is_x:group%ie_x,group%js_x:group%je_x, group%ksize_v)
+  MPP_TYPE_ :: fieldy(group%is_y:group%ie_y,group%js_y:group%je_y, group%ksize_v)
+  pointer(ptr, buffer )
+  pointer(ptr_field, field)
+  pointer(ptr_fieldx, fieldx)
+  pointer(ptr_fieldy, fieldy)
+
+  gridtype = group%gridtype
+  flags_v = group%flags_v
+  nscalar = group%nscalar
+  nvector = group%nvector
+  nrecv = group%nrecv
+  nsend = group%nsend
+  if(nscalar>0) then
+     ksize = group%ksize_s
+  else
+     ksize = group%ksize_v
+  endif
+  if(nvector > 0) recv_v = group%recv_v
+  ptr = LOC(mpp_domains_stack_nonblock)
+
+  if(num_nonblock_group_update < 1) call mpp_error(FATAL, &
+    'mpp_start_group_update must be called before calling mpp_end_group_update')  
+  num_nonblock_group_update = num_nonblock_group_update - 1
+  complete_group_update_on = .true.
+
+  if(nrecv>0) then
+     call mpp_clock_begin(nonblock_group_wait_clock)
+     call mpp_sync_self(check=EVENT_RECV, request=group%request_recv(1:nrecv), &
+                        msg_size=group%recv_size(1:nrecv), msg_type=group%type_recv(1:nrecv))
+     call mpp_clock_end(nonblock_group_wait_clock)
+  endif
+
+  !---unpack the buffer
+  nunpack = group%nunpack
+
+  call mpp_clock_begin(nonblock_group_unpk_clock)
+  buffer_start_pos = group%buffer_start_pos
+#include 
+  call mpp_clock_end(nonblock_group_unpk_clock)
+
+  ! ---northern boundary fold
+  shift = 0
+  if(domain%symmetry) shift = 1
+  if( nvector >0 .AND. BTEST(domain%fold,NORTH) .AND. (.NOT.BTEST(flags_v,SCALAR_BIT)) )then
+     j = domain%y(1)%global%end+shift
+     if( domain%y(1)%data%begin.LE.j .AND. j.LE.domain%y(1)%data%end+shift )then !fold is within domain
+        !poles set to 0: BGRID only
+        if( gridtype.EQ.BGRID_NE )then
+           midpoint = (domain%x(1)%global%begin+domain%x(1)%global%end-1+shift)/2
+           j  = domain%y(1)%global%end+shift
+           is = domain%x(1)%global%begin; ie = domain%x(1)%global%end+shift
+           if( .NOT. domain%symmetry ) is = is - 1
+           do i = is ,ie, midpoint
+              if( domain%x(1)%data%begin.LE.i .AND. i.LE. domain%x(1)%data%end+shift )then
+                 do l=1,nvector
+                    ptr_fieldx = group%addrs_x(l)
+                    ptr_fieldy = group%addrs_y(l)
+                    do k = 1,ksize
+                       fieldx(i,j,k) = 0.
+                       fieldy(i,j,k) = 0.
+                    end do
+                 end do
+              end if
+           end do
+        endif
+        ! the following code code block correct an error where the data in your halo coming from
+        ! other half may have the wrong sign
+        !off west edge, when update north or west direction
+        j = domain%y(1)%global%end+shift
+        if ( recv_v(7) .OR. recv_v(5) ) then
+           select case(gridtype)
+           case(BGRID_NE)
+              if(domain%symmetry) then
+                 is = domain%x(1)%global%begin
+              else
+                 is = domain%x(1)%global%begin - 1
+              end if
+              if( is.GT.domain%x(1)%data%begin )then
+
+                 if( 2*is-domain%x(1)%data%begin.GT.domain%x(1)%data%end+shift ) &
+                      call mpp_error( FATAL, 'MPP_DO_UPDATE_V: folded-north BGRID_NE west edge ubound error.' )
+                 do l=1,nvector
+                    ptr_fieldx = group%addrs_x(l)
+                    ptr_fieldy = group%addrs_y(l)
+                    do k = 1,ksize
+                       do i = domain%x(1)%data%begin,is-1
+                          fieldx(i,j,k) = fieldx(2*is-i,j,k)
+                          fieldy(i,j,k) = fieldy(2*is-i,j,k)
+                       end do
+                    end do
+                 end do
+              end if
+           case(CGRID_NE)
+              is = domain%x(1)%global%begin
+              if( is.GT.domain%x(1)%data%begin )then
+                 if( 2*is-domain%x(1)%data%begin-1.GT.domain%x(1)%data%end ) &
+                      call mpp_error( FATAL, 'MPP_DO_UPDATE_V: folded-north CGRID_NE west edge ubound error.' )
+                 do l=1,nvector
+                    ptr_fieldy = group%addrs_y(l)
+                    do k = 1,ksize
+                       do i = domain%x(1)%data%begin,is-1
+                          fieldy(i,j,k) = fieldy(2*is-i-1,j,k)
+                       end do
+                    end do
+                 end do
+              end if
+           end select
+        end if
+        !off east edge
+        is = domain%x(1)%global%end
+        if(domain%x(1)%cyclic .AND. is.LT.domain%x(1)%data%end )then
+           ie = domain%x(1)%data%end
+           is = is + 1
+           select case(gridtype)
+           case(BGRID_NE)
+              is = is + shift
+              ie = ie + shift
+              do l=1,nvector
+                 ptr_fieldx = group%addrs_x(l)
+                 ptr_fieldy = group%addrs_y(l)
+                 do k = 1,ksize
+                    do i = is,ie
+                       fieldx(i,j,k) = -fieldx(i,j,k)
+                       fieldy(i,j,k) = -fieldy(i,j,k)
+                    end do
+                 end do
+              end do
+           case(CGRID_NE)
+              do l=1,nvector
+                 ptr_fieldy = group%addrs_y(l)
+                 do k = 1,ksize
+                    do i = is, ie
+                       fieldy(i,j,k) = -fieldy(i,j,k)
+                    end do
+                 end do
+              end do
+           end select
+        end if
+     end if
+  else if( BTEST(domain%fold,SOUTH) .OR. BTEST(domain%fold,WEST) .OR. BTEST(domain%fold,EAST) ) then
+     call mpp_error(FATAL, "MPP_COMPLETE_GROUP_UPDATE: this interface does not support folded_south, " // &
+          "folded_west of folded_east, contact developer")
+  endif
+
+  if(nsend>0) then
+     call mpp_clock_begin(nonblock_group_wait_clock)
+     call mpp_sync_self(check=EVENT_SEND, request=group%request_send(1:nsend) )
+     call mpp_clock_end(nonblock_group_wait_clock)
+  endif
+
+  if( num_nonblock_group_update == 0) then
+     nonblock_group_buffer_pos   = 0
+  endif
+
+end subroutine MPP_COMPLETE_GROUP_UPDATE_
+
+subroutine MPP_RESET_GROUP_UPDATE_FIELD_2D_(group, field)
+  type(mpp_group_update_type), intent(inout) :: group
+  MPP_TYPE_,                   intent(in)    :: field(:,:)
+
+  group%reset_index_s = group%reset_index_s + 1
+
+  if(group%reset_index_s > group%nscalar) &
+     call mpp_error(FATAL, "MPP_RESET_GROUP_UPDATE_FIELD_2D_: group%reset_index_s > group%nscalar")
+  if(size(field,1) .NE. group%isize_s .OR. size(field,2) .NE. group%jsize_s .OR. group%ksize_s .NE. 1) &
+     call mpp_error(FATAL, "MPP_RESET_GROUP_UPDATE_FIELD_2D_: size of field does not match the size stored in group") 
+
+  group%addrs_s(group%reset_index_s) = LOC(field)
+
+end subroutine MPP_RESET_GROUP_UPDATE_FIELD_2D_
+
+subroutine MPP_RESET_GROUP_UPDATE_FIELD_3D_(group, field)
+  type(mpp_group_update_type), intent(inout) :: group
+  MPP_TYPE_,                   intent(in)    :: field(:,:,:)
+
+  group%reset_index_s = group%reset_index_s + 1
+
+  if(group%reset_index_s > group%nscalar) &
+     call mpp_error(FATAL, "MPP_RESET_GROUP_UPDATE_FIELD_3D_: group%reset_index_s > group%nscalar")
+  if(size(field,1) .NE. group%isize_s .OR. size(field,2) .NE. group%jsize_s .OR. size(field,3) .NE. group%ksize_s) &
+     call mpp_error(FATAL, "MPP_RESET_GROUP_UPDATE_FIELD_3D_: size of field does not match the size stored in group") 
+
+  group%addrs_s(group%reset_index_s) = LOC(field)
+
+end subroutine MPP_RESET_GROUP_UPDATE_FIELD_3D_
+
+subroutine MPP_RESET_GROUP_UPDATE_FIELD_4D_(group, field)
+  type(mpp_group_update_type), intent(inout) :: group
+  MPP_TYPE_,                   intent(in)    :: field(:,:,:,:)
+
+  group%reset_index_s = group%reset_index_s + 1
+
+  if(group%reset_index_s > group%nscalar) &
+     call mpp_error(FATAL, "MPP_RESET_GROUP_UPDATE_FIELD_4D_: group%reset_index_s > group%nscalar")
+  if(size(field,1) .NE. group%isize_s .OR. size(field,2) .NE. group%jsize_s .OR. &
+              size(field,3)*size(field,4) .NE. group%ksize_s) &
+     call mpp_error(FATAL, "MPP_RESET_GROUP_UPDATE_FIELD_4D_: size of field does not match the size stored in group") 
+
+  group%addrs_s(group%reset_index_s) = LOC(field)
+
+end subroutine MPP_RESET_GROUP_UPDATE_FIELD_4D_
+
+
+subroutine MPP_RESET_GROUP_UPDATE_FIELD_2D_V_(group, fieldx, fieldy)
+  type(mpp_group_update_type), intent(inout) :: group
+  MPP_TYPE_,                   intent(in)    :: fieldx(:,:), fieldy(:,:)
+  integer :: indx
+
+  group%reset_index_v = group%reset_index_v + 1
+
+  if(group%reset_index_v > group%nvector) &
+     call mpp_error(FATAL, "MPP_RESET_GROUP_UPDATE_FIELD_2D_V_: group%reset_index_v > group%nvector")
+  if(size(fieldx,1) .NE. group%isize_x .OR. size(fieldx,2) .NE. group%jsize_x .OR. group%ksize_v .NE. 1) &
+     call mpp_error(FATAL, "MPP_RESET_GROUP_UPDATE_FIELD_2D_V_: size of fieldx does not match the size stored in group") 
+  if(size(fieldy,1) .NE. group%isize_y .OR. size(fieldy,2) .NE. group%jsize_y ) &
+     call mpp_error(FATAL, "MPP_RESET_GROUP_UPDATE_FIELD_2D_V_: size of fieldy does not match the size stored in group") 
+
+  group%addrs_x(group%reset_index_v) = LOC(fieldx)
+  group%addrs_y(group%reset_index_v) = LOC(fieldy)
+
+end subroutine MPP_RESET_GROUP_UPDATE_FIELD_2D_V_
+
+
+subroutine MPP_RESET_GROUP_UPDATE_FIELD_3D_V_(group, fieldx, fieldy)
+  type(mpp_group_update_type), intent(inout) :: group
+  MPP_TYPE_,                   intent(in)    :: fieldx(:,:,:), fieldy(:,:,:)
+  integer :: indx
+
+  group%reset_index_v = group%reset_index_v + 1
+
+  if(group%reset_index_v > group%nvector) &
+     call mpp_error(FATAL, "MPP_RESET_GROUP_UPDATE_FIELD_3D_V_: group%reset_index_v > group%nvector")
+  if(size(fieldx,1) .NE. group%isize_x .OR. size(fieldx,2) .NE. group%jsize_x .OR. size(fieldx,3) .NE. group%ksize_v) &
+     call mpp_error(FATAL, "MPP_RESET_GROUP_UPDATE_FIELD_3D_V_: size of fieldx does not match the size stored in group") 
+  if(size(fieldy,1) .NE. group%isize_y .OR. size(fieldy,2) .NE. group%jsize_y .OR. size(fieldy,3) .NE. group%ksize_v) &
+     call mpp_error(FATAL, "MPP_RESET_GROUP_UPDATE_FIELD_3D_V_: size of fieldy does not match the size stored in group") 
+
+  group%addrs_x(group%reset_index_v) = LOC(fieldx)
+  group%addrs_y(group%reset_index_v) = LOC(fieldy)
+
+end subroutine MPP_RESET_GROUP_UPDATE_FIELD_3D_V_
+
+
+subroutine MPP_RESET_GROUP_UPDATE_FIELD_4D_V_(group, fieldx, fieldy)
+  type(mpp_group_update_type), intent(inout) :: group
+  MPP_TYPE_,                   intent(in)    :: fieldx(:,:,:,:), fieldy(:,:,:,:)
+  integer :: indx
+
+  group%reset_index_v = group%reset_index_v + 1
+
+  if(group%reset_index_v > group%nvector) &
+     call mpp_error(FATAL, "MPP_RESET_GROUP_UPDATE_FIELD_4D_V_: group%reset_index_v > group%nvector")
+  if(size(fieldx,1) .NE. group%isize_x .OR. size(fieldx,2) .NE. group%jsize_x .OR. &
+              size(fieldx,3)*size(fieldx,4) .NE. group%ksize_v) &
+     call mpp_error(FATAL, "MPP_RESET_GROUP_UPDATE_FIELD_4D_V_: size of fieldx does not match the size stored in group") 
+  if(size(fieldy,1) .NE. group%isize_y .OR. size(fieldy,2) .NE. group%jsize_y .OR. &
+              size(fieldy,3)*size(fieldy,4) .NE. group%ksize_v) &
+     call mpp_error(FATAL, "MPP_RESET_GROUP_UPDATE_FIELD_4D_V_: size of fieldy does not match the size stored in group") 
+
+  group%addrs_x(group%reset_index_v) = LOC(fieldx)
+  group%addrs_y(group%reset_index_v) = LOC(fieldy)
+
+end subroutine MPP_RESET_GROUP_UPDATE_FIELD_4D_V_
+
+
diff --git a/src/shared/mpp/include/mpp_io_connect.inc b/src/shared/mpp/include/mpp_io_connect.inc
index 0a05aedf1e..5db528c014 100644
--- a/src/shared/mpp/include/mpp_io_connect.inc
+++ b/src/shared/mpp/include/mpp_io_connect.inc
@@ -1,6 +1,6 @@
 
 ! -*-f90-*-
-! $Id: mpp_io_connect.inc,v 19.0 2012/01/06 22:03:20 fms Exp $
+! $Id$
 
 ! 
 
@@ -176,10 +176,11 @@
 
       character(len=16)  :: act, acc, for, pos
       character(len=128) :: mesg
+      character(len=256) :: text2
       integer :: action_flag, form_flag, access_flag, threading_flag, fileset_flag, length
       integer :: nfiles, tile_id(1), io_layout(2)
       logical :: exists, on_root_pe, dist_file
-      logical :: write_on_this_pe, io_domain_exist
+      logical :: write_on_this_pe, read_on_this_pe, io_domain_exist
       integer :: ios, nc_pos     !position of .nc in file name
       type(axistype) :: unlim    !used by netCDF with mpp_append
       type(domain2d), pointer :: io_domain=>NULL()
@@ -225,20 +226,21 @@
          if(associated(io_domain)) io_domain_exist = .true.
       endif
 
-      !--- when io_domain_exist, will ignore the threading_flag and fileset_flag.
-      if(io_domain_exist .AND. PRESENT(fileset) ) then
-         call mpp_error(NOTE, "mpp_io_connect.inc(mpp_open): io_domain exists for domain "// &
-            trim(mpp_get_domain_name(domain))//", optional argument fileset will be ignored")
-      endif
-
       write_on_this_pe = .true.
-      if( threading_flag.EQ.MPP_SINGLE .AND. .NOT. on_root_pe ) write_on_this_pe = .false.
+      read_on_this_pe = .true.
+      if( threading_flag.EQ.MPP_SINGLE .AND. .NOT.on_root_pe ) then
+         write_on_this_pe = .false.
+         read_on_this_pe = .false.
+      endif
       if(form_flag == MPP_NETCDF .AND. action_flag .NE. MPP_RDONLY) then
          if(fileset_flag .EQ.MPP_SINGLE .AND.  threading_flag.EQ.MPP_MULTI) then
             call mpp_error(FATAL, "mpp_io_connect.inc(mpp_open): multiple thread and single "// &
                 "file writing/appending is not supported for netCDF file")
          endif
-         if( fileset_flag  .EQ.MPP_SINGLE .AND. .NOT. on_root_pe ) write_on_this_pe = .false.
+         if( fileset_flag.EQ.MPP_SINGLE .AND. .NOT.on_root_pe ) then
+            write_on_this_pe = .false.
+            read_on_this_pe = .false.
+         endif   
       endif   
    
       if( io_domain_exist) then
@@ -272,6 +274,7 @@
       end if
       mpp_file(unit)%valid            = .true.
       mpp_file(unit)%write_on_this_pe = write_on_this_pe
+      mpp_file(unit)%read_on_this_pe  = read_on_this_pe
       mpp_file(unit)%io_domain_exist  = io_domain_exist
       if( PRESENT(domain) ) then
          allocate(mpp_file(unit)%domain)
@@ -291,10 +294,18 @@
             fileset_flag = MPP_MULTI
             threading_flag = MPP_MULTI
             tile_id = mpp_get_tile_id(io_domain)
-            if(mpp_npes() > 10000) then 
-               write( text,'(a,i6.6)' )trim(text)//'.', tile_id(1)
-            else
-               write( text,'(a,i4.4)' )trim(text)//'.', tile_id(1)
+            text2 = trim(text)
+            if(tile_id(1) .GE. 10000) call mpp_error(FATAL, 'mpp_open: tile_id should be less than 10000 when io_domain exist')
+            write( text,'(a,i4.4)' )trim(text)//'.', tile_id(1)
+            if( action_flag == MPP_RDONLY ) then
+               inquire(file=trim(text),EXIST=exists)
+               if(.not. exists) then
+                  write( text2,'(a,i6.6)' )trim(text2)//'.', tile_id(1)
+                  inquire(file=trim(text2),EXIST=exists)
+                  if(.not.exists) call mpp_error(FATAL, 'mpp_open: neither '// &
+                                  trim(text)//' nor '//trim(text2)//' exist and io domain exist')
+                  text = trim(text2)
+               endif
             endif
          else
             fileset_flag   = MPP_SINGLE
@@ -502,7 +513,7 @@
               if( verbose )print '(a,i6,i16,i4)', 'MPP_OPEN: append to existing netCDF file: pe, ncid, time_axis_id=',&
                    pe, mpp_file(unit)%ncid, mpp_file(unit)%id
               mpp_file(unit)%format=form_flag ! need this for mpp_read
-              call mpp_read_meta(unit)
+!              call mpp_read_meta(unit)
           else if( action_flag.EQ.MPP_RDONLY )then
                inquire(file=trim(mpp_file(unit)%name),EXIST=exists)
               if (.NOT.exists) call mpp_error(FATAL,'MPP_OPEN:'&
@@ -542,7 +553,7 @@
               if( verbose )print '(a,i6,i16,i4)', 'MPP_OPEN: append to existing netCDF file: pe, ncid, time_axis_id=',&
                    pe, mpp_file(unit)%ncid, mpp_file(unit)%id
               mpp_file(unit)%format=form_flag ! need this for mpp_read
-              call mpp_read_meta(unit)
+!              call mpp_read_meta(unit)
           else if( action_flag.EQ.MPP_RDONLY )then
                inquire(file=trim(mpp_file(unit)%name),EXIST=exists)
               if (.NOT.exists) call mpp_error(FATAL,'MPP_OPEN:'&
@@ -582,7 +593,7 @@
               if( verbose )print '(a,i6,i16,i4)', 'MPP_OPEN: append to existing netCDF file: pe, ncid, time_axis_id=',&
                    pe, mpp_file(unit)%ncid, mpp_file(unit)%id
               mpp_file(unit)%format=form_flag ! need this for mpp_read
-              call mpp_read_meta(unit)
+!              call mpp_read_meta(unit)
           else if( action_flag.EQ.MPP_RDONLY )then
                inquire(file=trim(mpp_file(unit)%name),EXIST=exists)
               if (.NOT.exists) call mpp_error(FATAL,'MPP_OPEN:'&
diff --git a/src/shared/mpp/include/mpp_io_misc.inc b/src/shared/mpp/include/mpp_io_misc.inc
index e2e6bf7dfc..aef7377768 100644
--- a/src/shared/mpp/include/mpp_io_misc.inc
+++ b/src/shared/mpp/include/mpp_io_misc.inc
@@ -1,6 +1,6 @@
 
 ! -*-f90-*-
-! $Id: mpp_io_misc.inc,v 20.0 2013/12/14 00:26:36 fms Exp $
+! $Id$
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 !                                                                      !
@@ -96,11 +96,12 @@
       default_field%type = -1
       default_field%natt = -1
       default_field%ndim = -1
+      default_field%checksum = 0
 !largest possible 4-byte reals
       default_field%min = -huge(1._4)
       default_field%max =  huge(1._4)
-      default_field%missing = -1e36
-      default_field%fill = -1e36
+      default_field%missing = MPP_FILL_DOUBLE ! now using netcdf:NF_FILL_DOUBLE instead of -1e36
+      default_field%fill = MPP_FILL_DOUBLE ! now using netcdf:NF_FILL_DOUBLE instead of -1e36
       default_field%scale = 1.0
       default_field%add = 0.0
       default_field%pack = 1
@@ -110,6 +111,7 @@
       default_axis%units = 'nounits'
       default_axis%longname = 'noname'
       default_axis%cartesian = 'none'
+      default_axis%compressed = 'unspecified'
       default_axis%calendar = 'unspecified'
       default_axis%sense = 0
       default_axis%len = -1
diff --git a/src/shared/mpp/include/mpp_io_read.inc b/src/shared/mpp/include/mpp_io_read.inc
index 770faa5eeb..1218965e8b 100644
--- a/src/shared/mpp/include/mpp_io_read.inc
+++ b/src/shared/mpp/include/mpp_io_read.inc
@@ -1,6 +1,6 @@
 
 ! -*-f90-*-
-! $Id: mpp_io_read.inc,v 20.0 2013/12/14 00:26:38 fms Exp $
+! $Id$
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 !                                                                      !
@@ -12,10 +12,22 @@
 #define MPP_READ_2DDECOMP_2D_ mpp_read_2ddecomp_r2d
 #undef MPP_READ_2DDECOMP_3D_
 #define MPP_READ_2DDECOMP_3D_ mpp_read_2ddecomp_r3d
+#undef MPP_READ_2DDECOMP_4D_
+#define MPP_READ_2DDECOMP_4D_ mpp_read_2ddecomp_r4d
 #undef MPP_TYPE_
 #define MPP_TYPE_ real
 #include 
 
+#undef MPP_READ_COMPRESSED_1D_
+#define MPP_READ_COMPRESSED_1D_ mpp_read_compressed_r1d
+#undef MPP_READ_COMPRESSED_2D_
+#define MPP_READ_COMPRESSED_2D_ mpp_read_compressed_r2d
+#undef MPP_TYPE_
+#define MPP_TYPE_ real
+#include 
+
+#include 
+
     subroutine read_record_core(unit, field, nwords, data, start, axsiz)
       integer,         intent(in)    :: unit
       type(fieldtype), intent(in)    :: field
@@ -113,7 +125,7 @@
     end subroutine read_record_core
 
 
-    subroutine read_record( unit, field, nwords, data, time_level, domain, position, tile_count )
+    subroutine read_record( unit, field, nwords, data, time_level, domain, position, tile_count, start_in, axsiz_in )
 !routine that is finally called by all mpp_read routines to perform the read
 !a non-netCDF record contains:
 !      field ID
@@ -136,6 +148,7 @@
       integer,        intent(in),    optional :: time_level
       type(domain2D), intent(in),    optional :: domain
       integer,        intent(in),    optional :: position, tile_count
+      integer,        intent(in),    optional :: start_in(:), axsiz_in(:)
       integer, dimension(size(field%axes(:))) :: start, axsiz
 
       integer :: tlevel !,subdomain(4)
@@ -152,14 +165,25 @@
 
       if( .NOT.module_is_initialized )call mpp_error( FATAL, 'READ_RECORD: must first call mpp_io_init.' )
       if( .NOT.mpp_file(unit)%opened )call mpp_error( FATAL, 'READ_RECORD: invalid unit number.' )
-      if( mpp_file(unit)%threading.EQ.MPP_SINGLE .AND. pe.NE.mpp_root_pe() )return
+      if( .NOT.mpp_file(unit)%read_on_this_pe )return
 
       if( .NOT.mpp_file(unit)%initialized ) call mpp_error( FATAL, 'MPP_READ: must first call mpp_read_meta.' )
+      if( mpp_file(unit)%format .NE. MPP_NETCDF ) call mpp_error( FATAL, 'Currently dont support non-NetCDF mpp read' )
 
-      if( verbose )print '(a,2i3,2i5)', 'MPP_READ: PE, unit, %id, %time_level =',&
-           pe, unit, mpp_file(unit)%id, tlevel
+      if (.not.PRESENT(time_level)) then
+          tlevel = 0
+      else
+          tlevel = time_level
+      endif
 
-      if( mpp_file(unit)%format.EQ.MPP_NETCDF )then
+      if( verbose )print '(a,2i6,2i5)', 'MPP_READ: PE, unit, %id, %time_level =',&
+           pe, unit, mpp_file(unit)%id, tlevel
+      if( PRESENT(start_in) .AND. PRESENT(axsiz_in) ) then
+         if(size(start(:)) > size(start_in(:)) )call mpp_error( FATAL, 'MPP_READ: size(start_in) < size(start)')
+         if(size(axsiz(:)) > size(axsiz_in(:)) )call mpp_error( FATAL, 'MPP_READ: size(axsiz_in) < size(axsiz)')
+         start(:) = start_in(1:size(start(:)))
+         axsiz(:) = axsiz_in(1:size(axsiz(:)))
+      else
 !define netCDF data block to be read:
 !  time axis: START = time level
 !             AXSIZ = 1
@@ -216,19 +240,29 @@
                  io_domain => NULL()
               end if
           end if  
-              
-          if( verbose )print '(a,2i6,i6,12i4)', 'READ_RECORD: PE, unit, nwords, start, axsiz=', pe, unit, nwords, start, axsiz
+      endif        
+      if( verbose )print '(a,2i6,i6,12i4)', 'READ_RECORD: PE, unit, nwords, start, axsiz=', pe, unit, nwords, start, axsiz
 
-          call read_record_core(unit, field, nwords, data, start, axsiz)
-      else                      !non-netCDF
-!subdomain contains (/is,ie,js,je/)
-          call mpp_error( FATAL, 'Currently dont support non-NetCDF mpp read' )
-          
-      end if
+      call read_record_core(unit, field, nwords, data, start, axsiz)
 
       return
     end subroutine read_record
 
+! 
+!   
+!   
+!   
+!   
+! 
+    subroutine mpp_read_r4D( unit, field, data, tindex)
+      integer, intent(in) :: unit
+      type(fieldtype), intent(in) :: field
+      real, intent(inout) :: data(:,:,:,:)
+      integer, intent(in), optional :: tindex
+
+      call read_record( unit, field, size(data(:,:,:,:)), data, tindex )
+    end subroutine mpp_read_r4D
+
 
 ! 
 !   
@@ -339,7 +373,7 @@
       lev = 1
       if(present(level)) lev = level
 
-      if( verbose )print '(a,2i3,2i5)', 'MPP_READ: PE, unit, %id, level =', pe, unit, mpp_file(unit)%id, lev
+      if( verbose )print '(a,2i6,2i5)', 'MPP_READ: PE, unit, %id, level =', pe, unit, mpp_file(unit)%id, lev
 
       if( mpp_file(unit)%format.EQ.MPP_NETCDF )then
          start = 1
@@ -428,8 +462,8 @@
       integer :: error, i, j, istat, check_exist
       integer         :: type, nvdims, nvatts, dimid
       integer, allocatable, dimension(:) :: dimids
-      character(len=128) :: name, attname, unlimname, attval
-      logical :: isdim
+      character(len=128) :: name, attname, unlimname, attval, bounds_name
+      logical :: isdim, found_bounds
       integer(LONG_KIND) :: checksumf
       character(len=64)  :: checksum_char
       integer :: num_checksumf, last, is, k
@@ -438,7 +472,7 @@
       integer(INT_KIND), allocatable :: ivals(:)
       real(FLOAT_KIND), allocatable  :: rvals(:)
       real(DOUBLE_KIND), allocatable :: r8vals(:)
-      
+
 #ifdef use_netCDF
 
       if( mpp_file(unit)%format.EQ.MPP_NETCDF )then
@@ -542,6 +576,8 @@
                  call netcdf_err( error, mpp_file(unit), attr=mpp_file(unit)%att(i) )
                  if( verbose .and. pe == 0 )print *, 'GLOBAL ATT ',trim(name),' ',ivals(1:len)
                  mpp_file(unit)%Att(i)%fatt(1:len)=ivals(1:len)
+                 if(lowercase(trim(name)) == 'time_axis' .and. ivals(1)==0) &
+                        mpp_file(unit)%time_level = -1 ! This file is an unlimited axis restart
                  deallocate(ivals)
               case (NF_FLOAT)
                  allocate(mpp_file(unit)%Att(i)%fatt(len), STAT=istat)
@@ -697,8 +733,13 @@
                           call mpp_error(FATAL, "mpp_io_mod(mpp_read_meta): Unable to allocate temporary array rvals.  STAT = "&
                                & //trim(text))
                        end if
-                       error = NF_GET_VAR_REAL(ncid,i,rvals)
-                       call netcdf_err( error, mpp_file(unit), mpp_file(unit)%Axis(dimid) )
+                       !z1l read from root pe and broadcast to other processor. 
+                       !In the future we will modify the code if there is performance issue for very high MPI ranks.
+                       if(mpp_pe()==mpp_root_pe()) then
+                          error = NF_GET_VAR_REAL(ncid,i,rvals)
+                          call netcdf_err( error, mpp_file(unit), mpp_file(unit)%Axis(dimid) )
+                       endif
+                       call mpp_broadcast(rvals, len, mpp_root_pe())
                        mpp_file(unit)%time_values(1:len) = rvals(1:len)
                        deallocate(rvals)
                     case (NF_DOUBLE)
@@ -708,8 +749,13 @@
                           call mpp_error(FATAL, "mpp_io_mod(mpp_read_meta): Unable to allocate temporary array r8vals.  STAT = "&
                             & //trim(text))
                        end if
-                       error = NF_GET_VAR_DOUBLE(ncid,i,r8vals)
-                       call netcdf_err( error, mpp_file(unit), mpp_file(unit)%Axis(dimid) )
+                       !z1l read from root pe and broadcast to other processor. 
+                       !In the future we will modify the code if there is performance issue for very high MPI ranks.
+                       if(mpp_pe()==mpp_root_pe()) then
+                         error = NF_GET_VAR_DOUBLE(ncid,i,r8vals)
+                         call netcdf_err( error, mpp_file(unit), mpp_file(unit)%Axis(dimid) )
+                       endif
+                       call mpp_broadcast(r8vals, len, mpp_root_pe())
                        mpp_file(unit)%time_values(1:len) = r8vals(1:len)
                        deallocate(r8vals)
                     case default
@@ -869,6 +915,106 @@
               enddo
            endif
         enddo 
+
+        ! assign axis bounds
+        do j = 1, mpp_file(unit)%ndim
+           if(.not. associated(mpp_file(unit)%Axis(j)%data)) cycle
+           len = size(mpp_file(unit)%Axis(j)%data(:))
+           allocate(mpp_file(unit)%Axis(j)%data_bounds(len+1))
+           mpp_file(unit)%Axis(j)%name_bounds = 'none'
+           bounds_name = 'none'
+           found_bounds = .false.
+           do i = 1, mpp_file(unit)%Axis(j)%natt
+              if(trim(mpp_file(unit)%Axis(j)%Att(i)%name) == 'bounds' .OR. &
+                   trim(mpp_file(unit)%Axis(j)%Att(i)%name) == 'edges' ) then
+                 bounds_name = mpp_file(unit)%Axis(j)%Att(i)%catt
+                 found_bounds = .true.
+                 exit 
+              endif
+           enddo
+           !-- loop through all the fields to locate bounds_name
+           if( found_bounds ) then
+              found_bounds = .false.
+              do i = 1, mpp_file(unit)%ndim
+                 if(.not. associated(mpp_file(unit)%Axis(i)%data)) cycle
+                 if(trim(mpp_file(unit)%Axis(i)%name) == trim(bounds_name)) then
+                    found_bounds = .true.
+                    if(size(mpp_file(unit)%Axis(i)%data(:)) .NE. len+1) &
+                         call mpp_error(FATAL, "mpp_read_meta: improperly size bounds for field "// &
+                            trim(bounds_name)//" in file "// trim(mpp_file(unit)%name) )
+                    mpp_file(unit)%Axis(j)%data_bounds(:) = mpp_file(unit)%Axis(i)%data(:)
+                    exit
+                 endif
+              enddo
+              if( .not. found_bounds ) then
+                 do i=1, nvar_total
+                    error=NF_INQ_VAR(ncid,i,name,type,nvdims,dimids,nvatts);call netcdf_err( error, mpp_file(unit) )
+                    if(trim(name) == trim(bounds_name)) then
+                       found_bounds = .true.
+                       if(nvdims .NE. 2) &
+                            call mpp_error(FATAL, "mpp_read_meta: field "//trim(bounds_name)//" in file "//&
+                              trim(mpp_file(unit)%name)//" must be 2-D field")
+                       if(mpp_file(unit)%Axis(dimids(1))%len .NE. 2) &
+                            call mpp_error(FATAL, "mpp_read_meta: first dimension size of field "// &
+                            trim(mpp_file(unit)%Var(i)%name)//" from file "//trim(mpp_file(unit)%name)// &
+                            " must be 2")
+                       if(mpp_file(unit)%Axis(dimids(2))%len .NE. len) &
+                            call mpp_error(FATAL, "mpp_read_meta: second dimension size of field "// &
+                            trim(mpp_file(unit)%Var(i)%name)//" from file "//trim(mpp_file(unit)%name)// &
+                            " is not correct")
+                       select case (type)
+                       case (NF_INT)
+                          allocate(ivals(2*len), STAT=istat)
+                          if ( istat .ne. 0 ) then
+                             write(text,'(A)') istat
+                             call mpp_error(FATAL, "mpp_io_mod(mpp_read_meta): Unable to allocate array ivals."//&
+                                  "  STAT = "//trim(text))
+                          end if            
+                          error = NF_GET_VAR_INT(ncid,i,ivals)
+                          call netcdf_err( error, mpp_file(unit), string=" Field="//trim(bounds_name) )
+                          mpp_file(unit)%Axis(j)%data_bounds(1:len) =ivals(1:(2*len-1):2)
+                          mpp_file(unit)%Axis(j)%data_bounds(len+1) = ivals(2*len)
+                          deallocate(ivals)
+                       case (NF_FLOAT)
+                          allocate(rvals(2*len), STAT=istat)
+                          if ( istat .ne. 0 ) then
+                             write(text,'(A)') istat
+                             call mpp_error(FATAL, "mpp_io_mod(mpp_read_meta): Unable to allocate array rvals. "// &
+                                  "  STAT = "//trim(text))
+                          end if            
+                          error = NF_GET_VAR_REAL(ncid,i,rvals)
+                          call netcdf_err( error, mpp_file(unit), string=" Field="//trim(bounds_name) )
+                          mpp_file(unit)%Axis(j)%data_bounds(1:len) =rvals(1:(2*len-1):2)
+                          mpp_file(unit)%Axis(j)%data_bounds(len+1) = rvals(2*len)
+                          deallocate(rvals)
+                       case (NF_DOUBLE)
+                          allocate(r8vals(2*len), STAT=istat)
+                          if ( istat .ne. 0 ) then
+                             write(text,'(A)') istat
+                             call mpp_error(FATAL, "mpp_io_mod(mpp_read_meta): Unable to allocate array r8vals. "//&
+                                  "  STAT = "//trim(text))
+                          end if            
+                          error = NF_GET_VAR_DOUBLE(ncid,i,r8vals)
+                          call netcdf_err( error, mpp_file(unit), string=" Field="//trim(bounds_name) )
+                          mpp_file(unit)%Axis(j)%data_bounds(1:len) =r8vals(1:(2*len-1):2)
+                          mpp_file(unit)%Axis(j)%data_bounds(len+1) = r8vals(2*len)
+                          deallocate(r8vals)
+                       case default
+                          call mpp_error( FATAL, 'mpp_io_mod(mpp_read_meta): Invalid data type for dimension' )
+                       end select
+                       exit
+                    endif
+                 enddo
+              endif
+           endif
+           if (found_bounds) then
+              mpp_file(unit)%Axis(j)%name_bounds = trim(bounds_name)
+           else
+              deallocate(mpp_file(unit)%Axis(j)%data_bounds)
+              mpp_file(unit)%Axis(j)%data_bounds =>NULL()
+           endif
+        enddo
+
 ! assign variable info
         nv = 0
         do i=1, nvar_total
@@ -912,7 +1058,7 @@
               allocate(mpp_file(unit)%Var(nv)%size(nvdims))
               
               do j=1,nvdims
-                 if( dimids(j).eq.mpp_file(unit)%recdimid )then
+                 if(dimids(j).eq.mpp_file(unit)%recdimid  .and. mpp_file(unit)%time_level/=-1)then
                     mpp_file(unit)%Var(nv)%time_axis_index = dimids(j)
                     mpp_file(unit)%Var(nv)%size(j)=1    ! dimid length set to 1 here for consistency w/ mpp_write
                  else
@@ -1033,8 +1179,12 @@
                        mpp_file(unit)%Var(nv)%missing=mpp_file(unit)%Var(nv)%Att(j)%fatt(1)
                     case('missing_value')
                        mpp_file(unit)%Var(nv)%missing=mpp_file(unit)%Var(nv)%Att(j)%fatt(1)
+                    case('_FillValue')                       
+                       mpp_file(unit)%Var(nv)%fill=mpp_file(unit)%Var(nv)%Att(j)%fatt(1)
                     case('add_offset')
                        mpp_file(unit)%Var(nv)%add=mpp_file(unit)%Var(nv)%Att(j)%fatt(1)
+                    case('packing')
+                       mpp_file(unit)%Var(nv)%pack=mpp_file(unit)%Var(nv)%Att(j)%fatt(1)
                     case('valid_range')
                        mpp_file(unit)%Var(nv)%min=mpp_file(unit)%Var(nv)%Att(j)%fatt(1)
                        mpp_file(unit)%Var(nv)%max=mpp_file(unit)%Var(nv)%Att(j)%fatt(2)
@@ -1063,7 +1213,7 @@
       else 
         call mpp_error( FATAL,  'MPP READ CURRENTLY DOES NOT SUPPORT NON-NETCDF' )
       endif
-        
+
       mpp_file(unit)%initialized = .TRUE.
 #else 
       call mpp_error( FATAL, 'MPP_READ currently requires use_netCDF option' )
diff --git a/src/shared/mpp/include/mpp_io_util.inc b/src/shared/mpp/include/mpp_io_util.inc
index 7894248d4a..2031d7efab 100644
--- a/src/shared/mpp/include/mpp_io_util.inc
+++ b/src/shared/mpp/include/mpp_io_util.inc
@@ -1,6 +1,6 @@
 
 ! -*-f90-*-
-! $Id: mpp_io_util.inc,v 20.0 2013/12/14 00:26:40 fms Exp $
+! $Id$
 
 
   !#####################################################################
@@ -148,11 +148,13 @@
    end subroutine mpp_get_field_atts
 
   !#####################################################################
-   subroutine mpp_get_axis_atts( axis, name, units, longname, cartesian, calendar, sense, len, natts, atts )
+   subroutine mpp_get_axis_atts( axis, name, units, longname, cartesian, &
+                                 calendar, sense, len, natts, atts, compressed )
 
      type(axistype), intent(in) :: axis
      character(len=*), intent(out) , optional :: name, units
-     character(len=*), intent(out), optional :: longname, cartesian,calendar
+     character(len=*), intent(out), optional :: longname, cartesian
+     character(len=*), intent(out), optional :: compressed, calendar
      integer,intent(out), optional :: sense, len , natts
      type(atttype), intent(inout), optional, dimension(:) :: atts
 
@@ -162,6 +164,7 @@
      if (PRESENT(units)) units = axis%units
      if (PRESENT(longname)) longname = axis%longname
      if (PRESENT(cartesian)) cartesian = axis%cartesian
+     if (PRESENT(compressed)) compressed = axis%compressed
      if (PRESENT(calendar)) calendar = axis%calendar
      if (PRESENT(sense)) sense = axis%sense
      if (PRESENT(len)) len = axis%len
@@ -249,6 +252,34 @@
    end subroutine mpp_get_axes
 
   !#####################################################################
+   function mpp_get_dimension_length(unit, dimname, found)
+     integer,           intent(in)  :: unit
+     character(len=*),  intent(in)  :: dimname
+     logical, optional, intent(out) :: found
+     integer                        :: mpp_get_dimension_length
+     logical                        :: found_dim
+     integer                        :: i
+
+
+     if( .NOT.module_is_initialized ) &
+       call mpp_error( FATAL, 'mpp_get_dimension_length: must first call mpp_io_init.' )
+     if( .NOT.mpp_file(unit)%opened )&
+       call mpp_error( FATAL, 'mpp_get_dimension_length: invalid unit number, file '//trim(mpp_file(unit)%name))
+     found_dim = .false.
+     mpp_get_dimension_length = -1
+     do i = 1, mpp_file(unit)%ndim
+        if(trim(dimname) == trim(mpp_file(unit)%Axis(i)%name)) then
+          mpp_get_dimension_length = mpp_file(unit)%Axis(i)%len
+          found_dim = .true.
+          exit
+        endif
+     enddo
+
+     if(present(found)) found = found_dim
+
+   end function mpp_get_dimension_length
+
+   !#####################################################################
     subroutine mpp_get_time_axis( unit, time_axis )
       integer, intent(in) :: unit
       type(axistype), intent(inout) :: time_axis
@@ -265,7 +296,7 @@
   !####################################################################
   function mpp_get_default_calendar( )
      character(len=len(default_axis%calendar)) :: mpp_get_default_calendar
-    
+
      mpp_get_default_calendar = default_axis%calendar
 
   end function mpp_get_default_calendar
@@ -410,6 +441,25 @@
      return
    end function mpp_get_axis_length
 
+  !#####################################################################
+  function mpp_get_axis_bounds(axis, data, name)
+     type(axistype), intent(in) :: axis
+     real, dimension(:), intent(out) :: data
+     character(len=*), optional, intent(out) :: name
+     logical                         :: mpp_get_axis_bounds
+
+     if (size(data(:)).lt.axis%len+1)&
+          call mpp_error(FATAL,'MPP_GET_AXIS_BOUNDS: data array not large enough, axis '//trim(axis%name))
+     if (.NOT.ASSOCIATED(axis%data_bounds)) then
+        mpp_get_axis_bounds = .false.
+     else
+        mpp_get_axis_bounds = .true.
+        data(1:axis%len+1) = axis%data_bounds(:)
+     endif
+     if(present(name)) name = trim(axis%name_bounds)
+
+     return
+  end function mpp_get_axis_bounds
 
   !#####################################################################
    subroutine mpp_get_axis_data( axis, data )
@@ -426,7 +476,7 @@
      else
         data(1:axis%len) = axis%data
      endif
- 
+
      return
    end subroutine mpp_get_axis_data
 
@@ -589,7 +639,7 @@
         v%min = f%att(imissing)%fatt(1)*f%scale + f%add
      else if (ifill>0) then
      !z1l ifdef is added in to be able to compile without using use_netCDF.
-#ifdef use_netCDF  
+#ifdef use_netCDF
         ! define min and max according to _FillValue
         if(f%att(ifill)%fatt(1)>0) then
             ! if _FillValue is positive, then it defines valid maximum
@@ -675,7 +725,7 @@
    function mpp_get_att_name(att)
       type(atttype),    intent(in) :: att
       character(len=len(att%name)) :: mpp_get_att_name
-     
+
       mpp_get_att_name = att%name
       return
 
@@ -687,7 +737,7 @@
    function mpp_get_att_type(att)
       type(atttype), intent(in) :: att
       integer                   :: mpp_get_att_type
-     
+
       mpp_get_att_type = att%type
       return
 
@@ -699,7 +749,7 @@
    function mpp_get_att_length(att)
       type(atttype), intent(in) :: att
       integer                   :: mpp_get_att_length
-     
+
       mpp_get_att_length = att%len
 
       return
@@ -712,7 +762,7 @@
    function mpp_get_att_char(att)
       type(atttype), intent(in) :: att
       character(len=att%len)    :: mpp_get_att_char
-     
+
       mpp_get_att_char = att%catt
       return
 
@@ -724,7 +774,7 @@
    function mpp_get_att_real(att)
       type(atttype), intent(in)          :: att
       real, dimension(size(att%fatt(:))) :: mpp_get_att_real
-     
+
       mpp_get_att_real = att%fatt
       return
 
@@ -756,7 +806,7 @@
    ! return the  file name of corresponding unit
    function mpp_get_file_name(unit)
       integer,                  intent(in) :: unit
-      character(len=len(mpp_file(1)%name)) :: mpp_get_file_name      
+      character(len=len(mpp_file(1)%name)) :: mpp_get_file_name
 
       mpp_get_file_name = mpp_file(unit)%name
       return
@@ -767,7 +817,7 @@
    ! return if certain file with unit is opened or not
    function mpp_file_is_opened(unit)
       integer,  intent(in) :: unit
-      logical              :: mpp_file_is_opened      
+      logical              :: mpp_file_is_opened
 
       mpp_file_is_opened = mpp_file(unit)%opened
       return
@@ -777,9 +827,9 @@
    !####################################################################
    ! return the attribute value of given field name
    subroutine mpp_get_field_att_text(unit, fieldname, attname, attvalue)
-     integer,           intent(in) :: unit  
+     integer,           intent(in) :: unit
      character(len=*),  intent(in) :: fieldname, attname
-     character(len=*), intent(out) :: attvalue 
+     character(len=*), intent(out) :: attvalue
      logical                       :: found_field,  found_att
      integer                       :: i, j, length
 
@@ -820,15 +870,69 @@
       mpp_io_clock_on = io_clocks_on
       return
 
-   end function mpp_io_clock_on   
+   end function mpp_io_clock_on
 
 
    function mpp_attribute_exist(field,name)
-      integer                      :: mpp_attribute_exist
+      logical                      :: mpp_attribute_exist
       type(fieldtype),  intent(in) :: field ! The field that you are searching for the attribute.
       character(len=*), intent(in) :: name ! name of the attributes
 
-      mpp_attribute_exist = mpp_find_att(field%Att(:),name)
-      
+      if(field%natt > 0) then
+         mpp_attribute_exist = ( mpp_find_att(field%Att(:),name) > 0 )
+      else
+         mpp_attribute_exist = .false.
+      endif
+
    end function mpp_attribute_exist
-   
+
+!#######################################################################
+subroutine mpp_dist_io_pelist(ssize,pelist)
+  integer,              intent(in)  :: ssize   ! Stripe size for dist read
+  integer, allocatable, intent(out) :: pelist(:)
+  integer :: i, lsize, ioroot
+  logical :: is_ioroot=.false.
+
+  ! Did you make a mistake?
+  if(ssize < 1) call mpp_error(FATAL,'mpp_dist_io_pelist: I/O stripe size < 1')
+
+  is_ioroot = mpp_is_dist_ioroot(ssize,ioroot=ioroot,lsize=lsize)
+
+  ! Did I make a mistake?
+  if(lsize < 1) call mpp_error(FATAL,'mpp_dist_io_pelist: size of pelist < 1')
+
+  allocate(pelist(lsize))
+  do i=1,lsize
+    pelist(i) = ioroot + i - 1
+  enddo
+end subroutine mpp_dist_io_pelist
+
+!#######################################################################
+logical function mpp_is_dist_ioroot(ssize,ioroot,lsize)
+  integer, intent(in)  :: ssize   ! Dist io set size
+  integer, intent(out), optional :: ioroot, lsize
+  integer :: pe, npes, mypos, maxpe, d_ioroot, d_lsize, last_ioroot
+  integer :: rootpe
+
+  if(ssize < 1) call mpp_error(FATAL,'mpp_is_dist_ioroot: I/O stripe size < 1')
+
+  mpp_is_dist_ioroot = .false.
+  rootpe = mpp_root_pe()
+  d_lsize = ssize
+  pe = mpp_pe()
+  mypos = modulo(pe-rootpe,ssize)  ! Which PE am I in the io group?
+  d_ioroot = pe - mypos            ! What is the io root for the group?
+  npes = mpp_npes()
+  maxpe = min(d_ioroot+ssize,npes+rootpe) - 1  ! Handle end case
+  d_lsize = maxpe - d_ioroot + 1
+  if(mod(npes,ssize) == 1)then  ! Ensure there are no sets with 1 member
+    last_ioroot = (npes-1) - ssize
+    if(pe >= last_ioroot) then
+       d_ioroot = last_ioroot
+       d_lsize = ssize + 1
+    endif
+  endif
+  if(pe == d_ioroot) mpp_is_dist_ioroot = .true.
+  if(PRESENT(ioroot)) ioroot = d_ioroot
+  if(PRESENT(lsize)) lsize = d_lsize
+end function mpp_is_dist_ioroot
diff --git a/src/shared/mpp/include/mpp_io_write.inc b/src/shared/mpp/include/mpp_io_write.inc
index 5fcf08e81d..cf13208df0 100644
--- a/src/shared/mpp/include/mpp_io_write.inc
+++ b/src/shared/mpp/include/mpp_io_write.inc
@@ -1,6 +1,6 @@
 
 ! -*-f90-*-
-! $Id: mpp_io_write.inc,v 20.0 2013/12/14 00:26:42 fms Exp $
+! $Id$
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 !                                                                            !
@@ -162,12 +162,13 @@
       return
     end subroutine mpp_write_meta_global_scalar_r
       
-    subroutine mpp_write_meta_global_scalar_i( unit, name, ival )
+    subroutine mpp_write_meta_global_scalar_i( unit, name, ival, pack )
       integer, intent(in) :: unit
       character(len=*), intent(in) :: name
       integer, intent(in) :: ival
+      integer, intent(in), optional :: pack
       
-      call mpp_write_meta_global( unit, name, ival=(/ival/) )
+      call mpp_write_meta_global( unit, name, ival=(/ival/), pack=pack )
       return
     end subroutine mpp_write_meta_global_scalar_i
       
@@ -213,12 +214,13 @@
       return
     end subroutine mpp_write_meta_scalar_r
       
-    subroutine mpp_write_meta_scalar_i( unit, id, name, ival )
+    subroutine mpp_write_meta_scalar_i( unit, id, name, ival,pack )
       integer, intent(in) :: unit, id
       character(len=*), intent(in) :: name
       integer, intent(in) :: ival
+      integer, intent(in), optional :: pack
       
-      call mpp_write_meta( unit, id, name, ival=(/ival/) )
+      call mpp_write_meta( unit, id, name, ival=(/ival/),pack=pack )
       return
     end subroutine mpp_write_meta_scalar_i
       
@@ -242,8 +244,98 @@
       endif
 #endif
     end subroutine mpp_write_axis_data
-    
-    subroutine mpp_write_meta_axis( unit, axis, name, units, longname, cartesian, sense, domain, data, min, compressed)
+
+    subroutine mpp_def_dim_nodata(unit,name,size)
+      integer, intent(in) :: unit
+      character(len=*), intent(in) :: name
+      integer, intent(in) :: size
+      integer :: error,did
+
+    ! This routine assumes the file is in define mode
+      if(.NOT. mpp_file(unit)%write_on_this_pe) return
+#ifdef use_netCDF
+      error = NF_DEF_DIM(mpp_file(unit)%ncid,name,size,did)
+      call netcdf_err(error, mpp_file(unit),string='Axis='//trim(name))
+#endif
+    end subroutine mpp_def_dim_nodata
+
+    subroutine mpp_def_dim_int(unit,name,dsize,longname,data)
+      integer, intent(in) :: unit
+      character(len=*), intent(in) :: name
+      integer, intent(in) :: dsize
+      character(len=*), intent(in) :: longname
+      integer, intent(in) :: data(:)
+      integer :: error,did,id
+
+    ! This routine assumes the file is in define mode
+#ifdef use_netCDF
+      if(.NOT. mpp_file(unit)%write_on_this_pe) return
+      error = NF_DEF_DIM(mpp_file(unit)%ncid,name,dsize,did)
+      call netcdf_err(error, mpp_file(unit),string='Axis='//trim(name))
+
+    ! Write dimension data.
+      error = NF_DEF_VAR( mpp_file(unit)%ncid, name, NF_INT, 1, did, id )
+      call netcdf_err( error, mpp_file(unit), string=' axis varable '//trim(name))
+
+      error = NF_PUT_ATT_TEXT( mpp_file(unit)%ncid, id, 'long_name', len_trim(longname), longname )
+      call netcdf_err( error, mpp_file(unit), string=' Attribute=long_name' )
+
+      if( mpp_file(unit)%action.EQ.MPP_WRONLY )then
+         if(header_buffer_val>0) then
+            error = NF__ENDDEF(mpp_file(unit)%ncid,header_buffer_val,4,0,4)
+         else
+            error = NF_ENDDEF(mpp_file(unit)%ncid)
+         endif
+      endif
+      call netcdf_err( error, mpp_file(unit), string=' subroutine mpp_def_dim')
+      error = NF_PUT_VARA_INT ( mpp_file(unit)%ncid, id, 1, size(data), data )
+      call netcdf_err( error, mpp_file(unit), string=' axis varable '//trim(name))
+      error = NF_REDEF(mpp_file(unit)%ncid)
+      call netcdf_err( error, mpp_file(unit), string=' subroutine mpp_def_dim')
+#endif
+    return
+  end subroutine mpp_def_dim_int
+
+    subroutine mpp_def_dim_real(unit,name,dsize,longname,data)
+      integer, intent(in) :: unit
+      character(len=*), intent(in) :: name
+      integer, intent(in) :: dsize
+      character(len=*), intent(in) :: longname
+      real, intent(in) :: data(:)
+      integer :: error,did,id
+
+    ! This routine assumes the file is in define mode
+#ifdef use_netCDF
+      if(.NOT. mpp_file(unit)%write_on_this_pe) return
+      error = NF_DEF_DIM(mpp_file(unit)%ncid,name,dsize,did)
+      call netcdf_err(error, mpp_file(unit),string='Axis='//trim(name))
+
+    ! Write dimension data.
+      error = NF_DEF_VAR( mpp_file(unit)%ncid, name, NF_INT, 1, did, id )
+      call netcdf_err( error, mpp_file(unit), string=' axis varable '//trim(name))
+
+      error = NF_PUT_ATT_TEXT( mpp_file(unit)%ncid, id, 'long_name', len_trim(longname), longname )
+      call netcdf_err( error, mpp_file(unit), string=' Attribute=long_name' )
+
+      if( mpp_file(unit)%action.EQ.MPP_WRONLY )then
+         if(header_buffer_val>0) then
+            error = NF__ENDDEF(mpp_file(unit)%ncid,header_buffer_val,4,0,4)
+         else
+            error = NF_ENDDEF(mpp_file(unit)%ncid)
+         endif
+      endif
+      call netcdf_err( error, mpp_file(unit), string=' subroutine mpp_def_dim')
+      error = NF_PUT_VARA_INT ( mpp_file(unit)%ncid, id, 1, size(data), data )
+      call netcdf_err( error, mpp_file(unit), string=' axis varable '//trim(name))
+      error = NF_REDEF(mpp_file(unit)%ncid)
+      call netcdf_err( error, mpp_file(unit), string=' subroutine mpp_def_dim')
+#endif
+    return
+  end subroutine mpp_def_dim_real
+
+
+
+    subroutine mpp_write_meta_axis_r1d( unit, axis, name, units, longname, cartesian, sense, domain, data, min, calendar)
 !load the values in an axistype (still need to call mpp_write)
 !write metadata attributes for axis
 !it is declared intent(inout) so you can nullify pointers in the incoming object if needed
@@ -256,7 +348,7 @@
       type(domain1D),   intent(in), optional :: domain
       real,             intent(in), optional :: data(:)
       real,             intent(in), optional :: min
-      character(len=*), intent(in), optional :: compressed
+      character(len=*), intent(in), optional :: calendar
 
       integer :: is, ie, isg, ieg
       integer :: istat      
@@ -314,8 +406,8 @@
       axis%name     = name
       axis%units    = units
       axis%longname = longname
+      if( PRESENT(calendar)  ) axis%calendar  = calendar
       if( PRESENT(sense)     ) axis%sense     = sense
-      if( PRESENT(compressed)) axis%compressed     = trim(compressed)
       if( PRESENT(data) )then
          if( mpp_file(unit)%fileset.EQ.MPP_MULTI .AND. domain_exist ) then
             axis%len = ie - is + 1
@@ -379,6 +471,10 @@
 !write axis attributes
       call mpp_write_meta( unit, axis%id, 'long_name', cval=axis%longname) ; axis%natt = axis%natt + 1
       call mpp_write_meta( unit, axis%id, 'units',     cval=axis%units) ; axis%natt = axis%natt + 1
+      if( PRESENT(calendar) ) then
+        call mpp_write_meta( unit, axis%id, 'calendar', cval=axis%calendar)
+        axis%natt = axis%natt + 1
+      endif 
       if( PRESENT(cartesian) ) then
         call mpp_write_meta( unit, axis%id, 'cartesian_axis', cval=axis%cartesian)
         axis%natt = axis%natt + 1
@@ -390,13 +486,10 @@
           else if( sense.EQ.1 )then
               call mpp_write_meta( unit, axis%id, 'positive', cval='up')
               axis%natt = axis%natt + 1
+          else
+             ! silently ignore values of sense other than +/-1.
           end if
-!silently ignore values of sense other than +/-1.
       end if
-      if( PRESENT(compressed) ) then
-        call mpp_write_meta( unit, axis%id, 'compress', cval=axis%compressed)
-        axis%natt = axis%natt + 1
-      endif 
       if( PRESENT(min) ) then
         call mpp_write_meta( unit, axis%id, 'valid_min', rval=min)
         axis%natt = axis%natt + 1
@@ -405,16 +498,143 @@
           call mpp_write_meta( unit, axis%id, 'domain_decomposition', ival=(/isg,ieg,is,ie/))
           axis%natt = axis%natt + 1
       end if
-      if( verbose )print '(a,2i3,x,a,2i3)', 'MPP_WRITE_META: Wrote axis metadata, pe, unit, axis%name, axis%id, axis%did=', &
+      if( verbose )print '(a,2i6,x,a,2i3)', 'MPP_WRITE_META: Wrote axis metadata, pe, unit, axis%name, axis%id, axis%did=', &
            pe, unit, trim(axis%name), axis%id, axis%did
       
       mpp_file(unit)%ndim = max(1,mpp_file(unit)%ndim + 1)
 
+!      call mpp_clock_end(mpp_write_clock)           
+      return
+    end subroutine mpp_write_meta_axis_r1d
+
+    subroutine mpp_write_meta_axis_i1d(unit, axis, name, units, longname, data, min, compressed)
+!load the values in an axistype (still need to call mpp_write)
+!write metadata attributes for axis
+!it is declared intent(inout) so you can nullify pointers in the incoming object if needed
+!the f90 standard doesn't guarantee that intent(out) on a type guarantees that its pointer components will be unassociated
+      integer,          intent(in)           :: unit
+      type(axistype),   intent(inout)        :: axis
+      character(len=*), intent(in)           :: name, units, longname
+      integer,          intent(in)           :: data(:)
+      integer,          intent(in), optional :: min
+      character(len=*), intent(in), optional :: compressed
+
+      integer :: istat      
+      logical :: domain_exist
+      type(domain2d), pointer :: io_domain => NULL()
+
+!      call mpp_clock_begin(mpp_write_clock)
+      if( .NOT.module_is_initialized    )call mpp_error( FATAL, 'MPP_WRITE_META_I1D: must first call mpp_io_init.' )
+      if( .NOT. mpp_file(unit)%write_on_this_pe) then
+!         call mpp_clock_end(mpp_write_clock)
+         return
+      endif
+      if( .NOT.mpp_file(unit)%opened )call mpp_error( FATAL, 'MPP_WRITE_META: invalid unit number.' )
+      if( mpp_file(unit)%initialized ) &
+           call mpp_error( FATAL, 'MPP_WRITE_META_I1D: cannot write metadata to file after an mpp_write.' )
+
+!pre-existing pointers need to be nullified
+      if( ASSOCIATED(axis%idata) ) then
+         DEALLOCATE(axis%idata, stat=istat)
+      endif
+!load axistype
+      axis%name     = name
+      axis%units    = units
+      axis%longname = longname
+      if( PRESENT(compressed)) axis%compressed = trim(compressed)
+      axis%len = size(data(:))
+      allocate(axis%idata(axis%len))
+       axis%idata = data
+!write metadata
+      if( mpp_file(unit)%format.EQ.MPP_NETCDF )then
+          error = NF_DEF_DIM( mpp_file(unit)%ncid, axis%name, axis%len, axis%did )
+          call netcdf_err( error, mpp_file(unit), axis )
+          error = NF_DEF_VAR( mpp_file(unit)%ncid, axis%name, NF_INT, 1, axis%did, axis%id )
+          call netcdf_err( error, mpp_file(unit), axis )
+      else
+          call mpp_error( FATAL, 'MPP_WRITE_META_AXIS_I1D: Only netCDF format is currently supported.' )
+      end if  
+!write axis attributes
+      call mpp_write_meta( unit, axis%id, 'long_name', cval=axis%longname) ; axis%natt = axis%natt + 1
+      call mpp_write_meta( unit, axis%id, 'units',     cval=axis%units) ; axis%natt = axis%natt + 1
+      if( PRESENT(compressed) ) then
+        call mpp_write_meta( unit, axis%id, 'compress', cval=axis%compressed)
+        axis%natt = axis%natt + 1
+      endif 
+      if( PRESENT(min) ) then
+        call mpp_write_meta( unit, axis%id, 'valid_min', ival=min)
+        axis%natt = axis%natt + 1
+      endif 
+      if( verbose )print '(a,2i6,x,a,2i3)', 'MPP_WRITE_META: Wrote axis metadata, pe, unit, axis%name, axis%id, axis%did=', &
+           pe, unit, trim(axis%name), axis%id, axis%did
+      
+      mpp_file(unit)%ndim = max(1,mpp_file(unit)%ndim + 1)
 
 !      call mpp_clock_end(mpp_write_clock)           
       return
-    end subroutine mpp_write_meta_axis
+    end subroutine mpp_write_meta_axis_i1d
+
+
+    subroutine mpp_write_meta_axis_unlimited(unit, axis, name, data, unlimited, units, longname)
+!load the values in an axistype (still need to call mpp_write)
+!write metadata attributes for axis
+!it is declared intent(inout) so you can nullify pointers in the incoming object if needed
+!the f90 standard doesn't guarantee that intent(out) on a type guarantees that its pointer components will be unassociated
+      integer,          intent(in)           :: unit
+      type(axistype),   intent(inout)        :: axis
+      character(len=*), intent(in)           :: name
+      integer,          intent(in)           :: data  ! Number of elements to be written
+      logical,          intent(in)           :: unlimited  ! Provides unique arg signature
+      character(len=*), intent(in), optional :: units, longname
+
+      integer :: istat      
+      logical :: domain_exist
+      type(domain2d), pointer :: io_domain => NULL()
+
+!      call mpp_clock_begin(mpp_write_clock)
+      if( .NOT.module_is_initialized    )call mpp_error( FATAL, 'MPP_WRITE_META_I1D: must first call mpp_io_init.' )
+      if( .NOT. mpp_file(unit)%write_on_this_pe) then
+!         call mpp_clock_end(mpp_write_clock)
+         return
+      endif
+      if( .NOT.mpp_file(unit)%opened )call mpp_error( FATAL, 'MPP_WRITE_META: invalid unit number.' )
+      if( mpp_file(unit)%initialized ) &
+           call mpp_error( FATAL, 'MPP_WRITE_META_I1D: cannot write metadata to file after an mpp_write.' )
+
+!load axistype
+      axis%name     = name
+      if(present(units)) axis%units    = units
+      if(present(longname)) axis%longname = longname
+      axis%len = 1
+      allocate(axis%idata(1))
+      axis%idata = data
+!write metadata
+      if( mpp_file(unit)%format.EQ.MPP_NETCDF )then
+          error = NF_DEF_DIM( mpp_file(unit)%ncid, axis%name, NF_UNLIMITED, axis%did )
+          call netcdf_err( error, mpp_file(unit), axis )
+          error = NF_DEF_VAR( mpp_file(unit)%ncid, axis%name, NF_INT, 0, axis%did, axis%id )
+          call netcdf_err( error, mpp_file(unit), axis )
+      else
+          call mpp_error( FATAL, 'MPP_WRITE_META_AXIS_UNLIMITED: Only netCDF format is currently supported.' )
+      end if  
+!write axis attributes
+      if(present(longname)) then
+         call mpp_write_meta(unit,axis%id,'long_name',cval=axis%longname); axis%natt=axis%natt+1
+      endif
+      if(present(units)) then
+         call mpp_write_meta(unit,axis%id,'units',     cval=axis%units); axis%natt=axis%natt+1
+      endif
+      if( verbose )print '(a,2i6,x,a,2i3)', &
+          'MPP_WRITE_META_UNLIMITED: Wrote axis metadata, pe, unit, axis%name, axis%id, axis%did=', &
+           pe, unit, trim(axis%name), axis%id, axis%did
       
+      mpp_file(unit)%ndim = max(1,mpp_file(unit)%ndim + 1)
+
+!      call mpp_clock_end(mpp_write_clock)           
+      return
+    end subroutine mpp_write_meta_axis_unlimited
+
+
     subroutine mpp_write_meta_field( unit, field, axes, name, units, longname,&
          min, max, missing, fill, scale, add, pack, time_method,standard_name, checksum)
 !define field: must have already called mpp_write_meta(axis) for each axis
@@ -475,9 +695,8 @@
 !           call mpp_error( FATAL, 'MPP_WRITE_META: cannot write metadata to file after an mpp_write.' )
            
 !pre-existing pointers need to be nullified
-      if( ASSOCIATED(field%axes) ) then
-         DEALLOCATE(field%axes, stat=istat)
-      endif
+      if( ASSOCIATED(field%axes) ) DEALLOCATE(field%axes, stat=istat)
+      if( ASSOCIATED(field%size) ) DEALLOCATE(field%size, stat=istat)
 !fill in field metadata
       field%name = name
       field%units = units
@@ -541,6 +760,7 @@
                   call mpp_error( FATAL, 'MPP_WRITE_META_FIELD: only legal packing values are 1,2,4,8.' )
           end select
           call netcdf_err( error, mpp_file(unit), field=field )
+          deallocate(axis_id)
 #ifndef use_netCDF3
           if(shuffle .NE. 0 .OR. deflate .NE. 0) then
              error = NF_DEF_VAR_DEFLATE(mpp_file(unit)%ncid, field%id, shuffle, deflate, deflate_level)
@@ -598,6 +818,22 @@
       if ( present(fill) ) then
          if( field%pack.EQ.1 .OR. field%pack.EQ.2 )then
             call mpp_write_meta( unit, field%id, '_FillValue', rval=field%fill, pack=pack )
+         else if (field%pack==0) then ! some safety checks for integer fills
+            if ( present(scale).OR.present(add) ) then
+               call mpp_error(FATAL,"add,scale not currently implimented for pack=0 int handling, try reals instead.")
+            else
+               if (KIND(field%fill)==DOUBLE_KIND) then
+                  !! check if type safe cast
+                  if ( field%fill /= TRANSFER( IBITS( TRANSFER(field%fill,INT(0,DOUBLE_KIND)) ,0 , BIT_SIZE(INT(0,INT_KIND))) , field%fill ) ) then
+                     call mpp_error(FATAL,"mpp_io(mpp_io_write.inc): For pack=0, fill and data should both be safely castable"// &
+                          "to NF_INT type, _FillValue was actually using more precision than INT_KIND, cannot safely cast.")
+                  end if  
+               else if (KIND(field%fill)/=INT_KIND) then
+                  call mpp_error(FATAL,"mpp_io(mpp_io_write.inc): Unexpected KIND of fill, use either 4, or 8 byte KIND.")
+               end if
+               !! then safe to write ival _FillValue and pack
+               call mpp_write_meta( unit, field%id, '_FillValue', ival=TRANSFER( field%fill, INT(0,INT_KIND)), pack=pack )
+            end if                     
          else
             a = nint((fill-add)/scale)
             call mpp_write_meta( unit, field%id, '_FillValue', rval=a, pack=pack )
@@ -624,7 +860,7 @@
       if ( PRESENT(standard_name)) &
            call mpp_write_meta(unit,field%id,'standard_name ', cval=field%standard_name) 
 
-      if( verbose )print '(a,2i3,x,a,i3)', 'MPP_WRITE_META: Wrote field metadata: pe, unit, field%name, field%id=', &
+      if( verbose )print '(a,2i6,x,a,i3)', 'MPP_WRITE_META: Wrote field metadata: pe, unit, field%name, field%id=', &
            pe, unit, trim(field%name), field%id
           
 !      call mpp_clock_end(mpp_write_clock) 
@@ -660,7 +896,7 @@
               write(unit)trim(text)//char(10)
           else                  !MPP_DIRECT
               write( unit,rec=mpp_file(unit)%record )trim(text)//char(10)
-              if( verbose )print '(a,i3,a,i3)', 'WRITE_ATTRIBUTE: PE=', pe, ' wrote record ', mpp_file(unit)%record
+              if( verbose )print '(a,i6,a,i3)', 'WRITE_ATTRIBUTE: PE=', pe, ' wrote record ', mpp_file(unit)%record
               mpp_file(unit)%record = mpp_file(unit)%record + 1
           end if
       end if  
@@ -679,9 +915,20 @@
       integer, allocatable :: rval_i(:)
 #ifdef use_netCDF
       if( PRESENT(rval) )then
-!pack is only meaningful for FP numbers
+!pack was only meaningful for FP numbers, but is now extended by the ival branch of this routine
           if( PRESENT(pack) )then
-              if( pack.EQ.1 )then
+             if( pack== 0 ) then !! here be dragons, use ival branch!...
+                if( KIND(rval).EQ.DOUBLE_KIND )then
+                   call mpp_error( FATAL, &
+                        'WRITE_ATTRIBUTE_NETCDF: attempting to write internal NF_INT, currently int32, as double.' )
+                   error = NF_PUT_ATT_DOUBLE( mpp_file(unit)%ncid, id, name, NF_DOUBLE, size(rval(:)), rval )
+                else if( KIND(rval).EQ.FLOAT_KIND )then
+                   call mpp_error( FATAL, &
+                        'WRITE_ATTRIBUTE_NETCDF: attempting to write internal NF_INT, currently int32, as float.' )
+                   error = NF_PUT_ATT_REAL  ( mpp_file(unit)%ncid, id, name, NF_DOUBLE, size(rval(:)), rval )
+                end if
+                call netcdf_err( error, mpp_file(unit), string=' Attribute='//name )
+             else if( pack.EQ.1 )then
                   if( KIND(rval).EQ.DOUBLE_KIND )then
                       error = NF_PUT_ATT_DOUBLE( mpp_file(unit)%ncid, id, name, NF_DOUBLE, size(rval(:)), rval )
                   else if( KIND(rval).EQ.FLOAT_KIND )then
@@ -730,8 +977,20 @@
               call netcdf_err( error, mpp_file(unit), string=' Attribute='//name )
           end if
       else if( PRESENT(ival) )then
+          if( PRESENT(pack) ) then
+             if (pack ==0) then
+                if (KIND(ival).EQ.LONG_KIND ) then
+                   call mpp_error(FATAL,'only use NF_INTs with pack=0 for now')
+                end if
+                error = NF_PUT_ATT_INT( mpp_file(unit)%ncid, id, name, NF_INT, size(ival(:)), ival ) !!XXX int32_t..
+                call netcdf_err( error, mpp_file(unit), string=' Attribute='//name )
+             else
+                call mpp_error( FATAL, 'WRITE_ATTRIBUTE_NETCDF: only implimented ints when pack=0, else use reals.' )
+             endif
+          else
           error = NF_PUT_ATT_INT ( mpp_file(unit)%ncid, id, name, NF_INT, size(ival(:)), ival )
           call netcdf_err( error, mpp_file(unit), string=' Attribute='//name )
+          end if
       else if( present(cval) )then
           error = NF_PUT_ATT_TEXT( mpp_file(unit)%ncid, id, name, len_trim(cval), cval )
           call netcdf_err( error, mpp_file(unit), string=' Attribute='//name )
@@ -802,6 +1061,20 @@
 #define MPP_TYPE_ real
 #include 
 
+#undef MPP_WRITE_COMPRESSED_1D_
+#define MPP_WRITE_COMPRESSED_1D_ mpp_write_compressed_r1d
+#undef MPP_WRITE_COMPRESSED_2D_
+#define MPP_WRITE_COMPRESSED_2D_ mpp_write_compressed_r2d
+#undef MPP_TYPE_
+#define MPP_TYPE_ real
+#include 
+
+#undef MPP_WRITE_UNLIMITED_AXIS_1D_
+#define MPP_WRITE_UNLIMITED_AXIS_1D_ mpp_write_unlimited_axis_r1d
+#undef MPP_TYPE_
+#define MPP_TYPE_ real
+#include 
+
 #undef MPP_WRITE_
 #define MPP_WRITE_ mpp_write_r0D
 #undef MPP_TYPE_
@@ -876,11 +1149,22 @@
       field%longname = axis%longname
       field%units = axis%units
 
-      allocate( field%axes(1)%data(size(axis%data) ))
-      field%axes(1)%data = axis%data
-      
+      if(ASSOCIATED(axis%data))then
+        allocate( field%axes(1)%data(size(axis%data) ))
+        field%axes(1)%data = axis%data
+        call write_record( unit, field, axis%len, axis%data )
+      elseif(ASSOCIATED(axis%idata))then
+        allocate( field%axes(1)%data(size(axis%idata) ))
+        field%axes(1)%data = REAL(axis%idata)
+        field%pack=4
+        call write_record( unit, field, axis%len, REAL(axis%idata) )
+      else
+        call mpp_error( FATAL, 'MPP_WRITE_AXIS: No data associated with axis.' )
+      endif
+
+      deallocate(field%axes(1)%data)
+      deallocate(field%axes,field%size)
 
-      call write_record( unit, field, axis%len, axis%data )
       call mpp_clock_end(mpp_write_clock)
       return
     end subroutine mpp_write_axis
@@ -946,7 +1230,7 @@
               call mpp_write_meta( unit, 'END', cval='metadata' )
           end if
           mpp_file(unit)%initialized = .TRUE.
-          if( verbose )print '(a,i3,a)', 'MPP_WRITE: PE=', pe, ' initialized file '//trim(mpp_file(unit)%name)//'.'
+          if( verbose )print '(a,i6,a)', 'MPP_WRITE: PE=', pe, ' initialized file '//trim(mpp_file(unit)%name)//'.'
       end if
           
 !initialize time: by default assume NULLTIME
@@ -960,7 +1244,7 @@
           mpp_file(unit)%time = time
           newtime = .TRUE.
       end if
-      if( verbose )print '(a,2i3,2i5,es13.5)', 'MPP_WRITE: PE, unit, %id, %time_level, %time=',&
+      if( verbose )print '(a,2i6,2i5,es13.5)', 'MPP_WRITE: PE, unit, %id, %time_level, %time=',&
            pe, unit, mpp_file(unit)%id, mpp_file(unit)%time_level, mpp_file(unit)%time
            
       if( mpp_file(unit)%format.EQ.MPP_NETCDF )then
@@ -1004,7 +1288,10 @@
                   error = NF_PUT_VAR1_REAL  ( mpp_file(unit)%ncid, mpp_file(unit)%id, mpp_file(unit)%time_level, time )
               end if
           end if  
-          if( field%pack.LE.2 )then
+          if( field%pack == 0 )then
+              packed_data = CEILING(data)
+              error = NF_PUT_VARA_INT   ( mpp_file(unit)%ncid, field%id, start, axsiz, packed_data )
+          elseif( field%pack.GT.0 .and. field%pack.LE.2 )then
               if( KIND(data).EQ.DOUBLE_KIND )then
                   error = NF_PUT_VARA_DOUBLE( mpp_file(unit)%ncid, field%id, start, axsiz, data )
               else if( KIND(data).EQ.FLOAT_KIND )then
@@ -1051,7 +1338,7 @@
 #else                 
                   write( unit, rec=mpp_file(unit)%record )field%id, subdomain, time_level, time, data
 #endif            
-                  if( debug )print '(a,i3,a,i3)', 'MPP_WRITE: PE=', pe, ' wrote record ', mpp_file(unit)%record
+                  if( debug )print '(a,i6,a,i6)', 'MPP_WRITE: PE=', pe, ' wrote record ', mpp_file(unit)%record
               end if
           end if  
       end if  
@@ -1215,7 +1502,7 @@
       if( mpp_file(unit)%threading.EQ.MPP_MULTI .AND. mpp_file(unit)%fileset.EQ.MPP_MULTI .AND. axis%domain.NE.NULL_DOMAIN1D )then
           call mpp_write_meta( unit, axis%id, 'domain_decomposition', ival=(/isg,ieg,is,ie/) )
       end if
-      if( verbose )print '(a,2i3,x,a,2i3)', 'MPP_WRITE_META: Wrote axis metadata, pe, unit, axis%name, axis%id, axis%did=', &
+      if( verbose )print '(a,2i6,x,a,2i3)', 'MPP_WRITE_META: Wrote axis metadata, pe, unit, axis%name, axis%id, axis%did=', &
            pe, unit, trim(axis%name), axis%id, axis%did
 #else      
       call mpp_error( FATAL, 'MPP_READ currently requires use_netCDF option' )
@@ -1295,6 +1582,7 @@
               case default
                   call mpp_error( FATAL, 'MPP_WRITE_META_FIELD: only legal packing values are 1,2,4,8.' )
           end select
+          deallocate( axis_id )
 #endif            
       else
           varnum = varnum + 1
@@ -1355,7 +1643,7 @@
           if( field%scale.NE.default_field%scale )call mpp_write_meta( unit, field%id, 'scale_factor',  rval=field%scale )
           if( field%add.NE.default_field%add   )call mpp_write_meta( unit, field%id, 'add_offset',    rval=field%add   )
       end if
-      if( verbose )print '(a,2i3,x,a,i3)', 'MPP_WRITE_META: Wrote field metadata: pe, unit, field%name, field%id=', &
+      if( verbose )print '(a,2i6,x,a,i3)', 'MPP_WRITE_META: Wrote field metadata: pe, unit, field%name, field%id=', &
            pe, unit, trim(field%name), field%id
 
 !      call mpp_clock_end(mpp_write_clock)           
diff --git a/src/shared/mpp/include/mpp_read_2Ddecomp.h b/src/shared/mpp/include/mpp_read_2Ddecomp.h
index 65964344e5..f76a8f1c23 100644
--- a/src/shared/mpp/include/mpp_read_2Ddecomp.h
+++ b/src/shared/mpp/include/mpp_read_2Ddecomp.h
@@ -132,3 +132,18 @@
 
       return
     end subroutine MPP_READ_2DDECOMP_3D_
+
+
+    subroutine MPP_READ_2DDECOMP_4D_( unit, field, domain, data, tindex, tile_count )
+      integer, intent(in) :: unit
+      type(fieldtype), intent(in) :: field
+      type(domain2D), intent(in) :: domain
+      MPP_TYPE_, intent(inout) :: data(:,:,:,:)
+      integer, intent(in), optional :: tindex, tile_count
+      MPP_TYPE_ :: data3D(size(data,1),size(data,2),size(data,3)*size(data,4))
+      pointer( ptr, data3D )
+      ptr = LOC(data)
+      call mpp_read( unit, field, domain, data3D, tindex, tile_count)
+      return
+    end subroutine MPP_READ_2DDECOMP_4D_
+
diff --git a/src/shared/mpp/include/mpp_read_compressed.h b/src/shared/mpp/include/mpp_read_compressed.h
new file mode 100644
index 0000000000..d32c0f5f57
--- /dev/null
+++ b/src/shared/mpp/include/mpp_read_compressed.h
@@ -0,0 +1,93 @@
+    subroutine MPP_READ_COMPRESSED_1D_(unit, field, domain, data, tindex)
+      integer, intent(in) :: unit 
+      type(fieldtype), intent(in) :: field
+      type(domain2D),  intent(in) :: domain
+      MPP_TYPE_, intent(inout) :: data(:)
+      integer,           intent(in), optional :: tindex
+
+      MPP_TYPE_ :: data2D(size(data,1),1)
+      pointer( ptr, data2D )
+      ptr = LOC(data)
+
+      call mpp_read(unit, field, domain, data2D, tindex)
+      return
+    end subroutine MPP_READ_COMPRESSED_1D_
+
+    subroutine MPP_READ_COMPRESSED_2D_(unit, field, domain, data, tindex, start, nread, threading)
+      integer,           intent(in)           :: unit
+      type(fieldtype),   intent(in)           :: field
+      type(domain2D),    intent(in)           :: domain
+      MPP_TYPE_,         intent(inout)        :: data(:,:)
+      integer,           intent(in), optional :: tindex
+      integer,           intent(in), optional :: start(:), nread(:)
+      integer,           intent(in), optional :: threading
+
+      integer, allocatable :: pelist(:)
+      integer :: npes, p, threading_flag
+      type(domain2d), pointer :: io_domain=>NULL()
+      logical :: compute_chksum
+      integer(LONG_KIND) ::chk
+
+      call mpp_clock_begin(mpp_read_clock)
+
+      data = 0 !! zero out data so other tiles do not contribute junk to chksum
+
+      if( .NOT.module_is_initialized )call mpp_error( FATAL, 'MPP_READ_COMPRESSED_2D_: must first call mpp_io_init.' )
+      if( .NOT.mpp_file(unit)%valid )call mpp_error( FATAL, 'MPP_READ_COMPRESSED_2D_: invalid unit number.' )
+
+      threading_flag = MPP_SINGLE
+      if( PRESENT(threading) )threading_flag = threading
+      if( threading_flag == MPP_MULTI ) then
+         call read_record(unit,field,size(data(:,:)),data,tindex,start_in=start, axsiz_in=nread)
+      else if( threading_flag == MPP_SINGLE ) then
+
+         io_domain=>mpp_get_io_domain(domain) 
+         if(.not. ASSOCIATED(io_domain)) call mpp_error( FATAL, 'MPP_READ_COMPRESSED_2D_: io_domain must be defined.' )
+         npes = mpp_get_domain_npes(io_domain)
+         allocate(pelist(npes))
+         call mpp_get_pelist(io_domain,pelist)
+         
+         if(mpp_pe() == pelist(1)) call read_record(unit,field,size(data(:,:)),data,tindex,start_in=start, axsiz_in=nread)
+
+         !--- z1l replace mpp_broadcast with mpp_send/mpp_recv to avoid hang in calling MPI_COMM_CREATE
+         !---     because size(pelist) might be different for different rank.
+         !--- prepost receive
+         if( mpp_pe() == pelist(1) ) then
+            do p = 2, npes
+               call mpp_send(data(1,1), plen=size(data(:,:)), to_pe=pelist(p), tag=COMM_TAG_1)
+            enddo
+            call mpp_sync_self()
+         else
+            call mpp_recv(data(1,1), glen=size(data(:,:)), from_pe=pelist(1), block=.false., tag=COMM_TAG_1)
+            call mpp_sync_self(check=EVENT_RECV)
+         endif
+
+         deallocate(pelist)
+      else
+         call mpp_error( FATAL, 'MPP_READ_COMPRESSED_2D_: threading should be MPP_SINGLE or MPP_MULTI')
+      endif
+
+      compute_chksum = .FALSE.
+      if (ANY(field%checksum /= default_field%checksum) ) compute_chksum = .TRUE.
+
+      if (compute_chksum) then 
+         if (field%type==NF_INT) then 
+            if (CEILING(field%fill) /= MPP_FILL_INT ) then
+               call mpp_error(NOTE,"During mpp_io(read_compressed) int field "//trim(field%name)// & 
+							 " found integer fill /= MPP_FILL_INT. Confirm this is what you want.")
+               chk = mpp_chksum( ceiling(data), mask_val=field%fill)  
+            else
+               chk = mpp_chksum( ceiling(data), mask_val=CEILING(field%fill) )  
+            end if    
+         else !!real
+            chk = mpp_chksum(data,mask_val=field%fill)
+         end if
+         !!compare
+         if (mpp_pe()==mpp_root_pe()) print '(A,Z16)', "mpp_read_compressed chksum: "//trim(field%name)//" = ",  chk
+         !! discuss making fatal after testing/review to match other routines.
+         if (chk /= field%checksum(1) ) call mpp_error(NOTE,"mpp_read_compressed chksum: "//trim(field%name)//" failed!")
+      end if
+
+      call mpp_clock_end(mpp_read_clock)
+      return
+    end subroutine MPP_READ_COMPRESSED_2D_
diff --git a/src/shared/mpp/include/mpp_read_distributed_ascii.h b/src/shared/mpp/include/mpp_read_distributed_ascii.h
new file mode 100644
index 0000000000..8334d3bcae
--- /dev/null
+++ b/src/shared/mpp/include/mpp_read_distributed_ascii.h
@@ -0,0 +1,29 @@
+subroutine MPP_READ_DISTRIBUTED_ASCII_1D_ (unit,fmt,ssize,data,iostat)
+  integer, intent(in)               :: unit
+  character(*), intent(in)          :: fmt
+  integer, intent(in)               :: ssize
+  MPP_TYPE_, dimension(:), intent(inout) :: data
+  integer, intent(out)              :: iostat
+
+  integer, allocatable :: pelist(:)
+  logical              :: is_ioroot=.false.
+
+  if(.not.module_is_initialized) call mpp_error(FATAL,'MPP_READ_DISTRIBUTED_ASCII_1D_:  module not initialized')
+
+  iostat = 0
+  call mpp_dist_io_pelist(ssize,pelist)  ! ALLOCATE and create pelist if size of group > 1
+  if(.not. ALLOCATED(pelist)) &
+           call mpp_error(FATAL,'MPP_READ_DISTRIBUTED_ASCII_1D_:: pelist allocation failed')
+  is_ioroot = mpp_is_dist_ioroot(ssize)
+  if(is_ioroot) then
+    if(trim(fmt)=='*')then
+       read(unit,*,iostat=iostat) data
+    else
+       read(unit,fmt=trim(fmt),iostat=iostat) data
+    endif
+    if(iostat /= 0) return  ! Calling routine must handle error
+  endif
+
+  call mpp_broadcast(data,size(data),pelist(1),pelist)
+  deallocate(pelist)  ! Don't forget to deallocate pelist
+end subroutine MPP_READ_DISTRIBUTED_ASCII_1D_
diff --git a/src/shared/mpp/include/mpp_read_distributed_ascii.inc b/src/shared/mpp/include/mpp_read_distributed_ascii.inc
new file mode 100644
index 0000000000..7eccf03b65
--- /dev/null
+++ b/src/shared/mpp/include/mpp_read_distributed_ascii.inc
@@ -0,0 +1,42 @@
+#undef MPP_READ_DISTRIBUTED_ASCII_1D_
+#define MPP_READ_DISTRIBUTED_ASCII_1D_ mpp_read_distributed_ascii_r1D
+#undef MPP_TYPE_
+#define MPP_TYPE_ real
+#include 
+
+#undef MPP_READ_DISTRIBUTED_ASCII_1D_
+#define MPP_READ_DISTRIBUTED_ASCII_1D_ mpp_read_distributed_ascii_i1D
+#undef MPP_TYPE_
+#define MPP_TYPE_ integer
+#include 
+
+subroutine mpp_read_distributed_ascii_a1D(unit,fmt,ssize,data,iostat)
+  integer, intent(in)               :: unit
+  character(*), intent(in)          :: fmt
+  integer, intent(in)               :: ssize
+  character(len=*), dimension(:), intent(inout) :: data
+  integer, intent(out)              :: iostat
+
+  integer, allocatable :: pelist(:)
+  logical              :: is_ioroot=.false.
+
+
+  if(.not.module_is_initialized) call mpp_error(FATAL,'mpp_read_distributed_ascii_a1D:  module not initialized')
+
+  iostat = 0
+  call mpp_dist_io_pelist(ssize,pelist)
+  if(.not. ALLOCATED(pelist)) &
+           call mpp_error(FATAL,'mpp_read_distributed_ascii_a1D: pelist allocation failed')
+  is_ioroot = mpp_is_dist_ioroot(ssize)
+  if(is_ioroot) then
+    if(trim(fmt)=='*')then
+       read(unit,*,iostat=iostat) data
+    else
+       read(unit,fmt=trim(fmt),iostat=iostat) data
+    endif
+    if(iostat /= 0) return  ! Calling routine must handle error
+  endif
+
+  call mpp_broadcast(data,len(data(1)),pelist(1),pelist)
+  deallocate(pelist)  ! Don't forget to deallocate pelist
+end subroutine mpp_read_distributed_ascii_a1D
diff --git a/src/shared/mpp/include/mpp_reduce_mpi.h b/src/shared/mpp/include/mpp_reduce_mpi.h
index 2f79767566..3fe50c8567 100644
--- a/src/shared/mpp/include/mpp_reduce_mpi.h
+++ b/src/shared/mpp/include/mpp_reduce_mpi.h
@@ -9,10 +9,10 @@
       if( .NOT.module_is_initialized )call mpp_error( FATAL, 'MPP_REDUCE: You must first call mpp_init.' )
       n = get_peset(pelist); if( peset(n)%count.EQ.1 )return
 
-      if( current_clock.NE.0 )call SYSTEM_CLOCK(start_tick)
+      if( debug .and. (current_clock.NE.0) )call SYSTEM_CLOCK(start_tick)
       if( verbose )call mpp_error( NOTE, 'MPP_REDUCE_: using MPI_ALLREDUCE...' )
       call MPI_ALLREDUCE( a, work, 1, MPI_TYPE_, MPI_REDUCE_, peset(n)%id, error )
       a = work
-      if( current_clock.NE.0 )call increment_current_clock( EVENT_ALLREDUCE, MPP_TYPE_BYTELEN_ )
+      if( debug .and. (current_clock.NE.0) )call increment_current_clock( EVENT_ALLREDUCE, MPP_TYPE_BYTELEN_ )
       return
     end subroutine MPP_REDUCE_
diff --git a/src/shared/mpp/include/mpp_reduce_sma.h b/src/shared/mpp/include/mpp_reduce_sma.h
index 595746dc85..a6cfa992c8 100644
--- a/src/shared/mpp/include/mpp_reduce_sma.h
+++ b/src/shared/mpp/include/mpp_reduce_sma.h
@@ -13,7 +13,7 @@
       if( .NOT.module_is_initialized )call mpp_error( FATAL, 'MPP_REDUCE: You must first call mpp_init.' )
       n = get_peset(pelist); if( peset(n)%count.EQ.1 )return
 
-      if( current_clock.NE.0 )call SYSTEM_CLOCK(start_tick)
+      if( debug .and. (current_clock.NE.0) )call SYSTEM_CLOCK(start_tick)
 !allocate space from the stack for pwrk and b
       ptr = LOC(mpp_stack)
       words = size(work(:))*size(transfer(work(1),word))
@@ -27,6 +27,6 @@
       call SHMEM_REDUCE_( work, work, 1, peset(n)%start, peset(n)%log2stride, peset(n)%count, work(2), sync )
       call mpp_sync(pelist)
       a = work(1)
-      if( current_clock.NE.0 )call increment_current_clock( EVENT_ALLREDUCE, MPP_TYPE_BYTELEN_ )
+      if( debug .and. (current_clock.NE.0) )call increment_current_clock( EVENT_ALLREDUCE, MPP_TYPE_BYTELEN_ )
       return
     end subroutine MPP_REDUCE_
diff --git a/src/shared/mpp/include/mpp_scatter.h b/src/shared/mpp/include/mpp_scatter.h
new file mode 100644
index 0000000000..f3d73dcc45
--- /dev/null
+++ b/src/shared/mpp/include/mpp_scatter.h
@@ -0,0 +1,131 @@
+subroutine MPP_SCATTER_PELIST_2D_(is, ie, js, je, pelist, array_seg, data, is_root_pe, &
+                                  ishift, jshift)
+   integer,                           intent(in)    :: is, ie, js, je
+   integer,   dimension(:),           intent(in)    :: pelist
+   MPP_TYPE_, dimension(is:ie,js:je), intent(inout)    :: array_seg
+   MPP_TYPE_, dimension(:,:),         intent(in) :: data
+   logical,                           intent(in)    :: is_root_pe
+   integer,   optional,               intent(in)    :: ishift, jshift
+
+   MPP_TYPE_ ::  arr3D(size(array_seg,1),size(array_seg,2),1)
+   MPP_TYPE_ :: data3D(size(     data,1),size(     data,2),1)
+   pointer( aptr,  arr3D )
+   pointer( dptr, data3D )
+   aptr = LOC(array_seg)
+   dptr = LOC(     data)
+
+   call mpp_scatter(is, ie, js, je, 1, pelist, arr3D, data3D, is_root_pe, &
+                    ishift, jshift)
+   return
+
+end subroutine MPP_SCATTER_PELIST_2D_
+
+
+subroutine MPP_SCATTER_PELIST_3D_(is, ie, js, je, nk, pelist, array_seg, data, is_root_pe, &
+                                  ishift, jshift)
+   integer,                                intent(in)    :: is, ie, js, je, nk
+   integer,   dimension(:),                intent(in)    :: pelist
+   MPP_TYPE_, dimension(is:ie,js:je,1:nk), intent(inout)    :: array_seg
+   MPP_TYPE_, dimension(:,:,:),            intent(in) :: data
+   logical,                                intent(in)    :: is_root_pe
+   integer,   optional,                    intent(in)    :: ishift, jshift
+
+   integer :: i, msgsize, root_pe, root_pe_test
+   integer :: i1, i2, j1, j2, ioff, joff
+   integer :: my_ind(4), gind(4,size(pelist))
+   type array3D
+     MPP_TYPE_, dimension(:,:,:), allocatable :: data
+   endtype array3D
+   type(array3d), dimension(size(pelist)) :: temp
+
+   if (.not.ANY(mpp_pe().eq.pelist(:))) return
+
+   if (is_root_pe) then
+     root_pe = mpp_pe()
+     root_pe_test = 999
+     if (.not.ANY(pelist(:).eq.root_pe)) call mpp_error(FATAL, &
+                "fms_io(mpp_scatter_pelist): root_pe not a member of pelist")
+   else
+     root_pe = 0
+     root_pe_test = -999
+   endif
+! need this check in case MPI-rank 0 is a member of the pelist
+   call mpp_max(root_pe_test, pelist)
+   if (root_pe_test.lt.0) call mpp_error(FATAL, &
+                "fms_io(mpp_scatter_pelist): root_pe not specified or not a member of the pelist")
+! need to make sure only one root_pe has been specified
+   call mpp_sum(root_pe, pelist)
+   if ((is_root_pe) .and. (mpp_pe().ne.root_pe)) call mpp_error(FATAL, &
+                "fms_io(mpp_scatter_pelist): too many root_pes specified")
+
+
+   ioff=0
+   joff=0
+   if (present(ishift)) ioff=ishift
+   if (present(jshift)) joff=jshift
+
+   my_ind(1) = is
+   my_ind(2) = ie
+   my_ind(3) = js
+   my_ind(4) = je
+
+! scatter indices into global index on root_pe
+   if (is_root_pe) then
+     do i = 1, size(pelist)
+! root_pe data copy - no send to self
+       if (pelist(i).eq.root_pe) then
+         gind(:,i) = my_ind(:)
+       else
+         call mpp_recv(gind(:,i:i), 4, pelist(i), .FALSE., COMM_TAG_1)
+       endif
+     enddo
+     call mpp_sync_self(check=EVENT_RECV)
+     gind(1,:)=gind(1,:)+ioff
+     gind(2,:)=gind(2,:)+ioff
+     gind(3,:)=gind(3,:)+joff
+     gind(4,:)=gind(4,:)+joff
+! check indices to make sure they are within the range of "data"
+     if ((minval(gind).lt.1) .OR. (maxval(gind(1:2,:)).gt.size(data,1)) .OR. (maxval(gind(3:4,:)).gt.size(data,2))) &
+         call mpp_error(FATAL,"fms_io(mpp_scatter_pelist): specified indices (with shift) are outside of the & 
+                        &range of the receiving array")
+   else
+! non root_pe's send indices to root_pe
+     call mpp_send(my_ind(:), 4, root_pe, COMM_TAG_1)
+     call mpp_sync_self(check=EVENT_SEND)
+   endif
+
+!  scatter segments into data based on indices
+   if (is_root_pe) then
+     do i = 1, size(pelist)
+       if (pelist(i).ne.root_pe) then    ! no send to self
+         i1 = gind(1,i)
+         i2 = gind(2,i)
+         j1 = gind(3,i)
+         j2 = gind(4,i)
+         msgsize = (i2-i1+1)*(j2-j1+1)*nk
+! allocate and copy data into a contiguous memory space
+         allocate(temp(i)%data(i1:i2,j1:j2,1:nk))
+         temp(i)%data(i1:i2,j1:j2,1:nk)=data(i1:i2,j1:j2,1:nk)
+         call mpp_send(temp(i)%data, msgsize, pelist(i), COMM_TAG_2)
+       else
+!        data copy - no send to self
+         array_seg(is:ie,js:je,1:nk) = data(is+ioff:ie+ioff,js+joff:je+joff,1:nk)
+       endif
+     enddo
+     call mpp_sync_self(check=EVENT_SEND)
+! deallocate the temporary array used for the send
+     do i = 1, size(pelist)
+       if (allocated(temp(i)%data)) deallocate(temp(i)%data)
+     enddo
+   else
+!    non root_pe's recv data from root_pe
+     msgsize = (my_ind(2)-my_ind(1)+1) * (my_ind(4)-my_ind(3)+1) * nk
+     call mpp_recv(array_seg, msgsize, root_pe, .FALSE., COMM_TAG_2)
+     call mpp_sync_self(check=EVENT_RECV)
+   endif
+
+   call mpp_sync_self()
+
+   return
+
+end subroutine MPP_SCATTER_PELIST_3D_
diff --git a/src/shared/mpp/include/mpp_sum.inc b/src/shared/mpp/include/mpp_sum.inc
index 35037d3f3b..bb6aa7b4b1 100644
--- a/src/shared/mpp/include/mpp_sum.inc
+++ b/src/shared/mpp/include/mpp_sum.inc
@@ -1,6 +1,6 @@
 
 ! -*-f90-*-
-! $Id: mpp_sum.inc,v 18.0 2010/03/02 23:57:50 fms Exp $
+! $Id$
 
 !#######################################################################
 
diff --git a/src/shared/mpp/include/mpp_sum_mpi.h b/src/shared/mpp/include/mpp_sum_mpi.h
index 7574e1c1af..4047492194 100644
--- a/src/shared/mpp/include/mpp_sum_mpi.h
+++ b/src/shared/mpp/include/mpp_sum_mpi.h
@@ -12,7 +12,7 @@
       if( .NOT.module_is_initialized )call mpp_error( FATAL, 'MPP_SUM: You must first call mpp_init.' )
       n = get_peset(pelist); if( peset(n)%count.EQ.1 )return
 
-      if( current_clock.NE.0 )call SYSTEM_CLOCK(start_tick)
+      if( debug .and. (current_clock.NE.0) )call SYSTEM_CLOCK(start_tick)
       if( verbose )call mpp_error( NOTE, 'MPP_SUM: using MPI_ALLREDUCE...' )
       if( debug ) then
           errunit = stderr()
@@ -20,7 +20,7 @@
       endif
       call MPI_ALLREDUCE( a, work, length, MPI_TYPE_, MPI_SUM, peset(n)%id, error )
       a(1:length) = work(1:length)
-      if( current_clock.NE.0 )call increment_current_clock( EVENT_ALLREDUCE, length*MPP_TYPE_BYTELEN_ )
+      if( debug .and. (current_clock.NE.0) )call increment_current_clock( EVENT_ALLREDUCE, length*MPP_TYPE_BYTELEN_ )
       return
     end subroutine MPP_SUM_
 
diff --git a/src/shared/mpp/include/mpp_sum_sma.h b/src/shared/mpp/include/mpp_sum_sma.h
index 28fc24957c..53396e0bf3 100644
--- a/src/shared/mpp/include/mpp_sum_sma.h
+++ b/src/shared/mpp/include/mpp_sum_sma.h
@@ -16,7 +16,7 @@
       if( .NOT.module_is_initialized )call mpp_error( FATAL, 'MPP_SUM: You must first call mpp_init.' )
       n = get_peset(pelist); if( peset(n)%count.EQ.1 )return
 
-      if( current_clock.NE.0 )call SYSTEM_CLOCK(start_tick)
+      if( debug .and. (current_clock.NE.0) )call SYSTEM_CLOCK(start_tick)
 !allocate space from the stack for pwrk and b
       ptr = LOC(mpp_stack)
       words = size(work(:))*size(transfer(work(1),word))
@@ -29,7 +29,7 @@
       call mpp_sync(pelist)
       call SHMEM_SUM_( work, work, length, peset(n)%start, peset(n)%log2stride, peset(n)%count, work(length+1), sync )
       a(1:length) = work(1:length)
-      if( current_clock.NE.0 )call increment_current_clock( EVENT_ALLREDUCE, length*MPP_TYPE_BYTELEN_ )
+      if( debug .and. (current_clock.NE.0) )call increment_current_clock( EVENT_ALLREDUCE, length*MPP_TYPE_BYTELEN_ )
       return
     end subroutine MPP_SUM_
 
diff --git a/src/shared/mpp/include/mpp_transmit.inc b/src/shared/mpp/include/mpp_transmit.inc
index de2c45a7c6..82f7073527 100644
--- a/src/shared/mpp/include/mpp_transmit.inc
+++ b/src/shared/mpp/include/mpp_transmit.inc
@@ -1,6 +1,6 @@
 
 ! -*-f90-*-
-! $Id: mpp_transmit.inc,v 19.0 2012/01/06 22:04:03 fms Exp $
+! $Id$
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 !                                                                             !
diff --git a/src/shared/mpp/include/mpp_transmit_mpi.h b/src/shared/mpp/include/mpp_transmit_mpi.h
index 4dcaa9184d..1f9009df6b 100644
--- a/src/shared/mpp/include/mpp_transmit_mpi.h
+++ b/src/shared/mpp/include/mpp_transmit_mpi.h
@@ -44,7 +44,7 @@
       out_unit = stdout()
       if( debug )then
           call SYSTEM_CLOCK(tick)
-          write( out_unit,'(a,i18,a,i5,a,2i5,2i8)' )&
+          write( out_unit,'(a,i18,a,i6,a,2i6,2i8)' )&
                'T=',tick, ' PE=',pe, ' MPP_TRANSMIT begin: to_pe, from_pe, put_len, get_len=', to_pe, from_pe, put_len, get_len
       end if
 
@@ -54,7 +54,7 @@
 !do put first and then get
       if( to_pe.GE.0 .AND. to_pe.LT.npes )then
 !use non-blocking sends
-          if( current_clock.NE.0 )call SYSTEM_CLOCK(start_tick)
+          if( debug .and. (current_clock.NE.0) )call SYSTEM_CLOCK(start_tick)
 !z1l: truly non-blocking send.
 !          if( request(to_pe).NE.MPI_REQUEST_NULL )then !only one message from pe->to_pe in queue 
 !              if( debug )write( stderr(),* )'PE waiting for sending', pe, to_pe
@@ -68,7 +68,7 @@
                 "MPP_TRANSMIT: cur_send_request is greater than max_request, increase mpp_nml request_multiply")
              call MPI_ISEND( put_data, put_len, MPI_TYPE_, to_pe, comm_tag, mpp_comm_private, request_send(cur_send_request), error)
           endif
-          if( current_clock.NE.0 )call increment_current_clock( EVENT_SEND, put_len*MPP_TYPE_BYTELEN_ )
+          if( debug .and. (current_clock.NE.0) )call increment_current_clock( EVENT_SEND, put_len*MPP_TYPE_BYTELEN_ )
       else if( to_pe.EQ.ALL_PES )then !this is a broadcast from from_pe
           if( from_pe.LT.0 .OR. from_pe.GE.npes )call mpp_error( FATAL, 'MPP_TRANSMIT: broadcasting from invalid PE.' )
           if( put_len.GT.get_len )call mpp_error( FATAL, 'MPP_TRANSMIT: size mismatch between put_data and get_data.' )
@@ -92,7 +92,7 @@
 !do the get: for libSMA, a get means do a wait to ensure put on remote PE is complete
       if( from_pe.GE.0 .AND. from_pe.LT.npes )then
 !receive from from_pe
-          if( current_clock.NE.0 )call SYSTEM_CLOCK(start_tick)
+          if( debug .and. (current_clock.NE.0) )call SYSTEM_CLOCK(start_tick)
           if( block_comm ) then
              call MPI_RECV( get_data, get_len, MPI_TYPE_, from_pe, comm_tag, mpp_comm_private, stat, error )
              call MPI_GET_COUNT( stat, MPI_TYPE_, rsize, error)
@@ -117,12 +117,12 @@
                 type_recv(cur_recv_request) = MPI_TYPE_
              endif
           endif
-          if( current_clock.NE.0 )call increment_current_clock( EVENT_RECV, get_len*MPP_TYPE_BYTELEN_ )
+          if( debug .and. (current_clock.NE.0) )call increment_current_clock( EVENT_RECV, get_len*MPP_TYPE_BYTELEN_ )
       else if( from_pe.EQ.ANY_PE )then
 !receive from MPI_ANY_SOURCE
-          if( current_clock.NE.0 )call SYSTEM_CLOCK(start_tick)
+          if( debug .and. (current_clock.NE.0) )call SYSTEM_CLOCK(start_tick)
           call MPI_RECV( get_data, get_len, MPI_TYPE_, MPI_ANY_SOURCE, comm_tag, mpp_comm_private, stat, error )
-          if( current_clock.NE.0 )call increment_current_clock( EVENT_RECV, get_len*MPP_TYPE_BYTELEN_ )
+          if( debug .and. (current_clock.NE.0) )call increment_current_clock( EVENT_RECV, get_len*MPP_TYPE_BYTELEN_ )
       else if( from_pe.EQ.ALL_PES )then
           call mpp_error( FATAL, 'MPP_TRANSMIT: from_pe=ALL_PES has ambiguous meaning, and hence is not implemented.' )
       else if( from_pe.NE.NULL_PE )then !only remaining valid choice is NULL_PE
@@ -131,7 +131,7 @@
 
       if( debug )then
           call SYSTEM_CLOCK(tick)
-          write( out_unit,'(a,i18,a,i5,a,2i5,2i8)' )&
+          write( out_unit,'(a,i18,a,i6,a,2i6,2i8)' )&
                'T=',tick, ' PE=',pe, ' MPP_TRANSMIT end: to_pe, from_pe, put_len, get_len=', to_pe, from_pe, put_len, get_len
       end if
       return
@@ -158,14 +158,14 @@
       out_unit = stdout()
       if( debug )then
           call SYSTEM_CLOCK(tick)
-          write( out_unit,'(a,i18,a,i5,a,2i5,2i8)' )&
+          write( out_unit,'(a,i18,a,i6,a,2i6,2i8)' )&
                'T=',tick, ' PE=',pe, ' MPP_BROADCAST begin: from_pe, length=', from_pe, length
       end if
 
       if( .NOT.ANY(from_pe.EQ.peset(current_peset_num)%list) ) &
            call mpp_error( FATAL, 'MPP_BROADCAST: broadcasting from invalid PE.' )
 
-      if( current_clock.NE.0 )call SYSTEM_CLOCK(start_tick)
+      if( debug .and. (current_clock.NE.0) )call SYSTEM_CLOCK(start_tick)
  ! find the rank of from_pe in the pelist.     
       do i = 1, mpp_npes()
          if(peset(n)%list(i) == from_pe) then
@@ -174,7 +174,7 @@
          endif
       enddo
       if( mpp_npes().GT.1 )call MPI_BCAST( data, length, MPI_TYPE_, from_rank, peset(n)%id, error )
-      if( current_clock.NE.0 )call increment_current_clock( EVENT_BROADCAST, length*MPP_TYPE_BYTELEN_ )
+      if( debug .and. (current_clock.NE.0) )call increment_current_clock( EVENT_BROADCAST, length*MPP_TYPE_BYTELEN_ )
       return
     end subroutine MPP_BROADCAST_
 
diff --git a/src/shared/mpp/include/mpp_transmit_sma.h b/src/shared/mpp/include/mpp_transmit_sma.h
index 7b2acce659..5cb9a92f0a 100644
--- a/src/shared/mpp/include/mpp_transmit_sma.h
+++ b/src/shared/mpp/include/mpp_transmit_sma.h
@@ -62,20 +62,20 @@
 #ifdef _CRAYT90
           call SHMEM_UDCFLUSH !invalidate data cache
 #endif
-          if( current_clock.NE.0 )call SYSTEM_CLOCK(start_tick)
+          if( debug .and. (current_clock.NE.0) )call SYSTEM_CLOCK(start_tick)
           call SHMEM_INT8_WAIT( status(to_pe), MPP_WAIT )
           status(to_pe) = MPP_WAIT !prohibit puts to to_pe until it has retrieved this message
-          if( current_clock.NE.0 )call increment_current_clock(EVENT_WAIT)
+          if( debug .and. (current_clock.NE.0) )call increment_current_clock(EVENT_WAIT)
 #ifdef __ia64
           data_loc = shmem_ptr(put_data,pe)
 !          write(0,*)'pe, data_loc, loc(put_data)=', pe, data_loc, loc(put_data)
 #else
           data_loc = LOC(put_data)
 #endif
-          if( current_clock.NE.0 )call SYSTEM_CLOCK(start_tick)
+          if( debug .and. (current_clock.NE.0) )call SYSTEM_CLOCK(start_tick)
           call SHMEM_INTEGER_PUT( mpp_from_pe, pe, 1, to_pe )
           call SHMEM_PUT8( remote_data_loc(pe), data_loc, 1, to_pe )
-          if( current_clock.NE.0 )call increment_current_clock( EVENT_SEND, put_len*MPP_TYPE_BYTELEN_ )
+          if( debug .and. (current_clock.NE.0) )call increment_current_clock( EVENT_SEND, put_len*MPP_TYPE_BYTELEN_ )
       else if( to_pe.EQ.ALL_PES )then !this is a broadcast from from_pe
           if( from_pe.LT.0 .OR. from_pe.GE.npes )call mpp_error( FATAL, 'MPP_TRANSMIT: broadcasting from invalid PE.' )
           if( put_len.GT.get_len )call mpp_error( FATAL, 'MPP_TRANSMIT: size mismatch between put_data and get_data.' )
@@ -92,10 +92,10 @@
 
       else if( to_pe.EQ.ANY_PE )then !we don't have a destination to do puts to, so only do gets
           if( from_pe.LT.0 .OR. from_pe.GE.npes )call mpp_error( FATAL, 'MPP_TRANSMIT: invalid from_pe along with to_pe=ANY_PE.' )
-          if( current_clock.NE.0 )call SYSTEM_CLOCK(start_tick)
+          if( debug .and. (current_clock.NE.0) )call SYSTEM_CLOCK(start_tick)
           call SHMEM_GET_( get_data, put_data, get_len, from_pe )
           call SHMEM_PUT8( status(pe), MPP_READY, 1, from_pe ) !tell from_pe that you have retrieved this message
-          if( current_clock.NE.0 )call increment_current_clock( EVENT_RECV, get_len*MPP_TYPE_BYTELEN_ )
+          if( debug .and. (current_clock.NE.0) )call increment_current_clock( EVENT_RECV, get_len*MPP_TYPE_BYTELEN_ )
           return
       else if( to_pe.NE.NULL_PE )then  !no other valid cases except NULL_PE
           call mpp_error( FATAL, 'MPP_TRANSMIT: invalid to_pe.' )
@@ -106,14 +106,14 @@
 #ifdef _CRAYT90
           call SHMEM_UDCFLUSH !invalidate data cache
 #endif
-          if( current_clock.NE.0 )call SYSTEM_CLOCK(start_tick)
+          if( debug .and. (current_clock.NE.0) )call SYSTEM_CLOCK(start_tick)
           if( debug )write( errunit,* )'pe, from_pe, remote_data_loc(from_pe)=', pe, from_pe, remote_data_loc(from_pe)
           call SHMEM_INT8_WAIT( remote_data_loc(from_pe), MPP_WAIT )
-          if( current_clock.NE.0 )call increment_current_clock(EVENT_WAIT)
+          if( debug .and. (current_clock.NE.0) )call increment_current_clock(EVENT_WAIT)
           ptr_remote_data = remote_data_loc(from_pe)
           remote_data_loc(from_pe) = MPP_WAIT !reset
 !          call SHMEM_PUT8( status(pe), MPP_READY, 1, from_pe ) !tell from_pe we have retrieved the location
-          if( current_clock.NE.0 )call SYSTEM_CLOCK(start_tick)
+          if( debug .and. (current_clock.NE.0) )call SYSTEM_CLOCK(start_tick)
 #if defined(CRAYPVP) || defined(sgi_mipspro) || defined(__ia64)
 !since we have the pointer to remote data, just retrieve it with a simple copy
           if( LOC(get_data).NE.LOC(remote_data) )then
@@ -128,7 +128,7 @@
           call SHMEM_GET_( get_data, remote_data, get_len, from_pe )
 #endif
           call SHMEM_PUT8( status(pe), MPP_READY, 1, from_pe ) !tell from_pe we have retrieved the location
-          if( current_clock.NE.0 )call increment_current_clock( EVENT_RECV, get_len*MPP_TYPE_BYTELEN_ )
+          if( debug .and. (current_clock.NE.0) )call increment_current_clock( EVENT_RECV, get_len*MPP_TYPE_BYTELEN_ )
 
       else if( from_pe.EQ.ANY_PE )then
 #ifdef _CRAYT90
@@ -137,15 +137,15 @@
 !since we don't know which PE is sending us data, we wait for remote PE to send us its ID
 !this is only required for !CRAYPVP  && !sgi_mipspro, but is done there too, so that we can send put_is_done back.
           call shmem_integer_wait( mpp_from_pe, ANY_PE )
-          if( current_clock.NE.0 )call SYSTEM_CLOCK(start_tick)
+          if( debug .and. (current_clock.NE.0) )call SYSTEM_CLOCK(start_tick)
           call SHMEM_INT8_WAIT( remote_data_loc(mpp_from_pe), MPP_WAIT )
-          if( current_clock.NE.0 )call increment_current_clock(EVENT_WAIT)
+          if( debug .and. (current_clock.NE.0) )call increment_current_clock(EVENT_WAIT)
           ptr_remote_data = remote_data_loc(mpp_from_pe)
           remote_data_loc(mpp_from_pe) = MPP_WAIT !reset
           call SHMEM_PUT8( status(pe), MPP_READY, 1, mpp_from_pe ) !tell mpp_from_pe we have retrieved the location
 #if defined(CRAYPVP) || defined(sgi_mipspro) || defined(__ia64)
 !since we have the pointer to remote data, just retrieve it with a simple copy
-          if( current_clock.NE.0 )call SYSTEM_CLOCK(start_tick)
+          if( debug .and. (current_clock.NE.0) )call SYSTEM_CLOCK(start_tick)
           if( LOC(get_data).NE.LOC(remote_data) )then
 !dir$ IVDEP
               do i = 1,get_len
@@ -155,7 +155,7 @@
 #else
           call SHMEM_GET_( get_data, remote_data, get_len, mpp_from_pe )
 #endif
-          if( current_clock.NE.0 )call increment_current_clock( EVENT_RECV, get_len*MPP_TYPE_BYTELEN_ )
+          if( debug .and. (current_clock.NE.0) )call increment_current_clock( EVENT_RECV, get_len*MPP_TYPE_BYTELEN_ )
           mpp_from_pe = ANY_PE   !reset
       else if( from_pe.EQ.ALL_PES )then
           call mpp_error( FATAL, 'MPP_TRANSMIT: from_pe=ALL_PES has ambiguous meaning, and hence is not implemented.' )
@@ -208,7 +208,7 @@
       if( .NOT.ANY(from_pe.EQ.peset(current_peset_num)%list) ) &
            call mpp_error( FATAL, 'MPP_BROADCAST: broadcasting from invalid PE.' )
 
-      if( current_clock.NE.0 )call SYSTEM_CLOCK(start_tick)
+      if( debug .and. (current_clock.NE.0) )call SYSTEM_CLOCK(start_tick)
       ptr = LOC(mpp_stack)
       words = size(bdata(:))*size(transfer(bdata(1),word))
       if( words.GT.mpp_stack_size )then
@@ -232,7 +232,7 @@
              data(i) = bdata(i)
           end do
       end if
-      if( current_clock.NE.0 )call increment_current_clock( EVENT_BROADCAST, length*MPP_TYPE_BYTELEN_ )
+      if( debug .and. (current_clock.NE.0) )call increment_current_clock( EVENT_BROADCAST, length*MPP_TYPE_BYTELEN_ )
       return
     end subroutine MPP_BROADCAST_
 
diff --git a/src/shared/mpp/include/mpp_update_domains2D.h b/src/shared/mpp/include/mpp_update_domains2D.h
index a03ac2b203..521706e22d 100644
--- a/src/shared/mpp/include/mpp_update_domains2D.h
+++ b/src/shared/mpp/include/mpp_update_domains2D.h
@@ -1,6 +1,6 @@
 ! -*-f90-*-
     subroutine MPP_UPDATE_DOMAINS_2D_( field, domain, flags, complete, position, &
-                                       whalo, ehalo, shalo, nhalo, name, tile_count, buffer)
+                                       whalo, ehalo, shalo, nhalo, name, tile_count)
 !updates data domain of 2D field whose computational domains have been computed
       MPP_TYPE_,        intent(inout)        :: field(:,:)
       type(domain2D),   intent(inout)        :: domain  
@@ -10,18 +10,17 @@
       integer,          intent(in), optional :: whalo, ehalo, shalo, nhalo
       character(len=*), intent(in), optional :: name
       integer,          intent(in), optional :: tile_count
-      MPP_TYPE_,     intent(inout), optional :: buffer(:)
 
       MPP_TYPE_ :: field3D(size(field,1),size(field,2),1)
       pointer( ptr, field3D )
       ptr = LOC(field)
       call mpp_update_domains( field3D, domain, flags, complete, position, &
-                               whalo, ehalo, shalo, nhalo, name, tile_count, buffer )
+                               whalo, ehalo, shalo, nhalo, name, tile_count )
       return
     end subroutine MPP_UPDATE_DOMAINS_2D_
 
     subroutine MPP_UPDATE_DOMAINS_3D_( field, domain, flags, complete, position, &
-                                       whalo, ehalo, shalo, nhalo, name, tile_count, buffer )
+                                       whalo, ehalo, shalo, nhalo, name, tile_count)
 !updates data domain of 3D field whose computational domains have been computed
       MPP_TYPE_,        intent(inout)        :: field(:,:,:)
       type(domain2D),   intent(inout)        :: domain  
@@ -31,17 +30,15 @@
       integer,          intent(in), optional :: whalo, ehalo, shalo, nhalo ! specify halo region to be updated.
       character(len=*), intent(in), optional :: name
       integer,          intent(in), optional :: tile_count
-      MPP_TYPE_,     intent(inout), optional :: buffer(:)
 
       integer                 :: update_position, update_whalo, update_ehalo, update_shalo, update_nhalo, ntile
 
       integer(LONG_KIND),dimension(MAX_DOMAIN_FIELDS, MAX_TILES),save :: f_addrs=-9999
-      integer(LONG_KIND),dimension(MAX_DOMAIN_FIELDS, MAX_TILES),save :: b_addrs=-9999
-      integer          :: tile, buffer_size, max_ntile
+      integer          :: tile, max_ntile
       character(len=3) :: text
       logical          :: set_mismatch, is_complete
       logical          :: do_update
-      integer, save    :: isize=0, jsize=0, ke=0, l_size=0, bsize=0, list=0
+      integer, save    :: isize=0, jsize=0, ke=0, l_size=0, list=0
       integer, save    :: pos, whalosz, ehalosz, shalosz, nhalosz
       MPP_TYPE_        :: d_type
       type(overlapSpec), pointer :: update => NULL()
@@ -107,17 +104,11 @@
          call mpp_error(FATAL,'MPP_UPDATE_3D: MAX_DOMAIN_FIELDS='//text//' exceeded for group update.' )
       endif
       f_addrs(list, tile) = LOC(field)
-      buffer_size = 0
-      if(present(buffer)) then
-         buffer_size = size(buffer(:))
-         b_addrs(list, tile) = LOC(buffer)
-      end if
       update_position = CENTER
       if(present(position)) update_position = position
       if(list == 1 .AND. tile == 1 )then
          isize=size(field,1); jsize=size(field,2); ke = size(field,3); pos = update_position
          whalosz = update_whalo; ehalosz = update_ehalo; shalosz = update_shalo; nhalosz = update_nhalo
-         bsize = buffer_size
       else
          set_mismatch = .false.
          set_mismatch = set_mismatch .OR. (isize /= size(field,1))
@@ -128,7 +119,6 @@
          set_mismatch = set_mismatch .OR. (update_ehalo /= ehalosz)
          set_mismatch = set_mismatch .OR. (update_shalo /= shalosz)
          set_mismatch = set_mismatch .OR. (update_nhalo /= nhalosz)
-         set_mismatch = set_mismatch .OR. (buffer_size  /= bsize)
          if(set_mismatch)then
             write( text,'(i2)' ) list
             call mpp_error(FATAL,'MPP_UPDATE_3D: Incompatible field at count '//text//' for group update.' )
@@ -152,21 +142,21 @@
             !                    b_addrs(1:l_size,1:ntile), bsize, flags)
 
             if ( PRESENT ( flags ) ) then
-                call mpp_do_update( f_addrs(1:l_size,1:ntile), domain, update, d_type, ke, b_addrs(1:l_size,1:ntile), bsize, flags )
+                call mpp_do_update( f_addrs(1:l_size,1:ntile), domain, update, d_type, ke, flags )
             else
-                call mpp_do_update( f_addrs(1:l_size,1:ntile), domain, update, d_type, ke, b_addrs(1:l_size,1:ntile), bsize )
+                call mpp_do_update( f_addrs(1:l_size,1:ntile), domain, update, d_type, ke )
             endif    
                 
 
          end if
-         l_size=0; f_addrs=-9999; bsize=0; b_addrs=-9999; isize=0;  jsize=0;  ke=0
+         l_size=0; f_addrs=-9999; isize=0;  jsize=0;  ke=0
       endif
       return
 
     end subroutine MPP_UPDATE_DOMAINS_3D_
 
     subroutine MPP_UPDATE_DOMAINS_4D_( field, domain, flags, complete, position, &
-                                       whalo, ehalo, shalo, nhalo, name, tile_count, buffer )
+                                       whalo, ehalo, shalo, nhalo, name, tile_count )
 !updates data domain of 4D field whose computational domains have been computed
       MPP_TYPE_,        intent(inout)        :: field(:,:,:,:)
       type(domain2D),   intent(inout)        :: domain  
@@ -176,18 +166,17 @@
       integer,          intent(in), optional :: whalo, ehalo, shalo, nhalo
       character(len=*), intent(in), optional :: name
       integer,          intent(in), optional :: tile_count
-      MPP_TYPE_,     intent(inout), optional :: buffer(:)
 
       MPP_TYPE_ :: field3D(size(field,1),size(field,2),size(field,3)*size(field,4))
       pointer( ptr, field3D )
       ptr = LOC(field)
       call mpp_update_domains( field3D, domain, flags, complete, position, &
-                               whalo, ehalo, shalo, nhalo, name, tile_count, buffer)
+                               whalo, ehalo, shalo, nhalo, name, tile_count)
       return
     end subroutine MPP_UPDATE_DOMAINS_4D_
 
     subroutine MPP_UPDATE_DOMAINS_5D_( field, domain, flags, complete, position, &
-                                       whalo, ehalo, shalo, nhalo, name, tile_count, buffer )
+                                       whalo, ehalo, shalo, nhalo, name, tile_count )
 !updates data domain of 5D field whose computational domains have been computed
       MPP_TYPE_,        intent(inout)        :: field(:,:,:,:,:)
       type(domain2D),   intent(inout)        :: domain  
@@ -197,14 +186,13 @@
       integer,          intent(in), optional :: whalo, ehalo, shalo, nhalo
       character(len=*), intent(in), optional :: name
       integer,          intent(in), optional :: tile_count
-      MPP_TYPE_,     intent(inout), optional :: buffer(:)
 
       MPP_TYPE_ :: field3D(size(field,1),size(field,2),size(field,3)*size(field,4)*size(field,5))
 
       pointer( ptr, field3D )
       ptr = LOC(field)
       call mpp_update_domains( field3D, domain, flags, complete, position, &
-                               whalo, ehalo, shalo, nhalo, name, tile_count, buffer )
+                               whalo, ehalo, shalo, nhalo, name, tile_count )
       return
     end subroutine MPP_UPDATE_DOMAINS_5D_
 
@@ -220,8 +208,11 @@
       type(DomainCommunicator2D),pointer,optional :: dc_handle
       pointer( ptr_in,  field3D_in  )
       pointer( ptr_out, field3D_out )
-      ptr_in  = LOC(field_in )
-      ptr_out = LOC(field_out)
+
+      ptr_in = 0
+      ptr_out = 0
+      if(domain_in%initialized) ptr_in  = LOC(field_in )
+      if(domain_out%initialized) ptr_out = LOC(field_out)
       call mpp_redistribute( domain_in, field3D_in, domain_out, field3D_out, complete, free, list_size, dc_handle, position )
 
       return
@@ -334,8 +325,11 @@
       type(DomainCommunicator2D),pointer,optional :: dc_handle
       pointer( ptr_in,  field3D_in  )
       pointer( ptr_out, field3D_out )
-      ptr_in  = LOC(field_in )
-      ptr_out = LOC(field_out)
+
+      ptr_in = 0
+      ptr_out = 0
+      if(domain_in%initialized) ptr_in  = LOC(field_in )
+      if(domain_out%initialized) ptr_out = LOC(field_out)
       call mpp_redistribute( domain_in, field3D_in, domain_out, field3D_out, complete, free, list_size, dc_handle, position  )
 
       return
@@ -354,8 +348,11 @@
       type(DomainCommunicator2D),pointer,optional :: dc_handle
       pointer( ptr_in,  field3D_in  )
       pointer( ptr_out, field3D_out )
-      ptr_in  = LOC(field_in )
-      ptr_out = LOC(field_out)
+
+      ptr_in = 0
+      ptr_out = 0
+      if(domain_in%initialized) ptr_in  = LOC(field_in )
+      if(domain_out%initialized) ptr_out = LOC(field_out)
       call mpp_redistribute( domain_in, field3D_in, domain_out, field3D_out, complete, free, list_size, dc_handle, position  )
 
       return
@@ -366,7 +363,7 @@
 !VECTOR_FIELD_ is set to false for MPP_TYPE_ integer.
 !vector fields
     subroutine MPP_UPDATE_DOMAINS_2D_V_( fieldx, fieldy, domain, flags, gridtype, complete, &
-                                         whalo, ehalo, shalo, nhalo, name, tile_count, bufferx, buffery)
+                                         whalo, ehalo, shalo, nhalo, name, tile_count)
 !updates data domain of 2D field whose computational domains have been computed
       MPP_TYPE_,        intent(inout)        :: fieldx(:,:), fieldy(:,:)
       type(domain2D),   intent(inout)        :: domain
@@ -375,7 +372,6 @@
       integer,          intent(in), optional :: whalo, ehalo, shalo, nhalo
       character(len=*), intent(in), optional :: name
       integer,          intent(in), optional :: tile_count
-      MPP_TYPE_,     intent(inout), optional :: bufferx(:), buffery(:)
 
       MPP_TYPE_ :: field3Dx(size(fieldx,1),size(fieldx,2),1)
       MPP_TYPE_ :: field3Dy(size(fieldy,1),size(fieldy,2),1)
@@ -384,13 +380,13 @@
       ptrx = LOC(fieldx)
       ptry = LOC(fieldy)
       call mpp_update_domains( field3Dx, field3Dy, domain, flags, gridtype, complete, &
-                               whalo, ehalo, shalo, nhalo, name, tile_count, bufferx, buffery )
+                               whalo, ehalo, shalo, nhalo, name, tile_count)
       return
     end subroutine MPP_UPDATE_DOMAINS_2D_V_
 
 
     subroutine MPP_UPDATE_DOMAINS_3D_V_( fieldx, fieldy, domain, flags, gridtype, complete, &
-                                         whalo, ehalo, shalo, nhalo, name, tile_count, bufferx, buffery )
+                                         whalo, ehalo, shalo, nhalo, name, tile_count)
 !updates data domain of 3D field whose computational domains have been computed
       MPP_TYPE_,        intent(inout)        :: fieldx(:,:,:), fieldy(:,:,:)
       type(domain2D),   intent(inout)        :: domain
@@ -399,19 +395,16 @@
       integer,          intent(in), optional :: whalo, ehalo, shalo, nhalo
       character(len=*), intent(in), optional :: name
       integer,          intent(in), optional :: tile_count
-      MPP_TYPE_,     intent(inout), optional :: bufferx(:), buffery(:)
 
       integer                                :: update_whalo, update_ehalo, update_shalo, update_nhalo, ntile    
       integer                                :: grid_offset_type
       logical                                :: exchange_uv
         
       integer(LONG_KIND),dimension(MAX_DOMAIN_FIELDS, MAX_TILES),save :: f_addrsx=-9999, f_addrsy=-9999
-      integer(LONG_KIND),dimension(MAX_DOMAIN_FIELDS, MAX_TILES),save :: b_addrsx=-9999, b_addrsy=-9999
       logical          :: do_update, is_complete
       integer, save    :: isize(2)=0,jsize(2)=0,ke=0,l_size=0, offset_type=0, list=0
       integer, save    :: whalosz, ehalosz, shalosz, nhalosz
-      integer, save    :: bsizex=1, bsizey=1
-      integer          :: bufferx_size, buffery_size, tile, max_ntile
+      integer          :: tile, max_ntile
       integer          :: position_x, position_y
       logical          :: set_mismatch
       character(len=3) :: text
@@ -490,20 +483,12 @@
 
       f_addrsx(list, tile) = LOC(fieldx)
       f_addrsy(list, tile) = LOC(fieldy)
-      bufferx_size = 1; buffery_size = 1
-      if(present(bufferx)) then
-         bufferx_size = size(bufferx(:))
-         buffery_size = size(buffery(:))
-         b_addrsx(list, tile) = LOC(bufferx)
-         b_addrsy(list, tile) = LOC(buffery)
-      end if
 
       if(list == 1 .AND. tile == 1)then
          isize(1)=size(fieldx,1); jsize(1)=size(fieldx,2); ke = size(fieldx,3)
          isize(2)=size(fieldy,1); jsize(2)=size(fieldy,2)
          offset_type = grid_offset_type
          whalosz = update_whalo; ehalosz = update_ehalo; shalosz = update_shalo; nhalosz = update_nhalo
-         bsizex = bufferx_size; bsizey = buffery_size
       else
          set_mismatch = .false.
          set_mismatch = set_mismatch .OR. (isize(1) /= size(fieldx,1))
@@ -517,8 +502,6 @@
          set_mismatch = set_mismatch .OR. (update_ehalo /= ehalosz)
          set_mismatch = set_mismatch .OR. (update_shalo /= shalosz)
          set_mismatch = set_mismatch .OR. (update_nhalo /= nhalosz)
-         set_mismatch = set_mismatch .OR. (bufferx_size  /= bsizex)
-         set_mismatch = set_mismatch .OR. (buffery_size  /= bsizey)
          if(set_mismatch)then
             write( text,'(i2)' ) list
             call mpp_error(FATAL,'MPP_UPDATE_3D_V: Incompatible field at count '//text//' for group vector update.' )
@@ -561,16 +544,13 @@
             updatey => search_update_overlap(domain, update_whalo, update_ehalo, update_shalo, update_nhalo, position_y)
             if(exchange_uv) then
                call mpp_do_update(f_addrsx(1:l_size,1:ntile),f_addrsy(1:l_size,1:ntile), domain, updatey, updatex, &
-                    d_type,ke, b_addrsx(1:l_size,1:ntile), b_addrsy(1:l_size,1:ntile),                             &
-                    bsizex, bsizey, grid_offset_type, flags)
+                    d_type,ke, grid_offset_type, flags)
             else
                call mpp_do_update(f_addrsx(1:l_size,1:ntile),f_addrsy(1:l_size,1:ntile), domain, updatex, updatey, &
-                    d_type,ke, b_addrsx(1:l_size,1:ntile), b_addrsy(1:l_size,1:ntile),                             &
-                    bsizex, bsizey, grid_offset_type, flags)
+                    d_type,ke,grid_offset_type, flags)
             end if
          end if
          l_size=0; f_addrsx=-9999; f_addrsy=-9999; isize=0;  jsize=0;  ke=0
-         bsizex=1; b_addrsx=-9999; bsizey=1; b_addrsy=-9999;
       end if
 
       return
@@ -578,7 +558,7 @@
 
 
     subroutine MPP_UPDATE_DOMAINS_4D_V_( fieldx, fieldy, domain, flags, gridtype, complete, &
-                                         whalo, ehalo, shalo, nhalo, name, tile_count, bufferx, buffery )
+                                         whalo, ehalo, shalo, nhalo, name, tile_count )
 !updates data domain of 4D field whose computational domains have been computed
       MPP_TYPE_,        intent(inout)        :: fieldx(:,:,:,:), fieldy(:,:,:,:)
       type(domain2D),   intent(inout)        :: domain
@@ -587,7 +567,6 @@
       integer,          intent(in), optional :: whalo, ehalo, shalo, nhalo
       character(len=*), intent(in), optional :: name
       integer,          intent(in), optional :: tile_count
-      MPP_TYPE_,     intent(inout), optional :: bufferx(:), buffery(:)
 
       MPP_TYPE_ :: field3Dx(size(fieldx,1),size(fieldx,2),size(fieldx,3)*size(fieldx,4))
       MPP_TYPE_ :: field3Dy(size(fieldy,1),size(fieldy,2),size(fieldy,3)*size(fieldy,4))
@@ -597,12 +576,12 @@
       ptrx = LOC(fieldx)
       ptry = LOC(fieldy)
       call mpp_update_domains( field3Dx, field3Dy, domain, flags, gridtype, complete, &
-                               whalo, ehalo, shalo, nhalo, name, tile_count, bufferx, buffery )
+                               whalo, ehalo, shalo, nhalo, name, tile_count)
       return
     end subroutine MPP_UPDATE_DOMAINS_4D_V_
 
     subroutine MPP_UPDATE_DOMAINS_5D_V_( fieldx, fieldy, domain, flags, gridtype, complete, &
-                                         whalo, ehalo, shalo, nhalo, name, tile_count, bufferx, buffery )
+                                         whalo, ehalo, shalo, nhalo, name, tile_count )
 !updates data domain of 5D field whose computational domains have been computed
       MPP_TYPE_,        intent(inout)        :: fieldx(:,:,:,:,:), fieldy(:,:,:,:,:)
       type(domain2D),   intent(inout)        :: domain
@@ -611,7 +590,6 @@
       integer,          intent(in), optional :: whalo, ehalo, shalo, nhalo
       character(len=*), intent(in), optional :: name
       integer,          intent(in), optional :: tile_count
-      MPP_TYPE_,     intent(inout), optional :: bufferx(:), buffery(:)
 
       MPP_TYPE_ :: field3Dx(size(fieldx,1),size(fieldx,2),size(fieldx,3)*size(fieldx,4)*size(fieldx,5))
       MPP_TYPE_ :: field3Dy(size(fieldy,1),size(fieldy,2),size(fieldy,3)*size(fieldy,4)*size(fieldy,5))
@@ -620,7 +598,7 @@
       ptrx = LOC(fieldx)
       ptry = LOC(fieldy)
       call mpp_update_domains( field3Dx, field3Dy, domain, flags, gridtype, complete, &
-                               whalo, ehalo, shalo, nhalo, name, tile_count, bufferx, buffery )
+                               whalo, ehalo, shalo, nhalo, name, tile_count)
 
       return
     end subroutine MPP_UPDATE_DOMAINS_5D_V_
diff --git a/src/shared/mpp/include/mpp_update_domains2D_nonblock.h b/src/shared/mpp/include/mpp_update_domains2D_nonblock.h
index 19c2d0736b..ece7fd0abd 100644
--- a/src/shared/mpp/include/mpp_update_domains2D_nonblock.h
+++ b/src/shared/mpp/include/mpp_update_domains2D_nonblock.h
@@ -159,8 +159,10 @@ function MPP_START_UPDATE_DOMAINS_3D_( field, domain, flags, position, &
   end if
   
   if(do_update) then
-     num_update = num_update + 1
+    if(num_nonblock_group_update>0) call mpp_error(FATAL, "MPP_START_UPDATE_DOMAINS: "// &
+          " can not be called in the middle of mpp_start_group_update/mpp_complete_group_update call")
 
+     num_update = num_update + 1
      if( PRESENT(update_id) ) then
         if( update_id < 1 .OR. update_id > MAX_NONBLOCK_UPDATE ) then
            write( text,'(a,i8,a,i8)' ) 'optional argument update_id =', update_id, &
@@ -267,7 +269,7 @@ end function MPP_START_UPDATE_DOMAINS_5D_
 
 !##################################################################################
 subroutine MPP_COMPLETE_UPDATE_DOMAINS_2D_( id_update, field, domain, flags, position, &
-                                            whalo, ehalo, shalo, nhalo, name, tile_count, buffer, complete )
+                                            whalo, ehalo, shalo, nhalo, name, tile_count, complete )
   integer,          intent(in)           :: id_update
   type(domain2D),   intent(inout)        :: domain  
   MPP_TYPE_,        intent(inout)        :: field(:,:)
@@ -277,19 +279,18 @@ subroutine MPP_COMPLETE_UPDATE_DOMAINS_2D_( id_update, field, domain, flags, pos
   character(len=*), intent(in), optional :: name
   integer,          intent(in), optional :: tile_count
   logical,          intent(in), optional :: complete
-  MPP_TYPE_,     intent(inout), optional :: buffer(:)
 
   MPP_TYPE_ :: field3D(size(field,1),size(field,2),1)
   pointer( ptr, field3D )
   ptr = LOC(field)
   call mpp_complete_update_domains(id_update, field3D, domain, flags, position, &
-                                   whalo, ehalo, shalo, nhalo, name, tile_count, buffer, complete )
+                                   whalo, ehalo, shalo, nhalo, name, tile_count, complete )
 
 end subroutine MPP_COMPLETE_UPDATE_DOMAINS_2D_
 
 !##################################################################################
 subroutine MPP_COMPLETE_UPDATE_DOMAINS_3D_( id_update, field, domain, flags, position, &
-                                            whalo, ehalo, shalo, nhalo, name, tile_count, buffer, complete )
+                                            whalo, ehalo, shalo, nhalo, name, tile_count, complete )
   integer,          intent(in)           :: id_update
   type(domain2D),   intent(inout)        :: domain  
   MPP_TYPE_,        intent(inout)        :: field(domain%x(1)%data%begin:,domain%y(1)%data%begin:,:)
@@ -299,7 +300,6 @@ subroutine MPP_COMPLETE_UPDATE_DOMAINS_3D_( id_update, field, domain, flags, pos
   character(len=*), intent(in), optional :: name
   integer,          intent(in), optional :: tile_count
   logical,          intent(in), optional :: complete
-  MPP_TYPE_,     intent(inout), optional :: buffer(:)
 
 
   integer                    :: update_whalo, update_ehalo, update_shalo, update_nhalo
@@ -308,14 +308,12 @@ subroutine MPP_COMPLETE_UPDATE_DOMAINS_3D_( id_update, field, domain, flags, pos
   integer                    :: tile, max_ntile, ntile, n
   logical                    :: is_complete
   logical                    :: do_update
-  integer                    :: ke_max, buffer_size
-  integer, save              :: list=0, bsize=0, l_size=0
+  integer                    :: ke_max
+  integer, save              :: list=0, l_size=0
   integer, save              :: ke_list(MAX_DOMAIN_FIELDS, MAX_TILES)=0
   integer(LONG_KIND), save   :: f_addrs(MAX_DOMAIN_FIELDS, MAX_TILES)=-9999
-  integer(LONG_KIND), save   :: b_addrs(MAX_DOMAIN_FIELDS, MAX_TILES)=-9999
   character(len=128)         :: text
-
-      MPP_TYPE_        :: d_type
+  MPP_TYPE_        :: d_type
 
   if(present(whalo)) then
      update_whalo = whalo
@@ -383,14 +381,7 @@ subroutine MPP_COMPLETE_UPDATE_DOMAINS_3D_( id_update, field, domain, flags, pos
      endif
   endif
   
-  buffer_size = 0
-  if(present(buffer)) then
-     buffer_size = size(buffer(:))
-     b_addrs(list, tile) = LOC(buffer)
-  end if
-
   ke_list(list,tile) = size(field,3)
-  bsize = max(bsize, buffer_size)
 
   !check to make sure the consistency of halo size, position and flags.
   if( nonblock_data(id_update)%update_flags .NE. update_flags ) call mpp_error(FATAL, "MPP_COMPLETE_UPDATE_DOMAINS_3D: "// &
@@ -419,11 +410,11 @@ subroutine MPP_COMPLETE_UPDATE_DOMAINS_3D_( id_update, field, domain, flags, pos
         update => search_update_overlap(domain, update_whalo, update_ehalo, update_shalo, update_nhalo, update_position)
         ke_max = maxval(ke_list(1:l_size,1:ntile))
         call mpp_complete_do_update(id_update, f_addrs(1:l_size,1:ntile), domain, update, d_type, &
-                                    ke_max, ke_list(1:l_size,1:ntile), b_addrs(1:l_size,1:ntile), bsize, update_flags) 
+                                    ke_max, ke_list(1:l_size,1:ntile), update_flags) 
      endif
      nonblock_data(id_update)%nfields = 0
      nonblock_data(id_update)%field_addrs(1:l_size) = 0
-     l_size=0; f_addrs=-9999; bsize=0; b_addrs=-9999; ke_list=0
+     l_size=0; f_addrs=-9999; ke_list=0
      !--- For the last call of mpp_complete_update_domains
      !--- reset everything to init state
      if( num_update == 0) then
@@ -439,7 +430,7 @@ end subroutine MPP_COMPLETE_UPDATE_DOMAINS_3D_
 
 !##################################################################################
 subroutine MPP_COMPLETE_UPDATE_DOMAINS_4D_( id_update, field, domain, flags, position, &
-                                            whalo, ehalo, shalo, nhalo, name, tile_count, buffer, complete )
+                                            whalo, ehalo, shalo, nhalo, name, tile_count, complete )
   integer,          intent(in)           :: id_update
   type(domain2D),   intent(inout)        :: domain  
   MPP_TYPE_,        intent(inout)        :: field(:,:,:,:)
@@ -449,19 +440,18 @@ subroutine MPP_COMPLETE_UPDATE_DOMAINS_4D_( id_update, field, domain, flags, pos
   character(len=*), intent(in), optional :: name
   integer,          intent(in), optional :: tile_count
   logical,          intent(in), optional :: complete
-  MPP_TYPE_,     intent(inout), optional :: buffer(:)
 
   MPP_TYPE_ :: field3D(size(field,1),size(field,2),size(field,3)*size(field,4))
   pointer( ptr, field3D )
   ptr = LOC(field)
   call mpp_complete_update_domains(id_update, field3D, domain, flags, position, &
-                                   whalo, ehalo, shalo, nhalo, name, tile_count, buffer, complete )
+                                   whalo, ehalo, shalo, nhalo, name, tile_count, complete )
 
 end subroutine MPP_COMPLETE_UPDATE_DOMAINS_4D_
 
 !##################################################################################
 subroutine MPP_COMPLETE_UPDATE_DOMAINS_5D_( id_update, field, domain, flags, position, &
-                                            whalo, ehalo, shalo, nhalo, name, tile_count, buffer, complete )
+                                            whalo, ehalo, shalo, nhalo, name, tile_count, complete )
   integer,          intent(in)           :: id_update
   type(domain2D),   intent(inout)        :: domain  
   MPP_TYPE_,        intent(inout)        :: field(:,:,:,:,:)
@@ -471,13 +461,12 @@ subroutine MPP_COMPLETE_UPDATE_DOMAINS_5D_( id_update, field, domain, flags, pos
   character(len=*), intent(in), optional :: name
   integer,          intent(in), optional :: tile_count
   logical,          intent(in), optional :: complete
-  MPP_TYPE_,     intent(inout), optional :: buffer(:)
 
   MPP_TYPE_ :: field3D(size(field,1),size(field,2),size(field,3)*size(field,4)*size(field,5))
   pointer( ptr, field3D )
   ptr = LOC(field)
   call mpp_complete_update_domains(id_update, field3D, domain, flags, position, &
-                                   whalo, ehalo, shalo, nhalo, name, tile_count, buffer, complete )
+                                   whalo, ehalo, shalo, nhalo, name, tile_count, complete )
 
 end subroutine MPP_COMPLETE_UPDATE_DOMAINS_5D_
 
@@ -656,6 +645,8 @@ function MPP_START_UPDATE_DOMAINS_3D_V_( fieldx, fieldy, domain, flags, gridtype
      list = 0
   end if
   if(do_update)then
+    if(num_nonblock_group_update>0) call mpp_error(FATAL, "MPP_START_UPDATE_DOMAINS_V: "// &       
+         " can not be called in the middle of mpp_start_group_update/mpp_complete_group_update call")
      num_update = num_update + 1
      if( PRESENT(update_id) ) then
         reuse_id_update = .true.
@@ -715,7 +706,7 @@ function MPP_START_UPDATE_DOMAINS_3D_V_( fieldx, fieldy, domain, flags, gridtype
            position_x = EAST
            position_y = NORTH
         case default
-           call mpp_error(FATAL, "mpp_update_domains2D.h: invalid value of grid_offset_type")
+           call mpp_error(FATAL, "mpp_update_domains2D_nonblock.h: invalid value of grid_offset_type")
         end select
         updatex => search_update_overlap(domain, update_whalo, update_ehalo, update_shalo, update_nhalo, position_x)
         updatey => search_update_overlap(domain, update_whalo, update_ehalo, update_shalo, update_nhalo, position_y)
@@ -798,7 +789,7 @@ end function MPP_START_UPDATE_DOMAINS_5D_V_
 
 !####################################################################################
 subroutine MPP_COMPLETE_UPDATE_DOMAINS_2D_V_( id_update, fieldx, fieldy, domain, flags, gridtype, &
-     whalo, ehalo, shalo, nhalo, name, tile_count, bufferx, buffery, complete )
+     whalo, ehalo, shalo, nhalo, name, tile_count, complete )
   !updates data domain of 3D field whose computational domains have been computed
   integer,          intent(in)           :: id_update
   MPP_TYPE_,        intent(inout)        :: fieldx(:,:), fieldy(:,:)
@@ -809,7 +800,6 @@ subroutine MPP_COMPLETE_UPDATE_DOMAINS_2D_V_( id_update, fieldx, fieldy, domain,
   integer,          intent(in), optional :: tile_count
   logical,          intent(in), optional :: complete
 
-  MPP_TYPE_,     intent(inout), optional :: bufferx(:), buffery(:)
   MPP_TYPE_ :: field3Dx(size(fieldx,1),size(fieldx,2),1)
   MPP_TYPE_ :: field3Dy(size(fieldy,1),size(fieldy,2),1)
   pointer( ptrx, field3Dx )
@@ -818,7 +808,7 @@ subroutine MPP_COMPLETE_UPDATE_DOMAINS_2D_V_( id_update, fieldx, fieldy, domain,
   ptry = LOC(fieldy)
 
   call mpp_complete_update_domains(id_update, field3Dx, field3Dy, domain, flags, gridtype, &
-     whalo, ehalo, shalo, nhalo, name, tile_count, bufferx, buffery, complete )
+     whalo, ehalo, shalo, nhalo, name, tile_count, complete )
 
   return
 
@@ -826,7 +816,7 @@ end subroutine MPP_COMPLETE_UPDATE_DOMAINS_2D_V_
 
 !####################################################################################
 subroutine MPP_COMPLETE_UPDATE_DOMAINS_3D_V_( id_update, fieldx, fieldy, domain, flags, gridtype, &
-     whalo, ehalo, shalo, nhalo, name, tile_count, bufferx, buffery, complete )
+     whalo, ehalo, shalo, nhalo, name, tile_count, complete )
   !updates data domain of 3D field whose computational domains have been computed
   integer,          intent(in)           :: id_update
   MPP_TYPE_,        intent(inout)        :: fieldx(:,:,:), fieldy(:,:,:)
@@ -835,22 +825,18 @@ subroutine MPP_COMPLETE_UPDATE_DOMAINS_3D_V_( id_update, fieldx, fieldy, domain,
   integer,          intent(in), optional :: whalo, ehalo, shalo, nhalo
   character(len=*), intent(in), optional :: name
   integer,          intent(in), optional :: tile_count
-  MPP_TYPE_,     intent(inout), optional :: bufferx(:), buffery(:)
   logical,          intent(in), optional :: complete
 
   integer                     :: update_whalo, update_ehalo, update_shalo, update_nhalo
   integer                     :: grid_offset_type, position_x, position_y, update_flags
   logical                     :: do_update, is_complete
   integer                     :: ntile, max_ntile, tile, ke_max, n
-  integer                     :: bufferx_size, buffery_size
   logical                     :: exchange_uv
   character(len=128)          :: text
-  integer,            save    :: l_size=0, list=0, bsizex=1, bsizey=1
+  integer,            save    :: l_size=0, list=0
   integer,            save    :: ke_list (MAX_DOMAIN_FIELDS, MAX_TILES)=0
   integer(LONG_KIND), save    :: f_addrsx(MAX_DOMAIN_FIELDS, MAX_TILES)=-9999
   integer(LONG_KIND), save    :: f_addrsy(MAX_DOMAIN_FIELDS, MAX_TILES)=-9999
-  integer(LONG_KIND) ,save    :: b_addrsx(MAX_DOMAIN_FIELDS, MAX_TILES)=-9999
-  integer(LONG_KIND) ,save    :: b_addrsy(MAX_DOMAIN_FIELDS, MAX_TILES)=-9999
   type(overlapSpec),  pointer :: updatex => NULL()
   type(overlapSpec),  pointer :: updatey => NULL()
   MPP_TYPE_                   :: d_type
@@ -941,17 +927,7 @@ subroutine MPP_COMPLETE_UPDATE_DOMAINS_3D_V_( id_update, fieldx, fieldy, domain,
      endif
   endif
 
-  bufferx_size = 1; buffery_size = 1
-  if(present(bufferx)) then
-     bufferx_size = size(bufferx(:))
-     buffery_size = size(buffery(:))
-     b_addrsx(list, tile) = LOC(bufferx)
-     b_addrsy(list, tile) = LOC(buffery)
-  end if
-
   ke_list(list, tile) = size(fieldx,3)
-  bsizex = max(bsizex, bufferx_size)
-  bsizey = max(bsizey, buffery_size)
 
   if(is_complete) then
      l_size = list
@@ -991,12 +967,10 @@ subroutine MPP_COMPLETE_UPDATE_DOMAINS_3D_V_( id_update, fieldx, fieldy, domain,
         if(exchange_uv) then
            call mpp_complete_do_update(id_update, f_addrsx(1:l_size,1:ntile), f_addrsy(1:l_size,1:ntile), domain, &
                                        updatey, updatex, d_type, ke_max, ke_list(1:l_size,1:ntile),               &
-                                       b_addrsx(1:l_size,1:ntile), b_addrsy(1:l_size,1:ntile), bsizex, bsizey,    &
                                        grid_offset_type, update_flags)
         else
            call mpp_complete_do_update(id_update, f_addrsx(1:l_size,1:ntile), f_addrsy(1:l_size,1:ntile), domain, &
                                        updatex, updatey, d_type, ke_max, ke_list(1:l_size,1:ntile),               &
-                                       b_addrsx(1:l_size,1:ntile), b_addrsy(1:l_size,1:ntile), bsizex, bsizey,    &
                                        grid_offset_type, update_flags)
         endif
      endif
@@ -1004,7 +978,6 @@ subroutine MPP_COMPLETE_UPDATE_DOMAINS_3D_V_( id_update, fieldx, fieldy, domain,
      nonblock_data(id_update)%field_addrs(1:l_size) = 0
      nonblock_data(id_update)%field_addrs2(1:l_size) = 0
      l_size=0; f_addrsx=-9999; f_addrsy=-9999; ke_list=0
-     bsizex=1; b_addrsx=-9999; bsizey=1; b_addrsy=-9999
      !--- For the last call of mpp_complete_update_domains
      !--- reset everything to init state
      if( num_update == 0) then
@@ -1021,7 +994,7 @@ end subroutine MPP_COMPLETE_UPDATE_DOMAINS_3D_V_
 
 !####################################################################################
 subroutine MPP_COMPLETE_UPDATE_DOMAINS_4D_V_( id_update, fieldx, fieldy, domain, flags, gridtype, &
-     whalo, ehalo, shalo, nhalo, name, tile_count, bufferx, buffery, complete )
+     whalo, ehalo, shalo, nhalo, name, tile_count, complete )
   !updates data domain of 3D field whose computational domains have been computed
   integer,          intent(in)           :: id_update
   MPP_TYPE_,        intent(inout)        :: fieldx(:,:,:,:), fieldy(:,:,:,:)
@@ -1032,7 +1005,6 @@ subroutine MPP_COMPLETE_UPDATE_DOMAINS_4D_V_( id_update, fieldx, fieldy, domain,
   integer,          intent(in), optional :: tile_count
   logical,          intent(in), optional :: complete
 
-  MPP_TYPE_,     intent(inout), optional :: bufferx(:), buffery(:)
   MPP_TYPE_ :: field3Dx(size(fieldx,1),size(fieldx,2),size(fieldx,3)*size(fieldx,4))
   MPP_TYPE_ :: field3Dy(size(fieldy,1),size(fieldy,2),size(fieldy,3)*size(fieldy,4))
   pointer( ptrx, field3Dx )
@@ -1041,7 +1013,7 @@ subroutine MPP_COMPLETE_UPDATE_DOMAINS_4D_V_( id_update, fieldx, fieldy, domain,
   ptry = LOC(fieldy)
 
   call mpp_complete_update_domains(id_update, field3Dx, field3Dy, domain, flags, gridtype, &
-     whalo, ehalo, shalo, nhalo, name, tile_count, bufferx, buffery, complete )
+     whalo, ehalo, shalo, nhalo, name, tile_count, complete )
 
   return
 
@@ -1049,7 +1021,7 @@ end subroutine MPP_COMPLETE_UPDATE_DOMAINS_4D_V_
 
 !####################################################################################
 subroutine MPP_COMPLETE_UPDATE_DOMAINS_5D_V_( id_update, fieldx, fieldy, domain, flags, gridtype, &
-     whalo, ehalo, shalo, nhalo, name, tile_count, bufferx, buffery, complete )
+     whalo, ehalo, shalo, nhalo, name, tile_count, complete )
   !updates data domain of 3D field whose computational domains have been computed
   integer,          intent(in)           :: id_update
   MPP_TYPE_,        intent(inout)        :: fieldx(:,:,:,:,:), fieldy(:,:,:,:,:)
@@ -1060,7 +1032,6 @@ subroutine MPP_COMPLETE_UPDATE_DOMAINS_5D_V_( id_update, fieldx, fieldy, domain,
   integer,          intent(in), optional :: tile_count
   logical,          intent(in), optional :: complete
 
-  MPP_TYPE_,     intent(inout), optional :: bufferx(:), buffery(:)
   MPP_TYPE_ :: field3Dx(size(fieldx,1),size(fieldx,2),size(fieldx,3)*size(fieldx,4)*size(fieldx,5))
   MPP_TYPE_ :: field3Dy(size(fieldy,1),size(fieldy,2),size(fieldy,3)*size(fieldy,4)*size(fieldy,5))
   pointer( ptrx, field3Dx )
@@ -1069,7 +1040,7 @@ subroutine MPP_COMPLETE_UPDATE_DOMAINS_5D_V_( id_update, fieldx, fieldy, domain,
   ptry = LOC(fieldy)
 
   call mpp_complete_update_domains(id_update, field3Dx, field3Dy, domain, flags, gridtype, &
-     whalo, ehalo, shalo, nhalo, name, tile_count, bufferx, buffery, complete )
+     whalo, ehalo, shalo, nhalo, name, tile_count, complete )
 
   return
 
diff --git a/src/shared/mpp/include/mpp_util.inc b/src/shared/mpp/include/mpp_util.inc
index 290ecf1ac3..7230f989d1 100644
--- a/src/shared/mpp/include/mpp_util.inc
+++ b/src/shared/mpp/include/mpp_util.inc
@@ -1,6 +1,6 @@
 
 ! -*-f90-*-
-! $Id: mpp_util.inc,v 20.0 2013/12/14 00:27:35 fms Exp $
+! $Id$
 
 
 #ifdef use_libSMA
@@ -816,10 +816,12 @@ end function rarray_to_char
        call mpp_sync()
     end if
 
-    num_clock_ids = num_clock_ids+1
-    if(num_clock_ids > MAX_CLOCKS)call mpp_error(FATAL,'MPP_CLOCK_BEGIN: max num previous_clock exceeded.' )
-    previous_clock(num_clock_ids) = current_clock
-    current_clock = id
+    if (debug) then
+      num_clock_ids = num_clock_ids+1
+      if(num_clock_ids > MAX_CLOCKS)call mpp_error(FATAL,'MPP_CLOCK_BEGIN: max num previous_clock exceeded.' )
+      previous_clock(num_clock_ids) = current_clock
+      current_clock = id
+    endif
     call SYSTEM_CLOCK( clocks(id)%tick )
     clocks(id)%is_on = .true.
 !$OMP END MASTER
@@ -851,11 +853,11 @@ end function rarray_to_char
        call mpp_error( WARNING, 'MPP_CLOCK_END: Clock rollover, assumed single roll.' )
     end if
     clocks(id)%total_ticks = clocks(id)%total_ticks + delta
-    if(num_clock_ids < 1)then
-       call mpp_error(FATAL,'MPP_CLOCK_END: min num previous_clock < 1.' )
+    if (debug) then
+      if(num_clock_ids < 1) call mpp_error(NOTE,'MPP_CLOCK_END: min num previous_clock < 1.' )
+      current_clock = previous_clock(num_clock_ids)
+      num_clock_ids = num_clock_ids-1
     endif
-    current_clock = previous_clock(num_clock_ids)
-    num_clock_ids = num_clock_ids-1
     clocks(id)%is_on = .false.
 !$OMP END MASTER
     return
@@ -885,7 +887,7 @@ end function rarray_to_char
     integer                       :: errunit
 
     if( .not. mpp_record_timing_data )return
-    if( current_clock.EQ.0 )return
+    if( .not.debug .or. (current_clock.EQ.0) )return
     if( current_clock.LT.0 .OR. current_clock.GT.clock_num )call mpp_error( FATAL, 'MPP_CLOCK_BEGIN: invalid current_clock.' )
     if( .NOT.clocks(current_clock)%detailed )return
     call SYSTEM_CLOCK(end_tick)
@@ -1259,7 +1261,7 @@ end function rarray_to_char
 
 !-----------------------------------------------------------------------
 !
-! AUTHOR: Rusty Benson (rusty.benson)
+! AUTHOR: Rusty Benson (rusty.benson@noaa.gov)
 !
 !
 ! THESE LINES MUST BE PRESENT IN MPP.F90
@@ -1284,8 +1286,8 @@ end function rarray_to_char
   subroutine read_input_nml()
 
 ! version and tagging information
-    character(len=128) :: version = '$Id: mpp_util.inc,v 20.0 2013/12/14 00:27:35 fms Exp $'
-    character(len=128) :: tagname = '$Name: tikal $' 
+    character(len=128) :: version = '$Id$'
+    character(len=128) :: tagname = '$Name$' 
 
 ! private variables
     integer :: log_unit
@@ -1388,8 +1390,8 @@ end function rarray_to_char
 
   !-----------------------------------------------------------------------
   !
-  ! AUTHOR: Rusty Benson , 
-  !         Seth Underwood 
+  ! AUTHOR: Rusty Benson , 
+  !         Seth Underwood 
   !
   !-----------------------------------------------------------------------
   ! subroutine READ_ASCII_FILE
@@ -1415,8 +1417,8 @@ end function rarray_to_char
     integer, intent(in), optional,   dimension(:) :: PELIST
 
     ! version and tagging information
-    character(len=*), parameter :: VERSION = '$Id: mpp_util.inc,v 20.0 2013/12/14 00:27:35 fms Exp $'
-    character(len=*), parameter :: TAGNAME = '$Name: tikal $' 
+    character(len=*), parameter :: VERSION = '$Id$'
+    character(len=*), parameter :: TAGNAME = '$Name$' 
 
     character(len=5) :: text
     logical :: file_exist
diff --git a/src/shared/mpp/include/mpp_util_mpi.inc b/src/shared/mpp/include/mpp_util_mpi.inc
index 758e413052..0e8e2b9c24 100644
--- a/src/shared/mpp/include/mpp_util_mpi.inc
+++ b/src/shared/mpp/include/mpp_util_mpi.inc
@@ -1,5 +1,5 @@
 ! -*-f90-*-
-! $Id: mpp_util_mpi.inc,v 20.0 2013/12/14 00:27:37 fms Exp $
+! $Id$
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 !                                                                             !
@@ -29,7 +29,7 @@ subroutine mpp_error_basic( errortype, errormsg )
      text = 'WARNING: non-existent errortype (must be NOTE|WARNING|FATAL)'
   end select
 
-  if( npes.GT.1 )write( text,'(a,i5)' )trim(text)//' from PE', pe   !this is the mpp part
+  if( npes.GT.1 )write( text,'(a,i6)' )trim(text)//' from PE', pe   !this is the mpp part
   if( PRESENT(errormsg) )text = trim(text)//': '//trim(errormsg)
 
   out_unit = stdout()
@@ -144,7 +144,7 @@ subroutine mpp_sync( pelist, do_self )
 
   n = get_peset(pelist); if( peset(n)%count.EQ.1 )return
 
-  if( current_clock.NE.0 )call SYSTEM_CLOCK(start_tick)
+  if( debug .and. (current_clock.NE.0) )call SYSTEM_CLOCK(start_tick)
 #ifdef use_MPI_SMA
   if( n.EQ.world_peset_num )then
      call SHMEM_BARRIER_ALL() !special call is faster
@@ -155,7 +155,7 @@ subroutine mpp_sync( pelist, do_self )
   call MPI_BARRIER( peset(n)%id, error )
 #endif
 
-  if( current_clock.NE.0 )call increment_current_clock(EVENT_WAIT)
+  if( debug .and. (current_clock.NE.0) )call increment_current_clock(EVENT_WAIT)
 
   return
 end subroutine mpp_sync
@@ -173,7 +173,7 @@ subroutine mpp_sync_self( pelist, check, request, msg_size, msg_type)
 
   integer                       :: i, m, n, stride, my_check, rsize
 
-  if( current_clock.NE.0 )call SYSTEM_CLOCK(start_tick)
+  if( debug .and. (current_clock.NE.0) )call SYSTEM_CLOCK(start_tick)
   my_check = EVENT_SEND
   if(present(check)) my_check = check
   if( my_check .NE. EVENT_SEND .AND. my_check .NE. EVENT_RECV ) then
@@ -230,7 +230,7 @@ subroutine mpp_sync_self( pelist, check, request, msg_size, msg_type)
         cur_recv_request = 0
      end select 
   endif
-  if( current_clock.NE.0 )call increment_current_clock(EVENT_WAIT)
+  if( debug .and. (current_clock.NE.0) )call increment_current_clock(EVENT_WAIT)
   return
 end subroutine mpp_sync_self
 
diff --git a/src/shared/mpp/include/mpp_util_nocomm.inc b/src/shared/mpp/include/mpp_util_nocomm.inc
index d8a81ee407..289cf4cb09 100644
--- a/src/shared/mpp/include/mpp_util_nocomm.inc
+++ b/src/shared/mpp/include/mpp_util_nocomm.inc
@@ -1,6 +1,6 @@
 
 ! -*-f90-*-
-! $Id: mpp_util_nocomm.inc,v 20.0 2013/12/14 00:27:39 fms Exp $
+! $Id$
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 !                                                                             !
diff --git a/src/shared/mpp/include/mpp_util_sma.inc b/src/shared/mpp/include/mpp_util_sma.inc
index addfd6b3db..125c56615d 100644
--- a/src/shared/mpp/include/mpp_util_sma.inc
+++ b/src/shared/mpp/include/mpp_util_sma.inc
@@ -1,6 +1,6 @@
 
 ! -*-f90-*-
-! $Id: mpp_util_sma.inc,v 20.0 2013/12/14 00:27:41 fms Exp $
+! $Id$
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 !                                                                             !
diff --git a/src/shared/mpp/include/mpp_write.h b/src/shared/mpp/include/mpp_write.h
index 79a2d9c08e..8c44e2a455 100644
--- a/src/shared/mpp/include/mpp_write.h
+++ b/src/shared/mpp/include/mpp_write.h
@@ -1,4 +1,4 @@
-    subroutine MPP_WRITE_( unit, field, data, tstamp)
+subroutine MPP_WRITE_( unit, field, data, tstamp)
       use mpp_parameter_mod, only : NULLUNIT
       integer, intent(in) :: unit
       type(fieldtype), intent(in) :: field
@@ -8,4 +8,4 @@
       if (unit == NULLUNIT) return
       MPP_WRITE_RECORD_
       return
-    end subroutine MPP_WRITE_
+    end subroutine MPP_WRITE_
\ No newline at end of file
diff --git a/src/shared/mpp/include/mpp_write_compressed.h b/src/shared/mpp/include/mpp_write_compressed.h
new file mode 100644
index 0000000000..51dbfa14e1
--- /dev/null
+++ b/src/shared/mpp/include/mpp_write_compressed.h
@@ -0,0 +1,101 @@
+    subroutine MPP_WRITE_COMPRESSED_1D_(unit, field, domain, data, nelems_io, tstamp, default_data)
+      integer, intent(in) :: unit
+      type(fieldtype), intent(inout) :: field
+      type(domain2D), intent(inout) :: domain
+      MPP_TYPE_, intent(inout) :: data(:)
+      integer, intent(in) :: nelems_io(:)  ! number of compressed elements
+      real,              intent(in), optional :: tstamp
+      MPP_TYPE_,         intent(in), optional :: default_data
+
+      MPP_TYPE_ :: data2D(size(data,1),1)
+      pointer( ptr, data2D )
+      ptr = LOC(data)
+
+      call mpp_write_compressed(unit, field, domain, data2D, nelems_io, tstamp, default_data)
+      return
+    end subroutine MPP_WRITE_COMPRESSED_1D_
+
+    subroutine MPP_WRITE_COMPRESSED_2D_(unit, field, domain, data, nelems_io, tstamp, default_data)
+      integer,           intent(in)           :: unit
+      type(fieldtype),   intent(inout)        :: field
+      type(domain2D),    intent(inout)        :: domain
+      MPP_TYPE_,         intent(inout)        :: data(:,:)
+      integer,           intent(in)           :: nelems_io(:)  ! number of compressed elements from each
+                                                               ! member of the io_domain. It MUST have the
+                                                               ! same order as the io_domain pelist.
+      real,              intent(in), optional :: tstamp
+      MPP_TYPE_,         intent(in), optional :: default_data
+
+!cdata is used to store the io-domain compressed data
+      MPP_TYPE_, allocatable, dimension(:,:) :: cdata
+      MPP_TYPE_, allocatable, dimension(:,:) :: sbuff,rbuff
+      MPP_TYPE_                            :: fill
+
+      MPP_TYPE_ :: sbuff1D(size(data))
+      MPP_TYPE_ :: rbuff1D(size(data,2)*sum(nelems_io(:)))
+      pointer(sptr,sbuff1D); pointer(rptr,rbuff1D)
+      integer, allocatable :: pelist(:)
+      integer, allocatable :: nz_gather(:)
+      integer :: i,j,nz,nelems,mynelems,idx,npes
+      type(domain2d), pointer :: io_domain=>NULL()
+
+      call mpp_clock_begin(mpp_write_clock)
+
+      if( .NOT.module_is_initialized )call mpp_error( FATAL, 'MPP_WRITE_COMPRESSED_2D_: must first call mpp_io_init.' )
+      if( .NOT.mpp_file(unit)%valid )call mpp_error( FATAL, 'MPP_WRITE_COMPRESSED_2D_: invalid unit number.' )
+
+      fill = 0
+      if(PRESENT(default_data)) fill = default_data
+
+      io_domain=>mpp_get_io_domain(domain) 
+      if (.not. ASSOCIATED(io_domain)) call mpp_error( FATAL, 'MPP_WRITE_COMPRESSED_2D_: io_domain must be defined.' )
+      npes = mpp_get_domain_npes(io_domain)
+      allocate(pelist(npes))
+      call mpp_get_pelist(io_domain,pelist)
+
+      mynelems = size(data,1)
+      nz = size(data,2)
+      nelems = sum(nelems_io(:))
+      ! Check that nz is consistent across all PEs in io_domain
+      allocate(nz_gather(npes))
+      call mpp_gather((/nz/), nz_gather, pelist)
+      if ( mpp_file(unit)%write_on_this_pe.and.maxloc(nz_gather,1).ne.minloc(nz_gather,1) ) then
+         call mpp_error( FATAL, 'MPP_WRITE_COMPRESSED_2D_: size(data,2) must be consistent across all PEs in io_domain' )
+      end if
+      deallocate(nz_gather)
+
+      if(mpp_file(unit)%write_on_this_pe ) allocate(rbuff(nz,nelems))
+      allocate(sbuff(nz,mynelems))
+
+      ! this matrix inversion makes for easy gather to the IO root
+      ! and a clear, concise unpack
+      do j=1,mynelems
+        do i=1,nz
+          sbuff(i,j) = data(j,i) 
+      enddo; enddo
+
+   !  Note that the gatherV implied here is asymmetric; only root needs to know the vector of recv size
+      sptr = LOC(sbuff)
+      rptr = 0
+      if(mpp_file(unit)%write_on_this_pe ) rptr = LOC(rbuff)
+      call mpp_gather(sbuff1D,size(sbuff),rbuff1D,nz*nelems_io(:),pelist)
+
+      if(mpp_file(unit)%write_on_this_pe ) then
+         allocate(cdata(nelems,nz))
+         cdata = fill
+         do j=1,nz
+           do i=1,nelems
+             cdata(i,j) = rbuff(j,i)
+         enddo; enddo
+         ! cludge for now; need resizing accessor
+         field%size(1) = nelems
+         call write_record( unit, field, nelems*nz, cdata, tstamp)
+         deallocate(rbuff,cdata)
+      endif
+
+      deallocate(sbuff,pelist)
+
+      call mpp_clock_end(mpp_write_clock)
+
+      return
+    end subroutine MPP_WRITE_COMPRESSED_2D_
diff --git a/src/shared/mpp/include/mpp_write_unlimited_axis.h b/src/shared/mpp/include/mpp_write_unlimited_axis.h
new file mode 100644
index 0000000000..5b4026fc54
--- /dev/null
+++ b/src/shared/mpp/include/mpp_write_unlimited_axis.h
@@ -0,0 +1,41 @@
+    subroutine MPP_WRITE_UNLIMITED_AXIS_1D_(unit,field,domain,data,nelems_io)
+      integer,           intent(in)           :: unit
+      type(fieldtype),   intent(inout)        :: field
+      type(domain2D),    intent(inout)        :: domain
+      MPP_TYPE_,         intent(inout)        :: data(:)
+      integer,           intent(in)           :: nelems_io(:)  ! number of compressed elements from each
+                                                               ! member of the io_domain. It MUST have the
+                                                               ! same order as the io_domain pelist.
+      integer, allocatable :: pelist(:)
+      integer :: i,j,nelems,npes
+      type(domain2d), pointer :: io_domain=>NULL()
+      MPP_TYPE_, allocatable, dimension(:) :: rbuff
+
+      call mpp_clock_begin(mpp_write_clock)
+
+      if( .NOT.module_is_initialized )call mpp_error( FATAL, 'MPP_WRITE_UNLIMITED_AXIS_1D_: must first call mpp_io_init.' )
+      if( .NOT.mpp_file(unit)%valid )call mpp_error( FATAL, 'MPP_WRITE_UNLIMITED_AXIS_1D_: invalid unit number.' )
+
+      io_domain=>mpp_get_io_domain(domain) 
+      if (.not. ASSOCIATED(io_domain)) call mpp_error( FATAL, 'MPP_WRITE_UNLIMITED_AXIS_1D_: io_domain must be defined.' )
+      npes = mpp_get_domain_npes(io_domain)
+      allocate(pelist(npes))
+      call mpp_get_pelist(io_domain,pelist)
+
+      nelems = sum(nelems_io(:))
+
+      if(mpp_file(unit)%write_on_this_pe) allocate(rbuff(nelems))
+
+   !  Note that the gatherV implied here is asymmetric; only root needs to know the vector of recv size
+      call mpp_gather(data,size(data),rbuff,nelems_io(:),pelist)
+
+      if(mpp_file(unit)%write_on_this_pe) then
+         field%size(1) = nelems  ! Correct the field size now that we have all the data
+         call write_record(unit, field, nelems, rbuff)
+         deallocate(rbuff)
+      endif
+      deallocate(pelist)
+
+      call mpp_clock_end(mpp_write_clock)
+      return
+    end subroutine MPP_WRITE_UNLIMITED_AXIS_1D_
diff --git a/src/shared/mpp/mpp.F90 b/src/shared/mpp/mpp.F90
index f21bf39435..26bb9ed54e 100644
--- a/src/shared/mpp/mpp.F90
+++ b/src/shared/mpp/mpp.F90
@@ -1,7 +1,7 @@
 !-----------------------------------------------------------------------
 !                 Communication for message-passing codes
 !
-! AUTHOR: V. Balaji (V.Balaji)
+! AUTHOR: V. Balaji (V.Balaji@noaa.gov)
 !         SGI/GFDL Princeton University
 !
 ! This program is free software; you can redistribute it and/or modify
@@ -21,12 +21,14 @@
 module mpp_mod
 !a generalized communication package for use with shmem and MPI
 !will add: co_array_fortran, MPI2
-!Balaji (V.Balaji) 11 May 1998
+!Balaji (V.Balaji@noaa.gov) 11 May 1998
 
-! 
+! 
 !   V. Balaji
 ! 
 
+! 
+! 
 
 ! 
 !   mpp_mod, is a set of simple calls to provide a uniform interface
@@ -163,8 +165,8 @@ module mpp_mod
   use mpp_parameter_mod, only : CLOCK_MODULE, CLOCK_ROUTINE, CLOCK_LOOP, CLOCK_INFRA
   use mpp_parameter_mod, only : MAX_EVENTS, MAX_BINS, MAX_EVENT_TYPES, MAX_CLOCKS
   use mpp_parameter_mod, only : MAXPES, EVENT_WAIT, EVENT_ALLREDUCE, EVENT_BROADCAST
-  use mpp_parameter_mod, only : EVENT_RECV, EVENT_SEND, MPP_READY, MPP_WAIT
   use mpp_parameter_mod, only : EVENT_ALLTOALL
+  use mpp_parameter_mod, only : EVENT_RECV, EVENT_SEND, MPP_READY, MPP_WAIT
   use mpp_parameter_mod, only : mpp_parameter_version=>version, mpp_parameter_tagname=>tagname
   use mpp_parameter_mod, only : DEFAULT_TAG
   use mpp_parameter_mod, only : COMM_TAG_1,  COMM_TAG_2,  COMM_TAG_3,  COMM_TAG_4
@@ -172,6 +174,7 @@ module mpp_mod
   use mpp_parameter_mod, only : COMM_TAG_9,  COMM_TAG_10, COMM_TAG_11, COMM_TAG_12
   use mpp_parameter_mod, only : COMM_TAG_13, COMM_TAG_14, COMM_TAG_15, COMM_TAG_16
   use mpp_parameter_mod, only : COMM_TAG_17, COMM_TAG_18, COMM_TAG_19, COMM_TAG_20
+  use mpp_parameter_mod, only : MPP_FILL_INT,MPP_FILL_DOUBLE
   use mpp_data_mod,      only : stat, mpp_stack, ptr_stack, status, ptr_status, sync, ptr_sync  
   use mpp_data_mod,      only : mpp_from_pe, ptr_from, remote_data_loc, ptr_remote
   use mpp_data_mod,      only : mpp_data_version=>version, mpp_data_tagname=>tagname
@@ -198,7 +201,7 @@ module mpp_mod
   public :: COMM_TAG_9,  COMM_TAG_10, COMM_TAG_11, COMM_TAG_12
   public :: COMM_TAG_13, COMM_TAG_14, COMM_TAG_15, COMM_TAG_16
   public :: COMM_TAG_17, COMM_TAG_18, COMM_TAG_19, COMM_TAG_20
-
+  public :: MPP_FILL_INT,MPP_FILL_DOUBLE
 
   !--- public data from mpp_data_mod ------------------------------
 !  public :: request
@@ -216,8 +219,7 @@ module mpp_mod
   !--- public interface from mpp_comm.h ------------------------------
   public :: mpp_chksum, mpp_max, mpp_min, mpp_sum, mpp_transmit, mpp_send, mpp_recv
   public :: mpp_broadcast, mpp_malloc, mpp_init, mpp_exit
-  public :: mpp_gather
-  public :: mpp_alltoall
+  public :: mpp_gather, mpp_scatter, mpp_alltoall
 #ifdef use_MPI_GSM
   public :: mpp_gsm_malloc, mpp_gsm_free
 #endif
@@ -663,8 +665,28 @@ module mpp_mod
      module procedure mpp_gather_int4_1dv
      module procedure mpp_gather_real4_1dv
      module procedure mpp_gather_real8_1dv
+     module procedure mpp_gather_pelist_int4_2d
+     module procedure mpp_gather_pelist_int4_3d
+     module procedure mpp_gather_pelist_real4_2d
+     module procedure mpp_gather_pelist_real4_3d
+     module procedure mpp_gather_pelist_real8_2d
+     module procedure mpp_gather_pelist_real8_3d
   end interface
 
+  !#####################################################################
+  ! 
+  !  
+  !    gather information onto root pe.
+  !  
+  ! 
+  interface mpp_scatter
+     module procedure mpp_scatter_pelist_int4_2d
+     module procedure mpp_scatter_pelist_int4_3d
+     module procedure mpp_scatter_pelist_real4_2d
+     module procedure mpp_scatter_pelist_real4_3d
+     module procedure mpp_scatter_pelist_real8_2d
+     module procedure mpp_scatter_pelist_real8_3d
+  end interface
 
   !#####################################################################
   ! 
@@ -684,6 +706,7 @@ module mpp_mod
      module procedure mpp_alltoall_real8_v
   end interface
 
+
   !#####################################################################
 
   ! 
@@ -1044,6 +1067,7 @@ module mpp_mod
   !     Integer checksums on FP data use the F90 TRANSFER()
   !     intrinsic.
   !
+  !     The serial checksum module is superseded
   !     by this function, and is no longer being actively maintained. This
   !     provides identical results on a single-processor job, and to perform
   !     serial checksums on a single processor of a parallel job, you only
@@ -1078,11 +1102,24 @@ module mpp_mod
      module procedure mpp_chksum_i8_2d
      module procedure mpp_chksum_i8_3d
      module procedure mpp_chksum_i8_4d
+     module procedure mpp_chksum_i8_5d
+     module procedure mpp_chksum_i8_1d_rmask
+     module procedure mpp_chksum_i8_2d_rmask
+     module procedure mpp_chksum_i8_3d_rmask
+     module procedure mpp_chksum_i8_4d_rmask
+     module procedure mpp_chksum_i8_5d_rmask
+
 #endif
      module procedure mpp_chksum_i4_1d
      module procedure mpp_chksum_i4_2d
      module procedure mpp_chksum_i4_3d
      module procedure mpp_chksum_i4_4d
+     module procedure mpp_chksum_i4_5d
+     module procedure mpp_chksum_i4_1d_rmask
+     module procedure mpp_chksum_i4_2d_rmask
+     module procedure mpp_chksum_i4_3d_rmask
+     module procedure mpp_chksum_i4_4d_rmask
+     module procedure mpp_chksum_i4_5d_rmask
      module procedure mpp_chksum_r8_0d
      module procedure mpp_chksum_r8_1d
      module procedure mpp_chksum_r8_2d
@@ -1207,7 +1244,7 @@ module mpp_mod
   character(len=128), public :: version= &
        '$Id mpp.F90 $'
   character(len=128), public :: tagname= &
-       '$Name: tikal $'
+       '$Name$'
 
   integer, parameter :: MAX_REQUEST_MIN  = 10000
   integer            :: request_multiply = 20
diff --git a/src/shared/mpp/mpp_data.F90 b/src/shared/mpp/mpp_data.F90
index cb3655c07c..6295467d51 100644
--- a/src/shared/mpp/mpp_data.F90
+++ b/src/shared/mpp/mpp_data.F90
@@ -13,7 +13,7 @@ module mpp_data_mod
   character(len=128), public :: version= &
        '$Id mpp_data.F90 $'
   character(len=128), public :: tagname= &
-       '$Name: tikal $'
+       '$Name$'
 
 #if defined(use_libSMA) || defined(use_MPI_SMA)
 #include 
diff --git a/src/shared/mpp/mpp_domains.F90 b/src/shared/mpp/mpp_domains.F90
index b3f063d3b6..6b9c8330e6 100644
--- a/src/shared/mpp/mpp_domains.F90
+++ b/src/shared/mpp/mpp_domains.F90
@@ -19,12 +19,14 @@
 !           675 Mass Ave, Cambridge, MA 02139, USA.  
 !-----------------------------------------------------------------------
 
-! 
+! 
 !   V. Balaji
 ! 
-! 
+! 
 !   Zhi Liang
 ! 
+! 
+! 
 
 ! 
 !   mpp_domains_mod is a set of simple calls for domain
@@ -120,7 +122,7 @@ module mpp_domains_mod
   use mpp_parameter_mod,      only : FOLD_WEST_EDGE, FOLD_EAST_EDGE, FOLD_SOUTH_EDGE, FOLD_NORTH_EDGE
   use mpp_parameter_mod,      only : WUPDATE, EUPDATE, SUPDATE, NUPDATE, XUPDATE, YUPDATE
   use mpp_parameter_mod,      only : NON_BITWISE_EXACT_SUM, BITWISE_EXACT_SUM, MPP_DOMAIN_TIME
-  use mpp_parameter_mod,      only : CENTER, CORNER, SCALAR_PAIR, SCALAR_BIT
+  use mpp_parameter_mod,      only : CENTER, CORNER, SCALAR_PAIR, SCALAR_BIT, BITWISE_EFP_SUM
   use mpp_parameter_mod,      only : NORTH, NORTH_EAST, EAST, SOUTH_EAST
   use mpp_parameter_mod,      only : SOUTH, SOUTH_WEST, WEST, NORTH_WEST
   use mpp_parameter_mod,      only : MAX_DOMAIN_FIELDS, NULL_PE, DOMAIN_ID_BASE
@@ -137,7 +139,8 @@ module mpp_domains_mod
   use mpp_mod,                only : input_nml_file
   use mpp_mod,                only : COMM_TAG_1, COMM_TAG_2, COMM_TAG_3, COMM_TAG_4
   use mpp_memutils_mod,       only : mpp_memuse_begin, mpp_memuse_end
-  use mpp_pset_mod, only: mpp_pset_init
+  use mpp_pset_mod,           only : mpp_pset_init
+  use mpp_efp_mod,            only : mpp_reproducing_sum
   implicit none
   private
 
@@ -150,7 +153,7 @@ module mpp_domains_mod
   public :: GLOBAL_DATA_DOMAIN, CYCLIC_GLOBAL_DOMAIN, BGRID_NE, BGRID_SW, CGRID_NE, CGRID_SW
   public :: DGRID_NE, DGRID_SW, FOLD_WEST_EDGE, FOLD_EAST_EDGE, FOLD_SOUTH_EDGE, FOLD_NORTH_EDGE
   public :: WUPDATE, EUPDATE, SUPDATE, NUPDATE, XUPDATE, YUPDATE
-  public :: NON_BITWISE_EXACT_SUM, BITWISE_EXACT_SUM, MPP_DOMAIN_TIME
+  public :: NON_BITWISE_EXACT_SUM, BITWISE_EXACT_SUM, MPP_DOMAIN_TIME, BITWISE_EFP_SUM
   public :: CENTER, CORNER, SCALAR_PAIR
   public :: NORTH, NORTH_EAST, EAST, SOUTH_EAST
   public :: SOUTH, SOUTH_WEST, WEST, NORTH_WEST
@@ -161,7 +164,7 @@ module mpp_domains_mod
   public :: NULL_DOMAIN1D, NULL_DOMAIN2D
 
   public :: domain_axis_spec, domain1D, domain2D, DomainCommunicator2D
-  public :: nest_domain_type
+  public :: nest_domain_type, mpp_group_update_type
 
   !--- public interface from mpp_domains_util.h
   public :: mpp_domains_set_stack_size, mpp_get_compute_domain, mpp_get_compute_domains
@@ -172,7 +175,6 @@ module mpp_domains_mod
   public :: mpp_set_compute_domain, mpp_set_data_domain, mpp_set_global_domain
   public :: mpp_get_memory_domain, mpp_get_domain_shift, mpp_domain_is_tile_root_pe
   public :: mpp_get_tile_id, mpp_get_domain_extents, mpp_get_current_ntile, mpp_get_ntile_count
-  public :: mpp_get_refine_overlap_number, mpp_get_mosaic_refine_overlap
   public :: mpp_get_tile_list
   public :: mpp_get_tile_npes, mpp_get_domain_root_pe, mpp_get_tile_pelist, mpp_get_tile_compute_domains
   public :: mpp_get_num_overlap, mpp_get_overlap
@@ -181,6 +183,8 @@ module mpp_domains_mod
   public :: mpp_copy_domain, mpp_set_domain_symmetry
   public :: mpp_get_update_pelist, mpp_get_update_size
   public :: mpp_get_domain_npes, mpp_get_domain_pelist
+  public :: mpp_clear_group_update
+  public :: mpp_group_update_initialized, mpp_group_update_is_set
 
   !--- public interface from mpp_domains_reduce.h
   public :: mpp_global_field, mpp_global_max, mpp_global_min, mpp_global_sum
@@ -189,21 +193,28 @@ module mpp_domains_mod
   public :: mpp_broadcast_domain, mpp_domains_init, mpp_domains_exit, mpp_redistribute
   public :: mpp_update_domains, mpp_check_field
   public :: mpp_start_update_domains, mpp_complete_update_domains
+  public :: mpp_create_group_update, mpp_do_group_update
+  public :: mpp_start_group_update, mpp_complete_group_update
+  public :: mpp_reset_group_update_field
   public :: mpp_update_nest_fine, mpp_update_nest_coarse
-!  public :: mpp_update_domains_ad   ! bnc
   public :: mpp_get_boundary
   !--- public interface from mpp_domains_define.h
   public :: mpp_define_layout, mpp_define_domains, mpp_modify_domain, mpp_define_mosaic
   public :: mpp_define_mosaic_pelist, mpp_define_null_domain, mpp_mosaic_defined
   public :: mpp_define_io_domain, mpp_deallocate_domain
-  public :: mpp_compute_extent
+  public :: mpp_compute_extent, mpp_compute_block_extent
 
   !--- public interface from mpp_define_domains.inc
   public :: mpp_define_nest_domains, mpp_get_C2F_index, mpp_get_F2C_index
 
   integer, parameter :: NAME_LENGTH = 64
   integer, parameter :: MAXLIST = 24
- 
+  integer, parameter :: MAXOVERLAP = 100
+  integer, parameter :: FIELD_S = 0
+  integer, parameter :: FIELD_X = 1
+  integer, parameter :: FIELD_Y = 2
+
+
   !--- data types used mpp_domains_mod.
   type domain_axis_spec        !type used to specify index limits along an axis of a domain
      private
@@ -250,13 +261,8 @@ module mpp_domains_mod
      integer,         pointer :: ie(:)           => NULL() ! ending   i-index 
      integer,         pointer :: js(:)           => NULL() ! starting j-index 
      integer,         pointer :: je(:)           => NULL() ! ending   j-index 
-     integer,         pointer :: isMe(:)         => NULL() ! starting i-index of my tile on current pe
-     integer,         pointer :: ieMe(:)         => NULL() ! ending   i-index of my tile on current pe
-     integer,         pointer :: jsMe(:)         => NULL() ! starting j-index of my tile on current pe
-     integer,         pointer :: jeMe(:)         => NULL() ! ending   j-index of my tile on current pe
      integer,         pointer :: dir(:)          => NULL() ! direction ( value 1,2,3,4 = E,S,W,N)
      integer,         pointer :: rotation(:)     => NULL() ! rotation angle.
-     logical,         pointer :: is_refined(:)   => NULL() ! indicate if the overlap is refined or not.
      integer,         pointer :: index(:)        => NULL() ! for refinement
      logical,         pointer :: from_contact(:) => NULL() ! indicate if the overlap is computed from define_contact_overlap
   end type overlap_type
@@ -269,7 +275,6 @@ module mpp_domains_mod
      integer                     :: sendsize, recvsize
      type(overlap_type), pointer :: send(:) => NULL()
      type(overlap_type), pointer :: recv(:) => NULL()
-     type(refineSpec),   pointer :: rSpec(:)=> NULL()
      type(overlapSpec),  pointer :: next
   end type overlapSpec
 
@@ -277,24 +282,6 @@ module mpp_domains_mod
      integer :: xbegin, xend, ybegin, yend
   end type tile_type
 
-  type refineSpec
-     private
-     integer          :: count                 ! number of ovrelapping
-     integer          :: total                 ! total number of points to be saved in buffer.
-     integer, pointer :: isMe(:)     => NULL() ! starting i-index on current pe and tile.
-     integer, pointer :: ieMe(:)     => NULL() ! ending i-index on current pe and tile.
-     integer, pointer :: jsMe(:)     => NULL() ! starting j-index on current pe and tile.
-     integer, pointer :: jeMe(:)     => NULL() ! ending j-index on current pe and tile.
-     integer, pointer :: isNbr(:)    => NULL() ! starting i-index on neighbor pe or tile
-     integer, pointer :: ieNbr(:)    => NULL() ! ending i-index on neighbor pe or tile
-     integer, pointer :: jsNbr(:)    => NULL() ! starting j-index on neighbor pe or tile
-     integer, pointer :: jeNbr(:)    => NULL() ! ending j-index on neighbor pe or tile
-     integer, pointer :: start(:)    => NULL() ! starting index in the buffer
-     integer, pointer :: end(:)      => NULL() ! ending index in the buffer
-     integer, pointer :: dir(:)      => NULL() ! direction 
-     integer, pointer :: rotation(:) => NULL() ! rotation angle.
-  end type refineSpec
-
 !domaintypes of higher rank can be constructed from type domain1D
 !typically we only need 1 and 2D, but could need higher (e.g 3D LES)
 !some elements are repeated below if they are needed once per domain, not once per axis
@@ -466,6 +453,60 @@ module mpp_domains_mod
      integer(LONG_KIND)              :: field_addrs2(MAX_DOMAIN_FIELDS)
      integer                         :: nfields 
   end type nonblock_type
+
+  type mpp_group_update_type
+     private
+     logical            :: initialized = .FALSE.
+     logical            :: k_loop_inside = .TRUE.
+     integer            :: nscalar = 0
+     integer            :: nvector = 0
+     integer            :: flags_s=0, flags_v=0
+     integer            :: whalo_s=0, ehalo_s=0, shalo_s=0, nhalo_s=0
+     integer            :: isize_s=0, jsize_s=0, ksize_s=1
+     integer            :: whalo_v=0, ehalo_v=0, shalo_v=0, nhalo_v=0
+     integer            :: isize_x=0, jsize_x=0, ksize_v=1
+     integer            :: isize_y=0, jsize_y=0
+     integer            :: position=0, gridtype=0
+     logical            :: recv_s(8), recv_v(8)
+     integer            :: is_s=0, ie_s=0, js_s=0, je_s=0
+     integer            :: is_x=0, ie_x=0, js_x=0, je_x=0
+     integer            :: is_y=0, ie_y=0, js_y=0, je_y=0
+     integer            :: nrecv=0, nsend=0
+     integer            :: npack=0, nunpack=0
+     integer            :: reset_index_s = 0
+     integer            :: reset_index_v = 0
+     integer            :: tot_msgsize = 0
+     integer            :: from_pe(MAXOVERLAP)
+     integer            :: to_pe(MAXOVERLAP)
+     integer            :: recv_size(MAXOVERLAP)
+     integer            :: send_size(MAXOVERLAP)
+     integer            :: buffer_pos_recv(MAXOVERLAP)
+     integer            :: buffer_pos_send(MAXOVERLAP)
+     integer            :: pack_type(MAXOVERLAP)
+     integer            :: pack_buffer_pos(MAXOVERLAP)
+     integer            :: pack_rotation(MAXOVERLAP)
+     integer            :: pack_size(MAXOVERLAP)
+     integer            :: pack_is(MAXOVERLAP)
+     integer            :: pack_ie(MAXOVERLAP)
+     integer            :: pack_js(MAXOVERLAP)
+     integer            :: pack_je(MAXOVERLAP)
+     integer            :: unpack_type(MAXOVERLAP)
+     integer            :: unpack_buffer_pos(MAXOVERLAP)
+     integer            :: unpack_rotation(MAXOVERLAP)
+     integer            :: unpack_size(MAXOVERLAP)
+     integer            :: unpack_is(MAXOVERLAP)
+     integer            :: unpack_ie(MAXOVERLAP)
+     integer            :: unpack_js(MAXOVERLAP)
+     integer            :: unpack_je(MAXOVERLAP)
+     integer(LONG_KIND) :: addrs_s(MAX_DOMAIN_FIELDS)
+     integer(LONG_KIND) :: addrs_x(MAX_DOMAIN_FIELDS)
+     integer(LONG_KIND) :: addrs_y(MAX_DOMAIN_FIELDS)
+     integer            :: buffer_start_pos = -1
+     integer            :: request_send(MAX_REQUEST)
+     integer            :: request_recv(MAX_REQUEST)
+     integer            :: type_recv(MAX_REQUEST)
+  end type mpp_group_update_type
+
 !#######################################################################
 
 !***********************************************************************
@@ -484,13 +525,16 @@ module mpp_domains_mod
   type(domain2D),save  :: NULL_DOMAIN2D
   integer              :: current_id_update = 0
   integer                         :: num_update = 0
+  integer                         :: num_nonblock_group_update = 0
   integer                         :: nonblock_buffer_pos = 0
+  integer                         :: nonblock_group_buffer_pos = 0
   logical                         :: start_update = .true.
   logical                         :: complete_update = .false.
   type(nonblock_type), allocatable :: nonblock_data(:)
   integer, parameter              :: MAX_NONBLOCK_UPDATE = 100
 
-
+  integer                         :: group_update_buffer_pos = 0
+  logical                         :: complete_group_update_on = .false.
   !-------- The following variables are used in mpp_domains_comm.h
   
   integer, parameter :: MAX_ADDRS=512
@@ -530,8 +574,6 @@ module mpp_domains_mod
   !     integer(LONG_KIND), parameter :: KE_BASE=2**48
   integer(LONG_KIND), parameter :: KE_BASE=Z'0001000000000000'  ! Workaround for 64bit int init problem
 
-  integer, parameter :: MAXOVERLAP = 100 
-
   integer(LONG_KIND) :: domain_cnt=0
 
   !--- the following variables are used in mpp_domains_misc.h
@@ -542,6 +584,9 @@ module mpp_domains_mod
   integer :: wait_clock_nonblock=0  
   integer :: nest_send_clock=0, nest_recv_clock=0, nest_unpk_clock=0
   integer :: nest_wait_clock=0, nest_pack_clock=0
+  integer :: group_recv_clock=0, group_send_clock=0, group_pack_clock=0, group_unpk_clock=0, group_wait_clock=0
+  integer :: nonblock_group_recv_clock=0, nonblock_group_send_clock=0, nonblock_group_pack_clock=0
+  integer :: nonblock_group_unpk_clock=0, nonblock_group_wait_clock=0
 
   !--- namelist interface
 ! 
@@ -553,10 +598,21 @@ module mpp_domains_mod
 !     processor/tile when updating doamin for symmetric domain and check the consistency on the north
 !     folded edge. 
 !   
+!   
+!     Set true to always do overflow_check when doing EFP bitwise mpp_global_sum. 
+!   
+!   
+!     Determine the loop order for packing and unpacking. When number of threads is greater than nthread_control_loop,
+!     k-loop will be moved outside and combined with number of pack and unpack. When number of threads is less 
+!     than or equal to nthread_control_loop, k-loop is moved inside but still outside of j,i loop.
+!   
 ! 
   character(len=32) :: debug_update_domain = "none"
   logical           :: debug_message_passing = .false.
-  namelist /mpp_domains_nml/ debug_update_domain, domain_clocks_on, debug_message_passing
+  integer           :: nthread_control_loop = 4
+  logical           :: efp_sum_overflow_check = .false.
+  namelist /mpp_domains_nml/ debug_update_domain, domain_clocks_on, debug_message_passing, nthread_control_loop, &
+                             efp_sum_overflow_check
 
   !***********************************************************************
 
@@ -1234,6 +1290,52 @@ module mpp_domains_mod
      module procedure mpp_complete_do_update_i4_3d
   end interface
 
+
+  interface mpp_create_group_update
+     module procedure mpp_create_group_update_r4_2d
+     module procedure mpp_create_group_update_r4_3d
+     module procedure mpp_create_group_update_r4_4d
+     module procedure mpp_create_group_update_r4_2dv
+     module procedure mpp_create_group_update_r4_3dv
+     module procedure mpp_create_group_update_r4_4dv
+     module procedure mpp_create_group_update_r8_2d
+     module procedure mpp_create_group_update_r8_3d
+     module procedure mpp_create_group_update_r8_4d
+     module procedure mpp_create_group_update_r8_2dv
+     module procedure mpp_create_group_update_r8_3dv
+     module procedure mpp_create_group_update_r8_4dv
+  end interface mpp_create_group_update
+
+  interface mpp_do_group_update
+     module procedure mpp_do_group_update_r4
+     module procedure mpp_do_group_update_r8
+  end interface mpp_do_group_update
+
+  interface mpp_start_group_update
+     module procedure mpp_start_group_update_r4
+     module procedure mpp_start_group_update_r8
+  end interface mpp_start_group_update
+
+  interface mpp_complete_group_update
+     module procedure mpp_complete_group_update_r4
+     module procedure mpp_complete_group_update_r8
+  end interface mpp_complete_group_update
+
+  interface mpp_reset_group_update_field
+     module procedure mpp_reset_group_update_field_r4_2d
+     module procedure mpp_reset_group_update_field_r4_3d
+     module procedure mpp_reset_group_update_field_r4_4d
+     module procedure mpp_reset_group_update_field_r4_2dv
+     module procedure mpp_reset_group_update_field_r4_3dv
+     module procedure mpp_reset_group_update_field_r4_4dv
+     module procedure mpp_reset_group_update_field_r8_2d
+     module procedure mpp_reset_group_update_field_r8_3d
+     module procedure mpp_reset_group_update_field_r8_4d
+     module procedure mpp_reset_group_update_field_r8_2dv
+     module procedure mpp_reset_group_update_field_r8_3dv
+     module procedure mpp_reset_group_update_field_r8_4dv
+  end interface mpp_reset_group_update_field
+
   ! 
   !   
   !     Set up a domain to pass data between coarse and fine grid of nested model.
@@ -1583,58 +1685,6 @@ module mpp_domains_mod
 end interface
 
 
-!--------------------------------------------------------------
-!bnc: for adjoint update
-!--------------------------------------------------------------
-!!$  interface mpp_update_domains_ad
-!!$     module procedure mpp_update_domain2D_ad_r8_2d
-!!$     module procedure mpp_update_domain2D_ad_r8_3d
-!!$     module procedure mpp_update_domain2D_ad_r8_4d
-!!$     module procedure mpp_update_domain2D_ad_r8_5d
-!!$     module procedure mpp_update_domain2D_ad_r8_2dv
-!!$     module procedure mpp_update_domain2D_ad_r8_3dv
-!!$     module procedure mpp_update_domain2D_ad_r8_4dv
-!!$     module procedure mpp_update_domain2D_ad_r8_5dv
-!!$#ifdef OVERLOAD_C8
-!!$     module procedure mpp_update_domain2D_ad_c8_2d
-!!$     module procedure mpp_update_domain2D_ad_c8_3d
-!!$     module procedure mpp_update_domain2D_ad_c8_4d
-!!$     module procedure mpp_update_domain2D_ad_c8_5d
-!!$#endif
-!!$#ifndef no_8byte_integers
-!!$     module procedure mpp_update_domain2D_ad_i8_2d
-!!$     module procedure mpp_update_domain2D_ad_i8_3d
-!!$     module procedure mpp_update_domain2D_ad_i8_4d
-!!$     module procedure mpp_update_domain2D_ad_i8_5d
-!!$     module procedure mpp_update_domain2D_ad_l8_2d
-!!$     module procedure mpp_update_domain2D_ad_l8_3d
-!!$     module procedure mpp_update_domain2D_ad_l8_4d
-!!$     module procedure mpp_update_domain2D_ad_l8_5d
-!!$#endif
-!!$#ifdef OVERLOAD_R4
-!!$     module procedure mpp_update_domain2D_ad_r4_2d
-!!$     module procedure mpp_update_domain2D_ad_r4_3d
-!!$     module procedure mpp_update_domain2D_ad_r4_4d
-!!$     module procedure mpp_update_domain2D_ad_r4_5d
-!!$     module procedure mpp_update_domain2D_ad_r4_2dv
-!!$     module procedure mpp_update_domain2D_ad_r4_3dv
-!!$     module procedure mpp_update_domain2D_ad_r4_4dv
-!!$     module procedure mpp_update_domain2D_ad_r4_5dv
-!!$#endif
-!!$#ifdef OVERLOAD_C4
-!!$     module procedure mpp_update_domain2D_ad_c4_2d
-!!$     module procedure mpp_update_domain2D_ad_c4_3d
-!!$     module procedure mpp_update_domain2D_ad_c4_4d
-!!$     module procedure mpp_update_domain2D_ad_c4_5d
-!!$#endif
-!!$     module procedure mpp_update_domain2D_ad_i4_2d
-!!$     module procedure mpp_update_domain2D_ad_i4_3d
-!!$     module procedure mpp_update_domain2D_ad_i4_4d
-!!$     module procedure mpp_update_domain2D_ad_i4_5d
-!!$  end interface
-!bnc
-
-
   interface mpp_do_update
      module procedure mpp_do_update_r8_3d
      module procedure mpp_do_update_r8_3dv
@@ -1674,25 +1724,6 @@ module mpp_domains_mod
   end interface
 
 
-!-------------------------------------------------------
-!bnc  for adjoint do_update
-!-------------------------------------------------------
-!!$  interface mpp_do_update_ad
-!!$     module procedure mpp_do_update_ad_r8_3d
-!!$     module procedure mpp_do_update_ad_r8_3dv
-!!$#ifdef OVERLOAD_C8
-!!$     module procedure mpp_do_update_ad_c8_3d
-!!$#endif
-!!$#ifndef no_8byte_integers
-!!$     module procedure mpp_do_update_ad_i8_3d
-!!$#endif
-!!$#ifdef OVERLOAD_R4
-!!$     module procedure mpp_do_update_ad_r4_3d
-!!$     module procedure mpp_do_update_ad_r4_3dv
-!!$#endif
-!!$#ifdef OVERLOAD_C4
-!!$     module procedure mpp_do_update_ad_c4_3d
-!!$#endif
 !!$     module procedure mpp_do_update_ad_i4_3d
 !!$  end interface
 !bnc
@@ -2082,24 +2113,12 @@ module mpp_domains_mod
      module procedure mpp_global_sum_r8_3d
      module procedure mpp_global_sum_r8_4d
      module procedure mpp_global_sum_r8_5d
-#ifdef OVERLOAD_C8
-     module procedure mpp_global_sum_c8_2d
-     module procedure mpp_global_sum_c8_3d
-     module procedure mpp_global_sum_c8_4d
-     module procedure mpp_global_sum_c8_5d
-#endif
 #ifdef OVERLOAD_R4
      module procedure mpp_global_sum_r4_2d
      module procedure mpp_global_sum_r4_3d
      module procedure mpp_global_sum_r4_4d
      module procedure mpp_global_sum_r4_5d
 #endif
-#ifdef OVERLOAD_C4
-     module procedure mpp_global_sum_c4_2d
-     module procedure mpp_global_sum_c4_3d
-     module procedure mpp_global_sum_c4_4d
-     module procedure mpp_global_sum_c4_5d
-#endif
 #ifndef no_8byte_integers
      module procedure mpp_global_sum_i8_2d
      module procedure mpp_global_sum_i8_3d
@@ -2118,74 +2137,15 @@ module mpp_domains_mod
      module procedure mpp_global_sum_tl_r8_3d
      module procedure mpp_global_sum_tl_r8_4d
      module procedure mpp_global_sum_tl_r8_5d
-#ifdef OVERLOAD_C8
-     module procedure mpp_global_sum_tl_c8_2d
-     module procedure mpp_global_sum_tl_c8_3d
-     module procedure mpp_global_sum_tl_c8_4d
-     module procedure mpp_global_sum_tl_c8_5d
-#endif
 #ifdef OVERLOAD_R4
      module procedure mpp_global_sum_tl_r4_2d
      module procedure mpp_global_sum_tl_r4_3d
      module procedure mpp_global_sum_tl_r4_4d
      module procedure mpp_global_sum_tl_r4_5d
 #endif
-#ifdef OVERLOAD_C4
-     module procedure mpp_global_sum_tl_c4_2d
-     module procedure mpp_global_sum_tl_c4_3d
-     module procedure mpp_global_sum_tl_c4_4d
-     module procedure mpp_global_sum_tl_c4_5d
-#endif
-#ifndef no_8byte_integers
-     module procedure mpp_global_sum_tl_i8_2d
-     module procedure mpp_global_sum_tl_i8_3d
-     module procedure mpp_global_sum_tl_i8_4d
-     module procedure mpp_global_sum_tl_i8_5d
-#endif
-     module procedure mpp_global_sum_tl_i4_2d
-     module procedure mpp_global_sum_tl_i4_3d
-     module procedure mpp_global_sum_tl_i4_4d
-     module procedure mpp_global_sum_tl_i4_5d
   end interface
 !gag
 
-!bnc
-!!$  interface mpp_global_sum_ad
-!!$     module procedure mpp_global_sum_ad_r8_2d
-!!$     module procedure mpp_global_sum_ad_r8_3d
-!!$     module procedure mpp_global_sum_ad_r8_4d
-!!$     module procedure mpp_global_sum_ad_r8_5d
-!!$#ifdef OVERLOAD_C8
-!!$     module procedure mpp_global_sum_ad_c8_2d
-!!$     module procedure mpp_global_sum_ad_c8_3d
-!!$     module procedure mpp_global_sum_ad_c8_4d
-!!$     module procedure mpp_global_sum_ad_c8_5d
-!!$#endif
-!!$#ifdef OVERLOAD_R4
-!!$     module procedure mpp_global_sum_ad_r4_2d
-!!$     module procedure mpp_global_sum_ad_r4_3d
-!!$     module procedure mpp_global_sum_ad_r4_4d
-!!$     module procedure mpp_global_sum_ad_r4_5d
-!!$#endif
-!!$#ifdef OVERLOAD_C4
-!!$     module procedure mpp_global_sum_ad_c4_2d
-!!$     module procedure mpp_global_sum_ad_c4_3d
-!!$     module procedure mpp_global_sum_ad_c4_4d
-!!$     module procedure mpp_global_sum_ad_c4_5d
-!!$#endif
-!!$#ifndef no_8byte_integers
-!!$     module procedure mpp_global_sum_ad_i8_2d
-!!$     module procedure mpp_global_sum_ad_i8_3d
-!!$     module procedure mpp_global_sum_ad_i8_4d
-!!$     module procedure mpp_global_sum_ad_i8_5d
-!!$#endif
-!!$     module procedure mpp_global_sum_ad_i4_2d
-!!$     module procedure mpp_global_sum_ad_i4_3d
-!!$     module procedure mpp_global_sum_ad_i4_4d
-!!$     module procedure mpp_global_sum_ad_i4_5d
-!!$  end interface
-!bnc
-
 !***********************************************************************
 !
 !            public interface from mpp_domain_util.h
@@ -2463,9 +2423,9 @@ module mpp_domains_mod
 
   !--- version information variables
   character(len=128), public :: version= &
-       '$Id: mpp_domains.F90,v 20.0 2013/12/14 00:22:42 fms Exp $'
+       '$Id$'
   character(len=128), public :: tagname= &
-       '$Name: tikal $'
+       '$Name$'
 
 
 contains
diff --git a/src/shared/mpp/mpp_efp.F90 b/src/shared/mpp/mpp_efp.F90
new file mode 100644
index 0000000000..85c46d30e7
--- /dev/null
+++ b/src/shared/mpp/mpp_efp.F90
@@ -0,0 +1,665 @@
+module mpp_efp_mod
+#include 
+
+use mpp_mod, only : mpp_error, FATAL, WARNING, NOTE
+use mpp_mod, only : mpp_pe, mpp_root_pe, mpp_npes
+use mpp_mod, only : mpp_sum
+
+implicit none ; private
+
+public :: mpp_reproducing_sum, mpp_efp_list_sum_across_PEs
+public :: mpp_efp_plus, mpp_efp_minus, mpp_efp_to_real, mpp_real_to_efp, mpp_efp_real_diff
+public :: operator(+), operator(-), assignment(=)
+public :: mpp_query_efp_overflow_error, mpp_reset_efp_overlow_error
+
+!   This module provides interfaces to the non-domain-oriented communication
+! subroutines.
+integer, parameter :: NUMBIT = 46  ! number of bits used in the 64-bit signed integer representation.
+integer, parameter :: NUMINT = 6   ! The number of long integers to use to represent
+                                   ! a real number.
+
+integer(LONG_KIND), parameter :: prec=2_8**NUMBIT ! The precision of each integer.
+real(DOUBLE_KIND), parameter :: r_prec=2.0**NUMBIT  ! A real version of prec.
+real(DOUBLE_KIND), parameter :: I_prec=1.0/(2.0**NUMBIT) ! The inverse of prec.
+integer, parameter :: max_count_prec=2**(63-NUMBIT)-1
+                              ! The number of values that can be added together
+                              ! with the current value of prec before there will
+                              ! be roundoff problems.
+
+real(DOUBLE_KIND), parameter, dimension(NUMINT) :: &
+  pr = (/ r_prec**2, r_prec, 1.0, 1.0/r_prec, 1.0/r_prec**2, 1.0/r_prec**3 /)
+real(DOUBLE_KIND), parameter, dimension(NUMINT) :: &
+  I_pr = (/ 1.0/r_prec**2, 1.0/r_prec, 1.0, r_prec, r_prec**2, r_prec**3 /)
+
+logical :: overflow_error = .false., NaN_error = .false.
+logical :: debug = .false.    ! Making this true enables debugging output.
+
+interface mpp_reproducing_sum
+  module procedure mpp_reproducing_sum_r8_2d
+  module procedure mpp_reproducing_sum_r8_3d
+  module procedure mpp_reproducing_sum_r4_2d
+end interface mpp_reproducing_sum
+
+! The Extended Fixed Point (mpp_efp) type provides a public interface for doing
+! sums and taking differences with this type.
+type, public :: mpp_efp_type ; private
+  integer(kind=8), dimension(NUMINT) :: v
+end type mpp_efp_type
+
+interface operator (+); module procedure mpp_efp_plus  ; end interface
+interface operator (-); module procedure mpp_efp_minus ; end interface
+interface assignment(=); module procedure mpp_efp_assign ; end interface
+
+contains
+
+function mpp_reproducing_sum_r8_2d(array, isr, ier, jsr, jer, EFP_sum, reproducing, &
+                            overflow_check, err) result(sum)
+  real(DOUBLE_KIND), dimension(:,:), intent(in) :: array
+  integer,        optional,          intent(in) :: isr, ier, jsr, jer
+  type(mpp_efp_type), optional,     intent(out) :: EFP_sum
+  logical,        optional,          intent(in) :: reproducing
+  logical,        optional,          intent(in) :: overflow_check
+  integer,        optional,         intent(out) :: err
+  real(DOUBLE_KIND)                             :: sum  ! Result
+
+  !   This subroutine uses a conversion to an integer representation
+  ! of real numbers to give order-invariant sums that will reproduce
+  ! across PE count.  This idea comes from R. Hallberg and A. Adcroft.
+
+  integer(LONG_KIND), dimension(NUMINT)  :: ints_sum
+  integer(LONG_KIND) :: ival, prec_error
+  real(DOUBLE_KIND)  :: rsum(1), rs
+  real(DOUBLE_KIND)  :: max_mag_term
+  logical :: repro, over_check
+  character(len=256) :: mesg
+  integer :: i, j, n, is, ie, js, je, sgn
+
+  if (mpp_npes() > max_count_prec) call mpp_error(FATAL, &
+    "mpp_reproducing_sum: Too many processors are being used for the value of "//&
+    "prec.  Reduce prec to (2^63-1)/mpp_npes.")
+
+  prec_error = (2_8**62 + (2_8**62 - 1)) / mpp_npes()
+
+  is = 1 ; ie = size(array,1) ; js = 1 ; je = size(array,2 )
+  if (present(isr)) then
+    if (isr < is) call mpp_error(FATAL, &
+      "Value of isr too small in mpp_reproducing_sum_2d.")
+    is = isr
+  endif
+  if (present(ier)) then
+    if (ier > ie) call mpp_error(FATAL, &
+      "Value of ier too large in mpp_reproducing_sum_2d.")
+    ie = ier
+  endif
+  if (present(jsr)) then
+    if (jsr < js) call mpp_error(FATAL, &
+      "Value of jsr too small in mpp_reproducing_sum_2d.")
+    js = jsr
+  endif
+  if (present(jer)) then
+    if (jer > je) call mpp_error(FATAL, &
+      "Value of jer too large in mpp_reproducing_sum_2d.")
+    je = jer
+  endif
+
+  repro = .true. ; if (present(reproducing)) repro = reproducing
+  over_check = .true. ; if (present(overflow_check)) over_check = overflow_check
+
+  if (repro) then
+    overflow_error = .false. ; NaN_error = .false. ; max_mag_term = 0.0
+    ints_sum(:) = 0
+    if (over_check) then
+      if ((je+1-js)*(ie+1-is) < max_count_prec) then
+        do j=js,je ; do i=is,ie
+          call increment_ints_faster(ints_sum, array(i,j), max_mag_term);
+        enddo ; enddo
+        call carry_overflow(ints_sum, prec_error)
+      elseif ((ie+1-is) < max_count_prec) then
+        do j=js,je
+          do i=is,ie
+            call increment_ints_faster(ints_sum, array(i,j), max_mag_term);
+          enddo
+          call carry_overflow(ints_sum, prec_error)
+        enddo
+      else
+        do j=js,je ; do i=is,ie
+          call increment_ints(ints_sum, real_to_ints(array(i,j), prec_error), &
+                              prec_error);
+        enddo ; enddo
+      endif
+    else
+      do j=js,je ; do i=is,ie
+        sgn = 1 ; if (array(i,j)<0.0) sgn = -1
+        rs = abs(array(i,j))
+        do n=1,NUMINT
+          ival = int(rs*I_pr(n), 8)
+          rs = rs - ival*pr(n)
+          ints_sum(n) = ints_sum(n) + sgn*ival
+        enddo
+      enddo ; enddo
+      call carry_overflow(ints_sum, prec_error)
+    endif
+
+    if (present(err)) then
+      err = 0
+      if (overflow_error) &
+        err = err+2
+      if (NaN_error) &
+        err = err+4
+      if (err > 0) then ; do n=1,NUMINT ; ints_sum(n) = 0 ; enddo ; endif
+    else
+      if (NaN_error) then
+        call mpp_error(FATAL, "NaN in input field of mpp_reproducing_sum(_2d).")
+      endif
+      if (abs(max_mag_term) >= prec_error*pr(1)) then
+        write(mesg, '(ES13.5)') max_mag_term
+        call mpp_error(FATAL,"Overflow in mpp_reproducing_sum(_2d) conversion of "//trim(mesg))
+      endif
+      if (overflow_error) then
+        call mpp_error(FATAL, "Overflow in mpp_reproducing_sum(_2d).")
+      endif
+    endif
+
+    call mpp_sum(ints_sum, NUMINT)
+
+    call regularize_ints(ints_sum)
+    sum = ints_to_real(ints_sum)
+  else
+    rsum(1) = 0.0
+    do j=js,je ; do i=is,ie
+      rsum(1) = rsum(1) + array(i,j);
+    enddo ; enddo
+    call mpp_sum(rsum,1)
+    sum = rsum(1)
+
+    if (present(err)) then ; err = 0 ; endif
+
+    if (debug .or. present(EFP_sum)) then
+      overflow_error = .false.
+      ints_sum = real_to_ints(sum, prec_error, overflow_error)
+      if (overflow_error) then
+        if (present(err)) then
+          err = err + 2
+        else
+          write(mesg, '(ES13.5)') sum
+          call mpp_error(FATAL,"Repro_sum_2d: Overflow in real_to_ints conversion of "//trim(mesg))
+        endif
+      endif
+    endif
+  endif
+
+  if (present(EFP_sum)) EFP_sum%v(:) = ints_sum(:)
+
+  if (debug) then
+    write(mesg,'("2d RS: ", ES24.16, 6 Z17.16)') sum, ints_sum(1:NUMINT)
+    if(mpp_pe() == mpp_root_pe()) call mpp_error(NOTE, mesg)
+  endif
+
+end function mpp_reproducing_sum_r8_2d
+
+function mpp_reproducing_sum_r4_2d(array, isr, ier, jsr, jer, EFP_sum, reproducing, &
+                            overflow_check, err) result(sum)
+  real(FLOAT_KIND), dimension(:,:), intent(in) :: array
+  integer,        optional,          intent(in) :: isr, ier, jsr, jer
+  type(mpp_efp_type), optional,     intent(out) :: EFP_sum
+  logical,        optional,          intent(in) :: reproducing
+  logical,        optional,          intent(in) :: overflow_check
+  integer,        optional,         intent(out) :: err
+  real(FLOAT_KIND)                             :: sum  ! Result
+
+  real(DOUBLE_KIND) :: array_r8(size(array,1), size(array,2))
+
+  array_r8 = array
+  
+  sum = mpp_reproducing_sum_r8_2d(array_r8, isr, ier, jsr, jer, EFP_sum, reproducing, &
+                            overflow_check, err)
+
+  return
+
+end function mpp_reproducing_sum_r4_2d
+
+
+function mpp_reproducing_sum_r8_3d(array, isr, ier, jsr, jer, sums, EFP_sum, err) &
+                            result(sum)
+  real(DOUBLE_KIND), dimension(:,:,:),        intent(in) :: array
+  integer,    optional,                       intent(in) :: isr, ier, jsr, jer
+  real(DOUBLE_KIND), dimension(:), optional, intent(out) :: sums
+  type(mpp_efp_type),     optional,          intent(out) :: EFP_sum
+  integer,            optional,              intent(out) :: err
+  real(DOUBLE_KIND)                                      :: sum  ! Result
+
+  !   This subroutine uses a conversion to an integer representation
+  ! of real numbers to give order-invariant sums that will reproduce
+  ! across PE count.  This idea comes from R. Hallberg and A. Adcroft.
+
+  real(DOUBLE_KIND)    :: max_mag_term
+  integer(LONG_KIND), dimension(NUMINT)  :: ints_sum
+  integer(LONG_KIND), dimension(NUMINT,size(array,3))  :: ints_sums
+  integer(LONG_KIND) :: prec_error
+  character(len=256) :: mesg
+  integer :: i, j, k, is, ie, js, je, ke, isz, jsz, n
+
+  if (mpp_npes() > max_count_prec) call mpp_error(FATAL, &
+    "mpp_reproducing_sum: Too many processors are being used for the value of "//&
+    "prec.  Reduce prec to (2^63-1)/mpp_npes.")
+
+  prec_error = (2_8**62 + (2_8**62 - 1)) / mpp_npes()
+  max_mag_term = 0.0
+
+  is = 1 ; ie = size(array,1) ; js = 1 ; je = size(array,2) ; ke = size(array,3)
+  if (present(isr)) then
+    if (isr < is) call mpp_error(FATAL, &
+      "Value of isr too small in mpp_reproducing_sum(_3d).")
+    is = isr
+  endif
+  if (present(ier)) then
+    if (ier > ie) call mpp_error(FATAL, &
+      "Value of ier too large in mpp_reproducing_sum(_3d).")
+    ie = ier
+  endif
+  if (present(jsr)) then
+    if (jsr < js) call mpp_error(FATAL, &
+      "Value of jsr too small in mpp_reproducing_sum(_3d).")
+    js = jsr
+  endif
+  if (present(jer)) then
+    if (jer > je) call mpp_error(FATAL, &
+      "Value of jer too large in mpp_reproducing_sum(_3d).")
+    je = jer
+  endif
+  jsz = je+1-js; isz = ie+1-is
+
+  if (present(sums)) then
+    if (size(sums) > ke) call mpp_error(FATAL, "Sums is smaller than "//&
+      "the vertical extent of array in mpp_reproducing_sum(_3d).")
+    ints_sums(:,:) = 0
+    overflow_error = .false. ; NaN_error = .false. ; max_mag_term = 0.0
+    if (jsz*isz < max_count_prec) then
+      do k=1,ke
+        do j=js,je ; do i=is,ie
+          call increment_ints_faster(ints_sums(:,k), array(i,j,k), max_mag_term);
+        enddo ; enddo
+        call carry_overflow(ints_sums(:,k), prec_error)
+      enddo
+    elseif (isz < max_count_prec) then
+      do k=1,ke ; do j=js,je
+        do i=is,ie
+          call increment_ints_faster(ints_sums(:,k), array(i,j,k), max_mag_term);
+        enddo
+        call carry_overflow(ints_sums(:,k), prec_error)
+      enddo ; enddo
+    else
+      do k=1,ke ; do j=js,je ; do i=is,ie
+        call increment_ints(ints_sums(:,k), &
+                            real_to_ints(array(i,j,k), prec_error), prec_error);
+      enddo ; enddo ; enddo
+    endif
+    if (present(err)) then
+      err = 0
+      if (abs(max_mag_term) >= prec_error*pr(1)) err = err+1
+      if (overflow_error) err = err+2
+      if (NaN_error) err = err+2
+      if (err > 0) then ; do k=1,ke ; do n=1,NUMINT ; ints_sums(n,k) = 0 ; enddo ; enddo ; endif
+    else
+      if (NaN_error) call mpp_error(FATAL, "NaN in input field of mpp_reproducing_sum(_3d).")
+      if (abs(max_mag_term) >= prec_error*pr(1)) then
+        write(mesg, '(ES13.5)') max_mag_term
+        call mpp_error(FATAL,"Overflow in mpp_reproducing_sum(_3d) conversion of "//trim(mesg))
+      endif
+      if (overflow_error) call mpp_error(FATAL, "Overflow in mpp_reproducing_sum(_3d).")
+    endif
+
+    call mpp_sum(ints_sums(:,1:ke), NUMINT*ke)
+
+    sum = 0.0
+    do k=1,ke
+      call regularize_ints(ints_sums(:,k))
+      sums(k) = ints_to_real(ints_sums(:,k))
+      sum = sum + sums(k)
+    enddo
+
+    if (present(EFP_sum)) then
+      EFP_sum%v(:) = 0
+      do k=1,ke ; call increment_ints(EFP_sum%v(:), ints_sums(:,k)) ; enddo
+    endif
+
+    if (debug) then
+      do n=1,NUMINT ; ints_sum(n) = 0 ; enddo
+      do k=1,ke ; do n=1,NUMINT ; ints_sum(n) = ints_sum(n) + ints_sums(n,k) ; enddo ; enddo
+      write(mesg,'("3D RS: ", ES24.16, 6 Z17.16)') sum, ints_sum(1:NUMINT)
+      if(mpp_pe()==mpp_root_pe()) call mpp_error(NOTE, mesg)
+    endif
+  else
+    ints_sum(:) = 0
+    overflow_error = .false. ; NaN_error = .false. ; max_mag_term = 0.0
+    if (jsz*isz < max_count_prec) then
+      do k=1,ke
+        do j=js,je ; do i=is,ie
+          call increment_ints_faster(ints_sum, array(i,j,k), max_mag_term);
+        enddo ; enddo
+        call carry_overflow(ints_sum, prec_error)
+      enddo
+    elseif (isz < max_count_prec) then
+      do k=1,ke ; do j=js,je
+        do i=is,ie
+          call increment_ints_faster(ints_sum, array(i,j,k), max_mag_term);
+        enddo
+        call carry_overflow(ints_sum, prec_error)
+      enddo ; enddo
+    else
+      do k=1,ke ; do j=js,je ; do i=is,ie
+        call increment_ints(ints_sum, real_to_ints(array(i,j,k), prec_error), &
+                            prec_error);
+      enddo ; enddo ; enddo
+    endif
+    if (present(err)) then
+      err = 0
+      if (abs(max_mag_term) >= prec_error*pr(1)) err = err+1
+      if (overflow_error) err = err+2
+      if (NaN_error) err = err+2
+      if (err > 0) then ; do n=1,NUMINT ; ints_sum(n) = 0 ; enddo ; endif
+    else
+      if (NaN_error) call mpp_error(FATAL, "NaN in input field of mpp_reproducing_sum(_3d).")
+      if (abs(max_mag_term) >= prec_error*pr(1)) then
+        write(mesg, '(ES13.5)') max_mag_term
+        call mpp_error(FATAL,"Overflow in mpp_reproducing_sum(_3d) conversion of "//trim(mesg))
+      endif
+      if (overflow_error) call mpp_error(FATAL, "Overflow in mpp_reproducing_sum(_3d).")
+    endif
+
+    call mpp_sum(ints_sum, NUMINT)
+
+    call regularize_ints(ints_sum)
+    sum = ints_to_real(ints_sum)
+
+    if (present(EFP_sum)) EFP_sum%v(:) = ints_sum(:)
+
+    if (debug) then
+      write(mesg,'("3d RS: ", ES24.16, 6 Z17.16)') sum, ints_sum(1:NUMINT)
+      if(mpp_pe()==mpp_root_pe()) call mpp_error(NOTE, mesg)
+    endif
+  endif
+
+end function mpp_reproducing_sum_r8_3d
+
+function real_to_ints(r, prec_error, overflow) result(ints)
+  real(DOUBLE_KIND),            intent(in) :: r
+  integer(LONG_KIND), optional, intent(in) :: prec_error
+  logical,   optional,       intent(inout) :: overflow
+  integer(LONG_KIND),    dimension(NUMINT) :: ints
+  !   This subroutine converts a real number to an equivalent representation
+  ! using several long integers.
+
+  real(DOUBLE_KIND) :: rs
+  character(len=80) :: mesg
+  integer(LONG_KIND) :: ival, prec_err
+  integer :: sgn, i
+
+  prec_err = prec ; if (present(prec_error)) prec_err = prec_error
+  ints(:) = 0_8
+  if ((r >= 1e30) .eqv. (r < 1e30)) then ; NaN_error = .true. ; return ; endif
+
+  sgn = 1 ; if (r<0.0) sgn = -1
+  rs = abs(r)
+
+  if (present(overflow)) then
+    if (.not.(rs < prec_err*pr(1))) overflow = .true.
+    if ((r >= 1e30) .eqv. (r < 1e30)) overflow = .true.
+  elseif (.not.(rs < prec_err*pr(1))) then
+    write(mesg, '(ES13.5)') r
+    call mpp_error(FATAL,"Overflow in real_to_ints conversion of "//trim(mesg))
+  endif
+
+  do i=1,NUMINT
+    ival = int(rs*I_pr(i), 8)
+    rs = rs - ival*pr(i)
+    ints(i) = sgn*ival
+  enddo
+
+end function real_to_ints
+
+function ints_to_real(ints) result(r)
+  integer(LONG_KIND), dimension(NUMINT), intent(in) :: ints
+  real(DOUBLE_KIND) :: r
+  ! This subroutine reverses the conversion in real_to_ints.
+
+  integer :: i
+
+  r = 0.0
+  do i=1,NUMINT ; r = r + pr(i)*ints(i) ; enddo
+end function ints_to_real
+
+subroutine increment_ints(int_sum, int2, prec_error)
+  integer(LONG_KIND), dimension(NUMINT), intent(inout) :: int_sum
+  integer(LONG_KIND), dimension(NUMINT), intent(in)    :: int2
+  integer(LONG_KIND), optional,      intent(in)    :: prec_error
+
+  ! This subroutine increments a number with another, both using the integer
+  ! representation in real_to_ints.
+  integer :: i
+
+  do i=NUMINT,2,-1
+    int_sum(i) = int_sum(i) + int2(i)
+    ! Carry the local overflow.
+    if (int_sum(i) > prec) then
+      int_sum(i) = int_sum(i) - prec
+      int_sum(i-1) = int_sum(i-1) + 1
+    elseif (int_sum(i) < -prec) then
+      int_sum(i) = int_sum(i) + prec
+      int_sum(i-1) = int_sum(i-1) - 1
+    endif
+  enddo
+  int_sum(1) = int_sum(1) + int2(1)
+  if (present(prec_error)) then
+    if (abs(int_sum(1)) > prec_error) overflow_error = .true.
+  else
+    if (abs(int_sum(1)) > prec) overflow_error = .true.
+  endif
+
+end subroutine increment_ints
+
+subroutine increment_ints_faster(int_sum, r, max_mag_term)
+  integer(LONG_KIND), dimension(NUMINT), intent(inout) :: int_sum
+  real(DOUBLE_KIND),                        intent(in) :: r
+  real(DOUBLE_KIND),                     intent(inout) :: max_mag_term
+
+  ! This subroutine increments a number with another, both using the integer
+  ! representation in real_to_ints, but without doing any carrying of overflow.
+  ! The entire operation is embedded in a single call for greater speed.
+  real(DOUBLE_KIND) :: rs
+  integer(LONG_KIND) :: ival
+  integer :: sgn, i
+
+  if ((r >= 1e30) .eqv. (r < 1e30)) then ; NaN_error = .true. ; return ; endif
+  sgn = 1 ; if (r<0.0) sgn = -1
+  rs = abs(r)
+  if (rs > abs(max_mag_term)) max_mag_term = r
+
+  do i=1,NUMINT
+    ival = int(rs*I_pr(i), 8)
+    rs = rs - ival*pr(i)
+    int_sum(i) = int_sum(i) + sgn*ival
+  enddo
+
+end subroutine increment_ints_faster
+
+subroutine carry_overflow(int_sum, prec_error)
+  integer(LONG_KIND), dimension(NUMINT), intent(inout) :: int_sum
+  integer(LONG_KIND),                intent(in)    :: prec_error
+
+  ! This subroutine handles carrying of the overflow.
+  integer :: i, num_carry
+
+  do i=NUMINT,2,-1 ; if (abs(int_sum(i)) > prec) then
+    num_carry = int(int_sum(i) * I_prec)
+    int_sum(i) = int_sum(i) - num_carry*prec
+    int_sum(i-1) = int_sum(i-1) + num_carry
+  endif ; enddo
+  if (abs(int_sum(1)) > prec_error) then
+    overflow_error = .true.
+  endif
+
+end subroutine carry_overflow
+
+subroutine regularize_ints(int_sum)
+  integer(LONG_KIND), dimension(NUMINT), intent(inout) :: int_sum
+
+  ! This subroutine carries the overflow, and then makes sure that
+  ! all integers are of the same sign as the overall value.
+  logical :: positive
+  integer :: i, num_carry
+
+  do i=NUMINT,2,-1 ; if (abs(int_sum(i)) > prec) then
+    num_carry = int(int_sum(i) * I_prec)
+    int_sum(i) = int_sum(i) - num_carry*prec
+    int_sum(i-1) = int_sum(i-1) + num_carry
+  endif ; enddo
+
+  ! Determine the sign of the final number.
+  positive = .true.
+  do i=1,NUMINT
+    if (abs(int_sum(i)) > 0) then
+      if (int_sum(i) < 0) positive = .false.
+      exit
+    endif
+  enddo
+
+  if (positive) then
+    do i=NUMINT,2,-1 ; if (int_sum(i) < 0) then
+      int_sum(i) = int_sum(i) + prec
+      int_sum(i-1) = int_sum(i-1) - 1
+    endif ; enddo
+  else
+    do i=NUMINT,2,-1 ; if (int_sum(i) > 0) then
+      int_sum(i) = int_sum(i) - prec
+      int_sum(i-1) = int_sum(i-1) + 1
+    endif ; enddo
+  endif
+
+end subroutine regularize_ints
+
+function mpp_query_efp_overflow_error()
+  logical :: mpp_query_efp_overflow_error
+  mpp_query_efp_overflow_error = overflow_error
+end function mpp_query_efp_overflow_error
+
+subroutine mpp_reset_efp_overlow_error()
+  overflow_error = .false.
+end subroutine mpp_reset_efp_overlow_error
+
+function mpp_efp_plus(EFP1, EFP2)
+  type(mpp_efp_type)             :: mpp_efp_plus
+  type(mpp_efp_type), intent(in) :: EFP1, EFP2
+
+  mpp_efp_plus = EFP1
+
+  call increment_ints(mpp_efp_plus%v(:), EFP2%v(:))
+end function mpp_efp_plus
+
+function mpp_efp_minus(EFP1, EFP2)
+  type(mpp_efp_type)             :: mpp_efp_minus
+  type(mpp_efp_type), intent(in) :: EFP1, EFP2
+  integer :: i
+
+  do i=1,NUMINT ; mpp_efp_minus%v(i) = -1*EFP2%v(i) ; enddo
+
+  call increment_ints(mpp_efp_minus%v(:), EFP1%v(:))
+end function mpp_efp_minus
+
+subroutine mpp_efp_assign(EFP1, EFP2)
+  type(mpp_efp_type), intent(out) :: EFP1
+  type(mpp_efp_type), intent(in)  :: EFP2
+  integer i
+  ! This subroutine assigns all components of the extended fixed point type
+  ! variable on the RHS (EFP2) to the components of the variable on the LHS
+  ! (EFP1).
+
+  do i=1,NUMINT ; EFP1%v(i) = EFP2%v(i) ; enddo
+end subroutine mpp_efp_assign
+
+function mpp_efp_to_real(EFP1)
+  type(mpp_efp_type), intent(inout) :: EFP1
+  real(DOUBLE_KIND) :: mpp_efp_to_real
+
+  call regularize_ints(EFP1%v)
+  mpp_efp_to_real = ints_to_real(EFP1%v)
+end function mpp_efp_to_real
+
+function mpp_efp_real_diff(EFP1, EFP2)
+  type(mpp_efp_type), intent(in) :: EFP1, EFP2
+  real(DOUBLE_KIND) :: mpp_efp_real_diff
+
+  type(mpp_efp_type)             :: EFP_diff
+
+  EFP_diff = EFP1 - EFP2
+  mpp_efp_real_diff = mpp_efp_to_real(EFP_diff)
+
+end function mpp_efp_real_diff
+
+function mpp_real_to_efp(val, overflow)
+  real(DOUBLE_KIND),    intent(in) :: val
+  logical, optional, intent(inout) :: overflow
+  type(mpp_efp_type)               :: mpp_real_to_efp
+
+  logical :: over
+  character(len=80) :: mesg
+
+  if (present(overflow)) then
+    mpp_real_to_efp%v(:) = real_to_ints(val, overflow=overflow)
+  else
+    over = .false.
+    mpp_real_to_efp%v(:) = real_to_ints(val, overflow=over)
+    if (over) then
+      write(mesg, '(ES13.5)') val
+      call mpp_error(FATAL,"Overflow in mpp_real_to_efp conversion of "//trim(mesg))
+    endif
+  endif
+
+end function mpp_real_to_efp
+
+subroutine mpp_efp_list_sum_across_PEs(EFPs, nval, errors)
+  type(mpp_efp_type), dimension(:), intent(inout) :: EFPs
+  integer, intent(in) :: nval
+  logical, dimension(:), optional, intent(out) :: errors
+
+  !   This subroutine does a sum across PEs of a list of EFP variables,
+  ! returning the sums in place, with all overflows carried.
+
+  integer(LONG_KIND), dimension(NUMINT,nval) :: ints
+  integer(LONG_KIND) :: prec_error
+  logical :: error_found
+  character(len=256) :: mesg
+  integer :: i, n
+
+  if (mpp_npes() > max_count_prec) call mpp_error(FATAL, &
+    "mpp_efp_list_sum_across_PEs: Too many processors are being used for the value of "//&
+    "prec.  Reduce prec to (2^63-1)/mpp_npes.")
+
+  prec_error = (2_8**62 + (2_8**62 - 1)) / mpp_npes()
+  ! overflow_error is an overflow error flag for the whole module.
+  overflow_error = .false. ; error_found = .false.
+
+  do i=1,nval ; do n=1,NUMINT ; ints(n,i) = EFPs(i)%v(n) ; enddo ; enddo
+
+  call mpp_sum(ints(:,:), NUMINT*nval)
+
+  if (present(errors)) errors(:) = .false.
+  do i=1,nval
+    overflow_error = .false.
+    call carry_overflow(ints(:,i), prec_error)
+    do n=1,NUMINT ; EFPs(i)%v(n) = ints(n,i) ; enddo
+    if (present(errors)) errors(i) = overflow_error
+    if (overflow_error) then
+      write (mesg,'("mpp_efp_list_sum_across_PEs error at ",i6," val was ",ES12.6, ", prec_error = ",ES12.6)') &
+             i, mpp_efp_to_real(EFPs(i)), real(prec_error)
+      if(mpp_pe()==mpp_root_pe()) call mpp_error(WARNING, mesg)
+    endif
+    error_found = error_found .or. overflow_error
+  enddo
+  if (error_found .and. .not.(present(errors))) then
+    call mpp_error(FATAL, "Overflow in mpp_efp_list_sum_across_PEs.")
+  endif
+
+end subroutine mpp_efp_list_sum_across_PEs
+
+end module mpp_efp_mod
diff --git a/src/shared/mpp/mpp_io.F90 b/src/shared/mpp/mpp_io.F90
index 4c02397b87..b234df8417 100644
--- a/src/shared/mpp/mpp_io.F90
+++ b/src/shared/mpp/mpp_io.F90
@@ -19,10 +19,12 @@
 !           675 Mass Ave, Cambridge, MA 02139, USA.  
 !-----------------------------------------------------------------------
 
-! 
+! 
 !   V. Balaji
 ! 
 
+! 
+! 
 
 ! 
 !   mpp_io_mod, is a set of simple calls for parallel I/O on
@@ -317,15 +319,18 @@ module mpp_io_mod
 use mpp_mod,            only : mpp_pe, mpp_root_pe, mpp_npes, lowercase, mpp_transmit, mpp_sync_self
 use mpp_mod,            only : mpp_init, mpp_sync, mpp_clock_id, mpp_clock_begin, mpp_clock_end
 use mpp_mod,            only : MPP_CLOCK_SYNC, MPP_CLOCK_DETAILED, CLOCK_ROUTINE
-use mpp_mod,            only : input_nml_file
+use mpp_mod,            only : input_nml_file, mpp_gather, mpp_broadcast
+use mpp_mod,            only : mpp_send, mpp_recv, mpp_sync_self, EVENT_RECV, COMM_TAG_1
 use mpp_domains_mod,    only : domain1d, domain2d, NULL_DOMAIN1D, mpp_domains_init
 use mpp_domains_mod,    only : mpp_get_global_domain, mpp_get_compute_domain
-use mpp_domains_mod,    only :  mpp_get_data_domain, mpp_get_memory_domain
+use mpp_domains_mod,    only : mpp_get_data_domain, mpp_get_memory_domain, mpp_get_pelist
 use mpp_domains_mod,    only : mpp_update_domains, mpp_global_field, mpp_domain_is_symmetry
 use mpp_domains_mod,    only : operator( .NE. ), mpp_get_domain_shift
 use mpp_domains_mod,    only : mpp_get_io_domain, mpp_domain_is_tile_root_pe, mpp_get_domain_tile_root_pe
 use mpp_domains_mod,    only : mpp_get_tile_id, mpp_get_tile_npes, mpp_get_io_domain_layout
 use mpp_domains_mod,    only : mpp_get_domain_name, mpp_get_domain_npes
+use mpp_parameter_mod,  only : MPP_FILL_DOUBLE,MPP_FILL_INT
+use mpp_mod,            only : mpp_chksum
 
 implicit none
 private
@@ -353,18 +358,20 @@ module mpp_io_mod
   public :: mpp_io_set_stack_size, mpp_get_field_index, mpp_get_axis_index
   public :: mpp_get_field_name, mpp_get_att_value, mpp_get_att_length
   public :: mpp_get_att_type, mpp_get_att_name, mpp_get_att_real, mpp_get_att_char
-  public :: mpp_get_att_real_scalar, mpp_get_axis_length
+  public :: mpp_get_att_real_scalar, mpp_get_axis_length, mpp_is_dist_ioroot
   public :: mpp_get_file_name, mpp_file_is_opened, mpp_attribute_exist 
   public :: mpp_io_clock_on, mpp_get_time_axis, mpp_get_default_calendar
+  public :: mpp_get_dimension_length, mpp_get_axis_bounds
 
   !--- public interface from mpp_io_misc.h ----------------------
   public :: mpp_io_init, mpp_io_exit, netcdf_err, mpp_flush
 
   !--- public interface from mpp_io_write.h ---------------------
-  public :: mpp_write, mpp_write_meta, mpp_copy_meta, mpp_modify_meta, mpp_write_axis_data
+  public :: mpp_write, mpp_write_meta, mpp_copy_meta, mpp_modify_meta, mpp_write_axis_data, mpp_def_dim
 
   !--- public interface from mpp_io_read.h ---------------------
   public :: mpp_read, mpp_read_meta, mpp_get_tavg_info
+  public :: mpp_read_compressed, mpp_write_compressed, mpp_read_distributed_ascii, mpp_write_unlimited_axis
 
   !--- public interface from mpp_io_switch.h ---------------------
   public :: mpp_open, mpp_close
@@ -384,6 +391,7 @@ module mpp_io_mod
   type :: axistype
      private
      character(len=128) :: name
+     character(len=128) :: name_bounds
      character(len=128) :: units
      character(len=256) :: longname
      character(len=8)   :: cartesian
@@ -392,6 +400,8 @@ module mpp_io_mod
      integer            :: sense, len          !+/-1, depth or height?
      type(domain1D)     :: domain              !if pointer is associated, it is a distributed data axis
      real, pointer      :: data(:) =>NULL()    !axis values (not used if time axis)
+     real, pointer      :: data_bounds(:) =>NULL()    !axis bounds values
+     integer, pointer   :: idata(:) =>NULL()   !compressed axis valuesi
      integer            :: id, did, type, natt !id is the "variable ID", did is the "dimension ID": 
                                                !netCDF requires 2 IDs for axes
      integer            :: shift               !normally is 0. when domain is symmetry, its value maybe 1.
@@ -432,6 +442,7 @@ module mpp_io_mod
      real(DOUBLE_KIND)  :: time
      logical            :: valid
      logical            :: write_on_this_pe   ! indicate if will write out from this pe
+     logical            :: read_on_this_pe    ! indicate if will read from this pe
      logical            :: io_domain_exist    ! indicate if io_domain exist or not.
      integer            :: id       !variable ID of time axis associated with file (only one time axis per file)
      integer            :: recdimid !dim ID of time axis associated with file (only one time axis per file)
@@ -546,15 +557,94 @@ module mpp_io_mod
   interface mpp_read
      module procedure mpp_read_2ddecomp_r2d
      module procedure mpp_read_2ddecomp_r3d
+     module procedure mpp_read_2ddecomp_r4d
      module procedure mpp_read_r0D
      module procedure mpp_read_r1D
      module procedure mpp_read_r2D
      module procedure mpp_read_r3D
+     module procedure mpp_read_r4D
      module procedure mpp_read_text
      module procedure mpp_read_region_r2D
      module procedure mpp_read_region_r3D
   end interface
 
+!***********************************************************************
+!
+!      public interfaces from mpp_io_read_distributed_ascii.inc
+!
+!***********************************************************************
+! 
+!   
+!     Read from an opened, ascii file, translating data to the desired format
+!   
+!   
+!     These routines are part of the mpp_read family. It is intended to 
+!     provide a general purpose, distributed, list directed read
+!  
+!   
+!  
+!  
+!  
+!  
+!  
+!  
+!  
+!     mpp_read_distributed_ascii
+!     The stripe size must be greater than or equal to 1. The stripe
+!     size does not have to be a common denominator of the number of
+!     MPI ranks.
+!  
+! 
+  interface mpp_read_distributed_ascii
+     module procedure mpp_read_distributed_ascii_r1d
+     module procedure mpp_read_distributed_ascii_i1d
+     module procedure mpp_read_distributed_ascii_a1d
+  end interface
+
+
+!***********************************************************************
+!
+!      public interfaces from mpp_io_read_compressed.h
+!
+!***********************************************************************
+! 
+!   
+!     Read from an opened, sparse data, compressed file (e.g. land_model)
+!   
+!   
+!     These routines are similar to mpp_read except that they are designed
+!     to handle sparse, compressed vectors of data such as from the
+!     land model. Currently, the sparse vector may vary in z. Hence
+!     the need for the rank 2 treatment.
+!  
+!   
+!  
+!  
+!  
+!  
+!  
+!     time_index is an optional argument. It is to be omitted if the
+!     field was defined not to be a function of time. Results are
+!     unpredictable if the argument is supplied for a time- independent
+!     field, or omitted for a time-dependent field.
+!  
+!  
+!     mpp_read_meta must be called prior to calling 
+!     mpp_read_compressed.
+!     Since in general, the vector is distributed across the io-domain
+!     The read expects the io_domain to be defined.
+!  
+! 
+  interface mpp_read_compressed
+     module procedure mpp_read_compressed_r1d
+     module procedure mpp_read_compressed_r2d
+  end interface mpp_read_compressed
+
+
 !***********************************************************************
 !
 !    public interface from mpp_io_write.h
@@ -669,7 +759,9 @@ module mpp_io_mod
      module procedure mpp_write_meta_var
      module procedure mpp_write_meta_scalar_r
      module procedure mpp_write_meta_scalar_i
-     module procedure mpp_write_meta_axis
+     module procedure mpp_write_meta_axis_r1d
+     module procedure mpp_write_meta_axis_i1d
+     module procedure mpp_write_meta_axis_unlimited
      module procedure mpp_write_meta_field
      module procedure mpp_write_meta_global
      module procedure mpp_write_meta_global_scalar_r
@@ -781,6 +873,109 @@ module mpp_io_mod
      module procedure mpp_write_axis
   end interface
 
+
+!***********************************************************************
+! 
+!   
+!     Write to an opened, sparse data, compressed file (e.g. land_model)
+!   
+!   
+!     These routines are similar to mpp_write except that they are 
+!     designed to handle sparse, compressed vectors of data such 
+!     as from the land model. Currently, the sparse vector may vary in z.
+!     Hence the need for the rank 2 treatment.
+!  
+!   
+!  
+!  
+!  
+!  
+!  
+!    nelems is a vector containing the number of elements expected
+!    from each member of the io_domain. It MUST have the same order as
+!    the io_domain pelist.
+!  
+!  
+!    tstamp is an optional argument. It is to
+!    be omitted if the field was defined not to be a function of time.
+!    Results are unpredictable if the argument is supplied for a time-
+!    independent field, or omitted for a time-dependent field. Repeated
+!    writes of a time-independent field are also not recommended. One
+!    time level of one field is written per call. tstamp must be an 8-byte
+!    real, even if the default real type is 4-byte.
+!  
+!  
+!  
+!     mpp_write_meta must be called prior to calling 
+!     mpp_write_compressed.
+!     Since in general, the vector is distributed across the io-domain
+!     The write expects the io_domain to be defined.
+!  
+! 
+  interface mpp_write_compressed
+     module procedure mpp_write_compressed_r1d
+     module procedure mpp_write_compressed_r2d
+  end interface mpp_write_compressed
+
+!***********************************************************************
+! 
+!   
+!     Write to an opened file along the unlimited axis (e.g. icebergs)
+!   
+!   
+!     These routines are similar to mpp_write except that they are 
+!     designed to handle data written along the unlimited axis that
+!     is not time (e.g. icebergs).
+!  
+!   
+!  
+!  
+!  
+!  
+!  
+!    nelems is a vector containing the number of elements expected
+!    from each member of the io_domain. It MUST have the same order as
+!    the io_domain pelist.
+!  
+!  
+!     mpp_write_meta must be called prior to calling 
+!     mpp_write_unlimited_axis.
+!     Since in general, the vector is distributed across the io-domain
+!     The write expects the io_domain to be defined.
+!  
+! 
+  interface mpp_write_unlimited_axis
+     module procedure mpp_write_unlimited_axis_r1d
+  end interface mpp_write_unlimited_axis
+
+
+!***********************************************************************
+! 
+!   
+!     Define an dimension variable
+!   
+!   
+!     Similar to the mpp_write_meta routines, but simply defines the
+!     a dimension variable with the optional attributes
+!  
+!   
+!  
+!  
+!  
+!  
+! 
+  interface mpp_def_dim
+     module procedure mpp_def_dim_nodata
+     module procedure mpp_def_dim_int
+     module procedure mpp_def_dim_real
+  end interface mpp_def_dim
+
 !***********************************************************************
 !
 !            module variables
@@ -820,9 +1015,9 @@ module mpp_io_mod
   integer :: pack_size ! = 1 when compiling with -r8 and = 2 when compiling with -r4.
 
   character(len=128) :: version= &
-       '$Id: mpp_io.F90,v 20.0 2013/12/14 00:23:45 fms Exp $'
+       '$Id$'
   character(len=128) :: tagname= &
-       '$Name: tikal $'
+       '$Name$'
 
 contains
 
diff --git a/src/shared/mpp/mpp_parameter.F90 b/src/shared/mpp/mpp_parameter.F90
index 85f6deffe5..1fc59f5cb1 100644
--- a/src/shared/mpp/mpp_parameter.F90
+++ b/src/shared/mpp/mpp_parameter.F90
@@ -7,7 +7,7 @@ module mpp_parameter_mod
   character(len=128), public :: version= &
        '$Id mpp_parameter.F90 $'
   character(len=128), public :: tagname= &
-       '$Name: tikal $'
+       '$Name$'
 
   !--- public paramters which is used by mpp_mod and its components. 
   !--- All othere modules should import these parameters from mpp_mod. 
@@ -23,14 +23,14 @@ module mpp_parameter_mod
   public :: COMM_TAG_9,  COMM_TAG_10, COMM_TAG_11, COMM_TAG_12
   public :: COMM_TAG_13, COMM_TAG_14, COMM_TAG_15, COMM_TAG_16
   public :: COMM_TAG_17, COMM_TAG_18, COMM_TAG_19, COMM_TAG_20
-
+  public :: MPP_FILL_INT,MPP_FILL_DOUBLE
 
   !--- public paramters which is used by mpp_domains_mod and its components. 
   !--- All othere modules should import these parameters from mpp_domains_mod. 
   public :: GLOBAL_DATA_DOMAIN, CYCLIC_GLOBAL_DOMAIN, BGRID_NE, BGRID_SW, CGRID_NE, CGRID_SW
   public :: DGRID_NE, DGRID_SW, FOLD_WEST_EDGE, FOLD_EAST_EDGE, FOLD_SOUTH_EDGE, FOLD_NORTH_EDGE
   public :: WUPDATE, EUPDATE, SUPDATE, NUPDATE, XUPDATE, YUPDATE, BITWISE_EXACT_SUM, NON_BITWISE_EXACT_SUM
-  public :: MPP_DOMAIN_TIME, WEST, EAST, SOUTH, NORTH, SCALAR_BIT, SCALAR_PAIR
+  public :: MPP_DOMAIN_TIME, WEST, EAST, SOUTH, NORTH, SCALAR_BIT, SCALAR_PAIR, BITWISE_EFP_SUM
   public :: NORTH_EAST, SOUTH_EAST, SOUTH_WEST, NORTH_WEST
   public :: AGRID, GLOBAL, CYCLIC, DOMAIN_ID_BASE, CENTER, CORNER
   public :: MAX_DOMAIN_FIELDS, MAX_TILES
@@ -54,6 +54,10 @@ module mpp_parameter_mod
   integer, parameter :: EVENT_ALLTOALL=6
   integer, parameter :: MPP_CLOCK_SYNC=1, MPP_CLOCK_DETAILED=2
   integer            :: DEFAULT_TAG = 1
+  !--- implimented to centralize _FILL_ values for land_model.F90 into mpp_mod
+  !------- instead of multiple includes of netcdf.inc and manual assignments
+  integer, parameter :: MPP_FILL_INT =-2147483647               !NF_FILL_INT
+  real,    parameter :: MPP_FILL_DOUBLE= 9.9692099683868690e+36 !NF_FILL_DOUBLE
   !--- predefined clock granularities, but you can use any integer
   !--- using CLOCK_LOOP and above may distort coarser-grain measurements
   integer, parameter :: CLOCK_COMPONENT=1      !component level, e.g model, exchange
@@ -96,6 +100,7 @@ module mpp_parameter_mod
   integer(LONG_KIND), parameter :: DOMAIN_ID_BASE=Z'0000000100000000' ! Workaround for 64bit init problem
   integer, parameter :: NON_BITWISE_EXACT_SUM=0
   integer, parameter :: BITWISE_EXACT_SUM=1
+  integer, parameter :: BITWISE_EFP_SUM=2
   integer, parameter :: MPP_DOMAIN_TIME=MPP_DEBUG+1
   integer, parameter :: MAX_DOMAIN_FIELDS=100
   integer, parameter :: MAX_TILES=100
diff --git a/src/shared/mpp/mpp_pset.F90 b/src/shared/mpp/mpp_pset.F90
index 02914d55f0..c197366a49 100644
--- a/src/shared/mpp/mpp_pset.F90
+++ b/src/shared/mpp/mpp_pset.F90
@@ -1,7 +1,7 @@
 ! module within MPP for handling PSETs:
 ! PSET: Persistent Shared-memory Execution Thread
 !
-! AUTHOR: V. Balaji (v.balaji)
+! AUTHOR: V. Balaji (v.balaji@noaa.gov)
 ! DATE: 2006-01-15
 #include 
 #ifdef test_mpp_pset
diff --git a/src/shared/mpp/mpp_utilities.F90 b/src/shared/mpp/mpp_utilities.F90
index e78d348754..001513e1fc 100644
--- a/src/shared/mpp/mpp_utilities.F90
+++ b/src/shared/mpp/mpp_utilities.F90
@@ -1,8 +1,8 @@
 module mpp_utilities_mod
 
 !-----------------------------------------------------------------------
-  character(len=128) :: version = '$Id: mpp_utilities.F90,v 17.0 2009/07/21 03:21:23 fms Exp $'
-  character(len=128) :: tag = '$Name: tikal $'
+  character(len=128) :: version = '$Id$'
+  character(len=128) :: tag = '$Name$'
 !-----------------------------------------------------------------------
 
   public :: mpp_array_global_min_max
@@ -24,9 +24,9 @@ module mpp_utilities_mod
 ! Vectorized using maxloc() and minloc() intrinsic functions by 
 ! Russell.Fiedler@csiro.au (May 2005).
 !
-! Modified by Zhi.Liang (July 2005)
+! Modified by Zhi.Liang@noaa.gov (July 2005)
 !          
-! Modified by Niki.Zadeh (Feb. 2009)
+! Modified by Niki.Zadeh@noaa.gov (Feb. 2009)
 !
 ! 
 !
@@ -73,14 +73,14 @@ subroutine mpp_array_global_min_max(in_array, tmask,isd,jsd,isc,iec,jsc,jec,nk,
   end if
 
   ! use "fudge" to distinguish processors when tracer extreme is independent of processor
-  fudge = 1.0 + 1.e-12*mpp_pe() 
+  fudge = 1.0 + 1.e-12*real(mpp_pe() )
   tmax = tmax*fudge
   tmin = tmin*fudge
   if(tmax == 0.0) then 
-    tmax = tmax + 1.e-12*mpp_pe() 
+    tmax = tmax + 1.e-12*real(mpp_pe() )
   endif 
   if(tmin == 0.0) then 
-    tmin = tmin + 1.e-12*mpp_pe() 
+    tmin = tmin + 1.e-12*real(mpp_pe() )
   endif 
   
 
@@ -101,8 +101,8 @@ subroutine mpp_array_global_min_max(in_array, tmask,isd,jsd,isc,iec,jsc,jec,nk,
   !
   !mpp_max trick
   !-999 on all current PE's
-  xgmax=-999; ygmax=-999; zgmax=-999
-  xgmin=-999; ygmin=-999; zgmin=-999
+  xgmax=-999.; ygmax=-999.; zgmax=-999.
+  xgmin=-999.; ygmin=-999.; zgmin=-999.
 
 
   !except when
diff --git a/src/shared/mpp/test_mpp.F90 b/src/shared/mpp/test_mpp.F90
index ce34fb2e27..f182276499 100644
--- a/src/shared/mpp/test_mpp.F90
+++ b/src/shared/mpp/test_mpp.F90
@@ -14,7 +14,8 @@ program test   !test various aspects of mpp_mod
   use mpp_mod, only : mpp_clock_id, mpp_clock_begin, mpp_clock_end, mpp_sync, mpp_malloc
   use mpp_mod, only : mpp_declare_pelist, mpp_set_current_pelist, mpp_set_stack_size
   use mpp_mod, only : mpp_broadcast, mpp_transmit, mpp_sum, mpp_max, mpp_chksum, ALL_PES
-  use mpp_mod, only : mpp_error, FATAL, mpp_sync_self
+  use mpp_mod, only : mpp_gather, mpp_error, FATAL, mpp_sync_self
+  use mpp_io_mod, only: mpp_io_init, mpp_flush
 #ifdef use_MPI_GSM
   use mpp_mod, only : mpp_gsm_malloc, mpp_gsm_free
 #endif
@@ -36,16 +37,24 @@ program test   !test various aspects of mpp_mod
   real                            :: dt
 
   call mpp_init()
+  call mpp_io_init()
   call mpp_set_stack_size(3145746)
   pe = mpp_pe()
   npes = mpp_npes()
   root = mpp_root_pe()
 
   out_unit = stdout()
+  call test_gather(npes,pe,root,out_unit)
+  call test_gatherV(npes,pe,root,out_unit)
+  call test_gather2DV(npes,pe,root,out_unit)
+
+  if(.false.) then
+
   ! first test broadcast
   call test_broadcast()
 
   call SYSTEM_CLOCK( count_rate=ticks_per_sec )
+
   allocate( a(n), b(n) )
   id = mpp_clock_id( 'Random number' )
   call mpp_clock_begin(id)
@@ -210,6 +219,8 @@ program test   !test various aspects of mpp_mod
   end if
 #endif
 #endif
+  endif  ! if(.false.)
+
   call mpp_exit()
 
 contains
@@ -255,6 +266,247 @@ subroutine test_broadcast()
 
   end subroutine test_broadcast
 
+  subroutine test_gather(npes,pe,root,out_unit)
+     integer, intent(in) :: npes,pe,root,out_unit
+
+     integer :: pelist(npes)
+     integer :: i
+     real :: rdata(npes)
+     real :: val
+
+     if(npes < 3)then
+       write(out_unit,*) "Minimum of 3 ranks required. Not testing gather; too few ranks."
+       return
+     endif
+     write(out_unit,*)
+
+     val = pe
+     rdata = -1.0
+     do i=1,npes
+       pelist(i) = i-1
+     enddo
+
+     call mpp_gather((/val/),rdata)
+     if(pe == root)then 
+       do i=1,npes
+        if(INT(rdata(i)) /= pelist(i))then
+           write(6,*) "Gathered data ",INT(rdata(i)), " NE reference ",pelist(i), "at i=",i
+           call mpp_error(FATAL, "Test gather uniform vector with global pelist failed")
+        endif
+       enddo
+     endif
+
+     call mpp_sync()
+     write(out_unit,*) "Test gather uniform vector with global pelist successful"
+
+     rdata = -1.0
+     if(ANY(pe == pelist(2:npes)))call mpp_gather((/val/),rdata(2:npes),pelist(2:npes))
+     if(pe == pelist(2))then
+       do i=2,npes
+        if(INT(rdata(i)) /= pelist(i))then
+           write(6,*) "Gathered data ",INT(rdata(i)), " NE reference ",pelist(i), "at i=",i
+           call mpp_error(FATAL, "Test gather uniform vector with reduced pelist failed")
+        endif
+       enddo
+     endif
+     call mpp_sync()
+     write(out_unit,*) "Test gather uniform vector with reduced pelist successful"
+
+  end subroutine test_gather
+
+
+  subroutine test_gatherV(npes,pe,root,out_unit)
+  implicit none
+     integer, intent(in) :: npes,pe,root,out_unit
+
+     integer :: pelist(npes),rsize(npes)
+     integer :: i,j,k,dsize,ssize
+     real,allocatable :: sdata(:), rdata(:), ref(:)
+
+     if(npes < 3)then
+       write(out_unit,*) "Minimum of 3 ranks required. Not testing gatherV; too few ranks."
+       return
+     elseif(npes > 9999)then
+       write(out_unit,*) "Maximum of 9999 ranks supported. Not testing gatherV; too many ranks."
+       return
+     endif
+     write(out_unit,*)
+
+     ssize = pe+1
+     allocate(sdata(ssize))
+     do i=1,ssize
+       sdata(i) = pe + 0.0001*i
+     enddo
+     do i=1,npes
+       pelist(i) = i-1
+       rsize(i) = i
+     enddo
+
+     dsize = sum(rsize)
+     allocate(rdata(dsize),ref(dsize))
+     rdata = -1.0
+     k=1
+     do j=1,npes
+       do i=1,rsize(j)
+          ref(k) = pelist(j) + 0.0001*i
+          k = k+1
+     enddo;enddo
+
+     call mpp_gather(sdata,ssize,rdata,rsize)
+ 
+     if(pe == root)then
+       k = 1
+       do j=1,npes
+         do i=1,rsize(j)
+           if(rdata(k) /= ref(k))then
+              write(6,*) "Gathered data ",rdata(k), " NE reference ",ref(k), "at k=",k
+              call mpp_error(FATAL, "Test gatherV global pelist failed")
+           endif
+           k = k+1
+       enddo;enddo
+     endif
+
+     call mpp_sync()
+     write(out_unit,*) "Test gatherV with global pelist successful"
+
+     rdata = -1.0
+     ref(1) = -1.0
+
+     if(ANY(pe == pelist(2:npes)))call mpp_gather(sdata,ssize,rdata(2:),rsize(2:),pelist(2:npes))
+
+     if(pe == pelist(2))then
+       k = 1
+       do j=1,npes
+         do i=1,rsize(j)
+           if(rdata(k) /= ref(k))then
+              write(6,*) "Gathered data ",rdata(k), " NE reference ",ref(k), "at k=",k
+              call mpp_error(FATAL, "Test gatherV with reduced pelist failed")
+           endif
+           k = k+1
+       enddo;enddo
+     endif
+     call mpp_sync()
+
+     write(out_unit,*) "Test gatherV with reduced pelist successful"
+     deallocate(sdata,rdata,ref)
+  end subroutine test_gatherV
+
+subroutine test_gather2DV(npes,pe,root,out_unit)
+  implicit none
+     integer, intent(in) :: npes,pe,root,out_unit
+
+     integer :: pelist(npes),rsize(npes)
+     integer :: pelist2(npes),rsize2(npes)
+     integer :: i,j,k,l,nz,ssize,nelems
+     real,allocatable,dimension(:,:) :: data, cdata, sbuff,rbuff
+     real,allocatable :: ref(:,:)
+     integer, parameter :: KSIZE=10
+
+     real :: sbuff1D(size(sbuff))
+     real :: rbuff1D(size(rbuff))
+     pointer(sptr,sbuff1D); pointer(rptr,rbuff1D)
+
+
+     if(npes < 3)then
+       write(out_unit,*) "Minimum of 3 ranks required. Not testing gather2DV; too few ranks."
+       return
+     elseif(npes > 9999)then
+       write(out_unit,*) "Maximum of 9999 ranks supported. Not testing gather2DV; too many ranks."
+       return
+     endif
+     write(out_unit,*)
+
+     ssize = pe+1
+     allocate(data(ssize,KSIZE))
+     do k=1,KSIZE; do i=1,ssize
+       data(i,k) = 10000.0*k + pe + 0.0001*i
+     enddo; enddo
+     do i=1,npes
+       pelist(i) = i-1
+       rsize(i) = i
+     enddo
+
+     nz = KSIZE
+     nelems = sum(rsize(:))
+
+     allocate(rbuff(nz,nelems)); rbuff = -1.0
+     allocate(ref(nelems,nz),cdata(nelems,nz))
+     ref = 0.0; cdata = 0.0
+     if(pe == root)then
+       do k=1,KSIZE
+       l=1
+       do j=1,npes
+         do i=1,rsize(j)
+            ref(l,k) = 10000.0*k + pelist(j) + 0.0001*i
+            l = l+1
+       enddo; enddo;enddo
+     endif
+     allocate(sbuff(nz,ssize))
+     ! this matrix inversion makes for easy gather to the IO root
+     ! and a clear, concise unpack
+     do j=1,ssize
+       do i=1,nz
+         sbuff(i,j) = data(j,i)
+     enddo; enddo
+
+  !  Note that the gatherV implied here is asymmetric; only root needs to know the vector of recv size
+     sptr = LOC(sbuff); rptr = LOC(rbuff)
+     call mpp_gather(sbuff1D,size(sbuff),rbuff1D,nz*rsize(:))
+
+     if(pe == root)then
+        do j=1,nz
+           do i=1,nelems
+             cdata(i,j) = rbuff(j,i)
+        enddo; enddo
+        do j=1,nz
+           do i=1,nelems
+            if(cdata(i,j) /= ref(i,j))then
+               write(6,*) "Gathered data ",cdata(i,j), " NE reference ",ref(i,j), "at i,j=",i,j
+               call mpp_error(FATAL, "Test gather2DV global pelist failed")
+            endif
+       enddo;enddo
+     endif
+
+     call mpp_sync()
+     write(out_unit,*) "Test gather2DV with global pelist successful"
+
+     do i=1,npes
+       pelist2(i) = pelist(npes-i+1) 
+       rsize2(i) = rsize(npes-i+1)
+     enddo
+
+     rbuff = -1.0
+     ref = 0.0; cdata = 0.0
+     if(pe == pelist2(1))then
+       do k=1,KSIZE
+       l=1
+       do j=1,npes
+         do i=1,rsize2(j)
+            ref(l,k) = 10000.0*k + pelist2(j) + 0.0001*i
+            l = l+1
+       enddo; enddo;enddo
+     endif
+
+     call mpp_gather(sbuff1D,size(sbuff),rbuff1D,nz*rsize2(:),pelist2)
+
+     if(pe == pelist2(1))then
+        do j=1,nz
+           do i=1,nelems
+             cdata(i,j) = rbuff(j,i)
+        enddo; enddo
+        do j=1,nz
+           do i=1,nelems
+            if(cdata(i,j) /= ref(i,j))then
+               write(6,*) "Gathered data ",cdata(i,j), " NE reference ",ref(i,j), "at i,j=",i,j
+               call mpp_error(FATAL, "Test gather2DV with reversed pelist failed")
+            endif
+       enddo;enddo
+     endif
+     call mpp_sync()
+     write(out_unit,*) "Test gather2DV with reversed pelist successful"
+     deallocate(data,sbuff,rbuff,cdata,ref)
+  end subroutine test_gather2DV
+
   subroutine test_shared_pointers(locd,n)
     integer(LONG_KIND), intent(in) :: locd
     integer :: n
diff --git a/src/shared/mpp/test_mpp_domains.F90 b/src/shared/mpp/test_mpp_domains.F90
index 4a4adde0bf..6e9ad7706e 100644
--- a/src/shared/mpp/test_mpp_domains.F90
+++ b/src/shared/mpp/test_mpp_domains.F90
@@ -10,7 +10,7 @@ program test
   use mpp_domains_mod, only : GLOBAL_DATA_DOMAIN, BITWISE_EXACT_SUM, BGRID_NE, CGRID_NE, DGRID_NE
   use mpp_domains_mod, only : FOLD_SOUTH_EDGE, FOLD_NORTH_EDGE, FOLD_WEST_EDGE, FOLD_EAST_EDGE
   use mpp_domains_mod, only : MPP_DOMAIN_TIME, CYCLIC_GLOBAL_DOMAIN, NUPDATE,EUPDATE, XUPDATE, YUPDATE, SCALAR_PAIR
-  use mpp_domains_mod, only : domain1D, domain2D, DomainCommunicator2D
+  use mpp_domains_mod, only : domain1D, domain2D, DomainCommunicator2D, BITWISE_EFP_SUM
   use mpp_domains_mod, only : mpp_get_compute_domain, mpp_get_data_domain, mpp_domains_set_stack_size
   use mpp_domains_mod, only : mpp_global_field, mpp_global_sum, mpp_global_max, mpp_global_min
   use mpp_domains_mod, only : mpp_domains_init, mpp_domains_exit, mpp_broadcast_domain
@@ -19,13 +19,16 @@ program test
   use mpp_domains_mod, only : mpp_get_neighbor_pe, mpp_define_mosaic, mpp_nullify_domain_list
   use mpp_domains_mod, only : NORTH, NORTH_EAST, EAST, SOUTH_EAST, CORNER, CENTER
   use mpp_domains_mod, only : SOUTH, SOUTH_WEST, WEST, NORTH_WEST, mpp_define_mosaic_pelist
-  use mpp_domains_mod, only : mpp_get_refine_overlap_number, mpp_get_mosaic_refine_overlap
   use mpp_domains_mod, only : mpp_get_global_domain, ZERO, NINETY, MINUS_NINETY
   use mpp_domains_mod, only : mpp_get_boundary, mpp_start_update_domains, mpp_complete_update_domains
   use mpp_domains_mod, only : mpp_define_nest_domains, nest_domain_type
   use mpp_domains_mod, only : mpp_get_C2F_index, mpp_update_nest_fine
   use mpp_domains_mod, only : mpp_get_F2C_index, mpp_update_nest_coarse
   use mpp_domains_mod, only : mpp_get_domain_shift, EDGEUPDATE, mpp_deallocate_domain
+  use mpp_domains_mod, only : mpp_group_update_type, mpp_create_group_update
+  use mpp_domains_mod, only : mpp_do_group_update, mpp_clear_group_update
+  use mpp_domains_mod, only : mpp_start_group_update, mpp_complete_group_update
+  use mpp_domains_mod, only : WUPDATE, SUPDATE
   use mpp_memutils_mod, only : mpp_memuse_begin, mpp_memuse_end
 
   implicit none
@@ -47,12 +50,15 @@ program test
   logical :: test_interface = .true.
   logical :: test_nest_domain = .false.
   logical :: test_edge_update = .false.
+  logical :: test_group = .false.
   logical :: test_cubic_grid_redistribute = .false.
   logical :: check_parallel = .FALSE.  ! when check_parallel set to false,
   logical :: test_get_nbr = .FALSE.
   logical :: test_boundary = .false.
+  logical :: test_global_sum = .false.
   integer :: ensemble_size
   integer :: layout_cubic(2) = (/0,0/)
+  integer :: layout_tripolar(2) = (/0,0/)
   integer :: layout_ensemble(2) = (/0,0/)
   logical :: do_sleep = .false.
   integer :: num_iter = 1
@@ -77,7 +83,8 @@ program test
                                jstart_fine, jend_fine, istart_coarse, iend_coarse, jstart_coarse, &
                                jend_coarse, extra_halo, npes_fine, npes_coarse, mix_2D_3D, test_get_nbr, &
                                test_edge_update, test_cubic_grid_redistribute, ensemble_size, &
-                               layout_cubic, layout_ensemble, nthreads, test_boundary
+                               layout_cubic, layout_ensemble, nthreads, test_boundary, &
+                               layout_tripolar, test_group, test_global_sum
   integer :: i, j, k
   integer :: layout(2)
   integer :: id
@@ -168,16 +175,26 @@ program test
       call update_domains_performance('Cubic-Grid')
   endif
 
+  if( test_global_sum ) then
+      call test_mpp_global_sum('Folded-north')
+  endif
+
   if( test_cubic_grid_redistribute ) then
      call cubic_grid_redistribute()
   endif
 
   if(test_boundary) then
+      call test_get_boundary('torus')
       call test_get_boundary('Four-Tile')
       call test_get_boundary('Cubic-Grid')
       call test_get_boundary('Folded-north')
   endif
 
+  if( test_group) then
+     call test_group_update( 'Folded-north' )
+     call test_group_update( 'Cubic-Grid' )
+  endif
+
   if( test_interface ) then
       call test_modify_domain()
 !!$      call test_cyclic_offset('x_cyclic_offset')
@@ -193,9 +210,6 @@ program test
       call test_uniform_mosaic('Four-Tile')
       call test_uniform_mosaic('Cubic-Grid') ! 6 tiles.
       call test_nonuniform_mosaic('Five-Tile')
-      call test_refined_mosaic('Refined-Four-Tile')
-      call test_refined_mosaic('Refined-Symmetric-Four-Tile')
-      call test_refined_mosaic('Refined-Cubic-Grid')
 
       call test_halo_update( 'Simple' ) !includes global field, global sum tests
       call test_halo_update( 'Cyclic' )
@@ -686,7 +700,7 @@ subroutine cubic_grid_redistribute
               enddo
            enddo
         enddo
-       write(mesg,'(a,i)') "cubic_grid redistribute from ensemble", n
+       write(mesg,'(a,i4)') "cubic_grid redistribute from ensemble", n
         call compare_checksums( x(isc:iec,jsc:jec,:,n), y(isc:iec,jsc:jec,:), trim(mesg) )
      enddo
     
@@ -718,7 +732,7 @@ subroutine cubic_grid_redistribute
               enddo
            enddo
         endif
-        write(mesg,'(a,i)') "cubic_grid redistribute to ensemble", n
+        write(mesg,'(a,i4)') "cubic_grid redistribute to ensemble", n
         call compare_checksums( x_ens(isc_ens:iec_ens,jsc_ens:jec_ens,:), y(isc_ens:iec_ens,jsc_ens:jec_ens,:), trim(mesg) )
      enddo
 
@@ -2147,1313 +2161,1252 @@ subroutine update_domains_performance( type )
 
   end subroutine update_domains_performance
 
-  !#################################################################################
 
-  subroutine fill_halo_zero(data, whalo, ehalo, shalo, nhalo, xshift, yshift, isc, iec, jsc, jec, isd, ied, jsd, jed)
-    integer,                         intent(in) :: isc, iec, jsc, jec, isd, ied, jsd, jed
-    integer,                         intent(in) :: whalo, ehalo, shalo, nhalo, xshift, yshift
-    real, dimension(isd:,jsd:,:), intent(inout) :: data
+  !###############################################################
+  subroutine test_mpp_global_sum( type )
+    character(len=*), intent(in) :: type
 
-    if(whalo >=0) then
-       data(iec+ehalo+1+xshift:ied+xshift,jsd:jed+yshift,:) = 0
-       data(isd:isc-whalo-1,jsd:jed+yshift,:) = 0
-    else
-       data(iec+1+xshift:iec-ehalo+xshift,jsc+shalo:jec-nhalo+yshift,:) = 0
-       data(isc+whalo:isc-1,jsc+shalo:jec-nhalo+yshift,:) = 0
-    end if
+    type(domain2D) :: domain
+    integer        :: num_contact, ntiles, npes_per_tile
+    integer        :: i, j, k, l, n, shift
+    integer        :: isc, iec, jsc, jec, isd, ied, jsd, jed
+    integer        :: ism, iem, jsm, jem
 
-    if(shalo>=0) then
-       data(isd:ied+xshift, jec+nhalo+1+yshift:jed+yshift,:) = 0
-       data(isd:ied+xshift, jsd:jsc-shalo-1,:) = 0
-    else
-       data(isc+whalo:iec-ehalo+xshift,jec+1+yshift:jec-nhalo+yshift,:) = 0
-       data(isc+whalo:iec-ehalo+xshift,jsc+shalo:jsc-1,:) = 0
-    end if
+    integer, allocatable, dimension(:)       :: pe_start, pe_end, tile1, tile2
+    integer, allocatable, dimension(:)       :: istart1, iend1, jstart1, jend1
+    integer, allocatable, dimension(:)       :: istart2, iend2, jstart2, jend2
+    integer, allocatable, dimension(:,:)     :: layout2D, global_indices
+    real,    allocatable, dimension(:,:,:)   :: data_3D
+    real,    allocatable, dimension(:,:)     :: data_2D
 
-  end subroutine fill_halo_zero
+    integer(kind=8)    :: mold
+    logical            :: folded_north, cubic_grid 
+    character(len=3)   :: text
+    integer            :: nx_save, ny_save
+    integer            :: id1, id2, id3, id4
+    real               :: gsum1, gsum2, gsum3, gsum4
 
-  !##############################################################################
-  ! this routine fill the halo points for the regular mosaic. 
-  subroutine fill_regular_mosaic_halo(data, data_all, te, tse, ts, tsw, tw, tnw, tn, tne)
-    real, dimension(1-whalo:,1-shalo:,:), intent(inout) :: data
-    real, dimension(:,:,:,:),             intent(in)    :: data_all
-    integer,                              intent(in)    :: te, tse, ts, tsw, tw, tnw, tn, tne
+    folded_north       = .false.
+    cubic_grid         = .false.
 
-       data(nx+1:nx+ehalo, 1:ny,          :) = data_all(1:ehalo,       1:ny,          :, te) ! east
-       data(1:nx,          1-shalo:0,     :) = data_all(1:nx,          ny-shalo+1:ny, :, ts) ! south 
-       data(1-whalo:0,     1:ny,          :) = data_all(nx-whalo+1:nx, 1:ny,          :, tw) ! west
-       data(1:nx,          ny+1:ny+nhalo, :) = data_all(1:nx,          1:nhalo,       :, tn) ! north  
-       data(nx+1:nx+ehalo, 1-shalo:0,     :) = data_all(1:ehalo,       ny-shalo+1:ny, :,tse) ! southeast
-       data(1-whalo:0,     1-shalo:0,     :) = data_all(nx-whalo+1:nx, ny-shalo+1:ny, :,tsw) ! southwest
-       data(nx+1:nx+ehalo, ny+1:ny+nhalo, :) = data_all(1:ehalo,       1:nhalo,       :,tnw) ! northeast
-       data(1-whalo:0,     ny+1:ny+nhalo, :) = data_all(nx-whalo+1:nx, 1:nhalo,       :,tne) ! northwest    
+    nx_save = nx
+    ny_save = ny
+    !--- check the type
+    select case(type)
+    case ( 'Folded-north' )
+       ntiles = 1
+       shift = 0
+       num_contact = 2
+       folded_north = .true.
+       npes_per_tile = npes
+       if(layout_tripolar(1)*layout_tripolar(2) == npes ) then
+          layout = layout_tripolar
+       else
+          call mpp_define_layout( (/1,nx,1,ny/), npes_per_tile, layout )
+       endif      
+    case ( 'Cubic-Grid' )
+       if( nx_cubic == 0 ) then
+          call mpp_error(NOTE,'test_group_update: for Cubic_grid mosaic, nx_cubic is zero, '//&
+               'No test is done for Cubic-Grid mosaic. ' )
+          return
+       endif
+       if( nx_cubic .NE. ny_cubic ) then
+          call mpp_error(NOTE,'test_group_update: for Cubic_grid mosaic, nx_cubic does not equal ny_cubic, '//&
+               'No test is done for Cubic-Grid mosaic. ' )
+          return
+       endif
+       shift = 1
+       nx = nx_cubic
+       ny = ny_cubic
+       ntiles = 6
+       num_contact = 12
+       cubic_grid = .true.
+       if( mod(npes, ntiles) == 0 ) then
+          npes_per_tile = npes/ntiles
+          write(outunit,*)'NOTE from test_mpp_global_sum ==> For Mosaic "', trim(type), &
+               '", each tile will be distributed over ', npes_per_tile, ' processors.'
+       else
+          call mpp_error(NOTE,'test_group_update: npes should be multiple of ntiles No test is done for '//trim(type))
+          return
+       endif
+       if(layout_cubic(1)*layout_cubic(2) == npes_per_tile) then
+          layout = layout_cubic
+       else
+          call mpp_define_layout( (/1,nx,1,ny/), npes_per_tile, layout )
+       endif
+    case default
+       call mpp_error(FATAL, 'test_mpp_global_sum: no such test: '//type)
+    end select
 
+    allocate(layout2D(2,ntiles), global_indices(4,ntiles), pe_start(ntiles), pe_end(ntiles) )
+    do n = 1, ntiles
+       pe_start(n) = (n-1)*npes_per_tile
+       pe_end(n)   = n*npes_per_tile-1
+    end do
 
+    do n = 1, ntiles
+       global_indices(:,n) = (/1,nx,1,ny/)
+       layout2D(:,n)         = layout
+    end do
 
-  end subroutine fill_regular_mosaic_halo
+    allocate(tile1(num_contact), tile2(num_contact) )
+    allocate(istart1(num_contact), iend1(num_contact), jstart1(num_contact), jend1(num_contact) ) 
+    allocate(istart2(num_contact), iend2(num_contact), jstart2(num_contact), jend2(num_contact) ) 
 
-  !################################################################################
-  subroutine fill_folded_north_halo(data, ioff, joff, ishift, jshift, sign)
-    real, dimension(1-whalo:,1-shalo:,:), intent(inout) :: data
-    integer,                              intent(in   ) :: ioff, joff, ishift, jshift, sign    
-    integer  :: nxp, nyp, m1, m2
+    !--- define domain
+    if(folded_north) then
+       !--- Contact line 1, between tile 1 (EAST) and tile 1 (WEST)  --- cyclic
+       tile1(1) = 1; tile2(1) = 1
+       istart1(1) = nx; iend1(1) = nx; jstart1(1) = 1;  jend1(1) = ny
+       istart2(1) = 1;  iend2(1) = 1;  jstart2(1) = 1;  jend2(1) = ny
+       !--- Contact line 2, between tile 1 (NORTH) and tile 1 (NORTH)  --- folded-north-edge
+       tile1(2) = 1; tile2(2) = 1
+       istart1(2) = 1;  iend1(2) = nx/2;   jstart1(2) = ny;  jend1(2) = ny
+       istart2(2) = nx; iend2(2) = nx/2+1; jstart2(2) = ny;  jend2(2) = ny
+       call mpp_define_mosaic(global_indices, layout2D, domain, ntiles, num_contact, tile1, tile2, &
+            istart1, iend1, jstart1, jend1, istart2, iend2, jstart2, jend2,      &
+            pe_start, pe_end, whalo=whalo, ehalo=ehalo, shalo=shalo, nhalo=nhalo, &
+            name = type, symmetry = .false.  )
+    else if( cubic_grid ) then
+       call define_cubic_mosaic(type, domain, (/nx,nx,nx,nx,nx,nx/), (/ny,ny,ny,ny,ny,ny/), &
+            global_indices, layout2D, pe_start, pe_end )
+    endif
 
-    nxp = nx+ishift
-    nyp = ny+jshift
-    m1 = ishift - ioff
-    m2 = 2*ishift - ioff
+    !--- setup data
+    call mpp_get_compute_domain( domain, isc, iec, jsc, jec )
+    call mpp_get_data_domain   ( domain, isd, ied, jsd, jed )
 
-    data(1-whalo:0,                  1:nyp,:) =      data(nx-whalo+1:nx,        1:ny+jshift,:) ! west
-    data(nx+1:nx+ehalo+ishift,       1:nyp,:) =      data(1:ehalo+ishift,       1:ny+jshift,:) ! east
-    if(m1 .GE. 1-whalo) data(1-whalo:m1,  nyp+1:nyp+nhalo,:) = sign*data(whalo+m2:1+ishift:-1, nyp-joff:nyp-nhalo-joff+1:-1,:)
-    data(m1+1:nx+m2,       nyp+1:nyp+nhalo,:) = sign*data(nx+ishift:1:-1,       nyp-joff:nyp-nhalo-joff+1:-1,:)
-    data(nx+m2+1:nxp+ehalo,nyp+1:nyp+nhalo,:) = sign*data(nx:nx-ehalo+m1+1:-1,  nyp-joff:nyp-nhalo-joff+1:-1,:)
+    allocate(data_2d(isd:ied,jsd:jed))
+    allocate(data_3d(isd:ied,jsd:jed,nz))
 
-  end subroutine fill_folded_north_halo
+    do k = 1, nz
+       do j = jsd, jed
+          do i = isd, ied
+             data_3d(i,j,k) =  k*1e3 + i + j*1e-3      
+          enddo
+       enddo
+    enddo
 
-  !################################################################################
-  subroutine fill_folded_south_halo(data, ioff, joff, ishift, jshift, sign)
-    real, dimension(1-whalo:,1-shalo:,:), intent(inout) :: data
-    integer,                              intent(in   ) :: ioff, joff, ishift, jshift, sign    
-    integer  :: nxp, nyp, m1, m2
+    do j = jsd, jed
+       do i = isd, ied
+          data_2d(i,j) =  i*1e3 + j*1e-3      
+       enddo
+    enddo
 
-    nxp = nx+ishift
-    nyp = ny+jshift
-    m1 = ishift - ioff
-    m2 = 2*ishift - ioff
+    id1 = mpp_clock_id( type//' bitwise sum 3D', flags=MPP_CLOCK_SYNC )
+    id2 = mpp_clock_id( type//' EFP sum 3D', flags=MPP_CLOCK_SYNC )
+    id3 = mpp_clock_id( type//' EFP sum 3D check', flags=MPP_CLOCK_SYNC )
+    id4 = mpp_clock_id( type//' non-bitwise sum 3D', flags=MPP_CLOCK_SYNC )
+ 
+    call mpp_clock_begin(id1)
+    do n = 1, num_iter
+       gsum1 = mpp_global_sum(domain, data_3d, flags=BITWISE_EXACT_SUM)
+    enddo
+    call mpp_clock_end(id1)
 
+    call mpp_clock_begin(id2)
+    do n = 1, num_iter
+       gsum2 = mpp_global_sum(domain, data_3d, flags=BITWISE_EFP_SUM)
+    enddo
+    call mpp_clock_end(id2)
 
-    data(1-whalo:0,                  1:nyp,:) =      data(nx-whalo+1:nx,        1:nyp,:) ! west
-    data(nx+1:nx+ehalo+ishift,       1:nyp,:) =      data(1:ehalo+ishift,       1:nyp,:) ! east
-    if(m1 .GE. 1-whalo)data(1-whalo:m1, 1-shalo:0,:) = sign*data(whalo+m2:1+ishift:-1, shalo+jshift:1+jshift:-1,:)
-    data(m1+1:nx+m2,       1-shalo:0,:) = sign*data(nxp:1:-1,             shalo+jshift:1+jshift:-1,:)
-    data(nx+m2+1:nxp+ehalo,1-shalo:0,:) = sign*data(nx:nx-ehalo+m1+1:-1,  shalo+jshift:1+jshift:-1,:)
+    call mpp_clock_begin(id3)
+    do n = 1, num_iter
+       gsum3 = mpp_global_sum(domain, data_3d, flags=BITWISE_EFP_SUM, overflow_check=.true. )
+    enddo
+    call mpp_clock_end(id3)
 
-  end subroutine fill_folded_south_halo
+    call mpp_clock_begin(id4)
+    do n = 1, num_iter
+       gsum4= mpp_global_sum(domain, data_3d)
+    enddo
+    call mpp_clock_end(id4)
+ 
+    write(outunit, *) " ********************************************************************************"
+    write(outunit, *) " global sum for "//type//' bitwise exact sum 3D = ', gsum1
+    write(outunit, *) " global sum for "//type//' bitwise EFP sum 3D = ', gsum2
+    write(outunit, *) " global sum for "//type//' bitwise EFP sum 3D with overflow_check = ', gsum3
+    write(outunit, *) " global sum for "//type//' non-bitwise sum 3D = ', gsum4
+    write(outunit, *) " "
+    write(outunit, *) " chksum for "//type//' bitwise exact sum 3D = ', transfer(gsum1, mold)
+    write(outunit, *) " chksum for "//type//' bitwise EFP sum 3D = ', transfer(gsum2, mold)
+    write(outunit, *) " chksum for "//type//' bitwise EFP sum 3D with overflow_check = ', transfer(gsum3, mold)
+    write(outunit, *) " chksum for "//type//' non-bitwise sum 3D = ', transfer(gsum4, mold)
+    write(outunit, *) " ********************************************************************************"
+
+    id1 = mpp_clock_id( type//' bitwise sum 2D', flags=MPP_CLOCK_SYNC )
+    id2 = mpp_clock_id( type//' EFP sum 2D', flags=MPP_CLOCK_SYNC )
+    id3 = mpp_clock_id( type//' EFP sum 2D check', flags=MPP_CLOCK_SYNC )
+    id4 = mpp_clock_id( type//' non-bitwise sum 2D', flags=MPP_CLOCK_SYNC )
 
-  !################################################################################
-  subroutine fill_folded_west_halo(data, ioff, joff, ishift, jshift, sign)
-    real, dimension(1-whalo:,1-shalo:,:), intent(inout) :: data
-    integer,                              intent(in   ) :: ioff, joff, ishift, jshift, sign    
-    integer  :: nxp, nyp, m1, m2
+    call mpp_clock_begin(id1)
+    do n = 1, num_iter
+       gsum1 = mpp_global_sum(domain, data_2d, flags=BITWISE_EXACT_SUM)
+    enddo
+    call mpp_clock_end(id1)
 
-    nxp = nx+ishift
-    nyp = ny+jshift
-    m1 = jshift - joff
-    m2 = 2*jshift - joff
+    call mpp_clock_begin(id2)
+    do n = 1, num_iter
+       gsum2 = mpp_global_sum(domain, data_2d, flags=BITWISE_EFP_SUM)
+    enddo
+    call mpp_clock_end(id2)
 
-    data(1:nxp, 1-shalo:0, :)      = data(1:nxp, ny-shalo+1:ny, :) ! south
-    data(1:nxp, ny+1:nyp+nhalo, :) = data(1:nxp, 1:nhalo+jshift,:) ! north
-    if(m1 .GE. 1-shalo) data(1-whalo:0, 1-shalo:m1, :) = sign*data(whalo+ishift:1+ishift:-1, shalo+m2:1+jshift:-1,:)
-    data(1-whalo:0, m1+1:ny+m2, :) = sign*data(whalo+ishift:1+ishift:-1, nyp:1:-1, :)
-    data(1-whalo:0, ny+m2+1:nyp+nhalo,:) = sign*data(whalo+ishift:1+ishift:-1, ny:ny-nhalo+m1+1:-1,:)
+    call mpp_clock_begin(id3)
+    do n = 1, num_iter
+       gsum3 = mpp_global_sum(domain, data_2d, flags=BITWISE_EFP_SUM, overflow_check=.true. )
+    enddo
+    call mpp_clock_end(id3)
 
-  end subroutine fill_folded_west_halo
+    call mpp_clock_begin(id4)
+    do n = 1, num_iter
+       gsum4= mpp_global_sum(domain, data_2d)
+    enddo
+    call mpp_clock_end(id4)
+  
+    write(outunit, *) " ********************************************************************************"
+    write(outunit, *) " global sum for "//type//' bitwise exact sum 2D = ', gsum1
+    write(outunit, *) " global sum for "//type//' bitwise EFP sum 2D = ', gsum2
+    write(outunit, *) " global sum for "//type//' bitwise EFP sum 2D with overflow_check = ', gsum3
+    write(outunit, *) " global sum for "//type//' non-bitwise sum 2D = ', gsum4
+    write(outunit, *) " "
+    write(outunit, *) " chksum for "//type//' bitwise exact sum 2D = ', transfer(gsum1, mold)
+    write(outunit, *) " chksum for "//type//' bitwise EFP sum 2D = ', transfer(gsum2, mold)
+    write(outunit, *) " chksum for "//type//' bitwise EFP sum 2D with overflow_check = ', transfer(gsum3, mold)
+    write(outunit, *) " chksum for "//type//' non-bitwise sum 2D = ', transfer(gsum4, mold)
+    write(outunit, *) " ********************************************************************************"
 
-  !################################################################################
-  subroutine fill_folded_east_halo(data, ioff, joff, ishift, jshift, sign)
-    real, dimension(1-whalo:,1-shalo:,:), intent(inout) :: data
-    integer,                              intent(in   ) :: ioff, joff, ishift, jshift, sign    
-    integer  :: nxp, nyp, m1, m2
 
-    nxp = nx+ishift
-    nyp = ny+jshift
-    m1 = jshift - joff
-    m2 = 2*jshift - joff
 
-    data(1:nxp, 1-shalo:0, :)      = data(1:nxp, ny-shalo+1:ny, :) ! south
-    data(1:nxp, ny+1:nyp+nhalo, :) = data(1:nxp, 1:nhalo+jshift,:) ! north
-    if(m1 .GE. 1-shalo) data(nxp+1:nxp+ehalo, 1-shalo:m1, :) = sign*data(nxp-ioff:nxp-ehalo-ioff+1:-1, shalo+m2:1+jshift:-1,:)
-    data(nxp+1:nxp+ehalo, m1+1:ny+m2, :) = sign*data(nxp-ioff:nxp-ehalo-ioff+1:-1, nyp:1:-1, :)
-    data(nxp+1:nxp+ehalo, ny+m2+1:nyp+nhalo,:) = sign*data(nxp-ioff:nxp-ehalo-ioff+1:-1, ny:ny-nhalo+m1+1:-1,:)
+    nx = nx_save
+    ny = ny_save
 
-  end subroutine fill_folded_east_halo
+  end subroutine test_mpp_global_sum
 
-  !################################################################################
-  subroutine fill_four_tile_bound(data_all, is, ie, js, je, ioff, joff, tile, &
-                                   ebound, sbound, wbound, nbound )
-    real, dimension(:,:,:,:),       intent(in)    :: data_all
-    integer,                        intent(in)    :: is, ie, js, je
-    integer,                        intent(in)    :: tile, ioff, joff
-    real, dimension(:,:), optional, intent(inout) :: ebound, sbound, wbound, nbound
-    integer                                       :: tw, te, ts, tn
+  !###############################################################
+  subroutine test_group_update( type )
+    character(len=*), intent(in) :: type
 
-    if(tile == 1 .OR. tile == 3) te = tile + 1
-    if(tile == 2 .OR. tile == 4) te = tile - 1
-    if(tile == 1 .OR. tile == 2) ts = tile + 2
-    if(tile == 3 .OR. tile == 4) ts = tile - 2
-    tw = te;   tn = ts
-    if(present(ebound)) then
-       if( ie == nx ) then
-          ebound(:,:) = data_all(1, js:je+joff, :, te)
-       else
-          ebound(:,:) = data_all(ie+ioff, js:je+joff, :, tile)
-       end if
-    end if
+    type(domain2D) :: domain
+    integer        :: num_contact, ntiles, npes_per_tile
+    integer        :: i, j, k, l, n, shift
+    integer        :: isc, iec, jsc, jec, isd, ied, jsd, jed
+    integer        :: ism, iem, jsm, jem
 
-    if(present(wbound)) then
-       if( is == 1 ) then
-          wbound(:,:) = data_all(nx+ioff, js:je+joff, :, tw)
-       else
-          wbound(:,:) = data_all(is, js:je+joff, :, tile)
-       end if
-    end if
+    integer, allocatable, dimension(:)       :: pe_start, pe_end, tile1, tile2
+    integer, allocatable, dimension(:)       :: istart1, iend1, jstart1, jend1
+    integer, allocatable, dimension(:)       :: istart2, iend2, jstart2, jend2
+    integer, allocatable, dimension(:,:)     :: layout2D, global_indices
+    real,    allocatable, dimension(:,:,:,:) :: x1, y1, x2, y2
+    real,    allocatable, dimension(:,:,:,:) :: a1, a2
+    real,    allocatable, dimension(:,:,:)   :: base
+    integer            :: id1, id2, id3   
+    logical            :: folded_north
+    logical            :: cubic_grid
+    character(len=3)   :: text
+    integer            :: nx_save, ny_save
+    type(mpp_group_update_type) :: group_update
+    type(mpp_group_update_type), allocatable :: update_list(:)
 
-    if(present(sbound)) then
-       if( js == 1 ) then
-          sbound(:,:) = data_all(is:ie+ioff, ny+joff, :, ts)
-       else
-          sbound(:,:) = data_all(is:ie+ioff, js, :, tile)
-       end if
-    end if
+    folded_north       = .false.
+    cubic_grid         = .false.
 
-    if(present(nbound)) then
-       if( je == ny ) then
-          nbound(:,:) = data_all(is:ie+ioff, 1, :, tn)
+    nx_save = nx
+    ny_save = ny
+    !--- check the type
+    select case(type)
+    case ( 'Folded-north' )
+       ntiles = 1
+       shift = 0
+       num_contact = 2
+       folded_north = .true.
+       npes_per_tile = npes
+       if(layout_tripolar(1)*layout_tripolar(2) == npes ) then
+          layout = layout_tripolar
        else
-          nbound(:,:) = data_all(is:ie+ioff, je+joff, :, tile)
-       end if
-    end if
+          call mpp_define_layout( (/1,nx,1,ny/), npes_per_tile, layout )
+       endif      
+    case ( 'Cubic-Grid' )
+       if( nx_cubic == 0 ) then
+          call mpp_error(NOTE,'test_group_update: for Cubic_grid mosaic, nx_cubic is zero, '//&
+               'No test is done for Cubic-Grid mosaic. ' )
+          return
+       endif
+       if( nx_cubic .NE. ny_cubic ) then
+          call mpp_error(NOTE,'test_group_update: for Cubic_grid mosaic, nx_cubic does not equal ny_cubic, '//&
+               'No test is done for Cubic-Grid mosaic. ' )
+          return
+       endif
+       shift = 1
+       nx = nx_cubic
+       ny = ny_cubic
+       ntiles = 6
+       num_contact = 12
+       cubic_grid = .true.
+       if( mod(npes, ntiles) == 0 ) then
+          npes_per_tile = npes/ntiles
+          write(outunit,*)'NOTE from update_domains_performance ==> For Mosaic "', trim(type), &
+               '", each tile will be distributed over ', npes_per_tile, ' processors.'
+       else
+          call mpp_error(NOTE,'test_group_update: npes should be multiple of ntiles No test is done for '//trim(type))
+          return
+       endif
+       if(layout_cubic(1)*layout_cubic(2) == npes_per_tile) then
+          layout = layout_cubic
+       else
+          call mpp_define_layout( (/1,nx,1,ny/), npes_per_tile, layout )
+       endif
+    case default
+       call mpp_error(FATAL, 'test_group_update: no such test: '//type)
+    end select
 
-    return
+    allocate(layout2D(2,ntiles), global_indices(4,ntiles), pe_start(ntiles), pe_end(ntiles) )
+    do n = 1, ntiles
+       pe_start(n) = (n-1)*npes_per_tile
+       pe_end(n)   = n*npes_per_tile-1
+    end do
 
-  end subroutine fill_four_tile_bound
+    do n = 1, ntiles
+       global_indices(:,n) = (/1,nx,1,ny/)
+       layout2D(:,n)         = layout
+    end do
 
+    allocate(tile1(num_contact), tile2(num_contact) )
+    allocate(istart1(num_contact), iend1(num_contact), jstart1(num_contact), jend1(num_contact) ) 
+    allocate(istart2(num_contact), iend2(num_contact), jstart2(num_contact), jend2(num_contact) ) 
 
-  !################################################################################
-  subroutine fill_folded_north_bound(data_all, is, ie, js, je, ioff, joff, tile, &
-                                     sbound, wbound)
-    real, dimension(:,:,:),       intent(in)    :: data_all
-    integer,                        intent(in)    :: is, ie, js, je
-    integer,                        intent(in)    :: tile, ioff, joff
-    real, dimension(:,:), optional, intent(inout) :: sbound, wbound
-    integer                                       :: tw, te, ts, tn
+    !--- define domain
+    if(folded_north) then
+       !--- Contact line 1, between tile 1 (EAST) and tile 1 (WEST)  --- cyclic
+       tile1(1) = 1; tile2(1) = 1
+       istart1(1) = nx; iend1(1) = nx; jstart1(1) = 1;  jend1(1) = ny
+       istart2(1) = 1;  iend2(1) = 1;  jstart2(1) = 1;  jend2(1) = ny
+       !--- Contact line 2, between tile 1 (NORTH) and tile 1 (NORTH)  --- folded-north-edge
+       tile1(2) = 1; tile2(2) = 1
+       istart1(2) = 1;  iend1(2) = nx/2;   jstart1(2) = ny;  jend1(2) = ny
+       istart2(2) = nx; iend2(2) = nx/2+1; jstart2(2) = ny;  jend2(2) = ny
+       call mpp_define_mosaic(global_indices, layout2D, domain, ntiles, num_contact, tile1, tile2, &
+            istart1, iend1, jstart1, jend1, istart2, iend2, jstart2, jend2,      &
+            pe_start, pe_end, whalo=whalo, ehalo=ehalo, shalo=shalo, nhalo=nhalo, &
+            name = type, symmetry = .false.  )
+    else if( cubic_grid ) then
+       call define_cubic_mosaic(type, domain, (/nx,nx,nx,nx,nx,nx/), (/ny,ny,ny,ny,ny,ny/), &
+            global_indices, layout2D, pe_start, pe_end )
+    endif
 
-    if(tile .NE. 1) call mpp_error(FATAL, "fill_folded_north_bound: tile must be 1")
+    !--- setup data
+    call mpp_get_compute_domain( domain, isc, iec, jsc, jec )
+    call mpp_get_data_domain   ( domain, isd, ied, jsd, jed )
+    call mpp_get_memory_domain   ( domain, ism, iem, jsm, jem )
 
-    if(present(wbound)) then
-       if( is == 1 ) then
-          wbound(:,:) = data_all(nx+ioff, js:je+joff, :)
-       else
-          wbound(:,:) = data_all(is, js:je+joff, :)
-       end if
-    end if
+    if(num_fields<1) then
+       call mpp_error(FATAL, "test_mpp_domains: num_fields must be a positive integer")
+    endif
 
-    if(present(sbound)) then
-       if( js == 1 ) then
-          sbound(:,:) = 0
-       else
-          sbound(:,:) = data_all(is:ie+ioff, js, :)
-       end if
-    end if
+    allocate(update_list(num_fields))
+ 
+    id1 = mpp_clock_id( type//' group 2D', flags=MPP_CLOCK_SYNC )
+    id2 = mpp_clock_id( type//' non-group 2D', flags=MPP_CLOCK_SYNC )
+    id3 = mpp_clock_id( type//' non-block group 2D', flags=MPP_CLOCK_SYNC )
+
+    allocate( a1(ism:iem,      jsm:jem,       nz, num_fields) )  
+    allocate( x1(ism:iem+shift,jsm:jem,       nz, num_fields) ) 
+    allocate( y1(ism:iem,      jsm:jem+shift, nz, num_fields) ) 
+    allocate( a2(ism:iem,      jsm:jem,       nz, num_fields) )
+    allocate( x2(ism:iem+shift,jsm:jem,       nz, num_fields) )
+    allocate( y2(ism:iem,      jsm:jem+shift, nz, num_fields) )
+    allocate( base(isc:iec+shift,jsc:jec+shift,nz) )
+    a1 = 0; x1 = 0; y1 = 0
+
+    base = 0
+    do k = 1,nz
+       do j = jsc, jec+shift
+          do i = isc, iec+shift
+             base(i,j,k) = k + i*1e-3 + j*1e-6
+          end do
+       end do
+    end do
 
-    return
+    !--- Test for partial direction update
+    do l =1, num_fields
+       call mpp_create_group_update(group_update, a1(:,:,:,l), domain, flags=WUPDATE+SUPDATE)
+    end do    
 
-  end subroutine fill_folded_north_bound
+    do l = 1, num_fields
+       a1(isc:iec,jsc:jec,:,l) = base(isc:iec,jsc:jec,:) + l*1e3
+       do k=1,nz
+          do i=isc-1,iec+1
+             a1(i,jsc-1,k,l) = 999;
+             a1(i,jec+1,k,l) = 999;
+          enddo
+          do j=jsc,jec
+             a1(isc-1,j,k,l) = 999
+             a1(iec+1,j,k,l) = 999
+          enddo
+       enddo
+    enddo
+    
+    a2 = a1
+    call mpp_do_group_update(group_update, domain, a1(isc,jsc,1,1))
 
-  !################################################################################
-  subroutine fill_cubic_grid_bound(data1_all, data2_all, is, ie, js, je, ioff, joff, tile, sign1, sign2, &
-                                   ebound, sbound, wbound, nbound )
-    real, dimension(:,:,:,:),       intent(in)    :: data1_all, data2_all
-    integer,                        intent(in)    :: is, ie, js, je
-    integer,                        intent(in)    :: tile, ioff, joff, sign1, sign2
-    real, dimension(:,:), optional, intent(inout) :: ebound, sbound, wbound, nbound
-    integer                                       :: tw, te, ts, tn
+    do l = 1, num_fields
+       call mpp_update_domains( a2(:,:,:,l), domain, flags=WUPDATE+SUPDATE, complete=l==num_fields )
+    enddo
 
-    if(mod(tile,2) == 0) then ! tile 2, 4, 6
-       tw = tile - 1; te = tile + 2; ts = tile - 2; tn = tile + 1
-       if(te > 6 ) te = te - 6
-       if(ts < 1 ) ts = ts + 6
-       if(tn > 6 ) tn = tn - 6
-       !--- East bound
-       if(present(ebound)) then
-          if(ie == nx) then                
-             ebound(:,:) = sign1*data2_all(nx+joff-js+1:nx-je+1:-1,1,:,te)
-          else
-             ebound(:,:) = data1_all(ie+ioff, js:je+joff, :,tile)
-          end if
-       end if
-       !--- South bound
-       if(present(sbound)) then
-          if(js == 1) then                
-             sbound(:,:) = sign2*data2_all(nx+joff, ny+ioff-is+1:ny-ie+1:-1,:,ts)
-          else
-             sbound(:,:) = data1_all(is:ie+ioff, js, :,tile)
-          end if
-       end if
+    do l = 1, num_fields
+       write(text, '(i3.3)') l
+       call compare_checksums(a1(isd:ied,jsd:jed,:,l),a2(isd:ied,jsd:jed,:,l),type//' CENTER South West '//text)
+    enddo
 
-       !--- West bound
-       if(present(wbound)) then
-          if(is == 1) then
-             wbound(:,:) = data1_all(nx+ioff, js:je+joff,:,tw)
-          else
-             wbound(:,:) = data1_all(is, js:je+joff,:,tile)
-          end if
-       end if
+    call mpp_clear_group_update(group_update)
 
-       !--- north bound
-       if(present(nbound)) then
-          if(je == ny) then                
-             nbound(:,:) = data1_all(is:ie+ioff, 1,:,tn)
-          else
-             nbound(:,:) = data1_all(is:ie+ioff, je+joff, :,tile)
-          end if
-       end if
-    else ! tile 1, 3, 5
-       tw = tile - 2; te = tile + 1; ts = tile - 1; tn = tile + 2
-       if(tw < 1 ) tw = tw + 6
-       if(ts < 1 ) ts = ts + 6
-       if(tn > 6 ) tn = tn - 6
-       !--- East bound
-       if(present(ebound)) then
-          if(ie == nx) then                
-             ebound(:,:) = data1_all(1, js:je+joff, :,te) 
-          else
-             ebound(:,:) = data1_all(ie+ioff, js:je+joff, :,tile)
-          end if
-       end if
-       !--- South bound
-       if(present(sbound)) then
-          if(js == 1) then                
-             sbound(:,:) = data1_all(is:ie+ioff,ny+joff,:,ts)
-          else
-             sbound(:,:) = data1_all(is:ie+ioff, js, :,tile)
-          end if
-       end if
+    !--- Test for DGRID update
+    if(type == 'Cubic-Grid' ) then
+       x1 = 0; y1 = 0
+       do l =1, num_fields
+          call mpp_create_group_update(group_update, x1(:,:,:,l), y1(:,:,:,l), domain, gridtype=DGRID_NE)
+       end do
 
-       !--- West bound
-       if(present(wbound)) then
-          if(is == 1) then
-             wbound(:,:) = sign1*data2_all(nx+joff-js+1:nx-je+1:-1,ny+ioff,:,tw)
-          else
-             wbound(:,:) = data1_all(is, js:je+joff,:,tile)
-          end if
-       end if
+       do l = 1, num_fields
+          y1(isc:iec+shift,jsc:jec,      :,l) = base(isc:iec+shift,jsc:jec,      :) + l*1e3 + 1e6
+          x1(isc:iec,      jsc:jec+shift,:,l) = base(isc:iec,      jsc:jec+shift,:) + l*1e3 + 2*1e6
+       enddo
+       x2 = x1; y2 = y1
+       call mpp_start_group_update(group_update, domain, x1(isc,jsc,1,1))
+       call mpp_complete_group_update(group_update, domain, x1(isc,jsc,1,1))
 
-       !--- north bound
-       if(present(nbound)) then
-          if(je == ny) then                
-             nbound(:,:) = sign2*data2_all(1, ny+ioff-is+1:ny-ie+1:-1,:,tn)
-          else
-             nbound(:,:) = data1_all(is:ie+ioff, je+joff, :,tile)
-          end if
-       end if
+       do l = 1, num_fields
+          call mpp_update_domains( x2(:,:,:,l), y2(:,:,:,l), domain, gridtype=DGRID_NE, complete=l==num_fields )
+       enddo
 
-    end if
+    !--- compare checksum
+       do l = 1, num_fields
+          write(text, '(i3.3)') l
+          call compare_checksums(x1(isd:ied+shift,jsd:jed,      :,l),x2(isd:ied+shift,jsd:jed,      :,l),type//' DGRID X'//text)
+          call compare_checksums(y1(isd:ied,      jsd:jed+shift,:,l),y2(isd:ied,      jsd:jed+shift,:,l),type//' DGRID Y'//text)
+       enddo
 
-  end subroutine fill_cubic_grid_bound
+       call mpp_clear_group_update(group_update)
+    endif
+    !--- Test for CGRID
+    a1 = 0; x1 = 0; y1 = 0
+    do l =1, num_fields
+       call mpp_create_group_update(group_update, a1(:,:,:,l), domain)
+       call mpp_create_group_update(group_update, x1(:,:,:,l), y1(:,:,:,l), domain, gridtype=CGRID_NE)
+    end do
 
-  !##############################################################################
-  ! this routine fill the halo points for the cubic grid. ioff and joff is used to distinguish
-  ! T, C, E, or N-cell
-  subroutine fill_cubic_grid_halo(data, data1_all, data2_all, tile, ioff, joff, sign1, sign2)
-    real, dimension(1-whalo:,1-shalo:,:), intent(inout) :: data
-    real, dimension(:,:,:,:),             intent(in)    :: data1_all, data2_all
-    integer,                              intent(in)    :: tile, ioff, joff, sign1, sign2 
-    integer                                             :: lw, le, ls, ln
+    do n = 1, num_iter
+       a1 = 0; x1 = 0; y1 = 0
+       do l = 1, num_fields
+          a1(isc:iec,      jsc:jec,      :,l) = base(isc:iec,      jsc:jec,      :) + l*1e3
+          x1(isc:iec+shift,jsc:jec,      :,l) = base(isc:iec+shift,jsc:jec,      :) + l*1e3 + 1e6
+          y1(isc:iec,      jsc:jec+shift,:,l) = base(isc:iec,      jsc:jec+shift,:) + l*1e3 + 2*1e6
+       enddo
+       a2 = a1; x2 = x1; y2 = y1
+       call mpp_clock_begin(id1)
+       call mpp_do_group_update(group_update, domain, a1(isc,jsc,1,1))
+       call mpp_clock_end  (id1)
+
+       call mpp_clock_begin(id2)
+       do l = 1, num_fields
+          call mpp_update_domains( a2(:,:,:,l), domain, complete=l==num_fields )
+       enddo
+       do l = 1, num_fields
+          call mpp_update_domains( x2(:,:,:,l), y2(:,:,:,l), domain, gridtype=CGRID_NE, complete=l==num_fields )
+       enddo
+       call mpp_clock_end(id2)
+
+       !--- compare checksum
+       do l = 1, num_fields
+          write(text, '(i3.3)') l
+          call compare_checksums(a1(isd:ied,      jsd:jed,      :,l),a2(isd:ied,      jsd:jed,      :,l),type//' CENTER '//text)
+          call compare_checksums(x1(isd:ied+shift,jsd:jed,      :,l),x2(isd:ied+shift,jsd:jed,      :,l),type//' CGRID X'//text)
+          call compare_checksums(y1(isd:ied,      jsd:jed+shift,:,l),y2(isd:ied,      jsd:jed+shift,:,l),type//' CGRID Y'//text)
+       enddo
+       a1 = 0; x1 = 0; y1 = 0
+       do l = 1, num_fields
+          a1(isc:iec,      jsc:jec,      :,l) = base(isc:iec,      jsc:jec,      :) + l*1e3
+          x1(isc:iec+shift,jsc:jec,      :,l) = base(isc:iec+shift,jsc:jec,      :) + l*1e3 + 1e6
+          y1(isc:iec,      jsc:jec+shift,:,l) = base(isc:iec,      jsc:jec+shift,:) + l*1e3 + 2*1e6
+       enddo
+       call mpp_clock_begin(id3)
+       call mpp_start_group_update(group_update, domain, a1(isc,jsc,1,1))
+       call mpp_complete_group_update(group_update, domain, a1(isc,jsc,1,1))
+       call mpp_clock_end  (id3)
+       !--- compare checksum
+       do l = 1, num_fields
+          write(text, '(i3.3)') l
+          call compare_checksums(a1(isd:ied,      jsd:jed,      :,l),a2(isd:ied,      jsd:jed,      :,l), &
+                                 type//' nonblock CENTER '//text)
+          call compare_checksums(x1(isd:ied+shift,jsd:jed,      :,l),x2(isd:ied+shift,jsd:jed,      :,l), &
+                                 type//' nonblock CGRID X'//text)
+          call compare_checksums(y1(isd:ied,      jsd:jed+shift,:,l),y2(isd:ied,      jsd:jed+shift,:,l), &
+                                 type//' nonblock CGRID Y'//text)
+       enddo
 
-    if(mod(tile,2) == 0) then ! tile 2, 4, 6
-       lw = tile - 1; le = tile + 2; ls = tile - 2; ln = tile + 1
-       if(le > 6 ) le = le - 6
-       if(ls < 1 ) ls = ls + 6
-       if(ln > 6 ) ln = ln - 6
-       data(1-whalo:0, 1:ny+joff, :) = data1_all(nx-whalo+1:nx, 1:ny+joff, :, lw) ! west 
-       do i = 1, ehalo 
-          data(nx+i+ioff, 1:ny+joff, :)    = sign1*data2_all(nx+joff:1:-1, i+ioff, :, le) ! east 
-       end do
-       do i = 1, shalo 
-          data(1:nx+ioff, 1-i, :)     = sign2*data2_all(nx-i+1, ny+ioff:1:-1, :, ls) ! south 
-       end do
-       data(1:nx+ioff, ny+1+joff:ny+nhalo+joff, :) = data1_all(1:nx+ioff, 1+joff:nhalo+joff, :, ln) ! north
-    else ! tile 1, 3, 5
-       lw = tile - 2; le = tile + 1; ls = tile - 1; ln = tile + 2
-       if(lw < 1 ) lw = lw + 6
-       if(ls < 1 ) ls = ls + 6
-       if(ln > 6 ) ln = ln - 6
-       do i = 1, whalo 
-          data(1-i, 1:ny+joff, :)     = sign1*data2_all(nx+joff:1:-1, ny-i+1, :, lw) ! west 
-       end do
-       data(nx+1+ioff:nx+ehalo+ioff, 1:ny+joff, :) = data1_all(1+ioff:ehalo+ioff, 1:ny+joff, :, le) ! east 
-       data(1:nx+ioff, 1-shalo:0, :)     = data1_all(1:nx+ioff, ny-shalo+1:ny, :, ls) ! south 
-       do i = 1, nhalo 
-          data(1:nx+ioff, ny+i+joff, :)    = sign2*data2_all(i+joff, ny+ioff:1:-1, :, ln) ! north 
+    enddo
+
+    call mpp_clear_group_update(group_update)
+
+    !--- The following is to test overlapping start and complete
+    if( num_fields > 1 ) then
+       do l =1, num_fields
+          call mpp_create_group_update(update_list(l), a1(:,:,:,l), domain)
+          call mpp_create_group_update(update_list(l), x1(:,:,:,l), y1(:,:,:,l), domain, gridtype=CGRID_NE)
        end do
-    end if
 
-  end subroutine fill_cubic_grid_halo
-    
-   !#####################################################################
-  subroutine test_nonuniform_mosaic( type )
-    character(len=*), intent(in) :: type
+       do n = 1, num_iter
 
-    type(domain2D)               :: domain
-    integer                      :: num_contact, ntiles, ntile_per_pe
-    integer                      :: i, j, k, n, nxm, nym, ni, nj, shift
-    integer                      :: ism, iem, jsm, jem, isc, iec, jsc, jec
-    integer                      :: isd, ied, jsd, jed
-    integer                      :: indices(4), msize(2)
-    character(len=128)           :: type2
+          a1 = 0; x1 = 0; y1 = 0
+          do l = 1, num_fields
+             a1(isc:iec,      jsc:jec,      :,l) = base(isc:iec,      jsc:jec,      :) + l*1e3
+             x1(isc:iec+shift,jsc:jec,      :,l) = base(isc:iec+shift,jsc:jec,      :) + l*1e3 + 1e6
+             y1(isc:iec,      jsc:jec+shift,:,l) = base(isc:iec,      jsc:jec+shift,:) + l*1e3 + 2*1e6
+          enddo
+          do l = 1, num_fields-1
+             call mpp_start_group_update(update_list(l), domain, a1(isc,jsc,1,1))
+          enddo
 
-    integer, allocatable, dimension(:)       :: tile
-    integer, allocatable, dimension(:)       :: pe_start, pe_end, tile1, tile2
-    integer, allocatable, dimension(:)       :: istart1, iend1, jstart1, jend1
-    integer, allocatable, dimension(:)       :: istart2, iend2, jstart2, jend2
-    integer, allocatable, dimension(:,:)     :: layout2D, global_indices
-    real,    allocatable, dimension(:,:,:,:) :: global1_all, global2_all
-    real,    allocatable, dimension(:,:,:,:) :: global1, global2, x, y  
+          call mpp_complete_group_update(update_list(1), domain, a1(isc,jsc,1,1))
+          call mpp_start_group_update(update_list(num_fields), domain, a1(isc,jsc,1,1))
+          do l = 2, num_fields
+             call mpp_complete_group_update(update_list(l), domain, a1(isc,jsc,1,1))
+          enddo
+          !--- compare checksum
+          do l = 1, num_fields
+             write(text, '(i3.3)') l
+             call compare_checksums(a1(isd:ied,      jsd:jed,      :,l),a2(isd:ied,      jsd:jed,      :,l), &
+                                    type//' multiple nonblock CENTER '//text)
+             call compare_checksums(x1(isd:ied+shift,jsd:jed,      :,l),x2(isd:ied+shift,jsd:jed,      :,l), &
+                                    type//' multiple nonblock CGRID X'//text)
+             call compare_checksums(y1(isd:ied,      jsd:jed+shift,:,l),y2(isd:ied,      jsd:jed+shift,:,l), &
+                                    type//' multiple nonblock CGRID Y'//text)
+          enddo
 
-    shift = 0
-    select case(type)
-    case('Five-Tile') ! one tile will run on pe 0 and other four tiles will run on pe 1
-       shift = 1      ! one extra point for symmetry domain
-       ntiles = 5     ! tile 1 with resolution 2*nx and 2*ny and the tiles are nx and ny.
-       num_contact = 11
-       if(npes .NE. 2) then
-          call mpp_error(NOTE,'TEST_MPP_DOMAINS: Five-Tile mosaic will not be tested because npes is not 2')
-          return
-       end if 
-       nxm = 2*nx; nym = 2*ny
-       layout = 1
-       if( pe == 0) then
-          ntile_per_pe = 1
-          allocate(tile(ntile_per_pe))
-          tile = 1
-          indices = (/1,2*nx,1,2*ny/)
-          ni = 2*nx; nj = 2*ny
-       else
-          ntile_per_pe = 4
-          allocate(tile(ntile_per_pe))
-          do n = 1, ntile_per_pe
-             tile(n) = n + 1
-          end do
-          indices = (/1,nx,1,ny/)
-          ni = nx; nj = ny
-       end if
-       allocate(pe_start(ntiles), pe_end(ntiles) )
-       pe_start(1) = 0; pe_start(2:) = 1
-       pe_end = pe_start
-    case default
-       call mpp_error(FATAL, 'TEST_MPP_DOMAINS: no such test: '//type)
-    end select
+       enddo
+    endif
 
-    allocate(layout2D(2,ntiles), global_indices(4,ntiles) )
+    do l =1, num_fields
+      call mpp_clear_group_update(update_list(l))
+    enddo
+    deallocate(update_list)
 
-    do n = 1, ntiles
-       if(n==1) then
-          global_indices(:,n) = (/1,2*nx,1,2*ny/)
-       else
-          global_indices(:,n) = (/1,nx,1,ny/)
-       endif  
-!       global_indices(:,n) = indices
-       layout2D(:,n)       = layout
-    end do
+    !--- test scalar 4-D variable
+    call mpp_create_group_update(group_update, a1(:,:,:,:), domain)
 
-    allocate(tile1(num_contact), tile2(num_contact) )
-    allocate(istart1(num_contact), iend1(num_contact), jstart1(num_contact), jend1(num_contact) ) 
-    allocate(istart2(num_contact), iend2(num_contact), jstart2(num_contact), jend2(num_contact) ) 
+    a1 = 0; x1 = 0; y1 = 0
+    do l = 1, num_fields
+       a1(isc:iec,      jsc:jec,      :,l) = base(isc:iec,      jsc:jec,      :) + l*1e3
+    enddo
+    a2 = a1; x2 = x1; y2 = y1
+    call mpp_clock_begin(id1)
+    call mpp_do_group_update(group_update, domain, a1(isc,jsc,1,1))
+    call mpp_clock_end  (id1)
 
-    !--- define domain
-    select case(type)
-    case( 'Five-Tile' )
-       !--- Contact line 1, between tile 1 (EAST) and tile 2 (WEST)
-       tile1(1) = 1; tile2(1) = 2
-       istart1(1) = 2*nx; iend1(1) = 2*nx; jstart1(1) = 1;  jend1(1) = ny
-       istart2(1) = 1;    iend2(1) = 1;    jstart2(1) = 1;  jend2(1) = ny
-       !--- Contact line 2, between tile 1 (EAST) and tile 4 (WEST)
-       tile1(2) = 1; tile2(2) = 4
-       istart1(2) = 2*nx; iend1(2) = 2*nx; jstart1(2) = ny+1; jend1(2) = 2*ny
-       istart2(2) = 1;    iend2(2) = 1;    jstart2(2) = 1;    jend2(2) = ny
-       !--- Contact line 3, between tile 1 (SOUTH) and tile 1 (NORTH)
-       tile1(3) = 1; tile2(3) = 1
-       istart1(3) = 1; iend1(3) = 2*nx; jstart1(3) = 1;    jend1(3) = 1
-       istart2(3) = 1; iend2(3) = 2*nx; jstart2(3) = 2*ny; jend2(3) = 2*ny
-       !--- Contact line 4, between tile 1 (WEST) and tile 3 (EAST)
-       tile1(4) = 1; tile2(4) = 3
-       istart1(4) = 1;  iend1(4) = 1;  jstart1(4) = 1;  jend1(4) = ny
-       istart2(4) = nx; iend2(4) = nx; jstart2(4) = 1;  jend2(4) = ny
-       !--- Contact line 5, between tile 1 (WEST) and tile 5 (EAST)
-       tile1(5) = 1; tile2(5) = 5
-       istart1(5) = 1;  iend1(5) = 1;  jstart1(5) = ny+1;  jend1(5) = 2*ny
-       istart2(5) = nx; iend2(5) = nx; jstart2(5) = 1;     jend2(5) = ny
-       !--- Contact line 6, between tile 2 (EAST) and tile 3 (WEST)
-       tile1(6) = 2; tile2(6) = 3
-       istart1(6) = nx; iend1(6) = nx; jstart1(6) = 1;  jend1(6) = ny
-       istart2(6) = 1;  iend2(6) = 1;  jstart2(6) = 1;  jend2(6) = ny       
-       !--- Contact line 7, between tile 2 (SOUTH) and tile 4 (NORTH)  --- cyclic
-       tile1(7) = 2; tile2(7) = 4
-       istart1(7) = 1;  iend1(7) = nx; jstart1(7) = 1;   jend1(7) = 1
-       istart2(7) = 1;  iend2(7) = nx; jstart2(7) = ny;  jend2(7) = ny
-       !--- Contact line 8, between tile 2 (NORTH) and tile 4 (SOUTH) 
-       tile1(8) = 2; tile2(8) = 4
-       istart1(8) = 1;  iend1(8) = nx; jstart1(8) = ny;  jend1(8) = ny
-       istart2(8) = 1;  iend2(8) = nx; jstart2(8) = 1;   jend2(8) = 1
-       !--- Contact line 9, between tile 3 (SOUTH) and tile 5 (NORTH)  --- cyclic
-       tile1(9) = 3; tile2(9) = 5
-       istart1(9) = 1;  iend1(9) = nx; jstart1(9) = 1;   jend1(9) = 1
-       istart2(9) = 1;  iend2(9) = nx; jstart2(9) = ny;  jend2(9) = ny
-       !--- Contact line 10, between tile 3 (NORTH) and tile 5 (SOUTH) 
-       tile1(10) = 3; tile2(10) = 5
-       istart1(10) = 1;  iend1(10) = nx; jstart1(10) = ny;  jend1(10) = ny
-       istart2(10) = 1;  iend2(10) = nx; jstart2(10) = 1;   jend2(10) = 1
-       !--- Contact line 11, between tile 4 (EAST) and tile 5 (WEST)
-       tile1(11) = 4; tile2(11) = 5
-       istart1(11) = nx; iend1(11) = nx; jstart1(11) = 1;  jend1(11) = ny
-       istart2(11) = 1;  iend2(11) = 1;  jstart2(11) = 1;  jend2(11) = ny  
-       msize(1) = 2*nx + whalo + ehalo
-       msize(2) = 2*ny + shalo + nhalo
-       call mpp_define_mosaic(global_indices, layout2D, domain, ntiles, num_contact, tile1, tile2, &
-            istart1, iend1, jstart1, jend1, istart2, iend2, jstart2, jend2,      &
-            pe_start, pe_end, whalo=whalo, ehalo=ehalo, shalo=shalo, nhalo=nhalo,       &
-            name = type, memory_size = msize, symmetry = .true.  )
-    end select
-    
-    !--- setup data
-    allocate(global1_all(1:nxm,1:nym,nz, ntiles) )  
-    allocate(global1(1-whalo:ni+ehalo,1-shalo:nj+nhalo,nz, ntile_per_pe) )   
-    do n = 1, ntiles
-       do k = 1, nz
-          do j = 1, nym
-             do i = 1, nxm
-                global1_all(i,j,k,n) = n + i*1.0e-3 + j*1.0e-6 + k*1.0e-9
-             end do
-          end do
-       end do
-    end do
+    call mpp_clock_begin(id2)
+    call mpp_update_domains( a2(:,:,:,:), domain )
+    call mpp_clock_end(id2)
 
-    do n = 1, ntile_per_pe
-       global1(1:ni,1:nj,:,n) = global1_all(1:ni,1:nj,:,tile(n))
-    end do
+    !--- compare checksum
+    do l = 1, num_fields
+       write(text, '(i3.3)') l
+       call compare_checksums(a1(isd:ied, jsd:jed, :,l),a2(isd:ied, jsd:jed, :,l),type//' 4D CENTER '//text)
+    enddo
 
-    call mpp_get_compute_domain( domain, isc, iec, jsc, jec )
-    call mpp_get_data_domain   ( domain, isd, ied, jsd, jed )
-    call mpp_get_memory_domain   ( domain, ism, iem, jsm, jem )
+    a1 = 0
+    do l = 1, num_fields
+       a1(isc:iec,      jsc:jec,      :,l) = base(isc:iec,      jsc:jec,      :) + l*1e3
+    enddo
+    call mpp_clock_begin(id3)
+    call mpp_start_group_update(group_update, domain, a1(isc,jsc,1,1))
+    call mpp_complete_group_update(group_update, domain, a1(isc,jsc,1,1))
+    call mpp_clock_end  (id3)
 
-    allocate( x (ism:iem,jsm:jem,nz, ntile_per_pe) )
-    x = 0.
-    x(isc:iec,jsc:jec,:,:) = global1(isc:iec,jsc:jec,:,:)
+    !--- compare checksum
+    do l = 1, num_fields
+       write(text, '(i3.3)') l
+       call compare_checksums(a1(isd:ied, jsd:jed, :,l),a2(isd:ied, jsd:jed, :,l), &
+                              type//' nonblock 4D CENTER '//text)
+    enddo
 
-    !--- fill up the value at halo points
-    do n = 1, ntile_per_pe
-       call fill_five_tile_halo(global1(:,:,:,n), global1_all, tile(n), 0, 0 )
-    end do
 
-    ! full update
-    id = mpp_clock_id( type, flags=MPP_CLOCK_SYNC+MPP_CLOCK_DETAILED )
-    call mpp_clock_begin(id)
-    do n = 1, ntile_per_pe
-       call mpp_update_domains( x(:,:,:,n), domain, tile_count = n )
-    end do
-    call mpp_clock_end(id)
 
-   do n = 1, ntile_per_pe
-      write(type2, *)type, " at tile_count = ",n
-      call compare_checksums( x(isd:ied,jsd:jed,:,n), global1(isd:ied,jsd:jed,:,n), trim(type2) )
-   end do
+    !--- test for BGRID.
+    deallocate(a1, x1, y1)
+    deallocate(a2, x2, y2)
+    call mpp_clear_group_update(group_update)
 
-   deallocate(global1_all, global1, x)
+    allocate( a1(ism:iem+shift,jsm:jem+shift, nz, num_fields) )  
+    allocate( x1(ism:iem+shift,jsm:jem+shift, nz, num_fields) )  
+    allocate( y1(ism:iem+shift,jsm:jem+shift, nz, num_fields) )  
+    allocate( a2(ism:iem+shift,jsm:jem+shift, nz, num_fields) )  
+    allocate( x2(ism:iem+shift,jsm:jem+shift, nz, num_fields) )  
+    allocate( y2(ism:iem+shift,jsm:jem+shift, nz, num_fields) )  
 
-    !------------------------------------------------------------------
-    !  vector update : BGRID_NE, one extra point in each direction for Five-Tile
-    !------------------------------------------------------------------
-    !--- setup data
-    allocate(global1_all(nxm+shift,nym+shift,nz, ntiles), global2_all(nxm+shift,nym+shift,nz, ntiles) )  
-    allocate(global1(1-whalo:ni+ehalo+shift,1-shalo:nj+nhalo+shift,nz, ntile_per_pe) )  
-    allocate(global2(1-whalo:ni+ehalo+shift,1-shalo:nj+nhalo+shift,nz, ntile_per_pe) )   
-    do n = 1, ntiles
-       do k = 1, nz
-          do j = 1, nym+shift
-             do i = 1, nxm+shift
-                global1_all(i,j,k,n) = 1.0e3 + n + i*1.0e-3 + j*1.0e-6 + k*1.0e-9
-                global2_all(i,j,k,n) = 2.0e3 + n + i*1.0e-3 + j*1.0e-6 + k*1.0e-9
-             end do
-          end do
-       end do
+    do l =1, num_fields
+       call mpp_create_group_update(group_update, a1(:,:,:,l), domain, position=CORNER)
+       call mpp_create_group_update(group_update, x1(:,:,:,l), y1(:,:,:,l), domain, gridtype=BGRID_NE)
     end do
 
-    !------------------------------------------------------------------------
-    ! --- make sure consisency on the boundary for Five-Tile mosaic
-    ! --- east boundary will take the value of neighbor tile west,
-    ! --- north boundary will take the value of neighbor tile south.
-    !------------------------------------------------------------------------
-    if(type == 'Five-Tile') then
-       global1_all(nxm+1,    1:ny,:,1) = global1_all(1,    1:ny,:,2)  ! east
-       global1_all(nxm+1,ny+1:nym,:,1) = global1_all(1,    1:ny,:,4)  ! east
-       global1_all(1:nxm+1, nym+1,:,1) = global1_all(1:nxm+1, 1,:,1)  ! north
-       global1_all(nx+1,     1:ny,:,2) = global1_all(1,    1:ny,:,3)  ! east
-       global1_all(1:nx+1,   ny+1,:,2) = global1_all(1:nx+1,  1,:,4)  ! north
-       global1_all(nx+1,     1:ny,:,3) = global1_all(1,    1:ny,:,1)  ! east
-       global1_all(1:nx+1,   ny+1,:,3) = global1_all(1:nx+1,  1,:,5)  ! north
-       global1_all(nx+1,     1:ny,:,4) = global1_all(1,    1:ny,:,5)  ! east
-       global1_all(1:nx+1,   ny+1,:,4) = global1_all(1:nx+1,  1,:,2)  ! north
-       global1_all(nx+1,     1:ny,:,5) = global1_all(1,ny+1:nym,:,1)  ! east
-       global1_all(1:nx+1,   ny+1,:,5) = global1_all(1:nx+1,  1,:,3)  ! north
-       global1_all(nx+1,     ny+1,:,2) = global1_all(1,       1,:,5)  ! northeast 
-       global1_all(nx+1,     ny+1,:,3) = global1_all(1,    ny+1,:,1)  ! northeast 
-       global2_all(nxm+1,    1:ny,:,1) = global2_all(1,    1:ny,:,2)  ! east
-       global2_all(nxm+1,ny+1:nym,:,1) = global2_all(1,    1:ny,:,4)  ! east
-       global2_all(1:nxm+1, nym+1,:,1) = global2_all(1:nxm+1, 1,:,1)  ! north
-       global2_all(nx+1,     1:ny,:,2) = global2_all(1,    1:ny,:,3)  ! east
-       global2_all(1:nx+1,   ny+1,:,2) = global2_all(1:nx+1,  1,:,4)  ! north
-       global2_all(nx+1,     1:ny,:,3) = global2_all(1,    1:ny,:,1)  ! east
-       global2_all(1:nx+1,   ny+1,:,3) = global2_all(1:nx+1,  1,:,5)  ! north
-       global2_all(nx+1,     1:ny,:,4) = global2_all(1,    1:ny,:,5)  ! east
-       global2_all(1:nx+1,   ny+1,:,4) = global2_all(1:nx+1,  1,:,2)  ! north
-       global2_all(nx+1,     1:ny,:,5) = global2_all(1,ny+1:nym,:,1)  ! east
-       global2_all(1:nx+1,   ny+1,:,5) = global2_all(1:nx+1,  1,:,3)  ! north
-       global2_all(nx+1,     ny+1,:,2) = global2_all(1,       1,:,5)  ! northeast 
-       global2_all(nx+1,     ny+1,:,3) = global2_all(1,    ny+1,:,1)  ! northeast 
-    end if
+    do n = 1, num_iter
+       a1 = 0; x1 = 0; y1 = 0
+       do l = 1, num_fields
+          a1(isc:iec+shift,jsc:jec+shift,:,l) = base(isc:iec+shift,jsc:jec+shift,:) + l*1e3
+          x1(isc:iec+shift,jsc:jec+shift,:,l) = base(isc:iec+shift,jsc:jec+shift,:) + l*1e3 + 1e6
+          y1(isc:iec+shift,jsc:jec+shift,:,l) = base(isc:iec+shift,jsc:jec+shift,:) + l*1e3 + 2*1e6
+       enddo
+       a2 = a1; x2 = x1; y2 = y1
+       call mpp_clock_begin(id1)
+       call mpp_do_group_update(group_update, domain, a1(isc,jsc,1,1))
+       call mpp_clock_end  (id1)
+
+       call mpp_clock_begin(id2)
+       do l = 1, num_fields
+          call mpp_update_domains( a2(:,:,:,l), domain, position=CORNER, complete=l==num_fields )
+       enddo
+       do l = 1, num_fields
+          call mpp_update_domains( x2(:,:,:,l), y2(:,:,:,l), domain, gridtype=BGRID_NE, complete=l==num_fields )
+       enddo
+       call mpp_clock_end(id2)
+
+       !--- compare checksum
+       do l = 1, num_fields
+          write(text, '(i3.3)') l
+          call compare_checksums(a1(isd:ied+shift,jsd:jed+shift,:,l),a2(isd:ied+shift,jsd:jed+shift,:,l),type//' CORNER '//text)
+          call compare_checksums(x1(isd:ied+shift,jsd:jed+shift,:,l),x2(isd:ied+shift,jsd:jed+shift,:,l),type//' BGRID X'//text)
+          call compare_checksums(y1(isd:ied+shift,jsd:jed+shift,:,l),y2(isd:ied+shift,jsd:jed+shift,:,l),type//' BGRID Y'//text)
+       enddo
 
-    do n = 1, ntile_per_pe
-       global1(1:ni+shift,1:nj+shift,:,n) = global1_all(1:ni+shift,1:nj+shift,:,tile(n))
-       global2(1:ni+shift,1:nj+shift,:,n) = global2_all(1:ni+shift,1:nj+shift,:,tile(n))
-    end do
+       a1 = 0; x1 = 0; y1 = 0
+       do l = 1, num_fields
+          a1(isc:iec+shift,jsc:jec+shift,:,l) = base(isc:iec+shift,jsc:jec+shift,:) + l*1e3
+          x1(isc:iec+shift,jsc:jec+shift,:,l) = base(isc:iec+shift,jsc:jec+shift,:) + l*1e3 + 1e6
+          y1(isc:iec+shift,jsc:jec+shift,:,l) = base(isc:iec+shift,jsc:jec+shift,:) + l*1e3 + 2*1e6
+       enddo
+       call mpp_clock_begin(id3)
+       call mpp_start_group_update(group_update, domain, a1(isc,jsc,1,1))
+       call mpp_complete_group_update(group_update, domain, a1(isc,jsc,1,1))
+       call mpp_clock_end  (id3)
+       !--- compare checksum
+       do l = 1, num_fields
+          write(text, '(i3.3)') l
+          call compare_checksums(a1(isd:ied+shift,jsd:jed+shift,:,l),a2(isd:ied+shift,jsd:jed+shift,:,l), &
+                                 type//' nonblockCORNER '//text)
+          call compare_checksums(x1(isd:ied+shift,jsd:jed+shift,:,l),x2(isd:ied+shift,jsd:jed+shift,:,l), &
+                                 type//' nonblock BGRID X'//text)
+          call compare_checksums(y1(isd:ied+shift,jsd:jed+shift,:,l),y2(isd:ied+shift,jsd:jed+shift,:,l), &
+                                 type//' nonblock BGRID Y'//text)
+       enddo
 
-    allocate( x (ism:iem+shift,jsm:jem+shift,nz,ntile_per_pe) )
-    allocate( y (ism:iem+shift,jsm:jem+shift,nz,ntile_per_pe) )
+    enddo 
 
-    x = 0.; y = 0
-    x (isc:iec+shift,jsc:jec+shift,:,:) = global1(isc:iec+shift,jsc:jec+shift,:,:)
-    y (isc:iec+shift,jsc:jec+shift,:,:) = global2(isc:iec+shift,jsc:jec+shift,:,:)
+    call mpp_clear_group_update(group_update)
 
-    !-----------------------------------------------------------------------
-    !                   fill up the value at halo points.     
-    !-----------------------------------------------------------------------
-    do n = 1, ntile_per_pe
-       call fill_five_tile_halo(global1(:,:,:,n), global1_all, tile(n), shift, shift)
-       call fill_five_tile_halo(global2(:,:,:,n), global2_all, tile(n), shift, shift)
-    end do
 
-    id = mpp_clock_id( type//' BGRID_NE', flags=MPP_CLOCK_SYNC+MPP_CLOCK_DETAILED )
-    call mpp_clock_begin(id)
-    do n = 1, ntile_per_pe
-       call mpp_update_domains( x(:,:,:,n), y(:,:,:,n), domain, gridtype=BGRID_NE, tile_count = n )
-    end do
-    call mpp_clock_end(id)
 
-   do n = 1, ntile_per_pe
-      write(type2, *)type, " at tile_count = ",n
-      call compare_checksums( x(isd:ied+shift,jsd:jed+shift,:,n), global1(isd:ied+shift,jsd:jed+shift,:,n), &
-                              trim(type2)//' BGRID_NE X')
-      call compare_checksums( y(isd:ied+shift,jsd:jed+shift,:,n), global2(isd:ied+shift,jsd:jed+shift,:,n), &
-                              trim(type2)//' BGRID_NE Y')
-   end do
+   
+    deallocate(pe_start, pe_end, tile1, tile2)
+    deallocate(istart1, iend1, jstart1, jend1)
+    deallocate(istart2, iend2, jstart2, jend2)
+    deallocate(layout2D, global_indices)
 
-   deallocate(global1_all, global2_all, global1, global2, x, y)
+    deallocate(a1, x1, y1)
+    deallocate(a2, x2, y2)
+    deallocate(base)
+    call mpp_deallocate_domain(domain)
 
-    !------------------------------------------------------------------
-    !  vector update : CGRID_NE
-    !------------------------------------------------------------------
-    !--- setup data
-    allocate(global1_all(nxm+shift,nym,nz, ntiles), global2_all(nxm,nym+shift,nz, ntiles) )  
-    allocate(global1(1-whalo:ni+ehalo+shift, 1-shalo:nj+nhalo,       nz, ntile_per_pe) )  
-    allocate(global2(1-whalo:ni+ehalo,       1-shalo:nj+nhalo+shift, nz, ntile_per_pe) )   
-    do n = 1, ntiles
-       do k = 1, nz
-          do j = 1, nym
-             do i = 1, nxm+shift
-                global1_all(i,j,k,n) = 1.0e3 + n + i*1.0e-3 + j*1.0e-6 + k*1.0e-9
-             end do
-          end do
-          do j = 1, nym+shift
-             do i = 1, nxm
-                global2_all(i,j,k,n) = 2.0e3 + n + i*1.0e-3 + j*1.0e-6 + k*1.0e-9
-             end do
-          end do
-       end do
-    end do
+end subroutine test_group_update
 
-    !------------------------------------------------------------------------
-    ! --- make sure consisency on the boundary for Five-Tile mosaic
-    ! --- east boundary will take the value of neighbor tile west,
-    ! --- north boundary will take the value of neighbor tile south.
-    !------------------------------------------------------------------------
-    if(type == 'Five-Tile') then
-       global1_all(nxm+1,    1:ny,:,1) = global1_all(1,    1:ny,:,2)  ! east
-       global1_all(nxm+1,ny+1:nym,:,1) = global1_all(1,    1:ny,:,4)  ! east
-       global1_all(nx+1,     1:ny,:,2) = global1_all(1,    1:ny,:,3)  ! east
-       global1_all(nx+1,     1:ny,:,3) = global1_all(1,    1:ny,:,1)  ! east
-       global1_all(nx+1,     1:ny,:,4) = global1_all(1,    1:ny,:,5)  ! east
-       global1_all(nx+1,     1:ny,:,5) = global1_all(1,ny+1:nym,:,1)  ! east
-       global2_all(1:nxm,   nym+1,:,1) = global2_all(1:nxm,   1,:,1)  ! north
-       global2_all(1:nx,     ny+1,:,2) = global2_all(1:nx,    1,:,4)  ! north
-       global2_all(1:nx,     ny+1,:,3) = global2_all(1:nx,    1,:,5)  ! north
-       global2_all(1:nx,     ny+1,:,4) = global2_all(1:nx,    1,:,2)  ! north
-       global2_all(1:nx,     ny+1,:,5) = global2_all(1:nx,    1,:,3)  ! north
+
+  !#################################################################################
+
+  subroutine fill_halo_zero(data, whalo, ehalo, shalo, nhalo, xshift, yshift, isc, iec, jsc, jec, isd, ied, jsd, jed)
+    integer,                         intent(in) :: isc, iec, jsc, jec, isd, ied, jsd, jed
+    integer,                         intent(in) :: whalo, ehalo, shalo, nhalo, xshift, yshift
+    real, dimension(isd:,jsd:,:), intent(inout) :: data
+
+    if(whalo >=0) then
+       data(iec+ehalo+1+xshift:ied+xshift,jsd:jed+yshift,:) = 0
+       data(isd:isc-whalo-1,jsd:jed+yshift,:) = 0
+    else
+       data(iec+1+xshift:iec-ehalo+xshift,jsc+shalo:jec-nhalo+yshift,:) = 0
+       data(isc+whalo:isc-1,jsc+shalo:jec-nhalo+yshift,:) = 0
     end if
 
-    do n = 1, ntile_per_pe
-       global1(1:ni+shift,      1:nj,:,n) = global1_all(1:ni+shift,      1:nj,:,tile(n))
-       global2(1:ni,      1:nj+shift,:,n) = global2_all(1:ni,      1:nj+shift,:,tile(n))
-    end do
+    if(shalo>=0) then
+       data(isd:ied+xshift, jec+nhalo+1+yshift:jed+yshift,:) = 0
+       data(isd:ied+xshift, jsd:jsc-shalo-1,:) = 0
+    else
+       data(isc+whalo:iec-ehalo+xshift,jec+1+yshift:jec-nhalo+yshift,:) = 0
+       data(isc+whalo:iec-ehalo+xshift,jsc+shalo:jsc-1,:) = 0
+    end if
 
-    allocate( x (ism:iem+shift,      jsm:jem,nz,ntile_per_pe) )
-    allocate( y (ism:iem,      jsm:jem+shift,nz,ntile_per_pe) )
+  end subroutine fill_halo_zero
 
-    x = 0.; y = 0
-    x (isc:iec+shift,      jsc:jec,:,:) = global1(isc:iec+shift,      jsc:jec,:,:)
-    y (isc:iec,      jsc:jec+shift,:,:) = global2(isc:iec,      jsc:jec+shift,:,:)
+  !##############################################################################
+  ! this routine fill the halo points for the regular mosaic. 
+  subroutine fill_regular_mosaic_halo(data, data_all, te, tse, ts, tsw, tw, tnw, tn, tne)
+    real, dimension(1-whalo:,1-shalo:,:), intent(inout) :: data
+    real, dimension(:,:,:,:),             intent(in)    :: data_all
+    integer,                              intent(in)    :: te, tse, ts, tsw, tw, tnw, tn, tne
 
-    !-----------------------------------------------------------------------
-    !                   fill up the value at halo points.     
-    !-----------------------------------------------------------------------
-    do n = 1, ntile_per_pe
-       call fill_five_tile_halo(global1(:,:,:,n), global1_all, tile(n), shift, 0)
-       call fill_five_tile_halo(global2(:,:,:,n), global2_all, tile(n), 0, shift)
-    end do
+       data(nx+1:nx+ehalo, 1:ny,          :) = data_all(1:ehalo,       1:ny,          :, te) ! east
+       data(1:nx,          1-shalo:0,     :) = data_all(1:nx,          ny-shalo+1:ny, :, ts) ! south 
+       data(1-whalo:0,     1:ny,          :) = data_all(nx-whalo+1:nx, 1:ny,          :, tw) ! west
+       data(1:nx,          ny+1:ny+nhalo, :) = data_all(1:nx,          1:nhalo,       :, tn) ! north  
+       data(nx+1:nx+ehalo, 1-shalo:0,     :) = data_all(1:ehalo,       ny-shalo+1:ny, :,tse) ! southeast
+       data(1-whalo:0,     1-shalo:0,     :) = data_all(nx-whalo+1:nx, ny-shalo+1:ny, :,tsw) ! southwest
+       data(nx+1:nx+ehalo, ny+1:ny+nhalo, :) = data_all(1:ehalo,       1:nhalo,       :,tnw) ! northeast
+       data(1-whalo:0,     ny+1:ny+nhalo, :) = data_all(nx-whalo+1:nx, 1:nhalo,       :,tne) ! northwest    
 
-    id = mpp_clock_id( type//' CGRID_NE', flags=MPP_CLOCK_SYNC+MPP_CLOCK_DETAILED )
-    call mpp_clock_begin(id)
-    do n = 1, ntile_per_pe
-       call mpp_update_domains( x(:,:,:,n), y(:,:,:,n), domain, gridtype=CGRID_NE, tile_count = n )
-    end do
-    call mpp_clock_end(id)
 
-   do n = 1, ntile_per_pe
-      write(type2, *)type, " at tile_count = ",n
-      call compare_checksums( x(isd:ied+shift,jsd:jed,:,n), global1(isd:ied+shift,jsd:jed,:,n), &
-                              trim(type2)//' CGRID_NE X')
-      call compare_checksums( y(isd:ied,jsd:jed+shift,:,n), global2(isd:ied,jsd:jed+shift,:,n), &
-                              trim(type2)//' CGRID_NE Y')
-   end do
 
-   deallocate(global1_all, global2_all, global1, global2, x, y)
+  end subroutine fill_regular_mosaic_halo
 
-  end subroutine test_nonuniform_mosaic
+  !################################################################################
+  subroutine fill_folded_north_halo(data, ioff, joff, ishift, jshift, sign)
+    real, dimension(1-whalo:,1-shalo:,:), intent(inout) :: data
+    integer,                              intent(in   ) :: ioff, joff, ishift, jshift, sign    
+    integer  :: nxp, nyp, m1, m2
 
-  subroutine fill_five_tile_halo(data, data_all, tile, ioff, joff)
+    nxp = nx+ishift
+    nyp = ny+jshift
+    m1 = ishift - ioff
+    m2 = 2*ishift - ioff
+
+    data(1-whalo:0,                  1:nyp,:) =      data(nx-whalo+1:nx,        1:ny+jshift,:) ! west
+    data(nx+1:nx+ehalo+ishift,       1:nyp,:) =      data(1:ehalo+ishift,       1:ny+jshift,:) ! east
+    if(m1 .GE. 1-whalo) data(1-whalo:m1,  nyp+1:nyp+nhalo,:) = sign*data(whalo+m2:1+ishift:-1, nyp-joff:nyp-nhalo-joff+1:-1,:)
+    data(m1+1:nx+m2,       nyp+1:nyp+nhalo,:) = sign*data(nx+ishift:1:-1,       nyp-joff:nyp-nhalo-joff+1:-1,:)
+    data(nx+m2+1:nxp+ehalo,nyp+1:nyp+nhalo,:) = sign*data(nx:nx-ehalo+m1+1:-1,  nyp-joff:nyp-nhalo-joff+1:-1,:)
+
+  end subroutine fill_folded_north_halo
+
+  !################################################################################
+  subroutine fill_folded_south_halo(data, ioff, joff, ishift, jshift, sign)
     real, dimension(1-whalo:,1-shalo:,:), intent(inout) :: data
-    real, dimension(:,:,:,:),             intent(in)    :: data_all
-    integer,                              intent(in)    :: tile, ioff, joff
-    integer                                             :: nxm, nym
+    integer,                              intent(in   ) :: ioff, joff, ishift, jshift, sign    
+    integer  :: nxp, nyp, m1, m2
 
-    nxm = 2*nx; nym = 2*ny
+    nxp = nx+ishift
+    nyp = ny+jshift
+    m1 = ishift - ioff
+    m2 = 2*ishift - ioff
 
-    select case(tile)
-    case(1)
-       data(nxm+1+ioff:nxm+ehalo+ioff,                     1:ny,:) = data_all(1+ioff:ehalo+ioff,              1:ny,:,2) ! east
-       data(nxm+1+ioff:nxm+ehalo+ioff,            ny+1:nym+joff,:) = data_all(1+ioff:ehalo+ioff,         1:ny+joff,:,4) ! east
-       data(1-whalo:0,                                     1:ny,:) = data_all(nx-whalo+1:nx,                  1:ny,:,3) ! west
-       data(1-whalo:0,                            ny+1:nym+joff,:) = data_all(nx-whalo+1:nx,             1:ny+joff,:,5) ! west
-       data(1:nxm+ioff,                               1-shalo:0,:) = data_all(1:nxm+ioff,          nym-shalo+1:nym,:,1) ! south
-       data(1:nxm+ioff,               nym+1+joff:nym+nhalo+joff,:) = data_all(1:nxm+ioff,        1+joff:nhalo+joff,:,1) ! north
-       data(nxm+1+ioff:nxm+ehalo+ioff,                1-shalo:0,:) = data_all(1+ioff:ehalo+ioff,     ny-shalo+1:ny,:,4) ! southeast
-       data(1-whalo:0,                                1-shalo:0,:) = data_all(nx-whalo+1:nx,         ny-shalo+1:ny,:,5) ! southwest
-       data(nxm+1+ioff:nxm+ehalo+ioff,nym+1+joff:nym+nhalo+joff,:) = data_all(1+ioff:ehalo+ioff, 1+joff:nhalo+joff,:,2) ! northeast
-       data(1-whalo:0,                nym+1+joff:nym+nhalo+joff,:) = data_all(nx-whalo+1:nx,     1+joff:nhalo+joff,:,3) ! northwest
-    case(2)
-       data(nx+1+ioff:nx+ehalo+ioff,              1:ny+joff,:) = data_all(1+ioff:ehalo+ioff,              1:ny+joff,:,3) ! east
-       data(1-whalo:0,                            1:ny+joff,:) = data_all(nxm-whalo+1:nxm,                1:ny+joff,:,1) ! west
-       data(1:nx+ioff,                            1-shalo:0,:) = data_all(1:nx+ioff,                  ny-shalo+1:ny,:,4) ! south 
-       data(1:nx+ioff,              ny+1+joff:ny+nhalo+joff,:) = data_all(1:nx+ioff,              1+joff:nhalo+joff,:,4) ! north
-       data(nx+1+ioff:nx+ehalo+ioff,              1-shalo:0,:) = data_all(1+ioff:ehalo+ioff,          ny-shalo+1:ny,:,5) ! southeast
-       data(1-whalo:0,                            1-shalo:0,:) = data_all(nxm-whalo+1:nxm,          nym-shalo+1:nym,:,1) ! southwest
-       data(nx+1+ioff:nx+ehalo+ioff,ny+1+joff:ny+nhalo+joff,:) = data_all(1+ioff:ehalo+ioff,      1+joff:nhalo+joff,:,5) ! northeast
-       data(1-whalo:0,              ny+1+joff:ny+nhalo+joff,:) = data_all(nxm-whalo+1:nxm,  ny+1+joff:ny+nhalo+joff,:,1) ! northwest
-    case(3)
-       data(nx+1+ioff:nx+ehalo+ioff,              1:ny+joff,:) = data_all(1+ioff:ehalo+ioff,              1:ny+joff,:,1) ! east
-       data(1-whalo:0,                            1:ny+joff,:) = data_all(nx-whalo+1:nx,                  1:ny+joff,:,2) ! west
-       data(1:nx+ioff,                            1-shalo:0,:) = data_all(1:nx+ioff,                  ny-shalo+1:ny,:,5) ! south 
-       data(1:nx+ioff,              ny+1+joff:ny+nhalo+joff,:) = data_all(1:nx+ioff,              1+joff:nhalo+joff,:,5) ! north
-       data(nx+1+ioff:nx+ehalo+ioff,              1-shalo:0,:) = data_all(1+ioff:ehalo+ioff,        nym-shalo+1:nym,:,1) ! southeast
-       data(1-whalo:0,                            1-shalo:0,:) = data_all(nx-whalo+1:nx,              ny-shalo+1:ny,:,4) ! southwest
-       data(nx+1+ioff:nx+ehalo+ioff,ny+1+joff:ny+nhalo+joff,:) = data_all(1+ioff:ehalo+ioff,ny+1+joff:ny+nhalo+joff,:,1) ! northeast
-       data(1-whalo:0,              ny+1+joff:ny+nhalo+joff,:) = data_all(nx-whalo+1:nx,          1+joff:nhalo+joff,:,4) ! northwest
-    case(4)
-       data(nx+1+ioff:nx+ehalo+ioff,              1:ny+joff,:) = data_all(1+ioff:ehalo+ioff,        1:ny+joff,:,5) ! east
-       data(1-whalo:0,                            1:ny+joff,:) = data_all(nxm-whalo+1:nxm,     ny+1:2*ny+joff,:,1) ! west
-       data(1:nx+ioff,                            1-shalo:0,:) = data_all(1:nx+ioff,            ny-shalo+1:ny,:,2) ! south 
-       data(1:nx+ioff,              ny+1+joff:ny+nhalo+joff,:) = data_all(1:nx+ioff,        1+joff:nhalo+joff,:,2) ! north
-       data(nx+1+ioff:nx+ehalo+ioff,              1-shalo:0,:) = data_all(1+ioff:ehalo+ioff,    ny-shalo+1:ny,:,3) ! southeast
-       data(1-whalo:0,                            1-shalo:0,:) = data_all(nxm-whalo+1:nxm,      ny-shalo+1:ny,:,1) ! southwest
-       data(nx+1+ioff:nx+ehalo+ioff,ny+1+joff:ny+nhalo+joff,:) = data_all(1+ioff:ehalo+ioff,1+joff:nhalo+joff,:,3) ! northeast
-       data(1-whalo:0,              ny+1+joff:ny+nhalo+joff,:) = data_all(nxm-whalo+1:nxm,  1+joff:nhalo+joff,:,1) ! northwest
-    case(5)
-       data(nx+1+ioff:nx+ehalo+ioff,            1:  ny+joff,:) = data_all(1+ioff:ehalo+ioff,   ny+1:2*ny+joff,:,1) ! east
-       data(1-whalo:0,                            1:ny+joff,:) = data_all(nx-whalo+1:nx,            1:ny+joff,:,4) ! west
-       data(1:nx+ioff,                            1-shalo:0,:) = data_all(1:nx+ioff,            ny-shalo+1:ny,:,3) ! south 
-       data(1:nx+ioff,              ny+1+joff:ny+nhalo+joff,:) = data_all(1:nx+ioff,        1+joff:nhalo+joff,:,3) ! north
-       data(nx+1+ioff:nx+ehalo+ioff,              1-shalo:0,:) = data_all(1+ioff:ehalo+ioff,    ny-shalo+1:ny,:,1) ! southeast
-       data(1-whalo:0,                            1-shalo:0,:) = data_all(nx-whalo+1:nx,        ny-shalo+1:ny,:,2) ! southwest
-       data(nx+1+ioff:nx+ehalo+ioff,ny+1+joff:ny+nhalo+joff,:) = data_all(1+ioff:ehalo+ioff,1+joff:nhalo+joff,:,1) ! northeast
-       data(1-whalo:0,              ny+1+joff:ny+nhalo+joff,:) = data_all(nx-whalo+1:nx,    1+joff:nhalo+joff,:,2) ! northwest
-    end select
 
-  end subroutine fill_five_tile_halo
+    data(1-whalo:0,                  1:nyp,:) =      data(nx-whalo+1:nx,        1:nyp,:) ! west
+    data(nx+1:nx+ehalo+ishift,       1:nyp,:) =      data(1:ehalo+ishift,       1:nyp,:) ! east
+    if(m1 .GE. 1-whalo)data(1-whalo:m1, 1-shalo:0,:) = sign*data(whalo+m2:1+ishift:-1, shalo+jshift:1+jshift:-1,:)
+    data(m1+1:nx+m2,       1-shalo:0,:) = sign*data(nxp:1:-1,             shalo+jshift:1+jshift:-1,:)
+    data(nx+m2+1:nxp+ehalo,1-shalo:0,:) = sign*data(nx:nx-ehalo+m1+1:-1,  shalo+jshift:1+jshift:-1,:)
+
+  end subroutine fill_folded_south_halo
+
+  !################################################################################
+  subroutine fill_folded_west_halo(data, ioff, joff, ishift, jshift, sign)
+    real, dimension(1-whalo:,1-shalo:,:), intent(inout) :: data
+    integer,                              intent(in   ) :: ioff, joff, ishift, jshift, sign    
+    integer  :: nxp, nyp, m1, m2
+
+    nxp = nx+ishift
+    nyp = ny+jshift
+    m1 = jshift - joff
+    m2 = 2*jshift - joff
+
+    data(1:nxp, 1-shalo:0, :)      = data(1:nxp, ny-shalo+1:ny, :) ! south
+    data(1:nxp, ny+1:nyp+nhalo, :) = data(1:nxp, 1:nhalo+jshift,:) ! north
+    if(m1 .GE. 1-shalo) data(1-whalo:0, 1-shalo:m1, :) = sign*data(whalo+ishift:1+ishift:-1, shalo+m2:1+jshift:-1,:)
+    data(1-whalo:0, m1+1:ny+m2, :) = sign*data(whalo+ishift:1+ishift:-1, nyp:1:-1, :)
+    data(1-whalo:0, ny+m2+1:nyp+nhalo,:) = sign*data(whalo+ishift:1+ishift:-1, ny:ny-nhalo+m1+1:-1,:)
+
+  end subroutine fill_folded_west_halo
+
+  !################################################################################
+  subroutine fill_folded_east_halo(data, ioff, joff, ishift, jshift, sign)
+    real, dimension(1-whalo:,1-shalo:,:), intent(inout) :: data
+    integer,                              intent(in   ) :: ioff, joff, ishift, jshift, sign    
+    integer  :: nxp, nyp, m1, m2
+
+    nxp = nx+ishift
+    nyp = ny+jshift
+    m1 = jshift - joff
+    m2 = 2*jshift - joff
+
+    data(1:nxp, 1-shalo:0, :)      = data(1:nxp, ny-shalo+1:ny, :) ! south
+    data(1:nxp, ny+1:nyp+nhalo, :) = data(1:nxp, 1:nhalo+jshift,:) ! north
+    if(m1 .GE. 1-shalo) data(nxp+1:nxp+ehalo, 1-shalo:m1, :) = sign*data(nxp-ioff:nxp-ehalo-ioff+1:-1, shalo+m2:1+jshift:-1,:)
+    data(nxp+1:nxp+ehalo, m1+1:ny+m2, :) = sign*data(nxp-ioff:nxp-ehalo-ioff+1:-1, nyp:1:-1, :)
+    data(nxp+1:nxp+ehalo, ny+m2+1:nyp+nhalo,:) = sign*data(nxp-ioff:nxp-ehalo-ioff+1:-1, ny:ny-nhalo+m1+1:-1,:)
+
+  end subroutine fill_folded_east_halo
+
+  !################################################################################
+  subroutine fill_four_tile_bound(data_all, is, ie, js, je, ioff, joff, tile, &
+                                   ebound, sbound, wbound, nbound )
+    real, dimension(:,:,:,:),       intent(in)    :: data_all
+    integer,                        intent(in)    :: is, ie, js, je
+    integer,                        intent(in)    :: tile, ioff, joff
+    real, dimension(:,:), optional, intent(inout) :: ebound, sbound, wbound, nbound
+    integer                                       :: tw, te, ts, tn
+
+    if(tile == 1 .OR. tile == 3) te = tile + 1
+    if(tile == 2 .OR. tile == 4) te = tile - 1
+    if(tile == 1 .OR. tile == 2) ts = tile + 2
+    if(tile == 3 .OR. tile == 4) ts = tile - 2
+    tw = te;   tn = ts
+    if(present(ebound)) then
+       if( ie == nx ) then
+          ebound(:,:) = data_all(1, js:je+joff, :, te)
+       else
+          ebound(:,:) = data_all(ie+ioff, js:je+joff, :, tile)
+       end if
+    end if
+
+    if(present(wbound)) then
+       if( is == 1 ) then
+          wbound(:,:) = data_all(nx+ioff, js:je+joff, :, tw)
+       else
+          wbound(:,:) = data_all(is, js:je+joff, :, tile)
+       end if
+    end if
+
+    if(present(sbound)) then
+       if( js == 1 ) then
+          sbound(:,:) = data_all(is:ie+ioff, ny+joff, :, ts)
+       else
+          sbound(:,:) = data_all(is:ie+ioff, js, :, tile)
+       end if
+    end if
+
+    if(present(nbound)) then
+       if( je == ny ) then
+          nbound(:,:) = data_all(is:ie+ioff, 1, :, tn)
+       else
+          nbound(:,:) = data_all(is:ie+ioff, je+joff, :, tile)
+       end if
+    end if
+
+    return
+
+  end subroutine fill_four_tile_bound
+
+
+  !################################################################################
+  subroutine fill_torus_bound(data_all, is, ie, js, je, ioff, joff, tile, &
+                                     sbound, wbound)
+    real, dimension(:,:,:),       intent(in)    :: data_all
+    integer,                        intent(in)    :: is, ie, js, je
+    integer,                        intent(in)    :: tile, ioff, joff
+    real, dimension(:,:), optional, intent(inout) :: sbound, wbound
+    integer                                       :: tw, te, ts, tn
+    integer                                       :: js1, js2, is1, is2
+
+    if(tile .NE. 1) call mpp_error(FATAL, "fill_torus_bound: tile must be 1")
+
+    js2 = js
+    js1 = 1
+    if( js == 1 .AND. joff==1 )  then
+      js1 = 2
+      js2 = js+1
+    endif
+    is2 = is
+    is1 = 1
+    if( is == 1 .AND. ioff==1 )  then
+      is1 = 2
+      is2 = is+1
+    endif    
+
+    if(present(wbound)) then
+       if(ioff .NE. 1) call mpp_error(FATAL, "fill_torus_bound: ioff must be 1 when wbound present")
+       if( is == 1 ) then
+          wbound(js1:,:) = data_all(nx+ioff, js2:je+joff, :)
+       else
+          wbound(js1:,:) = data_all(is, js2:je+joff, :)
+       end if
+       if(js1 == 2) then
+          if( is == 1 ) then
+             wbound(1,:) = data_all(nx+1, ny+1, :) 
+          else
+             wbound(1,:) = data_all(is, ny+1, :) 
+          endif
+       endif
+    end if
+
+    if(present(sbound)) then
+       if(joff .NE. 1) call mpp_error(FATAL, "fill_torus_bound: joff must be 1 when sbound present")
+       if( js == 1 ) then
+          sbound(is1:,:) = data_all(is2:ie+ioff, ny+joff, :)
+       else
+          sbound(is1:,:) = data_all(is2:ie+ioff, js, :)
+       end if
+       if(is1 == 2) then
+          if( js == 1 ) then
+             sbound(1,:) = data_all(nx+1, ny+1, :) 
+          else
+             sbound(1,:) = data_all(nx+1, js, :) 
+          endif
+       endif
+    end if
+
+    return
+
+  end subroutine fill_torus_bound
+
+  !################################################################################
+  subroutine fill_folded_north_bound(data_all, is, ie, js, je, ioff, joff, tile, &
+                                     sbound, wbound)
+    real, dimension(:,:,:),       intent(in)    :: data_all
+    integer,                        intent(in)    :: is, ie, js, je
+    integer,                        intent(in)    :: tile, ioff, joff
+    real, dimension(:,:), optional, intent(inout) :: sbound, wbound
+    integer                                       :: tw, te, ts, tn
+    integer                                       :: js1, js2
+
+    if(tile .NE. 1) call mpp_error(FATAL, "fill_folded_north_bound: tile must be 1")
+
+    js2 = js
+    js1 = 1
+    if( js == 1 .AND. joff==1 )  then
+      js1 = 2
+      js2 = js+1
+    endif
+
+    if(present(wbound)) then
+       if( is == 1 ) then
+          wbound(js1:,:) = data_all(nx+ioff, js2:je+joff, :)
+       else
+          wbound(js1:,:) = data_all(is, js2:je+joff, :)
+       end if
+    end if
+
+    if(present(sbound)) then
+       if( js == 1 ) then
+          sbound(:,:) = 0
+       else
+          if( is == 1 .AND. ioff == 1 ) then
+             sbound(1,:)  = data_all(nx+1, js, :)
+             sbound(2:,:) = data_all(is+1:ie+ioff, js, :)
+          else
+             sbound(:,:) = data_all(is:ie+ioff, js, :)          
+          endif
+       end if
+    end if
+
+    return
+
+  end subroutine fill_folded_north_bound
+
+  !################################################################################
+  subroutine fill_cubic_grid_bound(data1_all, data2_all, is, ie, js, je, ioff, joff, tile, sign1, sign2, &
+                                   ebound, sbound, wbound, nbound )
+    real, dimension(:,:,:,:),       intent(in)    :: data1_all, data2_all
+    integer,                        intent(in)    :: is, ie, js, je
+    integer,                        intent(in)    :: tile, ioff, joff, sign1, sign2
+    real, dimension(:,:), optional, intent(inout) :: ebound, sbound, wbound, nbound
+    integer                                       :: tw, te, ts, tn
+
+    if(mod(tile,2) == 0) then ! tile 2, 4, 6
+       tw = tile - 1; te = tile + 2; ts = tile - 2; tn = tile + 1
+       if(te > 6 ) te = te - 6
+       if(ts < 1 ) ts = ts + 6
+       if(tn > 6 ) tn = tn - 6
+       !--- East bound
+       if(present(ebound)) then
+          if(ie == nx) then                
+             ebound(:,:) = sign1*data2_all(nx+joff-js+1:nx-je+1:-1,1,:,te)
+          else
+             ebound(:,:) = data1_all(ie+ioff, js:je+joff, :,tile)
+          end if
+       end if
+       !--- South bound
+       if(present(sbound)) then
+          if(js == 1) then                
+             sbound(:,:) = sign2*data2_all(nx+joff, ny+ioff-is+1:ny-ie+1:-1,:,ts)
+          else
+             sbound(:,:) = data1_all(is:ie+ioff, js, :,tile)
+          end if
+       end if
+
+       !--- West bound
+       if(present(wbound)) then
+          if(is == 1) then
+             wbound(:,:) = data1_all(nx+ioff, js:je+joff,:,tw)
+          else
+             wbound(:,:) = data1_all(is, js:je+joff,:,tile)
+          end if
+       end if
+
+       !--- north bound
+       if(present(nbound)) then
+          if(je == ny) then                
+             nbound(:,:) = data1_all(is:ie+ioff, 1,:,tn)
+          else
+             nbound(:,:) = data1_all(is:ie+ioff, je+joff, :,tile)
+          end if
+       end if
+    else ! tile 1, 3, 5
+       tw = tile - 2; te = tile + 1; ts = tile - 1; tn = tile + 2
+       if(tw < 1 ) tw = tw + 6
+       if(ts < 1 ) ts = ts + 6
+       if(tn > 6 ) tn = tn - 6
+       !--- East bound
+       if(present(ebound)) then
+          if(ie == nx) then                
+             ebound(:,:) = data1_all(1, js:je+joff, :,te) 
+          else
+             ebound(:,:) = data1_all(ie+ioff, js:je+joff, :,tile)
+          end if
+       end if
+       !--- South bound
+       if(present(sbound)) then
+          if(js == 1) then                
+             sbound(:,:) = data1_all(is:ie+ioff,ny+joff,:,ts)
+          else
+             sbound(:,:) = data1_all(is:ie+ioff, js, :,tile)
+          end if
+       end if
+
+       !--- West bound
+       if(present(wbound)) then
+          if(is == 1) then
+             wbound(:,:) = sign1*data2_all(nx+joff-js+1:nx-je+1:-1,ny+ioff,:,tw)
+          else
+             wbound(:,:) = data1_all(is, js:je+joff,:,tile)
+          end if
+       end if
+
+       !--- north bound
+       if(present(nbound)) then
+          if(je == ny) then                
+             nbound(:,:) = sign2*data2_all(1, ny+ioff-is+1:ny-ie+1:-1,:,tn)
+          else
+             nbound(:,:) = data1_all(is:ie+ioff, je+joff, :,tile)
+          end if
+       end if
+
+    end if
+
+  end subroutine fill_cubic_grid_bound
 
   !##############################################################################
-  !--- The following is used to test the refined mosaic. Three cases will be tested, 
-  !--- non-symmetric regular mosaic, symmetric regular mosaic cubic grid. The regular mosaic 
-  !--- contains 4 tiles. East of tile 1 connected with West of tile 2 (refine = 3)
-  !--- and vice verse; East of tile 3 connected with West of tile 4 (refine = 2)
-  !--- and vice verse; North of tile 1 connected with South of tile 3 (refine = 2)
-  !--- and vice verse; North of tile 2 connected with South of tile 4 (refine = 1)
-  !--- and vice verse; So there ar total 8 contacts. 
-  subroutine test_refined_mosaic(type)
+  ! this routine fill the halo points for the cubic grid. ioff and joff is used to distinguish
+  ! T, C, E, or N-cell
+  subroutine fill_cubic_grid_halo(data, data1_all, data2_all, tile, ioff, joff, sign1, sign2)
+    real, dimension(1-whalo:,1-shalo:,:), intent(inout) :: data
+    real, dimension(:,:,:,:),             intent(in)    :: data1_all, data2_all
+    integer,                              intent(in)    :: tile, ioff, joff, sign1, sign2 
+    integer                                             :: lw, le, ls, ln
+
+    if(mod(tile,2) == 0) then ! tile 2, 4, 6
+       lw = tile - 1; le = tile + 2; ls = tile - 2; ln = tile + 1
+       if(le > 6 ) le = le - 6
+       if(ls < 1 ) ls = ls + 6
+       if(ln > 6 ) ln = ln - 6
+       data(1-whalo:0, 1:ny+joff, :) = data1_all(nx-whalo+1:nx, 1:ny+joff, :, lw) ! west 
+       do i = 1, ehalo 
+          data(nx+i+ioff, 1:ny+joff, :)    = sign1*data2_all(nx+joff:1:-1, i+ioff, :, le) ! east 
+       end do
+       do i = 1, shalo 
+          data(1:nx+ioff, 1-i, :)     = sign2*data2_all(nx-i+1, ny+ioff:1:-1, :, ls) ! south 
+       end do
+       data(1:nx+ioff, ny+1+joff:ny+nhalo+joff, :) = data1_all(1:nx+ioff, 1+joff:nhalo+joff, :, ln) ! north
+    else ! tile 1, 3, 5
+       lw = tile - 2; le = tile + 1; ls = tile - 1; ln = tile + 2
+       if(lw < 1 ) lw = lw + 6
+       if(ls < 1 ) ls = ls + 6
+       if(ln > 6 ) ln = ln - 6
+       do i = 1, whalo 
+          data(1-i, 1:ny+joff, :)     = sign1*data2_all(nx+joff:1:-1, ny-i+1, :, lw) ! west 
+       end do
+       data(nx+1+ioff:nx+ehalo+ioff, 1:ny+joff, :) = data1_all(1+ioff:ehalo+ioff, 1:ny+joff, :, le) ! east 
+       data(1:nx+ioff, 1-shalo:0, :)     = data1_all(1:nx+ioff, ny-shalo+1:ny, :, ls) ! south 
+       do i = 1, nhalo 
+          data(1:nx+ioff, ny+i+joff, :)    = sign2*data2_all(i+joff, ny+ioff:1:-1, :, ln) ! north 
+       end do
+    end if
+
+  end subroutine fill_cubic_grid_halo
+    
+   !#####################################################################
+  subroutine test_nonuniform_mosaic( type )
     character(len=*), intent(in) :: type
 
-    type(domain2D)                           :: domain
-    integer,              dimension(4)       :: isMe1, ieMe1, jsMe1, jeMe1
-    integer,              dimension(4)       :: isNb1, ieNb1, jsNb1, jeNb1
-    integer,              dimension(4)       :: isMe2, ieMe2, jsMe2, jeMe2
-    integer,              dimension(4)       :: isNb2, ieNb2, jsNb2, jeNb2
-    integer,              dimension(4)       :: rotation1, rotation2, dirMe1, dirMe2
-    integer, allocatable, dimension(:,:)     :: from_tile1, from_tile2
-    integer                                  :: ntiles, num_contact, npes_on_tile
-    integer                                  :: totpoints, maxtotal, pos, ntiles_on_pe
-    integer                                  :: tNb, nimax, njmax, avgpoints
-    integer                                  :: n, m, l, te, tse, ts, tsw, tw, tnw, tn, tne, nn
-    integer                                  :: noverlap1, noverlap2, total1, total2
-    integer                                  :: isc, iec, jsc, jec, isg, ieg, jsg, jeg
-    integer                                  :: ism, iem, jsm, jem, isd, ied, jsd, jed
-    integer, allocatable, dimension(:)       :: tiles, ni, nj
+    type(domain2D)               :: domain
+    integer                      :: num_contact, ntiles, ntile_per_pe
+    integer                      :: i, j, k, n, nxm, nym, ni, nj, shift
+    integer                      :: ism, iem, jsm, jem, isc, iec, jsc, jec
+    integer                      :: isd, ied, jsd, jed
+    integer                      :: indices(4), msize(2)
+    character(len=128)           :: type2
+
+    integer, allocatable, dimension(:)       :: tile
     integer, allocatable, dimension(:)       :: pe_start, pe_end, tile1, tile2
     integer, allocatable, dimension(:)       :: istart1, iend1, jstart1, jend1
     integer, allocatable, dimension(:)       :: istart2, iend2, jstart2, jend2
-    integer, allocatable, dimension(:,:)     :: layout, global_indices
-    real,    allocatable, dimension(:,:,:,:) :: global_all, global1, global2
-    real,    allocatable, dimension(:,:,:,:) :: x, y, x1, y1, x2, y2
+    integer, allocatable, dimension(:,:)     :: layout2D, global_indices
     real,    allocatable, dimension(:,:,:,:) :: global1_all, global2_all
-    real,    allocatable, dimension(:,:)     :: buffer, buffer1, buffer2, bufferx, buffery
-    real,    allocatable, dimension(:,:)     :: bufferx1, buffery1, bufferx2, buffery2
-    integer                                  :: shift
-    character(len=128)                       :: type2
-    logical                                  :: found
+    real,    allocatable, dimension(:,:,:,:) :: global1, global2, x, y  
 
-    !--- check the type
+    shift = 0
     select case(type)
-    case ("Refined-Four-Tile", "Refined-Symmetric-Four-Tile" )
-       ntiles = 4
-       allocate(ni(ntiles), nj(ntiles))
-       ! "Four-Tile" test case will only run on one pe or multiple 0f 8 ( balanced).
-       if( npes .NE. 1 .AND. npes .NE. 8 .AND. npes .NE. 16 .AND. npes .NE. 32) then
-          call mpp_error(NOTE,'TEST_MPP_DOMAINS(test_refined_mosaic: ' // &
-                  type// ' mosaic will not be tested because npes is not 1, 8, 16 or 32')
-          return
-       end if            
-       ni(1) =   nx; nj(1) =   ny
-       ni(2) =   nx; nj(2) = 3*ny
-       ni(3) = 2*nx; nj(3) =   ny
-       ni(4) =   nx; nj(4) = 2*ny
-       num_contact = 8   
-    case ("Refined-Cubic-Grid")
-       ntiles = 6; num_contact = 12
-       allocate(ni(ntiles), nj(ntiles))
-       ! "Cubic-Grid" will be tested only when nx = ny
-       if( nx /= ny ) then
-           call mpp_error(NOTE,'TEST_MPP_DOMAINS(test_refined_mosaic: ' // &
-           type//' will not be tested because nx is not equal to ny' )
-           return
-       end if
-       ! "Cubic-Grid" test case will only run on one pe or multiple 0f 16 pes ( balanced).
-       if( npes .NE. 1 .AND. mod(npes,16) .NE. 0) then
-          call mpp_error(NOTE,'TEST_MPP_DOMAINS(test_refined_mosaic: ' // &
-                  type//' will not be tested because npes is not 1 and can not be divided by 16')
+    case('Five-Tile') ! one tile will run on pe 0 and other four tiles will run on pe 1
+       shift = 1      ! one extra point for symmetry domain
+       ntiles = 5     ! tile 1 with resolution 2*nx and 2*ny and the tiles are nx and ny.
+       num_contact = 11
+       if(npes .NE. 2) then
+          call mpp_error(NOTE,'TEST_MPP_DOMAINS: Five-Tile mosaic will not be tested because npes is not 2')
           return
+       end if 
+       nxm = 2*nx; nym = 2*ny
+       layout = 1
+       if( pe == 0) then
+          ntile_per_pe = 1
+          allocate(tile(ntile_per_pe))
+          tile = 1
+          indices = (/1,2*nx,1,2*ny/)
+          ni = 2*nx; nj = 2*ny
+       else
+          ntile_per_pe = 4
+          allocate(tile(ntile_per_pe))
+          do n = 1, ntile_per_pe
+             tile(n) = n + 1
+          end do
+          indices = (/1,nx,1,ny/)
+          ni = nx; nj = ny
        end if
-       ni(1) =   nx; nj(1) =   ny
-       ni(2) = 2*nx; nj(2) = 3*ny
-       ni(3) = 2*nx; nj(3) =   ny
-       ni(4) =   nx; nj(4) = 3*ny
-       ni(5) = 2*nx; nj(5) =   ny
-       ni(6) =   nx; nj(6) = 2*ny
+       allocate(pe_start(ntiles), pe_end(ntiles) )
+       pe_start(1) = 0; pe_start(2:) = 1
+       pe_end = pe_start
     case default
-       call mpp_error(FATAL, 'TEST_MPP_DOMAINS(test_refined_mosaic): no such test: '//type)
+       call mpp_error(FATAL, 'TEST_MPP_DOMAINS: no such test: '//type)
     end select
 
-    allocate(layout(2,ntiles), global_indices(4,ntiles), pe_start(ntiles), pe_end(ntiles) )
-    totpoints = sum(ni*nj)
-    if(mod(totpoints, npes) .NE. 0) call mpp_error(FATAL,    &
-        "TEST_MPP_DOMAINS(test_refined_mosaic): totpoints can not be divided by npes")
-    avgpoints = totpoints/npes
-    layout = 1
-    pe_start = 0; pe_end = 0; pos = 0
+    allocate(layout2D(2,ntiles), global_indices(4,ntiles) )
 
     do n = 1, ntiles
-       global_indices(:,n) = (/1, ni(n), 1, nj(n)/)
-       if(npes > 1) then  ! no sharing processor between tiles
-          if( mod(ni(n)*nj(n), avgpoints) .NE. 0) call mpp_error(FATAL, &
-               'TEST_MPP_DOMAINS(test_refined_mosaic): number of points should be divided by average of points in each pe')
-          npes_on_tile = ni(n)*nj(n)/avgpoints
-          call mpp_define_layout( (/1,ni(n),1,nj(n)/), npes_on_tile, layout(:,n) )
-          pe_start(n) = pos
-          pe_end(n)   = pos + npes_on_tile - 1       
-          pos         = pos + npes_on_tile
-       end if
+       if(n==1) then
+          global_indices(:,n) = (/1,2*nx,1,2*ny/)
+       else
+          global_indices(:,n) = (/1,nx,1,ny/)
+       endif  
+!       global_indices(:,n) = indices
+       layout2D(:,n)       = layout
     end do
 
-    ntiles_on_pe = 1
-    if(npes == 1) then
-       ntiles_on_pe = ntiles
-       allocate(tiles(ntiles_on_pe))
-       tiles = (/ (i, i=1,ntiles) /)
-    else
-       ntiles_on_pe = 1
-       allocate(tiles(ntiles_on_pe))
-       do n = 1, ntiles
-          if( pe .GE. pe_start(n) .AND. pe .LE. pe_end(n) ) tiles = n
-       end do
-    end if
-
     allocate(tile1(num_contact), tile2(num_contact) )
     allocate(istart1(num_contact), iend1(num_contact), jstart1(num_contact), jend1(num_contact) ) 
     allocate(istart2(num_contact), iend2(num_contact), jstart2(num_contact), jend2(num_contact) ) 
 
     !--- define domain
     select case(type)
-    case( 'Refined-Four-Tile', 'Refined-Symmetric-Four-Tile' )
-       call define_fourtile_mosaic(type, domain, ni, nj, global_indices, layout, pe_start, pe_end, &
-                                   type == 'Refined-Symmetric-Four-Tile'   )
-    case( 'Refined-Cubic-Grid' )
-       call define_cubic_mosaic(type, domain, ni, nj, global_indices, layout, pe_start, pe_end )
+    case( 'Five-Tile' )
+       !--- Contact line 1, between tile 1 (EAST) and tile 2 (WEST)
+       tile1(1) = 1; tile2(1) = 2
+       istart1(1) = 2*nx; iend1(1) = 2*nx; jstart1(1) = 1;  jend1(1) = ny
+       istart2(1) = 1;    iend2(1) = 1;    jstart2(1) = 1;  jend2(1) = ny
+       !--- Contact line 2, between tile 1 (EAST) and tile 4 (WEST)
+       tile1(2) = 1; tile2(2) = 4
+       istart1(2) = 2*nx; iend1(2) = 2*nx; jstart1(2) = ny+1; jend1(2) = 2*ny
+       istart2(2) = 1;    iend2(2) = 1;    jstart2(2) = 1;    jend2(2) = ny
+       !--- Contact line 3, between tile 1 (SOUTH) and tile 1 (NORTH)
+       tile1(3) = 1; tile2(3) = 1
+       istart1(3) = 1; iend1(3) = 2*nx; jstart1(3) = 1;    jend1(3) = 1
+       istart2(3) = 1; iend2(3) = 2*nx; jstart2(3) = 2*ny; jend2(3) = 2*ny
+       !--- Contact line 4, between tile 1 (WEST) and tile 3 (EAST)
+       tile1(4) = 1; tile2(4) = 3
+       istart1(4) = 1;  iend1(4) = 1;  jstart1(4) = 1;  jend1(4) = ny
+       istart2(4) = nx; iend2(4) = nx; jstart2(4) = 1;  jend2(4) = ny
+       !--- Contact line 5, between tile 1 (WEST) and tile 5 (EAST)
+       tile1(5) = 1; tile2(5) = 5
+       istart1(5) = 1;  iend1(5) = 1;  jstart1(5) = ny+1;  jend1(5) = 2*ny
+       istart2(5) = nx; iend2(5) = nx; jstart2(5) = 1;     jend2(5) = ny
+       !--- Contact line 6, between tile 2 (EAST) and tile 3 (WEST)
+       tile1(6) = 2; tile2(6) = 3
+       istart1(6) = nx; iend1(6) = nx; jstart1(6) = 1;  jend1(6) = ny
+       istart2(6) = 1;  iend2(6) = 1;  jstart2(6) = 1;  jend2(6) = ny       
+       !--- Contact line 7, between tile 2 (SOUTH) and tile 4 (NORTH)  --- cyclic
+       tile1(7) = 2; tile2(7) = 4
+       istart1(7) = 1;  iend1(7) = nx; jstart1(7) = 1;   jend1(7) = 1
+       istart2(7) = 1;  iend2(7) = nx; jstart2(7) = ny;  jend2(7) = ny
+       !--- Contact line 8, between tile 2 (NORTH) and tile 4 (SOUTH) 
+       tile1(8) = 2; tile2(8) = 4
+       istart1(8) = 1;  iend1(8) = nx; jstart1(8) = ny;  jend1(8) = ny
+       istart2(8) = 1;  iend2(8) = nx; jstart2(8) = 1;   jend2(8) = 1
+       !--- Contact line 9, between tile 3 (SOUTH) and tile 5 (NORTH)  --- cyclic
+       tile1(9) = 3; tile2(9) = 5
+       istart1(9) = 1;  iend1(9) = nx; jstart1(9) = 1;   jend1(9) = 1
+       istart2(9) = 1;  iend2(9) = nx; jstart2(9) = ny;  jend2(9) = ny
+       !--- Contact line 10, between tile 3 (NORTH) and tile 5 (SOUTH) 
+       tile1(10) = 3; tile2(10) = 5
+       istart1(10) = 1;  iend1(10) = nx; jstart1(10) = ny;  jend1(10) = ny
+       istart2(10) = 1;  iend2(10) = nx; jstart2(10) = 1;   jend2(10) = 1
+       !--- Contact line 11, between tile 4 (EAST) and tile 5 (WEST)
+       tile1(11) = 4; tile2(11) = 5
+       istart1(11) = nx; iend1(11) = nx; jstart1(11) = 1;  jend1(11) = ny
+       istart2(11) = 1;  iend2(11) = 1;  jstart2(11) = 1;  jend2(11) = ny  
+       msize(1) = 2*nx + whalo + ehalo
+       msize(2) = 2*ny + shalo + nhalo
+       call mpp_define_mosaic(global_indices, layout2D, domain, ntiles, num_contact, tile1, tile2, &
+            istart1, iend1, jstart1, jend1, istart2, iend2, jstart2, jend2,      &
+            pe_start, pe_end, whalo=whalo, ehalo=ehalo, shalo=shalo, nhalo=nhalo,       &
+            name = type, memory_size = msize, symmetry = .true.  )
     end select
-
-    !--- first test mpp_get_mosaic_refine_overlap
-    maxtotal = 0
-    allocate(from_tile1(4, ntiles_on_pe), from_tile2(4, ntiles_on_pe))
-    do n = 1, ntiles_on_pe
-       rotation2 = ZERO
-       noverlap1 = mpp_get_refine_overlap_number(domain, tile_count=n)
-       call mpp_get_mosaic_refine_overlap(domain, isMe1, ieMe1, jsMe1, jeMe1, isNb1, ieNb1, jsNb1, jeNb1, &
-                                          dirMe1, rotation1, tile_count = n)
-       total1 = sum( (ieNb1(1:noverlap1)-isNb1(1:noverlap1)+1) * (jeNb1(1:noverlap1)-jsNb1(1:noverlap1)+1)  )
-
-       !--- the following will figure out the overlapping
-       call mpp_get_compute_domain(domain, isc, iec, jsc, jec, tile_count=n)
-       call mpp_get_global_domain(domain, isg, ieg, jsg, jeg, tile_count=n)
-       noverlap2 = 0; total2 = 0
-       select case ( type )
-       case ( 'Refined-Four-Tile', 'Refined-Symmetric-Four-Tile' )
-          if( iec == ieg ) then   ! --- EAST
-             noverlap2 = noverlap2 + 1
-             if( mod(tiles(n),2) == 1) then ! tile 1, 3    
-                tNb = tiles(n) + 1
-             else                           ! tile 2, 4
-                tNb = tiles(n) - 1
-             endif
-             from_tile2(noverlap2,n) = tNb
-             dirMe2(noverlap2) = 1
-             isMe2(noverlap2) = iec + 1;    ieMe2(noverlap2) = iec + ehalo
-             jsMe2(noverlap2) = max(jsg,jsc - shalo); jeMe2(noverlap2) = min(jeg, jec+nhalo)
-             isNb2(noverlap2) = 1;          ieNb2(noverlap2) = ehalo
-             select case(tiles(n))
-             case(1)
-                jsNb2(noverlap2) = (jsMe2(noverlap2)-1)*3+1;  jeNb2(noverlap2) = jeMe2(noverlap2)*3
-             case(2)
-                jsNb2(noverlap2) = (jsMe2(noverlap2)-1)/3+1; jeNb2(noverlap2) = ceiling(jeMe2(noverlap2)/3.)
-             case(3)
-                jsNb2(noverlap2) = (jsMe2(noverlap2)-1)*2+1;  jeNb2(noverlap2) = jeMe2(noverlap2)*2
-             case(4)
-                jsNb2(noverlap2) = (jsMe2(noverlap2)-1)/2+1; jeNb2(noverlap2) = ceiling(jeMe2(noverlap2)/2.)
-             end select
-             total2 = total2 + (ieNb2(noverlap2) - isNb2(noverlap2) + 1) * (jeNb2(noverlap2) - jsNb2(noverlap2) + 1)
-          end if
-
-          if( jsc == jsg .AND. mod(tiles(n),2) == 1 ) then   ! --- SOUTH (only tile 1 and 3)
-             noverlap2 = noverlap2 + 1
-             tNb = mod(tiles(n)+2, ntiles)
-             from_tile2(noverlap2,n) = tNb
-             dirMe2(noverlap2) = 3
-             isMe2(noverlap2) = max(isg,isc-whalo); ieMe2(noverlap2) = min(ieg, iec+ehalo)
-             jsMe2(noverlap2) = jsc - shalo;        jeMe2(noverlap2) = jsc - 1
-             if(tiles(n) == 1) then  ! refinement is 2
-                isNb2(noverlap2) = (isMe2(noverlap2)-1)*2+1;    ieNb2(noverlap2) = ieMe2(noverlap2)*2
-             else                   ! refinement is 2
-                isNb2(noverlap2) = (isMe2(noverlap2)-1)/2+1;    ieNb2(noverlap2) = ceiling(ieMe2(noverlap2)/2.)
-             end if
-             jsNb2(noverlap2) = nj(tNb) - shalo + 1; jeNb2(noverlap2) = nj(tNb)
-             total2 = total2 + (ieNb2(noverlap2) - isNb2(noverlap2) + 1) * (jeNb2(noverlap2) - jsNb2(noverlap2) + 1)
-          end if
-
-          if( isc == isg ) then   ! --- WEST
-             noverlap2 = noverlap2 + 1
-             if( mod(tiles(n),2) == 1) then ! tile 1, 3    
-                tNb = tiles(n) + 1
-             else                           ! tile 2, 4
-                tNb = tiles(n) - 1
-             endif
-             from_tile2(noverlap2,n) = tNb
-             dirMe2(noverlap2) = 5
-             isMe2(noverlap2) = isc - whalo;          ieMe2(noverlap2) = isc - 1
-             jsMe2(noverlap2) = max(jsg,jsc - shalo); jeMe2(noverlap2) = min(jeg, jec+nhalo)
-             isNb2(noverlap2) = ni(tNb) - whalo + 1; ieNb2(noverlap2) = ni(tNb)
-             select case(tiles(n))
-             case(1)
-                jsNb2(noverlap2) = (jsMe2(noverlap2)-1)*3+1;  jeNb2(noverlap2) = jeMe2(noverlap2)*3
-             case(2)
-                jsNb2(noverlap2) = (jsMe2(noverlap2)-1)/3+1; jeNb2(noverlap2) = ceiling(jeMe2(noverlap2)/3.)
-             case(3)
-                jsNb2(noverlap2) = (jsMe2(noverlap2)-1)*2+1;  jeNb2(noverlap2) = jeMe2(noverlap2)*2
-             case(4)
-                jsNb2(noverlap2) = (jsMe2(noverlap2)-1)/2+1; jeNb2(noverlap2) = ceiling(jeMe2(noverlap2)/2.)
-             end select
-             total2 = total2 + (ieNb2(noverlap2) - isNb2(noverlap2) + 1) * (jeNb2(noverlap2) - jsNb2(noverlap2) + 1)
-          end if
-
-          if( jec == jeg .AND. mod(tiles(n),2) == 1 ) then   ! --- NORTH (only tile 1 and 3)
-             noverlap2 = noverlap2 + 1
-             tNb = mod(tiles(n)+2, ntiles)
-             from_tile2(noverlap2,n) = tNb
-             dirMe2(noverlap2) = 7
-             isMe2(noverlap2) = max(isg,isc-whalo); ieMe2(noverlap2) = min(ieg, iec+ehalo)
-             jsMe2(noverlap2) = jec + 1;        jeMe2(noverlap2) = jec + nhalo
-             if(tiles(n) == 1) then  ! refinement is 2
-                isNb2(noverlap2) = (isMe2(noverlap2)-1)*2+1;    ieNb2(noverlap2) = ieMe2(noverlap2)*2
-             else                   ! refinement is 2
-                isNb2(noverlap2) = (isMe2(noverlap2)-1)/2+1;    ieNb2(noverlap2) = ceiling(ieMe2(noverlap2)/2.)
-             end if
-             jsNb2(noverlap2) = 1;          jeNb2(noverlap2) = nhalo
-             total2 = total2 + (ieNb2(noverlap2) - isNb2(noverlap2) + 1) * (jeNb2(noverlap2) - jsNb2(noverlap2) + 1)
-          end if
-
-       case ( 'Refined-Cubic-Grid' )
-          select case( tiles(n) )
-          case ( 1 )  ! possible refined overlap will be at EAST, WEST
-             if( iec == ieg ) then   ! --- EAST
-                noverlap2 = noverlap2 + 1
-                tNb = 2
-                from_tile2(noverlap2,n) = tNb
-                dirMe2(noverlap2) = 1
-                isMe2(noverlap2) = iec + 1;                  ieMe2(noverlap2) = iec + ehalo
-                jsMe2(noverlap2) = max(jsg,jsc - shalo);     jeMe2(noverlap2) = min(jeg, jec+nhalo)
-                isNb2(noverlap2) = 1;                        ieNb2(noverlap2) = ehalo
-                jsNb2(noverlap2) = (jsMe2(noverlap2)-1)*3+1; jeNb2(noverlap2) = jeMe2(noverlap2)*3
-                total2 = total2 + (ieNb2(noverlap2) - isNb2(noverlap2) + 1) * (jeNb2(noverlap2) - jsNb2(noverlap2) + 1) 
-             end if
-             if( isc == isg ) then   ! --- WEST
-                noverlap2 = noverlap2 + 1
-                tNb = 5
-                from_tile2(noverlap2,n) = tNb
-                dirMe2(noverlap2) = 5
-                isMe2(noverlap2) = isc - whalo;                  ieMe2(noverlap2) = isc - 1
-                jsMe2(noverlap2) = max(jsg,jsc - shalo);         jeMe2(noverlap2) = min(jeg, jec+nhalo)
-                isNb2(noverlap2) = (nj(1)-jeMe2(noverlap2))*2+1; ieNb2(noverlap2) = (nj(1)-jsMe2(noverlap2)+1)*2
-                jsNb2(noverlap2) = nj(tNb) - whalo + 1;          jeNb2(noverlap2) = nj(tNb)
-                total2 = total2 + (ieNb2(noverlap2) - isNb2(noverlap2) + 1) * (jeNb2(noverlap2) - jsNb2(noverlap2) + 1)
-                rotation2(noverlap2) = NINETY
-             end if
-          case ( 2 )  ! possible refined overlap will be at EAST, WEST
-             if( iec == ieg ) then   ! --- EAST
-                noverlap2 = noverlap2 + 1
-                tNb = 4
-                from_tile2(noverlap2,n) = tNb
-                dirMe2(noverlap2) = 1
-                isMe2(noverlap2) = iec + 1;                      ieMe2(noverlap2) = iec + ehalo
-                jsMe2(noverlap2) = max(jsg,jsc - shalo);         jeMe2(noverlap2) = min(jeg, jec+nhalo)
-                isNb2(noverlap2) = (nj(2)-jeMe2(noverlap2))/3+1; ieNb2(noverlap2) = ceiling((nj(2)-jsMe2(noverlap2)+1)/3.)
-                jsNb2(noverlap2) = 1;                            jeNb2(noverlap2) = ehalo
-                total2 = total2 + (ieNb2(noverlap2) - isNb2(noverlap2) + 1) * (jeNb2(noverlap2) - jsNb2(noverlap2) + 1)
-                rotation2(noverlap2) = NINETY       
-             end if
-             if( isc == isg ) then   ! --- WEST
-                noverlap2 = noverlap2 + 1
-                tNb = 1
-                from_tile2(noverlap2,n) = tNb
-                dirMe2(noverlap2) = 5
-                isMe2(noverlap2) = isc - whalo;              ieMe2(noverlap2) = isc - 1
-                jsMe2(noverlap2) = max(jsg,jsc - shalo);     jeMe2(noverlap2) = min(jeg, jec+nhalo)
-                isNb2(noverlap2) = ni(tNb) - whalo + 1;      ieNb2(noverlap2) = ni(tNb)
-                jsNb2(noverlap2) = (jsMe2(noverlap2)-1)/3+1; jeNb2(noverlap2) = ceiling(jeMe2(noverlap2)/3.)
-                total2 = total2 + (ieNb2(noverlap2) - isNb2(noverlap2) + 1) * (jeNb2(noverlap2) - jsNb2(noverlap2) + 1)
-             end if
-          case ( 3 )  ! possible refined overlap will be at EAST, NORTH          
-             if( iec == ieg ) then   ! --- EAST
-                noverlap2 = noverlap2 + 1
-                tNb = 4
-                from_tile2(noverlap2,n) = tNb
-                dirMe2(noverlap2) = 1
-                isMe2(noverlap2) = iec + 1;                  ieMe2(noverlap2) = iec + ehalo
-                jsMe2(noverlap2) = max(jsg,jsc - shalo);     jeMe2(noverlap2) = min(jeg, jec+nhalo)
-                isNb2(noverlap2) = 1;                        ieNb2(noverlap2) = ehalo
-                jsNb2(noverlap2) = (jsMe2(noverlap2)-1)*3+1; jeNb2(noverlap2) = jeMe2(noverlap2)*3
-                total2 = total2 + (ieNb2(noverlap2) - isNb2(noverlap2) + 1) * (jeNb2(noverlap2) - jsNb2(noverlap2) + 1) 
-             end if
-             if( jec == jeg ) then   ! --- NORTH
-                noverlap2 = noverlap2 + 1
-                tNb = 5
-                from_tile2(noverlap2,n) = tNb
-                dirMe2(noverlap2) = 7
-                isMe2(noverlap2) = max(isg,isc - whalo);        ieMe2(noverlap2) = min(ieg, iec+ehalo)
-                jsMe2(noverlap2) = jec + 1;                     jeMe2(noverlap2) = jec + nhalo
-                isNb2(noverlap2) = 1;                           ieNb2(noverlap2) = nhalo
-                jsNb2(noverlap2) = (ni(3)-ieMe2(noverlap2))/2+1; jeNb2(noverlap2) = ceiling((ni(3)-isMe2(noverlap2)+1)/2.)
-                total2 = total2 + (ieNb2(noverlap2) - isNb2(noverlap2) + 1) * (jeNb2(noverlap2) - jsNb2(noverlap2) + 1) 
-                rotation2(noverlap2) = MINUS_NINETY
-             end if
-          case ( 4 )  ! possible refined overlap will be at NORTH, EAST, SOUTH, WEST
-             if( jec == jeg ) then   ! --- NORTH
-                noverlap2 = noverlap2 + 1
-                tNb = 5
-                from_tile2(noverlap2,n) = tNb
-                dirMe2(noverlap2) = 7
-                isMe2(noverlap2) = max(isg,isc - whalo);        ieMe2(noverlap2) = min(ieg, iec+ehalo)
-                jsMe2(noverlap2) = jec + 1;                     jeMe2(noverlap2) = jec + nhalo
-                isNb2(noverlap2) = (isMe2(noverlap2)-1)*2+1;    ieNb2(noverlap2) = ieMe2(noverlap2)*2
-                jsNb2(noverlap2) = 1;                           jeNb2(noverlap2) = nhalo
-                total2 = total2 + (ieNb2(noverlap2) - isNb2(noverlap2) + 1) * (jeNb2(noverlap2) - jsNb2(noverlap2) + 1) 
-             end if
-             if( iec == ieg ) then   ! --- EAST
-                noverlap2 = noverlap2 + 1
-                tNb = 6
-                from_tile2(noverlap2,n) = tNb
-                dirMe2(noverlap2) = 1
-                isMe2(noverlap2) = iec + 1;                        ieMe2(noverlap2) = iec + ehalo
-                jsMe2(noverlap2) = max(jsg,jsc - shalo);           jeMe2(noverlap2) = min(jeg, jec+nhalo)
-                isNb2(noverlap2) = (nj(4)-jeMe2(noverlap2)-1)/3+1; ieNb2(noverlap2) = ceiling((nj(4)-jsMe2(noverlap2)+1)/3.)
-                jsNb2(noverlap2) = 1;                              jeNb2(noverlap2) = ehalo
-                total2 = total2 + (ieNb2(noverlap2) - isNb2(noverlap2) + 1) * (jeNb2(noverlap2) - jsNb2(noverlap2) + 1)
-                rotation2(noverlap2) = NINETY
-             end if
-             if( jsc == jsg ) then   ! --- SOUTH
-                noverlap2 = noverlap2 + 1
-                tNb = 2
-                from_tile2(noverlap2,n) = tNb
-                dirMe2(noverlap2) = 3
-                isMe2(noverlap2) = max(isg,isc - whalo);        ieMe2(noverlap2) = min(ieg, iec+ehalo)
-                jsMe2(noverlap2) = jsc - shalo;                 jeMe2(noverlap2) = jsc  - 1
-                isNb2(noverlap2) = ni(tNb) - shalo + 1;         ieNb2(noverlap2) = ni(tNb)
-                jsNb2(noverlap2) = (ni(4)-ieMe2(noverlap2))*3+1; jeNb2(noverlap2) = (ni(4)-isMe2(noverlap2)+1)*3
-                total2 = total2 + (ieNb2(noverlap2) - isNb2(noverlap2) + 1) * (jeNb2(noverlap2) - jsNb2(noverlap2) + 1) 
-                rotation2(noverlap2) = MINUS_NINETY
-             end if
-             if( isc == isg ) then   ! --- WEST
-                noverlap2 = noverlap2 + 1
-                tNb = 3
-                from_tile2(noverlap2,n) = tNb
-                dirMe2(noverlap2) = 5
-                isMe2(noverlap2) = isc - whalo;              ieMe2(noverlap2) = isc - 1
-                jsMe2(noverlap2) = max(jsg,jsc - shalo);     jeMe2(noverlap2) = min(jeg, jec+nhalo)
-                isNb2(noverlap2) = ni(tNb) - whalo + 1;      ieNb2(noverlap2) = ni(tNb)
-                jsNb2(noverlap2) = (jsMe2(noverlap2)-1)/3+1; jeNb2(noverlap2) = ceiling(jeMe2(noverlap2)/3.)
-                total2 = total2 + (ieNb2(noverlap2) - isNb2(noverlap2) + 1) * (jeNb2(noverlap2) - jsNb2(noverlap2) + 1)
-             end if
-
-         case ( 5 )  ! possible refined overlap will be at EAST, NORTH, WEST, SOUTH       
-             if( iec == ieg ) then   ! --- EAST
-                noverlap2 = noverlap2 + 1
-                tNb = 6
-                from_tile2(noverlap2,n) = tNb
-                dirMe2(noverlap2) = 1
-                isMe2(noverlap2) = iec + 1;                  ieMe2(noverlap2) = iec + ehalo
-                jsMe2(noverlap2) = max(jsg,jsc - shalo);     jeMe2(noverlap2) = min(jeg, jec+nhalo)
-                isNb2(noverlap2) = 1;                        ieNb2(noverlap2) = ehalo
-                jsNb2(noverlap2) = (jsMe2(noverlap2)-1)*2+1; jeNb2(noverlap2) = jeMe2(noverlap2)*2
-                total2 = total2 + (ieNb2(noverlap2) - isNb2(noverlap2) + 1) * (jeNb2(noverlap2) - jsNb2(noverlap2) + 1) 
-             end if
-             if( jec == jeg ) then   ! --- NORTH
-                noverlap2 = noverlap2 + 1
-                tNb = 1
-                from_tile2(noverlap2,n) = tNb
-                dirMe2(noverlap2) = 7
-                isMe2(noverlap2) = max(isg,isc - whalo);         ieMe2(noverlap2) = min(ieg, iec+ehalo)
-                jsMe2(noverlap2) = jec + 1;                      jeMe2(noverlap2) = jec + nhalo
-                isNb2(noverlap2) = 1;                            ieNb2(noverlap2) = nhalo
-                jsNb2(noverlap2) = (ni(5)-ieMe2(noverlap2))/2+1; jeNb2(noverlap2) = ceiling((ni(5)-isMe2(noverlap2)+1)/2.)
-                total2 = total2 + (ieNb2(noverlap2) - isNb2(noverlap2) + 1) * (jeNb2(noverlap2) - jsNb2(noverlap2) + 1)
-                rotation2(noverlap2) = MINUS_NINETY 
-             end if
-             if( isc == isg ) then   ! --- WEST
-                noverlap2 = noverlap2 + 1
-                tNb = 3
-                from_tile2(noverlap2,n) = tNb
-                dirMe2(noverlap2) = 5
-                isMe2(noverlap2) = isc - whalo;                  ieMe2(noverlap2) = isc - 1
-                jsMe2(noverlap2) = max(jsg,jsc - shalo);         jeMe2(noverlap2) = min(jeg, jec+nhalo)
-                isNb2(noverlap2) = (nj(5)-jeMe2(noverlap2))*2+1; ieNb2(noverlap2) = (nj(5)-jsMe2(noverlap2)+1)*2
-                jsNb2(noverlap2) = nj(tNb) - whalo + 1;          jeNb2(noverlap2) = nj(tNb)
-                total2 = total2 + (ieNb2(noverlap2) - isNb2(noverlap2) + 1) * (jeNb2(noverlap2) - jsNb2(noverlap2) + 1)
-                rotation2(noverlap2) = NINETY
-             end if
-             if( jsc == jsg ) then   ! --- SOUTH
-                noverlap2 = noverlap2 + 1
-                tNb = 4
-                from_tile2(noverlap2,n) = tNb
-                dirMe2(noverlap2) = 3
-                isMe2(noverlap2) = max(isg,isc - whalo);        ieMe2(noverlap2) = min(ieg, iec+ehalo)
-                jsMe2(noverlap2) = jsc - shalo;                 jeMe2(noverlap2) = jsc  - 1
-                isNb2(noverlap2) = (isMe2(noverlap2)-1)/2+1;    ieNb2(noverlap2) = ceiling(ieMe2(noverlap2)/2.)
-                jsNb2(noverlap2) = nj(tNb) - shalo + 1;         jeNb2(noverlap2) = nj(tNb)
-                total2 = total2 + (ieNb2(noverlap2) - isNb2(noverlap2) + 1) * (jeNb2(noverlap2) - jsNb2(noverlap2) + 1) 
-             end if
-
-
-          case ( 6 )  ! possible refined overlap will be at SOUTH, WEST
-             if( jsc == jsg ) then   ! --- SOUTH
-                noverlap2 = noverlap2 + 1
-                tNb = 4
-                from_tile2(noverlap2,n) = tNb
-                dirMe2(noverlap2) = 3
-                isMe2(noverlap2) = max(isg,isc - whalo);        ieMe2(noverlap2) = min(ieg, iec+ehalo)
-                jsMe2(noverlap2) = jsc - shalo;                 jeMe2(noverlap2) = jsc  - 1
-                isNb2(noverlap2) = ni(tNb) - shalo + 1;         ieNb2(noverlap2) = ni(tNb)
-                jsNb2(noverlap2) = (ni(6)-ieMe2(noverlap2))*3+1; jeNb2(noverlap2) = (ni(6)-isMe2(noverlap2)+1)*3
-                total2 = total2 + (ieNb2(noverlap2) - isNb2(noverlap2) + 1) * (jeNb2(noverlap2) - jsNb2(noverlap2) + 1) 
-                rotation2(noverlap2) = MINUS_NINETY
-             end if
-             if( isc == isg ) then   ! --- WEST
-                noverlap2 = noverlap2 + 1
-                tNb = 5
-                from_tile2(noverlap2,n) = tNb
-                dirMe2(noverlap2) = 5
-                isMe2(noverlap2) = isc - whalo;              ieMe2(noverlap2) = isc - 1
-                jsMe2(noverlap2) = max(jsg,jsc - shalo);     jeMe2(noverlap2) = min(jeg, jec+nhalo)
-                isNb2(noverlap2) = ni(tNb) - whalo + 1;      ieNb2(noverlap2) = ni(tNb)
-                jsNb2(noverlap2) = (jsMe2(noverlap2)-1)/2+1; jeNb2(noverlap2) = ceiling(jeMe2(noverlap2)/2.)
-                total2 = total2 + (ieNb2(noverlap2) - isNb2(noverlap2) + 1) * (jeNb2(noverlap2) - jsNb2(noverlap2) + 1)
-             end if
-          end select
-       end select
-
-       if(total1 .NE. total2) call mpp_error(FATAL, "test_mpp_domains: mismatch on total number of points")
-       !--- we add one extra point in each direction for the consideration of symmetric domain.
-       total2 = sum( (ieNb2(1:noverlap2) - isNb2(1:noverlap2) + 2) * (jeNb2(1:noverlap2) - jsNb2(1:noverlap2) + 2) )
-       maxtotal = max(maxtotal, total2)
-       !--- comparing
-       if( noverlap1 .NE. noverlap2 ) call mpp_error(FATAL, "test_mpp_domains: mismatch on number of overlapping region")
-       do m = 1, noverlap1
-          found = .false.
-          do l = 1, noverlap2
-             if(dirMe1(m) == dirMe2(l)) then
-                found = .true.
-                exit
-             endif
-          enddo
-          from_tile1(m,n) = from_tile2(l,n)
-          if(.not. found) call mpp_error(FATAL, "test_mpp_domains: mismatch on direction")
-          if( (isMe1(m) .NE. isMe2(l)) .OR. (ieMe1(m) .NE. ieMe2(l))        &
-              .OR. (jsMe1(m) .NE. jsMe2(l)) .OR. (jeMe1(m) .NE. jeMe2(l)) ) &
-              call mpp_error(FATAL, "test_mpp_domains: mismatch on myself overlapping index")
-          if( (isNb1(m) .NE. isNb2(l)) .OR. (ieNb1(m) .NE. ieNb2(l))        &
-              .OR. (jsNb1(m) .NE. jsNb2(l)) .OR. (jeNb1(m) .NE. jeNb2(l)) ) &
-              call mpp_error(FATAL, "test_mpp_domains: mismatch on neighbor overlapping index") 
-          if(rotation1(m) .NE. rotation2(l)) call mpp_error(FATAL, "test_mpp_domains: mismatch on rotation angle");
-       end do
-    end do
-
+    
     !--- setup data
-    nimax = maxval(ni); njmax = maxval(nj)
-    allocate(global_all(1:nimax,1:njmax,nz,ntiles) )
-    allocate(global2(1-whalo:nimax+ehalo,1-shalo:njmax+nhalo,nz, ntiles_on_pe) ) 
-    global2 = 0
+    allocate(global1_all(1:nxm,1:nym,nz, ntiles) )  
+    allocate(global1(1-whalo:ni+ehalo,1-shalo:nj+nhalo,nz, ntile_per_pe) )   
     do n = 1, ntiles
        do k = 1, nz
-          do j = 1, njmax
-             do i = 1, nimax
-                global_all(i,j,k,n) = n + i*1.0e-3 + j*1.0e-6 + k*1.0e-9
+          do j = 1, nym
+             do i = 1, nxm
+                global1_all(i,j,k,n) = n + i*1.0e-3 + j*1.0e-6 + k*1.0e-9
              end do
           end do
        end do
     end do
 
-    do n = 1, ntiles_on_pe
-       nn = tiles(n)
-       global2(1:ni(nn),1:nj(nn),:,n) = global_all(1:ni(nn),1:nj(nn),:,nn)
+    do n = 1, ntile_per_pe
+       global1(1:ni,1:nj,:,n) = global1_all(1:ni,1:nj,:,tile(n))
     end do
 
+    call mpp_get_compute_domain( domain, isc, iec, jsc, jec )
+    call mpp_get_data_domain   ( domain, isd, ied, jsd, jed )
     call mpp_get_memory_domain   ( domain, ism, iem, jsm, jem )
-    allocate( x (ism:iem,jsm:jem,nz, ntiles_on_pe) )
-    allocate( x1(ism:iem,jsm:jem,nz, ntiles_on_pe) )
-    allocate( x2(ism:iem,jsm:jem,nz, ntiles_on_pe) )
-    x = 0
-
-    do n = 1, ntiles_on_pe
-       call mpp_get_compute_domain( domain, isc, iec, jsc, jec, tile_count=n  )
-       call mpp_get_data_domain   ( domain, isd, ied, jsd, jed, tile_count=n )
-       x(isc:iec,jsc:jec,:,:) = global2(isc:iec,jsc:jec,:,:)
-    end do
-    x1 = x; x2 = x
 
-    allocate(buffer (maxtotal*nz, ntiles_on_pe))
-    allocate(buffer1(maxtotal*nz, ntiles_on_pe))
-    allocate(buffer2(maxtotal*nz, ntiles_on_pe))
+    allocate( x (ism:iem,jsm:jem,nz, ntile_per_pe) )
+    x = 0.
+    x(isc:iec,jsc:jec,:,:) = global1(isc:iec,jsc:jec,:,:)
 
-    !--- call mpp_update_domains to update domain 
-    do n = 1, ntiles_on_pe
-       call mpp_update_domains( x(:,:,:,n), domain, buffer=buffer(:,n), tile_count=n )
+    !--- fill up the value at halo points
+    do n = 1, ntile_per_pe
+       call fill_five_tile_halo(global1(:,:,:,n), global1_all, tile(n), 0, 0 )
     end do
 
-    !--- multiple varaibles update
-    do n = 1, ntiles_on_pe
-       call mpp_update_domains( x1(:,:,:,n), domain, buffer=buffer1(:,n), complete=.false., tile_count=n )
-       call mpp_update_domains( x2(:,:,:,n), domain, buffer=buffer2(:,n), complete=.true., tile_count=n )
+    ! full update
+    id = mpp_clock_id( type, flags=MPP_CLOCK_SYNC+MPP_CLOCK_DETAILED )
+    call mpp_clock_begin(id)
+    do n = 1, ntile_per_pe
+       call mpp_update_domains( x(:,:,:,n), domain, tile_count = n )
     end do
+    call mpp_clock_end(id)
 
-    !--- fill up the value at halo points and compare the value at buffer.
-    do n = 1, ntiles_on_pe  
-
-       !--- comparing the buffer.
-       noverlap1 = mpp_get_refine_overlap_number(domain, tile_count=n)
-       call mpp_get_mosaic_refine_overlap(domain, isMe1, ieMe1, jsMe1, jeMe1, isNb1, ieNb1, jsNb1, jeNb1, &
-                                          dirMe1, rotation1, tile_count = n)             
-       pos = 0
-       do m = 1, noverlap1       
-          do k = 1, nz
-             do j = jsNb1(m), jeNb1(m)
-                do i = isNb1(m), ieNb1(m)
-                   pos = pos + 1
-                   if(global_all(i,j,k,from_tile1(m,n)) .NE. buffer(pos,n) ) then
-                      write(stdunit, 111) 'x', type, mpp_pe(), i, j, k, buffer(pos,n), global_all(i,j,k,from_tile1(m,n)) 
-                      call mpp_error(FATAL, "test_refined_mosaic: mismatch between buffer data and actual data for "//type )
-                   end if
-                   if(global_all(i,j,k,from_tile1(m,n)) .NE. buffer1(pos,n) ) then
-                      write(stdunit, 111) 'x1', type, mpp_pe(), i, j, k, buffer1(pos,n), global_all(i,j,k,from_tile1(m,n)) 
-                      call mpp_error(FATAL, "test_refined_mosaic: mismatch between buffer data and actual data for "//type )
-                   end if
-                   if(global_all(i,j,k,from_tile1(m,n)) .NE. buffer2(pos,n) ) then
-                      write(stdunit, 111) 'x2', type, mpp_pe(), i, j, k, buffer2(pos,n), global_all(i,j,k,from_tile1(m,n)) 
-                      call mpp_error(FATAL, "test_refined_mosaic: mismatch between buffer data and actual data for "//type )
-                   end if
-                end do
-             end do
-          end do
-       end do
+   do n = 1, ntile_per_pe
+      write(type2, *)type, " at tile_count = ",n
+      call compare_checksums( x(isd:ied,jsd:jed,:,n), global1(isd:ied,jsd:jed,:,n), trim(type2) )
+   end do
 
-       !--- fill the halo and compare
-       select case(type)
-       case('Refined-Four-Tile', 'Refined-Symmetric-Four-Tile')
-          te = 0; ts = 0; tn = 0; tw = 0
-          select case(tiles(n))       
-          case(1)
-             tsw = 4 
-          case(2)
-             tsw = 3; tn = 4; ts = 4
-          case(3)
-             tsw = 2
-          case(4)
-             tsw = 1; ts = 2; tn = 2
-          end select
-          tse = tsw; tnw = tsw; tne = tsw
-          call fill_regular_refinement_halo( global2(:,:,:,n), global_all, ni, nj, tiles(n), &
-                                             te, tse, ts, tsw, tw, tnw, tn, tne, 0, 0)
-       case('Refined-Cubic-Grid')       
-          call fill_cubicgrid_refined_halo(global2(:,:,:,n), global_all, global_all, ni, nj, tiles(n), 0, 0, 1, 1 )
-       end select
-       call mpp_get_data_domain   ( domain, isd, ied, jsd, jed )
-       type2 = type
-       if(ntiles_on_pe>1) write(type2, *)trim(type2), " at tile_count = ", tiles(n)
-       call compare_checksums( x (isd:ied,jsd:jed,:,n), global2(isd:ied,jsd:jed,:,n), trim(type2)//' X' )
-       call compare_checksums( x1(isd:ied,jsd:jed,:,n), global2(isd:ied,jsd:jed,:,n), trim(type2)//' X1' )
-       call compare_checksums( x2(isd:ied,jsd:jed,:,n), global2(isd:ied,jsd:jed,:,n), trim(type2)//' X2' )
-    end do  
+   deallocate(global1_all, global1, x)
 
-    deallocate(global2, global_all, x, x1, x2)
     !------------------------------------------------------------------
-    !              vector update : BGRID_NE, one extra point in each direction for cubic-grid
+    !  vector update : BGRID_NE, one extra point in each direction for Five-Tile
     !------------------------------------------------------------------
     !--- setup data
-    shift = 0
-    if( type == 'Refined-Symmetric-Four-Tile' .OR. type == 'Refined-Cubic-Grid' ) shift = 1
-
-    nimax = maxval(ni) + shift; njmax = maxval(nj) + shift
-    allocate(global1_all(1:nimax,1:njmax,nz,ntiles) )
-    allocate(global2_all(1:nimax,1:njmax,nz,ntiles) )
-    allocate(global1(1-whalo:nimax+ehalo,1-shalo:njmax+nhalo,nz, ntiles_on_pe) ) 
-    allocate(global2(1-whalo:nimax+ehalo,1-shalo:njmax+nhalo,nz, ntiles_on_pe) ) 
-    global1 = 0; global2 = 0
+    allocate(global1_all(nxm+shift,nym+shift,nz, ntiles), global2_all(nxm+shift,nym+shift,nz, ntiles) )  
+    allocate(global1(1-whalo:ni+ehalo+shift,1-shalo:nj+nhalo+shift,nz, ntile_per_pe) )  
+    allocate(global2(1-whalo:ni+ehalo+shift,1-shalo:nj+nhalo+shift,nz, ntile_per_pe) )   
     do n = 1, ntiles
        do k = 1, nz
-          do j = 1, njmax
-             do i = 1, nimax
+          do j = 1, nym+shift
+             do i = 1, nxm+shift
                 global1_all(i,j,k,n) = 1.0e3 + n + i*1.0e-3 + j*1.0e-6 + k*1.0e-9
                 global2_all(i,j,k,n) = 2.0e3 + n + i*1.0e-3 + j*1.0e-6 + k*1.0e-9
              end do
@@ -3461,489 +3414,222 @@ subroutine test_refined_mosaic(type)
        end do
     end do
 
-    !--- make sure consistency on the no-refinement boundary  for symmetric domain.
-    !--- For "Symmetric four tile" mosaic, north of tile 2 and 4 need to be filled.
-    !--- For "Cubic-Grid", The following need to be filled: north of tile 1,
-    !--- north of tile 2, east and north of tile 6. The velocity at the corner point will be 0.
-    select case( type ) 
-    case ( 'Refined-Symmetric-Four-Tile' ) 
-       global1_all(1:ni(2)+1,nj(2)+1,:,2) = global1_all(1:ni(2)+1,1,:,4)
-       global2_all(1:ni(2)+1,nj(2)+1,:,2) = global2_all(1:ni(2)+1,1,:,4)
-       global1_all(1:ni(4)+1,nj(4)+1,:,4) = global1_all(1:ni(4)+1,1,:,2)
-       global2_all(1:ni(4)+1,nj(4)+1,:,4) = global2_all(1:ni(4)+1,1,:,2)
-    case ( 'Refined-Cubic-Grid' )
-       global1_all(1:ni(1)+1,nj(1)+1,:,1) = -global2_all(1,nj(3)+1:1:-1,:,3) ! north
-       global2_all(1:ni(1)+1,nj(1)+1,:,1) =  global1_all(1,nj(3)+1:1:-1,:,3) ! north
-       global1_all(1:ni(2)+1,nj(2)+1,:,2) =  global1_all(1:ni(3)+1,1,   :,3) ! north
-       global2_all(1:ni(2)+1,nj(2)+1,:,2) =  global2_all(1:ni(3)+1,1,   :,3) ! north
-       global1_all(1:ni(6)+1,nj(6)+1,:,6) =  global1_all(1:ni(1)+1,1,   :,1) ! north
-       global2_all(1:ni(6)+1,nj(6)+1,:,6) =  global2_all(1:ni(1)+1,1,   :,1) ! north
-       global1_all(ni(6)+1,1:nj(6)+1,:,6) =  global2_all(ni(2)+1:1:-1,1,:,2)  ! east 
-       global2_all(ni(6)+1,1:nj(6)+1,:,6) = -global1_all(ni(2)+1:1:-1,1,:,2)  ! east 
-       do n = 1, ntiles
-          global1_all(1,      1,:,n) = 0;  global1_all(1,      nj(n)+1,:,n) = 0;
-          global1_all(ni(n)+1,1,:,n) = 0;  global1_all(ni(n)+1,nj(n)+1,:,n) = 0;
-          global2_all(1,      1,:,n) = 0;  global2_all(1,      nj(n)+1,:,n) = 0;
-          global2_all(ni(n)+1,1,:,n) = 0;  global2_all(ni(n)+1,nj(n)+1,:,n) = 0;
-       end do
-    end select
+    !------------------------------------------------------------------------
+    ! --- make sure consisency on the boundary for Five-Tile mosaic
+    ! --- east boundary will take the value of neighbor tile west,
+    ! --- north boundary will take the value of neighbor tile south.
+    !------------------------------------------------------------------------
+    if(type == 'Five-Tile') then
+       global1_all(nxm+1,    1:ny,:,1) = global1_all(1,    1:ny,:,2)  ! east
+       global1_all(nxm+1,ny+1:nym,:,1) = global1_all(1,    1:ny,:,4)  ! east
+       global1_all(1:nxm+1, nym+1,:,1) = global1_all(1:nxm+1, 1,:,1)  ! north
+       global1_all(nx+1,     1:ny,:,2) = global1_all(1,    1:ny,:,3)  ! east
+       global1_all(1:nx+1,   ny+1,:,2) = global1_all(1:nx+1,  1,:,4)  ! north
+       global1_all(nx+1,     1:ny,:,3) = global1_all(1,    1:ny,:,1)  ! east
+       global1_all(1:nx+1,   ny+1,:,3) = global1_all(1:nx+1,  1,:,5)  ! north
+       global1_all(nx+1,     1:ny,:,4) = global1_all(1,    1:ny,:,5)  ! east
+       global1_all(1:nx+1,   ny+1,:,4) = global1_all(1:nx+1,  1,:,2)  ! north
+       global1_all(nx+1,     1:ny,:,5) = global1_all(1,ny+1:nym,:,1)  ! east
+       global1_all(1:nx+1,   ny+1,:,5) = global1_all(1:nx+1,  1,:,3)  ! north
+       global1_all(nx+1,     ny+1,:,2) = global1_all(1,       1,:,5)  ! northeast 
+       global1_all(nx+1,     ny+1,:,3) = global1_all(1,    ny+1,:,1)  ! northeast 
+       global2_all(nxm+1,    1:ny,:,1) = global2_all(1,    1:ny,:,2)  ! east
+       global2_all(nxm+1,ny+1:nym,:,1) = global2_all(1,    1:ny,:,4)  ! east
+       global2_all(1:nxm+1, nym+1,:,1) = global2_all(1:nxm+1, 1,:,1)  ! north
+       global2_all(nx+1,     1:ny,:,2) = global2_all(1,    1:ny,:,3)  ! east
+       global2_all(1:nx+1,   ny+1,:,2) = global2_all(1:nx+1,  1,:,4)  ! north
+       global2_all(nx+1,     1:ny,:,3) = global2_all(1,    1:ny,:,1)  ! east
+       global2_all(1:nx+1,   ny+1,:,3) = global2_all(1:nx+1,  1,:,5)  ! north
+       global2_all(nx+1,     1:ny,:,4) = global2_all(1,    1:ny,:,5)  ! east
+       global2_all(1:nx+1,   ny+1,:,4) = global2_all(1:nx+1,  1,:,2)  ! north
+       global2_all(nx+1,     1:ny,:,5) = global2_all(1,ny+1:nym,:,1)  ! east
+       global2_all(1:nx+1,   ny+1,:,5) = global2_all(1:nx+1,  1,:,3)  ! north
+       global2_all(nx+1,     ny+1,:,2) = global2_all(1,       1,:,5)  ! northeast 
+       global2_all(nx+1,     ny+1,:,3) = global2_all(1,    ny+1,:,1)  ! northeast 
+    end if
 
-    do n = 1, ntiles_on_pe
-       nn = tiles(n)
-       global1(1:ni(nn)+shift,1:nj(nn)+shift,:,n) = global1_all(1:ni(nn)+shift,1:nj(nn)+shift,:,nn)
-       global2(1:ni(nn)+shift,1:nj(nn)+shift,:,n) = global2_all(1:ni(nn)+shift,1:nj(nn)+shift,:,nn)
+    do n = 1, ntile_per_pe
+       global1(1:ni+shift,1:nj+shift,:,n) = global1_all(1:ni+shift,1:nj+shift,:,tile(n))
+       global2(1:ni+shift,1:nj+shift,:,n) = global2_all(1:ni+shift,1:nj+shift,:,tile(n))
     end do
 
-    call mpp_get_memory_domain   ( domain, ism, iem, jsm, jem )
-    allocate( x  (ism:iem+shift,jsm:jem+shift,nz, ntiles_on_pe) )
-    allocate( y  (ism:iem+shift,jsm:jem+shift,nz, ntiles_on_pe) )
-    allocate( x1 (ism:iem+shift,jsm:jem+shift,nz, ntiles_on_pe) )
-    allocate( y1 (ism:iem+shift,jsm:jem+shift,nz, ntiles_on_pe) )
-    allocate( x2 (ism:iem+shift,jsm:jem+shift,nz, ntiles_on_pe) )
-    allocate( y2 (ism:iem+shift,jsm:jem+shift,nz, ntiles_on_pe) )
-    x = 0; y = 0
-
-    do n = 1, ntiles_on_pe
-       call mpp_get_compute_domain( domain, isc, iec, jsc, jec, tile_count=n )
-       call mpp_get_data_domain   ( domain, isd, ied, jsd, jed, tile_count=n )
-       x(isc:iec+shift,jsc:jec+shift,:,:) = global1(isc:iec+shift,jsc:jec+shift,:,:)
-       y(isc:iec+shift,jsc:jec+shift,:,:) = global2(isc:iec+shift,jsc:jec+shift,:,:)
-    end do
-    x1 = x; x2 =x; y1 = y; y2 = y
+    allocate( x (ism:iem+shift,jsm:jem+shift,nz,ntile_per_pe) )
+    allocate( y (ism:iem+shift,jsm:jem+shift,nz,ntile_per_pe) )
 
-    allocate(bufferx(maxtotal*nz, ntiles_on_pe),  buffery(maxtotal*nz, ntiles_on_pe) )
-    allocate(bufferx1(maxtotal*nz, ntiles_on_pe), buffery1(maxtotal*nz, ntiles_on_pe) )
-    allocate(bufferx2(maxtotal*nz, ntiles_on_pe), buffery2(maxtotal*nz, ntiles_on_pe) )
+    x = 0.; y = 0
+    x (isc:iec+shift,jsc:jec+shift,:,:) = global1(isc:iec+shift,jsc:jec+shift,:,:)
+    y (isc:iec+shift,jsc:jec+shift,:,:) = global2(isc:iec+shift,jsc:jec+shift,:,:)
 
-    !--- call mpp_update_domains to update domain 
-    do n = 1, ntiles_on_pe
-       call mpp_update_domains( x(:,:,:,n), y(:,:,:,n), domain, gridtype=BGRID_NE, &
-                                bufferx=bufferx(:,n), buffery=buffery(:,n), tile_count=n )
+    !-----------------------------------------------------------------------
+    !                   fill up the value at halo points.     
+    !-----------------------------------------------------------------------
+    do n = 1, ntile_per_pe
+       call fill_five_tile_halo(global1(:,:,:,n), global1_all, tile(n), shift, shift)
+       call fill_five_tile_halo(global2(:,:,:,n), global2_all, tile(n), shift, shift)
     end do
 
-    !--- multiple update
-    do n = 1, ntiles_on_pe
-       call mpp_update_domains( x1(:,:,:,n), y1(:,:,:,n), domain, gridtype=BGRID_NE, &
-                                bufferx=bufferx1(:,n), buffery=buffery1(:,n), complete=.false., tile_count=n )
-       call mpp_update_domains( x2(:,:,:,n), y2(:,:,:,n), domain, gridtype=BGRID_NE, &
-                                bufferx=bufferx2(:,n), buffery=buffery2(:,n), complete=.true., tile_count=n )
+    id = mpp_clock_id( type//' BGRID_NE', flags=MPP_CLOCK_SYNC+MPP_CLOCK_DETAILED )
+    call mpp_clock_begin(id)
+    do n = 1, ntile_per_pe
+       call mpp_update_domains( x(:,:,:,n), y(:,:,:,n), domain, gridtype=BGRID_NE, tile_count = n )
     end do
+    call mpp_clock_end(id)
 
-    !--- fill up the value at halo points and compare the value at buffer.
-    do n = 1, ntiles_on_pe  
-       !--- comparing the buffer.
-       noverlap1 = mpp_get_refine_overlap_number(domain, tile_count=n, position = CORNER)
-       call mpp_get_mosaic_refine_overlap(domain, isMe1, ieMe1, jsMe1, jeMe1, isNb1, ieNb1, jsNb1, jeNb1, &
-               dirMe1, rotation1, tile_count = n, position = CORNER)             
-       pos = 0
-
-       do m = 1, noverlap1  
-          select case( rotation1(m) )
-          case (ZERO)     
-             do k = 1, nz
-                do j = jsNb1(m), jeNb1(m)
-                   do i = isNb1(m), ieNb1(m)
-                      pos = pos + 1
-                      if(global1_all(i,j,k,from_tile1(m,n)) .NE. bufferx(pos,n) ) then
-                         write(stdunit,111)'x','BGRID '//type,mpp_pe(),i,j,k,bufferx(pos,n),global1_all(i,j,k,from_tile1(m,n))
-                         call mpp_error(FATAL,"test_refined_mosaic: mismatch between buffer and actual data: BGRID "//type//" X")
-                      end if
-                      if(global2_all(i,j,k,from_tile1(m,n)) .NE. buffery(pos,n) ) then
-                         write(stdunit,111)'y','BGRID '//type,mpp_pe(),i,j,k,buffery(pos,n),global2_all(i,j,k,from_tile1(m,n))
-                         call mpp_error(FATAL,"test_refined_mosaic: mismatch between buffer and actual data: BGRID "//type//" Y")
-                      end if
-                      if(global1_all(i,j,k,from_tile1(m,n)) .NE. bufferx1(pos,n) ) then
-                         write(stdunit,111)'x1','BGRID '//type,mpp_pe(),i,j,k,bufferx1(pos,n),global1_all(i,j,k,from_tile1(m,n))
-                         call mpp_error(FATAL,"test_refined_mosaic: mismatch between buffer and actual data: BGRID "//type//" X1")
-                      end if
-                      if(global2_all(i,j,k,from_tile1(m,n)) .NE. buffery1(pos,n) ) then
-                         write(stdunit,111)'y1','BGRID '//type,mpp_pe(),i,j,k,buffery1(pos,n),global2_all(i,j,k,from_tile1(m,n))
-                         call mpp_error(FATAL,"test_refined_mosaic: mismatch between buffer and actual data: BGRID "//type//" Y1")
-                      end if
-                      if(global1_all(i,j,k,from_tile1(m,n)) .NE. bufferx2(pos,n) ) then
-                         write(stdunit,111)'x2','BGRID '//type,mpp_pe(),i,j,k,bufferx2(pos,n),global1_all(i,j,k,from_tile1(m,n))
-                         call mpp_error(FATAL,"test_refined_mosaic: mismatch between buffer and actual data: BGRID "//type//" X2")
-                      end if
-                      if(global2_all(i,j,k,from_tile1(m,n)) .NE. buffery2(pos,n) ) then
-                         write(stdunit,111)'y2','BGRID '//type,mpp_pe(),i,j,k,buffery2(pos,n),global2_all(i,j,k,from_tile1(m,n))
-                         call mpp_error(FATAL,"test_refined_mosaic: mismatch between buffer and actual data: BGRID "//type//" Y2")
-                      end if
-                   end do
-                end do
-             end do
-          case (NINETY) ! S->E, N->W, u->-v, v->u    
-             do k = 1, nz
-                do j = jsNb1(m), jeNb1(m)
-                   do i = isNb1(m), ieNb1(m)
-                      pos = pos + 1
-                      if(global2_all(i,j,k,from_tile1(m,n)) .NE. bufferx(pos,n) ) then
-                         write(stdunit,111)'x','BGRID '//type,mpp_pe(),i,j,k,bufferx(pos,n),global2_all(i,j,k,from_tile1(m,n))
-                         call mpp_error(FATAL,"test_refined_mosaic: mismatch between buffer and actual data: BGRID "//type//" X")
-                      end if
-                      if(-global1_all(i,j,k,from_tile1(m,n)) .NE. buffery(pos,n) ) then
-                         write(stdunit,111)'y','BGRID '//type,mpp_pe(),i,j,k,buffery(pos,n),-global1_all(i,j,k,from_tile1(m,n))
-                         call mpp_error(FATAL,"test_refined_mosaic: mismatch between buffer and actual data: BGRID "//type//" Y")
-                      end if
-                      if(global2_all(i,j,k,from_tile1(m,n)) .NE. bufferx1(pos,n) ) then
-                         write(stdunit,111)'x1','BGRID '//type,mpp_pe(),i,j,k,bufferx1(pos,n),global2_all(i,j,k,from_tile1(m,n))
-                         call mpp_error(FATAL,"test_refined_mosaic: mismatch between buffer and actual data: BGRID "//type//" X1")
-                      end if
-                      if(-global1_all(i,j,k,from_tile1(m,n)) .NE. buffery1(pos,n) ) then
-                         write(stdunit,111)'y1','BGRID '//type,mpp_pe(),i,j,k,buffery1(pos,n),-global1_all(i,j,k,from_tile1(m,n))
-                         call mpp_error(FATAL,"test_refined_mosaic: mismatch between buffer and actual data: BGRID "//type//" Y1")
-                      end if
-                      if(global2_all(i,j,k,from_tile1(m,n)) .NE. bufferx2(pos,n) ) then
-                         write(stdunit,111)'x2','BGRID '//type,mpp_pe(),i,j,k,bufferx2(pos,n),global2_all(i,j,k,from_tile1(m,n))
-                         call mpp_error(FATAL,"test_refined_mosaic: mismatch between buffer and actual data: BGRID "//type//" X2")
-                      end if
-                      if(-global1_all(i,j,k,from_tile1(m,n)) .NE. buffery2(pos,n) ) then
-                         write(stdunit,111)'y2','BGRID '//type,mpp_pe(),i,j,k,buffery2(pos,n),-global1_all(i,j,k,from_tile1(m,n))
-                         call mpp_error(FATAL,"test_refined_mosaic: mismatch between buffer and actual data: BGRID "//type//" Y2")
-                      end if
-                   end do
-                end do
-             end do
-          case (MINUS_NINETY) ! S->E, N->W, u->-v, v->u    
-             do k = 1, nz
-                do j = jsNb1(m), jeNb1(m)
-                   do i = isNb1(m), ieNb1(m)
-                      pos = pos + 1
-                      if(-global2_all(i,j,k,from_tile1(m,n)) .NE. bufferx(pos,n) ) then
-                         write(stdunit,111)'x','BGRID '//type,mpp_pe(),i,j,k,bufferx(pos,n),-global2_all(i,j,k,from_tile1(m,n))
-                         call mpp_error(FATAL,"test_refined_mosaic: mismatch between buffer and actual data: BGRID "//type//" X")
-                      end if
-                      if(global1_all(i,j,k,from_tile1(m,n)) .NE. buffery(pos,n) ) then
-                         write(stdunit,111)'y','BGRID '//type,mpp_pe(),i,j,k,buffery(pos,n),global1_all(i,j,k,from_tile1(m,n))
-                         call mpp_error(FATAL,"test_refined_mosaic: mismatch between buffer and actual data: BGRID "//type//" Y")
-                      end if
-                      if(-global2_all(i,j,k,from_tile1(m,n)) .NE. bufferx1(pos,n) ) then
-                         write(stdunit,111)'x1','BGRID '//type,mpp_pe(),i,j,k,bufferx1(pos,n),-global2_all(i,j,k,from_tile1(m,n))
-                         call mpp_error(FATAL,"test_refined_mosaic: mismatch between buffer and actual data: BGRID "//type//" X1")
-                      end if
-                      if(global1_all(i,j,k,from_tile1(m,n)) .NE. buffery1(pos,n) ) then
-                         write(stdunit,111)'y1','BGRID '//type,mpp_pe(),i,j,k,buffery1(pos,n),global1_all(i,j,k,from_tile1(m,n))
-                         call mpp_error(FATAL,"test_refined_mosaic: mismatch between buffer and actual data: BGRID "//type//" Y1")
-                      end if
-                      if(-global2_all(i,j,k,from_tile1(m,n)) .NE. bufferx2(pos,n) ) then
-                         write(stdunit,111)'x2','BGRID '//type,mpp_pe(),i,j,k,bufferx2(pos,n),-global2_all(i,j,k,from_tile1(m,n))
-                         call mpp_error(FATAL,"test_refined_mosaic: mismatch between buffer and actual data: BGRID "//type//" X2")
-                      end if
-                      if(global1_all(i,j,k,from_tile1(m,n)) .NE. buffery2(pos,n) ) then
-                         write(stdunit,111)'y2','BGRID '//type,mpp_pe(),i,j,k,buffery2(pos,n),global1_all(i,j,k,from_tile1(m,n))
-                         call mpp_error(FATAL,"test_refined_mosaic: mismatch between buffer and actual data: BGRID "//type//" Y2")
-                      end if
-                   end do
-                end do
-             end do
-          end select
-       end do
-
-       !--- fill the halo data and compare
-       select case(type)
-       case('Refined-Four-Tile', 'Refined-Symmetric-Four-Tile')
-          te = 0; ts = 0; tn = 0; tw = 0
-          select case(tiles(n))       
-          case(1)
-             tsw = 4 
-          case(2)
-             tsw = 3; tn = 4; ts = 4
-          case(3)
-             tsw = 2
-          case(4)
-             tsw = 1; ts = 2; tn = 2
-          end select
-          tse = tsw; tnw = tsw; tne = tsw
-          call fill_regular_refinement_halo( global1(:,:,:,n), global1_all, ni, nj, tiles(n), &
-               te, tse, ts, tsw, tw, tnw, tn, tne, shift, shift )
-          call fill_regular_refinement_halo( global2(:,:,:,n), global2_all, ni, nj, tiles(n), &
-               te, tse, ts, tsw, tw, tnw, tn, tne, shift, shift )
-       case('Refined-Cubic-Grid')       
-          call fill_cubicgrid_refined_halo(global1(:,:,:,n), global1_all, global2_all, ni, nj, tiles(n), 1, 1, 1, -1 )
-          call fill_cubicgrid_refined_halo(global2(:,:,:,n), global2_all, global1_all, ni, nj, tiles(n), 1, 1, -1, 1 )
-       end select
+   do n = 1, ntile_per_pe
+      write(type2, *)type, " at tile_count = ",n
+      call compare_checksums( x(isd:ied+shift,jsd:jed+shift,:,n), global1(isd:ied+shift,jsd:jed+shift,:,n), &
+                              trim(type2)//' BGRID_NE X')
+      call compare_checksums( y(isd:ied+shift,jsd:jed+shift,:,n), global2(isd:ied+shift,jsd:jed+shift,:,n), &
+                              trim(type2)//' BGRID_NE Y')
+   end do
 
-       call mpp_get_data_domain   ( domain, isd, ied, jsd, jed )
-       write(type2, *)"BGRID ", type
-       if(ntiles_on_pe>1) write(type2, *)trim(type2), " at tile_count = ", tiles(n)
-       call compare_checksums( x (isd:ied+shift,jsd:jed+shift,:,n), global1(isd:ied+shift,jsd:jed+shift,:,n), trim(type2)//' X' )
-       call compare_checksums( x1(isd:ied+shift,jsd:jed+shift,:,n), global1(isd:ied+shift,jsd:jed+shift,:,n), trim(type2)//' X1')
-       call compare_checksums( x2(isd:ied+shift,jsd:jed+shift,:,n), global1(isd:ied+shift,jsd:jed+shift,:,n), trim(type2)//' X2')
-       write(type2, *)"BGRID ", type
-       if(ntiles_on_pe>1) write(type2, *)trim(type2), " at tile_count = ", tiles(n)
-       call compare_checksums( y (isd:ied+shift,jsd:jed+shift,:,n), global2(isd:ied+shift,jsd:jed+shift,:,n), trim(type2)//' Y' )
-       call compare_checksums( y1(isd:ied+shift,jsd:jed+shift,:,n), global2(isd:ied+shift,jsd:jed+shift,:,n), trim(type2)//' Y1')
-       call compare_checksums( y2(isd:ied+shift,jsd:jed+shift,:,n), global2(isd:ied+shift,jsd:jed+shift,:,n), trim(type2)//' Y2')
-    end do  
+   deallocate(global1_all, global2_all, global1, global2, x, y)
 
-    deallocate(global1_all, global2_all, global1, global2, x, y, x1, x2, y1, y2 )
     !------------------------------------------------------------------
-    !              vector update : CGRID_NE, one extra point may needed to symmetric domain
+    !  vector update : CGRID_NE
     !------------------------------------------------------------------
     !--- setup data
-
-    nimax = maxval(ni); njmax = maxval(nj)
-    allocate(global1_all(1:nimax+shift,1:njmax,nz,ntiles) )
-    allocate(global2_all(1:nimax,1:njmax+shift,nz,ntiles) )
-    allocate(global1(1-whalo:nimax+ehalo+shift,1-shalo:njmax+nhalo,nz, ntiles_on_pe) ) 
-    allocate(global2(1-whalo:nimax+ehalo,1-shalo:njmax+nhalo+shift,nz, ntiles_on_pe) ) 
-    global1 = 0; global2 = 0
+    allocate(global1_all(nxm+shift,nym,nz, ntiles), global2_all(nxm,nym+shift,nz, ntiles) )  
+    allocate(global1(1-whalo:ni+ehalo+shift, 1-shalo:nj+nhalo,       nz, ntile_per_pe) )  
+    allocate(global2(1-whalo:ni+ehalo,       1-shalo:nj+nhalo+shift, nz, ntile_per_pe) )   
     do n = 1, ntiles
        do k = 1, nz
-          do j = 1, njmax
-             do i = 1, nimax + shift
+          do j = 1, nym
+             do i = 1, nxm+shift
                 global1_all(i,j,k,n) = 1.0e3 + n + i*1.0e-3 + j*1.0e-6 + k*1.0e-9
              end do
           end do
-          do j = 1, njmax + shift
-             do i = 1, nimax
+          do j = 1, nym+shift
+             do i = 1, nxm
                 global2_all(i,j,k,n) = 2.0e3 + n + i*1.0e-3 + j*1.0e-6 + k*1.0e-9
              end do
           end do
        end do
     end do
 
-    !--- make sure consistency on the no-refinement boundary  for symmetric domain.
-    !--- For "Symmetric four tile" mosaic, north of tile 2 and 4 need to be filled.
-    !--- For "Cubic-Grid", The following need to be filled: north of tile 1,
-    !--- north of tile 2, east and north of tile 6. The velocity at the corner point will be 0.
-    select case( type ) 
-    case ('Refined-Symmetric-Four-Tile' ) 
-       global2_all(1:ni(2),nj(2)+1,:,2) = global2_all(1:ni(2),1,:,4)
-       global2_all(1:ni(4),nj(4)+1,:,4) = global2_all(1:ni(4),1,:,2)
-    case ('Refined-Cubic-Grid' )
-       global2_all(1:ni(1),nj(1)+1,:,1) =  global1_all(1,nj(3):1:-1,:,3) ! north
-       global2_all(1:ni(2),nj(2)+1,:,2) =  global2_all(1:ni(3),1,   :,3) ! north
-       global2_all(1:ni(6),nj(6)+1,:,6) =  global2_all(1:ni(1),1,   :,1) ! north
-       global1_all(ni(6)+1,1:nj(6),:,6) =  global2_all(ni(2):1:-1,1,:,2)  ! east 
-    end select
+    !------------------------------------------------------------------------
+    ! --- make sure consisency on the boundary for Five-Tile mosaic
+    ! --- east boundary will take the value of neighbor tile west,
+    ! --- north boundary will take the value of neighbor tile south.
+    !------------------------------------------------------------------------
+    if(type == 'Five-Tile') then
+       global1_all(nxm+1,    1:ny,:,1) = global1_all(1,    1:ny,:,2)  ! east
+       global1_all(nxm+1,ny+1:nym,:,1) = global1_all(1,    1:ny,:,4)  ! east
+       global1_all(nx+1,     1:ny,:,2) = global1_all(1,    1:ny,:,3)  ! east
+       global1_all(nx+1,     1:ny,:,3) = global1_all(1,    1:ny,:,1)  ! east
+       global1_all(nx+1,     1:ny,:,4) = global1_all(1,    1:ny,:,5)  ! east
+       global1_all(nx+1,     1:ny,:,5) = global1_all(1,ny+1:nym,:,1)  ! east
+       global2_all(1:nxm,   nym+1,:,1) = global2_all(1:nxm,   1,:,1)  ! north
+       global2_all(1:nx,     ny+1,:,2) = global2_all(1:nx,    1,:,4)  ! north
+       global2_all(1:nx,     ny+1,:,3) = global2_all(1:nx,    1,:,5)  ! north
+       global2_all(1:nx,     ny+1,:,4) = global2_all(1:nx,    1,:,2)  ! north
+       global2_all(1:nx,     ny+1,:,5) = global2_all(1:nx,    1,:,3)  ! north
+    end if
 
-    do n = 1, ntiles_on_pe
-       nn = tiles(n)
-       global1(1:ni(nn)+shift,1:nj(nn),:,n) = global1_all(1:ni(nn)+shift,1:nj(nn),:,nn)
-       global2(1:ni(nn),1:nj(nn)+shift,:,n) = global2_all(1:ni(nn),1:nj(nn)+shift,:,nn)
+    do n = 1, ntile_per_pe
+       global1(1:ni+shift,      1:nj,:,n) = global1_all(1:ni+shift,      1:nj,:,tile(n))
+       global2(1:ni,      1:nj+shift,:,n) = global2_all(1:ni,      1:nj+shift,:,tile(n))
     end do
 
-    call mpp_get_memory_domain   ( domain, ism, iem, jsm, jem )
-    allocate( x  (ism:iem+shift,jsm:jem,nz, ntiles_on_pe) )
-    allocate( y  (ism:iem,jsm:jem+shift,nz, ntiles_on_pe) )
-    allocate( x1 (ism:iem+shift,jsm:jem,nz, ntiles_on_pe) )
-    allocate( y1 (ism:iem,jsm:jem+shift,nz, ntiles_on_pe) )
-    allocate( x2 (ism:iem+shift,jsm:jem,nz, ntiles_on_pe) )
-    allocate( y2 (ism:iem,jsm:jem+shift,nz, ntiles_on_pe) )
-    x = 0; y = 0
-    bufferx  = 0; buffery = 0 
-    bufferx1 = 0; buffery1 = 0
-    bufferx2 = 0; buffery2 = 0
-    do n = 1, ntiles_on_pe
-       call mpp_get_compute_domain( domain, isc, iec, jsc, jec, tile_count=n )
-       call mpp_get_data_domain   ( domain, isd, ied, jsd, jed, tile_count=n )
-       x(isc:iec+shift,jsc:jec,:,:) = global1(isc:iec+shift,jsc:jec,:,:)
-       y(isc:iec,jsc:jec+shift,:,:) = global2(isc:iec,jsc:jec+shift,:,:)
-    end do
-    x1 = x; x2 =x; y1 = y; y2 = y   
+    allocate( x (ism:iem+shift,      jsm:jem,nz,ntile_per_pe) )
+    allocate( y (ism:iem,      jsm:jem+shift,nz,ntile_per_pe) )
 
-    !--- call mpp_update_domains to update domain 
-    do n = 1, ntiles_on_pe
-       call mpp_update_domains( x(:,:,:,n), y(:,:,:,n), domain, gridtype=CGRID_NE, &
-                                bufferx=bufferx(:,n), buffery=buffery(:,n), tile_count=n )
+    x = 0.; y = 0
+    x (isc:iec+shift,      jsc:jec,:,:) = global1(isc:iec+shift,      jsc:jec,:,:)
+    y (isc:iec,      jsc:jec+shift,:,:) = global2(isc:iec,      jsc:jec+shift,:,:)
+
+    !-----------------------------------------------------------------------
+    !                   fill up the value at halo points.     
+    !-----------------------------------------------------------------------
+    do n = 1, ntile_per_pe
+       call fill_five_tile_halo(global1(:,:,:,n), global1_all, tile(n), shift, 0)
+       call fill_five_tile_halo(global2(:,:,:,n), global2_all, tile(n), 0, shift)
     end do
 
-    !--- multiple update
-    do n = 1, ntiles_on_pe
-       call mpp_update_domains( x1(:,:,:,n), y1(:,:,:,n), domain, gridtype=CGRID_NE, &
-                                bufferx=bufferx1(:,n), buffery=buffery1(:,n), complete=.false., tile_count=n )
-       call mpp_update_domains( x2(:,:,:,n), y2(:,:,:,n), domain, gridtype=CGRID_NE, &
-                                bufferx=bufferx2(:,n), buffery=buffery2(:,n), complete=.true., tile_count=n )
+    id = mpp_clock_id( type//' CGRID_NE', flags=MPP_CLOCK_SYNC+MPP_CLOCK_DETAILED )
+    call mpp_clock_begin(id)
+    do n = 1, ntile_per_pe
+       call mpp_update_domains( x(:,:,:,n), y(:,:,:,n), domain, gridtype=CGRID_NE, tile_count = n )
     end do
+    call mpp_clock_end(id)
 
-    !--- fill up the value at halo points and compare the value at buffer.
-    do n = 1, ntiles_on_pe  
-
-       !--- comparing the buffer.
-       noverlap1 = mpp_get_refine_overlap_number(domain, tile_count=n)
-       call mpp_get_mosaic_refine_overlap(domain, isMe1, ieMe1, jsMe1, jeMe1, isNb1, ieNb1, jsNb1, jeNb1, &
-                                          dirMe1, rotation1, tile_count = n, position = EAST)             
-       pos = 0
-       do m = 1, noverlap1 
-          select case( rotation1(m) )
-          case (ZERO)     
-             do k = 1, nz
-                do j = jsNb1(m), jeNb1(m)
-                   do i = isNb1(m), ieNb1(m)
-                      pos = pos + 1
-                      if(global1_all(i,j,k,from_tile1(m,n)) .NE. bufferx(pos,n) ) then
-                         write(stdunit,111)'x','CGRID '//type,mpp_pe(),i,j,k,bufferx(pos,n),global1_all(i,j,k,from_tile1(m,n))
-                         call mpp_error(FATAL,"test_refined_mosaic: mismatch between buffer and actual data: CGRID "//type//" X")
-                      end if
-                      if(global1_all(i,j,k,from_tile1(m,n)) .NE. bufferx1(pos,n) ) then
-                         write(stdunit,111)'x1','CGRID '//type,mpp_pe(),i,j,k,bufferx1(pos,n),global1_all(i,j,k,from_tile1(m,n))
-                         call mpp_error(FATAL,"test_refined_mosaic: mismatch between buffer and actual data: CGRID "//type//" X1")
-                      end if
-                      if(global1_all(i,j,k,from_tile1(m,n)) .NE. bufferx2(pos,n) ) then
-                         write(stdunit,111)'x2','CGRID '//type,mpp_pe(),i,j,k,bufferx2(pos,n),global1_all(i,j,k,from_tile1(m,n))
-                         call mpp_error(FATAL,"test_refined_mosaic: mismatch between buffer and actual data: CGRID "//type//" X2")
-                      end if
-                   end do
-                end do
-             end do
-          case (NINETY) ! S->E, N->W, u->-v, v->u    
-             do k = 1, nz
-                do j = jsNb1(m), jeNb1(m)
-                   do i = isNb1(m), ieNb1(m)
-                      pos = pos + 1
-                      if(global2_all(i,j,k,from_tile1(m,n)) .NE. bufferx(pos,n) ) then
-                         write(stdunit,111)'x','CGRID '//type,mpp_pe(),i,j,k,bufferx(pos,n),global2_all(i,j,k,from_tile1(m,n))
-                         call mpp_error(FATAL,"test_refined_mosaic: mismatch between buffer and actual data: CGRID "//type//" X")
-                      end if
-                      if(global2_all(i,j,k,from_tile1(m,n)) .NE. bufferx1(pos,n) ) then
-                         write(stdunit,111)'x1','CGRID '//type,mpp_pe(),i,j,k,bufferx1(pos,n),global2_all(i,j,k,from_tile1(m,n))
-                         call mpp_error(FATAL,"test_refined_mosaic: mismatch between buffer and actual data: CGRID "//type//" X1")
-                      end if
-                      if(global2_all(i,j,k,from_tile1(m,n)) .NE. bufferx2(pos,n) ) then
-                         write(stdunit,111)'x2','CGRID '//type,mpp_pe(),i,j,k,bufferx2(pos,n),global2_all(i,j,k,from_tile1(m,n))
-                         call mpp_error(FATAL,"test_refined_mosaic: mismatch between buffer and actual data: CGRID "//type//" X2")
-                      end if
-                   end do
-                end do
-             end do
-          case (MINUS_NINETY) ! S->E, N->W, u->-v, v->u    
-             do k = 1, nz
-                do j = jsNb1(m), jeNb1(m)
-                   do i = isNb1(m), ieNb1(m)
-                      pos = pos + 1
-                      if(-global2_all(i,j,k,from_tile1(m,n)) .NE. bufferx(pos,n) ) then
-                         write(stdunit,111)'x','CGRID '//type,mpp_pe(),i,j,k,bufferx(pos,n),-global2_all(i,j,k,from_tile1(m,n))
-                         call mpp_error(FATAL,"test_refined_mosaic: mismatch between buffer and actual data: CGRID "//type//" X")
-                      end if
-                      if(-global2_all(i,j,k,from_tile1(m,n)) .NE. bufferx1(pos,n) ) then
-                         write(stdunit,111)'x1','CGRID '//type,mpp_pe(),i,j,k,bufferx1(pos,n),-global2_all(i,j,k,from_tile1(m,n))
-                         call mpp_error(FATAL,"test_refined_mosaic: mismatch between buffer and actual data: CGRID "//type//" X1")
-                      end if
-                      if(-global2_all(i,j,k,from_tile1(m,n)) .NE. bufferx2(pos,n) ) then
-                         write(stdunit,111)'x2','CGRID '//type,mpp_pe(),i,j,k,bufferx2(pos,n),-global2_all(i,j,k,from_tile1(m,n))
-                         call mpp_error(FATAL,"test_refined_mosaic: mismatch between buffer and actual data: CGRID "//type//" X2")
-                      end if
-                   end do
-                end do
-             end do
-          end select
-       end do
+   do n = 1, ntile_per_pe
+      write(type2, *)type, " at tile_count = ",n
+      call compare_checksums( x(isd:ied+shift,jsd:jed,:,n), global1(isd:ied+shift,jsd:jed,:,n), &
+                              trim(type2)//' CGRID_NE X')
+      call compare_checksums( y(isd:ied,jsd:jed+shift,:,n), global2(isd:ied,jsd:jed+shift,:,n), &
+                              trim(type2)//' CGRID_NE Y')
+   end do
 
-       call mpp_get_mosaic_refine_overlap(domain, isMe1, ieMe1, jsMe1, jeMe1, isNb1, ieNb1, jsNb1, jeNb1, &
-            dirMe1, rotation1, tile_count = n, position = NORTH) 
-       pos = 0
-       do m = 1, noverlap1
-          select case( rotation1(m) )
-          case (ZERO)               
-             do k = 1, nz  
-                do j = jsNb1(m), jeNb1(m)
-                   do i = isNb1(m), ieNb1(m)
-                      pos = pos + 1
-                      if(global2_all(i,j,k,from_tile1(m,n)) .NE. buffery(pos,n) ) then
-                         write(stdunit,111)'y','CGRID '//type,mpp_pe(),i,j,k,buffery(pos,n),global2_all(i,j,k,from_tile1(m,n))
-                         call mpp_error(FATAL,"test_refined_mosaic: mismatch between buffer and actual data: CGRID "//type//" Y")
-                      end if
-                      if(global2_all(i,j,k,from_tile1(m,n)) .NE. buffery1(pos,n) ) then
-                         write(stdunit,111)'y1','CGRID '//type,mpp_pe(),i,j,k,buffery1(pos,n),global2_all(i,j,k,from_tile1(m,n))
-                         call mpp_error(FATAL,"test_refined_mosaic: mismatch between buffer and actual data: CGRID "//type//" Y1")
-                      end if
-                      if(global2_all(i,j,k,from_tile1(m,n)) .NE. buffery2(pos,n) ) then
-                         write(stdunit,111)'y2','CGRID '//type,mpp_pe(),i,j,k,buffery2(pos,n),global2_all(i,j,k,from_tile1(m,n))
-                         call mpp_error(FATAL,"test_refined_mosaic: mismatch between buffer and actual data: CGRID "//type//" Y2")
-                      end if
-                   end do
-                end do
-             end do
-          case (NINETY) ! S->E, N->W, u->-v, v->u    
-             do k = 1, nz
-                do j = jsNb1(m), jeNb1(m)
-                   do i = isNb1(m), ieNb1(m)
-                      pos = pos + 1
-                      if(-global1_all(i,j,k,from_tile1(m,n)) .NE. buffery(pos,n) ) then
-                         write(stdunit,111)'y','CGRID '//type,mpp_pe(),i,j,k,buffery(pos,n),-global1_all(i,j,k,from_tile1(m,n))
-                         call mpp_error(FATAL,"test_refined_mosaic: mismatch between buffer and actual data: CGRID "//type//" Y")
-                      end if
-                      if(-global1_all(i,j,k,from_tile1(m,n)) .NE. buffery1(pos,n) ) then
-                         write(stdunit,111)'y1','CGRID '//type,mpp_pe(),i,j,k,buffery1(pos,n),-global1_all(i,j,k,from_tile1(m,n))
-                         call mpp_error(FATAL,"test_refined_mosaic: mismatch between buffer and actual data: CGRID "//type//" Y1")
-                      end if
-                      if(-global1_all(i,j,k,from_tile1(m,n)) .NE. buffery2(pos,n) ) then
-                         write(stdunit,111)'y2','CGRID '//type,mpp_pe(),i,j,k,buffery2(pos,n),-global1_all(i,j,k,from_tile1(m,n))
-                         call mpp_error(FATAL,"test_refined_mosaic: mismatch between buffer and actual data: CGRID "//type//" Y2")
-                      end if
-                   end do
-                end do
-             end do
-          case (MINUS_NINETY) ! S->E, N->W, u->-v, v->u    
-             do k = 1, nz
-                do j = jsNb1(m), jeNb1(m)
-                   do i = isNb1(m), ieNb1(m)
-                      pos = pos + 1
-                      if(global1_all(i,j,k,from_tile1(m,n)) .NE. buffery(pos,n) ) then
-                         write(stdunit,111)'y','CGRID '//type,mpp_pe(),i,j,k,buffery(pos,n),global1_all(i,j,k,from_tile1(m,n))
-                         call mpp_error(FATAL,"test_refined_mosaic: mismatch between buffer and actual data: CGRID "//type//" Y")
-                      end if
-                      if(global1_all(i,j,k,from_tile1(m,n)) .NE. buffery1(pos,n) ) then
-                         write(stdunit,111)'y1','CGRID '//type,mpp_pe(),i,j,k,buffery1(pos,n),global1_all(i,j,k,from_tile1(m,n))
-                         call mpp_error(FATAL,"test_refined_mosaic: mismatch between buffer and actual data: CGRID "//type//" Y1")
-                      end if
-                      if(global1_all(i,j,k,from_tile1(m,n)) .NE. buffery2(pos,n) ) then
-                         write(stdunit,111)'y2','CGRID '//type,mpp_pe(),i,j,k,buffery2(pos,n),global1_all(i,j,k,from_tile1(m,n))
-                         call mpp_error(FATAL,"test_refined_mosaic: mismatch between buffer and actual data: CGRID "//type//" Y2")
-                      end if
-                   end do
-                end do
-             end do
-          end select
-       end do
-       !--- fill the halo data and compare
-       select case(type)
-       case('Refined-Four-Tile', 'Refined-Symmetric-Four-Tile')
-          te = 0; ts = 0; tn = 0; tw = 0
-          select case(tiles(n))       
-          case(1)
-             tsw = 4 
-          case(2)
-             tsw = 3; tn = 4; ts = 4
-          case(3)
-             tsw = 2
-          case(4)
-             tsw = 1; ts = 2; tn = 2
-          end select
-          tse = tsw; tnw = tsw; tne = tsw
-          call fill_regular_refinement_halo( global1(:,:,:,n), global1_all, ni, nj, tiles(n), &
-               te, tse, ts, tsw, tw, tnw, tn, tne, shift, 0 )
-          call fill_regular_refinement_halo( global2(:,:,:,n), global2_all, ni, nj, tiles(n), &
-               te, tse, ts, tsw, tw, tnw, tn, tne, 0, shift )
-       case('Refined-Cubic-Grid')       
-          call fill_cubicgrid_refined_halo(global1(:,:,:,n), global1_all, global2_all, ni, nj, tiles(n), 1, 0, 1, -1 )
-          call fill_cubicgrid_refined_halo(global2(:,:,:,n), global2_all, global1_all, ni, nj, tiles(n), 0, 1, -1, 1 )
-       end select
+   deallocate(global1_all, global2_all, global1, global2, x, y)
 
-       call mpp_get_data_domain   ( domain, isd, ied, jsd, jed )
-       write(type2, *)"CGRID ", type
-       if(ntiles_on_pe>1) write(type2, *)trim(type2), " at tile_count = ", tiles(n)
-       call compare_checksums( x (isd:ied+shift,jsd:jed,:,n), global1(isd:ied+shift,jsd:jed,:,n), trim(type2)//' X' )
-       call compare_checksums( x1(isd:ied+shift,jsd:jed,:,n), global1(isd:ied+shift,jsd:jed,:,n), trim(type2)//' X1')
-       call compare_checksums( x2(isd:ied+shift,jsd:jed,:,n), global1(isd:ied+shift,jsd:jed,:,n), trim(type2)//' X2')
-       write(type2, *)"CGRID ", type
-       if(ntiles_on_pe>1) write(type2, *)trim(type2), " at tile_count = ", tiles(n)
-       call compare_checksums( y (isd:ied,jsd:jed+shift,:,n), global2(isd:ied,jsd:jed+shift,:,n), trim(type2)//' Y' )
-       call compare_checksums( y1(isd:ied,jsd:jed+shift,:,n), global2(isd:ied,jsd:jed+shift,:,n), trim(type2)//' Y1' )
-       call compare_checksums( y2(isd:ied,jsd:jed+shift,:,n), global2(isd:ied,jsd:jed+shift,:,n), trim(type2)//' Y2' )
-    end do  
+  end subroutine test_nonuniform_mosaic
+
+  subroutine fill_five_tile_halo(data, data_all, tile, ioff, joff)
+    real, dimension(1-whalo:,1-shalo:,:), intent(inout) :: data
+    real, dimension(:,:,:,:),             intent(in)    :: data_all
+    integer,                              intent(in)    :: tile, ioff, joff
+    integer                                             :: nxm, nym
 
-  111 format('For variable ', a, ', type = ', a, ', at pe = ', i3, ', at neighbor point (',i3,',',i3,',',i3, &
-             '), failed value = ', f14.9, ', but the value should be ', f14.9 )
+    nxm = 2*nx; nym = 2*ny
 
+    select case(tile)
+    case(1)
+       data(nxm+1+ioff:nxm+ehalo+ioff,                     1:ny,:) = data_all(1+ioff:ehalo+ioff,              1:ny,:,2) ! east
+       data(nxm+1+ioff:nxm+ehalo+ioff,            ny+1:nym+joff,:) = data_all(1+ioff:ehalo+ioff,         1:ny+joff,:,4) ! east
+       data(1-whalo:0,                                     1:ny,:) = data_all(nx-whalo+1:nx,                  1:ny,:,3) ! west
+       data(1-whalo:0,                            ny+1:nym+joff,:) = data_all(nx-whalo+1:nx,             1:ny+joff,:,5) ! west
+       data(1:nxm+ioff,                               1-shalo:0,:) = data_all(1:nxm+ioff,          nym-shalo+1:nym,:,1) ! south
+       data(1:nxm+ioff,               nym+1+joff:nym+nhalo+joff,:) = data_all(1:nxm+ioff,        1+joff:nhalo+joff,:,1) ! north
+       data(nxm+1+ioff:nxm+ehalo+ioff,                1-shalo:0,:) = data_all(1+ioff:ehalo+ioff,     ny-shalo+1:ny,:,4) ! southeast
+       data(1-whalo:0,                                1-shalo:0,:) = data_all(nx-whalo+1:nx,         ny-shalo+1:ny,:,5) ! southwest
+       data(nxm+1+ioff:nxm+ehalo+ioff,nym+1+joff:nym+nhalo+joff,:) = data_all(1+ioff:ehalo+ioff, 1+joff:nhalo+joff,:,2) ! northeast
+       data(1-whalo:0,                nym+1+joff:nym+nhalo+joff,:) = data_all(nx-whalo+1:nx,     1+joff:nhalo+joff,:,3) ! northwest
+    case(2)
+       data(nx+1+ioff:nx+ehalo+ioff,              1:ny+joff,:) = data_all(1+ioff:ehalo+ioff,              1:ny+joff,:,3) ! east
+       data(1-whalo:0,                            1:ny+joff,:) = data_all(nxm-whalo+1:nxm,                1:ny+joff,:,1) ! west
+       data(1:nx+ioff,                            1-shalo:0,:) = data_all(1:nx+ioff,                  ny-shalo+1:ny,:,4) ! south 
+       data(1:nx+ioff,              ny+1+joff:ny+nhalo+joff,:) = data_all(1:nx+ioff,              1+joff:nhalo+joff,:,4) ! north
+       data(nx+1+ioff:nx+ehalo+ioff,              1-shalo:0,:) = data_all(1+ioff:ehalo+ioff,          ny-shalo+1:ny,:,5) ! southeast
+       data(1-whalo:0,                            1-shalo:0,:) = data_all(nxm-whalo+1:nxm,          nym-shalo+1:nym,:,1) ! southwest
+       data(nx+1+ioff:nx+ehalo+ioff,ny+1+joff:ny+nhalo+joff,:) = data_all(1+ioff:ehalo+ioff,      1+joff:nhalo+joff,:,5) ! northeast
+       data(1-whalo:0,              ny+1+joff:ny+nhalo+joff,:) = data_all(nxm-whalo+1:nxm,  ny+1+joff:ny+nhalo+joff,:,1) ! northwest
+    case(3)
+       data(nx+1+ioff:nx+ehalo+ioff,              1:ny+joff,:) = data_all(1+ioff:ehalo+ioff,              1:ny+joff,:,1) ! east
+       data(1-whalo:0,                            1:ny+joff,:) = data_all(nx-whalo+1:nx,                  1:ny+joff,:,2) ! west
+       data(1:nx+ioff,                            1-shalo:0,:) = data_all(1:nx+ioff,                  ny-shalo+1:ny,:,5) ! south 
+       data(1:nx+ioff,              ny+1+joff:ny+nhalo+joff,:) = data_all(1:nx+ioff,              1+joff:nhalo+joff,:,5) ! north
+       data(nx+1+ioff:nx+ehalo+ioff,              1-shalo:0,:) = data_all(1+ioff:ehalo+ioff,        nym-shalo+1:nym,:,1) ! southeast
+       data(1-whalo:0,                            1-shalo:0,:) = data_all(nx-whalo+1:nx,              ny-shalo+1:ny,:,4) ! southwest
+       data(nx+1+ioff:nx+ehalo+ioff,ny+1+joff:ny+nhalo+joff,:) = data_all(1+ioff:ehalo+ioff,ny+1+joff:ny+nhalo+joff,:,1) ! northeast
+       data(1-whalo:0,              ny+1+joff:ny+nhalo+joff,:) = data_all(nx-whalo+1:nx,          1+joff:nhalo+joff,:,4) ! northwest
+    case(4)
+       data(nx+1+ioff:nx+ehalo+ioff,              1:ny+joff,:) = data_all(1+ioff:ehalo+ioff,        1:ny+joff,:,5) ! east
+       data(1-whalo:0,                            1:ny+joff,:) = data_all(nxm-whalo+1:nxm,     ny+1:2*ny+joff,:,1) ! west
+       data(1:nx+ioff,                            1-shalo:0,:) = data_all(1:nx+ioff,            ny-shalo+1:ny,:,2) ! south 
+       data(1:nx+ioff,              ny+1+joff:ny+nhalo+joff,:) = data_all(1:nx+ioff,        1+joff:nhalo+joff,:,2) ! north
+       data(nx+1+ioff:nx+ehalo+ioff,              1-shalo:0,:) = data_all(1+ioff:ehalo+ioff,    ny-shalo+1:ny,:,3) ! southeast
+       data(1-whalo:0,                            1-shalo:0,:) = data_all(nxm-whalo+1:nxm,      ny-shalo+1:ny,:,1) ! southwest
+       data(nx+1+ioff:nx+ehalo+ioff,ny+1+joff:ny+nhalo+joff,:) = data_all(1+ioff:ehalo+ioff,1+joff:nhalo+joff,:,3) ! northeast
+       data(1-whalo:0,              ny+1+joff:ny+nhalo+joff,:) = data_all(nxm-whalo+1:nxm,  1+joff:nhalo+joff,:,1) ! northwest
+    case(5)
+       data(nx+1+ioff:nx+ehalo+ioff,            1:  ny+joff,:) = data_all(1+ioff:ehalo+ioff,   ny+1:2*ny+joff,:,1) ! east
+       data(1-whalo:0,                            1:ny+joff,:) = data_all(nx-whalo+1:nx,            1:ny+joff,:,4) ! west
+       data(1:nx+ioff,                            1-shalo:0,:) = data_all(1:nx+ioff,            ny-shalo+1:ny,:,3) ! south 
+       data(1:nx+ioff,              ny+1+joff:ny+nhalo+joff,:) = data_all(1:nx+ioff,        1+joff:nhalo+joff,:,3) ! north
+       data(nx+1+ioff:nx+ehalo+ioff,              1-shalo:0,:) = data_all(1+ioff:ehalo+ioff,    ny-shalo+1:ny,:,1) ! southeast
+       data(1-whalo:0,                            1-shalo:0,:) = data_all(nx-whalo+1:nx,        ny-shalo+1:ny,:,2) ! southwest
+       data(nx+1+ioff:nx+ehalo+ioff,ny+1+joff:ny+nhalo+joff,:) = data_all(1+ioff:ehalo+ioff,1+joff:nhalo+joff,:,1) ! northeast
+       data(1-whalo:0,              ny+1+joff:ny+nhalo+joff,:) = data_all(nx-whalo+1:nx,    1+joff:nhalo+joff,:,2) ! northwest
+    end select
 
-  end subroutine test_refined_mosaic
+  end subroutine fill_five_tile_halo
 
   !#######################################################################################
   subroutine test_get_boundary(type)
      character(len=*), intent(in)  :: type
 
-     type(domain2D)       :: domain
+     type(domain2D)       :: domain, domain_nonsym
      integer              :: ntiles, num_contact, npes_per_tile, ntile_per_pe, layout(2)
      integer              :: n, l, isc, iec, jsc, jec, ism, iem, jsm, jem
      integer, allocatable, dimension(:)       :: tile, ni, nj, pe_start, pe_end
@@ -3963,7 +3649,9 @@ subroutine test_get_boundary(type)
      real,    allocatable, dimension(:,:,:,:) :: global_all, global1_all, global2_all
      real,    allocatable, dimension(:,:,:,:) :: global, global1, global2
      real,    allocatable, dimension(:,:,:,:) :: x, x1, x2, y, y1, y2
+     real,    allocatable, dimension(:,:)     :: u_nonsym, v_nonsym
      logical    :: folded_north = .false.
+     logical    :: is_torus = .false.
      integer    :: nx_save, ny_save
 
      nx_save    = nx
@@ -3982,6 +3670,9 @@ subroutine test_get_boundary(type)
     case ( 'Folded-north' )
        folded_north = .true.
        ntiles = 1       
+    case ( 'torus' )
+       is_torus = .true.
+       ntiles = 1
     case default
        call mpp_error(FATAL, 'TEST_MPP_DOMAINS: no such test: '//type)
     end select
@@ -4036,7 +3727,14 @@ subroutine test_get_boundary(type)
                           xflags=CYCLIC_GLOBAL_DOMAIN, yflags=FOLD_NORTH_EDGE, &
                           whalo=whalo, ehalo=ehalo, shalo=shalo, nhalo=nhalo, &
                           symmetry=.true., name='tripolar' )
-
+        call mpp_define_domains((/1,nx,1,ny/), layout, domain_nonsym, &
+                          xflags=CYCLIC_GLOBAL_DOMAIN, yflags=FOLD_NORTH_EDGE, &
+                          whalo=whalo, ehalo=ehalo, shalo=shalo, nhalo=nhalo, &
+                          symmetry=.false., name='tripolar' )        
+     case("torus")
+        call mpp_define_domains( (/1,nx,1,ny/), layout, domain, whalo=whalo, ehalo=ehalo, &
+                                 shalo=shalo, nhalo=nhalo, xflags=CYCLIC_GLOBAL_DOMAIN,   &
+                                 yflags=CYCLIC_GLOBAL_DOMAIN, symmetry=.true., name=type)        
      end select
 
     !--- Test the get_boundary of the data at C-cell center. 
@@ -4067,21 +3765,21 @@ subroutine test_get_boundary(type)
     x1 = x; x2 = x*10
 
     !--- buffer allocation
-    allocate(ebuffer(jec-jsc+2, nz, ntile_per_pe), wbuffer(jec-jsc+2, nz, ntile_per_pe))
-    allocate(sbuffer(iec-isc+2, nz, ntile_per_pe), nbuffer(iec-isc+2, nz, ntile_per_pe))
-    allocate(ebuffer1(jec-jsc+2, nz, ntile_per_pe), wbuffer1(jec-jsc+2, nz, ntile_per_pe))
-    allocate(sbuffer1(iec-isc+2, nz, ntile_per_pe), nbuffer1(iec-isc+2, nz, ntile_per_pe))
-    allocate(ebuffer2(jec-jsc+2, nz, ntile_per_pe), wbuffer2(jec-jsc+2, nz, ntile_per_pe))
-    allocate(sbuffer2(iec-isc+2, nz, ntile_per_pe), nbuffer2(iec-isc+2, nz, ntile_per_pe))
-    allocate(ebound(jec-jsc+2, nz, ntile_per_pe), wbound(jec-jsc+2, nz, ntile_per_pe))
-    allocate(sbound(iec-isc+2, nz, ntile_per_pe), nbound(iec-isc+2, nz, ntile_per_pe))
+    allocate(ebuffer(jsc:jec+1, nz, ntile_per_pe), wbuffer(jsc:jec+1, nz, ntile_per_pe))
+    allocate(sbuffer(isc:iec+1, nz, ntile_per_pe), nbuffer(isc:iec+1, nz, ntile_per_pe))
+    allocate(ebuffer1(jsc:jec+1, nz, ntile_per_pe), wbuffer1(jsc:jec+1, nz, ntile_per_pe))
+    allocate(sbuffer1(isc:iec+1, nz, ntile_per_pe), nbuffer1(isc:iec+1, nz, ntile_per_pe))
+    allocate(ebuffer2(jsc:jec+1, nz, ntile_per_pe), wbuffer2(jsc:jec+1, nz, ntile_per_pe))
+    allocate(sbuffer2(isc:iec+1, nz, ntile_per_pe), nbuffer2(isc:iec+1, nz, ntile_per_pe))
+    allocate(ebound(jsc:jec+1, nz, ntile_per_pe), wbound(jsc:jec+1, nz, ntile_per_pe))
+    allocate(sbound(isc:iec+1, nz, ntile_per_pe), nbound(isc:iec+1, nz, ntile_per_pe))
     ebound  = 0; ebuffer = 0; ebuffer1 = 0; ebuffer2 = 0
     sbound  = 0; sbuffer = 0; sbuffer1 = 0; sbuffer2 = 0
     wbound  = 0; wbuffer = 0; wbuffer1 = 0; wbuffer2 = 0
     nbound  = 0; nbuffer = 0; nbuffer1 = 0; nbuffer2 = 0
 
     do n = 1, ntile_per_pe 
-       if(folded_north) then
+       if(folded_north .or. is_torus ) then
           call mpp_get_boundary(x(:,:,:,n), domain, sbuffer=sbuffer(:,:,n), wbuffer=wbuffer(:,:,n), &
                                 position=CORNER, tile_count=n  )
        else
@@ -4092,7 +3790,7 @@ subroutine test_get_boundary(type)
 
     !--- multiple variable 
     do n = 1, ntile_per_pe 
-       if(folded_north) then
+       if(folded_north .or. is_torus) then
           call mpp_get_boundary(x1(:,:,:,n), domain, sbuffer=sbuffer1(:,:,n), wbuffer=wbuffer1(:,:,n), &
                position=CORNER, tile_count=n, complete = .false.  )
           call mpp_get_boundary(x2(:,:,:,n), domain, sbuffer=sbuffer2(:,:,n), wbuffer=wbuffer2(:,:,n), &
@@ -4118,13 +3816,20 @@ subroutine test_get_boundary(type)
                tile(n), 1, 1, ebound(:,:,n), sbound(:,:,n), wbound(:,:,n), nbound(:,:,n) )  
        end do
     case("Folded-north")
+       !---- folded line update
+       global_all(nx/2+2:nx, ny+1,:,1) = global_all(nx/2:2:-1, ny+1,:,1)
        do n = 1, ntile_per_pe
           call fill_folded_north_bound(global_all(:,:,:,1), isc, iec, jsc, jec, 1, 1, &
                tile(n), sbound(:,:,n), wbound(:,:,n) )  
        end do       
+    case("torus")
+       do n = 1, ntile_per_pe
+          call fill_torus_bound(global_all(:,:,:,1), isc, iec, jsc, jec, 1, 1, &
+               tile(n), sbound(:,:,n), wbound(:,:,n) )  
+       end do                    
     end select
 
-    if(.not. folded_north) then
+    if(.not. folded_north .AND. .not. is_torus) then
        call compare_checksums( ebound, ebuffer(:,:,:),  "east bound of "//trim(type) )
        call compare_checksums( nbound, nbuffer(:,:,:),  "north bound of "//trim(type) ) 
        call compare_checksums( ebound, ebuffer1(:,:,:),  "east bound of "//trim(type)//" X1" ) 
@@ -4177,28 +3882,33 @@ subroutine test_get_boundary(type)
     allocate( y1(ism:iem+1,jsm:jem+1,nz, ntile_per_pe) )
     allocate( y2(ism:iem+1,jsm:jem+1,nz, ntile_per_pe) )
     x = 0.; y = 0
-    x(isc:iec+1,jsc:jec+1,:,:) = global1(isc:iec+1,jsc:jec+1,:,:)
-    y(isc:iec+1,jsc:jec+1,:,:) = global2(isc:iec+1,jsc:jec+1,:,:)
+    if( trim(type) == "Folded-north" ) then
+       x(isc+1:iec+1,jsc+1:jec+1,:,:) = global1(isc+1:iec+1,jsc+1:jec+1,:,:)
+       y(isc+1:iec+1,jsc+1:jec+1,:,:) = global2(isc+1:iec+1,jsc+1:jec+1,:,:)
+    else
+       x(isc:iec+1,jsc:jec+1,:,:) = global1(isc:iec+1,jsc:jec+1,:,:)
+       y(isc:iec+1,jsc:jec+1,:,:) = global2(isc:iec+1,jsc:jec+1,:,:)
+    endif
     x1 = x; x2 = x*10
     y1 = y; y2 = y*10
 
     !--- buffer allocation
-    allocate(ebufferx(jec-jsc+2, nz, ntile_per_pe), wbufferx(jec-jsc+2, nz, ntile_per_pe))
-    allocate(sbufferx(iec-isc+2, nz, ntile_per_pe), nbufferx(iec-isc+2, nz, ntile_per_pe))
-    allocate(ebufferx1(jec-jsc+2, nz, ntile_per_pe), wbufferx1(jec-jsc+2, nz, ntile_per_pe))
-    allocate(sbufferx1(iec-isc+2, nz, ntile_per_pe), nbufferx1(iec-isc+2, nz, ntile_per_pe))
-    allocate(ebufferx2(jec-jsc+2, nz, ntile_per_pe), wbufferx2(jec-jsc+2, nz, ntile_per_pe))
-    allocate(sbufferx2(iec-isc+2, nz, ntile_per_pe), nbufferx2(iec-isc+2, nz, ntile_per_pe))
-    allocate(eboundx(jec-jsc+2, nz, ntile_per_pe), wboundx(jec-jsc+2, nz, ntile_per_pe))
-    allocate(sboundx(iec-isc+2, nz, ntile_per_pe), nboundx(iec-isc+2, nz, ntile_per_pe))
-    allocate(ebuffery(jec-jsc+2, nz, ntile_per_pe), wbuffery(jec-jsc+2, nz, ntile_per_pe))
-    allocate(sbuffery(iec-isc+2, nz, ntile_per_pe), nbuffery(iec-isc+2, nz, ntile_per_pe))
-    allocate(ebuffery1(jec-jsc+2, nz, ntile_per_pe), wbuffery1(jec-jsc+2, nz, ntile_per_pe))
-    allocate(sbuffery1(iec-isc+2, nz, ntile_per_pe), nbuffery1(iec-isc+2, nz, ntile_per_pe))
-    allocate(ebuffery2(jec-jsc+2, nz, ntile_per_pe), wbuffery2(jec-jsc+2, nz, ntile_per_pe))
-    allocate(sbuffery2(iec-isc+2, nz, ntile_per_pe), nbuffery2(iec-isc+2, nz, ntile_per_pe))
-    allocate(eboundy(jec-jsc+2, nz, ntile_per_pe), wboundy(jec-jsc+2, nz, ntile_per_pe))
-    allocate(sboundy(iec-isc+2, nz, ntile_per_pe), nboundy(iec-isc+2, nz, ntile_per_pe))
+    allocate(ebufferx(jsc:jec+1, nz, ntile_per_pe), wbufferx(jsc:jec+1, nz, ntile_per_pe))
+    allocate(sbufferx(isc:iec+1, nz, ntile_per_pe), nbufferx(isc:iec+1, nz, ntile_per_pe))
+    allocate(ebufferx1(jsc:jec+1, nz, ntile_per_pe), wbufferx1(jsc:jec+1, nz, ntile_per_pe))
+    allocate(sbufferx1(isc:iec+1, nz, ntile_per_pe), nbufferx1(isc:iec+1, nz, ntile_per_pe))
+    allocate(ebufferx2(jsc:jec+1, nz, ntile_per_pe), wbufferx2(jsc:jec+1, nz, ntile_per_pe))
+    allocate(sbufferx2(isc:iec+1, nz, ntile_per_pe), nbufferx2(isc:iec+1, nz, ntile_per_pe))
+    allocate(eboundx(jsc:jec+1, nz, ntile_per_pe), wboundx(jsc:jec+1, nz, ntile_per_pe))
+    allocate(sboundx(isc:iec+1, nz, ntile_per_pe), nboundx(isc:iec+1, nz, ntile_per_pe))
+    allocate(ebuffery(jsc:jec+1, nz, ntile_per_pe), wbuffery(jsc:jec+1, nz, ntile_per_pe))
+    allocate(sbuffery(isc:iec+1, nz, ntile_per_pe), nbuffery(isc:iec+1, nz, ntile_per_pe))
+    allocate(ebuffery1(jsc:jec+1, nz, ntile_per_pe), wbuffery1(jsc:jec+1, nz, ntile_per_pe))
+    allocate(sbuffery1(isc:iec+1, nz, ntile_per_pe), nbuffery1(isc:iec+1, nz, ntile_per_pe))
+    allocate(ebuffery2(jsc:jec+1, nz, ntile_per_pe), wbuffery2(jsc:jec+1, nz, ntile_per_pe))
+    allocate(sbuffery2(isc:iec+1, nz, ntile_per_pe), nbuffery2(isc:iec+1, nz, ntile_per_pe))
+    allocate(eboundy(jsc:jec+1, nz, ntile_per_pe), wboundy(jsc:jec+1, nz, ntile_per_pe))
+    allocate(sboundy(isc:iec+1, nz, ntile_per_pe), nboundy(isc:iec+1, nz, ntile_per_pe))
     eboundx  = 0; ebufferx = 0; ebufferx1 = 0; ebufferx2 = 0
     sboundx  = 0; sbufferx = 0; sbufferx1 = 0; sbufferx2 = 0
     wboundx  = 0; wbufferx = 0; wbufferx1 = 0; wbufferx2 = 0
@@ -4210,7 +3920,7 @@ subroutine test_get_boundary(type)
 
 
     do n = 1, ntile_per_pe 
-       if(folded_north) then
+       if(folded_north .or. is_torus) then
           call mpp_get_boundary(x(:,:,:,n), y(:,:,:,n), domain, sbufferx=sbufferx(:,:,n), wbufferx=wbufferx(:,:,n), &
                sbuffery=sbuffery(:,:,n), wbuffery=wbuffery(:,:,n), gridtype=BGRID_NE, tile_count=n, flags = SCALAR_PAIR  )
        else
@@ -4222,7 +3932,7 @@ subroutine test_get_boundary(type)
     end do
 
     do n = 1, ntile_per_pe 
-       if(folded_north) then
+       if(folded_north .or. is_torus) then
           call mpp_get_boundary(x1(:,:,:,n), y1(:,:,:,n), domain, sbufferx=sbufferx1(:,:,n), wbufferx=wbufferx1(:,:,n), &
                sbuffery=sbuffery1(:,:,n), wbuffery=wbuffery1(:,:,n),                           &
                gridtype=BGRID_NE, tile_count=n, flags = SCALAR_PAIR, complete = .false.  )
@@ -4258,15 +3968,24 @@ subroutine test_get_boundary(type)
                tile(n), 1, 1, eboundy(:,:,n), sboundy(:,:,n), wboundy(:,:,n), nboundy(:,:,n) )  
        end do
     case("Folded-north")
+       global1_all(nx/2+2:nx, ny+1,:,1) = global1_all(nx/2:2:-1, ny+1,:,1)
+       global2_all(nx/2+2:nx, ny+1,:,1) = global2_all(nx/2:2:-1, ny+1,:,1)
        do n = 1, ntile_per_pe
           call fill_folded_north_bound(global1_all(:,:,:,1), isc, iec, jsc, jec, 1, 1, &
                tile(n), sboundx(:,:,n), wboundx(:,:,n) )
           call fill_folded_north_bound(global2_all(:,:,:,1), isc, iec, jsc, jec, 1, 1, &
                tile(n), sboundy(:,:,n), wboundy(:,:,n) )  
        end do       
+    case("torus")
+       do n = 1, ntile_per_pe
+          call fill_torus_bound(global1_all(:,:,:,1), isc, iec, jsc, jec, 1, 1, &
+               tile(n), sboundx(:,:,n), wboundx(:,:,n) )
+          call fill_torus_bound(global2_all(:,:,:,1), isc, iec, jsc, jec, 1, 1, &
+               tile(n), sboundy(:,:,n), wboundy(:,:,n) )  
+       end do       
     end select
 
-    if(.not. folded_north) then
+    if(.not. folded_north .AND. .not. is_torus ) then
        call compare_checksums( eboundx, ebufferx(:,:,:),   "east bound of SCALAR_PAIR BGRID " //trim(type)//" X" )
        call compare_checksums( nboundx, nbufferx(:,:,:),   "north bound of SCALAR_PAIR BGRID "//trim(type)//" X" )
        call compare_checksums( eboundy, ebuffery(:,:,:),   "east bound of SCALAR_PAIR BGRID " //trim(type)//" Y" )
@@ -4308,9 +4027,16 @@ subroutine test_get_boundary(type)
           call fill_folded_north_bound(global2_all(:,:,:,1)*10, isc, iec, jsc, jec, 1, 1, &
                tile(n), sboundy(:,:,n), wboundy(:,:,n) )  
        end do       
+    case("torus")
+       do n = 1, ntile_per_pe
+          call fill_torus_bound(global1_all(:,:,:,1)*10, isc, iec, jsc, jec, 1, 1, &
+               tile(n), sboundx(:,:,n), wboundx(:,:,n) )
+          call fill_torus_bound(global2_all(:,:,:,1)*10, isc, iec, jsc, jec, 1, 1, &
+               tile(n), sboundy(:,:,n), wboundy(:,:,n) )  
+       end do       
     end select
 
-    if(.not. folded_north) then
+    if(.not. folded_north .AND. .not. is_torus ) then
        call compare_checksums( eboundx, ebufferx2(:,:,:),  "east bound of SCALAR_PAIR BGRID " //trim(type)//" X2" )
        call compare_checksums( nboundx, nbufferx2(:,:,:),  "north bound of SCALAR_PAIR BGRID "//trim(type)//" X2" )
        call compare_checksums( eboundy, ebuffery2(:,:,:),  "east bound of SCALAR_PAIR BGRID " //trim(type)//" Y2" )
@@ -4326,6 +4052,17 @@ subroutine test_get_boundary(type)
     !             Test 2-D Vector BGRID
     !
     !-------------------------------------------------------------------------------------------
+    do l = 1, ntiles
+       do k = 1, nz
+          do j = 1, ny+1
+             do i = 1, nx+1
+                global1_all(i,j,k,l) = 1.0e3 + l + i*1.0e-3 + j*1.0e-6 + k*1.0e-9
+                global2_all(i,j,k,l) = 2.0e3 + l + i*1.0e-3 + j*1.0e-6 + k*1.0e-9
+             end do
+          end do
+       end do
+    end do
+	
     x = 0.; y = 0
     eboundx  = 0; ebufferx = 0; ebufferx1 = 0; ebufferx2 = 0
     sboundx  = 0; sbufferx = 0; sbufferx1 = 0; sbufferx2 = 0
@@ -4340,9 +4077,10 @@ subroutine test_get_boundary(type)
     y(isc:iec+1,jsc:jec+1,1,:) = global2(isc:iec+1,jsc:jec+1,1,:)
 
     do n = 1, ntile_per_pe 
-       if(folded_north) then
+       if(folded_north .or. is_torus ) then
           call mpp_get_boundary(x(:,:,1,n), y(:,:,1,n), domain, sbufferx=sbufferx(:,1,n), wbufferx=wbufferx(:,1,n), &
-               sbuffery=sbuffery(:,1,n), wbuffery=wbuffery(:,1,n), gridtype=BGRID_NE, tile_count=n)
+               sbuffery=sbuffery(:,1,n), wbuffery=wbuffery(:,1,n), gridtype=BGRID_NE, tile_count=n) 
+
        else
           call mpp_get_boundary(x(:,:,1,n), y(:,:,1,n), domain, ebufferx=ebufferx(:,1,n), sbufferx=sbufferx(:,1,n), &
                wbufferx=wbufferx(:,1,n), nbufferx=nbufferx(:,1,n), ebuffery=ebuffery(:,1,n),       &
@@ -4351,6 +4089,53 @@ subroutine test_get_boundary(type)
        endif
     end do
 
+    if(folded_north) then
+       allocate(u_nonsym(ism:iem,jsm:jem), v_nonsym(ism:iem,jsm:jem))
+       u_nonsym = 0.0; v_nonsym = 0.0
+       u_nonsym(isc:iec,jsc:jec) = global1(isc+1:iec+1,jsc+1:jec+1,1,1)
+       v_nonsym(isc:iec,jsc:jec) = global2(isc+1:iec+1,jsc+1:jec+1,1,1)
+       call mpp_update_domains(u_nonsym, v_nonsym, domain_nonsym, gridtype=BGRID_NE)
+       !--- comparing boundary data
+       do i = isc,iec+1
+          if(i==1) cycle
+          if(sbufferx(i,1,1) .NE. u_nonsym(i-1,jsc-1)) then
+             print*,"pe ", mpp_pe(), i, jsc-1, sbufferx(i,1,1), u_nonsym(i-1,jsc-1)
+             call mpp_error(FATAL, "test_get_boundary: mismatch of sbufferx")
+          endif
+       enddo
+       call mpp_error(NOTE,"test_get_boundary: reproduce non-symmetric halo update for sbufferx")
+
+       do i = isc,iec+1
+          if(i==1) cycle
+          if(sbuffery(i,1,1) .NE. v_nonsym(i-1,jsc-1)) then
+             print*,"pe ", mpp_pe(), i, jsc-1, sbufferx(i,1,1), v_nonsym(i-1,jsc-1)
+             call mpp_error(FATAL, "test_get_boundary: mismatch of sbuffery")
+          endif
+       enddo
+      call mpp_error(NOTE,"test_get_boundary: reproduce non-symmetric halo update for sbuffery")
+
+       do j = jsc,jec+1
+          if(j == 1) cycle
+          if(wbufferx(j,1,1) .NE. u_nonsym(isc-1,j-1)) then
+             print*,"pe ", mpp_pe(), isc-1, j, wbufferx(j,1,1), u_nonsym(isc-1,j-1)
+             call mpp_error(FATAL, "test_get_boundary: mismatch of wbufferx")
+          endif
+       enddo
+      call mpp_error(NOTE,"test_get_boundary: reproduce non-symmetric halo update for wbufferx")
+
+       do j = jsc,jec+1
+          if(j==1) cycle
+          if(wbuffery(j,1,1) .NE. v_nonsym(isc-1,j-1)) then
+             print*,"pe ", mpp_pe(), isc-1, j, wbuffery(j,1,1), v_nonsym(isc-1,j-1)
+             call mpp_error(FATAL, "test_get_boundary: mismatch of wbuffery")
+          endif
+       enddo
+       call mpp_error(NOTE,"test_get_boundary: reproduce non-symmetric halo update for wbuffery")
+
+       deallocate(u_nonsym, v_nonsym)
+
+    endif
+
     !--- compare the buffer.
     select case(type)
     case("Four-Tile")
@@ -4368,15 +4153,39 @@ subroutine test_get_boundary(type)
                tile(n), -1, 1, eboundy(:,:,n), sboundy(:,:,n), wboundy(:,:,n), nboundy(:,:,n) )  
        end do
     case("Folded-north")
+       global1_all(nx/2+2:nx, ny+1,:,1) = -global1_all(nx/2:2:-1, ny+1,:,1)
+       global2_all(nx/2+2:nx, ny+1,:,1) = -global2_all(nx/2:2:-1, ny+1,:,1)
+       global1_all(1, ny+1,:,1) = 0
+       global2_all(1, ny+1,:,1) = 0
+       global1_all(nx/2+1, ny+1,:,1) = 0
+       global2_all(nx/2+1, ny+1,:,1) = 0
+       global1_all(nx+1, ny+1,:,1) = 0
+       global2_all(nx+1, ny+1,:,1) = 0
+
+
        do n = 1, ntile_per_pe
           call fill_folded_north_bound(global1_all(:,:,:,1), isc, iec, jsc, jec, 1, 1, &
                tile(n), sboundx(:,:,n), wboundx(:,:,n) )
           call fill_folded_north_bound(global2_all(:,:,:,1), isc, iec, jsc, jec, 1, 1, &
                tile(n), sboundy(:,:,n), wboundy(:,:,n) )  
+          ! set wboundx and wbouny to zero at pole (i=1, nx/2+1, nx+1)
+!          if( jec == ny ) then
+!             if( isc == 1 .OR. isc == nx/2+1 .OR. isc == nx+1 ) then
+!                wboundx(jec+1,:,n) = 0
+!                wboundy(jec+1,:,n) = 0
+!             endif
+!         endif
        end do       
+    case("torus")
+       do n = 1, ntile_per_pe
+          call fill_torus_bound(global1_all(:,:,:,1), isc, iec, jsc, jec, 1, 1, &
+               tile(n), sboundx(:,:,n), wboundx(:,:,n) )
+          call fill_torus_bound(global2_all(:,:,:,1), isc, iec, jsc, jec, 1, 1, &
+               tile(n), sboundy(:,:,n), wboundy(:,:,n) )  
+       enddo
     end select
 
-    if(.not. folded_north) then
+    if(.not. folded_north .AND. .not. is_torus ) then
        call compare_checksums( eboundx(:,1:1,:), ebufferx(:,1:1,:),   "east bound of 2-D BGRID " //trim(type)//" X" )
        call compare_checksums( nboundx(:,1:1,:), nbufferx(:,1:1,:),   "north bound of 2-D BGRID "//trim(type)//" X" )
        call compare_checksums( eboundy(:,1:1,:), ebuffery(:,1:1,:),   "east bound of 2-D BGRID " //trim(type)//" Y" )
@@ -4463,7 +4272,7 @@ subroutine test_get_boundary(type)
 
 
     do n = 1, ntile_per_pe 
-       if(folded_north) then
+       if(folded_north .or. is_torus) then
           call mpp_get_boundary(x(:,:,:,n), y(:,:,:,n), domain, wbufferx=wbufferx(:,:,n), &
                sbuffery=sbuffery(:,:,n), gridtype=CGRID_NE, tile_count=n  )
        else
@@ -4473,7 +4282,7 @@ subroutine test_get_boundary(type)
     end do
 
     do n = 1, ntile_per_pe 
-       if( folded_north ) then
+       if( folded_north .or. is_torus ) then
           call mpp_get_boundary(x1(:,:,:,n), y1(:,:,:,n), domain, wbufferx=wbufferx1(:,:,n), &
                sbuffery=sbuffery1(:,:,n), gridtype=CGRID_NE, tile_count=n,  &
                complete = .false.  )
@@ -4508,14 +4317,21 @@ subroutine test_get_boundary(type)
        end do
     case("Folded-north")
        do n = 1, ntile_per_pe
-          call fill_folded_north_bound(global1_all(:,:,:,1), isc, iec, jsc, jec, 1, 1, &
+          call fill_folded_north_bound(global1_all(:,:,:,1), isc, iec, jsc, jec, 1, 0, &
                tile(n), wbound=wboundx(:,:,n) )
-          call fill_folded_north_bound(global2_all(:,:,:,1), isc, iec, jsc, jec, 1, 1, &
+          call fill_folded_north_bound(global2_all(:,:,:,1), isc, iec, jsc, jec, 0, 1, &
+               tile(n), sbound=sboundy(:,:,n) )  
+       end do       
+    case("torus")
+       do n = 1, ntile_per_pe
+          call fill_torus_bound(global1_all(:,:,:,1), isc, iec, jsc, jec, 1, 0, &
+               tile(n), wbound=wboundx(:,:,n) )
+          call fill_torus_bound(global2_all(:,:,:,1), isc, iec, jsc, jec, 0, 1, &
                tile(n), sbound=sboundy(:,:,n) )  
        end do       
     end select
 
-    if(.not. folded_north) then
+    if(.not. folded_north .and. .not. is_torus ) then
        call compare_checksums( eboundx, ebufferx(:,:,:),   "east bound of CGRID " //trim(type)//" X" )
        call compare_checksums( nboundy, nbuffery(:,:,:),   "north bound of CGRID "//trim(type)//" Y" )
        call compare_checksums( eboundx, ebufferx1(:,:,:),  "east bound of CGRID " //trim(type)//" X1" )
@@ -4543,14 +4359,21 @@ subroutine test_get_boundary(type)
        end do
     case("Folded-north")
        do n = 1, ntile_per_pe
-          call fill_folded_north_bound(global1_all(:,:,:,1)*10, isc, iec, jsc, jec, 1, 1, &
+          call fill_folded_north_bound(global1_all(:,:,:,1)*10, isc, iec, jsc, jec, 1, 0, &
                tile(n), wbound=wboundx(:,:,n) )
-          call fill_folded_north_bound(global2_all(:,:,:,1)*10, isc, iec, jsc, jec, 1, 1, &
+          call fill_folded_north_bound(global2_all(:,:,:,1)*10, isc, iec, jsc, jec, 0, 1, &
+               tile(n), sbound=sboundy(:,:,n) )  
+       end do       
+    case("torus")
+       do n = 1, ntile_per_pe
+          call fill_torus_bound(global1_all(:,:,:,1)*10, isc, iec, jsc, jec, 1, 0, &
+               tile(n), wbound=wboundx(:,:,n) )
+          call fill_torus_bound(global2_all(:,:,:,1)*10, isc, iec, jsc, jec, 0, 1, &
                tile(n), sbound=sboundy(:,:,n) )  
        end do       
     end select
 
-    if(.not. folded_north) then
+    if(.not. folded_north .and. .not. is_torus ) then
        call compare_checksums( eboundx, ebufferx2(:,:,:),  "east bound of CGRID " //trim(type)//" X2" )
        call compare_checksums( nboundy, nbuffery2(:,:,:),  "north bound of CGRID "//trim(type)//" Y2" )
     endif
diff --git a/src/shared/mpp/test_mpp_io.F90 b/src/shared/mpp/test_mpp_io.F90
index 6579ea0e15..7774f2e97e 100644
--- a/src/shared/mpp/test_mpp_io.F90
+++ b/src/shared/mpp/test_mpp_io.F90
@@ -16,6 +16,10 @@ program test
   use mpp_io_mod,      only : mpp_get_info, mpp_get_axes, mpp_get_fields, mpp_get_times
   use mpp_io_mod,      only : mpp_read, mpp_io_exit
 
+#ifdef INTERNAL_FILE_NML
+  USE mpp_mod, ONLY: input_nml_file
+#endif
+
   implicit none
 
 #ifdef use_netCDF
@@ -61,7 +65,7 @@ program test
   npes = mpp_npes()
 
 #ifdef INTERNAL_FILE_NML
-  read (input_nml_file, test_mpp_io_nml, status=io_status)
+  read (input_nml_file, test_mpp_io_nml, iostat=io_status)
 #else
   do
      inquire( unit=unit, opened=opened )
diff --git a/src/shared/oda_tools/oda_core.F90 b/src/shared/oda_tools/oda_core.F90
index 335c1999d2..2be3564dac 100644
--- a/src/shared/oda_tools/oda_core.F90
+++ b/src/shared/oda_tools/oda_core.F90
@@ -1,6 +1,6 @@
 module oda_core_mod
 !
-! Matthew Harrison
+! Matthew Harrison
 !
 !
 !
@@ -141,7 +141,7 @@ module oda_core_mod
 
   integer, allocatable, dimension(:) :: nprof_in_comp_domain
   
-  namelist /oda_core_nml/ max_misfit, add_tidal_aliasing, min_obs_err_t,&
+  namelist /oda_core_nml/ max_misfit,add_tidal_aliasing,min_obs_err_t,&
                           min_obs_err_s, eta_tide_const, debug, max_profiles,&
                           max_sfc_obs, temp_obs_rmse, salt_obs_rmse, ndebug
                           
@@ -643,8 +643,8 @@ subroutine open_profile_dataset(filename, localize)
 ! NOTE: this only works for lat/lon grids. Lower left indices.           
 !       
            
-           ri0 = frac_index(lon, x_grid(isg-1:ieg+1)) - 1 
-           rj0 = frac_index(lat, y_grid(jsg-1:jeg+1)) - 1
+           ri0 = frac_index(lon, x_grid(isg-1:ieg+1)) - 1.
+           rj0 = frac_index(lat, y_grid(jsg-1:jeg+1)) - 1.
            i0 = floor(ri0)
            j0 = floor(rj0)
            Profiles(num_profiles)%i_index = ri0
@@ -1147,14 +1147,14 @@ subroutine backward_obs_profile(Obs,model_t,model_s)
     end do
 
 
-    where(sum_wgt > 0)
+    where(sum_wgt > 0.0)
         model_t = model_t /sum_wgt
     elsewhere
         model_t = 0.0
     end where
 
     if (PRESENT(model_s)) then
-        where(sum_wgt > 0)
+        where(sum_wgt > 0.0)
             model_s = model_s /sum_wgt
         elsewhere
             model_s = 0.0
@@ -1366,7 +1366,7 @@ subroutine mult_obs_I_mse_profile(Obs)
     
     do n=1,size(Obs)
        do k = 1, Obs(n)%levels
-          Ims = 1/Obs(n)%ms_t(k)
+          Ims = 1./Obs(n)%ms_t(k)
           if (Obs(n)%flag_t(k) .eq. 0) Obs(n)%data_t(k) = Ims*Obs(n)%data_t(k)
           if (ASSOCIATED(Obs(n)%data_s)) then
               Ims = 1/Obs(n)%ms_s(k)
@@ -1705,7 +1705,7 @@ subroutine create_ideal_profiles(localize)
            allocate(profiles(num_profiles)%flag_s(nlevels))
            profiles(num_profiles)%flag_s= 0
 
-           profiles(num_profiles)%probe = 0
+           profiles(num_profiles)%probe = 0.
            profiles(num_profiles)%levels = nlevels
            profiles(num_profiles)%lat = lat(i)
            profiles(num_profiles)%lon = lon(i)
@@ -1719,8 +1719,8 @@ subroutine create_ideal_profiles(localize)
 ! calculate interpolation coefficients (make sure to account for grid offsets here!)
 ! note that this only works for lat/lon grids
            
-           ri0 = frac_index(lon(i), x_grid(isg-1:ieg+1)) - 1 
-           rj0 = frac_index(lat(i), y_grid(jsg-1:jeg+1)) - 1
+           ri0 = frac_index(lon(i), x_grid(isg-1:ieg+1)) - 1.
+           rj0 = frac_index(lat(i), y_grid(jsg-1:jeg+1)) - 1.
            i0 = floor(ri0)
            j0 = floor(rj0)
            Profiles(num_profiles)%i_index = ri0
@@ -1818,13 +1818,13 @@ subroutine nullify_obs_prof(profile)
 
 
     profile%nvar = 0
-    profile%project=0
-    profile%probe=0
-    profile%ref_inst=0
+    profile%project=0.
+    profile%probe=0.
+    profile%ref_inst=0.
     profile%wod_cast_num=0
-    profile%fix_depth=0
-    profile%ocn_vehicle=0
-    profile%database_id=0
+    profile%fix_depth=0.
+    profile%ocn_vehicle=0.
+    profile%database_id=0.
     profile%levels=0
     profile%profile_flag=-1
     profile%profile_flag_s=-1
diff --git a/src/shared/oda_tools/oda_core_ecda.F90 b/src/shared/oda_tools/oda_core_ecda.F90
index 23225f8119..c4b4989adc 100644
--- a/src/shared/oda_tools/oda_core_ecda.F90
+++ b/src/shared/oda_tools/oda_core_ecda.F90
@@ -669,37 +669,40 @@ subroutine open_profile_dataset(filename, time_start, time_end, obs_variable, lo
        if ( profile_time > time_start .and. profile_time < time_end ) data_in_period = .true.
        if ( (data_in_period .and. data_is_local) .and. (.NOT.localize_data) ) then ! localize
 
-          if (isd_filt >= 1 .and. ied_filt <= ieg) then
-            if (lon >= x_grid(isd_filt,jsd_flt0) .and.&
-               & lon <= x_grid(ied_filt-1,jsd_flt0) .and.&
-               & lat >= y_grid(isd_filt,jsd_flt0) .and.&
-               & lat <=  y_grid(ied_filt-1,jed_flt0-1)) then
+          if (isd_filt < 1 .and. ied_filt > ieg) then
+             ! filter domain is a full x band
+             if (lon >= x_grid(1,jsd_flt0) .and. lon <= x_grid(ieg-1,jsd_flt0) .and.&
+                  lat >= y_grid(1,jsd_flt0) .and. lat <= y_grid(ieg-1,jsd_flt0)) then
                 prof_in_filt_domain = .true.
-            end if
-          end if
-          if (isd_filt < 1) then
-            isd_flt0 = isd_filt + ieg
-            if ( lon >= x_grid(1,jsd_flt0) .and. lon <= x_grid(ied_filt-1,jsd_flt0) .and.&
-                  & lat >= y_grid(1,jsd_flt0) .and. lat <=  y_grid(ied_filt-1,jed_flt0-1) ) then
+             end if
+          else if (isd_filt >= 1 .and. ied_filt <= ieg) then
+             ! Interior filter domain
+            if (lon >= x_grid(isd_filt,jsd_flt0) .and. lon <= x_grid(ied_filt-1,jsd_flt0) .and.&
+               & lat >= y_grid(isd_filt,jsd_flt0) .and. lat <=  y_grid(ied_filt-1,jed_flt0-1)) then
                 prof_in_filt_domain = .true.
             end if
-            if ( lon >= x_grid(isd_flt0,jsd_flt0) .and. lon <= x_grid(ieg-1,jsd_flt0) .and.&
-                  & lat >= y_grid(isd_flt0,jsd_flt0) .and. lat <=  y_grid(ieg-1,jed_flt0-1) ) then
+          else if (isd_filt < 1 .and. ied_filt <= ieg) then
+             ! lhs filter domain
+             isd_flt0 = isd_filt + ieg
+             if ((lon >= x_grid(1,jsd_flt0) .and. lon <= x_grid(ied_filt-1,jsd_flt0) .and.&
+                  & lat >= y_grid(1,jsd_flt0) .and. lat <=  y_grid(ied_filt-1,jed_flt0-1)).or.&
+                  & (lon >= x_grid(isd_flt0,jsd_flt0) .and. lon <= x_grid(ieg-1,jsd_flt0) .and.&
+                  & lat >= y_grid(isd_flt0,jsd_flt0) .and. lat <=  y_grid(ieg-1,jed_flt0-1))) then
                 prof_in_filt_domain = .true.
-            end if
-          end if
-          if (ied_filt > ieg) then
-            ied_flt0 = ied_filt - ieg
-            if ( lon >= x_grid(isd_filt,jsd_flt0) .and. lon <= x_grid(ieg-1,jsd_flt0) .and.&
+             end if
+          else if (isd_filt >= 1 .and. ied_filt > ieg) then
+             ! rhs filter domain
+             ied_flt0 = ied_filt - ieg
+             if ( lon >= x_grid(isd_filt,jsd_flt0) .and. lon <= x_grid(ieg-1,jsd_flt0) .and.&
                   & lat >= y_grid(isd_filt,jsd_flt0) .and. lat <=  y_grid(ieg-1,jed_flt0-1) ) then
                 prof_in_filt_domain = .true.
-            end if
-            if (ied_flt0-1 > 1) then
-            if ( lon >= x_grid(1,jsd_flt0) .and. lon <= x_grid(ied_flt0-1,jsd_flt0) .and.&
-                  & lat >= y_grid(1,jsd_flt0) .and. lat <=  y_grid(ied_flt0-1,jed_flt0-1) ) then
-                prof_in_filt_domain = .true.
-            end if
-            end if
+             end if
+             if (ied_flt0-1 > 1) then
+                if ( lon >= x_grid(1,jsd_flt0) .and. lon <= x_grid(ied_flt0-1,jsd_flt0) .and.&
+                     & lat >= y_grid(1,jsd_flt0) .and. lat <=  y_grid(ied_flt0-1,jed_flt0-1) ) then
+                   prof_in_filt_domain = .true.
+                end if
+             end if
           end if
 
           if ( var_id == TEMP_ID .and. flag_t == 0.0 ) then
@@ -939,7 +942,7 @@ subroutine open_profile_dataset(filename, time_start, time_end, obs_variable, lo
              if (i0 < 1 .or. j0 < 1) then
                 Profiles(num_profiles)%accepted = .false.
              end if
-             if( i0 < isd_filt .or. i0 > ied_filt .or. j0 < jsd_filt .or. j0 > jed_filt ) then
+             if( i0 < isd_filt .or. i0 >= ied_filt .or. j0 < jsd_filt .or. j0 >= jed_filt ) then
                 Profiles(num_profiles)%accepted = .false.
              end if
 
@@ -1288,37 +1291,40 @@ subroutine open_profile_dataset_argo(filename, time_start, time_end, obs_variabl
        if ( profile_time > time_start .and. profile_time < time_end ) data_in_period = .true.
        if ( (data_in_period .and. data_is_local) .and. (.NOT.localize_data) ) then
 
-          if (isd_filt >= 1 .and. ied_filt <= ieg) then
-            if (lon >= x_grid(isd_filt,jsd_flt0) .and.&
-               & lon <= x_grid(ied_filt-1,jsd_flt0) .and.&
-               & lat >= y_grid(isd_filt,jsd_flt0) .and.&
-               & lat <=  y_grid(ied_filt-1,jed_flt0-1)) then
+          if (isd_filt < 1 .and. ied_filt > ieg) then
+             ! filter domain is a full x band
+             if (lon >= x_grid(1,jsd_flt0) .and. lon <= x_grid(ieg-1,jsd_flt0) .and.&
+                  lat >= y_grid(1,jsd_flt0) .and. lat <= y_grid(ieg-1,jsd_flt0)) then
                 prof_in_filt_domain = .true.
-            end if
-          end if
-          if (isd_filt < 1) then
-            isd_flt0 = isd_filt + ieg
-            if ( lon >= x_grid(1,jsd_flt0) .and. lon <= x_grid(ied_filt-1,jsd_flt0) .and.&
-                  & lat >= y_grid(1,jsd_flt0) .and. lat <=  y_grid(ied_filt-1,jed_flt0-1) ) then
+             end if
+          else if (isd_filt >= 1 .and. ied_filt <= ieg) then
+             ! Interior filter domain
+            if (lon >= x_grid(isd_filt,jsd_flt0) .and. lon <= x_grid(ied_filt-1,jsd_flt0) .and.&
+               & lat >= y_grid(isd_filt,jsd_flt0) .and. lat <=  y_grid(ied_filt-1,jed_flt0-1)) then
                 prof_in_filt_domain = .true.
             end if
-            if ( lon >= x_grid(isd_flt0,jsd_flt0) .and. lon <= x_grid(ieg-1,jsd_flt0) .and.&
-                  & lat >= y_grid(isd_flt0,jsd_flt0) .and. lat <=  y_grid(ieg-1,jed_flt0-1) ) then
+          else if (isd_filt < 1 .and. ied_filt <= ieg) then
+             ! lhs filter domain
+             isd_flt0 = isd_filt + ieg
+             if ((lon >= x_grid(1,jsd_flt0) .and. lon <= x_grid(ied_filt-1,jsd_flt0) .and.&
+                  & lat >= y_grid(1,jsd_flt0) .and. lat <=  y_grid(ied_filt-1,jed_flt0-1)).or.&
+                  & (lon >= x_grid(isd_flt0,jsd_flt0) .and. lon <= x_grid(ieg-1,jsd_flt0) .and.&
+                  & lat >= y_grid(isd_flt0,jsd_flt0) .and. lat <=  y_grid(ieg-1,jed_flt0-1))) then
                 prof_in_filt_domain = .true.
-            end if
-          end if
-          if (ied_filt > ieg) then
-            ied_flt0 = ied_filt - ieg
-            if ( lon >= x_grid(isd_filt,jsd_flt0) .and. lon <= x_grid(ieg-1,jsd_flt0) .and.&
+             end if
+          else if (isd_filt >= 1 .and. ied_filt > ieg) then
+             ! rhs filter domain
+             ied_flt0 = ied_filt - ieg
+             if ( lon >= x_grid(isd_filt,jsd_flt0) .and. lon <= x_grid(ieg-1,jsd_flt0) .and.&
                   & lat >= y_grid(isd_filt,jsd_flt0) .and. lat <=  y_grid(ieg-1,jed_flt0-1) ) then
                 prof_in_filt_domain = .true.
-            end if
-            if (ied_flt0-1 > 1) then
-            if ( lon >= x_grid(1,jsd_flt0) .and. lon <= x_grid(ied_flt0-1,jsd_flt0) .and.&
-                  & lat >= y_grid(1,jsd_flt0) .and. lat <=  y_grid(ied_flt0-1,jed_flt0-1) ) then
-                prof_in_filt_domain = .true.
-            end if
-            end if
+             end if
+             if (ied_flt0-1 > 1) then
+                if ( lon >= x_grid(1,jsd_flt0) .and. lon <= x_grid(ied_flt0-1,jsd_flt0) .and.&
+                     & lat >= y_grid(1,jsd_flt0) .and. lat <=  y_grid(ied_flt0-1,jed_flt0-1) ) then
+                   prof_in_filt_domain = .true.
+                end if
+             end if
           end if
 
           if ( var_id == TEMP_ID ) then
@@ -1543,7 +1549,7 @@ subroutine open_profile_dataset_argo(filename, time_start, time_end, obs_variabl
            if ( i0 < 1 .or. j0 < 1 ) then
               Profiles(num_profiles)%accepted = .false.
            end if
-           if ( i0 < isd_filt .or. i0 > ied_filt .or. j0 < jsd_filt .or. j0 > jed_filt ) then
+           if ( i0 < isd_filt .or. i0 >= ied_filt .or. j0 < jsd_filt .or. j0 >= jed_filt ) then
               Profiles(num_profiles)%accepted = .false.
            end if
 
@@ -2185,37 +2191,40 @@ subroutine open_profile_dataset_woa05t(filename, obs_variable, localize)
           if ( abs(lat) >= 20.0 .and. (mod(i,4) /= 0 .or. mod(j,4) /= 0) ) data_is_local = .false.
           if ( abs(lat) >= 60.0 .and. (mod(i,6) /= 0 .or. mod(j,6) /= 0) ) data_is_local = .false.
 
-          if (isd_filt >= 1 .and. ied_filt <= ieg) then
-            if (lon >= x_grid(isd_filt,jsd_flt0) .and.&
-               & lon <= x_grid(ied_filt-1,jsd_flt0) .and.&
-               & lat >= y_grid(isd_filt,jsd_flt0) .and.&
-               & lat <=  y_grid(ied_filt-1,jed_flt0-1)) then
+          if (isd_filt < 1 .and. ied_filt > ieg) then
+             ! filter domain is a full x band
+             if (lon >= x_grid(1,jsd_flt0) .and. lon <= x_grid(ieg-1,jsd_flt0) .and.&
+                  lat >= y_grid(1,jsd_flt0) .and. lat <= y_grid(ieg-1,jsd_flt0)) then
                 prof_in_filt_domain = .true.
-            end if
-          end if
-          if (isd_filt < 1) then
-            isd_flt0 = isd_filt + ieg
-            if ( lon >= x_grid(1,jsd_flt0) .and. lon <= x_grid(ied_filt-1,jsd_flt0) .and.&
-                  & lat >= y_grid(1,jsd_flt0) .and. lat <=  y_grid(ied_filt-1,jed_flt0-1) ) then
+             end if
+          else if (isd_filt >= 1 .and. ied_filt <= ieg) then
+             ! Interior filter domain
+            if (lon >= x_grid(isd_filt,jsd_flt0) .and. lon <= x_grid(ied_filt-1,jsd_flt0) .and.&
+               & lat >= y_grid(isd_filt,jsd_flt0) .and. lat <=  y_grid(ied_filt-1,jed_flt0-1)) then
                 prof_in_filt_domain = .true.
             end if
-            if ( lon >= x_grid(isd_flt0,jsd_flt0) .and. lon <= x_grid(ieg-1,jsd_flt0) .and.&
-                  & lat >= y_grid(isd_flt0,jsd_flt0) .and. lat <=  y_grid(ieg-1,jed_flt0-1) ) then
+          else if (isd_filt < 1 .and. ied_filt <= ieg) then
+             ! lhs filter domain
+             isd_flt0 = isd_filt + ieg
+             if ((lon >= x_grid(1,jsd_flt0) .and. lon <= x_grid(ied_filt-1,jsd_flt0) .and.&
+                  & lat >= y_grid(1,jsd_flt0) .and. lat <=  y_grid(ied_filt-1,jed_flt0-1)).or.&
+                  & (lon >= x_grid(isd_flt0,jsd_flt0) .and. lon <= x_grid(ieg-1,jsd_flt0) .and.&
+                  & lat >= y_grid(isd_flt0,jsd_flt0) .and. lat <=  y_grid(ieg-1,jed_flt0-1))) then
                 prof_in_filt_domain = .true.
-            end if
-          end if
-          if (ied_filt > ieg) then
-            ied_flt0 = ied_filt - ieg
-            if ( lon >= x_grid(isd_filt,jsd_flt0) .and. lon <= x_grid(ieg-1,jsd_flt0) .and.&
+             end if
+          else if (isd_filt >= 1 .and. ied_filt > ieg) then
+             ! rhs filter domain
+             ied_flt0 = ied_filt - ieg
+             if ( lon >= x_grid(isd_filt,jsd_flt0) .and. lon <= x_grid(ieg-1,jsd_flt0) .and.&
                   & lat >= y_grid(isd_filt,jsd_flt0) .and. lat <=  y_grid(ieg-1,jed_flt0-1) ) then
                 prof_in_filt_domain = .true.
-            end if
-            if (ied_flt0-1 > 1) then
-            if ( lon >= x_grid(1,jsd_flt0) .and. lon <= x_grid(ied_flt0-1,jsd_flt0) .and.&
-                  & lat >= y_grid(1,jsd_flt0) .and. lat <=  y_grid(ied_flt0-1,jed_flt0-1) ) then
-                prof_in_filt_domain = .true.
-            end if
-            end if
+             end if
+             if (ied_flt0-1 > 1) then
+                if ( lon >= x_grid(1,jsd_flt0) .and. lon <= x_grid(ied_flt0-1,jsd_flt0) .and.&
+                     & lat >= y_grid(1,jsd_flt0) .and. lat <=  y_grid(ied_flt0-1,jed_flt0-1) ) then
+                   prof_in_filt_domain = .true.
+                end if
+             end if
           end if
 
           if ( data_is_local .and. (.NOT.localize_data) ) then ! global index
@@ -2334,7 +2343,7 @@ subroutine open_profile_dataset_woa05t(filename, obs_variable, localize)
              if ( i0 < 1 .or. j0 < 1 ) then
                 Profiles(num_profiles)%accepted = .false.
              end if
-             if ( i0 < isd_filt .or. i0 > ied_filt .or. j0 < jsd_filt .or. j0 > jed_filt ) then
+             if ( i0 < isd_filt .or. i0 >= ied_filt .or. j0 < jsd_filt .or. j0 >= jed_filt ) then
                 Profiles(num_profiles)%accepted = .false.
              end if
 
@@ -2540,37 +2549,40 @@ subroutine open_profile_dataset_woa05s(filename, obs_variable, localize)
           if ( abs(lat) >= 20.0 .and. (mod(i,4) /= 0 .or. mod(j,4) /= 0) ) data_is_local = .false.
           if ( abs(lat) >= 60.0 .and. (mod(i,6) /= 0 .or. mod(j,6) /= 0) ) data_is_local = .false.
 
-          if (isd_filt >= 1 .and. ied_filt <= ieg) then
-            if (lon >= x_grid(isd_filt,jsd_flt0) .and.&
-               & lon <= x_grid(ied_filt-1,jsd_flt0) .and.&
-               & lat >= y_grid(isd_filt,jsd_flt0) .and.&
-               & lat <=  y_grid(ied_filt-1,jed_flt0-1)) then
+          if (isd_filt < 1 .and. ied_filt > ieg) then
+             ! filter domain is a full x band
+             if (lon >= x_grid(1,jsd_flt0) .and. lon <= x_grid(ieg-1,jsd_flt0) .and.&
+                  lat >= y_grid(1,jsd_flt0) .and. lat <= y_grid(ieg-1,jsd_flt0)) then
                 prof_in_filt_domain = .true.
-            end if
-          end if
-          if (isd_filt < 1) then
-            isd_flt0 = isd_filt + ieg
-            if ( lon >= x_grid(1,jsd_flt0) .and. lon <= x_grid(ied_filt-1,jsd_flt0) .and.&
-                  & lat >= y_grid(1,jsd_flt0) .and. lat <=  y_grid(ied_filt-1,jed_flt0-1) ) then
+             end if
+          else if (isd_filt >= 1 .and. ied_filt <= ieg) then
+             ! Interior filter domain
+            if (lon >= x_grid(isd_filt,jsd_flt0) .and. lon <= x_grid(ied_filt-1,jsd_flt0) .and.&
+               & lat >= y_grid(isd_filt,jsd_flt0) .and. lat <=  y_grid(ied_filt-1,jed_flt0-1)) then
                 prof_in_filt_domain = .true.
             end if
-            if ( lon >= x_grid(isd_flt0,jsd_flt0) .and. lon <= x_grid(ieg-1,jsd_flt0) .and.&
-                  & lat >= y_grid(isd_flt0,jsd_flt0) .and. lat <=  y_grid(ieg-1,jed_flt0-1) ) then
+          else if (isd_filt < 1 .and. ied_filt <= ieg) then
+             ! lhs filter domain
+             isd_flt0 = isd_filt + ieg
+             if ((lon >= x_grid(1,jsd_flt0) .and. lon <= x_grid(ied_filt-1,jsd_flt0) .and.&
+                  & lat >= y_grid(1,jsd_flt0) .and. lat <=  y_grid(ied_filt-1,jed_flt0-1)).or.&
+                  & (lon >= x_grid(isd_flt0,jsd_flt0) .and. lon <= x_grid(ieg-1,jsd_flt0) .and.&
+                  & lat >= y_grid(isd_flt0,jsd_flt0) .and. lat <=  y_grid(ieg-1,jed_flt0-1))) then
                 prof_in_filt_domain = .true.
-            end if
-          end if
-          if (ied_filt > ieg) then
-            ied_flt0 = ied_filt - ieg
-            if ( lon >= x_grid(isd_filt,jsd_flt0) .and. lon <= x_grid(ieg-1,jsd_flt0) .and.&
+             end if
+          else if (isd_filt >= 1 .and. ied_filt > ieg) then
+             ! rhs filter domain
+             ied_flt0 = ied_filt - ieg
+             if ( lon >= x_grid(isd_filt,jsd_flt0) .and. lon <= x_grid(ieg-1,jsd_flt0) .and.&
                   & lat >= y_grid(isd_filt,jsd_flt0) .and. lat <=  y_grid(ieg-1,jed_flt0-1) ) then
                 prof_in_filt_domain = .true.
-            end if
-            if (ied_flt0-1 > 1) then
-            if ( lon >= x_grid(1,jsd_flt0) .and. lon <= x_grid(ied_flt0-1,jsd_flt0) .and.&
-                  & lat >= y_grid(1,jsd_flt0) .and. lat <=  y_grid(ied_flt0-1,jed_flt0-1) ) then
-                prof_in_filt_domain = .true.
-            end if
-            end if
+             end if
+             if (ied_flt0-1 > 1) then
+                if ( lon >= x_grid(1,jsd_flt0) .and. lon <= x_grid(ied_flt0-1,jsd_flt0) .and.&
+                     & lat >= y_grid(1,jsd_flt0) .and. lat <=  y_grid(ied_flt0-1,jed_flt0-1) ) then
+                   prof_in_filt_domain = .true.
+                end if
+             end if
           end if
 
           if ( data_is_local .and. (.NOT.localize_data) ) then ! global index
@@ -2689,7 +2701,7 @@ subroutine open_profile_dataset_woa05s(filename, obs_variable, localize)
              if ( i0 < 1 .or. j0 < 1 ) then
                 Profiles(num_profiles)%accepted = .false.
              end if
-             if ( i0 < isd_filt .or. i0 > ied_filt .or. j0 < jsd_filt .or. j0 > jed_filt ) then
+             if ( i0 < isd_filt .or. i0 >= ied_filt .or. j0 < jsd_filt .or. j0 >= jed_filt ) then
                 Profiles(num_profiles)%accepted = .false.
              end if
 
diff --git a/src/shared/oda_tools/oda_types.F90 b/src/shared/oda_tools/oda_types.F90
index af400d6123..6438f1aa96 100644
--- a/src/shared/oda_tools/oda_types.F90
+++ b/src/shared/oda_tools/oda_types.F90
@@ -9,22 +9,22 @@ module oda_types_mod
 
 !============================================================
 ! This module contains type declarations and default values
-! for oda modules.
+! for oda modules.  
 !============================================================
-
-! Contact: Matthew.Harrison
+  
+! Contact: Matthew.Harrison@gfdl.noaa.gov
 
   use time_manager_mod, only : time_type, set_time
   use mpp_mod, only : stdout
   use mpp_domains_mod, only : domain2d
-
+  
   implicit none
 
   private
 
-  integer, parameter, public :: MAX_LEVELS_FILE = MAX_LEVS_FILE_ !< Controls record length for optimal storage
+  integer, parameter, public :: MAX_LEVELS_FILE = MAX_LEVS_FILE_ !< Controls record length for optimal storage  
   integer, parameter, public :: MAX_NEIGHBORS = 100 !< Maximum number of neighbors for QC or analysis for profiles
-  integer, parameter, public :: MAX_LINKS = MAX_LINKS_ !< Maximum number of records per profile for storage for profiles
+  integer, parameter, public :: MAX_LINKS = MAX_LINKS_ !< Maximum number of records per profile for storage for profiles  
 
   ! Additional Pramaeters needed for snz's ECDA
   integer, parameter, public :: DROP_PROFILER = 10
@@ -46,7 +46,7 @@ module oda_types_mod
   integer, save, public :: TEMP_ID = 1
   integer, save, public :: SALT_ID = 2
 
-  ! List of variables for ODA
+  ! List of variables for ODA   
 #ifndef ENABLE_ECDA
   real, parameter, public :: MISSING_VALUE = -1.e20
 #else
@@ -57,7 +57,7 @@ module oda_types_mod
   type, public :: forward_model_type
      real, dimension(:,:,:,:), pointer :: wgt ! interpolation weights
   end type forward_model_type
-
+  
   type, public :: ocean_profile_type
      integer :: variable !< variable ids are defined by the ocean_types module (e.g. TEMP_ID, SALT_ID)
      integer :: inst_type !< instrument types are defined by platform class (e.g. MOORING, DROP, etc.) and instrument type (XBT, CDT, etc.)
@@ -71,31 +71,24 @@ module oda_types_mod
      real    :: database_id
      integer :: levels
      integer :: profile_flag ! an overall flag for the profile
-     integer :: profile_flag_s ! an overall flag for the profile salinity
+     integer :: profile_flag_s ! an overall flag for the profile salinity     
      real :: lat, lon
      logical :: accepted
      integer :: nlinks
      type(ocean_profile_type), pointer, dimension(:) :: next ! Large profiles are stored as linked list.
      integer, dimension(MAX_NEIGHBORS) :: nbr_index
-     real, dimension(MAX_NEIGHBORS) :: nbr_dist ! distance in radians
+     real, dimension(MAX_NEIGHBORS) :: nbr_dist ! distance in radians 
      real, dimension(:), pointer :: depth, data_t, data_s
      real, dimension(:), pointer :: data
      integer, dimension(:), pointer :: flag_t
      integer, dimension(:), pointer :: flag_s ! level-by-level flags for salinity
-#if !(defined(ENABLE_ECDA)||defined(__GFORTRAN__)||defined(__FUJITSU))
-     ! this #if needed due to GNU not doing a logical cast properly
-     ! Will be removed in a later patch to only allow logical flag.
-     integer, dimension(:), pointer :: flag
-#else
-     !::sdu:: For now ECDA use flag as a logical, will likely change in future releases.
      logical, dimension(:), pointer :: flag
-#endif
      real    :: temp_err, salt_err ! measurement error
      real, dimension(:), pointer :: ms_t ! ms temperature by level
-     real, dimension(:), pointer :: ms_s ! ms salinity by level
+     real, dimension(:), pointer :: ms_s ! ms salinity by level  
      real, dimension(:), pointer :: ms_inv
      real, dimension(:), pointer :: ms
-     type(time_type) :: time
+     type(time_type) :: time 
      integer         :: yyyy
      integer         :: mmdd
      type(time_type), pointer :: Model_time ! each profile can be associated with a first-guess field with an associated time and grid
@@ -121,7 +114,7 @@ module oda_types_mod
      real, pointer, dimension(:,:) :: data2 => NULL()
      real, dimension(:,:), pointer :: ms2 => NULL()
      real, dimension(:,:), pointer :: i_index2=>NULL(), j_index2=>NULL() ! model indices
-     real :: k_index
+     real :: k_index          
      type(forward_model_type) :: Forward_model
      type(time_type) :: time
      integer :: yyyy
@@ -129,7 +122,7 @@ module oda_types_mod
      character(len=8) :: wmo_id
      type(time_type), pointer :: Model_time => NULL()
      type(grid_type), pointer :: Model_grid => NULL()
-     ! positive difference between current model time
+     ! positive difference between current model time 
      ! and observation time
      type(time_type) :: tdiff
   end type ocean_surface_type
@@ -143,7 +136,7 @@ module oda_types_mod
      real, pointer, dimension(:,:) :: lw_flux => NULL()
      real, pointer, dimension(:,:) :: sw_flux_vis_dir => NULL()
      real, pointer, dimension(:,:) :: sw_flux_vis_dif => NULL()
-     real, pointer, dimension(:,:) :: sw_flux_nir_dir => NULL()
+     real, pointer, dimension(:,:) :: sw_flux_nir_dir => NULL() 
      real, pointer, dimension(:,:) :: sw_flux_nir_dif => NULL()
   end type da_flux_type
 
@@ -188,7 +181,7 @@ module oda_types_mod
      type(grid_type), pointer :: grid => NULL()
      real, pointer, dimension(:,:) :: ex=>NULL(), vr=>NULL()
   end type field_dist_type_2d
-
+     
   type, public :: ocean_dist_type
      type(field_dist_type_3d) :: temp,salt,u,v
      type(field_dist_type_2d) :: eta
@@ -199,11 +192,11 @@ module oda_types_mod
   end type obs_clim_type
 
   public init_obs
-
+  
   interface init_obs
      module procedure init_obs_profile
   end interface
-
+  
   contains
 
     subroutine init_obs_profile(profile)
@@ -238,7 +231,7 @@ subroutine init_obs_profile(profile)
       profile%salt_err = -1.0
       profile%time = set_time(0,0)
       profile%yyyy = 0
-      profile%mmdd = 0
+      profile%mmdd = 0      
       if (associated(profile%model_time)) deallocate(profile%model_time)
       if (associated(profile%model_grid)) deallocate(profile%model_grid)
       profile%i_index = -1
@@ -247,7 +240,7 @@ subroutine init_obs_profile(profile)
       profile%tdiff = set_time(0,0)
 
       return
-
+      
     end subroutine init_obs_profile
-
+    
 end module oda_types_mod
diff --git a/src/shared/sat_vapor_pres/sat_vapor_pres.F90 b/src/shared/sat_vapor_pres/sat_vapor_pres.F90
index 7e41d7dac0..83accd43ba 100644
--- a/src/shared/sat_vapor_pres/sat_vapor_pres.F90
+++ b/src/shared/sat_vapor_pres/sat_vapor_pres.F90
@@ -71,10 +71,11 @@ module sat_vapor_pres_mod
 !
 !-----------------------------------------------------------------------
 
-! 
+! 
 !   Bruce Wyman
 ! 
 
+! 
 
 ! 
 !   Routines for determining the saturation vapor pressure 
@@ -491,8 +492,8 @@ module sat_vapor_pres_mod
 !-----------------------------------------------------------------------
 !  cvs version and tag name
 
- character(len=128) :: version = '$Id: sat_vapor_pres.F90,v 20.0 2013/12/14 00:27:58 fms Exp $'
- character(len=128) :: tagname = '$Name: tikal $'
+ character(len=128) :: version = '$Id$'
+ character(len=128) :: tagname = '$Name$'
 
  logical :: module_is_initialized = .false.
 
@@ -523,7 +524,7 @@ module sat_vapor_pres_mod
  logical :: construct_table_wrt_liq = .false.
  logical :: construct_table_wrt_liq_and_ice = .false.
 
- namelist /sat_vapor_pres_nml/ show_bad_value_count_by_slice, show_all_bad_values, &
+ namelist / sat_vapor_pres_nml / show_bad_value_count_by_slice, show_all_bad_values, &
                                  use_exact_qs, do_simple, &
                                  construct_table_wrt_liq, &
                                  construct_table_wrt_liq_and_ice
@@ -2220,7 +2221,7 @@ subroutine sat_vapor_pres_init(err_msg)
 #endif
 
 ! write version number and namelist to log file
-  call write_version_number()
+  call write_version_number (version, tagname)
   unit = stdlog()
   if (mpp_pe() == mpp_root_pe()) write (unit, nml=sat_vapor_pres_nml)
 
diff --git a/src/shared/sat_vapor_pres/sat_vapor_pres_k.F90 b/src/shared/sat_vapor_pres/sat_vapor_pres_k.F90
index 7c6dab864a..b7daf62812 100644
--- a/src/shared/sat_vapor_pres/sat_vapor_pres_k.F90
+++ b/src/shared/sat_vapor_pres/sat_vapor_pres_k.F90
@@ -28,8 +28,8 @@ module sat_vapor_pres_k_mod
  implicit none
  private
 
- character(len=128), parameter :: version = '$Id: sat_vapor_pres_k.F90,v 18.0 2010/03/02 23:58:26 fms Exp $'
- character(len=128), parameter :: tagname = '$Name: tikal $'
+ character(len=128), parameter :: version = '$Id$'
+ character(len=128), parameter :: tagname = '$Name$'
 
  public :: sat_vapor_pres_init_k
  public :: lookup_es_k
@@ -191,7 +191,7 @@ subroutine sat_vapor_pres_init_k(table_size, tcmin, tcmax, TFREEZE, HLV, RVGAS,
    endif
 
       table_siz = table_size
-      dtres = (tcmax - tcmin)/(table_size-1)
+      dtres = (tcmax - tcmin)/real(table_size-1)
       tminl = real(tcmin)+TFREEZE  ! minimum valid temp in table
       dtinvl = 1./dtres
       tepsl = .5*dtres
@@ -203,7 +203,7 @@ subroutine sat_vapor_pres_init_k(table_size, tcmin, tcmax, TFREEZE, HLV, RVGAS,
 ! To be able to compute tables for any temperature range and resolution,
 ! and at the same time exactly reproduce answers from memphis revision,
 ! it is necessary to compute ftact differently than it is in memphis.
-      tfact = 5*dtinvl
+      tfact = 5.0*dtinvl
 
       hdtinv = dtinvl*0.5
 
@@ -337,9 +337,9 @@ function compute_es_k(tem, TFREEZE) result (es)
 !  see smithsonian meteorological tables page 350.
 
      if (tem(i) > -20.+TBASI) then
-         x = -7.90298*(TBASW/tem(i)-1) + 5.02808*log10(TBASW/tem(i)) &
-             -1.3816e-07*(10**((1-tem(i)/TBASW)*11.344)-1)        &
-             +8.1328e-03*(10**((TBASW/tem(i)-1)*(-3.49149))-1)    &
+         x = -7.90298*(TBASW/tem(i)-1.0) + 5.02808*log10(TBASW/tem(i)) &
+             -1.3816e-07*(10.0**((1.0-tem(i)/TBASW)*11.344)-1.0)        &
+             +8.1328e-03*(10.0**((TBASW/tem(i)-1.0)*(-3.49149))-1.0)    &
              +log10(ESBASW)
          esh2o = 10.**(x)
      else
@@ -379,9 +379,9 @@ function compute_es_liq_k(tem, TFREEZE) result (es)
 !  values over 100 c may not be valid
 !  see smithsonian meteorological tables page 350.
 
-         x = -7.90298*(TBASW/tem(i)-1) + 5.02808*log10(TBASW/tem(i)) &
-             -1.3816e-07*(10**((1-tem(i)/TBASW)*11.344)-1)        &
-             +8.1328e-03*(10**((TBASW/tem(i)-1)*(-3.49149))-1)    &
+         x = -7.90298*(TBASW/tem(i)-1.0) + 5.02808*log10(TBASW/tem(i)) &
+             -1.3816e-07*(10.0**((1.0-tem(i)/TBASW)*11.344)-1.0)        &
+             +8.1328e-03*(10.0**((TBASW/tem(i)-1.0)*(-3.49149))-1.0)    &
              +log10(ESBASW)
          esh2o = 10.**(x)
 
@@ -421,9 +421,9 @@ function compute_es_liq_ice_k(tem, TFREEZE) result (es)
 !  values over 100 c may not be valid
 !  see smithsonian meteorological tables page 350.
 
-         x = -7.90298*(TBASW/tem(i)-1) + 5.02808*log10(TBASW/tem(i)) &
-             -1.3816e-07*(10**((1-tem(i)/TBASW)*11.344)-1)        &
-             +8.1328e-03*(10**((TBASW/tem(i)-1)*(-3.49149))-1)    &
+         x = -7.90298*(TBASW/tem(i)-1.0) + 5.02808*log10(TBASW/tem(i)) &
+             -1.3816e-07*(10.0**((1.0-tem(i)/TBASW)*11.344)-1.0)        &
+             +8.1328e-03*(10.0**((TBASW/tem(i)-1.0)*(-3.49149))-1.0)    &
              +log10(ESBASW)
          es(i) = 10.**(x)
      endif
diff --git a/src/shared/station_data/station_data.F90 b/src/shared/station_data/station_data.F90
index a21902e6cd..ecb6bfefb2 100644
--- a/src/shared/station_data/station_data.F90
+++ b/src/shared/station_data/station_data.F90
@@ -1,5 +1,5 @@
 module station_data_mod 
-! 
+! 
 !   Giang Nong
 ! 
 ! 
@@ -288,7 +288,7 @@ subroutine station_data_init()
 94  continue
     call close_file(iunit)
     call check_duplicate_output_fields
-    call write_version_number()
+    call write_version_number (version, tagname)
     module_is_initialized = .true.
     return
 99  continue
diff --git a/src/shared/time_interp/time_interp.F90 b/src/shared/time_interp/time_interp.F90
index 4d080bb909..5c0ec0c2f3 100644
--- a/src/shared/time_interp/time_interp.F90
+++ b/src/shared/time_interp/time_interp.F90
@@ -1,10 +1,11 @@
 
 module time_interp_mod
 
-! 
+! 
 !   Bruce Wyman
 ! 
 
+! 
 
 ! 
 !   Computes a weight and dates/indices for linearly interpolating between two dates.
@@ -195,13 +196,13 @@ module time_interp_mod
    integer :: yrmod, momod, dymod
    logical :: mod_leapyear
 
-   character(len=128) :: version='$Id: time_interp.F90,v 20.0 2013/12/14 00:28:05 fms Exp $'
-   character(len=128) :: tagname='$Name: tikal $'
+   character(len=128) :: version='$Id$'
+   character(len=128) :: tagname='$Name$'
 
    logical :: module_is_initialized=.FALSE.
    logical :: perthlike_behavior=.FALSE.
 
-   namelist /time_interp_nml/ perthlike_behavior
+   namelist / time_interp_nml / perthlike_behavior
 
 contains
 
@@ -224,7 +225,7 @@ subroutine time_interp_init()
    20 call close_file (namelist_unit)
 #endif
 
-   call write_version_number()
+   call write_version_number( version, tagname )
    logunit = stdlog()
    write(logunit,time_interp_nml)
 
@@ -910,7 +911,7 @@ program test_time_interp
 
  integer :: nmin, nmax
 
- namelist /test_time_interp_nml/ timelist_len
+ namelist / test_time_interp_nml / timelist_len
 
  call fms_init
  outunit = stdout()
diff --git a/src/shared/time_interp/time_interp_external.F90 b/src/shared/time_interp/time_interp_external.F90
index 6156af3c3e..fe088b91c5 100644
--- a/src/shared/time_interp/time_interp_external.F90
+++ b/src/shared/time_interp/time_interp_external.F90
@@ -2,7 +2,7 @@
 
 module time_interp_external_mod
 !
-!M.J. Harrison
+!M.J. Harrison
 !
 !Harper Simmons
 !
@@ -50,8 +50,8 @@ module time_interp_external_mod
   private
 
   character(len=128), private :: version= &
-   'CVS $Id: time_interp_external.F90,v 20.0 2013/12/14 00:28:08 fms Exp $'
-  character(len=128), private :: tagname='Tag $Name: tikal $'
+   'CVS $Id$'
+  character(len=128), private :: tagname='Tag $Name$'
 
   integer, parameter, public  :: NO_REGION=0, INSIDE_REGION=1, OUTSIDE_REGION=2
   integer, parameter, private :: modulo_year= 0001
diff --git a/src/shared/time_manager/get_cal_time.F90 b/src/shared/time_manager/get_cal_time.F90
index aabe5952df..7ac082fd32 100644
--- a/src/shared/time_manager/get_cal_time.F90
+++ b/src/shared/time_manager/get_cal_time.F90
@@ -1,6 +1,6 @@
 module get_cal_time_mod
 
-!   
+!   
 !     fms
 !   
 !   
@@ -36,11 +36,11 @@ module get_cal_time_mod
 
 logical :: allow_calendar_conversion=.true.
 
-namelist /get_cal_time_nml/ allow_calendar_conversion
+namelist / get_cal_time_nml / allow_calendar_conversion
 ! 
 
-character(len=128) :: version='$Id: get_cal_time.F90,v 20.0 2013/12/14 00:28:11 fms Exp $'
-character(len=128) :: tagname='$Name: tikal $'
+character(len=128) :: version='$Id$'
+character(len=128) :: tagname='$Name$'
 
 contains
 !------------------------------------------------------------------------
@@ -178,7 +178,7 @@ function get_cal_time(time_increment, units, calendar, permit_calendar_conversio
   20 call close_file (namelist_unit)
 #endif
 
-  call write_version_number()
+  call write_version_number (version, tagname)
   logunit = stdlog()
   if(mpp_pe() == mpp_root_pe()) write (logunit, nml=get_cal_time_nml)
   module_is_initialized = .true.
diff --git a/src/shared/time_manager/time_manager.F90 b/src/shared/time_manager/time_manager.F90
index 35d81bbfea..3cf84431aa 100644
--- a/src/shared/time_manager/time_manager.F90
+++ b/src/shared/time_manager/time_manager.F90
@@ -1,9 +1,10 @@
 module time_manager_mod
 
-! 
+! 
 !   fms
 ! 
 
+! 
 
 ! 
 !   A software package that provides a set of simple interfaces for
@@ -179,8 +180,8 @@ module time_manager_mod
 
 !======================================================================
 
-character(len=128) :: version='$Id: time_manager.F90,v 20.0 2013/12/14 00:28:14 fms Exp $'
-character(len=128) :: tagname='$Name: tikal $'
+character(len=128) :: version='$Id$'
+character(len=128) :: tagname='$Name$'
 logical :: module_is_initialized = .false.
 
 !======================================================================
@@ -3089,7 +3090,7 @@ subroutine time_manager_init ( )
 
   if (module_is_initialized) return  ! silent return if already called
 
-  call write_version_number()
+  call write_version_number (version, tagname)
   module_is_initialized = .true.
 
 end subroutine time_manager_init
diff --git a/src/shared/topography/gaussian_topog.F90 b/src/shared/topography/gaussian_topog.F90
index 9f8c0f0bf2..77951e002d 100644
--- a/src/shared/topography/gaussian_topog.F90
+++ b/src/shared/topography/gaussian_topog.F90
@@ -1,10 +1,11 @@
 
 module gaussian_topog_mod
 
-! 
+! 
 !   Bruce Wyman
 ! 
 
+! 
 
 ! 
 !   Routines for creating Gaussian-shaped land surface topography
@@ -72,8 +73,8 @@ module gaussian_topog_mod
 
 !-----------------------------------------------------------------------
 
-character(len=128) :: version = '$Id: gaussian_topog.F90,v 20.0 2013/12/14 00:28:17 fms Exp $'
-character(len=128) :: tagname = '$Name: tikal $'
+character(len=128) :: version = '$Id$'
+character(len=128) :: tagname = '$Name$'
 
 logical :: do_nml = .true.
 logical :: module_is_initialized = .FALSE.
@@ -119,7 +120,7 @@ subroutine gaussian_topog_init ( lon, lat, zsurf )
 integer :: n
 
   if (.not.module_is_initialized) then
-     call write_version_number()
+     call write_version_number( version, tagname )
   endif
 
   if(any(shape(zsurf) /= (/size(lon(:)),size(lat(:))/))) then
diff --git a/src/shared/topography/topography.F90 b/src/shared/topography/topography.F90
index c3f5d9a242..b492f8b1cf 100644
--- a/src/shared/topography/topography.F90
+++ b/src/shared/topography/topography.F90
@@ -1,10 +1,11 @@
 
 module topography_mod
 
-! 
+! 
 !   Bruce Wyman
 ! 
 
+! 
 
 ! 
 !   Routines for creating land surface topography fields and land-water masks
@@ -114,8 +115,8 @@ module topography_mod
 
 !-----------------------------------------------------------------------
 
- character(len=128) :: version = '$Id: topography.F90,v 20.0 2013/12/14 00:28:20 fms Exp $'
- character(len=128) :: tagname = '$Name: tikal $'
+ character(len=128) :: version = '$Id$'
+ character(len=128) :: tagname = '$Name$'
 
  logical :: module_is_initialized = .FALSE.
 
@@ -129,7 +130,7 @@ subroutine topography_init ()
 
      if ( module_is_initialized ) return
 
-     call write_version_number()
+     call write_version_number (version,tagname)
      call read_namelist
      module_is_initialized = .TRUE.
 
diff --git a/src/shared/tracer_manager/tracer_manager.F90 b/src/shared/tracer_manager/tracer_manager.F90
index 8858127f65..a4304fb1a0 100644
--- a/src/shared/tracer_manager/tracer_manager.F90
+++ b/src/shared/tracer_manager/tracer_manager.F90
@@ -1,20 +1,21 @@
 module tracer_manager_mod
-! 
+! 
 !   William Cooke
 ! 
 
-! 
+! 
 !   Matt Harrison
 ! 
 
-! 
+! 
 !   Bruce Wyman
 ! 
 
-! 
+! 
 !   Peter Phillipps
 ! 
 
+! 
 
 ! 
 !   Code to manage the simple addition of tracers to the FMS code.
@@ -141,8 +142,8 @@ module tracer_manager_mod
 type(tracer_type), save  :: tracers(MAX_TRACER_FIELDS)
 type(inst_type)  , save  :: instantiations(MAX_TRACER_FIELDS)
 
-character(len=128) :: version = '$Id: tracer_manager.F90,v 20.0 2013/12/14 00:28:23 fms Exp $'
-character(len=128) :: tagname = '$Name: tikal $'
+character(len=128) :: version = '$Id$'
+character(len=128) :: tagname = '$Name$'
 logical            :: module_is_initialized = .false.
 
 logical            :: verbose_local
@@ -171,7 +172,7 @@ subroutine tracer_manager_init
   if(module_is_initialized) return
   module_is_initialized = .TRUE.
 
-  call write_version_number()
+  call write_version_number (version, tagname)
   call field_manager_init()
   TRACER_ARRAY = NOTRACER
   do model=1,NUM_MODELS 
diff --git a/src/shared/tridiagonal/tridiagonal.F90 b/src/shared/tridiagonal/tridiagonal.F90
index 069ac2cd14..e177044083 100644
--- a/src/shared/tridiagonal/tridiagonal.F90
+++ b/src/shared/tridiagonal/tridiagonal.F90
@@ -1,12 +1,13 @@
 module tridiagonal_mod
 
-! 
+! 
 !    Isaac Held 
 ! 
-! 
+! 
 !    Bruce Wyman
 ! 
 
+! 
 
 ! 
 !   Solves the tridiagonal system of equations.
@@ -120,7 +121,7 @@ subroutine tri_invert(x,d,a,b,c)
   a(:,:,size(x,3)) = 0.0
 
   do  k= 2,size(x,3)
-    g(:,:,k) = 1/(b(:,:,k)+c(:,:,k)*e(:,:,k-1))
+    g(:,:,k) = 1.0/(b(:,:,k)+c(:,:,k)*e(:,:,k-1))
     e(:,:,k) = - a(:,:,k)*g(:,:,k)
   end do
   cc = c