diff --git a/exp/FMS_compile.csh b/exp/FMS_compile.csh index 2f4582cee3..1e04fac8b3 100644 --- a/exp/FMS_compile.csh +++ b/exp/FMS_compile.csh @@ -3,97 +3,8 @@ set pathnames_shared = $code_dir/path_names_shared # path to file containing list of source paths -cat > $pathnames_shared < 0) diff --git a/src/atmos_param/entrain/entrain.F90 b/src/atmos_param/entrain/entrain.F90 index 4e3b6cd3d6..e39a28691d 100644 --- a/src/atmos_param/entrain/entrain.F90 +++ b/src/atmos_param/entrain/entrain.F90 @@ -371,7 +371,7 @@ subroutine entrain_init(lonb, latb, axes,time,idim,jdim,kdim) #endif if ( mpp_pe() == mpp_root_pe() ) then - call write_version_number() + call write_version_number(version, tagname) unit = stdlog() Write (unit,nml=entrain_nml) endif @@ -492,7 +492,7 @@ subroutine entrain_init(lonb, latb, axes,time,idim,jdim,kdim) threading='multi', form='formatted') do_print = .true. if ( mpp_pe() == mpp_root_pe() ) then - call write_version_number() + call write_version_number(version, tagname) Write (dpu ,nml=entrain_nml) endif endif ! (num_pts > 0) diff --git a/src/atmos_param/fsrad/co2_data.F90 b/src/atmos_param/fsrad/co2_data.F90 index a4a6750ce9..8292311730 100644 --- a/src/atmos_param/fsrad/co2_data.F90 +++ b/src/atmos_param/fsrad/co2_data.F90 @@ -808,7 +808,7 @@ Subroutine co2_data_init !------- write version number and namelist --------- if ( mpp_pe() == mpp_root_pe() ) then - call write_version_number() + call write_version_number(version, tagname) endif module_is_initialized = .true. diff --git a/src/atmos_param/fsrad/co2int.F90 b/src/atmos_param/fsrad/co2int.F90 index f5cc85a0cb..0704dfaa24 100644 --- a/src/atmos_param/fsrad/co2int.F90 +++ b/src/atmos_param/fsrad/co2int.F90 @@ -470,7 +470,7 @@ Subroutine co2int_init !------- write version number and namelist --------- if ( mpp_pe() == mpp_root_pe() ) then - call write_version_number() + call write_version_number(version, tagname) endif module_is_initialized = .true. diff --git a/src/atmos_param/fsrad/fs_profile.F90 b/src/atmos_param/fsrad/fs_profile.F90 index d23db0a0ee..da4b854e2f 100644 --- a/src/atmos_param/fsrad/fs_profile.F90 +++ b/src/atmos_param/fsrad/fs_profile.F90 @@ -291,7 +291,7 @@ Subroutine fs_profile_init !------- write version number and namelist --------- if ( mpp_pe() == mpp_root_pe() ) then - call write_version_number() + call write_version_number(version, tagname) endif module_is_initialized = .true. diff --git a/src/atmos_param/fsrad/fsrad.F90 b/src/atmos_param/fsrad/fsrad.F90 index db9577df43..dcd7a77bbc 100644 --- a/src/atmos_param/fsrad/fsrad.F90 +++ b/src/atmos_param/fsrad/fsrad.F90 @@ -176,7 +176,7 @@ Subroutine fsrad_init !------- write version number and namelist --------- if ( mpp_pe() == mpp_root_pe() ) then - call write_version_number() + call write_version_number(version, tagname) endif module_is_initialized = .true. diff --git a/src/atmos_param/fsrad/longwave.F90 b/src/atmos_param/fsrad/longwave.F90 index 3b18ba9a03..1a02f5db37 100644 --- a/src/atmos_param/fsrad/longwave.F90 +++ b/src/atmos_param/fsrad/longwave.F90 @@ -782,7 +782,7 @@ Subroutine longwave_init !------- write version number --------- if ( mpp_pe() == mpp_root_pe() ) then - call write_version_number() + call write_version_number(version, tagname) endif module_is_initialized = .true. diff --git a/src/atmos_param/fsrad/mcm_lw.F90 b/src/atmos_param/fsrad/mcm_lw.F90 index 223e76beab..7eafbafc60 100644 --- a/src/atmos_param/fsrad/mcm_lw.F90 +++ b/src/atmos_param/fsrad/mcm_lw.F90 @@ -177,7 +177,7 @@ subroutine mcm_lw_init(ix_in, jx_in, kx_in) !------- write version number and namelist --------- if ( mpp_pe() == mpp_root_pe() ) then - call write_version_number() + call write_version_number(version, tagname) endif module_is_initialized = .true. diff --git a/src/atmos_param/fsrad/mcm_sw_driver.F90 b/src/atmos_param/fsrad/mcm_sw_driver.F90 index 89dd66c287..154722568a 100644 --- a/src/atmos_param/fsrad/mcm_sw_driver.F90 +++ b/src/atmos_param/fsrad/mcm_sw_driver.F90 @@ -272,7 +272,7 @@ subroutine mcm_sw_driver_init(kx_in) !------- write version number and namelist --------- if ( mpp_pe() == mpp_root_pe() ) then - call write_version_number() + call write_version_number(version, tagname) endif module_is_initialized = .true. diff --git a/src/atmos_param/fsrad/mcm_swnew.F90 b/src/atmos_param/fsrad/mcm_swnew.F90 index e60a7e3b6a..74959302eb 100644 --- a/src/atmos_param/fsrad/mcm_swnew.F90 +++ b/src/atmos_param/fsrad/mcm_swnew.F90 @@ -486,7 +486,7 @@ subroutine mcm_swnew_init !------- write version number and namelist --------- if ( mpp_pe() == mpp_root_pe() ) then - call write_version_number() + call write_version_number(version, tagname) endif module_is_initialized = .true. diff --git a/src/atmos_param/fsrad/rad_diag.F90 b/src/atmos_param/fsrad/rad_diag.F90 index 638712fa66..3df0351273 100644 --- a/src/atmos_param/fsrad/rad_diag.F90 +++ b/src/atmos_param/fsrad/rad_diag.F90 @@ -88,7 +88,7 @@ subroutine RAD_DIAG_init !------- write version number and namelist --------- if ( mpp_pe() == mpp_root_pe() ) then - call write_version_number() + call write_version_number(version, tagname) endif module_is_initialized = .true. diff --git a/src/atmos_param/fsrad/rdparm.F90 b/src/atmos_param/fsrad/rdparm.F90 index cc09da30b4..05f169a04a 100644 --- a/src/atmos_param/fsrad/rdparm.F90 +++ b/src/atmos_param/fsrad/rdparm.F90 @@ -77,7 +77,7 @@ SUBROUTINE RDPARM_INIT (KDIM) !------- write version number and namelist --------- if ( mpp_pe() == mpp_root_pe() ) then - call write_version_number() + call write_version_number(version, tagname) endif module_is_initialized = .true. diff --git a/src/atmos_param/fsrad/shortwave.F90 b/src/atmos_param/fsrad/shortwave.F90 index bc158ef4ac..7e87cd9e23 100644 --- a/src/atmos_param/fsrad/shortwave.F90 +++ b/src/atmos_param/fsrad/shortwave.F90 @@ -1207,7 +1207,7 @@ SUBROUTINE SHORTWAVE_INIT !------- write version number and namelist --------- if ( mpp_pe() == mpp_root_pe() ) then - call write_version_number() + call write_version_number(version, tagname) endif module_is_initialized = .true. diff --git a/src/atmos_param/grey_radiation/grey_radiation.F90 b/src/atmos_param/grey_radiation/grey_radiation.F90 index f5d76459d5..5ce41c474e 100644 --- a/src/atmos_param/grey_radiation/grey_radiation.F90 +++ b/src/atmos_param/grey_radiation/grey_radiation.F90 @@ -141,7 +141,7 @@ subroutine grey_radiation_init(axes, Time) endif #endif -call write_version_number() +call write_version_number(version, tagname) if ( mpp_pe() == mpp_root_pe() ) then logunit = stdlog() write (logunit, nml=grey_radiation_nml) diff --git a/src/atmos_param/lscale_cond/lscale_cond.F90 b/src/atmos_param/lscale_cond/lscale_cond.F90 index a5bbe316aa..946409cb9f 100644 --- a/src/atmos_param/lscale_cond/lscale_cond.F90 +++ b/src/atmos_param/lscale_cond/lscale_cond.F90 @@ -265,7 +265,7 @@ subroutine lscale_cond_init () !---------- output namelist -------------------------------------------- if ( mpp_pe() == mpp_root_pe() ) then - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() write (logunit,nml=lscale_cond_nml) endif diff --git a/src/atmos_param/mg_drag/mg_drag.F90 b/src/atmos_param/mg_drag/mg_drag.F90 index a5cd44acdc..eb85e3c988 100644 --- a/src/atmos_param/mg_drag/mg_drag.F90 +++ b/src/atmos_param/mg_drag/mg_drag.F90 @@ -1005,7 +1005,7 @@ subroutine mg_drag_init( lonb, latb, hprime ) ! --- Output version !--------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if(mpp_pe() == mpp_root_pe()) write (logunit, nml=mg_drag_nml) diff --git a/src/atmos_param/moist_conv/moist_conv.F90 b/src/atmos_param/moist_conv/moist_conv.F90 index 0315641dad..472bd8388c 100644 --- a/src/atmos_param/moist_conv/moist_conv.F90 +++ b/src/atmos_param/moist_conv/moist_conv.F90 @@ -830,7 +830,7 @@ subroutine moist_conv_init (axes, Time, tracers_in_mca) !---------- output namelist -------------------------------------------- if ( mpp_pe() == mpp_root_pe() ) then - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() write (logunit,nml=moist_conv_nml) endif diff --git a/src/atmos_param/moist_processes/detr_ice_num.F90 b/src/atmos_param/moist_processes/detr_ice_num.F90 index d31f8c9727..43ab0b9bcd 100644 --- a/src/atmos_param/moist_processes/detr_ice_num.F90 +++ b/src/atmos_param/moist_processes/detr_ice_num.F90 @@ -73,7 +73,7 @@ SUBROUTINE detr_ice_num_init !--------- write version and namelist to standard log ------------ - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if ( mpp_pe() == mpp_root_pe() ) & write ( logunit, nml=detr_ice_num_nml ) diff --git a/src/atmos_param/moist_processes/moist_processes.F90 b/src/atmos_param/moist_processes/moist_processes.F90 index dcada305f8..471e6d9eac 100644 --- a/src/atmos_param/moist_processes/moist_processes.F90 +++ b/src/atmos_param/moist_processes/moist_processes.F90 @@ -3592,7 +3592,7 @@ subroutine moist_processes_init ( id, jd, kd, lonb, latb, lon, lat, phalf, pref, !--------- write version and namelist to standard log ------------ - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if ( mpp_pe() == mpp_root_pe() ) & write ( logunit, nml=moist_processes_nml ) diff --git a/src/atmos_param/monin_obukhov/monin_obukhov.F90 b/src/atmos_param/monin_obukhov/monin_obukhov.F90 index dfdbafa9eb..06f22fd496 100644 --- a/src/atmos_param/monin_obukhov/monin_obukhov.F90 +++ b/src/atmos_param/monin_obukhov/monin_obukhov.F90 @@ -111,7 +111,7 @@ subroutine monin_obukhov_init !---------- output namelist to log------------------------------------- if ( mpp_pe() == mpp_root_pe() ) then - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() write (logunit, nml=monin_obukhov_nml) endif diff --git a/src/atmos_param/my25_turb/my25_turb.F90 b/src/atmos_param/my25_turb/my25_turb.F90 index 6907f62295..de8a83137f 100644 --- a/src/atmos_param/my25_turb/my25_turb.F90 +++ b/src/atmos_param/my25_turb/my25_turb.F90 @@ -635,7 +635,7 @@ SUBROUTINE MY25_TURB_INIT( ix, jx, kx ) !--------------------------------------------------------------------- if ( mpp_pe() == mpp_root_pe() ) then - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() WRITE( logunit, nml = my25_turb_nml ) endif diff --git a/src/atmos_param/physics_driver/physics_driver.F90 b/src/atmos_param/physics_driver/physics_driver.F90 index 202e463b43..9670588a19 100644 --- a/src/atmos_param/physics_driver/physics_driver.F90 +++ b/src/atmos_param/physics_driver/physics_driver.F90 @@ -703,7 +703,7 @@ subroutine physics_driver_init (Time, lonb, latb, lon, lat, axes, pref, & !-------------------------------------------------------------------- ! write version number and namelist to log file. !-------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if (mpp_pe() == mpp_root_pe() ) & write(logunit, nml=physics_driver_nml) diff --git a/src/atmos_param/radiation_driver/radiation_driver.F90 b/src/atmos_param/radiation_driver/radiation_driver.F90 index 85673c00c1..570c073683 100644 --- a/src/atmos_param/radiation_driver/radiation_driver.F90 +++ b/src/atmos_param/radiation_driver/radiation_driver.F90 @@ -1212,7 +1212,7 @@ subroutine radiation_driver_init (lonb, latb, pref, axes, Time, & !--------------------------------------------------------------------- ! write version number and namelist to logfile. !--------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if (mpp_pe() == mpp_root_pe() ) & write (logunit, nml=radiation_driver_nml) diff --git a/src/atmos_param/ras/ras.F90 b/src/atmos_param/ras/ras.F90 index a92e5a8c98..d7ea7e1fb1 100644 --- a/src/atmos_param/ras/ras.F90 +++ b/src/atmos_param/ras/ras.F90 @@ -251,7 +251,7 @@ SUBROUTINE RAS_INIT( do_strat, do_liq_num, axes, Time, tracers_in_ras ) ! --- Write namelist !--------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() WRITE( logunit, nml = ras_nml ) diff --git a/src/atmos_param/rh_clouds/rh_clouds.F90 b/src/atmos_param/rh_clouds/rh_clouds.F90 index a0710d0e6b..077354c37e 100644 --- a/src/atmos_param/rh_clouds/rh_clouds.F90 +++ b/src/atmos_param/rh_clouds/rh_clouds.F90 @@ -180,7 +180,7 @@ subroutine rh_clouds_init (nlon, nlat, nlev) !---------- output namelist to log------------------------------------- if ( mpp_pe() == mpp_root_pe() ) then - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() write (logunit, nml=rh_clouds_nml) endif diff --git a/src/atmos_param/sea_esf_rad/aerosol.F90 b/src/atmos_param/sea_esf_rad/aerosol.F90 index ceca7f00e6..d35f4c945a 100644 --- a/src/atmos_param/sea_esf_rad/aerosol.F90 +++ b/src/atmos_param/sea_esf_rad/aerosol.F90 @@ -397,7 +397,7 @@ subroutine aerosol_init (lonb, latb, aerosol_names, & !--------------------------------------------------------------------- ! write version number and namelist to logfile. !--------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if (mpp_pe() == mpp_root_pe() ) & write (logunit, nml=aerosol_nml) diff --git a/src/atmos_param/sea_esf_rad/aerosolrad_package.F90 b/src/atmos_param/sea_esf_rad/aerosolrad_package.F90 index d3221f0b32..2c01d07d2e 100644 --- a/src/atmos_param/sea_esf_rad/aerosolrad_package.F90 +++ b/src/atmos_param/sea_esf_rad/aerosolrad_package.F90 @@ -645,7 +645,7 @@ subroutine aerosolrad_package_init (kmax, aerosol_names, lonb, latb) !--------------------------------------------------------------------- ! write version number and namelist to logfile. !--------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if (mpp_pe() == mpp_root_pe() ) & write (logunit, nml=aerosolrad_package_nml) diff --git a/src/atmos_param/sea_esf_rad/bulkphys_rad.F90 b/src/atmos_param/sea_esf_rad/bulkphys_rad.F90 index 8cbe6aaaa9..97e6faf76d 100644 --- a/src/atmos_param/sea_esf_rad/bulkphys_rad.F90 +++ b/src/atmos_param/sea_esf_rad/bulkphys_rad.F90 @@ -278,7 +278,7 @@ subroutine bulkphys_rad_init (min_cld_drop_rad_in, max_cld_drop_rad_in,& !--------------------------------------------------------------------- ! write namelist to logfile. !--------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if (mpp_pe() == mpp_root_pe() ) & write (logunit, nml=bulkphys_rad_nml) diff --git a/src/atmos_param/sea_esf_rad/cloud_spec.F90 b/src/atmos_param/sea_esf_rad/cloud_spec.F90 index 742e63b3bd..bf97dd400f 100644 --- a/src/atmos_param/sea_esf_rad/cloud_spec.F90 +++ b/src/atmos_param/sea_esf_rad/cloud_spec.F90 @@ -355,7 +355,7 @@ subroutine cloud_spec_init (pref, lonb, latb, axes, Time, & !---------------------------------------------------------------------- ! write version number and namelist to logfile. !--------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if (mpp_pe() == mpp_root_pe() ) & write (logunit, nml=cloud_spec_nml) diff --git a/src/atmos_param/sea_esf_rad/cloudrad_diagnostics.F90 b/src/atmos_param/sea_esf_rad/cloudrad_diagnostics.F90 index dd31ed5582..3b4f6a02ef 100644 --- a/src/atmos_param/sea_esf_rad/cloudrad_diagnostics.F90 +++ b/src/atmos_param/sea_esf_rad/cloudrad_diagnostics.F90 @@ -450,7 +450,7 @@ subroutine cloudrad_diagnostics_init (min_cld_drop_rad_in, & !--------------------------------------------------------------------- ! write namelist to logfile. !--------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if (mpp_pe() == mpp_root_pe() ) & write (logunit, nml=cloudrad_diagnostics_nml) diff --git a/src/atmos_param/sea_esf_rad/cloudrad_package.F90 b/src/atmos_param/sea_esf_rad/cloudrad_package.F90 index f76cfeeeec..8609452a4f 100644 --- a/src/atmos_param/sea_esf_rad/cloudrad_package.F90 +++ b/src/atmos_param/sea_esf_rad/cloudrad_package.F90 @@ -233,7 +233,7 @@ subroutine cloudrad_package_init (pref, lonb, latb, axes, Time, & !--------------------------------------------------------------------- ! write version number and namelist to logfile. !--------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if (mpp_pe() == mpp_root_pe() ) & write (logunit, nml=cloudrad_package_nml) diff --git a/src/atmos_param/sea_esf_rad/diag_clouds_W.F90 b/src/atmos_param/sea_esf_rad/diag_clouds_W.F90 index 60f8fb6179..02773300ed 100644 --- a/src/atmos_param/sea_esf_rad/diag_clouds_W.F90 +++ b/src/atmos_param/sea_esf_rad/diag_clouds_W.F90 @@ -142,7 +142,7 @@ subroutine diag_clouds_W_init (num_slingo_bands_out) #endif if ( mpp_pe() == mpp_root_pe() ) then - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() write (logunit,nml=diag_clouds_W_nml) endif diff --git a/src/atmos_param/sea_esf_rad/donner_deep_clouds_W.F90 b/src/atmos_param/sea_esf_rad/donner_deep_clouds_W.F90 index d14a1e2fd1..2bc56f0693 100644 --- a/src/atmos_param/sea_esf_rad/donner_deep_clouds_W.F90 +++ b/src/atmos_param/sea_esf_rad/donner_deep_clouds_W.F90 @@ -141,7 +141,7 @@ subroutine donner_deep_clouds_W_init (pref, lonb, latb, axes, Time) #endif if ( mpp_pe() == mpp_root_pe() ) then - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() write (logunit,nml=donner_deep_clouds_W_nml) endif diff --git a/src/atmos_param/sea_esf_rad/esfsw_driver.F90 b/src/atmos_param/sea_esf_rad/esfsw_driver.F90 index df0e48f2e3..60dfe6253e 100644 --- a/src/atmos_param/sea_esf_rad/esfsw_driver.F90 +++ b/src/atmos_param/sea_esf_rad/esfsw_driver.F90 @@ -416,7 +416,7 @@ subroutine esfsw_driver_init !--------------------------------------------------------------------- ! write version number and namelist to logfile. !--------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if (mpp_pe() == mpp_root_pe() ) & write (logunit, nml=esfsw_driver_nml) diff --git a/src/atmos_param/sea_esf_rad/esfsw_parameters.F90 b/src/atmos_param/sea_esf_rad/esfsw_parameters.F90 index 29135df08b..67cd0c4c86 100644 --- a/src/atmos_param/sea_esf_rad/esfsw_parameters.F90 +++ b/src/atmos_param/sea_esf_rad/esfsw_parameters.F90 @@ -207,7 +207,7 @@ subroutine esfsw_parameters_init ! write version number and namelist to logfile also write out ! some key parameters obtained from an input data file. !--------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if (mpp_pe() == mpp_root_pe() ) then write (logunit,9000) & diff --git a/src/atmos_param/sea_esf_rad/gas_tf.F90 b/src/atmos_param/sea_esf_rad/gas_tf.F90 index 2efd3b55f4..1e1e31ab76 100644 --- a/src/atmos_param/sea_esf_rad/gas_tf.F90 +++ b/src/atmos_param/sea_esf_rad/gas_tf.F90 @@ -442,7 +442,7 @@ subroutine gas_tf_init (pref) !--------------------------------------------------------------------- ! write version number and namelist to logfile. !--------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if (mpp_pe() == mpp_root_pe() ) & write (logunit, nml=gas_tf_nml) diff --git a/src/atmos_param/sea_esf_rad/isccp_clouds.F90 b/src/atmos_param/sea_esf_rad/isccp_clouds.F90 index 46bf755134..15b46879a1 100644 --- a/src/atmos_param/sea_esf_rad/isccp_clouds.F90 +++ b/src/atmos_param/sea_esf_rad/isccp_clouds.F90 @@ -308,7 +308,7 @@ subroutine isccp_clouds_init (axes, Time) !--------------------------------------------------------------------- ! write namelist to logfile. !--------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if (mpp_pe() == mpp_root_pe() ) & write (logunit, nml=isccp_clouds_nml) diff --git a/src/atmos_param/sea_esf_rad/lhsw_driver.F90 b/src/atmos_param/sea_esf_rad/lhsw_driver.F90 index 3826272c39..8829545d5f 100644 --- a/src/atmos_param/sea_esf_rad/lhsw_driver.F90 +++ b/src/atmos_param/sea_esf_rad/lhsw_driver.F90 @@ -210,7 +210,7 @@ subroutine lhsw_driver_init ( pref ) #endif if ( mpp_pe() == mpp_root_pe() ) then - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() write (logunit,nml=lhsw_driver_nml) endif diff --git a/src/atmos_param/sea_esf_rad/longwave_clouds.F90 b/src/atmos_param/sea_esf_rad/longwave_clouds.F90 index c9692f4c19..3a908eb263 100644 --- a/src/atmos_param/sea_esf_rad/longwave_clouds.F90 +++ b/src/atmos_param/sea_esf_rad/longwave_clouds.F90 @@ -162,7 +162,7 @@ subroutine longwave_clouds_init !--------------------------------------------------------------------- ! write version number and namelist to logfile. !--------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if (mpp_pe() == mpp_root_pe() ) & write (logunit, nml=longwave_clouds_nml) diff --git a/src/atmos_param/sea_esf_rad/longwave_driver.F90 b/src/atmos_param/sea_esf_rad/longwave_driver.F90 index fb57efe6c7..33b4ef1221 100644 --- a/src/atmos_param/sea_esf_rad/longwave_driver.F90 +++ b/src/atmos_param/sea_esf_rad/longwave_driver.F90 @@ -204,7 +204,7 @@ subroutine longwave_driver_init (latb, lonb, pref, Lw_tables) !--------------------------------------------------------------------- ! write version number and namelist to logfile. !--------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if (mpp_pe() == mpp_root_pe() ) & write (logunit, nml=longwave_driver_nml) diff --git a/src/atmos_param/sea_esf_rad/longwave_fluxes.F90 b/src/atmos_param/sea_esf_rad/longwave_fluxes.F90 index d06c260878..2bad7584f0 100644 --- a/src/atmos_param/sea_esf_rad/longwave_fluxes.F90 +++ b/src/atmos_param/sea_esf_rad/longwave_fluxes.F90 @@ -144,7 +144,7 @@ subroutine longwave_fluxes_init !--------------------------------------------------------------------- ! write version number and namelist to logfile. !--------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if (mpp_pe() == mpp_root_pe() ) & write (logunit, nml=longwave_fluxes_nml) diff --git a/src/atmos_param/sea_esf_rad/longwave_params.F90 b/src/atmos_param/sea_esf_rad/longwave_params.F90 index 9443148533..cefd2d8e05 100644 --- a/src/atmos_param/sea_esf_rad/longwave_params.F90 +++ b/src/atmos_param/sea_esf_rad/longwave_params.F90 @@ -166,7 +166,7 @@ subroutine longwave_params_init !--------------------------------------------------------------------- ! write version number and namelist to logfile. !--------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if (mpp_pe() == mpp_root_pe() ) then write (logunit, nml=longwave_params_nml) diff --git a/src/atmos_param/sea_esf_rad/longwave_tables.F90 b/src/atmos_param/sea_esf_rad/longwave_tables.F90 index 979ad6893f..af7c7848a1 100644 --- a/src/atmos_param/sea_esf_rad/longwave_tables.F90 +++ b/src/atmos_param/sea_esf_rad/longwave_tables.F90 @@ -295,7 +295,7 @@ subroutine longwave_tables_init (Lw_tables, tabsr, tab1, tab2, tab3, & !--------------------------------------------------------------------- ! write version number and namelist to logfile. !--------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if (mpp_pe() == mpp_root_pe() ) & write (logunit, nml=longwave_tables_nml) diff --git a/src/atmos_param/sea_esf_rad/lw_gases_stdtf.F90 b/src/atmos_param/sea_esf_rad/lw_gases_stdtf.F90 index 64dcf591af..bb38e5688e 100644 --- a/src/atmos_param/sea_esf_rad/lw_gases_stdtf.F90 +++ b/src/atmos_param/sea_esf_rad/lw_gases_stdtf.F90 @@ -403,7 +403,7 @@ subroutine lw_gases_stdtf_init ( pref) !--------------------------------------------------------------------- ! write version number and namelist to logfile. !--------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if (mpp_pe() == mpp_root_pe() ) & write (logunit, nml=lw_gases_stdtf_nml) diff --git a/src/atmos_param/sea_esf_rad/mgrp_prscr_clds.F90 b/src/atmos_param/sea_esf_rad/mgrp_prscr_clds.F90 index 0964c8c6f1..99561f7144 100644 --- a/src/atmos_param/sea_esf_rad/mgrp_prscr_clds.F90 +++ b/src/atmos_param/sea_esf_rad/mgrp_prscr_clds.F90 @@ -232,7 +232,7 @@ subroutine mgrp_prscr_clds_init ( pref, latb ) endif #endif - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if (mpp_pe() == mpp_root_pe() ) & write (logunit, nml = mgrp_prscr_clds_nml) diff --git a/src/atmos_param/sea_esf_rad/microphys_cloud.F90 b/src/atmos_param/sea_esf_rad/microphys_cloud.F90 index 5e670efd4d..fab5388e3e 100644 --- a/src/atmos_param/sea_esf_rad/microphys_cloud.F90 +++ b/src/atmos_param/sea_esf_rad/microphys_cloud.F90 @@ -238,7 +238,7 @@ subroutine microphys_cloud_init ! write version number and namelist to logfile. !------------------------------------------------------------------------ - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if (mpp_pe() == mpp_root_pe() ) & write (logunit, nml=microphys_cloud_nml) diff --git a/src/atmos_param/sea_esf_rad/microphys_rad.F90 b/src/atmos_param/sea_esf_rad/microphys_rad.F90 index ee3e034c3d..dba67ac5cc 100644 --- a/src/atmos_param/sea_esf_rad/microphys_rad.F90 +++ b/src/atmos_param/sea_esf_rad/microphys_rad.F90 @@ -544,7 +544,7 @@ subroutine microphys_rad_init (min_cld_drop_rad_in, max_cld_drop_rad_in, & !--------------------------------------------------------------------- ! write namelist and version number to logfile. !--------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if (mpp_pe() == mpp_root_pe() ) & write (logunit, nml=microphys_rad_nml) diff --git a/src/atmos_param/sea_esf_rad/optical_path.F90 b/src/atmos_param/sea_esf_rad/optical_path.F90 index b859a52dc5..e6c046c2fa 100644 --- a/src/atmos_param/sea_esf_rad/optical_path.F90 +++ b/src/atmos_param/sea_esf_rad/optical_path.F90 @@ -371,7 +371,7 @@ subroutine optical_path_init(pref) !--------------------------------------------------------------------- ! write version number and namelist to logfile. !--------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if (mpp_pe() == mpp_root_pe() ) & write (logunit, nml=optical_path_nml) diff --git a/src/atmos_param/sea_esf_rad/original_fms_rad.F90 b/src/atmos_param/sea_esf_rad/original_fms_rad.F90 index dab057d021..19fa1a13d4 100644 --- a/src/atmos_param/sea_esf_rad/original_fms_rad.F90 +++ b/src/atmos_param/sea_esf_rad/original_fms_rad.F90 @@ -324,7 +324,7 @@ subroutine original_fms_rad_init ( lonb, latb, pref, axes, Time , & ! write namelist to logfile. !--------------------------------------------------------------------- if ( mpp_pe() == mpp_root_pe() ) then - call write_version_number() + call write_version_number(version, tagname) write (unit, nml=original_fms_rad_nml) endif diff --git a/src/atmos_param/sea_esf_rad/ozone.F90 b/src/atmos_param/sea_esf_rad/ozone.F90 index 944eecdfab..03e32fa8c5 100644 --- a/src/atmos_param/sea_esf_rad/ozone.F90 +++ b/src/atmos_param/sea_esf_rad/ozone.F90 @@ -334,7 +334,7 @@ subroutine ozone_init (latb, lonb) !--------------------------------------------------------------------- ! write version number and namelist to logfile. !--------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if (mpp_pe() == mpp_root_pe() ) & write (logunit, nml=ozone_nml) diff --git a/src/atmos_param/sea_esf_rad/rad_output_file.F90 b/src/atmos_param/sea_esf_rad/rad_output_file.F90 index 2f63674dd9..af8eba8d18 100644 --- a/src/atmos_param/sea_esf_rad/rad_output_file.F90 +++ b/src/atmos_param/sea_esf_rad/rad_output_file.F90 @@ -301,7 +301,7 @@ subroutine rad_output_file_init (axes, Time, names, family_names) !--------------------------------------------------------------------- ! write version number and namelist to logfile. !--------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if (mpp_pe() == mpp_root_pe() ) & write (logunit, nml=rad_output_file_nml) diff --git a/src/atmos_param/sea_esf_rad/rad_utilities.F90 b/src/atmos_param/sea_esf_rad/rad_utilities.F90 index d27808f85a..6a09174b7e 100644 --- a/src/atmos_param/sea_esf_rad/rad_utilities.F90 +++ b/src/atmos_param/sea_esf_rad/rad_utilities.F90 @@ -1367,7 +1367,7 @@ subroutine rad_utilities_init !--------------------------------------------------------------------- ! write version number and namelist to logfile. !--------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if (mpp_pe() == mpp_root_pe() ) & write (logunit, nml=rad_utilities_nml) diff --git a/src/atmos_param/sea_esf_rad/radiation_diag.F90 b/src/atmos_param/sea_esf_rad/radiation_diag.F90 index 93e83c09db..b7a8715248 100644 --- a/src/atmos_param/sea_esf_rad/radiation_diag.F90 +++ b/src/atmos_param/sea_esf_rad/radiation_diag.F90 @@ -252,7 +252,7 @@ subroutine radiation_diag_init (latb, lonb, Lw_tables) !--------------------------------------------------------------------- ! write version number and namelist to logfile. !--------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if (mpp_pe() == mpp_root_pe() ) & write (logunit, nml=radiation_diag_nml) diff --git a/src/atmos_param/sea_esf_rad/radiative_gases.F90 b/src/atmos_param/sea_esf_rad/radiative_gases.F90 index 8baa0a37ce..4e22a15e13 100644 --- a/src/atmos_param/sea_esf_rad/radiative_gases.F90 +++ b/src/atmos_param/sea_esf_rad/radiative_gases.F90 @@ -536,7 +536,7 @@ subroutine radiative_gases_init (pref, latb, lonb) !--------------------------------------------------------------------- ! write version number and namelist to logfile. !--------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if (mpp_pe() == mpp_root_pe() ) & write (logunit, nml=radiative_gases_nml) diff --git a/src/atmos_param/sea_esf_rad/rh_based_clouds.F90 b/src/atmos_param/sea_esf_rad/rh_based_clouds.F90 index 0fc84f9a8b..709fe082b5 100644 --- a/src/atmos_param/sea_esf_rad/rh_based_clouds.F90 +++ b/src/atmos_param/sea_esf_rad/rh_based_clouds.F90 @@ -249,7 +249,7 @@ subroutine rh_based_clouds_init !---------------------------------------------------------------------- ! write version number and namelist to logfile. !--------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if (mpp_pe() == mpp_root_pe() ) & write (logunit, nml=rh_based_clouds_nml) diff --git a/src/atmos_param/sea_esf_rad/sea_esf_rad.F90 b/src/atmos_param/sea_esf_rad/sea_esf_rad.F90 index 1f2f8f3cd4..21f555992f 100644 --- a/src/atmos_param/sea_esf_rad/sea_esf_rad.F90 +++ b/src/atmos_param/sea_esf_rad/sea_esf_rad.F90 @@ -226,7 +226,7 @@ subroutine sea_esf_rad_init (lonb, latb, pref_r) !--------------------------------------------------------------------- ! write version number and namelist to logfile. !--------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if (mpp_pe() == mpp_root_pe() ) & write (logunit, nml=sea_esf_rad_nml) diff --git a/src/atmos_param/sea_esf_rad/sealw99.F90 b/src/atmos_param/sea_esf_rad/sealw99.F90 index dbd311b0a6..baedda2838 100644 --- a/src/atmos_param/sea_esf_rad/sealw99.F90 +++ b/src/atmos_param/sea_esf_rad/sealw99.F90 @@ -488,7 +488,7 @@ subroutine sealw99_init (latb, lonb, pref, Lw_tables) !--------------------------------------------------------------------- ! write version number and namelist to logfile. !--------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if (mpp_pe() == mpp_root_pe() ) & write (logunit, nml=sealw99_nml) diff --git a/src/atmos_param/sea_esf_rad/shortwave_driver.F90 b/src/atmos_param/sea_esf_rad/shortwave_driver.F90 index 0acda38c20..8ae80c80d0 100644 --- a/src/atmos_param/sea_esf_rad/shortwave_driver.F90 +++ b/src/atmos_param/sea_esf_rad/shortwave_driver.F90 @@ -207,7 +207,7 @@ subroutine shortwave_driver_init (latb, pref) !--------------------------------------------------------------------- ! write version number and namelist to logfile. !--------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if (mpp_pe() == mpp_root_pe() ) & write (logunit, nml=shortwave_driver_nml) diff --git a/src/atmos_param/sea_esf_rad/specified_clouds_W.F90 b/src/atmos_param/sea_esf_rad/specified_clouds_W.F90 index 039d4310d9..01e631bdfa 100644 --- a/src/atmos_param/sea_esf_rad/specified_clouds_W.F90 +++ b/src/atmos_param/sea_esf_rad/specified_clouds_W.F90 @@ -170,7 +170,7 @@ subroutine specified_clouds_W_init (lonb, latb) !------- write version number and namelist --------- if ( mpp_pe() == mpp_root_pe() ) then - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() write (logunit,nml=specified_clouds_W_nml) endif diff --git a/src/atmos_param/sea_esf_rad/standalone_clouds.F90 b/src/atmos_param/sea_esf_rad/standalone_clouds.F90 index b2b1944c4a..dc7469a313 100644 --- a/src/atmos_param/sea_esf_rad/standalone_clouds.F90 +++ b/src/atmos_param/sea_esf_rad/standalone_clouds.F90 @@ -272,7 +272,7 @@ subroutine standalone_clouds_init (pref, lonb, latb) !--------------------------------------------------------------------- ! write namelist to logfile. !--------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if (mpp_pe() == mpp_root_pe() ) & write (logunit, nml=standalone_clouds_nml) diff --git a/src/atmos_param/sea_esf_rad/strat_clouds_W.F90 b/src/atmos_param/sea_esf_rad/strat_clouds_W.F90 index df53e32a18..5a30139c1f 100644 --- a/src/atmos_param/sea_esf_rad/strat_clouds_W.F90 +++ b/src/atmos_param/sea_esf_rad/strat_clouds_W.F90 @@ -187,7 +187,7 @@ subroutine strat_clouds_W_init(latb, lonb) !---------------------------------------------------------------------- ! write version number and namelist to logfile. !--------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if (mpp_pe() == mpp_root_pe() ) & write (logunit, nml=strat_clouds_W_nml) diff --git a/src/atmos_param/sea_esf_rad/uw_clouds_W.F90 b/src/atmos_param/sea_esf_rad/uw_clouds_W.F90 index 09114bcb26..512e85c31a 100644 --- a/src/atmos_param/sea_esf_rad/uw_clouds_W.F90 +++ b/src/atmos_param/sea_esf_rad/uw_clouds_W.F90 @@ -152,7 +152,7 @@ subroutine uw_clouds_W_init (pref, lonb, latb, axes, Time) #endif if ( mpp_pe() == mpp_root_pe() ) then - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() write (logunit,nml=uw_clouds_W_nml) endif diff --git a/src/atmos_param/sea_esf_rad/zetac_clouds_W.F90 b/src/atmos_param/sea_esf_rad/zetac_clouds_W.F90 index 10eac377c4..7ccaacc0d2 100644 --- a/src/atmos_param/sea_esf_rad/zetac_clouds_W.F90 +++ b/src/atmos_param/sea_esf_rad/zetac_clouds_W.F90 @@ -160,7 +160,7 @@ subroutine zetac_clouds_W_init !---------------------------------------------------------------------- ! write version number and namelist to logfile. !--------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if (mpp_pe() == mpp_root_pe() ) & write (logunit, nml=zetac_clouds_W_nml) diff --git a/src/atmos_param/shallow_conv/shallow_conv.F90 b/src/atmos_param/shallow_conv/shallow_conv.F90 index 9f50d71437..5611243ec5 100644 --- a/src/atmos_param/shallow_conv/shallow_conv.F90 +++ b/src/atmos_param/shallow_conv/shallow_conv.F90 @@ -108,7 +108,7 @@ SUBROUTINE SHALLOW_CONV_INIT( kx ) !------- write version number and namelist --------- if ( mpp_pe() == mpp_root_pe() ) then - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() WRITE( logunit, nml = shallow_conv_nml ) endif diff --git a/src/atmos_param/shallow_cu/conv_closures.F90 b/src/atmos_param/shallow_cu/conv_closures.F90 index 6ce5356387..1e7b4d041e 100644 --- a/src/atmos_param/shallow_cu/conv_closures.F90 +++ b/src/atmos_param/shallow_cu/conv_closures.F90 @@ -426,7 +426,7 @@ subroutine conv_closures_init !--------------------------------------------------------------------- ! write version number and namelist to logfile. !--------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) !--------------------------------------------------------------------- ! mark the module as initialized. !--------------------------------------------------------------------- diff --git a/src/atmos_param/shallow_cu/uw_conv.F90 b/src/atmos_param/shallow_cu/uw_conv.F90 index d4a1985fef..60ee4d4ee6 100644 --- a/src/atmos_param/shallow_cu/uw_conv.F90 +++ b/src/atmos_param/shallow_cu/uw_conv.F90 @@ -336,7 +336,7 @@ SUBROUTINE UW_CONV_INIT(do_strat, axes, Time, kd, tracers_in_uw) !========Option for deep convection======================================= end if #endif - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() WRITE( logunit, nml = uw_closure_nml ) WRITE( logunit, nml = uw_conv_nml ) diff --git a/src/atmos_param/stable_bl_turb/stable_bl_turb.F90 b/src/atmos_param/stable_bl_turb/stable_bl_turb.F90 index da684668dc..c1caebf199 100644 --- a/src/atmos_param/stable_bl_turb/stable_bl_turb.F90 +++ b/src/atmos_param/stable_bl_turb/stable_bl_turb.F90 @@ -504,7 +504,7 @@ subroutine STABLE_BL_TURB_INIT ( axes, Time ) !--------------------------------------------------------------------- if ( mpp_pe() == mpp_root_pe() ) then - call write_version_number() + call write_version_number(version, tagname) unit = stdlog() WRITE( unit, nml = stable_bl_turb_nml ) endif diff --git a/src/atmos_param/strat_cloud/aerosol_cloud.F90 b/src/atmos_param/strat_cloud/aerosol_cloud.F90 index 34dbf03106..8b1ee7b62a 100644 --- a/src/atmos_param/strat_cloud/aerosol_cloud.F90 +++ b/src/atmos_param/strat_cloud/aerosol_cloud.F90 @@ -125,7 +125,7 @@ subroutine aerosol_cloud_init (Constants) !------------------------------------------------------------------------- ! write version and namelist to standard log. !------------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if (mpp_pe() == mpp_root_pe()) & write (logunit, nml=aerosol_cloud_nml) diff --git a/src/atmos_param/strat_cloud/check_nan.F90 b/src/atmos_param/strat_cloud/check_nan.F90 index 99686863fd..cc1b7cfb78 100644 --- a/src/atmos_param/strat_cloud/check_nan.F90 +++ b/src/atmos_param/strat_cloud/check_nan.F90 @@ -32,7 +32,7 @@ subroutine check_nan_init !------------------------------------------------------------------------ ! write version number to output file. !------------------------------------------------------------------------ - call write_version_number() + call write_version_number(version, tagname) !------------------------------------------------------------------------ ! declare module initialized. diff --git a/src/atmos_param/strat_cloud/cldwat2m_micro.F90 b/src/atmos_param/strat_cloud/cldwat2m_micro.F90 index b40a9cf8ae..e2388a496f 100644 --- a/src/atmos_param/strat_cloud/cldwat2m_micro.F90 +++ b/src/atmos_param/strat_cloud/cldwat2m_micro.F90 @@ -392,7 +392,7 @@ subroutine ini_micro !----------------------------------------------------------------------- ! write version and namelist to stdlog. !----------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if (mpp_pe() == mpp_root_pe()) & write (logunit, nml=cldwat2m_micro_nml) diff --git a/src/atmos_param/strat_cloud/gamma_mg.F90 b/src/atmos_param/strat_cloud/gamma_mg.F90 index 1e23c7a3e7..608d261acd 100644 --- a/src/atmos_param/strat_cloud/gamma_mg.F90 +++ b/src/atmos_param/strat_cloud/gamma_mg.F90 @@ -31,7 +31,7 @@ SUBROUTINE gamma_mg_init !----------------------------------------------------------------------- ! write version number to output file. !----------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) !----------------------------------------------------------------------- ! be sure needed modules are initialized. diff --git a/src/atmos_param/strat_cloud/mg_const.F90 b/src/atmos_param/strat_cloud/mg_const.F90 index f92a1c5281..e77af929ba 100644 --- a/src/atmos_param/strat_cloud/mg_const.F90 +++ b/src/atmos_param/strat_cloud/mg_const.F90 @@ -58,7 +58,7 @@ subroutine mg_const_init !------------------------------------------------------------------------- ! write version number to output file. !------------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) !------------------------------------------------------------------------- ! mark the module as initialized. diff --git a/src/atmos_param/strat_cloud/microphysics.F90 b/src/atmos_param/strat_cloud/microphysics.F90 index be0636d2c5..b91d9b8395 100644 --- a/src/atmos_param/strat_cloud/microphysics.F90 +++ b/src/atmos_param/strat_cloud/microphysics.F90 @@ -57,7 +57,7 @@ subroutine microphysics_init (Nml) !------------------------------------------------------------------------- ! write version number to output file. !------------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) !------------------------------------------------------------------------- ! make sure needed modules have been initialized. diff --git a/src/atmos_param/strat_cloud/morrison_gettelman_microp.F90 b/src/atmos_param/strat_cloud/morrison_gettelman_microp.F90 index 35ee347978..00d62b0bd3 100644 --- a/src/atmos_param/strat_cloud/morrison_gettelman_microp.F90 +++ b/src/atmos_param/strat_cloud/morrison_gettelman_microp.F90 @@ -272,7 +272,7 @@ SUBROUTINE morrison_gettelman_microp_init (do_pdf_clouds, qcvar_in) !----------------------------------------------------------------------- ! write version and namelist to stdlog. !----------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if (mpp_pe() == mpp_root_pe()) & write (logunit, nml=morrison_gettelman_microp_nml) diff --git a/src/atmos_param/strat_cloud/nc_cond.F90 b/src/atmos_param/strat_cloud/nc_cond.F90 index 86e0317c93..2970ec7441 100644 --- a/src/atmos_param/strat_cloud/nc_cond.F90 +++ b/src/atmos_param/strat_cloud/nc_cond.F90 @@ -102,7 +102,7 @@ SUBROUTINE nc_cond_init( do_pdf_clouds ) !------------------------------------------------------------------------- ! write version and namelist to standard log. !------------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if ( mpp_pe() == mpp_root_pe() ) & write ( logunit, nml=nc_cond_nml ) diff --git a/src/atmos_param/strat_cloud/polysvp.F90 b/src/atmos_param/strat_cloud/polysvp.F90 index 15f5af2356..79f95f80bf 100644 --- a/src/atmos_param/strat_cloud/polysvp.F90 +++ b/src/atmos_param/strat_cloud/polysvp.F90 @@ -54,7 +54,7 @@ SUBROUTINE polysvp_init !------------------------------------------------------------------------ ! write version number to output file. !------------------------------------------------------------------------ - call write_version_number() + call write_version_number(version, tagname) !------------------------------------------------------------------------ ! make sure needed modules have been initialized. diff --git a/src/atmos_param/strat_cloud/rotstayn_klein_mp.F90 b/src/atmos_param/strat_cloud/rotstayn_klein_mp.F90 index 9309a3cfbc..c055872cf6 100644 --- a/src/atmos_param/strat_cloud/rotstayn_klein_mp.F90 +++ b/src/atmos_param/strat_cloud/rotstayn_klein_mp.F90 @@ -110,7 +110,7 @@ SUBROUTINE rotstayn_klein_microp_init !------------------------------------------------------------------------- ! write version and namelist to standard log. !------------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if ( mpp_pe() == mpp_root_pe() ) & write (logunit, nml=rotstayn_klein_mp_nml) diff --git a/src/atmos_param/strat_cloud/simple_pdf.F90 b/src/atmos_param/strat_cloud/simple_pdf.F90 index 20c46f441c..e12978c4cb 100644 --- a/src/atmos_param/strat_cloud/simple_pdf.F90 +++ b/src/atmos_param/strat_cloud/simple_pdf.F90 @@ -40,7 +40,7 @@ SUBROUTINE simple_pdf_init !----------------------------------------------------------------------- ! write version number to output file. !----------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) !----------------------------------------------------------------------- ! make sure needed modules have been initialized. diff --git a/src/atmos_param/strat_cloud/strat_cloud.F90 b/src/atmos_param/strat_cloud/strat_cloud.F90 index fa5cddafc4..5e24c4a7d1 100644 --- a/src/atmos_param/strat_cloud/strat_cloud.F90 +++ b/src/atmos_param/strat_cloud/strat_cloud.F90 @@ -396,7 +396,7 @@ subroutine strat_cloud_init (axes, Time, idim, jdim, kdim, & !----------------------------------------------------------------------- ! write version and namelist to stdlog. !----------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if ( mpp_pe() == mpp_root_pe() ) & write (logunit, nml=strat_cloud_nml) diff --git a/src/atmos_param/strat_cloud/strat_cloud_legacy.F90 b/src/atmos_param/strat_cloud/strat_cloud_legacy.F90 index 8be3b2509b..a1242546d4 100644 --- a/src/atmos_param/strat_cloud/strat_cloud_legacy.F90 +++ b/src/atmos_param/strat_cloud/strat_cloud_legacy.F90 @@ -345,7 +345,7 @@ subroutine strat_cloud_legacy_init (do_pdf_clouds) !----------------------------------------------------------------------- ! write version number to output file. !----------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) !----------------------------------------------------------------------- ! make sure needed modules have been initialized. diff --git a/src/atmos_param/strat_cloud/strat_cloud_utilities.F90 b/src/atmos_param/strat_cloud/strat_cloud_utilities.F90 index 4e91a6f8ea..7a9f0ecd7f 100644 --- a/src/atmos_param/strat_cloud/strat_cloud_utilities.F90 +++ b/src/atmos_param/strat_cloud/strat_cloud_utilities.F90 @@ -474,7 +474,7 @@ subroutine strat_cloud_utilities_init !------------------------------------------------------------------------ ! write version number to output file. !------------------------------------------------------------------------ - call write_version_number() + call write_version_number(version, tagname) !------------------------------------------------------------------------ ! mark this module as initialized. diff --git a/src/atmos_param/strat_cloud/strat_netcdf.F90 b/src/atmos_param/strat_cloud/strat_netcdf.F90 index 9797d1c390..387717e935 100644 --- a/src/atmos_param/strat_cloud/strat_netcdf.F90 +++ b/src/atmos_param/strat_cloud/strat_netcdf.F90 @@ -52,7 +52,7 @@ subroutine strat_netcdf_init (axes, Time, diag_id, diag_pt, n_diag_4d, & !----------------------------------------------------------------------- ! write version info to standard log. !----------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) !------------------------------------------------------------------------ ! make sure needed modules have been initialized. diff --git a/src/atmos_param/topo_drag/topo_drag.F90 b/src/atmos_param/topo_drag/topo_drag.F90 index efdfd9f137..f76b7b5eb3 100644 --- a/src/atmos_param/topo_drag/topo_drag.F90 +++ b/src/atmos_param/topo_drag/topo_drag.F90 @@ -601,7 +601,7 @@ subroutine topo_drag_init (lonb, latb) ! write version number and namelist to logfile - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if (mpp_pe() == mpp_root_pe()) & write (logunit, nml=topo_drag_nml) diff --git a/src/atmos_param/vert_diff/vert_diff.F90 b/src/atmos_param/vert_diff/vert_diff.F90 index bac7a3c4f0..eb3452f59e 100644 --- a/src/atmos_param/vert_diff/vert_diff.F90 +++ b/src/atmos_param/vert_diff/vert_diff.F90 @@ -113,7 +113,7 @@ subroutine vert_diff_init (Tri_surf, idim, jdim, kdim, & character(len=128) :: scheme ! tracer diffusion scheme integer :: n, logunit - call write_version_number() + call write_version_number(version, tagname) ! get the number of prognostic tracers call get_number_tracers( MODEL_ATMOS, num_prog=ntprog) diff --git a/src/atmos_param/vert_diff_driver/vert_diff_driver.F90 b/src/atmos_param/vert_diff_driver/vert_diff_driver.F90 index 72bdf594d7..a8363ab5b2 100644 --- a/src/atmos_param/vert_diff_driver/vert_diff_driver.F90 +++ b/src/atmos_param/vert_diff_driver/vert_diff_driver.F90 @@ -411,7 +411,7 @@ subroutine vert_diff_driver_init ( Surf_diff, idim, jdim, kdim, & !--------- write version number and namelist ------------------ - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if(mpp_pe() == mpp_root_pe() ) write(logunit,nml=vert_diff_driver_nml) diff --git a/src/atmos_param/vert_turb_driver/vert_turb_driver.F90 b/src/atmos_param/vert_turb_driver/vert_turb_driver.F90 index b30917171f..cebd8d8571 100644 --- a/src/atmos_param/vert_turb_driver/vert_turb_driver.F90 +++ b/src/atmos_param/vert_turb_driver/vert_turb_driver.F90 @@ -643,7 +643,7 @@ subroutine vert_turb_driver_init (lonb, latb, id, jd, kd, axes, Time, & logunit = stdlog() if ( mpp_pe() == mpp_root_pe() ) then - call write_version_number() + call write_version_number(version, tagname) write (logunit,nml=vert_turb_driver_nml) endif diff --git a/src/atmos_shared/atmos_nudge/atmos_nudge.F90 b/src/atmos_shared/atmos_nudge/atmos_nudge.F90 index 53bf4c973f..73a0c50b87 100644 --- a/src/atmos_shared/atmos_nudge/atmos_nudge.F90 +++ b/src/atmos_shared/atmos_nudge/atmos_nudge.F90 @@ -314,7 +314,7 @@ subroutine atmos_nudge_init ( Time, axes, flag ) enddo 10 call close_file (unit) #endif - call write_version_number() + call write_version_number(version, tagname) logunit=stdlog() if (mpp_pe() == mpp_root_pe()) write (logunit, nml=atmos_nudge_nml) diff --git a/src/atmos_shared/interpolator/interpolator.F90 b/src/atmos_shared/interpolator/interpolator.F90 index 7efa88f232..c8283768dd 100644 --- a/src/atmos_shared/interpolator/interpolator.F90 +++ b/src/atmos_shared/interpolator/interpolator.F90 @@ -1026,7 +1026,7 @@ subroutine interpolator_init( clim_type, file_name, lonb_mod, latb_mod, & module_is_initialized = .true. -call write_version_number() +call write_version_number(version, tagname) end subroutine interpolator_init diff --git a/src/atmos_shared/tracer_driver/aer_ccn_act/aer_ccn_act.F90 b/src/atmos_shared/tracer_driver/aer_ccn_act/aer_ccn_act.F90 index 180a5fa493..58d6705769 100644 --- a/src/atmos_shared/tracer_driver/aer_ccn_act/aer_ccn_act.F90 +++ b/src/atmos_shared/tracer_driver/aer_ccn_act/aer_ccn_act.F90 @@ -202,7 +202,7 @@ subroutine aer_ccn_act_init () !--------------------------------------------------------------------- ! write version number and namelist to logfile. !-------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit=stdlog() if (mpp_pe() == mpp_root_pe() ) & write (logunit, nml=aer_ccn_act_nml) diff --git a/src/atmos_shared/tracer_driver/aer_ccn_act/aerosol_params.F90 b/src/atmos_shared/tracer_driver/aer_ccn_act/aerosol_params.F90 index ef4ba8b4d5..0c3194fbad 100644 --- a/src/atmos_shared/tracer_driver/aer_ccn_act/aerosol_params.F90 +++ b/src/atmos_shared/tracer_driver/aer_ccn_act/aerosol_params.F90 @@ -95,7 +95,7 @@ subroutine aerosol_params_init !------------------------------------------------------------------------- ! write version number to output file. !------------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) !------------------------------------------------------------------------- ! declare module to be initialized. diff --git a/src/atmos_shared/tracer_driver/aer_ccn_act/ice_nucl.F90 b/src/atmos_shared/tracer_driver/aer_ccn_act/ice_nucl.F90 index c5d820f056..f72980a4d9 100644 --- a/src/atmos_shared/tracer_driver/aer_ccn_act/ice_nucl.F90 +++ b/src/atmos_shared/tracer_driver/aer_ccn_act/ice_nucl.F90 @@ -141,7 +141,7 @@ SUBROUTINE ice_nucl_wpdf_init !--------- write version and namelist to standard log ------------ - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() if ( mpp_pe() == mpp_root_pe() ) & write ( logunit, nml=ice_nucl_nml ) diff --git a/src/atmos_shared/tracer_driver/atmos_age_tracer.F90 b/src/atmos_shared/tracer_driver/atmos_age_tracer.F90 index 2807e5889c..058334444f 100644 --- a/src/atmos_shared/tracer_driver/atmos_age_tracer.F90 +++ b/src/atmos_shared/tracer_driver/atmos_age_tracer.F90 @@ -237,7 +237,7 @@ subroutine atmos_age_tracer_init( r, axes, Time, nage, & !---- write namelist ------------------ - call write_version_number() + call write_version_number(version, tagname) ! if ( mpp_pe() == mpp_root_pe() ) & ! write ( stdlog(), nml=atmos_age_tracer_nml ) diff --git a/src/atmos_shared/tracer_driver/atmos_carbon_aerosol.F90 b/src/atmos_shared/tracer_driver/atmos_carbon_aerosol.F90 index 34dea6e12f..0a56b5b5c8 100644 --- a/src/atmos_shared/tracer_driver/atmos_carbon_aerosol.F90 +++ b/src/atmos_shared/tracer_driver/atmos_carbon_aerosol.F90 @@ -990,7 +990,7 @@ subroutine atmos_carbon_aerosol_init (lonb, latb, axes, Time, mask) !--------------------------------------------------------------------- ! write version number and namelist to logfile. !--------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit=stdlog() if (mpp_pe() == mpp_root_pe() ) & write (logunit, nml=carbon_aerosol_nml) @@ -2237,7 +2237,7 @@ subroutine atmos_carbon_aerosol_init (lonb, latb, axes, Time, mask) endif - call write_version_number() + call write_version_number(version, tagname) module_is_initialized = .TRUE. !----------------------------------------------------------------------- diff --git a/src/atmos_shared/tracer_driver/atmos_ch3i.F90 b/src/atmos_shared/tracer_driver/atmos_ch3i.F90 index 232b170404..7d094b123e 100644 --- a/src/atmos_shared/tracer_driver/atmos_ch3i.F90 +++ b/src/atmos_shared/tracer_driver/atmos_ch3i.F90 @@ -100,7 +100,7 @@ subroutine atmos_ch3i_init( lonb_mod, latb_mod, axes, Time, mask ) !----------------------------------------------------------------------- ! ... write version number !----------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) !----------------------------------------------------------------------- ! ... read namelist diff --git a/src/atmos_shared/tracer_driver/atmos_co2.F90 b/src/atmos_shared/tracer_driver/atmos_co2.F90 index 56aaa67829..ec739a6a3a 100644 --- a/src/atmos_shared/tracer_driver/atmos_co2.F90 +++ b/src/atmos_shared/tracer_driver/atmos_co2.F90 @@ -577,7 +577,7 @@ subroutine atmos_co2_init (Time, axes) if (module_is_initialized) return - call write_version_number() + call write_version_number(version, tagname) !----------------------------------------------------------------------- ! read namelist. @@ -659,7 +659,7 @@ subroutine atmos_co2_init (Time, axes) write (logunit,*)' not using CO2 emissions: do_co2_emissions= ',do_co2_emissions endif -call write_version_number() +call write_version_number(version, tagname) module_is_initialized = .TRUE. diff --git a/src/atmos_shared/tracer_driver/atmos_convection_tracer.F90 b/src/atmos_shared/tracer_driver/atmos_convection_tracer.F90 index d20a4a4b86..937add5aad 100644 --- a/src/atmos_shared/tracer_driver/atmos_convection_tracer.F90 +++ b/src/atmos_shared/tracer_driver/atmos_convection_tracer.F90 @@ -225,7 +225,7 @@ subroutine atmos_convection_tracer_init (r, phalf, axes, Time, & !---- write namelist ------------------ - call write_version_number() + call write_version_number(version, tagname) logunit=stdlog() if ( mpp_pe() == mpp_root_pe() ) & write ( logunit, nml=atmos_convection_tracer_nml ) diff --git a/src/atmos_shared/tracer_driver/atmos_dust.F90 b/src/atmos_shared/tracer_driver/atmos_dust.F90 index 753b6f90bc..ce5eb0a746 100644 --- a/src/atmos_shared/tracer_driver/atmos_dust.F90 +++ b/src/atmos_shared/tracer_driver/atmos_dust.F90 @@ -245,7 +245,7 @@ subroutine atmos_dust_init (lonb, latb, axes, Time, mask) if (module_is_initialized) return - call write_version_number() + call write_version_number(version, tagname) !----------------------------------------------------------------------- ! read namelist. !----------------------------------------------------------------------- @@ -313,7 +313,7 @@ subroutine atmos_dust_init (lonb, latb, axes, Time, mask) vert_interp=(/INTERP_WEIGHTED_P/) ) - call write_version_number() + call write_version_number(version, tagname) module_is_initialized = .TRUE. diff --git a/src/atmos_shared/tracer_driver/atmos_radon.F90 b/src/atmos_shared/tracer_driver/atmos_radon.F90 index 7be9b3e79c..cc9aed9e4b 100644 --- a/src/atmos_shared/tracer_driver/atmos_radon.F90 +++ b/src/atmos_shared/tracer_driver/atmos_radon.F90 @@ -260,7 +260,7 @@ subroutine atmos_radon_init (r, axes, Time, nradon, mask) !---- write namelist ------------------ - call write_version_number() + call write_version_number(version, tagname) logunit=stdlog() if ( mpp_pe() == mpp_root_pe() ) & write ( logunit, nml=atmos_radon_nml ) diff --git a/src/atmos_shared/tracer_driver/atmos_sea_salt.F90 b/src/atmos_shared/tracer_driver/atmos_sea_salt.F90 index 1d8232769e..b7bb5e0bd0 100644 --- a/src/atmos_shared/tracer_driver/atmos_sea_salt.F90 +++ b/src/atmos_shared/tracer_driver/atmos_sea_salt.F90 @@ -375,7 +375,7 @@ subroutine atmos_sea_salt_init (lonb, latb, axes, Time, mask) #endif endif !--------- write version and namelist to standard log ------------ - call write_version_number() + call write_version_number(version, tagname) logunit=stdlog() if ( mpp_pe() == mpp_root_pe() ) & write ( logunit, nml=ssalt_nml ) diff --git a/src/atmos_shared/tracer_driver/atmos_soa.F90 b/src/atmos_shared/tracer_driver/atmos_soa.F90 index 8c1206dca7..0e73dad831 100644 --- a/src/atmos_shared/tracer_driver/atmos_soa.F90 +++ b/src/atmos_shared/tracer_driver/atmos_soa.F90 @@ -126,7 +126,7 @@ subroutine atmos_SOA_init ( lonb, latb, nlev, axes, Time, mask) !--------------------------------------------------------------------- ! write version number and namelist to logfile. !--------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit=stdlog() if (mpp_pe() == mpp_root_pe() ) & write (logunit, nml=secondary_organics_nml) @@ -171,7 +171,7 @@ subroutine atmos_SOA_init ( lonb, latb, nlev, axes, Time, mask) 'column SOA production by C4H10 + OH', & 'kg/m2/s') - call write_version_number() + call write_version_number(version, tagname) module_is_initialized = .TRUE. diff --git a/src/atmos_shared/tracer_driver/atmos_sulfate.F90 b/src/atmos_shared/tracer_driver/atmos_sulfate.F90 index 0a6c0d883a..1c3a6bc3b1 100644 --- a/src/atmos_shared/tracer_driver/atmos_sulfate.F90 +++ b/src/atmos_shared/tracer_driver/atmos_sulfate.F90 @@ -329,7 +329,7 @@ subroutine atmos_sulfate_init ( lonb, latb, nlev, axes, Time, mask) !---- write namelist ------------------ - call write_version_number() + call write_version_number(version, tagname) ! read namelist. !----------------------------------------------------------------------- @@ -350,7 +350,7 @@ subroutine atmos_sulfate_init ( lonb, latb, nlev, axes, Time, mask) !--------------------------------------------------------------------- ! write version number and namelist to logfile. !--------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit=stdlog() if (mpp_pe() == mpp_root_pe() ) & write (logunit, nml=simple_sulfate_nml) @@ -928,7 +928,7 @@ subroutine atmos_sulfate_init ( lonb, latb, nlev, axes, Time, mask) 'simpleH2O2 chemical production', & 'kgH2O2/m2/s') - call write_version_number() + call write_version_number(version, tagname) module_is_initialized = .TRUE. diff --git a/src/atmos_shared/tracer_driver/atmos_sulfur_hex.F90 b/src/atmos_shared/tracer_driver/atmos_sulfur_hex.F90 index 63f9981d2e..4758e1215b 100644 --- a/src/atmos_shared/tracer_driver/atmos_sulfur_hex.F90 +++ b/src/atmos_shared/tracer_driver/atmos_sulfur_hex.F90 @@ -281,7 +281,7 @@ subroutine atmos_sulfur_hex_init (lonb, latb, r, axes, Time, mask) !---- write namelist ------------------ - call write_version_number() + call write_version_number(version, tagname) n = get_tracer_index(MODEL_ATMOS,'sf6') if (n>0) then diff --git a/src/atmos_shared/tracer_driver/atmos_tracer_driver.F90 b/src/atmos_shared/tracer_driver/atmos_tracer_driver.F90 index c8e498ae7e..f43fdf64cd 100644 --- a/src/atmos_shared/tracer_driver/atmos_tracer_driver.F90 +++ b/src/atmos_shared/tracer_driver/atmos_tracer_driver.F90 @@ -1251,7 +1251,7 @@ subroutine atmos_tracer_driver_init (lonb, latb, r, axes, Time, phalf, mask) #endif endif !--------- write version and namelist to standard log ------------ - call write_version_number() + call write_version_number(version, tagname) logunit=stdlog() if ( mpp_pe() == mpp_root_pe() ) & write ( logunit, nml=atmos_tracer_driver_nml ) diff --git a/src/atmos_shared/tracer_driver/atmos_tracer_utilities.F90 b/src/atmos_shared/tracer_driver/atmos_tracer_utilities.F90 index 842be76438..d143ee14aa 100644 --- a/src/atmos_shared/tracer_driver/atmos_tracer_utilities.F90 +++ b/src/atmos_shared/tracer_driver/atmos_tracer_utilities.F90 @@ -373,7 +373,7 @@ subroutine atmos_tracer_utilities_init(lonb, latb, mass_axes, Time) 'm/s', missing_value=-999. ) - call write_version_number() + call write_version_number(version, tagname) if ( mpp_pe() == mpp_root_pe() ) then logunit=stdlog() diff --git a/src/atmos_shared/tracer_driver/stratchem/strat_chem_driver.F90 b/src/atmos_shared/tracer_driver/stratchem/strat_chem_driver.F90 index 9972333413..3f83b5e260 100644 --- a/src/atmos_shared/tracer_driver/stratchem/strat_chem_driver.F90 +++ b/src/atmos_shared/tracer_driver/stratchem/strat_chem_driver.F90 @@ -120,7 +120,7 @@ function strat_chem_driver_init() ! write version number and namelist to logfile. !--------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) logunit=stdlog() if (mpp_pe() == mpp_root_pe()) write (logunit, nml=strat_chem_nml) diff --git a/src/atmos_shared/tracer_driver/tropchem/mo_fastjx.F90 b/src/atmos_shared/tracer_driver/tropchem/mo_fastjx.F90 index 85bf822e46..aad0d1a904 100644 --- a/src/atmos_shared/tracer_driver/tropchem/mo_fastjx.F90 +++ b/src/atmos_shared/tracer_driver/tropchem/mo_fastjx.F90 @@ -420,7 +420,7 @@ subroutine fastjx_init !---------------------------------------------------------------------- ! output version number and tagname to logfile. !---------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) ! ! Defaults & constants diff --git a/src/atmos_shared/tracer_driver/tropchem/mo_fphoto.F90 b/src/atmos_shared/tracer_driver/tropchem/mo_fphoto.F90 index 51653d5c02..a69aa6f93a 100644 --- a/src/atmos_shared/tracer_driver/tropchem/mo_fphoto.F90 +++ b/src/atmos_shared/tracer_driver/tropchem/mo_fphoto.F90 @@ -156,7 +156,7 @@ subroutine fprate_init(o3_column_top_in) !------------------------------------------------------------------------- ! write version number and tagname to stdlog. !------------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) indexer(TAB_NDX_JO2) = get_rxt_ndx( 'jo2' ) indexer(TAB_NDX_JO3) = get_rxt_ndx( 'jo3p' ) diff --git a/src/atmos_shared/tracer_driver/tropchem/tropchem_driver.F90 b/src/atmos_shared/tracer_driver/tropchem/tropchem_driver.F90 index c84f47d40d..a3f5def9f9 100644 --- a/src/atmos_shared/tracer_driver/tropchem/tropchem_driver.F90 +++ b/src/atmos_shared/tracer_driver/tropchem/tropchem_driver.F90 @@ -1379,7 +1379,7 @@ function tropchem_driver_init( r, mask, axes, Time, & !----------------------------------------------------------------------- ! ... write version number !----------------------------------------------------------------------- - call write_version_number() + call write_version_number(version, tagname) !----------------------------------------------------------------------- ! ... read namelist diff --git a/src/atmos_shared/vert_advection/vert_advection.F90 b/src/atmos_shared/vert_advection/vert_advection.F90 index c33e000ab5..35288f100b 100644 --- a/src/atmos_shared/vert_advection/vert_advection.F90 +++ b/src/atmos_shared/vert_advection/vert_advection.F90 @@ -107,7 +107,7 @@ subroutine vert_advection_3d ( dt, w, dz, r, rdt, mask, scheme, form, flags ) integer :: kk if(.not.module_is_initialized) then - call write_version_number() + call write_version_number(version, tagname) module_is_initialized = .true. endif diff --git a/src/atmos_spectral/driver/coupled/atmosphere.F90 b/src/atmos_spectral/driver/coupled/atmosphere.F90 index ea098713e2..04bdcd8b76 100644 --- a/src/atmos_spectral/driver/coupled/atmosphere.F90 +++ b/src/atmos_spectral/driver/coupled/atmosphere.F90 @@ -105,7 +105,7 @@ subroutine atmosphere_init(Time_init, Time, Time_step_in, Surf_diff) enddo 20 call close_file (unit) -call write_version_number() +call write_version_number(version, tagname) if(mpp_pe() == mpp_root_pe()) write (stdlog(), nml=atmosphere_nml) !----------------------------------------------------------------------------------------- diff --git a/src/atmos_spectral/driver/coupled/mcm_mca_lsc.F90 b/src/atmos_spectral/driver/coupled/mcm_mca_lsc.F90 index d02fa99d18..8f5f4be450 100644 --- a/src/atmos_spectral/driver/coupled/mcm_mca_lsc.F90 +++ b/src/atmos_spectral/driver/coupled/mcm_mca_lsc.F90 @@ -150,7 +150,7 @@ subroutine init_mcm_mca_lsc enddo 20 call close_file (unit) - call write_version_number() + call write_version_number(version, tagname) if(mpp_pe() == mpp_root_pe()) write (stdlog(),nml=mcm_mca_lsc_nml) ! ix and kx stay fixed througout run diff --git a/src/atmos_spectral/driver/coupled/mcm_moist_processes.F90 b/src/atmos_spectral/driver/coupled/mcm_moist_processes.F90 index 0c52fbac7b..10d4f1f99d 100644 --- a/src/atmos_spectral/driver/coupled/mcm_moist_processes.F90 +++ b/src/atmos_spectral/driver/coupled/mcm_moist_processes.F90 @@ -389,7 +389,7 @@ subroutine mcm_moist_processes_init ( id, jd, kd, axes, Time ) !--------- write namelist ------------------ - call write_version_number() + call write_version_number(version, tagname) if(mpp_pe() == mpp_root_pe()) write (stdlog(), nml=mcm_moist_processes_nml) !------------ initialize various schemes ---------- diff --git a/src/atmos_spectral/driver/coupled/spectral_physics.F90 b/src/atmos_spectral/driver/coupled/spectral_physics.F90 index fc9583653a..93c14dc9cd 100644 --- a/src/atmos_spectral/driver/coupled/spectral_physics.F90 +++ b/src/atmos_spectral/driver/coupled/spectral_physics.F90 @@ -82,7 +82,7 @@ subroutine spectral_physics_init(Time, axes, Surf_diff, nhum_in, p_half, do_mcm_ if(module_is_initialized) return -call write_version_number() +call write_version_number(version, tagname) call fms_init call time_manager_init diff --git a/src/atmos_spectral/driver/solo/atmosphere.F90 b/src/atmos_spectral/driver/solo/atmosphere.F90 index 7a429cf1cc..5f66046cd8 100644 --- a/src/atmos_spectral/driver/solo/atmosphere.F90 +++ b/src/atmos_spectral/driver/solo/atmosphere.F90 @@ -112,7 +112,7 @@ subroutine atmosphere_init(Time_init, Time, Time_step_in) if(module_is_initialized) return -call write_version_number() +call write_version_number(version, tagname) Time_step = Time_step_in call get_time(Time_step, seconds, days) diff --git a/src/atmos_spectral/driver/solo/idealized_moist_phys.F90 b/src/atmos_spectral/driver/solo/idealized_moist_phys.F90 index 7b07ca0525..bca75d17e6 100644 --- a/src/atmos_spectral/driver/solo/idealized_moist_phys.F90 +++ b/src/atmos_spectral/driver/solo/idealized_moist_phys.F90 @@ -169,7 +169,7 @@ subroutine idealized_moist_phys_init(Time, Time_step_in, nhum, rad_lat_2d, rad_l if(module_is_initialized) return -call write_version_number() +call write_version_number(version, tagname) #ifdef INTERNAL_FILE_NML read (input_nml_file, nml=idealized_moist_phys_nml, iostat=io) diff --git a/src/atmos_spectral/driver/solo/mixed_layer.F90 b/src/atmos_spectral/driver/solo/mixed_layer.F90 index 00c925e8ee..301f1509c5 100644 --- a/src/atmos_spectral/driver/solo/mixed_layer.F90 +++ b/src/atmos_spectral/driver/solo/mixed_layer.F90 @@ -131,7 +131,7 @@ subroutine mixed_layer_init(is, ie, js, je, num_levels, t_surf, axes, Time) if(module_is_initialized) return -call write_version_number() +call write_version_number(version, tagname) unit = open_namelist_file () ierr=1 diff --git a/src/atmos_spectral/init/ic_from_external_file.F90 b/src/atmos_spectral/init/ic_from_external_file.F90 index 5bee90a46f..72e83b3a0f 100644 --- a/src/atmos_spectral/init/ic_from_external_file.F90 +++ b/src/atmos_spectral/init/ic_from_external_file.F90 @@ -72,7 +72,7 @@ subroutine ic_from_external_file(triang_trunc, tracer_attributes, vors, divs, ts enddo 20 call close_file (unit) #endif -call write_version_number() +call write_version_number(version, tagname) write (stdlog(), nml=ic_from_external_file_nml) !------------------------------------------------------------------------------- diff --git a/src/atmos_spectral/init/jablonowski_2006.F90 b/src/atmos_spectral/init/jablonowski_2006.F90 index bae9eb8119..51d3a16854 100644 --- a/src/atmos_spectral/init/jablonowski_2006.F90 +++ b/src/atmos_spectral/init/jablonowski_2006.F90 @@ -107,7 +107,7 @@ subroutine jablonowski_2006(sea_level_press, triang_trunc, & enddo 20 call close_file (unit) #endif -call write_version_number() +call write_version_number(version, tagname) if(mpp_pe() == mpp_root_pe()) write (stdlog(), nml=jablonowski_2006_nml) call compute_vert_coord(vert_coord_option, scale_heights, surf_res, exponent, p_press, p_sigma, sea_level_press, pk, bk) diff --git a/src/atmos_spectral/init/polvani_2004.F90 b/src/atmos_spectral/init/polvani_2004.F90 index 45028a9e57..66f4b08478 100644 --- a/src/atmos_spectral/init/polvani_2004.F90 +++ b/src/atmos_spectral/init/polvani_2004.F90 @@ -102,7 +102,7 @@ subroutine polvani_2004(sea_level_press, triang_trunc, vert_difference_option, p ierr = check_nml_error (io, 'polvani_2004_nml') enddo 20 call close_file (unit) -call write_version_number() +call write_version_number(version, tagname) if(mpp_pe() == mpp_root_pe()) write (stdlog(), nml=polvani_2004_nml) T_standard(1) = 288.15 diff --git a/src/atmos_spectral/init/polvani_2007.F90 b/src/atmos_spectral/init/polvani_2007.F90 index 3281890d50..6e30c7d52e 100644 --- a/src/atmos_spectral/init/polvani_2007.F90 +++ b/src/atmos_spectral/init/polvani_2007.F90 @@ -139,7 +139,7 @@ subroutine polvani_2007(triang_trunc, vert_difference_option, pk, bk, & enddo 20 call close_file (unit) #endif -call write_version_number() +call write_version_number(version, tagname) if(mpp_pe() == mpp_root_pe()) write (stdlog(), nml=polvani_2007_nml) call get_lon_max(lon_max) diff --git a/src/atmos_spectral/init/spectral_init_cond.F90 b/src/atmos_spectral/init/spectral_init_cond.F90 index 694c3a7718..9867f55359 100644 --- a/src/atmos_spectral/init/spectral_init_cond.F90 +++ b/src/atmos_spectral/init/spectral_init_cond.F90 @@ -92,7 +92,7 @@ subroutine spectral_init_cond(initial_state_option, tracer_attributes, reference enddo 20 call close_file (unit) #endif -call write_version_number() +call write_version_number(version, tagname) if(mpp_pe() == mpp_root_pe()) write (stdlog(), nml=spectral_init_cond_nml) call compute_vert_coord(vert_coord_option, scale_heights, surf_res, exponent, p_press, p_sigma, reference_sea_level_press, pk,bk) diff --git a/src/atmos_spectral/init/spectral_initialize_fields.F90 b/src/atmos_spectral/init/spectral_initialize_fields.F90 index 3c4f56bbd6..3809af95f0 100644 --- a/src/atmos_spectral/init/spectral_initialize_fields.F90 +++ b/src/atmos_spectral/init/spectral_initialize_fields.F90 @@ -42,7 +42,7 @@ subroutine spectral_initialize_fields(reference_sea_level_press, triang_trunc, i integer :: ms, me, ns, ne, is, ie, js, je, num_levels -call write_version_number() +call write_version_number(version, tagname) num_levels = size(ug,3) call get_grid_domain(is, ie, js, je) diff --git a/src/atmos_spectral/init/topog_regularization.F90 b/src/atmos_spectral/init/topog_regularization.F90 index ed3db70e1c..635f887c0a 100644 --- a/src/atmos_spectral/init/topog_regularization.F90 +++ b/src/atmos_spectral/init/topog_regularization.F90 @@ -275,7 +275,7 @@ subroutine topog_regularization_init(ocean_mask) real, allocatable, dimension(:,:,:) :: legendre_global, legendre logical, allocatable, dimension(:,:) :: ocean_mask_global -call write_version_number() +call write_version_number(version, tagname) if(.not.transforms_are_initialized()) then call error_mesg('topog_regularization_init','Transforms are not initialized',FATAL) diff --git a/src/atmos_spectral/init/vert_coordinate.F90 b/src/atmos_spectral/init/vert_coordinate.F90 index d8f90c637a..ca939ad411 100644 --- a/src/atmos_spectral/init/vert_coordinate.F90 +++ b/src/atmos_spectral/init/vert_coordinate.F90 @@ -77,7 +77,7 @@ subroutine compute_vert_coord (vert_coord_option, scale_heights, surf_res, expon character(len=16) :: chtmp2, chtmp3 if(.not.entry_to_logfile_done) then - call write_version_number() + call write_version_number(version, tagname) entry_to_logfile_done = .true. endif diff --git a/src/atmos_spectral/model/every_step_diagnostics.F90 b/src/atmos_spectral/model/every_step_diagnostics.F90 index e81236c94f..98c058fc5f 100644 --- a/src/atmos_spectral/model/every_step_diagnostics.F90 +++ b/src/atmos_spectral/model/every_step_diagnostics.F90 @@ -67,7 +67,7 @@ subroutine every_step_diagnostics_init(Time, lon_max, lat_max, num_levels_in, re real, dimension(num_levels_in) :: p_full, ln_p_full real, dimension(num_levels_in+1) :: p_half, ln_p_half -call write_version_number() +call write_version_number(version, tagname) call get_deg_lon(lon) call get_deg_lat(lat) diff --git a/src/atmos_spectral/model/fv_advection.F90 b/src/atmos_spectral/model/fv_advection.F90 index 1bc3e95997..bbad7cf530 100644 --- a/src/atmos_spectral/model/fv_advection.F90 +++ b/src/atmos_spectral/model/fv_advection.F90 @@ -47,7 +47,7 @@ subroutine fv_advection_init(nx_in, ny_in, yy_in, degrees_lon, advection_layout) if (module_is_initialized) return - call write_version_number() + call write_version_number(version, tagname) pe = mpp_pe() npes = mpp_npes() diff --git a/src/atmos_spectral/model/global_integral.F90 b/src/atmos_spectral/model/global_integral.F90 index 5cb3254f09..5a27af3fee 100644 --- a/src/atmos_spectral/model/global_integral.F90 +++ b/src/atmos_spectral/model/global_integral.F90 @@ -42,7 +42,7 @@ function mass_weighted_global_integral(field, surf_press) integer :: k, num_levels if(.not.entry_to_logfile_done) then - call write_version_number() + call write_version_number(version, tagname) entry_to_logfile_done=.true. endif diff --git a/src/atmos_spectral/model/implicit.F90 b/src/atmos_spectral/model/implicit.F90 index e9e046d478..0723dd7c36 100644 --- a/src/atmos_spectral/model/implicit.F90 +++ b/src/atmos_spectral/model/implicit.F90 @@ -75,7 +75,7 @@ subroutine implicit_init(pk_in, bk_in, ref_temperature_implicit_in, ref_surf_p_i if(module_is_initialized) return -call write_version_number() +call write_version_number(version, tagname) call get_spec_domain(ms, me, ns, ne) diff --git a/src/atmos_spectral/model/leapfrog.F90 b/src/atmos_spectral/model/leapfrog.F90 index 6b97ff6c22..af41595e5d 100644 --- a/src/atmos_spectral/model/leapfrog.F90 +++ b/src/atmos_spectral/model/leapfrog.F90 @@ -42,7 +42,7 @@ subroutine leapfrog_2level_A_3d_complex (a, dt_a, previous, current, future, del real, intent(in) :: delta_t, robert_coeff if(.not.entry_to_logfile_done) then - call write_version_number() + call write_version_number(version, tagname) entry_to_logfile_done = .true. endif @@ -66,7 +66,7 @@ subroutine leapfrog_2level_B_3d_complex (a, current, future, robert_coeff) real, intent(in) :: robert_coeff if(.not.entry_to_logfile_done) then - call write_version_number() + call write_version_number(version, tagname) entry_to_logfile_done = .true. endif @@ -85,7 +85,7 @@ subroutine leapfrog_2level_A_3d_real (a, dt_a, previous, current, future, delta_ real, intent(in) :: delta_t, robert_coeff if(.not.entry_to_logfile_done) then - call write_version_number() + call write_version_number(version, tagname) entry_to_logfile_done = .true. endif @@ -109,7 +109,7 @@ subroutine leapfrog_2level_B_3d_real (a, current, future, robert_coeff) real, intent(in) :: robert_coeff if(.not.entry_to_logfile_done) then - call write_version_number() + call write_version_number(version, tagname) entry_to_logfile_done = .true. endif @@ -132,7 +132,7 @@ subroutine leapfrog_2level_A_2d_complex (a, dt_a, previous, current, future, del complex, dimension(size(a,1),size(a,2),1) :: dt_a_3d if(.not.entry_to_logfile_done) then - call write_version_number() + call write_version_number(version, tagname) entry_to_logfile_done = .true. endif @@ -152,7 +152,7 @@ subroutine leapfrog_2level_B_2d_complex (a, current, future, robert_coeff) complex, dimension(size(a,1),size(a,2),1,size(a,3)) :: a_3d if(.not.entry_to_logfile_done) then - call write_version_number() + call write_version_number(version, tagname) entry_to_logfile_done = .true. endif @@ -171,7 +171,7 @@ subroutine leapfrog_3d_complex(a, dt_a, previous, current, future, delta_t, robe real, intent(in) :: delta_t, robert_coeff if(.not.entry_to_logfile_done) then - call write_version_number() + call write_version_number(version, tagname) entry_to_logfile_done = .true. endif @@ -200,7 +200,7 @@ subroutine leapfrog_2d_complex(a, dt_a, previous, current, future, delta_t, robe complex, dimension(size(a,1),size(a,2),1) :: dt_a_3d if(.not.entry_to_logfile_done) then - call write_version_number() + call write_version_number(version, tagname) entry_to_logfile_done = .true. endif diff --git a/src/atmos_spectral/model/matrix_invert.F90 b/src/atmos_spectral/model/matrix_invert.F90 index 3ddd5bdcb6..6380c0e06e 100644 --- a/src/atmos_spectral/model/matrix_invert.F90 +++ b/src/atmos_spectral/model/matrix_invert.F90 @@ -52,7 +52,7 @@ subroutine invert(matrix, det) ! PROGRAM IS TERMINATED. if(.not.entry_to_logfile_done) then - call write_version_number() + call write_version_number(version, tagname) entry_to_logfile_done = .true. endif diff --git a/src/atmos_spectral/model/press_and_geopot.F90 b/src/atmos_spectral/model/press_and_geopot.F90 index c1fbe43b33..c1ab36748c 100644 --- a/src/atmos_spectral/model/press_and_geopot.F90 +++ b/src/atmos_spectral/model/press_and_geopot.F90 @@ -71,7 +71,7 @@ subroutine press_and_geopot_init(pk_in, bk_in, use_virtual_temperature_in, vert_ if(module_is_initialized) return -call write_version_number() +call write_version_number(version, tagname) num_levels = size(pk_in,1)-1 diff --git a/src/atmos_spectral/model/spectral_damping.F90 b/src/atmos_spectral/model/spectral_damping.F90 index de67d5d402..812e322e5a 100644 --- a/src/atmos_spectral/model/spectral_damping.F90 +++ b/src/atmos_spectral/model/spectral_damping.F90 @@ -45,7 +45,7 @@ subroutine spectral_damping_init (damping_coeff, damping_order, damping_option, if(module_is_initialized) return -call write_version_number() +call write_version_number(version, tagname) num_levels = num_levels_in call get_spec_domain(ms, me, ns, ne) diff --git a/src/atmos_spectral/model/spectral_dynamics.F90 b/src/atmos_spectral/model/spectral_dynamics.F90 index de958e0ae8..826c48d2b9 100644 --- a/src/atmos_spectral/model/spectral_dynamics.F90 +++ b/src/atmos_spectral/model/spectral_dynamics.F90 @@ -227,9 +227,9 @@ subroutine spectral_dynamics_init(Time, Time_step_in, tracer_attributes, dry_mod 20 call close_file (unit) #endif -call write_version_number() +call write_version_number(version, tagname) if(mpp_pe() == mpp_root_pe()) write (stdlog(), nml=spectral_dynamics_nml) -call write_version_number() +call write_version_number(version, tagname) Time_step = Time_step_in Alarm_interval = set_time(print_interval(2), print_interval(1)) diff --git a/src/atmos_spectral/model/water_borrowing.F90 b/src/atmos_spectral/model/water_borrowing.F90 index b3dcb10b6b..ff0b12f2a1 100644 --- a/src/atmos_spectral/model/water_borrowing.F90 +++ b/src/atmos_spectral/model/water_borrowing.F90 @@ -28,7 +28,7 @@ subroutine water_borrowing(dt_qg, qg, current, p_half, delta_t) real, dimension(size(qg,1), size(qg,3)) :: dp if(.not.entry_to_logfile_done) then - call write_version_number() + call write_version_number(version, tagname) entry_to_logfile_done = .true. endif diff --git a/src/atmos_spectral/tools/gauss_and_legendre.F90 b/src/atmos_spectral/tools/gauss_and_legendre.F90 index 557b624ed9..db155d2c7b 100644 --- a/src/atmos_spectral/tools/gauss_and_legendre.F90 +++ b/src/atmos_spectral/tools/gauss_and_legendre.F90 @@ -38,7 +38,7 @@ subroutine compute_legendre(legendre, num_fourier, fourier_inc, & real, dimension(n_lat) :: cos_lat if(.not.entry_to_logfile_done) then - call write_version_number() + call write_version_number(version, tagname) entry_to_logfile_done=.true. endif @@ -105,7 +105,7 @@ subroutine compute_gaussian(sin_hem, wts_hem, n_hem) real :: pp, p1, p2, p3, z, z1 if(.not.entry_to_logfile_done) then - call write_version_number() + call write_version_number(version, tagname) entry_to_logfile_done=.true. endif diff --git a/src/atmos_spectral/tools/grid_fourier.F90 b/src/atmos_spectral/tools/grid_fourier.F90 index 10c3c5d7ad..113bae9b41 100644 --- a/src/atmos_spectral/tools/grid_fourier.F90 +++ b/src/atmos_spectral/tools/grid_fourier.F90 @@ -56,7 +56,7 @@ subroutine grid_fourier_init(num_lon_in, fourier_inc, check, longitude_origin) if(module_is_initialized) return -call write_version_number() +call write_version_number(version, tagname) if(mod(num_lon_in,2) .ne. 0) then write(chtmp,'(i8)') num_lon_in diff --git a/src/atmos_spectral/tools/spec_mpp.F90 b/src/atmos_spectral/tools/spec_mpp.F90 index 28698a0a0f..feaed9acfe 100644 --- a/src/atmos_spectral/tools/spec_mpp.F90 +++ b/src/atmos_spectral/tools/spec_mpp.F90 @@ -35,7 +35,7 @@ subroutine spec_mpp_init( num_fourier, num_spherical, num_lon, lat_max, grid_lay pe = mpp_pe() npes = mpp_npes() - call write_version_number() + call write_version_number(version, tagname) !grid domain: by default, 1D decomposition along Y layout = (/1,npes/) diff --git a/src/atmos_spectral/tools/spherical.F90 b/src/atmos_spectral/tools/spherical.F90 index c639969f72..caeebb889a 100644 --- a/src/atmos_spectral/tools/spherical.F90 +++ b/src/atmos_spectral/tools/spherical.F90 @@ -120,7 +120,7 @@ subroutine spherical_init(radius, num_fourier_in, fourier_inc_in, num_spherical_ if(module_is_initialized) return -call write_version_number() +call write_version_number(version, tagname) call get_spec_domain(ms, me, ns, ne) diff --git a/src/atmos_spectral/tools/spherical_fourier.F90 b/src/atmos_spectral/tools/spherical_fourier.F90 index 0b8d153f56..4c0506bbdb 100644 --- a/src/atmos_spectral/tools/spherical_fourier.F90 +++ b/src/atmos_spectral/tools/spherical_fourier.F90 @@ -116,7 +116,7 @@ subroutine spherical_fourier_init(radius, lat_max_in, num_fourier_in, & integer, intent(in) :: num_spherical_in logical, intent(in), optional :: south_to_north -call write_version_number() +call write_version_number(version, tagname) if(present(south_to_north)) then south_to_north_local = south_to_north diff --git a/src/atmos_spectral/tools/transforms.F90 b/src/atmos_spectral/tools/transforms.F90 index 3b7b74d65a..68defe3786 100644 --- a/src/atmos_spectral/tools/transforms.F90 +++ b/src/atmos_spectral/tools/transforms.F90 @@ -235,7 +235,7 @@ subroutine transforms_init(radius, & 20 call close_file (namelist_unit) #endif -call write_version_number() +call write_version_number(version, tagname) if(mpp_pe() == mpp_root_pe()) write (stdlog(), nml=transforms_nml) npes = mpp_npes() diff --git a/src/coupler/coupler_main.F90 b/src/coupler/coupler_main.F90 index bad5368bdf..71a4bfa27c 100644 --- a/src/coupler/coupler_main.F90 +++ b/src/coupler/coupler_main.F90 @@ -229,7 +229,7 @@ program coupler_main !----------------------------------------------------------------------- character(len=128) :: version = '$Id: coupler_main.F90,v 20.0 2013/12/13 23:27:07 fms Exp $' - character(len=128) :: tag = '$Name: tikal $' + character(len=128) :: tagname = '$Name: tikal $' !----------------------------------------------------------------------- !---- model defined-types ---- @@ -830,7 +830,7 @@ subroutine coupler_init endif !----- write version to logfile ------- - call write_version_number() + call write_version_number(version, tagname) !----- read namelist ------- diff --git a/src/coupler/flux_exchange.F90 b/src/coupler/flux_exchange.F90 index cc4ad638a4..1153c5ec92 100644 --- a/src/coupler/flux_exchange.F90 +++ b/src/coupler/flux_exchange.F90 @@ -262,7 +262,7 @@ module flux_exchange_mod !----------------------------------------------------------------------- character(len=128) :: version = '$Id: flux_exchange.F90,v 20.0 2013/12/13 23:27:41 fms Exp $' - character(len=128) :: tag = '$Name: tikal $' + character(len=128) :: tagname = '$Name: tikal $' !----------------------------------------------------------------------- !---- exchange grid maps ----- @@ -588,7 +588,7 @@ subroutine flux_exchange_init ( Time, Atm, Land, Ice, Ocean, Ocean_state,& #endif !----- write namelist to logfile ----- - call write_version_number() + call write_version_number(version, tagname) if( mpp_pe() == mpp_root_pe() )write( logunit, nml=flux_exchange_nml ) !----- find out number of atmospheric prognostic tracers and index of specific diff --git a/src/coupler/surface_flux.F90 b/src/coupler/surface_flux.F90 index 8f5e49a57c..1f78542001 100644 --- a/src/coupler/surface_flux.F90 +++ b/src/coupler/surface_flux.F90 @@ -736,7 +736,7 @@ subroutine surface_flux_init #endif ! write version number - call write_version_number() + call write_version_number(version, tagname) unit = stdlog() if ( mpp_pe() == mpp_root_pe() ) write (unit, nml=surface_flux_nml) diff --git a/src/ice_param/ice_albedo.F90 b/src/ice_param/ice_albedo.F90 index 1eb6d414c1..d0a457ba94 100644 --- a/src/ice_param/ice_albedo.F90 +++ b/src/ice_param/ice_albedo.F90 @@ -90,7 +90,7 @@ subroutine ice_albedo_init ( t_freeze ) !------- write version number and namelist --------- if ( mpp_pe() == mpp_root_pe() ) then - call write_version_number() + call write_version_number(version, tagname) unit = stdlog() write (unit, nml=ice_albedo_nml) endif diff --git a/src/ice_param/ocean_albedo.F90 b/src/ice_param/ocean_albedo.F90 index cda23782b0..6d1b059bae 100644 --- a/src/ice_param/ocean_albedo.F90 +++ b/src/ice_param/ocean_albedo.F90 @@ -423,7 +423,7 @@ subroutine ocean_albedo_init(ocean,lat) !------- write version number and namelist --------- if ( mpp_pe() == mpp_root_pe() ) then - call write_version_number() + call write_version_number(version, tagname) unit = stdlog() write (unit, nml=ocean_albedo_nml) endif diff --git a/src/ice_param/ocean_rough.F90 b/src/ice_param/ocean_rough.F90 index 225a325aef..4c7e3c16b1 100644 --- a/src/ice_param/ocean_rough.F90 +++ b/src/ice_param/ocean_rough.F90 @@ -229,7 +229,7 @@ subroutine ocean_rough_init !------- write version number and namelist --------- if ( mpp_pe() == mpp_root_pe() ) then - call write_version_number() + call write_version_number(version, tagname) unit = stdlog() write (unit,nml=ocean_rough_nml) write (unit,11) diff --git a/src/ice_sis/ice_bergs.F90 b/src/ice_sis/ice_bergs.F90 index 05365d811b..ccca225966 100644 --- a/src/ice_sis/ice_bergs.F90 +++ b/src/ice_sis/ice_bergs.F90 @@ -2579,7 +2579,7 @@ subroutine icebergs_init(bergs, & if (really_debug) debug=.true. ! One implies the other... ! Log version and parameters - call write_version_number() + call write_version_number(version, tagname) write (stdlogunit, icebergs_nml) if( reproduce_siena ) then diff --git a/src/ice_sis/ice_spec.F90 b/src/ice_sis/ice_spec.F90 index 78adccb53d..4dc2718e9f 100644 --- a/src/ice_sis/ice_spec.F90 +++ b/src/ice_sis/ice_spec.F90 @@ -73,7 +73,7 @@ subroutine get_sea_surface(Time, ts, cn, iceh) write (stdoutunit, ice_spec_nml) write (stdlogunit, ice_spec_nml) - call write_version_number() + call write_version_number(version, tagname) module_is_initialized = .true. endif diff --git a/src/ice_sis/ice_type.F90 b/src/ice_sis/ice_type.F90 index 340ffc74a4..3e0005e79d 100644 --- a/src/ice_sis/ice_type.F90 +++ b/src/ice_sis/ice_type.F90 @@ -373,7 +373,7 @@ subroutine ice_model_init (Ice, Time_Init, Time, Time_step_fast, Time_step_slow write (stdoutunit, ice_model_nml) write (stdlogunit, ice_model_nml) - call write_version_number() + call write_version_number(version, tagname) if (spec_ice) then slab_ice = .true. diff --git a/src/land_lad/land_model.F90 b/src/land_lad/land_model.F90 index 72b0abf737..963bad8d96 100644 --- a/src/land_lad/land_model.F90 +++ b/src/land_lad/land_model.F90 @@ -512,7 +512,7 @@ subroutine init_land_with_area & module_is_initialized = .TRUE. ! write the version and tagname to the logfile - call write_version_number() + call write_version_number(version, tagname) ! initialize tracers #ifdef LAND_BND_TRACERS diff --git a/src/land_lad/land_types.F90 b/src/land_lad/land_types.F90 index adb6adc2db..3406ba824e 100644 --- a/src/land_lad/land_types.F90 +++ b/src/land_lad/land_types.F90 @@ -334,7 +334,7 @@ subroutine land_types_init() ! module_is_initialized =.TRUE. - call write_version_number() + call write_version_number(version, tagname) end subroutine land_types_init ! diff --git a/src/land_lad/numerics.F90 b/src/land_lad/numerics.F90 index 91a86acdc1..ebfd62b9d5 100644 --- a/src/land_lad/numerics.F90 +++ b/src/land_lad/numerics.F90 @@ -85,7 +85,7 @@ module numerics_mod subroutine numerics_init() module_is_initialized =.TRUE. - call write_version_number() + call write_version_number(version, tagname) end subroutine numerics_init ! diff --git a/src/land_lad/soil/land_properties.F90 b/src/land_lad/soil/land_properties.F90 index 546190dba7..b02076244f 100644 --- a/src/land_lad/soil/land_properties.F90 +++ b/src/land_lad/soil/land_properties.F90 @@ -706,7 +706,7 @@ subroutine land_properties_init ( & call close_file (unit) endif - call write_version_number() + call write_version_number(version, tagname) if (mpp_pe() == mpp_root_pe()) then unit = stdlog() write (unit, nml=land_properties_nml) diff --git a/src/land_lad/soil/rivers.F90 b/src/land_lad/soil/rivers.F90 index e9d1ca0f42..5e8027a5cc 100644 --- a/src/land_lad/soil/rivers.F90 +++ b/src/land_lad/soil/rivers.F90 @@ -297,7 +297,7 @@ subroutine rivers_init & ! write the namelist to the log file if ( mpp_pe() == mpp_root_pe() ) then - call write_version_number() + call write_version_number(version, tagname) logunit = stdlog() write (logunit, nml=rivers_nml) endif diff --git a/src/land_lad/soil/soil.F90 b/src/land_lad/soil/soil.F90 index 2d2109e69c..74c64a1625 100644 --- a/src/land_lad/soil/soil.F90 +++ b/src/land_lad/soil/soil.F90 @@ -629,7 +629,7 @@ subroutine soil_init ( soil, gblon, gblat, garea, gfrac, & call get_restart_io_mode(do_netcdf_restart) ! write version information to a log file - call write_version_number() + call write_version_number(version, tagname) ! write the namelist to a log file if( mpp_pe()==0 ) then diff --git a/src/land_lad/vegetation/vegetation.F90 b/src/land_lad/vegetation/vegetation.F90 index 6586a4c5ab..133ec04441 100644 --- a/src/land_lad/vegetation/vegetation.F90 +++ b/src/land_lad/vegetation/vegetation.F90 @@ -247,7 +247,7 @@ subroutine vegetation_init & call get_restart_io_mode(do_netcdf_restart) ! write version and tag information to logfile - call write_version_number() + call write_version_number(version, tagname) ! write the namelist to a log file if( mpp_pe()==0 ) then unit = stdlog( ) diff --git a/src/land_lad2/canopy_air/canopy_air.F90 b/src/land_lad2/canopy_air/canopy_air.F90 index ffbd399e5d..e79a93fc25 100644 --- a/src/land_lad2/canopy_air/canopy_air.F90 +++ b/src/land_lad2/canopy_air/canopy_air.F90 @@ -96,7 +96,7 @@ subroutine read_cana_namelist() integer :: io ! i/o status for the namelist integer :: ierr ! error code, returned by i/o routines - call write_version_number() + call write_version_number(version, tagname) #ifdef INTERNAL_FILE_NML read (input_nml_file, nml=cana_nml, iostat=io) ierr = check_nml_error(io, 'cana_nml') diff --git a/src/land_lad2/glacier/glac_tile.F90 b/src/land_lad2/glacier/glac_tile.F90 index e7ba6b6018..b616c863aa 100644 --- a/src/land_lad2/glacier/glac_tile.F90 +++ b/src/land_lad2/glacier/glac_tile.F90 @@ -229,7 +229,7 @@ subroutine read_glac_data_namelist(glac_n_lev, glac_dz) integer :: i real :: z - call write_version_number() + call write_version_number(version, tagname) #ifdef INTERNAL_FILE_NML read (input_nml_file, nml=glac_data_nml, iostat=io) ierr = check_nml_error(io, 'glac_data_nml') diff --git a/src/land_lad2/glacier/glacier.F90 b/src/land_lad2/glacier/glacier.F90 index 2e3229d9e4..611cc9e255 100644 --- a/src/land_lad2/glacier/glacier.F90 +++ b/src/land_lad2/glacier/glacier.F90 @@ -101,7 +101,7 @@ subroutine read_glac_namelist() call read_glac_data_namelist(num_l, dz) - call write_version_number() + call write_version_number(version, tagname) #ifdef INTERNAL_FILE_NML read (input_nml_file, nml=glac_nml, iostat=io) ierr = check_nml_error(io, 'glac_nml') diff --git a/src/land_lad2/lake/lake.F90 b/src/land_lad2/lake/lake.F90 index 0fda4bfac5..4f632e4c33 100644 --- a/src/land_lad2/lake/lake.F90 +++ b/src/land_lad2/lake/lake.F90 @@ -132,7 +132,7 @@ subroutine read_lake_namelist() call read_lake_data_namelist(num_l) - call write_version_number() + call write_version_number(version, tagname) #ifdef INTERNAL_FILE_NML read (input_nml_file, nml=lake_nml, iostat=io) ierr = check_nml_error(io, 'lake_nml') diff --git a/src/land_lad2/lake/lake_tile.F90 b/src/land_lad2/lake/lake_tile.F90 index c2760dba07..3604109954 100644 --- a/src/land_lad2/lake/lake_tile.F90 +++ b/src/land_lad2/lake/lake_tile.F90 @@ -257,7 +257,7 @@ subroutine read_lake_data_namelist(lake_n_lev) integer :: i real :: z - call write_version_number() + call write_version_number(version, tagname) #ifdef INTERNAL_FILE_NML read (input_nml_file, nml=lake_data_nml, iostat=io) ierr = check_nml_error(io, 'lake_data_nml') diff --git a/src/land_lad2/land_data.F90 b/src/land_lad2/land_data.F90 index 94bbdb70be..d86c387f0a 100644 --- a/src/land_lad2/land_data.F90 +++ b/src/land_lad2/land_data.F90 @@ -207,7 +207,7 @@ subroutine land_data_init(layout, io_layout, time, dt_fast, dt_slow) integer :: io_id(1) ! write the version and tag name to the logfile - call write_version_number() + call write_version_number(version, tagname) ! define the processor layout information according to the global grid size call get_grid_ntiles('LND',ntiles) diff --git a/src/land_lad2/land_model.F90 b/src/land_lad2/land_model.F90 index f041c2143e..c08dd7b4f7 100644 --- a/src/land_lad2/land_model.F90 +++ b/src/land_lad2/land_model.F90 @@ -271,7 +271,7 @@ subroutine land_model_init & module_is_initialized = .TRUE. ! [1] print out version number - call write_version_number() + call write_version_number(version, tagname) ! initialize land model clocks landClock = mpp_clock_id('Land' ,CLOCK_FLAG_DEFAULT,CLOCK_COMPONENT) diff --git a/src/land_lad2/river/river.F90 b/src/land_lad2/river/river.F90 index fd95671ea6..d65009d192 100644 --- a/src/land_lad2/river/river.F90 +++ b/src/land_lad2/river/river.F90 @@ -221,7 +221,7 @@ subroutine river_init( land_lon, land_lat, time, dt_fast, land_domain, & #endif !--- write version and namelist info to logfile -------------------- - call write_version_number() + call write_version_number(version, tagname) unit=stdlog() write(unit, river_nml) diff --git a/src/land_lad2/river/river_physics.F90 b/src/land_lad2/river/river_physics.F90 index 82788b53fe..e1716ee81b 100644 --- a/src/land_lad2/river/river_physics.F90 +++ b/src/land_lad2/river/river_physics.F90 @@ -149,7 +149,7 @@ subroutine river_physics_init(River, domain, id_lon, id_lat ) #endif !--- write version and namelist info to logfile -------------------- - call write_version_number() + call write_version_number(version, tagname) unit=stdlog() write (unit, river_physics_nml) diff --git a/src/land_lad2/shared/land_debug.F90 b/src/land_lad2/shared/land_debug.F90 index 45b819c763..4cf1cb8d07 100644 --- a/src/land_lad2/shared/land_debug.F90 +++ b/src/land_lad2/shared/land_debug.F90 @@ -81,7 +81,7 @@ subroutine land_debug_init() integer :: unit, ierr, io, ntiles integer :: max_threads - call write_version_number() + call write_version_number(version, tagname) #ifdef INTERNAL_FILE_NML read (input_nml_file, nml=land_debug_nml, iostat=io) diff --git a/src/land_lad2/shared/land_io.F90 b/src/land_lad2/shared/land_io.F90 index b0511def5f..2fbdaed315 100644 --- a/src/land_lad2/shared/land_io.F90 +++ b/src/land_lad2/shared/land_io.F90 @@ -62,7 +62,7 @@ subroutine read_land_io_namelist() module_is_initialized = .TRUE. ! [1] print out version number - call write_version_number() + call write_version_number(version, tagname) #ifdef INTERNAL_FILE_NML read (input_nml_file, nml=land_io_nml, iostat=io) diff --git a/src/land_lad2/shared/land_numerics.F90 b/src/land_lad2/shared/land_numerics.F90 index ffeb19fe32..318d48b621 100644 --- a/src/land_lad2/shared/land_numerics.F90 +++ b/src/land_lad2/shared/land_numerics.F90 @@ -94,7 +94,7 @@ module land_numerics_mod subroutine numerics_init() module_is_initialized =.TRUE. - call write_version_number() + call write_version_number(version, tagname) end subroutine numerics_init diff --git a/src/land_lad2/shared/land_tile_diag.F90 b/src/land_lad2/shared/land_tile_diag.F90 index fe7d563eb5..95f063fd0e 100644 --- a/src/land_lad2/shared/land_tile_diag.F90 +++ b/src/land_lad2/shared/land_tile_diag.F90 @@ -98,7 +98,7 @@ subroutine tile_diag_init() if (module_is_initialized) return module_is_initialized = .true. - call write_version_number() + call write_version_number(version, tagname) ! initialize diag selectors call tile_selectors_init() diff --git a/src/land_lad2/snow/snow.F90 b/src/land_lad2/snow/snow.F90 index c33b0b402d..e3dd93f76d 100644 --- a/src/land_lad2/snow/snow.F90 +++ b/src/land_lad2/snow/snow.F90 @@ -105,7 +105,7 @@ subroutine read_snow_namelist() call read_snow_data_namelist(num_l,dz,mc_fict) - call write_version_number() + call write_version_number(version, tagname) #ifdef INTERNAL_FILE_NML read (input_nml_file, nml=snow_nml, iostat=io) ierr = check_nml_error(io, 'snow_nml') diff --git a/src/land_lad2/snow/snow_tile.F90 b/src/land_lad2/snow/snow_tile.F90 index fa76e0ce74..bd1f42ba1c 100644 --- a/src/land_lad2/snow/snow_tile.F90 +++ b/src/land_lad2/snow/snow_tile.F90 @@ -141,7 +141,7 @@ subroutine read_snow_data_namelist(snow_num_l, snow_dz, snow_mc_fict) integer :: io ! i/o status for the namelist integer :: ierr ! error code, returned by i/o routines - call write_version_number() + call write_version_number(version, tagname) #ifdef INTERNAL_FILE_NML read (input_nml_file, nml=snow_data_nml, iostat=io) ierr = check_nml_error(io, 'snow_data_nml') diff --git a/src/land_lad2/soil/soil.F90 b/src/land_lad2/soil/soil.F90 index bcd9051d45..00cbfe87fa 100644 --- a/src/land_lad2/soil/soil.F90 +++ b/src/land_lad2/soil/soil.F90 @@ -189,7 +189,7 @@ subroutine read_soil_namelist() call read_soil_data_namelist(num_l,dz,use_single_geo,gw_option) - call write_version_number() + call write_version_number(version, tagname) #ifdef INTERNAL_FILE_NML read (input_nml_file, nml=soil_nml, iostat=io) ierr = check_nml_error(io, 'soil_nml') diff --git a/src/land_lad2/soil/soil_tile.F90 b/src/land_lad2/soil/soil_tile.F90 index f1bb390024..ef069a9879 100644 --- a/src/land_lad2/soil/soil_tile.F90 +++ b/src/land_lad2/soil/soil_tile.F90 @@ -427,7 +427,7 @@ subroutine read_soil_data_namelist(soil_num_l, soil_dz, soil_single_geo, & integer :: i, rcode, ncid, varid, dimids(3) real :: z - call write_version_number() + call write_version_number(version, tagname) #ifdef INTERNAL_FILE_NML read (input_nml_file, nml=soil_data_nml, iostat=io) ierr = check_nml_error(io, 'soil_data_nml') diff --git a/src/land_lad2/soil/uptake.F90 b/src/land_lad2/soil/uptake.F90 index c1292bd3e0..f92dcd9105 100644 --- a/src/land_lad2/soil/uptake.F90 +++ b/src/land_lad2/soil/uptake.F90 @@ -51,7 +51,7 @@ subroutine uptake_init(num_l_in, dz_in, zfull_in) dz_in(:), & ! layer thickness zfull_in(:) ! layer centers - call write_version_number() + call write_version_number(version, tagname) module_is_initialized =.TRUE. num_l = num_l_in diff --git a/src/land_lad2/topo_rough/topo_rough.F90 b/src/land_lad2/topo_rough/topo_rough.F90 index 863b4491eb..4b4711ca10 100644 --- a/src/land_lad2/topo_rough/topo_rough.F90 +++ b/src/land_lad2/topo_rough/topo_rough.F90 @@ -105,7 +105,7 @@ subroutine topo_rough_init(time, lonb, latb, domain, id_lon,id_lat) logical :: used, got_stdev ! write the version and tagname to the logfile - call write_version_number() + call write_version_number(version, tagname) ! read and write (to logfile) namelist variables #ifdef INTERNAL_FILE_NML diff --git a/src/land_lad2/transitions/transitions.F90 b/src/land_lad2/transitions/transitions.F90 index 741a40ea29..9c89a32187 100644 --- a/src/land_lad2/transitions/transitions.F90 +++ b/src/land_lad2/transitions/transitions.F90 @@ -142,7 +142,7 @@ subroutine land_transitions_init(id_lon, id_lat) if(module_is_initialized) return call horiz_interp_init - call write_version_number() + call write_version_number(version, tagname) #ifdef INTERNAL_FILE_NML read (input_nml_file, nml=landuse_nml, iostat=io) diff --git a/src/land_lad2/vegetation/vegetation.F90 b/src/land_lad2/vegetation/vegetation.F90 index 2d19918979..9324469fac 100644 --- a/src/land_lad2/vegetation/vegetation.F90 +++ b/src/land_lad2/vegetation/vegetation.F90 @@ -184,7 +184,7 @@ subroutine read_vegn_namelist() call read_vegn_data_namelist() call read_static_vegn_namelist(use_static_veg) - call write_version_number() + call write_version_number(version, tagname) #ifdef INTERNAL_FILE_NML read (input_nml_file, nml=vegn_nml, iostat=io) ierr = check_nml_error(io, 'vegn_nml') diff --git a/src/land_lad2/vegetation/vegn_data.F90 b/src/land_lad2/vegetation/vegn_data.F90 index c4599d552a..0fe0e46436 100644 --- a/src/land_lad2/vegetation/vegn_data.F90 +++ b/src/land_lad2/vegetation/vegn_data.F90 @@ -442,7 +442,7 @@ subroutine read_vegn_data_namelist() type(table_printer_type) :: table - call write_version_number() + call write_version_number(version, tagname) #ifdef INTERNAL_FILE_NML read (input_nml_file, nml=vegn_data_nml, iostat=io) ierr = check_nml_error(io, 'vegn_data_nml') diff --git a/src/land_lad2/vegetation/vegn_dynamics.F90 b/src/land_lad2/vegetation/vegn_dynamics.F90 index 307e1f89ba..1050a8eafc 100644 --- a/src/land_lad2/vegetation/vegn_dynamics.F90 +++ b/src/land_lad2/vegetation/vegn_dynamics.F90 @@ -65,7 +65,7 @@ subroutine vegn_dynamics_init(id_lon, id_lat, time, delta_time, soil_decomp_opti real , intent(in) :: delta_time ! fast time step, s character(*) , intent(in) :: soil_decomp_option - call write_version_number() + call write_version_number(version, tagname) ! set up global variables dt_fast_yr = delta_time/seconds_per_year diff --git a/src/land_lad2/vegetation/vegn_harvesting.F90 b/src/land_lad2/vegetation/vegn_harvesting.F90 index d90ab2dbe5..a5893c32df 100644 --- a/src/land_lad2/vegetation/vegn_harvesting.F90 +++ b/src/land_lad2/vegetation/vegn_harvesting.F90 @@ -67,7 +67,7 @@ module vegn_harvesting_mod subroutine vegn_harvesting_init integer :: unit, ierr, io - call write_version_number() + call write_version_number(version, tagname) #ifdef INTERNAL_FILE_NML read (input_nml_file, nml=harvesting_nml, iostat=io) diff --git a/src/land_lad2/vegetation/vegn_photosynthesis.F90 b/src/land_lad2/vegetation/vegn_photosynthesis.F90 index 3d55aaf738..b6ad56e82d 100644 --- a/src/land_lad2/vegetation/vegn_photosynthesis.F90 +++ b/src/land_lad2/vegetation/vegn_photosynthesis.F90 @@ -40,7 +40,7 @@ module vegn_photosynthesis_mod subroutine vegn_photosynthesis_init(photosynthesis_to_use) character(*), intent(in) :: photosynthesis_to_use - call write_version_number() + call write_version_number(version, tagname) ! convert symbolic names of photosynthesis options into numeric IDs to ! speed up selection during run-time diff --git a/src/land_lad2/vegetation/vegn_radiation.F90 b/src/land_lad2/vegetation/vegn_radiation.F90 index cff75d3581..f66f69b421 100644 --- a/src/land_lad2/vegetation/vegn_radiation.F90 +++ b/src/land_lad2/vegetation/vegn_radiation.F90 @@ -58,7 +58,7 @@ subroutine vegn_radiation_init(rad_to_use,snow_rad_to_use) character(*), intent(in) :: rad_to_use character(*), intent(in) :: snow_rad_to_use - call write_version_number() + call write_version_number(version, tagname) ! convert symbolic names of radiation options into numeric IDs to ! speed up selection during run-time diff --git a/src/land_lad2/vegetation/vegn_static_override.F90 b/src/land_lad2/vegetation/vegn_static_override.F90 index 062094dc80..b1818ef668 100644 --- a/src/land_lad2/vegetation/vegn_static_override.F90 +++ b/src/land_lad2/vegetation/vegn_static_override.F90 @@ -95,7 +95,7 @@ subroutine read_static_vegn_namelist(static_veg_used) ! ---- local vars integer :: unit, ierr, io - call write_version_number() + call write_version_number(version, tagname) #ifdef INTERNAL_FILE_NML read (input_nml_file, nml=static_veg_nml, iostat=io) diff --git a/src/land_null/land_model.F90 b/src/land_null/land_model.F90 index 8a7a78c3ec..31aefeb43c 100644 --- a/src/land_null/land_model.F90 +++ b/src/land_null/land_model.F90 @@ -53,6 +53,7 @@ module land_model_mod ! This is the null version use fms_mod, only : write_version_number, error_mesg, FATAL, mpp_npes, stdout use fms_mod, only : open_namelist_file, check_nml_error, file_exist, close_file +use fms_mod, only : set_domain use fms_io_mod, only : parse_mask_table use grid_mod, only : get_grid_ntiles, get_grid_size, define_cube_mosaic @@ -197,7 +198,7 @@ subroutine land_model_init (cplr2land, land2cplr, time_init, time, dt_fast, dt_s stdoutunit = stdout() - call write_version_number() + call write_version_number(version, tagname) ! define the processor layout information according to the global grid size call get_grid_ntiles('LND',ntiles) @@ -231,6 +232,7 @@ subroutine land_model_init (cplr2land, land2cplr, time_init, time, dt_fast, dt_s endif land2cplr%domain = domain + call set_domain(domain) npes_per_tile = mpp_npes()/ntiles face = (mpp_pe()-mpp_root_pe())/npes_per_tile + 1 diff --git a/src/land_param/climap_albedo.F90 b/src/land_param/climap_albedo.F90 index cf4306f827..00d62bff15 100644 --- a/src/land_param/climap_albedo.F90 +++ b/src/land_param/climap_albedo.F90 @@ -52,7 +52,7 @@ subroutine get_climap_albedo (lonb, latb, n, albedo) !------- write version number and namelist --------- if ( mpp_pe() == mpp_root_pe() ) then - call write_version_number() + call write_version_number(version, tagname) endif do_init = .false. @@ -141,7 +141,7 @@ subroutine get_climap_albedo_mcm (lonb, latb, nlong, nlatg, is, ie, js, je, n, & !--- write version id to logfile --- if (do_init) then if (mpp_pe() == mpp_root_pe()) & - call write_version_number() + call write_version_number(version, tagname) do_init = .false. diff --git a/src/mom5/ocean_blobs/ocean_blob.F90 b/src/mom5/ocean_blobs/ocean_blob.F90 index c5d4f730e9..f76b519354 100644 --- a/src/mom5/ocean_blobs/ocean_blob.F90 +++ b/src/mom5/ocean_blobs/ocean_blob.F90 @@ -683,7 +683,7 @@ subroutine ocean_blob_init (Grid, Domain, Time, T_prog, Dens, Thickness, & '==>Note: Using the Lagrangian buoyancy blobs scheme.' Ocean_options%lagrangian_blobs = 'Did use Lagrangian blobs.' - call write_version_number() + call write_version_number(version, tagname) ! Register diagnostic fields allocate(id_tend_blob(num_prog_tracers)) diff --git a/src/mom5/ocean_core/ocean_advection_velocity.F90 b/src/mom5/ocean_core/ocean_advection_velocity.F90 index 611213adbc..b7f0ec8b58 100644 --- a/src/mom5/ocean_core/ocean_advection_velocity.F90 +++ b/src/mom5/ocean_core/ocean_advection_velocity.F90 @@ -292,7 +292,7 @@ subroutine ocean_advection_velocity_init(Grid, Domain, Time, Time_steps, Thickne '==>Error from ocean_advection_velocity_mod: module already initialized.') endif - call write_version_number() + call write_version_number(version, tagname) module_is_initialized = .TRUE. diff --git a/src/mom5/ocean_core/ocean_barotropic.F90 b/src/mom5/ocean_core/ocean_barotropic.F90 index ff9f9e5d44..97564b40ee 100644 --- a/src/mom5/ocean_core/ocean_barotropic.F90 +++ b/src/mom5/ocean_core/ocean_barotropic.F90 @@ -987,7 +987,7 @@ subroutine ocean_barotropic_init(Grid, Domain, Time, Time_steps, Ocean_options, if (diag_step == 0) diag_step = 1 - call write_version_number() + call write_version_number(version, tagname) #ifdef INTERNAL_FILE_NML read (input_nml_file, nml=ocean_barotropic_nml, iostat=io_status) diff --git a/src/mom5/ocean_core/ocean_bbc.F90 b/src/mom5/ocean_core/ocean_bbc.F90 index e645ba81fd..c9a50cca9c 100644 --- a/src/mom5/ocean_core/ocean_bbc.F90 +++ b/src/mom5/ocean_core/ocean_bbc.F90 @@ -274,7 +274,7 @@ subroutine ocean_bbc_init(Grid, Domain, Time, T_prog, Velocity, & module_is_initialized = .TRUE. -call write_version_number() +call write_version_number(version, tagname) #ifdef INTERNAL_FILE_NML read (input_nml_file, nml=ocean_bbc_nml, iostat=io_status) diff --git a/src/mom5/ocean_core/ocean_coriolis.F90 b/src/mom5/ocean_core/ocean_coriolis.F90 index 3cde775f20..59def2138c 100644 --- a/src/mom5/ocean_core/ocean_coriolis.F90 +++ b/src/mom5/ocean_core/ocean_coriolis.F90 @@ -148,7 +148,7 @@ subroutine ocean_coriolis_init(Grid, Domain, Time, Time_steps, Ocean_options, ho module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) ! provide for namelist over-ride of defaults #ifdef INTERNAL_FILE_NML diff --git a/src/mom5/ocean_core/ocean_density.F90 b/src/mom5/ocean_core/ocean_density.F90 index 1eed623d3e..e4c238c423 100644 --- a/src/mom5/ocean_core/ocean_density.F90 +++ b/src/mom5/ocean_core/ocean_density.F90 @@ -800,7 +800,7 @@ subroutine ocean_density_init (Grid, Domain, Time, Time_steps, Thickness, T_prog if (T_prog(n)%longname == 'Preformed Salinity') salt_variable = PREFORMED_SALT enddo - call write_version_number() + call write_version_number(version, tagname) ! provide for namelist override of defaults #ifdef INTERNAL_FILE_NML diff --git a/src/mom5/ocean_core/ocean_domains.F90 b/src/mom5/ocean_core/ocean_domains.F90 index 73a5e78b61..cc0d6e98aa 100644 --- a/src/mom5/ocean_core/ocean_domains.F90 +++ b/src/mom5/ocean_core/ocean_domains.F90 @@ -94,7 +94,7 @@ subroutine ocean_domain_init() integer :: ierr,ioun,io_status,stdoutunit,stdlogunit stdoutunit=stdout();stdlogunit=stdlog() - call write_version_number() + call write_version_number(version, tagname) ! provide for namelist over-ride #ifdef INTERNAL_FILE_NML diff --git a/src/mom5/ocean_core/ocean_grids.F90 b/src/mom5/ocean_core/ocean_grids.F90 index 476fc019f7..13552d07b7 100644 --- a/src/mom5/ocean_core/ocean_grids.F90 +++ b/src/mom5/ocean_core/ocean_grids.F90 @@ -201,7 +201,7 @@ subroutine ocean_grids_init(ver_coordinate, ver_coordinate_class, hor_grid, debu vert_coordinate_class = ver_coordinate_class horz_grid = hor_grid - call write_version_number() + call write_version_number(version, tagname) if (PRESENT(debug)) debug_this_module = debug diff --git a/src/mom5/ocean_core/ocean_model.F90 b/src/mom5/ocean_core/ocean_model.F90 index fbbd441a3a..cbc9db7af2 100644 --- a/src/mom5/ocean_core/ocean_model.F90 +++ b/src/mom5/ocean_core/ocean_model.F90 @@ -771,7 +771,7 @@ subroutine ocean_model_init(Ocean, Ocean_state, Time_init, Time_in) call mpp_clock_begin(id_init) - call write_version_number() + call write_version_number(version, tagname) write(stdoutunit,'(/54x,a/)') '======== STARTING MOM INITIALIZATION ========' diff --git a/src/mom5/ocean_core/ocean_obc.F90 b/src/mom5/ocean_core/ocean_obc.F90 index 3ac3ff8cc7..f7f10b90a8 100644 --- a/src/mom5/ocean_core/ocean_obc.F90 +++ b/src/mom5/ocean_core/ocean_obc.F90 @@ -732,7 +732,7 @@ subroutine ocean_obc_init(have_obc, Time_steps, Domain, Grid, Ocean_options, ver write (stdlogunit, ocean_obc_nml) !--- write out version information --------------------------------- - call write_version_number() + call write_version_number(version, tagname) !--- if there is no open boundary, just return Obc%nobc = nobc diff --git a/src/mom5/ocean_core/ocean_pressure.F90 b/src/mom5/ocean_core/ocean_pressure.F90 index a23dd56198..74a86092ec 100644 --- a/src/mom5/ocean_core/ocean_pressure.F90 +++ b/src/mom5/ocean_core/ocean_pressure.F90 @@ -198,7 +198,7 @@ subroutine ocean_pressure_init(Grid, Domain, Time, ver_coordinate, & module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) vert_coordinate = ver_coordinate vert_coordinate_class = ver_coordinate_class diff --git a/src/mom5/ocean_core/ocean_sbc.F90 b/src/mom5/ocean_core/ocean_sbc.F90 index 2718845626..21ba6d1c4b 100644 --- a/src/mom5/ocean_core/ocean_sbc.F90 +++ b/src/mom5/ocean_core/ocean_sbc.F90 @@ -896,7 +896,7 @@ subroutine ocean_sbc_init(Grid, Domain, Time, T_prog, T_diag, & dtime = dtime_t horz_grid = hor_grid - call write_version_number() + call write_version_number(version, tagname) #ifdef INTERNAL_FILE_NML read (input_nml_file, nml=ocean_sbc_nml, iostat=io_status) diff --git a/src/mom5/ocean_core/ocean_thickness.F90 b/src/mom5/ocean_core/ocean_thickness.F90 index 49a0882541..50539d3901 100644 --- a/src/mom5/ocean_core/ocean_thickness.F90 +++ b/src/mom5/ocean_core/ocean_thickness.F90 @@ -391,7 +391,7 @@ subroutine ocean_thickness_init (Time, Time_steps, Domain, Grid, Ext_mode, Thic module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) ! provide for namelist over-ride of defaults #ifdef INTERNAL_FILE_NML diff --git a/src/mom5/ocean_core/ocean_types.F90 b/src/mom5/ocean_core/ocean_types.F90 index 5215f138b4..c22b8f8fc4 100644 --- a/src/mom5/ocean_core/ocean_types.F90 +++ b/src/mom5/ocean_core/ocean_types.F90 @@ -1354,7 +1354,7 @@ subroutine ocean_types_init() endif module_is_initialized = .true. - call write_version_number() + call write_version_number(version, tagname) return diff --git a/src/mom5/ocean_core/ocean_velocity.F90 b/src/mom5/ocean_core/ocean_velocity.F90 index 1e0cb3b48f..6dc6445bc1 100644 --- a/src/mom5/ocean_core/ocean_velocity.F90 +++ b/src/mom5/ocean_core/ocean_velocity.F90 @@ -346,7 +346,7 @@ subroutine ocean_velocity_init (Grid, Domain, Time, Time_steps, Ocean_options, & module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) have_obc = obc tendency = Time_steps%tendency diff --git a/src/mom5/ocean_core/ocean_velocity_advect.F90 b/src/mom5/ocean_core/ocean_velocity_advect.F90 index d8cb74ffca..09a2ca37b3 100644 --- a/src/mom5/ocean_core/ocean_velocity_advect.F90 +++ b/src/mom5/ocean_core/ocean_velocity_advect.F90 @@ -175,7 +175,7 @@ subroutine ocean_velocity_advect_init(Grid, Domain, Time, obc, hor_grid, debug) module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) Grd => Grid Dom => Domain diff --git a/src/mom5/ocean_diag/ocean_adv_vel_diag.F90 b/src/mom5/ocean_diag/ocean_adv_vel_diag.F90 index e179190772..5893b17739 100644 --- a/src/mom5/ocean_diag/ocean_adv_vel_diag.F90 +++ b/src/mom5/ocean_diag/ocean_adv_vel_diag.F90 @@ -175,7 +175,7 @@ subroutine ocean_adv_vel_diag_init(Grid, Domain, Time, Time_steps, T_prog, Dens, module_is_initialized = .TRUE. -call write_version_number() +call write_version_number(version, tagname) #ifdef INTERNAL_FILE_NML read (input_nml_file, nml=ocean_adv_vel_diag_nml, iostat=io_status) diff --git a/src/mom5/ocean_diag/ocean_diagnostics.F90 b/src/mom5/ocean_diag/ocean_diagnostics.F90 index 70d327e345..53e771e4c4 100644 --- a/src/mom5/ocean_diag/ocean_diagnostics.F90 +++ b/src/mom5/ocean_diag/ocean_diagnostics.F90 @@ -88,7 +88,7 @@ subroutine ocean_diag_init(Grid, Domain, Time, Time_steps, Thickness, T_prog, T_ module_is_initialized = .TRUE. -call write_version_number() +call write_version_number(version, tagname) #ifndef MOM_STATIC_ARRAYS call get_local_indices(Domain, isd, ied, jsd, jed, isc, iec, jsc, jec) diff --git a/src/mom5/ocean_diag/ocean_tracer_diag.F90 b/src/mom5/ocean_diag/ocean_tracer_diag.F90 index 6f1ebf0eb5..adb875167c 100644 --- a/src/mom5/ocean_diag/ocean_tracer_diag.F90 +++ b/src/mom5/ocean_diag/ocean_tracer_diag.F90 @@ -427,7 +427,7 @@ subroutine ocean_tracer_diag_init(Grid, Domain, Time, Time_steps, Thickness, T_p num_prog_tracers = size(T_prog,1) num_diag_tracers = size(T_diag,1) - call write_version_number() + call write_version_number(version, tagname) #ifdef INTERNAL_FILE_NML read (input_nml_file, nml=ocean_tracer_diag_nml, iostat=io_status) diff --git a/src/mom5/ocean_diag/ocean_velocity_diag.F90 b/src/mom5/ocean_diag/ocean_velocity_diag.F90 index 299bd9245e..a3a9416033 100644 --- a/src/mom5/ocean_diag/ocean_velocity_diag.F90 +++ b/src/mom5/ocean_diag/ocean_velocity_diag.F90 @@ -258,7 +258,7 @@ subroutine ocean_velocity_diag_init(Grid, Domain, Time, Time_steps, hor_grid) module_is_initialized = .TRUE. -call write_version_number() +call write_version_number(version, tagname) #ifdef INTERNAL_FILE_NML read (input_nml_file, nml=ocean_velocity_diag_nml, iostat=io_status) diff --git a/src/mom5/ocean_param/lateral/ocean_bih_friction.F90 b/src/mom5/ocean_param/lateral/ocean_bih_friction.F90 index ff37d6ea62..c703c8ed76 100644 --- a/src/mom5/ocean_param/lateral/ocean_bih_friction.F90 +++ b/src/mom5/ocean_param/lateral/ocean_bih_friction.F90 @@ -160,7 +160,7 @@ subroutine ocean_bih_friction_init(Grid, Domain, Time, Ocean_options, dtime, obc module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) ! provide for namelist over-ride of defaults #ifdef INTERNAL_FILE_NML diff --git a/src/mom5/ocean_param/lateral/ocean_bih_tracer.F90 b/src/mom5/ocean_param/lateral/ocean_bih_tracer.F90 index de407e1451..9cf1819571 100644 --- a/src/mom5/ocean_param/lateral/ocean_bih_tracer.F90 +++ b/src/mom5/ocean_param/lateral/ocean_bih_tracer.F90 @@ -170,7 +170,7 @@ subroutine ocean_bih_tracer_init(Grid, Domain, Time, T_prog, Ocean_options, dtim num_prog_tracers = size(T_prog(:)) - call write_version_number() + call write_version_number(version, tagname) ! provide for namelist over-ride of defaults #ifdef INTERNAL_FILE_NML diff --git a/src/mom5/ocean_param/lateral/ocean_bihcgrid_friction.F90 b/src/mom5/ocean_param/lateral/ocean_bihcgrid_friction.F90 index bac0f43579..0e322eff38 100644 --- a/src/mom5/ocean_param/lateral/ocean_bihcgrid_friction.F90 +++ b/src/mom5/ocean_param/lateral/ocean_bihcgrid_friction.F90 @@ -398,7 +398,7 @@ subroutine ocean_bihcgrid_friction_init(Grid, Domain, Time, Ocean_options, d_tim module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) ! provide for namelist over-ride of defaults diff --git a/src/mom5/ocean_param/lateral/ocean_bihcst_friction.F90 b/src/mom5/ocean_param/lateral/ocean_bihcst_friction.F90 index 35236d5535..4b1bc7a8c6 100644 --- a/src/mom5/ocean_param/lateral/ocean_bihcst_friction.F90 +++ b/src/mom5/ocean_param/lateral/ocean_bihcst_friction.F90 @@ -192,7 +192,7 @@ subroutine ocean_bihcst_friction_init(Grid, Domain, Time, Ocean_options, d_time, module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) ! provide for namelist over-ride of defaults #ifdef INTERNAL_FILE_NML diff --git a/src/mom5/ocean_param/lateral/ocean_bihgen_friction.F90 b/src/mom5/ocean_param/lateral/ocean_bihgen_friction.F90 index b136873d6f..68801031ad 100644 --- a/src/mom5/ocean_param/lateral/ocean_bihgen_friction.F90 +++ b/src/mom5/ocean_param/lateral/ocean_bihgen_friction.F90 @@ -476,7 +476,7 @@ subroutine ocean_bihgen_friction_init(Grid, Domain, Time, Ocean_options, d_time, module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) ! provide for namelist over-ride of defaults #ifdef INTERNAL_FILE_NML diff --git a/src/mom5/ocean_param/lateral/ocean_lap_friction.F90 b/src/mom5/ocean_param/lateral/ocean_lap_friction.F90 index 2eba63d3d4..ee18aa778c 100644 --- a/src/mom5/ocean_param/lateral/ocean_lap_friction.F90 +++ b/src/mom5/ocean_param/lateral/ocean_lap_friction.F90 @@ -158,7 +158,7 @@ subroutine ocean_lap_friction_init(Grid, Domain, Time, Ocean_options, dtime, obc module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) ! provide for namelist over-ride of defaults #ifdef INTERNAL_FILE_NML diff --git a/src/mom5/ocean_param/lateral/ocean_lap_tracer.F90 b/src/mom5/ocean_param/lateral/ocean_lap_tracer.F90 index d69a371504..3fdd95bffb 100644 --- a/src/mom5/ocean_param/lateral/ocean_lap_tracer.F90 +++ b/src/mom5/ocean_param/lateral/ocean_lap_tracer.F90 @@ -190,7 +190,7 @@ subroutine ocean_lap_tracer_init(Grid, Domain, Time, T_prog, Ocean_options, dtim num_prog_tracers = size(T_prog(:)) - call write_version_number() + call write_version_number(version, tagname) ! provide for namelist over-ride of defaults #ifdef INTERNAL_FILE_NML diff --git a/src/mom5/ocean_param/lateral/ocean_lapcgrid_friction.F90 b/src/mom5/ocean_param/lateral/ocean_lapcgrid_friction.F90 index 159f142e8d..d444b31df3 100644 --- a/src/mom5/ocean_param/lateral/ocean_lapcgrid_friction.F90 +++ b/src/mom5/ocean_param/lateral/ocean_lapcgrid_friction.F90 @@ -410,7 +410,7 @@ subroutine ocean_lapcgrid_friction_init(Grid, Domain, Time, Ocean_options, d_tim module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) #ifndef MOM_STATIC_ARRAYS call get_local_indices(Domain, isd, ied, jsd, jed, isc, iec, jsc, jec) diff --git a/src/mom5/ocean_param/lateral/ocean_lapcst_friction.F90 b/src/mom5/ocean_param/lateral/ocean_lapcst_friction.F90 index 295ad93b36..6634dbeffc 100644 --- a/src/mom5/ocean_param/lateral/ocean_lapcst_friction.F90 +++ b/src/mom5/ocean_param/lateral/ocean_lapcst_friction.F90 @@ -186,7 +186,7 @@ subroutine ocean_lapcst_friction_init(Grid, Domain, Time, Ocean_options, d_time, module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) ! provide for namelist over-ride of defaults #ifdef INTERNAL_FILE_NML diff --git a/src/mom5/ocean_param/lateral/ocean_lapgen_friction.F90 b/src/mom5/ocean_param/lateral/ocean_lapgen_friction.F90 index 4314bc8abd..bf09e3bbc6 100644 --- a/src/mom5/ocean_param/lateral/ocean_lapgen_friction.F90 +++ b/src/mom5/ocean_param/lateral/ocean_lapgen_friction.F90 @@ -511,7 +511,7 @@ subroutine ocean_lapgen_friction_init(Grid, Domain, Time, Ocean_options, d_time, module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) #ifndef MOM_STATIC_ARRAYS call get_local_indices(Domain, isd, ied, jsd, jed, isc, iec, jsc, jec) diff --git a/src/mom5/ocean_param/lateral/ocean_mixdownslope.F90 b/src/mom5/ocean_param/lateral/ocean_mixdownslope.F90 index 74b2e660d1..282d473491 100644 --- a/src/mom5/ocean_param/lateral/ocean_mixdownslope.F90 +++ b/src/mom5/ocean_param/lateral/ocean_mixdownslope.F90 @@ -267,7 +267,7 @@ subroutine ocean_mixdownslope_init(Grid, Domain, Time, Dens, T_prog, Ocean_optio module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) #ifndef MOM_STATIC_ARRAYS call get_local_indices(Domain,isd,ied,jsd,jed,isc,iec,jsc,jec) nk = Grid%nk diff --git a/src/mom5/ocean_param/lateral/ocean_sigma_transport.F90 b/src/mom5/ocean_param/lateral/ocean_sigma_transport.F90 index 610926269e..1f6affe206 100644 --- a/src/mom5/ocean_param/lateral/ocean_sigma_transport.F90 +++ b/src/mom5/ocean_param/lateral/ocean_sigma_transport.F90 @@ -432,7 +432,7 @@ subroutine ocean_sigma_transport_init(Grid, Domain, Time, Dens, T_prog, Ocean_op module_is_initialized = .TRUE. vert_coordinate = ver_coordinate - call write_version_number() + call write_version_number(version, tagname) ! provide for namelist over-ride of defaults #ifdef INTERNAL_FILE_NML diff --git a/src/mom5/ocean_param/lateral/ocean_submesoscale.F90 b/src/mom5/ocean_param/lateral/ocean_submesoscale.F90 index dea51a0071..7590fa7dec 100644 --- a/src/mom5/ocean_param/lateral/ocean_submesoscale.F90 +++ b/src/mom5/ocean_param/lateral/ocean_submesoscale.F90 @@ -586,7 +586,7 @@ subroutine ocean_submesoscale_init(Grid, Domain, Time, Dens, T_prog, Ocean_optio module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) #ifdef INTERNAL_FILE_NML read (input_nml_file, nml=ocean_submesoscale_nml, iostat=io_status) diff --git a/src/mom5/ocean_param/neutral/ocean_nphysics.F90 b/src/mom5/ocean_param/neutral/ocean_nphysics.F90 index 1d9661cfba..b114b45209 100644 --- a/src/mom5/ocean_param/neutral/ocean_nphysics.F90 +++ b/src/mom5/ocean_param/neutral/ocean_nphysics.F90 @@ -154,7 +154,7 @@ subroutine ocean_nphysics_init(Grid, Domain, Time, Time_steps, Thickness, Dens, module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) Dom => Domain Grd => Grid diff --git a/src/mom5/ocean_param/neutral/ocean_nphysicsA.F90 b/src/mom5/ocean_param/neutral/ocean_nphysicsA.F90 index ea3ec84b89..cb17870d57 100644 --- a/src/mom5/ocean_param/neutral/ocean_nphysicsA.F90 +++ b/src/mom5/ocean_param/neutral/ocean_nphysicsA.F90 @@ -563,7 +563,7 @@ subroutine ocean_nphysicsA_init(Grid, Domain, Time, Time_steps, Thickness, Dens, module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) num_prog_tracers = size(T_prog(:)) dtime = Time_steps%dtime_t diff --git a/src/mom5/ocean_param/neutral/ocean_nphysicsB.F90 b/src/mom5/ocean_param/neutral/ocean_nphysicsB.F90 index 44f9c79e9b..35c447a1e4 100644 --- a/src/mom5/ocean_param/neutral/ocean_nphysicsB.F90 +++ b/src/mom5/ocean_param/neutral/ocean_nphysicsB.F90 @@ -550,7 +550,7 @@ subroutine ocean_nphysicsB_init(Grid, Domain, Time, Time_steps, Thickness, Dens, module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) num_prog_tracers = size(T_prog(:)) dtime = Time_steps%dtime_t diff --git a/src/mom5/ocean_param/neutral/ocean_nphysicsC.F90 b/src/mom5/ocean_param/neutral/ocean_nphysicsC.F90 index e54f75dcb2..e946e45c48 100644 --- a/src/mom5/ocean_param/neutral/ocean_nphysicsC.F90 +++ b/src/mom5/ocean_param/neutral/ocean_nphysicsC.F90 @@ -692,7 +692,7 @@ subroutine ocean_nphysicsC_init(Grid, Domain, Time, Time_steps, Thickness, Dens, module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) num_prog_tracers = size(T_prog(:)) dtime = Time_steps%dtime_t diff --git a/src/mom5/ocean_param/neutral/ocean_nphysics_new.F90 b/src/mom5/ocean_param/neutral/ocean_nphysics_new.F90 index 20f69e461a..b72b4d6d5c 100644 --- a/src/mom5/ocean_param/neutral/ocean_nphysics_new.F90 +++ b/src/mom5/ocean_param/neutral/ocean_nphysics_new.F90 @@ -217,7 +217,7 @@ subroutine ocean_nphysics_new_init(Grid, Domain, Time, Time_steps, Thickness, & module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) ! provide for namelist over-ride of default values #ifdef INTERNAL_FILE_NML diff --git a/src/mom5/ocean_param/neutral/ocean_nphysics_util.F90 b/src/mom5/ocean_param/neutral/ocean_nphysics_util.F90 index 9f6e3f7745..6fa30707fd 100644 --- a/src/mom5/ocean_param/neutral/ocean_nphysics_util.F90 +++ b/src/mom5/ocean_param/neutral/ocean_nphysics_util.F90 @@ -938,7 +938,7 @@ subroutine ocean_nphysics_util_init(Grid, Domain, Time, Time_steps, Dens, T_prog module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) Dom => Domain Grd => Grid diff --git a/src/mom5/ocean_param/sources/ocean_increment_eta.F90 b/src/mom5/ocean_param/sources/ocean_increment_eta.F90 index f4d91bc99e..9a210bc502 100644 --- a/src/mom5/ocean_param/sources/ocean_increment_eta.F90 +++ b/src/mom5/ocean_param/sources/ocean_increment_eta.F90 @@ -147,7 +147,7 @@ subroutine ocean_increment_eta_init(Grid, Domain, Time) module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) ! provide for namelist over-ride of default values #ifdef INTERNAL_FILE_NML diff --git a/src/mom5/ocean_param/sources/ocean_increment_tracer.F90 b/src/mom5/ocean_param/sources/ocean_increment_tracer.F90 index dffd77b76b..e364a50941 100644 --- a/src/mom5/ocean_param/sources/ocean_increment_tracer.F90 +++ b/src/mom5/ocean_param/sources/ocean_increment_tracer.F90 @@ -154,7 +154,7 @@ subroutine ocean_increment_tracer_init(Grid, Domain, Time, T_prog) allocate( Increment(num_prog_tracers) ) - call write_version_number() + call write_version_number(version, tagname) ! provide for namelist over-ride of default values #ifdef INTERNAL_FILE_NML diff --git a/src/mom5/ocean_param/sources/ocean_increment_velocity.F90 b/src/mom5/ocean_param/sources/ocean_increment_velocity.F90 index 04764a27ea..e503f27981 100644 --- a/src/mom5/ocean_param/sources/ocean_increment_velocity.F90 +++ b/src/mom5/ocean_param/sources/ocean_increment_velocity.F90 @@ -151,7 +151,7 @@ subroutine ocean_increment_velocity_init(Grid, Domain, Time) allocate( Increment(2) ) - call write_version_number() + call write_version_number(version, tagname) ! provide for namelist over-ride of default values #ifdef INTERNAL_FILE_NML diff --git a/src/mom5/ocean_param/sources/ocean_momentum_source.F90 b/src/mom5/ocean_param/sources/ocean_momentum_source.F90 index a43e48cc5b..97074e21a4 100644 --- a/src/mom5/ocean_param/sources/ocean_momentum_source.F90 +++ b/src/mom5/ocean_param/sources/ocean_momentum_source.F90 @@ -147,7 +147,7 @@ subroutine ocean_momentum_source_init(Grid, Domain, Time, Ocean_options, debug) module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) call get_local_indices(Domain,isd,ied,jsd,jed,isc,iec,jsc,jec) nk = Grid%nk diff --git a/src/mom5/ocean_param/sources/ocean_overexchange.F90 b/src/mom5/ocean_param/sources/ocean_overexchange.F90 index 591c21d042..0f908d9713 100644 --- a/src/mom5/ocean_param/sources/ocean_overexchange.F90 +++ b/src/mom5/ocean_param/sources/ocean_overexchange.F90 @@ -277,7 +277,7 @@ subroutine ocean_overexchange_init(Grid, Domain, Time, Dens, T_prog, Ocean_optio module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) #ifndef MOM_STATIC_ARRAYS call get_local_indices(Domain,isd,ied,jsd,jed,isc,iec,jsc,jec) nk = Grid%nk diff --git a/src/mom5/ocean_param/sources/ocean_overflow.F90 b/src/mom5/ocean_param/sources/ocean_overflow.F90 index f1f2d08398..0e2bc693f0 100644 --- a/src/mom5/ocean_param/sources/ocean_overflow.F90 +++ b/src/mom5/ocean_param/sources/ocean_overflow.F90 @@ -233,7 +233,7 @@ subroutine ocean_overflow_init(Grid, Domain, Time, Dens, T_prog, Ocean_options, module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) #ifndef MOM_STATIC_ARRAYS call get_local_indices(Domain,isd,ied,jsd,jed,isc,iec,jsc,jec) nk = Grid%nk diff --git a/src/mom5/ocean_param/sources/ocean_overflow_OFP.F90 b/src/mom5/ocean_param/sources/ocean_overflow_OFP.F90 index dd5257b4ce..75f207eb5d 100644 --- a/src/mom5/ocean_param/sources/ocean_overflow_OFP.F90 +++ b/src/mom5/ocean_param/sources/ocean_overflow_OFP.F90 @@ -528,7 +528,7 @@ subroutine ocean_overflow_OFP_init(Grid, Domain, Time, Dens, T_prog, Ocean_optio module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) #ifndef MOM_STATIC_ARRAYS call get_local_indices(Domain,isd,ied,jsd,jed,isc,iec,jsc,jec) nk = Grid%nk diff --git a/src/mom5/ocean_param/sources/ocean_rivermix.F90 b/src/mom5/ocean_param/sources/ocean_rivermix.F90 index 232067b5ba..16c8064d47 100644 --- a/src/mom5/ocean_param/sources/ocean_rivermix.F90 +++ b/src/mom5/ocean_param/sources/ocean_rivermix.F90 @@ -401,7 +401,7 @@ subroutine ocean_rivermix_init(Grid, Domain, Time, Time_steps, Dens, & module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) call get_local_indices(Domain,isd,ied,jsd,jed,isc,iec,jsc,jec) nk = Grid%nk diff --git a/src/mom5/ocean_param/sources/ocean_riverspread.F90 b/src/mom5/ocean_param/sources/ocean_riverspread.F90 index aee085476b..2ec89676c4 100644 --- a/src/mom5/ocean_param/sources/ocean_riverspread.F90 +++ b/src/mom5/ocean_param/sources/ocean_riverspread.F90 @@ -203,7 +203,7 @@ subroutine ocean_riverspread_init(Grid, Domain, Ocean_options, dtime_t) module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) call get_local_indices(Domain,isd,ied,jsd,jed,isc,iec,jsc,jec) nk = Grid%nk diff --git a/src/mom5/ocean_param/sources/ocean_shortwave.F90 b/src/mom5/ocean_param/sources/ocean_shortwave.F90 index 41403421af..cd1256e74c 100644 --- a/src/mom5/ocean_param/sources/ocean_shortwave.F90 +++ b/src/mom5/ocean_param/sources/ocean_shortwave.F90 @@ -135,7 +135,7 @@ subroutine ocean_shortwave_init(Grid, Domain, Time, Dens, vert_coordinate, Ocean module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) #ifdef INTERNAL_FILE_NML read (input_nml_file, nml=ocean_shortwave_nml, iostat=io_status) diff --git a/src/mom5/ocean_param/sources/ocean_shortwave_csiro.F90 b/src/mom5/ocean_param/sources/ocean_shortwave_csiro.F90 index c759602bab..43f4072734 100644 --- a/src/mom5/ocean_param/sources/ocean_shortwave_csiro.F90 +++ b/src/mom5/ocean_param/sources/ocean_shortwave_csiro.F90 @@ -180,7 +180,7 @@ subroutine ocean_shortwave_csiro_init(Grid, Domain, Time, Ocean_options) module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) #ifdef INTERNAL_FILE_NML read (input_nml_file, nml=ocean_shortwave_csiro_nml, iostat=io_status) diff --git a/src/mom5/ocean_param/sources/ocean_shortwave_gfdl.F90 b/src/mom5/ocean_param/sources/ocean_shortwave_gfdl.F90 index 3b913f3453..bcbaea1659 100644 --- a/src/mom5/ocean_param/sources/ocean_shortwave_gfdl.F90 +++ b/src/mom5/ocean_param/sources/ocean_shortwave_gfdl.F90 @@ -283,7 +283,7 @@ subroutine ocean_shortwave_gfdl_init(Grid, Domain, Time, ver_coordinate, Ocean_o module_is_initialized = .TRUE. vert_coordinate = ver_coordinate - call write_version_number() + call write_version_number(version, tagname) #ifdef INTERNAL_FILE_NML read (input_nml_file, nml=ocean_shortwave_gfdl_nml, iostat=io_status) diff --git a/src/mom5/ocean_param/sources/ocean_shortwave_jerlov.F90 b/src/mom5/ocean_param/sources/ocean_shortwave_jerlov.F90 index 9bcc2f1cb3..7a33177f9f 100644 --- a/src/mom5/ocean_param/sources/ocean_shortwave_jerlov.F90 +++ b/src/mom5/ocean_param/sources/ocean_shortwave_jerlov.F90 @@ -257,7 +257,7 @@ subroutine ocean_shortwave_jerlov_init(Grid, Domain, ver_coordinate, Ocean_optio module_is_initialized = .TRUE. vert_coordinate = ver_coordinate - call write_version_number() + call write_version_number(version, tagname) #ifdef INTERNAL_FILE_NML read (input_nml_file, nml=ocean_shortwave_jerlov_nml, iostat=io_status) diff --git a/src/mom5/ocean_param/sources/ocean_sponges_eta.F90 b/src/mom5/ocean_param/sources/ocean_sponges_eta.F90 index 0e9189cd90..ad01f5881e 100644 --- a/src/mom5/ocean_param/sources/ocean_sponges_eta.F90 +++ b/src/mom5/ocean_param/sources/ocean_sponges_eta.F90 @@ -152,7 +152,7 @@ subroutine ocean_sponges_eta_init(Grid, Domain, Time, dtime, Ocean_options) allocate( Sponge(1) ) - call write_version_number() + call write_version_number(version, tagname) ! provide for namelist over-ride of default values #ifdef INTERNAL_FILE_NML diff --git a/src/mom5/ocean_param/sources/ocean_sponges_tracer.F90 b/src/mom5/ocean_param/sources/ocean_sponges_tracer.F90 index 7d6fa8929c..bf98f924c5 100644 --- a/src/mom5/ocean_param/sources/ocean_sponges_tracer.F90 +++ b/src/mom5/ocean_param/sources/ocean_sponges_tracer.F90 @@ -183,7 +183,7 @@ subroutine ocean_sponges_tracer_init(Grid, Domain, Time, T_prog, dtime, Ocean_op allocate( Sponge(num_prog_tracers) ) - call write_version_number() + call write_version_number(version, tagname) ! provide for namelist over-ride of default values #ifdef INTERNAL_FILE_NML diff --git a/src/mom5/ocean_param/sources/ocean_sponges_velocity.F90 b/src/mom5/ocean_param/sources/ocean_sponges_velocity.F90 index 96ff7a1168..3b344ee693 100644 --- a/src/mom5/ocean_param/sources/ocean_sponges_velocity.F90 +++ b/src/mom5/ocean_param/sources/ocean_sponges_velocity.F90 @@ -162,7 +162,7 @@ subroutine ocean_sponges_velocity_init(Grid, Domain, Time, dtime, Ocean_options) allocate( Sponge_u(1) ) allocate( Sponge_v(1) ) - call write_version_number() + call write_version_number(version, tagname) ! provide for namelist over-ride of default values #ifdef INTERNAL_FILE_NML diff --git a/src/mom5/ocean_param/sources/ocean_xlandinsert.F90 b/src/mom5/ocean_param/sources/ocean_xlandinsert.F90 index 8fcf2c7640..0b2376ffeb 100644 --- a/src/mom5/ocean_param/sources/ocean_xlandinsert.F90 +++ b/src/mom5/ocean_param/sources/ocean_xlandinsert.F90 @@ -261,7 +261,7 @@ subroutine ocean_xlandinsert_init(Grid, Domain, Time, Dens, T_prog, Ocean_option module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) call get_local_indices(Domain,isd,ied,jsd,jed,isc,iec,jsc,jec) call get_global_indices(Domain,isg,ieg,jsg,jeg) diff --git a/src/mom5/ocean_param/sources/ocean_xlandmix.F90 b/src/mom5/ocean_param/sources/ocean_xlandmix.F90 index 31e14cc6c4..f64e999853 100644 --- a/src/mom5/ocean_param/sources/ocean_xlandmix.F90 +++ b/src/mom5/ocean_param/sources/ocean_xlandmix.F90 @@ -295,7 +295,7 @@ subroutine ocean_xlandmix_init(Grid, Domain, Time, Dens, T_prog, Ocean_options, module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) call get_local_indices(Domain,isd,ied,jsd,jed,isc,iec,jsc,jec) call get_global_indices(Domain,isg,ieg,jsg,jeg) diff --git a/src/mom5/ocean_param/vertical/ocean_convect.F90 b/src/mom5/ocean_param/vertical/ocean_convect.F90 index fa952e52bc..3575cd86aa 100644 --- a/src/mom5/ocean_param/vertical/ocean_convect.F90 +++ b/src/mom5/ocean_param/vertical/ocean_convect.F90 @@ -220,7 +220,7 @@ subroutine ocean_convect_init (Grid, Domain, Time, Dens, T_prog, Ocean_options, module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) num_prog_tracers = size(T_prog(:)) diff --git a/src/mom5/ocean_param/vertical/ocean_form_drag.F90 b/src/mom5/ocean_param/vertical/ocean_form_drag.F90 index b55fce6bb1..9108e94b7e 100644 --- a/src/mom5/ocean_param/vertical/ocean_form_drag.F90 +++ b/src/mom5/ocean_param/vertical/ocean_form_drag.F90 @@ -348,7 +348,7 @@ subroutine ocean_form_drag_init(Grid, Domain, Time, Time_steps, Ocean_options, d module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) call get_local_indices(Domain,isd,ied,jsd,jed,isc,iec,jsc,jec) nk = Grid%nk diff --git a/src/mom5/ocean_param/vertical/ocean_vert_chen.F90 b/src/mom5/ocean_param/vertical/ocean_vert_chen.F90 index 42db6fdde3..aecbca1d7a 100644 --- a/src/mom5/ocean_param/vertical/ocean_vert_chen.F90 +++ b/src/mom5/ocean_param/vertical/ocean_vert_chen.F90 @@ -252,7 +252,7 @@ subroutine ocean_vert_chen_init (Grid, Domain, Time, Time_steps, T_prog, hor_gri endif module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) ! provide for namelist over-ride of defaults #ifdef INTERNAL_FILE_NML diff --git a/src/mom5/ocean_param/vertical/ocean_vert_const.F90 b/src/mom5/ocean_param/vertical/ocean_vert_const.F90 index 2db0c82d8c..d2455115db 100644 --- a/src/mom5/ocean_param/vertical/ocean_vert_const.F90 +++ b/src/mom5/ocean_param/vertical/ocean_vert_const.F90 @@ -120,7 +120,7 @@ subroutine ocean_vert_const_init (Grid, Domain, Time, Time_steps, T_prog) module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) ! provide for namelist over-ride of default values #ifdef INTERNAL_FILE_NML diff --git a/src/mom5/ocean_param/vertical/ocean_vert_gotm.F90 b/src/mom5/ocean_param/vertical/ocean_vert_gotm.F90 index 346cc9c654..376b3a14e2 100644 --- a/src/mom5/ocean_param/vertical/ocean_vert_gotm.F90 +++ b/src/mom5/ocean_param/vertical/ocean_vert_gotm.F90 @@ -333,7 +333,7 @@ subroutine ocean_vert_gotm_init (Grid, Domain, Time, Time_steps, T_prog, obc, de have_obc = obc module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) ! provide for namelist over-ride of defaults #ifdef INTERNAL_FILE_NML diff --git a/src/mom5/ocean_param/vertical/ocean_vert_kpp_mom4p0.F90 b/src/mom5/ocean_param/vertical/ocean_vert_kpp_mom4p0.F90 index 43d4bec95b..efcd684326 100644 --- a/src/mom5/ocean_param/vertical/ocean_vert_kpp_mom4p0.F90 +++ b/src/mom5/ocean_param/vertical/ocean_vert_kpp_mom4p0.F90 @@ -538,7 +538,7 @@ subroutine ocean_vert_kpp_mom4p0_init (Grid, Domain, Time, Time_steps, Dens, T_p rho_cp = rho0*cp_ocean inv_rho_cp = 1.0/rho_cp - call write_version_number() + call write_version_number(version, tagname) ! provide for namelist over-ride of defaults #ifdef INTERNAL_FILE_NML diff --git a/src/mom5/ocean_param/vertical/ocean_vert_kpp_mom4p1.F90 b/src/mom5/ocean_param/vertical/ocean_vert_kpp_mom4p1.F90 index 6188cb1113..cd8fa6ebc1 100644 --- a/src/mom5/ocean_param/vertical/ocean_vert_kpp_mom4p1.F90 +++ b/src/mom5/ocean_param/vertical/ocean_vert_kpp_mom4p1.F90 @@ -562,7 +562,7 @@ subroutine ocean_vert_kpp_mom4p1_init (Grid, Domain, Time, Time_steps, Dens, T_p rho_cp = rho0*cp_ocean inv_rho_cp = 1.0/rho_cp - call write_version_number() + call write_version_number(version, tagname) ! provide for namelist over-ride of defaults #ifdef INTERNAL_FILE_NML diff --git a/src/mom5/ocean_param/vertical/ocean_vert_kpp_test.F90 b/src/mom5/ocean_param/vertical/ocean_vert_kpp_test.F90 index a17ced3f7c..7f15b6b7d6 100644 --- a/src/mom5/ocean_param/vertical/ocean_vert_kpp_test.F90 +++ b/src/mom5/ocean_param/vertical/ocean_vert_kpp_test.F90 @@ -571,7 +571,7 @@ subroutine ocean_vert_kpp_test_init (Grid, Domain, Time, Time_steps, Dens, T_pro inv_rho_cp = 1.0/rho_cp horz_grid = hor_grid - call write_version_number() + call write_version_number(version, tagname) ! provide for namelist over-ride of defaults #ifdef INTERNAL_FILE_NML diff --git a/src/mom5/ocean_param/vertical/ocean_vert_mix.F90 b/src/mom5/ocean_param/vertical/ocean_vert_mix.F90 index 93393ae0ed..073e83bbfe 100644 --- a/src/mom5/ocean_param/vertical/ocean_vert_mix.F90 +++ b/src/mom5/ocean_param/vertical/ocean_vert_mix.F90 @@ -728,7 +728,7 @@ subroutine ocean_vert_mix_init (Grid, Domain, Time, Dens, Velocity, Time_steps, if (trim(T_prog(n)%name) == 'salt') index_salt = n enddo - call write_version_number() + call write_version_number(version, tagname) #ifdef INTERNAL_FILE_NML read (input_nml_file, nml=ocean_vert_mix_nml, iostat=io_status) diff --git a/src/mom5/ocean_param/vertical/ocean_vert_pp.F90 b/src/mom5/ocean_param/vertical/ocean_vert_pp.F90 index 7db995d77d..ff95b6e98e 100644 --- a/src/mom5/ocean_param/vertical/ocean_vert_pp.F90 +++ b/src/mom5/ocean_param/vertical/ocean_vert_pp.F90 @@ -184,7 +184,7 @@ subroutine ocean_vert_pp_init (Grid, Domain, Time, Time_steps, T_prog, hor_grid) module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) ! provide for namelist over-ride of defaults #ifdef INTERNAL_FILE_NML diff --git a/src/mom5/ocean_param/vertical/ocean_vert_tidal.F90 b/src/mom5/ocean_param/vertical/ocean_vert_tidal.F90 index 166263fe20..e7518bdefe 100644 --- a/src/mom5/ocean_param/vertical/ocean_vert_tidal.F90 +++ b/src/mom5/ocean_param/vertical/ocean_vert_tidal.F90 @@ -543,7 +543,7 @@ subroutine ocean_vert_tidal_init(Grid, Domain, Time, T_prog, Velocity, Ocean_opt module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) #ifdef INTERNAL_FILE_NML read (input_nml_file, nml=ocean_vert_tidal_nml, iostat=io_status) diff --git a/src/mom5/ocean_param/vertical/ocean_vert_tidal_test.F90 b/src/mom5/ocean_param/vertical/ocean_vert_tidal_test.F90 index 52340cef20..c1281e997e 100644 --- a/src/mom5/ocean_param/vertical/ocean_vert_tidal_test.F90 +++ b/src/mom5/ocean_param/vertical/ocean_vert_tidal_test.F90 @@ -561,7 +561,7 @@ subroutine ocean_vert_tidal_test_init(Grid, Domain, Time, T_prog, Velocity, Ocea module_is_initialized = .TRUE. - call write_version_number() + call write_version_number(version, tagname) #ifdef INTERNAL_FILE_NML read (input_nml_file, nml=ocean_vert_tidal_test_nml, iostat=io_status) diff --git a/src/mom5/ocean_tracers/ocean_passive.F90 b/src/mom5/ocean_tracers/ocean_passive.F90 index 8f2f7a04d4..62ecf1e009 100644 --- a/src/mom5/ocean_tracers/ocean_passive.F90 +++ b/src/mom5/ocean_tracers/ocean_passive.F90 @@ -433,7 +433,7 @@ subroutine ocean_passive_init(Domain, Grid, Ocean_options, debug) Dom => Domain Grd => Grid - call write_version_number() + call write_version_number(version, tagname) ! provide for namelist override of defaults #ifdef INTERNAL_FILE_NML diff --git a/src/mom5/ocean_tracers/ocean_tempsalt.F90 b/src/mom5/ocean_tracers/ocean_tempsalt.F90 index af5833c518..f7da00443b 100644 --- a/src/mom5/ocean_tracers/ocean_tempsalt.F90 +++ b/src/mom5/ocean_tracers/ocean_tempsalt.F90 @@ -326,7 +326,7 @@ subroutine ocean_tempsalt_init(Domain, Grid, Ocean_options, itemp, isalt, debug) Dom => Domain Grd => Grid - call write_version_number() + call write_version_number(version, tagname) ! provide for namelist override of defaults #ifdef INTERNAL_FILE_NML diff --git a/src/mom5/ocean_tracers/ocean_tracer_advect.F90 b/src/mom5/ocean_tracers/ocean_tracer_advect.F90 index 9367578c73..43b22b6ff5 100644 --- a/src/mom5/ocean_tracers/ocean_tracer_advect.F90 +++ b/src/mom5/ocean_tracers/ocean_tracer_advect.F90 @@ -524,7 +524,7 @@ subroutine ocean_tracer_advect_init (Grid, Domain, Time, Dens, T_prog, obc, debu have_obc = obc - call write_version_number() + call write_version_number(version, tagname) ! provide for namelist over-ride of defaults #ifdef INTERNAL_FILE_NML diff --git a/src/mom5/ocean_wave/ocean_wave.F90 b/src/mom5/ocean_wave/ocean_wave.F90 index b3a63b66dd..1d9999d6e8 100644 --- a/src/mom5/ocean_wave/ocean_wave.F90 +++ b/src/mom5/ocean_wave/ocean_wave.F90 @@ -167,7 +167,7 @@ subroutine ocean_wave_init(Grid, Domain, Waves, Time, Time_steps, Ocean_options, call mpp_clock_begin(id_init) - call write_version_number() + call write_version_number(version, tagname) #ifdef INTERNAL_FILE_NML read (input_nml_file, ocean_wave_nml, iostat=io_status) diff --git a/src/ocean_shared/generic_tracers/generic_BLING.F90 b/src/ocean_shared/generic_tracers/generic_BLING.F90 index e5cf9d3e5e..b572d9e883 100644 --- a/src/ocean_shared/generic_tracers/generic_BLING.F90 +++ b/src/ocean_shared/generic_tracers/generic_BLING.F90 @@ -606,7 +606,7 @@ end subroutine generic_BLING_register subroutine generic_BLING_init(tracer_list) type(g_tracer_type), pointer :: tracer_list - call write_version_number() + call write_version_number(version, tagname) !Specify and initialize all parameters used by this package call user_add_params diff --git a/src/ocean_shared/generic_tracers/generic_ERGOM.F90 b/src/ocean_shared/generic_tracers/generic_ERGOM.F90 index 6c5d6ed0e9..29fa9a058c 100644 --- a/src/ocean_shared/generic_tracers/generic_ERGOM.F90 +++ b/src/ocean_shared/generic_tracers/generic_ERGOM.F90 @@ -853,7 +853,7 @@ subroutine generic_ERGOM_register(tracer_list) stdoutunit=stdout();stdlogunit=stdlog() - call write_version_number() + call write_version_number(version, tagname) ! provide for namelist over-ride of defaults #ifdef INTERNAL_FILE_NML diff --git a/src/postprocessing/regrid/regrid.F90 b/src/postprocessing/regrid/regrid.F90 index d2b4176232..50f70a20ac 100644 --- a/src/postprocessing/regrid/regrid.F90 +++ b/src/postprocessing/regrid/regrid.F90 @@ -226,7 +226,7 @@ subroutine regrid_init call close_file (unit) !--- write out version information --------------------------------- - call write_version_number() + call write_version_number(version, tagname) if(num_flds == 0) call error_handler('regrid: nml num_fiels = 0 should be a positive number') if(num_flds .gt. max_flds) call error_handler('regrid: nml num_fiels is greater than maximum'// & diff --git a/src/preprocessing/generate_grids/atmos/atmos_grid.f90 b/src/preprocessing/generate_grids/atmos/atmos_grid.f90 index e42b44ed32..e7ed71a957 100644 --- a/src/preprocessing/generate_grids/atmos/atmos_grid.f90 +++ b/src/preprocessing/generate_grids/atmos/atmos_grid.f90 @@ -181,7 +181,7 @@ subroutine atmos_grid_init endif !--- write version info and namelist to logfile ---------------------- - call write_version_number() + call write_version_number(version, tagname) write (stdout(), nml=atmos_grid_nml) module_is_initialized = .true. diff --git a/src/preprocessing/generate_grids/atmos/atmos_grid_generator.f90 b/src/preprocessing/generate_grids/atmos/atmos_grid_generator.f90 index a16082bfa5..29a20f0d06 100644 --- a/src/preprocessing/generate_grids/atmos/atmos_grid_generator.f90 +++ b/src/preprocessing/generate_grids/atmos/atmos_grid_generator.f90 @@ -71,7 +71,7 @@ program grid_generator endif !--- write version info and namelist to logfile ---------------------- write (stdout(), nml=atmos_grid_generator_nml) - call write_version_number() + call write_version_number(version, tagname) !--- generate data --------------------------------------------------- call atmos_grid_init diff --git a/src/preprocessing/generate_grids/grid_transfer/grid_transfer.F90 b/src/preprocessing/generate_grids/grid_transfer/grid_transfer.F90 index 5a1f6d83d1..228616e562 100644 --- a/src/preprocessing/generate_grids/grid_transfer/grid_transfer.F90 +++ b/src/preprocessing/generate_grids/grid_transfer/grid_transfer.F90 @@ -107,7 +107,7 @@ subroutine grid_transfer_init endif !--- write version information - call write_version_number() + call write_version_number(version, tagname) end subroutine grid_transfer_init diff --git a/src/preprocessing/generate_grids/ocean/compare_grid.f90 b/src/preprocessing/generate_grids/ocean/compare_grid.f90 index f2c3215480..741e06fd6d 100644 --- a/src/preprocessing/generate_grids/ocean/compare_grid.f90 +++ b/src/preprocessing/generate_grids/ocean/compare_grid.f90 @@ -125,7 +125,7 @@ subroutine compare_grid_init endif !--- write out version information --------------------------------- - call write_version_number() + call write_version_number(version, tagname) end subroutine compare_grid_init diff --git a/src/preprocessing/generate_grids/ocean/edit_grid.F90 b/src/preprocessing/generate_grids/ocean/edit_grid.F90 index 62e8495881..641c87f028 100644 --- a/src/preprocessing/generate_grids/ocean/edit_grid.F90 +++ b/src/preprocessing/generate_grids/ocean/edit_grid.F90 @@ -198,7 +198,7 @@ subroutine edit_grid_init endif !--- write out version information --------------------------------- - call write_version_number() + call write_version_number(version, tagname) end subroutine edit_grid_init diff --git a/src/preprocessing/generate_grids/ocean/hgrid.f90 b/src/preprocessing/generate_grids/ocean/hgrid.f90 index 475d3f853c..0c65aafcf8 100644 --- a/src/preprocessing/generate_grids/ocean/hgrid.f90 +++ b/src/preprocessing/generate_grids/ocean/hgrid.f90 @@ -358,7 +358,7 @@ subroutine hgrid_init !--- write version info and namelist to logfile ---------------------- - call write_version_number() + call write_version_number(version, tagname) write (stdout(), nml=hgrid_nml) module_is_initialized = .true. diff --git a/src/preprocessing/generate_grids/ocean/ocean_grid_generator.f90 b/src/preprocessing/generate_grids/ocean/ocean_grid_generator.f90 index b9c2074302..675ac6cb1d 100644 --- a/src/preprocessing/generate_grids/ocean/ocean_grid_generator.f90 +++ b/src/preprocessing/generate_grids/ocean/ocean_grid_generator.f90 @@ -95,7 +95,7 @@ program ocean_grid_generator call mpp_error(FATAL,'ocean_grid_generator: file input.nml does not exist') endif - call write_version_number() + call write_version_number(version, tagname) if (mpp_pe() == mpp_root_pe()) write(stdout(), nml=ocean_grid_generator_nml) !--- generate data --------------------------------------------------- diff --git a/src/preprocessing/generate_grids/ocean/topog.f90 b/src/preprocessing/generate_grids/ocean/topog.f90 index 8810dc32fb..f71ef1c708 100644 --- a/src/preprocessing/generate_grids/ocean/topog.f90 +++ b/src/preprocessing/generate_grids/ocean/topog.f90 @@ -357,7 +357,7 @@ subroutine topog_init(Topog, Hgrid) !--- write version info and namelist to logfile ---------------------- - call write_version_number() + call write_version_number(version, tagname) write (stdout(), nml=topog_nml) write (stdout(), nml=obc_nml) diff --git a/src/preprocessing/generate_grids/ocean/vgrid.f90 b/src/preprocessing/generate_grids/ocean/vgrid.f90 index 88f3135b0c..0ac44d8b8a 100644 --- a/src/preprocessing/generate_grids/ocean/vgrid.f90 +++ b/src/preprocessing/generate_grids/ocean/vgrid.f90 @@ -183,7 +183,7 @@ subroutine vgrid_init !--- write version info and namelist to logfile ---------------------- - call write_version_number() + call write_version_number(version, tagname) if (mpp_pe() == mpp_root_pe()) then write (stdout(), nml=vgrid_nml) endif diff --git a/src/preprocessing/mom4_prep/idealized_bc/idealized_bc.f90 b/src/preprocessing/mom4_prep/idealized_bc/idealized_bc.f90 index 808969f45d..662694133f 100644 --- a/src/preprocessing/mom4_prep/idealized_bc/idealized_bc.f90 +++ b/src/preprocessing/mom4_prep/idealized_bc/idealized_bc.f90 @@ -167,7 +167,7 @@ subroutine idealized_bc_init ' generate_wind_bc, generate_water_bc should be true') !--- write out version information and namelist option --------------- - call write_version_number() + call write_version_number(version, tagname) if (mpp_pe() == mpp_root_pe()) then write (stdout(), nml= idealized_bc_nml) endif diff --git a/src/preprocessing/mom4_prep/idealized_ic/idealized_ic.f90 b/src/preprocessing/mom4_prep/idealized_ic/idealized_ic.f90 index e43e5b30bd..1c44194f01 100644 --- a/src/preprocessing/mom4_prep/idealized_ic/idealized_ic.f90 +++ b/src/preprocessing/mom4_prep/idealized_ic/idealized_ic.f90 @@ -424,7 +424,7 @@ subroutine idealized_ic_init 'idealized_ic_mod: nml "generate_temp_ic"="generate_salt_ic"="generate_salt_ic"=false. At least one should be true') !--- write out version information and namelist option --------------- - call write_version_number() + call write_version_number(version, tagname) if (mpp_pe() == mpp_root_pe()) then write (stdout(), nml= idealized_ic_nml) endif diff --git a/src/preprocessing/regrid/regrid.F90 b/src/preprocessing/regrid/regrid.F90 index aebeccf274..7961cedb02 100644 --- a/src/preprocessing/regrid/regrid.F90 +++ b/src/preprocessing/regrid/regrid.F90 @@ -223,7 +223,7 @@ subroutine regrid_init call close_file (unit) !--- write out version information --------------------------------- - call write_version_number() + call write_version_number(version, tagname) if(num_flds == 0) call error_handler('regrid: nml num_fiels = 0 should be a positive number') if(num_flds .gt. max_flds) call error_handler('regrid: nml num_fiels is greater than maximum'// & diff --git a/src/preprocessing/regrid_2d/regrid_2d.f90 b/src/preprocessing/regrid_2d/regrid_2d.f90 index 0bd4aabfe6..74d497338f 100644 --- a/src/preprocessing/regrid_2d/regrid_2d.f90 +++ b/src/preprocessing/regrid_2d/regrid_2d.f90 @@ -220,7 +220,7 @@ subroutine regrid_2d_init enddo !--- write version information - call write_version_number() + call write_version_number(version, tagname) end subroutine regrid_2d_init diff --git a/src/preprocessing/regrid_3d/regrid_3d.f90 b/src/preprocessing/regrid_3d/regrid_3d.f90 index 8422b97728..a98c751974 100644 --- a/src/preprocessing/regrid_3d/regrid_3d.f90 +++ b/src/preprocessing/regrid_3d/regrid_3d.f90 @@ -237,7 +237,7 @@ subroutine regrid_3d_init enddo !--- write version information - call write_version_number() + call write_version_number(version, tagname) end subroutine regrid_3d_init diff --git a/src/preprocessing/river_regrid/river_regrid.f90 b/src/preprocessing/river_regrid/river_regrid.f90 index 1aed800baa..ea2e0db370 100644 --- a/src/preprocessing/river_regrid/river_regrid.f90 +++ b/src/preprocessing/river_regrid/river_regrid.f90 @@ -164,7 +164,7 @@ subroutine river_regrid_init D2R = PI/180.0 !--- write out version information --------------------------------- - call write_version_number() + call write_version_number(version, tagname) !--- read the river data from river_input_file call read_river_src_data( ) diff --git a/src/shared/README.md b/src/shared/README.md new file mode 100644 index 0000000000..5d4b8aa8b3 --- /dev/null +++ b/src/shared/README.md @@ -0,0 +1,25 @@ +# Flexible Modeling System (FMS) + +The Flexible Modeling System (FMS) is a software framework for supporting the +efficient development, construction, execution, and scientific interpretation +of atmospheric, oceanic, and climate system models. + +More information is available on the [GFDL FMS page](http://www.gfdl.noaa.gov/fms). + +# Disclaimer + +The United States Department of Commerce (DOC) GitHub project code is provided +on an 'as is' basis and the user assumes responsibility for its use. DOC has +relinquished control of the information and no longer has responsibility to +protect the integrity, confidentiality, or availability of the information. Any +claims against the Department of Commerce stemming from the use of its GitHub +project will be governed by all applicable Federal law. Any reference to +specific commercial products, processes, or services by service mark, +trademark, manufacturer, or otherwise, does not constitute or imply their +endorsement, recommendation or favoring by the Department of Commerce. The +Department of Commerce seal and logo, or the seal and logo of a DOC bureau, +shall not be used in any manner to imply endorsement of any commercial product +or activity by DOC or the United States Government. + +This project code is made available through GitHub but is managed by NOAA-GFDL +at https://gitlab.gfdl.noaa.gov. diff --git a/src/shared/amip_interp/amip_interp.F90 b/src/shared/amip_interp/amip_interp.F90 index 5f7276ab8b..83c5ccc272 100644 --- a/src/shared/amip_interp/amip_interp.F90 +++ b/src/shared/amip_interp/amip_interp.F90 @@ -2,10 +2,11 @@ module amip_interp_mod -! +! ! Bruce Wyman ! +! ! ! Provides observed SST and ice mask data sets that have been @@ -62,6 +63,12 @@ module amip_interp_mod use time_manager_mod, only: time_type, operator(+), operator(>), & get_date, set_time, set_date +! add by JHC +use get_cal_time_mod, only: get_cal_time +use mpp_io_mod, only : mpp_open, mpp_read, MPP_RDONLY, MPP_NETCDF, & + MPP_MULTI, MPP_SINGLE, mpp_close, mpp_get_times +! end add by JHC + use horiz_interp_mod, only: horiz_interp_init, horiz_interp, & horiz_interp_new, horiz_interp_del, & horiz_interp_type, assignment(=) @@ -71,7 +78,7 @@ module amip_interp_mod open_namelist_file, open_ieee32_file, & mpp_pe, close_file, lowercase, mpp_root_pe, & NOTE, mpp_error, fms_error_handler -use fms_io_mod, only: read_data +use fms_io_mod, only: read_data, field_size ! add by JHC use constants_mod, only: TFREEZE, pi use platform_mod, only: R4_KIND, I2_KIND use mpp_mod, only: input_nml_file @@ -100,11 +107,13 @@ module amip_interp_mod ! ---- version number ----- -character(len=128) :: version = '$Id: amip_interp.F90,v 19.0 2012/01/06 21:54:21 fms Exp $' -character(len=128) :: tagname = '$Name: tikal $' +character(len=128) :: version = '$Id$' +character(len=128) :: tagname = '$Name$' real, allocatable:: temp1(:,:), temp2(:,:) - +! add by JHC + real, allocatable, dimension(:,:) :: tempamip +! end add by JHC !----------------------------------------------------------------------- !------ private defined data type -------- @@ -215,7 +224,7 @@ module amip_interp_mod ! type amip_interp_type private - type (horiz_interp_type) :: Hintrp + type (horiz_interp_type) :: Hintrp, Hintrp2 ! add by JHC real, pointer :: data1(:,:) =>NULL(), & data2(:,:) =>NULL() type (date_type) :: Date1, Date2 @@ -315,6 +324,8 @@ module amip_interp_mod ! 'reynolds_eof' ! 'reynolds_oi' ! 'hurrell' + ! add by JHC: + ! 'daily', when "use_daily=.T." character(len=16) :: date_out_of_range = 'fail' ! use 'fail' ! 'initclimo' @@ -337,6 +348,12 @@ module amip_interp_mod !global temperature perturbation used for sensitivity experiments real :: sst_pert = 0. +! add by JHC + character(len=6) :: sst_pert_type = 'fixed' ! use 'random' or 'fixed' + logical :: do_sst_pert = .false. + logical :: use_daily = .false. ! if '.true.', give 'data_set = 'daily'' +! end add by JHC + ! SJL: During nudging: use_ncep_sst = .T.; no_anom_sst = .T. ! during forecast: use_ncep_sst = .T.; no_anom_sst = .F. ! For seasonal forecast: use_ncep_ice = .F. @@ -349,7 +366,11 @@ module amip_interp_mod namelist /amip_interp_nml/ use_ncep_sst, no_anom_sst, use_ncep_ice, tice_crit, & interp_oi_sst, data_set, date_out_of_range, & use_zonal, teq, tdif, tann, tlag, amip_date, & - sst_pert, verbose, i_sst, j_sst, forecast_mode + ! add by JHC + sst_pert, sst_pert_type, do_sst_pert, & + use_daily, & + ! end add by JHC + verbose, i_sst, j_sst, forecast_mode ! @@ -364,7 +385,9 @@ module amip_interp_mod ! ! -subroutine get_amip_sst (Time, Interp, sst, err_msg) +! modified by JHC +subroutine get_amip_sst (Time, Interp, sst, err_msg, lon_model, lat_model) +!subroutine get_amip_sst (Time, Interp, sst, err_msg) type (time_type), intent(in) :: Time type (amip_interp_type), intent(inout) :: Interp @@ -380,6 +403,22 @@ subroutine get_amip_sst (Time, Interp, sst, err_msg) type(time_type) :: Amip_Time integer :: tod(3),dum +! add by JHC + real, intent(in), dimension(:,:), optional :: lon_model, lat_model + real :: pert + integer :: i, j, mobs_sst, nobs_sst + integer :: jhctod(6) + type (time_type) :: Udate + character(len=4) :: yyyy + integer :: nrecords, ierr, k, yr, mo, dy + integer :: siz(4) + integer, dimension(:), allocatable :: ryr, rmo, rdy + character(len=30) :: time_unit + real, dimension(:), allocatable :: timeval + character(len=maxc) :: ncfilename +! end add by JHC + + if(present(err_msg)) err_msg = '' if(.not.Interp%I_am_initialized) then if(fms_error_handler('get_amip_sst','The amip_interp_type variable is not initialized',err_msg)) return @@ -397,108 +436,243 @@ subroutine get_amip_sst (Time, Interp, sst, err_msg) Amip_Time = Time endif - if ( .not. allocated(temp1) ) allocate (temp1(mobs,nobs)) - if ( .not. allocated(temp2) ) allocate (temp2(mobs,nobs)) - -if (use_zonal) then - call zonal_sst (Amip_Time, sice, temp1) - call horiz_interp ( Interp%Hintrp, temp1, sst ) -else +! add by JHC +if ( .not.use_daily ) then +! end add by JHC + + if ( .not. allocated(temp1) ) allocate (temp1(mobs,nobs)) + if ( .not. allocated(temp2) ) allocate (temp2(mobs,nobs)) + + if (use_zonal) then + call zonal_sst (Amip_Time, sice, temp1) + call horiz_interp ( Interp%Hintrp, temp1, sst ) + else !----------------------------------------------------------------------- !---------- get new observed sea surface temperature ------------------- ! ---- time interpolation for months ----- - call time_interp (Amip_Time, fmonth, year1, year2, month1, month2) + call time_interp (Amip_Time, fmonth, year1, year2, month1, month2) ! ---- force climatology ---- - if (Interp % use_climo) then - year1=0; year2=0 - endif - if (Interp % use_annual) then - year1=0; year2=0 - month1=0; month2=0 - endif + if (Interp % use_climo) then + year1=0; year2=0 + endif + if (Interp % use_annual) then + year1=0; year2=0 + month1=0; month2=0 + endif ! --------------------------- - Date1 = date_type( year1, month1, 0 ) - Date2 = date_type( year2, month2, 0 ) + Date1 = date_type( year1, month1, 0 ) + Date2 = date_type( year2, month2, 0 ) ! -- open/rewind file -- - unit = -1 + unit = -1 !----------------------------------------------------------------------- - if (Date1 /= Interp % Date1) then + if (Date1 /= Interp % Date1) then ! ---- use Date2 for Date1 ---- - if (Date1 == Interp % Date2) then - Interp % Date1 = Interp % Date2 - Interp % data1 = Interp % data2 - temp1(:,:) = temp2(:,:) ! SJL BUG fix: June 24, 2011 - else - call read_record ('sst', Date1, Udate1, temp1) - if ( use_ncep_sst .and. (.not. no_anom_sst) ) then - temp1(:,:) = temp1(:,:) + sst_anom(:,:) - endif - call horiz_interp ( Interp%Hintrp, temp1, Interp%data1 ) - call clip_data ('sst', Interp%data1) - Interp % Date1 = Date1 - endif - endif + if (Date1 == Interp % Date2) then + Interp % Date1 = Interp % Date2 + Interp % data1 = Interp % data2 + temp1(:,:) = temp2(:,:) ! SJL BUG fix: June 24, 2011 + else + call read_record ('sst', Date1, Udate1, temp1) + if ( use_ncep_sst .and. (.not. no_anom_sst) ) then + temp1(:,:) = temp1(:,:) + sst_anom(:,:) + endif + call horiz_interp ( Interp%Hintrp, temp1, Interp%data1 ) + call clip_data ('sst', Interp%data1) + Interp % Date1 = Date1 + endif + endif !----------------------------------------------------------------------- - if (Date2 /= Interp % Date2) then - call read_record ('sst', Date2, Udate2, temp2) - if ( use_ncep_sst .and. (.not. no_anom_sst) ) then - temp2(:,:) = temp2(:,:) + sst_anom(:,:) - endif - call horiz_interp ( Interp%Hintrp, temp2, Interp%data2 ) - call clip_data ('sst', Interp%data2) - Interp % Date2 = Date2 - endif + if (Date2 /= Interp % Date2) then + call read_record ('sst', Date2, Udate2, temp2) + if ( use_ncep_sst .and. (.not. no_anom_sst) ) then + temp2(:,:) = temp2(:,:) + sst_anom(:,:) + endif + call horiz_interp ( Interp%Hintrp, temp2, Interp%data2 ) + call clip_data ('sst', Interp%data2) + Interp % Date2 = Date2 + endif ! ---- if the unit was opened, close it and print dates ---- - if (unit /= -1) then - call close_file (unit) - if (verbose > 0 .and. mpp_pe() == 0) & - call print_dates (Amip_Time, & - Interp % Date1, Udate1, & - Interp % Date2, Udate2, fmonth) - endif + if (unit /= -1) then + call close_file (unit) + if (verbose > 0 .and. mpp_pe() == 0) & + call print_dates (Amip_Time, & + Interp % Date1, Udate1, & + Interp % Date2, Udate2, fmonth) + endif !----------------------------------------------------------------------- !---------- time interpolation (between months) of sst's --------------- !----------------------------------------------------------------------- - sst = Interp % data1 + fmonth * (Interp % data2 - Interp % data1) + sst = Interp % data1 + fmonth * (Interp % data2 - Interp % data1) !------------------------------------------------------------------------------- ! SJL mods for NWP and TCSF --- ! Nudging runs: (Note: NCEP SST updated only every 6-hr) ! Compute SST anomaly from global SST datasets for subsequent forecast runs !------------------------------------------------------------------------------- - if ( use_ncep_sst .and. no_anom_sst ) then - sst_anom(:,:) = sst_ncep(:,:) - (temp1(:,:) + fmonth*(temp2(:,:) - temp1(:,:)) ) - call horiz_interp ( Interp%Hintrp, sst_ncep, sst ) - call clip_data ('sst', sst) + if ( use_ncep_sst .and. no_anom_sst ) then + sst_anom(:,:) = sst_ncep(:,:) - (temp1(:,:) + fmonth*(temp2(:,:) - temp1(:,:)) ) + call horiz_interp ( Interp%Hintrp, sst_ncep, sst ) + call clip_data ('sst', sst) + endif + +!!! DEBUG CODE +! call get_date(Amip_Time,jhctod(1),jhctod(2),jhctod(3),jhctod(4),jhctod(5),jhctod(6)) +! if (mpp_pe() == 0) then +! write (*,200) 'JHC: use_daily = F, AMIP_Time: ',jhctod(1),jhctod(2),jhctod(3),jhctod(4),jhctod(5),jhctod(6) +! write (*,300) 'JHC: use_daily = F, interped SST: ', sst(1,1),sst(5,5),sst(10,10) +! endif +!!! END DEBUG CODE + + endif +! add by JHC +else + call get_date(Amip_Time,jhctod(1),jhctod(2),jhctod(3),jhctod(4),jhctod(5),jhctod(6)) + if (mpp_pe() == mpp_root_pe()) write(*,200) 'amip_interp_mod: use_daily = T, Amip_Time = ',jhctod(1),jhctod(2),jhctod(3),jhctod(4),jhctod(5),jhctod(6) + + yr = jhctod(1); mo = jhctod(2); dy = jhctod(3) + + write (yyyy,'(i4)') jhctod(1) -endif + file_name_sst = 'INPUT/' // 'sst.day.mean.'//yyyy//'.v2.nc' + ncfilename = trim(file_name_sst) + time_unit = 'days since 1978-01-01 00:00:00' -! add on non-zero sea surface temperature perturbation (namelist option) -! this perturbation may be useful in accessing model sensitivities + mobs_sst = 1440; nobs_sst = 720 + + call set_sst_grid_edges_daily(mobs_sst, nobs_sst) + call horiz_interp_new ( Interp%Hintrp2, lon_bnd, lat_bnd, & + lon_model, lat_model, interp_method="bilinear" ) - if ( abs(sst_pert) > 0.0001 ) then - sst = sst + sst_pert - endif + if ( (.NOT. file_exist(ncfilename)) ) call mpp_error ('amip_interp_mod', & + 'cannot find daily SST input data file: '//trim(ncfilename), NOTE) + + if (file_exist(ncfilename)) then + if (mpp_pe() == mpp_root_pe()) call mpp_error ('amip_interp_mod', & + 'Reading NetCDF formatted daily SST from: '//trim(ncfilename), NOTE) + + call field_size(ncfilename, 'TIME', siz) + nrecords = siz (1) + if (nrecords < 1) call mpp_error('amip_interp_mod', & + 'Invalid number of SST records in daily SST data file: '//trim(ncfilename), FATAL) + allocate(timeval(nrecords), ryr(nrecords), rmo(nrecords), rdy(nrecords)) + + call mpp_open( unit, ncfilename, MPP_RDONLY, MPP_NETCDF, MPP_MULTI, MPP_SINGLE ) + call mpp_get_times(unit, timeval) + call mpp_close(unit) + +!!! DEBUG CODE +! if (mpp_pe() == 0) then +! print *, 'JHC: nrecords = ', nrecords +! print *, 'JHC: TIME = ', timeval +! endif +!!! END DEBUG CODE + + ierr = 1 + do k = 1, nrecords + + Udate = get_cal_time (timeval(k), time_unit, 'julian') + call get_date(Udate,jhctod(1),jhctod(2),jhctod(3),jhctod(4),jhctod(5),jhctod(6)) + ryr(k) = jhctod(1); rmo(k) = jhctod(2); rdy(k) = jhctod(3) + + if ( yr == ryr(k) .and. mo == rmo(k) .and. dy == rdy (k) ) ierr = 0 + if (ierr==0) exit + + enddo +!!! DEBUG CODE + if (mpp_pe() == 0) then + print *, 'JHC: k =', k + print *, 'JHC: ryr(k) rmo(k) rdy(k)',ryr(k), rmo(k), rdy(k) + print *, 'JHC: yr mo dy ',yr, mo, dy + endif +!!! END DEBUG CODE + if (ierr .ne. 0) call mpp_error('amip_interp_mod', & + 'Model time is out of range not in SST data: '//trim(ncfilename), FATAL) + endif ! if(file_exist(ncfilename)) + + + !---- read NETCDF data ---- + if ( .not. allocated(tempamip) ) allocate (tempamip(mobs_sst,nobs_sst)) + + if (file_exist(ncfilename)) then + call read_data(ncfilename, 'SST', tempamip, timelevel=k, no_domain=.true.) + tempamip = tempamip + TFREEZE + +!!! DEBUG CODE +! if (mpp_pe() == 0) then +! print*, 'JHC: TFREEZE = ', TFREEZE +! print*, lbound(sst) +! print*, ubound(sst) +! print*, lbound(tempamip) +! print*, ubound(tempamip) +! write(*,300) 'JHC: tempamip : ', tempamip(100,100), tempamip(200,200), tempamip(300,300) +! endif +!!! END DEBUG CODE + + call horiz_interp ( Interp%Hintrp2, tempamip, sst ) + call clip_data ('sst', sst) + + endif + +!!! DEBUG CODE +! if (mpp_pe() == 400) then +! write(*,300)'JHC: use_daily = T, daily SST: ', sst(1,1),sst(5,5),sst(10,10) +! print *,'JHC: use_daily = T, daily SST: ', sst +! endif +!!! END DEBUG CODE + +200 format(a35, 6(i5,1x)) +300 format(a35, 3(f7.3,2x)) + +endif +! end add by JHC + +! add by JHC: add on non-zero sea surface temperature perturbation (namelist option) +! This perturbation may be useful in accessing model sensitivities + + if ( do_sst_pert ) then + + if ( trim(sst_pert_type) == 'fixed' ) then + sst = sst + sst_pert + else if ( trim(sst_pert_type) == 'random' ) then + call random_seed() +!!! DEBUG CODE +! if (mpp_pe() == 0) then +! print*, 'mobs = ', mobs +! print*, 'nobs = ', nobs +! print*, lbound(sst) +! print*, ubound(sst) +! endif +!!! END DEBUG CODE + do i = 1, size(sst,1) + do j = 1, size(sst,2) + call random_number(pert) + sst (i,j) = sst (i,j) + sst_pert*((pert-0.5)*2) + end do + end do + endif + + endif +! end add by JHC + !----------------------------------------------------------------------- end subroutine get_amip_sst - !####################################################################### ! ! @@ -771,7 +945,7 @@ subroutine amip_interp_init() #endif ! ----- write namelist/version info ----- - call write_version_number() + call write_version_number (version, tagname) unit = stdlog ( ) if (mpp_pe() == 0) then @@ -819,7 +993,17 @@ subroutine amip_interp_init() tice_crit_k = 271.38 if (mpp_pe() == 0) & call error_mesg ('amip_interp_init', 'using HURRELL sst', NOTE) - Date_end = date_type( 2001, 12, 0 ) + Date_end = date_type( 2011, 8, 16 ) ! updated by JHC +! add by JHC + else if (lowercase(trim(data_set)) == 'daily') then + file_name_sst = 'INPUT/' // 'hurrell_sst.data' + file_name_ice = 'INPUT/' // 'hurrell_ice.data' + mobs = 360; nobs = 180 + call set_sst_grid_edges_oi + if (mpp_pe() == 0) & + call error_mesg ('amip_interp_init', 'using AVHRR daily sst', NOTE) + Date_end = date_type( 2011, 8, 16 ) +! end add by JHC else if (lowercase(trim(data_set)) == 'reynolds_eof') then file_name_sst = 'INPUT/' // 'reynolds_sst.data' file_name_ice = 'INPUT/' // 'reynolds_sst.data' @@ -937,12 +1121,15 @@ subroutine set_sst_grid_edges_amip1 end subroutine set_sst_grid_edges_amip1 !####################################################################### - subroutine set_sst_grid_edges_oi integer :: i, j real :: hpie, dlon, dlat, wb, sb +! add by JHC + if(allocated(lon_bnd)) deallocate(lon_bnd) + if(allocated(lat_bnd)) deallocate(lat_bnd) +! end add by JHC allocate ( lon_bnd(mobs+1), lat_bnd(nobs+1) ) ! ---- compute grid edges (do only once) ----- @@ -963,6 +1150,37 @@ subroutine set_sst_grid_edges_oi enddo end subroutine set_sst_grid_edges_oi +!####################################################################### +! add by JHC + subroutine set_sst_grid_edges_daily(mobs_sst, nobs_sst) + + integer :: i, j, mobs_sst, nobs_sst + real :: hpie, dlon, dlat, wb, sb + + if(allocated(lon_bnd)) deallocate(lon_bnd) + if(allocated(lat_bnd)) deallocate(lat_bnd) + allocate ( lon_bnd(mobs_sst+1), lat_bnd(nobs_sst+1) ) + +! ---- compute grid edges (do only once) ----- + + hpie = 0.5*pi + + dlon = 4.*hpie/float(mobs_sst); wb = 0.0 + lon_bnd(1) = wb + do i = 2, mobs_sst+1 + lon_bnd(i) = wb + dlon * float(i-1) + enddo + lon_bnd(mobs_sst+1) = lon_bnd(1) + 4.*hpie + + dlat = 2.*hpie/float(nobs_sst); sb = -hpie + lat_bnd(1) = sb; lat_bnd(nobs_sst+1) = hpie + do j = 2, nobs_sst + lat_bnd(j) = sb + dlat * float(j-1) + enddo + + end subroutine set_sst_grid_edges_daily +! end add by JHC +!####################################################################### subroutine a2a_bilinear(nx, ny, dat1, n1, n2, dat2) @@ -1191,7 +1409,8 @@ subroutine read_record (type, Date, Adate, dat) else if(type(1:3) == 'ice') then ncfilename = trim(file_name_ice)//'.nc' if (lowercase(trim(data_set)) == 'amip2' .or. & - lowercase(trim(data_set)) == 'hurrell') ncfieldname = 'ice' + lowercase(trim(data_set)) == 'hurrell' .or. & + lowercase(trim(data_set)) == 'daily') ncfieldname = 'ice' ! modified by JHC endif !---- make sure IEEE format file is open ---- diff --git a/src/shared/astronomy/astronomy.F90 b/src/shared/astronomy/astronomy.F90 index 19bf449615..09f2593028 100644 --- a/src/shared/astronomy/astronomy.F90 +++ b/src/shared/astronomy/astronomy.F90 @@ -1,9 +1,10 @@ module astronomy_mod -! +! ! fil ! ! ! +! ! ! astronomy_mod provides astronomical variables for use ! by other modules within fms. the only currently used interface is @@ -45,8 +46,8 @@ module astronomy_mod !--------------------------------------------------------------------- !----------- version number for this module -------------------------- -character(len=128) :: version = '$Id: astronomy.F90,v 20.0 2013/12/14 00:18:17 fms Exp $' -character(len=128) :: tagname = '$Name: tikal $' +character(len=128) :: version = '$Id$' +character(len=128) :: tagname = '$Name$' !--------------------------------------------------------------------- @@ -287,7 +288,7 @@ subroutine astronomy_init (latb, lonb) !--------------------------------------------------------------------- ! write version number and namelist to logfile. !--------------------------------------------------------------------- - call write_version_number() + call write_version_number (version, tagname) if (mpp_pe() == mpp_root_pe() ) then unit = stdlog() write (unit, nml=astronomy_nml) @@ -2988,9 +2989,9 @@ subroutine astronomy_end !---------------------------------------------------------------------- ! check if the module has been initialized. !---------------------------------------------------------------------- - if (.not. module_is_initialized) & - call error_mesg ( 'astronomy_mod', & - ' module has not been initialized', FATAL) + if (.not. module_is_initialized) return +! call error_mesg ( 'astronomy_mod', & +! ' module has not been initialized', FATAL) !---------------------------------------------------------------------- ! deallocate module variables. @@ -3152,7 +3153,7 @@ function r_inv_squared (ang) ! its square (r_inv_squared) to the calling routine. !-------------------------------------------------------------------- rad_per = per*deg_to_rad - r = (1 - ecc**2)/(1. + ecc*cos(ang - rad_per)) + r = (1. - ecc**2)/(1. + ecc*cos(ang - rad_per)) r_inv_squared = r**(-2) diff --git a/src/shared/axis_utils/axis_utils.F90 b/src/shared/axis_utils/axis_utils.F90 index 59f3b42db6..1947234315 100644 --- a/src/shared/axis_utils/axis_utils.F90 +++ b/src/shared/axis_utils/axis_utils.F90 @@ -1,8 +1,8 @@ module axis_utils_mod ! - !M.J. Harrison + !M.J. Harrison ! - !Bruce Wyman + !Bruce Wyman ! ! @@ -27,9 +27,9 @@ module axis_utils_mod use mpp_io_mod, only: axistype, atttype, default_axis, default_att, & mpp_get_atts, mpp_get_axis_data, mpp_modify_meta, & mpp_get_att_name, mpp_get_att_type, mpp_get_att_char, & - mpp_get_att_length + mpp_get_att_length, mpp_get_axis_bounds use mpp_mod, only: mpp_error, FATAL, stdout - use fms_mod, only: lowercase, string_array_index + use fms_mod, only: lowercase, string_array_index, fms_error_handler implicit none @@ -43,8 +43,8 @@ module axis_utils_mod integer, parameter :: maxatts = 100 real, parameter :: epsln= 1.e-10 real, parameter :: fp5 = 0.5, f360 = 360.0 - character(len=256) :: version = '$Id: axis_utils.F90,v 20.0 2013/12/14 00:18:21 fms Exp $' - character(len=256) :: tagname = '$Name: tikal $' + character(len=256) :: version = '$Id$' + character(len=256) :: tagname = '$Name$' interface interp_1d module procedure interp_1d_1d @@ -125,52 +125,39 @@ subroutine get_axis_cart(axis, cart) end subroutine get_axis_cart - subroutine get_axis_bounds(axis,axis_bound,axes) + subroutine get_axis_bounds(axis,axis_bound,axes,bnd_name,err_msg) type(axistype), intent(in) :: axis type(axistype), intent(inout) :: axis_bound type(axistype), intent(in), dimension(:) :: axes + character(len=*), intent(out), optional :: bnd_name, err_msg - type(atttype), dimension(:), allocatable :: att real, dimension(:), allocatable :: data, tmp integer :: i, len - character(len=128) :: bounds_name = 'none', name, units + character(len=128) :: name, units character(len=256) :: longname character(len=1) :: cartesian + logical :: bounds_found + if(present(err_msg)) then + err_msg = '' + endif axis_bound = default_axis - allocate(att(maxatts)) - att = default_att - call mpp_get_atts(axis,atts=att) - - do i=1,maxatts - if (mpp_get_att_type(att(i)) == NF_CHAR) then - ! if (str_contains(att(i)%name,'bounds') .or. str_contains(att(i)%name,'edge')) then - if (string_array_index('bounds',(/mpp_get_att_name(att(i))/)) .or. & - string_array_index('edge',(/mpp_get_att_name(att(i))/))) then - bounds_name = mpp_get_att_char(att(i)) - endif - endif - enddo + call mpp_get_atts(axis,units=units,longname=longname,& + cartesian=cartesian, len=len) + if(len .LE. 0) return + allocate(data(len+1)) - if (trim(bounds_name) /= 'none') then - do i=1,size(axes(:)) - call mpp_get_atts(axes(i),name=name) - if (lowercase(trim(name)) == lowercase(trim(bounds_name))) then - axis_bound = axes(i) - endif - enddo - call mpp_get_atts(axis_bound,len=len) - if (len < 1) call mpp_error(FATAL,'error locating boundary axis for '//bounds_name) - else - call mpp_get_atts(axis,name=name,units=units,longname=longname,& - cartesian=cartesian,len=len) + bounds_found = mpp_get_axis_bounds(axis, data, name=name) + longname = trim(longname)//' bounds' + + if(.not.bounds_found .and. len>1 ) then + ! The following calculation can not be done for len=1 + call mpp_get_atts(axis,name=name) name = trim(name)//'_bounds' - longname = trim(longname)//' bounds' allocate(tmp(len)) call mpp_get_axis_data(axis,tmp) - allocate(data(len+1)) do i=2,len data(i)= tmp(i-1)+fp5*(tmp(i)-tmp(i-1)) enddo @@ -180,11 +167,13 @@ subroutine get_axis_bounds(axis,axis_bound,axes) if (data(1) == 0.0) then if (abs(data(len+1)-360.) > epsln) data(len+1)=360.0 endif + endif + if(bounds_found .OR. len>1) then call mpp_modify_meta(axis_bound,name=name,units=units,longname=& - longname,cartesian=cartesian,data=data) - deallocate(tmp) - deallocate(data) + longname,cartesian=cartesian,data=data) endif + if(allocated(tmp)) deallocate(tmp) + deallocate(data) return end subroutine get_axis_bounds @@ -621,7 +610,7 @@ subroutine interp_1d_cubic_spline(grid1, grid2, data1, data2, yp1, ypn) h = grid1(khi)-grid1(klo) a = (grid1(khi) - grid2(k))/h b = (grid2(k) - grid1(klo))/h - data2(k) = a*data1(klo) + b*data1(khi)+ ((a**3-a)*y2(klo) + (b**3-b)*y2(khi))*(h**2)/6 + data2(k) = a*data1(klo) + b*data1(khi)+ ((a**3-a)*y2(klo) + (b**3-b)*y2(khi))*(h**2)/6. enddo end subroutine interp_1d_cubic_spline diff --git a/src/shared/block_control/block_control.F90 b/src/shared/block_control/block_control.F90 new file mode 100644 index 0000000000..518f92f354 --- /dev/null +++ b/src/shared/block_control/block_control.F90 @@ -0,0 +1,90 @@ +module block_control_mod +#include + +use mpp_mod, only: mpp_error, NOTE, FATAL +use mpp_domains_mod, only: mpp_compute_extent + + public block_control_type + type block_control_type + integer :: nx_block, ny_block + integer :: nblks + integer :: isc, iec, jsc, jec + integer :: npz + integer, dimension(:), _ALLOCATABLE :: ibs _NULL, & + ibe _NULL, & + jbs _NULL, & + jbe _NULL + end type block_control_type + +public :: define_blocks + +contains + +!---------------------------------------------------------------------- +! set up "blocks" used for OpenMP threading of column-based +! calculations using rad_n[x/y]xblock from coupler_nml +!--------------------------------------------------------------------- + subroutine define_blocks (component, Block, isc, iec, jsc, jec, kpts, & + nx_block, ny_block, message) + character(len=*), intent(in) :: component + type(block_control_type), intent(inout) :: Block + integer, intent(in) :: isc, iec, jsc, jec, kpts + integer, intent(in) :: nx_block, ny_block + logical, intent(inout) :: message +!--- local variables + integer :: blocks + integer, dimension(nx_block) :: i1, i2 + integer, dimension(ny_block) :: j1, j2 + character(len=132) :: text + integer :: i, j, nblks + + if (message) then + if ((mod(iec-isc+1,nx_block) .ne. 0) .or. (mod(jec-jsc+1,ny_block) .ne. 0)) then + write( text,'(a,a,2i4,a,2i4,a)' ) trim(component),'define_blocks: domain (',& + (iec-isc+1), (jec-jsc+1),') is not an even divisor with definition (',& + nx_block, ny_block,') - blocks will not be uniform' + call mpp_error( NOTE, trim(text) ) + endif + message = .false. + endif + +!--- set up blocks + if (iec-isc+1 .lt. nx_block) & + call mpp_error(FATAL, 'block_control: number of '//trim(component)//' nxblocks .gt. & + &number of elements in MPI-domain size') + if (jec-jsc+1 .lt. ny_block) & + call mpp_error(FATAL, 'block_control: number of '//trim(component)//' nyblocks .gt. & + &number of elements in MPI-domain size') + call mpp_compute_extent(isc,iec,nx_block,i1,i2) + call mpp_compute_extent(jsc,jec,ny_block,j1,j2) + + nblks = nx_block*ny_block + Block%isc = isc + Block%iec = iec + Block%jsc = jsc + Block%jec = jec + Block%npz = kpts + Block%nx_block = nx_block + Block%ny_block = ny_block + Block%nblks = nblks + + if (.not._ALLOCATED(Block%ibs)) & + allocate (Block%ibs(nblks), & + Block%ibe(nblks), & + Block%jbs(nblks), & + Block%jbe(nblks)) + + blocks=0 + do j = 1, ny_block + do i = 1, nx_block + blocks = blocks + 1 + Block%ibs(blocks) = i1(i) + Block%jbs(blocks) = j1(j) + Block%ibe(blocks) = i2(i) + Block%jbe(blocks) = j2(j) + enddo + enddo + + end subroutine define_blocks + +end module block_control_mod diff --git a/src/shared/column_diagnostics/column_diagnostics.F90 b/src/shared/column_diagnostics/column_diagnostics.F90 index 69937d910d..8e115901b1 100644 --- a/src/shared/column_diagnostics/column_diagnostics.F90 +++ b/src/shared/column_diagnostics/column_diagnostics.F90 @@ -34,8 +34,8 @@ module column_diagnostics_mod !----------- ****** VERSION NUMBER ******* --------------------------- -character(len=128) :: version = '$Id: column_diagnostics.F90,v 20.0 2013/12/14 00:18:24 fms Exp $' -character(len=128) :: tag = '$Name: tikal $' +character(len=128) :: version = '$Id$' +character(len=128) :: tag = '$Name$' @@ -145,7 +145,7 @@ subroutine column_diagnostics_init !--------------------------------------------------------------------- ! write version number and namelist to logfile. !--------------------------------------------------------------------- - call write_version_number() + call write_version_number (version, tag) if (mpp_pe() == mpp_root_pe()) then unit = stdlog() write (unit, nml=column_diagnostics_nml) diff --git a/src/shared/constants/constants.F90 b/src/shared/constants/constants.F90 index 7f02965fb6..539fbac2e0 100644 --- a/src/shared/constants/constants.F90 +++ b/src/shared/constants/constants.F90 @@ -1,10 +1,11 @@ module constants_mod -! +! ! Bruce Wyman ! +! ! ! Defines useful constants for Earth. @@ -18,8 +19,8 @@ module constants_mod implicit none private -character(len=128) :: version='$Id: constants.F90,v 17.0 2009/07/21 03:18:26 fms Exp $' -character(len=128) :: tagname='$Name: tikal $' +character(len=128) :: version='$Id$' +character(len=128) :: tagname='$Name$' !dummy variable to use in HUGE initializations real :: realnumber diff --git a/src/shared/coupler/atmos_ocean_fluxes.F90 b/src/shared/coupler/atmos_ocean_fluxes.F90 index 35c2053af2..fb6024eb5e 100644 --- a/src/shared/coupler/atmos_ocean_fluxes.F90 +++ b/src/shared/coupler/atmos_ocean_fluxes.F90 @@ -19,10 +19,10 @@ !----------------------------------------------------------------------- ! ! -! Richard D. Slater +! Richard D. Slater ! ! -! John P. Dunne +! John P. Dunne ! ! ! @@ -188,8 +188,8 @@ module atmos_ocean_fluxes_mod !{ !---------------------------------------------------------------------- ! -character(len=128) :: version = '$Id: atmos_ocean_fluxes.F90,v 18.0 2010/03/02 23:55:03 fms Exp $' -character(len=128) :: tagname = '$Name: tikal $' +character(len=128) :: version = '$Id$' +character(len=128) :: tagname = '$Name$' ! !----------------------------------------------------------------------- @@ -408,12 +408,12 @@ function aof_set_coupler_flux(name, flux_type, implementation, atm_tr_index, par endif else if (implementation .eq. implementation_test) then - long_err_msg = 'Undefined flux_type/implementation (flux_type given from field_table): ' - long_err_msg = long_err_msg // trim(flux_type_test) // '/implementation/' // trim(implementation_test) + long_err_msg = 'Undefined flux_type/implementation (flux_type given from field_table): ' + long_err_msg = long_err_msg // trim(flux_type_test) // '/implementation/' // trim(implementation_test) call mpp_error(FATAL, trim(error_header) // long_err_msg) else - long_err_msg = ' Undefined flux_type/implementation (both given from field_table): ' - long_err_msg = long_err_msg // trim(flux_type_test) // '/implementation/' // trim(implementation_test) + long_err_msg = ' Undefined flux_type/implementation (both given from field_table): ' + long_err_msg = long_err_msg // trim(flux_type_test) // '/implementation/' // trim(implementation_test) call mpp_error(FATAL, trim(error_header) // long_err_msg) endif endif @@ -968,10 +968,6 @@ subroutine atmos_ocean_fluxes_calc(gas_fields_atm, gas_fields_ice, & character(len=64), parameter :: sub_name = 'atmos_ocean_fluxes_calc' character(len=256), parameter :: error_header = & '==>Error from ' // trim(mod_name) // '(' // trim(sub_name) // '):' -character(len=256), parameter :: warn_header = & - '==>Warning from ' // trim(mod_name) // '(' // trim(sub_name) // '):' -character(len=256), parameter :: note_header = & - '==>Note from ' // trim(mod_name) // '(' // trim(sub_name) // '):' ! !----------------------------------------------------------------------- @@ -1037,14 +1033,14 @@ subroutine atmos_ocean_fluxes_calc(gas_fields_atm, gas_fields_ice, & if (gas_fluxes%bc(n)%implementation .eq. 'ocmip2') then !}{ do i = 1, length !{ - if (seawater(i) == 1) then !{ + if (seawater(i) == 1.) then !{ gas_fluxes%bc(n)%field(ind_kw)%values(i) = gas_fluxes%bc(n)%param(1) * gas_fields_atm%bc(n)%field(ind_u10)%values(i)**2 cair(i) = & gas_fields_ice%bc(n)%field(ind_alpha)%values(i) * & gas_fields_atm%bc(n)%field(ind_pCair)%values(i) * & gas_fields_atm%bc(n)%field(ind_psurf)%values(i) * gas_fluxes%bc(n)%param(2) gas_fluxes%bc(n)%field(ind_flux)%values(i) = gas_fluxes%bc(n)%field(ind_kw)%values(i) * & - sqrt(660 / (gas_fields_ice%bc(n)%field(ind_sc_no)%values(i) + epsln)) * & + sqrt(660. / (gas_fields_ice%bc(n)%field(ind_sc_no)%values(i) + epsln)) * & (gas_fields_ice%bc(n)%field(ind_csurf)%values(i) - cair(i)) gas_fluxes%bc(n)%field(ind_deltap)%values(i) = (gas_fields_ice%bc(n)%field(ind_csurf)%values(i) - cair(i)) / & (gas_fields_ice%bc(n)%field(ind_alpha)%values(i) * permeg + epsln) @@ -1077,7 +1073,7 @@ subroutine atmos_ocean_fluxes_calc(gas_fields_atm, gas_fields_ice, & if (gas_fluxes%bc(n)%implementation .eq. 'ocmip2_data') then !{ do i = 1, length !{ - if (seawater(i) == 1) then !{ + if (seawater(i) == 1.) then !{ kw(i) = gas_fluxes%bc(n)%param(1) * gas_fields_atm%bc(n)%field(ind_u10)%values(i) cair(i) = & gas_fields_ice%bc(n)%field(ind_alpha)%values(i) * & @@ -1095,7 +1091,7 @@ subroutine atmos_ocean_fluxes_calc(gas_fields_atm, gas_fields_ice, & elseif (gas_fluxes%bc(n)%implementation .eq. 'ocmip2') then !}{ do i = 1, length !{ - if (seawater(i) == 1) then !{ + if (seawater(i) == 1.) then !{ kw(i) = gas_fluxes%bc(n)%param(1) * gas_fields_atm%bc(n)%field(ind_u10)%values(i)**2 cair(i) = & gas_fields_ice%bc(n)%field(ind_alpha)%values(i) * & @@ -1113,7 +1109,7 @@ subroutine atmos_ocean_fluxes_calc(gas_fields_atm, gas_fields_ice, & elseif (gas_fluxes%bc(n)%implementation .eq. 'linear') then !}{ do i = 1, length !{ - if (seawater(i) == 1) then !{ + if (seawater(i) == 1.) then !{ kw(i) = gas_fluxes%bc(n)%param(1) * max(0.0, gas_fields_atm%bc(n)%field(ind_u10)%values(i) - gas_fluxes%bc(n)%param(2)) cair(i) = & gas_fields_ice%bc(n)%field(ind_alpha)%values(i) * & @@ -1147,7 +1143,7 @@ subroutine atmos_ocean_fluxes_calc(gas_fields_atm, gas_fields_ice, & if (gas_fluxes%bc(n)%implementation .eq. 'dry') then !{ do i = 1, length !{ - if (seawater(i) == 1) then !{ + if (seawater(i) == 1.) then !{ gas_fluxes%bc(n)%field(ind_flux)%values(i) = & gas_fields_atm%bc(n)%field(ind_deposition)%values(i) / gas_fluxes%bc(n)%param(1) else !}{ @@ -1158,7 +1154,7 @@ subroutine atmos_ocean_fluxes_calc(gas_fields_atm, gas_fields_ice, & elseif (gas_fluxes%bc(n)%implementation .eq. 'wet') then !}{ do i = 1, length !{ - if (seawater(i) == 1) then !{ + if (seawater(i) == 1.) then !{ gas_fluxes%bc(n)%field(ind_flux)%values(i) = & gas_fields_atm%bc(n)%field(ind_deposition)%values(i) / gas_fluxes%bc(n)%param(1) else !}{ @@ -1186,7 +1182,7 @@ subroutine atmos_ocean_fluxes_calc(gas_fields_atm, gas_fields_ice, & if (gas_fluxes%bc(n)%implementation .eq. 'river') then !{ do i = 1, length !{ - if (seawater(i) == 1) then !{ + if (seawater(i) == 1.) then !{ gas_fluxes%bc(n)%field(ind_flux)%values(i) = & gas_fields_atm%bc(n)%field(ind_deposition)%values(i) / gas_fluxes%bc(n)%param(1) else !}{ diff --git a/src/shared/coupler/coupler_types.F90 b/src/shared/coupler/coupler_types.F90 index 8cf23970df..713407ba34 100644 --- a/src/shared/coupler/coupler_types.F90 +++ b/src/shared/coupler/coupler_types.F90 @@ -19,11 +19,11 @@ module coupler_types_mod !{ ! or see: http://www.gnu.org/licenses/gpl.html !----------------------------------------------------------------------- ! -! +! ! Richard D. Slater ! ! -! +! ! John Dunne ! ! @@ -171,8 +171,8 @@ module coupler_types_mod !{ implicit none ! !----------------------------------------------------------------------- - character(len=128) :: version = '$Id: coupler_types.F90,v 18.0 2010/03/02 23:55:06 fms Exp $' - character(len=128) :: tag = '$Name: tikal $' + character(len=128) :: version = '$Id$' + character(len=128) :: tag = '$Name$' !----------------------------------------------------------------------- real, parameter :: bound_tol = 1e-7 @@ -253,7 +253,7 @@ module coupler_types_mod !{ logical :: use_atm_pressure logical :: use_10m_wind_speed logical :: pass_through_ice - real :: mol_wt = 0.0 + real :: mol_wt = 0.0 end type coupler_3d_field_type type, public :: coupler_3d_bc_type !{ @@ -289,7 +289,7 @@ module coupler_types_mod !{ logical :: use_atm_pressure logical :: use_10m_wind_speed logical :: pass_through_ice - real :: mol_wt = 0.0 + real :: mol_wt = 0.0 end type coupler_2d_field_type type, public :: coupler_2d_bc_type !{ @@ -325,7 +325,7 @@ module coupler_types_mod !{ logical :: use_atm_pressure logical :: use_10m_wind_speed logical :: pass_through_ice - real :: mol_wt = 0.0 + real :: mol_wt = 0.0 end type coupler_1d_field_type type, public :: coupler_1d_bc_type !{ @@ -439,10 +439,6 @@ subroutine coupler_types_init character(len=64), parameter :: sub_name = 'coupler_types_init' character(len=256), parameter :: error_header = & '==>Error from ' // trim(mod_name) // '(' // trim(sub_name) // '):' -character(len=256), parameter :: warn_header = & - '==>Warning from ' // trim(mod_name) // '(' // trim(sub_name) // '):' -character(len=256), parameter :: note_header = & - '==>Note from ' // trim(mod_name) // '(' // trim(sub_name) // '):' ! !----------------------------------------------------------------------- @@ -976,10 +972,6 @@ subroutine coupler_type_copy_1d_2d(var_in, var_out, is, ie, js, je, & character(len=64), parameter :: sub_name = 'coupler_type_copy_1d_2d' character(len=256), parameter :: error_header = & '==>Error from ' // trim(mod_name) // '(' // trim(sub_name) // '):' -character(len=256), parameter :: warn_header = & - '==>Warning from ' // trim(mod_name) // '(' // trim(sub_name) // '):' -character(len=256), parameter :: note_header = & - '==>Note from ' // trim(mod_name) // '(' // trim(sub_name) // '):' ! !----------------------------------------------------------------------- @@ -1156,10 +1148,6 @@ subroutine coupler_type_copy_1d_3d(var_in, var_out, is, ie, js, je, kd, & character(len=64), parameter :: sub_name = 'coupler_type_copy_1d_3d' character(len=256), parameter :: error_header = & '==>Error from ' // trim(mod_name) // '(' // trim(sub_name) // '):' -character(len=256), parameter :: warn_header = & - '==>Warning from ' // trim(mod_name) // '(' // trim(sub_name) // '):' -character(len=256), parameter :: note_header = & - '==>Note from ' // trim(mod_name) // '(' // trim(sub_name) // '):' ! !----------------------------------------------------------------------- diff --git a/src/shared/coupler/ensemble_manager.F90 b/src/shared/coupler/ensemble_manager.F90 index 394ceed49f..cf67973fab 100644 --- a/src/shared/coupler/ensemble_manager.F90 +++ b/src/shared/coupler/ensemble_manager.F90 @@ -6,7 +6,6 @@ module ensemble_manager_mod use mpp_mod, only : mpp_pe, mpp_declare_pelist use mpp_mod, only : input_nml_file use fms_io_mod, only : set_filename_appendix - use diag_manager_mod, only : set_diag_filename_appendix IMPLICIT NONE @@ -358,9 +357,6 @@ subroutine ensemble_pelist_setup(concurrent, atmos_npes, ocean_npes, land_npes, write( text,'(a,i2.2)' ) 'ens_', ensemble_id !Append ensemble_id to the restart filenames call set_filename_appendix(trim(text)) - !Append ensemble_id to the diag_out filenames - write( text,'(a,i2.2)' ) '.ens_', ensemble_id - call set_diag_filename_appendix(trim(text)) endif end subroutine ensemble_pelist_setup diff --git a/src/shared/data_override/data_override.F90 b/src/shared/data_override/data_override.F90 index 684a211027..10e4d1145c 100644 --- a/src/shared/data_override/data_override.F90 +++ b/src/shared/data_override/data_override.F90 @@ -24,15 +24,15 @@ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! module data_override_mod ! -! +! ! G.T. Nong ! ! -! +! ! M.J. Harrison ! ! -! +! ! M. Winton ! @@ -80,8 +80,8 @@ module data_override_mod implicit none private -character(len=128) :: version = '$Id: data_override.F90,v 20.0 2013/12/14 00:18:34 fms Exp $' -character(len=128) :: tagname = '$Name: tikal $' +character(len=128) :: version = '$Id$' +character(len=128) :: tagname = '$Name$' type data_type character(len=3) :: gridname @@ -224,7 +224,7 @@ subroutine data_override_init(Atm_domain_in, Ocean_domain_in, Ice_domain_in, Lan radian_to_deg = 180./PI deg_to_radian = PI/180. - call write_version_number() + call write_version_number (version, tagname) ! Initialize user-provided data table default_table%gridname = 'none' @@ -371,27 +371,27 @@ subroutine data_override_init(Atm_domain_in, Ocean_domain_in, Ice_domain_in, Lan if(file_open) call mpp_error(FATAL, trim(grid_file)//' already opened') if(field_exist(grid_file, "x_T" ) .OR. field_exist(grid_file, "geolon_t" ) ) then - if (atm_on) then + if (atm_on .and. .not. allocated(lon_local_atm) ) then call mpp_get_compute_domain( atm_domain,is,ie,js,je) allocate(lon_local_atm(is:ie,js:je), lat_local_atm(is:ie,js:je)) call get_grid_version_1(grid_file, 'atm', atm_domain, is, ie, js, je, lon_local_atm, lat_local_atm, & min_glo_lon_atm, max_glo_lon_atm ) endif - if (ocn_on) then + if (ocn_on .and. .not. allocated(lon_local_ocn) ) then call mpp_get_compute_domain( ocn_domain,is,ie,js,je) allocate(lon_local_ocn(is:ie,js:je), lat_local_ocn(is:ie,js:je)) call get_grid_version_1(grid_file, 'ocn', ocn_domain, is, ie, js, je, lon_local_ocn, lat_local_ocn, & min_glo_lon_ocn, max_glo_lon_ocn ) endif - if (lnd_on) then + if (lnd_on .and. .not. allocated(lon_local_lnd) ) then call mpp_get_compute_domain( lnd_domain,is,ie,js,je) allocate(lon_local_lnd(is:ie,js:je), lat_local_lnd(is:ie,js:je)) call get_grid_version_1(grid_file, 'lnd', lnd_domain, is, ie, js, je, lon_local_lnd, lat_local_lnd, & min_glo_lon_lnd, max_glo_lon_lnd ) endif - if (ice_on) then + if (ice_on .and. .not. allocated(lon_local_ice) ) then call mpp_get_compute_domain( ice_domain,is,ie,js,je) allocate(lon_local_ice(is:ie,js:je), lat_local_ice(is:ie,js:je)) call get_grid_version_1(grid_file, 'ice', ice_domain, is, ie, js, je, lon_local_ice, lat_local_ice, & @@ -406,28 +406,28 @@ subroutine data_override_init(Atm_domain_in, Ocean_domain_in, Ice_domain_in, Lan if(count .NE. 1) call mpp_error(FATAL, 'data_override_mod: the grid file is a solo mosaic, ' // & 'one and only one of atm_on, lnd_on or ice_on/ocn_on should be true') endif - if (atm_on) then + if (atm_on .and. .not. allocated(lon_local_atm) ) then call mpp_get_compute_domain(atm_domain,is,ie,js,je) allocate(lon_local_atm(is:ie,js:je), lat_local_atm(is:ie,js:je)) call get_grid_version_2(grid_file, 'atm', atm_domain, is, ie, js, je, lon_local_atm, lat_local_atm, & min_glo_lon_atm, max_glo_lon_atm ) endif - if (ocn_on) then + if (ocn_on .and. .not. allocated(lon_local_ocn) ) then call mpp_get_compute_domain( ocn_domain,is,ie,js,je) allocate(lon_local_ocn(is:ie,js:je), lat_local_ocn(is:ie,js:je)) call get_grid_version_2(grid_file, 'ocn', ocn_domain, is, ie, js, je, lon_local_ocn, lat_local_ocn, & min_glo_lon_ocn, max_glo_lon_ocn ) endif - if (lnd_on) then + if (lnd_on .and. .not. allocated(lon_local_lnd) ) then call mpp_get_compute_domain( lnd_domain,is,ie,js,je) allocate(lon_local_lnd(is:ie,js:je), lat_local_lnd(is:ie,js:je)) call get_grid_version_2(grid_file, 'lnd', lnd_domain, is, ie, js, je, lon_local_lnd, lat_local_lnd, & min_glo_lon_lnd, max_glo_lon_lnd ) endif - if (ice_on) then + if (ice_on .and. .not. allocated(lon_local_ice) ) then call mpp_get_compute_domain( ice_domain,is,ie,js,je) allocate(lon_local_ice(is:ie,js:je), lat_local_ice(is:ie,js:je)) call get_grid_version_2(grid_file, 'ocn', ice_domain, is, ie, js, je, lon_local_ice, lat_local_ice, & @@ -911,6 +911,9 @@ subroutine data_override_3d(gridname,fieldname_code,data,time,override,data_inde horz_interp=override_array(curr_position)%horz_interp(window_id), & is_in=is_in,ie_in=ie_in,js_in=js_in,je_in=je_in,window_id=window_id) data(:,:,1) = data(:,:,1)*factor + do i = 2, size(data,3) + data(:,:,i) = data(:,:,1) + enddo else allocate(mask_out(size(data,1), size(data,2),1)) mask_out = .false. @@ -921,11 +924,13 @@ subroutine data_override_3d(gridname,fieldname_code,data,time,override,data_inde where(mask_out(:,:,1)) data(:,:,1) = data(:,:,1)*factor end where + do i = 2, size(data,3) + where(mask_out(:,:,1)) + data(:,:,i) = data(:,:,1) + end where + enddo deallocate(mask_out) endif - do i = 2, size(data,3) - data(:,:,i) = data(:,:,1) - enddo else if( data_table(index1)%region_type == NO_REGION ) then call time_interp_external(id_time,time,data,verbose=.false., & @@ -1382,6 +1387,7 @@ program test call mpp_define_domains( (/1,nlon,1,nlat/), layout, Domain, name='test_data_override') call data_override_init(Ice_domain_in=Domain, Ocean_domain_in=Domain) + call data_override_init(Ice_domain_in=Domain, Ocean_domain_in=Domain) call mpp_get_compute_domain(Domain, is, ie, js, je) call get_grid diff --git a/src/shared/diag_manager/diag_axis.F90 b/src/shared/diag_manager/diag_axis.F90 index 036b606653..d7fa28a5af 100644 --- a/src/shared/diag_manager/diag_axis.F90 +++ b/src/shared/diag_manager/diag_axis.F90 @@ -1,15 +1,15 @@ MODULE diag_axis_mod - ! + ! ! Seth Underwood ! - ! diag_axis_mod is an integral part - ! of diag_manager_mod. It helps to create axis IDs - ! that are used in register_diag_field. + ! diag_axis_mod is an integral part + ! of diag_manager_mod. It helps to create axis IDs + ! that are used in register_diag_field. ! ! Users first create axis ID by calling - ! diag_axis_init, then use this axis ID in + ! diag_axis_init, then use this axis ID in ! register_diag_field. ! @@ -33,9 +33,9 @@ MODULE diag_axis_mod ! Module variables ! Parameters CHARACTER(len=128), PARAMETER :: version =& - & '$Id: diag_axis.F90,v 20.0 2013/12/14 00:18:37 fms Exp $' + & '$Id$' CHARACTER(len=128), PARAMETER :: tagname =& - & '$Name: tikal $' + & '$Name$' ! counter of number of axes defined INTEGER, DIMENSION(:), ALLOCATABLE :: num_subaxes @@ -95,7 +95,7 @@ INTEGER FUNCTION diag_axis_init(name, DATA, units, cart_name, long_name, directi CHARACTER(len=*), INTENT(in) :: name REAL, DIMENSION(:), INTENT(in) :: DATA CHARACTER(len=*), INTENT(in) :: units - CHARACTER(len=*), INTENT(in) :: cart_name + CHARACTER(len=*), INTENT(in) :: cart_name CHARACTER(len=*), INTENT(in), OPTIONAL :: long_name, set_name INTEGER, INTENT(in), OPTIONAL :: direction, edges TYPE(domain1d), INTENT(in), OPTIONAL :: Domain @@ -110,7 +110,7 @@ INTEGER FUNCTION diag_axis_init(name, DATA, units, cart_name, long_name, directi CHARACTER(len=128) :: emsg IF ( .NOT.module_is_initialized ) THEN - call write_version_number() + CALL write_version_number( version, tagname ) ENDIF IF ( PRESENT(tile_count)) THEN @@ -118,7 +118,7 @@ INTEGER FUNCTION diag_axis_init(name, DATA, units, cart_name, long_name, directi ELSE tile = 1 END IF - + ! Allocate the axes IF (.NOT. ALLOCATED(Axis_sets)) ALLOCATE(Axis_sets(max_num_axis_sets)) IF (.NOT. ALLOCATED(Axes)) ALLOCATE(Axes(max_axes)) @@ -174,7 +174,7 @@ INTEGER FUNCTION diag_axis_init(name, DATA, units, cart_name, long_name, directi END IF END IF END DO - + !---- register axis ---- num_def_axes = num_def_axes + 1 ! max_axes exceeded, increase it via diag_manager_nml @@ -189,13 +189,13 @@ INTEGER FUNCTION diag_axis_init(name, DATA, units, cart_name, long_name, directi & TRIM(uppercase(cart_name)) == 'T' .OR.& & TRIM(uppercase(cart_name)) == 'N' ) THEN Axes(diag_axis_init)%cart_name = TRIM(uppercase(cart_name)) - ELSE + ELSE ! Invalid cart_name name. CALL error_mesg('diag_axis_mod::diag_axis_init', 'Invalid cart_name name.', FATAL) END IF !---- allocate storage for coordinate values of axis ---- - IF ( Axes(diag_axis_init)%cart_name == 'T' ) THEN + IF ( Axes(diag_axis_init)%cart_name == 'T' ) THEN axlen = 0 ELSE axlen = SIZE(DATA(:)) @@ -205,7 +205,7 @@ INTEGER FUNCTION diag_axis_init(name, DATA, units, cart_name, long_name, directi ! Initialize Axes(diag_axis_init) Axes(diag_axis_init)%name = TRIM(name) Axes(diag_axis_init)%data = DATA(1:axlen) - Axes(diag_axis_init)%units = units + Axes(diag_axis_init)%units = units Axes(diag_axis_init)%length = axlen Axes(diag_axis_init)%set = set ! start and end are used in subaxes information only @@ -225,7 +225,7 @@ INTEGER FUNCTION diag_axis_init(name, DATA, units, cart_name, long_name, directi ELSE Axes(diag_axis_init)%aux = 'none' END IF - + !---- axis direction (-1, 0, or +1) ---- IF ( PRESENT(direction) )THEN IF ( ABS(direction) /= 1 .AND. direction /= 0 )& @@ -257,11 +257,11 @@ INTEGER FUNCTION diag_axis_init(name, DATA, units, cart_name, long_name, directi IF ( Axes(diag_axis_init)%cart_name == 'X' ) Axes(diag_axis_init)%Domain = domain_x IF ( Axes(diag_axis_init)%cart_name == 'Y' ) Axes(diag_axis_init)%Domain = domain_y ELSE IF ( PRESENT(Domain)) THEN - !---- domain1d type ---- + !---- domain1d type ---- Axes(diag_axis_init)%Domain2 = null_domain2d ! needed since not 2-D domain Axes(diag_axis_init)%Domain = Domain ELSE - Axes(diag_axis_init)%Domain2 = null_domain2d + Axes(diag_axis_init)%Domain2 = null_domain2d Axes(diag_axis_init)%Domain = null_domain1d END IF @@ -271,7 +271,7 @@ INTEGER FUNCTION diag_axis_init(name, DATA, units, cart_name, long_name, directi CALL mpp_get_compute_domain(Axes(diag_axis_init)%Domain, isc, iec) CALL mpp_get_global_domain(Axes(diag_axis_init)%Domain, isg, ieg) IF ( Axes(diag_axis_init)%length == ieg - isg + 2 ) THEN - Axes(diag_axis_init)%shift = 1 + Axes(diag_axis_init)%shift = 1 END IF END IF @@ -283,7 +283,7 @@ INTEGER FUNCTION diag_axis_init(name, DATA, units, cart_name, long_name, directi IF ( Axes(edges)%cart_name /= Axes(diag_axis_init)%cart_name) ierr=1 IF ( Axes(edges)%length /= Axes(diag_axis_init)%length+1 ) ierr=ierr+2 IF ( Axes(edges)%set /= Axes(diag_axis_init)%set ) ierr=ierr+4 - IF ( ierr > 0 ) THEN + IF ( ierr > 0 ) THEN ! Edges axis does not match axis (code ). WRITE (emsg,'("Edges axis does not match axis (code ",I1,").")') ierr CALL error_mesg('diag_axis_mod::diag_axis_init', emsg, FATAL) @@ -312,7 +312,7 @@ END FUNCTION diag_axis_init ! ! Given the ID of a parent axis, create a subaxis and fill it with data, ! and return the ID of the corresponding subaxis. - ! + ! ! The subaxis is defined on the parent axis from start_indx ! to end_indx. ! @@ -326,13 +326,13 @@ INTEGER FUNCTION diag_subaxes_init(axis, subdata, start_indx, end_indx, domain_2 INTEGER, INTENT(in) :: axis REAL, DIMENSION(:), INTENT(in) :: subdata INTEGER, INTENT(in) :: start_indx - INTEGER, INTENT(in) :: end_indx + INTEGER, INTENT(in) :: end_indx TYPE(domain2d), INTENT(in), OPTIONAL :: domain_2d INTEGER :: i, nsub_axis, direction INTEGER :: xbegin, xend, ybegin, yend INTEGER :: ad_xbegin, ad_xend, ad_ybegin, ad_yend - CHARACTER(len=128) :: name, nsub_name + CHARACTER(len=128) :: name, nsub_name CHARACTER(len=128) :: units CHARACTER(len=128) :: cart_name CHARACTER(len=128) :: long_name @@ -377,14 +377,14 @@ INTEGER FUNCTION diag_subaxes_init(axis, subdata, start_indx, end_indx, domain_2 Axes(axis)%end(nsub_axis) = end_indx if ( hasDomain ) Axes(axis)%subaxis_domain2(nsub_axis) = domain_2d END IF - + ! Create new name for the subaxis from name of parent axis - ! If subaxis already exists, get the index and return + ! If subaxis already exists, get the index and return IF(subaxis_set) THEN IF ( Axes(axis)%set > 0 ) THEN - diag_subaxes_init = get_axis_num(name, set_name=TRIM(Axis_sets(Axes(axis)%set))) + diag_subaxes_init = get_axis_num(name, set_name=TRIM(Axis_sets(Axes(axis)%set))) ELSE - diag_subaxes_init = get_axis_num(name) + diag_subaxes_init = get_axis_num(name) END IF ELSE ! get a new index for subaxis @@ -406,7 +406,7 @@ INTEGER FUNCTION diag_subaxes_init(axis, subdata, start_indx, end_indx, domain_2 END IF END FUNCTION diag_subaxes_init ! - + ! ! ! Return information about the axis with index ID @@ -452,7 +452,7 @@ SUBROUTINE get_diag_axis(id, name, units, long_name, cart_name,& direction = Axes(id)%direction edges = Axes(id)%edges Domain = Axes(id)%Domain - IF ( Axes(id)%length > SIZE(DATA(:)) ) THEN + IF ( Axes(id)%length > SIZE(DATA(:)) ) THEN ! array data is too small. CALL error_mesg('diag_axis_mod::get_diag_axis', 'array data is too small', FATAL) ELSE @@ -499,7 +499,7 @@ SUBROUTINE get_diag_axis_data(id, DATA) REAL, DIMENSION(:), INTENT(out) :: DATA CALL valid_id_check(id, 'get_diag_axis_data') - IF (Axes(id)%length > SIZE(DATA(:))) THEN + IF (Axes(id)%length > SIZE(DATA(:))) THEN ! array data is too small CALL error_mesg('diag_axis_mod::get_diag_axis_data', 'array data is too small', FATAL) ELSE @@ -564,7 +564,7 @@ END SUBROUTINE get_diag_axis_domain_name INTEGER FUNCTION get_axis_length(id) INTEGER, INTENT(in) :: id - INTEGER :: length + INTEGER :: length CALL valid_id_check(id, 'get_axis_length') IF ( Axes(id)%Domain .NE. null_domain1d ) THEN @@ -634,7 +634,7 @@ INTEGER FUNCTION get_tile_count(ids) INTEGER :: i, id, flag - IF ( SIZE(ids(:)) < 1 ) THEN + IF ( SIZE(ids(:)) < 1 ) THEN ! input argument has incorrect size. CALL error_mesg('diag_axis_mod::get_tile_count', 'input argument has incorrect size', FATAL) END IF @@ -695,7 +695,7 @@ TYPE(domain2d) FUNCTION get_domain2d(ids) INTEGER :: i, id, flag - IF ( SIZE(ids(:)) < 1 ) THEN + IF ( SIZE(ids(:)) < 1 ) THEN ! input argument has incorrect size. CALL error_mesg('diag_axis_mod::get_domain2d', 'input argument has incorrect size', FATAL) END IF @@ -729,14 +729,14 @@ END FUNCTION get_domain2d ! ! X shift value. ! Y shift value. - SUBROUTINE get_axes_shift(ids, ishift, jshift) + SUBROUTINE get_axes_shift(ids, ishift, jshift) INTEGER, DIMENSION(:), INTENT(in) :: ids INTEGER, INTENT(out) :: ishift, jshift INTEGER :: i, id !-- get the value of the shift. - ishift = 0 + ishift = 0 jshift = 0 DO i = 1, SIZE(ids(:)) id = ids(i) @@ -823,7 +823,7 @@ END FUNCTION get_axis_set_num ! SUBROUTINE valid_id_check(id, routine_name) ! ! - ! Check to see if the given axis id is a valid id. If the axis id is invalid, + ! Check to see if the given axis id is a valid id. If the axis id is invalid, ! call a FATAL error. If the ID is valid, just return. ! ! Axis id to check for validity diff --git a/src/shared/diag_manager/diag_data.F90 b/src/shared/diag_manager/diag_data.F90 index d3d9a4e29c..cad175ad9d 100644 --- a/src/shared/diag_manager/diag_data.F90 +++ b/src/shared/diag_manager/diag_data.F90 @@ -1,16 +1,16 @@ #include MODULE diag_data_mod - ! + ! ! Seth Underwood ! - + ! ! Type descriptions and global variables for the diag_manager modules. ! ! - ! Notation: + ! Notation: !
!
input field
!
The data structure describing the field as @@ -29,7 +29,7 @@ MODULE diag_data_mod ! ! Each input field associated with one or several output fields via array of ! indices output_fields; each output field points to the single "parent" input - ! field with the input_field index, and to the output file with the output_file + ! field with the input_field index, and to the output file with the output_file ! index ! @@ -46,7 +46,6 @@ MODULE diag_data_mod PUBLIC - ! ! ! Maximum number of fields per file. @@ -66,7 +65,15 @@ MODULE diag_data_mod ! ! ! - + ! + ! Value used in the region specification of the diag_table to indicate to use the full axis instead of a sub-axis + ! + ! + ! Alternate value used in the region specification of the diag_table to indicate to use the full axis instead of a sub-axis + ! + ! + ! Return value for a diag_field that isn't found in the diag_table + ! ! Specify storage limits for fixed size tables used for pointers, etc. INTEGER, PARAMETER :: MAX_FIELDS_PER_FILE = 300 !< Maximum number of fields per file. INTEGER, PARAMETER :: DIAG_OTHER = 0 @@ -79,7 +86,10 @@ MODULE diag_data_mod INTEGER, PARAMETER :: DIAG_SECONDS = 1, DIAG_MINUTES = 2, DIAG_HOURS = 3 INTEGER, PARAMETER :: DIAG_DAYS = 4, DIAG_MONTHS = 5, DIAG_YEARS = 6 INTEGER, PARAMETER :: MAX_SUBAXES = 10 + INTEGER, PARAMETER :: GLO_REG_VAL = -999 + INTEGER, PARAMETER :: GLO_REG_VAL_ALT = -1 REAL, PARAMETER :: CMOR_MISSING_VALUE = 1.0e20 !< CMOR standard missing value + INTEGER, PARAMETER :: DIAG_FIELD_NOT_FOUND = -1 ! ! @@ -101,12 +111,12 @@ MODULE diag_data_mod ! ID returned from diag_subaxes_init of 3 subaces. ! TYPE diag_grid - REAL, DIMENSION(3) :: start, END ! start and end coordinates (lat,lon,depth) of local domain to output + REAL, DIMENSION(3) :: start, END ! start and end coordinates (lat,lon,depth) of local domain to output INTEGER, DIMENSION(3) :: l_start_indx, l_end_indx ! start and end indices at each LOCAL PE INTEGER, DIMENSION(3) :: subaxes ! id returned from diag_subaxes_init of 3 subaxes END TYPE diag_grid ! - + ! ! ! Diagnostic field type @@ -134,6 +144,38 @@ MODULE diag_data_mod END TYPE diag_fieldtype ! + ! + ! + ! Attribute type for diagnostic fields + ! + ! + ! Data type of attribute values (NF_INT, NF_FLOAT, NF_CHAR) + ! + ! + ! Number of values in attribute, or if a character string then + ! length of the string. + ! + ! + ! Name of the attribute + ! + ! + ! Character string to hold character value of attribute + ! + ! + ! REAL array to hold value of REAL attributes. + ! + ! + ! INTEGER array to hold value of INTEGER attributes. + ! + type :: diag_atttype + INTEGER :: type + INTEGER :: len + CHARACTER(len=128) :: name + CHARACTER(len=1280) :: catt + REAL, _ALLOCATABLE, DIMENSION(:) :: fatt _NULL + INTEGER, _ALLOCATABLE, DIMENSION(:) :: iatt _NULL + end type diag_atttype + ! ! ! ! Define the region for field output. @@ -159,7 +201,7 @@ MODULE diag_data_mod REAL :: zend END TYPE coord_type ! - + ! ! ! Type to define the diagnostic files that will be written as defined by the diagnostic table. @@ -222,6 +264,12 @@ MODULE diag_data_mod ! ! ! + ! + ! Array to hold user definable attributes + ! + ! + ! Number of defined attibutes + ! TYPE file_type CHARACTER(len=128) :: name !< Name of the output file. CHARACTER(len=128) :: long_name @@ -245,9 +293,11 @@ MODULE diag_data_mod TYPE(time_type) :: start_time !< Time file opened. TYPE(time_type) :: close_time !< Time file closed. File does not allow data after close time TYPE(diag_fieldtype):: f_avg_start, f_avg_end, f_avg_nitems, f_bounds + TYPE(diag_atttype), _ALLOCATABLE, dimension(:) :: attributes _NULL + INTEGER :: num_attributes END TYPE file_type - ! - + ! + ! ! ! Type to hold the input field description @@ -290,23 +340,36 @@ MODULE diag_data_mod ! ! ! + ! + ! + ! + ! The current level of OpenMP nesting + ! ! ! ! ! + ! + ! + ! + ! Indicates if the mask_ignore_warning has been issued for this input + ! field. Once .TRUE. the warning message is suppressed on all subsequent + ! send_data calls. + ! TYPE input_field_type CHARACTER(len=128) :: module_name, field_name, long_name, units, standard_name CHARACTER(len=64) :: interp_method INTEGER, DIMENSION(3) :: axes - INTEGER :: num_axes + INTEGER :: num_axes LOGICAL :: missing_value_present, range_present REAL :: missing_value REAL, DIMENSION(2) :: range - INTEGER, _ALLOCATABLE, dimension(:) :: output_fields + INTEGER, _ALLOCATABLE, dimension(:) :: output_fields _NULL INTEGER :: num_output_fields INTEGER, DIMENSION(3) :: size LOGICAL :: static, register, mask_variant, local INTEGER :: numthreads + INTEGER :: active_omp_level INTEGER :: tile_count TYPE(coord_type) :: local_coord TYPE(time_type) :: time @@ -337,11 +400,17 @@ MODULE diag_data_mod ! ! .TRUE. if the output field is averaged over time interval. ! + ! + ! .TRUE. if the output field is the rms. In this case, time_average will also be true. + ! ! - ! .TRUE. if any of time_min, time_max, or time_average is true + ! .TRUE. if any of time_min, time_max, time_rms, or time_average is true ! ! ! + ! + ! Power to use When calculating the mean_pow(n) + ! ! ! Time method field from the input file ! @@ -403,29 +472,37 @@ MODULE diag_data_mod ! ! ! + ! + ! Array to hold user definable attributes + ! + ! + ! Number of defined attibutes + ! TYPE output_field_type INTEGER :: input_field ! index of the corresponding input field in the table INTEGER :: output_file ! index of the output file in the table CHARACTER(len=128) :: output_name LOGICAL :: time_average ! true if the output field is averaged over time interval + LOGICAL :: time_rms ! true if the output field is the rms. If true, then time_average is also LOGICAL :: static LOGICAL :: time_max ! true if the output field is maximum over time interval LOGICAL :: time_min ! true if the output field is minimum over time interval - LOGICAL :: time_ops ! true if any of time_min, time_max, or time_average is true + LOGICAL :: time_ops ! true if any of time_min, time_max, time_rms or time_average is true INTEGER :: pack - CHARACTER(len=50) :: time_method ! time method field from the input file - ! coordianes of the buffer and counter are (x, y, z, time-of-day) + INTEGER :: pow_value !< Power value to use for mean_pow(n) calculations + CHARACTER(len=50) :: time_method ! time method field from the input file + ! coordinates of the buffer and counter are (x, y, z, time-of-day) REAL, _ALLOCATABLE, DIMENSION(:,:,:,:) :: buffer _NULL REAL, _ALLOCATABLE, DIMENSION(:,:,:,:) :: counter _NULL - ! the following two counters are used in time-averaging for some - ! combination of the field options. Their size is the length of the + ! the following two counters are used in time-averaging for some + ! combination of the field options. Their size is the length of the ! diurnal axis; the counters must be tracked separately for each of - ! the diurnal interval, becaus the number of time slices accumulated + ! the diurnal interval, because the number of time slices accumulated ! in each can be different, depending on time step and the number of ! diurnal samples. REAL, _ALLOCATABLE, DIMENSION(:) :: count_0d INTEGER, _ALLOCATABLE, dimension(:) :: num_elements - + TYPE(time_type) :: last_output, next_output, next_next_output TYPE(diag_fieldtype) :: f_type INTEGER, DIMENSION(4) :: axes @@ -436,6 +513,8 @@ MODULE diag_data_mod LOGICAL :: reduced_k_range INTEGER :: imin, imax, jmin, jmax, kmin, kmax TYPE(time_type) :: Time_of_prev_field_data + TYPE(diag_atttype), _ALLOCATABLE, dimension(:) :: attributes _NULL + INTEGER :: num_attributes END TYPE output_field_type ! @@ -508,12 +587,12 @@ MODULE diag_data_mod CHARACTER(len=128) :: tile_name='N/A' END TYPE diag_global_att_type ! - + ! Private CHARACTER Arrays for the CVS version and tagname. CHARACTER(len=128),PRIVATE :: version =& - & '$Id: diag_data.F90,v 20.0 2013/12/14 00:18:41 fms Exp $' + & '$Id$' CHARACTER(len=128),PRIVATE :: tagname =& - & '$Name: tikal $' + & '$Name$' ! ! @@ -543,7 +622,7 @@ MODULE diag_data_mod ! ! Maximum number of input fields. Increase via the diag_manager_nml namelist. ! - ! + ! ! Maximum number of output_fields per input_field. ! ! @@ -556,20 +635,35 @@ MODULE diag_data_mod ! ! Indicates if we should overwrite the MISSING_VALUE to use the CMOR missing value. ! - ! + ! ! Issue warnings if the output field has values outside the given ! range for a variable. ! - ! + ! ! Cause a fatal error if the output field has a value outside the ! given range for a variable. ! + ! + ! Maximum number of user definable attributes per field. + ! + ! + ! Maximum number of user definable global attributes per file. + ! + ! + ! Indicates if the file start date will be prepended to the file name. .TRUE. is + ! only supported if the diag_manager_init routine is called with the optional time_init parameter. + ! This was usually done by FRE after the model run. + ! + ! + ! Will determine which value to use when checking a regional output if the region is the full axis or a sub-axis. + ! The values are defined as GLO_REG_VAL (-999) and GLO_REG_VAL_ALT (-1) in diag_data_mod. + ! LOGICAL :: append_pelist_name = .FALSE. LOGICAL :: mix_snapshot_average_fields =.FALSE. INTEGER :: max_files = 31 !< Maximum number of output files allowed. Increase via diag_manager_nml. INTEGER :: max_output_fields = 300 !< Maximum number of output fields. Increase via diag_manager_nml. INTEGER :: max_input_fields = 300 !< Maximum number of input fields. Increase via diag_manager_nml. - INTEGER :: MAX_OUT_PER_IN_FIELD = 150 !< Maximum number of output_fields per input_field. Increase via diag_manager_nml. + INTEGER :: max_out_per_in_field = 150 !< Maximum number of output_fields per input_field. Increase via diag_manager_nml. INTEGER :: max_axes = 60 !< Maximum number of independent axes. LOGICAL :: do_diag_field_log = .FALSE. LOGICAL :: write_bytes_in_file = .FALSE. @@ -579,7 +673,11 @@ MODULE diag_data_mod LOGICAL :: use_cmor = .FALSE. LOGICAL :: issue_oor_warnings = .TRUE. LOGICAL :: oor_warnings_fatal = .FALSE. + LOGICAL :: region_out_use_alt_value = .TRUE. + INTEGER :: max_field_attributes = 2 + INTEGER :: max_file_attributes = 2 + LOGICAL :: prepend_date = .TRUE. ! ! ! Fill value used. Value will be NF90_FILL_REAL if using the @@ -588,7 +686,7 @@ MODULE diag_data_mod #ifdef use_netCDF REAL :: FILL_VALUE = NF_FILL_REAL ! from file /usr/local/include/netcdf.inc #else - REAL :: FILL_VALUE = 9.9692099683868690e+36 + REAL :: FILL_VALUE = 9.9692099683868690e+36 #endif INTEGER :: pack_size = 1 ! 1 for double and 2 for float @@ -601,6 +699,10 @@ MODULE diag_data_mod REAL :: MAX_VALUE, MIN_VALUE ! + ! + ! Time diag_manager_init called. If init_time not included in + ! diag_manager_init call, then same as base_time + ! ! ! ! @@ -609,6 +711,7 @@ MODULE diag_data_mod ! ! ! + TYPE(time_type) :: diag_init_time TYPE(time_type) :: base_time INTEGER :: base_year, base_month, base_day, base_hour, base_minute, base_second CHARACTER(len = 256):: global_descriptor @@ -628,7 +731,6 @@ MODULE diag_data_mod ! ! - ! ! TYPE(time_type) :: time_zero LOGICAL :: first_send_data_call = .TRUE. @@ -636,8 +738,7 @@ MODULE diag_data_mod INTEGER :: diag_log_unit CHARACTER(len=10), DIMENSION(6) :: time_unit_list = (/'seconds ', 'minutes ',& & 'hours ', 'days ', 'months ', 'years '/) - CHARACTER(len=32), SAVE :: filename_appendix = '' CHARACTER(len=32) :: pelist_name INTEGER :: oor_warning = WARNING - + END MODULE diag_data_mod diff --git a/src/shared/diag_manager/diag_grid.F90 b/src/shared/diag_manager/diag_grid.F90 index f254b48c96..8c09ae722c 100644 --- a/src/shared/diag_manager/diag_grid.F90 +++ b/src/shared/diag_manager/diag_grid.F90 @@ -1,9 +1,10 @@ #include MODULE diag_grid_mod - ! + ! ! Seth Underwood ! + ! ! ! diag_grid_mod is a set of procedures to work with the ! model's global grid to allow regional output. @@ -30,7 +31,7 @@ MODULE diag_grid_mod !
  • Single point region in Cubed Sphere
    • !
  • Single tile regions in the cubed sphere
    • ! - !
    + ! !
    !     @@ -60,9 +61,9 @@ MODULE diag_grid_mod ! Parameters CHARACTER(len=128), PARAMETER :: version =& - & '$Id: diag_grid.F90,v 20.0 2013/12/14 00:18:45 fms Exp $' + & '$Id$' CHARACTER(len=128), PARAMETER :: tagname =& - & '$Name: tikal $' + & '$Name$' ! Derived data types ! @@ -177,7 +178,7 @@ MODULE diag_grid_mod ! ! ! Send the global grid to the diag_manager_mod for - ! regional output. + ! regional output. ! ! ! - ! Send data over to output fields. + ! Send data over to output fields. ! ! ! send_data is overloaded for fields having zero dimension @@ -243,11 +266,11 @@ MODULE diag_manager_mod ! By default, a field will be written out entirely in its global grid. ! Users can also specify regions in which the field will be output. The ! region is specified in diag-table just before the end of output_field - ! replacing "none". + ! replacing "none". ! ! For example, by default: ! - ! "ocean_mod","Vorticity","vorticity","file1","all",.false.,"none",2 + ! "ocean_mod","Vorticity","vorticity","file1","all",.false.,"none",2 ! ! for regional output: ! @@ -266,16 +289,16 @@ MODULE diag_manager_mod ! diag_manager_mod using the diag_grid_init ! subroutine. - ! - ! NOTE: When using regional output the files containing regional - ! outputs should be different from files containing global (default) output. - ! It is a FATAL error to have one file containing both regional and global - ! results. For maximum flexibility and independence from PE counts one file + ! + ! NOTE: When using regional output the files containing regional + ! outputs should be different from files containing global (default) output. + ! It is a FATAL error to have one file containing both regional and global + ! results. For maximum flexibility and independence from PE counts one file ! should contain just one region. - ! + ! ! Time averaging is supported in regional output. - ! - ! Physical fields (written in "physics windows" of atmospheric code) are + ! + ! Physical fields (written in "physics windows" of atmospheric code) are ! fully supported for regional outputs. ! ! NOTE: Most fields are defined in the data domain but use the @@ -317,7 +340,7 @@ MODULE diag_manager_mod ! ! ! Return field index for subsequent calls to @@ -344,8 +367,10 @@ MODULE diag_manager_mod ! ! ! - ! + ! ! + ! + ! INTERFACE register_diag_field MODULE PROCEDURE register_diag_field_scalar MODULE PROCEDURE register_diag_field_array @@ -354,13 +379,13 @@ MODULE diag_manager_mod ! ! - ! Send tile-averaged data over to output fields. + ! Send tile-averaged data over to output fields. ! ! ! - ! send_tile_averaged_data is overloaded for 3D and 4D arrays. + ! send_tile_averaged_data is overloaded for 3D and 4D arrays. ! diag_field_id corresponds to the ID returned by previous call ! to register_diag_field. Logical masks can be used to mask out ! undefined and/or unused values. Note that the dimension of output field @@ -378,6 +403,34 @@ MODULE diag_manager_mod END INTERFACE ! + ! + ! + ! Add a attribute to the output field + ! + ! + ! + ! Add an arbitrary attribute and value to the output variable. Any number + ! of attributes can be added to a given field. All attribute addition must + ! be done before first send_data call. + ! + ! If a real or integer attribute is already defined, a FATAL error will be called. + ! If a character attribute is already defined, then it will be prepended to the + ! existing attribute value. + ! + ! + ! + ! + INTERFACE diag_field_add_attribute + MODULE PROCEDURE diag_field_add_attribute_scalar_r + MODULE PROCEDURE diag_field_add_attribute_scalar_i + MODULE PROCEDURE diag_field_add_attribute_scalar_c + MODULE PROCEDURE diag_field_add_attribute_r1d + MODULE PROCEDURE diag_field_add_attribute_i1d + END INTERFACE diag_field_add_attribute + ! + CONTAINS ! @@ -392,9 +445,12 @@ MODULE diag_manager_mod ! ! ! + ! + ! ! INTEGER FUNCTION register_diag_field_scalar(module_name, field_name, init_time, & - & long_name, units, missing_value, range, standard_name, do_not_log, err_msg) + & long_name, units, missing_value, range, standard_name, do_not_log, err_msg,& + & area, volume) CHARACTER(len=*), INTENT(in) :: module_name, field_name TYPE(time_type), OPTIONAL, INTENT(in) :: init_time CHARACTER(len=*), OPTIONAL, INTENT(in) :: long_name, units, standard_name @@ -402,11 +458,13 @@ INTEGER FUNCTION register_diag_field_scalar(module_name, field_name, init_time, REAL, DIMENSION(2), OPTIONAL, INTENT(in) :: RANGE LOGICAL, OPTIONAL, INTENT(in) :: do_not_log ! if TRUE, field information is not logged CHARACTER(len=*), OPTIONAL, INTENT(out):: err_msg - + INTEGER, OPTIONAL, INTENT(in) :: area, volume + IF ( PRESENT(init_time) ) THEN register_diag_field_scalar = register_diag_field_array(module_name, field_name,& & (/null_axis_id/), init_time,long_name, units, missing_value, range, & - & standard_name=standard_name, do_not_log=do_not_log, err_msg=err_msg) + & standard_name=standard_name, do_not_log=do_not_log, err_msg=err_msg,& + & area=area, volume=volume) ELSE IF ( PRESENT(err_msg) ) err_msg = '' register_diag_field_scalar = register_static_field(module_name, field_name,& @@ -428,12 +486,14 @@ END FUNCTION register_diag_field_scalar ! ! ! - ! ! ! + ! diag_field_id containing the cell area field + ! diag_field_id containing the cell volume field + ! INTEGER FUNCTION register_diag_field_array(module_name, field_name, axes, init_time, & & long_name, units, missing_value, range, mask_variant, standard_name, verbose,& - & do_not_log, err_msg, interp_method, tile_count) + & do_not_log, err_msg, interp_method, tile_count, area, volume) CHARACTER(len=*), INTENT(in) :: module_name, field_name INTEGER, INTENT(in) :: axes(:) TYPE(time_type), INTENT(in) :: init_time @@ -444,11 +504,14 @@ INTEGER FUNCTION register_diag_field_array(module_name, field_name, axes, init_t CHARACTER(len=*), OPTIONAL, INTENT(out):: err_msg CHARACTER(len=*), OPTIONAL, INTENT(in) :: interp_method INTEGER, OPTIONAL, INTENT(in) :: tile_count + INTEGER, OPTIONAL, INTENT(in) :: area, volume INTEGER :: field, j, ind, file_num, freq + INTEGER :: i, cm_ind, cm_file_num INTEGER :: output_units INTEGER :: stdout_unit LOGICAL :: mask_variant1, verbose1 + LOGICAL :: cm_found CHARACTER(len=128) :: msg ! get stdout unit number @@ -460,20 +523,20 @@ INTEGER FUNCTION register_diag_field_array(module_name, field_name, axes, init_t mask_variant1 = .FALSE. END IF - IF ( PRESENT(verbose) ) THEN + IF ( PRESENT(verbose) ) THEN verbose1 = verbose - ELSE + ELSE verbose1 = .FALSE. END IF IF ( PRESENT(err_msg) ) err_msg = '' - + ! Call register static, then set static back to false register_diag_field_array = register_static_field(module_name, field_name, axes,& & long_name, units, missing_value, range, mask_variant1, standard_name=standard_name,& & DYNAMIC=.TRUE., do_not_log=do_not_log, interp_method=interp_method, tile_count=tile_count) - IF ( .NOT.first_send_data_call ) THEN + IF ( .NOT.first_send_data_call ) THEN ! ! module/output_field / registered AFTER first ! send_data call, TOO LATE @@ -481,23 +544,53 @@ INTEGER FUNCTION register_diag_field_array(module_name, field_name, axes, init_t IF ( mpp_pe() == mpp_root_pe() ) & & CALL error_mesg ('diag_manager_mod::register_diag_field', 'module/output_field '& &//TRIM(module_name)//'/'// TRIM(field_name)//& - &' registered AFTER first send_data call, TOO LATE', WARNING) + &' registered AFTER first send_data call, TOO LATE', WARNING) END IF IF ( register_diag_field_array < 0 ) THEN ! ! module/output_field / NOT found in diag_table ! - IF ( debug_diag_manager .OR. verbose1 ) THEN + IF ( debug_diag_manager .OR. verbose1 ) THEN IF ( mpp_pe() == mpp_root_pe() ) & & CALL error_mesg ('diag_manager_mod::register_diag_field', 'module/output_field '& &//TRIM(module_name)//'/'// TRIM(field_name)//' NOT found in diag_table',& - & WARNING) + & WARNING) END IF - ELSE + ELSE input_fields(register_diag_field_array)%static = .FALSE. field = register_diag_field_array - IF ( PRESENT(standard_name) ) input_fields(field)%standard_name = standard_name + + + ! Verify that area and volume do not point to the same variable + IF ( PRESENT(volume).AND.PRESENT(area) ) THEN + IF ( area.EQ.volume ) THEN + CALL error_mesg ('diag_manager_mod::register_diag_field', 'module/output_field '& + &//TRIM(module_name)//'/'// TRIM(field_name)//' AREA and VOLUME CANNOT be the same variable.& + & Contact the developers.',& + & FATAL) + END IF + END IF + + ! Check for the existence of the area/volume field(s) + IF ( PRESENT(area) ) THEN + IF ( area < 0 ) THEN + CALL error_mesg ('diag_manager_mod::register_diag_field', 'module/output_field '& + &//TRIM(module_name)//'/'// TRIM(field_name)//' AREA measures field NOT found in diag_table.& + & Contact the model liaison.',& + & FATAL) + END IF + END IF + IF ( PRESENT(volume) ) THEN + IF ( volume < 0 ) THEN + CALL error_mesg ('diag_manager_mod::register_diag_field', 'module/output_field '& + &//TRIM(module_name)//'/'// TRIM(field_name)//' VOLUME measures field NOT found in diag_table.& + & Contact the model liaison.',& + & FATAL) + END IF + END IF + + IF ( PRESENT(standard_name) ) input_fields(field)%standard_name = standard_name DO j = 1, input_fields(field)%num_output_fields ind = input_fields(field)%output_fields(j) @@ -508,7 +601,7 @@ INTEGER FUNCTION register_diag_field_array(module_name, field_name, axes, init_t file_num = output_fields(ind)%output_file IF ( file_num == max_files ) CYCLE IF ( output_fields(ind)%local_output ) THEN - IF ( output_fields(ind)%need_compute) THEN + IF ( output_fields(ind)%need_compute) THEN files(file_num)%local = .TRUE. END IF END IF @@ -542,6 +635,15 @@ INTEGER FUNCTION register_diag_field_array(module_name, field_name, axes, init_t WRITE(stdout_unit,* ) 'module/output_field '//TRIM(module_name)//'/'//TRIM(field_name)// & & ' will be output in region:'//TRIM(msg) END IF + + ! Set the cell_measures attribute in the out file + CALL init_field_cell_measures(output_fields(ind), area=area, volume=volume, err_msg=err_msg) + IF ( LEN_TRIM(err_msg).GT.0 ) THEN + CALL error_mesg ('diag_manager_mod::register_diag_field',& + & TRIM(err_msg)//' for module/field '//TRIM(module_name)//'/'//TRIM(field_name),& + & FATAL) + END IF + END DO END IF END FUNCTION register_diag_field_array @@ -572,9 +674,11 @@ END FUNCTION register_diag_field_array ! ! ! + ! Field ID for the area field associated with this field + ! Field ID for the volume field associated with this field INTEGER FUNCTION register_static_field(module_name, field_name, axes, long_name, units,& & missing_value, range, mask_variant, standard_name, DYNAMIC, do_not_log, interp_method,& - & tile_count) + & tile_count, area, volume) CHARACTER(len=*), INTENT(in) :: module_name, field_name INTEGER, DIMENSION(:), INTENT(in) :: axes CHARACTER(len=*), OPTIONAL, INTENT(in) :: long_name, units, standard_name @@ -584,7 +688,7 @@ INTEGER FUNCTION register_static_field(module_name, field_name, axes, long_name, LOGICAL, OPTIONAL, INTENT(in) :: DYNAMIC LOGICAL, OPTIONAL, INTENT(in) :: do_not_log ! if TRUE, field information is not logged CHARACTER(len=*), OPTIONAL, INTENT(in) :: interp_method - INTEGER, OPTIONAL, INTENT(in) :: tile_count + INTEGER, OPTIONAL, INTENT(in) :: tile_count, area, volume REAL :: missing_value_use INTEGER :: field, num_axes, j, out_num, k @@ -594,33 +698,33 @@ INTEGER FUNCTION register_static_field(module_name, field_name, axes, long_name, CHARACTER(len=128) :: msg ! Fatal error if the module has not been initialized. - IF ( .NOT.module_is_initialized ) THEN + IF ( .NOT.module_is_initialized ) THEN ! diag_manager has NOT been initialized CALL error_mesg ('diag_manager_mod::register_static_field', 'diag_manager has NOT been initialized', FATAL) END IF ! Check if OPTIONAL parameters were passed in. IF ( PRESENT(missing_value) ) THEN - IF ( use_cmor ) THEN + IF ( use_cmor ) THEN missing_value_use = CMOR_MISSING_VALUE ELSE missing_value_use = missing_value END IF END IF - - IF ( PRESENT(mask_variant) ) THEN + + IF ( PRESENT(mask_variant) ) THEN mask_variant1 = mask_variant - ELSE + ELSE mask_variant1 = .FALSE. END IF - + IF ( PRESENT(DYNAMIC) ) THEN dynamic1 = DYNAMIC ELSE dynamic1 = .FALSE. END IF - IF ( PRESENT(tile_count) ) THEN + IF ( PRESENT(tile_count) ) THEN tile = tile_count ELSE tile = 1 @@ -629,7 +733,7 @@ INTEGER FUNCTION register_static_field(module_name, field_name, axes, long_name, IF ( PRESENT(do_not_log) ) THEN allow_log = .NOT.do_not_log ELSE - allow_log = .TRUE. + allow_log = .TRUE. END IF ! Namelist do_diag_field_log is by default false. Thus to log the @@ -655,7 +759,7 @@ INTEGER FUNCTION register_static_field(module_name, field_name, axes, long_name, ! CALL error_mesg ('diag_manager_mod::register_diag_field', 'module/output_field '//trim(module_name)//'/'//& & TRIM(field_name)//' is not registered for tile_count = 1, should not register for tile_count > 1',& - & FATAL) + & FATAL) END IF CALL init_input_field(module_name, field_name, tile) @@ -672,7 +776,7 @@ INTEGER FUNCTION register_static_field(module_name, field_name, axes, long_name, & files(file_num)%name,output_fields(out_num)%time_method, output_fields(out_num)%pack, tile) END IF END DO - field = register_static_field + field = register_static_field END IF ! Store information for this input field into input field table @@ -687,7 +791,35 @@ INTEGER FUNCTION register_static_field(module_name, field_name, axes, long_name, ! not register twice ! CALL error_mesg ('diag_manager_mod::register_diag_field', 'module/output_field '//trim(module_name)//'/'//& - & TRIM(field_name)//' ALREADY registered, should not register twice', FATAL) + & TRIM(field_name)//' ALREADY registered, should not register twice', FATAL) + END IF + + ! Verify that area and volume do not point to the same variable + IF ( PRESENT(volume).AND.PRESENT(area) ) THEN + IF ( area.EQ.volume ) THEN + CALL error_mesg ('diag_manager_mod::register_static_field', 'module/output_field '& + &//TRIM(module_name)//'/'// TRIM(field_name)//' AREA and VOLUME CANNOT be the same variable.& + & Contact the developers.',& + & FATAL) + END IF + END IF + + ! Check for the existence of the area/volume field(s) + IF ( PRESENT(area) ) THEN + IF ( area < 0 ) THEN + CALL error_mesg ('diag_manager_mod::register_static_field', 'module/output_field '& + &//TRIM(module_name)//'/'// TRIM(field_name)//' AREA measures field NOT found in diag_table.& + & Contact the model liaison.n',& + & FATAL) + END IF + END IF + IF ( PRESENT(volume) ) THEN + IF ( volume < 0 ) THEN + CALL error_mesg ('diag_manager_mod::register_static_field', 'module/output_field '& + &//TRIM(module_name)//'/'// TRIM(field_name)//' VOLUME measures field NOT found in diag_table& + & Contact the model liaison.',& + & FATAL) + END IF END IF ! Set flag that this field was registered @@ -703,22 +835,22 @@ INTEGER FUNCTION register_static_field(module_name, field_name, axes, long_name, ELSE input_fields(field)%long_name = input_fields(field)%field_name END IF - - IF ( PRESENT(standard_name) ) input_fields(field)%standard_name = standard_name - + + IF ( PRESENT(standard_name) ) input_fields(field)%standard_name = standard_name + IF ( PRESENT(units) ) THEN input_fields(field)%units = TRIM(units) ELSE input_fields(field)%units = 'none' END IF - + IF ( PRESENT(missing_value) ) THEN input_fields(field)%missing_value = missing_value_use input_fields(field)%missing_value_present = .TRUE. ELSE input_fields(field)%missing_value_present = .FALSE. END IF - + IF ( PRESENT(range) ) THEN input_fields(field)%range = range ! don't use the range if it is not a valid range @@ -729,7 +861,7 @@ INTEGER FUNCTION register_static_field(module_name, field_name, axes, long_name, END IF IF ( PRESENT(interp_method) ) THEN - IF ( TRIM(interp_method) .NE. 'conserve_order1' ) THEN + IF ( TRIM(interp_method) .NE. 'conserve_order1' ) THEN ! ! when registering module/output_field / then optional ! argument interp_method = , but it should be "conserve_order1" @@ -740,7 +872,7 @@ INTEGER FUNCTION register_static_field(module_name, field_name, axes, long_name, & ', but it should be "conserve_order1"', FATAL) END IF input_fields(field)%interp_method = TRIM(interp_method) - ELSE + ELSE input_fields(field)%interp_method = '' END IF @@ -748,7 +880,7 @@ INTEGER FUNCTION register_static_field(module_name, field_name, axes, long_name, num_axes = SIZE(axes(:)) ! num_axes should be <= 3. input_fields(field)%axes(1:num_axes) = axes input_fields(field)%num_axes = num_axes - + siz = 1 DO j = 1, num_axes IF ( axes(j) .LE. 0 ) THEN @@ -756,7 +888,7 @@ INTEGER FUNCTION register_static_field(module_name, field_name, axes, long_name, ! module/output_field / has non-positive axis_id ! CALL error_mesg ('diag_manager_mod::register_diag_field', 'module/output_field '//trim(module_name)//'/'//& - & TRIM(field_name)//' has non-positive axis_id', FATAL) + & TRIM(field_name)//' has non-positive axis_id', FATAL) END IF siz(j) = get_axis_length(axes(j)) END DO @@ -772,7 +904,7 @@ INTEGER FUNCTION register_static_field(module_name, field_name, axes, long_name, ! Need to loop through all output_fields associated and allocate their buffers DO j = 1, input_fields(field)%num_output_fields out_num = input_fields(field)%output_fields(j) - ! Range is required when pack >= 4 + ! Range is required when pack >= 4 IF ( output_fields(out_num)%pack>=4 .AND. .NOT.input_fields(field)%range_present ) THEN IF(mpp_pe() .EQ. mpp_root_pe()) THEN ! @@ -788,10 +920,10 @@ INTEGER FUNCTION register_static_field(module_name, field_name, axes, long_name, ! if local_output (size of output_fields does NOT equal size of input_fields) IF ( output_fields(out_num)%reduced_k_range ) THEN CALL get_subfield_vert_size(axes, out_num) - + local_start(3) = output_fields(out_num)%output_grid%l_start_indx(3) local_end(3) = output_fields(out_num)%output_grid%l_end_indx(3) - local_siz(3) = local_end(3) - local_start(3) +1 + local_siz(3) = local_end(3) - local_start(3) +1 ALLOCATE(output_fields(out_num)%buffer(siz(1), siz(2), local_siz(3),& & output_fields(out_num)%n_diurnal_samples)) @@ -816,7 +948,7 @@ INTEGER FUNCTION register_static_field(module_name, field_name, axes, long_name, DO k = 1, num_axes local_start(k) = output_fields(out_num)%output_grid%l_start_indx(k) local_end(k) = output_fields(out_num)%output_grid%l_end_indx(k) - local_siz(k) = local_end(k) - local_start(k) +1 + local_siz(k) = local_end(k) - local_start(k) +1 END DO ALLOCATE(output_fields(out_num)%buffer(local_siz(1), local_siz(2), local_siz(3),& & output_fields(out_num)%n_diurnal_samples)) @@ -832,7 +964,7 @@ INTEGER FUNCTION register_static_field(module_name, field_name, axes, long_name, files(output_fields(out_num)%output_file)%local = .true. END IF ELSE ! the field is output globally - ! size of output_fields equal size of input_fields + ! size of output_fields equal size of input_fields ALLOCATE(output_fields(out_num)%buffer(siz(1), siz(2), siz(3),& & output_fields(out_num)%n_diurnal_samples)) IF(output_fields(out_num)%time_max) THEN @@ -844,7 +976,7 @@ INTEGER FUNCTION register_static_field(module_name, field_name, axes, long_name, END IF output_fields(out_num)%total_elements = siz(1)*siz(2)*siz(3) END IF - + ! Reset to false in register_field if this is not static output_fields(out_num)%static = .TRUE. ! check if time average is true for static field @@ -871,7 +1003,7 @@ INTEGER FUNCTION register_static_field(module_name, field_name, axes, long_name, ! this should be changed later to take into account time-of-day axis output_fields(out_num)%num_axes = input_fields(field)%num_axes ! Axes are copied from input_fields if output globally or from subaxes if output locally - IF ( .NOT.output_fields(out_num)%local_output ) THEN + IF ( .NOT.output_fields(out_num)%local_output ) THEN output_fields(out_num)%axes(1:input_fields(field)%num_axes) =& & input_fields(field)%axes(1:input_fields(field)%num_axes) ELSE @@ -879,7 +1011,7 @@ INTEGER FUNCTION register_static_field(module_name, field_name, axes, long_name, & output_fields(out_num)%output_grid%subaxes(1:input_fields(field)%num_axes) END IF - ! if necessary, initialize the diurnal time axis and append its index in the + ! if necessary, initialize the diurnal time axis and append its index in the ! output field axes array IF ( output_fields(out_num)%n_diurnal_samples > 1 ) THEN output_fields(out_num)%axes(output_fields(out_num)%num_axes+1) =& @@ -887,12 +1019,20 @@ INTEGER FUNCTION register_static_field(module_name, field_name, axes, long_name, output_fields(out_num)%num_axes = output_fields(out_num)%num_axes+1 END IF - IF ( output_fields(out_num)%reduced_k_range ) THEN + IF ( output_fields(out_num)%reduced_k_range ) THEN output_fields(out_num)%axes(3) = output_fields(out_num)%output_grid%subaxes(3) END IF ! Initialize a time variable used in an error check output_fields(out_num)%Time_of_prev_field_data = Time_zero + + ! Set the cell_measures attribute in the out file + CALL init_field_cell_measures(output_fields(out_num), area=area, volume=volume, err_msg=msg) + IF ( LEN_TRIM(msg).GT.0 ) THEN + CALL error_mesg ('diag_manager_mod::register_static_field',& + & TRIM(msg)//' for module/field '//TRIM(module_name)//'/'//TRIM(field_name),& + & FATAL) + END IF END DO IF ( input_fields(field)%mask_variant ) THEN @@ -908,6 +1048,205 @@ INTEGER FUNCTION register_static_field(module_name, field_name, axes, long_name, END FUNCTION register_static_field ! + ! + ! + ! Return the diagnostic field ID of a given variable. + ! + ! + ! + ! get_diag_field_id will return the ID returned during the register_diag_field call. If + ! the variable is not in the diag_table, then the value "DIAG_FIELD_NOT_FOUND" will be + ! returned. + ! + ! Module name that registered the variable + ! Variable name + INTEGER FUNCTION get_diag_field_id(module_name, field_name) + CHARACTER(len=*), INTENT(in) :: module_name, field_name + + ! find_input_field will return DIAG_FIELD_NOT_FOUND if the field is not + ! included in the diag_table + get_diag_field_id = find_input_field(module_name, field_name, tile_count=1) + END FUNCTION get_diag_field_id + ! + + ! + ! + ! Finds the corresponding related output field and file + ! + ! + ! + ! Finds the corresponding related output field and file for a given input field + ! + ! input field ID to find the corresponding + ! Output field that field must correspond to + ! output_field index of related output field + ! file index of the out_field_id output field + LOGICAL FUNCTION get_related_field(field, rel_field, out_field_id, out_file_id) + INTEGER, INTENT(in) :: field + TYPE(output_field_type), INTENT(in) :: rel_field + INTEGER, INTENT(out) :: out_field_id, out_file_id + + INTEGER :: i, cm_ind, cm_file_num + INTEGER :: rel_file + + ! Output file index of field to compare to + rel_file = rel_field%output_file + + ! Default return values + out_field_id = -1 + out_file_id = -1 + get_related_field = .FALSE. + + ! First check if any fields are in the same file as rel_field + DO i = 1, input_fields(field)%num_output_fields + cm_ind = input_fields(field)%output_fields(i) + cm_file_num = output_fields(cm_ind)%output_file + + IF ( cm_file_num.EQ.rel_file.AND.& + & (( (output_fields(cm_ind)%time_ops.EQV.rel_field%time_ops) .AND.& + & (output_fields(cm_ind)%next_output.EQ.rel_field%next_output) .AND.& + & (output_fields(cm_ind)%last_output.EQ.rel_field%last_output) ).OR.& + & (output_fields(cm_ind)%static.OR.rel_field%static) ) ) THEN + get_related_field = .TRUE. + out_field_id = cm_ind + out_file_id = cm_file_num + EXIT + END IF + END DO + + ! Now look for the field in a different file + IF ( .NOT.get_related_field ) THEN + DO i = 1, input_fields(field)%num_output_fields + cm_ind = input_fields(field)%output_fields(i) + cm_file_num = output_fields(cm_ind)%output_file + + ! If time_method, freq, output_units, next_output, and last_output the same, or + ! the output_field is static then valid for cell_measures +!!$ For now, only static fields can be in an external file +!!$ IF ( ( (files(cm_file_num)%output_freq.EQ.files(rel_file)%output_freq) .AND.& +!!$ & (files(cm_file_num)%output_units.EQ.files(rel_file)%output_units) .AND.& +!!$ & (output_fields(cm_ind)%time_ops.EQV.rel_field%time_ops) .AND.& +!!$ & (output_fields(cm_ind)%next_output.EQ.rel_field%next_output) .AND.& +!!$ & (output_fields(cm_ind)%last_output.EQ.rel_field%last_output) ).OR.& +!!$ & ( output_fields(cm_ind)%static.OR.rel_field%static ) ) THEN + IF ( output_fields(cm_ind)%static.OR.rel_field%static ) THEN + get_related_field = .TRUE. + out_field_id = cm_ind + out_file_id = cm_file_num + EXIT + END IF + END DO + END IF + END FUNCTION get_related_field + ! + + ! + ! + ! If needed, add cell_measures and associated_file attribute to out field/file + ! + ! + ! + ! If needed, add cell_measures and associated_file attribute to out field/file + ! + ! Output field that needs the cell_measures + ! Field ID for area + ! Field ID for volume + ! + SUBROUTINE init_field_cell_measures(output_field, area, volume, err_msg) + TYPE(output_field_type), INTENT(inout) :: output_field + INTEGER, INTENT(in), OPTIONAL :: area, volume + CHARACTER(len=*), INTENT(out), OPTIONAL :: err_msg + + INTEGER :: cm_ind, cm_file_num, file_num + INTEGER :: year, month, day, hour, minute, second + CHARACTER(len=25) :: date_prefix + + IF ( PRESENT(err_msg) ) THEN + err_msg = '' + END IF + + ! Verify that area/volume are defined (.gt.0 + IF ( PRESENT(area) ) THEN + IF ( area.LE.0 ) THEN + IF ( fms_error_handler('diag_manager_mod::init_field_cell_measure',& + & 'AREA field not in diag_table for field '//TRIM(input_fields(output_field%input_field)%module_name)//& + & '/'//TRIM(input_fields(output_field%input_field)%field_name), err_msg) ) RETURN + END IF + END IF + + IF ( PRESENT(volume) ) THEN + IF ( volume.LE.0 ) THEN + IF ( fms_error_handler('diag_manager_mod::init_field_cell_measure',& + & 'VOLUME field not in diag_table for field '//TRIM(input_fields(output_field%input_field)%module_name)//& + & '/'//TRIM(input_fields(output_field%input_field)%field_name), err_msg) ) RETURN + END IF + END IF + + ! Get the file number that the output_field will be written to + file_num = output_field%output_file + + ! Create the date_string + IF ( prepend_date ) THEN + call get_date(diag_init_time, year, month, day, hour, minute, second) + write (date_prefix, '(1I20.4, 2I2.2,".")') year, month, day + date_prefix=adjustl(date_prefix) + ELSE + date_prefix='' + END IF + + ! Take care of the cell_measures attribute + IF ( PRESENT(area) ) THEN + IF ( get_related_field(area, output_field, cm_ind, cm_file_num) ) THEN + CALL prepend_attribute(output_field, 'cell_measures',& + & 'area: '//TRIM(output_fields(cm_ind)%output_name)) + IF ( cm_file_num.NE.file_num ) THEN + ! Not in the same file, set the global attribute associated_files + ! Should look like :associated_files = " output_name: output_file_name " ; + ! Need to append *.nc as files()%name does not include this. + CALL prepend_attribute(files(file_num), 'associated_files',& + & TRIM(output_fields(cm_ind)%output_name)//': '//& + & TRIM(date_prefix)//TRIM(files(cm_file_num)%name)//'.nc') + END IF + ELSE + IF ( fms_error_handler('diag_manager_mod::init_field_cell_measures',& + & 'AREA measures field "'//TRIM(input_fields(area)%module_name)//'/'//& + & TRIM(input_fields(area)%field_name)//& + & '" NOT in diag_table with correct output frequency for field '//& + & TRIM(input_fields(output_field%input_field)%module_name)//& + & '/'//TRIM(input_fields(output_field%input_field)%field_name), err_msg) ) RETURN + END IF + END IF + + + IF ( PRESENT(volume) ) THEN + IF ( get_related_field(volume, output_field, cm_ind, cm_file_num) ) THEN + CALL prepend_attribute(output_field, 'cell_measures',& + & 'volume: '//TRIM(output_fields(cm_ind)%output_name)) + IF ( cm_file_num.NE.file_num ) THEN + ! Not in the same file, set the global attribute associated_files + ! Should look like :associated_files = " output_name: output_file_name " ; + CALL prepend_attribute(files(file_num), 'associated_files',& + & TRIM(output_fields(cm_ind)%output_name)//': '//& + & TRIM(date_prefix)//TRIM(files(cm_file_num)%name)//'.nc') + END IF + ELSE + IF ( fms_error_handler('diag_manager_mod::init_field_cell_measures',& + & 'VOLUME measures field "'//TRIM(input_fields(volume)%module_name)//'/'//& + & TRIM(input_fields(volume)%field_name)//& + & '" NOT in diag_table with correct output frequency for field '//& + & TRIM(input_fields(output_field%input_field)%module_name)//& + & '/'//TRIM(input_fields(output_field%input_field)%field_name), err_msg) ) RETURN + END IF + END IF + END SUBROUTINE init_field_cell_measures + ! + ! ! ! @@ -978,7 +1317,7 @@ LOGICAL FUNCTION send_data_1d(diag_field_id, field, time, is_in, mask, rmask, ie field_out(:, 1, 1) = field ! Default values for mask - IF ( PRESENT(mask) ) THEN + IF ( PRESENT(mask) ) THEN mask_out(:, 1, 1) = mask ELSE mask_out = .TRUE. @@ -1042,12 +1381,12 @@ LOGICAL FUNCTION send_data_2d(diag_field_id, field, time, is_in, js_in, & field_out(:, :, 1) = field ! Default values for mask - IF ( PRESENT(mask) ) THEN + IF ( PRESENT(mask) ) THEN mask_out(:, :, 1) = mask - ELSE + ELSE mask_out = .TRUE. END IF - + IF ( PRESENT(rmask) ) WHERE ( rmask < 0.5 ) mask_out(:, :, 1) = .FALSE. IF ( PRESENT(mask) .OR. PRESENT(rmask) ) THEN send_data_2d = send_data_3d(diag_field_id, field_out, time, is_in=is_in, js_in=js_in, ks_in=1, mask=mask_out,& @@ -1079,13 +1418,14 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & REAL, DIMENSION(:,:,:), INTENT(in) :: field REAL, INTENT(in), OPTIONAL :: weight TYPE (time_type), INTENT(in), OPTIONAL :: time - INTEGER, INTENT(in), OPTIONAL :: is_in, js_in, ks_in,ie_in,je_in, ke_in + INTEGER, INTENT(in), OPTIONAL :: is_in, js_in, ks_in,ie_in,je_in, ke_in LOGICAL, DIMENSION(:,:,:), INTENT(in), OPTIONAL :: mask REAL, DIMENSION(:,:,:), INTENT(in), OPTIONAL :: rmask CHARACTER(len=*), INTENT(out), OPTIONAL :: err_msg REAL :: weight1 REAL :: missvalue + INTEGER :: pow_value INTEGER :: ksr, ker INTEGER :: i, out_num, file_num, n1, n2, n3, number_of_outputs, ii,f1,f2,f3,f4 INTEGER :: freq, units, is, js, ks, ie, je, ke, i1, j1,k1, j, k @@ -1095,12 +1435,14 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & INTEGER :: day,second,tick ! components of the current date INTEGER :: status INTEGER :: numthreads + INTEGER :: active_omp_level #if defined(_OPENMP) INTEGER :: omp_get_num_threads !< OMP function + INTEGER :: omp_get_level !< OMP function #endif LOGICAL :: average, phys_window, need_compute LOGICAL :: reduced_k_range, local_output - LOGICAL :: time_max, time_min + LOGICAL :: time_max, time_min, time_rms LOGICAL :: missvalue_present LOGICAL, ALLOCATABLE, DIMENSION(:,:,:) :: oor_mask CHARACTER(len=256) :: err_msg_local @@ -1120,6 +1462,14 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & IF ( fms_error_handler('diag_manager_mod::send_data_3d', 'diag_manager NOT initialized', err_msg) ) RETURN END IF err_msg_local = '' + ! The following lines are commented out as they have not been included in the code prior to now, + ! and there are a lot of send_data calls before register_diag_field calls. A method to do this safely + ! needs to be developed. + ! + ! Set first_send_data_call to .FALSE. on first non-static field. +!!$ IF ( .NOT.input_fields(diag_field_id)%static .AND. first_send_data_call ) THEN +!!$ first_send_data_call = .FALSE. +!!$ END IF ! oor_mask is only used for checking out of range values. ALLOCATE(oor_mask(SIZE(field,1),SIZE(field,2),SIZE(field,3)), STAT=status) @@ -1129,9 +1479,9 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & IF ( fms_error_handler('diag_manager_mod::send_data_3d', err_msg_local, err_msg) ) RETURN END IF - IF ( PRESENT(mask) ) THEN + IF ( PRESENT(mask) ) THEN oor_mask = mask - ELSE + ELSE oor_mask = .TRUE. END IF IF ( PRESENT(rmask) ) WHERE ( rmask < 0.5 ) oor_mask = .FALSE. @@ -1149,14 +1499,14 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & ! of presence/absence of is,ie,js,je. The checks below should catch improper combinations. IF ( PRESENT(ie_in) ) THEN IF ( .NOT.PRESENT(is_in) ) THEN - IF ( fms_error_handler('diag_manager_mod::send_data_3d', 'ie_in present without is_in', err_msg) ) THEN + IF ( fms_error_handler('diag_manager_mod::send_data_3d', 'ie_in present without is_in', err_msg) ) THEN DEALLOCATE(oor_mask) RETURN END IF END IF IF ( PRESENT(js_in) .AND. .NOT.PRESENT(je_in) ) THEN IF ( fms_error_handler('diag_manager_modsend_data_3d',& - & 'is_in and ie_in present, but js_in present without je_in', err_msg) ) THEN + & 'is_in and ie_in present, but js_in present without je_in', err_msg) ) THEN DEALLOCATE(oor_mask) RETURN END IF @@ -1164,14 +1514,14 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & END IF IF ( PRESENT(je_in) ) THEN IF ( .NOT.PRESENT(js_in) ) THEN - IF ( fms_error_handler('diag_manager_mod::send_data_3d', 'je_in present without js_in', err_msg) ) THEN + IF ( fms_error_handler('diag_manager_mod::send_data_3d', 'je_in present without js_in', err_msg) ) THEN DEALLOCATE(oor_mask) RETURN END IF END IF IF ( PRESENT(is_in) .AND. .NOT.PRESENT(ie_in) ) THEN IF ( fms_error_handler('diag_manager_mod::send_data_3d',& - & 'js_in and je_in present, but is_in present without ie_in', err_msg)) THEN + & 'js_in and je_in present, but is_in present without ie_in', err_msg)) THEN DEALLOCATE(oor_mask) RETURN END IF @@ -1196,14 +1546,14 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & IF ( PRESENT(ke_in) ) ke = ke_in twohi = n1-(ie-is+1) IF ( MOD(twohi,2) /= 0 ) THEN - IF ( fms_error_handler('diag_manager_mod::send_data_3d', 'non-symmetric halos in first dimension', err_msg) ) THEN + IF ( fms_error_handler('diag_manager_mod::send_data_3d', 'non-symmetric halos in first dimension', err_msg) ) THEN DEALLOCATE(oor_mask) RETURN END IF END IF twohj = n2-(je-js+1) IF ( MOD(twohj,2) /= 0 ) THEN - IF ( fms_error_handler('diag_manager_mod::send_data_3d', 'non-symmetric halos in second dimension', err_msg) ) THEN + IF ( fms_error_handler('diag_manager_mod::send_data_3d', 'non-symmetric halos in second dimension', err_msg) ) THEN DEALLOCATE(oor_mask) RETURN END IF @@ -1227,12 +1577,12 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & f4=n2-hj ! weight is for time averaging where each time level may has a different weight - IF ( PRESENT(weight) ) THEN + IF ( PRESENT(weight) ) THEN weight1 = weight ELSE weight1 = 1. END IF - + ! Is there a missing_value? missvalue_present = input_fields(diag_field_id)%missing_value_present IF ( missvalue_present ) missvalue = input_fields(diag_field_id)%missing_value @@ -1240,10 +1590,13 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & number_of_outputs = input_fields(diag_field_id)%num_output_fields !$OMP CRITICAL input_fields(diag_field_id)%numthreads = 1 + active_omp_level=0 #if defined(_OPENMP) input_fields(diag_field_id)%numthreads = omp_get_num_threads() + input_fields(diag_field_id)%active_omp_level = omp_get_level() #endif numthreads = input_fields(diag_field_id)%numthreads + active_omp_level = input_fields(diag_field_id)%active_omp_level !$OMP END CRITICAL if(present(time)) input_fields(diag_field_id)%time = time @@ -1318,16 +1671,21 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & units = files(file_num)%output_units ! Is this output field being time averaged? average = output_fields(out_num)%time_average + ! Is this output field the rms? + ! If so, then average is also .TRUE. + time_rms = output_fields(out_num)%time_rms + ! Power value for rms or pow(x) calculations + pow_value = output_fields(out_num)%pow_value ! Looking for max and min value of this field over the sampling interval? time_max = output_fields(out_num)%time_max - time_min = output_fields(out_num)%time_min + time_min = output_fields(out_num)%time_min IF ( output_fields(out_num)%total_elements > SIZE(field(f1:f2,f3:f4,ks:ke)) ) THEN output_fields(out_num)%phys_window = .TRUE. ELSE output_fields(out_num)%phys_window = .FALSE. END IF phys_window = output_fields(out_num)%phys_window - IF ( need_compute ) THEN + IF ( need_compute ) THEN l_start = output_fields(out_num)%output_grid%l_start_indx l_end = output_fields(out_num)%output_grid%l_end_indx END IF @@ -1339,7 +1697,7 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & CALL get_time(time,second,day,tick) ! current date sample = set_time(second,0,tick)/dt + 1 END IF - + ! Get the vertical layer start and end index. IF ( reduced_k_range ) THEN l_start(3) = output_fields(out_num)%output_grid%l_start_indx(3) @@ -1358,19 +1716,19 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & & TRIM(input_fields(diag_field_id)%module_name),& & TRIM(output_fields(out_num)%output_name) IF ( fms_error_handler('diag_manager_mod::send_data_3d', 'module/output_field '//TRIM(error_string)//& - & ', time must be present when output frequency = EVERY_TIME', err_msg)) THEN + & ', time must be present when output frequency = EVERY_TIME', err_msg)) THEN DEALLOCATE(oor_mask) RETURN END IF END IF END IF - END IF + END IF IF ( .NOT.output_fields(out_num)%static .AND. .NOT.PRESENT(time) ) THEN WRITE (error_string,'(a,"/",a)')& & TRIM(input_fields(diag_field_id)%module_name), & & TRIM(output_fields(out_num)%output_name) IF ( fms_error_handler('diag_manager_mod::send_data_3d', 'module/output_field '//TRIM(error_string)//& - & ', time must be present for nonstatic field', err_msg)) THEN + & ', time must be present for nonstatic field', err_msg)) THEN DEALLOCATE(oor_mask) RETURN END IF @@ -1378,18 +1736,18 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & ! Is it time to output for this field; CAREFUL ABOUT > vs >= HERE !--- The fields send out within openmp parallel region will be written out in - !--- diag_send_complete. - IF ( numthreads == 1) then + !--- diag_send_complete. + IF ( (numthreads == 1) .AND. (active_omp_level.LE.1) ) then IF ( .NOT.output_fields(out_num)%static .AND. freq /= END_OF_RUN ) THEN IF ( time > output_fields(out_num)%next_output ) THEN ! A non-static field that has skipped a time level is an error IF ( time > output_fields(out_num)%next_next_output .AND. freq > 0 ) THEN - IF ( mpp_pe() .EQ. mpp_root_pe() ) THEN + IF ( mpp_pe() .EQ. mpp_root_pe() ) THEN WRITE (error_string,'(a,"/",a)')& & TRIM(input_fields(diag_field_id)%module_name), & & TRIM(output_fields(out_num)%output_name) IF ( fms_error_handler('diag_manager_mod::send_data_3d', 'module/output_field '//TRIM(error_string)//& - & ' is skipped one time level in output data', err_msg)) THEN + & ' is skipped one time level in output data', err_msg)) THEN DEALLOCATE(oor_mask) RETURN END IF @@ -1400,7 +1758,7 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & IF(status == -1) THEN IF ( mpp_pe() .EQ. mpp_root_pe() ) THEN IF(fms_error_handler('diag_manager_mod::send_data_3d','module/output_field '//TRIM(error_string)//& - & ', write EMPTY buffer', err_msg)) THEN + & ', write EMPTY buffer', err_msg)) THEN DEALLOCATE(oor_mask) RETURN END IF @@ -1408,57 +1766,63 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & END IF END IF !time > output_fields(out_num)%next_output END IF !.not.output_fields(out_num)%static .and. freq /= END_OF_RUN - ! Finished output of previously buffered data, now deal with buffering new data + ! Finished output of previously buffered data, now deal with buffering new data END IF IF ( .NOT.output_fields(out_num)%static .AND. .NOT.need_compute .AND. debug_diag_manager ) THEN CALL check_bounds_are_exact_dynamic(out_num, diag_field_id, Time, err_msg=err_msg_local) IF ( err_msg_local /= '' ) THEN - IF ( fms_error_handler('diag_manager_mod::send_data_3d', err_msg_local, err_msg) ) THEN + IF ( fms_error_handler('diag_manager_mod::send_data_3d', err_msg_local, err_msg) ) THEN DEALLOCATE(oor_mask) RETURN END IF END IF END IF - + ! Take care of submitted field data IF ( average ) THEN IF ( input_fields(diag_field_id)%mask_variant ) THEN IF ( need_compute ) THEN WRITE (error_string,'(a,"/",a)') & & TRIM(input_fields(diag_field_id)%module_name), & - & TRIM(output_fields(out_num)%output_name) + & TRIM(output_fields(out_num)%output_name) IF ( fms_error_handler('diag_manager_mod::send_data_3d', 'module/output_field '//TRIM(error_string)//& - & ', regional output NOT supported with mask_variant', err_msg)) THEN + & ', regional output NOT supported with mask_variant', err_msg)) THEN DEALLOCATE(oor_mask) RETURN END IF END IF ! Should reduced_k_range data be supported with the mask_variant option ????? - ! If not, error message should be produced and the reduced_k_range loop below eliminated + ! If not, error message should be produced and the reduced_k_range loop below eliminated IF ( PRESENT(mask) ) THEN - IF ( missvalue_present ) THEN + IF ( missvalue_present ) THEN IF ( debug_diag_manager ) THEN CALL update_bounds(out_num, is-hi, ie-hi, js-hj, je-hj, ks, ke) CALL check_out_of_bounds(out_num, diag_field_id, err_msg=err_msg_local) IF ( err_msg_local /= '' ) THEN - IF ( fms_error_handler('diag_manager_mod::send_data_3d', err_msg_local, err_msg) ) THEN + IF ( fms_error_handler('diag_manager_mod::send_data_3d', err_msg_local, err_msg) ) THEN DEALLOCATE(oor_mask) RETURN END IF END IF END IF IF( numthreads>1 .AND. phys_window ) then - IF ( reduced_k_range ) THEN + IF ( reduced_k_range ) THEN DO k= ksr, ker - k1= k - ksr + 1 + k1= k - ksr + 1 DO j=js, je DO i=is, ie IF ( mask(i-is+1+hi, j-js+1+hj, k) ) THEN - output_fields(out_num)%buffer(i-hi,j-hj,k1,sample) =& - & output_fields(out_num)%buffer(i-hi,j-hj,k1,sample) +& - & field(i-is+1+hi, j-js+1+hj, k) * weight1 + IF ( pow_value /= 1 ) THEN + output_fields(out_num)%buffer(i-hi,j-hj,k1,sample) =& + & output_fields(out_num)%buffer(i-hi,j-hj,k1,sample) +& + & (field(i-is+1+hi, j-js+1+hj, k) * weight1)**(pow_value) + ELSE + output_fields(out_num)%buffer(i-hi,j-hj,k1,sample) =& + & output_fields(out_num)%buffer(i-hi,j-hj,k1,sample) +& + & field(i-is+1+hi, j-js+1+hj, k) * weight1 + END IF output_fields(out_num)%counter(i-hi,j-hj,k1,sample) =& & output_fields(out_num)%counter(i-hi,j-hj,k1,sample) + weight1 END IF @@ -1466,13 +1830,19 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & END DO END DO ELSE - DO k=ks, ke + DO k=ks, ke DO j=js, je DO i=is, ie IF ( mask(i-is+1+hi, j-js+1+hj, k) ) THEN - output_fields(out_num)%buffer(i-hi,j-hj,k,sample) =& - & output_fields(out_num)%buffer(i-hi,j-hj,k,sample) +& - & field(i-is+1+hi,j-js+1+hj,k)*weight1 + IF ( pow_value /= 1 ) THEN + output_fields(out_num)%buffer(i-hi,j-hj,k,sample) =& + & output_fields(out_num)%buffer(i-hi,j-hj,k,sample) +& + & (field(i-is+1+hi,j-js+1+hj,k)*weight1)**(pow_value) + ELSE + output_fields(out_num)%buffer(i-hi,j-hj,k,sample) =& + & output_fields(out_num)%buffer(i-hi,j-hj,k,sample) +& + & field(i-is+1+hi,j-js+1+hj,k)*weight1 + END IF output_fields(out_num)%counter(i-hi,j-hj,k,sample) =& &output_fields(out_num)%counter(i-hi,j-hj,k,sample) + weight1 END IF @@ -1482,15 +1852,21 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & END IF ELSE !$OMP CRITICAL - IF ( reduced_k_range ) THEN + IF ( reduced_k_range ) THEN DO k= ksr, ker - k1= k - ksr + 1 + k1= k - ksr + 1 DO j=js, je DO i=is, ie IF ( mask(i-is+1+hi, j-js+1+hj, k) ) THEN - output_fields(out_num)%buffer(i-hi,j-hj,k1,sample) =& - & output_fields(out_num)%buffer(i-hi,j-hj,k1,sample) +& - & field(i-is+1+hi, j-js+1+hj, k) * weight1 + IF ( pow_value /= 1 ) THEN + output_fields(out_num)%buffer(i-hi,j-hj,k1,sample) =& + & output_fields(out_num)%buffer(i-hi,j-hj,k1,sample) +& + & (field(i-is+1+hi, j-js+1+hj, k) * weight1)**(pow_value) + ELSE + output_fields(out_num)%buffer(i-hi,j-hj,k1,sample) =& + & output_fields(out_num)%buffer(i-hi,j-hj,k1,sample) +& + & field(i-is+1+hi, j-js+1+hj, k) * weight1 + END IF output_fields(out_num)%counter(i-hi,j-hj,k1,sample) =& & output_fields(out_num)%counter(i-hi,j-hj,k1,sample) + weight1 END IF @@ -1498,13 +1874,19 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & END DO END DO ELSE - DO k=ks, ke + DO k=ks, ke DO j=js, je DO i=is, ie IF ( mask(i-is+1+hi, j-js+1+hj, k) ) THEN - output_fields(out_num)%buffer(i-hi,j-hj,k,sample) =& - & output_fields(out_num)%buffer(i-hi,j-hj,k,sample) +& - & field(i-is+1+hi,j-js+1+hj,k)*weight1 + IF ( pow_value /= 1 ) THEN + output_fields(out_num)%buffer(i-hi,j-hj,k,sample) =& + & output_fields(out_num)%buffer(i-hi,j-hj,k,sample) +& + & (field(i-is+1+hi,j-js+1+hj,k)*weight1)**(pow_value) + ELSE + output_fields(out_num)%buffer(i-hi,j-hj,k,sample) =& + & output_fields(out_num)%buffer(i-hi,j-hj,k,sample) +& + & field(i-is+1+hi,j-js+1+hj,k)*weight1 + END IF output_fields(out_num)%counter(i-hi,j-hj,k,sample) =& &output_fields(out_num)%counter(i-hi,j-hj,k,sample) + weight1 END IF @@ -1519,7 +1901,7 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & & TRIM(input_fields(diag_field_id)%module_name), & & TRIM(output_fields(out_num)%output_name) IF ( fms_error_handler('diag_manager_mod::send_data_3d', 'module/output_field '//TRIM(error_string)//& - & ', variable mask but no missing value defined', err_msg)) THEN + & ', variable mask but no missing value defined', err_msg)) THEN DEALLOCATE(oor_mask) RETURN END IF @@ -1529,7 +1911,7 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & & TRIM(input_fields(diag_field_id)%module_name), & & TRIM(output_fields(out_num)%output_name) IF(fms_error_handler('diag_manager_mod::send_data_3d','module/output_field '//TRIM(error_string)//& - & ', variable mask but no mask given', err_msg)) THEN + & ', variable mask but no mask given', err_msg)) THEN DEALLOCATE(oor_mask) RETURN END IF @@ -1541,17 +1923,23 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & IF (numthreads>1 .AND. phys_window) then DO k = l_start(3), l_end(3) k1 = k-l_start(3)+1 - DO j = js, je + DO j = js, je DO i = is, ie IF ( l_start(1)+hi <= i .AND. i <= l_end(1)+hi .AND. l_start(2)+hj <= j .AND. j <= l_end(2)+hj ) THEN - i1 = i-l_start(1)-hi+1 + i1 = i-l_start(1)-hi+1 j1= j-l_start(2)-hj+1 IF ( mask(i-is+1+hi, j-js+1+hj, k) ) THEN - output_fields(out_num)%buffer(i1,j1,k1,sample) =& - & output_fields(out_num)%buffer(i1,j1,k1,sample) +& - & field(i-is+1+hi,j-js+1+hj,k) * weight1 + IF ( pow_value /= 1 ) THEN + output_fields(out_num)%buffer(i1,j1,k1,sample) =& + & output_fields(out_num)%buffer(i1,j1,k1,sample) +& + & (field(i-is+1+hi,j-js+1+hj,k) * weight1)**(pow_value) + ELSE + output_fields(out_num)%buffer(i1,j1,k1,sample) =& + & output_fields(out_num)%buffer(i1,j1,k1,sample) +& + & field(i-is+1+hi,j-js+1+hj,k) * weight1 + END IF ELSE - output_fields(out_num)%buffer(i1,j1,k1,sample) = missvalue + output_fields(out_num)%buffer(i1,j1,k1,sample) = missvalue END IF END IF END DO @@ -1561,17 +1949,23 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & !$OMP CRITICAL DO k = l_start(3), l_end(3) k1 = k-l_start(3)+1 - DO j = js, je + DO j = js, je DO i = is, ie IF ( l_start(1)+hi <= i .AND. i <= l_end(1)+hi .AND. l_start(2)+hj <= j .AND. j <= l_end(2)+hj ) THEN - i1 = i-l_start(1)-hi+1 + i1 = i-l_start(1)-hi+1 j1= j-l_start(2)-hj+1 IF ( mask(i-is+1+hi, j-js+1+hj, k) ) THEN - output_fields(out_num)%buffer(i1,j1,k1,sample) =& - & output_fields(out_num)%buffer(i1,j1,k1,sample) +& - & field(i-is+1+hi,j-js+1+hj,k) * weight1 + IF ( pow_value /= 1 ) THEN + output_fields(out_num)%buffer(i1,j1,k1,sample) =& + & output_fields(out_num)%buffer(i1,j1,k1,sample) +& + & (field(i-is+1+hi,j-js+1+hj,k) * weight1)**(pow_value) + ELSE + output_fields(out_num)%buffer(i1,j1,k1,sample) =& + & output_fields(out_num)%buffer(i1,j1,k1,sample) +& + & field(i-is+1+hi,j-js+1+hj,k) * weight1 + END IF ELSE - output_fields(out_num)%buffer(i1,j1,k1,sample) = missvalue + output_fields(out_num)%buffer(i1,j1,k1,sample) = missvalue END IF END IF END DO @@ -1580,7 +1974,7 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & !$OMP END CRITICAL ENDIF !$OMP CRITICAL - DO j = js, je + DO j = js, je DO i = is, ie IF ( l_start(1)+hi <= i .AND. i <= l_end(1)+hi .AND. l_start(2)+hj <= j .AND. j <= l_end(2)+hj ) THEN output_fields(out_num)%num_elements(sample) = & @@ -1589,16 +1983,22 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & END DO END DO !$OMP END CRITICAL - ELSE IF ( reduced_k_range ) THEN + ELSE IF ( reduced_k_range ) THEN IF (numthreads>1 .AND. phys_window) then DO k=ksr, ker - k1 = k - ksr + 1 + k1 = k - ksr + 1 DO j=js, je DO i=is, ie IF ( mask(i-is+1+hi,j-js+1+hj,k) ) THEN - output_fields(out_num)%buffer(i-hi,j-hj,k1,sample) =& - & output_fields(out_num)%buffer(i-hi,j-hj,k1,sample) +& - & field(i-is+1+hi,j-js+1+hj,k) * weight1 + IF ( pow_value /= 1 ) THEN + output_fields(out_num)%buffer(i-hi,j-hj,k1,sample) =& + & output_fields(out_num)%buffer(i-hi,j-hj,k1,sample) +& + & (field(i-is+1+hi,j-js+1+hj,k) * weight1)**(pow_value) + ELSE + output_fields(out_num)%buffer(i-hi,j-hj,k1,sample) =& + & output_fields(out_num)%buffer(i-hi,j-hj,k1,sample) +& + & field(i-is+1+hi,j-js+1+hj,k) * weight1 + END IF ELSE output_fields(out_num)%buffer(i-hi,j-hj,k1,sample)= missvalue END IF @@ -1608,13 +2008,19 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & ELSE !$OMP CRITICAL DO k=ksr, ker - k1 = k - ksr + 1 + k1 = k - ksr + 1 DO j=js, je DO i=is, ie IF ( mask(i-is+1+hi,j-js+1+hj,k) ) THEN - output_fields(out_num)%buffer(i-hi,j-hj,k1,sample) =& - & output_fields(out_num)%buffer(i-hi,j-hj,k1,sample) +& - & field(i-is+1+hi,j-js+1+hj,k) * weight1 + IF ( pow_value /= 1 ) THEN + output_fields(out_num)%buffer(i-hi,j-hj,k1,sample) =& + & output_fields(out_num)%buffer(i-hi,j-hj,k1,sample) +& + & (field(i-is+1+hi,j-js+1+hj,k) * weight1)**(pow_value) + ELSE + output_fields(out_num)%buffer(i-hi,j-hj,k1,sample) =& + & output_fields(out_num)%buffer(i-hi,j-hj,k1,sample) +& + & field(i-is+1+hi,j-js+1+hj,k) * weight1 + END IF ELSE output_fields(out_num)%buffer(i-hi,j-hj,k1,sample)= missvalue END IF @@ -1628,7 +2034,7 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & CALL update_bounds(out_num, is-hi, ie-hi, js-hj, je-hj, ks, ke) CALL check_out_of_bounds(out_num, diag_field_id, err_msg=err_msg_local) IF ( err_msg_local /= '' ) THEN - IF ( fms_error_handler('diag_manager_mod::send_data_3d', err_msg_local, err_msg) ) THEN + IF ( fms_error_handler('diag_manager_mod::send_data_3d', err_msg_local, err_msg) ) THEN DEALLOCATE(oor_mask) RETURN END IF @@ -1639,9 +2045,15 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & DO j=js, je DO i=is, ie IF ( mask(i-is+1+hi,j-js+1+hj,k) ) THEN - output_fields(out_num)%buffer(i-hi,j-hj,k,sample) =& - & output_fields(out_num)%buffer(i-hi,j-hj,k,sample) +& - & field(i-is+1+hi,j-js+1+hj,k) * weight1 + IF ( pow_value /= 1 ) THEN + output_fields(out_num)%buffer(i-hi,j-hj,k,sample) =& + & output_fields(out_num)%buffer(i-hi,j-hj,k,sample) +& + & (field(i-is+1+hi,j-js+1+hj,k) * weight1)**(pow_value) + ELSE + output_fields(out_num)%buffer(i-hi,j-hj,k,sample) =& + & output_fields(out_num)%buffer(i-hi,j-hj,k,sample) +& + & field(i-is+1+hi,j-js+1+hj,k) * weight1 + END IF ELSE output_fields(out_num)%buffer(i-hi,j-hj,k,sample)= missvalue END IF @@ -1654,16 +2066,22 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & DO j=js, je DO i=is, ie IF ( mask(i-is+1+hi,j-js+1+hj,k) ) THEN - output_fields(out_num)%buffer(i-hi,j-hj,k,sample) =& - & output_fields(out_num)%buffer(i-hi,j-hj,k,sample) +& - & field(i-is+1+hi,j-js+1+hj,k) * weight1 + IF ( pow_value /= 1 ) THEN + output_fields(out_num)%buffer(i-hi,j-hj,k,sample) =& + & output_fields(out_num)%buffer(i-hi,j-hj,k,sample) +& + & (field(i-is+1+hi,j-js+1+hj,k) * weight1)**(pow_value) + ELSE + output_fields(out_num)%buffer(i-hi,j-hj,k,sample) =& + & output_fields(out_num)%buffer(i-hi,j-hj,k,sample) +& + & field(i-is+1+hi,j-js+1+hj,k) * weight1 + END IF ELSE output_fields(out_num)%buffer(i-hi,j-hj,k,sample)= missvalue END IF END DO END DO - END DO -!$OMP END CRITICAL + END DO +!$OMP END CRITICAL END IF END IF !$OMP CRITICAL @@ -1675,7 +2093,7 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & IF ( ANY(mask(f1:f2,f3:f4,ks:ke)) ) output_fields(out_num)%count_0d(sample) =& & output_fields(out_num)%count_0d(sample)+weight1 END IF -!$OMP END CRITICAL +!$OMP END CRITICAL ELSE ! missing value NOT present IF ( (.NOT.ALL(mask(f1:f2,f3:f4,ks:ke)) .AND. mpp_pe() .EQ. mpp_root_pe()).AND.& @@ -1690,34 +2108,44 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & & trim(input_fields(diag_field_id)%module_name), WARNING) input_fields(diag_field_id)%issued_mask_ignore_warning = .TRUE. END IF - IF ( need_compute ) THEN + IF ( need_compute ) THEN IF (numthreads>1 .AND. phys_window) then - DO j = js, je + DO j = js, je DO i = is, ie IF ( l_start(1)+hi <= i .AND. i <= l_end(1)+hi .AND. l_start(2)+hj <= j .AND. j <= l_end(2)+hj ) THEN - i1 = i-l_start(1)-hi+1 + i1 = i-l_start(1)-hi+1 j1 = j-l_start(2)-hj+1 - output_fields(out_num)%buffer(i1,j1,:,sample)= output_fields(out_num)%buffer(i1,j1,:,sample)+ & - & field(i-is+1+hi,j-js+1+hj,l_start(3):l_end(3))*weight1 + IF ( pow_value /= 1 ) THEN + output_fields(out_num)%buffer(i1,j1,:,sample)= output_fields(out_num)%buffer(i1,j1,:,sample)+ & + & (field(i-is+1+hi,j-js+1+hj,l_start(3):l_end(3))*weight1)**(pow_value) + ELSE + output_fields(out_num)%buffer(i1,j1,:,sample)= output_fields(out_num)%buffer(i1,j1,:,sample)+ & + & field(i-is+1+hi,j-js+1+hj,l_start(3):l_end(3))*weight1 + END IF END IF END DO END DO ELSE !$OMP CRITICAL - DO j = js, je + DO j = js, je DO i = is, ie IF ( l_start(1)+hi <= i .AND. i <= l_end(1)+hi .AND. l_start(2)+hj <= j .AND. j <= l_end(2)+hj ) THEN - i1 = i-l_start(1)-hi+1 + i1 = i-l_start(1)-hi+1 j1 = j-l_start(2)-hj+1 - output_fields(out_num)%buffer(i1,j1,:,sample)= output_fields(out_num)%buffer(i1,j1,:,sample)+ & - & field(i-is+1+hi,j-js+1+hj,l_start(3):l_end(3))*weight1 + IF ( pow_value /= 1 ) THEN + output_fields(out_num)%buffer(i1,j1,:,sample)= output_fields(out_num)%buffer(i1,j1,:,sample)+ & + & (field(i-is+1+hi,j-js+1+hj,l_start(3):l_end(3))*weight1)**(pow_value) + ELSE + output_fields(out_num)%buffer(i1,j1,:,sample)= output_fields(out_num)%buffer(i1,j1,:,sample)+ & + & field(i-is+1+hi,j-js+1+hj,l_start(3):l_end(3))*weight1 + END IF END IF END DO END DO !$OMP END CRITICAL END IF !$OMP CRITICAL - DO j = js, je + DO j = js, je DO i = is, ie IF ( l_start(1)+hi <= i .AND. i <= l_end(1)+hi .AND. l_start(2)+hj <= j .AND. j <= l_end(2)+hj ) THEN output_fields(out_num)%num_elements(sample)=& @@ -1727,20 +2155,32 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & END DO END DO !$OMP END CRITICAL - ELSE IF ( reduced_k_range ) THEN + ELSE IF ( reduced_k_range ) THEN IF (numthreads>1 .AND. phys_window) then ksr= l_start(3) ker= l_end(3) - output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,:,sample) =& - & output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,:,sample)& - & + field(f1:f2,f3:f4,ksr:ker)*weight1 - ELSE -!$OMP CRITICAL + IF ( pow_value /= 1 ) THEN + output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,:,sample) =& + & output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,:,sample) +& + & (field(f1:f2,f3:f4,ksr:ker)*weight1)**(pow_value) + ELSE + output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,:,sample) =& + & output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,:,sample) +& + & field(f1:f2,f3:f4,ksr:ker)*weight1 + END IF + ELSE +!$OMP CRITICAL ksr= l_start(3) ker= l_end(3) - output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,:,sample) =& - & output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,:,sample)& - & + field(f1:f2,f3:f4,ksr:ker)*weight1 + IF ( pow_value /= 1 ) THEN + output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,:,sample) =& + & output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,:,sample) +& + & (field(f1:f2,f3:f4,ksr:ker)*weight1)**(pow_value) + ELSE + output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,:,sample) =& + & output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,:,sample) +& + & field(f1:f2,f3:f4,ksr:ker)*weight1 + END IF !$OMP END CRITICAL END IF ELSE @@ -1748,44 +2188,62 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & CALL update_bounds(out_num, is-hi, ie-hi, js-hj, je-hj, ks, ke) CALL check_out_of_bounds(out_num, diag_field_id, err_msg=err_msg_local) IF ( err_msg_local /= '') THEN - IF ( fms_error_handler('diag_manager_mod::send_data_3d', err_msg_local, err_msg) ) THEN + IF ( fms_error_handler('diag_manager_mod::send_data_3d', err_msg_local, err_msg) ) THEN DEALLOCATE(oor_mask) RETURN END IF END IF END IF IF (numthreads>1 .AND. phys_window) then - output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,ks:ke,sample) =& - & output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,ks:ke,sample)& - & + field(f1:f2,f3:f4,ks:ke)*weight1 + IF ( pow_value /= 1 ) THEN + output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,ks:ke,sample) =& + & output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,ks:ke,sample) +& + & (field(f1:f2,f3:f4,ks:ke)*weight1)**(pow_value) + ELSE + output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,ks:ke,sample) =& + & output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,ks:ke,sample) +& + & field(f1:f2,f3:f4,ks:ke)*weight1 + END IF ELSE -!$OMP CRITICAL - output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,ks:ke,sample) =& - & output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,ks:ke,sample)& - & + field(f1:f2,f3:f4,ks:ke)*weight1 -!$OMP END CRITICAL +!$OMP CRITICAL + IF ( pow_value /= 1 ) THEN + output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,ks:ke,sample) =& + & output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,ks:ke,sample) +& + & (field(f1:f2,f3:f4,ks:ke)*weight1)**(pow_value) + ELSE + output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,ks:ke,sample) =& + & output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,ks:ke,sample) +& + & field(f1:f2,f3:f4,ks:ke)*weight1 + END IF +!$OMP END CRITICAL END IF END IF -!$OMP CRITICAL +!$OMP CRITICAL IF ( .NOT.phys_window ) output_fields(out_num)%count_0d(sample) =& & output_fields(out_num)%count_0d(sample) + weight1 -!$OMP END CRITICAL +!$OMP END CRITICAL END IF ELSE ! mask NOT present IF ( missvalue_present ) THEN - IF ( need_compute ) THEN + IF ( need_compute ) THEN if( numthreads>1 .AND. phys_window ) then DO k = l_start(3), l_end(3) - k1 = k - l_start(3) + 1 - DO j = js, je + k1 = k - l_start(3) + 1 + DO j = js, je DO i = is, ie IF ( l_start(1)+hi <= i .AND. i <= l_end(1)+hi .AND. l_start(2)+hj <= j .AND. j <= l_end(2)+hj) THEN - i1 = i-l_start(1)-hi+1 - j1= j-l_start(2)-hj+1 + i1 = i-l_start(1)-hi+1 + j1= j-l_start(2)-hj+1 IF ( field(i-is+1+hi,j-js+1+hj,k) /= missvalue ) THEN - output_fields(out_num)%buffer(i1,j1,k1,sample) =& - & output_fields(out_num)%buffer(i1,j1,k1,sample) +& - & field(i-is+1+hi,j-js+1+hj,k) * weight1 + IF ( pow_value /= 1 ) THEN + output_fields(out_num)%buffer(i1,j1,k1,sample) =& + & output_fields(out_num)%buffer(i1,j1,k1,sample) +& + & (field(i-is+1+hi,j-js+1+hj,k) * weight1)**(pow_value) + ELSE + output_fields(out_num)%buffer(i1,j1,k1,sample) =& + & output_fields(out_num)%buffer(i1,j1,k1,sample) +& + & field(i-is+1+hi,j-js+1+hj,k) * weight1 + END IF ELSE output_fields(out_num)%buffer(i1,j1,k1,sample) = missvalue END IF @@ -1796,16 +2254,22 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & ELSE !$OMP CRITICAL DO k = l_start(3), l_end(3) - k1 = k - l_start(3) + 1 - DO j = js, je + k1 = k - l_start(3) + 1 + DO j = js, je DO i = is, ie IF ( l_start(1)+hi <= i .AND. i <= l_end(1)+hi .AND. l_start(2)+hj <= j .AND. j <= l_end(2)+hj) THEN - i1 = i-l_start(1)-hi+1 - j1= j-l_start(2)-hj+1 + i1 = i-l_start(1)-hi+1 + j1= j-l_start(2)-hj+1 IF ( field(i-is+1+hi,j-js+1+hj,k) /= missvalue ) THEN - output_fields(out_num)%buffer(i1,j1,k1,sample) =& - & output_fields(out_num)%buffer(i1,j1,k1,sample) +& - & field(i-is+1+hi,j-js+1+hj,k) * weight1 + IF ( pow_value /= 1 ) THEN + output_fields(out_num)%buffer(i1,j1,k1,sample) =& + & output_fields(out_num)%buffer(i1,j1,k1,sample) +& + & (field(i-is+1+hi,j-js+1+hj,k) * weight1)**(pow_value) + ELSE + output_fields(out_num)%buffer(i1,j1,k1,sample) =& + & output_fields(out_num)%buffer(i1,j1,k1,sample) +& + & field(i-is+1+hi,j-js+1+hj,k) * weight1 + END IF ELSE output_fields(out_num)%buffer(i1,j1,k1,sample) = missvalue END IF @@ -1816,7 +2280,7 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & !$OMP END CRITICAL END IF !$OMP CRITICAL - DO j = js, je + DO j = js, je DO i = is, ie IF ( l_start(1)+hi <= i .AND. i <= l_end(1)+hi .AND. l_start(2)+hj <= j .AND. j <= l_end(2)+hj) THEN output_fields(out_num)%num_elements(sample) =& @@ -1825,14 +2289,11 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & END DO END DO IF ( .NOT.phys_window ) THEN - !rab if(any(field(l_start(1)+hi:l_end(1)+hi,l_start(2)+hj:l_end(2)+hj,l_start(3):l_end(3)) /= & - !rab & missvalue)) & - !rab & output_fields(out_num)%count_0d = output_fields(out_num)%count_0d + weight1 outer0: DO k = l_start(3), l_end(3) DO j=l_start(2)+hj, l_end(2)+hj DO i=l_start(1)+hi, l_end(1)+hi IF ( field(i,j,k) /= missvalue ) THEN - output_fields(out_num)%count_0d(sample) = output_fields(out_num)%count_0d(sample) + weight1 + output_fields(out_num)%count_0d(sample) = output_fields(out_num)%count_0d(sample) + weight1 EXIT outer0 END IF END DO @@ -1840,7 +2301,7 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & END DO outer0 END IF !$OMP END CRITICAL - ELSE IF ( reduced_k_range ) THEN + ELSE IF ( reduced_k_range ) THEN if( numthreads>1 .AND. phys_window ) then ksr= l_start(3) ker= l_end(3) @@ -1849,9 +2310,15 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & DO j=js, je DO i=is, ie IF ( field(i-is+1+hi,j-js+1+hj,k) /= missvalue ) THEN - output_fields(out_num)%buffer(i-hi,j-hj,k1,sample) =& - & output_fields(out_num)%buffer(i-hi,j-hj,k1,sample) +& - & field(i-is+1+hi,j-js+1+hj,k) * weight1 + IF ( pow_value /= 1 ) THEN + output_fields(out_num)%buffer(i-hi,j-hj,k1,sample) =& + & output_fields(out_num)%buffer(i-hi,j-hj,k1,sample) +& + & (field(i-is+1+hi,j-js+1+hj,k) * weight1)**(pow_value) + ELSE + output_fields(out_num)%buffer(i-hi,j-hj,k1,sample) =& + & output_fields(out_num)%buffer(i-hi,j-hj,k1,sample) +& + & field(i-is+1+hi,j-js+1+hj,k) * weight1 + END IF ELSE output_fields(out_num)%buffer(i-hi,j-hj,k1,sample) = missvalue END IF @@ -1867,9 +2334,15 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & DO j=js, je DO i=is, ie IF ( field(i-is+1+hi,j-js+1+hj,k) /= missvalue ) THEN - output_fields(out_num)%buffer(i-hi,j-hj,k1,sample) =& - & output_fields(out_num)%buffer(i-hi,j-hj,k1,sample) +& - & field(i-is+1+hi,j-js+1+hj,k) * weight1 + IF ( pow_value /= 1 ) THEN + output_fields(out_num)%buffer(i-hi,j-hj,k1,sample) =& + & output_fields(out_num)%buffer(i-hi,j-hj,k1,sample) +& + & (field(i-is+1+hi,j-js+1+hj,k) * weight1)**(pow_value) + ELSE + output_fields(out_num)%buffer(i-hi,j-hj,k1,sample) =& + & output_fields(out_num)%buffer(i-hi,j-hj,k1,sample) +& + & field(i-is+1+hi,j-js+1+hj,k) * weight1 + END IF ELSE output_fields(out_num)%buffer(i-hi,j-hj,k1,sample) = missvalue END IF @@ -1878,16 +2351,13 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & END DO !$OMP END CRITICAL END IF - !rab - !rab if(any(field(f1:f2,f3:f4,ks:ke) /= missvalue)) & - !rab & output_fields(out_num)%count_0d = output_fields(out_num)%count_0d + weight1 !$OMP CRITICAL outer3: DO k = ksr, ker k1=k-ksr+1 - DO j=f3, f4 - DO i=f1, f2 + DO j=f3, f4 + DO i=f1, f2 IF ( field(i,j,k) /= missvalue ) THEN - output_fields(out_num)%count_0d = output_fields(out_num)%count_0d + weight1 + output_fields(out_num)%count_0d = output_fields(out_num)%count_0d + weight1 EXIT outer3 END IF END DO @@ -1899,7 +2369,7 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & CALL update_bounds(out_num, is-hi, ie-hi, js-hj, je-hj, ks, ke) CALL check_out_of_bounds(out_num, diag_field_id, err_msg=err_msg_local) IF ( err_msg_local /= '' ) THEN - IF ( fms_error_handler('diag_manager_mod::send_data_3d', err_msg_local, err_msg) ) THEN + IF ( fms_error_handler('diag_manager_mod::send_data_3d', err_msg_local, err_msg) ) THEN DEALLOCATE(oor_mask) RETURN END IF @@ -1910,9 +2380,15 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & DO j=js, je DO i=is, ie IF ( field(i-is+1+hi,j-js+1+hj,k) /= missvalue ) THEN - output_fields(out_num)%buffer(i-hi,j-hj,k,sample) =& - & output_fields(out_num)%buffer(i-hi,j-hj,k,sample) +& - & field(i-is+1+hi,j-js+1+hj,k) * weight1 + IF ( pow_value /= 1 ) THEN + output_fields(out_num)%buffer(i-hi,j-hj,k,sample) =& + & output_fields(out_num)%buffer(i-hi,j-hj,k,sample) +& + & (field(i-is+1+hi,j-js+1+hj,k) * weight1)**(pow_value) + ELSE + output_fields(out_num)%buffer(i-hi,j-hj,k,sample) =& + & output_fields(out_num)%buffer(i-hi,j-hj,k,sample) +& + & field(i-is+1+hi,j-js+1+hj,k) * weight1 + END IF ELSE output_fields(out_num)%buffer(i-hi,j-hj,k,sample) = missvalue END IF @@ -1925,9 +2401,15 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & DO j=js, je DO i=is, ie IF ( field(i-is+1+hi,j-js+1+hj,k) /= missvalue ) THEN - output_fields(out_num)%buffer(i-hi,j-hj,k,sample) =& - & output_fields(out_num)%buffer(i-hi,j-hj,k,sample) +& - & field(i-is+1+hi,j-js+1+hj,k) * weight1 + IF ( pow_value /= 1 ) THEN + output_fields(out_num)%buffer(i-hi,j-hj,k,sample) =& + & output_fields(out_num)%buffer(i-hi,j-hj,k,sample) +& + & (field(i-is+1+hi,j-js+1+hj,k) * weight1)**(pow_value) + ELSE + output_fields(out_num)%buffer(i-hi,j-hj,k,sample) =& + & output_fields(out_num)%buffer(i-hi,j-hj,k,sample) +& + & field(i-is+1+hi,j-js+1+hj,k) * weight1 + END IF ELSE output_fields(out_num)%buffer(i-hi,j-hj,k,sample) = missvalue END IF @@ -1937,11 +2419,11 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & !$OMP END CRITICAL END IF !$OMP CRITICAL - outer1: DO k=ks, ke - DO j=f3, f4 - DO i=f1, f2 + outer1: DO k=ks, ke + DO j=f3, f4 + DO i=f1, f2 IF ( field(i,j,k) /= missvalue ) THEN - output_fields(out_num)%count_0d(sample) = output_fields(out_num)%count_0d(sample) + weight1 + output_fields(out_num)%count_0d(sample) = output_fields(out_num)%count_0d(sample) + weight1 EXIT outer1 END IF END DO @@ -1952,25 +2434,35 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & ELSE ! no missing value defined, No mask IF ( need_compute ) THEN IF( numthreads > 1 .AND. phys_window ) then - DO j = js, je - DO i = is, ie + DO j = js, je + DO i = is, ie IF ( l_start(1)+hi <= i .AND. i <= l_end(1)+hi .AND. l_start(2)+hj <= j .AND. j <= l_end(2)+hj ) THEN - i1 = i-l_start(1)-hi+1 + i1 = i-l_start(1)-hi+1 j1= j-l_start(2)-hj+1 - output_fields(out_num)%buffer(i1,j1,:,sample)= output_fields(out_num)%buffer(i1,j1,:,sample) +& - & field(i-is+1+hi,j-js+1+hj,l_start(3):l_end(3))*weight1 + IF ( pow_value /= 1 ) THEN + output_fields(out_num)%buffer(i1,j1,:,sample)= output_fields(out_num)%buffer(i1,j1,:,sample) +& + & (field(i-is+1+hi,j-js+1+hj,l_start(3):l_end(3))*weight1)**(pow_value) + ELSE + output_fields(out_num)%buffer(i1,j1,:,sample)= output_fields(out_num)%buffer(i1,j1,:,sample) +& + & field(i-is+1+hi,j-js+1+hj,l_start(3):l_end(3))*weight1 + END IF END IF END DO END DO ELSE !$OMP CRITICAL - DO j = js, je - DO i = is, ie + DO j = js, je + DO i = is, ie IF ( l_start(1)+hi <= i .AND. i <= l_end(1)+hi .AND. l_start(2)+hj <= j .AND. j <= l_end(2)+hj ) THEN - i1 = i-l_start(1)-hi+1 + i1 = i-l_start(1)-hi+1 j1= j-l_start(2)-hj+1 - output_fields(out_num)%buffer(i1,j1,:,sample)= output_fields(out_num)%buffer(i1,j1,:,sample) +& - & field(i-is+1+hi,j-js+1+hj,l_start(3):l_end(3))*weight1 + IF ( pow_value /= 1 ) THEN + output_fields(out_num)%buffer(i1,j1,:,sample)= output_fields(out_num)%buffer(i1,j1,:,sample) +& + & (field(i-is+1+hi,j-js+1+hj,l_start(3):l_end(3))*weight1)**(pow_value) + ELSE + output_fields(out_num)%buffer(i1,j1,:,sample)= output_fields(out_num)%buffer(i1,j1,:,sample) +& + & field(i-is+1+hi,j-js+1+hj,l_start(3):l_end(3))*weight1 + END IF END IF END DO END DO @@ -1978,8 +2470,8 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & END IF !$OMP CRITICAL - DO j = js, je - DO i = is, ie + DO j = js, je + DO i = is, ie IF ( l_start(1)+hi <= i .AND. i <= l_end(1)+hi .AND. l_start(2)+hj <= j .AND. j <= l_end(2)+hj ) THEN output_fields(out_num)%num_elements(sample) =& & output_fields(out_num)%num_elements(sample)+l_end(3)-l_start(3)+1 @@ -1987,41 +2479,65 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & END DO END DO !$OMP END CRITICAL - ! Accumulate time average - ELSE IF ( reduced_k_range ) THEN + ! Accumulate time average + ELSE IF ( reduced_k_range ) THEN ksr= l_start(3) ker= l_end(3) IF( numthreads > 1 .AND. phys_window ) then - output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,:,sample) =& - & output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,:,sample) + & - & field(f1:f2,f3:f4,ksr:ker)*weight1 + IF ( pow_value /= 1 ) THEN + output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,:,sample) =& + & output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,:,sample) + & + & (field(f1:f2,f3:f4,ksr:ker)*weight1)**(pow_value) + ELSE + output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,:,sample) =& + & output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,:,sample) + & + & field(f1:f2,f3:f4,ksr:ker)*weight1 + END IF ELSE !$OMP CRITICAL - output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,:,sample) =& - & output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,:,sample) + & - & field(f1:f2,f3:f4,ksr:ker)*weight1 + IF ( pow_value /= 1 ) THEN + output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,:,sample) =& + & output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,:,sample) + & + & (field(f1:f2,f3:f4,ksr:ker)*weight1)**(pow_value) + ELSE + output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,:,sample) =& + & output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,:,sample) + & + & field(f1:f2,f3:f4,ksr:ker)*weight1 + END IF !$OMP END CRITICAL END IF - ELSE + ELSE IF ( debug_diag_manager ) THEN CALL update_bounds(out_num, is-hi, ie-hi, js-hj, je-hj, ks, ke) CALL check_out_of_bounds(out_num, diag_field_id, err_msg=err_msg_local) IF ( err_msg_local /= '' ) THEN - IF ( fms_error_handler('diag_manager_mod::send_data_3d', err_msg_local, err_msg) ) THEN + IF ( fms_error_handler('diag_manager_mod::send_data_3d', err_msg_local, err_msg) ) THEN DEALLOCATE(oor_mask) RETURN END IF END IF END IF IF( numthreads > 1 .AND. phys_window ) then - output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,ks:ke,sample) =& - & output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,ks:ke,sample) +& - & field(f1:f2,f3:f4,ks:ke)*weight1 + IF ( pow_value /= 1 ) THEN + output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,ks:ke,sample) =& + & output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,ks:ke,sample) +& + & (field(f1:f2,f3:f4,ks:ke)*weight1)**(pow_value) + ELSE + output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,ks:ke,sample) =& + & output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,ks:ke,sample) +& + & field(f1:f2,f3:f4,ks:ke)*weight1 + END IF ELSE !$OMP CRITICAL - output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,ks:ke,sample) =& - & output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,ks:ke,sample) +& - & field(f1:f2,f3:f4,ks:ke)*weight1 + IF ( pow_value /= 1 ) THEN + output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,ks:ke,sample) =& + & output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,ks:ke,sample) +& + & (field(f1:f2,f3:f4,ks:ke)*weight1)**(pow_value) + ELSE + output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,ks:ke,sample) =& + & output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,ks:ke,sample) +& + & field(f1:f2,f3:f4,ks:ke)*weight1 + END IF !$OMP END CRITICAL END IF END IF @@ -2049,7 +2565,7 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & DO j = js, je DO i = is, ie IF ( l_start(1)+hi <= i .AND. i <= l_end(1)+hi .AND. l_start(2)+hj <= j .AND. j <= l_end(2)+hj ) THEN - i1 = i-l_start(1)-hi+1 + i1 = i-l_start(1)-hi+1 j1= j-l_start(2)-hj+1 IF ( mask(i-is+1+hi,j-js+1+hj,k) .AND.& & field(i-is+1+hi,j-js+1+hj,k)>output_fields(out_num)%buffer(i1,j1,k1,sample)) THEN @@ -2059,8 +2575,8 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & END DO END DO END DO - ! Maximum time value with masking - ELSE IF ( reduced_k_range ) THEN + ! Maximum time value with masking + ELSE IF ( reduced_k_range ) THEN ksr = l_start(3) ker = l_end(3) WHERE ( mask(f1:f2,f3:f4,ksr:ker) .AND. & @@ -2071,7 +2587,7 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & CALL update_bounds(out_num, is-hi, ie-hi, js-hj, je-hj, ks, ke) CALL check_out_of_bounds(out_num, diag_field_id, err_msg=err_msg_local) IF ( err_msg_local /= '' ) THEN - IF ( fms_error_handler('diag_manager_mod::send_data_3d', err_msg_local, err_msg) ) THEN + IF ( fms_error_handler('diag_manager_mod::send_data_3d', err_msg_local, err_msg) ) THEN DEALLOCATE(oor_mask) RETURN END IF @@ -2088,7 +2604,7 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & DO j = js, je DO i = is, ie IF(l_start(1)+hi <= i .AND. i <= l_end(1)+hi .AND. l_start(2)+hj <= j .AND. j <= l_end(2)+hj ) THEN - i1 = i-l_start(1)-hi+1 + i1 = i-l_start(1)-hi+1 j1 = j-l_start(2)-hj+1 IF ( field(i-is+1+hi,j-js+1+hj,k) > output_fields(out_num)%buffer(i1,j1,k1,sample) ) THEN output_fields(out_num)%buffer(i1,j1,k1,sample) = field(i-is+1+hi,j-js+1+hj,k) @@ -2097,8 +2613,8 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & END DO END DO END DO - ! Maximum time value - ELSE IF ( reduced_k_range ) THEN + ! Maximum time value + ELSE IF ( reduced_k_range ) THEN ksr = l_start(3) ker = l_end(3) WHERE ( field(f1:f2,f3:f4,ksr:ker) > output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,:,sample) ) & @@ -2108,7 +2624,7 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & CALL update_bounds(out_num, is-hi, ie-hi, js-hj, je-hj, ks, ke) CALL check_out_of_bounds(out_num, diag_field_id, err_msg=err_msg_local) IF ( err_msg_local /= '' ) THEN - IF ( fms_error_handler('diag_manager_mod::send_data_3d', err_msg_local, err_msg) ) THEN + IF ( fms_error_handler('diag_manager_mod::send_data_3d', err_msg_local, err_msg) ) THEN DEALLOCATE(oor_mask) RETURN END IF @@ -2127,7 +2643,7 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & DO j = js, je DO i = is, ie IF ( l_start(1)+hi <= i .AND. i <= l_end(1)+hi .AND. l_start(2)+hj <= j .AND. j <= l_end(2)+hj ) THEN - i1 = i-l_start(1)-hi+1 + i1 = i-l_start(1)-hi+1 j1 = j-l_start(2)-hj+1 IF ( mask(i-is+1+hi,j-js+1+hj,k) .AND.& & field(i-is+1+hi,j-js+1+hj,k) < output_fields(out_num)%buffer(i1,j1,k1,sample) ) THEN @@ -2137,8 +2653,8 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & END DO END DO END DO - ! Minimum time value with masking - ELSE IF ( reduced_k_range ) THEN + ! Minimum time value with masking + ELSE IF ( reduced_k_range ) THEN ksr= l_start(3) ker= l_end(3) WHERE ( mask(f1:f2,f3:f4,ksr:ker) .AND.& @@ -2149,7 +2665,7 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & CALL update_bounds(out_num, is-hi, ie-hi, js-hj, je-hj, ks, ke) CALL check_out_of_bounds(out_num, diag_field_id, err_msg=err_msg_local) IF ( err_msg_local /= '' ) THEN - IF ( fms_error_handler('diag_manager_mod::send_data_3d', err_msg_local, err_msg) ) THEN + IF ( fms_error_handler('diag_manager_mod::send_data_3d', err_msg_local, err_msg) ) THEN DEALLOCATE(oor_mask) RETURN END IF @@ -2157,7 +2673,7 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & END IF WHERE ( mask(f1:f2,f3:f4,ks:ke) .AND.& & field(f1:f2,f3:f4,ks:ke) < output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,ks:ke,sample) ) & - & output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,ks:ke,sample) = field(f1:f2,f3:f4,ks:ke) + & output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,ks:ke,sample) = field(f1:f2,f3:f4,ks:ke) END IF ELSE IF ( need_compute ) THEN @@ -2166,7 +2682,7 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & DO j = js, je DO i = is, ie IF ( l_start(1)+hi <=i.AND.i<=l_end(1)+hi.AND.l_start(2)+hj<=j.AND.j<=l_end(2)+hj) THEN - i1 = i-l_start(1)-hi+1 + i1 = i-l_start(1)-hi+1 j1= j-l_start(2)-hj+1 IF ( field(i-is+1+hi,j-js+1+hj,k) < output_fields(out_num)%buffer(i1,j1,k1,sample) ) THEN output_fields(out_num)%buffer(i1,j1,k1,sample) = field(i-is+1+hi,j-js+1+hj,k) @@ -2175,8 +2691,8 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & END DO END DO END DO - ! Minimum time value - ELSE IF ( reduced_k_range ) THEN + ! Minimum time value + ELSE IF ( reduced_k_range ) THEN ksr= l_start(3) ker= l_end(3) WHERE ( field(f1:f2,f3:f4,ksr:ker) < output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,:,sample) ) & @@ -2186,7 +2702,7 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & CALL update_bounds(out_num, is-hi, ie-hi, js-hj, je-hj, ks, ke) CALL check_out_of_bounds(out_num, diag_field_id, err_msg=err_msg_local) IF ( err_msg_local /= '' ) THEN - IF ( fms_error_handler('diag_manager_mod::send_data_3d', err_msg_local, err_msg) ) THEN + IF ( fms_error_handler('diag_manager_mod::send_data_3d', err_msg_local, err_msg) ) THEN DEALLOCATE(oor_mask) RETURN END IF @@ -2200,17 +2716,17 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & ELSE ! ( not average, not min, max) output_fields(out_num)%count_0d(sample) = 1 IF ( need_compute ) THEN - DO j = js, je - DO i = is, ie + DO j = js, je + DO i = is, ie IF ( l_start(1)+hi <= i .AND. i <= l_end(1)+hi .AND. l_start(2)+hj <= j .AND. j <= l_end(2)+hj ) THEN - i1 = i-l_start(1)-hi+1 + i1 = i-l_start(1)-hi+1 j1 = j-l_start(2)-hj+1 output_fields(out_num)%buffer(i1,j1,:,sample) = field(i-is+1+hi,j-js+1+hj,l_start(3):l_end(3)) END IF END DO END DO ! instantaneous output - ELSE IF ( reduced_k_range ) THEN + ELSE IF ( reduced_k_range ) THEN ksr = l_start(3) ker = l_end(3) output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,:,sample) = field(f1:f2,f3:f4,ksr:ker) @@ -2219,7 +2735,7 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & CALL update_bounds(out_num, is-hi, ie-hi, js-hj, je-hj, ks, ke) CALL check_out_of_bounds(out_num, diag_field_id, err_msg=err_msg_local) IF ( err_msg_local /= '' ) THEN - IF ( fms_error_handler('diag_manager_mod::send_data_3d', err_msg_local, err_msg) ) THEN + IF ( fms_error_handler('diag_manager_mod::send_data_3d', err_msg_local, err_msg) ) THEN DEALLOCATE(oor_mask) RETURN END IF @@ -2227,26 +2743,26 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & END IF output_fields(out_num)%buffer(is-hi:ie-hi,js-hj:je-hj,ks:ke,sample) = field(f1:f2,f3:f4,ks:ke) END IF - + IF ( PRESENT(mask) .AND. missvalue_present ) THEN IF ( need_compute ) THEN DO k = l_start(3), l_end(3) k1 = k - l_start(3) + 1 DO j = js, je - DO i = is, ie + DO i = is, ie IF ( l_start(1)+hi <= i .AND. i <= l_end(1)+hi .AND. l_start(2)+hj <= j .AND. j <= l_end(2)+hj ) THEN - i1 = i-l_start(1)-hi+1 - j1 = j-l_start(2)-hj+1 + i1 = i-l_start(1)-hi+1 + j1 = j-l_start(2)-hj+1 IF ( .NOT.mask(i-is+1+hi,j-js+1+hj,k) )& - & output_fields(out_num)%buffer(i1,j1,k1,sample) = missvalue + & output_fields(out_num)%buffer(i1,j1,k1,sample) = missvalue END IF END DO END DO END DO - ELSE IF ( reduced_k_range ) THEN + ELSE IF ( reduced_k_range ) THEN ksr= l_start(3) ker= l_end(3) - DO k=ksr, ker + DO k=ksr, ker k1= k - ksr + 1 DO j=js, je DO i=is, ie @@ -2271,33 +2787,33 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & IF ( output_fields(out_num)%static .AND. .NOT.need_compute .AND. debug_diag_manager ) THEN CALL check_bounds_are_exact_static(out_num, diag_field_id, err_msg=err_msg_local) IF ( err_msg_local /= '' ) THEN - IF ( fms_error_handler('diag_manager_mod::send_data_3d', err_msg_local, err_msg)) THEN + IF ( fms_error_handler('diag_manager_mod::send_data_3d', err_msg_local, err_msg)) THEN DEALLOCATE(oor_mask) RETURN END IF END IF END IF - - ! If rmask and missing value present, then insert missing value + + ! If rmask and missing value present, then insert missing value IF ( PRESENT(rmask) .AND. missvalue_present ) THEN IF ( need_compute ) THEN DO k = l_start(3), l_end(3) k1 = k - l_start(3) + 1 - DO j = js, je - DO i = is, ie + DO j = js, je + DO i = is, ie IF ( l_start(1)+hi <= i .AND. i <= l_end(1)+hi .AND. l_start(2)+hj <= j .AND. j <= l_end(2)+hj ) THEN - i1 = i-l_start(1)-hi+1 + i1 = i-l_start(1)-hi+1 j1 = j-l_start(2)-hj+1 IF ( rmask(i-is+1+hi,j-js+1+hj,k) < 0.5 ) & - & output_fields(out_num)%buffer(i1,j1,k1,sample) = missvalue + & output_fields(out_num)%buffer(i1,j1,k1,sample) = missvalue END IF END DO END DO END DO - ELSE IF ( reduced_k_range ) THEN + ELSE IF ( reduced_k_range ) THEN ksr= l_start(3) ker= l_end(3) - DO k= ksr, ker + DO k= ksr, ker k1 = k - ksr + 1 DO j=js, je DO i=is, ie @@ -2319,7 +2835,7 @@ LOGICAL FUNCTION send_data_3d(diag_field_id, field, time, is_in, js_in, ks_in, & END IF END DO num_out_fields - + DEALLOCATE(oor_mask) END FUNCTION send_data_3d ! @@ -2331,7 +2847,7 @@ END FUNCTION send_data_3d ! ! LOGICAL FUNCTION send_tile_averaged_data2d ( id, field, area, time, mask ) - INTEGER, INTENT(in) :: id ! id od the diagnostic field + INTEGER, INTENT(in) :: id ! id od the diagnostic field REAL, INTENT(in) :: field(:,:,:) ! field to average and send REAL, INTENT(in) :: area (:,:,:) ! area of tiles (== averaging weights), arbitrary units TYPE(time_type), INTENT(in) :: time ! current time @@ -2389,10 +2905,10 @@ END FUNCTION send_tile_averaged_data3d ! (lon, lat) result of averaging SUBROUTINE average_tiles(diag_field_id, x, area, mask, out) INTEGER, INTENT(in) :: diag_field_id - REAL, DIMENSION(:,:,:), INTENT(in) :: x + REAL, DIMENSION(:,:,:), INTENT(in) :: x REAL, DIMENSION(:,:,:), INTENT(in) :: area LOGICAL, DIMENSION(:,:,:), INTENT(in) :: mask - REAL, DIMENSION(:,:), INTENT(out) :: out + REAL, DIMENSION(:,:), INTENT(out) :: out INTEGER :: it ! iterator over tile number REAL, DIMENSION(SIZE(x,1),SIZE(x,2)) :: s ! area accumulator @@ -2401,22 +2917,22 @@ SUBROUTINE average_tiles(diag_field_id, x, area, mask, out) ! Initialize local_missing_value IF ( input_fields(diag_field_id)%missing_value_present ) THEN local_missing_value = input_fields(diag_field_id)%missing_value - ELSE + ELSE local_missing_value = 0.0 END IF - + ! Initialize s and out to zero. s(:,:) = 0.0 out(:,:) = 0.0 - + DO it = 1, SIZE(area,3) - WHERE ( mask(:,:,it) ) + WHERE ( mask(:,:,it) ) out(:,:) = out(:,:) + x(:,:,it)*area(:,:,it) s(:,:) = s(:,:) + area(:,:,it) END WHERE END DO - WHERE ( s(:,:) > 0 ) + WHERE ( s(:,:) > 0 ) out(:,:) = out(:,:)/s(:,:) ELSEWHERE out(:,:) = local_missing_value @@ -2432,14 +2948,14 @@ INTEGER FUNCTION writing_field(out_num, at_diag_end, error_string, time) TYPE(time_type) :: middle_time LOGICAL :: time_max, time_min, reduced_k_range, missvalue_present - LOGICAL :: average, need_compute, phys_window + LOGICAL :: average, time_rms, need_compute, phys_window INTEGER :: in_num, file_num, freq, units INTEGER :: b1,b2,b3,b4 ! size of buffer along x,y,z,and diurnal axes INTEGER :: i, j, k, m REAL :: missvalue, num writing_field = 0 - + need_compute = output_fields(out_num)%need_compute in_num = output_fields(out_num)%input_field @@ -2451,9 +2967,12 @@ INTEGER FUNCTION writing_field(out_num, at_diag_end, error_string, time) phys_window = output_fields(out_num)%phys_window ! Is this output field being time averaged? average = output_fields(out_num)%time_average + ! Are we taking the rms of the field? + ! If so, then average is also .TRUE. + time_rms = output_fields(out_num)%time_rms ! Looking for max and min value of this field over the sampling interval? time_max = output_fields(out_num)%time_max - time_min = output_fields(out_num)%time_min + time_min = output_fields(out_num)%time_min file_num = output_fields(out_num)%output_file freq = files(file_num)%output_freq units = files(file_num)%output_units @@ -2461,17 +2980,19 @@ INTEGER FUNCTION writing_field(out_num, at_diag_end, error_string, time) ! If average get size: Average intervals are last_output, next_output IF ( average ) THEN b1=SIZE(output_fields(out_num)%buffer,1) - b2=SIZE(output_fields(out_num)%buffer,2) + b2=SIZE(output_fields(out_num)%buffer,2) b3=SIZE(output_fields(out_num)%buffer,3) b4=SIZE(output_fields(out_num)%buffer,4) - IF ( input_fields(in_num)%mask_variant ) THEN - DO m=1, b4 + IF ( input_fields(in_num)%mask_variant ) THEN + DO m=1, b4 DO k=1, b3 DO j=1, b2 - DO i=1, b1 + DO i=1, b1 IF ( output_fields(out_num)%counter(i,j,k,m) > 0. )THEN output_fields(out_num)%buffer(i,j,k,m) = & & output_fields(out_num)%buffer(i,j,k,m)/output_fields(out_num)%counter(i,j,k,m) + IF ( time_rms ) output_fields(out_num)%buffer(i,j,k,m) = & + SQRT(output_fields(out_num)%buffer(i,j,k,m)) ELSE output_fields(out_num)%buffer(i,j,k,m) = missvalue END IF @@ -2495,13 +3016,18 @@ INTEGER FUNCTION writing_field(out_num, at_diag_end, error_string, time) DO k=1, b3 DO j=1, b2 DO i=1, b1 - IF ( output_fields(out_num)%buffer(i,j,k,m) /= missvalue ) & - & output_fields(out_num)%buffer(i,j,k,m) = output_fields(out_num)%buffer(i,j,k,m)/num + IF ( output_fields(out_num)%buffer(i,j,k,m) /= missvalue ) THEN + output_fields(out_num)%buffer(i,j,k,m) = output_fields(out_num)%buffer(i,j,k,m)/num + IF ( time_rms ) output_fields(out_num)%buffer(i,j,k,m) =& + & SQRT(output_fields(out_num)%buffer(i,j,k,m)) + END IF END DO END DO END DO ELSE output_fields(out_num)%buffer(:,:,:,m) = output_fields(out_num)%buffer(:,:,:,m)/num + IF ( time_rms ) output_fields(out_num)%buffer(:,:,:,m) =& + & SQRT(output_fields(out_num)%buffer(:,:,:,m)) END IF ELSE IF ( .NOT. at_diag_end ) THEN IF ( missvalue_present ) THEN @@ -2518,7 +3044,7 @@ INTEGER FUNCTION writing_field(out_num, at_diag_end, error_string, time) END IF ! mask_variant ELSE IF ( time_min .OR. time_max ) THEN IF ( missvalue_present ) THEN - WHERE ( ABS(output_fields(out_num)%buffer) == MIN_VALUE ) + WHERE ( ABS(output_fields(out_num)%buffer) == MIN_VALUE ) output_fields(out_num)%buffer = missvalue END WHERE END IF ! if missvalue is NOT present buffer retains max_value or min_value @@ -2550,7 +3076,7 @@ INTEGER FUNCTION writing_field(out_num, at_diag_end, error_string, time) END IF output_fields(out_num)%count_0d(:) = 0.0 output_fields(out_num)%num_elements(:) = 0 - IF ( time_max ) THEN + IF ( time_max ) THEN output_fields(out_num)%buffer = MAX_VALUE ELSE IF ( time_min ) THEN output_fields(out_num)%buffer = MIN_VALUE @@ -2573,7 +3099,7 @@ END SUBROUTINE diag_manager_set_time_end SUBROUTINE diag_send_complete(time_step, err_msg) TYPE (time_type), INTENT(in) :: time_step character(len=*), INTENT(out), optional :: err_msg - + type(time_type) :: next_time, time integer :: file, j, out_num, in_num, freq, status logical :: local_output, need_compute @@ -2593,7 +3119,7 @@ SUBROUTINE diag_send_complete(time_step, err_msg) out_num = files(file)%fields(j) !this is position of output_field in array output_fields in_num = output_fields(out_num)%input_field - IF ( input_fields(in_num)%numthreads == 1 ) CYCLE + IF ( (input_fields(in_num)%numthreads == 1) .AND. (input_fields(in_num)%active_omp_level.LE.1) ) CYCLE IF ( output_fields(out_num)%static .OR. freq == END_OF_RUN ) CYCLE time = input_fields(in_num)%time IF ( time >= time_end ) CYCLE @@ -2609,7 +3135,7 @@ SUBROUTINE diag_send_complete(time_step, err_msg) IF ( next_time > output_fields(out_num)%next_output ) THEN ! A non-static field that has skipped a time level is an error IF ( next_time > output_fields(out_num)%next_next_output .AND. freq > 0 ) THEN - IF ( mpp_pe() .EQ. mpp_root_pe() ) THEN + IF ( mpp_pe() .EQ. mpp_root_pe() ) THEN WRITE (error_string,'(a,"/",a)')& & TRIM(input_fields(in_num)%module_name), & & TRIM(output_fields(out_num)%output_name) @@ -2654,7 +3180,7 @@ SUBROUTINE diag_manager_end(time) CALL mpp_close (diag_log_unit) END IF DO file = 1, num_files - CALL closing_file(file, time) + CALL closing_file(file, time) END DO END SUBROUTINE diag_manager_end ! @@ -2694,16 +3220,16 @@ SUBROUTINE closing_file(file, time) ! skip all PEs not participating in outputting this field IF ( local_output .AND. (.NOT. need_compute) ) CYCLE - ! skip fields that were not registered or non-static + ! skip fields that were not registered or non-static input_num = output_fields(i)%input_field IF ( input_fields(input_num)%static ) CYCLE - IF ( .NOT.input_fields(input_num)%register ) CYCLE + IF ( .NOT.input_fields(input_num)%register ) CYCLE freq = files(file)%output_freq IF ( freq /= END_OF_RUN .AND. files(file)%file_unit < 0 & & .AND. ALL(output_fields(i)%num_elements(:) == 0)& & .AND. ALL(output_fields(i)%count_0d(:) == 0) ) CYCLE ! Is it time to output for this field; CAREFUL ABOUT >= vs > HERE - ! For end should be >= because no more data is coming + ! For end should be >= because no more data is coming IF ( time >= output_fields(i)%next_output .OR. freq == END_OF_RUN ) THEN IF ( time >= output_fields(i)%next_next_output .AND. freq > 0 ) THEN WRITE (message,'(a,"/",a)') TRIM(input_fields(input_num)%module_name), & @@ -2712,7 +3238,7 @@ SUBROUTINE closing_file(file, time) ! / skip one time ! level, maybe send_data never called ! - IF ( mpp_pe() .EQ. mpp_root_pe() ) & + IF ( mpp_pe() .EQ. mpp_root_pe() ) & & CALL error_mesg('diag_manager_mod::closing_file', 'module/output_field ' //& & TRIM(message)//', skip one time level, maybe send_data never called', WARNING) ELSE @@ -2727,7 +3253,7 @@ SUBROUTINE closing_file(file, time) & TRIM(output_fields(i)%output_name)//' NOT available,'//& & ' check if output interval > runlength. Netcdf fill_values are written', NOTE) output_fields(i)%buffer = FILL_VALUE - CALL diag_data_out(file, i, output_fields(i)%buffer, time, .TRUE.) + CALL diag_data_out(file, i, output_fields(i)%buffer, time, .TRUE.) END IF END DO ! Now it's time to output static fields @@ -2754,9 +3280,11 @@ END SUBROUTINE closing_file ! Open and read diag_table. Select fields and files for diagnostic output. ! ! + ! Model time diag_manager initialized ! - SUBROUTINE diag_manager_init(diag_model_subset, err_msg) + SUBROUTINE diag_manager_init(diag_model_subset, time_init, err_msg) INTEGER, OPTIONAL, INTENT(IN) :: diag_model_subset + INTEGER, DIMENSION(6), OPTIONAL, INTENT(IN) :: time_init CHARACTER(len=*), INTENT(out), OPTIONAL :: err_msg CHARACTER(len=*), PARAMETER :: SEP = '|' @@ -2776,7 +3304,8 @@ SUBROUTINE diag_manager_init(diag_model_subset, err_msg) NAMELIST /diag_manager_nml/ append_pelist_name, mix_snapshot_average_fields, max_output_fields, & & max_input_fields, max_axes, do_diag_field_log, write_bytes_in_file, debug_diag_manager,& & max_num_axis_sets, max_files, use_cmor, issue_oor_warnings,& - & oor_warnings_fatal, max_out_per_in_field, conserve_water + & oor_warnings_fatal, max_out_per_in_field, conserve_water, region_out_use_alt_value, max_field_attributes,& + & max_file_attributes, prepend_date ! If the module was already initialized do nothing IF ( module_is_initialized ) RETURN @@ -2799,7 +3328,7 @@ SUBROUTINE diag_manager_init(diag_model_subset, err_msg) stdout_unit = stdout() ! version number to logfile - call write_version_number() + CALL write_version_number(version, tagname) Time_zero = set_time(0,0) !--- initialize time_end to time_zero @@ -2826,7 +3355,7 @@ SUBROUTINE diag_manager_init(diag_model_subset, err_msg) END IF #endif ! Check the status of reading the diag_manager_nml - + IF ( check_nml_error(IOSTAT=mystat, NML_NAME='DIAG_MANAGER_NML') < 0 ) THEN IF ( mpp_pe() == mpp_root_pe() ) THEN CALL error_mesg('diag_manager_mod::diag_manager_init', 'DIAG_MANAGER_NML not found in input.nml. Using defaults.',& @@ -2834,12 +3363,12 @@ SUBROUTINE diag_manager_init(diag_model_subset, err_msg) END IF END IF - IF ( mpp_pe() == mpp_root_pe() ) THEN + IF ( mpp_pe() == mpp_root_pe() ) THEN WRITE (stdlog_unit, diag_manager_nml) END IF ! Issue note about using the CMOR missing value. - IF ( use_cmor ) THEN + IF ( use_cmor ) THEN err_msg_local = '' WRITE (err_msg_local,'(ES8.1E2)') CMOR_MISSING_VALUE CALL error_mesg('diag_manager_mod::diag_manager_init', 'Using CMOR missing value ('//TRIM(err_msg_local)//').', NOTE) @@ -2856,7 +3385,7 @@ SUBROUTINE diag_manager_init(diag_model_subset, err_msg) END IF IF ( mix_snapshot_average_fields ) THEN - IF ( mpp_pe() == mpp_root_pe() ) THEN + IF ( mpp_pe() == mpp_root_pe() ) THEN CALL error_mesg('diag_manager_mod::diag_manager_init', 'Setting diag_manager_nml variable '//& & 'mix_snapshot_average_fields = .TRUE. will cause ERRORS in the time coordinates '//& & 'of all time averaged fields. Strongly recommend setting mix_snapshot_average_fields '//& @@ -2865,24 +3394,37 @@ SUBROUTINE diag_manager_init(diag_model_subset, err_msg) END IF ALLOCATE(output_fields(max_output_fields)) ALLOCATE(input_fields(max_input_fields)) - do j = 1, max_input_fields - allocate(input_fields(j)%output_fields(MAX_OUT_PER_IN_FIELD)) - enddo + DO j = 1, max_input_fields + ALLOCATE(input_fields(j)%output_fields(MAX_OUT_PER_IN_FIELD)) + END DO ALLOCATE(files(max_files)) ALLOCATE(pelist(mpp_npes())) CALL mpp_get_current_pelist(pelist, pelist_name) + ! set the diag_init_time if time_init present. Otherwise, set it to base_time + IF ( PRESENT(time_init) ) THEN + diag_init_time = set_date(time_init(1), time_init(2), time_init(3), time_init(4),& + & time_init(5), time_init(6)) + ELSE + diag_init_time = base_time + IF ( prepend_date .EQV. .TRUE. ) THEN + CALL error_mesg('diag_manager_mod::diag_manager_init',& + & 'prepend_date only supported when diag_manager_init is called with time_init present.', NOTE) + prepend_date = .FALSE. + END IF + END IF + CALL parse_diag_table(DIAG_SUBSET=diag_subset_output, ISTAT=mystat, ERR_MSG=err_msg_local) IF ( mystat /= 0 ) THEN IF ( fms_error_handler('diag_manager_mod::diag_manager_init',& & 'Error parsing diag_table. '//TRIM(err_msg_local), err_msg) ) RETURN END IF - + !initialize files%bytes_written to zero files(:)%bytes_written = 0 ! open diag field log file - IF ( do_diag_field_log ) THEN + IF ( do_diag_field_log.AND.mpp_pe().EQ.mpp_root_pe() ) THEN CALL mpp_open(diag_log_unit, 'diag_field_log.out', nohdrs=.TRUE.) WRITE (diag_log_unit,'(777a)') & & 'Module', SEP, 'Field', SEP, 'Long Name', SEP,& @@ -2901,7 +3443,7 @@ END SUBROUTINE diag_manager_init ! ! - ! Return base time for diagnostics. + ! Return base time for diagnostics. ! ! ! ! Writes axis data to file. This subroutine is to be called once per file - ! after all write_meta_data calls, and before the first + ! after all write_meta_data calls, and before the first ! diag_field_out call. ! ! Output file unit number SUBROUTINE done_meta_data(file_unit) - INTEGER, INTENT(in) :: file_unit + INTEGER, INTENT(in) :: file_unit INTEGER :: i @@ -657,14 +748,14 @@ END SUBROUTINE get_diag_global_att ! ! SUBROUTINE set_diag_global_att(component, gridType, tileName) - CHARACTER(len=*),INTENT(in) :: component, gridType, tileName + CHARACTER(len=*),INTENT(in) :: component, gridType, tileName ! The following two lines are set to remove compile time warnings ! about 'only used once'. CHARACTER(len=64) :: component_tmp component_tmp = component ! Don't know how to set these for specific component - ! Want to be able to say + ! Want to be able to say ! if(output_file has component) then diag_global_att%grid_type = gridType diag_global_att%tile_name = tileName @@ -673,4 +764,3 @@ END SUBROUTINE set_diag_global_att !     END MODULE diag_output_mod - diff --git a/src/shared/diag_manager/diag_table.F90 b/src/shared/diag_manager/diag_table.F90 index 7e8c0414a5..e3f9bc5b46 100644 --- a/src/shared/diag_manager/diag_table.F90 +++ b/src/shared/diag_manager/diag_table.F90 @@ -1,7 +1,8 @@ MODULE diag_table_mod - ! + ! ! Seth Underwood ! + ! ! ! diag_table_mod is a set of subroutines use to parse out the data from a diag_table. This module ! will also setup the arrays required to store the information by counting the number of input fields, output files, and @@ -124,9 +125,8 @@ MODULE diag_table_mod ! ! !
  • - ! Field Section: Field lines contain 8 fields. Field lines can be intermixed with file lines, but the file must - ! be defined before any fields that are to be written to the file. Fields line can contain fields that are not written - ! to any files. The file name for these fields is null. + ! Field Section: Field lines contain 8 fields. Field lines can be intermixed with file lines. Fields line can contain + ! fields that are not written to any files. The file name for these fields is null. ! ! Field lines have the following format:
    ! 
    @@ -156,6 +156,8 @@ MODULE diag_table_mod
   !             
    Average from the last time written to the current time.
    
   !             
    .FALSE., none
    
   !             
    No reduction performed.  Write current time step value only.
    
+  !             
    rms
     
    Calculate the root mean square from the last time written to the current time.
    
+  !             
    pow##
     
    Calculate the mean of the power ## from the last time written to the current time.
    
   !             
    min
     
    Minimum value from last write to current time.
    
   !             
    max
     
    Maximum value from last write to current time.
    
   !             
    diurnal##
     
    ## diurnal averages
    
@@ -214,27 +216,27 @@ MODULE diag_table_mod
   USE fms_mod, ONLY: fms_error_handler, error_mesg, file_exist, stdlog, mpp_pe, mpp_root_pe, FATAL, WARNING, lowercase, close_file
   USE time_manager_mod, ONLY: get_calendar_type, NO_CALENDAR, set_date, set_time, month_name, time_type
   USE constants_mod, ONLY: SECONDS_PER_HOUR, SECONDS_PER_MINUTE
-  
+
   USE diag_data_mod, ONLY: global_descriptor, base_time, base_year, base_month, base_day, base_hour, base_minute, base_second,&
-       & DIAG_OTHER, DIAG_OCEAN, DIAG_ALL, coord_type, append_pelist_name, pelist_name, filename_appendix
+       & DIAG_OTHER, DIAG_OCEAN, DIAG_ALL, coord_type, append_pelist_name, pelist_name
   USE diag_util_mod, ONLY: init_file, check_duplicate_output_fields, init_input_field, init_output_field
 
   IMPLICIT NONE
 
   PRIVATE
   PUBLIC :: parse_diag_table
-  
+
   TYPE field_description_type
      CHARACTER(len=128) :: module_name, field_name, output_name, file_name
      CHARACTER(len=50) :: time_sampling
-     CHARACTER(len=50) :: time_method   
+     CHARACTER(len=50) :: time_method
      CHARACTER(len=50) :: spatial_ops
      TYPE(coord_type) :: regional_coords
      INTEGER :: pack
   END TYPE field_description_type
 
   TYPE file_description_type
-     INTEGER :: output_freq 
+     INTEGER :: output_freq
      INTEGER :: file_format
      INTEGER :: new_file_freq
      INTEGER :: file_duration
@@ -256,7 +258,7 @@ MODULE diag_table_mod
   CHARACTER(len=*), PARAMETER :: UNALLOWED_ALL = UNALLOWED_QTE//","
 
 CONTAINS
-  
+
   ! 
   !   
   !     Parse the diag_table in preparation for diagnostic output.
@@ -344,6 +346,7 @@ SUBROUTINE parse_diag_table(diag_subset, istat, err_msg)
     INTEGER :: commentStart !< Index location of first '#' on line
     INTEGER :: diag_subset_output !< local value of diag_subset
     INTEGER :: nfields, nfiles !< Number of fields and files.  Not used yet.
+    INTEGER :: npass !< number of passes done while parsing the diag_table (1 for files, 2 for fields)
     INTEGER, TARGET :: mystat !< variable to hold return status of function/subroutine calls.
     INTEGER, POINTER :: pstat !< pointer that points to istat if preset, otherwise, points to mystat.
 
@@ -367,17 +370,17 @@ SUBROUTINE parse_diag_table(diag_subset, istat, err_msg)
 
     IF ( PRESENT(diag_subset) ) THEN
        diag_subset_output = diag_subset
-    ELSE 
+    ELSE
        diag_subset_output = DIAG_ALL
     END IF
-    
+
     ! get the stdlog unit number
     stdlog_unit = stdlog()
     num_lines = get_ascii_file_num_lines('diag_table', DT_LINE_LENGTH)
     allocate(diag_table(num_lines))
 
     call read_ascii_file('diag_table', DT_LINE_LENGTH, diag_table)
-    
+
     ! Read in the global file labeling string
     READ (UNIT=diag_table(1), FMT=*, IOSTAT=mystat) global_descriptor
     IF ( mystat /= 0 ) THEN
@@ -385,14 +388,14 @@ SUBROUTINE parse_diag_table(diag_subset, istat, err_msg)
        IF ( fms_error_handler('diag_table_mod::parse_diag_table', 'Error reading the global descriptor from the diagnostic table.',&
             & err_msg) ) RETURN
     END IF
-    
+
     ! Read in the base date
     READ (UNIT=diag_table(2), FMT=*, IOSTAT=mystat) base_year, base_month, base_day, base_hour, base_minute, base_second
     IF ( mystat /= 0 ) THEN
        pstat = mystat
        IF ( fms_error_handler('diag_manager_init', 'Error reading the base date from the diagnostic table.', err_msg) ) RETURN
     END IF
-    
+
     ! Set up the time type for base time
     IF ( get_calendar_type() /= NO_CALENDAR ) THEN
        IF ( base_year==0 .OR. base_month==0 .OR. base_day==0 ) THEN
@@ -416,88 +419,94 @@ SUBROUTINE parse_diag_table(diag_subset, istat, err_msg)
 
     nfiles=0
     nfields=0
-    parser: DO line_num=3, num_lines
-       ! Read in the entire line from the file.
-       ! If there is a read error, give a warning, and
-       ! cycle the parser loop.
-       READ (diag_table(line_num), FMT='(A)', IOSTAT=mystat) record_line
-       ! Increase line counter, and put in string for use in warning/error messages.
-       WRITE (line_number, '(I5)') line_num
-
-       IF ( mystat > 0 ) THEN
-          IF ( mpp_pe() == mpp_root_pe() ) &
-               & CALL error_mesg("diag_table_mod::parse_diag_table",&
-               & "Problem reading the diag_table (line:" //line_number//").", FATAL)
-          CYCLE parser
-       ELSE IF ( mystat < 0 ) THEN
-          EXIT parser
-       END IF
-       
-       ! How long is the read in string?
-       record_len = LEN_TRIM(record_line)
-
-       ! ignore blank lines and  lines with comments only (comment marker '#')
-       commentStart = INDEX(record_line,'#')
-       IF ( commentStart .NE. 0 ) record_line = record_line(1:commentStart-1)
-       IF ( LEN_TRIM(record_line) == 0 .OR. record_len == 0 ) CYCLE parser
-
-       init: IF ( is_a_file(TRIM(record_line)) ) THEN
-          temp_file = parse_file_line(LINE=record_line, ISTAT=mystat, ERR_MSG=local_err_msg)
-          
-          IF ( mystat > 0 ) THEN 
-             CALL error_mesg("diag_table_mod::parse_diag_table",&
-                  & TRIM(local_err_msg)//" (line:" //TRIM(line_number)//").", FATAL)
-          ELSE IF ( mystat < 0 ) THEN
-             IF ( mpp_pe() == mpp_root_pe() )&
+    pass: DO npass = 1, 2
+       parser: DO line_num=3, num_lines
+          ! Read in the entire line from the file.
+          ! If there is a read error, give a warning, and
+          ! cycle the parser loop.
+          READ (diag_table(line_num), FMT='(A)', IOSTAT=mystat) record_line
+          ! Increase line counter, and put in string for use in warning/error messages.
+          WRITE (line_number, '(I5)') line_num
+
+          IF ( mystat > 0 ) THEN
+             IF ( mpp_pe() == mpp_root_pe() ) &
                   & CALL error_mesg("diag_table_mod::parse_diag_table",&
-                  & TRIM(local_err_msg)//" (line: "//TRIM(line_number)//").", WARNING)
-             CYCLE parser
-          ELSE IF ( (diag_subset_output == DIAG_OTHER .AND. VERIFY('ocean', lowercase(temp_file%file_name)) == 0).OR.&
-               &    (diag_subset_output == DIAG_OCEAN .AND. VERIFY('ocean', lowercase(temp_file%file_name)) /= 0) ) THEN
+                  & "Problem reading the diag_table (line:" //line_number//").", FATAL)
              CYCLE parser
-          ELSE IF ( temp_file%new_file_freq > 0 ) THEN ! Call the init_file subroutine.  The '1' is for the tile_count
-             CALL init_file(temp_file%file_name, temp_file%output_freq, temp_file%iOutput_freq_units, temp_file%file_format,&
-                  & temp_file%iTime_units, temp_file%long_name, 1, temp_file%new_file_freq, temp_file%iNew_file_freq_units,&
-                  & temp_file%start_time, temp_file%file_duration, temp_file%iFile_duration_units)
-          ELSE
-             CALL init_file(temp_file%file_name, temp_file%output_freq, temp_file%iOutput_freq_units, temp_file%file_format,&
-                  & temp_file%iTime_units, temp_file%long_name, 1)
-          END IF
-          
-          ! Increment number of files
-          nfiles = nfiles + 1
-       ELSE ! We have a field.
-          temp_field = parse_field_line(LINE=record_line, ISTAT=mystat, ERR_MSG=local_err_msg)
-
-          ! Check for errors, then initialize the input and output field
-          IF (  mystat > 0 ) THEN
-             CALL error_mesg("diag_table_mod::parse_diag_table",&
-                  & TRIM(local_err_msg)//" (line: "//TRIM(line_number)//").",FATAL)
           ELSE IF ( mystat < 0 ) THEN
-             IF ( mpp_pe() == mpp_root_pe() )&
-                  & CALL error_mesg("diag_table_mod::Parse_diag_table",&
-                  & TRIM(local_err_msg)//" (line: "//TRIM(line_number)//").",WARNING)
-             CYCLE parser
-          ELSE IF ( (diag_subset_output == DIAG_OTHER .AND. VERIFY('ocean', lowercase(temp_field%file_name)) == 0).OR.&
-               &    (diag_subset_output == DIAG_OCEAN .AND. VERIFY('ocean', lowercase(temp_field%file_name)) /= 0) ) THEN 
-             CYCLE parser
-          ELSE IF ( lowercase(TRIM(temp_field%spatial_ops)) == 'none' ) THEN
-             CALL init_input_field(temp_field%module_name, temp_field%field_name, 1)
-             CALL init_output_field(temp_field%module_name, temp_field%field_name, temp_field%output_name, temp_field%file_name,&
-                  & temp_field%time_method, temp_field%pack, 1)
-          ELSE 
-             CALL init_input_field(temp_field%module_name, temp_field%field_name, 1)
-             CALL init_output_field(temp_field%module_name, temp_field%field_name, temp_field%output_name, temp_field%file_name,&
-                  & temp_field%time_method, temp_field%pack, 1, temp_field%regional_coords)
+             EXIT parser
           END IF
 
-          ! Increment number of fields
-          nfields = nfields + 1
-       END IF init
-    END DO parser
+          ! How long is the read in string?
+          record_len = LEN_TRIM(record_line)
+
+          ! ignore blank lines and  lines with comments only (comment marker '#')
+          commentStart = INDEX(record_line,'#')
+          IF ( commentStart .NE. 0 ) record_line = record_line(1:commentStart-1)
+          IF ( LEN_TRIM(record_line) == 0 .OR. record_len == 0 ) CYCLE parser
+
+          init: IF ( npass == 1 ) THEN ! Checking for files only
+             IF ( is_a_file(TRIM(record_line)) ) THEN
+                temp_file = parse_file_line(LINE=record_line, ISTAT=mystat, ERR_MSG=local_err_msg)
+
+                IF ( mystat > 0 ) THEN
+                   CALL error_mesg("diag_table_mod::parse_diag_table",&
+                        & TRIM(local_err_msg)//" (line:" //TRIM(line_number)//").", FATAL)
+                ELSE IF ( mystat < 0 ) THEN
+                   IF ( mpp_pe() == mpp_root_pe() )&
+                        & CALL error_mesg("diag_table_mod::parse_diag_table",&
+                        & TRIM(local_err_msg)//" (line: "//TRIM(line_number)//").", WARNING)
+                   CYCLE parser
+                ELSE IF ( (diag_subset_output == DIAG_OTHER .AND. VERIFY('ocean', lowercase(temp_file%file_name)) == 0).OR.&
+                     &    (diag_subset_output == DIAG_OCEAN .AND. VERIFY('ocean', lowercase(temp_file%file_name)) /= 0) ) THEN
+                   CYCLE parser
+                ELSE IF ( temp_file%new_file_freq > 0 ) THEN ! Call the init_file subroutine.  The '1' is for the tile_count
+                   CALL init_file(temp_file%file_name, temp_file%output_freq, temp_file%iOutput_freq_units, temp_file%file_format,&
+                        & temp_file%iTime_units, temp_file%long_name, 1, temp_file%new_file_freq, temp_file%iNew_file_freq_units,&
+                        & temp_file%start_time, temp_file%file_duration, temp_file%iFile_duration_units)
+                ELSE
+                   CALL init_file(temp_file%file_name, temp_file%output_freq, temp_file%iOutput_freq_units, temp_file%file_format,&
+                        & temp_file%iTime_units, temp_file%long_name, 1)
+                END IF
+
+                ! Increment number of files
+                nfiles = nfiles + 1
+             END IF
+          ELSE ! Looking for fields
+             IF ( .NOT.is_a_file(TRIM(record_line)) ) THEN
+                temp_field = parse_field_line(LINE=record_line, ISTAT=mystat, ERR_MSG=local_err_msg)
+
+                ! Check for errors, then initialize the input and output field
+                IF (  mystat > 0 ) THEN
+                   CALL error_mesg("diag_table_mod::parse_diag_table",&
+                        & TRIM(local_err_msg)//" (line: "//TRIM(line_number)//").",FATAL)
+                ELSE IF ( mystat < 0 ) THEN
+                   IF ( mpp_pe() == mpp_root_pe() )&
+                        & CALL error_mesg("diag_table_mod::Parse_diag_table",&
+                        & TRIM(local_err_msg)//" (line: "//TRIM(line_number)//").",WARNING)
+                   CYCLE parser
+                ELSE IF ( (diag_subset_output == DIAG_OTHER .AND. VERIFY('ocean', lowercase(temp_field%file_name)) == 0).OR.&
+                     &    (diag_subset_output == DIAG_OCEAN .AND. VERIFY('ocean', lowercase(temp_field%file_name)) /= 0) ) THEN
+                   CYCLE parser
+                ELSE IF ( lowercase(TRIM(temp_field%spatial_ops)) == 'none' ) THEN
+                   CALL init_input_field(temp_field%module_name, temp_field%field_name, 1)
+                   CALL init_output_field(temp_field%module_name, temp_field%field_name, temp_field%output_name, temp_field%file_name,&
+                        & temp_field%time_method, temp_field%pack, 1)
+                ELSE
+                   CALL init_input_field(temp_field%module_name, temp_field%field_name, 1)
+                   CALL init_output_field(temp_field%module_name, temp_field%field_name, temp_field%output_name, temp_field%file_name,&
+                        & temp_field%time_method, temp_field%pack, 1, temp_field%regional_coords)
+                END IF
+
+                ! Increment number of fields
+                nfields = nfields + 1
+             END IF
+          END IF init
+       END DO parser
+    END DO pass
 
     ! Close the diag_table file.
-    deallocate(diag_table)
+    DEALLOCATE(diag_table)
 
     ! check duplicate output_fields in the diag_table
     CALL check_duplicate_output_fields(ERR_MSG=local_err_msg)
@@ -537,15 +546,15 @@ SUBROUTINE open_diag_table(iunit, iostat, err_msg)
 
     IF ( PRESENT(iostat) ) THEN
        pstat => iostat
-    ELSE 
+    ELSE
        pstat => mystat
     END IF
-    
+
     IF ( .NOT.file_exist('diag_table') ) THEN
        pstat = 1
        IF ( fms_error_handler('diag_table_mod::open_diag_table',&
             & 'diag_table file does not exist.', err_msg) ) RETURN
-    ELSE 
+    ELSE
        pstat = 0
     END IF
 
@@ -667,7 +676,7 @@ TYPE(file_description_type) FUNCTION parse_file_line(line, istat, err_msg)
             & 'Unallowed character in file_duration_units in the diag_table.', err_msg) ) RETURN
     END IF
 
-            
+
     ! Fix the file name
     parse_file_line%file_name = fix_file_name(TRIM(parse_file_line%file_name))
 
@@ -677,7 +686,7 @@ TYPE(file_description_type) FUNCTION parse_file_line(line, istat, err_msg)
        IF ( fms_error_handler('diag_table_mod::parse_file_line', 'Invalid file format for file description in the diag_table.',&
             & err_msg) ) RETURN
     END IF
-    
+
     ! check for known units
     parse_file_line%iTime_units = find_unit_ivalue(parse_file_line%time_units)
     parse_file_line%iOutput_freq_units = find_unit_ivalue(parse_file_line%output_freq_units)
@@ -714,7 +723,7 @@ TYPE(file_description_type) FUNCTION parse_file_line(line, istat, err_msg)
     new_file_freq_present: IF ( parse_file_line%new_file_freq > 0 ) THEN ! New file frequency present.
        IF ( LEN_TRIM(parse_file_line%start_time_s) > 0 ) THEN ! start time present
           READ (parse_file_line%start_time_s, FMT=*, IOSTAT=mystat) year, month, day, hour, minute, second
-          IF ( mystat /= 0 ) THEN 
+          IF ( mystat /= 0 ) THEN
              pstat = 1
              IF ( fms_error_handler('diag_table_mod::parse_file_line',&
                   & 'Invalid start time in the file description in diag_table.', err_msg) ) RETURN
@@ -788,37 +797,37 @@ TYPE(field_description_type) FUNCTION parse_field_line(line, istat, err_msg)
     IF ( SCAN(parse_field_line%module_name, UNALLOWED_ALL) > 0 ) THEN
        pstat = 1
        IF ( fms_error_handler('diag_table_mod::parse_field_line',&
-            & 'Unallowed Unallowed character in module_name in the diag_table.', err_msg) ) RETURN
+            & 'Unallowed character in module_name in the diag_table.', err_msg) ) RETURN
     END IF
     IF ( SCAN(parse_field_line%field_name, UNALLOWED_ALL) > 0 ) THEN
        pstat = 1
        IF ( fms_error_handler('diag_table_mod::parse_field_line',&
-            & 'Unallowed Unallowed character in field_name in the diag_table.', err_msg) ) RETURN
+            & 'Unallowed character in field_name in the diag_table.', err_msg) ) RETURN
     END IF
     IF ( SCAN(parse_field_line%output_name, UNALLOWED_ALL) > 0 ) THEN
        pstat = 1
        IF ( fms_error_handler('diag_table_mod::parse_field_line',&
-            & 'Unallowed Unallowed character in output_name in the diag_table.', err_msg) ) RETURN
+            & 'Unallowed character in output_name in the diag_table.', err_msg) ) RETURN
     END IF
     IF ( SCAN(parse_field_line%file_name, UNALLOWED_ALL) > 0 ) THEN
        pstat = 1
        IF ( fms_error_handler('diag_table_mod::parse_field_line',&
-            & 'Unallowed Unallowed character in file_name in the diag_table.', err_msg) ) RETURN
+            & 'Unallowed character in file_name in the diag_table.', err_msg) ) RETURN
     END IF
     IF ( SCAN(parse_field_line%time_sampling, UNALLOWED_ALL) > 0 ) THEN
        pstat = 1
        IF ( fms_error_handler('diag_table_mod::parse_field_line',&
-            & 'Unallowed Unallowed character in time_sampling in the diag_table.', err_msg) ) RETURN
+            & 'Unallowed character in time_sampling in the diag_table.', err_msg) ) RETURN
     END IF
     IF ( SCAN(parse_field_line%time_method, UNALLOWED_ALL) > 0 ) THEN
        pstat = 1
        IF ( fms_error_handler('diag_table_mod::parse_field_line',&
-            & 'Unallowed Unallowed character in time_method in the diag_table.', err_msg) ) RETURN
+            & 'Unallowed character in time_method in the diag_table.', err_msg) ) RETURN
     END IF
     IF ( SCAN(parse_field_line%spatial_ops, UNALLOWED_QTE) > 0 ) THEN
        pstat = 1
        IF ( fms_error_handler('diag_table_mod::parse_field_line',&
-            & 'Unallowed Unallowed character in spatial_ops in the diag_table.', err_msg) ) RETURN
+            & 'Unallowed character in spatial_ops in the diag_table.', err_msg) ) RETURN
     END IF
 
     ! Fix the file name
@@ -830,7 +839,7 @@ TYPE(field_description_type) FUNCTION parse_field_line(line, istat, err_msg)
        IF ( fms_error_handler('diag_table_mod::parse_field_line',&
             & 'Packing is out of range for the field description in diag_table.', err_msg) ) RETURN
     END IF
-    
+
     IF ( lowercase(TRIM(parse_field_line%spatial_ops)) /= 'none' ) THEN
        READ (parse_field_line%spatial_ops, FMT=*, IOSTAT=mystat) parse_field_line%regional_coords
        IF ( mystat /= 0 ) THEN
@@ -859,7 +868,7 @@ PURE LOGICAL FUNCTION is_a_file(line)
     CHARACTER(len=*), INTENT(in) :: line
 
     CHARACTER(len=5) :: first
-    INTEGER :: second 
+    INTEGER :: second
     INTEGER :: mystat !< IO status from read
 
 #if defined __PATHSCALE__ || defined _CRAYFTN
@@ -893,9 +902,8 @@ END FUNCTION is_a_file
   !     PURE CHARACTER(len=128) FUNCTION fix_file_name(file_name_string)
   !   
   !   
-  !     Removes any trailing '.nc' and appends to the file name additional information
-  !     depending on if we are running an ensemble, or requesting append_pelist_name.
-  !     
+  !     Removes any trailing '.nc' and appends (if requested) append_pelist_name.
+  !
   !     Presently, the ensemble appendix will override the append_pelist_name variable.
   !   
   !   String containing the file name from the diag_table.
@@ -909,19 +917,16 @@ PURE CHARACTER(len=128) FUNCTION fix_file_name(file_name_string)
     file_name_len = LEN_TRIM(file_name_string)
 
     ! Remove trailing '.nc' from the file_name, and append suffixes
-    IF ( file_name_len > 2 ) THEN 
+    IF ( file_name_len > 2 ) THEN
        IF ( file_name_string(file_name_len-2:file_name_len) == '.nc' ) THEN
           fix_file_name = file_name_string(1:file_name_len-3)
           file_name_len = file_name_len - 3
        END IF
     END IF
-       
-    ! If using ensembles, then append the ensemble information
-    ! Or add the optional suffix based on the pe list name if the
+
+    ! Add the optional suffix based on the pe list name if the
     ! append_pelist_name == .TRUE.
-    IF ( LEN_TRIM(filename_appendix) > 0 ) THEN 
-       fix_file_name(file_name_len+1:) = TRIM(filename_appendix)    
-    ELSE IF ( append_pelist_name ) THEN
+    IF ( append_pelist_name ) THEN
        fix_file_name(file_name_len+1:) = TRIM(pelist_name)
     END IF
   END FUNCTION fix_file_name
@@ -961,7 +966,7 @@ PURE INTEGER FUNCTION find_unit_ivalue(unit_string)
        find_unit_ivalue = 3
     CASE ('days')
        find_unit_ivalue = 4
-    CASE ('months') 
+    CASE ('months')
        find_unit_ivalue = 5
     CASE ('years')
        find_unit_ivalue = 6
@@ -990,4 +995,3 @@ END SUBROUTINE initialize_output_arrays
   ! 
   ! 
 END MODULE diag_table_mod
-
diff --git a/src/shared/diag_manager/diag_util.F90 b/src/shared/diag_manager/diag_util.F90
index a471d618bf..67372e533f 100644
--- a/src/shared/diag_manager/diag_util.F90
+++ b/src/shared/diag_manager/diag_util.F90
@@ -1,5 +1,7 @@
+#include 
+
 MODULE diag_util_mod
-  ! 
+  ! 
   !   Seth Underwood
   ! 
   ! 
@@ -15,7 +17,7 @@ MODULE diag_util_mod
   ! 
   !   
   !     Make an interface check_bounds_are_exact for the subroutines check_bounds_are_exact_static and
-  !     check_bounds_are_exact_dynamic. 
+  !     check_bounds_are_exact_dynamic.
   !     
       !       INTERFACE check_bounds_are_exact
   !         MODULE PROCEDURE check_bounds_are_exact_static
@@ -31,7 +33,9 @@ MODULE diag_util_mod
        & base_second, num_files, max_files, max_fields_per_file, max_out_per_in_field,&
        & max_input_fields,num_input_fields, max_output_fields, num_output_fields, coord_type,&
        & mix_snapshot_average_fields, global_descriptor, CMOR_MISSING_VALUE, use_cmor, pack_size,&
-       & debug_diag_manager, conserve_water
+       & debug_diag_manager, conserve_water, output_field_type, max_field_attributes, max_file_attributes,&
+       & file_type, prepend_date, region_out_use_alt_value, GLO_REG_VAL, GLO_REG_VAL_ALT,&
+       & DIAG_FIELD_NOT_FOUND, diag_init_time
   USE diag_axis_mod, ONLY  : get_diag_axis_data, get_axis_global_length, get_diag_axis_cart,&
        & get_domain1d, get_domain2d, diag_subaxes_init, diag_axis_init, get_diag_axis, get_axis_aux,&
        & get_axes_shift, get_diag_axis_name, get_diag_axis_domain_name
@@ -48,19 +52,67 @@ MODULE diag_util_mod
        & OPERATOR(<), OPERATOR(>=), OPERATOR(<=)
   USE mpp_io_mod, ONLY : mpp_close
   USE mpp_mod, ONLY : mpp_npes
+  USE fms_io_mod, ONLY : get_instance_filename
   USE constants_mod, ONLY : SECONDS_PER_DAY, SECONDS_PER_HOUR, SECONDS_PER_MINUTE
 
+#ifdef use_netCDF
+  USE netcdf, ONLY: NF90_CHAR
+#endif
+
   IMPLICIT NONE
   PRIVATE
   PUBLIC get_subfield_size, log_diag_field_info, update_bounds, check_out_of_bounds,&
        & check_bounds_are_exact_dynamic, check_bounds_are_exact_static, init_file, diag_time_inc,&
        & find_input_field, init_input_field, init_output_field, diag_data_out, write_static,&
-       & check_duplicate_output_fields, get_date_dif, get_subfield_vert_size, sync_file_times
+       & check_duplicate_output_fields, get_date_dif, get_subfield_vert_size, sync_file_times,&
+       & prepend_attribute, attribute_init
+
+  ! 
+  !   
+  !     prepend a value to a string attribute in the output field or output file
+  !   
+  !   
+  !   
+  !     Prepend a character string to a character attribute for a give field, or to a global attribute
+  !     in a give file.
+  !   
+  !   
+  !   
+  !   
+  !   
+  INTERFACE prepend_attribute
+     MODULE PROCEDURE prepend_attribute_field
+     MODULE PROCEDURE prepend_attribute_file
+  END INTERFACE prepend_attribute
+  ! 
+
+  ! 
+  !   
+  !     Allocates the atttype in out_file
+  !   
+  !   
+  !   
+  !     Allocates memory in out_file for the attributes.  Will FATAL if err_msg is not included
+  !     in the subroutine call.
+  !   
+  !   output field to allocate memory for attribute
+  !   output file to allocate memory for attribute
+  !   Error message, passed back to calling function
+  INTERFACE attribute_init
+     MODULE PROCEDURE attribute_init_field
+     MODULE PROCEDURE attribute_init_file
+  END INTERFACE attribute_init
+  ! 
 
   CHARACTER(len=128),PRIVATE  :: version =&
-       & '$Id: diag_util.F90,v 20.0 2013/12/14 00:18:58 fms Exp $'
+       & '$Id$'
   CHARACTER(len=128),PRIVATE  :: tagname =&
-       & '$Name: tikal $'
+       & '$Name$'
 
 CONTAINS
 
@@ -72,7 +124,7 @@ MODULE diag_util_mod
   !     SUBROUTINE get_subfield_size(axes, outnum)
   !   
   !   
-  !     Get the size, start and end indices for output_fields(outnum), then  
+  !     Get the size, start and end indices for output_fields(outnum), then
   !     fill in output_fields(outnum)%output_grid%(start_indx, end_indx)
   !   
   !   Axes of the input_field.
@@ -83,23 +135,31 @@ SUBROUTINE get_subfield_size(axes, outnum)
 
     REAL, ALLOCATABLE   :: global_lat(:), global_lon(:), global_depth(:)
     INTEGER :: global_axis_size
-    INTEGER :: i,xbegin,xend,ybegin,yend,xbegin_l,xend_l,ybegin_l,yend_l 
+    INTEGER :: i,xbegin,xend,ybegin,yend,xbegin_l,xend_l,ybegin_l,yend_l
     CHARACTER(len=1) :: cart
     TYPE(domain2d) :: Domain2, Domain2_new
     TYPE(domain1d) :: Domain1, Domain1x, Domain1y
     REAL :: start(3), end(3) ! start and end coordinates in 3 axes
-    INTEGER :: gstart_indx(3), gend_indx(3) ! global start and end indices of output domain in 3 axes 
+    INTEGER :: gstart_indx(3), gend_indx(3) ! global start and end indices of output domain in 3 axes
     REAL, ALLOCATABLE :: subaxis_x(:), subaxis_y(:), subaxis_z(:) !containing local coordinates in x,y,z axes
     CHARACTER(len=128) :: msg
     INTEGER :: ishift, jshift
+    INTEGER :: grv !< Value used to determine if the region defined in the diag_table is for the whole axis, or a sub-axis
     CHARACTER(len=128), DIMENSION(2) :: axis_domain_name
 
     !initilization for local output
     ! initially out of (lat/lon/depth) range
     start = -1.e10
-    end = -1.e10 
+    end = -1.e10
     gstart_indx = -1
-    gend_indx=-1
+    gend_indx = -1
+
+    ! get the value to compare to determine if writing full axis data
+    IF ( region_out_use_alt_value ) THEN
+       grv = GLO_REG_VAL_ALT
+    ELSE
+       grv = GLO_REG_VAL
+    END IF
 
     ! get axis data (lat, lon, depth) and indices
     start = output_fields(outnum)%output_grid%start
@@ -121,23 +181,23 @@ SUBROUTINE get_subfield_size(axes, outnum)
                   & 'wrong order of axes, X should come first',FATAL)
              ALLOCATE(global_lon(global_axis_size))
              CALL get_diag_axis_data(axes(i),global_lon)
-             IF( INT( start(i)*END(i) ) == 1 ) THEN 
+             IF( INT(start(i)) == grv .AND. INT(end(i)) == grv ) THEN
                 gstart_indx(i) = 1
                 gend_indx(i) = global_axis_size
                 output_fields(outnum)%output_grid%subaxes(i) = axes(i)
-             ELSE 
+             ELSE
                 gstart_indx(i) = get_index(start(i),global_lon)
                 gend_indx(i) = get_index(END(i),global_lon)
              END IF
              ALLOCATE(subaxis_x(gstart_indx(i):gend_indx(i)))
-             subaxis_x=global_lon(gstart_indx(i):gend_indx(i))   
+             subaxis_x=global_lon(gstart_indx(i):gend_indx(i))
           CASE ('Y')
              ! wrong order of axes, Y should come second.
              IF( i.NE.2 ) CALL error_mesg('diag_util_mod::get_subfield_size',&
                   & 'wrong order of axes, Y should come second',FATAL)
              ALLOCATE(global_lat(global_axis_size))
              CALL get_diag_axis_data(axes(i),global_lat)
-             IF( INT( start(i)*END(i) ) == 1 ) THEN 
+             IF( INT(start(i)) == grv .AND. INT(END(i)) == grv ) THEN
                 gstart_indx(i) = 1
                 gend_indx(i) = global_axis_size
                 output_fields(outnum)%output_grid%subaxes(i) = axes(i)
@@ -149,9 +209,9 @@ SUBROUTINE get_subfield_size(axes, outnum)
              subaxis_y=global_lat(gstart_indx(i):gend_indx(i))
           CASE ('Z')
              ! wrong values in vertical axis of region
-             IF ( start(i)*END(i)<0 ) CALL error_mesg('diag_util_mod::get_subfield_size',&
+             IF ( start(i)*END(i)<0. ) CALL error_mesg('diag_util_mod::get_subfield_size',&
                   & 'wrong values in vertical axis of region',FATAL)
-             IF ( start(i)>=0 .AND. END(i)>0 ) THEN 
+             IF ( start(i)>=0. .AND. END(i)>0. ) THEN
                 ALLOCATE(global_depth(global_axis_size))
                 CALL get_diag_axis_data(axes(i),global_depth)
                 gstart_indx(i) = get_index(start(i),global_depth)
@@ -176,7 +236,7 @@ SUBROUTINE get_subfield_size(axes, outnum)
        END DO
 
        DO i = 1, SIZE(axes(:))
-          IF( gstart_indx(i) == -1 .OR. gend_indx(i) == -1 ) THEN
+          IF ( gstart_indx(i) == -1 .OR. gend_indx(i) == -1 ) THEN
              ! 
              !   can not find gstart_indx/gend_indx for ,
              !   check region bounds for axis .
@@ -219,9 +279,9 @@ SUBROUTINE get_subfield_size(axes, outnum)
           ! 
           !   wrong values in vertical axis of region
           ! 
-          IF ( start(3)*END(3)<0 ) CALL error_mesg('diag_util_mod::get_subfield_size',&
+          IF ( start(3)*END(3)<0. ) CALL error_mesg('diag_util_mod::get_subfield_size',&
                & 'wrong values in vertical axis of region',FATAL)
-          IF ( start(3)>=0 .AND. END(3)>0 ) THEN 
+          IF ( start(3)>=0. .AND. END(3)>0. ) THEN
              ALLOCATE(global_depth(global_axis_size))
              CALL get_diag_axis_data(axes(3),global_depth)
              gstart_indx(3) = get_index(start(3),global_depth)
@@ -229,7 +289,7 @@ SUBROUTINE get_subfield_size(axes, outnum)
              gend_indx(3) = get_index(END(3),global_depth)
              IF( start(3) >= MAXVAL(global_depth) ) gstart_indx(3)= global_axis_size
              IF( END(3)   >= MAXVAL(global_depth) ) gend_indx(3)  = global_axis_size
-             
+
              ALLOCATE(subaxis_z(gstart_indx(3):gend_indx(3)))
              subaxis_z=global_depth(gstart_indx(3):gend_indx(3))
              output_fields(outnum)%output_grid%subaxes(3) =&
@@ -242,19 +302,19 @@ SUBROUTINE get_subfield_size(axes, outnum)
           END IF
        END IF
     END IF
-    
+
     ! get domain and compute_domain(xbegin,xend,ybegin,yend)
-    xbegin=-1
-    xend=-1
-    ybegin=-1
-    yend=-1
+    xbegin = -1
+    xend = -1
+    ybegin = -1
+    yend = -1
 
     Domain2 = get_domain2d(axes)
     IF ( Domain2 .NE. NULL_DOMAIN2D ) THEN
-       CALL mpp_get_compute_domain(Domain2,xbegin,xend,ybegin,yend)
+       CALL mpp_get_compute_domain(Domain2, xbegin, xend, ybegin, yend)
        CALL mpp_get_domain_components(Domain2, Domain1x, Domain1y)
     ELSE
-       DO i = 1, MIN(SIZE(axes(:)),2)    
+       DO i = 1, MIN(SIZE(axes(:)),2)
           Domain1 = get_domain1d(axes(i))
           IF ( Domain1 .NE. NULL_DOMAIN1D ) THEN
              CALL get_diag_axis_cart(axes(i),cart)
@@ -278,17 +338,17 @@ SUBROUTINE get_subfield_size(axes, outnum)
     xend = xend+ishift
     yend = yend+jshift
 
-    IF ( xbegin== -1 .OR. xend==-1 .OR. ybegin==-1 .OR. yend==-1 ) THEN
+    IF ( xbegin == -1 .OR. xend == -1 .OR. ybegin == -1 .OR. yend == -1 ) THEN
        ! wrong compute domain indices
-       CALL error_mesg('diag_util_mod::get_subfield_size', 'wrong compute domain indices',FATAL)  
+       CALL error_mesg('diag_util_mod::get_subfield_size', 'wrong compute domain indices',FATAL)
     END IF
-      
+
     ! get the area containing BOTH compute domain AND local output area
-    IF(gstart_indx(1)> xend .OR. xbegin > gend_indx(1)) THEN
+    IF( gstart_indx(1) > xend .OR. xbegin > gend_indx(1) ) THEN
        output_fields(outnum)%output_grid%l_start_indx(1) = -1
        output_fields(outnum)%output_grid%l_end_indx(1) = -1
        output_fields(outnum)%need_compute = .FALSE. ! not involved
-    ELSEIF (gstart_indx(2)> yend .OR. ybegin > gend_indx(2)) THEN
+    ELSEIF ( gstart_indx(2) > yend .OR. ybegin > gend_indx(2) ) THEN
        output_fields(outnum)%output_grid%l_start_indx(2) = -1
        output_fields(outnum)%output_grid%l_end_indx(2) = -1
        output_fields(outnum)%need_compute = .FALSE. ! not involved
@@ -314,20 +374,20 @@ SUBROUTINE get_subfield_size(axes, outnum)
        output_fields(outnum)%output_grid%subaxes(2) =&
             & diag_subaxes_init(axes(2),subaxis_y, gstart_indx(2),gend_indx(2),Domain2_new)
        DO i = 1, SIZE(axes(:))
-          IF(output_fields(outnum)%output_grid%subaxes(i) == -1) THEN  
+          IF ( output_fields(outnum)%output_grid%subaxes(i) == -1 ) THEN
              ! 
              !    error at i = 
              ! 
              WRITE(msg,'(a,"/",I4)') 'at i = ',i
              CALL error_mesg('diag_util_mod::get_subfield_size '//TRIM(output_fields(outnum)%output_name),&
-                  'error '//TRIM(msg), FATAL)   
+                  'error '//TRIM(msg), FATAL)
           END IF
        END DO
 
        ! local start index should start from 1
-       output_fields(outnum)%output_grid%l_start_indx(1) = MAX(xbegin, gstart_indx(1)) - xbegin + 1   
+       output_fields(outnum)%output_grid%l_start_indx(1) = MAX(xbegin, gstart_indx(1)) - xbegin + 1
        output_fields(outnum)%output_grid%l_start_indx(2) = MAX(ybegin, gstart_indx(2)) - ybegin + 1
-       output_fields(outnum)%output_grid%l_end_indx(1) = MIN(xend, gend_indx(1)) - xbegin + 1 
+       output_fields(outnum)%output_grid%l_end_indx(1) = MIN(xend, gend_indx(1)) - xbegin + 1
        output_fields(outnum)%output_grid%l_end_indx(2) = MIN(yend, gend_indx(2)) - ybegin + 1
        IF ( SIZE(axes(:))>2 ) THEN
           output_fields(outnum)%output_grid%l_start_indx(3) = gstart_indx(3)
@@ -339,10 +399,9 @@ SUBROUTINE get_subfield_size(axes, outnum)
     END IF
     IF ( ALLOCATED(subaxis_x) ) DEALLOCATE(subaxis_x, global_lon)
     IF ( ALLOCATED(subaxis_y) ) DEALLOCATE(subaxis_y, global_lat)
-
   END SUBROUTINE get_subfield_size
   ! 
-  
+
   ! 
   !   
   !     Get size, start and end indices for output fields.
@@ -364,21 +423,21 @@ SUBROUTINE get_subfield_vert_size(axes, outnum)
     REAL, ALLOCATABLE, DIMENSION(:) :: global_depth
     REAL, ALLOCATABLE, DIMENSION(:) :: subaxis_z !containing local coordinates in x,y,z axes
     INTEGER :: i, global_axis_size
-    INTEGER, DIMENSION(3) :: gstart_indx, gend_indx ! global start and end indices of output domain in 3 axes 
+    INTEGER, DIMENSION(3) :: gstart_indx, gend_indx ! global start and end indices of output domain in 3 axes
     CHARACTER(len=1) :: cart
     CHARACTER(len=128) :: msg
 
     !initilization for local output
     start = -1.e10
     end = -1.e10 ! initially out of (lat/lon/depth) range
-    gstart_indx = -1 
+    gstart_indx = -1
     gend_indx=-1
 
     ! get axis data (lat, lon, depth) and indices
     start= output_fields(outnum)%output_grid%start
     end = output_fields(outnum)%output_grid%end
 
-    DO i = 1, SIZE(axes(:))   
+    DO i = 1, SIZE(axes(:))
        global_axis_size = get_axis_global_length(axes(i))
        output_fields(outnum)%output_grid%subaxes(i) = -1
        CALL get_diag_axis_cart(axes(i), cart)
@@ -399,9 +458,9 @@ SUBROUTINE get_subfield_vert_size(axes, outnum)
           output_fields(outnum)%output_grid%subaxes(i) = axes(i)
        CASE ('Z')
           ! wrong values in vertical axis of region
-          IF( start(i)*END(i) < 0 ) CALL error_mesg('diag_util_mod::get_subfield_vert_size',&
+          IF( start(i)*END(i) < 0. ) CALL error_mesg('diag_util_mod::get_subfield_vert_size',&
                & 'wrong values in vertical axis of region',FATAL)
-          IF( start(i) >= 0 .AND. END(i) > 0 ) THEN 
+          IF( start(i) >= 0. .AND. END(i) > 0. ) THEN
              ALLOCATE(global_depth(global_axis_size))
              CALL get_diag_axis_data(axes(i),global_depth)
              gstart_indx(i) = get_index(start(i),global_depth)
@@ -431,7 +490,7 @@ SUBROUTINE get_subfield_vert_size(axes, outnum)
     END DO
 
     DO i = 1,SIZE(axes(:))
-       IF ( gstart_indx(i)== -1 .OR. gend_indx(i)== -1 ) THEN
+       IF ( gstart_indx(i) == -1 .OR. gend_indx(i) == -1 ) THEN
           ! 
           !   can not find gstart_indx/gend_indx for 
           !   check region bounds for axis
@@ -456,7 +515,7 @@ SUBROUTINE get_subfield_vert_size(axes, outnum)
     END IF
   END SUBROUTINE get_subfield_vert_size
   ! 
-  
+
   ! 
   ! 
   !   
@@ -483,13 +542,13 @@ INTEGER FUNCTION get_index(number, array)
     DO i = 2, n-1
        IF( (array(i-1)array(i+1)) .OR. (array(i-1)>array(i).AND.array(i)array NOT monotonously ordered
-          CALL error_mesg('diag_util_mod::get_index', 'array NOT monotonously ordered',FATAL) 
+          CALL error_mesg('diag_util_mod::get_index', 'array NOT monotonously ordered',FATAL)
        END IF
     END DO
     get_index = -1
     found = .FALSE.
-    ! search in increasing array 
-    DO i = 1, n-1                
+    ! search in increasing array
+    DO i = 1, n-1
        IF ( (array(i)<=number).AND.(array(i+1)>= number) ) THEN
           IF( number - array(i) <= array(i+1) - number ) THEN
              get_index = i
@@ -506,7 +565,7 @@ INTEGER FUNCTION get_index(number, array)
        DO i = 1, n-1
           IF ( (array(i)>=number).AND.(array(i+1)<= number) ) THEN
              IF ( array(i)-number <= number-array(i+1) ) THEN
-                get_index = i 
+                get_index = i
                 found = .TRUE.
              ELSE
                 get_index = i+1
@@ -518,25 +577,28 @@ INTEGER FUNCTION get_index(number, array)
     END IF
     ! if still not found, is it less than the first element
     ! or greater than last element? (Increasing Array)
+    ! But it must be within 2x the axis spacing
+    ! i.e. array(1)-(array(3)-array(1)).LT.number .AND. or 2*array(1)-array(3).LT.number
     IF ( .NOT. found ) THEN
-       IF ( array(1).GT.number ) THEN
+       IF ( 2*array(1)-array(3).LT.number .AND. number.LT.array(1) ) THEN
           get_index = 1
           found = .TRUE.
-       ELSE IF ( array(n).LT.number ) THEN
+       ELSE IF ( array(n).LT.number .AND. number.LT.2*array(n)-array(n-2) ) THEN
           get_index = n
           found = .TRUE.
        ELSE
           found = .FALSE.
        END IF
     END IF
-   
-   ! if still not found, is it greater than the first element
-   ! or less than the last element? (Decreasing Array)
+
+    ! if still not found, is it greater than the first element
+    ! or less than the last element? (Decreasing Array)
+    ! But it must be within 2x the axis spacing (see above)
     IF ( .NOT. found ) THEN
-       IF ( array(1).LT.number ) THEN
+       IF ( 2*array(1)-array(3).GT.number .AND. number.GT.array(1) ) THEN
           get_index = 1
           found = .TRUE.
-       ELSE IF ( array(n).GT.number ) THEN
+       ELSE IF ( array(n).GT.number .AND. number.GT.2*array(n)-array(n-2) ) THEN
           get_index = n
           found = .TRUE.
        ELSE
@@ -600,16 +662,16 @@ SUBROUTINE log_diag_field_info(module_name, field_name, axes, long_name, units,&
 
     IF ( PRESENT(long_name) ) THEN
        lname  = TRIM(long_name)
-    ELSE 
+    ELSE
        lname  = ''
     END IF
-    
+
     IF ( PRESENT(units) ) THEN
        lunits = TRIM(units)
     ELSE
        lunits = ''
     END IF
- 
+
     WRITE (numaxis,'(i1)') SIZE(axes)
 
     IF (PRESENT(missing_value)) THEN
@@ -675,7 +737,7 @@ END SUBROUTINE log_diag_field_info
   !   Upper k bound.
   SUBROUTINE update_bounds(out_num, lower_i, upper_i, lower_j, upper_j, lower_k, upper_k)
     INTEGER, INTENT(in) :: out_num, lower_i, upper_i, lower_j, upper_j, lower_k, upper_k
-    
+
     output_fields(out_num)%imin = MIN(output_fields(out_num)%imin, lower_i)
     output_fields(out_num)%imax = MAX(output_fields(out_num)%imax, upper_i)
     output_fields(out_num)%jmin = MIN(output_fields(out_num)%jmin, lower_j)
@@ -890,7 +952,7 @@ SUBROUTINE check_bounds_are_exact_static(out_num, diag_field_id, err_msg)
     output_fields(out_num)%jmin = VERY_LARGE_AXIS_LENGTH
     output_fields(out_num)%kmax = 0
     output_fields(out_num)%kmin = VERY_LARGE_AXIS_LENGTH
-    
+
   END SUBROUTINE check_bounds_are_exact_static
   ! 
 
@@ -944,32 +1006,32 @@ SUBROUTINE init_file(name, output_freq, output_units, FORMAT, time_units, long_n
             & in the namelist diag_manager_nml.', FATAL)
     END IF
 
-    IF ( PRESENT(new_file_freq) ) THEN 
+    IF ( PRESENT(new_file_freq) ) THEN
        new_file_freq1 = new_file_freq
-    ELSE 
+    ELSE
        new_file_freq1 = VERY_LARGE_FILE_FREQ
     END IF
-    
-    IF ( PRESENT(new_file_freq_units) ) THEN 
-       new_file_freq_units1 = new_file_freq_units 
+
+    IF ( PRESENT(new_file_freq_units) ) THEN
+       new_file_freq_units1 = new_file_freq_units
     ELSE IF ( get_calendar_type() == NO_CALENDAR ) THEN
        new_file_freq_units1 = DIAG_DAYS
-    ELSE 
+    ELSE
        new_file_freq_units1 = DIAG_YEARS
     END IF
-    
+
     IF ( PRESENT(file_duration) ) THEN
-       file_duration1 = file_duration 
+       file_duration1 = file_duration
     ELSE
        file_duration1 = new_file_freq1
     END IF
-    
-    IF ( PRESENT(file_duration_units) ) THEN 
+
+    IF ( PRESENT(file_duration_units) ) THEN
        file_duration_units1 = file_duration_units
-    ELSE 
+    ELSE
        file_duration_units1 = new_file_freq_units1
     END IF
-    
+
     files(num_files)%tile_count = tile_count
     files(num_files)%name = TRIM(name)
     files(num_files)%output_freq = output_freq
@@ -985,7 +1047,7 @@ SUBROUTINE init_file(name, output_freq, output_units, FORMAT, time_units, long_n
     files(num_files)%new_file_freq_units = new_file_freq_units1
     files(num_files)%duration = file_duration1
     files(num_files)%duration_units = file_duration_units1
-    IF ( PRESENT(start_time) ) THEN 
+    IF ( PRESENT(start_time) ) THEN
        files(num_files)%start_time = start_time
     ELSE
        files(num_files)%start_time = base_time
@@ -1000,7 +1062,7 @@ SUBROUTINE init_file(name, output_freq, output_units, FORMAT, time_units, long_n
        CALL error_mesg('diag_util_mod::init_file', 'close time GREATER than next_open time, check file duration,&
             & file frequency in '//files(num_files)%name, FATAL)
     END IF
-    
+
     ! add time_axis_id and time_bounds_id here
     WRITE(time_units_str, 11) TRIM(time_unit_list(files(num_files)%time_units)), base_year,&
          & base_month, base_day, base_hour, base_minute, base_second
@@ -1132,7 +1194,7 @@ TYPE(time_type) FUNCTION diag_time_inc(time, output_freq, output_units, err_msg)
        ELSE
           diag_time_inc = increment_date(time, output_freq, 0, 0, 0, 0, 0, err_msg=error_message_local)
        END IF
-    ELSE 
+    ELSE
        error_message_local = 'illegal output units'
     END IF
 
@@ -1192,11 +1254,11 @@ INTEGER FUNCTION find_input_field(module_name, field_name, tile_count)
 
     INTEGER :: i
 
-    find_input_field = -1 ! Default return value if not found.
+    find_input_field = DIAG_FIELD_NOT_FOUND ! Default return value if not found.
     DO i = 1, num_input_fields
        IF(tile_count == input_fields(i)%tile_count .AND.&
             & TRIM(input_fields(i)%module_name) == TRIM(module_name) .AND.&
-            & lowercase(TRIM(input_fields(i)%field_name)) == lowercase(TRIM(field_name))) THEN 
+            & lowercase(TRIM(input_fields(i)%field_name)) == lowercase(TRIM(field_name))) THEN
           find_input_field = i
           RETURN
        END IF
@@ -1243,6 +1305,7 @@ SUBROUTINE init_input_field(module_name, field_name, tile_count)
     input_fields(num_input_fields)%standard_name = 'none'
     input_fields(num_input_fields)%tile_count = tile_count
     input_fields(num_input_fields)%numthreads = 1
+    input_fields(num_input_fields)%active_omp_level = 0
     input_fields(num_input_fields)%time = time_zero
   END SUBROUTINE init_input_field
   ! 
@@ -1277,9 +1340,20 @@ SUBROUTINE init_output_field(module_name, field_name, output_name, output_file,&
     INTEGER :: out_num, in_num, file_num, file_num_tile1
     INTEGER :: num_fields, i, method_selected, l1
     INTEGER :: ioerror
+    REAL :: pow_value
+    INTEGER :: grv !< Value used to determine if the region defined in the diag_table is for the whole axis, or a sub-axis
     CHARACTER(len=128) :: error_msg
     CHARACTER(len=50) :: t_method
 
+    ! Value to use to determine if a region is to be output on the full axis, or sub-axis
+    ! get the value to compare to determine if writing full axis data
+    IF ( region_out_use_alt_value ) THEN
+       grv = GLO_REG_VAL_ALT
+    ELSE
+       grv = GLO_REG_VAL
+    END IF
+
+
     ! Get a number for this output field
     num_output_fields = num_output_fields + 1
     IF ( num_output_fields > max_output_fields ) THEN
@@ -1375,6 +1449,7 @@ SUBROUTINE init_output_field(module_name, field_name, output_name, output_file,&
     ! Enter the other data for this output field
     output_fields(out_num)%output_name = TRIM(output_name)
     output_fields(out_num)%pack = pack
+    output_fields(out_num)%pow_value = 1
     output_fields(out_num)%num_axes = 0
     output_fields(out_num)%total_elements = 0
     output_fields(out_num)%region_elements = 0
@@ -1391,8 +1466,9 @@ SUBROUTINE init_output_field(module_name, field_name, output_name, output_file,&
     ! Initialize all time method to false
     method_selected = 0
     output_fields(out_num)%time_average = .FALSE.
+    output_fields(out_num)%time_rms = .FALSE.
     output_fields(out_num)%time_min = .FALSE.
-    output_fields(out_num)%time_max = .FALSE. 
+    output_fields(out_num)%time_max = .FALSE.
     output_fields(out_num)%time_ops = .FALSE.
     output_fields(out_num)%written_once = .FALSE.
 
@@ -1421,12 +1497,32 @@ SUBROUTINE init_output_field(module_name, field_name, output_name, output_file,&
        output_fields(out_num)%time_average = .TRUE.
        method_selected = method_selected+1
        t_method='mean'
+    ELSEIF ( INDEX(t_method,'pow') == 1 ) THEN
+       ! Get the integer number from the t_method
+       READ (UNIT=t_method(4:LEN_TRIM(t_method)), FMT=*, IOSTAT=ioerror) pow_value
+       IF ( ioerror /= 0 .OR. output_fields(out_num)%pow_value < 1  .OR. FLOOR(pow_value) /= CEILING(pow_value) ) THEN
+          ! 
+          !   Invalid power number in time operation "".  Must be a positive integer.
+          ! 
+          CALL error_mesg('diag_util_mod::init_output_field',&
+               & 'Invalid power number in time operation "'//TRIM(t_method)//'".  Must be a positive integer', FATAL)
+       END IF
+       output_fields(out_num)%pow_value = INT(pow_value)
+       output_fields(out_num)%time_average = .TRUE.
+       method_selected = method_selected+1
+       t_method = 'mean_pow('//t_method(4:LEN_TRIM(t_method))//')'
     ELSE
        SELECT CASE(TRIM(t_method))
        CASE ( '.true.', 'mean', 'average', 'avg' )
           output_fields(out_num)%time_average = .TRUE.
           method_selected = method_selected+1
           t_method = 'mean'
+       CASE ( 'rms' )
+          output_fields(out_num)%time_average = .TRUE.
+          output_fields(out_num)%time_rms = .TRUE.
+          output_fields(out_num)%pow_value = 2.0
+          method_selected = method_selected+1
+          t_method = 'root_mean_square'
        CASE ( '.false.', 'none', 'point' )
           output_fields(out_num)%time_average = .FALSE.
           method_selected = method_selected+1
@@ -1437,17 +1533,17 @@ SUBROUTINE init_output_field(module_name, field_name, output_name, output_file,&
           IF ( output_fields(out_num)%output_name(l1-2:l1) /= 'max' ) &
                output_fields(out_num)%output_name = TRIM(output_name)//'_max'
           method_selected = method_selected+1
-          t_method = 'max'        
+          t_method = 'max'
        CASE ( 'minimum', 'min' )
           output_fields(out_num)%time_min = .TRUE.
           l1 = LEN_TRIM(output_fields(out_num)%output_name)
           IF ( output_fields(out_num)%output_name(l1-2:l1) /= 'min' )&
                & output_fields(out_num)%output_name = TRIM(output_name)//'_min'
           method_selected = method_selected+1
-          t_method = 'min'        
+          t_method = 'min'
        END SELECT
     END IF
-    
+
     ! reconcile logical flags
     output_fields(out_num)%time_ops = output_fields(out_num)%time_min.OR.output_fields(out_num)%time_max&
          & .OR.output_fields(out_num)%time_average
@@ -1457,8 +1553,8 @@ SUBROUTINE init_output_field(module_name, field_name, output_name, output_file,&
     IF ( PRESENT(local_coord) ) THEN
        input_fields(in_num)%local = .TRUE.
        input_fields(in_num)%local_coord = local_coord
-       IF ( INT(local_coord%xbegin * local_coord%xbegin) == 1 .AND.&
-            & INT(local_coord%ybegin * local_coord%ybegin) ==1 ) THEN
+       IF ( INT(local_coord%xbegin) ==  grv .AND. INT(local_coord%xend) == grv .AND.&
+            & INT(local_coord%ybegin) == grv .AND. INT(local_coord%yend) == grv ) THEN
           output_fields(out_num)%local_output = .FALSE.
           output_fields(out_num)%need_compute = .FALSE.
           output_fields(out_num)%reduced_k_range = .TRUE.
@@ -1493,16 +1589,17 @@ SUBROUTINE init_output_field(module_name, field_name, output_name, output_file,&
 
     output_fields(out_num)%time_method = TRIM(t_method)
 
-    ! allocate counters: NOTE that for simplicity we always allocate them, even 
-    ! if they are superceeded by 4D "counter" array. This isn't most memory 
+    ! allocate counters: NOTE that for simplicity we always allocate them, even
+    ! if they are superceeded by 4D "counter" array. This isn't most memory
     ! efficient, approach, but probably tolerable since they are so small anyway
     ALLOCATE(output_fields(out_num)%count_0d(output_fields(out_num)%n_diurnal_samples))
     ALLOCATE(output_fields(out_num)%num_elements(output_fields(out_num)%n_diurnal_samples))
     output_fields(out_num)%count_0d(:) = 0
     output_fields(out_num)%num_elements(:) = 0
+    output_fields(out_num)%num_attributes = 0
   END SUBROUTINE init_output_field
   ! 
-  
+
   ! 
   ! 
   !   
@@ -1519,28 +1616,30 @@ END SUBROUTINE init_output_field
   SUBROUTINE opening_file(file, time)
     ! WARNING: Assumes that all data structures are fully initialized
     INTEGER, INTENT(in) :: file
-    TYPE(time_type), INTENT(in) :: time  
+    TYPE(time_type), INTENT(in) :: time
 
     REAL, DIMENSION(2) :: DATA
-    INTEGER :: j, field_num, input_field_num, num_axes, k 
+    INTEGER :: j, field_num, input_field_num, num_axes, k
     INTEGER :: field_num1
     INTEGER :: position
     INTEGER :: dir, edges
     INTEGER :: ntileMe
+    INTEGER :: year, month, day, hour, minute, second
     INTEGER, ALLOCATABLE :: tile_id(:)
     INTEGER, DIMENSION(1) :: time_axis_id, time_bounds_id
     ! size of this axes array must be at least max num. of
     ! axes per field + 2; the last two elements are for time
     ! and time bounds dimensions
-    INTEGER, DIMENSION(6) :: axes 
+    INTEGER, DIMENSION(6) :: axes
     LOGICAL :: time_ops, aux_present, match_aux_name
     LOGICAL :: all_scalar_or_1d
     CHARACTER(len=7) :: prefix
-    CHARACTER (len = 7) :: avg_name = 'average'
+    CHARACTER(len=7) :: avg_name = 'average'
     CHARACTER(len=128) :: time_units, timeb_units, avg, error_string, filename, aux_name,fieldname
     CHARACTER(len=128) :: suffix, base_name
     CHARACTER(len=32) :: time_name, timeb_name,time_longname, timeb_longname, cart_name
     CHARACTER(len=256) :: fname
+    CHARACTER(len=24) :: start_date
     TYPE(domain1d) :: domain
     TYPE(domain2d) :: domain2
 
@@ -1549,7 +1648,7 @@ SUBROUTINE opening_file(file, time)
     ! it's unlikely that a file starts with word "rregion", need to check anyway.
     IF ( LEN(files(file)%name) >=7 .AND. .NOT.files(file)%local ) THEN
        prefix = files(file)%name(1:7)
-       IF ( lowercase(prefix) == 'rregion' ) THEN 
+       IF ( lowercase(prefix) == 'rregion' ) THEN
           ! 
           !   file name should not start with word "rregion"
           ! 
@@ -1557,7 +1656,7 @@ SUBROUTINE opening_file(file, time)
                & 'file name should not start with word "rregion"', WARNING)
        END IF
     END IF
-    
+
     ! Here is where time_units string must be set up; time since base date
     WRITE (time_units, 11) TRIM(time_unit_list(files(file)%time_units)), base_year,&
          & base_month, base_day, base_hour, base_minute, base_second
@@ -1572,18 +1671,23 @@ SUBROUTINE opening_file(file, time)
           !   filename  does not contain % for time stamp string
           ! 
           CALL error_mesg('diag_util_mod::opening_file',&
-               & 'file name '//TRIM(files(file)%name)//' does not contain % for time stamp string', FATAL) 
+               & 'file name '//TRIM(files(file)%name)//' does not contain % for time stamp string', FATAL)
        END IF
        suffix = get_time_string(files(file)%name, time)
     ELSE
        suffix = ' '
     END IF
+
+    ! Add ensemble ID to filename
+    fname=base_name
+    call get_instance_filename(fname, base_name)
+
     ! Add CVS tag as prefix of filename  (currently not implemented)
     !  i1 = INDEX(tagname,':') + 2
     !  i2 = len_trim(tagname) - 2
     !  if(i2 <=i1)  call error_mesg('diag_util opening_file','error in CVS tagname index',FATAL)
     !  prefix2 = tagname(i1:i2)//'_'
-    IF ( files(file)%local ) THEN      
+    IF ( files(file)%local ) THEN
        ! prepend "rregion" to all local files for post processing, the prefix will be removed in postprocessing
        filename = 'rregion'//TRIM(base_name)//TRIM(suffix)
     ELSE
@@ -1591,6 +1695,14 @@ SUBROUTINE opening_file(file, time)
        filename = TRIM(base_name)//TRIM(suffix)
     END IF
 
+    ! prepend the file start date if prepend_date == .TRUE.
+    IF ( prepend_date ) THEN
+       call get_date(diag_init_time, year, month, day, hour, minute, second)
+       write (start_date, '(1I20.4, 2I2.2)') year, month, day
+
+       filename = TRIM(adjustl(start_date))//'.'//TRIM(filename)
+    END IF
+
     ! Loop through all fields with this file to output axes
     ! JWD: This is a klooge; need something more robust
     domain2 = NULL_DOMAIN2D
@@ -1624,9 +1736,15 @@ SUBROUTINE opening_file(file, time)
        END IF
     END IF
 
-    CALL diag_output_init(filename, files(file)%format, global_descriptor,&
-         & files(file)%file_unit, all_scalar_or_1d, domain2) 
-    files(file)%bytes_written = 0 
+    IF ( _ALLOCATED(files(file)%attributes) ) THEN
+       CALL diag_output_init(filename, files(file)%format, global_descriptor,&
+            & files(file)%file_unit, all_scalar_or_1d, domain2,&
+            & attributes=files(file)%attributes(1:files(file)%num_attributes))
+    ELSE
+       CALL diag_output_init(filename, files(file)%format, global_descriptor,&
+            & files(file)%file_unit, all_scalar_or_1d, domain2)
+    END IF
+    files(file)%bytes_written = 0
     ! Does this file contain time_average fields?
     time_ops = .FALSE.
     DO j = 1, files(file)%num_fields
@@ -1649,7 +1767,7 @@ SUBROUTINE opening_file(file, time)
              !   NOT registered
              ! 
              CALL error_mesg('diag_util_mod::opening_file',&
-                  & 'module/field_name ('//TRIM(error_string)//') NOT registered', WARNING)  
+                  & 'module/field_name ('//TRIM(error_string)//') NOT registered', WARNING)
           END IF
           CYCLE
        END IF
@@ -1684,7 +1802,7 @@ SUBROUTINE opening_file(file, time)
        CALL write_axis_meta_data(files(file)%file_unit, axes(1:num_axes + 1), time_ops)
        IF ( time_ops ) THEN
           axes(num_axes + 2) = files(file)%time_bounds_id
-          CALL write_axis_meta_data(files(file)%file_unit, axes(1:num_axes + 2))     
+          CALL write_axis_meta_data(files(file)%file_unit, axes(1:num_axes + 2))
        END IF
     END DO
 
@@ -1721,7 +1839,7 @@ SUBROUTINE opening_file(file, time)
        ! check if any field has the same name as aux_name
        IF ( aux_present .AND. .NOT.match_aux_name ) THEN
           fieldname = output_fields(field_num)%output_name
-          IF ( INDEX(aux_name, TRIM(fieldname)) > 0 ) match_aux_name = .TRUE.   
+          IF ( INDEX(aux_name, TRIM(fieldname)) > 0 ) match_aux_name = .TRUE.
        END IF
 
        ! Put the time axis in the axis field
@@ -1750,7 +1868,9 @@ SUBROUTINE opening_file(file, time)
                   & input_fields(input_field_num)%missing_value, avg_name = avg,&
                   & time_method=output_fields(field_num)%time_method,&
                   & standard_name = input_fields(input_field_num)%standard_name,&
-                  & interp_method = input_fields(input_field_num)%interp_method)
+                  & interp_method = input_fields(input_field_num)%interp_method,&
+                  & attributes=output_fields(field_num)%attributes,&
+                  & num_attributes=output_fields(field_num)%num_attributes)
           ELSE
              output_fields(field_num)%f_type = write_field_meta_data(files(file)%file_unit,&
                   & output_fields(field_num)%output_name, axes(1:num_axes),&
@@ -1759,7 +1879,9 @@ SUBROUTINE opening_file(file, time)
                   & input_fields(input_field_num)%range, output_fields(field_num)%pack,&
                   & input_fields(input_field_num)%missing_value, avg_name = avg,&
                   & time_method=output_fields(field_num)%time_method,&
-                  & standard_name = input_fields(input_field_num)%standard_name)
+                  & standard_name = input_fields(input_field_num)%standard_name,&
+                  & attributes=output_fields(field_num)%attributes,&
+                  & num_attributes=output_fields(field_num)%num_attributes)
           END IF
           ! NEED TO TAKE CARE OF TIME AVERAGING INFO TOO BOTH CASES
        ELSE
@@ -1772,7 +1894,9 @@ SUBROUTINE opening_file(file, time)
                   & avg_name = avg,&
                   & time_method=output_fields(field_num)%time_method,&
                   & standard_name = input_fields(input_field_num)%standard_name,&
-                  & interp_method = input_fields(input_field_num)%interp_method)
+                  & interp_method = input_fields(input_field_num)%interp_method,&
+                  & attributes=output_fields(field_num)%attributes,&
+                  & num_attributes=output_fields(field_num)%num_attributes)
           ELSE
              output_fields(field_num)%f_type = write_field_meta_data(files(file)%file_unit,&
                   & output_fields(field_num)%output_name, axes(1:num_axes),&
@@ -1781,7 +1905,9 @@ SUBROUTINE opening_file(file, time)
                   & input_fields(input_field_num)%range, output_fields(field_num)%pack,&
                   & avg_name = avg,&
                   & time_method=output_fields(field_num)%time_method,&
-                  & standard_name = input_fields(input_field_num)%standard_name)
+                  & standard_name = input_fields(input_field_num)%standard_name,&
+                  & attributes=output_fields(field_num)%attributes,&
+                  & num_attributes=output_fields(field_num)%num_attributes)
           END IF
        END IF
     END DO
@@ -1798,7 +1924,7 @@ SUBROUTINE opening_file(file, time)
             & "End time for average period", pack=pack_size)
        files(file)%f_avg_nitems = write_field_meta_data(files(file)%file_unit,&
             & avg_name // '_DT', time_axis_id,&
-            & TRIM(time_unit_list(files(file)%time_units)),& 
+            & TRIM(time_unit_list(files(file)%time_units)),&
             & "Length of average period", pack=pack_size)
     END IF
 
@@ -1808,11 +1934,11 @@ SUBROUTINE opening_file(file, time)
        CALL get_diag_axis( time_axis_id(1), time_name, time_units, time_longname,&
             & cart_name, dir, edges, Domain, DATA)
        CALL get_diag_axis( time_bounds_id(1), timeb_name, timeb_units, timeb_longname,&
-            & cart_name, dir, edges, Domain, DATA)     
+            & cart_name, dir, edges, Domain, DATA)
        files(file)%f_bounds =  write_field_meta_data(files(file)%file_unit,&
             & TRIM(time_name)//'_bounds', (/time_bounds_id,time_axis_id/),&
             & TRIM(time_unit_list(files(file)%time_units)),&
-            & TRIM(time_name)//' axis boundaries', pack=pack_size)      
+            & TRIM(time_name)//' axis boundaries', pack=pack_size)
     END IF
     ! Let lower levels know that all meta data has been sent
     CALL done_meta_data(files(file)%file_unit)
@@ -1827,7 +1953,7 @@ SUBROUTINE opening_file(file, time)
   END SUBROUTINE opening_file
   ! 
   ! 
-  
+
   ! 
   ! 
   !   
@@ -1863,31 +1989,31 @@ CHARACTER(len=128) FUNCTION get_time_string(filename, current_time)
     len = LEN_TRIM(filename)
     first_percent = INDEX(filename, '%')
     filetail = filename(first_percent:len)
-    ! compute year string 
+    ! compute year string
     position = INDEX(filetail, 'yr')
     IF ( position > 0 ) THEN
        width = filetail(position-1:position-1)
        yr1_s = yr1
        format(7:9) = width//'.'//width
-       WRITE(yr, format) yr1_s   
+       WRITE(yr, format) yr1_s
        yr2 = 0
-    ELSE  
+    ELSE
        yr = ' '
        yr2 = yr1 - 1
     END IF
-    ! compute month string 
+    ! compute month string
     position = INDEX(filetail, 'mo')
-    IF ( position > 0 ) THEN   
+    IF ( position > 0 ) THEN
        width = filetail(position-1:position-1)
-       mo1_s = yr2*12 + mo1  
+       mo1_s = yr2*12 + mo1
        format(7:9) = width//'.'//width
        WRITE(mo, format) mo1_s
     ELSE
        mo = ' '
     END IF
-    ! compute day string        
+    ! compute day string
     IF ( LEN_TRIM(mo) > 0 ) THEN ! month present
-       dy1_s = dy1 
+       dy1_s = dy1
        dy2 = dy1_s - 1
     ELSE IF ( LEN_TRIM(yr) >0 )  THEN ! no month, year present
        ! compute julian day
@@ -1905,11 +2031,11 @@ CHARACTER(len=128) FUNCTION get_time_string(filename, current_time)
        dy2 = dy1_s - 1
     ELSE ! no month, no year
        CALL get_time(current_time, abs_sec, abs_day)
-       dy1_s = abs_day  
-       dy2 = dy1_s 
+       dy1_s = abs_day
+       dy2 = dy1_s
     END IF
     position = INDEX(filetail, 'dy')
-    IF ( position > 0 ) THEN 
+    IF ( position > 0 ) THEN
        width = filetail(position-1:position-1)
        FORMAT(7:9) = width//'.'//width
        WRITE(dy, FORMAT) dy1_s
@@ -2064,7 +2190,7 @@ SUBROUTINE diag_data_out(file, field, dat, time, final_call_in, static_write_in)
          & (SIZE(dat,1)*SIZE(dat,2)*SIZE(dat,3))*(8/output_fields(field)%pack)
     IF ( .NOT.output_fields(field)%written_once ) output_fields(field)%written_once = .TRUE.
     ! *** inserted this line because start_dif < 0 for static fields ***
-    IF ( .NOT.output_fields(field)%static ) THEN 
+    IF ( .NOT.output_fields(field)%static ) THEN
        start_dif = get_date_dif(output_fields(field)%last_output, base_time,files(file)%time_units)
        IF ( .NOT.mix_snapshot_average_fields ) THEN
           end_dif = get_date_dif(output_fields(field)%next_output, base_time, files(file)%time_units)
@@ -2089,7 +2215,7 @@ SUBROUTINE diag_data_out(file, field, dat, time, final_call_in, static_write_in)
              CALL diag_field_out(files(file)%file_unit, files(file)%f_avg_nitems, dt_time(1:1,:,:,:), dif)
 
              ! Include boundary variable for CF compliance
-             CALL diag_field_out(files(file)%file_unit, files(file)%f_bounds, time_data(1:2,:,:,:), dif)         
+             CALL diag_field_out(files(file)%file_unit, files(file)%f_bounds, time_data(1:2,:,:,:), dif)
              EXIT
           END IF
        END IF
@@ -2131,7 +2257,7 @@ SUBROUTINE check_and_open(file, time, do_write)
     TYPE(time_type), INTENT(in) :: time
     LOGICAL, INTENT(out) :: do_write
 
-    IF ( time >= files(file)%start_time ) THEN 
+    IF ( time >= files(file)%start_time ) THEN
        IF ( files(file)%file_unit < 0 ) THEN ! need to open a new file
           CALL opening_file(file, time)
           do_write = .TRUE.
@@ -2139,16 +2265,16 @@ SUBROUTINE check_and_open(file, time, do_write)
           do_write = .TRUE.
           IF ( time > files(file)%close_time .AND. time < files(file)%next_open ) THEN
              do_write = .FALSE. ! file still open but receives NO MORE data
-          ELSE IF ( time > files(file)%next_open ) THEN ! need to close current file and open a new one 
+          ELSE IF ( time > files(file)%next_open ) THEN ! need to close current file and open a new one
              CALL write_static(file)  ! write all static fields and close this file
-             CALL opening_file(file, time)        
+             CALL opening_file(file, time)
              files(file)%start_time = files(file)%next_open
              files(file)%close_time =&
-                  & diag_time_inc(files(file)%start_time,files(file)%duration, files(file)%duration_units)  
+                  & diag_time_inc(files(file)%start_time,files(file)%duration, files(file)%duration_units)
              files(file)%next_open =&
                   & diag_time_inc(files(file)%next_open, files(file)%new_file_freq,&
                   & files(file)%new_file_freq_units)
-             IF ( files(file)%close_time > files(file)%next_open ) THEN 
+             IF ( files(file)%close_time > files(file)%next_open ) THEN
                 ! 
                 !    has close time GREATER than next_open time,
                 !   check file duration and frequency
@@ -2191,7 +2317,7 @@ SUBROUTINE write_static(file)
        IF ( .NOT.output_fields(i)%static ) CYCLE
        CALL diag_data_out(file, i, output_fields(i)%buffer, files(file)%last_flush, .TRUE., .TRUE.)
     END DO
-    ! Close up this file   
+    ! Close up this file
     CALL mpp_close(files(file)%file_unit)
     files(file)%file_unit = -1
   END SUBROUTINE write_static
@@ -2218,7 +2344,7 @@ SUBROUTINE check_duplicate_output_fields(err_msg)
 
     IF ( PRESENT(err_msg) ) err_msg=''
     ! Do the checking when more than 1 output_fileds present
-    IF ( num_output_fields <= 1 ) RETURN 
+    IF ( num_output_fields <= 1 ) RETURN
     err_msg_local = ''
 
     i_loop: DO i = 1, num_output_fields-1
@@ -2238,4 +2364,293 @@ SUBROUTINE check_duplicate_output_fields(err_msg)
     END IF
   END SUBROUTINE check_duplicate_output_fields
   ! 
+
+  ! 
+  !   
+  !     Allocates the atttype in out_field
+  !   
+  !   
+  !   
+  !     Allocates memory in out_field for the attributes.  Will FATAL if err_msg is not included
+  !     in the subroutine call.
+  !   
+  !   output field to allocate memory for attribute
+  !   Error message, passed back to calling function
+  SUBROUTINE attribute_init_field(out_field, err_msg)
+    TYPE(output_field_type), INTENT(inout) :: out_field
+    CHARACTER(LEN=*), INTENT(out), OPTIONAL :: err_msg
+
+    INTEGER :: istat
+
+    ! Need to initialize err_msg if present
+    IF ( PRESENT(err_msg) ) err_msg = ''
+
+    ! Allocate memory for the attributes
+    IF ( .NOT._ALLOCATED(out_field%attributes) ) THEN
+       ALLOCATE(out_field%attributes(max_field_attributes), STAT=istat)
+       IF ( istat.NE.0 ) THEN
+          ! 
+          !   Unable to allocate memory for attribute  to module/input_field /
+          ! 
+          IF ( fms_error_handler('diag_util_mod::attribute_init_field',&
+               & 'Unable to allocate memory for attributes', err_msg) ) THEN
+             RETURN
+          END IF
+       ELSE
+          ! Set equal to 0.  It will be increased when attributes added
+          out_field%num_attributes = 0
+       END IF
+    END IF
+  END SUBROUTINE attribute_init_field
+  ! 
+
+  ! 
+  !   
+  !     Prepends the attribute value to an already existing attribute.  If the
+  !     attribute isn't yet defined, then creates a new attribute
+  !   
+  !   
+  !   
+  !     Checks if the attribute already exists in the out_field.  If it exists,
+  !     then prepend the prepend_value, otherwise, create the attribute
+  !     with the prepend_value. err_msg indicates no duplicates found.
+  !   
+  !   output field that will get the attribute
+  !   Name of the attribute
+  !   Value to prepend
+  !   Error message, passed back to calling routine
+  SUBROUTINE prepend_attribute_field(out_field, att_name, prepend_value, err_msg)
+    TYPE(output_field_type), INTENT(inout) :: out_field
+    CHARACTER(len=*), INTENT(in) :: att_name, prepend_value
+    CHARACTER(len=*), INTENT(out) , OPTIONAL :: err_msg
+
+    INTEGER :: length, i, this_attribute
+    CHARACTER(len=512) :: err_msg_local
+
+    ! Initialize string characters
+    err_msg_local=''
+    IF ( PRESENT(err_msg) ) err_msg = ''
+
+    ! Make sure the attributes for this out field have been initialized
+    CALL attribute_init_field(out_field, err_msg_local)
+    IF ( TRIM(err_msg_local) .NE. '' ) THEN
+       IF ( fms_error_handler('diag_util_mod::prepend_attribute_field', TRIM(err_msg_local), err_msg) ) THEN
+          RETURN
+       END IF
+    END IF
+
+    ! Find if attribute exists
+    this_attribute = 0
+    DO i=1, out_field%num_attributes
+       IF ( TRIM(out_field%attributes(i)%name) .EQ. TRIM(att_name) ) THEN
+          this_attribute = i
+          EXIT
+       END IF
+    END DO
+
+    IF ( this_attribute > 0 ) THEN
+       IF ( out_field%attributes(this_attribute)%type .NE. NF90_CHAR ) THEN
+          ! 
+          !   Attribute  is not a character attribute.
+          ! 
+          IF ( fms_error_handler('diag_util_mod::prepend_attribute_field', &
+               & 'Attribute "'//TRIM(att_name)//'" is not a character attribute.',&
+               & err_msg) ) THEN
+             RETURN
+          END IF
+       END IF
+    ELSE
+       ! Defining a new attribute
+       ! Increase the number of field attributes
+       this_attribute = out_field%num_attributes + 1
+       IF ( this_attribute .GT. max_field_attributes ) THEN
+          ! 
+          !   Number of attributes exceeds max_field_attributes for attribute .
+          !   Increase diag_manager_nml:max_field_attributes.
+          ! 
+          IF ( fms_error_handler('diag_util_mod::prepend_attribute_field',&
+               & 'Number of attributes exceeds max_field_attributes for attribute "'&
+               & //TRIM(att_name)//'".  Increase diag_manager_nml:max_field_attributes.',&
+               & err_msg) ) THEN
+             RETURN
+          END IF
+       ELSE
+          out_field%num_attributes = this_attribute
+          ! Set name and type
+          out_field%attributes(this_attribute)%name = att_name
+          out_field%attributes(this_attribute)%type = NF90_CHAR
+          ! Initialize catt to a blank string, as len_trim doesn't always work on an uninitialized string
+          out_field%attributes(this_attribute)%catt = ''
+       END IF
+    END IF
+
+    ! Check if string is already included, and return if found
+    IF ( INDEX(TRIM(out_field%attributes(this_attribute)%catt), TRIM(prepend_value)).EQ.0 ) THEN
+       ! Check if new string length goes beyond the length of catt
+       length = LEN_TRIM(TRIM(prepend_value)//" "//TRIM(out_field%attributes(this_attribute)%catt))
+       IF ( length.GT.LEN(out_field%attributes(this_attribute)%catt) ) THEN
+          ! 
+          !   Prepend length for attribute  is longer than allowed.
+          ! 
+          IF ( fms_error_handler('diag_util_mod::prepend_attribute_field',&
+               & 'Prepend length for attribute "'//TRIM(att_name)//'" is longer than allowed.',&
+               & err_msg) ) THEN
+             RETURN
+          END IF
+       END IF
+       ! Set fields
+       out_field%attributes(this_attribute)%catt =&
+            & TRIM(prepend_value)//' '//TRIM(out_field%attributes(this_attribute)%catt)
+       out_field%attributes(this_attribute)%len = length
+    END IF
+  END SUBROUTINE prepend_attribute_field
+  ! 
+
+  ! 
+  !   
+  !     Allocates the atttype in out_file
+  !   
+  !   
+  !   
+  !     Allocates memory in out_file for the attributes.  Will FATAL if err_msg is not included
+  !     in the subroutine call.
+  !   
+  !   output file to allocate memory for attribute
+  !   Error message, passed back to calling function
+  SUBROUTINE attribute_init_file(out_file, err_msg)
+    TYPE(file_type), INTENT(inout) :: out_file
+    CHARACTER(LEN=*), INTENT(out), OPTIONAL :: err_msg
+
+    INTEGER :: istat
+
+    ! Initialize err_msg
+    IF ( PRESENT(err_msg) ) err_msg = ''
+
+    ! Allocate memory for the attributes
+    IF ( .NOT._ALLOCATED(out_file%attributes) ) THEN
+       ALLOCATE(out_file%attributes(max_field_attributes), STAT=istat)
+       IF ( istat.NE.0 ) THEN
+          ! 
+          !   Unable to allocate memory for file attributes
+          ! 
+          IF ( fms_error_handler('diag_util_mod::attribute_init_file', 'Unable to allocate memory for file attributes', err_msg) ) THEN
+             RETURN
+          END IF
+       ELSE
+          ! Set equal to 0.  It will be increased when attributes added
+          out_file%num_attributes = 0
+       END IF
+    END IF
+  END SUBROUTINE attribute_init_file
+  ! 
+
+  ! 
+  !   
+  !     Prepends the attribute value to an already existing attribute.  If the
+  !     attribute isn't yet defined, then creates a new attribute
+  !   
+  !   
+  !   
+  !     Checks if the attribute already exists in the out_file.  If it exists,
+  !     then prepend the prepend_value, otherwise, create the attribute
+  !     with the prepend_value. err_msg indicates no duplicates found.
+  !   
+  !   output file that will get the attribute
+  !   Name of the attribute
+  !   Value to prepend
+  !   Error message, passed back to calling routine
+  SUBROUTINE prepend_attribute_file(out_file, att_name, prepend_value, err_msg)
+    TYPE(file_type), INTENT(inout) :: out_file
+    CHARACTER(len=*), INTENT(in) :: att_name, prepend_value
+    CHARACTER(len=*), INTENT(out) , OPTIONAL :: err_msg
+
+    INTEGER :: length, i, this_attribute
+    CHARACTER(len=512) :: err_msg_local
+
+    ! Initialize string variables
+    err_msg_local = ''
+    IF ( PRESENT(err_msg) ) err_msg = ''
+
+    ! Make sure the attributes for this out file have been initialized
+    CALL attribute_init_file(out_file, err_msg_local)
+    IF ( TRIM(err_msg_local) .NE. '' ) THEN
+       IF ( fms_error_handler('diag_util_mod::prepend_attribute_file', TRIM(err_msg_local), err_msg) ) THEN
+          RETURN
+       END IF
+    END IF
+
+    ! Find if attribute exists
+    this_attribute = 0
+    DO i=1, out_file%num_attributes
+       IF ( TRIM(out_file%attributes(i)%name) .EQ. TRIM(att_name) ) THEN
+          this_attribute = i
+          EXIT
+       END IF
+    END DO
+
+    IF ( this_attribute > 0 ) THEN
+       IF ( out_file%attributes(this_attribute)%type .NE. NF90_CHAR ) THEN
+          ! 
+          !   Attribute  is not a character attribute.
+          ! 
+          IF ( fms_error_handler('diag_util_mod::prepend_attribute_file',&
+               & 'Attribute "'//TRIM(att_name)//'" is not a character attribute.',&
+               & err_msg) ) THEN
+             RETURN
+          END IF
+       END IF
+    ELSE
+       ! Defining a new attribute
+       ! Increase the number of file attributes
+       this_attribute = out_file%num_attributes + 1
+       IF ( this_attribute .GT. max_file_attributes ) THEN
+          ! 
+          !   Number of attributes exceeds max_file_attributes for attribute .
+          !   Increase diag_manager_nml:max_file_attributes.
+          ! 
+          IF ( fms_error_handler('diag_util_mod::prepend_attribute_file',&
+               & 'Number of attributes exceeds max_file_attributes for attribute "'&
+               &//TRIM(att_name)//'".  Increase diag_manager_nml:max_file_attributes.',&
+               & err_msg) ) THEN
+             RETURN
+          END IF
+       ELSE
+          out_file%num_attributes = this_attribute
+          ! Set name and type
+          out_file%attributes(this_attribute)%name = att_name
+          out_file%attributes(this_attribute)%type = NF90_CHAR
+          ! Initialize catt to a blank string, as len_trim doesn't always work on an uninitialized string
+          out_file%attributes(this_attribute)%catt = ''
+       END IF
+    END IF
+
+    ! Only add string only if not already defined
+    IF ( INDEX(TRIM(out_file%attributes(this_attribute)%catt), TRIM(prepend_value)).EQ.0 ) THEN
+       ! Check if new string length goes beyond the length of catt
+       length = LEN_TRIM(TRIM(prepend_value)//" "//TRIM(out_file%attributes(this_attribute)%catt))
+       IF ( length.GT.LEN(out_file%attributes(this_attribute)%catt) ) THEN
+          ! 
+          !   Prepend length for attribute  is longer than allowed.
+          ! 
+          IF ( fms_error_handler('diag_util_mod::prepend_attribute_file',&
+               & 'Prepend length for attribute "'//TRIM(att_name)//'" is longer than allowed.',&
+               & err_msg) ) THEN
+             RETURN
+          END IF
+       END IF
+       ! Set files
+       out_file%attributes(this_attribute)%catt =&
+            & TRIM(prepend_value)//' '//TRIM(out_file%attributes(this_attribute)%catt)
+       out_file%attributes(this_attribute)%len = length
+    END IF
+  END SUBROUTINE prepend_attribute_file
+  ! 
 END MODULE diag_util_mod
diff --git a/src/shared/drifters/cloud_interpolator.F90 b/src/shared/drifters/cloud_interpolator.F90
index 82a0428635..05f8cf74c8 100644
--- a/src/shared/drifters/cloud_interpolator.F90
+++ b/src/shared/drifters/cloud_interpolator.F90
@@ -13,8 +13,8 @@ MODULE cloud_interpolator_mod
   public :: cld_ntrp_expand_index, cld_ntrp_contract_indices
 #endif
 
-character(128), parameter :: version = '$Id: cloud_interpolator.F90,v 14.0 2007/03/15 22:38:35 fms Exp $'
-real, parameter           :: tol = 10*epsilon(1.)
+character(128), parameter :: version = '$Id$'
+real, parameter           :: tol = 10.0*epsilon(1.)
 
 CONTAINS
 
@@ -77,7 +77,7 @@ _PURE subroutine cld_ntrp_linear_cell_interp(fvals, ts, f, ier)
     real    basis
 
     ier = 0
-    f   = 0
+    f   = 0.
     nd   = size(ts)
     if(size(fvals) /= 2**nd) then
        ier = 1
@@ -85,10 +85,10 @@ _PURE subroutine cld_ntrp_linear_cell_interp(fvals, ts, f, ier)
     endif
     
     do Ic = 0, 2**nd - 1
-       basis = 1
+       basis = 1.
        call cld_ntrp_expand_index(Ic, ie, iflag)
        do j = 1, nd
-          basis = basis * (  (1-ie(j))*(1.0-ts(j)) + ie(j)*ts(j) )
+          basis = basis * (  (1.0-real(ie(j)))*(1.0-ts(j)) + real(ie(j))*ts(j) )
        end do
        f = f + fvals(Ic)*basis
     end do
@@ -133,7 +133,7 @@ _PURE subroutine cld_ntrp_locate_cell(axis, x, index, ier)
        return
     endif
 
-    index = floor((n-1)*(x - axis_1)/(axis_n-axis_1)) + 1
+    index = floor(real(n-1)*(x - axis_1)/(axis_n-axis_1)) + 1
     index  = min(n-1, index)
     index1 = index+1
 
@@ -221,7 +221,7 @@ _PURE subroutine cld_ntrp_get_cell_values(nsizes, fnodes, indices, fvals, ier)
     integer id, nt, nd, flat_index, Ic, iflag
     integer, dimension(size(nsizes)) :: cell_indices, node_indices
     ier = 0
-    fvals = 0
+    fvals = 0.
 
     nd = size(nsizes)
     if(nd /= size(indices)) then
diff --git a/src/shared/drifters/drifters.F90 b/src/shared/drifters/drifters.F90
index bc40760037..a5b1168c76 100644
--- a/src/shared/drifters/drifters.F90
+++ b/src/shared/drifters/drifters.F90
@@ -1,17 +1,18 @@
 !FDOC_TAG_GFDL fdoc.pl generated xml skeleton
-! $Id: drifters.F90,v 20.0 2013/12/14 00:19:02 fms Exp $
+! $Id$
 
 #include 
 #include "fms_switches.h"
 #define _FLATTEN(A) reshape((A), (/size((A))/) )
 
 module drifters_mod
-! 
+! 
 !   Alexander Pletzer
 ! 
 ! 
 !   
 ! 
+! 
 ! 
 !   
 ! 
@@ -109,7 +110,7 @@ module drifters_mod
   public :: drifters_print_checksums, drifters_save, drifters_write_restart, drifters_distribute
 
   integer, parameter, private :: MAX_STR_LEN = 128
-  character(len=MAX_STR_LEN), parameter, private :: version = '$Id: drifters.F90,v 20.0 2013/12/14 00:19:02 fms Exp $'
+  character(len=MAX_STR_LEN), parameter, private :: version = '$Id$'
   real :: DRFT_EMPTY_ARRAY(0)
 
   type drifters_type
diff --git a/src/shared/drifters/drifters_combine b/src/shared/drifters/drifters_combine
index 297effcc1b..f2ef5e6e0f 100755
--- a/src/shared/drifters/drifters_combine
+++ b/src/shared/drifters/drifters_combine
@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 
-VERSION = "$Id: drifters_combine,v 13.0 2006/03/28 21:38:16 fms Exp $"
+VERSION = "$Id$"
 
 from Scientific.IO.NetCDF import NetCDFFile
 import sys
diff --git a/src/shared/drifters/drifters_comm.F90 b/src/shared/drifters/drifters_comm.F90
index c60dec4c02..0a7d674e2d 100644
--- a/src/shared/drifters/drifters_comm.F90
+++ b/src/shared/drifters/drifters_comm.F90
@@ -1,7 +1,7 @@
 #include 
 #include "fms_switches.h"
 
-! $Id: drifters_comm.F90,v 20.0 2013/12/14 00:19:06 fms Exp $
+! $Id$
 
 module drifters_comm_mod
 
@@ -834,7 +834,7 @@ program main
      
      if(drfts%np > 0) then 
         do i=1,drfts%np
-           print '(a,i2,a,i3,a,i3,a, i3, a,2f10.6)', 'PE: ',pe, ' it=', it, ' np=', drfts%np, ' ip=', i, &
+           print '(a,i6,a,i3,a,i3,a, i3, a,2f10.6)', 'PE: ',pe, ' it=', it, ' np=', drfts%np, ' ip=', i, &
                 & ' x,y=', drfts%positions(1,i), drfts%positions(2,i)
         enddo
      endif
diff --git a/src/shared/drifters/drifters_compute_k.h b/src/shared/drifters/drifters_compute_k.h
index 084426cb79..90842efb54 100644
--- a/src/shared/drifters/drifters_compute_k.h
+++ b/src/shared/drifters/drifters_compute_k.h
@@ -1,5 +1,5 @@
 ! -*-f90-*-
-! $Id: drifters_compute_k.h,v 13.0 2006/03/28 21:38:20 fms Exp $
+! $Id$
 
 subroutine drifters_compute_k_XXX(self, positions, u, v, &
 #if _DIMS >= 3
diff --git a/src/shared/drifters/drifters_core.F90 b/src/shared/drifters/drifters_core.F90
index 4c3ac36594..d59699f2a9 100644
--- a/src/shared/drifters/drifters_core.F90
+++ b/src/shared/drifters/drifters_core.F90
@@ -1,4 +1,4 @@
-! $Id: drifters_core.F90,v 14.0 2007/03/15 22:38:50 fms Exp $
+! $Id$
 !
 ! nf95 -r8 -g -I ~/regression/ia64/23-Jun-2005/CM2.1U_Control-1990_E1.k32pe/include/ -D_TEST_DRIFTERS -D_F95 quicksort.F90 drifters_core.F90
 
@@ -16,7 +16,7 @@ module drifters_core_mod
 
   ! Globals
   integer, parameter, private   :: MAX_STR_LEN = 128
-  character(MAX_STR_LEN), parameter, private :: version = '$Id: drifters_core.F90,v 14.0 2007/03/15 22:38:50 fms Exp $'
+  character(MAX_STR_LEN), parameter, private :: version = '$Id$'
 
   type drifters_core_type
      ! Be sure to update drifters_core_new, drifters_core_del and drifters_core_copy_new
@@ -49,7 +49,7 @@ subroutine drifters_core_new(self, nd, npdim, ermesg)
 
     allocate(self%positions(nd, npdim), stat=iflag)
     if(iflag/=0) ier = ier + 1
-    self%positions   = 0
+    self%positions   = 0.
 
     allocate(self%ids(npdim), stat=iflag)
     if(iflag/=0) ier = ier + 1
@@ -129,7 +129,7 @@ subroutine drifters_core_resize(self, npdim, ermesg)
 
     allocate(self%positions(self%nd, npdim), stat=iflag)
     allocate(self%ids(npdim), stat=iflag)
-    self%positions = 0
+    self%positions = 0.0
     ! default id numbers
     self%ids       = (/ (i, i=1,npdim) /)
     self%positions(:, 1:self%np) = positions
diff --git a/src/shared/drifters/drifters_input.F90 b/src/shared/drifters/drifters_input.F90
index 1868d55b67..ea3e2a73bf 100644
--- a/src/shared/drifters/drifters_input.F90
+++ b/src/shared/drifters/drifters_input.F90
@@ -1,4 +1,4 @@
-! $Id: drifters_input.F90,v 14.0 2007/03/15 22:38:53 fms Exp $
+! $Id$
 
 #include 
 
@@ -11,7 +11,7 @@ module drifters_input_mod
 
   ! Globals
   integer, parameter, private   :: MAX_STR_LEN = 128
-  character(MAX_STR_LEN), parameter, private :: version = '$Id: drifters_input.F90,v 14.0 2007/03/15 22:38:53 fms Exp $'
+  character(MAX_STR_LEN), parameter, private :: version = '$Id$'
   character, parameter, private :: SEPARATOR = ' '
 
   type drifters_input_type
diff --git a/src/shared/drifters/drifters_io.F90 b/src/shared/drifters/drifters_io.F90
index f120bdf52c..7002a04366 100644
--- a/src/shared/drifters/drifters_io.F90
+++ b/src/shared/drifters/drifters_io.F90
@@ -1,4 +1,4 @@
-! $Id: drifters_io.F90,v 14.0 2007/03/15 22:38:56 fms Exp $
+! $Id$
 
 !!#include 
 
@@ -13,9 +13,9 @@ module drifters_io_mod
   ! Globals
   integer, parameter, private   :: MAX_STR_LEN = 128
   character(MAX_STR_LEN), parameter, private :: &
-       & version = '$Id: drifters_io.F90,v 14.0 2007/03/15 22:38:56 fms Exp $'
+       & version = '$Id$'
 
-  real :: drfts_eps_t = 10*epsilon(1.)
+  real :: drfts_eps_t = 10.*epsilon(1.)
   
 
   type drifters_io_type
diff --git a/src/shared/drifters/drifters_push.h b/src/shared/drifters/drifters_push.h
index ddd2f0e031..d7b51c36a7 100644
--- a/src/shared/drifters/drifters_push.h
+++ b/src/shared/drifters/drifters_push.h
@@ -1,5 +1,5 @@
 ! -*-f90-*-
-! $Id: drifters_push.h,v 13.0 2006/03/28 21:38:35 fms Exp $
+! $Id$
 !============================================================================
 subroutine drifters_push_XXX(self, u, v, &
 #if _DIMS >= 3
diff --git a/src/shared/drifters/drifters_set_field.h b/src/shared/drifters/drifters_set_field.h
index 4c79c9e202..2e388d8e5b 100644
--- a/src/shared/drifters/drifters_set_field.h
+++ b/src/shared/drifters/drifters_set_field.h
@@ -1,5 +1,5 @@
 ! -*-f90-*-
-! $Id: drifters_set_field.h,v 13.0 2006/03/28 21:38:37 fms Exp $
+! $Id$
 
 subroutine drifters_set_field_XXX(self, index_field, x, y, &
 #if _DIMS >= 3
diff --git a/src/shared/drifters/readme.txt b/src/shared/drifters/readme.txt
index d4e3484e72..b51dca7824 100644
--- a/src/shared/drifters/readme.txt
+++ b/src/shared/drifters/readme.txt
@@ -1,9 +1,9 @@
 Drifters
 ========
 
-$Id: readme.txt,v 13.0 2006/03/28 21:38:43 fms Exp $
+$Id$
 
-Alexander.Pletzer
+Alexander.Pletzer@noaa.gov
 
 Overview:
 ---------
diff --git a/src/shared/exchange/stock_constants.F90 b/src/shared/exchange/stock_constants.F90
index 972c8affbf..50b63889b2 100644
--- a/src/shared/exchange/stock_constants.F90
+++ b/src/shared/exchange/stock_constants.F90
@@ -7,7 +7,7 @@ module stock_constants_mod
 
   implicit none
 
-  character(len=128), parameter :: version = '$Id: stock_constants.F90,v 17.0 2009/07/21 03:19:07 fms Exp $'
+  character(len=128), parameter :: version = '$Id$'
 
 
   integer,public,    parameter                :: NELEMS=3
diff --git a/src/shared/exchange/xgrid.F90 b/src/shared/exchange/xgrid.F90
index c1c0f6ae5a..fa1da52499 100644
--- a/src/shared/exchange/xgrid.F90
+++ b/src/shared/exchange/xgrid.F90
@@ -38,18 +38,19 @@
 !             PE.  For the make_exchange_reproduce option, a special side 1 get
 !             is used.  This get communicates individual exchange cells.  The
 !             cells are summed in the order they appear in the grid spec. file.
-!                                    Michael Winton (Michael.Winton) Oct 2001
+!                                    Michael Winton (Michael.Winton@noaa.gov) Oct 2001
 !
 !~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 module xgrid_mod
 
-! 
+! 
 !   Michael Winton
 ! 
-! 
+! 
 !   Zhi Liang
 ! 
 
+! 
 
 ! 
 !    xgrid_mod implements exchange grids for coupled models running on
@@ -117,8 +118,8 @@ module xgrid_mod
                            mpp_clock_begin, mpp_clock_end, MPP_CLOCK_SYNC,    &
                            COMM_TAG_1, COMM_TAG_2, COMM_TAG_3, COMM_TAG_4,    &
                            COMM_TAG_5, COMM_TAG_6, COMM_TAG_7, COMM_TAG_8,    &
-			   COMM_TAG_9, COMM_TAG_10
-use mpp_mod,         only: input_nml_file, mpp_set_current_pelist, mpp_sum, mpp_sync        
+                           COMM_TAG_9, COMM_TAG_10
+use mpp_mod,         only: input_nml_file, mpp_set_current_pelist, mpp_sum, mpp_sync
 use mpp_domains_mod, only: mpp_get_compute_domain, mpp_get_compute_domains, &
                            Domain2d, mpp_global_sum, mpp_update_domains,    &
                            mpp_modify_domain, mpp_get_data_domain, XUPDATE, &
@@ -175,21 +176,16 @@ module xgrid_mod
 !     information. The purpose of this namelist is to improve performance of setup_xmap when running
 !     on highr processor count and solve receiving size mismatch issue on high processor count.
 !     Try to set nsubset = mpp_npes/MPI_rank_per_node.
-!     This parameter is not used when do_alltoall and do_alltoallv are enabled.
-!   
-!   
-!    Use the MPI_Alltoall collective in place of fixed-size point-to-point
-!    operations during the initial exchange grid calculation.  This parameter
-!    removes any issues due to large numbers of messages (such as hangs) and
-!    shows improved performance at higher PE counts.  When enabled, the nsubset
-!    parameter is ignored.
+!
+!     This parameter is not used when `collective_setup` is enabled.
 !   
-!   
-!    Use the MPI_Alltoallv collective in place of variable-size point-to-point
-!    operations during the initial exchange grid calculation.  This parameter
-!    removes any issues due to large numbers of messages (such as hangs) and
-!    shows improved performance at higher PE counts.  When enabled, the nsubset
-!    parameter is ignored.
+!   
+!    Use the MPI_Alltoall collectives in place of point-to-point operations
+!    during the xgrid setup.  This parameter can resolve potential issues
+!    related to large numbers of messages, such as model hangs.  It also shows
+!    improved performance at higher PE counts.
+!
+!    When enabled, the `nsubset` parameter is ignored.
 !   
 logical :: make_exchange_reproduce = .false. ! exactly same on different # PEs
 logical :: xgrid_log = .false. 
@@ -198,10 +194,10 @@ module xgrid_mod
 logical :: xgrid_clocks_on = .false.
 logical :: monotonic_exchange = .false.
 integer :: nsubset = 0 ! 0 means mpp_npes()
-logical :: do_alltoall = .true.
-logical :: do_alltoallv = .true.
-namelist /xgrid_nml/ make_exchange_reproduce, interp_method, debug_stocks, xgrid_log, xgrid_clocks_on, &
-    monotonic_exchange, nsubset, do_alltoall, do_alltoallv
+logical :: collective_setup = .true.
+namelist /xgrid_nml/ &
+    make_exchange_reproduce, interp_method, debug_stocks, xgrid_log, &
+    xgrid_clocks_on, monotonic_exchange, nsubset, collective_setup
 ! 
 logical :: init = .true.
 integer :: remapping_method
@@ -285,6 +281,7 @@ module xgrid_mod
 
 type xcell_type
   integer :: i1, j1, i2, j2 ! indices of cell in model arrays on both sides
+  integer :: recv_pos       ! position in the receive buffer.
   integer :: pe             ! other side pe that has this cell
   integer :: tile           ! tile index of side 1 mosaic.
   real    :: area           ! geographic area of exchange cell
@@ -363,11 +360,13 @@ module xgrid_mod
    integer          :: count
    integer          :: pe
    integer          :: buffer_pos
-   integer, pointer :: i(:)  =>NULL()
-   integer, pointer :: j(:)  =>NULL()
-   integer, pointer :: tile(:)  =>NULL()
-   real,    pointer :: di(:) =>NULL()
-   real,    pointer :: dj(:) =>NULL()
+   integer, _ALLOCATABLE :: i(:) _NULL
+   integer, _ALLOCATABLE :: j(:) _NULL
+   integer, _ALLOCATABLE :: g(:) _NULL
+   integer, _ALLOCATABLE :: xLoc(:) _NULL
+   integer, _ALLOCATABLE :: tile(:) _NULL
+   real,    _ALLOCATABLE :: di(:) _NULL
+   real,    _ALLOCATABLE :: dj(:) _NULL
 end type overlap_type
 
 type comm_type
@@ -415,11 +414,12 @@ module xgrid_mod
   integer, pointer, dimension(:) :: ind_put1 =>NULL() ! indx for side1 put and side 2get.
   type(comm_type), pointer       :: put1 =>NULL()      ! for put_1_to_xgrid
   type(comm_type), pointer       :: get1 =>NULL()      ! for get_1_from_xgrid
+  type(comm_type), pointer       :: get1_repro =>NULL()! for get_1_from_xgrid_repro
 end type xmap_type
 
 !-----------------------------------------------------------------------
- character(len=128) :: version = '$Id: xgrid.F90,v 20.0 2013/12/14 00:19:20 fms Exp $'
- character(len=128) :: tagname = '$Name: tikal $'
+ character(len=128) :: version = '$Id$'
+ character(len=128) :: tagname = '$Name$'
 
  real, parameter                              :: EPS = 1.0e-10
  real, parameter                              :: LARGE_NUMBER = 1.e20
@@ -427,6 +427,7 @@ module xgrid_mod
  integer :: id_put_1_to_xgrid_order_1 = 0
  integer :: id_put_1_to_xgrid_order_2 = 0
  integer :: id_get_1_from_xgrid = 0
+ integer :: id_get_1_from_xgrid_repro = 0
  integer :: id_get_2_from_xgrid = 0
  integer :: id_put_2_to_xgrid = 0
  integer :: id_setup_xmap = 0
@@ -503,7 +504,7 @@ subroutine xgrid_init(remap_method)
 #endif
 
 !--------- write version number and namelist ------------------
-  call write_version_number()
+  call write_version_number (version, tagname)
 
   unit = stdlog ( )
   out_unit = stdout()
@@ -535,6 +536,7 @@ subroutine xgrid_init(remap_method)
      id_put_1_to_xgrid_order_1 = mpp_clock_id("put_1_to_xgrid_order_1", flags=MPP_CLOCK_SYNC)
      id_put_1_to_xgrid_order_2 = mpp_clock_id("put_1_to_xgrid_order_2", flags=MPP_CLOCK_SYNC)
      id_get_1_from_xgrid       = mpp_clock_id("get_1_from_xgrid", flags=MPP_CLOCK_SYNC) 
+     id_get_1_from_xgrid_repro = mpp_clock_id("get_1_from_xgrid_repro", flags=MPP_CLOCK_SYNC)
      id_get_2_from_xgrid       = mpp_clock_id("get_2_from_xgrid", flags=MPP_CLOCK_SYNC) 
      id_put_2_to_xgrid         = mpp_clock_id("put_2_to_xgrid", flags=MPP_CLOCK_SYNC) 
      id_setup_xmap             = mpp_clock_id("setup_xmap", flags=MPP_CLOCK_SYNC) 
@@ -729,7 +731,7 @@ subroutine load_xgrid (xmap, grid, grid_file, grid1_id, grid_id, tile1, tile2, u
         ! check the units of "xgrid_area 
         call get_var_att_value(grid_file, "xgrid_area", "units", attvalue)
         if( trim(attvalue) == 'm2' ) then
-           garea = 4*PI*RADIUS*RADIUS;
+           garea = 4.0*PI*RADIUS*RADIUS;
            area_tmp = tmp(:,1)/garea
         else if( trim(attvalue) == 'none' ) then
            area_tmp = tmp(:,1)
@@ -865,9 +867,9 @@ subroutine load_xgrid (xmap, grid, grid_file, grid1_id, grid_id, tile1, tile2, u
 
      !--- send the size of the data on side 1 to be sent over.
 
-     if (do_alltoall) then
-        call mpp_alltoall(nsend1, nrecv1)
-        call mpp_alltoall(nsend2, nrecv2)
+     if (collective_setup) then
+        call mpp_alltoall(nsend1, 1, nrecv1, 1)
+        call mpp_alltoall(nsend2, 1, nrecv2, 1)
      else
         do n = 0, npes-1
            p = mod(mypos+npes-n, npes)
@@ -910,7 +912,7 @@ subroutine load_xgrid (xmap, grid, grid_file, grid1_id, grid_id, tile1, tile2, u
      nxgrid2 = sum(nrecv2)
      if(nxgrid1>0 .OR. nxgrid2>0) allocate(recv_buffer(nxgrid1*nset1+nxgrid2*nset2))
 
-     if (do_alltoallv) then
+     if (collective_setup) then
         ! Construct the send and receive counters
         send_cnt(:) = nset1 * nsend1(:) + nset2 * nsend2(:)
         recv_cnt(:) = nset1 * nrecv1(:) + nset2 * nrecv2(:)
@@ -1042,7 +1044,7 @@ subroutine load_xgrid (xmap, grid, grid_file, grid1_id, grid_id, tile1, tile2, u
         deallocate(x_local)
      else
         allocate( grid%x( grid%size ) )
-        grid%x%di = 0; grid%x%dj = 0
+        grid%x%di = 0.0; grid%x%dj = 0.0
      end if
   end if
 
@@ -1059,7 +1061,7 @@ subroutine load_xgrid (xmap, grid, grid_file, grid1_id, grid_id, tile1, tile2, u
            if(scale_exist) then
               grid%x(ll)%scale = scale(l)
            else
-              grid%x(ll)%scale = 1
+              grid%x(ll)%scale = 1.0
            endif
            if(use_higher_order) then
               grid%x(ll)%di  = di(l)
@@ -1193,7 +1195,7 @@ subroutine load_xgrid (xmap, grid, grid_file, grid1_id, grid_id, tile1, tile2, u
         deallocate(x_local)
      else
         allocate( grid%x_repro( grid%size_repro ) )
-        grid%x_repro%di = 0; grid%x_repro%dj = 0
+        grid%x_repro%di = 0.0; grid%x_repro%dj = 0.0
      end if
      do l=1,nxgrid1
         if (in_box(i1_side1(l),j1_side1(l), grid1%is_me,grid1%ie_me, grid1%js_me,grid1%je_me) ) then
@@ -1374,7 +1376,7 @@ subroutine get_area_elements(file, name, domain, data)
      call error_mesg('xgrid_mod', 'no field named '//trim(name)//' in grid file '//trim(file)// &
                      ' Will set data to negative values...', NOTE)
      ! area elements no present in grid_spec file, set to negative values....
-     data = -1
+     data = -1.0
   endif    
 
 end subroutine get_area_elements
@@ -1833,6 +1835,11 @@ subroutine setup_xmap(xmap, grid_ids, grid_domains, grid_file, atm_grid)
 
   call set_comm_put1(xmap)
 
+  if(make_exchange_reproduce) then
+    allocate(xmap%get1_repro)
+    call set_comm_get1_repro(xmap)
+  endif
+
   call mpp_clock_end(id_set_comm)
 
   call mpp_clock_begin(id_regen)
@@ -1841,12 +1848,12 @@ subroutine setup_xmap(xmap, grid_ids, grid_domains, grid_file, atm_grid)
 
   call mpp_clock_begin(id_conservation_check)
 
-  xxx = conservation_check(grid1%area*0+1.0, grid1%id, xmap)
+  xxx = conservation_check(grid1%area*0.0+1.0, grid1%id, xmap)
   write(out_unit,* )"Checked data is array of constant 1"
   write(out_unit,* )grid1%id,'(',xmap%grids(:)%id,')=', xxx 
 
   do g=2,size(xmap%grids(:))
-     xxx = conservation_check(xmap%grids(g)%frac_area*0+1.0, xmap%grids(g)%id, xmap )
+     xxx = conservation_check(xmap%grids(g)%frac_area*0.0+1.0, xmap%grids(g)%id, xmap )
      write( out_unit,* )xmap%grids(g)%id,'(',xmap%grids(:)%id,')=', xxx 
   enddo
   ! create an random number 2d array
@@ -1933,7 +1940,7 @@ function get_nest_contact(mosaic_file, tile_nest_out, tile_parent_out, is_nest_o
     t2 = tile2(n);
     ! For nesting, the contact index of one tile must match its global domain 
     if( (nx(t1) .NE. nx1_contact .OR. ny(t1) .NE. ny1_contact ) .AND. &
-	(nx(t2) .NE. nx2_contact .OR. ny(t2) .NE. ny2_contact ) ) cycle
+        (nx(t2) .NE. nx2_contact .OR. ny(t2) .NE. ny2_contact ) ) cycle
     if(nx1_contact == nx2_contact .AND. ny1_contact == ny2_contact) then
       call error_mesg('xgrid_mod', 'There is no refinement for the overlapping region', FATAL)
     endif
@@ -1976,7 +1983,102 @@ function get_nest_contact(mosaic_file, tile_nest_out, tile_parent_out, is_nest_o
   
 end function get_nest_contact
 
+!#######################################################################
+subroutine set_comm_get1_repro(xmap)
+  type (xmap_type), intent(inout) :: xmap
+  integer, dimension(xmap%npes) :: pe_ind, cnt
+  integer, dimension(0:xmap%npes-1) :: send_ind, recv_ind, pl
+  integer :: npes, nsend, nrecv, mypos
+  integer :: m, p, pos, n, g, l
+  type(comm_type), pointer, save :: comm => NULL()
+
+  comm => xmap%get1_repro
+  npes = xmap%npes
+
+  nrecv = 0
+  mypos = mpp_pe() - mpp_root_pe()
+  do m=0,npes-1 
+    p = mod(mypos+npes-m, npes)
+    if( xmap%recv_count_repro(p) > 0 ) then
+      nrecv = nrecv + 1
+      pe_ind(nrecv) = p
+    endif
+  enddo
+
+  comm%nrecv = nrecv
+  if( nrecv > 0 ) then
+    allocate(comm%recv(nrecv))
+    pos = 0
+    do n = 1, nrecv
+      p = pe_ind(n)
+      comm%recv(n)%count = xmap%recv_count_repro(p)
+      comm%recv(n)%pe = p + xmap%root_pe
+      comm%recv(n)%buffer_pos = pos 
+      pos = pos + comm%recv(n)%count
+    enddo
+  endif
+
+
+  ! send information
+  nsend = 0
+  mypos = mpp_pe() - mpp_root_pe()
+  do m=0,xmap%npes-1 
+    p = mod(mypos+m, npes)
+    if( xmap%send_count_repro(p) > 0 ) then
+      nsend = nsend + 1
+      pe_ind(nsend) = p
+      send_ind(p) = nsend
+    endif
+  enddo
+
+  comm%nsend = nsend
+  if( nsend > 0 ) then
+     allocate(comm%send(nsend))
+     pos = 0
+     cnt(:) = 0
+     do n = 1, nsend
+        p = pe_ind(n)
+        comm%send(n)%count = xmap%send_count_repro(p)
+        comm%send(n)%pe = p + xmap%root_pe 
+        comm%send(n)%buffer_pos = pos
+        pos = pos + comm%send(n)%count
+        allocate(comm%send(n)%i(comm%send(n)%count))
+        allocate(comm%send(n)%j(comm%send(n)%count))
+        allocate(comm%send(n)%g(comm%send(n)%count))
+        allocate(comm%send(n)%xLoc(comm%send(n)%count))
+     enddo
+
+     do g=2,size(xmap%grids(:))
+        do l=1,xmap%grids(g)%size ! index into this side 2 grid's patterns
+           p = xmap%grids(g)%x(l)%pe-xmap%root_pe 
+           n = send_ind(p)
+           cnt(n) = cnt(n) + 1
+           pos = cnt(n)
+           comm%send(n)%i(pos) = xmap%grids(g)%x(l)%i2
+           comm%send(n)%j(pos) = xmap%grids(g)%x(l)%j2
+           comm%send(n)%g(pos) = g
+        enddo
+     enddo
+     !--- make sure the count is correct
+     do n = 1, nsend
+        if( comm%send(n)%count .NE. cnt(n) ) call error_mesg('xgrid_mod', &
+             'comm%send(n)%count .NE. cnt(n)', FATAL)
+     enddo
+   endif
+
+   !--- set up the recv_pos for unpack the data.
+   pl(:) = 1
+   do g=2,size(xmap%grids(:))
+      do l=1,xmap%grids(g)%size_repro ! index into side1 grid's patterns
+         p = xmap%grids(g)%x_repro(l)%pe-xmap%root_pe
+         xmap%grids(g)%x_repro(l)%recv_pos = pl(p)
+         pl(p) = pl(p) + 1
+      end do
+   end do
+
+
 
+end subroutine set_comm_get1_repro
 
 !#######################################################################
 subroutine set_comm_get1(xmap)
@@ -2114,37 +2216,10 @@ subroutine set_comm_get1(xmap)
      enddo
   endif
 
-
   mypos = mpp_pe()-mpp_root_pe()
-!  do n = 0, npes-1
-!     p = mod(mypos+npes-n, npes)
-!     call mpp_recv(recv_size(p), glen=1, from_pe=pelist(p), block=.false., tag=COMM_TAG_3)
-!  enddo
-
-  !--- send data
-!  do n = 0, npes-1
-!     p = mod(mypos+n, npes)
-!     call mpp_send(send_size(p), plen=1, to_pe=pelist(p), tag=COMM_TAG_3)
-!  enddo
-
-!  call mpp_sync_self(check=EVENT_RECV)
-!  call mpp_sync_self()
-
-!  call mpp_sync()
-  
-!  do p = 0, npes-1
-        
-!     if(recv_size(p) .NE. xmap%your2my1_size(p)) then
-!        print*, "My = ", mpp_pe(), p, recv_size(p), xmap%your2my1_size(p)
-!          call error_mesg("xgrid_mod", &
-!             "recv_size(p) .NE. xmap%your2my1_size(p)", FATAL)
-!     endif
-!  end do
  
+  ! send/recv for get_1_from_xgrid_recv
   recv_size(:) = xmap%your2my1_size(:)
-
-
-  !--- set up for send for get_1_from_xgrid, also is recv for put_1_to_xgrid
   nsend = count( send_size> 0)  
   comm%nsend = nsend
   if(nsend>0) then
@@ -2230,7 +2305,7 @@ subroutine set_comm_get1(xmap)
      do p = 0, npes-1
         recv_buffer_pos(p) = buffer_pos
         buffer_pos = buffer_pos +  recv_size(p)
-     enddo      
+     enddo
      pos = 0
      buffer_pos = 0        
      do m=0,npes-1
@@ -2274,8 +2349,8 @@ subroutine set_comm_get1(xmap)
         endif
      enddo
   endif
-
   call mpp_sync_self()
+
   if(allocated(send_buf) ) deallocate(send_buf)
   if(allocated(recv_buf) ) deallocate(recv_buf)
   if(allocated(pelist)   ) deallocate(pelist)
@@ -2583,6 +2658,9 @@ subroutine regen(xmap)
   integer              :: tile1
   integer              :: ll
   logical              :: overlap_with_nest
+  integer              :: cnt(xmap%get1%nsend)
+  integer              :: i,j,n,xloc,pos,nsend,m,npes, mypos
+  integer              :: send_ind(0:xmap%npes-1)
 
   max_size = 0
 
@@ -2692,6 +2770,34 @@ subroutine regen(xmap)
      xmap%grids(g)%last_get = xmap%size_get2
   end do
 
+  !---set up information for get_1_from_xgrid_repro
+  if(make_exchange_reproduce .AND. xmap%get1_repro%nsend >0) then
+     xloc = 0
+     nsend = 0
+     npes = xmap%npes
+     mypos = mpp_pe() - mpp_root_pe()
+     cnt(:) = 0
+     do m=0,npes-1 
+        p = mod(mypos+m, npes)
+        if( xmap%send_count_repro(p) > 0 ) then
+          nsend = nsend + 1
+          send_ind(p) = nsend
+        endif
+     enddo
+     do g=2,size(xmap%grids(:))
+        do l=1,xmap%grids(g)%size ! index into this side 2 grid's patterns
+           i = xmap%grids(g)%x(l)%i2
+           j = xmap%grids(g)%x(l)%j2
+           p = xmap%grids(g)%x(l)%pe-xmap%root_pe 
+           n = send_ind(p) 
+           cnt(n) = cnt(n) + 1
+           pos = cnt(n) 
+           xmap%get1_repro%send(n)%xLoc(pos) = xloc
+           xloc = xloc + count(xmap%grids(g)%frac_area(i,j,:)/=0.0)
+        enddo
+     enddo
+  endif
+
 end subroutine regen
 
 !#######################################################################
@@ -3173,6 +3279,8 @@ subroutine put_1_to_xgrid_order_1(d_addrs, x_addrs, xmap, isize, jsize, xsize, l
      end do
   else
      start_pos = 0
+!$OMP parallel do default(none) shared(lsize,x_addrs,comm,recv_buffer,xmap) &
+!$OMP                          private(ptr_x,count,ibegin,istart,iend,pos,unpack_buffer)
      do l = 1, lsize
         ptr_x = x_addrs(l)
         do p = 1, comm%nrecv
@@ -3222,7 +3330,7 @@ subroutine put_1_to_xgrid_order_2(d_addrs, x_addrs, xmap, isize, jsize, xsize, l
   real                            :: recv_buffer(xmap%put1%recvsize*lsize*3)
   real                            :: send_buffer(xmap%put1%sendsize*lsize*3)
   real                            :: unpack_buffer(xmap%put1%recvsize*3)
-
+  logical                         :: on_west_edge, on_east_edge, on_south_edge, on_north_edge
   real, dimension(isize, jsize)   :: d
   real, dimension(xsize)          :: x
   pointer(ptr_d, d)
@@ -3236,26 +3344,34 @@ subroutine put_1_to_xgrid_order_2(d_addrs, x_addrs, xmap, isize, jsize, xsize, l
   isd = grid1%isd_me
   jsd = grid1%jsd_me
 
-  tmp = LARGE_NUMBER
+!$OMP parallel do default(none) shared(lsize,tmp,d_addrs,isize,jsize) private(ptr_d)
   do l = 1, lsize
+     tmp(:,:,l) = LARGE_NUMBER
      ptr_d = d_addrs(l)
      tmp(1:isize,1:jsize,l) = d(:,:)
   enddo
 
   if(grid1%is_latlon) then
      call mpp_update_domains(tmp,grid1%domain_with_halo)
+!$OMP parallel do default(none) shared(lsize,tmp,grid1,is,ie,js,je,isd,jsd,tmpx,tmpy)
      do l = 1, lsize
         tmpy(:,:,l) = grad_merid_latlon(tmp(:,:,l), grid1%lat, is, ie, js, je, isd, jsd)
         tmpx(:,:,l) = grad_zonal_latlon(tmp(:,:,l), grid1%lon, grid1%lat, is, ie, js, je, isd, jsd)
      enddo
   else
      call mpp_update_domains(tmp,grid1%domain)
+     on_west_edge  = (is==1)
+     on_east_edge  = (ie==grid1%im)
+     on_south_edge = (js==1)
+     on_north_edge = (je==grid1%jm)
+!$OMP parallel do default(none) shared(lsize,tmp,grid1,tmpx,tmpy, &
+!$OMP                                  on_west_edge,on_east_edge,on_south_edge,on_north_edge)
      do l = 1, lsize
-        call gradient_cubic(tmp(:,:,l), xmap%grids(1)%box%dx, xmap%grids(1)%box%dy, xmap%grids(1)%box%area, &
-                            xmap%grids(1)%box%edge_w, xmap%grids(1)%box%edge_e, xmap%grids(1)%box%edge_s,   &
-                            xmap%grids(1)%box%edge_n, xmap%grids(1)%box%en1, xmap%grids(1)%box%en2,         &
-                            xmap%grids(1)%box%vlon, xmap%grids(1)%box%vlat, tmpx(:,:,l), tmpy(:,:,l),       &
-                            is==1, ie==grid1%im, js==1, je==grid1%jm)
+        call gradient_cubic(tmp(:,:,l), grid1%box%dx, grid1%box%dy, grid1%box%area,   &
+                            grid1%box%edge_w, grid1%box%edge_e, grid1%box%edge_s,     &
+                            grid1%box%edge_n, grid1%box%en1, grid1%box%en2,           &
+                            grid1%box%vlon, grid1%box%vlat, tmpx(:,:,l), tmpy(:,:,l), &
+                            on_west_edge, on_east_edge, on_south_edge, on_north_edge)
      enddo
   end if     
 
@@ -3275,6 +3391,7 @@ subroutine put_1_to_xgrid_order_2(d_addrs, x_addrs, xmap, isize, jsize, xsize, l
 
   !--- compute d_bar_max and d_bar_min.
   if(monotonic_exchange) then
+!$OMP parallel do default(none) shared(lsize,isize,jsize,d_bar_max,d_bar_min,d_max,d_min,tmp)
      do l = 1, lsize
         do j = 1, jsize
            do i = 1, isize
@@ -3391,11 +3508,14 @@ subroutine put_1_to_xgrid_order_2(d_addrs, x_addrs, xmap, isize, jsize, xsize, l
   else
      if( lsize == 1) then
         ptr_x = x_addrs(1)
+!$OMP parallel do default(none) shared(xmap,recv_buffer,ptr_x) private(pos)
         do l=1,xmap%size_put1
            pos = xmap%x1_put(l)%pos
            x(l) = recv_buffer(3*pos-2) + recv_buffer(3*pos-1)*xmap%x1_put(l)%dj + recv_buffer(3*pos)*xmap%x1_put(l)%di
         end do
      else
+!$OMP parallel do default(none) shared(lsize,comm,xmap,recv_buffer,x_addrs) &
+!$OMP                          private(ptr_x,pos,ibegin,istart,iend,count,unpack_buffer)
         do l = 1, lsize
            ptr_x = x_addrs(l)
            pos = 0
@@ -3454,7 +3574,6 @@ subroutine get_1_from_xgrid(d_addrs, x_addrs, xmap, isize, jsize, xsize, lsize)
   comm => xmap%get1
   grid1 => xmap%grids(1)
 
-
   do p = 1, comm%nrecv
      recv => comm%recv(p)
      msgsize = recv%count*lsize
@@ -3463,6 +3582,7 @@ subroutine get_1_from_xgrid(d_addrs, x_addrs, xmap, isize, jsize, xsize, lsize)
   enddo
 
   dg = 0.0;
+!$OMP parallel do default(none) shared(lsize,xmap,dg,x_addrs) private(dgp,ptr_x)
   do l = 1, lsize
      ptr_x = x_addrs(l)
      do i=1,xmap%size
@@ -3471,6 +3591,7 @@ subroutine get_1_from_xgrid(d_addrs, x_addrs, xmap, isize, jsize, xsize, lsize)
      enddo
   enddo
 
+
   !--- send the data
   buffer_pos = 0
   istart = 1
@@ -3496,16 +3617,19 @@ subroutine get_1_from_xgrid(d_addrs, x_addrs, xmap, isize, jsize, xsize, lsize)
   !--- unpack the buffer
   do l = 1, lsize
      ptr_d = d_addrs(l)
-     d = 0
+     d = 0.0
   enddo
   !--- To bitwise reproduce old results, first copy the data onto its own pe.
 
   do p = 1, comm%nrecv
      recv => comm%recv(p)
      count = recv%count
-     pos = recv%buffer_pos*lsize
+     buffer_pos = recv%buffer_pos*lsize
      if( recv%pe == xmap%me ) then
+!$OMP parallel do default(none) shared(lsize,recv,recv_buffer,buffer_pos,d_addrs,count) &
+!$OMP                          private(ptr_d,i,j,pos)
         do l = 1, lsize
+           pos = buffer_pos + (l-1)*count
            ptr_d = d_addrs(l)
            do n = 1,count
               i = recv%i(n)
@@ -3525,8 +3649,11 @@ subroutine get_1_from_xgrid(d_addrs, x_addrs, xmap, isize, jsize, xsize, lsize)
      if( recv%pe == xmap%me ) then
         cycle
      endif
-     pos = recv%buffer_pos*lsize
+     buffer_pos = recv%buffer_pos*lsize
+!$OMP parallel do default(none) shared(lsize,recv,recv_buffer,buffer_pos,d_addrs) &
+!$OMP                          private(ptr_d,i,j,pos)
      do l = 1, lsize
+        pos = buffer_pos + (l-1)*recv%count
         ptr_d = d_addrs(l)
         do n = 1, recv%count
            i = recv%i(n)
@@ -3540,6 +3667,7 @@ subroutine get_1_from_xgrid(d_addrs, x_addrs, xmap, isize, jsize, xsize, lsize)
   !
   ! normalize with side 1 grid cell areas
   !
+!$OMP parallel do default(none) shared(lsize,d_addrs,grid1) private(ptr_d)
   do l = 1, lsize
      ptr_d = d_addrs(l)
      d = d * grid1%area_inv
@@ -3557,66 +3685,64 @@ subroutine get_1_from_xgrid_repro(d_addrs, x_addrs, xmap, xsize, lsize)
   type (xmap_type),              intent(inout) :: xmap
   integer,                          intent(in) :: xsize, lsize
 
-  real,    dimension(:), allocatable :: x_psum
-  integer, dimension(:), allocatable :: pe_psum
-  integer                            :: l1, l2, l3, g, i, j, k, p, l
-  integer,  dimension(0:xmap%npes-1) :: pl
-  type (grid_type),    pointer, save :: grid =>NULL()
+  integer                            :: g, i, j, k, p, l, n, l2, m, l3
+  integer                            :: msgsize, buffer_pos, pos
+  type (grid_type), pointer, save :: grid =>NULL()
+  type(comm_type),  pointer, save :: comm => NULL()
+  type(overlap_type), pointer, save  :: send => NULL()
+  type(overlap_type), pointer, save  :: recv => NULL()
+    integer,  dimension(0:xmap%npes-1) :: pl, ml
   real                               :: recv_buffer(xmap%recv_count_repro_tot*lsize)
   real                               :: send_buffer(xmap%send_count_repro_tot*lsize)
-  real                               :: unpack_buffer(xmap%get1%recvsize)
   real                               :: d(xmap%grids(1)%is_me:xmap%grids(1)%ie_me, &
                                           xmap%grids(1)%js_me:xmap%grids(1)%je_me)
   real, dimension(xsize)             :: x
   pointer(ptr_d, d)
   pointer(ptr_x, x)
 
+  call mpp_clock_begin(id_get_1_from_xgrid_repro)
+  comm => xmap%get1_repro
   !--- pre-post receiving
-  l2 = 0;
-  do p=0,xmap%npes-1
-    l1 = l2 + 1
-    l2 = l2 + xmap%recv_count_repro(p)*lsize
-    if (xmap%recv_count_repro(p)>0) then ! can receive from myself
-      ! Force use of "scalar", integer pointer mpp interface.
-      call mpp_recv(recv_buffer(l1), glen=l2-l1+1, from_pe=p+xmap%root_pe, block=.false., tag=COMM_TAG_10);
-      pl(p) = l1
-    end if
-  end do
+  do p = 1, comm%nrecv
+     recv => comm%recv(p)
+     msgsize = recv%count*lsize
+     buffer_pos = recv%buffer_pos*lsize
+     call mpp_recv(recv_buffer(buffer_pos+1), glen=msgsize, from_pe = recv%pe, block=.false., tag=COMM_TAG_10)
+     n = recv%pe -xmap%root_pe
+     pl(n) = buffer_pos 
+     ml(n) = recv%count
+  enddo
 
-  allocate ( x_psum  (xmap%send_count_repro_tot*lsize) )
-  allocate ( pe_psum (xmap%send_count_repro_tot*lsize) )
-  x_psum = 0.0
-  l1 = 0 ! index into partition summed exchange grid variable
-  do l = 1, lsize
-     l2 = 0 ! index into exchange grid variable
-     ptr_x = x_addrs(l)
-     do g=2,size(xmap%grids(:))
-        do l3=1,xmap%grids(g)%size ! index into this side 2 grid's patterns
-           l1 = l1 + 1
-           do k=1,xmap%grids(g)%km
-              i = xmap%grids(g)%x(l3)%i2
-              j = xmap%grids(g)%x(l3)%j2
-              if (xmap%grids(g)%frac_area(i,j,k)/=0.0) then
-                 l2 = l2 + 1
-                 x_psum (l1) = x_psum(l1) + xmap%x1(l2)%area * x(l2)
-                 pe_psum(l1) = xmap%grids(g)%x(l3)%pe
-              end if
-           end do
-        end do
-     end do
+  !pack the data
+  send_buffer(:) = 0.0
+!$OMP parallel do default(none) shared(lsize,x_addrs,comm,xmap,send_buffer) &
+!$OMP                          private(ptr_x,i,j,g,l2,pos,send)
+  do p = 1, comm%nsend
+     pos = comm%send(p)%buffer_pos*lsize
+     send => comm%send(p)
+     do l = 1,lsize
+        ptr_x = x_addrs(l)
+        do n = 1, send%count
+           i = send%i(n)
+           j = send%j(n)
+           g = send%g(n)
+           l2 = send%xloc(n)
+           pos = pos + 1
+           do k =1, xmap%grids(g)%km
+             if(xmap%grids(g)%frac_area(i,j,k)/=0.0) then
+              l2 = l2+1
+              send_buffer(pos) = send_buffer(pos) + xmap%x1(l2)%area *x(l2)
+             endif
+           enddo
+         enddo
+      enddo
+   enddo
+
+  do p =1, comm%nsend
+     buffer_pos = comm%send(p)%buffer_pos*lsize
+     msgsize = comm%send(p)%count*lsize
+     call mpp_send(send_buffer(buffer_pos+1), plen=msgsize, to_pe=comm%send(p)%pe, tag=COMM_TAG_10)
   enddo
-  l2 = 0;
-
-  do p=0,xmap%npes-1
-    l1 = l2 + 1
-    l2 = l2 + xmap%send_count_repro(p)*lsize
-    if (xmap%send_count_repro(p)>0) then ! can send to myself
-      send_buffer(l1:l2) = pack(x_psum, pe_psum==p+xmap%root_pe)
-      ! Force use of "scalar", integer pointer mpp interface.
-      call mpp_send(send_buffer(l1), plen=l2-l1+1, to_pe=p+xmap%root_pe, tag=COMM_TAG_10);
-    end if
-  end do
-  deallocate ( x_psum, pe_psum)
 
   do l = 1, lsize
      ptr_d = d_addrs(l)
@@ -3625,8 +3751,9 @@ subroutine get_1_from_xgrid_repro(d_addrs, x_addrs, xmap, xsize, lsize)
 
   call mpp_sync_self(check=EVENT_RECV)
 
-
-  do l = 1, lsize  
+!$OMP parallel do default(none) shared(lsize,d_addrs,xmap,recv_buffer,pl,ml) &
+!$OMP                          private(ptr_d,grid,i,j,p,pos)
+  do l = 1, lsize
      ptr_d = d_addrs(l)
      do g=2,size(xmap%grids(:))
         grid => xmap%grids(g)
@@ -3634,8 +3761,8 @@ subroutine get_1_from_xgrid_repro(d_addrs, x_addrs, xmap, xsize, lsize)
            i = grid%x_repro(l3)%i1
            j = grid%x_repro(l3)%j1
            p = grid%x_repro(l3)%pe-xmap%root_pe
-           d(i,j) = d(i,j) + recv_buffer(pl(p))
-           pl(p) = pl(p) + 1
+           pos = pl(p) + (l-1)*ml(p) + grid%x_repro(l3)%recv_pos
+           d(i,j) = d(i,j) + recv_buffer(pos)
         end do
      end do
      ! normalize with side 1 grid cell areas
@@ -3644,6 +3771,8 @@ subroutine get_1_from_xgrid_repro(d_addrs, x_addrs, xmap, xsize, lsize)
 
   call mpp_sync_self()       
 
+  call mpp_clock_end(id_get_1_from_xgrid_repro)
+
 end subroutine get_1_from_xgrid_repro
 
 !#######################################################################
@@ -3736,7 +3865,7 @@ function conservation_check_side2(d, grid_id, xmap,remap_method) ! this one for
       endif
       call put_to_xgrid(d, grid_id, x_over, xmap)  ! put from this side 2
     else
-      call put_to_xgrid(0 * grid2%frac_area, grid2%id, x_over, xmap) ! zero rest
+      call put_to_xgrid(0.0 * grid2%frac_area, grid2%id, x_over, xmap) ! zero rest
     end if
   end do
 
@@ -3912,7 +4041,7 @@ subroutine stock_move_3d(from, to, grid_index, data, xmap, &
      return
   endif
 
-     from_dq = delta_t * 4*PI*radius**2 * sum( sum(xmap%grids(grid_index)%area * &
+     from_dq = delta_t * 4.0*PI*radius**2 * sum( sum(xmap%grids(grid_index)%area * &
           & sum(xmap%grids(grid_index)%frac_area * data, DIM=3), DIM=1))
      to_dq = from_dq
 
@@ -3923,8 +4052,8 @@ subroutine stock_move_3d(from, to, grid_index, data, xmap, &
   if(present(verbose).and.debug_stocks) then
      call mpp_sum(from_dq)
      call mpp_sum(to_dq)
-     from_dq = from_dq/(4*PI*radius**2)
-     to_dq   = to_dq  /(4*PI*radius**2)
+     from_dq = from_dq/(4.0*PI*radius**2)
+     to_dq   = to_dq  /(4.0*PI*radius**2)
      if(mpp_pe()==mpp_root_pe()) then
         write(stocks_file,'(a,es19.12,a,es19.12,a)') verbose, from_dq,' [*/m^2]'
      endif
@@ -3966,7 +4095,7 @@ subroutine stock_move_2d(from, to, grid_index, data, xmap, &
   if( .not. present(grid_index) .or. grid_index==1 ) then
 
      ! only makes sense if grid_index == 1
-     from_dq = delta_t * 4*PI*radius**2 * sum(sum(xmap%grids(1)%area * data, DIM=1))
+     from_dq = delta_t * 4.0*PI*radius**2 * sum(sum(xmap%grids(1)%area * data, DIM=1))
      to_dq = from_dq
 
   else
@@ -3983,8 +4112,8 @@ subroutine stock_move_2d(from, to, grid_index, data, xmap, &
   if(debug_stocks) then
      call mpp_sum(from_dq)
      call mpp_sum(to_dq)
-     from_dq = from_dq/(4*PI*radius**2)
-     to_dq   = to_dq  /(4*PI*radius**2)
+     from_dq = from_dq/(4.0*PI*radius**2)
+     to_dq   = to_dq  /(4.0*PI*radius**2)
      if(mpp_pe()==mpp_root_pe()) then
         write(stocks_file,'(a,es19.12,a,es19.12,a)') verbose, from_dq,' [*/m^2]'
      endif
@@ -4007,14 +4136,14 @@ subroutine stock_integrate_2d(data, xmap, delta_t, radius, res, ier)
   integer, intent(out)            :: ier
 
   ier = 0
-  res = 0
+  res = 0.0
 
   if(.not. associated(xmap%grids) ) then
      ier = 6
      return
   endif
 
-  res = delta_t * 4*PI*radius**2 * sum(sum(xmap%grids(1)%area * data, DIM=1))
+  res = delta_t * 4.0*PI*radius**2 * sum(sum(xmap%grids(1)%area * data, DIM=1))
 
 end subroutine stock_integrate_2d
 !#######################################################################
@@ -4065,7 +4194,7 @@ subroutine stock_print(stck, Time, comp_name, index, ref_value, radius, pelist)
 
   if(mpp_pe() == mpp_root_pe()) then
      ! normalize to 1 earth m^2
-     planet_area = 4*PI*radius**2
+     planet_area = 4.0*PI*radius**2
      f_value       = f_value     / planet_area
      c_value       = c_value     / planet_area
 
@@ -4170,6 +4299,7 @@ end module xgrid_mod
 ! 
 
 !      
+!      A  guide to grid coupling in FMS.
 !   
 !   
 !      A simple xgrid  example. 
diff --git a/src/shared/fft/fft.F90 b/src/shared/fft/fft.F90
index 529d3cddc1..7e8f51eab6 100644
--- a/src/shared/fft/fft.F90
+++ b/src/shared/fft/fft.F90
@@ -1,10 +1,11 @@
 
 module fft_mod
 
-! 
+! 
 !   Bruce Wyman
 ! 
 
+! 
 
 ! 
 !     Performs simultaneous fast Fourier transforms (FFTs) between
@@ -180,8 +181,8 @@ module fft_mod
 logical :: module_is_initialized=.false.
 
 !  cvs version and tag name
-character(len=128) :: version = '$Id: fft.F90,v 13.0 2006/03/28 21:38:54 fms Exp $'
-character(len=128) :: tagname = '$Name: tikal $'
+character(len=128) :: version = '$Id$'
+character(len=128) :: tagname = '$Name$'
 
 !-----------------------------------------------------------------------
 !
@@ -846,7 +847,7 @@ subroutine fft_init (n)
 
 !  write version and tag name to log file
    if (do_log) then
-      call write_version_number()
+      call write_version_number (version, tagname)
       do_log = .false.
    endif
 
diff --git a/src/shared/field_manager/field_manager.F90 b/src/shared/field_manager/field_manager.F90
index 4da6ab1293..dd80961e34 100644
--- a/src/shared/field_manager/field_manager.F90
+++ b/src/shared/field_manager/field_manager.F90
@@ -1,26 +1,27 @@
 module field_manager_mod
 #ifndef MAXFIELDS_ 
-#define MAXFIELDS_ 250
+#define MAXFIELDS_ 150
 #endif
 
 #ifndef MAXFIELDMETHODS_
-#define MAXFIELDMETHODS_ 250
+#define MAXFIELDMETHODS_ 150
 #endif
 
 !
-!  William Cooke
+!  William Cooke
 ! 
 ! 
-!  Richard D. Slater
+!  Richard D. Slater
 ! 
 !
-!  Matthew Harrison
+!  Matthew Harrison
 ! 
 !
-!  John P. Dunne
+!  John P. Dunne
 ! 
 !
 ! 
 
 ! 
 
@@ -181,8 +182,8 @@ module field_manager_mod
 private
 
 
-character(len=128) :: version = '$Id: field_manager.F90,v 20.0 2013/12/14 00:19:26 fms Exp $'
-character(len=128) :: tagname = '$Name: tikal $'
+character(len=128) :: version = '$Id$'
+character(len=128) :: tagname = '$Name$'
 logical            :: module_is_initialized  = .false.
 
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
@@ -613,6 +614,16 @@ subroutine field_manager_init(nfields, table_name)
    if(present(nfields)) nfields = num_fields
    return
 endif
+
+#ifdef PRESERVE_UNIT_CASE
+! 
+!   The case of the units in the field_table is preserved.  This option is
+!   still experimental.  It is possible other model components expect the units
+!   to be lowercase.  Please notify the developers if any issues are discovered.
+! 
+call mpp_error(NOTE,trim(note_header)//"Preserving the unit's case is experimental.")
+#endif
+
 num_fields = 0
 call initialize
 
@@ -641,7 +652,7 @@ subroutine field_manager_init(nfields, table_name)
 
 call mpp_open(iunit,file=trim(tbl_name), form=MPP_ASCII, action=MPP_RDONLY)
 !write_version_number should precede all writes to stdlog from field_manager
-call write_version_number()
+call write_version_number (version, tagname)
 log_unit = stdlog()
 do while (.TRUE.)
    read(iunit,'(a)',end=89,err=99) record
@@ -803,8 +814,22 @@ subroutine field_manager_init(nfields, table_name)
         case(4)
 ! If there is no control string then the last string can be omitted and there are only 4 '"' in the record.
           read(record,*,end=99,err=99) text_method_short
-          fields(num_fields)%methods(m)%method_type = lowercase(trim(text_method_short%method_type))
-          fields(num_fields)%methods(m)%method_name = lowercase(trim(text_method_short%method_name))
+          fields(num_fields)%methods(m)%method_type =&
+               & lowercase(trim(text_method_short%method_type))
+#ifdef PRESERVE_UNIT_CASE
+
+          if ( trim(fields(num_fields)%methods(m)%method_type) == 'units' ) then
+             ! Do not lowercase if units
+             fields(num_fields)%methods(m)%method_name =&
+                  & trim(text_method_short%method_name)
+          else
+             fields(num_fields)%methods(m)%method_name =&
+                  & lowercase(trim(text_method_short%method_name))
+          end if
+#else
+          fields(num_fields)%methods(m)%method_name =&
+               & lowercase(trim(text_method_short%method_name))
+#endif
           fields(num_fields)%methods(m)%method_control = " "
 
           type_str    = text_method_short%method_type
@@ -1272,16 +1297,12 @@ subroutine field_manager_end
 !        local parameters
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 character(len=17), parameter :: sub_name     = 'field_manager_end'
-character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
-                                               '(' // trim(sub_name) // '): '
-character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
-                                               '(' // trim(sub_name) // '): '
 character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
                                                '(' // trim(sub_name) // '): '
 
 integer :: unit 
 
-call write_version_number()
+call write_version_number (version, tagname)
 if ( mpp_pe() == mpp_root_pe() ) then
    unit = stdlog()
    write (unit,'(/,(a))') trim(note_header), 'Exiting field_manager, have a nice day ...'
@@ -1314,13 +1335,6 @@ subroutine strip_front_blanks(name)
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 !        local parameters
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-character(len=18), parameter :: sub_name     = 'strip_front_blanks'
-character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
-                                               '(' // trim(sub_name) // '): '
-character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
-                                               '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
 
 integer :: i, j
 
@@ -1372,13 +1386,6 @@ function find_field_index_old(model, field_name)
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 !        local parameters
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-character(len=16), parameter :: sub_name     = 'find_field_index'
-character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
-                                               '(' // trim(sub_name) // '): '
-character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
-                                               '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
 integer :: i
 
 find_field_index_old = NO_FIELD
@@ -1407,13 +1414,6 @@ function find_field_index_new(field_name)
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 !        local parameters
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-character(len=16), parameter :: sub_name     = 'find_field_index'
-character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
-                                               '(' // trim(sub_name) // '): '
-character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
-                                               '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
 integer :: i
 
 find_field_index_new = NO_FIELD
@@ -1469,10 +1469,6 @@ subroutine get_field_info(n,fld_type,fld_name,model,num_methods)
 character(len=14), parameter :: sub_name     = 'get_field_info'
 character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
                                                '(' // trim(sub_name) // '): '
-character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
-                                               '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
 
 !   
 !     The field index is invalid because it is less than 1 or greater than the 
@@ -1523,10 +1519,6 @@ subroutine get_field_method(n,m,method)
 character(len=16), parameter :: sub_name     = 'get_field_method'
 character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
                                                '(' // trim(sub_name) // '): '
-character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
-                                               '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
 
 !   
 !     The field index is invalid because it is less than 1 or greater than the 
@@ -1577,10 +1569,6 @@ subroutine get_field_methods(n,methods)
 character(len=17), parameter :: sub_name     = 'get_field_methods'
 character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
                                                '(' // trim(sub_name) // '): '
-character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
-                                               '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
 
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 !        local variables
@@ -1764,8 +1752,6 @@ function  create_field(parent_p, name)                        &
                                                '(' // trim(sub_name) // '): '
 character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
                                                '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
 integer                      :: ier
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 !        local variables
@@ -1902,8 +1888,6 @@ recursive function dump_list(list_p, recursive, depth)                &
 integer,                  parameter :: max_depth    = 128
 character(len=max_depth), parameter :: blank        = '    '
 character(len=9),  parameter :: sub_name     = 'dump_list'
-character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
-                                               '(' // trim(sub_name) // '): '
 character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
                                                '(' // trim(sub_name) // '): '
 character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
@@ -2154,16 +2138,6 @@ subroutine find_base(name, path, base)  !{
 character(len=*), intent(out) :: path
 character(len=*), intent(out) :: base
 
-!+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-!        local parameters
-!+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-character(len=9),  parameter :: sub_name     = 'find_base'
-character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
-                                               '(' // trim(sub_name) // '): '
-character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
-                                               '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 !        local variables
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
@@ -2267,16 +2241,6 @@ function find_field(name, this_list_p)                                &
 character(len=*), intent(in) :: name
 type (field_def), pointer    :: this_list_p
 
-!+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-!        local parameters
-!+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-character(len=10), parameter :: sub_name     = 'find_field'
-character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
-                                               '(' // trim(sub_name) // '): '
-character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
-                                               '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 !        local variables
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
@@ -2358,16 +2322,6 @@ subroutine find_head(name, head, rest)  !{
 character(len=*), intent(out) :: head
 character(len=*), intent(out) :: rest
 
-!+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-!        local parameters
-!+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-character(len=9),  parameter :: sub_name     = 'find_head'
-character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
-                                               '(' // trim(sub_name) // '): '
-character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
-                                               '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 !        local variables
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
@@ -2464,8 +2418,6 @@ function find_list(path, relative_p, create)                    &
 !        local parameters
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 character(len=9),  parameter :: sub_name     = 'find_list'
-character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
-                                               '(' // trim(sub_name) // '): '
 character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
                                                '(' // trim(sub_name) // '): '
 character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
@@ -2620,16 +2572,6 @@ function fm_change_list(name)                                        &
 !
 character(len=*), intent(in)  :: name
 
-!+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-!        local parameters
-!+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-character(len=14), parameter :: sub_name     = 'fm_change_list'
-character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
-                                               '(' // trim(sub_name) // '): '
-character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
-                                               '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 !        local variables
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
@@ -2700,12 +2642,8 @@ function  fm_change_root(name)                                        &
 !        local parameters
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 character(len=14), parameter :: sub_name     = 'fm_change_root'
-character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
-                                               '(' // trim(sub_name) // '): '
 character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
                                                '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 !        local variables
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
@@ -2821,12 +2759,8 @@ function  fm_dump_list(name, recursive)                        &
 !        local parameters
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 character(len=12), parameter :: sub_name     = 'fm_dump_list'
-character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
-                                               '(' // trim(sub_name) // '): '
 character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
                                                '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 !        local variables
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
@@ -2920,16 +2854,6 @@ function fm_exists(name)                                                &
 !
 character(len=*), intent(in) :: name
 
-!+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-!        local parameters
-!+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-character(len=9),  parameter :: sub_name     = 'fm_exists'
-character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
-                                               '(' // trim(sub_name) // '): '
-character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
-                                               '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 !        local variables
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
@@ -2989,12 +2913,8 @@ function  fm_get_index(name)                        &
 !        local parameters
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 character(len=12), parameter :: sub_name     = 'fm_get_index'
-character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
-                                               '(' // trim(sub_name) // '): '
 character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
                                                '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 !        local variables
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
@@ -3070,16 +2990,6 @@ function  fm_get_current_list()                                        &
 !        arguments
 !
 
-!+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-!        local parameters
-!+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-character(len=19), parameter :: sub_name     = 'fm_get_current_list'
-character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
-                                               '(' // trim(sub_name) // '): '
-character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
-                                               '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 !        local variables
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
@@ -3171,12 +3081,8 @@ function  fm_get_length(name)                        &
 !        local parameters
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 character(len=13), parameter :: sub_name     = 'fm_get_length'
-character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
-                                               '(' // trim(sub_name) // '): '
 character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
                                                '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 !        local variables
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
@@ -3268,12 +3174,8 @@ function  fm_get_type(name)                        &
 !        local parameters
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 character(len=11), parameter :: sub_name     = 'fm_get_type'
-character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
-                                               '(' // trim(sub_name) // '): '
 character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
                                                '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 !        local variables
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
@@ -3370,12 +3272,8 @@ function  fm_get_value_integer(name, value, index)                 &
 !        local parameters
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 character(len=20), parameter :: sub_name     = 'fm_get_value_integer'
-character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
-                                               '(' // trim(sub_name) // '): '
 character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
                                                '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 !        local variables
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
@@ -3497,12 +3395,8 @@ function  fm_get_value_logical(name, value, index)                 &
 !        local parameters
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 character(len=20), parameter :: sub_name     = 'fm_get_value_logical'
-character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
-                                               '(' // trim(sub_name) // '): '
 character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
                                                '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 !        local variables
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
@@ -3627,12 +3521,8 @@ function  fm_get_value_real(name, value, index)                 &
 !        local parameters
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 character(len=17), parameter :: sub_name     = 'fm_get_value_real'
-character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
-                                               '(' // trim(sub_name) // '): '
 character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
                                                '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 !        local variables
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
@@ -3760,12 +3650,8 @@ function  fm_get_value_string(name, value, index)                 &
 !        local parameters
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 character(len=19), parameter :: sub_name     = 'fm_get_value_string'
-character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
-                                               '(' // trim(sub_name) // '): '
 character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
                                                '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 !        local variables
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
@@ -3922,8 +3808,6 @@ function fm_intersection(lists, dim)                        &
                                                '(' // trim(sub_name) // '): '
 character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
                                                '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 !        local variables
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
@@ -4130,12 +4014,8 @@ function  fm_loop_over_list(list, name, field_type, index)        &
 !        local parameters
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 character(len=17), parameter :: sub_name     = 'fm_loop_over_list'
-character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
-                                               '(' // trim(sub_name) // '): '
 character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
                                                '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 !        local variables
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
@@ -4271,12 +4151,8 @@ function  fm_new_list(name, create, keep)                        &
 !        local parameters
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 character(len=11), parameter :: sub_name     = 'fm_new_list'
-character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
-                                               '(' // trim(sub_name) // '): '
 character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
                                                '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 !        local variables
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
@@ -4427,12 +4303,8 @@ function  fm_new_value_integer(name, value, create, index, append)     &
 !        local parameters
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 character(len=20), parameter :: sub_name     = 'fm_new_value_integer'
-character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
-                                               '(' // trim(sub_name) // '): '
 character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
                                                '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 !        local variables
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
@@ -4660,12 +4532,8 @@ function  fm_new_value_logical(name, value, create, index, append) &
 !        local parameters
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 character(len=20), parameter :: sub_name     = 'fm_new_value_logical'
-character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
-                                               '(' // trim(sub_name) // '): '
 character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
                                                '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 !        local variables
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
@@ -4899,12 +4767,8 @@ function  fm_new_value_real(name, value, create, index, append) &
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 
 character(len=17), parameter :: sub_name     = 'fm_new_value_real'
-character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
-                                               '(' // trim(sub_name) // '): '
 character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
                                                '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 !        local variables
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
@@ -5126,12 +4990,8 @@ function  fm_new_value_string(name, value, create, index, append) &
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 
 character(len=19), parameter :: sub_name     = 'fm_new_value_string'
-character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
-                                               '(' // trim(sub_name) // '): '
 character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
                                                '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 !        local variables
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
@@ -5367,17 +5227,6 @@ subroutine  fm_reset_loop
 !        arguments
 !
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-!        local parameters
-!+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-
-character(len=13), parameter :: sub_name     = 'fm_reset_loop'
-character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
-                                               '(' // trim(sub_name) // '): '
-character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
-                                               '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
-!+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 !        local variables
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 
@@ -5420,17 +5269,6 @@ subroutine  fm_return_root  !{
 !        arguments
 !
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-!        local parameters
-!+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-
-character(len=14), parameter :: sub_name     = 'fm_return_root'
-character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
-                                               '(' // trim(sub_name) // '): '
-character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
-                                               '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
-!+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 !        local variables
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 !
@@ -5495,16 +5333,6 @@ function get_field(name, this_list_p)                                        &
 character(len=*), intent(in)     :: name
 type (field_def), pointer        :: this_list_p
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-!        local parameters
-!+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-character(len=9),  parameter :: sub_name     = 'get_field'
-character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
-                                               '(' // trim(sub_name) // '): '
-character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
-                                               '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
-!+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 !        local variables
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 character(len=fm_path_name_len)  :: path
@@ -5574,16 +5402,6 @@ function fm_modify_name(oldname, newname)
 character(len=*), intent(in)     :: oldname
 character(len=*), intent(in)     :: newname
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-!        local parameters
-!+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-character(len=14), parameter :: sub_name     = 'fm_modify_name'
-character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
-                                               '(' // trim(sub_name) // '): '
-character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
-                                               '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
-!+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 !        local variables
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 character(len=fm_path_name_len)  :: path
@@ -5643,16 +5461,6 @@ subroutine initialize  !{
 !        arguments
 !
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-!        local parameters
-!+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-character(len=10), parameter :: sub_name     = 'initialize'
-character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
-                                               '(' // trim(sub_name) // '): '
-character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
-                                               '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
-!+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 !        local variables
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 integer :: ier
@@ -5744,12 +5552,8 @@ function  make_list(this_list_p, name)                        &
 !        local parameters
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 character(len=9),  parameter :: sub_name     = 'make_list'
-character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
-                                               '(' // trim(sub_name) // '): '
 character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
                                                '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 !        local variables
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
@@ -5851,12 +5655,8 @@ function fm_query_method(name, method_name, method_control)                &
 !        local parameters
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 character(len=15), parameter :: sub_name     = 'fm_query_method'
-character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
-                                               '(' // trim(sub_name) // '): '
 character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
                                                '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 !        local variables
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
@@ -5973,14 +5773,8 @@ recursive function query_method(list_p, recursive, name, method_name, method_con
 !        local parameters
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 character(len=12), parameter :: sub_name     = 'query_method'
-character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
-                                               '(' // trim(sub_name) // '): '
 character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
                                                '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
-integer,                  parameter :: max_depth = 64
-character(len=max_depth), parameter :: blank = '    '
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 !        local variables
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
@@ -6128,8 +5922,6 @@ function fm_copy_list(list_name, suffix, create ) &
                                                '(' // trim(sub_name) // '): '
 character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
                                                '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 !        local variables
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
@@ -6287,12 +6079,8 @@ function fm_find_methods(list_name, methods, control ) &
 !        local parameters
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 character(len=15), parameter :: sub_name     = 'fm_find_methods'
-character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
-                                               '(' // trim(sub_name) // '): '
 character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
                                                '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 !        local variables
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
@@ -6403,17 +6191,8 @@ recursive function find_method(list_p, recursive, num_meth, method, control)   &
 !        local parameters
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 character(len=11), parameter :: sub_name     = 'find_method'
-character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
-                                               '(' // trim(sub_name) // '): '
 character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
                                                '(' // trim(sub_name) // '): '
-character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
-                                               '(' // trim(sub_name) // '): '
-!+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-!        local parameters
-!+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-integer, parameter                          :: max_depth = 64
-character(len=max_depth), parameter         :: blank = '    '
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 !        local variables
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
@@ -6558,10 +6337,6 @@ subroutine  fm_set_verbosity(verbosity)  !{
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 
 character(len=16), parameter :: sub_name     = 'fm_set_verbosity'
-character(len=64), parameter :: error_header = '==>Error from ' // trim(module_name)   //  &
-                                               '(' // trim(sub_name) // '): '
-character(len=64), parameter :: warn_header  = '==>Warning from ' // trim(module_name) //  &
-                                               '(' // trim(sub_name) // '): '
 character(len=64), parameter :: note_header  = '==>Note from ' // trim(module_name)    //  &
                                                '(' // trim(sub_name) // '): '
 !+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
diff --git a/src/shared/field_manager/fm_util.F90 b/src/shared/field_manager/fm_util.F90
index 578c1dff6b..5bc1cb1824 100644
--- a/src/shared/field_manager/fm_util.F90
+++ b/src/shared/field_manager/fm_util.F90
@@ -1,9 +1,9 @@
 module fm_util_mod  !{
 ! 
-! Richard D. Slater
+! Richard D. Slater
 !
 !
-! John P. Dunne
+! John P. Dunne
 !
 !
 !
@@ -87,8 +87,8 @@ module fm_util_mod  !{
 character(len=fm_path_name_len) :: save_current_list
 character(len=fm_path_name_len) :: save_path
 character(len=fm_path_name_len) :: save_name
-character(len=128) :: version = '$Id: fm_util.F90,v 17.0 2009/07/21 03:19:16 fms Exp $'
-character(len=128) :: tagname = '$Name: tikal $'
+character(len=128) :: version = '$Id$'
+character(len=128) :: tagname = '$Name$'
 
 !
 !        Interface definitions for overloaded routines
@@ -146,12 +146,6 @@ subroutine fm_util_set_caller(caller)  !{
 
 character(len=*), intent(in)          :: caller
 
-!
-!       Local parameters
-!
-
-character(len=48), parameter  :: sub_name = 'fm_util_set_caller'
-
 !
 !       Local variables
 !
@@ -196,12 +190,6 @@ subroutine fm_util_reset_caller  !{
 !       arguments
 !
 
-!
-!       Local parameters
-!
-
-character(len=48), parameter  :: sub_name = 'fm_util_reset_caller'
-
 !
 !       Local variables
 !
@@ -239,12 +227,6 @@ subroutine fm_util_set_good_name_list(good_name_list)  !{
 
 character(len=*), intent(in)          :: good_name_list
 
-!
-!       Local parameters
-!
-
-character(len=48), parameter  :: sub_name = 'fm_util_set_good_name_list'
-
 !
 !       Local variables
 !
@@ -285,12 +267,6 @@ subroutine fm_util_reset_good_name_list  !{
 !       arguments
 !
 
-!
-!       Local parameters
-!
-
-character(len=48), parameter  :: sub_name = 'fm_util_reset_good_name_list'
-
 !
 !       Local variables
 !
@@ -328,12 +304,6 @@ subroutine fm_util_set_no_overwrite(no_overwrite)  !{
 
 logical, intent(in)          :: no_overwrite
 
-!
-!       Local parameters
-!
-
-character(len=48), parameter  :: sub_name = 'fm_util_set_no_overwrite'
-
 !
 !       Local variables
 !
@@ -374,12 +344,6 @@ subroutine fm_util_reset_no_overwrite  !{
 !       arguments
 !
 
-!
-!       Local parameters
-!
-
-character(len=48), parameter  :: sub_name = 'fm_util_reset_no_overwrite'
-
 !
 !       Local variables
 !
diff --git a/src/shared/fms/fms.F90 b/src/shared/fms/fms.F90
index f0146bd219..6f46b00d13 100644
--- a/src/shared/fms/fms.F90
+++ b/src/shared/fms/fms.F90
@@ -1,10 +1,11 @@
 
 module fms_mod
 
-! 
+! 
 !   Bruce Wyman
 ! 
 
+! 
 
 ! 
 !   The fms module provides routines that are commonly used
@@ -21,10 +22,12 @@ module fms_mod
 !     These include namelist files, restart files, and 32-bit IEEE
 !     data files. There also is a matching interface to close the files.
 !     If other file types are needed the mpp_open and mpp_close
+!     interfaces in module mpp_io must be used.
    
 !    3. Read and write distributed data to simple native unformatted files.
 !     This type of file (called a restart file) is used to checkpoint
 !     model integrations for a subsequent restart of the run.
    
 !    4. For convenience there are several routines published from
+!     the mpp module. These are routines for getting processor
 !     numbers, commonly used I/O unit numbers, error handling, and timing sections of code.
 ! 
 
@@ -134,14 +137,15 @@ module fms_mod
                        MPP_SINGLE, MPP_MULTI, MPP_DELETE, mpp_io_exit, &
                        fieldtype, mpp_get_atts, mpp_get_info, mpp_get_fields
 
-use fms_io_mod, only : read_data, write_data, fms_io_init, fms_io_exit, field_size, &
+use fms_io_mod, only : fms_io_init, fms_io_exit, field_size, &
+                       read_data, write_data, read_compressed, read_distributed, &
                        open_namelist_file, open_restart_file, open_ieee32_file, close_file, &
                        set_domain, get_domain_decomp, nullify_domain, &
                        open_file, open_direct_file, string, get_mosaic_tile_grid, &
                        get_mosaic_tile_file, get_global_att_value, file_exist, field_exist
 
 use memutils_mod, only: print_memuse_stats, memutils_init
-use version_mod, only: MOM_VERSION
+use constants_mod, only: constants_version=>version, constants_tagname=>tagname !pjp: PI not computed
 
 
 implicit none
@@ -156,7 +160,7 @@ module fms_mod
           open_file, open_direct_file
 
 ! routines for reading/writing distributed data
-public :: set_domain, read_data, write_data
+public :: set_domain, read_data, write_data, read_compressed, read_distributed
 public :: get_domain_decomp, field_size, nullify_domain
 public :: get_global_att_value
 
@@ -221,6 +225,7 @@ module fms_mod
 !     module: mpp_clock_id, mpp_clock_begin, and mpp_clock_end.
 !     The fms module makes these routines public.
 !     A list of timed code sections will be printed to STDOUT.
+!     See the MPP
 !     module for more details.
 !       
 !   
@@ -230,6 +235,7 @@ module fms_mod
 !     DETAILED also turns on detailed message-passing performance diagnosis.
 !     Both SYNC and DETAILED will  work correctly on innermost clock nest
 !     and distort outer clocks, and possibly the overall code time.
+!     See the MPP
 !     module for more details.
 !   
 !   
@@ -274,6 +280,12 @@ module fms_mod
    logical, private :: do_nml_error_init = .true.
    private  nml_error_init
 
+
+!  ---- version number -----
+
+  character(len=128) :: version = '$Id$'
+  character(len=128) :: tagname = '$Name$'
+
   logical :: module_is_initialized = .FALSE.
 
 
@@ -401,7 +413,7 @@ subroutine fms_init (localcomm )
 
 !--- write version info and namelist to logfile ---
 
-    call write_version_number()
+    call write_version_number (version, tagname)
     if (mpp_pe() == mpp_root_pe()) then
       unit = stdlog()
       write (unit, nml=fms_nml)
@@ -411,6 +423,8 @@ subroutine fms_init (localcomm )
     call memutils_init( print_memory_usage )
     call print_memuse_stats('fms_init')
 
+    call write_version_number (constants_version,constants_tagname)
+
 end subroutine fms_init
 ! 
 
@@ -748,41 +762,60 @@ END SUBROUTINE nml_error_init
 ! 
 
 !   
-!     Prints to the log file (or a specified unit) the version (git hash)
+!     Prints to the log file (or a specified unit) the (cvs) version id string and
+!     (cvs) tag name.
 !   
 !   
-!     Prints to the log file (stdlog) or a specified unit the version (git hash)
+!     Prints to the log file (stdlog) or a specified unit the (cvs) version id string
+!      and (cvs) tag name.
 !   
 !   
+
+!   
+!    string that contains routine name and version number.
+!   
+!   
+!    The tag/name string, this is usually the Name string
+!    returned by CVS when checking out the code.
+!   
 !   
 !    The Fortran unit number of an open formatted file. If this unit number 
 !    is not supplied the log file unit number is used (stdlog). 
 !   
 ! prints module version number to the log file of specified unit number
 
- subroutine write_version_number (unit)
+ subroutine write_version_number (version, tag, unit)
 
+!   in:  version = string that contains routine name and version number
+!
 !   optional in:
-!        unit    = alternate unit number to direct output
+!        tag = cvs tag name that code was checked out with
+!        unit    = alternate unit number to direct output  
 !                  (default: unit=stdlog)
 
-   integer,          intent(in), optional :: unit
+   character(len=*), intent(in) :: version
+   character(len=*), intent(in), optional :: tag 
+   integer,          intent(in), optional :: unit 
 
-   integer :: logunit
+   integer :: logunit 
 
-   if (.not.module_is_initialized) call fms_init ()
+   if (.not.module_is_initialized) call fms_init ( )
 
      logunit = stdlog()
      if (present(unit)) then
          logunit = unit
-     else
+     else    
        ! only allow stdlog messages on root pe
          if ( mpp_pe() /= mpp_root_pe() ) return
-     endif
+     endif   
 
-     write (logunit,'(/,80("="),/(a))') trim(MOM_VERSION)
+     if (present(tag)) then
+         write (logunit,'(/,80("="),/(a))') trim(version), trim(tag)
+     else    
+         write (logunit,'(/,80("="),/(a))') trim(version)
+     endif   
 
  end subroutine write_version_number
 ! 
diff --git a/src/shared/fms/fms_io.F90 b/src/shared/fms/fms_io.F90
index cee4b0f52c..d73787e3c3 100644
--- a/src/shared/fms/fms_io.F90
+++ b/src/shared/fms/fms_io.F90
@@ -4,19 +4,19 @@ module fms_io_mod
 
 !
 !
-! 
+! 
 ! Zhi Liang
 ! 
 
-! 
-! M.J. Harrison 
+! 
+! M.J. Harrison
 ! 
 !
-! 
-! M.J. Harrison 
+! 
+! M.J. Harrison
 ! 
 
-! 
+! 
 ! B. Wyman
 ! 
 
@@ -36,27 +36,14 @@ module fms_io_mod
 ! fms_netcdf_override
 ! fms_netcdf_restart
 !
-! because default values of both flags are .true., the default behavior of the entire model is 
+! because default values of both flags are .true., the default behavior of the entire model is
 ! to use netCDF IO mode. To turn off netCDF restart, simply set fms_netcdf_restart to .false.
 !
-! Fei.Liu
-! 05222006
-! Read distributed files in NetCDF is available. Details can be found in read_data_3d_new
-! 
    -!threading_read='multi', threading_write='multi', fileset_write='multi' (default)
-!threading_read='multi', threading_write='single', fileset_write='single'
-! 
    
 !
-!   
+! 
 ! 
 ! threading_read can be 'single' or 'multi'
 ! 
-! 
-! threading_write can be 'single' or 'multi'
-! 
-! 
-! fileset_write can be 'single' or 'multi'
-! 
 ! 
 !   .true. : fms_netcdf_restart overrides individual do_netcdf_restart value (default behavior)
 !   .false.: individual module settings has a precedence over the global setting, therefore fms_netcdf_restart is ignored
@@ -68,18 +55,18 @@ module fms_io_mod
 !   module setting takes over.
 ! 
 ! 
-!   .true. : time_stamp will be added to the restart file name as a prefix when 
-!            optional argument time_stamp is passed into routine save_restart. 
+!   .true. : time_stamp will be added to the restart file name as a prefix when
+!            optional argument time_stamp is passed into routine save_restart.
 !   .false.: time_stmp will not be added to the restart file name even though
 !            time_stamp is passed into save_restart.
 !    default is true.
 ! 
 ! 
 !    set print_chksum (default is false) to true to print out chksum of fields that are
-!    read and written through save_restart/restore_state. The chksum is accross all the 
-!    processors, so there will be only one chksum even there are multiple-tiles in the 
+!    read and written through save_restart/restore_state. The chksum is accross all the
+!    processors, so there will be only one chksum even there are multiple-tiles in the
 !    grid. For the multiple case, the filename appeared in the message will contain
-!    tile1 because the message is print out from root pe and on root pe the tile id is tile1. 
+!    tile1 because the message is print out from root pe and on root pe the tile id is tile1.
 ! 
 ! 
 !    set debug_mask_list (default is false) to true to print out mask_list reading from mask_table.
@@ -88,7 +75,7 @@ module fms_io_mod
 !    Set checksum_required (default is true) to true to compare checksums stored in the attribute of a
 !    field against the checksum after reading in the data. This check mitigates the possibility of data
 !    that gets corrupted on write or read from being used in a n ongoing fashion. The checksum is across
-!    all the  processors, so there will be only one checksum even if there are multiple-tiles in the 
+!    all the  processors, so there will be only one checksum even if there are multiple-tiles in the
 !    grid. For the decomposed file case, the filename appearing in the message will contain tile1
 !    because the message is printed out from the root pe and on root pe the tile id is tile1.
 !
@@ -99,20 +86,26 @@ module fms_io_mod
 
 use mpp_io_mod,      only: mpp_open, mpp_close, mpp_io_init, mpp_io_exit, mpp_read, mpp_write
 use mpp_io_mod,      only: mpp_write_meta, mpp_get_info, mpp_get_atts, mpp_get_fields
+use mpp_io_mod,      only: mpp_read_compressed, mpp_write_compressed, mpp_def_dim
+use mpp_io_mod,      only: mpp_write_unlimited_axis, mpp_read_distributed_ascii
 use mpp_io_mod,      only: mpp_get_axes, mpp_get_axis_data, mpp_get_att_char, mpp_get_att_name
-use mpp_io_mod,      only: mpp_get_att_real_scalar, mpp_attribute_exist
+use mpp_io_mod,      only: mpp_get_att_real_scalar, mpp_attribute_exist, mpp_is_dist_ioroot
 use mpp_io_mod,      only: fieldtype, axistype, atttype, default_field, default_axis, default_att
 use mpp_io_mod,      only: MPP_NETCDF, MPP_ASCII, MPP_MULTI, MPP_SINGLE, MPP_OVERWR, MPP_RDONLY
 use mpp_io_mod,      only: MPP_IEEE32, MPP_NATIVE, MPP_DELETE, MPP_APPEND, MPP_SEQUENTIAL, MPP_DIRECT
 use mpp_io_mod,      only: MAX_FILE_SIZE, mpp_get_att_value
-use mpp_domains_mod, only: domain2d, domain1d, NULL_DOMAIN1D, NULL_DOMAIN2D, operator( .EQ. ), CENTER
+use mpp_io_mod,      only: mpp_get_dimension_length
+use mpp_domains_mod, only: domain2d, domain1d, NULL_DOMAIN1D, NULL_DOMAIN2D, operator( .EQ. )
+use mpp_domains_mod, only: CENTER, EAST, WEST, NORTH, SOUTH, CORNER
 use mpp_domains_mod, only: mpp_get_domain_components, mpp_get_compute_domain, mpp_get_data_domain
 use mpp_domains_mod, only: mpp_get_domain_shift, mpp_get_global_domain, mpp_global_field, mpp_domain_is_tile_root_pe
-use mpp_domains_mod, only: mpp_get_ntile_count, mpp_get_current_ntile, mpp_get_tile_id, mpp_mosaic_defined
-use mpp_domains_mod, only: mpp_get_io_domain
+use mpp_domains_mod, only: mpp_get_ntile_count, mpp_get_current_ntile, mpp_get_tile_id
+use mpp_domains_mod, only: mpp_get_pelist, mpp_get_io_domain, mpp_get_domain_npes
 use mpp_mod,         only: mpp_error, FATAL, NOTE, WARNING, mpp_pe, mpp_root_pe, mpp_npes, stdlog, stdout
 use mpp_mod,         only: mpp_broadcast, ALL_PES, mpp_chksum, mpp_get_current_pelist, mpp_npes, lowercase
 use mpp_mod,         only: input_nml_file, mpp_get_current_pelist_name, uppercase
+use mpp_mod,         only: mpp_gather, mpp_scatter, mpp_send, mpp_recv, mpp_sync_self, COMM_TAG_1, EVENT_RECV
+use mpp_mod,	     only: MPP_FILL_DOUBLE,MPP_FILL_INT
 
 use platform_mod, only: r8_kind
 
@@ -129,29 +122,77 @@ module fms_io_mod
 integer, parameter, private :: MAX_TIME_LEVEL_WRITE = 20
 integer, parameter          :: max_axis_size=10000
 
+! Index postions for axes in restart_file_type
+! This is done so the user may define the axes
+! in any order but a check can be performed
+! to ensure no registration of duplicate axis
+integer, parameter, private :: XIDX=1
+integer, parameter, private :: YIDX=2
+integer, parameter, private :: CIDX=3
+integer, parameter, private :: ZIDX=4
+integer, parameter, private :: HIDX=5
+integer, parameter, private :: TIDX=6
+integer, parameter, private :: UIDX=7
+integer, parameter, private :: NIDX=7
+
+type meta_type
+  type(meta_type), pointer :: prev=>null(), next=>null()
+!!$ Gfortran on gaea does not yet support deferred length character strings
+!!$  character(len=:),allocatable  :: name
+  character(len=256)   :: name
+  real,    allocatable :: rval(:)
+  integer, allocatable :: ival(:)
+!!$ Gfortran on gaea does not yet support deferred length character strings
+!!$  character(len=:), allocatable :: cval
+  character(len=256)   :: cval
+end type meta_type
+
+type ax_type
+   private
+   character(len=128) :: name = ''
+   character(len=128) :: units = ''
+   character(len=128) :: longname = ''
+   character(len=8)   :: cartesian = ''
+   character(len=256) :: compressed = ''
+   character(len=128) :: dimlen_name = ''
+   character(len=128) :: dimlen_lname = ''
+   character(len=128) :: calendar = ''
+   integer            :: sense              !Orientation of z axis definition
+   integer            :: dimlen             !max dim of elements across global domain
+   real               :: min             !valid min for real axis data
+   integer            :: imin            !valid min for integer axis data
+   integer,allocatable :: idx(:)         !compressed io-domain index vector
+   integer,allocatable :: nelems(:)      !num elements for each rank in io domain
+   real, pointer      :: data(:) =>NULL()    !real axis values (not used if time axis)
+   type(domain2d),pointer :: domain =>NULL() ! domain associated with compressed axis
+end type ax_type
+
 type var_type
    private
-   character(len=128)                     :: name
-   character(len=128)                     :: longname
-   character(len=128)                     :: units
-   real, dimension(:,:,:,:), _ALLOCATABLE :: buffer _NULL 
-   logical                                :: domain_present
-   logical                                :: write_on_this_pe
-   integer                                :: domain_idx
-   logical                                :: is_dimvar
+   character(len=128)                     :: name = ''
+   character(len=128)                     :: longname = ''
+   character(len=128)                     :: units = ''
+   real, dimension(:,:,:,:), _ALLOCATABLE :: buffer _NULL
+   logical                                :: domain_present = .FALSE.
+   integer                                :: domain_idx = -1
+   logical                                :: is_dimvar = .FALSE.
+   logical                                :: read_only = .FALSE.
    type(fieldtype)                        :: field
    type(axistype)                         :: axis
    integer                                :: position
    integer                                :: ndim
-   integer                                :: siz(4)      ! X/Y/Z/T extent of fields (data domain 
+   integer                                :: siz(5)      ! X/Y/Z/T/A extent of fields (data domain
                                                          ! size for distributed writes;global size for reads)
-   integer                                :: gsiz(4)     ! global X/Y/Z/T extent of fields 
-   integer                                :: csiz(4)     ! actual data size in the file
-   integer                                :: id_axes(3)  ! store index for x/y/z axistype.
+   integer                                :: gsiz(4)     ! global X/Y/Z/A extent of fields
+   integer                                :: id_axes(4)  ! store index for x/y/z/a axistype.
    logical                                :: initialized ! indicate if the field is read or not in routine save_state.
    logical                                :: mandatory   ! indicate if the field is mandatory to be when restart.
    integer                                :: is, ie, js, je  ! index of the data in compute domain
    real                                   :: default_data 
+   character(len=8)                       :: compressed_axis !< If on a compressed axis, which axis
+   integer, dimension(:), allocatable     :: pelist
+   integer                                :: ishift, jshift ! can be used to shift indices when no_domain=T
+   integer                                :: x_halo, y_halo ! can be used to indicate halo size when no_domain=T
 end type var_type
 
 type Ptr0Dr
@@ -170,6 +211,10 @@ module fms_io_mod
    real, dimension(:,:,:), pointer :: p => NULL()
 end type Ptr3Dr
 
+type Ptr4Dr
+   real, dimension(:,:,:,:), pointer :: p => NULL()
+end type Ptr4Dr
+
 type Ptr0Di
    integer,                   pointer :: p => NULL()
 end type Ptr0Di
@@ -188,16 +233,24 @@ module fms_io_mod
 
 type restart_file_type
    private
-   integer                                  :: unit ! mpp_io unit for netcdf file
-   character(len=128)                       :: name
-   integer                                  :: nvar, natt, max_ntime
-   logical                                  :: is_root_pe
-   integer                                  :: tile_count
+   integer                                  :: unit = -1 ! mpp_io unit for netcdf file
+   character(len=128)                       :: name = ''
+   integer                                  :: register_id = 0
+   integer                                  :: nvar = 0
+   integer                                  :: natt = 0
+   integer                                  :: max_ntime = 0
+   logical                                  :: is_root_pe = .FALSE.
+   logical                                  :: is_compressed = .FALSE.
+   logical                                  :: unlimited_axis = .FALSE.
+   integer                                  :: tile_count = 1
+   type(ax_type),  allocatable              :: axes(:)  ! Currently define X,Y,Compressed, unlimited and maybe Z
+   type(meta_type),                pointer  :: first =>NULL() ! pointer to first additional global metadata element
    type(var_type), dimension(:),   pointer  :: var  => NULL()
    type(Ptr0Dr),   dimension(:,:), pointer  :: p0dr => NULL()
    type(Ptr1Dr),   dimension(:,:), pointer  :: p1dr => NULL()
    type(Ptr2Dr),   dimension(:,:), pointer  :: p2dr => NULL()
    type(Ptr3Dr),   dimension(:,:), pointer  :: p3dr => NULL()
+   type(Ptr4Dr),   dimension(:,:), pointer  :: p4dr => NULL()
    type(Ptr0Di),   dimension(:,:), pointer  :: p0di => NULL()
    type(Ptr1Di),   dimension(:,:), pointer  :: p1di => NULL()
    type(Ptr2Di),   dimension(:,:), pointer  :: p2di => NULL()
@@ -222,6 +275,24 @@ module fms_io_mod
    module procedure read_data_2d_region
 end interface
 
+interface read_distributed
+   module procedure read_distributed_r1D
+   module procedure read_distributed_r3D
+   module procedure read_distributed_r5D
+   module procedure read_distributed_i1D
+   module procedure read_distributed_iscalar
+   module procedure read_distributed_a1D
+end interface
+
+! Only need read compressed att; write is handled in with
+! mpp_io calls in save_compressed_restart
+interface read_compressed
+   module procedure read_compressed_i1d
+   module procedure read_compressed_i2d
+   module procedure read_compressed_1d
+   module procedure read_compressed_2d
+end interface read_compressed
+
 interface write_data
    module procedure write_data_3d_new
    module procedure write_data_2d_new
@@ -243,6 +314,7 @@ module fms_io_mod
    module procedure register_restart_field_r1d
    module procedure register_restart_field_r2d
    module procedure register_restart_field_r3d
+   module procedure register_restart_field_r4d
    module procedure register_restart_field_i0d
    module procedure register_restart_field_i1d
    module procedure register_restart_field_i2d
@@ -255,6 +327,14 @@ module fms_io_mod
    module procedure register_restart_field_i1d_2level
    module procedure register_restart_field_i2d_2level
    module procedure register_restart_field_i3d_2level
+   module procedure register_restart_region_r2d
+   module procedure register_restart_region_r3d
+end interface
+
+interface register_restart_axis
+   module procedure register_restart_axis_r1d
+   module procedure register_restart_axis_i1d
+   module procedure register_restart_axis_unlimited
 end interface
 
 interface reset_field_pointer
@@ -262,6 +342,7 @@ module fms_io_mod
    module procedure reset_field_pointer_r1d
    module procedure reset_field_pointer_r2d
    module procedure reset_field_pointer_r3d
+   module procedure reset_field_pointer_r4d
    module procedure reset_field_pointer_i0d
    module procedure reset_field_pointer_i1d
    module procedure reset_field_pointer_i2d
@@ -286,6 +367,7 @@ module fms_io_mod
    module procedure query_initialized_name
    module procedure query_initialized_r2d
    module procedure query_initialized_r3d
+   module procedure query_initialized_r4d
 end interface
 
 interface set_initialized
@@ -293,6 +375,7 @@ module fms_io_mod
    module procedure set_initialized_name
    module procedure set_initialized_r2d
    module procedure set_initialized_r3d
+   module procedure set_initialized_r4d
 end interface
 
 interface get_global_att_value
@@ -304,17 +387,20 @@ module fms_io_mod
   module procedure get_var_att_value_text
 end interface
 
+interface parse_mask_table
+  module procedure parse_mask_table_2d
+  module procedure parse_mask_table_3d
+end interface
+
 integer :: num_files_r = 0 ! number of currently opened files for reading
 integer :: num_files_w = 0 ! number of currently opened files for writing
 integer :: num_domains = 0 ! number of domains in array_domain
-integer :: num_registered_files ! mumber of files registered by calling register_restart_file
+integer :: num_registered_files = 0 ! mumber of files registered by calling register_restart_file
 
-integer :: thread_r, thread_w, fset_w, form
+integer :: thread_r, form
 logical :: module_is_initialized = .FALSE.
 
-character(len=32) :: pelist_name
-character(len=7)  :: pe_name
-character(len=128):: error_msg  
+character(len=128):: error_msg
 logical           :: great_circle_algorithm=.FALSE.
 
 !------ private data, pointer to current 2d domain ------
@@ -324,24 +410,27 @@ module fms_io_mod
 integer, private :: is,ie,js,je      ! compute domain
 integer, private :: isd,ied,jsd,jed  ! data domain
 integer, private :: isg,ieg,jsg,jeg  ! global domain
-character(len=128),      dimension(:), allocatable         :: registered_file ! file names registered through register_restart_file 
+character(len=128),      dimension(:), allocatable         :: registered_file ! file names registered through register_restart_file
 type(restart_file_type), dimension(:), allocatable         :: files_read  ! store files that are read through read_data
 type(restart_file_type), dimension(:), allocatable, target :: files_write ! store files that are written through write_data
-type(domain2d), dimension(max_domains), save       :: array_domain
+type(domain2d), dimension(max_domains), target, save  :: array_domain
 type(domain1d), dimension(max_domains), save       :: domain_x, domain_y
-public  :: read_data, write_data, fms_io_init, fms_io_exit, field_size
-public  :: open_namelist_file, open_restart_file, open_ieee32_file, close_file 
+public  :: read_data, read_compressed, write_data, read_distributed
+public  :: fms_io_init, fms_io_exit, field_size, get_field_size
+public  :: open_namelist_file, open_restart_file, open_ieee32_file, close_file
 public  :: set_domain, nullify_domain, get_domain_decomp, return_domain
 public  :: open_file, open_direct_file
 public  :: get_restart_io_mode, get_tile_string, string
-public  :: get_mosaic_tile_grid, get_mosaic_tile_file
+public  :: get_mosaic_tile_grid, get_mosaic_tile_file, get_file_name
 public  :: get_global_att_value, get_var_att_value
 public  :: file_exist, field_exist
-public  :: register_restart_field, save_restart, restore_state
-public  :: restart_file_type, query_initialized, set_initialized
+public  :: register_restart_field, register_restart_axis, save_restart, restore_state
+public  :: set_meta_global
+public  :: save_restart_border, restore_state_border
+public  :: restart_file_type, query_initialized, set_initialized, free_restart_type
 public  :: reset_field_name, reset_field_pointer
 private :: lookup_field_r, lookup_axis, unique_axes
-
+public  :: dimension_size
 public  :: set_filename_appendix, get_instance_filename
 public  :: parse_mask_table
 public  :: get_great_circle_algorithm
@@ -357,29 +446,27 @@ module fms_io_mod
 logical           :: fms_netcdf_override = .true.
 logical           :: fms_netcdf_restart  = .true.
 character(len=32) :: threading_read      = 'multi'
-character(len=32) :: threading_write     = 'multi'
-character(len=32) :: fileset_write       = 'multi'
-character(len=32) :: format              = 'netcdf' 
+character(len=32) :: format              = 'netcdf'
 logical           :: read_all_pe         = .TRUE.
 character(len=64) :: iospec_ieee32       = '-N ieee_32'
 integer           :: max_files_w         = 40
 integer           :: max_files_r         = 40
+integer           :: dr_set_size         = 10
 logical           :: read_data_bug       = .false.
 logical           :: time_stamp_restart  = .true.
 logical           :: print_chksum        = .false.
 logical           :: show_open_namelist_file_warning = .false.
-logical           :: debug_mask_list     = .false. 
-logical           :: checksum_required   = .true. 
+logical           :: debug_mask_list     = .false.
+logical           :: checksum_required   = .true.
   namelist /fms_io_nml/ fms_netcdf_override, fms_netcdf_restart, &
-       threading_read, threading_write, &
-       fileset_write, format, read_all_pe, iospec_ieee32,max_files_w,max_files_r, &
+       threading_read, format, read_all_pe, iospec_ieee32,max_files_w,max_files_r, &
        read_data_bug, time_stamp_restart, print_chksum, show_open_namelist_file_warning, &
-       debug_mask_list, checksum_required 
+       debug_mask_list, checksum_required, dr_set_size
 
 integer            :: pack_size  ! = 1 for double = 2 for float
 
-character(len=128) :: version = '$Id: fms_io.F90,v 20.0 2013/12/14 00:20:08 fms Exp $'
-character(len=128) :: tagname = '$Name: tikal $'
+character(len=128) :: version = '$Id$'
+character(len=128) :: tagname = '$Name$'
 
 contains
 
@@ -392,9 +479,9 @@ module fms_io_mod
 ! fms_netcdf_override
 ! fms_netcdf_restart
 !
-! because default values of both flags are .true., the default behavior of the entire model is 
+! because default values of both flags are .true., the default behavior of the entire model is
 ! to use netCDF IO mode. To turn off netCDF restart, simply set fms_netcdf_restart to .false.
-! 
+!
 ! 
 ! 
 subroutine fms_io_init()
-    
+
   integer                            :: i, unit, io_status, logunit
   integer, allocatable, dimension(:) :: pelist
   real(DOUBLE_KIND)                  :: doubledata = 0
@@ -457,7 +544,7 @@ subroutine fms_io_init()
   pack_size = size(transfer(doubledata, realarray))
   if( pack_size .NE. 1 .AND. pack_size .NE. 2) call mpp_error(FATAL,'=>fms_io_init: pack_size should be 1 or 2')
 
-  select case (threading_read) 
+  select case (threading_read)
   case ('multi')
      thread_r = MPP_MULTI
   case ('single')
@@ -467,24 +554,6 @@ subroutine fms_io_init()
   end select
 ! take namelist options if present
 
-  select case (fileset_write) 
-  case ('multi')
-     fset_w = MPP_MULTI
-  case ('single')
-     fset_w = MPP_SINGLE
-  case default
-     call mpp_error(FATAL,'fms_io_init: fileset_write should be multi/single but you chose'//trim(fileset_write))
-  end select
-
-  select case (threading_write) 
-  case ('multi')
-     thread_w = MPP_MULTI
-  case ('single')
-     thread_w = MPP_SINGLE
-  case default
-     call mpp_error(FATAL,'fms_io_init: threading_write should be multi/single but you chose'//trim(threading_write))
-  end select
-
   select case(format)
   case ('netcdf')
      form=MPP_NETCDF
@@ -496,15 +565,6 @@ subroutine fms_io_init()
   allocate(files_write(max_files_w),files_read(max_files_r))
   allocate(registered_file(max_files_w))
 
-  allocate(pelist(mpp_npes()))        
-  call mpp_get_current_pelist(pelist,pelist_name)
-  if(mpp_npes()>10000) then
-     write(pe_name,'(a,i6.6)' )'.', mpp_pe()    
-  else
-     write(pe_name,'(a,i4.4)' )'.', mpp_pe()    
-  endif
-  deallocate(pelist)
-
   do i = 1, max_domains
      array_domain(i) = NULL_DOMAIN2D
   enddo
@@ -513,7 +573,7 @@ subroutine fms_io_init()
 
   !This is set here instead of at the end of the routine to prevent the read_data call below from stopping the model
   module_is_initialized = .TRUE.
-  
+
   !--- read INPUT/grid_spec.nc to decide the value of great_circle_algorithm
   !--- great_circle_algorithm could be true only for mosaic grid.
   great_circle_algorithm = .false.
@@ -545,7 +605,7 @@ subroutine fms_io_init()
 
   if(great_circle_algorithm .AND. (mpp_pe() == mpp_root_pe()) ) then
      call mpp_error(NOTE,"fms_io_mod: great_circle algorithm will be used in the model run")
-  endif  
+  endif
 
 end subroutine fms_io_init
 
@@ -563,7 +623,7 @@ subroutine fms_io_exit()
     integer                             :: num_x_axes, num_y_axes, num_z_axes
     integer                             :: unit
     real, dimension(max_axis_size)      :: axisdata
-    real                                :: tlev  
+    real                                :: tlev
     integer,        dimension(max_axes) :: id_x_axes, siz_x_axes
     integer,        dimension(max_axes) :: id_y_axes, siz_y_axes
     integer,        dimension(max_axes) :: id_z_axes, siz_z_axes
@@ -574,6 +634,8 @@ subroutine fms_io_exit()
     character(len=256)                  :: filename
     character(len=10)                   :: axisname
     logical                             :: domain_present
+    logical                             :: write_on_this_pe
+    type(domain2d), pointer :: io_domain =>NULL()
 
     if( .NOT.module_is_initialized )return !make sure it's only called once per PE
 
@@ -604,26 +666,36 @@ subroutine fms_io_exit()
        num_z_axes = unique_axes(files_write(i), 3, id_z_axes, siz_z_axes          )
 
        if( domain_present ) then
-          call mpp_open(unit,trim(filename),action=MPP_OVERWR,form=form,threading=thread_w,&
-               fileset=fset_w, is_root_pe=files_write(i)%is_root_pe, domain=array_domain(files_write(i)%var(j)%domain_idx))
+          call mpp_open(unit,trim(filename),action=MPP_OVERWR,form=form, &
+               is_root_pe=files_write(i)%is_root_pe, domain=array_domain(files_write(i)%var(j)%domain_idx))
        else  ! global data
           call mpp_open(unit,trim(filename),action=MPP_OVERWR,form=form,threading=MPP_SINGLE,&
                fileset=MPP_SINGLE, is_root_pe=files_write(i)%is_root_pe)
        end if
 
+       write_on_this_pe = .false.
+       if(domain_present) then
+          io_domain => mpp_get_io_domain(array_domain(files_write(i)%var(j)%domain_idx))
+          if(associated(io_domain)) then
+             if(mpp_domain_is_tile_root_pe(io_domain)) write_on_this_pe = .true.
+          endif
+       endif
+       !--- always write out from root pe
+       if( files_write(i)%is_root_pe ) write_on_this_pe = .true.
+
        do j = 1, num_x_axes
          if (j < 10) then
              write(axisname,'(a,i1)') 'xaxis_',j
           else
              write(axisname,'(a,i2)') 'xaxis_',j
-          endif          
+          endif
           if(id_x_axes(j) > 0) then
              call mpp_write_meta(unit,x_axes(j),axisname,'none',axisname, &
                   data=axisdata(1:siz_x_axes(j)),domain=domain_x(id_x_axes(j)),cartesian='X')
           else
              call mpp_write_meta(unit,x_axes(j),axisname,'none',axisname, &
                   data=axisdata(1:siz_x_axes(j)),cartesian='X')
-          endif             
+          endif
        end do
 
        do j = 1, num_y_axes
@@ -631,14 +703,14 @@ subroutine fms_io_exit()
              write(axisname,'(a,i1)') 'yaxis_',j
           else
              write(axisname,'(a,i2)') 'yaxis_',j
-          endif          
+          endif
           if(id_y_axes(j) > 0) then
              call mpp_write_meta(unit,y_axes(j),axisname,'none',axisname, &
                   data=axisdata(1:siz_y_axes(j)),domain=domain_y(id_y_axes(j)),cartesian='Y')
           else
              call mpp_write_meta(unit,y_axes(j),axisname,'none',axisname, &
                   data=axisdata(1:siz_y_axes(j)),cartesian='Y')
-          endif             
+          endif
        end do
 
        do j = 1, num_z_axes
@@ -646,7 +718,7 @@ subroutine fms_io_exit()
              write(axisname,'(a,i1)') 'zaxis_',j
           else
              write(axisname,'(a,i2)') 'zaxis_',j
-          endif          
+          endif
           call mpp_write_meta(unit,z_axes(j),axisname,'none',axisname, &
                data=axisdata(1:siz_z_axes(j)),cartesian='Z')
        end do
@@ -692,8 +764,7 @@ subroutine fms_io_exit()
              if(cur_var%domain_present) then
                 call mpp_write(unit, cur_var%field,array_domain(cur_var%domain_idx), cur_var%buffer(:,:,:,kk), tlev, &
                                default_data=cur_var%default_data)
-             else if (thread_w == MPP_MULTI .or. cur_var%write_on_this_pe .OR. &
-                      (files_write(i)%is_root_pe.and.thread_w == MPP_SINGLE)) then
+             else if (write_on_this_pe) then
                 call mpp_write(unit, cur_var%field, cur_var%buffer(:,:,:,kk), tlev)
              end if
           enddo ! end j loop
@@ -701,7 +772,7 @@ subroutine fms_io_exit()
        call mpp_close(unit)
     enddo ! end i loop
 
-    !--- release the memory 
+    !--- release the memory
 
     do i = 1,  num_files_w
        do j = 1, files_write(i)%nvar
@@ -712,7 +783,7 @@ subroutine fms_io_exit()
   cur_var=>NULL()
   module_is_initialized = .false.
   num_files_w = 0
-  num_files_r = 0    
+  num_files_r = 0
 
 end subroutine fms_io_exit
 !.....................................................................
@@ -720,7 +791,7 @@ end subroutine fms_io_exit
 
 ! 
     !
-    ! This subroutine performs writing "fieldname" to file "filename". All values of "fieldname" 
+    ! This subroutine performs writing "fieldname" to file "filename". All values of "fieldname"
     ! will be written to a temporary file. The final NETCDF file will be created only at a later step
     ! when the user calls fms_io_exit. Therefore, make sure that fms_io_exit is called after all
     ! fields have been written by this subroutine.
@@ -744,14 +815,14 @@ end subroutine fms_io_exit
 subroutine write_data_i3d_new(filename, fieldname, data, domain,                  &
                               no_domain, position, tile_count, data_default)
 
-  character(len=*), intent(in) :: filename, fieldname 
+  character(len=*), intent(in) :: filename, fieldname
   integer, dimension(:,:,:), intent(in) :: data
   type(domain2d), intent(in), optional :: domain
   logical, intent(in), optional :: no_domain
   integer, intent(in), optional :: position, tile_count, data_default
   real :: default_data
 
-  default_data = 0
+  default_data = TRANSFER(MPP_FILL_INT,default_data)
   if(present(data_default)) default_data = real(data_default)
 
   call write_data_3d_new(filename, fieldname, real(data), domain,  &
@@ -761,14 +832,14 @@ end subroutine write_data_i3d_new
 subroutine write_data_i2d_new(filename, fieldname, data, domain, &
                               no_domain, position, tile_count, data_default)
 
-  character(len=*), intent(in) :: filename, fieldname 
+  character(len=*), intent(in) :: filename, fieldname
   integer, dimension(:,:), intent(in) :: data
   type(domain2d), intent(in), optional :: domain
   logical, intent(in), optional :: no_domain
   integer, intent(in), optional :: position, tile_count, data_default
   real :: default_data
 
-  default_data = 0
+  default_data = TRANSFER(MPP_FILL_INT,default_data)
   if(present(data_default)) default_data = real(data_default)
   call write_data_2d_new(filename, fieldname, real(data), domain, &
                          no_domain, position, tile_count, data_default=default_data)
@@ -778,13 +849,13 @@ end subroutine write_data_i2d_new
 subroutine write_data_i1d_new(filename, fieldname, data, domain, &
                               no_domain, tile_count, data_default)
   type(domain2d), intent(in), optional :: domain
-  character(len=*), intent(in) :: filename, fieldname 
+  character(len=*), intent(in) :: filename, fieldname
   integer, dimension(:), intent(in) :: data
   logical, intent(in), optional :: no_domain
   integer, intent(in), optional :: tile_count, data_default
   real :: default_data
 
-  default_data = 0
+  default_data = TRANSFER(MPP_FILL_INT,default_data)
   if(present(data_default)) default_data = real(data_default)
   call write_data_1d_new(filename, fieldname, real(data), domain, &
                          no_domain, tile_count, data_default=default_data)
@@ -793,13 +864,13 @@ end subroutine write_data_i1d_new
 subroutine write_data_iscalar_new(filename, fieldname, data, domain, &
                                   no_domain, tile_count, data_default)
   type(domain2d), intent(in), optional :: domain
-  character(len=*), intent(in) :: filename, fieldname 
+  character(len=*), intent(in) :: filename, fieldname
   integer, intent(in) :: data
   logical, intent(in), optional :: no_domain
   integer, intent(in), optional :: tile_count, data_default
   real :: default_data
 
-  default_data = 0
+  default_data = TRANSFER(MPP_FILL_INT,default_data)
   if(present(data_default)) default_data = real(data_default)
   call write_data_scalar_new(filename, fieldname, real(data), domain, &
                              no_domain, tile_count, data_default=default_data)
@@ -809,11 +880,11 @@ end subroutine write_data_iscalar_new
 subroutine write_data_3d_new(filename, fieldname, data, domain, no_domain, scalar_or_1d, &
                              position, tile_count, data_default)
 
-  character(len=*),         intent(in)         :: filename, fieldname 
+  character(len=*),         intent(in)         :: filename, fieldname
   real, dimension(:,:,:),   intent(in)         :: data
   type(domain2d), optional, intent(in), target :: domain
   real,           optional, intent(in)         :: data_default
-  logical,        optional, intent(in)         :: no_domain   
+  logical,        optional, intent(in)         :: no_domain
   logical,        optional, intent(in)         :: scalar_or_1d
   integer,        optional, intent(in)         :: position, tile_count
 
@@ -832,16 +903,15 @@ subroutine write_data_3d_new(filename, fieldname, data, domain, no_domain, scala
   type(domain2d), pointer, save   :: d_ptr   =>NULL()
   type(var_type), pointer, save   :: cur_var =>NULL()
   type(restart_file_type), pointer, save :: cur_file =>NULL()
-  type(domain2d), pointer,save            :: io_domain=>NULL()
 
 ! Initialize files to default values
-  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(write_data_3d_new): need to call fms_io_init')  
+  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(write_data_3d_new): need to call fms_io_init')
 
 
   if(PRESENT(data_default))then
      default_data=data_default
   else
-     default_data=0.
+     default_data=MPP_FILL_DOUBLE
   endif
 
   if(present(tile_count) .AND. .not. present(domain)) call mpp_error(FATAL, &
@@ -857,7 +927,7 @@ subroutine write_data_3d_new(filename, fieldname, data, domain, no_domain, scala
 
   if(is_no_domain) then
      if(PRESENT(domain)) &
-       call mpp_error(FATAL, 'fms_io(write_data_3d_new): no_domain cannot be .true. when optional argument domain is present.')  
+       call mpp_error(FATAL, 'fms_io(write_data_3d_new): no_domain cannot be .true. when optional argument domain is present.')
   else if(PRESENT(domain))then
      d_ptr => domain
   else if (ASSOCIATED(Current_domain)) then
@@ -877,7 +947,7 @@ subroutine write_data_3d_new(filename, fieldname, data, domain, no_domain, scala
   !Append a string to the file name
   append_string=''
 
-  !If the filename_appendix  is set override the passed argument. 
+  !If the filename_appendix  is set override the passed argument.
   if(len_trim(filename_appendix) > 0)  then
      append_pelist = .true.
      append_string = filename_appendix
@@ -900,14 +970,14 @@ subroutine write_data_3d_new(filename, fieldname, data, domain, no_domain, scala
      endif
   enddo
 
-  if (index_file < 0) then 
+  if (index_file < 0) then
      if(num_files_w == max_files_w) &  ! need to have bigger max_files_w
-          call mpp_error(FATAL,'fms_io(write_data_3d_new): max_files_w exceeded, increase it via fms_io_nml')    
+          call mpp_error(FATAL,'fms_io(write_data_3d_new): max_files_w exceeded, increase it via fms_io_nml')
      ! record the file name in array files_write
      num_files_w=num_files_w + 1
      index_file = num_files_w
      cur_file => files_write(index_file)
-     cur_file%name = trim(fname)         
+     cur_file%name = trim(fname)
      cur_file%tile_count=1
      if(present(tile_count)) cur_file%tile_count = tile_count
      if(ASSOCIATED(d_ptr))then
@@ -922,7 +992,7 @@ subroutine write_data_3d_new(filename, fieldname, data, domain, no_domain, scala
      do i = 1, max_fields
         cur_file%var(i)%name           = 'none'
         cur_file%var(i)%domain_present = .false.
-        cur_file%var(i)%write_on_this_pe = .false.
+        cur_file%var(i)%read_only      = .false.
         cur_file%var(i)%domain_idx     = -1
         cur_file%var(i)%is_dimvar      = .false.
         cur_file%var(i)%position       = CENTER
@@ -938,7 +1008,7 @@ subroutine write_data_3d_new(filename, fieldname, data, domain, no_domain, scala
      if(trim(cur_file%var(i)%name) == trim(fieldname)) then
         index_field = i
         exit
-     end if 
+     end if
   end do
 
   if(index_field > 0) then
@@ -949,10 +1019,10 @@ subroutine write_data_3d_new(filename, fieldname, data, domain, no_domain, scala
      if( cur_var%siz(4) > MAX_TIME_LEVEL_WRITE ) call mpp_error(FATAL, 'fms_io(write_data_3d_new): ' // &
           'the time level of field '//trim(cur_var%name)//' in file '//trim(cur_file%name)// &
           ' is greater than MAX_TIME_LEVEL_WRITE(=20), increase MAX_TIME_LEVEL_WRITE or check your code')
-  else 
+  else
      cur_file%nvar = cur_file%nvar +1
      if(cur_file%nvar>max_fields) then
-        write(error_msg,'(I3,"/",I3)') cur_file%nvar, max_fields 
+        write(error_msg,'(I3,"/",I3)') cur_file%nvar, max_fields
         call  mpp_error(FATAL,'fms_io(write_data_3d_new): max_fields exceeded, needs increasing, nvar/max_fields=' &
              //trim(error_msg))
      endif
@@ -967,14 +1037,7 @@ subroutine write_data_3d_new(filename, fieldname, data, domain, no_domain, scala
      cur_var%default_data = default_data
      cur_var%ndim = 3
      if(present(position)) cur_var%position = position
-     
-     if(ASSOCIATED(d_ptr)) then
-        io_domain => mpp_get_io_domain(d_ptr)
-        if(associated(io_domain)) then
-           if(mpp_domain_is_tile_root_pe(io_domain)) cur_var%write_on_this_pe = .true.
-        endif
-     endif
-     
+
      if(ASSOCIATED(d_ptr) .AND. .NOT. is_scalar_or_1d)then
         cur_var%domain_present = .true.
         domain_idx = lookup_domain(d_ptr)
@@ -1012,9 +1075,9 @@ subroutine write_data_3d_new(filename, fieldname, data, domain, no_domain, scala
      endif
   end if
 
-  ! copy the data to the buffer 
-  ! if the time level is greater than the size(cur_var%buffer,4), 
-  ! need to increase the buffer size 
+  ! copy the data to the buffer
+  ! if the time level is greater than the size(cur_var%buffer,4),
+  ! need to increase the buffer size
 
   if(cur_var%siz(4) == 1) then
      allocate(cur_var%buffer(cur_var%siz(1), cur_var%siz(2), cur_var%siz(3), cur_var%siz(4)) )
@@ -1023,18 +1086,318 @@ subroutine write_data_3d_new(filename, fieldname, data, domain, no_domain, scala
      tmp_buffer = cur_var%buffer
      deallocate(cur_var%buffer)
      allocate(cur_var%buffer(cur_var%siz(1), cur_var%siz(2), cur_var%siz(3), cur_var%siz(4)) )
-     cur_var%buffer(:,:,:,1:size(tmp_buffer,4)) = tmp_buffer 
+     cur_var%buffer(:,:,:,1:size(tmp_buffer,4)) = tmp_buffer
      deallocate(tmp_buffer)
   endif
 
-  cur_var%buffer(:,:,:,cur_var%siz(4)) = data ! copy current data to buffer for future write out 
+  cur_var%buffer(:,:,:,cur_var%siz(4)) = data ! copy current data to buffer for future write out
 
   d_ptr =>NULL()
   cur_var =>NULL()
   cur_file =>NULL()
 
 end subroutine write_data_3d_new
-!   
+! 
+
+!-------------------------------------------------------------------------------
+!
+!   This routine will register an integer restart file axis
+!
+!-------------------------------------------------------------------------------
+subroutine register_restart_axis_r1d(fileObj,filename,fieldname,data,cartesian,units,longname,sense,min,calendar)
+  type(restart_file_type),    intent(inout)      :: fileObj
+  character(len=*),           intent(in)         :: filename, fieldname
+  real,                       intent(in), target :: data(:)
+  character(len=*),           intent(in)         :: cartesian
+  character(len=*), optional, intent(in)         :: units, longname
+  integer,          optional, intent(in)         :: sense
+  real,             optional, intent(in)         :: min !valid min for real axis data
+  character(len=*), optional, intent(in)         :: calendar
+
+  integer :: idx
+
+
+  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(register_restart_axis_r1d): need to call fms_io_init')
+
+  select case(trim(cartesian))
+    case('X')
+      idx = XIDX
+    case('Y')
+      idx = YIDX
+    case('Z')
+      idx = ZIDX
+    case('T')
+      idx = TIDX
+    case default
+      call mpp_error(FATAL,'fms_io(register_restart_axis_r1d): Axis must be one of X,Y or Z ' // &
+                                                           'but has value '//trim(cartesian))
+  end select
+  if(.not. ALLOCATED(fileObj%axes)) allocate(fileObj%axes(NIDX))
+  if(ASSOCIATED(fileObj%axes(idx)%data)) &
+       call mpp_error(FATAL,'fms_io(register_restart_axis_r1d): '//trim(cartesian)//' axis has already been defined')
+  fileObj%name = filename
+  fileObj%axes(idx)%name = fieldname
+  fileObj%axes(idx)%data =>data
+  fileObj%axes(idx)%cartesian = cartesian
+  fileObj%axes(idx)%dimlen = -1   ! This is not a compressed axis
+  if(PRESENT(units)) fileObj%axes(idx)%units = units
+  if(PRESENT(longname)) fileObj%axes(idx)%longname = longname
+  if(PRESENT(min)) fileObj%axes(idx)%min = min
+  if(idx == TIDX) then
+     if(PRESENT(calendar)) fileObj%axes(idx)%calendar = trim(calendar)
+  endif
+  if(PRESENT(sense)) then
+     if(idx /= ZIDX) call mpp_error(FATAL,'fms_io(register_restart_axis_r1d): Only the Z axis may define sense; ' // &
+                                    'Axis = '//trim(cartesian))
+     if(abs(sense) /= 1) call mpp_error(FATAL,'fms_io(register_restart_axis_r1d): Value of sense must be +/- 1')
+     fileObj%axes(idx)%sense = sense
+  endif
+end subroutine register_restart_axis_r1d
+
+!-------------------------------------------------------------------------------
+!
+!   This routine will register the compressed index restart file axis
+!
+!-------------------------------------------------------------------------------
+subroutine register_restart_axis_i1d(fileObj,filename,fieldname,data,compressed, &
+                                     compressed_axis,dimlen,dimlen_name,dimlen_lname,units,longname,imin)
+  type(restart_file_type),    intent(inout)      :: fileObj
+  character(len=*),           intent(in)         :: filename, fieldname
+  integer,                    intent(in)         :: data(:)
+  character(len=*),           intent(in)         :: compressed
+  character(len=*),           intent(in)         :: compressed_axis !< which compressed axis (C or H)
+  integer,                    intent(in)         :: dimlen
+  character(len=*), optional, intent(in)         :: dimlen_name, dimlen_lname !< dimlen axis name and longname
+  character(len=*), optional, intent(in)         :: units, longname
+  integer,          optional, intent(in)         :: imin !valid min for integer axis data
+
+  integer :: ssize,rsize,npes
+  integer :: idx
+  integer, allocatable :: pelist(:)
+  type(domain2d), pointer :: io_domain=>NULL()
+
+
+  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(register_restart_axis_i1d): need to call fms_io_init')
+
+  select case(trim(compressed_axis))
+  case('C')
+     idx = CIDX
+  case('H')
+     idx = HIDX
+  case default
+     call mpp_error(FATAL,'fms_io(register_restart_axis_r1d): Axis must be one of C or H ' // &
+          'but has value '//trim(compressed_axis))
+  end select
+  
+  if(.not. ALLOCATED(fileObj%axes)) allocate(fileObj%axes(NIDX))
+  if(ALLOCATED(fileObj%axes(idx)%idx)) &
+                 call mpp_error(FATAL,'fms_io(register_restart_axis_i1d): Compressed axis ' //&
+                 trim(compressed_axis) // ' has already been defined')
+  fileObj%name = filename
+  fileObj%is_compressed = .true.
+  fileObj%unlimited_axis = .false.
+  fileObj%axes(idx)%name = fieldname
+  if(ASSOCIATED(current_domain)) then
+     fileObj%axes(idx)%domain =>current_domain
+     io_domain =>mpp_get_io_domain(current_domain)
+     if(.not. ASSOCIATED(io_domain)) &
+                 call mpp_error(FATAL,'fms_io(register_restart_axis_i1d): The io domain must be defined')
+     npes = mpp_get_domain_npes(io_domain)
+     allocate(fileObj%axes(idx)%nelems(npes)); fileObj%axes(idx)%nelems = 0
+     allocate(pelist(npes))
+     call mpp_get_pelist(io_domain,pelist)
+     ssize = size(data)
+     call mpp_gather((/ssize/),fileObj%axes(idx)%nelems,pelist)
+     rsize = sum(fileObj%axes(idx)%nelems)
+     allocate( fileObj%axes(idx)%idx(rsize) )
+  !  Note that the gatherV implied here is asymmetric; only root needs to know the vector of recv sizes
+     call mpp_gather(data,ssize,fileObj%axes(idx)%idx,fileObj%axes(idx)%nelems,pelist)
+     deallocate(pelist); io_domain=>NULL()
+  else
+     call mpp_error(FATAL,'fms_io(register_restart_axis_i1d): The domain must be defined through set_domain')
+  endif
+  fileObj%axes(idx)%compressed = compressed
+  fileObj%axes(idx)%dimlen = dimlen
+  if(PRESENT(dimlen_name)) fileObj%axes(idx)%dimlen_name = dimlen_name
+  if(PRESENT(dimlen_lname)) fileObj%axes(idx)%dimlen_lname = dimlen_lname
+  if(PRESENT(units)) fileObj%axes(idx)%units = units
+  if(PRESENT(longname)) fileObj%axes(idx)%longname = longname
+  if(PRESENT(imin)) fileObj%axes(idx)%imin = imin
+end subroutine register_restart_axis_i1d
+
+!-------------------------------------------------------------------------------
+
+subroutine register_restart_axis_unlimited(fileObj,filename,fieldname,nelem,units,longname)
+  type(restart_file_type),    intent(inout)      :: fileObj
+  character(len=*),           intent(in)         :: filename, fieldname
+  integer                                        :: nelem  ! Number of elements on rank
+  character(len=*), optional, intent(in)         :: units, longname
+
+  integer :: idx,npes
+  integer, allocatable :: pelist(:)
+  type(domain2d), pointer :: io_domain=>NULL()
+
+
+  if(.not.module_is_initialized) &
+                call mpp_error(FATAL,'fms_io(register_restart_axis_unlimited): need to call fms_io_init')
+  idx = UIDX
+
+  if(.not. ALLOCATED(fileObj%axes)) allocate(fileObj%axes(NIDX))
+  if(ALLOCATED(fileObj%axes(idx)%idx)) &
+               call mpp_error(FATAL,'fms_io(register_restart_axis_unlimited): Unlimited axis has already been defined')
+  fileObj%name = filename
+  fileObj%is_compressed = .false.
+  fileObj%unlimited_axis = .true.
+  fileObj%axes(idx)%name = fieldname
+  if(ASSOCIATED(current_domain)) then
+     fileObj%axes(idx)%domain =>current_domain
+     io_domain =>mpp_get_io_domain(current_domain)
+     if(.not. ASSOCIATED(io_domain)) &
+                 call mpp_error(FATAL,'fms_io(register_restart_axis_i1d): The io domain must be defined')
+     npes = mpp_get_domain_npes(io_domain)
+     allocate(fileObj%axes(idx)%nelems(npes)); fileObj%axes(idx)%nelems = 0
+     allocate(pelist(npes))
+     call mpp_get_pelist(io_domain,pelist)
+     call mpp_gather((/nelem/),fileObj%axes(idx)%nelems,pelist)
+     deallocate(pelist); io_domain=>NULL()
+  else
+     call mpp_error(FATAL,'fms_io(register_restart_axis_unlimited): The domain must be defined through set_domain')
+  endif
+  if(PRESENT(units)) fileObj%axes(idx)%units = units
+  if(PRESENT(longname)) fileObj%axes(idx)%longname = longname
+end subroutine register_restart_axis_unlimited
+
+!
+!   This routine is the destructor for the file object
+!
+!-------------------------------------------------------------------------------
+subroutine free_restart_type(fileObj)
+  type(restart_file_type), intent(inout)      :: fileObj
+  type(meta_type),pointer                :: this
+  type(meta_type),pointer                :: this_p
+  integer :: id, n
+
+  !--- remove file name from registered_file
+  id = fileObj%register_id
+  if( id > num_registered_files .OR. id < 1 ) then
+     print*, " register_id = ", id, " and num_registered_files = ", num_registered_files
+     call mpp_error(FATAL, &
+        'fms_io(free_restart_type): fileObj%register_id should be between 1 and num_registered_files')
+  endif
+  if( trim(fileObj%name) .NE. trim(registered_file(id)) ) &
+     call mpp_error(FATAL, 'fms_io(free_restart_type): fileObj%name .NE. registered_file(id)')
+  do n = id+1, num_registered_files
+     registered_file(n-1) = trim(registered_file(n))
+  enddo
+  registered_file(num_registered_files) = ''
+  num_registered_files = num_registered_files - 1
+
+  fileObj%register_id = 0
+  fileObj%unit = -1
+  fileObj%name = ''
+  fileObj%nvar = -1
+  fileObj%natt = -1
+  fileObj%max_ntime = -1
+  fileObj%tile_count = -1
+  if(ALLOCATED(fileObj%axes)) deallocate(fileObj%axes)
+  if(ASSOCIATED(fileObj%var)) deallocate(fileObj%var)
+  if(ASSOCIATED(fileObj%p0dr)) deallocate(fileObj%p0dr)
+  if(ASSOCIATED(fileObj%p1dr)) deallocate(fileObj%p1dr)
+  if(ASSOCIATED(fileObj%p2dr)) deallocate(fileObj%p2dr)
+  if(ASSOCIATED(fileObj%p3dr)) deallocate(fileObj%p3dr)
+  if(ASSOCIATED(fileObj%p0di)) deallocate(fileObj%p0di)
+  if(ASSOCIATED(fileObj%p1di)) deallocate(fileObj%p1di)
+  if(ASSOCIATED(fileObj%p2di)) deallocate(fileObj%p2di)
+  if(ASSOCIATED(fileObj%p3di)) deallocate(fileObj%p3di)
+  if(ASSOCIATED(fileObj%first)) then
+     this =>fileObj%first
+     do while(associated(this%next))
+        this =>this%next  ! Find the last element
+     enddo
+     do while(associated(this))  ! Deallocate from the last element to the first
+       this_p =>this%prev
+!!$ Gfortran on gaea does not yet support deferred length character strings
+!!$       deallocate(this%name)
+       this%name=''  ! Remove this line when Gfortran supports deferred length character strings
+       if(allocated(this%rval)) deallocate(this%rval)
+       if(allocated(this%ival)) deallocate(this%ival)
+!!$ Gfortran on gaea does not yet support deferred length character strings
+!!$       if(allocated(this%cval)) deallocate(this%cval)
+       this%cval=''  ! Remove this line when Gfortran supports deferred length character strings
+       deallocate(this)
+       this =>this_p
+     enddo
+     fileObj%first =>NULL()
+  endif
+end subroutine free_restart_type
+
+!-------------------------------------------------------------------------------
+!
+!   The routine sets up a list of global metadata expressions for save_restart
+!
+!-------------------------------------------------------------------------------
+subroutine set_meta_global(fileObj, name, rval, ival, cval)
+  type(restart_file_type), intent(inout) :: fileObj
+  character(len=*), intent(in)           :: name
+  real,             intent(in), optional :: rval(:)
+  integer,          intent(in), optional :: ival(:)
+  character(len=*), intent(in), optional :: cval
+  type(meta_type),pointer                :: this
+  type(meta_type),pointer                :: this_n
+
+  this =>fileObj%first
+  if(associated(this))then
+     do while(associated(this%next))
+        this =>this%next
+     enddo
+     allocate(this_n); this%next =>this_n; this_n%prev =>this; this =>this_n
+  else
+     allocate(this)
+     fileObj%first =>this
+  endif
+
+! Per mpp_write_meta_global, only one type of data can be associated with the metadata
+!!$ Gfortran on gaea does not yet support deferred length character strings
+!!$  allocate(character(len(name)) :: this%name); this%name = name
+  this%name = name  ! Remove this line when Gfortran supports deferred length character stings
+  if(present(rval))then
+     allocate(this%rval(size(rval))); this%rval=rval
+  elseif(present(ival))then
+     allocate(this%ival(size(ival))); this%ival=ival
+  elseif(present(cval))then
+!!$ Gfortran on gaea does not yet support deferred length character strings
+!!$     allocate(character(len(cval)) :: this%cval); this%cval = cval
+     this%cval=cval  ! Remove this line when Gfortran supports deferred length character stings
+  endif
+end subroutine set_meta_global
+
+
+!-------------------------------------------------------------------------------
+!
+!   The routine writes the global metadata
+!
+!-------------------------------------------------------------------------------
+subroutine write_meta_global(unit,fileObj)
+  integer,                 intent(in) :: unit
+  type(restart_file_type), intent(in) :: fileObj
+  type(meta_type), pointer            :: this
+
+  this =>fileObj%first
+  do while(associated(this))
+     if(allocated(this%rval))then
+        call mpp_write_meta(unit,this%name,rval=this%rval)
+     elseif(allocated(this%ival))then
+        call mpp_write_meta(unit,this%name,ival=this%ival)
+!!$ Gfortran on gaea does not yet support deferred length character strings
+!!$     elseif(allocated(this%cval))then
+     elseif(len_trim(this%cval).GT.0)then  ! Remove this line when Gfortran supports deferred length character stings
+        call mpp_write_meta(unit,this%name,cval=this%cval)
+     else
+        call mpp_write_meta(unit,this%name)
+     endif
+     this =>this%next
+  enddo
+end subroutine write_meta_global
 
 !-------------------------------------------------------------------------------
 !
@@ -1042,28 +1405,29 @@ end subroutine write_data_3d_new
 !
 !-------------------------------------------------------------------------------
 function register_restart_field_r0d(fileObj, filename, fieldname, data, domain, mandatory, &
-                                    no_domain, position, tile_count, data_default, longname, units)
+                                    no_domain, position, tile_count, data_default, longname, units, read_only)
   type(restart_file_type),    intent(inout)      :: fileObj
   character(len=*),           intent(in)         :: filename, fieldname
-  real,                       intent(in), target :: data 
+  real,                       intent(in), target :: data
   type(domain2d),   optional, intent(in), target :: domain
   logical,          optional, intent(in)         :: no_domain
   real,             optional, intent(in)         :: data_default
   logical,          optional, intent(in)         :: mandatory
   integer,          optional, intent(in)         :: position, tile_count
   character(len=*), optional, intent(in)         :: longname, units
+  logical,          optional, intent(in)         :: read_only
   integer                                        :: index_field
   integer                                        :: register_restart_field_r0d
 
-  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(register_restart_field_r0d): need to call fms_io_init')  
+  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(register_restart_field_r0d): need to call fms_io_init')
   call setup_one_field(fileObj, filename, fieldname, (/1, 1, 1, 1/), index_field, domain, mandatory, &
                        no_domain, scalar_or_1d=.true., position=position, tile_count=tile_count, &
-                       data_default=data_default, longname=longname, units=units)
+                       data_default=data_default, longname=longname, units=units, read_only=read_only)
   fileObj%p0dr(fileObj%var(index_field)%siz(4), index_field)%p => data
   fileObj%var(index_field)%ndim = 0
-  register_restart_field_r0d = index_field  
+  register_restart_field_r0d = index_field
 
-  return    
+  return
 
 end function register_restart_field_r0d
 
@@ -1073,30 +1437,32 @@ end function register_restart_field_r0d
 !
 !-------------------------------------------------------------------------------
 function register_restart_field_r1d(fileObj, filename, fieldname, data, domain, mandatory, &
-                             no_domain, position, tile_count, data_default, longname, units)
+                             no_domain, position, tile_count, data_default, longname, units, compressed_axis, read_only)
   type(restart_file_type), intent(inout)         :: fileObj
   character(len=*),           intent(in)         :: filename, fieldname
-  real, dimension(:),         intent(in), target :: data 
+  real, dimension(:),         intent(in), target :: data
   type(domain2d),   optional, intent(in), target :: domain
   logical,          optional, intent(in)         :: no_domain
   real,             optional, intent(in)         :: data_default
   integer,          optional, intent(in)         :: position, tile_count
   logical,          optional, intent(in)         :: mandatory
-  character(len=*), optional, intent(in)         :: longname, units
+  character(len=*), optional, intent(in)         :: longname, units, compressed_axis
+  logical,          optional, intent(in)         :: read_only
   integer                                        :: index_field
   integer                                        :: register_restart_field_r1d
 
-  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(register_restart_field_r1d): need to call fms_io_init')  
+  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(register_restart_field_r1d): need to call fms_io_init')
   call setup_one_field(fileObj, filename, fieldname, (/size(data,1), 1, 1, 1/), index_field, domain, mandatory, &
                        no_domain, scalar_or_1d=.true., position=position, tile_count=tile_count, &
-                       data_default=data_default, longname=longname, units=units )
+                       data_default=data_default, longname=longname, units=units, compressed_axis=compressed_axis, &
+                       read_only=read_only )
 
   fileObj%p1dr(fileObj%var(index_field)%siz(4), index_field)%p => data
   fileObj%var(index_field)%ndim = 1
-  register_restart_field_r1d = index_field  
+  register_restart_field_r1d = index_field
+
+  return
 
-  return    
-  
 end function register_restart_field_r1d
 
 !-------------------------------------------------------------------------------
@@ -1104,30 +1470,35 @@ end function register_restart_field_r1d
 !   The routine will register a 2-D real restart file field with one time level
 !
 !-------------------------------------------------------------------------------
-function register_restart_field_r2d(fileObj, filename, fieldname, data, domain, mandatory, &
-                                    no_domain, position, tile_count, data_default, longname, units)
+function register_restart_field_r2d(fileObj, filename, fieldname, data, domain, mandatory, no_domain, &
+                                    compressed, position, tile_count, data_default, longname, units, &
+                                    compressed_axis, read_only)
   type(restart_file_type), intent(inout)         :: fileObj
   character(len=*),           intent(in)         :: filename, fieldname
-  real,     dimension(:,:),   intent(in), target :: data 
+  real,     dimension(:,:),   intent(in), target :: data
   type(domain2d),   optional, intent(in), target :: domain
   real,             optional, intent(in)         :: data_default
-  logical,          optional, intent(in)         :: no_domain   
+  logical,          optional, intent(in)         :: no_domain
+  logical,          optional, intent(in)         :: compressed
   integer,          optional, intent(in)         :: position, tile_count
   logical,          optional, intent(in)         :: mandatory
-  character(len=*), optional, intent(in)         :: longname, units
-
+  character(len=*), optional, intent(in)         :: longname, units, compressed_axis
+  logical,          optional, intent(in)         :: read_only
+  logical                                        :: is_compressed
   integer                                        :: index_field
   integer                                        :: register_restart_field_r2d
 
-  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(register_restart_field_r2d): need to call fms_io_init')  
+  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(register_restart_field_r2d): need to call fms_io_init')
+  is_compressed = .false.
+  if(present(compressed)) is_compressed=compressed
   call setup_one_field(fileObj, filename, fieldname, (/size(data,1), size(data,2), 1, 1/), &
-                       index_field, domain, mandatory, no_domain, .false., &
-                       position, tile_count, data_default, longname, units)
+                       index_field, domain, mandatory, no_domain, is_compressed, &
+                       position, tile_count, data_default, longname, units, compressed_axis, read_only=read_only)
   fileObj%p2dr(fileObj%var(index_field)%siz(4), index_field)%p => data
   fileObj%var(index_field)%ndim = 2
-  register_restart_field_r2d = index_field    
+  register_restart_field_r2d = index_field
 
-  return    
+  return
 
 end function register_restart_field_r2d
 
@@ -1138,60 +1509,103 @@ end function register_restart_field_r2d
 !
 !-------------------------------------------------------------------------------
 function register_restart_field_r3d(fileObj, filename, fieldname, data, domain, mandatory, &
-                             no_domain, position, tile_count, data_default, longname, units)
+                             no_domain, position, tile_count, data_default, longname, units, read_only)
   type(restart_file_type), intent(inout)         :: fileObj
   character(len=*),           intent(in)         :: filename, fieldname
-  real,     dimension(:,:,:), intent(in), target :: data 
+  real,     dimension(:,:,:), intent(in), target :: data
   type(domain2d),   optional, intent(in), target :: domain
   real,             optional, intent(in)         :: data_default
-  logical,          optional, intent(in)         :: no_domain   
+  logical,          optional, intent(in)         :: no_domain
   integer,          optional, intent(in)         :: position, tile_count
   logical,          optional, intent(in)         :: mandatory
   character(len=*), optional, intent(in)         :: longname, units
+  logical,          optional, intent(in)         :: read_only
   integer                                        :: index_field
   integer                                        :: register_restart_field_r3d
 
-  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(register_restart_field_r3d): need to call fms_io_init')  
+  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(register_restart_field_r3d): need to call fms_io_init')
   call setup_one_field(fileObj, filename, fieldname, (/size(data,1), size(data,2), size(data,3), 1/), &
                        index_field, domain, mandatory, no_domain, .false., &
-                       position, tile_count, data_default, longname, units)
+                       position, tile_count, data_default, longname, units, read_only=read_only)
   fileObj%p3dr(fileObj%var(index_field)%siz(4), index_field)%p => data
   fileObj%var(index_field)%ndim = 3
-  register_restart_field_r3d = index_field   
+  register_restart_field_r3d = index_field
 
-  return    
+  return
 
 end function register_restart_field_r3d
 
+
+!-------------------------------------------------------------------------------
+!
+!   The routine will register a 4-D real restart file field with one time level
+!
+!-------------------------------------------------------------------------------
+function register_restart_field_r4d(fileObj, filename, fieldname, data, domain, mandatory, &
+                             no_domain, position, tile_count, data_default, longname, units, read_only)
+  type(restart_file_type),   intent(inout)         :: fileObj
+  character(len=*),             intent(in)         :: filename, fieldname
+  real,     dimension(:,:,:,:), intent(in), target :: data
+  type(domain2d),   optional,   intent(in), target :: domain
+  real,             optional,   intent(in)         :: data_default
+  logical,          optional,   intent(in)         :: no_domain
+  integer,          optional,   intent(in)         :: position, tile_count
+  logical,          optional,   intent(in)         :: mandatory
+  character(len=*), optional,   intent(in)         :: longname, units
+  logical,          optional,   intent(in)         :: read_only
+  integer                                          :: index_field
+  integer                                          :: register_restart_field_r4d
+
+  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(register_restart_field_r4d): need to call fms_io_init')
+  call setup_one_field(fileObj, filename, fieldname, (/size(data,1), size(data,2), size(data,3), 1, size(data,4)/), &
+                       index_field, domain, mandatory, no_domain, .false., &
+                       position, tile_count, data_default, longname, units, read_only=read_only)
+  fileObj%p4dr(fileObj%var(index_field)%siz(4), index_field)%p => data
+  fileObj%var(index_field)%ndim = 4
+  register_restart_field_r4d = index_field
+
+  return
+
+end function register_restart_field_r4d
+
+
 !-------------------------------------------------------------------------------
 !
 !   The routine will register a scalar integer restart file field with one time level
 !
 !-------------------------------------------------------------------------------
 function register_restart_field_i0d(fileObj, filename, fieldname, data, domain, mandatory, &
-                             no_domain, position, tile_count, data_default, longname, units)
+                             no_domain, position, tile_count, data_default, longname, units, read_only)
   type(restart_file_type), intent(inout)         :: fileObj
   character(len=*),           intent(in)         :: filename, fieldname
-  integer,                    intent(in), target :: data 
+  integer,                    intent(in), target :: data
   type(domain2d),   optional, intent(in), target :: domain
-  real,             optional, intent(in)         :: data_default
+  integer,             optional, intent(in)      :: data_default
   integer,          optional, intent(in)         :: position, tile_count
   logical,          optional, intent(in)         :: mandatory
-  logical,          optional, intent(in)         :: no_domain  
+  logical,          optional, intent(in)         :: no_domain
   character(len=*), optional, intent(in)         :: longname, units
+  logical,          optional, intent(in)         :: read_only
   integer                                        :: index_field
   integer                                        :: register_restart_field_i0d
+  real                                           :: data_default_r
+
+  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(register_restart_field_i0d): need to call fms_io_init')
+
+  if (KIND(data_default)/=KIND(data)) call mpp_error(FATAL,'fms_io(register_restart_field_i0d): data_default and data different KIND()')
+  data_default_r = TRANSFER(MPP_FILL_INT,data_default_r) 
+  if (present(data_default)) data_default_r = TRANSFER(data_default ,data_default_r)
 
-  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(register_restart_field_i0d): need to call fms_io_init')  
   call setup_one_field(fileObj, filename, fieldname, (/1, 1, 1, 1/), index_field, domain, &
                        mandatory, no_domain=no_domain, scalar_or_1d=.true., position=position, tile_count=tile_count, &
-                       data_default=data_default, longname=longname, units=units)
+                          data_default=data_default_r, longname=longname, units=units, read_only=read_only)
+  
   fileObj%p0di(fileObj%var(index_field)%siz(4), index_field)%p => data
   fileObj%var(index_field)%ndim = 0
   register_restart_field_i0d = index_field
-  
+
   return
-   
+
 end function register_restart_field_i0d
 
 !-------------------------------------------------------------------------------
@@ -1200,29 +1614,38 @@ end function register_restart_field_i0d
 !
 !-------------------------------------------------------------------------------
 function register_restart_field_i1d(fileObj, filename, fieldname, data, domain, mandatory, &
-                             no_domain, position, tile_count, data_default, longname, units)
+                             no_domain, position, tile_count, data_default, longname, units, &
+                             compressed_axis, read_only)
   type(restart_file_type), intent(inout)         :: fileObj
   character(len=*),           intent(in)         :: filename, fieldname
-  integer, dimension(:),      intent(in), target :: data 
+  integer, dimension(:),      intent(in), target :: data
   type(domain2d),   optional, intent(in), target :: domain
-  real,             optional, intent(in)         :: data_default
+  integer,          optional, intent(in)         :: data_default
   integer,          optional, intent(in)         :: position, tile_count
   logical,          optional, intent(in)         :: mandatory
   logical,          optional, intent(in)         :: no_domain
-  character(len=*), optional, intent(in)         :: longname, units
+  character(len=*), optional, intent(in)         :: longname, units, compressed_axis
+  logical,          optional, intent(in)         :: read_only
   integer                                        :: index_field
   integer                                        :: register_restart_field_i1d
+  real                                           :: data_default_r
+
+  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(register_restart_field_i1d): need to call fms_io_init')
+
+  if (KIND(data_default)/=KIND(data)) call mpp_error(FATAL,'fms_io(register_restart_field_i1d): data_default and data different KIND()')
+  data_default_r = TRANSFER(MPP_FILL_INT,data_default_r) 
+  if (present(data_default)) data_default_r = TRANSFER(data_default ,data_default_r)
 
-  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(register_restart_field_i1d): need to call fms_io_init')  
   call setup_one_field(fileObj, filename, fieldname, (/size(data,1), 1, 1, 1/), index_field, domain, &
                        mandatory, no_domain=no_domain, scalar_or_1d=.true., position=position, tile_count=tile_count, &
-                       data_default=data_default, longname=longname, units=units)
+                       data_default=data_default_r, longname=longname, units=units, compressed_axis=compressed_axis, &
+                       read_only=read_only)
   fileObj%p1di(fileObj%var(index_field)%siz(4), index_field)%p => data
   fileObj%var(index_field)%ndim = 1
   register_restart_field_i1d = index_field
-  
+
   return
-  
+
 end function register_restart_field_i1d
 
 
@@ -1231,29 +1654,42 @@ end function register_restart_field_i1d
 !   The routine will register a 2-D real restart file field with one time level
 !
 !-------------------------------------------------------------------------------
-function register_restart_field_i2d(fileObj, filename, fieldname, data, domain, mandatory, &
-                             no_domain, position, tile_count, data_default, longname, units)
+function register_restart_field_i2d(fileObj, filename, fieldname, data, domain, mandatory, no_domain, &
+                             compressed, position, tile_count, data_default, longname, units, &
+                             compressed_axis, read_only)
   type(restart_file_type), intent(inout)         :: fileObj
   character(len=*),           intent(in)         :: filename, fieldname
-  integer,  dimension(:,:),   intent(in), target :: data 
+  integer,  dimension(:,:),   intent(in), target :: data
   type(domain2d),   optional, intent(in), target :: domain
-  real,             optional, intent(in)         :: data_default
-  logical,          optional, intent(in)         :: no_domain   
+  integer,          optional, intent(in)         :: data_default
+  logical,          optional, intent(in)         :: no_domain
+  logical,          optional, intent(in)         :: compressed
   integer,          optional, intent(in)         :: position, tile_count
   logical,          optional, intent(in)         :: mandatory
-  character(len=*), optional, intent(in)         :: longname, units
+  character(len=*), optional, intent(in)         :: longname, units, compressed_axis
+  logical,          optional, intent(in)         :: read_only
+  logical                                        :: is_compressed
   integer                                        :: index_field
   integer                                        :: register_restart_field_i2d
+  real                                           :: data_default_r
+
+  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(register_restart_field_i2d): need to call fms_io_init')
+  is_compressed = .false.
+  if(present(compressed)) is_compressed=compressed
 
-  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(register_restart_field_i2d): need to call fms_io_init')  
+  if (KIND(data_default)/=KIND(data)) call mpp_error(FATAL,'fms_io(register_restart_field_i2d): data_default and data different KIND()')
+  data_default_r = TRANSFER(MPP_FILL_INT,data_default_r) 
+  if (present(data_default)) data_default_r = TRANSFER(data_default ,data_default_r)
+  
   call setup_one_field(fileObj, filename, fieldname, (/size(data,1), size(data,2), 1, 1/), &
-                       index_field, domain, mandatory, no_domain, .false., &
-                       position, tile_count, data_default, longname, units)
+                       index_field, domain, mandatory, no_domain, is_compressed, &
+                       position, tile_count, data_default_r, longname, units, compressed_axis, &
+                       read_only=read_only)
   fileObj%p2di(fileObj%var(index_field)%siz(4), index_field)%p => data
   fileObj%var(index_field)%ndim = 2
   register_restart_field_i2d = index_field
-  
-  return 
+
+  return
 
 end function register_restart_field_i2d
 
@@ -1263,28 +1699,35 @@ end function register_restart_field_i2d
 !
 !-------------------------------------------------------------------------------
 function register_restart_field_i3d(fileObj, filename, fieldname, data, domain, mandatory, &
-                             no_domain, position, tile_count, data_default, longname, units)
+                             no_domain, position, tile_count, data_default, longname, units, read_only)
   type(restart_file_type), intent(inout)         :: fileObj
   character(len=*),           intent(in)         :: filename, fieldname
-  integer,  dimension(:,:,:), intent(in), target :: data 
+  integer,  dimension(:,:,:), intent(in), target :: data
   type(domain2d),   optional, intent(in), target :: domain
-  real,             optional, intent(in)         :: data_default
-  logical,          optional, intent(in)         :: no_domain   
+  integer,             optional, intent(in)         :: data_default
+  logical,          optional, intent(in)         :: no_domain
   integer,          optional, intent(in)         :: position, tile_count
   logical,          optional, intent(in)         :: mandatory
   character(len=*), optional, intent(in)         :: longname, units
+  logical,          optional, intent(in)         :: read_only
   integer                                        :: index_field
   integer                                        :: register_restart_field_i3d
+  real                                           :: data_default_r
+
+  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(register_restart_field_i3d): need to call fms_io_init')
 
-  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(register_restart_field_i3d): need to call fms_io_init')  
+  if (KIND(data_default)/=KIND(data)) call mpp_error(FATAL,'fms_io(register_restart_field_i3d): data_default and data different KIND()')
+  data_default_r = TRANSFER(MPP_FILL_INT,data_default_r) 
+  if (present(data_default)) data_default_r = TRANSFER(data_default ,data_default_r)
+  
   call setup_one_field(fileObj, filename, fieldname, (/size(data,1), size(data,2), size(data,3), 1/), &
                        index_field, domain, mandatory, no_domain, .false., &
-                       position, tile_count, data_default, longname, units)
+                       position, tile_count, data_default_r, longname, units, read_only=read_only)
   fileObj%p3di(fileObj%var(index_field)%siz(4), index_field)%p => data
   fileObj%var(index_field)%ndim = 3
   register_restart_field_i3d = index_field
-  
-  return  
+
+  return
 
 end function register_restart_field_i3d
 
@@ -1294,30 +1737,31 @@ end function register_restart_field_i3d
 !
 !-------------------------------------------------------------------------------
 function register_restart_field_r0d_2level(fileObj, filename, fieldname, data1, data2, domain, mandatory, &
-                             no_domain, position, tile_count, data_default, longname, units)
+                             no_domain, position, tile_count, data_default, longname, units, read_only)
   type(restart_file_type), intent(inout)         :: fileObj
   character(len=*),           intent(in)         :: filename, fieldname
-  real,                       intent(in), target :: data1, data2 
+  real,                       intent(in), target :: data1, data2
   type(domain2d),   optional, intent(in), target :: domain
   real,             optional, intent(in)         :: data_default
   integer,          optional, intent(in)         :: position, tile_count
   logical,          optional, intent(in)         :: mandatory
   logical,          optional, intent(in)         :: no_domain
   character(len=*), optional, intent(in)         :: longname, units
+  logical,          optional, intent(in)         :: read_only
   integer                                        :: index_field
   integer                                        :: register_restart_field_r0d_2level
 
   if(.not.module_is_initialized) call mpp_error(FATAL, &
-      'fms_io(register_restart_field_r0d_2level): need to call fms_io_init')  
+      'fms_io(register_restart_field_r0d_2level): need to call fms_io_init')
   call setup_one_field(fileObj, filename, fieldname, (/1, 1, 1, 2/), index_field, domain, &
                        mandatory, no_domain=no_domain, scalar_or_1d=.true., position=position, tile_count=tile_count, &
-                       data_default=data_default, longname=longname, units=units)
+                       data_default=data_default, longname=longname, units=units, read_only=read_only)
   fileObj%p0dr(1, index_field)%p => data1
   fileObj%p0dr(2, index_field)%p => data2
   fileObj%var(index_field)%ndim = 0
   register_restart_field_r0d_2level = index_field
-  
-  return  
+
+  return
 
 end function register_restart_field_r0d_2level
 
@@ -1327,30 +1771,31 @@ end function register_restart_field_r0d_2level
 !
 !-------------------------------------------------------------------------------
 function register_restart_field_r1d_2level(fileObj, filename, fieldname, data1, data2, domain, mandatory, &
-                             no_domain, position, tile_count, data_default, longname, units)
+                             no_domain, position, tile_count, data_default, longname, units, read_only)
   type(restart_file_type), intent(inout)         :: fileObj
   character(len=*),           intent(in)         :: filename, fieldname
-  real,     dimension(:),     intent(in), target :: data1, data2 
+  real,     dimension(:),     intent(in), target :: data1, data2
   type(domain2d),   optional, intent(in), target :: domain
   real,             optional, intent(in)         :: data_default
   integer,          optional, intent(in)         :: position, tile_count
   logical,          optional, intent(in)         :: mandatory
   logical,          optional, intent(in)         :: no_domain
   character(len=*), optional, intent(in)         :: longname, units
+  logical,          optional, intent(in)         :: read_only
   integer                                        :: index_field
   integer                                        :: register_restart_field_r1d_2level
 
   if(.not.module_is_initialized) call mpp_error(FATAL, &
-      'fms_io(register_restart_field_r1d_2level): need to call fms_io_init')  
+      'fms_io(register_restart_field_r1d_2level): need to call fms_io_init')
   call setup_one_field(fileObj, filename, fieldname, (/size(data1,1), 1, 1, 2/), index_field, domain, &
                        mandatory, no_domain=no_domain, scalar_or_1d=.true., position=position, tile_count=tile_count, &
-                       data_default=data_default, longname=longname, units=units)
+                       data_default=data_default, longname=longname, units=units, read_only=read_only)
   fileObj%p1dr(1, index_field)%p => data1
   fileObj%p1dr(2, index_field)%p => data2
   fileObj%var(index_field)%ndim = 1
   register_restart_field_r1d_2level = index_field
-  
-  return    
+
+  return
 
 end function register_restart_field_r1d_2level
 
@@ -1360,30 +1805,31 @@ end function register_restart_field_r1d_2level
 !
 !-------------------------------------------------------------------------------
 function register_restart_field_r2d_2level(fileObj, filename, fieldname, data1, data2, domain, mandatory, &
-                             no_domain, position, tile_count, data_default, longname, units)
+                             no_domain, position, tile_count, data_default, longname, units, read_only)
   type(restart_file_type), intent(inout)         :: fileObj
   character(len=*),           intent(in)         :: filename, fieldname
-  real,     dimension(:,:),   intent(in), target :: data1, data2 
+  real,     dimension(:,:),   intent(in), target :: data1, data2
   type(domain2d),   optional, intent(in), target :: domain
   real,             optional, intent(in)         :: data_default
-  logical,          optional, intent(in)         :: no_domain   
+  logical,          optional, intent(in)         :: no_domain
   integer,          optional, intent(in)         :: position, tile_count
   logical,          optional, intent(in)         :: mandatory
   character(len=*), optional, intent(in)         :: longname, units
+  logical,          optional, intent(in)         :: read_only
   integer                                        :: index_field
   integer                                        :: register_restart_field_r2d_2level
 
   if(.not.module_is_initialized) call mpp_error(FATAL, &
-      'fms_io(register_restart_field_r2d_2level): need to call fms_io_init')  
+      'fms_io(register_restart_field_r2d_2level): need to call fms_io_init')
   call setup_one_field(fileObj, filename, fieldname, (/size(data1,1), size(data1,2), 1, 2/), &
                        index_field, domain, mandatory, no_domain, .false., &
-                       position, tile_count, data_default, longname, units)
+                       position, tile_count, data_default, longname, units, read_only=read_only)
   fileObj%p2dr(1, index_field)%p => data1
   fileObj%p2dr(2, index_field)%p => data2
   fileObj%var(index_field)%ndim = 2
   register_restart_field_r2d_2level = index_field
-  
-  return    
+
+  return
 
 end function register_restart_field_r2d_2level
 
@@ -1393,30 +1839,31 @@ end function register_restart_field_r2d_2level
 !
 !-------------------------------------------------------------------------------
 function register_restart_field_r3d_2level(fileObj, filename, fieldname, data1, data2, domain, mandatory, &
-                             no_domain, position, tile_count, data_default, longname, units)
+                             no_domain, position, tile_count, data_default, longname, units, read_only)
   type(restart_file_type), intent(inout)         :: fileObj
   character(len=*),           intent(in)         :: filename, fieldname
-  real,     dimension(:,:,:), intent(in), target :: data1, data2 
+  real,     dimension(:,:,:), intent(in), target :: data1, data2
   type(domain2d),   optional, intent(in), target :: domain
   real,             optional, intent(in)         :: data_default
-  logical,          optional, intent(in)         :: no_domain   
+  logical,          optional, intent(in)         :: no_domain
   integer,          optional, intent(in)         :: position, tile_count
   logical,          optional, intent(in)         :: mandatory
   character(len=*), optional, intent(in)         :: longname, units
+  logical,          optional, intent(in)         :: read_only
   integer                                        :: index_field
   integer                                        :: register_restart_field_r3d_2level
 
   if(.not.module_is_initialized) call mpp_error(FATAL, &
-      'fms_io(register_restart_field_r3d_2level): need to call fms_io_init')  
+      'fms_io(register_restart_field_r3d_2level): need to call fms_io_init')
   call setup_one_field(fileObj, filename, fieldname, (/size(data1,1), size(data1,2), size(data1,3), 2/), &
                        index_field, domain, mandatory, no_domain, .false., &
-                       position, tile_count, data_default, longname, units)
+                       position, tile_count, data_default, longname, units, read_only=read_only)
   fileObj%p3dr(1, index_field)%p => data1
   fileObj%p3dr(2, index_field)%p => data2
   fileObj%var(index_field)%ndim = 3
   register_restart_field_r3d_2level = index_field
-  
-  return    
+
+  return
 
 end function register_restart_field_r3d_2level
 
@@ -1426,31 +1873,32 @@ end function register_restart_field_r3d_2level
 !
 !-------------------------------------------------------------------------------
 function register_restart_field_i0d_2level(fileObj, filename, fieldname, data1, data2, domain, mandatory, &
-                             no_domain, position, tile_count, data_default, longname, units)
+                             no_domain, position, tile_count, data_default, longname, units, read_only)
   type(restart_file_type), intent(inout)         :: fileObj
   character(len=*),           intent(in)         :: filename, fieldname
-  integer,                    intent(in), target :: data1, data2 
+  integer,                    intent(in), target :: data1, data2
   type(domain2d),   optional, intent(in), target :: domain
   real,             optional, intent(in)         :: data_default
   integer,          optional, intent(in)         :: position, tile_count
   logical,          optional, intent(in)         :: mandatory
   logical,          optional, intent(in)         :: no_domain
   character(len=*), optional, intent(in)         :: longname, units
+  logical,          optional, intent(in)         :: read_only
   integer                                        :: index_field
   integer                                        :: register_restart_field_i0d_2level
 
   if(.not.module_is_initialized) call mpp_error(FATAL, &
-      'fms_io(register_restart_field_i0d_2level): need to call fms_io_init')  
+      'fms_io(register_restart_field_i0d_2level): need to call fms_io_init')
   call setup_one_field(fileObj, filename, fieldname, (/1, 1, 1, 2/), index_field, domain, &
                        mandatory, no_domain=no_domain, scalar_or_1d=.true., position=position, tile_count=tile_count, &
-                       data_default=data_default, longname=longname, units=units)
+                       data_default=data_default, longname=longname, units=units, read_only=read_only)
   fileObj%p0di(1, index_field)%p => data1
   fileObj%p0di(2, index_field)%p => data2
   fileObj%var(index_field)%ndim = 0
   register_restart_field_i0d_2level = index_field
-  
-  return   
- 
+
+  return
+
 end function register_restart_field_i0d_2level
 
 !-------------------------------------------------------------------------------
@@ -1459,31 +1907,32 @@ end function register_restart_field_i0d_2level
 !
 !-------------------------------------------------------------------------------
 function register_restart_field_i1d_2level(fileObj, filename, fieldname, data1, data2, domain, mandatory, &
-                             no_domain, position, tile_count, data_default, longname, units)
+                             no_domain, position, tile_count, data_default, longname, units, read_only)
   type(restart_file_type), intent(inout)         :: fileObj
   character(len=*),           intent(in)         :: filename, fieldname
-  integer,  dimension(:),     intent(in), target :: data1, data2 
+  integer,  dimension(:),     intent(in), target :: data1, data2
   type(domain2d),   optional, intent(in), target :: domain
   real,             optional, intent(in)         :: data_default
   integer,          optional, intent(in)         :: position, tile_count
   logical,          optional, intent(in)         :: mandatory
   logical,          optional, intent(in)         :: no_domain
   character(len=*), optional, intent(in)         :: longname, units
+  logical,          optional, intent(in)         :: read_only
   integer                                        :: index_field
   integer                                        :: register_restart_field_i1d_2level
 
   if(.not.module_is_initialized) call mpp_error(FATAL, &
-      'fms_io(register_restart_field_i1d_2level): need to call fms_io_init')  
+      'fms_io(register_restart_field_i1d_2level): need to call fms_io_init')
   call setup_one_field(fileObj, filename, fieldname, (/size(data1,1), 1, 1, 2/), index_field, domain, &
                        mandatory, no_domain=no_domain, scalar_or_1d=.true., position=position, tile_count=tile_count, &
-                       data_default=data_default, longname=longname, units=units)
+                       data_default=data_default, longname=longname, units=units, read_only=read_only)
   fileObj%p1di(1, index_field)%p => data1
   fileObj%p1di(2, index_field)%p => data2
   fileObj%var(index_field)%ndim = 1
   register_restart_field_i1d_2level = index_field
-  
-  return  
-  
+
+  return
+
 end function register_restart_field_i1d_2level
 
 !-------------------------------------------------------------------------------
@@ -1492,30 +1941,31 @@ end function register_restart_field_i1d_2level
 !
 !-------------------------------------------------------------------------------
 function register_restart_field_i2d_2level(fileObj, filename, fieldname, data1, data2, domain, mandatory, &
-                             no_domain, position, tile_count, data_default, longname, units)
+                             no_domain, position, tile_count, data_default, longname, units, read_only)
   type(restart_file_type), intent(inout)         :: fileObj
   character(len=*),           intent(in)         :: filename, fieldname
-  integer,  dimension(:,:),   intent(in), target :: data1, data2 
+  integer,  dimension(:,:),   intent(in), target :: data1, data2
   type(domain2d),   optional, intent(in), target :: domain
   real,             optional, intent(in)         :: data_default
-  logical,          optional, intent(in)         :: no_domain   
+  logical,          optional, intent(in)         :: no_domain
   integer,          optional, intent(in)         :: position, tile_count
   logical,          optional, intent(in)         :: mandatory
   character(len=*), optional, intent(in)         :: longname, units
+  logical,          optional, intent(in)         :: read_only
   integer                                        :: index_field
   integer                                        :: register_restart_field_i2d_2level
 
   if(.not.module_is_initialized) call mpp_error(FATAL, &
-      'fms_io(register_restart_field_i2d_2level): need to call fms_io_init')  
+      'fms_io(register_restart_field_i2d_2level): need to call fms_io_init')
   call setup_one_field(fileObj, filename, fieldname, (/size(data1,1), size(data1,2), 1, 2/), &
                        index_field, domain, mandatory, no_domain, .false., &
-                       position, tile_count, data_default, longname, units)
+                       position, tile_count, data_default, longname, units, read_only=read_only)
   fileObj%p2di(1, index_field)%p => data1
   fileObj%p2di(2, index_field)%p => data2
   fileObj%var(index_field)%ndim = 2
   register_restart_field_i2d_2level = index_field
-  
-  return    
+
+  return
 
 end function register_restart_field_i2d_2level
 
@@ -1525,423 +1975,1422 @@ end function register_restart_field_i2d_2level
 !
 !-------------------------------------------------------------------------------
 function register_restart_field_i3d_2level(fileObj, filename, fieldname, data1, data2, domain, mandatory, &
-                             no_domain, position, tile_count, data_default, longname, units)
+                             no_domain, position, tile_count, data_default, longname, units, read_only)
   type(restart_file_type), intent(inout)         :: fileObj
   character(len=*),           intent(in)         :: filename, fieldname
-  integer,  dimension(:,:,:), intent(in), target :: data1, data2 
+  integer,  dimension(:,:,:), intent(in), target :: data1, data2
   type(domain2d),   optional, intent(in), target :: domain
   real,             optional, intent(in)         :: data_default
-  logical,          optional, intent(in)         :: no_domain   
+  logical,          optional, intent(in)         :: no_domain
   integer,          optional, intent(in)         :: position, tile_count
   logical,          optional, intent(in)         :: mandatory
   character(len=*), optional, intent(in)         :: longname, units
+  logical,          optional, intent(in)         :: read_only
   integer                                        :: index_field
   integer                                        :: register_restart_field_i3d_2level
 
   if(.not.module_is_initialized) call mpp_error(FATAL, &
-      'fms_io(register_restart_field_i3d_2level): need to call fms_io_init')  
+      'fms_io(register_restart_field_i3d_2level): need to call fms_io_init')
   call setup_one_field(fileObj, filename, fieldname, (/size(data1,1), size(data1,2), size(data1,3), 2/), &
                        index_field, domain, mandatory, no_domain, .false., &
-                       position, tile_count, data_default, longname, units)
+                       position, tile_count, data_default, longname, units, read_only=read_only)
   fileObj%p3di(1, index_field)%p => data1
   fileObj%p3di(2, index_field)%p => data2
   fileObj%var(index_field)%ndim = 3
   register_restart_field_i3d_2level = index_field
-  
-  return    
+
+  return
 
 end function register_restart_field_i3d_2level
 
+!-------------------------------------------------------------------------------
+!
+!   The routine will register a 2-D real for a generic region defined
+!   by the global_size variable.
+!
+!-------------------------------------------------------------------------------
+function register_restart_region_r2d (fileObj, filename, fieldname, data, indices, global_size, &
+                                      pelist, is_root_pe, longname, units, position, &
+                                      x_halo, y_halo, ishift, jshift, read_only)
+  type(restart_file_type), intent(inout)         :: fileObj
+  character(len=*),           intent(in)         :: filename, fieldname
+  real,       dimension(:,:), intent(in), target :: data
+  integer,      dimension(:), intent(in)         :: indices, global_size, pelist
+  logical,                    intent(in)         :: is_root_pe
+  character(len=*), optional, intent(in)         :: longname, units
+  integer,          optional, intent(in)         :: position, x_halo, y_halo, ishift, jshift
+  logical,          optional, intent(in)         :: read_only
+  integer :: index_field, l_position
+  integer :: register_restart_region_r2d
+
+  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(register_restart_region_r2d): need to call fms_io_init')
+  if ((is_root_pe) .and. (.not.ANY(mpp_pe().eq.pelist))) call mpp_error(FATAL, &
+                    'fms_io(register_restart_region_r2d) designated root_pe is not a member of pelist')
+  l_position = CENTER
+  if (present(position)) l_position = position
+  call setup_one_field(fileObj, filename, fieldname, (/size(data,1), size(data,2), 1, 1/), &
+                       index_field, no_domain=.true., position=l_position, longname=longname, units=units, &
+                       read_only=read_only)
+  fileObj%p2dr(fileObj%var(index_field)%siz(4), index_field)%p => data
+  fileObj%var(index_field)%ndim = 2
+  fileObj%var(index_field)%is = indices(1)
+  fileObj%var(index_field)%ie = indices(2)
+  fileObj%var(index_field)%js = indices(3)
+  fileObj%var(index_field)%je = indices(4)
+  fileObj%var(index_field)%gsiz(1) = global_size(1)
+  fileObj%var(index_field)%gsiz(2) = global_size(2)
+  fileObj%is_root_pe = is_root_pe
+  fileObj%var(index_field)%x_halo = 0
+  fileObj%var(index_field)%y_halo = 0
+  fileObj%var(index_field)%ishift = 0
+  fileObj%var(index_field)%jshift = 0
+  if (present(x_halo)) fileObj%var(index_field)%x_halo = x_halo
+  if (present(y_halo)) fileObj%var(index_field)%y_halo = y_halo
+  if (present(ishift)) fileObj%var(index_field)%ishift = ishift
+  if (present(jshift)) fileObj%var(index_field)%jshift = jshift
+  if (allocated(fileObj%var(index_field)%pelist)) deallocate(fileObj%var(index_field)%pelist)
+  if (allocated(fileObj%var(index_field)%pelist)) deallocate(fileObj%var(index_field)%pelist)
+  allocate(fileObj%var(index_field)%pelist(size(pelist)))
+  fileObj%var(index_field)%pelist = pelist
+  register_restart_region_r2d = index_field
+
+  return
+end function register_restart_region_r2d
+
+!-------------------------------------------------------------------------------
+!
+!   The routine will register a 3-D real for a generic region defined
+!   by the global_size variable.
+!
+!-------------------------------------------------------------------------------
+function register_restart_region_r3d (fileObj, filename, fieldname, data, indices, global_size, &
+                                      pelist, is_root_pe, longname, units, position, &
+                                      x_halo, y_halo, ishift, jshift, read_only)
+  type(restart_file_type), intent(inout)         :: fileObj
+  character(len=*),           intent(in)         :: filename, fieldname
+  real,     dimension(:,:,:), intent(in), target :: data
+  integer,      dimension(:), intent(in)         :: indices, global_size, pelist
+  logical,                    intent(in)         :: is_root_pe
+  character(len=*), optional, intent(in)         :: longname, units
+  logical,          optional, intent(in)         :: read_only
+  integer,          optional, intent(in)         :: position, x_halo, y_halo, ishift, jshift
+  integer :: index_field, l_position
+  integer :: register_restart_region_r3d
+
+  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(register_restart_region_r3d): need to call fms_io_init')
+  if ((is_root_pe) .and. (.not.ANY(mpp_pe().eq.pelist))) call mpp_error(FATAL, &
+                    'fms_io(register_restart_region_r3d) designated root_pe is not a member of pelist')
+  l_position = CENTER
+  if (present(position)) l_position = position
+  call setup_one_field(fileObj, filename, fieldname, (/size(data,1), size(data,2), size(data,3), 1/), &
+                       index_field, no_domain=.true., position=l_position, longname=longname, units=units, &
+                       read_only=read_only)
+  fileObj%p3dr(fileObj%var(index_field)%siz(4), index_field)%p => data
+  fileObj%var(index_field)%ndim = 3
+  fileObj%var(index_field)%is = indices(1)
+  fileObj%var(index_field)%ie = indices(2)
+  fileObj%var(index_field)%js = indices(3)
+  fileObj%var(index_field)%je = indices(4)
+  fileObj%var(index_field)%gsiz(1) = global_size(1)
+  fileObj%var(index_field)%gsiz(2) = global_size(2)
+  fileObj%var(index_field)%gsiz(3) = global_size(3)
+  fileObj%is_root_pe = is_root_pe
+  fileObj%var(index_field)%x_halo = 0
+  fileObj%var(index_field)%y_halo = 0
+  fileObj%var(index_field)%ishift = 0
+  fileObj%var(index_field)%jshift = 0
+  if (present(x_halo)) fileObj%var(index_field)%x_halo = x_halo
+  if (present(y_halo)) fileObj%var(index_field)%y_halo = y_halo
+  if (present(ishift)) fileObj%var(index_field)%ishift = ishift
+  if (present(jshift)) fileObj%var(index_field)%jshift = jshift
+  if (allocated(fileObj%var(index_field)%pelist)) deallocate(fileObj%var(index_field)%pelist)
+  allocate(fileObj%var(index_field)%pelist(size(pelist)))
+  fileObj%var(index_field)%pelist = pelist
+  register_restart_region_r3d = index_field
+
+  return
+end function register_restart_region_r3d
+
 !-------------------------------------------------------------------------------
 !
 !  saves all registered variables to restart files. Those variables are set 
 !  through register_restart_field
 !
 !-------------------------------------------------------------------------------
-
-subroutine save_restart(fileObj, time_stamp, directory )
+subroutine save_restart(fileObj, time_stamp, directory, append, time_level)
   type(restart_file_type), intent(inout) :: fileObj
   character(len=*), intent(in), optional :: directory
   character(len=*), intent(in), optional :: time_stamp
-  ! Arguments: 
+  ! Arguments:
   !  (in)      directory  - The directory where the restart file goes.
   !  (in)      time_stamp - character format of the time of this restart file.
+  logical, intent(in), optional :: append
+  real,    intent(in), optional :: time_level
   character(len=256) :: dir
-  character(len=256) :: restartpath          ! The restart file path (dir/file).
-  character(len=258) :: restartpath_var      ! The restart file path for the current variable
   character(len=80)  :: restartname          ! The restart file name (no dir).
-  character(len=8)   :: suffix               ! A suffix (like _2) that is appended to the name of files after the first.
-  integer            :: var_sz, size_in_file ! The size in bytes of each variable and of the variables already in a file.
-  integer            :: start_var, next_var  ! The starting variables of the current and next files.
-  integer            :: unit                 ! The mpp unit of the open file.
-  real, dimension(max_axis_size)      :: axisdata
-  integer,        dimension(max_axes) :: id_x_axes, siz_x_axes
-  integer,        dimension(max_axes) :: id_y_axes, siz_y_axes
-  integer,        dimension(max_axes) :: id_z_axes, siz_z_axes
-  integer,        dimension(max_axes) :: x_axes_indx, y_axes_indx, z_axes_indx
-  type(axistype), dimension(max_axes) :: x_axes, y_axes, z_axes
-  type(axistype)                      :: t_axes            
-  integer                             :: num_var_axes
-  type(axistype), dimension(4)        :: var_axes
-  type(var_type), pointer, save       :: cur_var=>NULL()
-  integer                             :: num_x_axes, num_y_axes, num_z_axes
-  integer                             :: naxes_x, naxes_y, naxes_z
-  integer                             :: nfiles, i, j, k, l, siz, ind_dom
-  logical                             :: domain_present
-  real                                :: tlev  
-  character(len=10)                   :: axisname  
-  integer                             :: meta_size
-  type(domain2d)                      :: domain
+  character(len=336) :: restartpath          ! The restart file path (dir/file).
 
-  real, allocatable, dimension(:,:,:) :: r3d
-  real, allocatable, dimension(:,:)   :: r2d, global_r2d
-  real, allocatable, dimension(:)     :: r1d  
-  real                                :: r0d
-  integer(LONG_KIND), allocatable, dimension(:)    :: check_val
-  character(len=256)                  :: checksum_char
-integer :: isc, iec, jsc, jec
-integer :: isg, ieg, jsg, jeg
-integer :: ishift, jshift, iadd, jadd
+  ! This approach is taken rather than interface overloading in order to preserve
+  ! use of the register_restart_field infrastructure
 
   if (.not.associated(fileObj%var)) call mpp_error(FATAL, "fms_io(save_restart): " // &
       "restart_file_type data must be initialized by calling register_restart_field before using it")
 
-  do i=1,max_axis_size
-     axisdata(i) = i
-  enddo
-
   dir = "RESTART"
   if(present(directory)) dir = directory
 
   restartname = fileObj%name
-  nfiles = 0
   if(time_stamp_restart) then
      if (PRESENT(time_stamp)) then
+        if(len_trim(restartname)+len_trim(time_stamp) > 79) call mpp_error(FATAL, "fms_io(save_restart): " // &
+          "Length of restart file name + time_stamp is greater than allowed character length of 79")
         restartname = trim(time_stamp)//"."//trim(restartname)
      endif
   end if
   if(len_trim(dir) > 0) then
+     if(len_trim(dir)+len_trim(restartname) > 335) call mpp_error(FATAL, "fms_io(save_restart): " // &
+       "Length of full restart path + file name is greater than allowed character length of 355")
      restartpath = trim(dir)//"/"// trim(restartname)
   else
      restartpath = trim(restartname)
   end if
-  !--- check if any field in this file present domain.
-  domain_present = .false.
-  do j = 1, fileObj%nvar
-     if (fileObj%var(j)%domain_present) then
-        domain_present = .true.
-        ind_dom = j
-        exit
-     end if
+
+  if(fileObj%is_compressed .AND. ALLOCATED(fileObj%axes)) then
+     ! fileObj%axes must also be allocated if the file contains compressed axes
+     ! But will this always be true in the future?
+     call save_compressed_restart(fileObj,restartpath,append,time_level)
+  elseif(fileObj%unlimited_axis .AND. ALLOCATED(fileObj%axes)) then
+     call save_unlimited_axis_restart(fileObj,restartpath)
+  else
+     call save_default_restart(fileObj,restartpath)
+  endif
+
+  if(print_chksum) call write_chksum(fileObj, MPP_OVERWR)
+end subroutine save_restart
+
+!---- return true if all fields in fileObj is read only
+function all_field_read_only(fileObj)
+  type(restart_file_type), intent(in) :: fileObj
+  logical                             :: all_field_read_only
+  integer :: j
+
+  all_field_read_only = .TRUE.
+  do j = 1, fileObj%nvar
+     if( .not. fileObj%var(j)%read_only) then
+        all_field_read_only = .FALSE.
+        exit
+     endif
+  enddo
+
+  return
+
+end function all_field_read_only
+
+!-------------------------------------------------------------------------------
+!
+!  saves all registered variables to restart files. Those variables are set
+!  through register_restart_field
+!
+!-------------------------------------------------------------------------------
+
+subroutine save_compressed_restart(fileObj,restartpath,append,time_level)
+  type(restart_file_type), intent(inout),target :: fileObj
+  character(len=336)                     :: restartpath ! The restart file path (dir/file).
+
+  ! Optional arguments:
+
+  ! If neither append or time_level is present:
+  !   routine writes both meta data and field data.
+
+  ! If append is present and append=.true.:
+  !   Only field data is written.
+  !   The field data is appended to a new time level.
+  !   time_level must also be present and it must be >= 0.0
+  !   The value of time_level is written as a new value of the time axis data.
+
+  ! If time_level is present and time_level < 0.0:
+  !   A new file is opened and only the meta data is written.
+
+  ! If append is present and append=.false.:
+  !   Behaves the same was as if it were not present. That is, meta data is
+  !   written and whether or not field data is written is determined time_level.
+
+  logical, intent(in), optional :: append
+  real,    intent(in), optional :: time_level
+
+  integer            :: unit                 ! The mpp unit of the open file.
+  type(axistype)                      :: x_axis, y_axis, z_axis
+  type(axistype)                      :: t_axis, c_axis, h_axis  ! time & sparse compressed vector axes
+  type(axistype)                      :: comp_axis
+  logical                             :: naxis_z=.false.
+  type(axistype), dimension(4)        :: var_axes
+  type(var_type), pointer, save       :: cur_var=>NULL()
+  integer                             :: i, j, k, l, num_var_axes, cpack, idx, mpp_action
+  real                                :: tlev
+  real, allocatable, dimension(:,:)   :: r2d
+  real, allocatable, dimension(:)     :: r1d
+  real                                :: r0d
+  integer(LONG_KIND), allocatable, dimension(:)    :: check_val
+  character(len=256)                  :: checksum_char
+  logical                             :: domain_present, write_meta_data, write_field_data
+  logical                             :: c_axis_defined, h_axis_defined
+  type(domain2d), pointer :: domain =>NULL()
+  type(ax_type),  pointer :: axis   =>NULL()
+
+  !-- no need to proceed if all the variables are read only.
+  if( all_field_read_only(fileObj) ) return
+
+  if (.not.ALLOCATED(fileObj%axes(CIDX)%idx) .and. .not.ALLOCATED(fileObj%axes(HIDX)%idx) ) then
+     call mpp_error(FATAL, "fms_io(save_compressed_restart): A compressed axis has "// &
+          "not been defined for file "//trim(fileObj%name))
+  else if (ALLOCATED(fileObj%axes(CIDX)%idx)) then
+     domain =>fileObj%axes(CIDX)%domain
+  else
+     domain =>fileObj%axes(HIDX)%domain
+  endif
+
+  if(present(append)) then
+    if(append .and. .not.present(time_level)) then
+      call mpp_error(FATAL, 'fms_io(save_compressed_restart): time_level must be present when append=.true.'// &
+                    ' for file '//trim(fileObj%name))
+    endif
+  endif
+
+  mpp_action = MPP_OVERWR
+  write_meta_data  = .true.
+  if(present(append)) then
+    if(append) then
+      mpp_action = MPP_APPEND
+      write_meta_data = .false. ! Assuming meta data is already written when routine is called to append to field data.
+      if(time_level < 0.0) then
+        call mpp_error(FATAL, 'fms_io(save_compressed_restart): time_level cannot be negative when append is .true.'// &
+                      ' for file '//trim(fileObj%name))
+      endif
+    endif
+  endif
+
+  write_field_data = .true.
+  if(present(time_level)) then
+    write_field_data = time_level >= 0.0 ! Using negative value of time_level as a flag that there is no valid field data to write.
+  endif
+
+  call mpp_open(unit,trim(restartpath),action=mpp_action,form=form, &
+          is_root_pe=fileObj%is_root_pe, domain=domain)
+
+  if(write_meta_data) then
+    ! User has defined axes and these are assumed to be unique
+    ! Unfortunately it has proven difficult to write a generalized form because
+    ! of the variations possible across all of the axes
+    ! Currently support only 1 user defined axis of each type
+    ! In fact, this config is specifically designed to support the land model
+    ! sparse, compressed tile data
+    axis => fileobj%axes(XIDX)
+    if(.not. ASSOCIATED(axis)) call mpp_error(FATAL, "fms_io(save_compressed_restart): "// &
+                " The X axis has not been defined for "// &
+                " file "//trim(fileObj%name) )
+    call mpp_write_meta(unit,x_axis,axis%name,axis%units,axis%longname,data=axis%data,cartesian='X')
+
+    axis => fileobj%axes(YIDX)
+    if(.not. ASSOCIATED(axis)) call mpp_error(FATAL, "fms_io(save_compressed_restart): "// &
+                " The Y axis has not been defined for "// &
+                " file "//trim(fileObj%name) )
+    call mpp_write_meta(unit,y_axis,axis%name,axis%units,axis%longname,data=axis%data,cartesian='Y')
+
+    axis => fileobj%axes(ZIDX)
+    naxis_z = .false.
+    if(ASSOCIATED(axis%data))then
+       call mpp_write_meta(unit,z_axis,axis%name,axis%units,axis%longname, &
+            data=axis%data,cartesian='Z')
+       naxis_z = .true.
+    endif
+
+    ! The compressed axis
+    axis => fileObj%axes(CIDX)
+    if(ALLOCATED(axis%idx)) then
+       call mpp_def_dim(unit,trim(axis%dimlen_name),axis%dimlen,trim(axis%dimlen_lname), (/(i,i=1,axis%dimlen)/))
+       call mpp_write_meta(unit,c_axis,axis%name,axis%units,axis%longname, &
+                           data=axis%idx,compressed=axis%compressed,min=axis%imin)
+       c_axis_defined = .TRUE.
+    else
+       c_axis_defined = .FALSE.
+    endif
+
+    axis => fileObj%axes(HIDX)
+    if (ALLOCATED(axis%idx)) then
+       call mpp_def_dim(unit,trim(axis%dimlen_name),axis%dimlen,trim(axis%dimlen_lname), (/(i,i=1,axis%dimlen)/))
+       call mpp_write_meta(unit,h_axis,axis%name,axis%units,axis%longname, &
+                         data=axis%idx,compressed=axis%compressed,min=axis%imin)
+       h_axis_defined = .TRUE.
+    else
+       h_axis_defined = .FALSE.
+    endif
+
+    ! write out time axis
+    axis => fileobj%axes(TIDX)
+    if(ASSOCIATED(axis%data))then
+       call mpp_write_meta(unit,t_axis, axis%name, units=axis%units, longname=axis%longname, cartesian='T', calendar=axis%calendar)
+    else
+       call mpp_write_meta(unit,t_axis, 'Time','time level','Time',cartesian='T')
+    endif
+
+    ! write metadata for fields
+    do j = 1,fileObj%nvar
+       cur_var => fileObj%var(j)
+       if(cur_var%read_only) cycle
+       if(cur_var%siz(4) > 1 .AND. cur_var%siz(4) .NE. fileObj%max_ntime ) call mpp_error(FATAL, &
+        "fms_io(save_restart): "//trim(cur_var%name)//" in file "//trim(fileObj%name)// &
+        " has more than one time level, but number of time level is not equal to max_ntime")
+
+       select case (trim(cur_var%compressed_axis))
+       case ('C')
+          comp_axis = c_axis
+       case ('H')
+          comp_axis = h_axis
+       case default
+          if (ALLOCATED(fileObj%axes(CIDX)%idx)) then
+             comp_axis = c_axis
+          else
+             comp_axis = h_axis
+          endif
+       end select
+
+       if(cur_var%ndim == 0) then
+          num_var_axes = 1
+          var_axes(1) = t_axis
+       elseif(cur_var%ndim == 1) then
+          num_var_axes = 1
+          var_axes(1) = comp_axis
+          if(cur_var%siz(4) == fileObj%max_ntime) then
+             num_var_axes = 2
+             var_axes(2) = t_axis
+          endif
+       elseif(cur_var%ndim == 2) then
+          num_var_axes = 2
+          var_axes(1) = comp_axis
+          var_axes(2) = z_axis
+          if(cur_var%siz(4) == fileObj%max_ntime) then
+             num_var_axes = 3
+             var_axes(3) = t_axis
+          endif
+       else
+        call mpp_error(FATAL, "fms_io(save_compressed_restart): "//trim(cur_var%name)//" in file "// &
+           trim(fileObj%name)//" has more than two dimension (not including time level)")        
+       endif
+
+       cpack = pack_size  ! Default size of real
+        allocate(check_val(max(1,cur_var%siz(4))))
+        do k = 1, cur_var%siz(4)
+           if ( Associated(fileObj%p0dr(k,j)%p) ) then
+              check_val(k) = mpp_chksum(fileObj%p0dr(k,j)%p, (/mpp_pe()/), mask_val=cur_var%default_data)
+           else if ( Associated(fileObj%p1dr(k,j)%p) ) then
+              check_val(k) = mpp_chksum(fileObj%p1dr(k,j)%p(:), mask_val=cur_var%default_data)
+           else if ( Associated(fileObj%p2dr(k,j)%p) ) then
+              check_val(k) = mpp_chksum(fileObj%p2dr(k,j)%p(:,:), mask_val=cur_var%default_data)
+           else if ( Associated(fileObj%p3dr(k,j)%p) ) then
+              call mpp_error(FATAL, "fms_io(save_compressed_restart): real 3D restart fields are not currently supported"// &
+                   trim(cur_var%name)//" of file "//trim(fileObj%name) )
+           else if ( Associated(fileObj%p0di(k,j)%p) ) then
+              check_val(k) = fileObj%p0di(k,j)%p
+              cpack = 0  ! Write data as integer*4
+           else if ( Associated(fileObj%p1di(k,j)%p) ) then
+              check_val(k) = mpp_chksum(fileObj%p1di(k,j)%p(:), mask_val=cur_var%default_data)
+              cpack = 0  ! Write data as integer*4
+           else if ( Associated(fileObj%p2di(k,j)%p) ) then
+              check_val(k) = mpp_chksum(fileObj%p2di(k,j)%p(:,:), mask_val=cur_var%default_data)
+              cpack = 0  ! Write data as integer*4
+           else if ( Associated(fileObj%p3di(k,j)%p) ) then
+              call mpp_error(FATAL, "fms_io(save_compressed_restart): integer 3D restart fields are not currently supported"// &
+                   trim(cur_var%name)//" of file "//trim(fileObj%name) )
+           else
+              call mpp_error(FATAL, "fms_io(save_restart): There is no pointer associated with the data of field "// &
+                   trim(cur_var%name)//" of file "//trim(fileObj%name) )
+           end if
+        enddo
+! The chksum could not reproduce when running on different processor count. So commenting out now.
+! Also the chksum of compressed data is not read.
+       if(write_field_data) then ! Write checksums only if valid field data exists
+          call mpp_write_meta(unit,cur_var%field, var_axes(1:num_var_axes), cur_var%name, &
+                cur_var%units,cur_var%longname,pack=cpack,checksum=check_val,fill=cur_var%default_data)
+       else
+          call mpp_write_meta(unit,cur_var%field, var_axes(1:num_var_axes), cur_var%name, &
+                 cur_var%units,cur_var%longname,pack=cpack,fill=cur_var%default_data)
+       endif
+        deallocate(check_val)
+    enddo
+
+    ! write values for ndim of spatial and compressed axes
+    call mpp_write(unit,x_axis)
+    call mpp_write(unit,y_axis)
+    if (c_axis_defined) call mpp_write(unit,c_axis)
+    if (h_axis_defined) call mpp_write(unit,h_axis)
+    if(naxis_z) call mpp_write(unit,z_axis)
+
+  endif ! End of section to write meta data. Write meta data only if not appending.
+
+  if(write_field_data) then
+    ! write data of each field
+    do k = 1, fileObj%max_ntime
+       if(present(time_level)) then
+          tlev = time_level
+       else
+          tlev = k
+       endif
+       do j=1,fileObj%nvar
+          cur_var => fileObj%var(j)
+          if(cur_var%read_only) cycle
+
+          select case (trim(cur_var%compressed_axis))
+          case ('C')
+             idx = CIDX
+          case ('H')
+             idx = HIDX
+          case default
+             if (ALLOCATED(fileObj%axes(CIDX)%idx)) then
+                idx = CIDX
+             else
+                idx = HIDX
+             endif
+          end select
+
+          ! If some fields only have one time level, we do not need to write the second level, just keep
+          ! the data missing.
+          if(k <= cur_var%siz(4)) then
+             if ( Associated(fileObj%p0dr(k,j)%p) ) then
+                call mpp_write(unit, cur_var%field, fileObj%p0dr(k,j)%p, tlev)
+             elseif ( Associated(fileObj%p1dr(k,j)%p) ) then
+                call mpp_write_compressed(unit, cur_var%field, domain, fileObj%p1dr(k,j)%p, &
+                     fileObj%axes(idx)%nelems(:), tstamp=tlev, default_data=cur_var%default_data)
+             elseif ( Associated(fileObj%p2dr(k,j)%p) ) then
+                call mpp_write_compressed(unit, cur_var%field, domain, fileObj%p2dr(k,j)%p, &
+                     fileObj%axes(idx)%nelems(:), tstamp=tlev, default_data=cur_var%default_data)
+             elseif ( Associated(fileObj%p0di(k,j)%p) ) then
+                r0d =  fileObj%p0di(k,j)%p
+                call mpp_write(unit, cur_var%field, r0d, tlev)
+             elseif ( Associated(fileObj%p1di(k,j)%p) ) then
+                allocate(r1d(cur_var%siz(1)) )
+                r1d = fileObj%p1di(k,j)%p
+                call mpp_write_compressed(unit, cur_var%field, domain, r1d, &
+                     fileObj%axes(idx)%nelems(:), tstamp=tlev, default_data=cur_var%default_data)
+                deallocate(r1d)
+             elseif ( Associated(fileObj%p2di(k,j)%p) ) then
+                allocate(r2d(cur_var%siz(1), cur_var%siz(2)) )
+                r2d = fileObj%p2di(k,j)%p
+                call mpp_write_compressed(unit, cur_var%field, domain, r2d, &
+                     fileObj%axes(idx)%nelems(:), tstamp=tlev, default_data=cur_var%default_data)
+                deallocate(r2d)
+             else
+                call mpp_error(FATAL, "fms_io(save_restart): There is no pointer associated with the data of field "// &
+                       trim(cur_var%name)//" of file "//trim(fileObj%name) )
+             endif
+          endif
+       enddo ! end j loop
+    enddo ! end k loop
+    cur_var =>NULL()
+  endif
+  call mpp_close(unit)
+end subroutine save_compressed_restart
+
+!-------------------------------------------------------------------------------
+!
+!  saves all registered variables to restart files. Those variables are set
+!  through register_restart_field
+!
+!-------------------------------------------------------------------------------
+
+subroutine save_unlimited_axis_restart(fileObj,restartpath)
+  type(restart_file_type), intent(inout),target :: fileObj
+  character(len=336)                     :: restartpath ! The restart file path (dir/file).
+
+  integer            :: unit                 ! The mpp unit of the open file.
+  type(axistype)                      :: u_axis
+  type(axistype), dimension(4)        :: var_axes
+  type(var_type), pointer, save       :: cur_var=>NULL()
+  integer                             :: i, j, k, l, num_var_axes, cpack, idx
+  real, allocatable, dimension(:)     :: r1d
+  integer(LONG_KIND)                  :: check_val
+  character(len=256)                  :: checksum_char
+  type(domain2d), pointer :: domain =>NULL()
+  type(ax_type),  pointer :: axis   =>NULL()
+
+
+  if ( .NOT.fileObj%unlimited_axis ) then
+     call mpp_error(FATAL, "fms_io(save_unlimited_axis_restart): An unlimited axis has "// &
+          "not been defined for file "//trim(fileObj%name))
+  endif
+  domain =>fileObj%axes(UIDX)%domain
+
+  call mpp_open(unit,trim(restartpath),action=MPP_OVERWR,form=form, &
+                is_root_pe=fileObj%is_root_pe, domain=domain)
+
+  ! Set unlimited axis
+  axis => fileobj%axes(UIDX)
+  call mpp_write_meta(unit,u_axis,axis%name,data=sum(axis%nelems(:)),unlimited=.true.)
+  call write_meta_global(unit,fileObj)  ! Write any additional global metadata
+  call mpp_write(unit,u_axis)
+
+  ! write metadata for fields
+  do j = 1,fileObj%nvar
+     cur_var => fileObj%var(j)
+     if(cur_var%siz(4) > 1) call mpp_error(FATAL, &
+      "fms_io(save_restart): "//trim(cur_var%name)//" in file "//trim(fileObj%name)// &
+      " has more than one time level. Only single time level is currrently supported")
+
+     if(cur_var%ndim == 1) then
+        num_var_axes = 1
+        var_axes(1) = u_axis
+        else
+        call mpp_error(FATAL, 'fms_io(save_unlimited_axis_restart): Only vectors are currently supported')
+     endif
+
+     cpack = pack_size  ! Default size of real
+     if ( Associated(fileObj%p1dr(1,j)%p) ) then
+        check_val = mpp_chksum(fileObj%p1dr(1,j)%p(:))
+     else if ( Associated(fileObj%p1di(1,j)%p) ) then
+           ! Fill values are -HUGE(i4) which don't behave as desired for checksum algorithm
+        check_val = mpp_chksum(INT(fileObj%p1di(1,j)%p(:),8))
+           cpack = 0  ! Write data as integer*4
+        else
+        call mpp_error(FATAL, "fms_io(save_unlimited_axis_restart): There is no pointer associated with the record data of field "// &
+                trim(cur_var%name)//" of file "//trim(fileObj%name) )
+        end if
+     call mpp_write_meta(unit,cur_var%field, var_axes(1:num_var_axes), cur_var%name, &
+              cur_var%units,cur_var%longname,pack=cpack,checksum=(/check_val/))
+  enddo ! end j loop
+
+  ! write data of each field
+     do j=1,fileObj%nvar
+        cur_var => fileObj%var(j)
+     if ( Associated(fileObj%p1dr(1,j)%p) ) then
+        call mpp_write_unlimited_axis(unit,cur_var%field,domain,fileObj%p1dr(1,j)%p,fileObj%axes(UIDX)%nelems(:))
+     elseif ( Associated(fileObj%p1di(1,j)%p) ) then
+              allocate(r1d(cur_var%siz(1)) )
+        r1d = fileObj%p1di(1,j)%p
+        call mpp_write_unlimited_axis(unit,cur_var%field,domain,r1d,fileObj%axes(UIDX)%nelems(:))
+              deallocate(r1d)
+           else
+              call mpp_error(FATAL, "fms_io(save_restart): There is no pointer associated with the data of field "// &
+                     trim(cur_var%name)//" of file "//trim(fileObj%name) )
+           endif
+     enddo ! end j loop
+  call mpp_close(unit)
+  cur_var =>NULL()
+end subroutine save_unlimited_axis_restart
+
+!-------------------------------------------------------------------------------
+!
+!  saves all registered variables to restart files. Those variables are set
+!  through register_restart_field
+!
+!-------------------------------------------------------------------------------
+
+subroutine save_default_restart(fileObj,restartpath)
+  type(restart_file_type), intent(inout) :: fileObj
+  character(len=336)                     :: restartpath ! The restart file path (dir/file).
+
+  character(len=8)   :: suffix               ! A suffix (like _2) that is appended to the name of files after the first.
+  integer            :: var_sz, size_in_file ! The size in bytes of each variable and of the variables already in a file.
+  integer            :: unit                 ! The mpp unit of the open file.
+  real, dimension(max_axis_size)      :: axisdata
+  integer,        dimension(max_axes) :: id_x_axes, siz_x_axes
+  integer,        dimension(max_axes) :: id_y_axes, siz_y_axes
+  integer,        dimension(max_axes) :: id_z_axes, siz_z_axes
+  integer,        dimension(max_axes) :: id_a_axes, siz_a_axes
+  integer,        dimension(max_axes) :: x_axes_indx, y_axes_indx, z_axes_indx, a_axes_indx
+  type(axistype), dimension(max_axes) :: x_axes, y_axes, z_axes, a_axes
+  type(axistype)                      :: t_axes
+  integer                             :: num_var_axes
+  type(axistype), dimension(5)        :: var_axes
+  type(var_type), pointer, save       :: cur_var=>NULL()
+  integer                             :: num_x_axes, num_y_axes, num_z_axes, num_a_axes
+  integer                             :: naxes_x, naxes_y, naxes_z, naxes_a
+  integer                             :: i, j, k, l, siz, ind_dom
+  logical                             :: domain_present
+  real                                :: tlev
+  character(len=10)                   :: axisname
+  integer                             :: meta_size
+  type(domain2d)                      :: domain
+
+  real, allocatable, dimension(:,:,:) :: r3d
+  real, allocatable, dimension(:,:)   :: r2d
+  real, allocatable, dimension(:)     :: r1d
+  real                                :: r0d
+  integer(LONG_KIND), allocatable, dimension(:)    :: check_val
+  character(len=256)                  :: checksum_char
+  integer :: isc, iec, jsc, jec
+  integer :: isg, ieg, jsg, jeg
+  integer :: ishift, jshift, iadd, jadd
+  logical :: write_on_this_pe
+  type(domain2d), pointer :: io_domain =>NULL()
+
+  if (.not.associated(fileObj%var)) call mpp_error(FATAL, "fms_io(save_restart): " // &
+      "restart_file_type data must be initialized by calling register_restart_field before using it")
+
+  !-- no need to proceed if all the variables are read only.
+  if( all_field_read_only(fileObj) ) return
+
+  do i=1,max_axis_size
+     axisdata(i) = i
+  enddo
+
+  !--- check if any field in this file present domain.
+  domain_present = .false.
+  do j = 1, fileObj%nvar
+     if (fileObj%var(j)%domain_present) then
+        domain_present = .true.
+        ind_dom = j
+        exit
+     end if
   end do
   num_x_axes = unique_axes(fileObj, 1, id_x_axes, siz_x_axes, domain_x)
   num_y_axes = unique_axes(fileObj, 2, id_y_axes, siz_y_axes, domain_y)
   num_z_axes = unique_axes(fileObj, 3, id_z_axes, siz_z_axes          )
-  next_var = 1
-  meta_size = 0
+  num_a_axes = unique_axes(fileObj, 4, id_a_axes, siz_a_axes          )
+
+  write_on_this_pe = .false.
+  if(domain_present) then
+     io_domain => mpp_get_io_domain(array_domain(fileObj%var(ind_dom)%domain_idx))
+     if(associated(io_domain)) then
+       if(mpp_domain_is_tile_root_pe(io_domain)) write_on_this_pe = .true.
+     endif
+  endif
+  !--- always write out from root pe
+  if( fileObj%is_root_pe ) write_on_this_pe = .true.
+  
+  if( domain_present ) then
+     call mpp_open(unit,trim(restartpath),action=MPP_OVERWR,form=form,&
+          is_root_pe=fileObj%is_root_pe, domain=array_domain(fileObj%var(ind_dom)%domain_idx) )
+  else  ! global data
+     call mpp_open(unit,trim(restartpath),action=MPP_OVERWR,form=form,threading=MPP_SINGLE,&
+          fileset=MPP_SINGLE, is_root_pe=fileObj%is_root_pe)
+  end if
+
+  naxes_x = 0
+  x_axes_indx = 0
+  y_axes_indx = 0
+  z_axes_indx = 0
+  a_axes_indx = 0
+
+  ! write_out x_axes
   do j = 1, num_x_axes
-     meta_size = meta_size + siz_x_axes(j)
+     ! make sure this axis is used by some variable
+     do l=1,fileObj%nvar
+        if(fileObj%var(l)%read_only) cycle
+        if( fileObj%var(l)%id_axes(1) == j ) exit
+     end do
+     if( l > fileObj%nvar ) cycle
+     naxes_x = naxes_x + 1
+     x_axes_indx(naxes_x) = j
+     if (naxes_x < 10) then
+        write(axisname,'(a,i1)') 'xaxis_',naxes_x
+     else
+        write(axisname,'(a,i2)') 'xaxis_',naxes_x
+     endif
+     if(id_x_axes(j) > 0) then
+        call mpp_write_meta(unit,x_axes(j),axisname,'none',axisname, &
+             data=axisdata(1:siz_x_axes(j)),domain=domain_x(id_x_axes(j)),cartesian='X')
+     else
+        call mpp_write_meta(unit,x_axes(j),axisname,'none',axisname, &
+             data=axisdata(1:siz_x_axes(j)),cartesian='X')
+     endif
   end do
+
+  ! write out y_axes
+  naxes_y = 0
   do j = 1, num_y_axes
-     meta_size = meta_size + siz_y_axes(j)
+     ! make sure this axis is used by some variable
+     do l=1,fileObj%nvar
+        if(fileObj%var(l)%read_only) cycle
+        if( fileObj%var(l)%id_axes(2) == j ) exit
+     end do
+     if( l > fileObj%nvar ) cycle
+     naxes_y = naxes_y + 1
+     y_axes_indx(naxes_y) = j
+     if (naxes_y < 10) then
+        write(axisname,'(a,i1)') 'yaxis_',naxes_y
+     else
+        write(axisname,'(a,i2)') 'yaxis_',naxes_y
+     endif
+     if(id_y_axes(j) > 0) then
+        call mpp_write_meta(unit,y_axes(j),axisname,'none',axisname, &
+             data=axisdata(1:siz_y_axes(j)),domain=domain_y(id_y_axes(j)),cartesian='Y')
+     else
+        call mpp_write_meta(unit,y_axes(j),axisname,'none',axisname, &
+             data=axisdata(1:siz_y_axes(j)),cartesian='Y')
+     endif
   end do
+
+  ! write out z_axes
+  naxes_z = 0
   do j = 1, num_z_axes
-     meta_size = meta_size + siz_z_axes(j)
+     ! make sure this axis is used by some variable
+     do l=1,fileObj%nvar
+        if(fileObj%var(l)%read_only) cycle
+        if( fileObj%var(l)%id_axes(3) == j ) exit
+     end do
+     if( l > fileObj%nvar ) cycle
+     naxes_z = naxes_z + 1
+     z_axes_indx(naxes_z) = j
+     if (naxes_z < 10) then
+        write(axisname,'(a,i1)') 'zaxis_',naxes_z
+     else
+        write(axisname,'(a,i2)') 'zaxis_',naxes_z
+     endif
+     call mpp_write_meta(unit,z_axes(j),axisname,'none',axisname, &
+          data=axisdata(1:siz_z_axes(j)),cartesian='Z')
+  end do
+
+  ! write out a_axes
+  naxes_a = 0
+  do j = 1, num_a_axes
+     ! make sure this axis is used by some variable
+     do l=1,fileObj%nvar
+        if(fileObj%var(l)%read_only) cycle
+        if( fileObj%var(l)%id_axes(4) == j ) exit
+     end do
+     if( l > fileObj%nvar ) cycle
+     naxes_a = naxes_a + 1
+     a_axes_indx(naxes_a) = j
+     if (naxes_a < 10) then
+        write(axisname,'(a,i1)') 'aaxis_',naxes_a
+     else
+        write(axisname,'(a,i2)') 'aaxis_',naxes_a
+     endif
+     call mpp_write_meta(unit,a_axes(j),axisname,'none',axisname, &
+          data=axisdata(1:siz_a_axes(j)),cartesian='N')
   end do
-  meta_size = 8*(meta_size*2+1000)
 
-  do while (next_var <= fileObj%nvar )
-     start_var = next_var
-     size_in_file = meta_size
-     restartpath_var = restartpath
-     do j=start_var,fileObj%nvar
-        cur_var => fileObj%var(j)
-        var_sz = 8*cur_var%csiz(1)*cur_var%csiz(2)*cur_var%csiz(3)
-        if ((j==start_var) .OR. (size_in_file < MAX_FILE_SIZE-var_sz)) then
-           size_in_file = size_in_file + var_sz
-        else 
-           exit
-        endif
-     enddo
-     next_var = j
-     ! For distribute write, normally will not over the limit. 
-     if( nfiles > 0 ) then
-        if(fset_w == MPP_MULTI .AND. domain_present) call mpp_error(FATAL, "fms_io_mod(save_restart): "// &
-             "For distribute write(fileset_write='multi'), the file size should not be very large and need to be split")
-        if (nfiles < 10) then
-           write(suffix,'("_",I1)') nfiles
-        else if(nfiles < 100) then
-           write(suffix,'("_",I2)') nfiles
-        else
-           call mpp_error(FATAL, "fms_io(save_restart): num_files should be less than 100")
-        endif
-        !--- remove .nc from restartpath and attach suffix.
-        siz = len_trim(restartpath)
-        if(restartpath(siz-2:siz) == ".nc") then
-           restartpath_var = restartpath(1:siz-3)//trim(suffix)
-        else      
-           restartpath_var = trim(restartpath) // trim(suffix)
-        end if
-     end if
-     if( domain_present ) then
-        call mpp_open(unit,trim(restartpath_var),action=MPP_OVERWR,form=form,threading=thread_w,&
-             fileset=fset_w, is_root_pe=fileObj%is_root_pe, domain=array_domain(fileObj%var(ind_dom)%domain_idx) )
-     else  ! global data
-        call mpp_open(unit,trim(restartpath_var),action=MPP_OVERWR,form=form,threading=MPP_SINGLE,&
-             fileset=MPP_SINGLE, is_root_pe=fileObj%is_root_pe)
-     end if
+  ! write out time axis
+  call mpp_write_meta(unit,t_axes,&
+       'Time','time level','Time',cartesian='T')
+  ! write metadata for fields
+  do j = 1,fileObj%nvar
+     cur_var => fileObj%var(j)
+     if(cur_var%read_only) cycle
+     if(cur_var%siz(4) > 1 .AND. cur_var%siz(4) .NE. fileObj%max_ntime ) call mpp_error(FATAL, &
+      "fms_io(save_restart): "//trim(cur_var%name)//" in file "//trim(fileObj%name)// &
+      " has more than one time level, but number of time level is not equal to max_ntime")
+
+     if(cur_var%ndim == 0) then
+        num_var_axes = 1
+        var_axes(1) = t_axes
+     else if(cur_var%ndim == 1) then
+        num_var_axes = 1
+        var_axes(1) = x_axes(cur_var%id_axes(1))
+        if(cur_var%siz(4) == fileObj%max_ntime) then
+           num_var_axes = 2
+           var_axes(2) = t_axes
+        end if
+     else if(cur_var%ndim == 2) then
+        num_var_axes = 2
+        var_axes(1) = x_axes(cur_var%id_axes(1))
+        var_axes(2) = y_axes(cur_var%id_axes(2))
+        if(cur_var%siz(4) == fileObj%max_ntime) then
+           num_var_axes = 3
+           var_axes(3) = t_axes
+        end if
+     else if(cur_var%ndim == 3) then
+        num_var_axes = 3
+        var_axes(1) = x_axes(cur_var%id_axes(1))
+        var_axes(2) = y_axes(cur_var%id_axes(2))
+        var_axes(3) = z_axes(cur_var%id_axes(3))
+        if(cur_var%siz(4) == fileObj%max_ntime) then
+           num_var_axes = 4
+           var_axes(4) = t_axes
+        end if
+     else if(cur_var%ndim == 4) then
+        num_var_axes = 4
+        var_axes(1) = x_axes(cur_var%id_axes(1))
+        var_axes(2) = y_axes(cur_var%id_axes(2))
+        var_axes(3) = z_axes(cur_var%id_axes(3))
+        var_axes(4) = a_axes(cur_var%id_axes(4))
+        if(cur_var%siz(4) == fileObj%max_ntime) then
+           num_var_axes = 5
+           var_axes(5) = t_axes
+        end if
+     end if
+
+     if ( cur_var%domain_idx > 0) then
+       call mpp_get_compute_domain(array_domain(cur_var%domain_idx), isc, iec, jsc, jec)
+       call mpp_get_global_domain(array_domain(cur_var%domain_idx), isg, ieg, jsg, jeg)
+       call mpp_get_domain_shift(array_domain(cur_var%domain_idx), ishift, jshift, cur_var%position)
+     else if (ASSOCIATED(Current_domain)) then
+       call mpp_get_compute_domain(Current_domain, isc, iec, jsc, jec)
+       call mpp_get_global_domain(Current_domain, isg, ieg, jsg, jeg)
+       call mpp_get_domain_shift(Current_domain, ishift, jshift, cur_var%position)
+     else
+       iec = cur_var%ie
+       isc = cur_var%is
+       ieg = cur_var%ie
+       jec = cur_var%je
+       jsc = cur_var%js
+       jeg = cur_var%je
+       ishift = 0
+       jshift = 0
+     endif
+!     call return_domain(domain)
+     iadd = iec-isc ! Size of the i-dimension on this processor (-1 as it is an increment)
+     jadd = jec-jsc ! Size of the j-dimension on this processor
+     if(iec == ieg) iadd = iadd + ishift
+     if(jec == jeg) jadd = jadd + jshift
+
+     allocate(check_val(max(1,cur_var%siz(4))))
+     do k = 1, cur_var%siz(4)
+        if ( Associated(fileObj%p0dr(k,j)%p) ) then
+           check_val(k) = mpp_chksum(fileObj%p0dr(k,j)%p, (/mpp_pe()/) )
+        else if ( Associated(fileObj%p1dr(k,j)%p) ) then
+           check_val(k) = mpp_chksum(fileObj%p1dr(k,j)%p, (/mpp_pe()/) )
+        else if ( Associated(fileObj%p2dr(k,j)%p) ) then
+           check_val(k) = mpp_chksum(fileObj%p2dr(k,j)%p(cur_var%is:cur_var%is+iadd, cur_var%js:cur_var%js+jadd) )
+        else if ( Associated(fileObj%p3dr(k,j)%p) ) then
+           check_val(k) = mpp_chksum(fileObj%p3dr(k,j)%p(cur_var%is:cur_var%is+iadd, cur_var%js:cur_var%js+jadd, :) )
+        else if ( Associated(fileObj%p4dr(k,j)%p) ) then
+           check_val(k) = mpp_chksum(fileObj%p4dr(k,j)%p(cur_var%is:cur_var%is+iadd, cur_var%js:cur_var%js+jadd, :, :) )
+        else if ( Associated(fileObj%p0di(k,j)%p) ) then
+           check_val(k) = fileObj%p0di(k,j)%p
+        else if ( Associated(fileObj%p1di(k,j)%p) ) then
+           check_val(k) = mpp_chksum(fileObj%p1di(k,j)%p, (/mpp_pe()/) )
+        else if ( Associated(fileObj%p2di(k,j)%p) ) then
+           check_val(k) = mpp_chksum(fileObj%p2di(k,j)%p(cur_var%is:cur_var%is+iadd, cur_var%js:cur_var%js+jadd) )
+        else if ( Associated(fileObj%p3di(k,j)%p) ) then
+           check_val(k) = mpp_chksum(fileObj%p3di(k,j)%p(cur_var%is:cur_var%is+iadd, cur_var%js:cur_var%js+jadd, :))
+        else
+           call mpp_error(FATAL, "fms_io(save_restart): There is no pointer associated with the data of  field "// &
+                trim(cur_var%name)//" of file "//trim(fileObj%name) )
+        end if
+     enddo
+     call mpp_write_meta(unit,cur_var%field, var_axes(1:num_var_axes), cur_var%name, &
+              cur_var%units,cur_var%longname,pack=pack_size,checksum=check_val)
+     deallocate(check_val)
+  enddo
+
+  ! write values for ndim of spatial axes
+  do j = 1, naxes_x
+     call mpp_write(unit,x_axes(x_axes_indx(j)))
+  enddo
+  do j = 1, naxes_y
+     call mpp_write(unit,y_axes(y_axes_indx(j)))
+  enddo
+  do j = 1, naxes_z
+     call mpp_write(unit,z_axes(z_axes_indx(j)))
+  enddo
+
+  do j = 1, naxes_a
+     call mpp_write(unit,a_axes(a_axes_indx(j)))
+  enddo
+
+  ! write data of each field
+  do k = 1, fileObj%max_ntime
+     do j=1,fileObj%nvar
+        cur_var => fileObj%var(j)
+        if(cur_var%read_only) cycle
+        tlev=k
+        ! If some fields only have one time level, we do not need to write the second level, just keep
+        ! the data missing.
+        if(k <= cur_var%siz(4)) then
+           if(cur_var%domain_present) then  ! one 2-D or 3-D case possible present domain
+              if( Associated(fileObj%p2dr(k,j)%p) ) then
+                 call mpp_write(unit, cur_var%field, array_domain(cur_var%domain_idx), fileObj%p2dr(k,j)%p, tlev, &
+                                default_data=cur_var%default_data)
+              else if( Associated(fileObj%p3dr(k,j)%p) ) then
+                 call mpp_write(unit, cur_var%field, array_domain(cur_var%domain_idx), fileObj%p3dr(k,j)%p, tlev, &
+                                default_data=cur_var%default_data)
+              else if( Associated(fileObj%p4dr(k,j)%p) ) then
+                 call mpp_write(unit, cur_var%field, array_domain(cur_var%domain_idx), fileObj%p4dr(k,j)%p, tlev, &
+                                default_data=cur_var%default_data)
+              else if( Associated(fileObj%p2di(k,j)%p) ) then
+                 allocate(r2d(cur_var%siz(1), cur_var%siz(2)) )
+                 r2d = fileObj%p2di(k,j)%p
+                 call mpp_write(unit, cur_var%field, array_domain(cur_var%domain_idx), r2d, tlev, &
+                                default_data=cur_var%default_data)
+                 deallocate(r2d)
+              else if( Associated(fileObj%p3di(k,j)%p) ) then
+                 allocate(r3d(cur_var%siz(1), cur_var%siz(2), cur_var%siz(3)) )
+                 r3d = fileObj%p3di(k,j)%p
+                 call mpp_write(unit, cur_var%field, array_domain(cur_var%domain_idx), r3d, tlev, &
+                                default_data=cur_var%default_data)
+                 deallocate(r3d)
+              else
+                 call mpp_error(FATAL, "fms_io(save_restart): domain is present, "// &
+                      "field "//trim(cur_var%name)//" of file "//trim(fileObj%name)// &
+                      ", but none of p2dr, p3dr, p2di and p3di is associated")
+              end if
+           else if (write_on_this_pe) then
+              if ( Associated(fileObj%p0dr(k,j)%p) ) then
+                 call mpp_write(unit, cur_var%field, fileObj%p0dr(k,j)%p, tlev)
+              else if ( Associated(fileObj%p1dr(k,j)%p) ) then
+                 call mpp_write(unit, cur_var%field, fileObj%p1dr(k,j)%p, tlev)
+              else if ( Associated(fileObj%p2dr(k,j)%p) ) then
+                 call mpp_write(unit, cur_var%field, fileObj%p2dr(k,j)%p, tlev)
+              else if ( Associated(fileObj%p3dr(k,j)%p) ) then
+                 call mpp_write(unit, cur_var%field, fileObj%p3dr(k,j)%p, tlev)
+              else if ( Associated(fileObj%p4dr(k,j)%p) ) then
+                 call mpp_write(unit, cur_var%field, fileObj%p4dr(k,j)%p, tlev)
+              else if ( Associated(fileObj%p0di(k,j)%p) ) then
+                 r0d =  fileObj%p0di(k,j)%p
+                 call mpp_write(unit, cur_var%field, r0d,                  tlev)
+              else if ( Associated(fileObj%p1di(k,j)%p) ) then
+                 allocate(r1d(cur_var%siz(1)) )
+                 r1d = fileObj%p1di(k,j)%p
+                 call mpp_write(unit, cur_var%field, r1d,                  tlev)
+                 deallocate(r1d)
+              else if ( Associated(fileObj%p2di(k,j)%p) ) then
+                 allocate(r2d(cur_var%siz(1), cur_var%siz(2)) )
+                 r2d = fileObj%p2di(k,j)%p
+                 call mpp_write(unit, cur_var%field, r2d,                  tlev)
+                 deallocate(r2d)
+              else if ( Associated(fileObj%p3di(k,j)%p) ) then
+                 allocate(r3d(cur_var%siz(1), cur_var%siz(2), cur_var%siz(3)) )
+                 r3d = fileObj%p3di(k,j)%p
+                 call mpp_write(unit, cur_var%field, r3d,                  tlev)
+                 deallocate(r3d)
+              else
+                 call mpp_error(FATAL, "fms_io(save_restart): There is no pointer associated with the data of  field "// &
+                      trim(cur_var%name)//" of file "//trim(fileObj%name) )
+              end if
+           end if
+        end if
+     enddo ! end j loop
+  enddo ! end k loop
+  call mpp_close(unit)
+  cur_var =>NULL()
+end subroutine save_default_restart
+!-------------------------------------------------------------------------------
+!
+!  saves all registered border/halo variables to restart files. Those variables 
+!  are set through register_restart_field (region option)
+!
+!-------------------------------------------------------------------------------
+subroutine save_restart_border (fileObj, time_stamp, directory)
+  type(restart_file_type), intent(inout) :: fileObj
+  character(len=*),        intent(in), optional :: directory
+  character(len=*),        intent(in), optional :: time_stamp
+
+  character(len=256) :: dir
+  character(len=256) :: restartpath          ! The restart file path (dir/file).
+  character(len=80)  :: restartname          ! The restart file name (no dir).
+!rab  integer            :: start_var, next_var  ! The starting variables of the current and next files.
+  integer            :: unit                 ! The mpp unit of the open file.
+  real, dimension(max_axis_size)      :: axisdata
+  integer,        dimension(max_axes) :: id_x_axes, siz_x_axes
+  integer,        dimension(max_axes) :: id_y_axes, siz_y_axes
+  integer,        dimension(max_axes) :: id_z_axes, siz_z_axes
+  integer,        dimension(max_axes) :: x_axes_indx, y_axes_indx, z_axes_indx
+  type(axistype), dimension(max_axes) :: x_axes, y_axes, z_axes
+  type(axistype)                      :: t_axes
+  integer                             :: num_var_axes
+  type(axistype), dimension(4)        :: var_axes
+  type(var_type), pointer, save       :: cur_var=>NULL()
+  integer                             :: num_x_axes, num_y_axes, num_z_axes
+  integer                             :: naxes_x, naxes_y, naxes_z
+  integer                             :: i, j, k, l
+  integer                             :: isc, iec, jsc, jec
+  integer                             :: is, ie, js, je
+  integer                             :: i_add, i1, i2
+  integer                             :: j_add, j1, j2
+  integer                             :: i_glob, j_glob, k_glob
+  real                                :: tlev
+  character(len=10)                   :: axisname
+
+  real, allocatable, dimension(:,:)   :: r2d
+  real, allocatable, dimension(:,:,:) :: r3d
+  integer(LONG_KIND), allocatable, dimension(:)    :: check_val
+
+  !-- no need to proceed if all the variables are read only.
+  if( all_field_read_only(fileObj) ) return
+
+  do i=1,max_axis_size
+   axisdata(i) = i
+  enddo
+
+  dir = "RESTART"
+  if(present(directory)) dir = directory
+
+  restartname = fileObj%name
+  if (time_stamp_restart) then
+    if (PRESENT(time_stamp)) then
+      restartname = trim(time_stamp)//"."//trim(restartname)
+    endif
+  end if
+  if (len_trim(dir) > 0) then
+    restartpath = trim(dir)//"/"// trim(restartname)
+  else
+    restartpath = trim(restartname)
+  end if
+
+  num_x_axes = unique_axes(fileObj, 1, id_x_axes, siz_x_axes)
+  num_y_axes = unique_axes(fileObj, 2, id_y_axes, siz_y_axes)
+  num_z_axes = unique_axes(fileObj, 3, id_z_axes, siz_z_axes)
+
+  call mpp_open(unit,trim(restartpath),action=MPP_OVERWR,form=MPP_NETCDF,threading=MPP_SINGLE,&
+                fileset=MPP_SINGLE, is_root_pe=fileObj%is_root_pe)
+
+! write out axes
+  naxes_x = 0
+  x_axes_indx = 0
+  y_axes_indx = 0
+  z_axes_indx = 0
+
+! write out x_axes metadata
+  do j = 1, num_x_axes
+    ! make sure this axis is used by some variable
+    do l=1, fileObj%nvar
+      if(fileObj%var(l)%read_only) cycle
+      if (fileObj%var(l)%id_axes(1) == j) exit
+    end do
+    if( l > fileObj%nvar ) cycle  
+    naxes_x = naxes_x + 1
+    x_axes_indx(naxes_x) = j
+    if (naxes_x < 10) then
+      write(axisname,'(a,i1)') 'xaxis_',naxes_x
+    else
+      write(axisname,'(a,i2)') 'xaxis_',naxes_x
+    endif
+    call mpp_write_meta(unit,x_axes(j),axisname,'none',axisname, &
+                        data=axisdata(1:siz_x_axes(j)),cartesian='X')
+  end do
+
+! write out y_axes metadata
+  naxes_y = 0
+  do j = 1, num_y_axes
+    ! make sure this axis is used by some variable
+    do l=1, fileObj%nvar
+      if(fileObj%var(l)%read_only) cycle
+      if (fileObj%var(l)%id_axes(2) == j) exit
+    end do
+    if( l > fileObj%nvar ) cycle
+    naxes_y = naxes_y + 1
+    y_axes_indx(naxes_y) = j
+    if (naxes_y < 10) then
+      write(axisname,'(a,i1)') 'yaxis_',naxes_y
+    else
+      write(axisname,'(a,i2)') 'yaxis_',naxes_y
+    endif
+    call mpp_write_meta(unit,y_axes(j),axisname,'none',axisname, &
+                        data=axisdata(1:siz_y_axes(j)),cartesian='Y')
+  end do
+
+! write out z_axes metadata
+  naxes_z = 0
+  do j = 1, num_z_axes
+    ! make sure this axis is used by some variable
+    do l=1, fileObj%nvar
+      if(fileObj%var(l)%read_only) cycle
+      if (fileObj%var(l)%id_axes(3) == j) exit
+    end do
+    if( l > fileObj%nvar ) cycle
+    naxes_z = naxes_z + 1
+    z_axes_indx(naxes_z) = j
+    if (naxes_z < 10) then
+      write(axisname,'(a,i1)') 'zaxis_',naxes_z
+    else
+      write(axisname,'(a,i2)') 'zaxis_',naxes_z
+    endif
+    call mpp_write_meta(unit,z_axes(j),axisname,'none',axisname, &
+                        data=axisdata(1:siz_z_axes(j)),cartesian='Z')
+  end do
+
+! write out time axis
+  call mpp_write_meta(unit,t_axes,'Time','time level', &
+                      'Time',cartesian='T')
+
+! write metadata for fields
+  do j = 1, fileObj%nvar
+    cur_var => fileObj%var(j)
+    if(cur_var%read_only) cycle
+    if ((cur_var%siz(4) > 1) .AND. (cur_var%siz(4).NE.fileObj%max_ntime)) call mpp_error(FATAL, &
+     "fms_io(save_restart_border): "//trim(cur_var%name)//" in file "//trim(fileObj%name)// &
+     " has more than one time level, but number of time level is not equal to max_ntime")
+
+    if (cur_var%ndim == 2) then
+      num_var_axes = 2
+      var_axes(1) = x_axes(cur_var%id_axes(1))
+      var_axes(2) = y_axes(cur_var%id_axes(2))
+      if(cur_var%siz(4) == fileObj%max_ntime) then
+        num_var_axes = 3
+        var_axes(3) = t_axes
+      end if
+    else if (cur_var%ndim == 3) then
+      num_var_axes = 3
+      var_axes(1) = x_axes(cur_var%id_axes(1))
+      var_axes(2) = y_axes(cur_var%id_axes(2))
+      var_axes(3) = z_axes(cur_var%id_axes(3))
+      if(cur_var%siz(4) == fileObj%max_ntime) then
+        num_var_axes = 4
+        var_axes(4) = t_axes
+      end if
+    else
+      call mpp_error(FATAL, "fms_io(save_restart_border): "//trim(cur_var%name)//" in file "// &
+         trim(fileObj%name)//" has more than three dimension (not including time level)")      
+    end if
+   
+! cycle the loop for pes not a member of the current pelist
+    if (.not.ANY(mpp_pe().eq.cur_var%pelist(:))) cycle
+
+! IN ORDER TO GET CHECKSUM INFO, PERFORM THE GATHER AS IF YOU WILL BE DOING THE WRITE
+! BUT INSTEAD CHECKSUM THE RESULTING TEMPORARY ARRAY
+    allocate(check_val(max(1,cur_var%siz(4))))
+    do k = 1, cur_var%siz(4)
+! cycle the loop for pes not a member of the current pelist
+      if (.not.ANY(mpp_pe().eq.cur_var%pelist(:))) cycle
+      isc = cur_var%is
+      iec = cur_var%ie
+      jsc = cur_var%js
+      jec = cur_var%je
+! set up indices for local array segment pointer (pointer is 1-based)
+      i1 = 1 + cur_var%x_halo
+      i2 = i1 + (iec-isc)
+      j1 = 1 + cur_var%y_halo
+      j2 = j1 + (jec-jsc)
+! set up index shifts for global array r*d (1-based, but potentially needs offsets: i_add, j_add)
+      i_add = cur_var%ishift
+      j_add = cur_var%jshift
+! If some fields only have one time level, we do not need to write the second level, just keep
+! the data missing.
+      if(k <= cur_var%siz(4)) then
+        if ( Associated(fileObj%p2dr(k,j)%p) ) then
+          i_glob = cur_var%gsiz(1)
+          j_glob = cur_var%gsiz(2)
+          if (fileObj%is_root_pe) allocate(r2d(i_glob, j_glob))
+          call mpp_gather(isc+i_add, iec+i_add, jsc+j_add, jec+j_add, cur_var%pelist, & 
+                          fileObj%p2dr(k,j)%p(i1:i2,j1:j2), & 
+                          r2d, fileObj%is_root_pe)
+          check_val(k) = mpp_chksum(r2d, (/mpp_pe()/))
+          if (allocated(r2d)) deallocate(r2d)
+        else if ( Associated(fileObj%p3dr(k,j)%p) ) then
+          i_glob = cur_var%gsiz(1)
+          j_glob = cur_var%gsiz(2)
+          k_glob = cur_var%gsiz(3)
+          if (fileObj%is_root_pe) allocate(r3d(i_glob, j_glob, k_glob))
+          call mpp_gather(isc+i_add, iec+i_add, jsc+j_add, jec+j_add, k_glob, cur_var%pelist, &
+                          fileObj%p3dr(k,j)%p(i1:i2,j1:j2,:), r3d, fileObj%is_root_pe)
+          check_val(k) = mpp_chksum(r3d, (/mpp_pe()/))
+          if (allocated(r3d)) deallocate(r3d)
+        else
+          call mpp_error(FATAL, "fms_io(save_restart_border): no pointer associated with data of field "// &
+               trim(cur_var%name)//" in file "//trim(fileObj%name) )
+        end if
+      end if
+    enddo ! end k loop
+    call mpp_write_meta(unit,cur_var%field, var_axes(1:num_var_axes), cur_var%name, &
+                        cur_var%units,cur_var%longname,pack=pack_size,checksum=check_val)
+    if (allocated(check_val)) deallocate(check_val)
+  enddo
+
+! write values for ndim of spatial axes
+  do j = 1, naxes_x
+     call mpp_write(unit,x_axes(x_axes_indx(j)))
+  enddo
+  do j = 1, naxes_y
+     call mpp_write(unit,y_axes(y_axes_indx(j)))
+  enddo
+  do j = 1, naxes_z
+     call mpp_write(unit,z_axes(z_axes_indx(j)))
+  enddo
+
+! write data of each field
+  do k = 1, fileObj%max_ntime
+    tlev=k
+    do j=1, fileObj%nvar
+      cur_var => fileObj%var(j)
+      if(cur_var%read_only) cycle
+! cycle the loop for pes not a member of the current pelist
+      if (.not.ANY(mpp_pe().eq.cur_var%pelist(:))) cycle
+      isc = cur_var%is
+      iec = cur_var%ie
+      jsc = cur_var%js
+      jec = cur_var%je
+! set up indices for local array segment pointer (pointer is 1-based)
+      i1 = 1 + cur_var%x_halo
+      i2 = i1 + (iec-isc)
+      j1 = 1 + cur_var%y_halo
+      j2 = j1 + (jec-jsc)
+! set up index shifts for global array r*d (1-based, but potentially needs offsets: i_add, j_add)
+      i_add = cur_var%ishift
+      j_add = cur_var%jshift
+! If some fields only have one time level, we do not need to write the second level, just keep
+! the data missing.
+      if(k <= cur_var%siz(4)) then
+        if (Associated(fileObj%p2dr(k,j)%p)) then
+          i_glob = cur_var%gsiz(1)
+          j_glob = cur_var%gsiz(2)
+          if (fileObj%is_root_pe) allocate(r2d(i_glob, j_glob))
+          call mpp_gather(isc+i_add, iec+i_add, jsc+j_add, jec+j_add, cur_var%pelist, & 
+                          fileObj%p2dr(k,j)%p(i1:i2,j1:j2),   r2d, fileObj%is_root_pe)
+          call mpp_write(unit, cur_var%field, r2d, tlev)
+          if (allocated(r2d)) deallocate(r2d)
+        else if (Associated(fileObj%p3dr(k,j)%p)) then
+          i_glob = cur_var%gsiz(1)
+          j_glob = cur_var%gsiz(2)
+          k_glob = cur_var%gsiz(3)
+          if (fileObj%is_root_pe) allocate(r3d(i_glob, j_glob, k_glob))
+          call mpp_gather(isc+i_add, iec+i_add, jsc+j_add, jec+j_add, k_glob, cur_var%pelist, &
+                          fileObj%p3dr(k,j)%p(i1:i2,j1:j2,:), r3d, fileObj%is_root_pe)
+          call mpp_write(unit, cur_var%field, r3d, tlev)
+          if (allocated(r3d)) deallocate(r3d)
+        else
+          call mpp_error(FATAL, "fms_io(save_restart_border): no pointer associated with data of field "// &
+               trim(cur_var%name)//" in file "//trim(fileObj%name) )
+        end if
+      end if
+    enddo ! end j loop
+  enddo ! end k loop
+  call mpp_close(unit)
+
+  cur_var =>NULL()
+
+  if(print_chksum) call write_chksum(fileObj, MPP_OVERWR)
+  return
+
+end subroutine save_restart_border
+
+
+!-------------------------------------------------------------------------------
+!
+!  restores all registered border/halo variables to restart files. Those 
+!  variables are set through register_restart_field (region option)
+!
+!-------------------------------------------------------------------------------
+subroutine restore_state_border(fileObj, directory)
+  type(restart_file_type), intent(inout)       :: fileObj
+  character(len=*),      intent(in), optional  :: directory
+! Arguments: 
+!  (in)      directory - The directory where the restart or save
+!                        files should be found. The default is 'INPUT'
+  character(len=128) :: dir    
+  character(len=256) :: restartpath ! The restart file path (dir/file). 
+  character(len=200) :: filepath    ! The path (dir/file) to the file being opened.
+  character(len=80)  :: varname     ! A variable's name.
+  character(len=256) :: mesg        ! Message to be constructed for checksum error.
+  type(var_type), pointer, save       :: cur_var=>NULL()
+  integer                             :: ndim, nvar, natt, ntime, tlev, siz
+  type(fieldtype), allocatable        :: fields(:)
+  logical                             :: fexist
+  integer                             :: j, n, l, k, unit
+  real, allocatable, dimension(:,:,:) :: r3d
+  real, allocatable, dimension(:,:)   :: r2d
+  integer                             :: isc, iec, jsc, jec 
+  logical                             :: check_exist
+  integer                             :: i1, i2, j1, j2
+  integer                             :: ishift, jshift, i_add, j_add
+  integer                             :: i_glob, j_glob, k_glob
+  integer(LONG_KIND), dimension(3)    :: checksum_file
+  integer(LONG_KIND)                  :: checksum_data
+  logical                             :: is_there_a_checksum
 
-     ! write_out x_axes
-     naxes_x = 0
-     x_axes_indx = 0
-     y_axes_indx = 0
-     z_axes_indx = 0
+  if (.not.associated(fileObj%var)) call mpp_error(FATAL, "fms_io(restore_state_border): " // &
+      "restart_file_type data must be initialized by calling register_restart_field before using it")
 
-     do j = 1, num_x_axes
-        ! make sure this axis is used by some variable 
-        do l=start_var,next_var-1
-           if( fileObj%var(l)%id_axes(1) == j ) exit
-        end do
-        if(l == next_var) cycle  
-        naxes_x = naxes_x + 1
-        x_axes_indx(naxes_x) = j
-        if (naxes_x < 10) then
-           write(axisname,'(a,i1)') 'xaxis_',naxes_x
-        else
-           write(axisname,'(a,i2)') 'xaxis_',naxes_x
-        endif
-        if(id_x_axes(j) > 0) then
-           call mpp_write_meta(unit,x_axes(j),axisname,'none',axisname, &
-                data=axisdata(1:siz_x_axes(j)),domain=domain_x(id_x_axes(j)),cartesian='X')
-        else
-           call mpp_write_meta(unit,x_axes(j),axisname,'none',axisname, &
-                data=axisdata(1:siz_x_axes(j)),cartesian='X')
-        endif
-     end do
+  dir = 'INPUT'
+  if(present(directory)) dir = directory
 
-     ! write out y_axes
-     naxes_y = 0
-     do j = 1, num_y_axes
-        ! make sure this axis is used by some variable 
-        do l=start_var,next_var-1
-           if( fileObj%var(l)%id_axes(2) == j ) exit
-        end do
-        if(l == next_var) cycle  
-        naxes_y = naxes_y + 1
-        y_axes_indx(naxes_y) = j
-        if (naxes_y < 10) then
-           write(axisname,'(a,i1)') 'yaxis_',naxes_y
-        else
-           write(axisname,'(a,i2)') 'yaxis_',naxes_y
-        endif
-        if(id_y_axes(j) > 0) then
-           call mpp_write_meta(unit,y_axes(j),axisname,'none',axisname, &
-                data=axisdata(1:siz_y_axes(j)),domain=domain_y(id_y_axes(j)),cartesian='Y')
-        else
-           call mpp_write_meta(unit,y_axes(j),axisname,'none',axisname, &
-                data=axisdata(1:siz_y_axes(j)),cartesian='Y')
-        endif
-     end do
+  if(len_trim(dir) > 0) then
+     restartpath = trim(dir)//"/"// trim(fileObj%name)
+  else
+     restartpath = trim(fileObj%name)
+  end if
 
-     ! write out z_axes
-     naxes_z = 0
-     do j = 1, num_z_axes
-        ! make sure this axis is used by some variable 
-        do l=start_var,next_var-1
-           if( fileObj%var(l)%id_axes(3) == j ) exit
-        end do
-        if(l == next_var) cycle  
-        naxes_z = naxes_z + 1
-        z_axes_indx(naxes_z) = j
-        if (naxes_z < 10) then
-           write(axisname,'(a,i1)') 'zaxis_',naxes_z
-        else
-           write(axisname,'(a,i2)') 'zaxis_',naxes_z
-        endif
-        call mpp_write_meta(unit,z_axes(j),axisname,'none',axisname, &
-             data=axisdata(1:siz_z_axes(j)),cartesian='Z')
-     end do
+!--- first open the restart files
+!--- NOTE: For distributed restart file, we are assuming there is only one file exist.
 
-     ! write out time axis  
-     call mpp_write_meta(unit,t_axes,&
-          'Time','time level','Time',cartesian='T')
-     ! write metadata for fields
-     do j = start_var,next_var-1
-        cur_var => fileObj%var(j)
-        if(cur_var%siz(4) > 1 .AND. cur_var%siz(4) .NE. fileObj%max_ntime ) call mpp_error(FATAL, &
-         "fms_io(save_restart): "//trim(cur_var%name)//" in file "//trim(fileObj%name)// &
-         " has more than one time level, but number of time level is not equal to max_ntime")
-
-        if(cur_var%ndim == 0) then
-           num_var_axes = 1
-           var_axes(1) = t_axes
-        else if(cur_var%ndim == 1) then
-           num_var_axes = 1
-           var_axes(1) = x_axes(cur_var%id_axes(1))
-           if(cur_var%siz(4) == fileObj%max_ntime) then
-              num_var_axes = 2
-              var_axes(2) = t_axes
-           end if
-        else if(cur_var%ndim == 2) then
-           num_var_axes = 2
-           var_axes(1) = x_axes(cur_var%id_axes(1))
-           var_axes(2) = y_axes(cur_var%id_axes(2))
-           if(cur_var%siz(4) == fileObj%max_ntime) then
-              num_var_axes = 3
-              var_axes(3) = t_axes
-           end if
-        else if(cur_var%ndim == 3) then
-           num_var_axes = 3
-           var_axes(1) = x_axes(cur_var%id_axes(1))
-           var_axes(2) = y_axes(cur_var%id_axes(2))
-           var_axes(3) = z_axes(cur_var%id_axes(3))
-           if(cur_var%siz(4) == fileObj%max_ntime) then
-              num_var_axes = 4
-              var_axes(4) = t_axes
-           end if
-        end if
-        
-        if ( cur_var%domain_idx > 0) then
-          call mpp_get_compute_domain(array_domain(cur_var%domain_idx), isc, iec, jsc, jec)
-          call mpp_get_global_domain(array_domain(cur_var%domain_idx), isg, ieg, jsg, jeg)
-          call mpp_get_domain_shift(array_domain(cur_var%domain_idx), ishift, jshift, cur_var%position)
-        else if (ASSOCIATED(Current_domain)) then
-          call mpp_get_compute_domain(Current_domain, isc, iec, jsc, jec)
-          call mpp_get_global_domain(Current_domain, isg, ieg, jsg, jeg)
-          call mpp_get_domain_shift(Current_domain, ishift, jshift, cur_var%position)
-       else
-          iec = cur_var%ie
-          isc = cur_var%is
-          ieg = cur_var%ie
-          jec = cur_var%je
-          jsc = cur_var%js
-          jeg = cur_var%je
-          ishift = 0
-          jshift = 0
-        endif
-!        call return_domain(domain)
-        iadd = iec-isc ! Size of the i-dimension on this processor (-1 as it is an increment)
-        jadd = jec-jsc ! Size of the j-dimension on this processor
-        if(iec == ieg) iadd = iadd + ishift
-        if(jec == jeg) jadd = jadd + jshift
+  inquire (file=trim(restartpath), exist=fexist)
+  if (.not.fexist) call mpp_error(FATAL, "fms_io(restore_state_border): unable to find any restart &
+                                 &files specified by "//trim(restartpath))
+  call mpp_open(unit,trim(restartpath),action=MPP_RDONLY,form=MPP_NETCDF,threading=MPP_SINGLE,&
+                fileset=MPP_SINGLE, is_root_pe=fileObj%is_root_pe)
 
+! Read each variable from the first file in which it is found.
+  call mpp_get_info(unit, ndim, nvar, natt, ntime)
 
-        allocate(check_val(max(1,cur_var%siz(4))))
-        do k = 1, cur_var%siz(4)
-           if ( Associated(fileObj%p0dr(k,j)%p) ) then
-              check_val(k) = mpp_chksum(fileObj%p0dr(k,j)%p, (/mpp_pe()/) )
-           else if ( Associated(fileObj%p1dr(k,j)%p) ) then
-              check_val(k) = mpp_chksum(fileObj%p1dr(k,j)%p, (/mpp_pe()/) )
-           else if ( Associated(fileObj%p2dr(k,j)%p) ) then
-              check_val(k) = mpp_chksum(fileObj%p2dr(k,j)%p(cur_var%is:cur_var%is+iadd, cur_var%js:cur_var%js+jadd) )
-           else if ( Associated(fileObj%p3dr(k,j)%p) ) then
-              check_val(k) = mpp_chksum(fileObj%p3dr(k,j)%p(cur_var%is:cur_var%is+iadd, cur_var%js:cur_var%js+jadd, :) )
-           else if ( Associated(fileObj%p0di(k,j)%p) ) then
-              check_val(k) = fileObj%p0di(k,j)%p
-           else if ( Associated(fileObj%p1di(k,j)%p) ) then
-              check_val(k) = mpp_chksum(fileObj%p1di(k,j)%p, (/mpp_pe()/) )
-           else if ( Associated(fileObj%p2di(k,j)%p) ) then
-              check_val(k) = mpp_chksum(fileObj%p2di(k,j)%p(cur_var%is:cur_var%is+iadd, cur_var%js:cur_var%js+jadd) )
-           else if ( Associated(fileObj%p3di(k,j)%p) ) then
-              check_val(k) = mpp_chksum(fileObj%p3di(k,j)%p(cur_var%is:cur_var%is+iadd, cur_var%js:cur_var%js+jadd, :))
-           else
-              call mpp_error(FATAL, "fms_io(save_restart): There is no pointer associated with the data of  field "// &
-                   trim(cur_var%name)//" of file "//trim(fileObj%name) )
-           end if
-        enddo
-        call mpp_write_meta(unit,cur_var%field, var_axes(1:num_var_axes), cur_var%name, &
-                 cur_var%units,cur_var%longname,pack=pack_size,checksum=check_val)
-        deallocate(check_val)
-     enddo
+  allocate(fields(nvar))
+  call mpp_get_fields(unit,fields(1:nvar))
 
-     ! write values for ndim of spatial axes
-     do j = 1, naxes_x
-        call mpp_write(unit,x_axes(x_axes_indx(j)))
-     enddo
-     do j = 1, naxes_y
-        call mpp_write(unit,y_axes(y_axes_indx(j)))
-     enddo
-     do j = 1, naxes_z
-        call mpp_write(unit,z_axes(z_axes_indx(j)))
-     enddo
+  do j=1,fileObj%nvar
+    cur_var => fileObj%var(j)
+! cycle the loop for pes not a member of the current pelist
+    if (.not.ANY(mpp_pe().eq.cur_var%pelist(:))) cycle
+    isc = cur_var%is
+    iec = cur_var%ie
+    jsc = cur_var%js
+    jec = cur_var%je
+! set up indices for local array segment pointer (pointer is 1-based)
+    i1 = 1 + cur_var%x_halo
+    i2 = i1 + (iec-isc)
+    j1 = 1 + cur_var%y_halo
+    j2 = j1 + (jec-jsc)
+! set up index shifts for global array r*d (1-based, but potentially needs offsets: i_add, j_add)
+    i_add = cur_var%ishift
+    j_add = cur_var%jshift
+    do l=1, nvar
+      call mpp_get_atts(fields(l),name=varname)
+      if (lowercase(trim(varname)) == lowercase(trim(cur_var%name))) then
+        cur_var%initialized = .true.
+        check_exist = mpp_attribute_exist(fields(l),"checksum")
+        checksum_file = 0
+        is_there_a_checksum = .false.
+        if ( check_exist  ) then
+          call mpp_get_atts(fields(l),checksum=checksum_file)
+          is_there_a_checksum = .true.
+        endif
+        if (.NOT. checksum_required) is_there_a_checksum = .false. ! Do not need to do data checksumming.
 
-     ! write data of each field
-     do k = 1, fileObj%max_ntime
-        do j=start_var,next_var-1
-           cur_var => fileObj%var(j)
-           tlev=k
-           ! If some fields only have one time level, we do not need to write the second level, just keep
-           ! the data missing.
-           if(k <= cur_var%siz(4)) then
-              if(cur_var%domain_present) then  ! one 2-D or 3-D case possible present domain
-                 if( Associated(fileObj%p2dr(k,j)%p) ) then
-                    call mpp_write(unit, cur_var%field, array_domain(cur_var%domain_idx), fileObj%p2dr(k,j)%p, tlev, &
-                                   default_data=cur_var%default_data)
-                 else if( Associated(fileObj%p3dr(k,j)%p) ) then
-                    call mpp_write(unit, cur_var%field, array_domain(cur_var%domain_idx), fileObj%p3dr(k,j)%p, tlev, &
-                                   default_data=cur_var%default_data)
-                 else if( Associated(fileObj%p2di(k,j)%p) ) then
-                    allocate(r2d(cur_var%siz(1), cur_var%siz(2)) )
-                    r2d = fileObj%p2di(k,j)%p
-                    call mpp_write(unit, cur_var%field, array_domain(cur_var%domain_idx), r2d, tlev, &
-                                   default_data=cur_var%default_data)
-                    deallocate(r2d)
-                 else if( Associated(fileObj%p3di(k,j)%p) ) then
-                    allocate(r3d(cur_var%siz(1), cur_var%siz(2), cur_var%siz(3)) )
-                    r3d = fileObj%p3di(k,j)%p
-                    call mpp_write(unit, cur_var%field, array_domain(cur_var%domain_idx), r3d, tlev, &
-                                   default_data=cur_var%default_data)
-                    deallocate(r3d)
-                 else
-                    call mpp_error(FATAL, "fms_io(save_restart): domain is present and thread_w  "// &
-                         "is MPP_MULTI, field "//trim(cur_var%name)//" of file "//trim(fileObj%name)// &
-                         ", but none of p2dr, p3dr, p2di and p3di is associated") 
-                 end if
-              else if (thread_w == MPP_MULTI .or. cur_var%write_on_this_pe .or. &
-                       (fileObj%is_root_pe.and.thread_w == MPP_SINGLE)) then     
-                 if ( Associated(fileObj%p0dr(k,j)%p) ) then
-                    call mpp_write(unit, cur_var%field, fileObj%p0dr(k,j)%p, tlev)
-                 else if ( Associated(fileObj%p1dr(k,j)%p) ) then
-                    call mpp_write(unit, cur_var%field, fileObj%p1dr(k,j)%p, tlev)
-                 else if ( Associated(fileObj%p2dr(k,j)%p) ) then
-                    call mpp_write(unit, cur_var%field, fileObj%p2dr(k,j)%p, tlev)
-                 else if ( Associated(fileObj%p3dr(k,j)%p) ) then
-                    call mpp_write(unit, cur_var%field, fileObj%p3dr(k,j)%p, tlev)
-                 else if ( Associated(fileObj%p0di(k,j)%p) ) then
-                    r0d =  fileObj%p0di(k,j)%p
-                    call mpp_write(unit, cur_var%field, r0d,                  tlev)
-                 else if ( Associated(fileObj%p1di(k,j)%p) ) then
-                    allocate(r1d(cur_var%siz(1)) )
-                    r1d = fileObj%p1di(k,j)%p
-                    call mpp_write(unit, cur_var%field, r1d,                  tlev)
-                    deallocate(r1d)
-                 else if ( Associated(fileObj%p2di(k,j)%p) ) then
-                    allocate(r2d(cur_var%siz(1), cur_var%siz(2)) )
-                    r2d = fileObj%p2di(k,j)%p
-                    call mpp_write(unit, cur_var%field, r2d,                  tlev)
-                    deallocate(global_r2d, r2d)
-                 else if ( Associated(fileObj%p3di(k,j)%p) ) then
-                    allocate(r3d(cur_var%siz(1), cur_var%siz(2), cur_var%siz(3)) )
-                    r3d = fileObj%p3di(k,j)%p
-                    call mpp_write(unit, cur_var%field, r3d,                  tlev)
-                    deallocate(r3d)
-                 else
-                    call mpp_error(FATAL, "fms_io(save_restart): There is no pointer associated with the data of  field "// &
-                         trim(cur_var%name)//" of file "//trim(fileObj%name) )
-                 end if
-              end if
-           end if
-        enddo ! end j loop
-     enddo ! end k loop
-     call mpp_close(unit)
-     nfiles = nfiles+1
+        do k = 1, cur_var%siz(4)
+          tlev = k
+! read the field and scatter it to the rest of the pelist
+          if (Associated(fileObj%p2dr(k,j)%p)) then
+            i_glob = cur_var%gsiz(1)
+            j_glob = cur_var%gsiz(2)
+            if (fileObj%is_root_pe) allocate(r2d(i_glob, j_glob))
+            call mpp_read(unit, fields(l), r2d, tlev)
+            call mpp_scatter(isc+i_add, iec+i_add, jsc+j_add, jec+j_add, cur_var%pelist, &
+                             fileObj%p2dr(k,j)%p(i1:i2,j1:j2), r2d, fileObj%is_root_pe)
+            if ((fileObj%is_root_pe) .and. (is_there_a_checksum)) checksum_data = mpp_chksum(r2d, (/mpp_pe()/) )
+            if (allocated(r2d)) deallocate(r2d)
+          else if (Associated(fileObj%p3dr(k,j)%p)) then
+            i_glob = cur_var%gsiz(1)
+            j_glob = cur_var%gsiz(2)
+            k_glob = cur_var%gsiz(3)
+            if (fileObj%is_root_pe) allocate(r3d(i_glob, j_glob, k_glob))
+            call mpp_read(unit, fields(l), r3d, tlev)
+            call mpp_scatter(isc+i_add, iec+i_add, jsc+j_add, jec+j_add, k_glob, cur_var%pelist, &
+                             fileObj%p3dr(k,j)%p(i1:i2,j1:j2,:), r3d, fileObj%is_root_pe)
+            if ((fileObj%is_root_pe) .and. (is_there_a_checksum)) checksum_data = mpp_chksum(r3d, (/mpp_pe()/) )
+            if (allocated(r3d)) deallocate(r3d)
+          else
+            call mpp_error(FATAL, "fms_io(retore_state_border): no pointer associated with data of field "// &
+                  trim(cur_var%name)//" in file "//trim(fileObj%name) )
+          end if
+          if ((fileObj%is_root_pe) .and. (is_there_a_checksum) .and. (checksum_file(k)/=checksum_data)) then
+            write (mesg,'(a,Z16,a,Z16,a)') "Checksum of input field "// uppercase(trim(varname))//" ", checksum_data,&
+                         " does not match value ", checksum_file(k), " stored in "//uppercase(trim(fileObj%name)//"." )
+            call mpp_error(FATAL, "fms_io(restore_state_border): "//trim(mesg) )
+          endif
+        end do
+        exit ! Start search for next restart variable.
+      endif
+    enddo
   enddo
 
+  deallocate(fields)
+
+  call close_file(unit)
+
   cur_var =>NULL()
 
-  if(print_chksum) call write_chksum(fileObj, MPP_OVERWR)
+  ! check whether all fields have been found
+  do j = 1, fileObj%nvar
+    if (.not.ANY(mpp_pe().eq.fileObj%var(j)%pelist(:))) cycle
+    if (.NOT. fileObj%var(j)%initialized) then
+      if (fileObj%var(j)%mandatory) then
+        call mpp_error(FATAL, "fms_io(restore_state_border): unable to find mandatory variable "// &
+                       trim(fileObj%var(j)%name)//" in restart file "//trim(fileObj%name) )
+      end if
+    end if
+  end do
 
-end subroutine save_restart
+  if(print_chksum) call write_chksum(fileObj, MPP_RDONLY )
+  return
+
+end subroutine restore_state_border
 
 !-------------------------------------------------------------------------------
 !    This subroutine will calculate chksum and print out chksum information.
@@ -1991,7 +3440,7 @@ subroutine write_chksum(fileObj, action)
      if(iec == ieg) iadd = iadd + ishift
      if(jec == jeg) jadd = jadd + jshift
 
-     if(action == MPP_OVERWR .OR. (action == MPP_RDONLY .AND. cur_var%initialized) ) then 
+     if(action == MPP_OVERWR .OR. (action == MPP_RDONLY .AND. cur_var%initialized) ) then
         do k = 1, cur_var%siz(4)
            if ( Associated(fileObj%p0dr(k,j)%p) ) then
               data_chksum = mpp_chksum(fileObj%p0dr(k,j)%p, (/mpp_pe()/) )
@@ -2001,6 +3450,8 @@ subroutine write_chksum(fileObj, action)
               data_chksum = mpp_chksum(fileObj%p2dr(k,j)%p(cur_var%is:cur_var%is+iadd,cur_var%js:cur_var%js+jadd) )
            else if ( Associated(fileObj%p3dr(k,j)%p) ) then
               data_chksum = mpp_chksum(fileObj%p3dr(k,j)%p(cur_var%is:cur_var%is+iadd,cur_var%js:cur_var%js+jadd, :) )
+           else if ( Associated(fileObj%p4dr(k,j)%p) ) then
+              data_chksum = mpp_chksum(fileObj%p4dr(k,j)%p(cur_var%is:cur_var%is+iadd,cur_var%js:cur_var%js+jadd, :, :) )
            else if ( Associated(fileObj%p0di(k,j)%p) ) then
               data_chksum = fileObj%p0di(k,j)%p
            else if ( Associated(fileObj%p1di(k,j)%p) ) then
@@ -2015,7 +3466,7 @@ subroutine write_chksum(fileObj, action)
            end if
            outunit = stdout()
            write(outunit,'(a, I1, a, Z16)')'fms_io('//trim(routine_name)//'): At time level = ', k, ', chksum for "'// &
-                trim(cur_var%name)// '" of "'// trim(fileObj%name)// '" = ', data_chksum    
+                trim(cur_var%name)// '" of "'// trim(fileObj%name)// '" = ', data_chksum
 
         enddo
      endif
@@ -2034,12 +3485,12 @@ subroutine restore_state_all(fileObj, directory)
   type(restart_file_type), intent(inout)       :: fileObj
   character(len=*),      intent(in), optional  :: directory
 
-! Arguments: 
+! Arguments:
 !  (in)      directory - The directory where the restart or save
 !                        files should be found. The default is 'INPUT'
 
-  character(len=128) :: dir    
-  character(len=256) :: restartpath ! The restart file path (dir/file). 
+  character(len=128) :: dir
+  character(len=256) :: restartpath ! The restart file path (dir/file).
   character(len=200) :: filepath    ! The path (dir/file) to the file being opened.
   character(len=8)   :: suffix      ! A suffix (like "_2") that is added to any
                                     ! additional restart files.
@@ -2059,10 +3510,11 @@ subroutine restore_state_all(fileObj, directory)
   integer                             :: tile_id(1)
   real, allocatable, dimension(:,:,:) :: r3d
   real, allocatable, dimension(:,:)   :: r2d
-  real, allocatable, dimension(:)     :: r1d  
+  real, allocatable, dimension(:)     :: r1d
   real                                :: r0d
   type(domain2d), pointer, save       :: io_domain=>NULL()
-  integer                             :: isc, iec, jsc, jec, check_exist
+  integer                             :: isc, iec, jsc, jec
+  logical                             :: check_exist
   integer                             :: isg, ieg, jsg, jeg
   integer                             :: ishift, jshift, iadd, jadd
   integer(LONG_KIND), dimension(3)    :: checksum_file
@@ -2094,9 +3546,8 @@ subroutine restore_state_all(fileObj, directory)
 
   !--- first open all the restart files
   !--- NOTE: For distributed restart file, we are assuming there is only one file exist.
-
-  inquire (file=trim(restartpath)//trim(pe_name), exist=fexist)
-  if(.NOT. fexist .and. domain_present) then
+  fexist = .FALSE.
+  if(domain_present) then
      io_domain => mpp_get_io_domain(array_domain(domain_idx))
      if(associated(io_domain)) then
         tile_id = mpp_get_tile_id(io_domain)
@@ -2111,13 +3562,9 @@ subroutine restore_state_all(fileObj, directory)
   endif
   if(fexist) then
      nfile = 1
-     if(domain_present) then
-        call mpp_open(unit(nfile), trim(restartpath), form=form,action=MPP_RDONLY,threading=thread_r, &
-             fileset=MPP_MULTI, domain=array_domain(domain_idx) )
-     else
-        call mpp_open(unit(nfile), trim(restartpath), form=form,action=MPP_RDONLY,threading=thread_r, &
-             fileset=MPP_MULTI)
-     endif
+     !--- domain_present is true
+     call mpp_open(unit(nfile), trim(restartpath), form=form,action=MPP_RDONLY, &
+           threading=MPP_MULTI, domain=array_domain(domain_idx) )
   else
      do while(.true.)
         if (num_restart < 10) then
@@ -2129,19 +3576,19 @@ subroutine restore_state_all(fileObj, directory)
            siz = len_trim(restartpath)
            if(restartpath(siz-2:siz) == ".nc") then
               filepath = restartpath(1:siz-3)//trim(suffix)
-           else      
+           else
               filepath = trim(restartpath) // trim(suffix)
            end if
         else
            filepath = trim(restartpath)
         end if
-        inquire (file=trim(filepath), exist=fexist) 
+        inquire (file=trim(filepath), exist=fexist)
         if(.not. fexist) inquire(file=trim(filepath)//".nc", exist=fexist)
         if(fexist) then
            nfile = nfile + 1
            if(nfile > max_split_file) call mpp_error(FATAL, &
                 "fms_io(restore_state_all): nfile is larger than max_split_file, increase max_split_file")
-           call mpp_open(unit(nfile), trim(filepath), form=form,action=MPP_RDONLY,threading=thread_r, &
+           call mpp_open(unit(nfile), trim(filepath), form=form,action=MPP_RDONLY,threading=MPP_MULTI, &
                 fileset=MPP_SINGLE)
         else
            exit
@@ -2201,7 +3648,7 @@ subroutine restore_state_all(fileObj, directory)
               check_exist = mpp_attribute_exist(fields(l),"checksum")
               checksum_file = 0
               is_there_a_checksum = .false.
-              if ( check_exist > 0 ) then
+              if ( check_exist ) then
                 call mpp_get_atts(fields(l),checksum=checksum_file)
                 is_there_a_checksum = .true.
               endif
@@ -2209,7 +3656,7 @@ subroutine restore_state_all(fileObj, directory)
 
               do k = 1, cur_var%siz(4)
                  tlev = k
-                 if(domain_present) then 
+                 if(domain_present) then
                     if( Associated(fileObj%p0dr(k,j)%p) ) then
                        call mpp_read(unit(n), fields(l), fileObj%p0dr(k,j)%p, tlev)
                        if ( is_there_a_checksum ) checksum_data = mpp_chksum(fileObj%p0dr(k,j)%p, (/mpp_pe()/) )
@@ -2224,6 +3671,10 @@ subroutine restore_state_all(fileObj, directory)
                        call mpp_read(unit(n), fields(l), array_domain(domain_idx), fileObj%p3dr(k,j)%p, tlev)
                        if ( is_there_a_checksum ) &
                          checksum_data = mpp_chksum(fileObj%p3dr(k,j)%p(cur_var%is:cur_var%is+iadd,cur_var%js:cur_var%js+jadd, :) )
+                    else if( Associated(fileObj%p4dr(k,j)%p) ) then
+                       call mpp_read(unit(n), fields(l), array_domain(domain_idx), fileObj%p4dr(k,j)%p, tlev)
+                       if ( is_there_a_checksum ) &
+                         checksum_data = mpp_chksum(fileObj%p4dr(k,j)%p(cur_var%is:cur_var%is+iadd,cur_var%js:cur_var%js+jadd,:,:))
                     else if( Associated(fileObj%p0di(k,j)%p) ) then
                        call mpp_read(unit(n), fields(l), r0d, tlev)
                        fileObj%p0di(k,j)%p = r0d
@@ -2246,13 +3697,13 @@ subroutine restore_state_all(fileObj, directory)
                        allocate(r3d(cur_var%siz(1), cur_var%siz(2), cur_var%siz(3)) )
                        r3d = 0
                        call mpp_read(unit(n), fields(l), array_domain(domain_idx), r3d, tlev)
-                       fileObj%p3di(k,j)%p(isc:iec,jsc:jec,:) = r3d(isc:iec,jsc:jec,:) 
+                       fileObj%p3di(k,j)%p(isc:iec,jsc:jec,:) = r3d(isc:iec,jsc:jec,:)
                        if ( is_there_a_checksum ) &
                          checksum_data = mpp_chksum(fileObj%p3di(k,j)%p(cur_var%is:cur_var%is+iadd,cur_var%js:cur_var%js+jadd, :))
                        deallocate(r3d)
                     else
                        call mpp_error(FATAL, "fms_io(restore_state_all): domain is present for the field "//trim(varname)// &
-                            " of file "//trim(fileObj%name)//", but none of p2dr, p3dr, p2di and p3di is associated") 
+                            " of file "//trim(fileObj%name)//", but none of p2dr, p3dr, p2di and p3di is associated")
                     end if
                  else
                     if( Associated(fileObj%p0dr(k,j)%p) ) then
@@ -2269,20 +3720,24 @@ subroutine restore_state_all(fileObj, directory)
                        call mpp_read(unit(n), fields(l), fileObj%p3dr(k,j)%p, tlev)
                        if ( is_there_a_checksum ) &
                          checksum_data = mpp_chksum(fileObj%p3dr(k,j)%p(cur_var%is:cur_var%is+iadd,cur_var%js:cur_var%js+jadd, :) )
+                    else if( Associated(fileObj%p4dr(k,j)%p) ) then
+                       call mpp_read(unit(n), fields(l), fileObj%p4dr(k,j)%p, tlev)
+                       if ( is_there_a_checksum ) &
+                         checksum_data = mpp_chksum(fileObj%p4dr(k,j)%p(cur_var%is:cur_var%is+iadd,cur_var%js:cur_var%js+jadd,:,:))
                     else if( Associated(fileObj%p0di(k,j)%p) ) then
                        call mpp_read(unit(n), fields(l), r0d, tlev)
                        fileObj%p0di(k,j)%p = r0d
                        if ( is_there_a_checksum ) checksum_data = fileObj%p0di(k,j)%p
                     else if( Associated(fileObj%p1di(k,j)%p) ) then
                        allocate(r1d(cur_var%siz(1)) )
-                       call mpp_read(unit(n), fields(l), r1d, tlev)                
+                       call mpp_read(unit(n), fields(l), r1d, tlev)
                        fileObj%p1di(k,j)%p = r1d
                        if ( is_there_a_checksum ) checksum_data = mpp_chksum(fileObj%p1di(k,j)%p, (/mpp_pe()/) )
                        deallocate(r1d)
                     else if( Associated(fileObj%p2di(k,j)%p) ) then
                        allocate(r2d(cur_var%siz(1), cur_var%siz(2)) )
                        r2d = 0
-                       call mpp_read(unit(n), fields(l), r2d, tlev)                
+                       call mpp_read(unit(n), fields(l), r2d, tlev)
                        fileObj%p2di(k,j)%p = r2d
                        if ( is_there_a_checksum ) &
                          checksum_data = mpp_chksum(fileObj%p2di(k,j)%p(cur_var%is:cur_var%is+iadd,cur_var%js:cur_var%js+jadd) )
@@ -2290,7 +3745,7 @@ subroutine restore_state_all(fileObj, directory)
                     else if( Associated(fileObj%p3di(k,j)%p) ) then
                        allocate(r3d(cur_var%siz(1), cur_var%siz(2), cur_var%siz(3)) )
                        r3d = 0
-                       call mpp_read(unit(n), fields(l), r3d, tlev)                
+                       call mpp_read(unit(n), fields(l), r3d, tlev)
                        fileObj%p3di(k,j)%p = r3d
                        if ( is_there_a_checksum ) &
                          checksum_data = mpp_chksum(fileObj%p3di(k,j)%p(cur_var%is:cur_var%is+iadd,cur_var%js:cur_var%js+jadd, :))
@@ -2346,12 +3801,12 @@ subroutine restore_state_one_field(fileObj, id_field, directory)
   integer,                 intent(in)          :: id_field
   character(len=*),      intent(in), optional  :: directory
 
-! Arguments: 
+! Arguments:
 !  (in)      directory - The directory where the restart or save
 !                        files should be found. The default is 'INPUT'
 
-  character(len=128) :: dir    
-  character(len=256) :: restartpath ! The restart file path (dir/file). 
+  character(len=128) :: dir
+  character(len=256) :: restartpath ! The restart file path (dir/file).
   character(len=200) :: filepath    ! The path (dir/file) to the file being opened.
   character(len=8)   :: suffix      ! A suffix (like "_2") that is added to any
                                     ! additional restart files.
@@ -2371,10 +3826,11 @@ subroutine restore_state_one_field(fileObj, id_field, directory)
   integer                             :: j, n, l, k, missing_fields, domain_idx
   real, allocatable, dimension(:,:,:) :: r3d
   real, allocatable, dimension(:,:)   :: r2d
-  real, allocatable, dimension(:)     :: r1d  
+  real, allocatable, dimension(:)     :: r1d
   real                                :: r0d
   type(domain2d), pointer, save       :: io_domain=>NULL()
-  integer                             :: isc, iec, jsc, jec, check_exist
+  integer                             :: isc, iec, jsc, jec
+  logical                             :: check_exist
   integer                             :: isg, ieg, jsg, jeg
   integer                             :: ishift, jshift, iadd, jadd
   integer(LONG_KIND), dimension(3)    :: checksum_file ! There should be no more than 3 timelevels in a restart file.
@@ -2422,8 +3878,8 @@ subroutine restore_state_one_field(fileObj, id_field, directory)
   end if
   !--- first open all the restart files
   !--- NOTE: For distributed restart file, we are assuming there is only one file exist.
-  inquire (file=trim(restartpath)//trim(pe_name), exist=fexist)     
-  if(.NOT. fexist .and. domain_present) then
+  fexist = .FALSE.
+  if(domain_present) then
      io_domain => mpp_get_io_domain(array_domain(domain_idx))
      if(associated(io_domain)) then
         tile_id = mpp_get_tile_id(io_domain)
@@ -2439,13 +3895,9 @@ subroutine restore_state_one_field(fileObj, id_field, directory)
 
   if(fexist) then
      nfile = 1
-     if(domain_present) then
-        call mpp_open(unit(nfile), trim(restartpath), form=form,action=MPP_RDONLY,threading=thread_r, &
-             fileset=MPP_MULTI, domain=array_domain(domain_idx) )
-     else
-        call mpp_open(unit(nfile), trim(restartpath), form=form,action=MPP_RDONLY,threading=thread_r, &
-             fileset=MPP_MULTI)
-     endif
+     !--- domain_present is true here.
+     call mpp_open(unit(nfile), trim(restartpath), form=form,action=MPP_RDONLY, &
+             threading=MPP_MULTI, domain=array_domain(domain_idx) )
   else
      do while(.true.)
         if (num_restart < 10) then
@@ -2457,19 +3909,19 @@ subroutine restore_state_one_field(fileObj, id_field, directory)
            siz = len_trim(restartpath)
            if(restartpath(siz-2:siz) == ".nc") then
               filepath = restartpath(1:siz-3)//trim(suffix)
-           else      
+           else
               filepath = trim(restartpath) // trim(suffix)
            end if
         else
            filepath = trim(restartpath)
         end if
-        inquire (file=trim(filepath), exist=fexist) 
+        inquire (file=trim(filepath), exist=fexist)
         if(.not. fexist) inquire(file=trim(filepath)//".nc", exist=fexist)
         if(fexist) then
            nfile = nfile + 1
            if(nfile > max_split_file) call mpp_error(FATAL, &
                 "fms_io(restore_state_one_field): nfile is larger than max_split_file, increase max_split_file")
-           call mpp_open(unit(nfile), trim(filepath), form=form,action=MPP_RDONLY,threading=thread_r, &
+           call mpp_open(unit(nfile), trim(filepath), form=form,action=MPP_RDONLY,threading=MPP_MULTI, &
                 fileset=MPP_SINGLE)
         else
            exit
@@ -2497,7 +3949,7 @@ subroutine restore_state_one_field(fileObj, id_field, directory)
            check_exist = mpp_attribute_exist(fields(l),"checksum")
            checksum_file = 0
            is_there_a_checksum = .false.
-           if ( check_exist > 0 ) then
+           if ( check_exist ) then
              call mpp_get_atts(fields(l),checksum=checksum_file)
              is_there_a_checksum = .true.
            endif
@@ -2508,12 +3960,12 @@ subroutine restore_state_one_field(fileObj, id_field, directory)
            jec = cur_var%je
            do k = 1, cur_var%siz(4)
               tlev = k
-              if(domain_present) then        
+              if(domain_present) then
                  if( Associated(fileObj%p0dr(k,j)%p) ) then
                     call mpp_read(unit(n), fields(l), fileObj%p0dr(k,j)%p, tlev)
                     if ( is_there_a_checksum ) checksum_data = mpp_chksum(fileObj%p0dr(k,j)%p, (/mpp_pe()/) )
                  else if( Associated(fileObj%p1dr(k,j)%p) ) then
-                    call mpp_read(unit(n), fields(l), fileObj%p1dr(k,j)%p, tlev)       
+                    call mpp_read(unit(n), fields(l), fileObj%p1dr(k,j)%p, tlev)
                     if ( is_there_a_checksum ) checksum_data = mpp_chksum(fileObj%p1dr(k,j)%p, (/mpp_pe()/) )
                  else if( Associated(fileObj%p2dr(k,j)%p) ) then
                     call mpp_read(unit(n), fields(l), array_domain(domain_idx), fileObj%p2dr(k,j)%p, tlev)
@@ -2523,6 +3975,10 @@ subroutine restore_state_one_field(fileObj, id_field, directory)
                     call mpp_read(unit(n), fields(l), array_domain(domain_idx), fileObj%p3dr(k,j)%p, tlev)
                     if ( is_there_a_checksum ) checksum_data =&
                          & mpp_chksum(fileObj%p3dr(k,j)%p(cur_var%is:cur_var%is+iadd,cur_var%js:cur_var%js+jadd, :) )
+                 else if( Associated(fileObj%p4dr(k,j)%p) ) then
+                    call mpp_read(unit(n), fields(l), array_domain(domain_idx), fileObj%p4dr(k,j)%p, tlev)
+                    if ( is_there_a_checksum ) checksum_data =&
+                         & mpp_chksum(fileObj%p4dr(k,j)%p(cur_var%is:cur_var%is+iadd,cur_var%js:cur_var%js+jadd, :,:) )
                  else if( Associated(fileObj%p0di(k,j)%p) ) then
                     call mpp_read(unit(n), fields(l), r0d, tlev)
                     fileObj%p0di(k,j)%p = r0d
@@ -2537,7 +3993,7 @@ subroutine restore_state_one_field(fileObj, id_field, directory)
                     allocate(r2d(cur_var%siz(1), cur_var%siz(2)) )
                     r2d = 0
                     call mpp_read(unit(n), fields(l), array_domain(domain_idx), r2d, tlev)
-                    fileObj%p2di(k,j)%p(isc:iec,jsc:jec) = r2d(isc:iec,jsc:jec) 
+                    fileObj%p2di(k,j)%p(isc:iec,jsc:jec) = r2d(isc:iec,jsc:jec)
                     if ( is_there_a_checksum ) checksum_data =&
                          & mpp_chksum(fileObj%p2di(k,j)%p(cur_var%is:cur_var%is+iadd,cur_var%js:cur_var%js+jadd) )
                     deallocate(r2d)
@@ -2545,13 +4001,13 @@ subroutine restore_state_one_field(fileObj, id_field, directory)
                     allocate(r3d(cur_var%siz(1), cur_var%siz(2), cur_var%siz(3)) )
                     r3d = 0
                     call mpp_read(unit(n), fields(l), array_domain(domain_idx), r3d, tlev)
-                    fileObj%p3di(k,j)%p(isc:iec,jsc:jec,:) = r3d(isc:iec,jsc:jec,:) 
+                    fileObj%p3di(k,j)%p(isc:iec,jsc:jec,:) = r3d(isc:iec,jsc:jec,:)
                     if ( is_there_a_checksum ) checksum_data =&
                          & mpp_chksum(fileObj%p3di(k,j)%p(cur_var%is:cur_var%is+iadd,cur_var%js:cur_var%js+jadd, :))
                     deallocate(r3d)
                  else
                     call mpp_error(FATAL, "fms_io(restore_state_one_field): domain is present for the field "//trim(varname)// &
-                         " of file "//trim(fileObj%name)//", but none of p2dr, p3dr, p2di and p3di is associated") 
+                         " of file "//trim(fileObj%name)//", but none of p2dr, p3dr, p2di and p3di is associated")
                  end if
               else
                  if( Associated(fileObj%p0dr(k,j)%p) ) then
@@ -2568,20 +4024,24 @@ subroutine restore_state_one_field(fileObj, id_field, directory)
                     call mpp_read(unit(n), fields(l), fileObj%p3dr(k,j)%p, tlev)
                     if ( is_there_a_checksum ) checksum_data =&
                          & mpp_chksum(fileObj%p3dr(k,j)%p(cur_var%is:cur_var%is+iadd,cur_var%js:cur_var%js+jadd, :) )
+                 else if( Associated(fileObj%p4dr(k,j)%p) ) then
+                    call mpp_read(unit(n), fields(l), fileObj%p4dr(k,j)%p, tlev)
+                    if ( is_there_a_checksum ) checksum_data =&
+                         & mpp_chksum(fileObj%p4dr(k,j)%p(cur_var%is:cur_var%is+iadd,cur_var%js:cur_var%js+jadd, :, :) )
                  else if( Associated(fileObj%p0di(k,j)%p) ) then
                     call mpp_read(unit(n), fields(l), r0d, tlev)
                     fileObj%p0di(k,j)%p = r0d
                     if ( is_there_a_checksum ) checksum_data = fileObj%p0di(k,j)%p
                  else if( Associated(fileObj%p1di(k,j)%p) ) then
                     allocate(r1d(cur_var%siz(1)) )
-                    call mpp_read(unit(n), fields(l), r1d, tlev)                
+                    call mpp_read(unit(n), fields(l), r1d, tlev)
                     fileObj%p1di(k,j)%p = r1d
                     if ( is_there_a_checksum ) checksum_data = fileObj%p0di(k,j)%p
                     deallocate(r1d)
                  else if( Associated(fileObj%p2di(k,j)%p) ) then
                     allocate(r2d(cur_var%siz(1), cur_var%siz(2)) )
                     r2d = 0
-                    call mpp_read(unit(n), fields(l), r2d, tlev)                
+                    call mpp_read(unit(n), fields(l), r2d, tlev)
                     fileObj%p2di(k,j)%p = r2d
                     if ( is_there_a_checksum ) checksum_data =&
                          & mpp_chksum(fileObj%p2di(k,j)%p(cur_var%is:cur_var%is+iadd,cur_var%js:cur_var%js+jadd) )
@@ -2589,7 +4049,7 @@ subroutine restore_state_one_field(fileObj, id_field, directory)
                  else if( Associated(fileObj%p3di(k,j)%p) ) then
                     allocate(r3d(cur_var%siz(1), cur_var%siz(2), cur_var%siz(3)) )
                     r3d = 0
-                    call mpp_read(unit(n), fields(l), r3d, tlev)                
+                    call mpp_read(unit(n), fields(l), r3d, tlev)
                     fileObj%p3di(k,j)%p = r3d
                     if ( is_there_a_checksum ) checksum_data =&
                          & mpp_chksum(fileObj%p3di(k,j)%p(cur_var%is:cur_var%is+iadd,cur_var%js:cur_var%js+jadd, :))
@@ -2630,22 +4090,24 @@ end subroutine restore_state_one_field
 
 !-------------------------------------------------------------------------------
 !
-!     This routine will setup one entry to be written out 
+!     This routine will setup one entry to be written out
 !
 !-------------------------------------------------------------------------------
 subroutine setup_one_field(fileObj, filename, fieldname, field_siz, index_field,  domain, mandatory, &
-                           no_domain, scalar_or_1d, position, tile_count, data_default, longname, units)
+                           no_domain, scalar_or_1d, position, tile_count, data_default, longname, units, &
+                           compressed_axis, read_only)
   type(restart_file_type), intent(inout)         :: fileObj
-  character(len=*),         intent(in)           :: filename, fieldname 
+  character(len=*),         intent(in)           :: filename, fieldname
   integer, dimension(:),    intent(in)           :: field_siz
   integer,                  intent(out)          :: index_field
   type(domain2d), optional, intent(in), target   :: domain
   real,           optional, intent(in)           :: data_default
   logical,        optional, intent(in)           :: no_domain
-  logical,        optional, intent(in)           :: scalar_or_1d   
+  logical,        optional, intent(in)           :: scalar_or_1d
   integer,        optional, intent(in)           :: position, tile_count
   logical,          optional, intent(in)         :: mandatory
-  character(len=*), optional, intent(in)         :: longname, units
+  character(len=*), optional, intent(in)         :: longname, units, compressed_axis
+  logical,        optional, intent(in)           :: read_only  !The variable will not be written to restart file.
 
   !--- local variables
   integer                         :: i, domain_idx
@@ -2659,17 +4121,16 @@ subroutine setup_one_field(fileObj, filename, fieldname, field_siz, index_field,
   character(len=256)              :: fname, filename2, append_string
   type(domain2d), pointer, save   :: d_ptr   =>NULL()
   type(var_type), pointer, save   :: cur_var =>NULL()
-  type(domain2d), pointer, save   :: io_domain =>NULL()
-  integer                         :: length
+  integer                         :: length, n_field_siz
 
-  if(ANY(field_siz < 1)) then
-     call mpp_error(FATAL, "fms_io(setup_one_field): each entry of field_size should be a positive integer")
+  if(ANY(field_siz < 0)) then
+     call mpp_error(FATAL, "fms_io(setup_one_field): each entry of field_size should be a non-negative integer")
   end if
 
   if(PRESENT(data_default))then
      default_data=data_default
   else
-     default_data=0.
+     default_data = MPP_FILL_DOUBLE
   endif
 
   if(present(tile_count) .AND. .not. present(domain)) call mpp_error(FATAL, &
@@ -2702,7 +4163,7 @@ subroutine setup_one_field(fileObj, filename, fieldname, field_siz, index_field,
 
   !Append a string to the file name
   append_string=''
-  !If the filename_appendix  is set override the passed argument. 
+  !If the filename_appendix  is set override the passed argument.
   if(len_trim(filename_appendix) > 0)   append_string = filename_appendix
 
   if(len_trim(append_string) > 0) filename2 = trim(filename2)//'.'//trim(append_string)
@@ -2722,18 +4183,25 @@ subroutine setup_one_field(fileObj, filename, fieldname, field_siz, index_field,
      allocate(fileObj%p1dr(MAX_TIME_LEVEL_REGISTER, max_fields))
      allocate(fileObj%p2dr(MAX_TIME_LEVEL_REGISTER, max_fields))
      allocate(fileObj%p3dr(MAX_TIME_LEVEL_REGISTER, max_fields))
+     allocate(fileObj%p4dr(MAX_TIME_LEVEL_REGISTER, max_fields))
      allocate(fileObj%p0di(MAX_TIME_LEVEL_REGISTER, max_fields))
      allocate(fileObj%p1di(MAX_TIME_LEVEL_REGISTER, max_fields))
      allocate(fileObj%p2di(MAX_TIME_LEVEL_REGISTER, max_fields))
      allocate(fileObj%p3di(MAX_TIME_LEVEL_REGISTER, max_fields))
      !--- make sure fname is not used in other restart_file_type object.
      do i = 1, num_registered_files
-        if(trim(fname) == trim(registered_file(i)) ) call mpp_error(FATAL, &
-          'fms_io(setup_one_field): '//trim(fname)//' is already registered with other restart_file_type data')
+        if(trim(fname) == trim(registered_file(i)) ) then
+           call mpp_error(NOTE, &
+             'fms_io(setup_one_field): '//trim(fname)//' is already registered with other restart_file_type data')
+           exit
+        endif
      end do
      num_registered_files = num_registered_files + 1
+     if( num_registered_files > max_files_w ) call mpp_error(WARNING, &
+         'fms_io(setup_one_field): num_registered_files > max_files_w, increase fms_io_nml max_files_w')
      registered_file(num_registered_files) = trim(fname)
-     fileObj%name = trim(fname)         
+     fileObj%register_id = num_registered_files
+     fileObj%name = trim(fname)
      fileObj%tile_count=1
      if(present(tile_count)) fileObj%tile_count = tile_count
      if(ASSOCIATED(d_ptr))then
@@ -2747,17 +4215,18 @@ subroutine setup_one_field(fileObj, filename, fieldname, field_siz, index_field,
      do i = 1, max_fields
         fileObj%var(i)%name           = 'none'
         fileObj%var(i)%domain_present = .false.
-        fileObj%var(i)%write_on_this_pe = .false.
         fileObj%var(i)%domain_idx     = -1
         fileObj%var(i)%is_dimvar      = .false.
         fileObj%var(i)%position       = CENTER
         fileObj%var(i)%siz(:)         = 0
         fileObj%var(i)%gsiz(:)        = 0
         fileObj%var(i)%id_axes(:)     = -1
-        fileObj%var(i)%longname       = "";
-        fileObj%var(i)%units          = "none";
+        fileObj%var(i)%longname       = '';
+        fileObj%var(i)%units          = 'none';
         fileObj%var(i)%mandatory      = .true.
         fileObj%var(i)%initialized    = .false.
+        fileObj%var(i)%compressed_axis = ''
+        fileObj%var(i)%read_only      = .false.
      end do
   endif
 
@@ -2767,7 +4236,7 @@ subroutine setup_one_field(fileObj, filename, fieldname, field_siz, index_field,
      if(trim(fileObj%var(i)%name) == trim(fieldname)) then
         index_field = i
         exit
-     end if 
+     end if
   end do
 
   if(index_field > 0) then
@@ -2778,25 +4247,27 @@ subroutine setup_one_field(fileObj, filename, fieldname, field_siz, index_field,
 
      cur_var%siz(4) =  cur_var%siz(4) + field_siz(4)
      if(fileObj%max_ntime < cur_var%siz(4) ) fileObj%max_ntime = cur_var%siz(4)
-     ! the time level should be no larger than MAX_TIME_LEVEL_REGISTER ( = 2) 
+     ! the time level should be no larger than MAX_TIME_LEVEL_REGISTER ( = 2)
      if( cur_var%siz(4) > MAX_TIME_LEVEL_REGISTER ) call mpp_error(FATAL, 'fms_io(setup_one_field): ' // &
           'the time level of field '//trim(cur_var%name)//' in file '//trim(fileObj%name)// &
           ' is greater than MAX_TIME_LEVEL_REGISTER(=2), increase MAX_TIME_LEVEL_REGISTER or check your code')
-  else 
+  else
      fileObj%nvar = fileObj%nvar +1
      if(fileObj%nvar>max_fields) then
-        write(error_msg,'(I3,"/",I3)') fileObj%nvar, max_fields 
+        write(error_msg,'(I3,"/",I3)') fileObj%nvar, max_fields
         call  mpp_error(FATAL,'fms_io(setup_one_field): max_fields exceeded, needs increasing, nvar/max_fields=' &
              //trim(error_msg))
      endif
      index_field =  fileObj%nvar
      cur_var   => fileObj%var(index_field)
-     cur_var%siz(:)  = field_siz(:)
+     n_field_siz = size(field_siz(:))
+     cur_var%siz(1:n_field_siz)  = field_siz(1:n_field_siz)
      cur_var%gsiz(3) = field_siz(3)
-     cur_var%csiz(3) = field_siz(3)
+     if(n_field_siz == 5) cur_var%gsiz(4) = field_siz(5)
      cur_var%name = fieldname
      cur_var%default_data = default_data
      if(present(mandatory)) cur_var%mandatory = mandatory
+     if(present(read_only)) cur_var%read_only = read_only
      if(present(longname)) then
         cur_var%longname = longname
      else
@@ -2804,15 +4275,10 @@ subroutine setup_one_field(fileObj, filename, fieldname, field_siz, index_field,
      end if
      if(present(units))    cur_var%units    = units
      if(present(position)) cur_var%position = position
-     cur_var%is = 1; cur_var%ie =  cur_var%siz(1) 
+     if(present(compressed_axis)) cur_var%compressed_axis = compressed_axis
+     cur_var%is = 1; cur_var%ie =  cur_var%siz(1)
      cur_var%js = 1; cur_var%je =  cur_var%siz(2)
-     if(ASSOCIATED(d_ptr)) then
-        io_domain => mpp_get_io_domain(d_ptr)
-        if(associated(io_domain)) then
-           if(mpp_domain_is_tile_root_pe(io_domain)) cur_var%write_on_this_pe = .true.
-        endif
-     endif
-     
+
      if(ASSOCIATED(d_ptr) .AND. .NOT. is_scalar_or_1d ) then
         cur_var%domain_present = .true.
         domain_idx = lookup_domain(d_ptr)
@@ -2847,23 +4313,9 @@ subroutine setup_one_field(fileObj, filename, fieldname, field_siz, index_field,
         cur_var%je   = cur_var%js + cysize - 1;
         cur_var%gsiz(1)   = gxsize
         cur_var%gsiz(2)   = gysize
-        io_domain => mpp_get_io_domain(array_domain(domain_idx))
-        if(associated(io_domain)) then
-           call mpp_get_global_domain( io_domain, xsize=cxsize, ysize=cysize,tile_count=tile_count)
-           cur_var%csiz(1)   = cxsize
-           cur_var%csiz(2)   = cysize
-        else if(thread_w == MPP_MULTI) then
-           call mpp_get_compute_domain(array_domain(domain_idx), xsize=cxsize,ysize=cysize,tile_count=tile_count)
-           cur_var%csiz(1)   = cxsize
-           cur_var%csiz(2)   = cysize
-        else
-           cur_var%csiz(1)   = cur_var%gsiz(1)
-           cur_var%csiz(2)   = cur_var%gsiz(2)
-        end if
      else
         cur_var%domain_present=.false.
         cur_var%gsiz(1:2) = field_siz(1:2)
-        cur_var%csiz(1:2) = field_siz(1:2)
      endif
   end if
 
@@ -2877,14 +4329,14 @@ end subroutine setup_one_field
 subroutine write_data_2d_new(filename, fieldname, data, domain,    &
                              no_domain, position,tile_count, data_default)
 
-  character(len=*), intent(in)                 :: filename, fieldname 
+  character(len=*), intent(in)                 :: filename, fieldname
   real, dimension(:,:), intent(in)             :: data
   real, dimension(size(data,1),size(data,2),1) :: data_3d
   real, intent(in), optional                   :: data_default
   type(domain2d), intent(in), optional         :: domain
   logical, intent(in), optional                :: no_domain
   integer, intent(in), optional                :: position, tile_count
- 
+
   if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(write_data_2d_new):need to call fms_io_init first')
   data_3d(:,:,1) = data(:,:)
 
@@ -2896,19 +4348,19 @@ end subroutine write_data_2d_new
 ! ........................................................
 subroutine write_data_1d_new(filename, fieldname, data,domain, &
                              no_domain, tile_count, data_default)
-  
+
   type(domain2d), intent(in), optional   :: domain
-  character(len=*), intent(in)           :: filename, fieldname 
+  character(len=*), intent(in)           :: filename, fieldname
   real, dimension(:), intent(in)         :: data
   real, dimension(size(data(:)),1,1)     :: data_3d
   real, intent(in), optional             :: data_default
   logical, intent(in), optional          :: no_domain
   integer, intent(in), optional          :: tile_count
-  
-  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(write_data_1d_new): module not initialized')  
-  data_3d(:,1,1) = data(:)  
+
+  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(write_data_1d_new): module not initialized')
+  data_3d(:,1,1) = data(:)
   call write_data_3d_new(filename, fieldname, data_3d,domain,   &
-                         no_domain=no_domain, scalar_or_1d=.true., tile_count=tile_count, data_default=data_default)  
+                         no_domain=no_domain, scalar_or_1d=.true., tile_count=tile_count, data_default=data_default)
 end subroutine write_data_1d_new
 
 ! ..........................................................
@@ -2916,14 +4368,14 @@ subroutine write_data_scalar_new(filename, fieldname, data, domain, &
                                  no_domain, tile_count, data_default)
 
   type(domain2d), intent(in), optional   :: domain
-  character(len=*), intent(in)           :: filename, fieldname 
+  character(len=*), intent(in)           :: filename, fieldname
   real, intent(in)                       :: data
   real, dimension(1,1,1)                 :: data_3d
   real, intent(in), optional             :: data_default
   logical, intent(in), optional          :: no_domain
   integer, intent(in), optional          :: tile_count
-    
-  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(write_data_scalar_new):  module not initialized: '//fieldname)  
+
+  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(write_data_scalar_new):  module not initialized: '//fieldname)
 
   data_3d(1,1,1) = data
   call write_data_3d_new(filename, fieldname, data_3d,domain, &
@@ -2944,7 +4396,7 @@ function lookup_field_r(nfile,fieldname)
   do j = 1, files_read(nfile)%nvar
      if (trim(files_read(nfile)%var(j)%name) == trim(fieldname)) then
         lookup_field_r = j
-        exit 
+        exit
      endif
   enddo
   return
@@ -2969,7 +4421,7 @@ end function lookup_domain
 !.........................................................
 function lookup_axis(axis_sizes,siz,domains,dom)
 
-! Given axis size (global), this function returns the axis id  
+! Given axis size (global), this function returns the axis id
 
   integer, intent(in)      :: axis_sizes(:), siz
   type(domain1d), optional :: domains(:)
@@ -2982,7 +4434,7 @@ function lookup_axis(axis_sizes,siz,domains,dom)
   do j=1,size(axis_sizes(:))
      if (siz == axis_sizes(j)) then
         if (PRESENT(domains)) then
-           if (dom .EQ. domains(j)) then 
+           if (dom .EQ. domains(j)) then
               lookup_axis = j
               exit
            endif
@@ -2995,44 +4447,106 @@ function lookup_axis(axis_sizes,siz,domains,dom)
   if (lookup_axis == -1) call mpp_error(FATAL,'fms_io(lookup_axis): could not find axis in set of axes')
 end function lookup_axis
 !.....................................................................
-! 
+! 
+!
+! Given filename and fieldname, this subroutine returns the size of field
+!
+!   
+!   
+!    File name
+!   
+!   
+!    Field  name
+!   
+!   
+!    siz must be a dimension(4) array to retrieve the size of the field
+!   
+!   
+!    if this flag is present, field_size will not abort if
+!    called for a non-existent field.
+!    Instead it will return T or F depending on
+!    whether or not the field was found.
+!   
+subroutine field_size(filename, fieldname, siz, field_found, domain, no_domain )
+
+  character(len=*), intent(in)                 :: filename, fieldname
+  integer,       intent(inout)                 :: siz(:)
+  logical,       intent(out), optional         :: field_found
+  type(domain2d), intent(in), optional, target :: domain
+  logical,       intent(in),  optional         :: no_domain
+
+  integer                              :: nfile, unit
+  logical                              :: found, found_file
+  character(len=256)                   :: actual_file
+  logical                              :: read_dist, io_domain_exist, is_no_domain
+
+  if (size(siz(:)) < 4) call mpp_error(FATAL,'fms_io(field_size): size array must be >=4 to receive field size of ' &
+       //trim(fieldname)//' in file '// trim(filename))
+
+  is_no_domain = .false.
+  if(present(no_domain)) is_no_domain = no_domain
+
+!--- first need to get the filename, when is_no_domain is true, only check file without tile
+!--- if is_no_domain is false, first check no_domain=.false., then check no_domain = .true.
+  found_file = get_file_name(filename, actual_file, read_dist, io_domain_exist, no_domain=is_no_domain, &
+                             domain=domain)
+  !--- when is_no_domain is true and file is not found, send out error message.
+  if(is_no_domain .AND. .NOT. found_file) call mpp_error(FATAL, &
+         'fms_io_mod(field_size): file '//trim(filename)//' and corresponding distributed file are not found')
+  found = .false.
+  if(found_file) then
+     call get_file_unit(actual_file, unit, nfile, read_dist, io_domain_exist, domain=domain)
+     call get_size(unit,fieldname,siz,found)
+  endif
+
+  if(.not.found .AND. .not. is_no_domain) then
+    found_file =  get_file_name(filename, actual_file, read_dist, io_domain_exist, no_domain=.true.)
+    if(found_file) then
+      call get_file_unit(actual_file, unit, nfile, read_dist, io_domain_exist, domain=domain)
+      call get_size(unit,fieldname,siz,found)
+    endif
+  endif
+
+! If field_found is present we assume that it is being checked on exit.
+! If not present and the field was not found, exit with a FATAL error.
+  if( PRESENT(field_found) )then
+     field_found = found
+  else if (.not. found )then
+     call mpp_error(FATAL, 'fms_io(field_size): field '//trim(fieldname)//' NOT found in file '//trim(actual_file))
+  end if
+
+  return
+end subroutine field_size
+! 
+
+!.....................................................................
+! 
 !
-! Given filename and fieldname, this subroutine returns the size of field
+! Given filename and dimension name, this function returns the size of field
 !
 !   
 !   
 !    File name
 !   
-!   
+!   
 !    Field  name
 !   
-!   
-!    siz must be a dimension(4) array to retrieve the size of the field
-!   
-!   
-!    if this flag is present, field_size will not abort if
-!    called for a non-existent field.
-!    Instead it will return T or F depending on
-!    whether or not the field was found.
-!   
-subroutine field_size(filename, fieldname, siz, field_found, domain, no_domain )
+function dimension_size(filename, dimname, domain, no_domain )
 
-  character(len=*), intent(in)                 :: filename, fieldname
-  integer,       intent(inout)                 :: siz(:)
-  logical,       intent(out), optional         :: field_found  
+  character(len=*), intent(in)                 :: filename, dimname
   type(domain2d), intent(in), optional, target :: domain
   logical,       intent(in),  optional         :: no_domain
+  integer                                      :: dimension_size
 
   integer                              :: nfile, unit
   logical                              :: found, found_file
   character(len=256)                   :: actual_file
   logical                              :: read_dist, io_domain_exist, is_no_domain
 
-  if (size(siz(:)) < 4) call mpp_error(FATAL,'fms_io(field_size): size array must be >=4 to receive field size of ' &
-       //trim(fieldname)//' in file '// trim(filename))
-
   is_no_domain = .false.
   if(present(no_domain)) is_no_domain = no_domain
 
@@ -3042,31 +4556,112 @@ subroutine field_size(filename, fieldname, siz, field_found, domain, no_domain )
                              domain=domain)
   !--- when is_no_domain is true and file is not found, send out error message.
   if(is_no_domain .AND. .NOT. found_file) call mpp_error(FATAL, &
-         'fms_io_mod(field_size): file '//trim(filename)//' and corresponding distributed file are not found')
+         'fms_io_mod(dimesion_size): file '//trim(filename)//' and corresponding distributed file are not found')
   found = .false.
   if(found_file) then
      call get_file_unit(actual_file, unit, nfile, read_dist, io_domain_exist, domain=domain)
-     call get_size(unit,fieldname,siz,found)    
+     dimension_size = mpp_get_dimension_length(unit, dimname, found)
   endif
 
   if(.not.found .AND. .not. is_no_domain) then
     found_file =  get_file_name(filename, actual_file, read_dist, io_domain_exist, no_domain=.true.)
     if(found_file) then
       call get_file_unit(actual_file, unit, nfile, read_dist, io_domain_exist, domain=domain)
-      call get_size(unit,fieldname,siz,found)
+      dimension_size = mpp_get_dimension_length(unit, dimname, found)
     endif
   endif
 
+  if(.not. found) call mpp_error(FATAL, &
+         'fms_io_mod(dimesion_size): failed at inquiring size of dimesion '//trim(dimname)//' from file '//trim(filename))
+
+  return
+end function dimension_size
+! 
+
+
+!.....................................................................
+! 
+!
+! Given filename and fieldname, this subroutine returns the size of field
+! This is the io subset interface to field_size
+!
+!   
+!   
+!    File name
+!   
+!   
+!    Field  name
+!   
+!   
+!    siz must be a dimension(4) array to retrieve the size of the field
+!   
+!   
+!    if this flag is present, field_size will not abort if
+!    called for a non-existent field.
+!    Instead it will return T or F depending on
+!    whether or not the field was found.
+!   
+subroutine get_field_size(filename, fieldname, siz, field_found, domain, no_domain)
+
+  character(len=*), intent(in)                 :: filename, fieldname
+  integer,       intent(inout)                 :: siz(:)
+  logical,       intent(out), optional         :: field_found
+  type(domain2d), intent(in), optional, target :: domain
+  logical,       intent(in),  optional         :: no_domain
+
+  integer :: npes, p
+  integer, allocatable :: pelist(:)
+  logical :: found
+  type(domain2d), pointer :: domain_in =>NULL()
+  type(domain2d), pointer :: io_domain =>NULL()
+
+
+  if(PRESENT(domain)) then
+     domain_in =>domain
+  elseif(ASSOCIATED(current_domain)) then
+     domain_in =>current_domain
+  else
+     call mpp_error(FATAL,'fms_io(get_field_size): The domain must be defined')
+  endif
+
+  io_domain =>mpp_get_io_domain(domain)
+  if(.not. ASSOCIATED(io_domain)) call mpp_error(FATAL,'fms_io(get_field_size): The io domain must be defined')
+
+  npes = mpp_get_domain_npes(io_domain)
+  allocate(pelist(npes))
+  call mpp_get_pelist(io_domain,pelist)
+
+  if(mpp_pe() == pelist(1)) then
+     call field_size(filename, fieldname, siz, found, domain, no_domain)
+     if(.not. found) siz(:) = -1
+  endif
+  !--- z1l replace mpp_broadcast with mpp_send/mpp_recv to avoid hang in calling MPI_COMM_CREATE
+  !---     because size(pelist) might be different for different rank.
+  !--- prepost receive
+  if( mpp_pe() == pelist(1) ) then
+     do p = 2, npes
+        call mpp_send(siz(1), plen=size(siz(:)), to_pe=pelist(p), tag=COMM_TAG_1)
+     enddo
+     call mpp_sync_self()
+  else
+     call mpp_recv(siz(1), glen=size(siz(:)), from_pe=pelist(1), block=.false., tag=COMM_TAG_1)
+     call mpp_sync_self(check=EVENT_RECV)
+  endif
+
+  found = .true.
+  if(siz(1) == -1) found=.false.
+
 ! If field_found is present we assume that it is being checked on exit.
 ! If not present and the field was not found, exit with a FATAL error.
   if( PRESENT(field_found) )then
      field_found = found
   else if (.not. found )then
-     call mpp_error(FATAL, 'fms_io(field_size): field '//trim(fieldname)//' NOT found in file '//trim(actual_file))
-  end if
-
-  return
-end subroutine field_size
+  ! Force the error to be trapped with the correct file name
+     if(mpp_pe() == pelist(1)) call field_size(filename, fieldname, siz, domain=domain, no_domain=no_domain)
+  endif
+end subroutine get_field_size
 ! 
 
 subroutine get_size(unit, fieldname, siz, found)
@@ -3083,7 +4678,7 @@ subroutine get_size(unit, fieldname, siz, found)
      found = .false.
      call mpp_get_info(unit,ndim,nvar,natt,ntime)
      if (nvar > max_fields) then
-        write(error_msg,'(I3,"/",I3)') nvar,max_fields 
+        write(error_msg,'(I3,"/",I3)') nvar,max_fields
         call  mpp_error(FATAL,'fms_io(field_size): max_fields too small, needs increasing, nvar/max_fields=' &
              //trim(error_msg))!//' in file '//trim(filename))
      endif
@@ -3157,63 +4752,60 @@ end subroutine get_size
 !   
 !=====================================================================================
 subroutine read_data_i3d_new(filename,fieldname,data,domain,timelevel, &
-                             no_domain,position, tile_count, is_compressed)
+                                        no_domain,position, tile_count)
   character(len=*),           intent(in)   :: filename, fieldname
-  integer, dimension(:,:,:), intent(inout) :: data ! 3 dimensional data    
+  integer, dimension(:,:,:), intent(inout) :: data ! 3 dimensional data
   type(domain2d), intent(in),   optional   :: domain
   integer, intent(in),          optional   :: timelevel
   logical, intent(in),          optional   :: no_domain
   integer, intent(in) ,         optional   :: position, tile_count
-  logical, intent(in),          optional   :: is_compressed
 
   real, dimension(size(data,1),size(data,2),size(data,3)) :: r_data
   r_data = 0
   call read_data_3d_new(filename,fieldname,r_data,domain,timelevel, &
-                        no_domain, .false., position, tile_count, is_compressed)
+                        no_domain, .false., position, tile_count)
   data = CEILING(r_data)
 end subroutine read_data_i3d_new
 
 subroutine read_data_i2d_new(filename,fieldname,data,domain,timelevel, &
-                             no_domain,position, tile_count, is_compressed)
+                             no_domain,position, tile_count)
   character(len=*),         intent(in)   :: filename, fieldname
-  integer, dimension(:,:), intent(inout) :: data ! 2 dimensional data    
+  integer, dimension(:,:), intent(inout) :: data ! 2 dimensional data
   type(domain2d), intent(in), optional   :: domain
   integer, intent(in),        optional   :: timelevel
   logical, intent(in),        optional   :: no_domain
   integer, intent(in) ,       optional   :: position, tile_count
-  logical, intent(in),        optional   :: is_compressed
   real, dimension(size(data,1),size(data,2)) :: r_data
 
   r_data = 0
   call read_data_2d_new(filename,fieldname,r_data,domain,timelevel, &
-                        no_domain, position, tile_count, is_compressed)
+                        no_domain, position, tile_count)
   data = CEILING(r_data)
 end subroutine read_data_i2d_new
 !.....................................................................
 subroutine read_data_i1d_new(filename,fieldname,data,domain,timelevel, &
-                             no_domain, tile_count, is_compressed)
+                             no_domain, tile_count)
   character(len=*), intent(in)           :: filename, fieldname
-  integer, dimension(:), intent(inout)   :: data ! 1 dimensional data    
+  integer, dimension(:), intent(inout)   :: data ! 1 dimensional data
   type(domain2d), intent(in), optional   :: domain
   integer, intent(in) , optional         :: timelevel
   logical, intent(in), optional          :: no_domain
   integer, intent(in), optional          :: tile_count
-  logical, intent(in), optional          :: is_compressed
 
   real, dimension(size(data,1))        :: r_data
 
   call read_data_1d_new(filename,fieldname,r_data,domain,timelevel, &
-                        no_domain, tile_count, is_compressed)
+                        no_domain, tile_count)
   data = CEILING(r_data)
 end subroutine read_data_i1d_new
 !.....................................................................
 subroutine read_data_iscalar_new(filename,fieldname,data,domain,timelevel, &
                                  no_domain, tile_count)
   character(len=*), intent(in)           :: filename, fieldname
-  integer, intent(inout)                 :: data     
+  integer, intent(inout)                 :: data
   type(domain2d), intent(in), optional   :: domain
   integer, intent(in) , optional         :: timelevel
-  logical, intent(in), optional          :: no_domain 
+  logical, intent(in), optional          :: no_domain
   integer, intent(in), optional          :: tile_count
 
   real                                 :: r_data
@@ -3223,15 +4815,14 @@ subroutine read_data_iscalar_new(filename,fieldname,data,domain,timelevel, &
 end subroutine read_data_iscalar_new
 !=====================================================================================
 subroutine read_data_3d_new(filename,fieldname,data,domain,timelevel, &
-      no_domain, scalar_or_1d, position, tile_count, is_compressed)
+                            no_domain, scalar_or_1d, position, tile_count)
   character(len=*),                  intent(in) :: filename, fieldname
-  real, dimension(:,:,:),         intent(inout) :: data ! 3 dimensional data    
+  real, dimension(:,:,:),         intent(inout) :: data ! 3 dimensional data
   type(domain2d), target, optional,  intent(in) :: domain
   integer,                optional,  intent(in) :: timelevel
-  logical,                optional,  intent(in) :: no_domain 
+  logical,                optional,  intent(in) :: no_domain
   logical,                optional,  intent(in) :: scalar_or_1d
   integer,                optional,  intent(in) :: position, tile_count
-  logical,                optional,  intent(in) :: is_compressed
 
   character(len=256)            :: fname
   integer                       :: unit, siz_in(4)
@@ -3244,7 +4835,6 @@ subroutine read_data_3d_new(filename,fieldname,data,domain,timelevel, &
   integer                       :: ishift, jshift
   logical                       :: is_scalar_or_1d = .false.
   logical                       :: is_no_domain = .false.
-  logical                       :: compressed 
   logical                       :: read_dist, io_domain_exist, found_file
   type(domain2d), pointer, save :: d_ptr =>NULL()
   type(domain2d), pointer, save :: io_domain =>NULL()
@@ -3252,19 +4842,16 @@ subroutine read_data_3d_new(filename,fieldname,data,domain,timelevel, &
 
 ! read disttributed files is used when reading restart files that are NOT mppnccombined. In this
 ! case PE 0 will read file_res.nc.0000, PE 1 will read file_res.nc.0001 and so forth.
-! 
-! namelist to be used with read_dist_files: threading_read=multi,
-! threading_write=multi, fileset_write=multi.
-
+!
 ! Initialize files to default values
-  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(read_data_3d_new):  module not initialized')  
+  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(read_data_3d_new):  module not initialized')
   is_no_domain = .false.
   if (PRESENT(no_domain)) THEN
      if(PRESENT(domain) .AND. no_domain) &
        call mpp_error(FATAL, 'fms_io(read_data_3d_new): no_domain cannot be .true. when optional argument domain is present.')
      is_no_domain = no_domain
-  endif  
- 
+  endif
+
   if(PRESENT(domain))then
      d_ptr => domain
   elseif (ASSOCIATED(Current_domain) .AND. .NOT. is_no_domain ) then
@@ -3274,18 +4861,15 @@ subroutine read_data_3d_new(filename,fieldname,data,domain,timelevel, &
   is_scalar_or_1d = .false.
   if(present(scalar_or_1d)) is_scalar_or_1d = scalar_or_1d
 
-  compressed = .false.
-  if(present(is_compressed)) compressed = is_compressed
-
   if(.not. PRESENT(domain) .and. .not. ASSOCIATED(Current_domain) ) is_no_domain = .true.
 
   found_file = get_file_name(filename, fname, read_dist, io_domain_exist, is_no_domain, domain,  tile_count)
   if(.not.found_file) call mpp_error(FATAL, 'fms_io_mod(read_data_3d_new): file ' //trim(filename)// &
-          '(with the consideration of tile number) and corresponding distributed file are not found')  
+          '(with the consideration of tile number) and corresponding distributed file are not found')
   call get_file_unit(fname, unit, file_index, read_dist, io_domain_exist, domain=domain)
 
   siz_in(3) = size(data,3)
-  if(is_no_domain .or. .NOT. associated(d_ptr) .or. is_scalar_or_1d .or. compressed) then
+  if(is_no_domain .or. .NOT. associated(d_ptr) .or. is_scalar_or_1d) then
      gxsize = size(data,1)
      gysize = size(data,2)
   else if(read_dist) then
@@ -3306,48 +4890,45 @@ subroutine read_data_3d_new(filename,fieldname,data,domain,timelevel, &
        call mpp_error(FATAL,'fms_io(read_data_3d_new): data should be on either computer domain '//&
                             'or data domain when domain is present. '//&
                             'shape(data)=',shape(data),'  cxsize,cysize,dxsize,dysize=',(/cxsize,cysize,dxsize,dysize/))
-     end if 
+     end if
   endif
 
   if (PRESENT(timelevel)) then
      tlev = timelevel
   else
      tlev = 1
-  endif  
-
-  if ((thread_r == MPP_MULTI).or.(mpp_pe()==mpp_root_pe())) then
-     call get_field_id(unit, file_index, fieldname, index_field, is_no_domain, .false. )
-     siz_in = files_read(file_index)%var(index_field)%siz
-     if(files_read(file_index)%var(index_field)%is_dimvar ) then
-        if (.not. read_dist) then
-           if (siz_in(1) /= gxsize) &
-                call mpp_error(FATAL,'fms_io(read_data_3d_new), field '//trim(fieldname)// &
-                ' in file '//trim(filename)//' field size mismatch 2')
-        endif
-     else
-        if (siz_in(1) /= gxsize .or. siz_in(2) /= gysize .or. siz_in(3) /= size(data,3)) then
-           PRINT *, gxsize, gysize, size(data, 3), siz_in(1), siz_in(2), siz_in(3)
-           call mpp_error(FATAL,'fms_io(read_data_3d_new), field '//trim(fieldname)// &
-                ' in file '//trim(filename)//': field size mismatch 1')
-        endif
-     end if
-     if ( tlev < 1 .or. files_read(file_index)%max_ntime < tlev)  then
-        write(error_msg,'(I5,"/",I5)') tlev, files_read(file_index)%max_ntime
-        call mpp_error(FATAL,'fms_io(read_data_3d_new): time level out of range, time level/max_time_level=' &
-             //trim(error_msg)//' in field/file: '//trim(fieldname)//'/'//trim(filename))
+  endif
+
+  call get_field_id(unit, file_index, fieldname, index_field, is_no_domain, .false. )
+  siz_in(1:4) = files_read(file_index)%var(index_field)%siz(1:4)
+  if(files_read(file_index)%var(index_field)%is_dimvar ) then
+     if (.not. read_dist) then
+        if (siz_in(1) /= gxsize) &
+             call mpp_error(FATAL,'fms_io(read_data_3d_new), field '//trim(fieldname)// &
+             ' in file '//trim(filename)//' field size mismatch 2')
+     endif
+  else
+     if (siz_in(1) /= gxsize .or. siz_in(2) /= gysize .or. siz_in(3) /= size(data,3)) then
+        PRINT *, gxsize, gysize, size(data, 3), siz_in(1), siz_in(2), siz_in(3)
+        call mpp_error(FATAL,'fms_io(read_data_3d_new), field '//trim(fieldname)// &
+             ' in file '//trim(filename)//': field size mismatch 1')
      endif
+  end if
+  if ( tlev < 1 .or. files_read(file_index)%max_ntime < tlev)  then
+     write(error_msg,'(I5,"/",I5)') tlev, files_read(file_index)%max_ntime
+     call mpp_error(FATAL,'fms_io(read_data_3d_new): time level out of range, time level/max_time_level=' &
+          //trim(error_msg)//' in field/file: '//trim(fieldname)//'/'//trim(filename))
+  endif
 
-     
-     if(is_no_domain .OR. is_scalar_or_1d .or. compressed) then
-        if (files_read(file_index)%var(index_field)%is_dimvar) then
-           call mpp_get_axis_data(files_read(file_index)%var(index_field)%axis,data(:,1,1))
-        else
-           call mpp_read(unit,files_read(file_index)%var(index_field)%field,data(:,:,:),tlev)
-        endif
-     else 
-        call mpp_read(unit,files_read(file_index)%var(index_field)%field,d_ptr,data,tlev,tile_count)
+  if(is_no_domain .OR. is_scalar_or_1d) then
+     if (files_read(file_index)%var(index_field)%is_dimvar) then
+        call mpp_get_axis_data(files_read(file_index)%var(index_field)%axis,data(:,1,1))
+     else
+        call mpp_read(unit,files_read(file_index)%var(index_field)%field,data(:,:,:),tlev)
      endif
-  endif  
+  else
+     call mpp_read(unit,files_read(file_index)%var(index_field)%field,d_ptr,data,tlev,tile_count)
+  endif
 
   d_ptr =>NULL()
 
@@ -3355,16 +4936,185 @@ subroutine read_data_3d_new(filename,fieldname,data,domain,timelevel, &
 end subroutine read_data_3d_new
 
 
+!=====================================================================================
+subroutine read_compressed_i1d(filename,fieldname,data,domain,timelevel,start,nread,threading)
+  character(len=*), intent(in)           :: filename, fieldname
+  integer, dimension(:), intent(inout)   :: data ! 1 dimensional data
+  type(domain2d), intent(in), optional   :: domain
+  integer, intent(in) , optional         :: timelevel
+  integer, intent(in) , optional         :: start(:), nread(:)
+  integer, intent(in) , optional         :: threading
+  real, dimension(size(data))        :: r_data
+
+  r_data = 0.0
+  call read_compressed_1d(filename,fieldname,r_data,domain,timelevel,start,nread,threading)
+  data = CEILING(r_data)
+end subroutine read_compressed_i1d
+!.....................................................................
+subroutine read_compressed_i2d(filename,fieldname,data,domain,timelevel,start,nread,threading)
+  character(len=*),         intent(in)   :: filename, fieldname
+  integer, dimension(:,:), intent(inout) :: data ! 2 dimensional data
+  type(domain2d), intent(in), optional   :: domain
+  integer, intent(in),        optional   :: timelevel
+  integer, intent(in) , optional         :: start(:), nread(:)
+  integer, intent(in) , optional         :: threading
+  real, dimension(size(data,1),size(data,2)) :: r_data
+
+  r_data = 0.0
+  call read_compressed_2d(filename,fieldname,r_data,domain,timelevel,start,nread,threading)
+  data = CEILING(r_data)
+end subroutine read_compressed_i2d
+!.....................................................................
+subroutine read_compressed_1d(filename,fieldname,data,domain,timelevel,start,nread,threading)
+  character(len=*), intent(in)           :: filename, fieldname
+  real, dimension(:), intent(inout)      :: data     !1 dimensional data
+  real, dimension(size(data,1),1)        :: data_2d
+  type(domain2d), intent(in), optional   :: domain
+  integer, intent(in) , optional         :: timelevel
+  integer, intent(in) , optional         :: start(:), nread(:)
+  integer, intent(in) , optional         :: threading
+#ifdef use_CRI_pointers
+  pointer( p, data_2d )
+  p = LOC(data)
+#endif
+  call read_compressed_2d(filename,fieldname,data_2d,domain,timelevel,start,nread,threading)
+end subroutine read_compressed_1d
+!.....................................................................
+subroutine read_compressed_2d(filename,fieldname,data,domain,timelevel,start,nread,threading)
+  character(len=*), intent(in)           :: filename, fieldname
+  real, dimension(:,:), intent(inout)    :: data     !2 dimensional data
+  type(domain2d), target, optional, intent(in) :: domain
+  integer, intent(in) , optional         :: timelevel
+  integer, intent(in) , optional         :: start(:), nread(:)
+  integer, intent(in) , optional         :: threading
+
+  character(len=256)            :: fname
+  integer                       :: unit, siz_in(4)
+  integer                       :: file_index  ! index of the opened file in array files
+  integer                       :: index_field ! position of the fieldname in the list of variables
+  logical                       :: read_dist, io_domain_exist, found_file
+  type(domain2d), pointer, save :: d_ptr =>NULL()
+  type(domain2d), pointer, save :: io_domain =>NULL()
+
+! Initialize files to default values
+  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(read_compressed_2d):  module not initialized')
+
+  if(PRESENT(domain))then
+     d_ptr => domain
+  elseif (ASSOCIATED(Current_domain)) then
+     d_ptr => Current_domain
+  else
+     call mpp_error(FATAL,'fms_io(read_compressed_2d): Domain must be an argument or set by set_domain()')
+  endif
+
+  found_file = get_file_name(filename, fname, read_dist, io_domain_exist, domain=d_ptr)
+  if(.not. found_file) then
+     found_file = get_file_name(filename, fname, read_dist, io_domain_exist, no_domain=.true. )
+  endif
+  if(.not.found_file) call mpp_error(FATAL, 'fms_io_mod(read_compressed_2d): file ' //trim(filename)// &
+          '(with the consideration of tile number) and corresponding distributed file are not found')
+  call get_file_unit(fname, unit, file_index, read_dist, io_domain_exist, domain=d_ptr)
+  call get_field_id(unit, file_index, fieldname, index_field, .false., .false. )
+
+  if (files_read(file_index)%var(index_field)%is_dimvar) then
+     call mpp_get_axis_data(files_read(file_index)%var(index_field)%axis,data(:,1))
+  else
+     call mpp_read_compressed(unit,files_read(file_index)%var(index_field)%field,d_ptr,data,timelevel,start,nread,threading)
+  endif
+  d_ptr =>NULL()
+end subroutine read_compressed_2d
+
+!.....................................................................
+subroutine read_distributed_a1D(unit,fmt,iostat,data)
+  integer, intent(in)               :: unit
+  character(*), intent(in)          :: fmt
+  integer, intent(out)              :: iostat
+  character(len=*), dimension(:), intent(inout) :: data
+
+  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(read_distributed_a1D):  module not initialized')
+  call mpp_read_distributed_ascii(unit,fmt,dr_set_size,data,iostat)
+end subroutine read_distributed_a1D
+
+!.....................................................................
+subroutine read_distributed_i1D(unit,fmt,iostat,data)
+  integer, intent(in)               :: unit
+  character(*), intent(in)          :: fmt
+  integer, intent(out)              :: iostat
+  integer, dimension(:), intent(inout) :: data
+
+  integer, allocatable :: pelist(:)
+  integer              :: i,lsize
+  logical              :: is_ioroot=.false.
+
+  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(read_distributed_i1D):  module not initialized')
+  call mpp_read_distributed_ascii(unit,fmt,dr_set_size,data,iostat)
+end subroutine read_distributed_i1D
+
+!.....................................................................
+subroutine read_distributed_iscalar(unit,fmt,iostat,data)
+  integer, intent(in)               :: unit
+  character(*), intent(in)          :: fmt
+  integer, intent(out)              :: iostat
+  integer, intent(inout) :: data
+
+  integer                           :: idata(1)
+  pointer(ptr,idata)
+
+  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(read_distributed_iscalar):  module not initialized')
+  ptr = LOC(data)
+  call read_distributed(unit,fmt,iostat,idata)
+end subroutine read_distributed_iscalar
+
+!.....................................................................
+subroutine read_distributed_r3D(unit,fmt,iostat,data)
+  integer, intent(in)               :: unit
+  character(*), intent(in)          :: fmt
+  integer, intent(out)              :: iostat
+  real, dimension(:,:,:), intent(inout) :: data
+
+  real :: data1D(size(data))
+  pointer(ptr,data1D)
+
+  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(read_distributed_r5D):  module not initialized')
+  ptr = LOC(data)
+  call read_distributed(unit,fmt,iostat,data1D)
+end subroutine read_distributed_r3D
+
+!.....................................................................
+subroutine read_distributed_r5D(unit,fmt,iostat,data)
+  integer, intent(in)               :: unit
+  character(*), intent(in)          :: fmt
+  integer, intent(out)              :: iostat
+  real, dimension(:,:,:,:,:), intent(inout) :: data
+
+  real :: data1D(size(data))
+  pointer(ptr,data1D)
+
+  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(read_distributed_r5D):  module not initialized')
+  ptr = LOC(data)
+  call read_distributed(unit,fmt,iostat,data1D)
+end subroutine read_distributed_r5D
+
+!.....................................................................
+subroutine read_distributed_r1D(unit,fmt,iostat,data)
+  integer, intent(in)               :: unit
+  character(*), intent(in)          :: fmt
+  integer, intent(out)              :: iostat
+  real, dimension(:), intent(inout) :: data
+
+  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(read_distributed_r1D):  module not initialized')
+  call mpp_read_distributed_ascii(unit,fmt,dr_set_size,data,iostat)
+end subroutine read_distributed_r1D
+
 !=====================================================================================
 subroutine read_data_2d_region(filename,fieldname,data,start,nread,domain, &
-                                 no_domain, tile_count, is_compressed)
+                                 no_domain, tile_count)
   character(len=*),                  intent(in) :: filename, fieldname
-  real, dimension(:,:),           intent(inout) :: data ! 3 dimensional data    
+  real, dimension(:,:),           intent(inout) :: data ! 3 dimensional data
   integer, dimension(:),             intent(in) :: start, nread
   type(domain2d), target,  optional, intent(in) :: domain
   logical,                 optional, intent(in) :: no_domain
   integer,                 optional, intent(in) :: tile_count
-  logical,                 optional, intent(in) :: is_compressed
   character(len=256)            :: fname
   integer                       :: unit, siz_in(4)
   integer                       :: file_index  ! index of the opened file in array files
@@ -3375,10 +5125,10 @@ subroutine read_data_2d_region(filename,fieldname,data,start,nread,domain, &
 
 
 ! Initialize files to default values
-  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(read_data_2d_region):  module not initialized')  
+  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(read_data_2d_region):  module not initialized')
   is_no_domain = .false.
   if (PRESENT(no_domain)) is_no_domain = no_domain
- 
+
   if(PRESENT(domain))then
      d_ptr => domain
   elseif (ASSOCIATED(Current_domain) .AND. .NOT. is_no_domain ) then
@@ -3389,18 +5139,16 @@ subroutine read_data_2d_region(filename,fieldname,data,start,nread,domain, &
 
   found_file = get_file_name(filename, fname, read_dist, io_domain_exist, is_no_domain, domain,  tile_count)
   if(.not.found_file) call mpp_error(FATAL, 'fms_io_mod(read_data_2d_region): file ' //trim(filename)// &
-          '(with the consideration of tile number) and corresponding distributed file are not found')  
+          '(with the consideration of tile number) and corresponding distributed file are not found')
   call get_file_unit(fname, unit, file_index, read_dist, io_domain_exist, domain=domain)
 
 
-  if ((thread_r == MPP_MULTI).or.(mpp_pe()==mpp_root_pe())) then
-     call get_field_id(unit, file_index, fieldname, index_field, is_no_domain, .false. )
-     siz_in = files_read(file_index)%var(index_field)%siz
-     if(files_read(file_index)%var(index_field)%is_dimvar) then
-        call mpp_error(FATAL, 'fms_io_mod(read_data_2d_region): the field should not be a dimension variable')
-     endif
-     call mpp_read(unit,files_read(file_index)%var(index_field)%field,data,start, nread)
-  endif  
+  call get_field_id(unit, file_index, fieldname, index_field, is_no_domain, .false. )
+  siz_in(1:4) = files_read(file_index)%var(index_field)%siz(1:4)
+  if(files_read(file_index)%var(index_field)%is_dimvar) then
+     call mpp_error(FATAL, 'fms_io_mod(read_data_2d_region): the field should not be a dimension variable')
+  endif
+  call mpp_read(unit,files_read(file_index)%var(index_field)%field,data,start, nread)
 
   d_ptr =>NULL()
 
@@ -3419,102 +5167,95 @@ subroutine read_data_text(filename,fieldname,data,level)
   character(len=256)             :: fname
 
 ! Initialize files to default values
-  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(read_data_text):  module not initialized')  
+  if(.not.module_is_initialized) call mpp_error(FATAL,'fms_io(read_data_text):  module not initialized')
 
   file_opened=.false.
   if (PRESENT(level)) then
      lev = level
   else
      lev = 1
-  endif  
+  endif
 
   found_file = get_file_name(filename, fname, read_dist, io_domain_exist, no_domain=.true. )
  if(.not.found_file) call mpp_error(FATAL, 'fms_io_mod(read_data_text): file ' //trim(filename)// &
-          '(with the consideration of tile number) and corresponding distributed file are not found')  
+          '(with the consideration of tile number) and corresponding distributed file are not found')
   call get_file_unit(fname, unit, file_index, read_dist, io_domain_exist )
 
-! Get info of this file and field   
-  if ((thread_r == MPP_MULTI).or.(mpp_pe()==mpp_root_pe())) then
-     call get_field_id(unit, file_index, fieldname, index_field, .true., .true. )     
+! Get info of this file and field
+  call get_field_id(unit, file_index, fieldname, index_field, .true., .true. )
 
-     if ( lev < 1 .or. lev > files_read(file_index)%var(index_field)%siz(1) )  then
-        write(error_msg,'(I5,"/",I5)') lev, files_read(file_index)%var(index_field)%siz(1)
-        call mpp_error(FATAL,'fms_io(read_data_text): text level out of range, level/max_level=' &
-             //trim(error_msg)//' in field/file: '//trim(fieldname)//'/'//trim(filename))
-     endif
+  if ( lev < 1 .or. lev > files_read(file_index)%var(index_field)%siz(1) )  then
+     write(error_msg,'(I5,"/",I5)') lev, files_read(file_index)%var(index_field)%siz(1)
+     call mpp_error(FATAL,'fms_io(read_data_text): text level out of range, level/max_level=' &
+          //trim(error_msg)//' in field/file: '//trim(fieldname)//'/'//trim(filename))
+  endif
 
-     call mpp_read(unit,files_read(file_index)%var(index_field)%field,data, level=level)
-  endif  
+  call mpp_read(unit,files_read(file_index)%var(index_field)%field,data, level=level)
   return
 end subroutine read_data_text
-!.............................................................. 
+!..............................................................
 ! 
 subroutine read_data_2d_new(filename,fieldname,data,domain,timelevel,&
-      no_domain,position,tile_count, is_compressed)
+                            no_domain,position,tile_count)
   character(len=*), intent(in)                 :: filename, fieldname
-  real, dimension(:,:), intent(inout)          :: data     !2 dimensional data 
+  real, dimension(:,:), intent(inout)          :: data     !2 dimensional data
   real, dimension(size(data,1),size(data,2),1) :: data_3d
   type(domain2d), intent(in), optional         :: domain
   integer, intent(in) , optional               :: timelevel
   logical, intent(in), optional                :: no_domain
   integer, intent(in) , optional               :: position, tile_count
-  logical, intent(in), optional                :: is_compressed
+
 
   integer                                      :: isc,iec,jsc,jec,isd,ied,jsd,jed
   integer                                      :: isg,ieg,jsg,jeg
   integer                                      :: xsize_c,ysize_c,xsize_d,ysize_d
   integer                                      :: xsize_g,ysize_g, ishift, jshift
-  logical                                      :: compressed
 
 !#ifdef use_CRI_pointers
 !  pointer( p, data_3d )
 !  p = LOC(data)
 !#endif
 
-  compressed= .false.
-  if (PRESENT(is_compressed)) compressed=is_compressed
-
   call read_data_3d_new(filename,fieldname,data_3d,domain,timelevel,&
-        no_domain,.false., position,tile_count,is_compressed=compressed)
-  
+                        no_domain,.false., position,tile_count)
+
   if(PRESENT(domain)) then
      call mpp_get_global_domain( domain,isg,ieg,jsg,jeg,xsize=xsize_g,ysize=ysize_g, tile_count=tile_count, position=position)
      call mpp_get_compute_domain( domain,isc,iec,jsc,jec,xsize=xsize_c,ysize=ysize_c, tile_count=tile_count, position=position)
      call mpp_get_data_domain( domain,isd,ied,jsd,jed,xsize=xsize_d,ysize=ysize_d, tile_count=tile_count, position=position)
      call mpp_get_domain_shift  (domain, ishift, jshift, position)
-     if(((size(data,1)==xsize_c) .and. (size(data,2)==ysize_c)) .or. compressed) then !on_comp_domain
+     if(((size(data,1)==xsize_c) .and. (size(data,2)==ysize_c))) then !on_comp_domain
         data(:,:) = data_3d(:,:,1)
      else if((size(data,1)==xsize_d) .and. (size(data,2)==ysize_d)) then !on_data_domain
         data(isc-isd+1:iec-isd+1,jsc-jsd+1:jec-jsd+1) = data_3d(isc-isd+1:iec-isd+1,jsc-jsd+1:jec-jsd+1,1)
      else if((size(data,1)==xsize_g) .and. (size(data,2)==ysize_g)) then !on_global_domain
         data(:,:) = data_3d(:,:,1)
-     else 
+     else
         call mpp_error(FATAL,'error in read_data_2d_new, field '//trim(fieldname)// &
                       ' in file '//trim(filename)//' data must be in compute or data domain')
      endif
-  else     
+  else
      data(:,:) = data_3d(:,:,1)
   endif
 
 end subroutine read_data_2d_new
 !.....................................................................
 subroutine read_data_1d_new(filename,fieldname,data,domain,timelevel,&
-      no_domain, tile_count, is_compressed)
+                                               no_domain, tile_count)
   character(len=*), intent(in)           :: filename, fieldname
-  real, dimension(:), intent(inout)      :: data     !1 dimensional data 
+  real, dimension(:), intent(inout)      :: data     !1 dimensional data
   real, dimension(size(data,1),1,1)      :: data_3d
   type(domain2d), intent(in), optional   :: domain
   integer, intent(in) , optional         :: timelevel
   logical, intent(in), optional          :: no_domain
   integer, intent(in), optional          :: tile_count
-  logical, intent(in), optional          :: is_compressed
 #ifdef use_CRI_pointers
   pointer( p, data_3d )
   p = LOC(data)
 #endif
 
   call read_data_3d_new(filename,fieldname,data_3d,domain,timelevel,&
-        no_domain=no_domain, scalar_or_1d=.true., tile_count=tile_count, is_compressed=is_compressed)
+        no_domain=no_domain, scalar_or_1d=.true., tile_count=tile_count)
 
 end subroutine read_data_1d_new
 !.....................................................................
@@ -3524,7 +5265,7 @@ subroutine read_data_scalar_new(filename,fieldname,data,domain,timelevel,&
 
 ! this subroutine is for reading a single number
   character(len=*), intent(in)           :: filename, fieldname
-  real, intent(inout)                    :: data     !zero dimension data 
+  real, intent(inout)                    :: data     !zero dimension data
   real, dimension(1,1,1)                 :: data_3d
   type(domain2d), intent(in), optional   :: domain
   integer, intent(in) , optional         :: timelevel
@@ -3554,13 +5295,14 @@ function unique_axes(file, index, id_axes, siz_axes, dom)
   type(var_type), pointer, save :: cur_var => NULL()
   integer :: i,j
   logical :: found
-  
+
   unique_axes=0
 
-  if(index <0 .OR. index > 3) call mpp_error(FATAL,"unique_axes(fms_io_mod): index should be 1, 2 or 3")
+  if(index <0 .OR. index > 4) call mpp_error(FATAL,"unique_axes(fms_io_mod): index should be 1, 2, 3 or 4")
 
   do i = 1, file%nvar
      cur_var => file%var(i)
+     if(cur_var%read_only) cycle
      if(cur_var%ndim < index) cycle
      found = .false.
      do j = 1, unique_axes
@@ -3579,7 +5321,7 @@ function unique_axes(file, index, id_axes, siz_axes, dom)
               found = .true.
               exit
            end if
-        end if  
+        end if
      end do
      if(found) then
         cur_var%id_axes(index) = j
@@ -3603,10 +5345,10 @@ function unique_axes(file, index, id_axes, siz_axes, dom)
         endif
         cur_var%id_axes(index) = unique_axes
      end if
-  end do       
+  end do
 
   cur_var => NULL()
-  
+
   return
 
 end function unique_axes
@@ -3619,7 +5361,7 @@ end function unique_axes
   !
   ! reading can be done either by all PEs (default) or by only the root PE
   ! this is controlled by namelist variable "read_all_pe".
-  
+
   ! By default, array data is expected to be declared in data domain and no_halo
   !is NOT needed, however IF data is decalared in COMPUTE domain then optional NO_HALO should be .true.
 
@@ -3629,12 +5371,12 @@ subroutine read_data_2d ( unit, data, end)
 
   integer, intent(in)                        :: unit
   real,    intent(out), dimension(isd:,jsd:) :: data
-  logical, intent(out), optional             :: end  
+  logical, intent(out), optional             :: end
   real, dimension(isg:ieg,jsg:jeg)           :: gdata
   integer                                    :: len
   logical                                    :: no_halo
 
-  include "read_data_2d.inc"  
+  include "read_data_2d.inc"
 end subroutine read_data_2d
 
 !#######################################################################
@@ -3643,7 +5385,7 @@ subroutine read_ldata_2d ( unit, data, end)
 
   integer, intent(in)                        :: unit
   logical, intent(out), dimension(isd:,jsd:) :: data
-  logical, intent(out), optional             :: end  
+  logical, intent(out), optional             :: end
   logical, dimension(isg:ieg,jsg:jeg)        :: gdata
   integer                                    :: len
   logical                                    :: no_halo
@@ -3668,7 +5410,7 @@ end subroutine read_idata_2d
 
 #ifdef OVERLOAD_C8
 subroutine read_cdata_2d ( unit, data, end)
-  
+
   integer, intent(in)                           :: unit
   complex,    intent(out), dimension(isd:,jsd:) :: data
   logical, intent(out), optional                :: end
@@ -3686,7 +5428,7 @@ subroutine read_data_3d ( unit, data, end)
 
   integer, intent(in)                           :: unit
   real,    intent(out), dimension(isd:,jsd:,:)  :: data
-  logical, intent(out), optional                :: end  
+  logical, intent(out), optional                :: end
   real, dimension(isg:ieg,jsg:jeg,size(data,3)) :: gdata
   integer                                       :: len
   logical                                       :: no_halo
@@ -3706,7 +5448,7 @@ subroutine read_cdata_3d ( unit, data, end)
   integer                                          :: len
   logical                                          :: no_halo
 
-  include "read_data_3d.inc"  
+  include "read_data_3d.inc"
 end subroutine read_cdata_3d
 #endif
 
@@ -3721,8 +5463,8 @@ subroutine read_data_4d ( unit, data, end)
   integer                                                    :: len
   logical                                                    :: no_halo
 ! WARNING: memory usage with this routine could be costly
-   
-  include "read_data_4d.inc"  
+
+  include "read_data_4d.inc"
 end subroutine read_data_4d
 
 !#######################################################################
@@ -3737,7 +5479,7 @@ subroutine read_cdata_4d ( unit, data, end)
   integer                                                       :: len
   logical                                                       :: no_halo
 ! WARNING: memory usage with this routine could be costly
- 
+
   include "read_data_4d.inc"
 end subroutine read_cdata_4d
 #endif
@@ -3749,7 +5491,7 @@ end subroutine read_cdata_4d
 !#######################################################################
 subroutine write_data_2d ( unit, data )
   integer, intent(in)                       :: unit
-  real,    intent(in), dimension(isd:,jsd:) :: data  
+  real,    intent(in), dimension(isd:,jsd:) :: data
   real, dimension(isg:ieg,jsg:jeg) :: gdata
 
   include "write_data.inc"
@@ -3796,7 +5538,7 @@ subroutine write_data_3d ( unit, data )
   integer, intent(in) :: unit
   real,    intent(in), dimension(isd:,jsd:,:) :: data
   real, dimension(isg:ieg,jsg:jeg,size(data,3)) :: gdata
-    
+
   include "write_data.inc"
 end subroutine write_data_3d
 
@@ -3860,7 +5602,7 @@ end subroutine write_cdata_4d
 
 subroutine read_eof (end_found)
   logical, intent(out), optional :: end_found
-  
+
   if (present(end_found))then
      end_found = .true.
   else
@@ -3893,14 +5635,14 @@ subroutine reset_field_name(fileObj, id_field, name)
 
   fileObj%var(id_field)%name = trim(name)
 
-end subroutine reset_field_name 
+end subroutine reset_field_name
 
 !#######################################################################
 
 subroutine reset_field_pointer_r0d(fileObj, id_field, data)
   type(restart_file_type), intent(inout)      :: fileObj
   integer,                 intent(in)         :: id_field
-  real,                    intent(in), target :: data   
+  real,                    intent(in), target :: data
 
   if (.not.associated(fileObj%var)) call mpp_error(FATAL, "fms_io(reset_field_pointer_r0d): " // &
       "restart_file_type data must be initialized by calling register_restart_field before using it")
@@ -3921,7 +5663,7 @@ end subroutine reset_field_pointer_r0d
 subroutine reset_field_pointer_r1d(fileObj, id_field, data)
   type(restart_file_type), intent(inout)      :: fileObj
   integer,                 intent(in)         :: id_field
-  real, dimension(:),      intent(in), target :: data   
+  real, dimension(:),      intent(in), target :: data
 
   if (.not.associated(fileObj%var)) call mpp_error(FATAL, "fms_io(reset_field_pointer_r1d): " // &
       "restart_file_type data must be initialized by calling register_restart_field before using it")
@@ -3942,7 +5684,7 @@ end subroutine reset_field_pointer_r1d
 subroutine reset_field_pointer_r2d(fileObj, id_field, data)
   type(restart_file_type), intent(inout)      :: fileObj
   integer,                 intent(in)         :: id_field
-  real, dimension(:,:),    intent(in), target :: data   
+  real, dimension(:,:),    intent(in), target :: data
 
   if (.not.associated(fileObj%var)) call mpp_error(FATAL, "fms_io(reset_field_pointer_r2d): " // &
       "restart_file_type data must be initialized by calling register_restart_field before using it")
@@ -3963,7 +5705,7 @@ end subroutine reset_field_pointer_r2d
 subroutine reset_field_pointer_r3d(fileObj, id_field, data)
   type(restart_file_type), intent(inout)      :: fileObj
   integer,                 intent(in)         :: id_field
-  real, dimension(:,:,:),  intent(in), target :: data   
+  real, dimension(:,:,:),  intent(in), target :: data
 
   if (.not.associated(fileObj%var)) call mpp_error(FATAL, "fms_io(reset_field_pointer_r3d): " // &
       "restart_file_type data must be initialized by calling register_restart_field before using it")
@@ -3979,12 +5721,34 @@ subroutine reset_field_pointer_r3d(fileObj, id_field, data)
 
 end subroutine reset_field_pointer_r3d
 
+!#######################################################################
+
+subroutine reset_field_pointer_r4d(fileObj, id_field, data)
+  type(restart_file_type),   intent(inout)      :: fileObj
+  integer,                   intent(in)         :: id_field
+  real, dimension(:,:,:,:),  intent(in), target :: data
+
+  if (.not.associated(fileObj%var)) call mpp_error(FATAL, "fms_io(reset_field_pointer_r4d): " // &
+      "restart_file_type data must be initialized by calling register_restart_field before using it")
+
+  if(id_field < 0 .OR. id_field > fileObj%nvar) call mpp_error(FATAL, &
+         "fms_io(reset_field_pointer_r4d): id_field should be positive integer and "// &
+         "no larger than number of fields in the file "//trim(fileObj%name) )
+  if(fileObj%var(id_field)%siz(4) .NE. 1) call mpp_error(FATAL, &
+         "fms_io(reset_field_pointer_r4d): one-level reset_field_pointer is called, but "//&
+         "field "//trim(fileObj%var(id_field)%name)//" of file "//trim(fileObj%name)//" is not one level" )
+
+  fileObj%p4dr(1, id_field)%p => data
+
+end subroutine reset_field_pointer_r4d
+
+
 !#######################################################################
 
 subroutine reset_field_pointer_i0d(fileObj, id_field, data)
   type(restart_file_type), intent(inout)      :: fileObj
   integer,                 intent(in)         :: id_field
-  integer,                 intent(in), target :: data   
+  integer,                 intent(in), target :: data
 
   if (.not.associated(fileObj%var)) call mpp_error(FATAL, "fms_io(reset_field_pointer_i0d): " // &
       "restart_file_type data must be initialized by calling register_restart_field before using it")
@@ -4005,7 +5769,7 @@ end subroutine reset_field_pointer_i0d
 subroutine reset_field_pointer_i1d(fileObj, id_field, data)
   type(restart_file_type), intent(inout)      :: fileObj
   integer,                 intent(in)         :: id_field
-  integer, dimension(:),   intent(in), target :: data   
+  integer, dimension(:),   intent(in), target :: data
 
   if (.not.associated(fileObj%var)) call mpp_error(FATAL, "fms_io(reset_field_pointer_i1d): " // &
       "restart_file_type data must be initialized by calling register_restart_field before using it")
@@ -4026,7 +5790,7 @@ end subroutine reset_field_pointer_i1d
 subroutine reset_field_pointer_i2d(fileObj, id_field, data)
   type(restart_file_type), intent(inout)      :: fileObj
   integer,                 intent(in)         :: id_field
-  integer, dimension(:,:), intent(in), target :: data   
+  integer, dimension(:,:), intent(in), target :: data
 
   if (.not.associated(fileObj%var)) call mpp_error(FATAL, "fms_io(reset_field_pointer_i2d): " // &
       "restart_file_type data must be initialized by calling register_restart_field before using it")
@@ -4047,7 +5811,7 @@ end subroutine reset_field_pointer_i2d
 subroutine reset_field_pointer_i3d(fileObj, id_field, data)
   type(restart_file_type),   intent(inout)      :: fileObj
   integer,                   intent(in)         :: id_field
-  integer, dimension(:,:,:), intent(in), target :: data   
+  integer, dimension(:,:,:), intent(in), target :: data
 
   if (.not.associated(fileObj%var)) call mpp_error(FATAL, "fms_io(reset_field_pointer_i3d): " // &
       "restart_file_type data must be initialized by calling register_restart_field before using it")
@@ -4068,7 +5832,7 @@ end subroutine reset_field_pointer_i3d
 subroutine reset_field_pointer_r0d_2level(fileObj, id_field, data1, data2)
   type(restart_file_type), intent(inout)      :: fileObj
   integer,                 intent(in)         :: id_field
-  real,                    intent(in), target :: data1, data2   
+  real,                    intent(in), target :: data1, data2
 
   if (.not.associated(fileObj%var)) call mpp_error(FATAL, "fms_io(reset_field_pointer_r0d_2level): " // &
       "restart_file_type data must be initialized by calling register_restart_field before using it")
@@ -4090,7 +5854,7 @@ end subroutine reset_field_pointer_r0d_2level
 subroutine reset_field_pointer_r1d_2level(fileObj, id_field, data1, data2)
   type(restart_file_type), intent(inout)      :: fileObj
   integer,                 intent(in)         :: id_field
-  real, dimension(:),      intent(in), target :: data1, data2   
+  real, dimension(:),      intent(in), target :: data1, data2
 
   if (.not.associated(fileObj%var)) call mpp_error(FATAL, "fms_io(reset_field_pointer_r1d_2level): " // &
       "restart_file_type data must be initialized by calling register_restart_field before using it")
@@ -4112,7 +5876,7 @@ end subroutine reset_field_pointer_r1d_2level
 subroutine reset_field_pointer_r2d_2level(fileObj, id_field, data1, data2)
   type(restart_file_type), intent(inout)      :: fileObj
   integer,                 intent(in)         :: id_field
-  real, dimension(:,:),    intent(in), target :: data1, data2   
+  real, dimension(:,:),    intent(in), target :: data1, data2
 
   if (.not.associated(fileObj%var)) call mpp_error(FATAL, "fms_io(reset_field_pointer_r2d_2level): " // &
       "restart_file_type data must be initialized by calling register_restart_field before using it")
@@ -4134,7 +5898,7 @@ end subroutine reset_field_pointer_r2d_2level
 subroutine reset_field_pointer_r3d_2level(fileObj, id_field, data1, data2)
   type(restart_file_type), intent(inout)      :: fileObj
   integer,                 intent(in)         :: id_field
-  real, dimension(:,:,:),  intent(in), target :: data1, data2   
+  real, dimension(:,:,:),  intent(in), target :: data1, data2
 
   if (.not.associated(fileObj%var)) call mpp_error(FATAL, "fms_io(reset_field_pointer_r3d_2level): " // &
       "restart_file_type data must be initialized by calling register_restart_field before using it")
@@ -4156,7 +5920,7 @@ end subroutine reset_field_pointer_r3d_2level
 subroutine reset_field_pointer_i0d_2level(fileObj, id_field, data1, data2)
   type(restart_file_type), intent(inout)      :: fileObj
   integer,                 intent(in)         :: id_field
-  integer,                 intent(in), target :: data1, data2   
+  integer,                 intent(in), target :: data1, data2
 
   if (.not.associated(fileObj%var)) call mpp_error(FATAL, "fms_io(reset_field_pointer_i0d_2level): " // &
       "restart_file_type data must be initialized by calling register_restart_field before using it")
@@ -4178,7 +5942,7 @@ end subroutine reset_field_pointer_i0d_2level
 subroutine reset_field_pointer_i1d_2level(fileObj, id_field, data1, data2)
   type(restart_file_type), intent(inout)      :: fileObj
   integer,                 intent(in)         :: id_field
-  integer, dimension(:),   intent(in), target :: data1, data2   
+  integer, dimension(:),   intent(in), target :: data1, data2
 
   if (.not.associated(fileObj%var)) call mpp_error(FATAL, "fms_io(reset_field_pointer_i1d_2level): " // &
       "restart_file_type data must be initialized by calling register_restart_field before using it")
@@ -4200,7 +5964,7 @@ end subroutine reset_field_pointer_i1d_2level
 subroutine reset_field_pointer_i2d_2level(fileObj, id_field, data1, data2)
   type(restart_file_type), intent(inout)      :: fileObj
   integer,                 intent(in)         :: id_field
-  integer, dimension(:,:), intent(in), target :: data1, data2   
+  integer, dimension(:,:), intent(in), target :: data1, data2
 
   if (.not.associated(fileObj%var)) call mpp_error(FATAL, "fms_io(reset_field_pointer_i2d_2level): " // &
       "restart_file_type data must be initialized by calling register_restart_field before using it")
@@ -4222,7 +5986,7 @@ end subroutine reset_field_pointer_i2d_2level
 subroutine reset_field_pointer_i3d_2level(fileObj, id_field, data1, data2)
   type(restart_file_type),   intent(inout)      :: fileObj
   integer,                   intent(in)         :: id_field
-  integer, dimension(:,:,:), intent(in), target :: data1, data2   
+  integer, dimension(:,:,:), intent(in), target :: data1, data2
 
   if (.not.associated(fileObj%var)) call mpp_error(FATAL, "fms_io(reset_field_pointer_i3d_2level): " // &
       "restart_file_type data must be initialized by calling register_restart_field before using it")
@@ -4241,7 +6005,7 @@ end subroutine reset_field_pointer_i3d_2level
 
 !#########################################################################
 !   This function returns .true. if the field referred to by id has
-! initialized from a restart file, and .false. otherwise. 
+! initialized from a restart file, and .false. otherwise.
 !
 ! Arguments: id - A integer that is the index of the field in fileObj.
 !  (in)  fileObj - The control structure returned by a previous call to
@@ -4266,7 +6030,7 @@ end function query_initialized_id
 
 !#########################################################################
 !   This function returns .true. if the field referred to by name has
-! initialized from a restart file, and .false. otherwise. 
+! initialized from a restart file, and .false. otherwise.
 !
 ! Arguments: name - A pointer to the field that is being queried.
 !  (in)  fileObj - The control structure returned by a previous call to
@@ -4309,7 +6073,7 @@ end function query_initialized_name
 !  (in)      CS - The control structure returned by a previous call to
 !                 restart_init.
 function query_initialized_r2d(fileObj, f_ptr, name)
-  type(restart_file_type),   intent(inout) :: fileObj 
+  type(restart_file_type),   intent(inout) :: fileObj
   real, dimension(:,:), target, intent(in) :: f_ptr
   character(len=*),             intent(in) :: name
 
@@ -4331,14 +6095,57 @@ function query_initialized_r2d(fileObj, f_ptr, name)
   if (m>fileObj%nvar) then
      if (mpp_pe() == mpp_root_pe() ) call mpp_error(NOTE, "fms_io(query_initialized_r2d): Unable to find "// &
           trim(name)//" queried by pointer, "//"probably because of the suspect comparison of pointers by ASSOCIATED.")
-     query_initialized_r2d = query_initialized_name(fileObj, name)
-     if (mpp_pe() == mpp_root_pe() .AND. query_initialized_r2d) call mpp_error(NOTE, &
-          "fms_io(query_initialized_r2d): "//trim(name)// " initialization confirmed by name.")
+     query_initialized_r2d = query_initialized_name(fileObj, name)
+     if (mpp_pe() == mpp_root_pe() .AND. query_initialized_r2d) call mpp_error(NOTE, &
+          "fms_io(query_initialized_r2d): "//trim(name)// " initialization confirmed by name.")
+  endif
+
+  return
+
+end function query_initialized_r2d
+
+!#########################################################################
+!   This function returns 1 if the field pointed to by f_ptr has
+! initialized from a restart file, and 0 otherwise.  If f_ptr is
+! NULL, it tests whether the entire restart file has been success-
+! fully read.
+!
+! Arguments: f_ptr - A pointer to the field that is being queried.
+!  (in)      name - The name of the field that is being queried.
+!  (in)      CS - The control structure returned by a previous call to
+!                 restart_init.
+function query_initialized_r3d(fileObj, f_ptr, name)
+  type(restart_file_type),     intent(inout) :: fileObj
+  real, dimension(:,:,:), target, intent(in) :: f_ptr
+  character(len=*),               intent(in) :: name
+
+  logical :: query_initialized_r3d
+  integer :: m
+
+  if (.not.associated(fileObj%var)) call mpp_error(FATAL, "fms_io(query_initialized_r3d): " // &
+      "restart_file_type data must be initialized by calling register_restart_field before using it")
+
+  query_initialized_r3d = .false.
+  do m=1, fileObj%nvar
+     if (ASSOCIATED(fileObj%p3dr(1,m)%p,f_ptr)) then
+        if (fileObj%var(m)%initialized) query_initialized_r3d = .true.
+        exit
+     endif
+  enddo
+  ! Assume that you are going to initialize it now, so set flag to initialized if
+  ! queried again.
+  if (m>fileObj%nvar) then
+     if (mpp_pe() == mpp_root_pe() ) call mpp_error(NOTE, "fms_io(query_initialized_r3d): Unable to find "// &
+          trim(name)//" queried by pointer, "//"probably because of the suspect comparison of pointers by ASSOCIATED.")
+     query_initialized_r3d = query_initialized_name(fileObj, name)
+     if (mpp_pe() == mpp_root_pe() .AND. query_initialized_r3d) call mpp_error(NOTE, &
+          "fms_io(query_initialized_r3d): "//trim(name)// " initialization confirmed by name.")
   endif
 
   return
 
-end function query_initialized_r2d
+end function query_initialized_r3d
+
 
 !#########################################################################
 !   This function returns 1 if the field pointed to by f_ptr has
@@ -4350,41 +6157,40 @@ end function query_initialized_r2d
 !  (in)      name - The name of the field that is being queried.
 !  (in)      CS - The control structure returned by a previous call to
 !                 restart_init.
-function query_initialized_r3d(fileObj, f_ptr, name)
-  type(restart_file_type),     intent(inout) :: fileObj 
-  real, dimension(:,:,:), target, intent(in) :: f_ptr
-  character(len=*),               intent(in) :: name
+function query_initialized_r4d(fileObj, f_ptr, name)
+  type(restart_file_type),       intent(inout) :: fileObj
+  real, dimension(:,:,:,:), target, intent(in) :: f_ptr
+  character(len=*),                 intent(in) :: name
 
-  logical :: query_initialized_r3d
+  logical :: query_initialized_r4d
   integer :: m
 
-  if (.not.associated(fileObj%var)) call mpp_error(FATAL, "fms_io(query_initialized_r3d): " // &
+  if (.not.associated(fileObj%var)) call mpp_error(FATAL, "fms_io(query_initialized_r4d): " // &
       "restart_file_type data must be initialized by calling register_restart_field before using it")
 
-  query_initialized_r3d = .false.
+  query_initialized_r4d = .false.
   do m=1, fileObj%nvar
-     if (ASSOCIATED(fileObj%p3dr(1,m)%p,f_ptr)) then
-        if (fileObj%var(m)%initialized) query_initialized_r3d = .true.
+     if (ASSOCIATED(fileObj%p4dr(1,m)%p,f_ptr)) then
+        if (fileObj%var(m)%initialized) query_initialized_r4d = .true.
         exit
      endif
   enddo
   ! Assume that you are going to initialize it now, so set flag to initialized if
   ! queried again.
   if (m>fileObj%nvar) then
-     if (mpp_pe() == mpp_root_pe() ) call mpp_error(NOTE, "fms_io(query_initialized_r3d): Unable to find "// &
+     if (mpp_pe() == mpp_root_pe() ) call mpp_error(NOTE, "fms_io(query_initialized_r4d): Unable to find "// &
           trim(name)//" queried by pointer, "//"probably because of the suspect comparison of pointers by ASSOCIATED.")
-     query_initialized_r3d = query_initialized_name(fileObj, name)
-     if (mpp_pe() == mpp_root_pe() .AND. query_initialized_r3d) call mpp_error(NOTE, &
-          "fms_io(query_initialized_r3d): "//trim(name)// " initialization confirmed by name.")
+     query_initialized_r4d = query_initialized_name(fileObj, name)
+     if (mpp_pe() == mpp_root_pe() .AND. query_initialized_r4d) call mpp_error(NOTE, &
+          "fms_io(query_initialized_r4d): "//trim(name)// " initialization confirmed by name.")
   endif
 
   return
 
-end function query_initialized_r3d
-
+end function query_initialized_r4d
 
 !#########################################################################
-!   This function sets that a variable has been initialized for future queries. 
+!   This function sets that a variable has been initialized for future queries.
 !
 ! Arguments: name - A pointer to the field whose initialization status is being set.
 !  (in)  fileObj - The control structure returned by a previous call to
@@ -4412,7 +6218,7 @@ subroutine set_initialized_id(fileObj, id, is_set)
 end subroutine set_initialized_id
 
 !#########################################################################
-!   This function sets that a variable has been initialized for future queries. 
+!   This function sets that a variable has been initialized for future queries.
 !
 ! Arguments: name - A pointer to the field whose initialization status is being set.
 !  (in)  fileObj - The control structure returned by a previous call to
@@ -4446,7 +6252,7 @@ subroutine set_initialized_name(fileObj, name, is_set)
 end subroutine set_initialized_name
 
 !#########################################################################
-!   This function sets that a variable has been initialized for future queries. 
+!   This function sets that a variable has been initialized for future queries.
 !
 ! Arguments: name - A pointer to the field whose initialization status is being set.
 !  (in)  fileObj - The control structure returned by a previous call to
@@ -4493,7 +6299,7 @@ subroutine set_initialized_r2d(fileObj, f_ptr, name, is_set)
 end subroutine set_initialized_r2d
 
 !#########################################################################
-!   This function sets that a variable has been initialized for future queries. 
+!   This function sets that a variable has been initialized for future queries.
 !
 ! Arguments: name - A pointer to the field whose initialization status is being set.
 !  (in)  fileObj - The control structure returned by a previous call to
@@ -4540,13 +6346,60 @@ subroutine set_initialized_r3d(fileObj, f_ptr, name, is_set)
 end subroutine set_initialized_r3d
 
 
+!#########################################################################
+!   This function sets that a variable has been initialized for future queries.
+!
+! Arguments: name - A pointer to the field whose initialization status is being set.
+!  (in)  fileObj - The control structure returned by a previous call to
+!                  register_restart_field
+subroutine set_initialized_r4d(fileObj, f_ptr, name, is_set)
+  type(restart_file_type),       intent(inout) :: fileObj
+  real, dimension(:,:,:,:), target, intent(in) :: f_ptr
+  character(len=*),                 intent(in) :: name
+  logical,          optional,       intent(in) :: is_set
+  logical :: set_val
+  integer :: m
+
+  set_val = .true.
+  if (present(is_set)) set_val = is_set
+
+  if (.not.associated(fileObj%var)) call mpp_error(FATAL, "fms_io(set_initialized_r4d): " // &
+      "restart_file_type data must be initialized by calling set_restart_field before using it")
+
+  do m=1, fileObj%nvar
+     if (ASSOCIATED(fileObj%p4dr(1,m)%p,f_ptr)) then
+        fileObj%var(m)%initialized = set_val
+        return
+     endif
+  enddo
+
+  if (m>fileObj%nvar .AND. mpp_pe() == mpp_root_pe() ) then
+    call mpp_error(NOTE,"fms_io(set_initialized_r4d): Unable to find "// &
+          trim(name)//" queried by pointer, "//"probably because of the suspect comparison of pointers by ASSOCIATED"//&
+                        " when attempting to set initialization.")
+  end if
+
+  do m=1,fileObj%nvar
+    if (trim(name) == fileObj%var(m)%name) then
+      fileObj%var(m)%initialized = set_val
+      return
+    endif
+  enddo
+
+  if (m>fileObj%nvar .AND. mpp_pe() == mpp_root_pe() ) then
+    call mpp_error(NOTE,"fms_io(set_initialized_r4d): Unknown restart variable "//name// &
+                        " attempted to set initialization.")
+  end if
+
+end subroutine set_initialized_r4d
+
 !#######################################################################
 !#######################################################################
 !
 ! routines for opening specific types of files:
 !
-!                       form        action 
-! open_namelist_file  MPP_ASCII   MPP_RDONLY  
+!                       form        action
+! open_namelist_file  MPP_ASCII   MPP_RDONLY
 ! open restart_file   MPP_NATIVE
 ! open_ieee32_file    MPP_IEEE32
 !
@@ -4581,14 +6434,14 @@ function open_namelist_file (file) result (unit)
   if(show_open_namelist_file_warning) call mpp_error(WARNING, "fms_io_mod: open_namelist_file should not be called when INTERNAL_FILE_NML is defined")
 #endif
 
-  if (.not.module_is_initialized) call fms_io_init ( )    
+  if (.not.module_is_initialized) call fms_io_init ( )
   if (present(file)) then
      call mpp_open ( unit, file, form=MPP_ASCII, action=MPP_RDONLY, &
           access=MPP_SEQUENTIAL, threading=MPP_SINGLE )
   else
 !  the following code is necessary for using alternate namelist files (nests, stretched grids, etc)
      pelist_name = mpp_get_current_pelist_name()
-     if ( file_exist('input_'//trim(pelist_name)//'.nml') ) then
+     if ( file_exist('input_'//trim(pelist_name)//'.nml', no_domain=.true.) ) then
         filename='input_'//trim(pelist_name)//'.nml'
      else
         filename='input.nml'
@@ -4600,7 +6453,7 @@ end function open_namelist_file
 !     
 
 ! 
-!    
+!   
 ! Opens single restart file for reading by all PEs or
 ! writing by root PE only
 ! the file has native format and no mpp header records.
@@ -4616,7 +6469,7 @@ end function open_namelist_file
 ! 
 function open_restart_file (file, action) result (unit)
   character(len=*), intent(in) :: file, action
-  integer :: unit  
+  integer :: unit
   integer :: mpp_action
 
   if (.not.module_is_initialized) call fms_io_init ( )
@@ -4631,7 +6484,7 @@ function open_restart_file (file, action) result (unit)
   case default
      call mpp_error(FATAL,'fms_io(open_restart_file): action should be either read or write in file'//trim(file))
   end select
-  
+
   call mpp_open ( unit, file, form=MPP_NATIVE, action=mpp_action, &
        access=MPP_SEQUENTIAL, threading=MPP_SINGLE, nohdrs=.true. )
 
@@ -4640,7 +6493,7 @@ end function open_restart_file
 
 
 ! 
-!    
+!   
 ! Opens single direct access file for reading by all PEs or
 ! writing by root PE only
 ! the file has native format and no mpp header records.
@@ -4673,8 +6526,8 @@ end function open_direct_file
 ! 
 
 ! 
-!     
-! Opens single 32-bit ieee file for reading by all PEs or 
+!   
+! Opens single 32-bit ieee file for reading by all PEs or
 ! writing by root PE only (writing is not recommended)
 ! the file has no mpp header records.
 !   
@@ -4689,7 +6542,7 @@ end function open_direct_file
 ! 
 function open_ieee32_file (file, action) result (unit)
   character(len=*), intent(in) :: file, action
-  integer :: unit  
+  integer :: unit
   integer :: mpp_action
 
   if (.not.module_is_initialized) call fms_io_init ( )
@@ -4703,7 +6556,7 @@ function open_ieee32_file (file, action) result (unit)
   case default
      call mpp_error (FATAL,'fms_io(open_ieee32_file): action should be either read or write in file'//trim(file))
   end select
-  
+
   if (iospec_ieee32(1:1) == ' ') then
      call mpp_open ( unit, file, form=MPP_IEEE32, action=mpp_action, &
           access=MPP_SEQUENTIAL, threading=MPP_SINGLE,    &
@@ -4729,12 +6582,20 @@ end function open_ieee32_file
 ! action to be performed: can be 'delete'
 ! 
 
-subroutine close_file (unit, status)
+subroutine close_file (unit, status, dist)
   integer,          intent(in)           :: unit
   character(len=*), intent(in), optional :: status
-  
+  logical,          intent(in), optional :: dist
+
   if (.not.module_is_initialized) call fms_io_init ( )
-  if (unit == stdlog()) return    
+  if(PRESENT(dist))then
+    ! If distributed, return if not I/O root
+    if(dist)then
+      if(.not. mpp_is_dist_ioroot(dr_set_size)) return
+    endif
+  endif
+
+  if (unit == stdlog()) return
   if (present(status)) then
      if (lowercase(trim(status)) == 'delete') then
         call mpp_close (unit, action=MPP_DELETE)
@@ -4748,7 +6609,7 @@ end subroutine close_file
 ! 
 
 !#######################################################################
-  
+
 
 ! 
 !   
@@ -4760,7 +6621,7 @@ end subroutine close_file
 ! this Domain2
 ! 
 subroutine set_domain (Domain2)
-    
+
   type(domain2D), intent(in), target :: Domain2
 
   if (.NOT.module_is_initialized) call fms_io_init ( )
@@ -4768,9 +6629,9 @@ subroutine set_domain (Domain2)
 !  --- set_domain must be called before a read_data or write_data ---
   if (associated(Current_domain)) nullify (Current_domain)
   Current_domain => Domain2
-  
+
   !  --- module indexing to shorten read/write routines ---
-  
+
   call mpp_get_compute_domain (Current_domain,is ,ie ,js ,je )
   call mpp_get_data_domain    (Current_domain,isd,ied,jsd,jed)
   call mpp_get_global_domain  (Current_domain,isg,ieg,jsg,jeg)
@@ -4796,7 +6657,7 @@ end subroutine nullify_domain
 
 ! 
 !   
-! This routine is the reverse of set_domain above. This routine is called when 
+! This routine is the reverse of set_domain above. This routine is called when
 ! users want to retrieve the domain2d that is used in fms_io_mod
 !   
 ! 
@@ -4808,7 +6669,7 @@ subroutine return_domain(domain2)
   if (associated(Current_domain)) then
      domain2 = Current_domain
   else
-     domain2 = NULL_DOMAIN2D       
+     domain2 = NULL_DOMAIN2D
   endif
 end subroutine return_domain
 ! 
@@ -4832,7 +6693,7 @@ end subroutine return_domain
 subroutine get_domain_decomp ( x, y )
 
   integer, intent(out), dimension(4) :: x, y
-  
+
   if (mpp_pe() == mpp_root_pe())  call mpp_error(NOTE, &
        'subroutine get_domain_decomp will be removed with the next release')
   x = (/ isg, ieg, is, ie /)
@@ -4841,7 +6702,7 @@ subroutine get_domain_decomp ( x, y )
 end subroutine get_domain_decomp
 ! 
 
-subroutine get_axis_cart(axis, cart)      
+subroutine get_axis_cart(axis, cart)
 
   type(axistype), intent(in) :: axis
   character(len=1), intent(out) :: cart
@@ -4862,9 +6723,9 @@ subroutine get_axis_cart(axis, cart)
   lon_units = (/'degrees_e   ', 'degrees_east'/)
   lat_units = (/'degrees_n    ', 'degrees_north'/)
   z_units = (/'cm ','m  ','pa ','hpa'/)
-  t_units = (/'sec', 'min','hou','day'/)  
+  t_units = (/'sec', 'min','hou','day'/)
   call mpp_get_atts(axis,cartesian=axis_cart)
-  cart = 'N'  
+  cart = 'N'
   if (axis_cart == 'x' ) cart = 'X'
   if (axis_cart == 'y' ) cart = 'Y'
   if (axis_cart == 'z' ) cart = 'Z'
@@ -4902,22 +6763,21 @@ subroutine get_axis_cart(axis, cart)
         if (name(1:3) == trim(t_units(i))) cart = 'T'
      enddo
   end if
-  
+
   return
 end subroutine get_axis_cart
 
-
 ! The following function is here as a last resort.
-! This is copied from what was utilities_mod in order that redundant code 
+! This is copied from what was utilities_mod in order that redundant code
 ! could be deleted.
 
- function open_file ( file, form, action, access, threading, recl ) &
-             result ( unit )
+function open_file(file, form, action, access, threading, recl, dist) result(unit)
 
- character(len=*), intent(in) :: file 
+ character(len=*), intent(in) :: file
  character(len=*), intent(in), optional :: form, action, access, threading
- integer         , intent(in), optional :: recl 
- integer  :: unit 
+ integer         , intent(in), optional :: recl
+ logical         , intent(in), optional :: dist  ! Distributed open?
+ integer  :: unit
 
  character(len=32) :: form_local, action_local, access_local, thread_local
  character(len=32) :: action_ieee32
@@ -4925,9 +6785,23 @@ function open_file ( file, form, action, access, threading, recl ) &
  integer :: mpp_format, mpp_action, mpp_access, mpp_thread
 !-----------------------------------------------------------------------
 
-   if ( .not. module_is_initialized ) then
-        call fms_io_init ( )
-!        do_init = .false.
+   if ( .not. module_is_initialized ) call fms_io_init ( )
+
+   if (present(action)) then    ! must be present
+      action_local = action
+   else
+      call mpp_error (FATAL, 'open_file in fms_mod : argument action not present')
+   endif
+
+   unit = 0  ! Initialize return value. Note that mpp_open will call mpi_abort on error
+   if(PRESENT(dist))then
+     if(lowercase(trim(action_local)) /= 'read') &
+       call mpp_error(FATAL,'open_file in fms_mod: distributed'//lowercase(trim(action_local))// &
+                              ' not currently supported')
+     ! If distributed, return if not I/O root
+     if(dist) then
+       if(.not. mpp_is_dist_ioroot(dr_set_size)) return
+     endif
    endif
 
 !   ---- return stdlog if this is the logfile ----
@@ -4937,7 +6811,7 @@ function open_file ( file, form, action, access, threading, recl ) &
        return
     endif
 
-!   ---- is this file open and connected to a unit ?? ---- 
+!   ---- is this file open and connected to a unit ?? ----
 
    inquire (file=trim(file), opened=open, number=unit)
 
@@ -4947,7 +6821,7 @@ function open_file ( file, form, action, access, threading, recl ) &
    if ( open .and. unit >= 0 ) then
       call mpp_error (FATAL, 'open_file in fms_mod : '// &
                        'file '//trim(file)//' is already open')
-   endif   
+   endif
 
 !  --- defaults ---
 
@@ -4957,13 +6831,6 @@ function open_file ( file, form, action, access, threading, recl ) &
    no_headers   = .true.
    do_ieee32    = .false.
 
-   if (present(action)) then    ! must be present
-      action_local = action
-   else
-      call mpp_error (FATAL, 'open_file in fms_mod : argument action not present')
-   endif
-
-
 !   --- file format ---
 
     select case (lowercase(trim(form_local)))
@@ -5028,7 +6895,7 @@ function open_file ( file, form, action, access, threading, recl ) &
     if ( .not.do_ieee32 ) then
        call mpp_open ( unit, file, form=mpp_format, action=mpp_action, &
                        access=mpp_access, threading=mpp_thread,        &
-                       nohdrs=no_headers, recl=recl )
+                       fileset=MPP_SINGLE,nohdrs=no_headers, recl=recl )
     else
      ! special open for ieee32 file
      ! fms_mod has iospec value
@@ -5112,22 +6979,13 @@ subroutine get_mosaic_tile_file(file_in, file_out, is_no_domain, domain, tile_co
     character(len=*), intent(out)                  :: file_out
     logical,          intent(in)                   :: is_no_domain
     type(domain2D),   intent(in), optional, target :: domain
-    integer,          intent(in), optional         :: tile_count 
+    integer,          intent(in), optional         :: tile_count
     character(len=256)                             :: basefile, tilename
     integer                                        :: lens, ntiles, ntileMe, tile, my_tile_id
     integer, dimension(:), allocatable             :: tile_id
     type(domain2d), pointer, save                  :: d_ptr =>NULL()
     logical                                        :: domain_exist
 
-    !--- deal with the situation that the file is alreday in the full name.
-    lens = len_trim(file_in)
-    if(lens > 8) then
-       if(file_in(lens-7:lens) == '.nc'//trim(pe_name) ) then
-         file_out = file_in
-         return
-        endif
-    endif
-
     if(index(file_in, '.nc', back=.true.)==0) then
        basefile = trim(file_in)
     else
@@ -5137,39 +6995,37 @@ subroutine get_mosaic_tile_file(file_in, file_out, is_no_domain, domain, tile_co
        basefile = file_in(1:lens-3)
     end if
 
-    if(mpp_mosaic_defined())then
-       !--- get the tile name
-       ntiles = 1
-       my_tile_id = 1
-       domain_exist = .false.
-       if(PRESENT(domain))then
-          domain_exist = .true.
-          ntiles = mpp_get_ntile_count(domain)
-          d_ptr => domain
-       elseif (ASSOCIATED(Current_domain) .AND. .NOT. is_no_domain ) then
-          domain_exist = .true.
-          ntiles = mpp_get_ntile_count(Current_domain)
-          d_ptr => Current_domain
-       endif
-       
-       if(domain_exist) then
-          ntileMe = mpp_get_current_ntile(d_ptr)
-          allocate(tile_id(ntileMe))
-          tile_id = mpp_get_tile_id(d_ptr)
-          tile = 1
-          if(present(tile_count)) tile = tile_count
-          my_tile_id = tile_id(tile)
-       endif
+    !--- get the tile name
+    ntiles = 1
+    my_tile_id = 1
+    domain_exist = .false.
+    if(PRESENT(domain))then
+       domain_exist = .true.
+       ntiles = mpp_get_ntile_count(domain)
+       d_ptr => domain
+    elseif (ASSOCIATED(Current_domain) .AND. .NOT. is_no_domain ) then
+       domain_exist = .true.
+       ntiles = mpp_get_ntile_count(Current_domain)
+       d_ptr => Current_domain
+    endif
 
-       if(ntiles > 1 .or. my_tile_id > 1 )then
-          tilename = 'tile'//string(my_tile_id)
-          if(index(basefile,'.'//trim(tilename),back=.true.) == 0)then
-             basefile = trim(basefile)//'.'//trim(tilename);
-          end if
-       end if
-       if(allocated(tile_id)) deallocate(tile_id)
+    if(domain_exist) then
+       ntileMe = mpp_get_current_ntile(d_ptr)
+       allocate(tile_id(ntileMe))
+       tile_id = mpp_get_tile_id(d_ptr)
+       tile = 1
+       if(present(tile_count)) tile = tile_count
+       my_tile_id = tile_id(tile)
     endif
 
+    if(ntiles > 1 .or. my_tile_id > 1 )then
+       tilename = 'tile'//string(my_tile_id)
+       if(index(basefile,'.'//trim(tilename),back=.true.) == 0)then
+          basefile = trim(basefile)//'.'//trim(tilename);
+       end if
+    end if
+    if(allocated(tile_id)) deallocate(tile_id)
+
     file_out = trim(basefile)//'.nc'
 
     d_ptr =>NULL()
@@ -5177,11 +7033,11 @@ subroutine get_mosaic_tile_file(file_in, file_out, is_no_domain, domain, tile_co
   end subroutine get_mosaic_tile_file
 
   !#############################################################################
-  subroutine get_mosaic_tile_grid(grid_file, mosaic_file, domain, tile_count) 
+  subroutine get_mosaic_tile_grid(grid_file, mosaic_file, domain, tile_count)
     character(len=*), intent(out)          :: grid_file
     character(len=*), intent(in)           :: mosaic_file
     type(domain2D),   intent(in)           :: domain
-    integer,          intent(in), optional :: tile_count 
+    integer,          intent(in), optional :: tile_count
     integer                                :: tile, ntileMe
     integer, dimension(:), allocatable     :: tile_id
 
@@ -5189,7 +7045,7 @@ subroutine get_mosaic_tile_grid(grid_file, mosaic_file, domain, tile_count)
     if(present(tile_count)) tile = tile_count
     ntileMe = mpp_get_current_ntile(domain)
     allocate(tile_id(ntileMe))
-    tile_id = mpp_get_tile_id(domain)     
+    tile_id = mpp_get_tile_id(domain)
     call read_data(mosaic_file, "gridfiles", grid_file, level=tile_id(tile) )
     grid_file = 'INPUT/'//trim(grid_file)
     deallocate(tile_id)
@@ -5206,7 +7062,7 @@ subroutine get_var_att_value_text(file, varname, attname, attvalue)
     call mpp_open(unit,trim(file),MPP_RDONLY,MPP_NETCDF,threading=MPP_MULTI,fileset=MPP_SINGLE)
     call mpp_get_att_value(unit, varname, attname, attvalue)
     call mpp_close(unit)
- 
+
     return
 
   end subroutine get_var_att_value_text
@@ -5278,7 +7134,7 @@ function get_file_name(orig_file, actual_file, read_dist, io_domain_exist, no_do
     logical,                         intent(out) :: io_domain_exist
     logical,                optional, intent(in) :: no_domain
     type(domain2D), target, optional, intent(in) :: domain
-    integer,                optional, intent(in) :: tile_count  
+    integer,                optional, intent(in) :: tile_count
     logical                                      :: get_file_name
 
     type(domain2d), pointer, save :: d_ptr, io_domain
@@ -5289,11 +7145,6 @@ function get_file_name(orig_file, actual_file, read_dist, io_domain_exist, no_do
     is_no_domain=.false.
     if(PRESENT(no_domain)) is_no_domain = no_domain
 
-    if(present(domain)) then
-       d_ptr => domain
-    elseif (ASSOCIATED(Current_domain) .AND. .NOT. is_no_domain ) then
-       d_ptr => Current_domain
-    endif
 
     fexist          = .false.
     read_dist       = .false.
@@ -5308,9 +7159,16 @@ function get_file_name(orig_file, actual_file, read_dist, io_domain_exist, no_do
           get_file_name = .true.
           return
        endif
-    endif 
-    
-    !JWD:  This is likely a temporary fix. Since fms_io needs to know tile_count, 
+    endif
+
+    if(present(domain)) then
+       d_ptr => domain
+    elseif (ASSOCIATED(Current_domain) .AND. .NOT. is_no_domain ) then
+       d_ptr => Current_domain
+    endif
+
+
+    !JWD:  This is likely a temporary fix. Since fms_io needs to know tile_count,
     !JWD:  I just don't see how the physics can remain "tile neutral"
     call get_mosaic_tile_file(orig_file, actual_file, is_no_domain, domain, tile_count)
 
@@ -5318,7 +7176,7 @@ function get_file_name(orig_file, actual_file, read_dist, io_domain_exist, no_do
     if(ASSOCIATED(d_ptr)) then
        io_domain => mpp_get_io_domain(d_ptr)
        if(associated(io_domain)) then
-          tile_id = mpp_get_tile_id(io_domain)       
+          tile_id = mpp_get_tile_id(io_domain)
           write(fname, '(a,i4.4)' ) trim(actual_file)//'.', tile_id(1)
           inquire (file=trim(fname), exist=fexist)
           if(.not. fexist) then
@@ -5326,11 +7184,9 @@ function get_file_name(orig_file, actual_file, read_dist, io_domain_exist, no_do
              inquire (file=trim(fname), exist=fexist)
           endif
           if(fexist) io_domain_exist = .true.
-       endif   
+       endif
        io_domain=>NULL()
-    endif 
-
-    if(.not. fexist) inquire (file=trim(actual_file)//trim(pe_name), exist=fexist)
+    endif
 
     if(fexist) then
        read_dist = .true.
@@ -5344,59 +7200,62 @@ function get_file_name(orig_file, actual_file, read_dist, io_domain_exist, no_do
        d_ptr => NULL()
        get_file_name = .true.
        return
-    endif    
+    endif
 
     !Perhaps the file has an ensemble instance appendix
-    call get_instance_filename(orig_file, actual_file)
-    if(index(orig_file, '.nc', back=.true.) == 0) then
-       inquire (file=trim(actual_file), exist=fexist)
+    if(len_trim(filename_appendix) > 0) then
+       call get_instance_filename(orig_file, actual_file)
+       if(index(orig_file, '.nc', back=.true.) == 0) then
+          inquire (file=trim(actual_file), exist=fexist)
+          if(fexist) then
+             d_ptr => NULL()
+             get_file_name = .true.
+             return
+          endif
+       endif
+
+       call get_mosaic_tile_file(actual_file, actual_file, is_no_domain, domain, tile_count)
+       !--- check if the file is group redistribution.
+       if(ASSOCIATED(d_ptr)) then
+          io_domain => mpp_get_io_domain(d_ptr)
+          if(associated(io_domain)) then
+             tile_id = mpp_get_tile_id(io_domain)
+             if(mpp_npes()>10000) then
+                write(fname, '(a,i6.6)' ) trim(actual_file)//'.', tile_id(1)
+             else
+                write(fname, '(a,i4.4)' ) trim(actual_file)//'.', tile_id(1)
+             endif
+             inquire (file=trim(fname), exist=fexist)
+             if(fexist) io_domain_exist = .true.
+          endif
+          io_domain=>NULL()
+       endif
+
        if(fexist) then
+          read_dist = .true.
+          d_ptr => NULL()
           get_file_name = .true.
           return
        endif
-    endif 
 
-    call get_mosaic_tile_file(actual_file, actual_file, is_no_domain, domain, tile_count)
-    !--- check if the file is group redistribution.
-    if(ASSOCIATED(d_ptr)) then
-       io_domain => mpp_get_io_domain(d_ptr)
-       if(associated(io_domain)) then
-          tile_id = mpp_get_tile_id(io_domain)       
-          if(mpp_npes()>10000) then
-             write(fname, '(a,i6.6)' ) trim(actual_file)//'.', tile_id(1)
-          else
-             write(fname, '(a,i4.4)' ) trim(actual_file)//'.', tile_id(1)
-          endif
-          inquire (file=trim(fname), exist=fexist)
-          if(fexist) io_domain_exist = .true.
-       endif   
-       io_domain=>NULL()
-    endif 
+       inquire (file=trim(actual_file), exist=fexist)
 
-    if(.not. fexist) inquire (file=trim(actual_file)//trim(pe_name), exist=fexist)  
-    if(fexist) then
-       read_dist = .true. 
-       d_ptr => NULL()
-       get_file_name = .true.
-       return
-    endif    
-    inquire (file=trim(actual_file), exist=fexist)
-    
-    if(fexist) then
-       d_ptr => NULL()
-       get_file_name = .true.
-       return
+       if(fexist) then
+          d_ptr => NULL()
+          get_file_name = .true.
+          return
+       endif
     endif
 
   end function get_file_name
-  
+
 
   !#############################################################################
   subroutine get_file_unit(filename, unit, index_file, read_dist, io_domain_exist, domain )
     character(len=*),         intent(in) :: filename
-    integer,                 intent(out) :: unit, index_file 
+    integer,                 intent(out) :: unit, index_file
     logical,                  intent(in) :: read_dist, io_domain_exist
-    type(domain2d), optional, intent(in) :: domain    
+    type(domain2d), optional, intent(in) :: domain
 
     logical  :: file_opened
     integer  :: i
@@ -5407,19 +7266,15 @@ subroutine get_file_unit(filename, unit, index_file, read_dist, io_domain_exist,
        if (files_read(i)%name == trim(filename))  then
           index_file = i
           unit = files_read(index_file)%unit
-          return 
+          return
        endif
     enddo
 
     ! need to open the file now
-    ! Increase num_files_r and set file_type 
+    ! Increase num_files_r and set file_type
     if(num_files_r == max_files_r) &  ! need to have bigger max_files_r
          call mpp_error(FATAL,'fms_io(get_file_unit): max_files_r exceeded, increase it via fms_io_nml')
-    num_files_r=num_files_r + 1 
-    if (read_dist .and. thread_r == MPP_SINGLE) then
-       call mpp_error(FATAL,'fms_io(get_file_unit): single-threaded read from distributed fileset not allowed' &
-            //'change threading_read to MULTI')
-    endif
+    num_files_r=num_files_r + 1
     if(read_dist) then
        if(io_domain_exist) then
           if(present(domain)) then
@@ -5433,18 +7288,18 @@ subroutine get_file_unit(filename, unit, index_file, read_dist, io_domain_exist,
                    'either domain is present or current_domain is associated')
           endif
        else
-          call mpp_open(unit,trim(filename),form=form,action=MPP_RDONLY,threading=thread_r, &
+          call mpp_open(unit,trim(filename),form=form,action=MPP_RDONLY,threading=MPP_MULTI, &
             fileset=MPP_MULTI)
        endif
     else
-       call mpp_open(unit,trim(filename),form=form,action=MPP_RDONLY,threading=thread_r, &
+       call mpp_open(unit,trim(filename),form=form,action=MPP_RDONLY,threading=MPP_MULTI, &
             fileset=MPP_SINGLE)
     end if
     files_read(num_files_r)%name = trim(filename)
     allocate(files_read(num_files_r)%var (max_fields) )
     files_read(num_files_r)%nvar = 0
     index_file = num_files_r
-    files_read(index_file)%unit = unit   
+    files_read(index_file)%unit = unit
 
   end subroutine get_file_unit
 
@@ -5471,10 +7326,10 @@ subroutine get_field_id(unit, index_file, fieldname, index_field, is_no_domain,
        endif
     enddo
 
-    !--- fieldname is not read, so need to get fieldname from file 
+    !--- fieldname is not read, so need to get fieldname from file
     files_read(index_file)%nvar = files_read(index_file)%nvar + 1
     if(files_read(index_file)%nvar > max_fields) then
-       write(error_msg,'(I3,"/",I3)') files_read(index_file)%nvar, max_fields 
+       write(error_msg,'(I3,"/",I3)') files_read(index_file)%nvar, max_fields
        call  mpp_error(FATAL,'fms_io(get_field_id): max_fields exceeded, needs increasing, nvar/max_fields=' &
             //trim(error_msg))
     endif
@@ -5485,13 +7340,15 @@ subroutine get_field_id(unit, index_file, fieldname, index_field, is_no_domain,
        call mpp_error(FATAL,'fms_io(get_field_id): max_fields too small needs increasing,nvar/max_fields=' &
             //trim(error_msg)//'in file'//trim(files_read(index_file)%name))
     endif
-    call mpp_get_fields(unit, fields(1:nvar))     
+    call mpp_get_fields(unit, fields(1:nvar))
     siz_in = 1
     index_field = files_read(index_file)%nvar
     files_read(index_file)%var(index_field)%is_dimvar = .false.
 
     do i=1, nvar
        call mpp_get_atts(fields(i),name=name,ndim=var_dim,siz=siz_in)
+       if(var_dim .GT. 4) call mpp_error(FATAL, 'fms_io(get_field_id): number of dimension of field '// &
+                trim(name)//' in file '//trim(files_read(index_file)%name)//' should not be greater than 4')
        if (lowercase(trim(name)) == lowercase(trim(fieldname))) then ! found the variable
           if(var_dim .lt.3) then
              do j=var_dim+1,3
@@ -5500,8 +7357,8 @@ subroutine get_field_id(unit, index_file, fieldname, index_field, is_no_domain,
           endif
           files_read(index_file)%var(index_field)%name    = fieldname
           files_read(index_file)%var(index_field)%field   = fields(i)
-          files_read(index_file)%var(index_field)%siz(:)  = siz_in
-          files_read(index_file)%var(index_field)%gsiz(:) = siz_in
+          files_read(index_file)%var(index_field)%siz(1:4)  = siz_in(1:4)
+          files_read(index_file)%var(index_field)%gsiz(1:3) = siz_in(1:3)
           return
        endif
     enddo
@@ -5509,21 +7366,21 @@ subroutine get_field_id(unit, index_file, fieldname, index_field, is_no_domain,
     !--- the fieldname may be a dimension variable.
     if( .not. is_not_dim) then
        if (ndim > max_axes) then
-          write(error_msg,'(I3,"/",I3)') ndim, max_axes 
+          write(error_msg,'(I3,"/",I3)') ndim, max_axes
           call  mpp_error(FATAL,'fms_io(get_field_id): max_axes exceeded, needs increasing, ndim/max_fields=' &
-               //trim(error_msg)//' in file '//trim(files_read(index_file)%name))             
+               //trim(error_msg)//' in file '//trim(files_read(index_file)%name))
        endif
        call mpp_get_axes(unit, axes(1:ndim))
        do i=1,ndim
-          call mpp_get_atts(axes(i), name=name, len = siz_in(1)) 
+          call mpp_get_atts(axes(i), name=name, len = siz_in(1))
           if (lowercase(trim(name)) == lowercase(trim(fieldname))) then
 !             if(.not. is_no_domain) call mpp_error(FATAL, &
 !                  'fms_io(get_field_id): the field is a dimension variable, no_domain should be true.')
              files_read(index_file)%var(index_field)%is_dimvar = .true.
              files_read(index_file)%var(index_field)%name      = fieldname
              files_read(index_file)%var(index_field)%axis      = axes(i)
-             files_read(index_file)%var(index_field)%siz(:)    = siz_in
-             files_read(index_file)%var(index_field)%gsiz(:)   = siz_in
+             files_read(index_file)%var(index_field)%siz(1:4)    = siz_in(1:4)
+             files_read(index_file)%var(index_field)%gsiz(1:3)   = siz_in(1:3)
              return
           endif
        enddo
@@ -5556,7 +7413,7 @@ end subroutine get_field_id
 !     A file name (or path name) that is checked for existence.
 !   
 !   
-!     This function returns a logical result.  If file_name exists the result 
+!     This function returns a logical result.  If file_name exists the result
 !     is true, otherwise false is returned.
 !     If the length of character string "file_name" is zero or the first
 !     character is blank, then the returned value will be false.
@@ -5564,14 +7421,14 @@ end subroutine get_field_id
 !     routine open_file.
 !   
 !   
-!     Before calling write_data you must first call set_domain with domain2d data 
+!     Before calling write_data you must first call set_domain with domain2d data
 !     type associated with the distributed data you are writing.
 !   
 
  function file_exist (file_name, domain, no_domain)
   character(len=*), intent(in)         :: file_name
   type(domain2d), intent(in), optional :: domain
-  logical,        intent(iN), optional :: no_domain  
+  logical,        intent(iN), optional :: no_domain
 
   logical                              :: file_exist, is_no_domain
   character(len=256)                   :: fname
@@ -5582,10 +7439,10 @@ function file_exist (file_name, domain, no_domain)
    !--- to deal with mosaic file, in this case, the file is assumed to be in netcdf format
    file_exist = get_file_name(file_name, fname, read_dist, io_domain_exist, no_domain=is_no_domain, domain=domain)
    if(is_no_domain) return
-   if(.not.file_exist) file_exist=get_file_name(file_name, fname, read_dist, io_domain_exist, no_domain=.true.)   
+   if(.not.file_exist) file_exist=get_file_name(file_name, fname, read_dist, io_domain_exist, no_domain=.true.)
 
    return
- 
+
  end function file_exist
 ! 
 
@@ -5594,10 +7451,10 @@ end function file_exist
 ! 
 
 !   
-!     check if a given field name exists in a given file name. 
+!     check if a given field name exists in a given file name.
 !   
 !   
-!     check if a given field name exists in a given file name. 
+!     check if a given field name exists in a given file name.
 !     If the field_name string has zero length or the
 !     first character is blank return a false result.
 !     if the file file_name don't exist, return a false result.
@@ -5613,7 +7470,7 @@ end function file_exist
 !     A field name that is checked for existence.
 !   
 !   
-!     This function returns a logical result.  If field exists in the 
+!     This function returns a logical result.  If field exists in the
 !     file file_name, the result is true, otherwise false is returned.
 !     If the length of character string "field_name" is zero or the first
 !     character is blank, then the returned value will be false.
@@ -5637,7 +7494,7 @@ function field_exist (file_name, field_name, domain, no_domain)
    if (field_name(1:1) == ' ')    return
 
    is_no_domain = .false.
-   if(present(no_domain)) is_no_domain = no_domain   
+   if(present(no_domain)) is_no_domain = no_domain
 
    file_exist=get_file_name(file_name, fname, read_dist, io_domain_exist, no_domain=is_no_domain, domain=domain)
    if(file_exist) then
@@ -5673,17 +7530,25 @@ end function field_exist
 
 subroutine set_filename_appendix(string_in)
   character(len=*) , intent(in) :: string_in
-  filename_appendix = trim(string_in)
+
+  integer :: index_num
+
+  ! Check if string has already been added
+  index_num = index(filename_appendix, string_in)
+  if ( index_num .le. 0 ) then
+     filename_appendix = trim(filename_appendix)//trim(string_in)
+  end if
+
 end subroutine set_filename_appendix
 
 subroutine get_instance_filename(name_in,name_out)
   character(len=*)  , intent(in)  :: name_in
   character(len=*), intent(inout) :: name_out
   integer :: length
-  
+
   length = len_trim(name_in)
   name_out = name_in(1:length)
-  
+
   if(len_trim(filename_appendix) > 0) then
      if(name_in(length-2:length) == '.nc') then
         name_out = name_in(1:length-3)//'.'//trim(filename_appendix)//'.nc'
@@ -5691,12 +7556,11 @@ subroutine get_instance_filename(name_in,name_out)
         name_out = name_in(1:length)  //'.'//trim(filename_appendix)
      end if
   end if
-  
-end subroutine get_instance_filename
 
+end subroutine get_instance_filename
 
 !#######################################################################
-subroutine parse_mask_table(mask_table, maskmap, modelname)
+subroutine parse_mask_table_2d(mask_table, maskmap, modelname)
 
   character(len=*), intent(in) :: mask_table
   logical,         intent(out) :: maskmap(:,:)
@@ -5713,21 +7577,21 @@ subroutine parse_mask_table(mask_table, maskmap, modelname)
      call mpp_open(unit, mask_table, action=MPP_RDONLY)
      read(unit, FMT=*, IOSTAT=mystat) nmask
      if( mystat /= 0 ) call mpp_error(FATAL, &
-          "fms_io(parse_mask_table): Error reading nmask from file " //trim(mask_table))
+          "fms_io(parse_mask_table_2d): Error reading nmask from file " //trim(mask_table))
      write(stdoutunit,*)"parse_mask_table: Number of domain regions masked in ", trim(modelname), " = ", nmask
      if( nmask > 0 ) then
         !--- read layout from mask_table and confirm it matches the shape of maskmap
         read(unit, FMT=*, IOSTAT=mystat) layout
         if( mystat /= 0 ) call mpp_error(FATAL, &
-             "fms_io(parse_mask_talbe): Error reading layout from file " //trim(mask_table))
+             "fms_io(parse_mask_talbe_2d): Error reading layout from file " //trim(mask_table))
         if( (layout(1) .NE. size(maskmap,1)) .OR. (layout(2) .NE. size(maskmap,2)) )then
            write(stdoutunit,*)"layout=", layout, ", size(maskmap) = ", size(maskmap,1), size(maskmap,2)
-           call mpp_error(FATAL, "fms_io(parse_mask_table): layout in file "//trim(mask_table)// &
+           call mpp_error(FATAL, "fms_io(parse_mask_table_2d): layout in file "//trim(mask_table)// &
                   "does not match size of maskmap for "//trim(modelname))
         endif
         !--- make sure mpp_npes() == layout(1)*layout(2) - nmask
         if( mpp_npes() .NE. layout(1)*layout(2) - nmask ) call mpp_error(FATAL, &
-           "fms_io(parse_mask_table): mpp_npes() .NE. layout(1)*layout(2) - nmask for "//trim(modelname))
+           "fms_io(parse_mask_table_2d): mpp_npes() .NE. layout(1)*layout(2) - nmask for "//trim(modelname))
       endif
    endif
 
@@ -5748,31 +7612,121 @@ subroutine parse_mask_table(mask_table, maskmap, modelname)
         if (record(1:10) == '          ') cycle
         n = n + 1
         if( n > nmask ) then
-           call mpp_error(FATAL, "fms_io(parse_mask_table): number of mask_list entry "// &
-                "is greater than nmask in file "//trim(mask_table) ) 
+           call mpp_error(FATAL, "fms_io(parse_mask_table_2d): number of mask_list entry "// &
+                "is greater than nmask in file "//trim(mask_table) )
         endif
-        read(record,*,err=888) mask_list(n,1), mask_list(n,2) 
+        read(record,*,err=888) mask_list(n,1), mask_list(n,2)
      enddo
-888  call mpp_error(FATAL, "fms_io(parse_mask_table):  Error in reading mask_list from file "//trim(mask_table))    
+888  call mpp_error(FATAL, "fms_io(parse_mask_table_2d):  Error in reading mask_list from file "//trim(mask_table))
 
 999  continue
      !--- make sure the number of entry for mask_list is nmask
      if( n .NE. nmask) call mpp_error(FATAL, &
-        "fms_io(parse_mask_table): number of mask_list entry does not match nmask in file "//trim(mask_table)) 
+        "fms_io(parse_mask_table_2d): number of mask_list entry does not match nmask in file "//trim(mask_table))
      call mpp_close(unit)
   endif
-  
+
   call mpp_broadcast(mask_list, 2*nmask, mpp_root_pe())
   do n = 1, nmask
      if(debug_mask_list) then
-       write(stdoutunit,*) "==>NOTE from parse_mask_table: ", trim(modelname), " mask_list = ", mask_list(n,1), mask_list(n,2)
+       write(stdoutunit,*) "==>NOTE from parse_mask_table_2d: ", trim(modelname), " mask_list = ", mask_list(n,1), mask_list(n,2)
      endif
      maskmap(mask_list(n,1),mask_list(n,2)) = .false.
   enddo
 
   deallocate(mask_list)
 
-end subroutine parse_mask_table
+end subroutine parse_mask_table_2d
+
+
+!#######################################################################
+subroutine parse_mask_table_3d(mask_table, maskmap, modelname)
+
+  character(len=*), intent(in) :: mask_table
+  logical,         intent(out) :: maskmap(:,:,:)
+  character(len=*), intent(in) :: modelname
+  integer                      :: nmask, layout(2)
+  integer, allocatable         :: mask_list(:,:)
+  integer                      :: unit, mystat, n, stdoutunit, ntiles
+  character(len=128)           :: record
+
+  maskmap = .true.
+  nmask = 0
+  stdoutunit = stdout()
+  if( mpp_pe() == mpp_root_pe() ) then
+     call mpp_open(unit, mask_table, action=MPP_RDONLY)
+     read(unit, FMT=*, IOSTAT=mystat) nmask
+     if( mystat /= 0 ) call mpp_error(FATAL, &
+          "fms_io(parse_mask_table_3d): Error reading nmask from file " //trim(mask_table))
+     write(stdoutunit,*)"parse_mask_table: Number of domain regions masked in ", trim(modelname), " = ", nmask
+     if( nmask > 0 ) then
+        !--- read layout from mask_table and confirm it matches the shape of maskmap
+        read(unit, FMT=*, IOSTAT=mystat) layout(1), layout(2), ntiles
+        if( mystat /= 0 ) call mpp_error(FATAL, &
+             "fms_io(parse_mask_talbe_3d): Error reading layout from file " //trim(mask_table))
+        if( (layout(1) .NE. size(maskmap,1)) .OR. (layout(2) .NE. size(maskmap,2)) )then
+           write(stdoutunit,*)"layout=", layout, ", size(maskmap) = ", size(maskmap,1), size(maskmap,2)
+           call mpp_error(FATAL, "fms_io(parse_mask_table_3d): layout in file "//trim(mask_table)// &
+                  "does not match size of maskmap for "//trim(modelname))
+        endif
+        if( ntiles .NE. size(maskmap,3) ) then
+           write(stdoutunit,*)"ntiles=", ntiles, ", size(maskmap,3) = ", size(maskmap,3)
+           call mpp_error(FATAL, "fms_io(parse_mask_table_3d): ntiles in file "//trim(mask_table)// &
+                  "does not match size of maskmap for "//trim(modelname))
+        endif
+        !--- make sure mpp_npes() == layout(1)*layout(2) - nmask
+        if( mpp_npes() .NE. layout(1)*layout(2)*ntiles - nmask ) then
+           print*, "layout=", layout, nmask, mpp_npes()
+           call mpp_error(FATAL, &
+              "fms_io(parse_mask_table_3d): mpp_npes() .NE. layout(1)*layout(2) - nmask for "//trim(modelname))
+        endif
+      endif
+   endif
+
+   call mpp_broadcast(nmask, mpp_root_pe())
+
+   if(nmask==0) then
+      if( mpp_pe() == mpp_root_pe() ) call mpp_close(unit)
+      return
+   endif
+
+   allocate(mask_list(nmask,3))
+
+   if( mpp_pe() == mpp_root_pe() ) then
+     n = 0
+     do while( .true. )
+        read(unit,'(a)',end=999) record
+        if (record(1:1) == '#') cycle
+        if (record(1:10) == '          ') cycle
+        n = n + 1
+        if( n > nmask ) then
+           call mpp_error(FATAL, "fms_io(parse_mask_table_3d): number of mask_list entry "// &
+                "is greater than nmask in file "//trim(mask_table) )
+        endif
+        read(record,*,err=888) mask_list(n,1), mask_list(n,2), mask_list(n,3)
+     enddo
+888  call mpp_error(FATAL, "fms_io(parse_mask_table_3d):  Error in reading mask_list from file "//trim(mask_table))
+
+999  continue
+     !--- make sure the number of entry for mask_list is nmask
+     if( n .NE. nmask) call mpp_error(FATAL, &
+        "fms_io(parse_mask_table_3d): number of mask_list entry does not match nmask in file "//trim(mask_table))
+     call mpp_close(unit)
+  endif
+
+  call mpp_broadcast(mask_list, 3*nmask, mpp_root_pe())
+  do n = 1, nmask
+     if(debug_mask_list) then
+       write(stdoutunit,*) "==>NOTE from parse_mask_table_3d: ", trim(modelname), " mask_list = ", &
+                           mask_list(n,1), mask_list(n,2), mask_list(n,3)
+     endif
+     maskmap(mask_list(n,1),mask_list(n,2),mask_list(n,3)) = .false.
+  enddo
+
+  deallocate(mask_list)
+
+end subroutine parse_mask_table_3d
+
 
 function get_great_circle_algorithm()
    logical :: get_great_circle_algorithm
@@ -5786,5 +7740,3 @@ end function get_great_circle_algorithm
 
 
 end module fms_io_mod
-
-
diff --git a/src/shared/fms/read_data_2d.inc b/src/shared/fms/read_data_2d.inc
index 5a00047840..6970f62b6b 100644
--- a/src/shared/fms/read_data_2d.inc
+++ b/src/shared/fms/read_data_2d.inc
@@ -1,5 +1,5 @@
 ! -*-f90-*-
-! $Id: read_data_2d.inc,v 13.0 2006/03/28 21:39:18 fms Exp $
+! $Id$
 
 
    if((size(data,1)== ied-isd+1).and.(size(data,2)==jed-jsd+1)) then
diff --git a/src/shared/fms/read_data_3d.inc b/src/shared/fms/read_data_3d.inc
index 04ab5f8a60..f2dc5f4a80 100644
--- a/src/shared/fms/read_data_3d.inc
+++ b/src/shared/fms/read_data_3d.inc
@@ -1,5 +1,5 @@
 ! -*-f90-*-
-! $Id: read_data_3d.inc,v 13.0 2006/03/28 21:39:21 fms Exp $
+! $Id$
 
    if((size(data,1)== ied-isd+1).and.(size(data,2)==jed-jsd+1)) then
      no_halo = .false.
diff --git a/src/shared/fms/read_data_4d.inc b/src/shared/fms/read_data_4d.inc
index f0c3419afc..d3160a367f 100644
--- a/src/shared/fms/read_data_4d.inc
+++ b/src/shared/fms/read_data_4d.inc
@@ -1,5 +1,5 @@
 ! -*-f90-*-
-! $Id: read_data_4d.inc,v 13.0 2006/03/28 21:39:24 fms Exp $
+! $Id$
 
    if((size(data,1)== ied-isd+1).and.(size(data,2)==jed-jsd+1)) then
      no_halo = .false.
diff --git a/src/shared/fms/test_fms_io.F90 b/src/shared/fms/test_fms_io.F90
index 2c617fc542..7529e07b51 100644
--- a/src/shared/fms/test_fms_io.F90
+++ b/src/shared/fms/test_fms_io.F90
@@ -22,6 +22,7 @@ program fms_io_test
  integer :: sizey_latlon_grid = 90
  integer :: size_cubic_grid = 48
  integer :: nz = 10, nt = 2, halo = 1
+ integer :: nl = 5
  integer :: stackmax =4000000
  integer :: num_step = 4 ! number of time steps to run, this is used for intermediate run.
                          ! set num_step = 0 for no intermediate run.
@@ -31,22 +32,25 @@ program fms_io_test
  integer :: layout_latlon(2) = (/0,0/)  
  integer :: io_layout(2) = (/0,0/) ! set ndivs_x and ndivs_y to divide each tile into io_layout(1)*io_layout(2)
                                    ! group and write out data from the root pe of each group.
+ logical :: read_only = .False.
 
  namelist /test_fms_io_nml/ sizex_latlon_grid, sizey_latlon_grid, size_cubic_grid, &
-                            nz, nt, halo, num_step, stackmax, do_write, layout_cubic, layout_latlon, io_layout
+                            nz, nt, halo, num_step, stackmax, do_write, layout_cubic, layout_latlon, io_layout, &
+                            read_only, nl
 
  integer           :: unit, io_status, step
  character(len=20) :: time_stamp
 
  type data_storage_type
-    real,    allocatable, dimension(:,:,:,:) :: data1_r3d, data2_r3d, data1_r3d_read, data2_r3d_read
-    real,    allocatable, dimension(:,:,:)   :: data1_r2d, data2_r2d, data1_r2d_read, data2_r2d_read
-    real,    allocatable, dimension(:,:)     :: data1_r1d, data2_r1d, data1_r1d_read, data2_r1d_read
-    real,    allocatable, dimension(:)       :: data1_r0d, data2_r0d, data1_r0d_read, data2_r0d_read
-    integer, allocatable, dimension(:,:,:,:) :: data1_i3d, data2_i3d, data1_i3d_read, data2_i3d_read
-    integer, allocatable, dimension(:,:,:)   :: data1_i2d, data2_i2d, data1_i2d_read, data2_i2d_read
-    integer, allocatable, dimension(:,:)     :: data1_i1d, data2_i1d, data1_i1d_read, data2_i1d_read
-    integer, allocatable, dimension(:)       :: data1_i0d, data2_i0d, data1_i0d_read, data2_i0d_read
+    real,    allocatable, dimension(:,:,:,:,:) :: data1_r4d, data2_r4d, data1_r4d_read, data2_r4d_read
+    real,    allocatable, dimension(:,:,:,:)   :: data1_r3d, data2_r3d, data1_r3d_read, data2_r3d_read
+    real,    allocatable, dimension(:,:,:)     :: data1_r2d, data2_r2d, data1_r2d_read, data2_r2d_read
+    real,    allocatable, dimension(:,:)       :: data1_r1d, data2_r1d, data1_r1d_read, data2_r1d_read
+    real,    allocatable, dimension(:)         :: data1_r0d, data2_r0d, data1_r0d_read, data2_r0d_read
+    integer, allocatable, dimension(:,:,:,:)   :: data1_i3d, data2_i3d, data1_i3d_read, data2_i3d_read
+    integer, allocatable, dimension(:,:,:)     :: data1_i2d, data2_i2d, data1_i2d_read, data2_i2d_read
+    integer, allocatable, dimension(:,:)       :: data1_i1d, data2_i1d, data1_i1d_read, data2_i1d_read
+    integer, allocatable, dimension(:)         :: data1_i0d, data2_i0d, data1_i0d_read, data2_i0d_read
  end type data_storage_type
  
  type(data_storage_type), save :: latlon_data
@@ -99,11 +103,11 @@ program fms_io_test
 
  if(file_exist('INPUT/'//trim(file_latlon), domain_latlon)) then
     call restore_state(restart_latlon)
-    call compare_restart("latlon_grid save_restore", latlon_data)
+    call compare_restart("latlon_grid save_restore", latlon_data, .true.)
  end if
  if(file_exist('INPUT/'//trim(file_cubic), domain_cubic) ) then
     call restore_state(restart_cubic)
-    call compare_restart("cubic_grid save_restore", cubic_data)
+    call compare_restart("cubic_grid save_restore", cubic_data, .true. )
  end if
  
  !---copy data
@@ -149,7 +153,7 @@ subroutine setup_test_restart(restart_data, type, ntiles, storage, file, layout_
     integer, dimension(1)                    :: tile1, tile2
     integer, dimension(1)                    :: istart1, iend1, jstart1, jend1
     integer, dimension(1)                    :: istart2, iend2, jstart2, jend2
-    integer                                  :: i, j, k, nx, ny
+    integer                                  :: i, j, k, nx, ny, l
     integer                                  :: isc, iec, jsc, jec
     integer                                  :: isd, ied, jsd, jed
     integer                                  :: id_restart
@@ -205,6 +209,10 @@ subroutine setup_test_restart(restart_data, type, ntiles, storage, file, layout_
 
     allocate(storage%data1_r3d(isd:ied, jsd:jed, nz, nt), storage%data1_r3d_read(isd:ied, jsd:jed, nz, nt) )
     allocate(storage%data2_r3d(isd:ied, jsd:jed, nz, nt), storage%data2_r3d_read(isd:ied, jsd:jed, nz, nt) )
+    allocate(storage%data1_r4d(isd:ied, jsd:jed, nz, nl, nt) )
+    allocate(storage%data1_r4d_read(isd:ied, jsd:jed, nz, nl, nt) )
+    allocate(storage%data2_r4d(isd:ied, jsd:jed, nz, nl, nt) )
+    allocate(storage%data2_r4d_read(isd:ied, jsd:jed, nz, nl, nt) )
     allocate(storage%data1_i3d(isd:ied, jsd:jed, nz, nt), storage%data1_i3d_read(isd:ied, jsd:jed, nz, nt) )
     allocate(storage%data2_i3d(isd:ied, jsd:jed, nz, nt), storage%data2_i3d_read(isd:ied, jsd:jed, nz, nt) )
     allocate(storage%data1_r2d(isd:ied, jsd:jed,     nt), storage%data1_r2d_read(isd:ied, jsd:jed,     nt) )
@@ -228,11 +236,22 @@ subroutine setup_test_restart(restart_data, type, ntiles, storage, file, layout_
     storage%data1_i1d = 0; storage%data1_i1d_read = 0; storage%data2_i1d = 0; storage%data2_i1d_read = 0
     storage%data1_r0d = 0; storage%data1_r0d_read = 0; storage%data2_r0d = 0; storage%data2_r0d_read = 0
     storage%data1_i0d = 0; storage%data1_i0d_read = 0; storage%data2_i0d = 0; storage%data2_i0d_read = 0
+    storage%data1_r4d = 0; storage%data1_r4d_read = 0; storage%data2_r4d = 0; storage%data2_r4d_read = 0
     do n = 1, nt
        storage%data1_r0d(n) =  tile + n*1e-3
        storage%data2_r0d(n) = -tile - n*1e-3
        storage%data1_i0d(n) =  tile*1e3 + n
        storage%data2_i0d(n) = -tile*1e3 - n
+       do l = 1, nl
+          do k = 1, nz
+             do j = jsc, jec
+                do i = isc, iec
+                   storage%data1_r4d(i,j,k,l,n) =  l*1e9 + tile*1e6 + n*1e3 + k + i*1e-3 + j*1e-6;
+                   storage%data2_r4d(i,j,k,l,n) =  -l*1e9 - tile*1e6 - n*1e3 + k - i*1e-3 - j*1e-6;
+                enddo
+             enddo
+          enddo
+       enddo
        do k = 1, nz
           storage%data1_r1d(k,n) =   tile*1e3 + n + k*1e-3
           storage%data2_r1d(k,n) =  -tile*1e3 - n - k*1e-3
@@ -276,7 +295,7 @@ subroutine setup_test_restart(restart_data, type, ntiles, storage, file, layout_
           call read_data(file_r, "data1_i0d", storage%data1_i0d_read(      n), domain, timelevel = n )
           call read_data(file_r, "data2_i0d", storage%data2_i0d_read(      n), domain, timelevel = n )
        end do
-       call compare_restart(type//" read_write", storage)
+       call compare_restart(type//" read_write", storage, .false.)
     end if
 
 
@@ -305,13 +324,20 @@ subroutine setup_test_restart(restart_data, type, ntiles, storage, file, layout_
     !--- test register_restart_field, save_restart, restore_state
 
     id_restart = register_restart_field(restart_data, file, "data1_r3d", storage%data1_r3d_read(:,:,:,1), &
-                                domain, longname="first data_r3d",units="none")
+                                domain, longname="first data_r3d",units="none", read_only=read_only)
     id_restart = register_restart_field(restart_data, file, "data1_r3d", storage%data1_r3d_read(:,:,:,2), &
-                                domain, longname="first data_r3d",units="none")
+                                domain, longname="first data_r3d",units="none", read_only=read_only)
     id_restart = register_restart_field(restart_data, file, "data2_r3d", storage%data2_r3d_read(:,:,:,1), &
                                 storage%data2_r3d_read(:,:,:,2), &
                                 domain, longname="second data_i3d", units="none")
 
+    id_restart = register_restart_field(restart_data, file, "data1_r4d", storage%data1_r4d_read(:,:,:,:,1), &
+                                domain, longname="first data_r4d",units="none")
+    id_restart = register_restart_field(restart_data, file, "data1_r4d", storage%data1_r4d_read(:,:,:,:,2), &
+                                domain, longname="first data_r4d",units="none")
+    id_restart = register_restart_field(restart_data, file, "data2_r4d", storage%data2_r4d_read(:,:,:,:,1), &
+                                domain, longname="second data_r4d",units="none")
+
     id_restart = register_restart_field(restart_data, file, "data1_i3d", storage%data1_i3d_read(:,:,:,1), &
                                 domain, longname="first data_i3d",units="none")
     id_restart = register_restart_field(restart_data, file, "data1_i3d", storage%data1_i3d_read(:,:,:,2), &
@@ -321,20 +347,20 @@ subroutine setup_test_restart(restart_data, type, ntiles, storage, file, layout_
                                 domain, longname="second data_i3d", units="none")
 
     id_restart = register_restart_field(restart_data, file, "data1_r2d", storage%data1_r2d_read(:,:,  1), &
-                                domain, longname="first data_r2d",units="none")
+                                domain, longname="first data_r2d",units="none", read_only=read_only)
     id_restart = register_restart_field(restart_data, file, "data1_r2d", storage%data1_r2d_read(:,:,  2), &
-                                domain, longname="first data_r2d",units="none")
+                                domain, longname="first data_r2d",units="none", read_only=read_only)
     id_restart = register_restart_field(restart_data, file, "data2_r2d", storage%data2_r2d_read(:,:,  1), &
                                 storage%data2_r2d_read(:,:,2), &
                                 domain, longname="second data_i2d", units="none")
 
     id_restart = register_restart_field(restart_data, file, "data1_i2d", storage%data1_i2d_read(:,:,  1), &
-                                domain, longname="first data_i2d",units="none")
+                                domain, longname="first data_i2d",units="none", read_only=read_only)
     id_restart = register_restart_field(restart_data, file, "data1_i2d", storage%data1_i2d_read(:,:,  2), &
-                                domain, longname="first data_i2d",units="none")
+                                domain, longname="first data_i2d",units="none", read_only=read_only)
     id_restart = register_restart_field(restart_data, file, "data2_i2d", storage%data2_i2d_read(:,:,  1), &
                                 storage%data2_i2d_read(:,:,2), &
-                                domain, longname="second data_i2d", units="none")
+                                domain, longname="second data_i2d", units="none", read_only=read_only)
 
     id_restart = register_restart_field(restart_data, file, "data1_r1d", storage%data1_r1d_read(:,    1), &
                                 domain, longname="first data_r1d",units="none")
@@ -345,18 +371,18 @@ subroutine setup_test_restart(restart_data, type, ntiles, storage, file, layout_
                                 domain, longname="second data_i1d", units="none")
 
     id_restart = register_restart_field(restart_data, file, "data1_i1d", storage%data1_i1d_read(:,    1), &
-                                domain, longname="first data_i1d",units="none")
+                                domain, longname="first data_i1d",units="none", read_only=read_only)
     id_restart = register_restart_field(restart_data, file, "data1_i1d", storage%data1_i1d_read(:,    2), &
-                                domain, longname="first data_i1d",units="none")
+                                domain, longname="first data_i1d",units="none", read_only=read_only)
     id_restart = register_restart_field(restart_data, file, "data2_i1d", storage%data2_i1d_read(:,    1), &
                                 storage%data2_i1d_read(:,  2), &
                                 domain, longname="second data_i1d", units="none")
 
 
     id_restart = register_restart_field(restart_data, file, "data1_r0d", storage%data1_r0d_read(      1), &
-                                domain, longname="first data_r0d",units="none")
+                                domain, longname="first data_r0d",units="none", read_only=read_only)
     id_restart = register_restart_field(restart_data, file, "data1_r0d", storage%data1_r0d_read(      2), &
-                                domain, longname="first data_r0d",units="none")
+                                domain, longname="first data_r0d",units="none", read_only=read_only)
     id_restart = register_restart_field(restart_data, file, "data2_r0d", storage%data2_r0d_read(      1), &
                                 storage%data2_r0d_read(    2), &
                                 domain, longname="second data_i0d", units="none")
@@ -371,10 +397,15 @@ subroutine setup_test_restart(restart_data, type, ntiles, storage, file, layout_
 
   end subroutine setup_test_restart
 
-  subroutine compare_restart(type, storage)
+  subroutine compare_restart(type, storage, compare_r4d)
     character(len=*), intent(in)             :: type
     type(data_storage_type), intent(inout)   :: storage
+    logical,                 intent(in   )   :: compare_r4d
 
+       if(compare_r4d) then   
+          call compare_data_r5d(storage%data1_r4d, storage%data1_r4d_read, type//" data1_r4d")
+          call compare_data_r5d(storage%data2_r4d(:,:,:,:,1:1), storage%data2_r4d_read(:,:,:,:,1:1), type//" data2_r4d")
+       endif
        call compare_data_r4d(storage%data1_r3d, storage%data1_r3d_read, type//" data1_r3d")
        call compare_data_r4d(storage%data2_r3d, storage%data2_r3d_read, type//" data2_r3d")
        call compare_data_i4d(storage%data1_i3d, storage%data1_i3d_read, type//" data1_i3d")
@@ -405,11 +436,13 @@ subroutine release_storage_memory(storage)
     deallocate(storage%data1_i1d, storage%data2_i1d, storage%data1_i1d_read, storage%data2_i1d_read)
     deallocate(storage%data1_r0d, storage%data2_r0d, storage%data1_r0d_read, storage%data2_r0d_read)
     deallocate(storage%data1_i0d, storage%data2_i0d, storage%data1_i0d_read, storage%data2_i0d_read)
+    deallocate(storage%data1_r4d, storage%data2_r4d, storage%data1_r4d_read, storage%data2_r4d_read)
 
   end subroutine release_storage_memory
 
   subroutine copy_restart_data(storage)
     type(data_storage_type), intent(inout)   :: storage
+    integer :: n, l
 
     storage%data1_r3d_read = storage%data1_r3d; storage%data2_r3d_read = storage%data2_r3d
     storage%data1_i3d_read = storage%data1_i3d; storage%data2_i3d_read = storage%data2_i3d
@@ -419,11 +452,52 @@ subroutine copy_restart_data(storage)
     storage%data1_i1d_read = storage%data1_i1d; storage%data2_i1d_read = storage%data2_i1d
     storage%data1_r0d_read = storage%data1_r0d; storage%data2_r0d_read = storage%data2_r0d
     storage%data1_i0d_read = storage%data1_i0d; storage%data2_i0d_read = storage%data2_i0d
+    storage%data1_r4d_read = storage%data1_r4d; storage%data2_r4d_read = storage%data2_r4d
 
     return
 
   end subroutine copy_restart_data
 
+  subroutine compare_data_r5d( a, b, string )
+    real, intent(in), dimension(:,:,:,:,:) :: a, b
+    character(len=*), intent(in)         :: string
+    integer(LONG_KIND)                   :: sum1, sum2
+    integer                              :: i, j, k, l, n
+    integer, parameter                   :: stdunit = 6
+
+    if(size(a,1) .ne. size(b,1) .or. size(a,2) .ne. size(b,2) .or. size(a,3) .ne. size(b,3) &
+      .or. size(a,4) .ne. size(b,4) .or. size(a,5) .ne. size(b,5) ) &
+         call mpp_error(FATAL,'compare_data_r5d: size of a and b does not match')
+    do n = 1, size(a,5)
+       do l = 1, size(a,4)
+          do k = 1, size(a,3)
+             do j = 1, size(a,2)
+                do i = 1, size(a,1)
+                   if(a(i,j,k,l,n) .ne. b(i,j,k,l,n)) then
+                      write(stdunit,'(a,i3,a,i3,a,i3,a,i3,a,i3,a,i3,a,f18.6,a,f18.6)')" at pe ", mpp_pe(), &
+                           ", at point (",i,", ", j, ", ", k, ", ", l, ", ", n, "), a = ", a(i,j,k,l,n), ",&
+                           b = ", b(i,j,k,l,n)
+                      call mpp_error(FATAL, trim(string)//': point by point comparison are not OK.')
+                   endif
+                enddo
+             enddo
+          enddo
+       enddo
+    enddo
+    sum1 = mpp_chksum( a, (/mpp_pe()/) )
+    sum2 = mpp_chksum( b, (/mpp_pe()/) )
+
+    if( sum1.EQ.sum2 )then
+       if( mpp_pe() .EQ. mpp_root_pe() )call mpp_error( NOTE, trim(string)//': OK.' )
+       !--- in some case, even though checksum agree, the two arrays 
+       !    actually are different, like comparing (1.1,-1.2) with (-1.1,1.2)
+       !--- hence we need to check the value point by point.
+    else
+       call mpp_error( FATAL, trim(string)//': chksums are not OK.' )
+    end if
+  end subroutine compare_data_r5d
+
+
   subroutine compare_data_r4d( a, b, string )
     real, intent(in), dimension(:,:,:,:) :: a, b
     character(len=*), intent(in)         :: string
@@ -439,7 +513,7 @@ subroutine compare_data_r4d( a, b, string )
           do j = 1, size(a,2)
              do i = 1, size(a,1)
                 if(a(i,j,k,l) .ne. b(i,j,k,l)) then
-                   write(stdunit,'(a,i3,a,i3,a,i3,a,i3,a,i3,a,f18.9,a,f18.9)')" at pe ", mpp_pe(), &
+                   write(stdunit,'(a,i3,a,i3,a,i3,a,i3,a,i3,a,f18.6,a,f18.6)')" at pe ", mpp_pe(), &
                         ", at point (",i,", ", j, ", ", k, ", ", l, "), a = ", a(i,j,k,l), ", b = ", b(i,j,k,l)
                    call mpp_error(FATAL, trim(string)//': point by point comparison are not OK.')
                 endif
@@ -533,7 +607,7 @@ subroutine compare_data_r2d( a, b, string )
     do l = 1, size(a,2)
        do i = 1, size(a,1)
           if(a(i,l) .ne. b(i,l)) then
-             write(stdunit,'(a,i3,a,i3,a,i3,a,f16.9,a,f16.9)')" at pe ", mpp_pe(), &
+             write(stdunit,'(a,i3,a,i3,a,i3,a,f16.6,a,f16.6)')" at pe ", mpp_pe(), &
                   ", at point (",i, ", ", l, "), a = ", a(i,l), ", b = ", b(i,l)
              call mpp_error(FATAL, trim(string)//': point by point comparison are not OK.')
           endif
@@ -576,7 +650,7 @@ subroutine compare_data_r1d( a, b, string )
 
     do l = 1, size(a(:))
        if(a(l) .ne. b(l)) then
-          write(stdunit,'(a,i3,a,i3,a,f16.9,a,f16.9)')" at pe ", mpp_pe(), &
+          write(stdunit,'(a,i3,a,i3,a,f16.6,a,f16.6)')" at pe ", mpp_pe(), &
                ", at point (",l, "), a = ", a(l), ", b = ", b(l)
           call mpp_error(FATAL, trim(string)//': point by point comparison are not OK.')
        endif
diff --git a/src/shared/horiz_interp/horiz_interp.F90 b/src/shared/horiz_interp/horiz_interp.F90
index 38357a8159..9deee50af3 100644
--- a/src/shared/horiz_interp/horiz_interp.F90
+++ b/src/shared/horiz_interp/horiz_interp.F90
@@ -24,9 +24,10 @@
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 module horiz_interp_mod
 
-!  Zhi Liang 
-!  Bruce Wyman 
+!  Zhi Liang 
+!  Bruce Wyman 
 
+! 
 
 ! 
 !   Performs spatial interpolation between grids.
@@ -235,8 +236,8 @@ module horiz_interp_mod
  namelist /horiz_interp_nml/ reproduce_siena
 
 !-----------------------------------------------------------------------
- character(len=128) :: version = '$Id: horiz_interp.F90,v 20.0 2013/12/14 00:20:17 fms Exp $'
- character(len=128) :: tagname = '$Name: tikal $'
+ character(len=128) :: version = '$Id$'
+ character(len=128) :: tagname = '$Name$'
  logical            :: module_is_initialized = .FALSE.
 !-----------------------------------------------------------------------
 
@@ -255,7 +256,7 @@ subroutine horiz_interp_init
   integer :: unit, ierr, io
 
   if(module_is_initialized) return
-  call write_version_number()
+  call write_version_number (version, tagname)
 
 #ifdef INTERNAL_FILE_NML
   read (input_nml_file, horiz_interp_nml, iostat=io)
@@ -277,13 +278,9 @@ subroutine horiz_interp_init
   endif
 
   if( reproduce_siena ) then
-     if( mpp_pe() == mpp_root_pe() ) then
-        call mpp_error(WARNING, "horiz_interp_mod: You have overridden the default value of reproduce_siena " // &
-                             "and set it to .true. in horiz_interp_nml. This is a temporary workaround to " // &
-                             "allow for consistency in continuing experiments.  Please use the default " //&
-                             "value (.false.) as this option will be removed in a future release. ")
-     endif
-     call set_reproduce_siena_true( )
+     call mpp_error(FATAL, "horiz_interp_mod: You have overridden the default value of reproduce_siena " // &
+                           "and set it to .true. in horiz_interp_nml. This is a temporary workaround to " // &
+                           "allow for consistency in continuing experiments. Please remove this namelist " )
   endif
 
   call horiz_interp_conserve_init
@@ -720,7 +717,8 @@ end subroutine horiz_interp_new_1d_dst
 
 ! 
  subroutine horiz_interp_base_2d ( Interp, data_in, data_out, verbose, &
-                                   mask_in, mask_out, missing_value, missing_permit, err_msg )
+                                   mask_in, mask_out, missing_value, missing_permit, &
+                                   err_msg, new_missing_handle )
 !
 !-----------------------------------------------------------------------
    type (horiz_interp_type), intent(in) :: Interp
@@ -732,6 +730,7 @@ subroutine horiz_interp_base_2d ( Interp, data_in, data_out, verbose, &
       real, intent(in),                   optional :: missing_value
       integer, intent(in),                optional :: missing_permit
    character(len=*), intent(out),         optional :: err_msg
+      logical, intent(in),                optional :: new_missing_handle
 !-----------------------------------------------------------------------
    if(present(err_msg)) err_msg = ''
    if(.not.Interp%I_am_initialized) then
@@ -743,7 +742,7 @@ subroutine horiz_interp_base_2d ( Interp, data_in, data_out, verbose, &
       call horiz_interp_conserve(Interp,data_in, data_out, verbose, mask_in, mask_out)
    case(BILINEAR)
       call horiz_interp_bilinear(Interp,data_in, data_out, verbose, mask_in, mask_out, &
-                             missing_value, missing_permit )
+                             missing_value, missing_permit, new_missing_handle )
    case(BICUBIC)
       call horiz_interp_bicubic(Interp,data_in, data_out, verbose, mask_in, mask_out, &
                              missing_value, missing_permit )
diff --git a/src/shared/horiz_interp/horiz_interp_bicubic.F90 b/src/shared/horiz_interp/horiz_interp_bicubic.F90
index b2a738f269..7afbbfc315 100644
--- a/src/shared/horiz_interp/horiz_interp_bicubic.F90
+++ b/src/shared/horiz_interp/horiz_interp_bicubic.F90
@@ -41,8 +41,8 @@ module horiz_interp_bicubic_mod
     module procedure horiz_interp_bicubic_new_1d_s
   end interface
 
-   character(len=128) :: version="$Id: horiz_interp_bicubic.F90,v 19.0 2012/01/06 21:57:52 fms Exp $"
-   character(len=128) :: tagname = '$Name: tikal $'
+   character(len=128) :: version="$Id$"
+   character(len=128) :: tagname = '$Name$'
    logical            :: module_is_initialized = .FALSE.
    integer            :: verbose_bicubic = 0
    
@@ -84,7 +84,7 @@ module horiz_interp_bicubic_mod
   subroutine horiz_interp_bicubic_init
 
      if(module_is_initialized) return
-     call write_version_number()
+     call write_version_number (version, tagname)
      module_is_initialized = .true.
      tpi = 2.0*PI
 
@@ -545,13 +545,17 @@ subroutine bcucof(y,y1,y2,y12,d1,d2,c)
       integer i,j,k,l
       real d1d2,xx,cl(16),wt(16,16),x(16)
       save wt
-      data wt/1,0,-3,2,4*0,-3,0,9,-6,2,0,-6,4,8*0,3,0,-9,6,-2,0,6,-4,10* &
-       0,9,-6,2*0,-6,4,2*0,3,-2,6*0,-9,6,2*0,6,-4,4*0,1,0,-3,2,-2,0,6,-4, &
-       1,0,-3,2,8*0,-1,0,3,-2,1,0,-3,2,10*0,-3,2,2*0,3,-2,6*0,3,-2,2*0,   &
-      -6,4,2*0,3,-2,0,1,-2,1,5*0,-3,6,-3,0,2,-4,2,9*0,3,-6,3,0,-2,4,-2,  &
-       10*0,-3,3,2*0,2,-2,2*0,-1,1,6*0,3,-3,2*0,-2,2,5*0,1,-2,1,0,-2,4,   &
-      -2,0,1,-2,1,9*0,-1,2,-1,0,1,-2,1,10*0,1,-1,2*0,-1,1,6*0,-1,1,2*0,  &
-       2,-2,2*0,-1,1/
+      data wt/1., 0., -3., 2., 4*0., -3., 0., 9., -6., 2., 0., -6., 4., 8*0., &
+              3., 0., -9., 6., -2., 0., 6., -4., 10*0., 9., -6., 2*0., -6., &
+              4., 2*0., 3., -2., 6*0., -9., 6., 2*0., 6., -4., 4*0., 1., 0., &
+              -3., 2., -2., 0., 6., -4., 1., 0., -3., 2., 8*0., -1., 0., 3., &
+              -2., 1., 0., -3., 2., 10*0., -3., 2., 2*0., 3., -2., 6*0., 3., &
+              -2., 2*0., -6., 4., 2*0., 3., -2., 0., 1., -2., 1., 5*0., -3., &
+              6., -3., 0., 2., -4., 2., 9*0., 3., -6., 3., 0., -2., 4., -2., &
+              10*0., -3., 3., 2*0., 2., -2., 2*0., -1., 1., 6*0., 3., -3., &
+              2*0., -2., 2., 5*0., 1., -2., 1., 0., -2., 4., -2., 0., 1., -2., &
+              1., 9*0., -1., 2., -1., 0., 1., -2., 1., 10*0., 1., -1., 2*0., &
+              -1., 1., 6*0., -1., 1., 2*0., 2., -2., 2*0., -1., 1./
 
       d1d2=d1*d2
       do i=1,4
@@ -647,7 +651,7 @@ subroutine fill_xy(fi, ics, ice, jcs, jce, mask, maxpass)
                  jnu = min(j+1,jce)
                  do js=jnl,jnu
                    do is=inl,inr
-                     if (work_old(is,js) .ne. blank .and. mask(is,js).ne.0) then
+                     if (work_old(is,js) .ne. blank .and. mask(is,js).ne.0.) then
                        tavr = tavr + work_old(is,js)
                        iavr = iavr+1
                      endif
diff --git a/src/shared/horiz_interp/horiz_interp_bilinear.F90 b/src/shared/horiz_interp/horiz_interp_bilinear.F90
index 006eab302e..a632cde5e5 100644
--- a/src/shared/horiz_interp/horiz_interp_bilinear.F90
+++ b/src/shared/horiz_interp/horiz_interp_bilinear.F90
@@ -1,7 +1,8 @@
 module horiz_interp_bilinear_mod
 
-  !  Zhi Liang 
+  !  Zhi Liang 
 
+  ! 
 
   ! 
   !   Performs spatial interpolation between grids using bilinear interpolation
@@ -38,8 +39,8 @@ module horiz_interp_bilinear_mod
   integer, parameter :: DUMMY = -999
 
   !-----------------------------------------------------------------------
-  character(len=128) :: version = '$Id: horiz_interp_bilinear.F90,v 20.0 2013/12/14 00:20:22 fms Exp $'
-  character(len=128) :: tagname = '$Name: tikal $'
+  character(len=128) :: version = '$Id$'
+  character(len=128) :: tagname = '$Name$'
   logical            :: module_is_initialized = .FALSE.
 
 contains
@@ -56,7 +57,7 @@ module horiz_interp_bilinear_mod
   subroutine horiz_interp_bilinear_init
 
     if(module_is_initialized) return
-    call write_version_number()
+    call write_version_number (version, tagname)
     module_is_initialized = .true.
 
   end subroutine horiz_interp_bilinear_init
@@ -395,7 +396,7 @@ subroutine horiz_interp_bilinear_new_2d ( Interp, lon_in, lat_in, lon_out, lat_o
           a  = b2*c1-b1*c2
           b  = a1*b2-a2*b1+c1*d2-c2*d1+c2*lon-c1*lat
           c  = a2*lon-a1*lat+a1*d2-a2*d1
-          quadra = b*b-4*a*c
+          quadra = b*b-4.*a*c
           if(abs(quadra) < epsln) quadra = 0.0
           if(quadra < 0.0) call mpp_error(FATAL, &
                "horiz_interp_bilinear_mod: No solution existed for this quadratic equation")
@@ -440,7 +441,7 @@ subroutine horiz_interp_bilinear_new_2d ( Interp, lon_in, lat_in, lon_out, lat_o
              if (x > 1.0) x = 1.0
              if (y > 1.0) y = 1.0
            endif 
-           if( x>1 .or. x<0 .or. y>1 .or. y < 0) call mpp_error(FATAL, &
+           if( x>1. .or. x<0. .or. y>1. .or. y < 0.) call mpp_error(FATAL, &
                 "horiz_interp_bilinear_mod: weight should be between 0 and 1")
            Interp % wti(m,n,1)=1.0-x; Interp % wti(m,n,2)=x   
            Interp % wtj(m,n,1)=1.0-y; Interp % wtj(m,n,2)=y          
@@ -957,7 +958,7 @@ end function intersect
   !   
 
   subroutine horiz_interp_bilinear ( Interp, data_in, data_out, verbose, mask_in,mask_out, &
-       missing_value, missing_permit)
+       missing_value, missing_permit, new_handle_missing )
     !-----------------------------------------------------------------------
     type (horiz_interp_type), intent(in)        :: Interp
     real, intent(in),  dimension(:,:)           :: data_in
@@ -967,6 +968,7 @@ subroutine horiz_interp_bilinear ( Interp, data_in, data_out, verbose, mask_in,m
     real, intent(out), dimension(:,:), optional :: mask_out
     real, intent(in),                  optional :: missing_value
     integer, intent(in),               optional :: missing_permit
+    logical, intent(in),               optional :: new_handle_missing
     !-----------------------------------------------------------------------
     integer :: nlon_in, nlat_in, nlon_out, nlat_out, n, m,         &
          is, ie, js, je, iverbose, max_missing, num_missing, &
@@ -974,6 +976,8 @@ subroutine horiz_interp_bilinear ( Interp, data_in, data_out, verbose, mask_in,m
     real    :: dwtsum, wtsum, min_in, max_in, avg_in, &
          min_out, max_out, avg_out, wtw, wte, wts, wtn
     real    :: mask(size(data_in,1), size(data_in,2) )
+    logical :: set_to_missing, is_missing(4), new_handler
+    real    :: f1, f2, f3, f4, middle, w, s
 
     num_missing = 0
 
@@ -998,6 +1002,12 @@ subroutine horiz_interp_bilinear ( Interp, data_in, data_out, verbose, mask_in,m
        max_missing = 0
     endif
 
+    if(present(new_handle_missing)) then
+       new_handler = new_handle_missing
+    else
+       new_handler = .false.
+    endif
+
     if(max_missing .gt. 3 .or. max_missing .lt. 0) call mpp_error(FATAL, &
          'horiz_interp_bilinear_mod: missing_permit should be between 0 and 3')
 
@@ -1007,60 +1017,217 @@ subroutine horiz_interp_bilinear ( Interp, data_in, data_out, verbose, mask_in,m
     if (size(data_out,1) /= nlon_out .or. size(data_out,2) /= nlat_out) &
          call mpp_error(FATAL,'horiz_interp_bilinear_mod: size of output array incorrect')
 
-    do n = 1, nlat_out
-       do m = 1, nlon_out
-          is = Interp % i_lon (m,n,1); ie = Interp % i_lon (m,n,2)
-          js = Interp % j_lat (m,n,1); je = Interp % j_lat (m,n,2)
-          wtw = Interp % wti   (m,n,1)
-          wte = Interp % wti   (m,n,2)
-          wts = Interp % wtj   (m,n,1)
-          wtn = Interp % wtj   (m,n,2)
-
-          if(present(missing_value) ) then
+    if(new_handler) then
+       if( .not. present(missing_value) )  call mpp_error(FATAL, &
+            "horiz_interp_bilinear_mod: misisng_value must be present when new_handle_missing is .true.")
+       if( present(mask_in) ) call mpp_error(FATAL, &
+            "horiz_interp_bilinear_mod: mask_in should not be present when new_handle_missing is .true.")
+       do n = 1, nlat_out
+          do m = 1, nlon_out
+             is = Interp % i_lon (m,n,1); ie = Interp % i_lon (m,n,2)
+             js = Interp % j_lat (m,n,1); je = Interp % j_lat (m,n,2)
+             wtw = Interp % wti   (m,n,1)
+             wte = Interp % wti   (m,n,2)
+             wts = Interp % wtj   (m,n,1)
+             wtn = Interp % wtj   (m,n,2)
+
+             is_missing = .false.
              num_missing = 0
+             set_to_missing = .false.
              if(data_in(is,js) == missing_value) then
                 num_missing = num_missing+1
-                mask(is,js) = 0.0
+                is_missing(1) = .true.
+                if(wtw .GE. 0.5 .AND. wts .GE. 0.5) set_to_missing = .true.
              endif
+
              if(data_in(ie,js) == missing_value) then
                 num_missing = num_missing+1
-                mask(ie,js) = 0.0
+                is_missing(2) = .true.
+                if(wte .GE. 0.5 .AND. wts .GE. 0.5) set_to_missing = .true.
              endif
              if(data_in(ie,je) == missing_value) then
                 num_missing = num_missing+1
-                mask(ie,je) = 0.0
+                is_missing(3) = .true.
+                if(wte .GE. 0.5 .AND. wtn .GE. 0.5) set_to_missing = .true.
              endif
              if(data_in(is,je) == missing_value) then
                 num_missing = num_missing+1
-                mask(is,je) = 0.0
+                is_missing(4) = .true.
+                if(wtw .GE. 0.5 .AND. wtn .GE. 0.5) set_to_missing = .true.
              endif
-          endif
 
-          dwtsum = data_in(is,js)*mask(is,js)*wtw*wts &
-               + data_in(ie,js)*mask(ie,js)*wte*wts &
-               + data_in(ie,je)*mask(ie,je)*wte*wtn &
-               + data_in(is,je)*mask(is,je)*wtw*wtn 
-          wtsum  = mask(is,js)*wtw*wts + mask(ie,js)*wte*wts  &
-               + mask(ie,je)*wte*wtn + mask(is,je)*wtw*wtn
+             if( num_missing == 4 .OR. set_to_missing ) then
+                data_out(m,n) = missing_value
+                if(present(mask_out)) mask_out(m,n) = 0.0
+                 cycle
+             else if(num_missing == 0) then
+                f1 = data_in(is,js)
+                f2 = data_in(ie,js)
+                f3 = data_in(ie,je)
+                f4 = data_in(is,je)
+                w = wtw
+                s = wts
+             else if(num_missing == 3) then  !--- three missing value 
+                if(.not. is_missing(1) ) then
+                   data_out(m,n) = data_in(is,js)
+                else if(.not. is_missing(2) ) then
+                   data_out(m,n) = data_in(ie,js)
+                else if(.not. is_missing(3) ) then
+                   data_out(m,n) = data_in(ie,je)
+                else if(.not. is_missing(4) ) then
+                   data_out(m,n) = data_in(is,je)
+                endif
+                if(present(mask_out) ) mask_out(m,n) = 1.0
+                cycle
+             else   !--- one or two missing value
+                if( num_missing == 1) then
+                   if( is_missing(1) .OR. is_missing(3) ) then
+                      middle = 0.5*(data_in(ie,js)+data_in(is,je))
+                   else
+                      middle = 0.5*(data_in(is,js)+data_in(ie,je))
+                   endif
+                else ! num_missing = 2
+                   if( is_missing(1) .AND. is_missing(2) ) then
+                      middle = 0.5*(data_in(ie,je)+data_in(is,je))
+                   else if( is_missing(1) .AND. is_missing(3) ) then
+                      middle = 0.5*(data_in(ie,js)+data_in(is,je))
+                   else if( is_missing(1) .AND. is_missing(4) ) then
+                      middle = 0.5*(data_in(ie,js)+data_in(ie,je))
+                   else if( is_missing(2) .AND. is_missing(3) ) then
+                      middle = 0.5*(data_in(is,js)+data_in(is,je))
+                   else if( is_missing(2) .AND. is_missing(4) ) then
+                      middle = 0.5*(data_in(is,js)+data_in(ie,je))
+                   else if( is_missing(3) .AND. is_missing(4) ) then
+                      middle = 0.5*(data_in(is,js)+data_in(ie,js))
+                   endif
+                endif
 
-          if(.not. present(mask_in) .and. .not. present(missing_value)) wtsum = 1.0
+                if( wtw .GE. 0.5 .AND. wts .GE. 0.5 ) then  ! zone 1
+                   w = 2.0*(wtw-0.5)
+                   s = 2.0*(wts-0.5)
+                   f1 = data_in(is,js)
+                   if(is_missing(2)) then
+                      f2 = f1
+                   else
+                      f2 = 0.5*(data_in(is,js)+data_in(ie,js))
+                   endif
+                   f3 = middle
+                   if(is_missing(4)) then
+                      f4 = f1
+                   else
+                      f4 = 0.5*(data_in(is,js)+data_in(is,je))
+                   endif
+                else if( wte .GE. 0.5 .AND. wts .GE. 0.5 ) then  ! zone 2
+                   w = 2.0*(1.0-wte)
+                   s = 2.0*(wts-0.5)
+                   f2 = data_in(ie,js)
+                   if(is_missing(1)) then
+                      f1 = f2
+                   else
+                      f1 = 0.5*(data_in(is,js)+data_in(ie,js))
+                   endif
+                   f4 = middle
+                   if(is_missing(3)) then
+                      f3 = f2
+                   else
+                      f3 = 0.5*(data_in(ie,js)+data_in(ie,je))
+                   endif
+                else if( wte .GE. 0.5 .AND. wtn .GE. 0.5 ) then  ! zone 3
+                   w = 2.0*(1.0-wte)
+                   s = 2.0*(1.0-wtn)
+                   f3 = data_in(ie,je)
+                   if(is_missing(2)) then
+                      f2 = f3
+                   else
+                      f2 = 0.5*(data_in(ie,js)+data_in(ie,je))
+                   endif
+                   f1 = middle
+                   if(is_missing(4)) then
+                      f4 = f3
+                   else
+                      f4 = 0.5*(data_in(ie,je)+data_in(is,je))
+                   endif
+                else if( wtw .GE. 0.5 .AND. wtn .GE. 0.5 ) then  ! zone 4
+                   w = 2.0*(wtw-0.5)
+                   s = 2.0*(1.0-wtn)
+                   f4 = data_in(is,je)
+                   if(is_missing(1)) then
+                      f1 = f4
+                   else
+                      f1 = 0.5*(data_in(is,js)+data_in(is,je))
+                   endif
+                   f2 = middle
+                   if(is_missing(3)) then
+                      f3 = f4
+                   else
+                      f3 = 0.5*(data_in(ie,je)+data_in(is,je))
+                   endif
+                else 
+                   call mpp_error(FATAL, &
+                      "horiz_interp_bilinear_mod: the point should be in one of the four zone")
+                endif
+             endif
+
+             data_out(m,n) = f3 + (f4-f3)*w + (f2-f3)*s + ((f1-f2)+(f3-f4))*w*s 
+            if(present(mask_out)) mask_out(m,n) = 1.0
+          enddo
+       enddo       
+    else
+       do n = 1, nlat_out
+          do m = 1, nlon_out
+             is = Interp % i_lon (m,n,1); ie = Interp % i_lon (m,n,2)
+             js = Interp % j_lat (m,n,1); je = Interp % j_lat (m,n,2)
+             wtw = Interp % wti   (m,n,1)
+             wte = Interp % wti   (m,n,2)
+             wts = Interp % wtj   (m,n,1)
+             wtn = Interp % wtj   (m,n,2)
+
+             if(present(missing_value) ) then
+                num_missing = 0
+                if(data_in(is,js) == missing_value) then
+                   num_missing = num_missing+1
+                   mask(is,js) = 0.0
+                endif
+                if(data_in(ie,js) == missing_value) then
+                   num_missing = num_missing+1
+                   mask(ie,js) = 0.0
+                endif
+                if(data_in(ie,je) == missing_value) then
+                   num_missing = num_missing+1
+                   mask(ie,je) = 0.0
+                endif
+                if(data_in(is,je) == missing_value) then
+                   num_missing = num_missing+1
+                   mask(is,je) = 0.0
+                endif
+             endif
+
+             dwtsum = data_in(is,js)*mask(is,js)*wtw*wts &
+                  + data_in(ie,js)*mask(ie,js)*wte*wts &
+                  + data_in(ie,je)*mask(ie,je)*wte*wtn &
+                  + data_in(is,je)*mask(is,je)*wtw*wtn 
+             wtsum  = mask(is,js)*wtw*wts + mask(ie,js)*wte*wts  &
+                  + mask(ie,je)*wte*wtn + mask(is,je)*wtw*wtn
+
+             if(.not. present(mask_in) .and. .not. present(missing_value)) wtsum = 1.0
 
-          if(num_missing .gt. max_missing ) then
-             data_out(m,n) = missing_value
-             if(present(mask_out)) mask_out(m,n) = 0.0
-          else if(wtsum .lt. epsln) then 
-             if(present(missing_value)) then
+             if(num_missing .gt. max_missing ) then
                 data_out(m,n) = missing_value
+                if(present(mask_out)) mask_out(m,n) = 0.0
+             else if(wtsum .lt. epsln) then 
+                if(present(missing_value)) then
+                   data_out(m,n) = missing_value
+                else
+                   data_out(m,n) = 0.0
+                endif
+                if(present(mask_out)) mask_out(m,n) = 0.0      
              else
-                data_out(m,n) = 0.0
+                data_out(m,n) = dwtsum/wtsum
+                if(present(mask_out)) mask_out(m,n) = wtsum
              endif
-             if(present(mask_out)) mask_out(m,n) = 0.0      
-          else
-             data_out(m,n) = dwtsum/wtsum
-             if(present(mask_out)) mask_out(m,n) = wtsum
-          endif
+          enddo
        enddo
-    enddo
+    endif
     !***********************************************************************
     ! compute statistics: minimum, maximum, and mean
     !-----------------------------------------------------------------------
diff --git a/src/shared/horiz_interp/horiz_interp_conserve.F90 b/src/shared/horiz_interp/horiz_interp_conserve.F90
index c705798499..a58d372cac 100644
--- a/src/shared/horiz_interp/horiz_interp_conserve.F90
+++ b/src/shared/horiz_interp/horiz_interp_conserve.F90
@@ -1,8 +1,9 @@
 module horiz_interp_conserve_mod
 
-  !  Bruce Wyman 
-  !  Zhi Liang 
+  !  Bruce Wyman 
+  !  Zhi Liang 
 
+  ! 
 
   ! 
   !   Performs spatial interpolation between grids using conservative interpolation
@@ -90,8 +91,8 @@ module horiz_interp_conserve_mod
 
   integer :: pe, root_pe
   !-----------------------------------------------------------------------
-  character(len=128) :: version = '$Id: horiz_interp_conserve.F90,v 20.0 2013/12/14 00:20:25 fms Exp $'
-  character(len=128) :: tagname = '$Name: tikal $'
+  character(len=128) :: version = '$Id$'
+  character(len=128) :: tagname = '$Name$'
   logical            :: module_is_initialized = .FALSE.
 
   logical         :: great_circle_algorithm = .false.
@@ -110,7 +111,7 @@ module horiz_interp_conserve_mod
   subroutine horiz_interp_conserve_init
 
     if(module_is_initialized) return
-    call write_version_number()
+    call write_version_number (version, tagname)
 
     great_circle_algorithm = get_great_circle_algorithm()
 
@@ -350,7 +351,7 @@ subroutine horiz_interp_conserve_new_1dx2d ( Interp, lon_in, lat_in, lon_out, la
     nlon_in  = size(lon_in(:)) - 1;  nlat_in  = size(lat_in(:)) - 1
     nlon_out = size(lon_out,1) - 1;  nlat_out = size(lon_out,2) - 1
 
-    mask_src = 1
+    mask_src = 1.
     if(present(mask_in)) then
        if( (size(mask_in,1) .NE. nlon_in) .OR.  (size(mask_in,2) .NE. nlat_in)) call mpp_error(FATAL, &
          'horiz_interp_conserve_mod: size mismatch between mask_in and lon_in/lat_in')
@@ -388,7 +389,7 @@ subroutine horiz_interp_conserve_new_1dx2d ( Interp, lon_in, lat_in, lon_out, la
     Interp%j_dst = j_dst(1:nxgrid)+1
 
     ! sum over exchange grid area to get destination grid area
-    dst_area = 0
+    dst_area = 0.
     do i = 1, nxgrid
        dst_area(Interp%i_dst(i), Interp%j_dst(i)) = dst_area(Interp%i_dst(i), Interp%j_dst(i)) + xgrid_area(i)       
     end do    
@@ -440,7 +441,7 @@ subroutine horiz_interp_conserve_new_2dx1d ( Interp, lon_in, lat_in, lon_out, la
     nlon_in  = size(lon_in,1)   - 1;  nlat_in  = size(lon_in,2)   - 1
     nlon_out = size(lon_out(:)) - 1;  nlat_out = size(lat_out(:)) - 1
 
-    mask_src = 1
+    mask_src = 1.
     if(present(mask_in)) then
        if( (size(mask_in,1) .NE. nlon_in) .OR.  (size(mask_in,2) .NE. nlat_in)) call mpp_error(FATAL, &
          'horiz_interp_conserve_mod: size mismatch between mask_in and lon_in/lat_in')
@@ -478,7 +479,7 @@ subroutine horiz_interp_conserve_new_2dx1d ( Interp, lon_in, lat_in, lon_out, la
     Interp%j_dst = j_dst(1:nxgrid)+1
 
     ! sum over exchange grid area to get destination grid area
-    dst_area = 0
+    dst_area = 0.
     do i = 1, nxgrid
        dst_area(Interp%i_dst(i), Interp%j_dst(i)) = dst_area(Interp%i_dst(i), Interp%j_dst(i)) + xgrid_area(i)       
     end do    
@@ -531,7 +532,7 @@ subroutine horiz_interp_conserve_new_2dx2d ( Interp, lon_in, lat_in, lon_out, la
     nlon_in  = size(lon_in,1)  - 1;  nlat_in  = size(lon_in,2)  - 1
     nlon_out = size(lon_out,1) - 1;  nlat_out = size(lon_out,2) - 1
 
-    mask_src = 1
+    mask_src = 1.
     if(present(mask_in)) then
        if( (size(mask_in,1) .NE. nlon_in) .OR.  (size(mask_in,2) .NE. nlat_in)) call mpp_error(FATAL, &
          'horiz_interp_conserve_mod: size mismatch between mask_in and lon_in/lat_in')
@@ -563,7 +564,7 @@ subroutine horiz_interp_conserve_new_2dx2d ( Interp, lon_in, lat_in, lon_out, la
     Interp%j_dst = j_dst(1:nxgrid)+1
 
     ! sum over exchange grid area to get destination grid area
-    dst_area = 0
+    dst_area = 0.
     do i = 1, nxgrid
        dst_area(Interp%i_dst(i), Interp%j_dst(i)) = dst_area(Interp%i_dst(i), Interp%j_dst(i)) + xgrid_area(i)       
     end do    
diff --git a/src/shared/horiz_interp/horiz_interp_spherical.F90 b/src/shared/horiz_interp/horiz_interp_spherical.F90
index 63e62a54e3..00b3c26491 100644
--- a/src/shared/horiz_interp/horiz_interp_spherical.F90
+++ b/src/shared/horiz_interp/horiz_interp_spherical.F90
@@ -1,8 +1,9 @@
 module horiz_interp_spherical_mod
 
-  !  Matthew Harrison 
-  !  Zhi Liang 
+  !  Matthew Harrison 
+  !  Zhi Liang 
 
+  ! 
 
   ! 
   !   Performs spatial interpolation between grids using inverse-distance-weighted scheme.
@@ -59,8 +60,8 @@ module horiz_interp_spherical_mod
   namelist /horiz_interp_spherical_nml/ search_method
 
   !-----------------------------------------------------------------------
-  character(len=128) :: version = '$Id: horiz_interp_spherical.F90,v 20.0 2013/12/14 00:20:28 fms Exp $'
-  character(len=128) :: tagname = '$Name: tikal $'
+  character(len=128) :: version = '$Id$'
+  character(len=128) :: tagname = '$Name$'
   logical            :: module_is_initialized = .FALSE.
 
 contains
@@ -79,7 +80,7 @@ subroutine horiz_interp_spherical_init
 
 
     if(module_is_initialized) return
-    call write_version_number()
+    call write_version_number (version, tagname)
 #ifdef INTERNAL_FILE_NML
       read (input_nml_file, horiz_interp_spherical_nml, iostat=io)
       ierr = check_nml_error(io,'horiz_interp_spherical_nml') 
@@ -171,10 +172,10 @@ subroutine horiz_interp_spherical_new(Interp, lon_in,lat_in,lon_out,lat_out, &
     logical :: src_is_modulo
     real    :: min_theta_dst, max_theta_dst, min_phi_dst, max_phi_dst
     real    :: min_theta_src, max_theta_src, min_phi_src, max_phi_src 
-    integer, dimension(:),        allocatable        :: ilon, jlat
-    integer, dimension(:,:,:),    allocatable        :: map_src_add
-    integer, dimension(:,:),      allocatable        :: num_found
-    real, dimension(:,:,:),       allocatable        :: map_src_dist
+    integer :: map_src_add(size(lon_out,1),size(lon_out,2),max_neighbors)
+    real    :: map_src_dist(size(lon_out,1),size(lon_out,2),max_neighbors)
+    integer :: num_found(size(lon_out,1),size(lon_out,2))
+    integer :: ilon(max_neighbors), jlat(max_neighbors)
     real, dimension(size(lon_out,1),size(lon_out,2)) :: theta_dst, phi_dst
     real, dimension(size(lon_in,1)*size(lon_in,2))   :: theta_src, phi_src
 
@@ -247,16 +248,19 @@ subroutine horiz_interp_spherical_new(Interp, lon_in,lat_in,lon_out,lat_out, &
        if (min_theta_dst < min_theta_src) print *, '=> WARNING : longitude of dest grid exceeds src'
        if (max_theta_dst > max_theta_src) print *, '=> WARNING : longitude of dest grid exceeds src'
     endif
-    allocate(map_src_add(map_dst_xsize,map_dst_ysize,max_neighbors),    &
-         map_src_dist(map_dst_xsize,map_dst_ysize,max_neighbors),   &
-         num_found(map_dst_xsize,map_dst_ysize),                    &
-         ilon(max_neighbors),jlat(max_neighbors)  )
 
     ! allocate memory to data type
-    allocate(Interp%i_lon(map_dst_xsize,map_dst_ysize,max_neighbors), &
-         Interp%j_lat(map_dst_xsize,map_dst_ysize,max_neighbors),     &
-         Interp%src_dist(map_dst_xsize,map_dst_ysize,max_neighbors),  &
-         Interp%num_found(map_dst_xsize,map_dst_ysize)       )
+    if(ASSOCIATED(Interp%i_lon)) then
+       if(size(Interp%i_lon,1) .NE. map_dst_xsize .OR.     &
+          size(Interp%i_lon,2) .NE. map_dst_ysize )  call mpp_error(FATAL, &
+          'horiz_interp_spherical_mod: size(Interp%i_lon(:),1) .NE. map_dst_xsize .OR. '// &
+          'size(Interp%i_lon(:),2) .NE. map_dst_ysize')
+    else
+       allocate(Interp%i_lon(map_dst_xsize,map_dst_ysize,max_neighbors), &
+            Interp%j_lat(map_dst_xsize,map_dst_ysize,max_neighbors),     &
+            Interp%src_dist(map_dst_xsize,map_dst_ysize,max_neighbors),  &
+            Interp%num_found(map_dst_xsize,map_dst_ysize)       )
+    endif
 
     map_src_add         = 0
     map_src_dist        = large
@@ -300,7 +304,6 @@ subroutine horiz_interp_spherical_new(Interp, lon_in,lat_in,lon_out,lat_out, &
     Interp%nlon_src = map_src_xsize; Interp%nlat_src = map_src_ysize
     Interp%nlon_dst = map_dst_xsize; Interp%nlat_dst = map_dst_ysize
 
-    deallocate(map_src_add, map_src_dist, ilon, jlat)
     return
 
   end subroutine horiz_interp_spherical_new
@@ -944,8 +947,8 @@ function spherical_distance(theta1,phi1,theta2,phi2)
     endif
   
     dot = cos(phi1)*cos(phi2)*cos(theta1-theta2) + sin(phi1)*sin(phi2)
-    if(dot > 1 ) dot = 1.
-    if(dot < -1) dot = -1
+    if(dot > 1. ) dot = 1.
+    if(dot < -1.) dot = -1.
     spherical_distance = acos(dot)
 
     return
diff --git a/src/shared/horiz_interp/horiz_interp_type.F90 b/src/shared/horiz_interp/horiz_interp_type.F90
index 5f8080e4d0..7f2a41dbf9 100644
--- a/src/shared/horiz_interp/horiz_interp_type.F90
+++ b/src/shared/horiz_interp/horiz_interp_type.F90
@@ -23,8 +23,9 @@
 !!                                                                   !!
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 module horiz_interp_type_mod
-!  Zhi Liang 
+!  Zhi Liang 
 
+! 
 
 ! 
 !     define derived data type that contains indices and weights used for subsequent 
@@ -146,7 +147,7 @@ subroutine stats ( dat, low, high, avg, miss, missing_value, mask )
          miss = miss + buffer_int(1)
          npts = npts + buffer_int(2)
       enddo         
-      if(npts == 0) then
+      if(npts == 0.) then
          print*, 'Warning: no points is valid'
       else
          avg = dsum/real(npts)
diff --git a/src/shared/horiz_interp/test_horiz_interp.F90 b/src/shared/horiz_interp/test_horiz_interp.F90
new file mode 100644
index 0000000000..9efa898ed4
--- /dev/null
+++ b/src/shared/horiz_interp/test_horiz_interp.F90
@@ -0,0 +1,251 @@
+#ifdef TEST_HORIZ_INTERP
+!z1l: currently only test bilinear interpolation.
+
+program test
+
+  use mpp_mod,          only : mpp_error, mpp_pe,  mpp_npes, mpp_root_pe
+  use mpp_mod,          only : FATAL, stdout, stdlog
+  use mpp_io_mod,       only : mpp_open, mpp_close, mpp_read
+  use mpp_io_mod,       only : axistype, fieldtype
+  use mpp_io_mod,       only : mpp_get_info, mpp_get_fields, mpp_get_times
+  use mpp_io_mod,       only : mpp_get_axes, mpp_get_axis_data, mpp_get_atts
+  use mpp_io_mod,       only : MPP_RDONLY, MPP_NETCDF, MPP_MULTI, MPP_SINGLE, MPP_OVERWR
+  use mpp_io_mod,       only : mpp_get_att_name, mpp_get_att_char, mpp_get_att_type, mpp_get_att_real
+  use mpp_io_mod,       only : mpp_write_meta, axistype, fieldtype, mpp_write, mpp_close 
+  use mpp_domains_mod,  only : mpp_update_domains, mpp_define_domains, domain1d
+  use mpp_domains_mod,  only : domain2d, mpp_define_layout, mpp_get_compute_domain
+  use mpp_domains_mod,  only : mpp_get_domain_components
+  use horiz_interp_mod, only : horiz_interp_new, horiz_interp, horiz_interp_end, horiz_interp_type
+  use horiz_interp_mod, only : horiz_interp_init
+  use axis_utils_mod,   only : get_axis_cart
+  use fms_io_mod,       only : read_data, write_data
+  use fms_io_mod,       only : field_size, fms_io_exit
+  use fms_mod,          only : fms_init, fms_end, open_namelist_file, close_file, file_exist, field_exist
+  use fms_mod,          only : check_nml_error, write_version_number, lowercase
+  use constants_mod,    only : constants_init, PI
+  use horiz_interp_mod, only : horiz_interp_new, horiz_interp, horiz_interp_end, horiz_interp_type
+
+implicit none
+
+  character(len=256) :: src_file = ""
+  character(len=256) :: dst_grid = "INPUT/grid_spec.nc"
+  character(len=256) :: field_name = ""
+  character(len=256) :: dst_file = "output.nc"
+  logical            :: new_missing_handle = .false.
+  character(len=256) :: interp_method = "bilinear"
+
+  real, allocatable, dimension(:)            :: x_src, y_src
+  real, allocatable, dimension(:,:)          :: x_dst, y_dst
+  type(axistype), allocatable, dimension(:)  :: axes
+  type(fieldtype), allocatable, dimension(:) :: fields
+  type(domain2d)                             :: Domain
+  integer :: unit, ierr, io, src_unit, src_field_index, nk
+  integer :: nx_src, ny_src, nx_dst, ny_dst, is, ie, js, je
+  real    :: missing_value
+  integer :: n, ntimes
+
+  namelist /test_horiz_interp_nml/ src_file, field_name, dst_file, dst_grid, new_missing_handle, &
+           interp_method
+
+  call fms_init
+  call horiz_interp_init
+  call constants_init
+
+#ifdef INTERNAL_FILE_NML
+  read (input_nml_file, test_horiz_interp_nml, iostat=io)
+  ierr = check_nml_error(io, 'test_horiz_interp_nml')
+#else
+  if (file_exist('input.nml')) then
+     unit = open_namelist_file ( )
+     ierr=1
+     do while (ierr /= 0)
+        read(unit, nml=test_horiz_interp_nml, iostat=io, end=10)
+        ierr = check_nml_error(io, 'test_horiz_interp_nml')
+     enddo
+10   call close_file (unit)
+  endif
+#endif
+
+
+  if( .not. file_exist(src_file) ) call mpp_error(FATAL, &
+       "test_horiz_interp: src_file = "//trim(src_file)//" does not exist")
+  if( .not. field_exist(src_file, field_name) ) call mpp_error(FATAL, &
+       "test_horiz_interp: field_name = "//trim(field_name)//" does not exist in file "//trim(src_file) )
+
+
+  ! reading the grid information and missing value from src_file
+  call mpp_open(src_unit, trim(src_file), &
+         action=MPP_RDONLY, form=MPP_NETCDF, threading=MPP_MULTI, fileset=MPP_SINGLE)
+  call read_src_file()
+
+  ! reading the dst_grid file
+  call read_dst_grid()
+
+  !--- currently only test for first time level. The following will read the input data,
+  !--- do the remapping and write out data
+  call process_data()
+  
+  call mpp_close(src_unit)
+
+  call fms_io_exit
+
+  call fms_end()
+
+contains
+
+
+  ! read the input data, do the remapping and write out data
+  subroutine process_data()
+     real, allocatable, dimension(:,:,:) :: src_data, dst_data
+     type(horiz_interp_type)             :: Interp
+     type(axistype)     :: xaxis, yaxis, zaxis, taxis
+     type(domain1D)     :: xdom, ydom
+     type(fieldtype)    :: field
+     real    :: D2R
+     integer :: k, i
+
+     call mpp_get_domain_components( domain, xdom, ydom )
+     !--- set up meta data
+     call mpp_open(unit,dst_file,action=MPP_OVERWR,form=MPP_NETCDF,threading=MPP_MULTI, fileset=MPP_MULTI)
+     call mpp_write_meta( unit, xaxis, 'lon', 'none', 'X index', 'X', domain=xdom, data=(/(i-1.,i=1,nx_dst)/) )
+     call mpp_write_meta( unit, yaxis, 'lat', 'none', 'Y index', 'Y', domain=ydom, data=(/(i-1.,i=1,ny_dst)/) )
+     call mpp_write_meta( unit, zaxis, 'level', 'none', 'Z index', 'Z', data=(/(i-1.,i=1,nk)/) )
+     call mpp_write_meta( unit, taxis, 'time', 'none', 'T index', 'T' )
+     call mpp_write_meta( unit, field, (/xaxis, yaxis, zaxis, taxis/), field_name, 'none', 'none')
+     call mpp_write( unit, xaxis )
+     call mpp_write( unit, yaxis )
+     call mpp_write( unit, zaxis )
+
+
+     D2R = PI/180.
+
+     allocate(src_data(nx_src,ny_src,nk))
+     allocate(dst_data(is:ie,js:je,nk))
+
+     call horiz_interp_new(Interp, x_src*D2R, y_src*D2R, x_dst*D2R, y_dst*D2R, &
+          interp_method = trim(interp_method), grid_at_center = .true. )    
+
+     do n = 1, ntimes
+        call mpp_read(src_unit,fields(src_field_index),src_data, tindex=n)
+
+        do k = 1, nk
+           call horiz_interp(interp, src_data(:,:,k), dst_data(:,:,k), &
+                missing_value=missing_value, new_missing_handle=new_missing_handle)
+        enddo
+        call mpp_write(unit, field, domain, dst_data, n*1.0)
+     enddo
+
+     call mpp_close(unit)
+     deallocate(src_data, dst_data)
+  
+  end subroutine process_data
+
+
+  subroutine read_dst_grid
+    integer :: layout(2), start(4), nread(4)
+    character(len=256) :: tile_file
+    integer :: i, j, siz(4)
+    real, allocatable :: tmp(:,:)
+
+    if (field_exist(dst_grid, "gridfiles" )) then
+       call read_data(dst_grid, "gridfiles", tile_file)
+       tile_file = 'INPUT/'//trim(tile_file)
+    else
+       call mpp_error(FATAL, "test_horiz_interp: field gridfiles does not exist in file "//trim(dst_grid) )
+    endif
+
+    call field_size(tile_file, "x", siz)
+    nx_dst = (siz(1)-1)/2
+    ny_dst = (siz(2)-1)/2
+
+    if(layout(1)*layout(2) .NE. mpp_npes() ) then
+       call mpp_define_layout( (/1,nx_dst,1,ny_dst/), mpp_npes(), layout )
+    end if
+
+    call mpp_define_domains( (/1,nx_dst,1,ny_dst/), layout, Domain, name='test_data_override')
+
+    call mpp_get_compute_domain(Domain, is, ie, js, je)
+
+    allocate(tmp(2*is-1:2*ie+1,2*js-1:2*je+1))
+    allocate(x_dst(is:ie,js:je), y_dst(is:ie,js:je))
+ 
+    start = 1; nread = 1
+    start(1) = 2*is-1; nread(1) = 2*(ie-is+1)+1
+    start(2) = 2*js-1; nread(2) = 2*(je-js+1)+1
+
+    call read_data(tile_file, "x", tmp, start, nread, domain) 
+    do j = js, je
+       do i = is, ie
+          x_dst(i,j) = tmp(2*i,2*j)
+       enddo
+    enddo
+    call read_data(tile_file, "y", tmp, start, nread, domain)   
+    do j = js, je
+       do i = is, ie
+          y_dst(i,j) = tmp(2*i,2*j)
+       enddo
+    enddo
+
+    deallocate(tmp)
+
+  end subroutine read_dst_grid
+
+  subroutine read_src_file
+
+    integer                                    :: ndim, nvar, natt, n
+    integer                                    :: nt, i, j, k, jj, len1
+    character(len=1)                           :: cart
+    character(len=32)                          :: name
+
+    call mpp_get_info(src_unit, ndim, nvar, natt, ntimes)
+    
+    allocate(fields(nvar))
+    call mpp_get_fields(src_unit, fields)
+    src_field_index = 0
+    do i=1,nvar
+       call mpp_get_atts(fields(i),name=name)
+       if (lowercase(trim(field_name)) == lowercase(trim(name))) then
+             src_field_index = i
+       endif
+    enddo
+    if(src_field_index == 0) call mpp_error(FATAL, 'test_horiz_interp: field '&
+            //trim(field_name)//' is not in the file '//trim(src_file) )
+    !--- get the src grid
+    call mpp_get_atts(fields(src_field_index),ndim=ndim)
+    allocate(axes(ndim))
+    call mpp_get_atts(fields(src_field_index),axes=axes)
+    nx_src=0; ny_src=0; nk=1
+    do j=1,ndim
+       call mpp_get_atts(axes(j),len=len1)
+       call get_axis_cart(axes(j),cart)
+       select case (cart)
+       case ('X')
+          nx_src = len1
+          allocate(x_src(nx_src))
+          call mpp_get_axis_data(axes(j),x_src)
+       case('Y')
+          ny_src = len1
+          allocate(y_src(ny_src))
+          call mpp_get_axis_data(axes(j),y_src)
+       case('Z')
+          nk = len1
+       end select
+    enddo
+    if(nx_src==0) call mpp_error(FATAL,'test_horiz_interp: file ' &
+         //trim(src_file)//' does not contain axis with cartesian attributes = "X" ')
+    if(ny_src==0) call mpp_error(FATAL,'test_horiz_interp: file '&
+         //trim(src_file)//' does not contain axis with cartesian attributes = "Y" ')
+
+    !--- get the missing value
+    call mpp_get_atts(fields(src_field_index),missing=missing_value)
+
+  end subroutine read_src_file
+
+
+end program test
+
+#else
+module null_test_horiz_interp
+end module  
+
+#endif 
diff --git a/src/shared/memutils/memutils.F90 b/src/shared/memutils/memutils.F90
index 1f8ab610bb..80771b9f56 100644
--- a/src/shared/memutils/memutils.F90
+++ b/src/shared/memutils/memutils.F90
@@ -1,5 +1,5 @@
 module memutils_mod
-!Author: Balaji (V.Balaji)
+!Author: Balaji (V.Balaji@noaa.gov)
 !Various operations for memory management
 !these currently include efficient methods for memory-to-memory copy
 !including strided data and arbitrary gather-scatter vectors
diff --git a/src/shared/mosaic/gradient.F90 b/src/shared/mosaic/gradient.F90
index 446a089d41..d41af3035f 100644
--- a/src/shared/mosaic/gradient.F90
+++ b/src/shared/mosaic/gradient.F90
@@ -1,8 +1,9 @@
 module gradient_mod
-! 
+! 
 !   Zhi Liang
 ! 
 
+! 
 
 ! 
 !    gradient_mod implements some utility routines to calculate gradient.
@@ -23,8 +24,8 @@ module gradient_mod
 public :: gradient_cubic
 public :: calc_cubic_grid_info
 
-character(len=128) :: version = '$Id: gradient.F90,v 16.0 2008/07/30 22:46:00 fms Exp $'
-character(len=128) :: tagname = '$Name: tikal $'
+character(len=128) :: version = '$Id$'
+character(len=128) :: tagname = '$Name$'
 
 contains
 
diff --git a/src/shared/mosaic/grid.F90 b/src/shared/mosaic/grid.F90
index d9b604b6fa..262efd14a4 100644
--- a/src/shared/mosaic/grid.F90
+++ b/src/shared/mosaic/grid.F90
@@ -48,8 +48,8 @@ module grid_mod
 ! ==== module constants ======================================================
 character(len=*), parameter :: &
      module_name = 'grid_mod', &
-     version     = '$Id: grid.F90,v 20.0 2013/12/14 00:20:48 fms Exp $', &
-     tagname     = '$Name: tikal $'
+     version     = '$Id$', &
+     tagname     = '$Name$'
 
 character(len=*), parameter :: &
      grid_dir  = 'INPUT/',     &      ! root directory for all grid files
@@ -62,6 +62,9 @@ module grid_mod
      VERSION_1 = 1,   &
      VERSION_2 = 2
 
+integer, parameter :: BUFSIZE = 1048576  ! This is used to control memory usage in get_grid_comp_area
+                                         ! We may change this to a namelist variable is needed.
+
 ! ==== module variables ======================================================
 integer :: grid_version = -1
 logical :: great_circle_algorithm = .FALSE.
@@ -194,7 +197,7 @@ subroutine get_grid_cell_area(component, tile, cellarea, domain)
              FATAL)
      end select
      ! convert area to m2
-     cellarea = cellarea*4*PI*radius**2
+     cellarea = cellarea*4.*PI*radius**2
   case(VERSION_2)
      if (present(domain)) then
         call mpp_get_compute_domain(domain,xsize=nlon,ysize=nlat)
@@ -244,6 +247,7 @@ subroutine get_grid_comp_area(component,tile,area,domain)
   integer :: num_nest_tile, ntiles
   logical :: is_nest
   integer :: found_xgrid_files ! how many xgrid files we actually found in the grid spec
+  integer :: ibegin, iend, bsize, l
 
   select case (get_grid_version())
   case(VERSION_0,VERSION_1)
@@ -315,7 +319,7 @@ subroutine get_grid_comp_area(component,tile,area,domain)
            endif
         end if
      end do
-     area(:,:) = 0
+     area(:,:) = 0.
      if(field_exist(grid_file,xgrid_name)) then
         ! get the number of the exchange-grid files
         call field_size(grid_file,xgrid_name,siz)
@@ -343,15 +347,26 @@ subroutine get_grid_comp_area(component,tile,area,domain)
 
            ! finally read the exchange grid
            nxgrid = get_mosaic_xgrid_size(grid_dir//xgrid_file)
-           allocate(i1(nxgrid), j1(nxgrid), i2(nxgrid), j2(nxgrid), xgrid_area(nxgrid))
-           call get_mosaic_xgrid(grid_dir//xgrid_file, i1, j1, i2, j2, xgrid_area)
-           ! and sum the exchange grid areas
-           do m = 1, nxgrid
-              i = i2(m); j = j2(m)
-              if (iie) cycle
-              if (jje) cycle
-              area(i+i0,j+j0) = area(i+i0,j+j0) + xgrid_area(m)
-           end do
+           if(nxgrid < BUFSIZE) then
+              allocate(i1(nxgrid), j1(nxgrid), i2(nxgrid), j2(nxgrid), xgrid_area(nxgrid))
+           else
+              allocate(i1(BUFSIZE), j1(BUFSIZE), i2(BUFSIZE), j2(BUFSIZE), xgrid_area(BUFSIZE))
+           endif
+           ibegin = 1
+           do l = 1,nxgrid,BUFSIZE
+              bsize = min(BUFSIZE, nxgrid-l+1)
+              iend = ibegin + bsize - 1
+              call get_mosaic_xgrid(grid_dir//xgrid_file, i1(1:bsize), j1(1:bsize), i2(1:bsize), j2(1:bsize), &
+                                    xgrid_area(1:bsize), ibegin, iend)
+              ! and sum the exchange grid areas
+              do m = 1, bsize
+                 i = i2(m); j = j2(m)
+                 if (iie) cycle
+                 if (jje) cycle
+                 area(i+i0,j+j0) = area(i+i0,j+j0) + xgrid_area(m)
+              end do
+              ibegin = iend + 1
+           enddo
            deallocate(i1, j1, i2, j2, xgrid_area)
         enddo
         if (found_xgrid_files == 0) &
@@ -362,7 +377,7 @@ subroutine get_grid_comp_area(component,tile,area,domain)
      deallocate(nest_tile_name)
   end select ! version
   ! convert area to m2
-  area = area*4*PI*radius**2
+  area = area*4.*PI*radius**2
 end subroutine
 
 ! ============================================================================
@@ -446,7 +461,7 @@ subroutine get_grid_cell_vertices_1D(component, tile, glonb, glatb)
 
      start = 1; nread = 1
      nread(2) = 2*nlat+1
-     call read_data(filename2, "x", tmp, start, nread, no_domain=.TRUE.)
+     call read_data(filename2, "y", tmp, start, nread, no_domain=.TRUE.)
      glatb(1:nlat+1) = tmp(1,1:2*nlat+1:2)
      deallocate(tmp)
   end select
@@ -821,11 +836,12 @@ end subroutine get_grid_cell_centers_2D
 ! domain for current processor
 ! ============================================================================
 ! this subroutine probably does not belong in the grid_mod 
-subroutine define_cube_mosaic ( component, domain, layout, halo )
+subroutine define_cube_mosaic ( component, domain, layout, halo, maskmap )
   character(len=*) , intent(in)    :: component
   type(domain2d)   , intent(inout) :: domain
   integer          , intent(in)    :: layout(2)
   integer, optional, intent(in)    :: halo 
+  logical, optional, intent(in)    :: maskmap(:,:,:)
 
   ! ---- local constants
   
@@ -835,7 +851,7 @@ subroutine define_cube_mosaic ( component, domain, layout, halo )
   integer :: ntiles     ! number of tiles
   integer :: ncontacts  ! number of contacts between mosaic tiles
   integer :: n
-  integer :: ng         ! halo size
+  integer :: ng, pe_pos, npes         ! halo size
   integer, allocatable :: nlon(:), nlat(:), global_indices(:,:)
   integer, allocatable :: pe_start(:), pe_end(:), layout_2d(:,:)
   integer, allocatable :: tile1(:),tile2(:)
@@ -849,11 +865,18 @@ subroutine define_cube_mosaic ( component, domain, layout, halo )
   allocate(layout_2d(2,ntiles))
   call get_grid_size(component,nlon,nlat)
 
+  pe_pos = mpp_root_pe()
   do n = 1, ntiles
      global_indices(:,n) = (/ 1, nlon(n), 1, nlat(n) /)
      layout_2d     (:,n) = layout
-     pe_start        (n) = mpp_root_pe() + (n-1)*layout(1)*layout(2)
-     pe_end          (n) = mpp_root_pe() +     n*layout(1)*layout(2) - 1
+     if(present(maskmap)) then
+        npes = count(maskmap(:,:,n))
+     else
+        npes = layout(1)*layout(2)
+     endif
+     pe_start(n) = pe_pos
+     pe_end  (n) = pe_pos + npes - 1
+     pe_pos      = pe_end(n) + 1
   enddo
 
   varname=trim(lowercase(component))//'_mosaic_file'
@@ -877,6 +900,7 @@ subroutine define_cube_mosaic ( component, domain, layout, halo )
        is2, ie2, js2, je2, &
        pe_start=pe_start, pe_end=pe_end, symmetry=.true.,  &
        shalo = ng, nhalo = ng, whalo = ng, ehalo = ng,     &
+       maskmap = maskmap,                                  &
        name = trim(component)//'Cubic-Sphere Grid' )
 
   deallocate(nlon,nlat,global_indices,pe_start,pe_end,layout_2d)
diff --git a/src/shared/mosaic/interp.h b/src/shared/mosaic/interp.h
index d6b0543d78..d704bee065 100644
--- a/src/shared/mosaic/interp.h
+++ b/src/shared/mosaic/interp.h
@@ -8,7 +8,7 @@
 /*********************************************************************
                      interp.h
    This header files contains defition of some interpolation routine  (1-D or 2-D).
-   contact: Zhi.Liang
+   contact: Zhi.Liang@noaa.gov
 *********************************************************************/
 void cubic_spline_sp(int size1, int size2, const double *grid1, const double *grid2, const double *data1,
                   double *data2 );
diff --git a/src/shared/mosaic/mosaic.F90 b/src/shared/mosaic/mosaic.F90
index 8645be6847..794c8219a0 100644
--- a/src/shared/mosaic/mosaic.F90
+++ b/src/shared/mosaic/mosaic.F90
@@ -1,9 +1,10 @@
 module mosaic_mod
 
-! 
+! 
 !   Zhi Liang
 ! 
 
+! 
 
 ! 
 !    mosaic_mod implements some utility routines to read mosaic information.
@@ -16,11 +17,22 @@ module mosaic_mod
 !    grid information. Each routine will call a C-version routine to get these information.
 ! 
 
-use mpp_mod, only : mpp_error, FATAL
+use mpp_mod,    only : mpp_error, FATAL, mpp_pe, mpp_root_pe
+use mpp_io_mod, only : MPP_MULTI
+use fms_io_mod, only : dimension_size, field_exist, read_data, read_compressed
+use constants_mod, only : PI, RADIUS
 
 implicit none
 private
 
+character(len=*), parameter :: &
+     grid_dir  = 'INPUT/'      ! root directory for all grid files
+
+integer, parameter :: &
+     MAX_NAME = 256,  & ! max length of the variable names
+     MAX_FILE = 1024, & ! max length of the file names
+     X_REFINE = 2,    & ! supergrid size/model grid size in x-direction
+     Y_REFINE = 2       ! supergrid size/model grid size in y-direction
 
 ! --- public interface
 
@@ -37,8 +49,8 @@ module mosaic_mod
 
 logical :: module_is_initialized = .true.
 ! version information varaible
- character(len=128) :: version = '$Id: mosaic.F90,v 20.0 2013/12/14 00:22:25 fms Exp $'
- character(len=128) :: tagname = '$Name: tikal $'
+ character(len=128) :: version = '$Id$'
+ character(len=128) :: tagname = '$Name$'
 
 contains
 
@@ -61,7 +73,7 @@ subroutine mosaic_init()
   module_is_initialized = .TRUE.
 
 !--------- write version number and namelist ------------------
-!  call write_version_number()
+!  call write_version_number (version, tagname)
 
 end subroutine mosaic_init
 ! 
@@ -83,17 +95,8 @@ end subroutine mosaic_init
   function get_mosaic_xgrid_size(xgrid_file)
     character(len=*), intent(in)          :: xgrid_file
     integer                               :: get_mosaic_xgrid_size
-    character(len=len_trim(xgrid_file)+1) :: xfile    
-    integer                               :: read_mosaic_xgrid_size
-    integer                               :: strlen
-
-    !---- transfer to C-stype string
-    strlen = len_trim(xgrid_file)
-    xfile(1:strlen) = xgrid_file(1:strlen)
-    strlen = strlen+1
-    xfile(strlen:strlen) = CHAR(0)
 
-    get_mosaic_xgrid_size = read_mosaic_xgrid_size(xfile)
+    get_mosaic_xgrid_size = dimension_size(xgrid_file, "ncells", no_domain=.TRUE.)
 
     return   
 
@@ -125,46 +128,53 @@ end function get_mosaic_xgrid_size
 !   
 !     area of the exchange grid. The area is scaled to represent unit earth area.
 !   
-  subroutine get_mosaic_xgrid(xgrid_file, i1, j1, i2, j2, area, di, dj, istart, iend)
+  subroutine get_mosaic_xgrid(xgrid_file, i1, j1, i2, j2, area, ibegin, iend)
     character(len=*), intent(in) :: xgrid_file
     integer,       intent(inout) :: i1(:), j1(:), i2(:), j2(:)
     real,          intent(inout) :: area(:)
-    real, optional,intent(inout) :: di(:), dj(:)
-    integer, optional,intent(in) :: istart, iend 
-
-    character(len=len_trim(xgrid_file)+1) :: xfile
-    integer :: n, strlen, nxgrid
-
-    !---- transfer to C-stype string
-    strlen = len_trim(xgrid_file)
-    xfile(1:strlen) = xgrid_file(1:strlen)
-    strlen = strlen+1
-    xfile(strlen:strlen) = CHAR(0)
-
-    !--- order 2 xgrid will be implemented later 
-    nxgrid = size(i1(:))
-
-    if(PRESENT(di)) then
-       if( PRESENT(istart) .OR. PRESENT(iend) ) &
-          call mpp_error(FATAL, "mosaic_mod: istart and iend should not present when di is present, contact developer")
-       if(.NOT. PRESENT(dj) ) call mpp_error(FATAL, "mosaic_mod: when di is present, dj should be present")
-       call read_mosaic_xgrid_order2(xfile, i1, j1, i2, j2, area, di, dj)
+    integer, optional, intent(in) :: ibegin, iend
+
+    integer                            :: start(4), nread(4), istart
+    real,    dimension(2, size(i1(:))) :: tile1_cell, tile2_cell
+    integer                            :: nxgrid, n
+    real                               :: garea
+    real                               :: get_global_area;
+
+    garea = get_global_area();
+
+    ! When start and nread present, make sure nread(1) is the same as the size of the data
+    if(present(ibegin) .and. present(iend)) then
+       istart = ibegin
+       nxgrid = iend - ibegin + 1
+       if(nxgrid .NE. size(i1(:))) call mpp_error(FATAL, "get_mosaic_xgrid: nxgrid .NE. size(i1(:))")
+       if(nxgrid .NE. size(j1(:))) call mpp_error(FATAL, "get_mosaic_xgrid: nxgrid .NE. size(j1(:))")
+       if(nxgrid .NE. size(i2(:))) call mpp_error(FATAL, "get_mosaic_xgrid: nxgrid .NE. size(i2(:))")
+       if(nxgrid .NE. size(j2(:))) call mpp_error(FATAL, "get_mosaic_xgrid: nxgrid .NE. size(j2(:))")
+       if(nxgrid .NE. size(area(:))) call mpp_error(FATAL, "get_mosaic_xgrid: nxgrid .NE. size(area(:))")
     else
-       if( PRESENT(istart) .AND. PRESENT(iend) ) then
-          if( iend-istart+1 .NE. nxgrid) call mpp_error(FATAL, "mosaic_mod: iend-istart+1 must equal size(i1)")
-          call read_mosaic_xgrid_order1_region(xfile, i1, j1, i2, j2, area, istart-1, iend-1) ! convert to c-index
-       else
-          call read_mosaic_xgrid_order1(xfile, i1, j1, i2, j2, area)
-       endif
-    end if
-
-    ! in C, programming, the starting index is 0, so need add 1 to the index.
-    do n = 1, nxgrid
-       i1(n) = i1(n) + 1
-       j1(n) = j1(n) + 1
-       i2(n) = i2(n) + 1
-       j2(n) = j2(n) + 1
+       istart = 1
+       nxgrid = size(i1(:))
+    endif
+
+    start  = 1; nread = 1
+    start(1) = istart; nread(1) = nxgrid
+    call read_compressed(xgrid_file, 'xgrid_area', area, start=start, nread=nread, threading=MPP_MULTI)
+    start = 1; nread = 1
+    nread(1) = 2
+    start(2) = istart; nread(2) = nxgrid
+    call read_compressed(xgrid_file, 'tile1_cell', tile1_cell, start=start, nread=nread, threading=MPP_MULTI)
+    call read_compressed(xgrid_file, 'tile2_cell', tile2_cell, start=start, nread=nread, threading=MPP_MULTI)
+
+     do n = 1, nxgrid
+       i1(n) = tile1_cell(1,n) 
+       j1(n) = tile1_cell(2,n)
+       i2(n) = tile2_cell(1,n) 
+       j2(n) = tile2_cell(2,n)
+       area(n) = area(n)/garea
     end do
+    
+    return
+
   end subroutine get_mosaic_xgrid
 ! 
 
@@ -186,17 +196,9 @@ function get_mosaic_ntiles(mosaic_file)
     character(len=*), intent(in) :: mosaic_file
     integer                      :: get_mosaic_ntiles 
 
-    character(len=len_trim(mosaic_file)+1) :: mfile    
-    integer                                :: strlen
-    integer                                :: read_mosaic_ntiles
+    get_mosaic_ntiles = dimension_size(mosaic_file, "ntiles")
 
-    !---- transfer to C-stype string
-    strlen = len_trim(mosaic_file)
-    mfile(1:strlen) = mosaic_file(1:strlen)
-    strlen = strlen+1
-    mfile(strlen:strlen) = CHAR(0)
-
-    get_mosaic_ntiles = read_mosaic_ntiles(mfile)
+    return
 
   end function get_mosaic_ntiles
 ! 
@@ -223,13 +225,13 @@ function get_mosaic_ncontacts( mosaic_file)
     integer                                :: strlen
     integer                                :: read_mosaic_ncontacts
 
-    !---- transfer to C-stype string
-    strlen = len_trim(mosaic_file)
-    mfile(1:strlen) = mosaic_file(1:strlen)
-    strlen = strlen+1
-    mfile(strlen:strlen) = CHAR(0)
+    if(field_exist(mosaic_file, "contacts") ) then
+      get_mosaic_ncontacts = dimension_size(mosaic_file, "ncontact", no_domain=.TRUE.)
+    else
+      get_mosaic_ncontacts = 0
+    endif
 
-    get_mosaic_ncontacts = read_mosaic_ncontacts(mfile)
+    return
 
   end function get_mosaic_ncontacts
 ! 
@@ -259,16 +261,25 @@ subroutine get_mosaic_grid_sizes( mosaic_file, nx, ny)
     character(len=*),         intent(in) :: mosaic_file
     integer, dimension(:), intent(inout) :: nx, ny
 
-    character(len=len_trim(mosaic_file)+1) :: mfile    
-    integer                                :: strlen
-
-    !---- transfer to C-stype string
-    strlen = len_trim(mosaic_file)
-    mfile(1:strlen) = mosaic_file(1:strlen)
-    strlen = strlen+1
-    mfile(strlen:strlen) = CHAR(0)
-
-    call read_mosaic_grid_sizes(mfile, nx, ny)
+    character(len=MAX_FILE) :: gridfile
+    integer                 :: ntiles, n    
+
+    ntiles = get_mosaic_ntiles(mosaic_file)
+    if(ntiles .NE. size(nx(:)) .OR. ntiles .NE. size(ny(:)) ) then
+      call mpp_error(FATAL, "get_mosaic_grid_sizes: size of nx/ny does not equal to ntiles")
+    endif
+    do n = 1, ntiles
+      call read_data(mosaic_file, 'gridfiles', gridfile, level=n)
+      gridfile = grid_dir//trim(gridfile)
+      nx(n) = dimension_size(gridfile, "nx")
+      ny(n) = dimension_size(gridfile, "ny")
+      if(mod(nx(n),x_refine) .NE. 0) call mpp_error(FATAL, "get_mosaic_grid_sizes: nx is not divided by x_refine");
+      if(mod(ny(n),y_refine) .NE. 0) call mpp_error(FATAL, "get_mosaic_grid_sizes: ny is not divided by y_refine");
+      nx(n) = nx(n)/x_refine;
+      ny(n) = ny(n)/y_refine;
+    enddo
+
+    return
 
   end subroutine get_mosaic_grid_sizes
 ! 
@@ -324,30 +335,141 @@ subroutine get_mosaic_contact( mosaic_file, tile1, tile2, istart1, iend1, jstart
     integer, dimension(:), intent(inout) :: tile1, tile2
     integer, dimension(:), intent(inout) :: istart1, iend1, jstart1, jend1
     integer, dimension(:), intent(inout) :: istart2, iend2, jstart2, jend2
-    character(len=len_trim(mosaic_file)+1) :: mfile    
-    integer                                :: strlen
-
-    !---- transfer to C-stype string
-    strlen = len_trim(mosaic_file)
-    mfile(1:strlen) = mosaic_file(1:strlen)
-    strlen = strlen+1
-    mfile(strlen:strlen) = CHAR(0)
-
-    call read_mosaic_contact(mfile, tile1, tile2, istart1, iend1, jstart1, jend1, &
-                            istart2, iend2, jstart2, jend2)
-    !--- transfer C-index to Fortran-index.
-    istart1 = istart1 + 1
-    iend1   = iend1   + 1
-    jstart1 = jstart1 + 1
-    jend1   = jend1   + 1
-    istart2 = istart2 + 1
-    iend2   = iend2   + 1
-    jstart2 = jstart2 + 1
-    jend2   = jend2   + 1
+    character(len=MAX_NAME), allocatable :: gridtiles(:)
+    character(len=MAX_NAME)              :: contacts
+    character(len=MAX_NAME)              :: strlist(8)
+    integer :: ntiles, n, m, ncontacts, nstr, ios
+    integer :: i1_type, j1_type, i2_type, j2_type
+    logical :: found
+
+    ntiles = get_mosaic_ntiles(mosaic_file)
+    allocate(gridtiles(ntiles))    
+    do n = 1, ntiles
+      call read_data(mosaic_file, 'gridtiles', gridtiles(n), level=n)
+    enddo
+
+    ncontacts = get_mosaic_ncontacts(mosaic_file)
+
+    do n = 1, ncontacts
+      call read_data(mosaic_file, "contacts", contacts, level=n)
+      nstr = parse_string(contacts, ":", strlist)      
+      if(nstr .NE. 4) call mpp_error(FATAL, &
+         "mosaic_mod(get_mosaic_contact): number of elements in contact seperated by :/:: should be 4")
+      found = .false.
+      do m = 1, ntiles
+        if(trim(gridtiles(m)) == trim(strlist(2)) ) then !found the tile name 
+          found = .true.
+          tile1(n) = m
+          exit
+        endif
+      enddo
+    
+      if(.not.found) call mpp_error(FATAL, &
+         "mosaic_mod(get_mosaic_contact):the first tile name specified in contact is not found in tile list")
+
+      found = .false.
+      do m = 1, ntiles
+        if(trim(gridtiles(m)) == trim(strlist(4)) ) then !found the tile name 
+          found = .true.
+          tile2(n) = m
+          exit
+        endif
+      enddo
+    
+      if(.not.found) call mpp_error(FATAL, &
+         "mosaic_mod(get_mosaic_contact):the second tile name specified in contact is not found in tile list")
+
+      call read_data(mosaic_file, "contact_index", contacts, level=n)
+      nstr = parse_string(contacts, ":,", strlist)
+      if(nstr .NE. 8) then
+        if(mpp_pe()==mpp_root_pe()) then 
+          print*, "nstr is ", nstr
+          print*, "contacts is ", contacts
+          do m = 1, nstr
+            print*, "strlist is ", trim(strlist(m))
+          enddo
+        endif
+        call mpp_error(FATAL, &
+               "mosaic_mod(get_mosaic_contact): number of elements in contact_index seperated by :/, should be 8")
+      endif
+      read(strlist(1), *, iostat=ios) istart1(n)
+      if(ios .NE. 0) call mpp_error(FATAL, &
+         "mosaic_mod(get_mosaic_contact): Error in reading istart1")
+      read(strlist(2), *, iostat=ios) iend1(n)
+      if(ios .NE. 0) call mpp_error(FATAL, &
+         "mosaic_mod(get_mosaic_contact): Error in reading iend1")
+      read(strlist(3), *, iostat=ios) jstart1(n)
+      if(ios .NE. 0) call mpp_error(FATAL, &
+         "mosaic_mod(get_mosaic_contact): Error in reading jstart1")
+      read(strlist(4), *, iostat=ios) jend1(n)
+      if(ios .NE. 0) call mpp_error(FATAL, &
+         "mosaic_mod(get_mosaic_contact): Error in reading jend1")
+      read(strlist(5), *, iostat=ios) istart2(n)
+      if(ios .NE. 0) call mpp_error(FATAL, &
+         "mosaic_mod(get_mosaic_contact): Error in reading istart2")
+      read(strlist(6), *, iostat=ios) iend2(n)
+      if(ios .NE. 0) call mpp_error(FATAL, &
+         "mosaic_mod(get_mosaic_contact): Error in reading iend2")
+      read(strlist(7), *, iostat=ios) jstart2(n)
+      if(ios .NE. 0) call mpp_error(FATAL, &
+         "mosaic_mod(get_mosaic_contact): Error in reading jstart2")
+      read(strlist(8), *, iostat=ios) jend2(n)
+      if(ios .NE. 0) call mpp_error(FATAL, &
+         "mosaic_mod(get_mosaic_contact): Error in reading jend2")
+
+      i1_type = transfer_to_model_index(istart1(n), iend1(n), x_refine)
+      j1_type = transfer_to_model_index(jstart1(n), jend1(n), y_refine)
+      i2_type = transfer_to_model_index(istart2(n), iend2(n), x_refine)
+      j2_type = transfer_to_model_index(jstart2(n), jend2(n), y_refine)
+
+      if( i1_type == 0 .AND. j1_type == 0 ) call mpp_error(FATAL, &
+         "mosaic_mod(get_mosaic_contact): istart1==iend1 and jstart1==jend1")
+      if( i2_type == 0 .AND. j2_type == 0 ) call mpp_error(FATAL, &
+         "mosaic_mod(get_mosaic_contact): istart2==iend2 and jstart2==jend2")
+      if( i1_type + j1_type .NE. i2_type + j2_type ) call mpp_error(FATAL, &
+         "mosaic_mod(get_mosaic_contact): It is not a line or overlap contact")
+
+   enddo
+
+      deallocate(gridtiles)
 
   end subroutine get_mosaic_contact
 ! 
 
+
+function transfer_to_model_index(istart, iend, refine_ratio)
+   integer, intent(inout) :: istart, iend
+   integer                :: refine_ratio
+   integer                :: transfer_to_model_index
+   integer                :: istart_in, iend_in
+
+   istart_in = istart
+   iend_in = iend
+
+   if( istart_in == iend_in ) then
+      transfer_to_model_index = 0
+      istart = (istart_in + 1)/refine_ratio
+      iend   = istart
+   else 
+      transfer_to_model_index = 1
+      if( iend_in > istart_in ) then
+        istart = istart_in + 1
+        iend   = iend_in
+      else 
+        istart = istart_in
+        iend   = iend_in + 1
+      endif
+      if( mod(istart, refine_ratio) .NE. 0 .OR. mod(iend,refine_ratio) .NE. 0) call mpp_error(FATAL, &
+         "mosaic_mod(transfer_to_model_index): mismatch between refine_ratio and istart/iend")
+      istart = istart/refine_ratio
+      iend = iend/refine_ratio
+
+   endif
+
+   return
+
+end function transfer_to_model_index
+
   !###############################################################################
   ! 
   !   
@@ -453,6 +575,52 @@ function is_inside_polygon(lon1, lat1, lon2, lat2 )
 
   end function is_inside_polygon
 
+  function parse_string(string, set, value)
+  character(len=*),  intent(in) :: string
+  character(len=*),  intent(in) :: set
+  character(len=*), intent(out) :: value(:)
+  integer                       :: parse_string
+  integer :: nelem, length, first, last
+
+     nelem = size(value(:))
+     length = len_trim(string)
+
+     first = 1; last = 0
+     parse_string = 0
+
+     do while(first .LE. length)
+       parse_string = parse_string + 1
+       if(parse_string>nelem) then
+         call mpp_error(FATAL, "mosaic_mod(parse_string) : number of element is greater than size(value(:))")
+       endif
+       last = first - 1 + scan(string(first:length), set)
+       if(last == first-1 ) then  ! not found, end of string
+         value(parse_string) = string(first:length)
+         exit
+       else
+         if(last <= first) then
+           call mpp_error(FATAL, "mosaic_mod(parse_string) : last <= first")
+         endif
+         value(parse_string) = string(first:(last-1))
+         first = last + 1
+         ! scan to make sure the next is not the character in the set
+         do while (first == last+1)
+           last = first - 1 + scan(string(first:length), set)
+           if(last == first) then
+             first = first+1
+           else
+             exit
+           endif
+         end do
+       endif
+     enddo
+
+     return
+
+  end function parse_string
+
+
+     
 end module mosaic_mod
 
 
diff --git a/src/shared/mosaic/mosaic_util.h b/src/shared/mosaic/mosaic_util.h
index 0641640981..c7c0cefc42 100644
--- a/src/shared/mosaic/mosaic_util.h
+++ b/src/shared/mosaic/mosaic_util.h
@@ -2,7 +2,7 @@
                       mosaic_util.h
     This header file provide some utilities routine that will be used in many tools.
     
-    contact: Zhi.Liang
+    contact: Zhi.Liang@noaa.gov
 ***********************************************************************/
 #ifndef MOSAIC_UTIL_H_
 #define MOSAIC_UTIL_H_
diff --git a/src/shared/mosaic/read_mosaic.c b/src/shared/mosaic/read_mosaic.c
index 27ed0b2e4a..460f2c42c4 100644
--- a/src/shared/mosaic/read_mosaic.c
+++ b/src/shared/mosaic/read_mosaic.c
@@ -375,6 +375,35 @@ int read_mosaic_xgrid_size( const char *xgrid_file )
   return ncells;
 }
 
+#ifdef OVERLOAD_R4
+float get_global_area(void) 
+{
+  float garea;
+#else
+double get_global_area(void) 
+{
+  double garea;
+#endif
+  garea = 4*M_PI*RADIUS*RADIUS;
+
+  return garea;
+};
+
+#ifndef __AIX
+#ifdef OVERLOAD_R4
+float get_global_area_(void)
+{
+  float garea;
+#else
+double get_global_area_(void)
+{
+  double garea;
+#endif
+  garea = 4*M_PI*RADIUS*RADIUS;
+
+  return garea;
+};
+#endif
 
 
 /****************************************************************************/
diff --git a/src/shared/mosaic/read_mosaic.h b/src/shared/mosaic/read_mosaic.h
index b6256d2dcc..cd56d15d1c 100644
--- a/src/shared/mosaic/read_mosaic.h
+++ b/src/shared/mosaic/read_mosaic.h
@@ -7,11 +7,13 @@ void read_mosaic_xgrid_order1(const char *xgrid_file, int *i1, int *j1, int *i2,
 void read_mosaic_xgrid_order1_region(const char *xgrid_file, int *i1, int *j1, int *i2, int *j2, float *area, int *isc, int *iec );
 void read_mosaic_xgrid_order2(const char *xgrid_file, int *i1, int *j1, int *i2, int *j2, 
                               float *area, float *di, float *dj );
+float get_global_area(void);
 #else
 void read_mosaic_xgrid_order1(const char *xgrid_file, int *i1, int *j1, int *i2, int *j2, double *area );
 void read_mosaic_xgrid_order1_region(const char *xgrid_file, int *i1, int *j1, int *i2, int *j2, double *area, int *isc, int *iec );
 void read_mosaic_xgrid_order2(const char *xgrid_file, int *i1, int *j1, int *i2, int *j2, 
                               double *area, double *di, double *dj );
+double get_global_area(void);
 #endif
 int read_mosaic_ntiles(const char *mosaic_file);
 int read_mosaic_ncontacts(const char *mosaic_file);
@@ -20,4 +22,5 @@ void read_mosaic_contact(const char *mosaic_file, int *tile1, int *tile2, int *i
 			 int *jstart1, int *jend1, int *istart2, int *iend2, int *jstart2, int *jend2);
 void read_mosaic_grid_data(const char *mosaic_file, const char *name, int nx, int ny,
                            double *data, int level, int ioff, int joff);
+
 #endif
diff --git a/src/shared/mpp/include/group_update_pack.inc b/src/shared/mpp/include/group_update_pack.inc
new file mode 100644
index 0000000000..db5f44ec59
--- /dev/null
+++ b/src/shared/mpp/include/group_update_pack.inc
@@ -0,0 +1,413 @@
+if( group%k_loop_inside ) then
+!$OMP parallel do default(none) shared(npack,group,ptr,nvector,ksize,buffer_start_pos) &
+!$OMP                          private(buffer_pos,pos,m,is,ie,js,je,rotation, &
+!$OMP                                  ptr_field, ptr_fieldx, ptr_fieldy,n,k)
+  do n = 1, npack
+     buffer_pos = group%pack_buffer_pos(n) + buffer_start_pos
+     pos  = buffer_pos
+     is = group%pack_is(n); ie = group%pack_ie(n)
+     js = group%pack_js(n); je = group%pack_je(n)
+     rotation = group%pack_rotation(n)
+     if( group%pack_type(n) == FIELD_S ) then
+        select case( rotation )
+        case(ZERO)
+           do l=1, group%nscalar  ! loop over number of fields
+              ptr_field = group%addrs_s(l)
+              do k = 1, ksize
+                 do j = js, je
+                    do i = is, ie
+                       pos = pos + 1
+                       buffer(pos) = field(i,j,k)
+                    end do
+                 end do
+              enddo
+           enddo
+        case( MINUS_NINETY )
+           do l=1,group%nscalar  ! loop over number of fields
+              ptr_field = group%addrs_s(l)
+              do k = 1, ksize
+                 do i = is, ie
+                    do j = je, js, -1
+                       pos = pos + 1
+                       buffer(pos) = field(i,j,k)
+                    end do
+                 end do
+              end do
+           end do
+        case( NINETY )
+           do l=1,group%nscalar  ! loop over number of fields
+              ptr_field = group%addrs_s(l)
+              do k = 1, ksize
+                 do i = ie, is, -1
+                    do j = js, je
+                       pos = pos + 1
+                       buffer(pos) = field(i,j,k)
+                    end do
+                 end do
+              end do
+           end do
+        case( ONE_HUNDRED_EIGHTY )
+           do l=1,group%nscalar  ! loop over number of fields
+              ptr_field = group%addrs_s(l)
+              do k = 1, ksize
+                 do j = je, js, -1
+                    do i = ie, is, -1
+                       pos = pos + 1
+                       buffer(pos) = field(i,j,k)
+                    end do
+                 end do
+              end do
+           end do
+        end select
+     else if( group%pack_type(n) == FIELD_X  ) then
+        select case( rotation )
+        case(ZERO)
+           do l=1, nvector  ! loop over number of fields
+              ptr_fieldx = group%addrs_x(l)
+              do k = 1, ksize
+                 do j = js, je
+                    do i = is, ie
+                       pos = pos + 1
+                       buffer(pos) = fieldx(i,j,k)
+                    end do
+                 end do
+              end do
+           end do
+        case( MINUS_NINETY )
+           if( BTEST(group%flags_v,SCALAR_BIT) ) then
+              do l=1,nvector  ! loop over number of fields
+                 ptr_fieldy = group%addrs_y(l)
+                 do k = 1, ksize
+                    do i = is, ie
+                       do j = je, js, -1
+                          pos = pos + 1
+                          buffer(pos) = fieldy(i,j,k)
+                       end do
+                    end do
+                 end do
+              end do
+           else
+              do l=1,nvector  ! loop over number of fields
+                 ptr_fieldy = group%addrs_y(l)
+                 do k = 1, ksize
+                    do i = is, ie
+                       do j = je, js, -1
+                          pos = pos + 1
+                          buffer(pos) = -fieldy(i,j,k)
+                       end do
+                    end do
+                 end do
+              end do
+           end if
+        case( NINETY )
+           do l=1, nvector  ! loop over number of fields
+              ptr_fieldy = group%addrs_y(l)
+              do k = 1, ksize
+                 do i = ie, is, -1
+                    do j = js, je
+                       pos = pos + 1
+                       buffer(pos) = fieldy(i,j,k)
+                    end do
+                 end do
+              end do
+           end do
+        case( ONE_HUNDRED_EIGHTY )
+           if( BTEST(group%flags_v,SCALAR_BIT) ) then
+              do l=1,nvector  ! loop over number of fields
+                 ptr_fieldx = group%addrs_x(l)
+                 do k = 1, ksize
+                    do j = je, js, -1
+                       do i = ie, is, -1
+                          pos = pos + 1
+                          buffer(pos) =  fieldx(i,j,k)
+                       end do
+                    end do
+                 end do
+              end do
+           else
+              do l=1,nvector  ! loop over number of fields
+                 ptr_fieldx = group%addrs_x(l)
+                 do k = 1, ksize
+                    do j = je, js, -1
+                       do i = ie, is, -1
+                          pos = pos + 1
+                          buffer(pos) =  -fieldx(i,j,k)
+                       end do
+                    end do
+                 end do
+              end do
+           end if
+        end select ! select case( rotation(n) )
+     else if( group%pack_type(n) == FIELD_Y ) then
+        select case( rotation )
+        case(ZERO)
+           do l=1, nvector  ! loop over number of fields
+              ptr_fieldy = group%addrs_y(l)
+              do k = 1, ksize
+                 do j = js, je
+                    do i = is, ie
+                       pos = pos + 1
+                       buffer(pos) = fieldy(i,j,k)
+                    end do
+                 end do
+              end do
+           end do
+        case( MINUS_NINETY )
+           do l=1,nvector  ! loop over number of fields
+              ptr_fieldx = group%addrs_x(l)
+              do k = 1, ksize
+                 do i = is, ie
+                    do j = je, js, -1
+                       pos = pos + 1
+                       buffer(pos) = fieldx(i,j,k)
+                    end do
+                 end do
+              end do
+           end do
+        case( NINETY )
+           if( BTEST(group%flags_v,SCALAR_BIT) ) then
+              do l=1, nvector  ! loop over number of fields
+                 ptr_fieldx = group%addrs_x(l)
+                 do k = 1, ksize
+                    do i = ie, is, -1
+                       do j = js, je
+                          pos = pos + 1
+                          buffer(pos) = fieldx(i,j,k)
+                       end do
+                    end do
+                 end do
+              end do
+           else
+              do l=1,nvector  ! loop over number of fields
+                 ptr_fieldx = group%addrs_x(l)
+                 do k = 1, ksize
+                    do i = ie, is, -1
+                       do j = js, je
+                          pos = pos + 1
+                          buffer(pos) = -fieldx(i,j,k)
+                       end do
+                    end do
+                 end do
+              end do
+           end if
+        case( ONE_HUNDRED_EIGHTY )
+           if( BTEST(group%flags_v,SCALAR_BIT) ) then
+              do l=1,nvector  ! loop over number of fields
+                 ptr_fieldy = group%addrs_y(l)
+                 do k = 1, ksize
+                    do j = je, js, -1
+                       do i = ie, is, -1
+                          pos = pos + 1
+                          buffer(pos) =  fieldy(i,j,k)
+                       end do
+                    end do
+                 end do
+              end do
+           else
+              do l=1,nvector  ! loop over number of fields
+                 ptr_fieldy = group%addrs_y(l)
+                 do k = 1, ksize
+                    do j = je, js, -1
+                       do i = ie, is, -1
+                          pos = pos + 1
+                          buffer(pos) =  -fieldy(i,j,k)
+                       end do
+                    end do
+                 end do
+              end do
+           end if
+        end select ! select case( rotation(n) )
+     endif
+  enddo
+else
+!$OMP parallel do default(none) shared(npack,group,ptr,nvector,ksize,buffer_start_pos) &
+!$OMP                          private(buffer_pos,pos,m,is,ie,js,je,rotation, &
+!$OMP                                  ptr_field, ptr_fieldx, ptr_fieldy,n,k)
+  do nk = 1, npack*ksize
+     n = (nk-1)/ksize + 1
+     k = mod((nk-1), ksize) + 1
+     buffer_pos = group%pack_buffer_pos(n) + buffer_start_pos
+     pos  = buffer_pos + (k-1)*group%pack_size(n)
+     is = group%pack_is(n); ie = group%pack_ie(n)
+     js = group%pack_js(n); je = group%pack_je(n)
+     rotation = group%pack_rotation(n)
+     if( group%pack_type(n) == FIELD_S ) then
+        select case( rotation )
+        case(ZERO)
+           do l=1, group%nscalar  ! loop over number of fields
+              ptr_field = group%addrs_s(l)
+              do j = js, je
+                 do i = is, ie
+                    pos = pos + 1
+                    buffer(pos) = field(i,j,k)
+                 end do
+              end do
+           enddo
+        case( MINUS_NINETY )
+           do l=1,group%nscalar  ! loop over number of fields
+              ptr_field = group%addrs_s(l)
+              do i = is, ie
+                 do j = je, js, -1
+                    pos = pos + 1
+                    buffer(pos) = field(i,j,k)
+                 end do
+              end do
+           end do
+        case( NINETY )
+           do l=1,group%nscalar  ! loop over number of fields
+              ptr_field = group%addrs_s(l)
+              do i = ie, is, -1
+                 do j = js, je
+                    pos = pos + 1
+                    buffer(pos) = field(i,j,k)
+                 end do
+              end do
+           end do
+        case( ONE_HUNDRED_EIGHTY )
+           do l=1,group%nscalar  ! loop over number of fields
+              ptr_field = group%addrs_s(l)
+              do j = je, js, -1
+                 do i = ie, is, -1
+                    pos = pos + 1
+                    buffer(pos) = field(i,j,k)
+                 end do
+              end do
+           end do
+        end select
+     else if( group%pack_type(n) == FIELD_X  ) then
+        select case( rotation )
+        case(ZERO)
+           do l=1, nvector  ! loop over number of fields
+              ptr_fieldx = group%addrs_x(l)
+              do j = js, je
+                 do i = is, ie
+                    pos = pos + 1
+                    buffer(pos) = fieldx(i,j,k)
+                 end do
+              end do
+           end do
+        case( MINUS_NINETY )
+           if( BTEST(group%flags_v,SCALAR_BIT) ) then
+              do l=1,nvector  ! loop over number of fields
+                 ptr_fieldy = group%addrs_y(l)
+                 do i = is, ie
+                    do j = je, js, -1
+                       pos = pos + 1
+                       buffer(pos) = fieldy(i,j,k)
+                    end do
+                 end do
+              end do
+           else
+              do l=1,nvector  ! loop over number of fields
+                 ptr_fieldy = group%addrs_y(l)
+                 do i = is, ie
+                    do j = je, js, -1
+                       pos = pos + 1
+                       buffer(pos) = -fieldy(i,j,k)
+                    end do
+                 end do
+              end do
+           end if
+        case( NINETY )
+           do l=1, nvector  ! loop over number of fields
+              ptr_fieldy = group%addrs_y(l)
+              do i = ie, is, -1
+                 do j = js, je
+                    pos = pos + 1
+                    buffer(pos) = fieldy(i,j,k)
+                 end do
+              end do
+           end do
+        case( ONE_HUNDRED_EIGHTY )
+           if( BTEST(group%flags_v,SCALAR_BIT) ) then
+              do l=1,nvector  ! loop over number of fields
+                 ptr_fieldx = group%addrs_x(l)
+                 do j = je, js, -1
+                    do i = ie, is, -1
+                       pos = pos + 1
+                       buffer(pos) =  fieldx(i,j,k)
+                    end do
+                 end do
+              end do
+           else
+              do l=1,nvector  ! loop over number of fields
+                 ptr_fieldx = group%addrs_x(l)
+                 do j = je, js, -1
+                    do i = ie, is, -1
+                       pos = pos + 1
+                       buffer(pos) =  -fieldx(i,j,k)
+                    end do
+                 end do
+              end do
+           end if
+        end select ! select case( rotation(n) )
+     else if( group%pack_type(n) == FIELD_Y ) then
+        select case( rotation )
+        case(ZERO)
+           do l=1, nvector  ! loop over number of fields
+              ptr_fieldy = group%addrs_y(l)
+              do j = js, je
+                 do i = is, ie
+                    pos = pos + 1
+                    buffer(pos) = fieldy(i,j,k)
+                 end do
+              end do
+           end do
+        case( MINUS_NINETY )
+           do l=1,nvector  ! loop over number of fields
+              ptr_fieldx = group%addrs_x(l)
+              do i = is, ie
+                 do j = je, js, -1
+                    pos = pos + 1
+                    buffer(pos) = fieldx(i,j,k)
+                 end do
+              end do
+           end do
+        case( NINETY )
+           if( BTEST(group%flags_v,SCALAR_BIT) ) then
+              do l=1, nvector  ! loop over number of fields
+                 ptr_fieldx = group%addrs_x(l)
+                 do i = ie, is, -1
+                    do j = js, je
+                       pos = pos + 1
+                       buffer(pos) = fieldx(i,j,k)
+                    end do
+                 end do
+              end do
+           else
+              do l=1,nvector  ! loop over number of fields
+                 ptr_fieldx = group%addrs_x(l)
+                 do i = ie, is, -1
+                    do j = js, je
+                       pos = pos + 1
+                       buffer(pos) = -fieldx(i,j,k)
+                    end do
+                 end do
+              end do
+           end if
+        case( ONE_HUNDRED_EIGHTY )
+           if( BTEST(group%flags_v,SCALAR_BIT) ) then
+              do l=1,nvector  ! loop over number of fields
+                 ptr_fieldy = group%addrs_y(l)
+                 do j = je, js, -1
+                    do i = ie, is, -1
+                       pos = pos + 1
+                       buffer(pos) =  fieldy(i,j,k)
+                    end do
+                 end do
+              end do
+           else
+              do l=1,nvector  ! loop over number of fields
+                 ptr_fieldy = group%addrs_y(l)
+                 do j = je, js, -1
+                    do i = ie, is, -1
+                       pos = pos + 1
+                       buffer(pos) =  -fieldy(i,j,k)
+                    end do
+                 end do
+              end do
+           end if
+        end select ! select case( rotation(n) )
+     endif
+  enddo
+endif
diff --git a/src/shared/mpp/include/group_update_unpack.inc b/src/shared/mpp/include/group_update_unpack.inc
new file mode 100644
index 0000000000..bf6f9f00c5
--- /dev/null
+++ b/src/shared/mpp/include/group_update_unpack.inc
@@ -0,0 +1,91 @@
+if( group%k_loop_inside ) then
+!$OMP parallel do default(none) shared(nunpack,group,nscalar,ptr,nvector,ksize,buffer_start_pos) &
+!$OMP                          private(buffer_pos,pos,m,is, ie, js, je,rotation, &
+!$OMP                                  ptr_field, ptr_fieldx, ptr_fieldy, n,k )
+  do n = nunpack, 1, -1
+     buffer_pos = group%unpack_buffer_pos(n) + buffer_start_pos
+     pos = buffer_pos 
+     is = group%unpack_is(n); ie = group%unpack_ie(n)
+     js = group%unpack_js(n); je = group%unpack_je(n)
+     if( group%unpack_type(n) == FIELD_S ) then
+        do l=1,nscalar  ! loop over number of fields
+           ptr_field = group%addrs_s(l)
+           do k = 1, ksize
+              do j = js, je
+                 do i = is, ie
+                    pos = pos + 1
+                    field(i,j,k) = buffer(pos)
+                 end do
+              end do
+           end do
+        end do
+     else if( group%unpack_type(n) == FIELD_X ) then
+        do l=1,nvector  ! loop over number of fields
+           ptr_fieldx = group%addrs_x(l)
+           do k = 1, ksize
+              do j = js, je
+                 do i = is, ie
+                    pos = pos + 1
+                    fieldx(i,j,k) = buffer(pos)
+                 end do
+              end do
+           end do
+        end do
+     else if( group%unpack_type(n) == FIELD_Y ) then
+        do l=1,nvector  ! loop over number of fields
+           ptr_fieldy = group%addrs_y(l)
+           do k = 1, ksize
+              do j = js, je
+                 do i = is, ie
+                    pos = pos + 1
+                    fieldy(i,j,k) = buffer(pos)
+                 end do
+              end do
+           end do
+        end do
+     endif
+  enddo
+else
+!$OMP parallel do default(none) shared(nunpack,group,nscalar,ptr,nvector,ksize,buffer_start_pos) &
+!$OMP                          private(buffer_pos,pos,m,is, ie, js, je,rotation, &
+!$OMP                                  ptr_field, ptr_fieldx, ptr_fieldy,n,k)
+  do nk = nunpack*ksize, 1, -1
+     n = (nk-1)/ksize + 1
+     k = mod((nk-1), ksize) + 1
+     buffer_pos = group%unpack_buffer_pos(n) + buffer_start_pos
+     pos = buffer_pos + (k-1)*group%unpack_size(n)
+     is = group%unpack_is(n); ie = group%unpack_ie(n)
+     js = group%unpack_js(n); je = group%unpack_je(n)
+     if( group%unpack_type(n) == FIELD_S ) then
+        do l=1,nscalar  ! loop over number of fields
+           ptr_field = group%addrs_s(l)
+           do j = js, je
+              do i = is, ie
+                 pos = pos + 1
+                 field(i,j,k) = buffer(pos)
+              end do
+           end do
+        end do
+     else if( group%unpack_type(n) == FIELD_X ) then
+        do l=1,nvector  ! loop over number of fields
+           ptr_fieldx = group%addrs_x(l)
+           do j = js, je
+              do i = is, ie
+                 pos = pos + 1
+                 fieldx(i,j,k) = buffer(pos)
+              end do
+           end do
+        end do
+     else if( group%unpack_type(n) == FIELD_Y ) then
+        do l=1,nvector  ! loop over number of fields
+           ptr_fieldy = group%addrs_y(l)
+           do j = js, je
+              do i = is, ie
+                 pos = pos + 1
+                 fieldy(i,j,k) = buffer(pos)
+              end do
+           end do
+        end do
+     endif
+  enddo
+endif
diff --git a/src/shared/mpp/include/mpp_alltoall_mpi.h b/src/shared/mpp/include/mpp_alltoall_mpi.h
index bd129ea7bc..a0949650ec 100644
--- a/src/shared/mpp/include/mpp_alltoall_mpi.h
+++ b/src/shared/mpp/include/mpp_alltoall_mpi.h
@@ -1,7 +1,9 @@
-subroutine MPP_ALLTOALL_(sbuf, rbuf, pelist)
+subroutine MPP_ALLTOALL_(sbuf, scount, rbuf, rcount, pelist)
 
     MPP_TYPE_, intent(in) :: sbuf(:)
     MPP_TYPE_, intent(inout) :: rbuf(:)
+    integer,   intent(in) :: scount, rcount
+   
 
     integer, intent(in), optional :: pelist(0:)
     integer :: n
@@ -17,7 +19,7 @@ subroutine MPP_ALLTOALL_(sbuf, rbuf, pelist)
     if (verbose) call mpp_error(NOTE, 'MPP_ALLTOALL_: using MPI_Alltoall...')
 
     ! TODO: Message lengths greater than 1
-    call MPI_Alltoall(sbuf, 1, MPI_TYPE_, rbuf, 1, MPI_TYPE_, &
+    call MPI_Alltoall(sbuf, scount, MPI_TYPE_, rbuf, rcount, MPI_TYPE_, &
                       peset(n)%id, error)
 
     if (current_clock .NE. 0) &
diff --git a/src/shared/mpp/include/mpp_alltoall_nocomm.h b/src/shared/mpp/include/mpp_alltoall_nocomm.h
index 328cee07e9..70118a3d35 100644
--- a/src/shared/mpp/include/mpp_alltoall_nocomm.h
+++ b/src/shared/mpp/include/mpp_alltoall_nocomm.h
@@ -1,7 +1,8 @@
-subroutine MPP_ALLTOALL_(sbuf, rbuf, pelist)
+subroutine MPP_ALLTOALL_(sbuf, scount, rbuf, rcount, pelist)
 
     MPP_TYPE_, dimension(:), intent(in) :: sbuf
     MPP_TYPE_, dimension(:), intent(inout) :: rbuf
+    integer,   intent(in) :: scount, rcount
 
     integer, intent(in), optional :: pelist(0:)
 
diff --git a/src/shared/mpp/include/mpp_alltoall_sma.h b/src/shared/mpp/include/mpp_alltoall_sma.h
index 5a8c1e6c88..afaece1459 100644
--- a/src/shared/mpp/include/mpp_alltoall_sma.h
+++ b/src/shared/mpp/include/mpp_alltoall_sma.h
@@ -1,7 +1,8 @@
-subroutine MPP_ALLTOALL_(sbuf, rbuf, pelist)
+subroutine MPP_ALLTOALL_(sbuf, scount, rbuf, rcount, pelist)
 
     MPP_TYPE_, dimension(:), intent(in) :: sbuf
     MPP_TYPE_, dimension(:), intent(inout) :: rbuf
+    integer,   intent(in) :: scount, rcount
 
     integer, intent(in), optional :: pelist(0:)
 
diff --git a/src/shared/mpp/include/mpp_chksum.h b/src/shared/mpp/include/mpp_chksum.h
index 372924d9c6..ffa7086a3d 100644
--- a/src/shared/mpp/include/mpp_chksum.h
+++ b/src/shared/mpp/include/mpp_chksum.h
@@ -1,11 +1,20 @@
-    function MPP_CHKSUM_( var, pelist )
+function MPP_CHKSUM_( var, pelist , mask_val)
 !mold is a dummy array to be used by TRANSFER()
 !must be same TYPE as result
 !result is LONG_KIND, which will actually be int ifdef no_8byte_integers
-      integer(LONG_KIND) :: MPP_CHKSUM_, mold(1)
+  !optional mask_val is masked away in checksum_int.h function via PACK()
+  integer(LONG_KIND) :: MPP_CHKSUM_
+  integer(LONG_KIND) :: mold(1)
       MPP_TYPE_, intent(in) :: var MPP_RANK_
       integer, intent(in), optional :: pelist(:)
+  MPP_TYPE_, intent(in),optional :: mask_val
 
+  if ( PRESENT(mask_val) ) then
+     MPP_CHKSUM_ = mpp_chksum( TRANSFER(var,mold), pelist, &
+          mask_val= TRANSFER(mask_val,mold(1) ) )
+  else
       MPP_CHKSUM_ = mpp_chksum( TRANSFER(var,mold), pelist )
+  end if
+  
       return
     end function MPP_CHKSUM_
diff --git a/src/shared/mpp/include/mpp_chksum_int.h b/src/shared/mpp/include/mpp_chksum_int.h
index bce804fd00..2d2109d09e 100644
--- a/src/shared/mpp/include/mpp_chksum_int.h
+++ b/src/shared/mpp/include/mpp_chksum_int.h
@@ -1,8 +1,52 @@
-    function MPP_CHKSUM_INT_( var, pelist )
-      MPP_TYPE_ :: MPP_CHKSUM_INT_
+function MPP_CHKSUM_INT_( var, pelist, mask_val )
+  integer(LONG_KIND) :: MPP_CHKSUM_INT_
       MPP_TYPE_, intent(in) :: var MPP_RANK_
       integer, optional :: pelist(:)
-      MPP_CHKSUM_INT_ = sum(var)
+  MPP_TYPE_, intent(in), optional :: mask_val
+  
+  if ( PRESENT(mask_val) ) then
+     !PACK on var/=mask_val ignores values in var
+     !equiv to setting those values=0, but on sparse arrays
+     !pack should return much smaller array to sum
+     MPP_CHKSUM_INT_ = sum( INT( PACK(var,var/=mask_val),LONG_KIND) )
+  else
+     MPP_CHKSUM_INT_ = sum(INT(var,LONG_KIND))
+  end if
+
       call mpp_sum( MPP_CHKSUM_INT_, pelist )
       return
+
     end function MPP_CHKSUM_INT_
+
+
+!Handles real mask for easier implimentation
+! until exists full integer vartypes...
+function MPP_CHKSUM_INT_RMASK_( var, pelist, mask_val )
+  integer(LONG_KIND) :: MPP_CHKSUM_INT_RMASK_
+  MPP_TYPE_, intent(in) :: var MPP_RANK_
+  integer, optional :: pelist(:)
+  real, intent(in) :: mask_val
+  integer(KIND(var))::tmpVarP
+  integer(KIND(mask_val)) :: tmpFullP,tmpZeroMasked
+
+
+  if ( KIND(mask_val) == KIND(var)) then
+     !same numBytes
+     !cast to MPP_TYPE_
+     tmpVarP = TRANSFER(mask_val , tmpVarP)
+  else if (KIND(mask_val) /=  KIND(var) ) then
+     !check if still safe to duck type to lower precision
+     tmpFullP = TRANSFER(mask_val,tmpFullP ) ! transfer bits to int mold of same numBytes
+     tmpZeroMasked = IBITS(tmpFullP,0,BIT_SIZE(tmpVarP))
+     if (tmpFullP /= tmpZeroMasked ) then
+        !as an int, mask_val is actually /using/ higher bits than var, and so not a valid mask...
+        call mpp_error(FATAL, "mpp_chksum_int.h was called with real mask_val, and mask_val can not be safely cast to int type of var (nonzero high bits).")
+     end if
+     tmpVarP = INT(tmpFullP, KIND(var) ) ! attempt cast of mask_val as int down to the int precision of var, could do transfer...
+  end if
+
+  MPP_CHKSUM_INT_RMASK_ = mpp_chksum(var,pelist,mask_val=tmpVarP)
+
+  return
+
+end function MPP_CHKSUM_INT_RMASK_
diff --git a/src/shared/mpp/include/mpp_chksum_scalar.h b/src/shared/mpp/include/mpp_chksum_scalar.h
index be992534b5..1cad513250 100644
--- a/src/shared/mpp/include/mpp_chksum_scalar.h
+++ b/src/shared/mpp/include/mpp_chksum_scalar.h
@@ -1,14 +1,21 @@
-    function MPP_CHKSUM_( var, pelist )
+function MPP_CHKSUM_( var, pelist, mask_val )
 !mold is a dummy array to be used by TRANSFER()
 !must be same TYPE as result
 !result is LONG_KIND, which will actually be int ifdef no_8byte_integers
+  !mold and mask_val must be same numBytes, otherwise undefined behavior
       integer(LONG_KIND) :: MPP_CHKSUM_
       MPP_TYPE_, intent(in) :: var
       integer, intent(in), optional :: pelist(:)
       integer(LONG_KIND) :: mold(1)
+  MPP_TYPE_, intent(in), optional :: mask_val
       pointer( p, mold )
 
       p = LOC(var)
+
+  if ( PRESENT(mask_val) ) then
+     MPP_CHKSUM_ = mpp_chksum( mold, pelist, TRANSFER(mask_val, mold(1)) )
+  else
       MPP_CHKSUM_ = mpp_chksum( mold, pelist )
+  end if
       return
     end function MPP_CHKSUM_
diff --git a/src/shared/mpp/include/mpp_comm.inc b/src/shared/mpp/include/mpp_comm.inc
index aff06aa91d..95a3d2a601 100644
--- a/src/shared/mpp/include/mpp_comm.inc
+++ b/src/shared/mpp/include/mpp_comm.inc
@@ -1,7 +1,6 @@
 
 ! -*-f90-*-
-! $Id: mpp_comm.inc,v 19.0 2012/01/06 22:00:37 fms Exp $
-
+! $Id$
 
 #ifdef use_libSMA
 #include 
@@ -14,6 +13,8 @@
 #ifndef no_8byte_integers
 #undef MPP_CHKSUM_INT_
 #define MPP_CHKSUM_INT_ mpp_chksum_i8_1d
+#undef MPP_CHKSUM_INT_RMASK_
+#define MPP_CHKSUM_INT_RMASK_ mpp_chksum_i8_1d_rmask
 #undef MPP_TYPE_
 #define MPP_TYPE_ integer(LONG_KIND)
 #undef MPP_RANK_
@@ -22,6 +23,8 @@
 
 #undef MPP_CHKSUM_INT_
 #define MPP_CHKSUM_INT_ mpp_chksum_i8_2d
+#undef MPP_CHKSUM_INT_RMASK_
+#define MPP_CHKSUM_INT_RMASK_ mpp_chksum_i8_2d_rmask
 #undef MPP_TYPE_
 #define MPP_TYPE_ integer(LONG_KIND)
 #undef MPP_RANK_
@@ -30,6 +33,8 @@
 
 #undef MPP_CHKSUM_INT_
 #define MPP_CHKSUM_INT_ mpp_chksum_i8_3d
+#undef MPP_CHKSUM_INT_RMASK_
+#define MPP_CHKSUM_INT_RMASK_ mpp_chksum_i8_3d_rmask
 #undef MPP_TYPE_
 #define MPP_TYPE_ integer(LONG_KIND)
 #undef MPP_RANK_
@@ -38,6 +43,8 @@
 
 #undef MPP_CHKSUM_INT_
 #define MPP_CHKSUM_INT_ mpp_chksum_i8_4d
+#undef MPP_CHKSUM_INT_RMASK_
+#define MPP_CHKSUM_INT_RMASK_ mpp_chksum_i8_4d_rmask
 #undef MPP_TYPE_
 #define MPP_TYPE_ integer(LONG_KIND)
 #undef MPP_RANK_
@@ -46,6 +53,8 @@
 
 #undef MPP_CHKSUM_INT_
 #define MPP_CHKSUM_INT_ mpp_chksum_i8_5d
+#undef MPP_CHKSUM_INT_RMASK_
+#define MPP_CHKSUM_INT_RMASK_ mpp_chksum_i8_5d_rmask
 #undef MPP_TYPE_
 #define MPP_TYPE_ integer(LONG_KIND)
 #undef MPP_RANK_
@@ -55,6 +64,8 @@
 
 #undef MPP_CHKSUM_INT_
 #define MPP_CHKSUM_INT_ mpp_chksum_i4_1d
+#undef MPP_CHKSUM_INT_RMASK_
+#define MPP_CHKSUM_INT_RMASK_ mpp_chksum_i4_1d_rmask
 #undef MPP_TYPE_
 #define MPP_TYPE_ integer(INT_KIND)
 #undef MPP_RANK_
@@ -63,6 +74,8 @@
 
 #undef MPP_CHKSUM_INT_
 #define MPP_CHKSUM_INT_ mpp_chksum_i4_2d
+#undef MPP_CHKSUM_INT_RMASK_
+#define MPP_CHKSUM_INT_RMASK_ mpp_chksum_i4_2d_rmask
 #undef MPP_TYPE_
 #define MPP_TYPE_ integer(INT_KIND)
 #undef MPP_RANK_
@@ -71,6 +84,8 @@
 
 #undef MPP_CHKSUM_INT_
 #define MPP_CHKSUM_INT_ mpp_chksum_i4_3d
+#undef MPP_CHKSUM_INT_RMASK_
+#define MPP_CHKSUM_INT_RMASK_ mpp_chksum_i4_3d_rmask
 #undef MPP_TYPE_
 #define MPP_TYPE_ integer(INT_KIND)
 #undef MPP_RANK_
@@ -79,6 +94,8 @@
 
 #undef MPP_CHKSUM_INT_
 #define MPP_CHKSUM_INT_ mpp_chksum_i4_4d
+#undef MPP_CHKSUM_INT_RMASK_
+#define MPP_CHKSUM_INT_RMASK_ mpp_chksum_i4_4d_rmask
 #undef MPP_TYPE_
 #define MPP_TYPE_ integer(INT_KIND)
 #undef MPP_RANK_
@@ -87,6 +104,8 @@
 
 #undef MPP_CHKSUM_INT_
 #define MPP_CHKSUM_INT_ mpp_chksum_i4_5d
+#undef MPP_CHKSUM_INT_RMASK_
+#define MPP_CHKSUM_INT_RMASK_ mpp_chksum_i4_5d_rmask
 #undef MPP_TYPE_
 #define MPP_TYPE_ integer(INT_KIND)
 #undef MPP_RANK_
@@ -293,23 +312,62 @@
 
 !#################################################
 #undef MPP_GATHER_1D_
+#undef MPP_GATHER_1DV_
 #undef MPP_TYPE_
 #define MPP_TYPE_ integer(INT_KIND)
 #define MPP_GATHER_1D_ mpp_gather_int4_1d
 #define MPP_GATHER_1DV_ mpp_gather_int4_1dv
+#undef  MPP_GATHER_PELIST_2D_
+#undef  MPP_GATHER_PELIST_3D_
+#define MPP_GATHER_PELIST_2D_ mpp_gather_pelist_int4_2d
+#define MPP_GATHER_PELIST_3D_ mpp_gather_pelist_int4_3d
 #include 
 
 #undef MPP_GATHER_1D_
+#undef MPP_GATHER_1DV_
 #undef MPP_TYPE_
 #define MPP_TYPE_ real(FLOAT_KIND)
 #define MPP_GATHER_1D_ mpp_gather_real4_1d
 #define MPP_GATHER_1DV_ mpp_gather_real4_1dv
+#undef  MPP_GATHER_PELIST_2D_
+#undef  MPP_GATHER_PELIST_3D_
+#define MPP_GATHER_PELIST_2D_ mpp_gather_pelist_real4_2d
+#define MPP_GATHER_PELIST_3D_ mpp_gather_pelist_real4_3d
 #include 
 
 #undef MPP_GATHER_1D_
+#undef MPP_GATHER_1DV_
 #undef MPP_TYPE_
 #define MPP_TYPE_ real(DOUBLE_KIND)
 #define MPP_GATHER_1D_ mpp_gather_real8_1d
 #define MPP_GATHER_1DV_ mpp_gather_real8_1dv
+#undef  MPP_GATHER_PELIST_2D_
+#undef  MPP_GATHER_PELIST_3D_
+#define MPP_GATHER_PELIST_2D_ mpp_gather_pelist_real8_2d
+#define MPP_GATHER_PELIST_3D_ mpp_gather_pelist_real8_3d
 #include 
 
+!#################################################
+#undef  MPP_SCATTER_PELIST_2D_
+#undef  MPP_SCATTER_PELIST_3D_
+#undef MPP_TYPE_
+#define MPP_TYPE_ integer(INT_KIND)
+#define MPP_SCATTER_PELIST_2D_ mpp_scatter_pelist_int4_2d
+#define MPP_SCATTER_PELIST_3D_ mpp_scatter_pelist_int4_3d
+#include 
+
+#undef  MPP_SCATTER_PELIST_2D_
+#undef  MPP_SCATTER_PELIST_3D_
+#undef MPP_TYPE_
+#define MPP_TYPE_ real(FLOAT_KIND)
+#define MPP_SCATTER_PELIST_2D_ mpp_scatter_pelist_real4_2d
+#define MPP_SCATTER_PELIST_3D_ mpp_scatter_pelist_real4_3d
+#include 
+
+#undef  MPP_SCATTER_PELIST_2D_
+#undef  MPP_SCATTER_PELIST_3D_
+#undef MPP_TYPE_
+#define MPP_TYPE_ real(DOUBLE_KIND)
+#define MPP_SCATTER_PELIST_2D_ mpp_scatter_pelist_real8_2d
+#define MPP_SCATTER_PELIST_3D_ mpp_scatter_pelist_real8_3d
+#include 
diff --git a/src/shared/mpp/include/mpp_comm_mpi.inc b/src/shared/mpp/include/mpp_comm_mpi.inc
index 3fcd991453..a8a878497f 100644
--- a/src/shared/mpp/include/mpp_comm_mpi.inc
+++ b/src/shared/mpp/include/mpp_comm_mpi.inc
@@ -1,6 +1,6 @@
 
 ! -*-f90-*-
-! $Id: mpp_comm_mpi.inc,v 20.0 2013/12/14 00:24:07 fms Exp $
+! $Id$
 
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
@@ -392,7 +392,7 @@ end subroutine mpp_gsm_free
       if( .NOT.ANY(from_pe.EQ.peset(current_peset_num)%list) ) &
            call mpp_error( FATAL, 'mpp_broadcast_text: broadcasting from invalid PE.' )
 
-      if( current_clock.NE.0 )call SYSTEM_CLOCK(start_tick)
+      if( debug .and. (current_clock.NE.0) )call SYSTEM_CLOCK(start_tick)
  ! find the rank of from_pe in the pelist.     
       do i = 1, mpp_npes()
          if(peset(n)%list(i) == from_pe) then
@@ -402,7 +402,7 @@ end subroutine mpp_gsm_free
       enddo
       lptr = LOC (data)
       if( mpp_npes().GT.1 ) call MPI_BCAST( data, length*size(data(:)), MPI_CHARACTER, from_rank, peset(n)%id, error )
-      if( current_clock.NE.0 )call increment_current_clock( EVENT_BROADCAST, length )
+      if( debug .and. (current_clock.NE.0) )call increment_current_clock( EVENT_BROADCAST, length )
       return
     end subroutine mpp_broadcast_char
 
diff --git a/src/shared/mpp/include/mpp_comm_nocomm.inc b/src/shared/mpp/include/mpp_comm_nocomm.inc
index 268a26e7f7..7aa2cee176 100644
--- a/src/shared/mpp/include/mpp_comm_nocomm.inc
+++ b/src/shared/mpp/include/mpp_comm_nocomm.inc
@@ -1,6 +1,6 @@
 
 ! -*-f90-*-
-! $Id: mpp_comm_nocomm.inc,v 20.0 2013/12/14 00:24:09 fms Exp $
+! $Id$
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 !                                                                             !
diff --git a/src/shared/mpp/include/mpp_comm_sma.inc b/src/shared/mpp/include/mpp_comm_sma.inc
index f47b299d79..5f152e3a22 100644
--- a/src/shared/mpp/include/mpp_comm_sma.inc
+++ b/src/shared/mpp/include/mpp_comm_sma.inc
@@ -1,6 +1,6 @@
 
 ! -*-f90-*-
-! $Id: mpp_comm_sma.inc,v 20.0 2013/12/14 00:24:11 fms Exp $
+! $Id$
 
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
@@ -339,7 +339,7 @@ end subroutine mpp_malloc
       if( .NOT.ANY(from_pe.EQ.peset(current_peset_num)%list) ) &
            call mpp_error( FATAL, 'mpp_broadcast_text: broadcasting from invalid PE.' )
 
-      if( current_clock.NE.0 )call SYSTEM_CLOCK(start_tick)
+      if( debug .and. (current_clock.NE.0) )call SYSTEM_CLOCK(start_tick)
 
       ptr = LOC(mpp_stack)
       words = size(bdata(:))*size(transfer(bdata(1),word))
@@ -365,7 +365,7 @@ end subroutine mpp_malloc
           end do
       end if
 
-      if( current_clock.NE.0 )call increment_current_clock( EVENT_BROADCAST, length )
+      if( debug .and. (current_clock.NE.0) )call increment_current_clock( EVENT_BROADCAST, length )
       return
     end subroutine mpp_broadcast_char
 
diff --git a/src/shared/mpp/include/mpp_data_mpi.inc b/src/shared/mpp/include/mpp_data_mpi.inc
index 09053cbce0..0351dbb1dc 100644
--- a/src/shared/mpp/include/mpp_data_mpi.inc
+++ b/src/shared/mpp/include/mpp_data_mpi.inc
@@ -1,6 +1,6 @@
 
 ! -*-f90-*-
-! $Id: mpp_data_mpi.inc,v 19.0 2012/01/06 22:00:45 fms Exp $
+! $Id$
 
 !----------------------------------------------------------------!
 ! The following data is used in mpp_mod and its components       !
diff --git a/src/shared/mpp/include/mpp_data_nocomm.inc b/src/shared/mpp/include/mpp_data_nocomm.inc
index 5417b286af..702edcc0b6 100644
--- a/src/shared/mpp/include/mpp_data_nocomm.inc
+++ b/src/shared/mpp/include/mpp_data_nocomm.inc
@@ -1,6 +1,6 @@
 
 ! -*-f90-*-
-! $Id: mpp_data_nocomm.inc,v 19.0 2012/01/06 22:01:17 fms Exp $
+! $Id$
 
 !----------------------------------------------------------------!
 ! The following data is used in mpp_mod and its components       !
diff --git a/src/shared/mpp/include/mpp_data_sma.inc b/src/shared/mpp/include/mpp_data_sma.inc
index 4c1f28a29e..29afd40361 100644
--- a/src/shared/mpp/include/mpp_data_sma.inc
+++ b/src/shared/mpp/include/mpp_data_sma.inc
@@ -1,6 +1,6 @@
 
 ! -*-f90-*-
-! $Id: mpp_data_sma.inc,v 19.0 2012/01/06 22:01:19 fms Exp $
+! $Id$
 
 !----------------------------------------------------------------!
 ! The following data is used in mpp_mod and its components       !
diff --git a/src/shared/mpp/include/mpp_define_nest_domains.inc b/src/shared/mpp/include/mpp_define_nest_domains.inc
index 8b88199551..77c38a8095 100644
--- a/src/shared/mpp/include/mpp_define_nest_domains.inc
+++ b/src/shared/mpp/include/mpp_define_nest_domains.inc
@@ -1,5 +1,5 @@
 ! -*-f90-*-
-! $Id: mpp_define_nest_domains.inc,v 20.0 2013/12/14 00:24:16 fms Exp $
+! $Id$
 
 
 !#############################################################################
diff --git a/src/shared/mpp/include/mpp_do_get_boundary.h b/src/shared/mpp/include/mpp_do_get_boundary.h
index 3cd3aa3045..908bc12831 100644
--- a/src/shared/mpp/include/mpp_do_get_boundary.h
+++ b/src/shared/mpp/include/mpp_do_get_boundary.h
@@ -309,7 +309,7 @@ end subroutine MPP_DO_GET_BOUNDARY_3D_
 
 
 subroutine MPP_DO_GET_BOUNDARY_3D_V_(f_addrsx, f_addrsy, domain, boundx, boundy, b_addrsx, b_addrsy, &
-                                        bsizex, bsizey, ke, d_type, flags)
+                                        bsizex, bsizey, ke, d_type, flags, gridtype)
   type(domain2D),     intent(in)  :: domain
   type(overlapSpec),  intent(in)  :: boundx, boundy
   integer(LONG_KIND), intent(in)  :: f_addrsx(:,:), f_addrsy(:,:)
@@ -317,6 +317,7 @@ subroutine MPP_DO_GET_BOUNDARY_3D_V_(f_addrsx, f_addrsy, domain, boundx, boundy,
   integer,            intent(in)  :: bsizex(:), bsizey(:), ke
   MPP_TYPE_, intent(in)           :: d_type  ! creates unique interface
   integer, intent(in)             :: flags
+  integer, intent(in)             :: gridtype
 
   MPP_TYPE_ :: fieldx(boundx%xbegin:boundx%xend, boundx%ybegin:boundx%yend,ke)
   MPP_TYPE_ :: fieldy(boundy%xbegin:boundy%xend, boundy%ybegin:boundy%yend,ke)
@@ -342,7 +343,7 @@ subroutine MPP_DO_GET_BOUNDARY_3D_V_(f_addrsx, f_addrsy, domain, boundx, boundy,
   integer                 :: rank_x, rank_y, cur_rank, ind_x, ind_y
   integer                 :: nsend_x, nsend_y, nrecv_x, nrecv_y, num
   character(len=8)        :: text
-  integer                 :: outunit
+  integer                 :: outunit, shift, midpoint
 
   MPP_TYPE_ :: buffer(size(mpp_domains_stack(:)))
   pointer( ptr, buffer )
@@ -989,6 +990,33 @@ subroutine MPP_DO_GET_BOUNDARY_3D_V_(f_addrsx, f_addrsy, domain, boundx, boundy,
      cur_rank = min(rank_x, rank_y)
   end do
 
+  !--- domain always is symmetry
+  shift = 1
+  tMe = 1
+  if( BTEST(domain%fold,NORTH) .AND. (.NOT.BTEST(flags,SCALAR_BIT)) )then
+     j = domain%y(1)%global%end+shift
+     if( domain%y(1)%data%begin.LE.j .AND. j.LE.domain%y(1)%data%end+shift )then !fold is within domain
+        !poles set to 0: BGRID only
+        if( gridtype.EQ.BGRID_NE )then
+           midpoint = (domain%x(1)%global%begin+domain%x(1)%global%end-1+shift)/2
+           j  = domain%y(1)%global%end+shift - domain%y(1)%compute%begin + 1
+           is = domain%x(1)%global%begin; ie = domain%x(1)%global%end+shift
+           do i = is ,ie, midpoint
+              if( domain%x(1)%compute%begin == i )then
+                 do l=1,l_size
+                    ptr_wbufferx = b_addrsx(3, l, tMe)     
+                    ptr_wbuffery = b_addrsy(3, l, tMe)     
+                    do k = 1,ke
+                       wbufferx(j,k) = 0
+                       wbuffery(j,k) = 0
+                    end do
+                 end do
+              end if
+           end do
+        endif
+     endif
+  endif
+
   call mpp_sync_self( )
 
 
diff --git a/src/shared/mpp/include/mpp_do_update.h b/src/shared/mpp/include/mpp_do_update.h
index 9b74e67068..48a58348e4 100644
--- a/src/shared/mpp/include/mpp_do_update.h
+++ b/src/shared/mpp/include/mpp_do_update.h
@@ -1,19 +1,15 @@
 ! -*-f90-*- 
-    subroutine MPP_DO_UPDATE_3D_( f_addrs, domain, update, d_type, ke, b_addrs, b_size, flags)
+    subroutine MPP_DO_UPDATE_3D_( f_addrs, domain, update, d_type, ke, flags)
 !updates data domain of 3D field whose computational domains have been computed
       integer(LONG_KIND),         intent(in) :: f_addrs(:,:)
       type(domain2D),             intent(in) :: domain
       type(overlapSpec),          intent(in) :: update
       MPP_TYPE_,                  intent(in) :: d_type  ! creates unique interface
       integer,                    intent(in) :: ke
-      integer(LONG_KIND),         intent(in) :: b_addrs(:,:)
-      integer,                    intent(in) :: b_size
       integer, optional,          intent(in) :: flags
 
       MPP_TYPE_ :: field(update%xbegin:update%xend, update%ybegin:update%yend,ke)
-      MPP_TYPE_ :: fillbuffer(b_size)
       pointer(ptr_field, field)
-      pointer(ptr_buffer, fillbuffer)
       integer                     :: update_flags
       type(overlap_type), pointer :: overPtr => NULL()      
       character(len=8)            :: text
@@ -174,7 +170,8 @@
                tMe = overPtr%tileMe(n)
                is = overPtr%is(n); ie = overPtr%ie(n)
                js = overPtr%js(n); je = overPtr%je(n)
-               if( overptr%is_refined(n) ) then
+               select case( overPtr%rotation(n) )
+               case(ZERO)
                   do l=1,l_size  ! loop over number of fields
                      ptr_field = f_addrs(l, tMe)
                      do k = 1,ke  
@@ -186,58 +183,43 @@
                         end do
                      end do
                   end do
-               else
-                  select case( overPtr%rotation(n) )
-                  case(ZERO)
-                     do l=1,l_size  ! loop over number of fields
-                        ptr_field = f_addrs(l, tMe)
-                        do k = 1,ke  
-                           do j = js, je
-                              do i = is, ie
-                                 pos = pos + 1
-                                 buffer(pos) = field(i,j,k)
-                              end do
+               case( MINUS_NINETY ) 
+                  do l=1,l_size  ! loop over number of fields
+                     ptr_field = f_addrs(l, tMe)
+                     do k = 1,ke  
+                        do i = is, ie
+                           do j = je, js, -1
+                              pos = pos + 1
+                              buffer(pos) = field(i,j,k)
                            end do
                         end do
                      end do
-                  case( MINUS_NINETY ) 
-                     do l=1,l_size  ! loop over number of fields
-                        ptr_field = f_addrs(l, tMe)
-                        do k = 1,ke  
-                           do i = is, ie
-                              do j = je, js, -1
-                                 pos = pos + 1
-                                 buffer(pos) = field(i,j,k)
-                              end do
+                  end do
+               case( NINETY ) 
+                  do l=1,l_size  ! loop over number of fields
+                     ptr_field = f_addrs(l, tMe)
+                     do k = 1,ke  
+                        do i = ie, is, -1
+                           do j = js, je
+                              pos = pos + 1
+                              buffer(pos) = field(i,j,k)
                            end do
                         end do
                      end do
-                  case( NINETY ) 
-                     do l=1,l_size  ! loop over number of fields
-                        ptr_field = f_addrs(l, tMe)
-                        do k = 1,ke  
+                  end do
+               case( ONE_HUNDRED_EIGHTY ) 
+                  do l=1,l_size  ! loop over number of fields
+                     ptr_field = f_addrs(l, tMe)
+                     do k = 1,ke  
+                        do j = je, js, -1
                            do i = ie, is, -1
-                              do j = js, je
-                                 pos = pos + 1
-                                 buffer(pos) = field(i,j,k)
-                              end do
-                           end do
-                        end do
-                     end do
-                  case( ONE_HUNDRED_EIGHTY ) 
-                     do l=1,l_size  ! loop over number of fields
-                        ptr_field = f_addrs(l, tMe)
-                        do k = 1,ke  
-                           do j = je, js, -1
-                              do i = ie, is, -1
-                                 pos = pos + 1
-                                 buffer(pos) = field(i,j,k)
-                              end do
+                              pos = pos + 1
+                              buffer(pos) = field(i,j,k)
                            end do
                         end do
                      end do
-                  end select
-               end if
+                  end do
+               end select
             endif
          end do ! do n = 1, overPtr%count
          send_msgsize(m) = pos-buffer_pos
@@ -278,44 +260,18 @@
                msgsize = (ie-is+1)*(je-js+1)*ke*l_size
                pos = buffer_pos - msgsize
                buffer_pos = pos
-               if(OverPtr%is_refined(n)) then
-                  index = overPtr%index(n)
-                  is1 = update%rSpec(tMe)%isNbr(index); ie1 = update%rSpec(tMe)%ieNbr(index)
-                  js1 = update%rSpec(tMe)%jsNbr(index); je1 = update%rSpec(tMe)%jeNbr(index)
-                  ni = ie1 - is1 + 1
-                  nj = je1 - js1 + 1
-                  total = ni*nj
-                  start = (update%rSpec(tMe)%start(index)-1)*ke
-                  if(start+total*ke>b_size ) call mpp_error(FATAL, &
-                       "MPP_DO_UPDATE: b_size is less than the size of the data to be filled.")
-                  msgsize = ie - is + 1
-                  do l=1, l_size  ! loop over number of fields
-                     ptr_buffer = b_addrs(l, tMe)
-                     start1 = start + (js-js1)*ni + is - is1
-                     do k = 1, ke
-                        start2 = start1
-                        do j = js, je
-                           fillbuffer(start2+1:start2+msgsize) = buffer(pos+1:pos+msgsize)
-                           start2 = start2 + ni
-                           pos   = pos + msgsize
+               do l=1,l_size  ! loop over number of fields
+                  ptr_field = f_addrs(l, tMe)
+                  do k = 1,ke
+                     do j = js, je
+                        do i = is, ie
+                           pos = pos + 1
+                           field(i,j,k) = buffer(pos)
                         end do
-                        start1 = start1 + total
                      end do
                   end do
-               else
-                  do l=1,l_size  ! loop over number of fields
-                     ptr_field = f_addrs(l, tMe)
-                     do k = 1,ke
-                        do j = js, je
-                           do i = is, ie
-                              pos = pos + 1
-                              field(i,j,k) = buffer(pos)
-                           end do
-                        end do
-                     end do
-                  end do
-               endif
-            end if
+               end do
+            endif
          end do ! do n = 1, overPtr%count
       end do
       call mpp_clock_end(unpk_clock)
diff --git a/src/shared/mpp/include/mpp_do_updateV.h b/src/shared/mpp/include/mpp_do_updateV.h
index 648843b553..90c4e41d13 100644
--- a/src/shared/mpp/include/mpp_do_updateV.h
+++ b/src/shared/mpp/include/mpp_do_updateV.h
@@ -1,37 +1,30 @@
 ! -*-f90-*- 
     subroutine MPP_DO_UPDATE_3D_V_(f_addrsx,f_addrsy, domain, update_x, update_y, &
-                                   d_type, ke, b_addrsx, b_addrsy, b_sizex, b_sizey, gridtype, flags)
+                                   d_type, ke, gridtype, flags)
 !updates data domain of 3D field whose computational domains have been computed
       integer(LONG_KIND),  intent(in)        :: f_addrsx(:,:), f_addrsy(:,:)
       type(domain2d),      intent(in)        :: domain
       type(overlapSpec),   intent(in)        :: update_x, update_y
       integer,             intent(in)        :: ke
       MPP_TYPE_, intent(in)                  :: d_type  ! creates unique interface
-      integer(LONG_KIND), intent(in)         :: b_addrsx(:,:), b_addrsy(:,:)
-      integer,            intent(in)         :: b_sizex, b_sizey
       integer, intent(in)                    :: gridtype
       integer, intent(in),          optional :: flags
 
       MPP_TYPE_ :: fieldx(update_x%xbegin:update_x%xend, update_x%ybegin:update_x%yend,ke)
       MPP_TYPE_ :: fieldy(update_y%xbegin:update_y%xend, update_y%ybegin:update_y%yend,ke)
-      MPP_TYPE_ :: bufferx(b_sizex)
-      MPP_TYPE_ :: buffery(b_sizey)
       pointer(ptr_fieldx, fieldx)
       pointer(ptr_fieldy, fieldy)
-      pointer(ptr_bufferx, bufferx)
-      pointer(ptr_buffery, buffery)
 
       integer :: update_flags
       integer :: l_size, l, i, j, k, is, ie, js, je, n, m
       integer :: pos, nlist, msgsize
       integer :: to_pe, from_pe, midpoint
       integer :: tMe, dir
-      integer :: index, is1, ie1, js1, je1, ni, nj, total, start1, start, start2
 
       integer :: send_start_pos, nsend
       integer :: send_msgsize(2*MAXLIST)
       integer :: send_pe(2*MAXLIST)
-      integer, allocatable :: msg1(:), msg2(:)
+      integer,    allocatable :: msg1(:), msg2(:)
       logical :: send(8), recv(8), update_edge_only
       MPP_TYPE_ :: buffer(size(mpp_domains_stack(:)))
       pointer(ptr,buffer )
@@ -322,221 +315,112 @@
 
                      is = update_x%send(ind_x)%is(n); ie = update_x%send(ind_x)%ie(n)
                      js = update_x%send(ind_x)%js(n); je = update_x%send(ind_x)%je(n)
-                     if( update_x%send(ind_x)%is_refined(n) ) then
-                        select case( update_x%send(ind_x)%rotation(n) )
-                        case(ZERO)
+                     select case( update_x%send(ind_x)%rotation(n) )
+                     case(ZERO)
+                        do l=1,l_size  ! loop over number of fields
+                           ptr_fieldx = f_addrsx(l,tMe)
+                           ptr_fieldy = f_addrsy(l,tMe)
+                           do k = 1,ke
+                              do j = js, je
+                                 do i = is, ie
+                                    pos = pos + 2
+                                    buffer(pos-1) = fieldx(i,j,k)
+                                    buffer(pos)   = fieldy(i,j,k)
+                                 end do
+                              end do
+                           end do
+                        end do
+                     case( MINUS_NINETY ) 
+                        if( BTEST(update_flags,SCALAR_BIT) ) then
                            do l=1,l_size  ! loop over number of fields
                               ptr_fieldx = f_addrsx(l,tMe)
                               ptr_fieldy = f_addrsy(l,tMe)
                               do k = 1,ke
-                                 do j = js, je
-                                    do i = is, ie
+                                 do i = is, ie
+                                    do j = je, js, -1
                                        pos = pos + 2
-                                       buffer(pos-1) = fieldx(i,j,k)
-                                       buffer(pos)   = fieldy(i,j,k)
+                                       buffer(pos-1) = fieldy(i,j,k)
+                                       buffer(pos)   = fieldx(i,j,k)
                                     end do
                                  end do
                               end do
                            end do
-                        case(MINUS_NINETY)
-                           if( BTEST(update_flags,SCALAR_BIT) ) then
-                              do l=1,l_size  ! loop over number of fields
-                                 ptr_fieldx = f_addrsx(l,tMe)
-                                 ptr_fieldy = f_addrsy(l,tMe)
-                                 do k = 1,ke
-                                    do j = js, je
-                                       do i = is, ie
-                                          pos = pos + 2
-                                          buffer(pos-1) =  fieldy(i,j,k)
-                                          buffer(pos)   =  fieldx(i,j,k)
-                                       end do
-                                    end do
-                                 end do
-                              end do
-                           else
-                              do l=1,l_size  ! loop over number of fields
-                                 ptr_fieldx = f_addrsx(l,tMe)
-                                 ptr_fieldy = f_addrsy(l,tMe)
-                                 do k = 1,ke
-                                    do j = js, je
-                                       do i = is, ie
-                                          pos = pos + 2
-                                          buffer(pos-1) = -fieldy(i,j,k)
-                                          buffer(pos)   =  fieldx(i,j,k)
-                                       end do
-                                    end do
-                                 end do
-                              end do
-                           end if
-                        case(NINETY)
-                           if( BTEST(update_flags,SCALAR_BIT) ) then
-                              do l=1,l_size  ! loop over number of fields
-                                 ptr_fieldx = f_addrsx(l,tMe)
-                                 ptr_fieldy = f_addrsy(l,tMe)
-                                 do k = 1,ke
-                                    do j = js, je
-                                       do i = is, ie
-                                          pos = pos + 2
-                                          buffer(pos-1) =  fieldy(i,j,k)
-                                          buffer(pos)   =  fieldx(i,j,k)
-                                       end do
-                                    end do
-                                 end do
-                              end do
-                           else
-                              do l=1,l_size  ! loop over number of fields
-                                 ptr_fieldx = f_addrsx(l,tMe)
-                                 ptr_fieldy = f_addrsy(l,tMe)
-                                 do k = 1,ke
-                                    do j = js, je
-                                       do i = is, ie
-                                          pos = pos + 2
-                                          buffer(pos-1) =  fieldy(i,j,k)
-                                          buffer(pos)   = -fieldx(i,j,k)
-                                       end do
-                                    end do
-                                 end do
-                              end do
-                           end if
-                        case(ONE_HUNDRED_EIGHTY)
-                           if( BTEST(update_flags,SCALAR_BIT) ) then
-                              do l=1,l_size  ! loop over number of fields
-                                 ptr_fieldx = f_addrsx(l,tMe)
-                                 ptr_fieldy = f_addrsy(l,tMe)
-                                 do k = 1,ke
-                                    do j = js, je
-                                       do i = is, ie
-                                          pos = pos + 2
-                                          buffer(pos-1) = fieldx(i,j,k)
-                                          buffer(pos)   = fieldy(i,j,k)
-                                       end do
+                        else
+                           do l=1,l_size  ! loop over number of fields
+                              ptr_fieldx = f_addrsx(l,tMe)
+                              ptr_fieldy = f_addrsy(l,tMe)
+                              do k = 1,ke
+                                 do i = is, ie
+                                    do j = je, js, -1
+                                       pos = pos + 2
+                                       buffer(pos-1) = -fieldy(i,j,k)
+                                       buffer(pos)   =  fieldx(i,j,k)
                                     end do
                                  end do
                               end do
-                           else
-                              do l=1,l_size  ! loop over number of fields
-                                 ptr_fieldx = f_addrsx(l,tMe)
-                                 ptr_fieldy = f_addrsy(l,tMe)
-                                 do k = 1,ke
+                           end do
+                        end if
+                     case( NINETY )
+                        if( BTEST(update_flags,SCALAR_BIT) ) then
+                           do l=1,l_size  ! loop over number of fields
+                              ptr_fieldx = f_addrsx(l,tMe)
+                              ptr_fieldy = f_addrsy(l,tMe)
+                              do k = 1,ke
+                                 do i = ie, is, -1
                                     do j = js, je
-                                       do i = is, ie
-                                          pos = pos + 2
-                                          buffer(pos-1) = -fieldx(i,j,k)
-                                          buffer(pos)   = -fieldy(i,j,k)
-                                       end do
+                                       pos = pos + 2
+                                       buffer(pos-1) = fieldy(i,j,k)
+                                       buffer(pos)   = fieldx(i,j,k)
                                     end do
                                  end do
                               end do
-                           end if
-                        end select  ! select case( rotation(n) )
-                     else   ! if( is_refined(n) )
-                        select case( update_x%send(ind_x)%rotation(n) )
-                        case(ZERO)
+                           end do
+                        else
                            do l=1,l_size  ! loop over number of fields
                               ptr_fieldx = f_addrsx(l,tMe)
                               ptr_fieldy = f_addrsy(l,tMe)
                               do k = 1,ke
-                                 do j = js, je
-                                    do i = is, ie
+                                 do i = ie, is, -1
+                                    do j = js, je
                                        pos = pos + 2
-                                       buffer(pos-1) = fieldx(i,j,k)
-                                       buffer(pos)   = fieldy(i,j,k)
+                                       buffer(pos-1) = fieldy(i,j,k)
+                                       buffer(pos)   = -fieldx(i,j,k)
                                     end do
                                  end do
                               end do
                            end do
-                        case( MINUS_NINETY ) 
-                           if( BTEST(update_flags,SCALAR_BIT) ) then
-                              do l=1,l_size  ! loop over number of fields
-                                 ptr_fieldx = f_addrsx(l,tMe)
-                                 ptr_fieldy = f_addrsy(l,tMe)
-                                 do k = 1,ke
-                                    do i = is, ie
-                                       do j = je, js, -1
-                                          pos = pos + 2
-                                          buffer(pos-1) = fieldy(i,j,k)
-                                          buffer(pos)   = fieldx(i,j,k)
-                                       end do
-                                    end do
-                                 end do
-                              end do
-                           else
-                              do l=1,l_size  ! loop over number of fields
-                                 ptr_fieldx = f_addrsx(l,tMe)
-                                 ptr_fieldy = f_addrsy(l,tMe)
-                                 do k = 1,ke
-                                    do i = is, ie
-                                       do j = je, js, -1
-                                          pos = pos + 2
-                                          buffer(pos-1) = -fieldy(i,j,k)
-                                          buffer(pos)   =  fieldx(i,j,k)
-                                       end do
-                                    end do
-                                 end do
-                              end do
-                           end if
-                        case( NINETY )
-                           if( BTEST(update_flags,SCALAR_BIT) ) then
-                              do l=1,l_size  ! loop over number of fields
-                                 ptr_fieldx = f_addrsx(l,tMe)
-                                 ptr_fieldy = f_addrsy(l,tMe)
-                                 do k = 1,ke
+                        end if
+                     case( ONE_HUNDRED_EIGHTY )
+                        if( BTEST(update_flags,SCALAR_BIT) ) then
+                           do l=1,l_size  ! loop over number of fields
+                              ptr_fieldx = f_addrsx(l,tMe)
+                              ptr_fieldy = f_addrsy(l,tMe)
+                              do k = 1,ke
+                                 do j = je, js, -1
                                     do i = ie, is, -1
-                                       do j = js, je
-                                          pos = pos + 2
-                                          buffer(pos-1) = fieldy(i,j,k)
-                                          buffer(pos)   = fieldx(i,j,k)
-                                       end do
+                                       pos = pos + 2
+                                       buffer(pos-1) =  fieldx(i,j,k)
+                                       buffer(pos)   =  fieldy(i,j,k)
                                     end do
                                  end do
                               end do
-                           else
-                              do l=1,l_size  ! loop over number of fields
-                                 ptr_fieldx = f_addrsx(l,tMe)
-                                 ptr_fieldy = f_addrsy(l,tMe)
-                                 do k = 1,ke
+                           end do
+                        else
+                           do l=1,l_size  ! loop over number of fields
+                              ptr_fieldx = f_addrsx(l,tMe)
+                              ptr_fieldy = f_addrsy(l,tMe)
+                              do k = 1,ke
+                                 do j = je, js, -1
                                     do i = ie, is, -1
-                                       do j = js, je
-                                          pos = pos + 2
-                                          buffer(pos-1) = fieldy(i,j,k)
-                                          buffer(pos)   = -fieldx(i,j,k)
-                                       end do
-                                    end do
-                                 end do
-                              end do
-                           end if
-                        case( ONE_HUNDRED_EIGHTY )
-                           if( BTEST(update_flags,SCALAR_BIT) ) then
-                              do l=1,l_size  ! loop over number of fields
-                                 ptr_fieldx = f_addrsx(l,tMe)
-                                 ptr_fieldy = f_addrsy(l,tMe)
-                                 do k = 1,ke
-                                    do j = je, js, -1
-                                       do i = ie, is, -1
-                                          pos = pos + 2
-                                          buffer(pos-1) =  fieldx(i,j,k)
-                                          buffer(pos)   =  fieldy(i,j,k)
-                                       end do
-                                    end do
-                                 end do
-                              end do
-                           else
-                              do l=1,l_size  ! loop over number of fields
-                                 ptr_fieldx = f_addrsx(l,tMe)
-                                 ptr_fieldy = f_addrsy(l,tMe)
-                                 do k = 1,ke
-                                    do j = je, js, -1
-                                       do i = ie, is, -1
-                                          pos = pos + 2
-                                          buffer(pos-1) =  -fieldx(i,j,k)
-                                          buffer(pos)   =  -fieldy(i,j,k)
-                                       end do
+                                       pos = pos + 2
+                                       buffer(pos-1) =  -fieldx(i,j,k)
+                                       buffer(pos)   =  -fieldy(i,j,k)
                                     end do
                                  end do
                               end do
-                           end if
-                        end select ! select case( rotation(n) )
-                     end if ! if( is_refined(n) )
+                           end do
+                        end if
+                     end select ! select case( rotation(n) )
                   end if ! if( send(dir) ) 
                end do ! do n = 1, update_x%send(ind_x)%count
                ind_x = ind_x+1
@@ -558,162 +442,90 @@
                      tMe = update_x%send(ind_x)%tileMe(n)
                      is = update_x%send(ind_x)%is(n); ie = update_x%send(ind_x)%ie(n)
                      js = update_x%send(ind_x)%js(n); je = update_x%send(ind_x)%je(n)
-                     if( update_x%send(ind_x)%is_refined(n) ) then
-                        select case( update_x%send(ind_x)%rotation(n) )
-                        case(ZERO)
-                           do l=1,l_size  ! loop over number of fields
-                              ptr_fieldx = f_addrsx(l, tMe)
-                              ptr_fieldy = f_addrsy(l, tMe)
-                              do k = 1,ke
-                                 do j = js, je
-                                    do i = is, ie
-                                       pos = pos + 1
-                                       buffer(pos) = fieldx(i,j,k)
-                                    end do
+                     select case( update_x%send(ind_x)%rotation(n) )
+                     case(ZERO)
+                        do l=1,l_size  ! loop over number of fields
+                           ptr_fieldx = f_addrsx(l, tMe)
+                           ptr_fieldy = f_addrsy(l, tMe)
+                           do k = 1,ke
+                              do j = js, je
+                                 do i = is, ie
+                                    pos = pos + 1
+                                    buffer(pos) = fieldx(i,j,k)
                                  end do
                               end do
                            end do
-                        case(MINUS_NINETY)
-                           if( BTEST(update_flags,SCALAR_BIT) ) then
-                              do l=1,l_size  ! loop over number of fields
-                                 ptr_fieldx = f_addrsx(l, tMe)
-                                 ptr_fieldy = f_addrsy(l, tMe)
-                                 do k = 1,ke
-                                    do j = js, je
-                                       do i = is, ie
-                                          pos = pos + 1
-                                          buffer(pos) = fieldy(i,j,k)
-                                       end do
-                                    end do
-                                 end do
-                              end do
-                           else
-                              do l=1,l_size  ! loop over number of fields
-                                 ptr_fieldx = f_addrsx(l, tMe)
-                                 ptr_fieldy = f_addrsy(l, tMe)
-                                 do k = 1,ke
-                                    do j = js, je
-                                       do i = is, ie
-                                          pos = pos + 1
-                                          buffer(pos) = -fieldy(i,j,k)
-                                       end do
-                                    end do
-                                 end do
-                              end do
-                           end if
-                        case(NINETY)
+                        end do
+                     case(MINUS_NINETY)
+                        if( BTEST(update_flags,SCALAR_BIT) ) then
                            do l=1,l_size  ! loop over number of fields
                               ptr_fieldx = f_addrsx(l, tMe)
                               ptr_fieldy = f_addrsy(l, tMe)
-                              do k = 1, ke
-                                 do j = js, je
-                                    do i = is, ie
+                              do k = 1,ke
+                                 do i = is, ie
+                                    do j = je, js, -1
                                        pos = pos + 1
                                        buffer(pos) = fieldy(i,j,k)
                                     end do
                                  end do
                               end do
                            end do
-                        case(ONE_HUNDRED_EIGHTY)
+                        else
                            do l=1,l_size  ! loop over number of fields
                               ptr_fieldx = f_addrsx(l, tMe)
                               ptr_fieldy = f_addrsy(l, tMe)
                               do k = 1,ke
-                                 do j = js, je
-                                    do i = is, ie
+                                 do i = is, ie
+                                    do j = je, js, -1
                                        pos = pos + 1
-                                       buffer(pos) = fieldx(i,j,k)
+                                       buffer(pos) = -fieldy(i,j,k)
                                     end do
                                  end do
                               end do
                            end do
-                        end select
-                     else
-                        select case( update_x%send(ind_x)%rotation(n) )
-                        case(ZERO)
+                        end if
+                     case(NINETY)
+                        do l=1,l_size  ! loop over number of fields
+                           ptr_fieldx = f_addrsx(l, tMe)
+                           ptr_fieldy = f_addrsy(l, tMe)
+                           do k = 1, ke
+                              do i = ie, is, -1
+                                 do j = js, je
+                                    pos = pos + 1
+                                    buffer(pos) = fieldy(i,j,k)
+                                 end do
+                              end do
+                           end do
+                        end do
+                     case(ONE_HUNDRED_EIGHTY)
+                        if( BTEST(update_flags,SCALAR_BIT) ) then
                            do l=1,l_size  ! loop over number of fields
                               ptr_fieldx = f_addrsx(l, tMe)
                               ptr_fieldy = f_addrsy(l, tMe)
                               do k = 1,ke
-                                 do j = js, je
-                                    do i = is, ie
+                                 do j = je, js, -1
+                                    do i = ie, is, -1
                                        pos = pos + 1
                                        buffer(pos) = fieldx(i,j,k)
                                     end do
                                  end do
                               end do
                            end do
-                        case(MINUS_NINETY)
-                           if( BTEST(update_flags,SCALAR_BIT) ) then
-                              do l=1,l_size  ! loop over number of fields
-                                 ptr_fieldx = f_addrsx(l, tMe)
-                                 ptr_fieldy = f_addrsy(l, tMe)
-                                 do k = 1,ke
-                                    do i = is, ie
-                                       do j = je, js, -1
-                                          pos = pos + 1
-                                          buffer(pos) = fieldy(i,j,k)
-                                       end do
-                                    end do
-                                 end do
-                              end do
-                           else
-                              do l=1,l_size  ! loop over number of fields
-                                 ptr_fieldx = f_addrsx(l, tMe)
-                                 ptr_fieldy = f_addrsy(l, tMe)
-                                 do k = 1,ke
-                                    do i = is, ie
-                                       do j = je, js, -1
-                                          pos = pos + 1
-                                          buffer(pos) = -fieldy(i,j,k)
-                                       end do
-                                    end do
-                                 end do
-                              end do
-                           end if
-                        case(NINETY)
+                        else
                            do l=1,l_size  ! loop over number of fields
                               ptr_fieldx = f_addrsx(l, tMe)
                               ptr_fieldy = f_addrsy(l, tMe)
-                              do k = 1, ke
-                                 do i = ie, is, -1
-                                    do j = js, je
+                              do k = 1,ke
+                                 do j = je, js, -1
+                                    do i = ie, is, -1
                                        pos = pos + 1
-                                       buffer(pos) = fieldy(i,j,k)
+                                       buffer(pos) = -fieldx(i,j,k)
                                     end do
                                  end do
                               end do
                            end do
-                        case(ONE_HUNDRED_EIGHTY)
-                           if( BTEST(update_flags,SCALAR_BIT) ) then
-                              do l=1,l_size  ! loop over number of fields
-                                 ptr_fieldx = f_addrsx(l, tMe)
-                                 ptr_fieldy = f_addrsy(l, tMe)
-                                 do k = 1,ke
-                                    do j = je, js, -1
-                                       do i = ie, is, -1
-                                          pos = pos + 1
-                                          buffer(pos) = fieldx(i,j,k)
-                                       end do
-                                    end do
-                                 end do
-                              end do
-                           else
-                              do l=1,l_size  ! loop over number of fields
-                                 ptr_fieldx = f_addrsx(l, tMe)
-                                 ptr_fieldy = f_addrsy(l, tMe)
-                                 do k = 1,ke
-                                    do j = je, js, -1
-                                       do i = ie, is, -1
-                                          pos = pos + 1
-                                          buffer(pos) = -fieldx(i,j,k)
-                                       end do
-                                    end do
-                                 end do
-                              end do
-                           end if
-                        end select
-                     end if
+                        end if
+                     end select
                   end if
                end do
                ind_x = ind_x+1
@@ -732,162 +544,90 @@
                      tMe = update_y%send(ind_y)%tileMe(n)
                      is = update_y%send(ind_y)%is(n); ie = update_y%send(ind_y)%ie(n)
                      js = update_y%send(ind_y)%js(n); je = update_y%send(ind_y)%je(n)
-                     if( update_y%send(ind_y)%is_refined(n) ) then
-                        select case( update_y%send(ind_y)%rotation(n) )
-                        case(ZERO)
-                           do l=1,l_size  ! loop over number of fields
-                              ptr_fieldx = f_addrsx(l, tMe)
-                              ptr_fieldy = f_addrsy(l, tMe)
-                              do k = 1,ke
-                                 do j = js, je
-                                    do i = is, ie
-                                       pos = pos + 1
-                                       buffer(pos) = fieldy(i,j,k)
-                                    end do
+                     select case( update_y%send(ind_y)%rotation(n) )
+                     case(ZERO)
+                        do l=1,l_size  ! loop over number of fields
+                           ptr_fieldx = f_addrsx(l, tMe)
+                           ptr_fieldy = f_addrsy(l, tMe)
+                           do k = 1,ke
+                              do j = js, je
+                                 do i = is, ie
+                                    pos = pos + 1
+                                    buffer(pos) = fieldy(i,j,k)
                                  end do
                               end do
                            end do
-                        case(MINUS_NINETY)
+                        end do
+                     case(MINUS_NINETY)
+                        do l=1,l_size  ! loop over number of fields
+                           ptr_fieldx = f_addrsx(l, tMe)
+                           ptr_fieldy = f_addrsy(l, tMe)
+                           do k = 1,ke
+                              do i = is, ie
+                                 do j = je, js, -1
+                                    pos = pos + 1
+                                    buffer(pos) = fieldx(i,j,k)
+                                 end do
+                              end do
+                           end do
+                        end do
+                     case(NINETY)
+                        if( BTEST(update_flags,SCALAR_BIT) ) then
                            do l=1,l_size  ! loop over number of fields
                               ptr_fieldx = f_addrsx(l, tMe)
                               ptr_fieldy = f_addrsy(l, tMe)
                               do k = 1,ke
-                                 do j = js, je
-                                    do i = is, ie
+                                 do i = ie, is, -1
+                                    do j = js, je
                                        pos = pos + 1
                                        buffer(pos) = fieldx(i,j,k)
                                     end do
                                  end do
                               end do
                            end do
-                        case(NINETY)
-                           if( BTEST(update_flags,SCALAR_BIT) ) then
-                              do l=1,l_size  ! loop over number of fields
-                                 ptr_fieldx = f_addrsx(l, tMe)
-                                 ptr_fieldy = f_addrsy(l, tMe)
-                                 do k = 1,ke
-                                    do j = js, je
-                                       do i = is, ie
-                                          pos = pos + 1
-                                          buffer(pos) = fieldx(i,j,k)
-                                       end do
-                                    end do
-                                 end do
-                              end do
-                           else
-                              do l=1,l_size  ! loop over number of fields
-                                 ptr_fieldx = f_addrsx(l, tMe)
-                                 ptr_fieldy = f_addrsy(l, tMe)
-                                 do k = 1,ke
-                                    do j = js, je
-                                       do i = is, ie
-                                          pos = pos + 1
-                                          buffer(pos) = -fieldx(i,j,k)
-                                       end do
-                                    end do
-                                 end do
-                              end do
-                           end if
-                        case(ONE_HUNDRED_EIGHTY)
+                        else
                            do l=1,l_size  ! loop over number of fields
                               ptr_fieldx = f_addrsx(l, tMe)
                               ptr_fieldy = f_addrsy(l, tMe)
                               do k = 1,ke
-                                 do j = js, je
-                                    do i = is, ie
+                                 do i = ie, is, -1
+                                    do j = js, je
                                        pos = pos + 1
-                                       buffer(pos) = fieldy(i,j,k)
+                                       buffer(pos) = -fieldx(i,j,k)
                                     end do
                                  end do
                               end do
                            end do
-                        end select
-                     else
-                        select case( update_y%send(ind_y)%rotation(n) )
-                        case(ZERO)
+                        end if
+                     case(ONE_HUNDRED_EIGHTY)
+                        if( BTEST(update_flags,SCALAR_BIT) ) then
                            do l=1,l_size  ! loop over number of fields
                               ptr_fieldx = f_addrsx(l, tMe)
                               ptr_fieldy = f_addrsy(l, tMe)
                               do k = 1,ke
-                                 do j = js, je
-                                    do i = is, ie
+                                 do j = je, js, -1
+                                    do i = ie, is, -1
                                        pos = pos + 1
                                        buffer(pos) = fieldy(i,j,k)
                                     end do
                                  end do
                               end do
                            end do
-                        case(MINUS_NINETY)
+                        else
                            do l=1,l_size  ! loop over number of fields
                               ptr_fieldx = f_addrsx(l, tMe)
                               ptr_fieldy = f_addrsy(l, tMe)
                               do k = 1,ke
-                                 do i = is, ie
-                                    do j = je, js, -1
+                                 do j = je, js, -1
+                                    do i = ie, is, -1
                                        pos = pos + 1
-                                       buffer(pos) = fieldx(i,j,k)
+                                       buffer(pos) = -fieldy(i,j,k)
                                     end do
                                  end do
                               end do
                            end do
-                        case(NINETY)
-                           if( BTEST(update_flags,SCALAR_BIT) ) then
-                              do l=1,l_size  ! loop over number of fields
-                                 ptr_fieldx = f_addrsx(l, tMe)
-                                 ptr_fieldy = f_addrsy(l, tMe)
-                                 do k = 1,ke
-                                    do i = ie, is, -1
-                                       do j = js, je
-                                          pos = pos + 1
-                                          buffer(pos) = fieldx(i,j,k)
-                                       end do
-                                    end do
-                                 end do
-                              end do
-                           else
-                              do l=1,l_size  ! loop over number of fields
-                                 ptr_fieldx = f_addrsx(l, tMe)
-                                 ptr_fieldy = f_addrsy(l, tMe)
-                                 do k = 1,ke
-                                    do i = ie, is, -1
-                                       do j = js, je
-                                          pos = pos + 1
-                                          buffer(pos) = -fieldx(i,j,k)
-                                       end do
-                                    end do
-                                 end do
-                              end do
-                           end if
-                        case(ONE_HUNDRED_EIGHTY)
-                           if( BTEST(update_flags,SCALAR_BIT) ) then
-                              do l=1,l_size  ! loop over number of fields
-                                 ptr_fieldx = f_addrsx(l, tMe)
-                                 ptr_fieldy = f_addrsy(l, tMe)
-                                 do k = 1,ke
-                                    do j = je, js, -1
-                                       do i = ie, is, -1
-                                          pos = pos + 1
-                                          buffer(pos) = fieldy(i,j,k)
-                                       end do
-                                    end do
-                                 end do
-                              end do
-                           else
-                              do l=1,l_size  ! loop over number of fields
-                                 ptr_fieldx = f_addrsx(l, tMe)
-                                 ptr_fieldy = f_addrsy(l, tMe)
-                                 do k = 1,ke
-                                    do j = je, js, -1
-                                       do i = ie, is, -1
-                                          pos = pos + 1
-                                          buffer(pos) = -fieldy(i,j,k)
-                                       end do
-                                    end do
-                                 end do
-                              end do
-                           end if
-                        end select
-                     end if
+                        end if
+                     end select
                   endif
                enddo
                ind_y = ind_y+1
@@ -941,50 +681,19 @@
                      msgsize = (ie-is+1)*(je-js+1)*ke*2*l_size
                      pos = buffer_pos - msgsize
                      buffer_pos = pos
-                     if(update_x%recv(ind_x)%is_refined(n)) then
-                        index = update_x%recv(ind_x)%index(n)
-                        is1 = update_x%rSpec(tMe)%isNbr(index); ie1 = update_x%rSpec(tMe)%ieNbr(index)
-                        js1 = update_x%rSpec(tMe)%jsNbr(index); je1 = update_x%rSpec(tMe)%jeNbr(index)
-                        ni = ie1 - is1 + 1
-                        nj = je1 - js1 + 1
-                        total = ni*nj
-                        start = (update_x%rSpec(tMe)%start(index)-1)*ke
-
-                        if(start+total*ke>b_sizex ) call mpp_error(FATAL, &
-                             "MPP_DO_UPDATE_V: b_size is less than the size of the data to be filled.")
-                        msgsize = ie - is + 1
-                        do l=1, l_size  ! loop over number of fields
-                           ptr_bufferx = b_addrsx(l, tMe)
-                           ptr_buffery = b_addrsy(l, tMe)
-                           start1 = start + (js-js1)*ni + is - is1 
-                           do k = 1, ke
-                              start2 = start1
-                              do j = js, je
-                                 do i = start2+1, start2+msgsize
-                                    pos = pos + 2
-                                    bufferx(i) = buffer(pos-1)
-                                    buffery(i) = buffer(pos)
-                                 end do
-                                 start2 = start2 + ni
-                              end do
-                              start1 = start1 + total
-                           end do
-                        end do
-                     else
-                        do l=1, l_size  ! loop over number of fields
-                           ptr_fieldx = f_addrsx(l, tMe)
-                           ptr_fieldy = f_addrsy(l, tMe)
-                           do k = 1,ke
-                              do j = js, je
-                                 do i = is, ie
-                                    pos = pos + 2
-                                    fieldx(i,j,k) = buffer(pos-1)
-                                    fieldy(i,j,k) = buffer(pos)
-                                 end do
+                     do l=1, l_size  ! loop over number of fields
+                        ptr_fieldx = f_addrsx(l, tMe)
+                        ptr_fieldy = f_addrsy(l, tMe)
+                        do k = 1,ke
+                           do j = js, je
+                              do i = is, ie
+                                 pos = pos + 2
+                                 fieldx(i,j,k) = buffer(pos-1)
+                                 fieldy(i,j,k) = buffer(pos)
                               end do
                            end do
                         end do
-                     end if
+                     end do
                   end if ! end if( recv(dir) )
                end do  ! do dir=8,1,-1 
                ind_x = ind_x-1
@@ -1008,47 +717,18 @@
                      msgsize = (ie-is+1)*(je-js+1)*ke*l_size
                      pos = buffer_pos - msgsize
                      buffer_pos = pos
-                     if(update_y%recv(ind_y)%is_refined(n)) then
-                        index = update_y%recv(ind_y)%index(n)
-                        is1 = update_y%rSpec(tMe)%isNbr(index); ie1 = update_y%rSpec(tMe)%ieNbr(index)
-                        js1 = update_y%rSpec(tMe)%jsNbr(index); je1 = update_y%rSpec(tMe)%jeNbr(index)
-                        ni = ie1 - is1 + 1
-                        nj = je1 - js1 + 1
-                        total = ni*nj
-                        start = (update_y%rSpec(tMe)%start(index)-1)*ke
-                        if(start+total*ke>b_sizex ) call mpp_error(FATAL, &
-                             "MPP_DO_UPDATE_V: b_size is less than the size of the data to be filled.")
-                        msgsize = ie - is + 1
-                        do l=1,l_size  ! loop over number of fields
-                           ptr_bufferx = b_addrsx(l, tMe)
-                           ptr_buffery = b_addrsy(l, tMe)  
-                           start1 = start + (js-js1)*ni + is - is1 
-                           do k = 1, ke
-                              start2 = start1
-                              do j = js, je
-                                 do i = start2+1, start2+msgsize
-                                    pos = pos + 1
-                                    buffery(i) = buffer(pos)
-                                 end do
-                                 start2 = start2 + ni
-                              end do
-                              start1 = start1 + total
-                           end do
-                        end do
-                     else
-                        do l=1,l_size  ! loop over number of fields
-                           ptr_fieldx = f_addrsx(l, tMe)
-                           ptr_fieldy = f_addrsy(l, tMe)
-                           do k = 1,ke
-                              do j = js, je
-                                 do i = is, ie
-                                    pos = pos + 1
-                                    fieldy(i,j,k) = buffer(pos)
-                                 end do
+                     do l=1,l_size  ! loop over number of fields
+                        ptr_fieldx = f_addrsx(l, tMe)
+                        ptr_fieldy = f_addrsy(l, tMe)
+                        do k = 1,ke
+                           do j = js, je
+                              do i = is, ie
+                                 pos = pos + 1
+                                 fieldy(i,j,k) = buffer(pos)
                               end do
                            end do
                         end do
-                     end if
+                     end do
                   end if
                end do
                ind_y = ind_y-1
@@ -1069,47 +749,18 @@
                      msgsize = (ie-is+1)*(je-js+1)*ke*l_size
                      pos = buffer_pos - msgsize
                      buffer_pos = pos
-                     if(update_x%recv(ind_x)%is_refined(n)) then
-                        index = update_x%recv(ind_x)%index(n)
-                        is1 = update_x%rSpec(tMe)%isNbr(index); ie1 = update_x%rSpec(tMe)%ieNbr(index)
-                        js1 = update_x%rSpec(tMe)%jsNbr(index); je1 = update_x%rSpec(tMe)%jeNbr(index)
-                        ni = ie1 - is1 + 1
-                        nj = je1 - js1 + 1
-                        total = ni*nj
-                        start = (update_x%rSpec(tMe)%start(index)-1)*ke
-                        if(start+total*ke>b_sizex ) call mpp_error(FATAL, &
-                             "MPP_DO_UPDATE_V: b_size is less than the size of the data to be filled.")
-                        msgsize = ie - is + 1
-                        do l=1,l_size  ! loop over number of fields
-                           ptr_bufferx = b_addrsx(l, tMe)
-                           ptr_buffery = b_addrsy(l, tMe)  
-                           start1 = start + (js-js1)*ni + is - is1 
-                           do k = 1, ke
-                              start2 = start1
-                              do j = js, je
-                                 do i = start2+1, start2+msgsize
-                                    pos = pos + 1
-                                    bufferx(i) = buffer(pos)
-                                 end do
-                                 start2 = start2 + ni
-                              end do
-                              start1 = start1 + total
-                           end do
-                        end do
-                     else
-                        do l=1,l_size  ! loop over number of fields
-                           ptr_fieldx = f_addrsx(l, tMe)
-                           ptr_fieldy = f_addrsy(l, tMe)
-                           do k = 1,ke
-                              do j = js, je
-                                 do i = is, ie
-                                    pos = pos + 1
-                                    fieldx(i,j,k) = buffer(pos)
-                                 end do
+                     do l=1,l_size  ! loop over number of fields
+                        ptr_fieldx = f_addrsx(l, tMe)
+                        ptr_fieldy = f_addrsy(l, tMe)
+                        do k = 1,ke
+                           do j = js, je
+                              do i = is, ie
+                                 pos = pos + 1
+                                 fieldx(i,j,k) = buffer(pos)
                               end do
                            end do
                         end do
-                     end if
+                     end do
                   end if
                end do
                ind_x = ind_x-1
diff --git a/src/shared/mpp/include/mpp_do_updateV_nonblock.h b/src/shared/mpp/include/mpp_do_updateV_nonblock.h
index 2a3f2703fe..be2b12e300 100644
--- a/src/shared/mpp/include/mpp_do_updateV_nonblock.h
+++ b/src/shared/mpp/include/mpp_do_updateV_nonblock.h
@@ -23,6 +23,7 @@ subroutine MPP_START_DO_UPDATE_3D_V_(id_update, f_addrsx, f_addrsy, domain, upda
   integer            :: ind_x, ind_y
   integer            :: nsend, nrecv, sendsize, recvsize
   integer            :: request
+  integer            :: send_msgsize(update_x%nsend+update_y%nsend)
   integer            :: ind_send_x(update_x%nsend+update_y%nsend), ind_send_y(update_x%nsend+update_y%nsend)
   integer            :: ind_recv_x(update_x%nrecv+update_y%nrecv), ind_recv_y(update_x%nrecv+update_y%nrecv)
   integer            :: from_pe_list(update_x%nrecv+update_y%nrecv), to_pe_list(update_x%nsend+update_y%nsend)
@@ -126,7 +127,7 @@ subroutine MPP_START_DO_UPDATE_3D_V_(id_update, f_addrsx, f_addrsy, domain, upda
         end do
      endif
      if( msgsize.GT.0 )then
-        msgsize = msgsize*ke_sum
+     msgsize = msgsize*ke_sum
         sendsize = sendsize + msgsize
         nonblock_data(id_update)%buffer_pos_send(m) = buffer_pos
         buffer_pos = buffer_pos + msgsize
@@ -157,35 +158,32 @@ subroutine MPP_START_DO_UPDATE_3D_V_(id_update, f_addrsx, f_addrsy, domain, upda
 
   !--- recv
   call mpp_clock_begin(recv_clock_nonblock)
-  !$OMP parallel do schedule(dynamic) default(shared) private(msgsize,ind_x,ind_y,from_pe,dir,buffer_pos,request)  
   do m = 1, nrecv
      msgsize = nonblock_data(id_update)%size_recv(m)
      from_pe = from_pe_list(m)
      if( msgsize .GT. 0 )then
         buffer_pos = nonblock_data(id_update)%buffer_pos_recv(m)
-!$OMP CRITICAL
         call mpp_recv( buffer(buffer_pos+1), glen=msgsize, from_pe=from_pe, block=.false., &
                    tag=id_update, request=request)
-!$OMP END CRITICAL
         nonblock_data(id_update)%request_recv(m) = request
 #ifdef use_libMPI
         nonblock_data(id_update)%type_recv(m) = MPI_TYPE_
 #endif
      end if
   end do
-!$OMP end parallel do
+
   call mpp_clock_end(recv_clock_nonblock)
 
   !--- send
 
   call mpp_clock_begin(send_pack_clock_nonblock)
 
-!$OMP parallel do schedule(dynamic) default(shared) private(ind_x,ind_y,to_pe,buffer_pos,pos,dir,tMe, &
-!$OMP          is,ie,js,je,ptr_fieldx,ptr_fieldy,msgsize,request)
+!$OMP parallel do schedule(dynamic) default(shared) private(ind_x,ind_y,buffer_pos,pos,dir,tMe, &
+!$OMP          is,ie,js,je,ptr_fieldx,ptr_fieldy)
   do m = 1, nsend
+     send_msgsize(m) = 0
      ind_x = ind_send_x(m)
      ind_y = ind_send_y(m)
-     to_pe = to_pe_list(m)
      buffer_pos = nonblock_data(id_update)%buffer_pos_send(m)
      pos = buffer_pos
      
@@ -198,221 +196,112 @@ subroutine MPP_START_DO_UPDATE_3D_V_(id_update, f_addrsx, f_addrsy, domain, upda
                  tMe = update_x%send(ind_x)%tileMe(n)
                  is = update_x%send(ind_x)%is(n); ie = update_x%send(ind_x)%ie(n)
                  js = update_x%send(ind_x)%js(n); je = update_x%send(ind_x)%je(n)
-                 if( update_x%send(ind_x)%is_refined(n) ) then
-                    select case( update_x%send(ind_x)%rotation(n) )
-                    case(ZERO)
+                 select case( update_x%send(ind_x)%rotation(n) )
+                 case(ZERO)
+                    do l=1,l_size  ! loop over number of fields
+                       ptr_fieldx = f_addrsx(l,tMe)
+                       ptr_fieldy = f_addrsy(l,tMe)
+                       do k = 1,ke_list(l,tMe)
+                          do j = js, je
+                             do i = is, ie
+                                pos = pos + 2
+                                buffer(pos-1) = fieldx(i,j,k)
+                                buffer(pos)   = fieldy(i,j,k)
+                             end do
+                          end do
+                       end do
+                    end do
+                 case( MINUS_NINETY ) 
+                    if( BTEST(flags,SCALAR_BIT) ) then
                        do l=1,l_size  ! loop over number of fields
                           ptr_fieldx = f_addrsx(l,tMe)
                           ptr_fieldy = f_addrsy(l,tMe)
                           do k = 1,ke_list(l,tMe)
-                             do j = js, je
-                                do i = is, ie
+                             do i = is, ie
+                                do j = je, js, -1
                                    pos = pos + 2
-                                   buffer(pos-1) = fieldx(i,j,k)
-                                   buffer(pos)   = fieldy(i,j,k)
+                                   buffer(pos-1) = fieldy(i,j,k)
+                                   buffer(pos)   = fieldx(i,j,k)
                                 end do
                              end do
                           end do
                        end do
-                    case(MINUS_NINETY)
-                       if( BTEST(flags,SCALAR_BIT) ) then
-                          do l=1,l_size  ! loop over number of fields
-                             ptr_fieldx = f_addrsx(l,tMe)
-                             ptr_fieldy = f_addrsy(l,tMe)
-                             do k = 1,ke_list(l,tMe)
-                                do j = js, je
-                                   do i = is, ie
-                                      pos = pos + 2
-                                      buffer(pos-1) =  fieldy(i,j,k)
-                                      buffer(pos)   =  fieldx(i,j,k)
-                                   end do
-                                end do
-                             end do
-                          end do
-                       else
-                          do l=1,l_size  ! loop over number of fields
-                             ptr_fieldx = f_addrsx(l,tMe)
-                             ptr_fieldy = f_addrsy(l,tMe)
-                             do k = 1,ke_list(l,tMe)
-                                do j = js, je
-                                   do i = is, ie
-                                      pos = pos + 2
-                                      buffer(pos-1) = -fieldy(i,j,k)
-                                      buffer(pos)   =  fieldx(i,j,k)
-                                   end do
-                                end do
-                             end do
-                          end do
-                       end if
-                    case(NINETY)
-                       if( BTEST(flags,SCALAR_BIT) ) then
-                          do l=1,l_size  ! loop over number of fields
-                             ptr_fieldx = f_addrsx(l,tMe)
-                             ptr_fieldy = f_addrsy(l,tMe)
-                             do k = 1,ke_list(l,tMe)
-                                do j = js, je
-                                   do i = is, ie
-                                      pos = pos + 2
-                                      buffer(pos-1) =  fieldy(i,j,k)
-                                      buffer(pos)   =  fieldx(i,j,k)
-                                   end do
-                                end do
-                             end do
-                          end do
-                       else
-                          do l=1,l_size  ! loop over number of fields
-                             ptr_fieldx = f_addrsx(l,tMe)
-                             ptr_fieldy = f_addrsy(l,tMe)
-                             do k = 1,ke_list(l,tMe)
-                                do j = js, je
-                                   do i = is, ie
-                                      pos = pos + 2
-                                      buffer(pos-1) =  fieldy(i,j,k)
-                                      buffer(pos)   = -fieldx(i,j,k)
-                                   end do
-                                end do
-                             end do
-                          end do
-                       end if
-                    case(ONE_HUNDRED_EIGHTY)
-                       if( BTEST(flags,SCALAR_BIT) ) then
-                          do l=1,l_size  ! loop over number of fields
-                             ptr_fieldx = f_addrsx(l,tMe)
-                             ptr_fieldy = f_addrsy(l,tMe)
-                             do k = 1,ke_list(l,tMe)
-                                do j = js, je
-                                   do i = is, ie
-                                      pos = pos + 2
-                                      buffer(pos-1) = fieldx(i,j,k)
-                                      buffer(pos)   = fieldy(i,j,k)
-                                   end do
+                    else
+                       do l=1,l_size  ! loop over number of fields
+                          ptr_fieldx = f_addrsx(l,tMe)
+                          ptr_fieldy = f_addrsy(l,tMe)
+                          do k = 1,ke_list(l,tMe)
+                             do i = is, ie
+                                do j = je, js, -1
+                                   pos = pos + 2
+                                   buffer(pos-1) = -fieldy(i,j,k)
+                                   buffer(pos)   =  fieldx(i,j,k)
                                 end do
                              end do
                           end do
-                       else
-                          do l=1,l_size  ! loop over number of fields
-                             ptr_fieldx = f_addrsx(l,tMe)
-                             ptr_fieldy = f_addrsy(l,tMe)
-                             do k = 1,ke_list(l,tMe)
+                       end do
+                    end if
+                 case( NINETY )
+                    if( BTEST(flags,SCALAR_BIT) ) then
+                       do l=1,l_size  ! loop over number of fields
+                          ptr_fieldx = f_addrsx(l,tMe)
+                          ptr_fieldy = f_addrsy(l,tMe)
+                          do k = 1,ke_list(l,tMe)
+                             do i = ie, is, -1
                                 do j = js, je
-                                   do i = is, ie
-                                      pos = pos + 2
-                                      buffer(pos-1) = -fieldx(i,j,k)
-                                      buffer(pos)   = -fieldy(i,j,k)
-                                   end do
+                                   pos = pos + 2
+                                   buffer(pos-1) = fieldy(i,j,k)
+                                   buffer(pos)   = fieldx(i,j,k)
                                 end do
                              end do
                           end do
-                       end if
-                    end select  ! select case( rotation(n) )
-                 else   ! if( is_refined(n) )
-                    select case( update_x%send(ind_x)%rotation(n) )
-                    case(ZERO)
+                       end do
+                    else
                        do l=1,l_size  ! loop over number of fields
                           ptr_fieldx = f_addrsx(l,tMe)
                           ptr_fieldy = f_addrsy(l,tMe)
                           do k = 1,ke_list(l,tMe)
-                             do j = js, je
-                                do i = is, ie
+                             do i = ie, is, -1
+                                do j = js, je
                                    pos = pos + 2
-                                   buffer(pos-1) = fieldx(i,j,k)
-                                   buffer(pos)   = fieldy(i,j,k)
+                                   buffer(pos-1) = fieldy(i,j,k)
+                                   buffer(pos)   = -fieldx(i,j,k)
                                 end do
                              end do
                           end do
                        end do
-                    case( MINUS_NINETY ) 
-                       if( BTEST(flags,SCALAR_BIT) ) then
-                          do l=1,l_size  ! loop over number of fields
-                             ptr_fieldx = f_addrsx(l,tMe)
-                             ptr_fieldy = f_addrsy(l,tMe)
-                             do k = 1,ke_list(l,tMe)
-                                do i = is, ie
-                                   do j = je, js, -1
-                                      pos = pos + 2
-                                      buffer(pos-1) = fieldy(i,j,k)
-                                      buffer(pos)   = fieldx(i,j,k)
-                                   end do
-                                end do
-                             end do
-                          end do
-                       else
-                          do l=1,l_size  ! loop over number of fields
-                             ptr_fieldx = f_addrsx(l,tMe)
-                             ptr_fieldy = f_addrsy(l,tMe)
-                             do k = 1,ke_list(l,tMe)
-                                do i = is, ie
-                                   do j = je, js, -1
-                                      pos = pos + 2
-                                      buffer(pos-1) = -fieldy(i,j,k)
-                                      buffer(pos)   =  fieldx(i,j,k)
-                                   end do
-                                end do
-                             end do
-                          end do
-                       end if
-                    case( NINETY )
-                       if( BTEST(flags,SCALAR_BIT) ) then
-                          do l=1,l_size  ! loop over number of fields
-                             ptr_fieldx = f_addrsx(l,tMe)
-                             ptr_fieldy = f_addrsy(l,tMe)
-                             do k = 1,ke_list(l,tMe)
+                    end if
+                 case( ONE_HUNDRED_EIGHTY )
+                    if( BTEST(flags,SCALAR_BIT) ) then
+                       do l=1,l_size  ! loop over number of fields
+                          ptr_fieldx = f_addrsx(l,tMe)
+                          ptr_fieldy = f_addrsy(l,tMe)
+                          do k = 1,ke_list(l,tMe)
+                             do j = je, js, -1
                                 do i = ie, is, -1
-                                   do j = js, je
-                                      pos = pos + 2
-                                      buffer(pos-1) = fieldy(i,j,k)
-                                      buffer(pos)   = fieldx(i,j,k)
-                                   end do
+                                   pos = pos + 2
+                                   buffer(pos-1) =  fieldx(i,j,k)
+                                   buffer(pos)   =  fieldy(i,j,k)
                                 end do
                              end do
                           end do
-                       else
-                          do l=1,l_size  ! loop over number of fields
-                             ptr_fieldx = f_addrsx(l,tMe)
-                             ptr_fieldy = f_addrsy(l,tMe)
-                             do k = 1,ke_list(l,tMe)
+                       end do
+                    else
+                       do l=1,l_size  ! loop over number of fields
+                          ptr_fieldx = f_addrsx(l,tMe)
+                          ptr_fieldy = f_addrsy(l,tMe)
+                          do k = 1,ke_list(l,tMe)
+                             do j = je, js, -1
                                 do i = ie, is, -1
-                                   do j = js, je
-                                      pos = pos + 2
-                                      buffer(pos-1) = fieldy(i,j,k)
-                                      buffer(pos)   = -fieldx(i,j,k)
-                                   end do
-                                end do
-                             end do
-                          end do
-                       end if
-                    case( ONE_HUNDRED_EIGHTY )
-                       if( BTEST(flags,SCALAR_BIT) ) then
-                          do l=1,l_size  ! loop over number of fields
-                             ptr_fieldx = f_addrsx(l,tMe)
-                             ptr_fieldy = f_addrsy(l,tMe)
-                             do k = 1,ke_list(l,tMe)
-                                do j = je, js, -1
-                                   do i = ie, is, -1
-                                      pos = pos + 2
-                                      buffer(pos-1) =  fieldx(i,j,k)
-                                      buffer(pos)   =  fieldy(i,j,k)
-                                   end do
-                                end do
-                             end do
-                          end do
-                       else
-                          do l=1,l_size  ! loop over number of fields
-                             ptr_fieldx = f_addrsx(l,tMe)
-                             ptr_fieldy = f_addrsy(l,tMe)
-                             do k = 1,ke_list(l,tMe)
-                                do j = je, js, -1
-                                   do i = ie, is, -1
-                                      pos = pos + 2
-                                      buffer(pos-1) =  -fieldx(i,j,k)
-                                      buffer(pos)   =  -fieldy(i,j,k)
-                                   end do
+                                   pos = pos + 2
+                                   buffer(pos-1) =  -fieldx(i,j,k)
+                                   buffer(pos)   =  -fieldy(i,j,k)
                                 end do
                              end do
                           end do
-                       end if
-                    end select ! select case( rotation(n) )
-                 end if ! if( is_refined(n) )
+                       end do
+                    end if
+                 end select ! select case( rotation(n) )
               end if ! if( send(dir) ) 
            end do ! do n = 1, update_x%send(ind_x)%count
         endif
@@ -424,162 +313,90 @@ subroutine MPP_START_DO_UPDATE_3D_V_(id_update, f_addrsx, f_addrsy, domain, upda
                  tMe = update_x%send(ind_x)%tileMe(n)
                  is = update_x%send(ind_x)%is(n); ie = update_x%send(ind_x)%ie(n)
                  js = update_x%send(ind_x)%js(n); je = update_x%send(ind_x)%je(n)
-                 if( update_x%send(ind_x)%is_refined(n) ) then
-                    select case( update_x%send(ind_x)%rotation(n) )
-                    case(ZERO)
-                       do l=1,l_size  ! loop over number of fields
-                          ptr_fieldx = f_addrsx(l,tMe)
-                          ptr_fieldy = f_addrsy(l,tMe)
-                          do k = 1,ke_list(l,tMe)
-                             do j = js, je
-                                do i = is, ie
-                                   pos = pos + 1
-                                   buffer(pos) = fieldx(i,j,k)
-                                end do
+                 select case( update_x%send(ind_x)%rotation(n) )
+                 case(ZERO)
+                    do l=1,l_size  ! loop over number of fields
+                       ptr_fieldx = f_addrsx(l,tMe)
+                       ptr_fieldy = f_addrsy(l,tMe)
+                       do k = 1,ke_list(l,tMe)
+                          do j = js, je
+                             do i = is, ie
+                                pos = pos + 1
+                                buffer(pos) = fieldx(i,j,k)
                              end do
                           end do
                        end do
-                    case(MINUS_NINETY)
-                       if( BTEST(flags,SCALAR_BIT) ) then
-                          do l=1,l_size  ! loop over number of fields
-                             ptr_fieldx = f_addrsx(l,tMe)
-                             ptr_fieldy = f_addrsy(l,tMe)
-                             do k = 1,ke_list(l,tMe)
-                                do j = js, je
-                                   do i = is, ie
-                                      pos = pos + 1
-                                      buffer(pos) = fieldy(i,j,k)
-                                   end do
-                                end do
-                             end do
-                          end do
-                       else
-                          do l=1,l_size  ! loop over number of fields
-                             ptr_fieldx = f_addrsx(l,tMe)
-                             ptr_fieldy = f_addrsy(l,tMe)
-                             do k = 1,ke_list(l,tMe)
-                                do j = js, je
-                                   do i = is, ie
-                                      pos = pos + 1
-                                      buffer(pos) = -fieldy(i,j,k)
-                                   end do
-                                end do
-                             end do
-                          end do
-                       end if
-                    case(NINETY)
+                    end do
+                 case(MINUS_NINETY)
+                    if( BTEST(flags,SCALAR_BIT) ) then
                        do l=1,l_size  ! loop over number of fields
                           ptr_fieldx = f_addrsx(l,tMe)
                           ptr_fieldy = f_addrsy(l,tMe)
-                          do k = 1, ke_list(l,tMe)
-                             do j = js, je
-                                do i = is, ie
+                          do k = 1,ke_list(l,tMe)
+                             do i = is, ie
+                                do j = je, js, -1
                                    pos = pos + 1
                                    buffer(pos) = fieldy(i,j,k)
                                 end do
                              end do
                           end do
                        end do
-                    case(ONE_HUNDRED_EIGHTY)
+                    else
                        do l=1,l_size  ! loop over number of fields
                           ptr_fieldx = f_addrsx(l,tMe)
                           ptr_fieldy = f_addrsy(l,tMe)
                           do k = 1,ke_list(l,tMe)
-                             do j = js, je
-                                do i = is, ie
+                             do i = is, ie
+                                do j = je, js, -1
                                    pos = pos + 1
-                                   buffer(pos) = fieldx(i,j,k)
+                                   buffer(pos) = -fieldy(i,j,k)
                                 end do
                              end do
                           end do
                        end do
-                    end select
-                 else
-                    select case( update_x%send(ind_x)%rotation(n) )
-                    case(ZERO)
+                    end if
+                 case(NINETY)
+                    do l=1,l_size  ! loop over number of fields
+                       ptr_fieldx = f_addrsx(l,tMe)
+                       ptr_fieldy = f_addrsy(l,tMe)
+                       do k = 1, ke_list(l,tMe)
+                          do i = ie, is, -1
+                             do j = js, je
+                                pos = pos + 1
+                                buffer(pos) = fieldy(i,j,k)
+                             end do
+                          end do
+                       end do
+                    end do
+                 case(ONE_HUNDRED_EIGHTY)
+                    if( BTEST(flags,SCALAR_BIT) ) then
                        do l=1,l_size  ! loop over number of fields
                           ptr_fieldx = f_addrsx(l,tMe)
                           ptr_fieldy = f_addrsy(l,tMe)
                           do k = 1,ke_list(l,tMe)
-                             do j = js, je
-                                do i = is, ie
+                             do j = je, js, -1
+                                do i = ie, is, -1
                                    pos = pos + 1
                                    buffer(pos) = fieldx(i,j,k)
                                 end do
                              end do
                           end do
                        end do
-                    case(MINUS_NINETY)
-                       if( BTEST(flags,SCALAR_BIT) ) then
-                          do l=1,l_size  ! loop over number of fields
-                             ptr_fieldx = f_addrsx(l,tMe)
-                             ptr_fieldy = f_addrsy(l,tMe)
-                             do k = 1,ke_list(l,tMe)
-                                do i = is, ie
-                                   do j = je, js, -1
-                                      pos = pos + 1
-                                      buffer(pos) = fieldy(i,j,k)
-                                   end do
-                                end do
-                             end do
-                          end do
-                       else
-                          do l=1,l_size  ! loop over number of fields
-                             ptr_fieldx = f_addrsx(l,tMe)
-                             ptr_fieldy = f_addrsy(l,tMe)
-                             do k = 1,ke_list(l,tMe)
-                                do i = is, ie
-                                   do j = je, js, -1
-                                      pos = pos + 1
-                                      buffer(pos) = -fieldy(i,j,k)
-                                   end do
-                                end do
-                             end do
-                          end do
-                       end if
-                    case(NINETY)
+                    else
                        do l=1,l_size  ! loop over number of fields
                           ptr_fieldx = f_addrsx(l,tMe)
                           ptr_fieldy = f_addrsy(l,tMe)
-                          do k = 1, ke_list(l,tMe)
-                             do i = ie, is, -1
-                                do j = js, je
+                          do k = 1,ke_list(l,tMe)
+                             do j = je, js, -1
+                                do i = ie, is, -1
                                    pos = pos + 1
-                                   buffer(pos) = fieldy(i,j,k)
+                                   buffer(pos) = -fieldx(i,j,k)
                                 end do
                              end do
                           end do
                        end do
-                    case(ONE_HUNDRED_EIGHTY)
-                       if( BTEST(flags,SCALAR_BIT) ) then
-                          do l=1,l_size  ! loop over number of fields
-                             ptr_fieldx = f_addrsx(l,tMe)
-                             ptr_fieldy = f_addrsy(l,tMe)
-                             do k = 1,ke_list(l,tMe)
-                                do j = je, js, -1
-                                   do i = ie, is, -1
-                                      pos = pos + 1
-                                      buffer(pos) = fieldx(i,j,k)
-                                   end do
-                                end do
-                             end do
-                          end do
-                       else
-                          do l=1,l_size  ! loop over number of fields
-                             ptr_fieldx = f_addrsx(l,tMe)
-                             ptr_fieldy = f_addrsy(l,tMe)
-                             do k = 1,ke_list(l,tMe)
-                                do j = je, js, -1
-                                   do i = ie, is, -1
-                                      pos = pos + 1
-                                      buffer(pos) = -fieldx(i,j,k)
-                                   end do
-                                end do
-                             end do
-                          end do
-                       end if
-                    end select
-                 end if
+                    end if
+                 end select
               end if
            end do
         endif
@@ -590,177 +407,107 @@ subroutine MPP_START_DO_UPDATE_3D_V_(id_update, f_addrsx, f_addrsy, domain, upda
                  tMe = update_y%send(ind_y)%tileMe(n)
                  is = update_y%send(ind_y)%is(n); ie = update_y%send(ind_y)%ie(n)
                  js = update_y%send(ind_y)%js(n); je = update_y%send(ind_y)%je(n)
-                 if( update_y%send(ind_y)%is_refined(n) ) then
-                    select case( update_y%send(ind_y)%rotation(n) )
-                    case(ZERO)
-                       do l=1,l_size  ! loop over number of fields
-                          ptr_fieldx = f_addrsx(l,tMe)
-                          ptr_fieldy = f_addrsy(l,tMe)
-                          do k = 1,ke_list(l,tMe)
-                             do j = js, je
-                                do i = is, ie
-                                   pos = pos + 1
-                                   buffer(pos) = fieldy(i,j,k)
-                                end do
+                 select case( update_y%send(ind_y)%rotation(n) )
+                 case(ZERO)
+                    do l=1,l_size  ! loop over number of fields
+                       ptr_fieldx = f_addrsx(l,tMe)
+                       ptr_fieldy = f_addrsy(l,tMe)
+                       do k = 1,ke_list(l,tMe)
+                          do j = js, je
+                             do i = is, ie
+                                pos = pos + 1
+                                buffer(pos) = fieldy(i,j,k)
+                             end do
+                          end do
+                       end do
+                    end do
+                 case(MINUS_NINETY)
+                    do l=1,l_size  ! loop over number of fields
+                       ptr_fieldx = f_addrsx(l,tMe)
+                       ptr_fieldy = f_addrsy(l,tMe)
+                       do k = 1,ke_list(l,tMe)
+                          do i = is, ie
+                             do j = je, js, -1
+                                pos = pos + 1
+                                buffer(pos) = fieldx(i,j,k)
                              end do
                           end do
-                       enddo
-                    case(MINUS_NINETY)
+                       end do
+                    end do
+                 case(NINETY)
+                    if( BTEST(flags,SCALAR_BIT) ) then
                        do l=1,l_size  ! loop over number of fields
                           ptr_fieldx = f_addrsx(l,tMe)
                           ptr_fieldy = f_addrsy(l,tMe)
                           do k = 1,ke_list(l,tMe)
-                             do j = js, je
-                                do i = is, ie
+                             do i = ie, is, -1
+                                do j = js, je
                                    pos = pos + 1
                                    buffer(pos) = fieldx(i,j,k)
                                 end do
                              end do
                           end do
                        end do
-                    case(NINETY)
-                       if( BTEST(flags,SCALAR_BIT) ) then
-                          do l=1,l_size  ! loop over number of fields
-                             ptr_fieldx = f_addrsx(l,tMe)
-                             ptr_fieldy = f_addrsy(l,tMe)
-                             do k = 1,ke_list(l,tMe)
-                                do j = js, je
-                                   do i = is, ie
-                                      pos = pos + 1
-                                      buffer(pos) = fieldx(i,j,k)
-                                   end do
-                                end do
-                             end do
-                          end do
-                       else
-                          do l=1,l_size  ! loop over number of fields
-                             ptr_fieldx = f_addrsx(l,tMe)
-                             ptr_fieldy = f_addrsy(l,tMe)
-                             do k = 1,ke_list(l,tMe)
-                                do j = js, je
-                                   do i = is, ie
-                                      pos = pos + 1
-                                      buffer(pos) = -fieldx(i,j,k)
-                                   end do
-                                end do
-                             end do
-                          end do
-                       end if
-                    case(ONE_HUNDRED_EIGHTY)
+                    else
                        do l=1,l_size  ! loop over number of fields
                           ptr_fieldx = f_addrsx(l,tMe)
                           ptr_fieldy = f_addrsy(l,tMe)
                           do k = 1,ke_list(l,tMe)
-                             do j = js, je
-                                do i = is, ie
+                             do i = ie, is, -1
+                                do j = js, je
                                    pos = pos + 1
-                                   buffer(pos) = fieldy(i,j,k)
+                                   buffer(pos) = -fieldx(i,j,k)
                                 end do
                              end do
                           end do
                        end do
-                    end select
-                 else
-                    select case( update_y%send(ind_y)%rotation(n) )
-                    case(ZERO)
+                    end if
+                 case(ONE_HUNDRED_EIGHTY)
+                    if( BTEST(flags,SCALAR_BIT) ) then
                        do l=1,l_size  ! loop over number of fields
                           ptr_fieldx = f_addrsx(l,tMe)
                           ptr_fieldy = f_addrsy(l,tMe)
                           do k = 1,ke_list(l,tMe)
-                             do j = js, je
-                                do i = is, ie
+                             do j = je, js, -1
+                                do i = ie, is, -1
                                    pos = pos + 1
                                    buffer(pos) = fieldy(i,j,k)
                                 end do
                              end do
                           end do
                        end do
-                    case(MINUS_NINETY)
+                    else
                        do l=1,l_size  ! loop over number of fields
                           ptr_fieldx = f_addrsx(l,tMe)
                           ptr_fieldy = f_addrsy(l,tMe)
                           do k = 1,ke_list(l,tMe)
-                             do i = is, ie
-                                do j = je, js, -1
+                             do j = je, js, -1
+                                do i = ie, is, -1
                                    pos = pos + 1
-                                   buffer(pos) = fieldx(i,j,k)
+                                   buffer(pos) = -fieldy(i,j,k)
                                 end do
                              end do
                           end do
                        end do
-                    case(NINETY)
-                       if( BTEST(flags,SCALAR_BIT) ) then
-                          do l=1,l_size  ! loop over number of fields
-                             ptr_fieldx = f_addrsx(l,tMe)
-                             ptr_fieldy = f_addrsy(l,tMe)
-                             do k = 1,ke_list(l,tMe)
-                                do i = ie, is, -1
-                                   do j = js, je
-                                      pos = pos + 1
-                                      buffer(pos) = fieldx(i,j,k)
-                                   end do
-                                end do
-                             end do
-                          end do
-                       else
-                          do l=1,l_size  ! loop over number of fields
-                             ptr_fieldx = f_addrsx(l,tMe)
-                             ptr_fieldy = f_addrsy(l,tMe)
-                             do k = 1,ke_list(l,tMe)
-                                do i = ie, is, -1
-                                   do j = js, je
-                                      pos = pos + 1
-                                      buffer(pos) = -fieldx(i,j,k)
-                                   end do
-                                end do
-                             end do
-                          end do
-                       end if
-                    case(ONE_HUNDRED_EIGHTY)
-                       if( BTEST(flags,SCALAR_BIT) ) then
-                          do l=1,l_size  ! loop over number of fields
-                             ptr_fieldx = f_addrsx(l,tMe)
-                             ptr_fieldy = f_addrsy(l,tMe)
-                             do k = 1,ke_list(l,tMe)
-                                do j = je, js, -1
-                                   do i = ie, is, -1
-                                      pos = pos + 1
-                                      buffer(pos) = fieldy(i,j,k)
-                                   end do
-                                end do
-                             end do
-                          end do
-                       else
-                          do l=1,l_size  ! loop over number of fields
-                             ptr_fieldx = f_addrsx(l,tMe)
-                             ptr_fieldy = f_addrsy(l,tMe)
-                             do k = 1,ke_list(l,tMe)
-                                do j = je, js, -1
-                                   do i = ie, is, -1
-                                      pos = pos + 1
-                                      buffer(pos) = -fieldy(i,j,k)
-                                   end do
-                                end do
-                             end do
-                          end do
-                       end if
-                    end select
-                 end if
+                    end if
+                 end select
               endif
            enddo
         endif
      end select
-
-     msgsize = pos - buffer_pos
+     send_msgsize(m) = pos - buffer_pos
+  enddo
+!$OMP end parallel do
+  do m = 1, nsend
+     msgsize = send_msgsize(m)
+     to_pe = to_pe_list(m)
+     buffer_pos = nonblock_data(id_update)%buffer_pos_send(m)
      if( msgsize .GT.0 )then
-!$OMP CRITICAL
         call mpp_send( buffer(buffer_pos+1), plen=msgsize, to_pe=to_pe, &
                        tag=id_update, request=request )
-!$OMP END CRITICAL
         nonblock_data(id_update)%request_send(m) = request
      end if
   end do
-!$OMP end parallel do
 
   call mpp_clock_end(send_pack_clock_nonblock)
 
@@ -769,8 +516,7 @@ end subroutine MPP_START_DO_UPDATE_3D_V_
 
 !###############################################################################
 subroutine MPP_COMPLETE_DO_UPDATE_3D_V_(id_update, f_addrsx, f_addrsy, domain, update_x, update_y,     &
-                                        d_type, ke_max, ke_list, b_addrsx, b_addrsy, b_sizex, b_sizey, &
-                                        gridtype, flags) 
+                                        d_type, ke_max, ke_list, gridtype, flags) 
   integer,             intent(in) :: id_update
   integer(LONG_KIND),  intent(in) :: f_addrsx(:,:), f_addrsy(:,:)
   type(domain2d),      intent(in) :: domain
@@ -778,8 +524,6 @@ subroutine MPP_COMPLETE_DO_UPDATE_3D_V_(id_update, f_addrsx, f_addrsy, domain, u
   integer,             intent(in) :: ke_max
   integer,             intent(in) :: ke_list(:,:)
   MPP_TYPE_,           intent(in) :: d_type  ! creates unique interface
-  integer(LONG_KIND),  intent(in) :: b_addrsx(:,:), b_addrsy(:,:)
-  integer,             intent(in) :: b_sizex, b_sizey
   integer,             intent(in) :: gridtype
   integer,             intent(in) :: flags
 
@@ -787,13 +531,8 @@ subroutine MPP_COMPLETE_DO_UPDATE_3D_V_(id_update, f_addrsx, f_addrsy, domain, u
   !--- local variables
   MPP_TYPE_ :: fieldx(update_x%xbegin:update_x%xend, update_x%ybegin:update_x%yend,ke_max)
   MPP_TYPE_ :: fieldy(update_y%xbegin:update_y%xend, update_y%ybegin:update_y%yend,ke_max)
-  MPP_TYPE_ :: bufferx(b_sizex)
-  MPP_TYPE_ :: buffery(b_sizey)
   pointer(ptr_fieldx, fieldx)
   pointer(ptr_fieldy, fieldy)
-  pointer(ptr_bufferx, bufferx)
-  pointer(ptr_buffery, buffery)
-
 
   MPP_TYPE_ :: recv_buffer(size(mpp_domains_stack_nonblock(:)))
   pointer( ptr, recv_buffer )
@@ -801,7 +540,6 @@ subroutine MPP_COMPLETE_DO_UPDATE_3D_V_(id_update, f_addrsx, f_addrsy, domain, u
   integer :: i, j, k, l, is, ie, js, je, n, ke_sum, l_size, m
   integer :: pos, nlist, msgsize, tile, buffer_pos
   integer :: ind_x, ind_y, nrecv, nsend
-  integer :: index, is1, ie1, js1, je1, ni, nj, total, start1, start, start2
   integer :: ind_recv_x(update_x%nrecv+update_y%nrecv), ind_recv_y(update_x%nrecv+update_y%nrecv)
   integer :: start_pos_recv(update_x%nrecv+update_y%nrecv)
   integer :: from_pe_list(update_x%nrecv+update_y%nrecv)
@@ -853,7 +591,7 @@ subroutine MPP_COMPLETE_DO_UPDATE_3D_V_(id_update, f_addrsx, f_addrsy, domain, u
 
   call mpp_clock_begin(unpk_clock_nonblock)
 !$OMP parallel do schedule(dynamic) default(shared) private(ind_x,ind_y,buffer_pos,pos,dir,tMe,is,ie,js,je, &
-!$OMP          msgsize,index,is1,ie1,js1,je1,ni,nj,total,start,ptr_bufferx,ptr_buffery,start1,start2,ptr_fieldx,ptr_fieldy)
+!$OMP          msgsize,ptr_fieldx,ptr_fieldy)
   do m = nrecv,1,-1
      ind_x = ind_recv_x(m)
      ind_y = ind_recv_y(m)
@@ -871,51 +609,19 @@ subroutine MPP_COMPLETE_DO_UPDATE_3D_V_(id_update, f_addrsx, f_addrsy, domain, u
                  msgsize = (ie-is+1)*(je-js+1)*ke_sum*2
                  pos = buffer_pos - msgsize
                  buffer_pos = pos
-                 if(update_x%recv(ind_x)%is_refined(n)) then
-                    index = update_x%recv(ind_x)%index(n)
-                    is1 = update_x%rSpec(tMe)%isNbr(index); ie1 = update_x%rSpec(tMe)%ieNbr(index)
-                    js1 = update_x%rSpec(tMe)%jsNbr(index); je1 = update_x%rSpec(tMe)%jeNbr(index)
-                    ni = ie1 - is1 + 1
-                    nj = je1 - js1 + 1
-                    total = ni*nj
-                    start = (update_x%rSpec(tMe)%start(index)-1)*ke_max
-
-                    if(start+total*ke_max>size(bufferx) .or. start+total*ke_max>size(buffery) ) call mpp_error(FATAL, &
-                         "MPP_COMPLETE_DO_UPDATE_3D_V: size of bufferx or buffery is less than the size of the data to be filled.")
-                    msgsize = ie - is + 1
-                    do l=1, l_size  ! loop over number of fields
-                       ptr_bufferx = b_addrsx(l, tMe)
-                       ptr_buffery = b_addrsy(l, tMe)
-                       if(l==1) start = (update_x%rSpec(tMe)%start(index)-1)*ke_list(l,tMe)
-                       start1 = start + (js-js1)*ni + is - is1 
-                       do k = 1, ke_list(l,tMe)
-                          start2 = start1
-                          do j = js, je
-                             do i = start2+1, start2+msgsize
-                                pos = pos + 2
-                                bufferx(i) = recv_buffer(pos-1)
-                                buffery(i) = recv_buffer(pos)
-                             end do
-                             start2 = start2 + ni
+                 do l=1, l_size  ! loop over number of fields
+                    ptr_fieldx = f_addrsx(l, tMe)
+                    ptr_fieldy = f_addrsy(l, tMe)
+                    do k = 1,ke_list(l,tMe)
+                       do j = js, je
+                          do i = is, ie
+                             pos = pos + 2
+                             fieldx(i,j,k) = recv_buffer(pos-1)
+                             fieldy(i,j,k) = recv_buffer(pos)
                           end do
-                          start1 = start1 + total
                        end do
                     enddo
-                 else
-                    do l=1, l_size  ! loop over number of fields
-                       ptr_fieldx = f_addrsx(l, tMe)
-                       ptr_fieldy = f_addrsy(l, tMe)
-                       do k = 1,ke_list(l,tMe)
-                          do j = js, je
-                             do i = is, ie
-                                pos = pos + 2
-                                fieldx(i,j,k) = recv_buffer(pos-1)
-                                fieldy(i,j,k) = recv_buffer(pos)
-                             end do
-                          end do
-                       enddo
-                    end do
-                 end if
+                 end do
               end if ! end if( recv(dir) )
            end do  ! do dir=8,1,-1 
         endif
@@ -931,47 +637,17 @@ subroutine MPP_COMPLETE_DO_UPDATE_3D_V_(id_update, f_addrsx, f_addrsy, domain, u
                  msgsize = (ie-is+1)*(je-js+1)*ke_sum
                  pos = buffer_pos - msgsize
                  buffer_pos = pos
-                 if(update_y%recv(ind_y)%is_refined(n)) then
-                    index = update_y%recv(ind_y)%index(n)
-                    is1 = update_y%rSpec(tMe)%isNbr(index); ie1 = update_y%rSpec(tMe)%ieNbr(index)
-                    js1 = update_y%rSpec(tMe)%jsNbr(index); je1 = update_y%rSpec(tMe)%jeNbr(index)
-                    ni = ie1 - is1 + 1
-                    nj = je1 - js1 + 1
-                    total = ni*nj
-                    start = (update_y%rSpec(tMe)%start(index)-1)*ke_max
-                    if(start+total*ke_max>size(buffery) ) call mpp_error(FATAL, &
-                      "MPP_COMPLETE_DO_UPDATE_3D_V: size of buffery is less than the size of the data to be filled.")
-
-                    msgsize = ie - is + 1
-                    do l = 1, l_size
-                       ptr_buffery = b_addrsy(l, tMe)
-                       if(l==1) start = (update_y%rSpec(tMe)%start(index)-1)*ke_list(l,tMe)
-                       start1 = start + (js-js1)*ni + is - is1 
-                       do k = 1, ke_list(l,tMe)
-                          start2 = start1
-                          do j = js, je
-                             do i = start2+1, start2+msgsize
-                                pos = pos + 1
-                                buffery(i) = recv_buffer(pos)
-                             end do
-                             start2 = start2 + ni
+                 do l=1, l_size  ! loop over number of fields
+                    ptr_fieldy = f_addrsy(l, tMe)
+                    do k = 1,ke_list(l,tMe)
+                       do j = js, je
+                          do i = is, ie
+                             pos = pos + 1
+                             fieldy(i,j,k) = recv_buffer(pos)
                           end do
-                          start1 = start1 + total
                        end do
-                    enddo
-                 else
-                    do l=1, l_size  ! loop over number of fields
-                       ptr_fieldy = f_addrsy(l, tMe)
-                       do k = 1,ke_list(l,tMe)
-                          do j = js, je
-                             do i = is, ie
-                                pos = pos + 1
-                                fieldy(i,j,k) = recv_buffer(pos)
-                             end do
-                          end do
-                       end do 
                     end do
-                 end if
+                 end do
               end if
            end do
         endif
@@ -985,47 +661,17 @@ subroutine MPP_COMPLETE_DO_UPDATE_3D_V_(id_update, f_addrsx, f_addrsy, domain, u
                  msgsize = (ie-is+1)*(je-js+1)*ke_sum
                  pos = buffer_pos - msgsize
                  buffer_pos = pos
-                 if(update_x%recv(ind_x)%is_refined(n)) then
-                    index = update_x%recv(ind_x)%index(n)
-                    is1 = update_x%rSpec(tMe)%isNbr(index); ie1 = update_x%rSpec(tMe)%ieNbr(index)
-                    js1 = update_x%rSpec(tMe)%jsNbr(index); je1 = update_x%rSpec(tMe)%jeNbr(index)
-                    ni = ie1 - is1 + 1
-                    nj = je1 - js1 + 1
-                    total = ni*nj
-                    start = (update_x%rSpec(tMe)%start(index)-1)*ke_max
-                    if(start+total*ke_max>size(bufferx) ) call mpp_error(FATAL, &
-                      "MPP_COMPLETE_DO_UPDATE_3D_V: size of bufferx is less than the size of the data to be filled.")
-
-                    msgsize = ie - is + 1
-                    do l=1, l_size  ! loop over number of fields
-                       ptr_bufferx = b_addrsx(l, tMe)
-                       if(l==1) start = (update_x%rSpec(tMe)%start(index)-1)*ke_list(l,tMe)
-                       start1 = start + (js-js1)*ni + is - is1 
-                       do k = 1, ke_list(l,tMe)
-                          start2 = start1
-                          do j = js, je
-                             do i = start2+1, start2+msgsize
-                                pos = pos + 1
-                                bufferx(i) = recv_buffer(pos)
-                             end do
-                             start2 = start2 + ni
-                          end do
-                          start1 = start1 + total
-                       end do
-                    end do
-                 else
-                    do l=1, l_size  ! loop over number of fields
-                       ptr_fieldx = f_addrsx(l, tMe)
-                       do k = 1,ke_list(l,tMe)
-                          do j = js, je
-                             do i = is, ie
-                                pos = pos + 1
-                                fieldx(i,j,k) = recv_buffer(pos)
-                             end do
+                 do l=1, l_size  ! loop over number of fields
+                    ptr_fieldx = f_addrsx(l, tMe)
+                    do k = 1,ke_list(l,tMe)
+                       do j = js, je
+                          do i = is, ie
+                             pos = pos + 1
+                             fieldx(i,j,k) = recv_buffer(pos)
                           end do
                        end do
                     end do
-                 end if
+                 end do
               end if
            end do
         endif
diff --git a/src/shared/mpp/include/mpp_do_update_nonblock.h b/src/shared/mpp/include/mpp_do_update_nonblock.h
index c800c45416..4d0a553108 100644
--- a/src/shared/mpp/include/mpp_do_update_nonblock.h
+++ b/src/shared/mpp/include/mpp_do_update_nonblock.h
@@ -18,6 +18,7 @@ subroutine MPP_START_DO_UPDATE_3D_(id_update, f_addrs, domain, update, d_type, k
   logical                     :: send(8), recv(8), update_edge_only
   integer                     :: l_size, ke_sum, my_id_update
   integer                     :: request
+  integer                     :: send_msgsize(MAXLIST)
   character(len=128)          :: text
   MPP_TYPE_                   :: buffer(size(mpp_domains_stack_nonblock(:)))
   MPP_TYPE_                   :: field(update%xbegin:update%xend, update%ybegin:update%yend,ke_max)
@@ -123,16 +124,13 @@ subroutine MPP_START_DO_UPDATE_3D_(id_update, f_addrs, domain, update, d_type, k
 
   ! pre-postrecv
   call mpp_clock_begin(recv_clock_nonblock)
-!$OMP parallel do schedule(dynamic) default(shared) private(dir,from_pe,buffer_pos, request, msgsize)
   do m = 1, update%nrecv
      msgsize = nonblock_data(id_update)%size_recv(m)
      if( msgsize.GT.0 )then
         from_pe =  update%recv(m)%pe
         buffer_pos = nonblock_data(id_update)%buffer_pos_recv(m)
-!$OMP CRITICAL
         call mpp_recv( buffer(buffer_pos+1), glen=msgsize, from_pe=from_pe, block=.FALSE., &
              tag=id_update, request=request)
-!$OMP END CRITICAL
         nonblock_data(id_update)%request_recv(m) = request
 
 #ifdef use_libMPI
@@ -140,7 +138,6 @@ subroutine MPP_START_DO_UPDATE_3D_(id_update, f_addrs, domain, update, d_type, k
 #endif
      end if
   end do ! end do m = 1, update%nrecv  
-!$OMP end parallel do
 
   call mpp_clock_end(recv_clock_nonblock)
 
@@ -149,8 +146,9 @@ subroutine MPP_START_DO_UPDATE_3D_(id_update, f_addrs, domain, update, d_type, k
 !$OMP parallel do schedule(dynamic) default(shared) private(buffer_pos,pos,dir,tMe,is,ie,js,je,ptr_field,to_pe, &
 !$OMP msgsize,request)
   do m = 1, update%nsend
+     send_msgsize(m) = 0
      if( update%send(m)%count == 0 )cycle
-  
+     
      buffer_pos = nonblock_data(id_update)%buffer_pos_send(m)
      pos = buffer_pos
 
@@ -160,10 +158,11 @@ subroutine MPP_START_DO_UPDATE_3D_(id_update, f_addrs, domain, update, d_type, k
            tMe = update%send(m)%tileMe(n)
            is = update%send(m)%is(n); ie = update%send(m)%ie(n)
            js = update%send(m)%js(n); je = update%send(m)%je(n)
-           if( update%send(m)%is_refined(n) ) then
+           select case( update%send(m)%rotation(n) )
+           case(ZERO)
               do l=1,l_size  ! loop over number of fields
                  ptr_field = f_addrs(l, tMe)
-                 do k = 1,ke_list(l,tMe)
+                 do k = 1,ke_list(l,tMe)  
                     do j = js, je
                        do i = is, ie
                           pos = pos + 1
@@ -172,75 +171,60 @@ subroutine MPP_START_DO_UPDATE_3D_(id_update, f_addrs, domain, update, d_type, k
                     end do
                  end do
               enddo
-           else
-              select case( update%send(m)%rotation(n) )
-              case(ZERO)
-                 do l=1,l_size  ! loop over number of fields
-                    ptr_field = f_addrs(l, tMe)
-                    do k = 1,ke_list(l,tMe)  
-                       do j = js, je
-                          do i = is, ie
-                             pos = pos + 1
-                             buffer(pos) = field(i,j,k)
-                          end do
-                       end do
-                    end do
-                 enddo
-              case( MINUS_NINETY ) 
-                 do l=1,l_size  ! loop over number of fields
-                    ptr_field = f_addrs(l, tMe)
-                    do k = 1,ke_list(l,tMe)  
-                       do i = is, ie
-                          do j = je, js, -1
-                             pos = pos + 1
-                             buffer(pos) = field(i,j,k)
-                          end do
+           case( MINUS_NINETY ) 
+              do l=1,l_size  ! loop over number of fields
+                 ptr_field = f_addrs(l, tMe)
+                 do k = 1,ke_list(l,tMe)  
+                    do i = is, ie
+                       do j = je, js, -1
+                          pos = pos + 1
+                          buffer(pos) = field(i,j,k)
                        end do
                     end do
                  end do
-              case( NINETY ) 
-                 do l=1,l_size  ! loop over number of fields
-                    ptr_field = f_addrs(l, tMe)
+              end do
+           case( NINETY ) 
+              do l=1,l_size  ! loop over number of fields
+                 ptr_field = f_addrs(l, tMe)
 
-                    do k = 1,ke_list(l,tMe)  
-                       do i = ie, is, -1
-                          do j = js, je
-                             pos = pos + 1
-                             buffer(pos) = field(i,j,k)
-                          end do
+                 do k = 1,ke_list(l,tMe)  
+                    do i = ie, is, -1
+                       do j = js, je
+                          pos = pos + 1
+                          buffer(pos) = field(i,j,k)
                        end do
                     end do
                  end do
-              case( ONE_HUNDRED_EIGHTY ) 
-                 do l=1,l_size  ! loop over number of fields
-                    ptr_field = f_addrs(l, tMe)
-
-                    do k = 1,ke_list(l,tMe)  
-                       do j = je, js, -1
-                          do i = ie, is, -1
-                             pos = pos + 1
-                             buffer(pos) = field(i,j,k)
-                          end do
+              end do
+           case( ONE_HUNDRED_EIGHTY ) 
+              do l=1,l_size  ! loop over number of fields
+                 ptr_field = f_addrs(l, tMe)
+                 do k = 1,ke_list(l,tMe)  
+                    do j = je, js, -1
+                       do i = ie, is, -1
+                          pos = pos + 1
+                          buffer(pos) = field(i,j,k)
                        end do
                     end do
                  end do
-              end select
-           end if
-        endif
-     end do ! do n = 1, update%send(m)%count
-
+              end do
+           end select
+       endif
+    end do ! do n = 1, update%send(m)%count
+    send_msgsize(m) = pos - buffer_pos
+ enddo
+ !$OMP end parallel do
 
-     msgsize = pos - buffer_pos
+  do m = 1, update%nsend
+    msgsize = send_msgsize(m)
      if( msgsize .GT.0 )then
+        buffer_pos = nonblock_data(id_update)%buffer_pos_send(m)
         to_pe = update%send(m)%pe
-!$OMP CRITICAL
-        call mpp_send( buffer(buffer_pos+1), plen= msgsize, to_pe=to_pe, &
+        call mpp_send( buffer(buffer_pos+1), plen=msgsize , to_pe=to_pe, &
                        tag=id_update, request=request)
-!$OMP END CRITICAL
         nonblock_data(id_update)%request_send(m) = request
      end if
   end do ! end do ist = 0,nlist-1
-!$OMP end parallel do
 
   call mpp_clock_end(send_pack_clock_nonblock)
 
@@ -251,8 +235,7 @@ end subroutine MPP_START_DO_UPDATE_3D_
 
 !###############################################################################
 
-subroutine MPP_COMPLETE_DO_UPDATE_3D_(id_update, f_addrs, domain, update, d_type, ke_max, ke_list, &
-                                      b_addrs, b_size, flags) 
+subroutine MPP_COMPLETE_DO_UPDATE_3D_(id_update, f_addrs, domain, update, d_type, ke_max, ke_list, flags) 
   integer,             intent(in) :: id_update
   integer(LONG_KIND),  intent(in) :: f_addrs(:,:)
   type(domain2d),      intent(in) :: domain
@@ -260,26 +243,19 @@ subroutine MPP_COMPLETE_DO_UPDATE_3D_(id_update, f_addrs, domain, update, d_type
   integer,             intent(in) :: ke_max
   integer,             intent(in) :: ke_list(:,:)
   MPP_TYPE_,           intent(in) :: d_type  ! creates unique interface
-  integer(LONG_KIND),  intent(in) :: b_addrs(:,:)
-  integer,             intent(in) :: b_size
   integer,             intent(in) :: flags
 
   !--- local variables
-  integer                     :: i, j, k, m, n, l, dir, count, tMe
+  integer                     :: i, j, k, m, n, l, dir, count, tMe, tNbr
   integer                     :: buffer_pos, msgsize, from_pe, pos
   integer                     :: is, ie, js, je
-  integer                     :: start, start1, start2, index
-  integer                     :: is1, ie1, js1, je1, ni, nj, total
   logical                     :: send(8), recv(8), update_edge_only
   integer                     :: l_size, ke_sum, sendsize, recvsize
   character(len=128)          :: text
   MPP_TYPE_                   :: recv_buffer(size(mpp_domains_stack_nonblock(:)))
   MPP_TYPE_                   :: field(update%xbegin:update%xend, update%ybegin:update%yend,ke_max)
-  MPP_TYPE_                   :: buffer(b_size)
   pointer( ptr, recv_buffer )
   pointer(ptr_field, field)
-  pointer(ptr_buffer, buffer) 
-
 
   update_edge_only = BTEST(flags, EDGEONLY)
   recv(1) = BTEST(flags,EAST)
@@ -323,7 +299,7 @@ subroutine MPP_COMPLETE_DO_UPDATE_3D_(id_update, f_addrs, domain, update, d_type
   !--unpack the data
   call mpp_clock_begin(unpk_clock_nonblock)
 !$OMP parallel do schedule(dynamic) default(shared) private(dir,buffer_pos,pos,tMe,is,ie,js,je,msgsize, &
-!$OMP          ptr_field, index,is1,ie1,js1,je1,total,start,ptr_buffer,start1,start2)
+!$OMP          ptr_field)
   do m = update%nrecv, 1, -1
      if( update%recv(m)%count == 0 )cycle
      buffer_pos = nonblock_data(id_update)%buffer_pos_recv(m) + nonblock_data(id_update)%size_recv(m)
@@ -338,44 +314,17 @@ subroutine MPP_COMPLETE_DO_UPDATE_3D_(id_update, f_addrs, domain, update, d_type
            msgsize = (ie-is+1)*(je-js+1)*ke_sum
            pos = buffer_pos - msgsize
            buffer_pos = pos
-           if(update%recv(m)%is_refined(n)) then
-              index = update%recv(m)%index(n)
-              is1 = update%rSpec(tMe)%isNbr(index); ie1 = update%rSpec(tMe)%ieNbr(index)
-              js1 = update%rSpec(tMe)%jsNbr(index); je1 = update%rSpec(tMe)%jeNbr(index)
-              ni = ie1 - is1 + 1
-              nj = je1 - js1 + 1
-              total = ni*nj
-              start = (update%rSpec(tMe)%start(index)-1)*ke_max
-              if(start+total*ke_max>size(buffer) ) call mpp_error(FATAL, &
-                   "MPP_COMPETE_UPDATE_DOMAINS: size of buffer is less than the size of the data to be filled.")
-              msgsize = ie - is + 1
-              do l=1, l_size  ! loop over number of fields
-                 ptr_buffer = b_addrs(l, tMe)
-                 if(l==1) start = (update%rSpec(tMe)%start(index)-1)*ke_list(l,tMe)
-                 start1 = start + (js-js1)*ni + is - is1
-                 do k = 1, ke_list(l,tMe)
-                    start2 = start1
-                    do j = js, je
-                       buffer(start2+1:start2+msgsize) = recv_buffer(pos+1:pos+msgsize)
-                       start2 = start2 + ni
-                       pos   = pos + msgsize
-                    end do
-                    start1 = start1 + total
-                 end do
-              enddo
-           else
-              do l=1, l_size  ! loop over number of fields
-                 ptr_field = f_addrs(l, tMe)
-                 do k = 1,ke_list(l,tMe)
-                    do j = js, je
-                       do i = is, ie
-                          pos = pos + 1
-                          field(i,j,k) = recv_buffer(pos)
-                       end do
+           do l=1, l_size  ! loop over number of fields
+              ptr_field = f_addrs(l, tMe)
+              do k = 1,ke_list(l,tMe)
+                 do j = js, je
+                    do i = is, ie
+                       pos = pos + 1
+                       field(i,j,k) = recv_buffer(pos)
                     end do
                  end do
               end do
-           endif
+           end do
         end if
      end do ! do n = 1, update%recv(m)%count
   end do
diff --git a/src/shared/mpp/include/mpp_domains_comm.inc b/src/shared/mpp/include/mpp_domains_comm.inc
index 9b2a3842fc..928d85e5ce 100644
--- a/src/shared/mpp/include/mpp_domains_comm.inc
+++ b/src/shared/mpp/include/mpp_domains_comm.inc
@@ -1,6 +1,6 @@
 
 ! -*-f90-*-
-! $Id: mpp_domains_comm.inc,v 20.0 2013/12/14 00:24:40 fms Exp $
+! $Id$
 
 
     function mpp_redistribute_init_comm(domain_in,l_addrs_in, domain_out,l_addrs_out, &
diff --git a/src/shared/mpp/include/mpp_domains_define.inc b/src/shared/mpp/include/mpp_domains_define.inc
index 3510ef8134..a97e743e54 100644
--- a/src/shared/mpp/include/mpp_domains_define.inc
+++ b/src/shared/mpp/include/mpp_domains_define.inc
@@ -1,6 +1,6 @@
 
 ! -*-f90-*-
-! $Id: mpp_domains_define.inc,v 20.0 2013/12/14 00:24:42 fms Exp $
+! $Id$
 
 
   ! 
@@ -140,6 +140,32 @@
 
   end subroutine mpp_define_mosaic_pelist
 
+  !-- The following implementation is different from mpp_compute_extents 
+  !-- The last block might have most points 
+  subroutine mpp_compute_block_extent(isg,ieg,ndivs,ibegin,iend)
+    integer,                 intent(in) :: isg, ieg, ndivs
+    integer, dimension(:), intent(out)  :: ibegin, iend
+
+    integer :: ndiv, imax, ndmax
+    integer :: is, ie, n
+
+    ie = ieg
+    do ndiv=ndivs,1,-1
+      !domain is sized by dividing remaining points by remaining domains
+      is = ie - CEILING( REAL(ie-isg+1)/ndiv ) + 1
+      ibegin(ndiv) = is
+      iend(ndiv) = ie
+      
+      if( ie.LT.is )call mpp_error( FATAL,  &
+          'MPP_DEFINE_DOMAINS(mpp_compute_block_extent): domain extents must be positive definite.' )
+      if( ndiv.EQ.1 .AND. ibegin(ndiv) .NE. isg ) &
+          call mpp_error( FATAL, 'mpp_compute_block_extent: domain extents do not span space completely.' )
+      ie = is - 1
+     end do
+
+  end subroutine mpp_compute_block_extent
+
+
   !#####################################################################
   subroutine mpp_compute_extent(isg,ieg,ndivs,ibegin,iend, extent )
     integer,                 intent(in)          :: isg, ieg, ndivs
@@ -491,7 +517,7 @@
     ndivx = size(domain%pearray,1)
     ndivy = size(domain%pearray,2)
     allocate(posarray(0:ndivx-1, 0:ndivy-1))
-    n = 0
+    n = domain%tile_root_pe - mpp_root_pe()
     posarray = -1
     do j = 0,ndivy-1
        do i = 0,ndivx-1
@@ -622,8 +648,8 @@
     type(overlapSpec), pointer :: update=>NULL()
     character(len=1)     :: position
     integer              :: msgsize, l, p, is, ie, js, je, from_pe
-    integer, allocatable :: msg1(:), msg2(:), msg3(:)
     integer              :: outunit
+    logical              :: send(8), recv(8)
 
     outunit = stdout()
     if( .NOT.module_is_initialized )call mpp_error( FATAL, 'MPP_DEFINE_DOMAINS2D: You must first call mpp_domains_init.' )
@@ -957,7 +983,7 @@
        !set up domain%list
        if( mpp_pe().EQ.pes(0) .AND. PRESENT(name) )then
           logunit = stdlog()
-          write( logunit, '(/a,i3,a,i3)' )trim(name)//' domain decomposition: ', ndivx, ' X', ndivy
+          write( logunit, '(/a,i5,a,i5)' )trim(name)//' domain decomposition: ', ndivx, ' X', ndivy
           write( logunit, '(3x,a)' )'pe,   is,  ie,  js,  je,    isd, ied, jsd, jed'
        end if
     end if   !     if( ANY(pes == mpp_pe()) ) 
@@ -1004,7 +1030,7 @@
 
 
        !--- when ncontacts is nonzero, set_check_overlap will be called in mpp_define
-       if(domain%symmetry .AND. domain%ncontacts == 0 ) then
+       if(domain%symmetry .AND. (domain%ncontacts == 0 .OR. domain%ntiles == 1) ) then
           call set_check_overlap( domain, CORNER )
           call set_check_overlap( domain, EAST   )
           call set_check_overlap( domain, NORTH  )
@@ -1022,66 +1048,16 @@
     !--- check the send and recv size are matching.
     !--- or ntiles>1 mosaic, 
     !--- the check will be done in mpp_define_mosaic
-    if(debug_message_passing .and. domain%ntiles == 1 ) then
-       allocate(msg1(0:nlist-1), msg2(0:nlist-1), msg3(0:nlist-1) )
-       do p = 1, 4
-          msg1 = 0
-          msg2 = 0
-          select case(p)
-          case(1)
-             update=>domain%update_T
-             position = "T"
-          case(2)
-             update=>domain%update_C
-             position = "C"
-          case(3)
-             update=>domain%update_E
-             position = "E"
-          case(4)
-             update=>domain%update_N
-             position = "N"
-          end select
-          do m = 1, update%nrecv
-             msgsize = 0
-             do n = 1, update%recv(m)%count
-                is = update%recv(m)%is(n); ie = update%recv(m)%ie(n)
-                js = update%recv(m)%js(n); je = update%recv(m)%je(n)
-                msgsize = msgsize + (ie-is+1)*(je-js+1)
-             end do
-             from_pe = update%recv(m)%pe
-             l = from_pe-mpp_root_pe()
-             call mpp_recv( msg1(l), glen=1, from_pe=from_pe, block=.FALSE., tag=COMM_TAG_1)
-             msg2(l) = msgsize
-          enddo
-
-          do m = 1, update%nsend
-             msgsize = 0
-             do n = 1, update%send(m)%count
-                is = update%send(m)%is(n); ie = update%send(m)%ie(n)
-                js = update%send(m)%js(n); je = update%send(m)%je(n)
-                msgsize = msgsize + (ie-is+1)*(je-js+1)
-             end do
-             l = update%send(m)%pe-mpp_root_pe()
-             msg3(l) = msgsize
-             call mpp_send( msg3(l), plen=1, to_pe=update%send(m)%pe, tag=COMM_TAG_1)
-          enddo
-          call mpp_sync_self(check=EVENT_RECV)
-
-          do m = 0, nlist-1
-             if(msg1(m) .NE. msg2(m)) then
-                print*, "My pe = ", mpp_pe(), ",domain name =", trim(domain%name), ",at position=",position,",from pe=", &
-                     domain%list(m)%pe, ":send size = ", msg1(m), ", recv size = ", msg2(m)
-                call mpp_error(FATAL, "mpp_define_domains2D: mismatch on send and recv size")
-             endif
-          enddo
-          call mpp_sync_self()
-       enddo
-       write(outunit,*)"NOTE from mpp_define_domains2D: message sizes are matched between send and recv for domain " &
-            //trim(domain%name)
-       deallocate(msg1, msg2, msg3)
-       update=>NULL()
+    if(debug_message_passing .and. (domain%ncontacts == 0 .OR. domain%ntiles == 1) ) then
+       send = .true.
+       recv = .true.
+       call check_message_size(domain, domain%update_T, send, recv, 'T')
+       call check_message_size(domain, domain%update_E, send, recv, 'E')
+       call check_message_size(domain, domain%update_C, send, recv, 'C')
+       call check_message_size(domain, domain%update_N, send, recv, 'N')
     endif
 
+
     !print out decomposition, this didn't consider maskmap.
     if( mpp_pe() .EQ. pes(0) .AND. PRESENT(name) )then
        write(*,*) trim(name)//' domain decomposition'
@@ -1095,8 +1071,70 @@
     deallocate( pes, pesall)
 
 
-    return
-  end subroutine mpp_define_domains2D
+  return
+end subroutine mpp_define_domains2D
+
+
+!#####################################################################
+subroutine check_message_size(domain, update, send, recv, position)
+  type(domain2d),       intent(in) :: domain 
+  type(overlapSpec),    intent(in) :: update
+  logical,              intent(in) :: send(:)
+  logical,              intent(in) :: recv(:)
+  character,            intent(in) :: position     
+
+  integer, dimension(0:size(domain%list(:))-1) :: msg1, msg2, msg3
+  integer :: m, n, l, dir, is, ie, js, je, from_pe, msgsize
+  integer :: nlist
+
+  nlist = size(domain%list(:))     
+
+
+  msg1 = 0
+  msg2 = 0
+  do m = 1, update%nrecv
+     msgsize = 0
+     do n = 1, update%recv(m)%count
+        dir = update%recv(m)%dir(n)
+        if( recv(dir) ) then
+           is = update%recv(m)%is(n); ie = update%recv(m)%ie(n)
+           js = update%recv(m)%js(n); je = update%recv(m)%je(n)
+           msgsize = msgsize + (ie-is+1)*(je-js+1)
+        endif
+     end do
+     from_pe = update%recv(m)%pe
+     l = from_pe-mpp_root_pe()
+     call mpp_recv( msg1(l), glen=1, from_pe=from_pe, block=.FALSE., tag=COMM_TAG_1)
+     msg2(l) = msgsize
+  enddo
+
+  do m = 1, update%nsend
+     msgsize = 0
+     do n = 1, update%send(m)%count
+        dir = update%send(m)%dir(n)
+        if(send(dir))then
+           is = update%send(m)%is(n); ie = update%send(m)%ie(n)
+           js = update%send(m)%js(n); je = update%send(m)%je(n)
+           msgsize = msgsize + (ie-is+1)*(je-js+1)
+        endif
+     end do
+     l = update%send(m)%pe-mpp_root_pe()
+     msg3(l) = msgsize
+     call mpp_send( msg3(l), plen=1, to_pe=update%send(m)%pe, tag=COMM_TAG_1)
+  enddo
+  call mpp_sync_self(check=EVENT_RECV)
+
+  do m = 0, nlist-1
+     if(msg1(m) .NE. msg2(m)) then
+        print*, "My pe = ", mpp_pe(), ",domain name =", trim(domain%name), ",at position=",position,",from pe=", &
+             domain%list(m)%pe, ":send size = ", msg1(m), ", recv size = ", msg2(m)
+        call mpp_error(FATAL, "mpp_define_domains2D: mismatch on send and recv size")
+     endif
+  enddo
+  call mpp_sync_self()
+
+
+end subroutine check_message_size
 
   !#####################################################################
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
@@ -1152,6 +1190,7 @@
     integer              :: msgsize, l, p, is, ie, js, je, from_pe
     integer, allocatable :: msg1(:), msg2(:), msg3(:)
     integer              :: outunit
+    logical              :: send(8), recv(8)
 
     outunit = stdout()
     mosaic_defined = .true.
@@ -1449,61 +1488,20 @@
     !--- currently only check T and C-cell. For ntiles>1 mosaic, 
     !--- the check will be done in mpp_define_mosaic
     if(debug_message_passing) then
-       allocate(msg1(0:nlist-1), msg2(0:nlist-1), msg3(0:nlist-1) )
-       do p = 1, 2
-          msg1 = 0
-          msg2 = 0
-          select case(p)
-          case(1)
-             update=>domain%update_T
-             position = "T"
-          case(2)
-             update=>domain%update_C
-             position = "C"
-          end select
-          do m = 1, update%nrecv
-             msgsize = 0
-             do n = 1, update%recv(m)%count
-                is = update%recv(m)%is(n); ie = update%recv(m)%ie(n)
-                js = update%recv(m)%js(n); je = update%recv(m)%je(n)
-                msgsize = msgsize + (ie-is+1)*(je-js+1)
-             end do
-             from_pe = update%recv(m)%pe
-             l = from_pe-mpp_root_pe()
-             call mpp_recv( msg1(l), glen=1, from_pe=from_pe, block=.FALSE., tag=COMM_TAG_2)
-             msg2(l) = msgsize
-          enddo
-
-          do m = 1, update%nsend
-             msgsize = 0
-             do n = 1, update%send(m)%count
-                is = update%send(m)%is(n); ie = update%send(m)%ie(n)
-                js = update%send(m)%js(n); je = update%send(m)%je(n)
-                msgsize = msgsize + (ie-is+1)*(je-js+1)
-             end do
-             msg3(m) = msgsize
-             call mpp_send( msg3(m), plen=1, to_pe=update%send(m)%pe, tag=COMM_TAG_2)
-          enddo
-          call mpp_sync_self(check=EVENT_RECV)
-
-          do m = 0, nlist-1
-             if(msg1(m) .NE. msg2(m)) then
-                print*, "My pe = ", mpp_pe(), ",domain name =", trim(domain%name), ",at position=",position,",from pe=", &
-                     domain%list(m)%pe, ":send size = ", msg1(m), ", recv size = ", msg2(m)
-                call mpp_error(FATAL, "mpp_define_mosaic: mismatch on send and recv size")
-             endif
-          enddo
-          call mpp_sync_self()
-       enddo
-       write(outunit,*)"NOTE from mpp_define_mosaic: message sizes are matched between send and recv for domain " &
-            //trim(domain%name)//"(T and C-cell)"
-       deallocate(msg1, msg2)
-       update=>NULL()
+       send = .true.
+       recv = .true.
+       call check_message_size(domain, domain%update_T, send, recv, 'T')
+       call check_message_size(domain, domain%update_C, send, recv, 'C')
+       call check_message_size(domain, domain%update_E, send, recv, 'E')
+       call check_message_size(domain, domain%update_N, send, recv, 'N')
     endif
 
+
     !--- release memory
     deallocate(align1, align2, is1, ie1, js1, je1, is2, ie2, js2, je2  )
     deallocate(isgList, iegList, jsgList, jegList, refine1, refine2 )
+
+
   end subroutine mpp_define_mosaic
 
 !#####################################################################
@@ -5144,7 +5142,6 @@
     rotate = ZERO
     if(present(rotation)) rotate = rotation
     overlap_out%rotation  (count) = overlap_in%rotation(index)
-    overlap_out%is_refined(count) = overlap_in%is_refined(index)
     overlap_out%dir       (count) = dir
     overlap_out%tileMe    (count) = overlap_in%tileMe(index)   
     overlap_out%tileNbr   (count) = overlap_in%tileNbr(index)   
@@ -5493,12 +5490,8 @@
                    case ( WEST, EAST )
                       ioff = isd - is2Send(n) 
                       joff = jsd - js2Send(n)
-                      joff = get_refine_count( joff, refineSend(n), .true., dir )
-                      nyd  = get_refine_count( nyd, refineSend(n), .false., dir  )
                    case ( SOUTH, NORTH )
                       ioff = isd - is2Send(n) 
-                      ioff = get_refine_count( ioff, refineSend(n), .true., dir  )
-                      nxd  = get_refine_count( nxd, refineSend(n), .false., dir  )
                       joff = jsd - js2Send(n)
                    end select
 
@@ -5518,8 +5511,8 @@
                    js = max(jsc1,jsc2); je = min(jec1,jec2)
                    if(ie.GE.is .AND. je.GE.js )then
                       if(.not. associated(overlapSend(m)%tileMe)) call allocate_update_overlap(overlapSend(m), MAXOVERLAP)
-                      call insert_mosaic_update_overlap(overlapSend(m), domain%list(m)%pe, tMe, tNbr, &
-                                                        is, ie, js, je, dir, rotateSend(n), .true., refineSend(n) .NE. 1 )
+                    call insert_overlap_type(overlapSend(m), domain%list(m)%pe, tMe, tNbr, &
+                         is, ie, js, je, dir, rotateSend(n), .true. )
                    endif
                 end do ! end do dir = 1, 8
              end do ! end do tNbr = 1, ntileNbr
@@ -5549,12 +5542,8 @@
                       ioff = ie2Recv(n) - iec
                    endif
                    joff = jsc - js2Recv(n)
-                   joff = get_refine_count( joff, 1/refineRecv(n), .true., dir  )
-                   nyc  = get_refine_count( nyc, 1/refineRecv(n), .false., dir  )
                 case ( NORTH, SOUTH )
                    ioff = isc - is2Recv(n) 
-                   ioff = get_refine_count( ioff, 1/refineRecv(n), .true., dir  )
-                   nxc  = get_refine_count( nxc, 1/refineRecv(n), .false., dir  )
                    if(align2Recv(n) == SOUTH) then
                       joff = jsc - js2Recv(n)
                    else
@@ -5625,48 +5614,9 @@
                    js = max(jsd1,jsd2); je = min(jed1,jed2)
                    if(ie.GE.is .AND. je.GE.js )then
                       if(.not. associated(overlapRecv(m)%tileMe)) call allocate_update_overlap(overlapRecv(m), MAXOVERLAP)
-                      call insert_mosaic_update_overlap(overlapRecv(m), domain%list(m)%pe, tMe, tNbr, &
-                                                        is, ie, js, je, dir, rotateRecv(n), .true., refineSend(n) .NE. 1 )
+                    call insert_overlap_type(overlapRecv(m), domain%list(m)%pe, tMe, tNbr, &
+                         is, ie, js, je, dir, rotateRecv(n), .true.)
                       count                              = overlapRecv(m)%count
-                      overlapRecv(m)%isMe        (count) = is
-                      overlapRecv(m)%ieMe        (count) = ie
-                      overlapRecv(m)%jsMe        (count) = js
-                      overlapRecv(m)%jeMe        (count) = je
-                      !--- get the index on the pe sending data when there is refinement
-                      if( overlapRecv(m)%is_refined(count) ) then
-                         select case ( align1Recv(n) )
-                         case ( WEST, EAST )
-                            isoff = 0;               ieoff = ie - is + 1
-                            jsoff = js - js1Recv(n); jeoff = je - js1Recv(n) + 1
-                            jsoff = get_refine_count( jsoff, refineRecv(n), .true., dir  )
-                            jeoff = get_refine_count( jeoff, refineRecv(n), .false., dir  ) - 1
-                         case ( NORTH, SOUTH )
-                            isoff = is - is1Recv(n); ieoff = ie - is1Recv(n) + 1
-                            jsoff = 0;               jeoff = je - js
-                            isoff = get_refine_count( isoff, refineRecv(n), .true., dir  )
-                            ieoff = get_refine_count( ieoff, refineRecv(n), .false., dir  ) - 1
-                         end select
-
-                         !--- get the index in neighbor pe.
-                         select case ( rotateRecv(n) )
-                         case ( ZERO )
-                            isd2 = is2Recv(n) + isoff; ied2 = is2Recv(n) + ieoff
-                            jsd2 = js2Recv(n) + jsoff; jed2 = js2Recv(n) + jeoff
-                         case ( NINETY )                      ! N -> W or S -> E
-                            isd2 = ie2Recv(n) - jeoff; ied2 = ie2Recv(n) - jsoff
-                            jsd2 = js2Recv(n);         jed2 = js2Recv(n) + ieoff - isoff
-                         case ( MINUS_NINETY )                ! W -> N or E -> S
-                            isd2 = is2Recv(n);         ied2 = is2Recv(n) + jeoff - jsoff
-                            jsd2 = je2Recv(n) - ieoff; jed2 = je2Recv(n) - isoff
-                         end select
-                         isd2= max(isd2,isc); ied2 = min(ied2, iec); jsd2= max(jsd2,jsc); jed2 = min(jed2, jec)
-                         if( isd2 > ied2 .OR. jsd2 > jed2 ) call mpp_error(FATAL, &
-                              "mpp_domains_define.inc:  the neighbor index is out of range")
-                         overlapRecv(m)%is(count) = isd2
-                         overlapRecv(m)%ie(count) = ied2
-                         overlapRecv(m)%js(count) = jsd2
-                         overlapRecv(m)%je(count) = jed2
-                      end if
                    endif
                 end do ! end do dir = 1, 8
              end do ! end do tNbr = 1, ntileNbr
@@ -5732,10 +5682,10 @@
                          if(overlapSend(m)%tileMe(l) .NE. tMe) cycle
                          if(overlapSend(m)%tileNbr(l) .NE. tNbr) cycle
                          if(overlapSend(m)%dir(l) .NE. dirlist(n) ) cycle
-                         call insert_mosaic_update_overlap(domain%update_T%send(nsend2), overlapSend(m)%pe,                     &
+                       call insert_overlap_type(domain%update_T%send(nsend2), overlapSend(m)%pe,                     &
                               overlapSend(m)%tileMe(l), overlapSend(m)%tileNbr(l), overlapSend(m)%is(l), overlapSend(m)%ie(l),  &
                               overlapSend(m)%js(l), overlapSend(m)%je(l), overlapSend(m)%dir(l), overlapSend(m)%rotation(l),    &
-                              overlapSend(m)%from_contact(l),  overlapSend(m)%is_refined(l) )                   
+                            overlapSend(m)%from_contact(l)  )                   
                       end do
                    end do
                 end do
@@ -5798,16 +5748,11 @@
                          if(overlapRecv(m)%tileMe(l) .NE. tMe) cycle
                          if(overlapRecv(m)%tileNbr(l) .NE. tNbr) cycle
                          if(overlapRecv(m)%dir(l) .NE. dirlist(n) ) cycle
-                         call insert_mosaic_update_overlap(domain%update_T%recv(nrecv2), overlapRecv(m)%pe, &
+                       call insert_overlap_type(domain%update_T%recv(nrecv2), overlapRecv(m)%pe, &
                               overlapRecv(m)%tileMe(l), overlapRecv(m)%tileNbr(l), overlapRecv(m)%is(l), overlapRecv(m)%ie(l),  &
                               overlapRecv(m)%js(l), overlapRecv(m)%je(l), overlapRecv(m)%dir(l), overlapRecv(m)%rotation(l),    &
-                              overlapRecv(m)%from_contact(l),  overlaprecv(m)%is_refined(l) )                   
+                            overlapRecv(m)%from_contact(l))                   
                          count = domain%update_T%recv(nrecv2)%count
-                         domain%update_T%recv(nrecv2)%isMe        (count) = overlapRecv(m)%isMe        (l)
-                         domain%update_T%recv(nrecv2)%ieMe        (count) = overlapRecv(m)%ieMe        (l)
-                         domain%update_T%recv(nrecv2)%jsMe        (count) = overlapRecv(m)%jsMe        (l)
-                         domain%update_T%recv(nrecv2)%jeMe        (count) = overlapRecv(m)%jeMe        (l)
-                         domain%update_T%recv(nrecv2)%index       (count) = overlapRecv(m)%index       (l)
                       end do
                    end do
                 end do
@@ -5840,170 +5785,11 @@
        deallocate(nCont(n)%is2, nCont(n)%ie2, nCont(n)%js2, nCont(n)%je2 )
     end do
 
-    !--- define the refined overlapping and group the overlapping 
-    !--- between same tiles( in different direction or different pe ). 
-    !--- set up the index position in the return buffer.
-    call define_refine_overlap( domain, CENTER )  
-
     domain%initialized = .true.
 
 
   end subroutine define_contact_point
 
-  !--- currently assume no grid is divided over two pes.
-  !--- we may remove this restriction in the future. 
-  function get_refine_count(num , refine, use_floor, dir )
-    integer, intent(in) :: num
-    real,    intent(in) :: refine
-    logical, intent(in) :: use_floor
-    integer, intent(in) :: dir
-    integer             :: get_refine_count
-    real, parameter     :: EPSLN = 1.e-10
-    real                :: rval
-
-    rval = num*refine
-
-    if(mod(dir,2) == 1) then ! WEST, EAST, SOUTH, NORTH
-       get_refine_count = nint(rval)
-       if(abs(rval-get_refine_count) > EPSLN) call mpp_error(FATAL, &
-            "get_refine_count: The two adjacent tiles with refinement does not align well in E/W/S/N direction")
-    else
-       get_refine_count = floor(rval)
-       if(use_floor) then
-          if( rval - get_refine_count > 1 - EPSLN ) get_refine_count = get_refine_count + 1 ! Deal with truncation error
-       else
-          if(rval - get_refine_count  > EPSLN) get_refine_count = get_refine_count + 1 ! Deal with truncation error
-       end if
-    end if
-
-
-    return
-
-  end function get_refine_count
-
-  !####################################################################################
-  !--- The following will define the refined overlapping and group the overlapping 
-  !--- between same tiles( in different direction or different pe ). 
-  !--- set up the index position in the return buffer.
-  subroutine define_refine_overlap(domain, position)
-    type(domain2d), intent(inout) :: domain
-    integer,        intent(in)    :: position
-
-    logical                        :: new_overlap
-    integer,           parameter   :: MAXOVERLAP = 100
-    integer, dimension(MAXOVERLAP) :: isMe, ieMe, jsMe, jeMe, isNbr, ieNbr, jsNbr, jeNbr
-    integer, dimension(MAXOVERLAP) :: rotation, tileList, dirList
-    integer                        :: count, tMe, ntileMe, nlist, pos
-    integer                        :: m, n, l, dir, istart, iend, dirdiff
-    type(overlap_type), pointer    :: OverPtr => NULL()
-    type(overlapSpec), pointer     :: update => NULL()
-
-    select case(position)
-    case (CENTER)
-       update => domain%update_T
-    case (CORNER)
-       update => domain%update_C
-    case (EAST)
-       update => domain%update_E
-    case (NORTH)
-       update => domain%update_N
-    case default
-       call mpp_error(FATAL, &
-         "mpp_domains_define.inc(define_refine_overlap): position should be CENTER, CORNER, EAST or NORTH")
-    end select
-
-    ntileMe = size(domain%x(:))
-    nlist = size(domain%list(:))    
-
-    allocate(update%rSpec(ntileMe) )
-
-    do tMe = 1, ntileMe
-       count = 0
-       do m = 1, update%nrecv
-          overPtr => update%recv(m)
-          if( overptr%count == 0)cycle
-          pos = overPtr%pe - mpp_root_pe()
-          do l = 1, overPtr%count
-             if(OverPtr%is_refined(l) .AND. overPtr%tileMe(l) == tMe ) then
-                dir = overPtr%dir(l)
-                new_overlap = .true.
-                do n = 1, count
-                   if( tileList(n) == domain%list(pos)%tile_id(overPtr%tileNbr(l)) ) then ! same tile
-                      dirdiff = dir - dirList(n)   ! dir always no less than dirList(n).
-                      if( (dir == 8 .AND. dirList(n) == 1) .OR. (dir == 1 .AND. dirList(n) == 8) ) dirdiff = 1
-                      dirdiff = abs(dirdiff)                      
-                      if( dirdiff .LE. 1 ) then    ! same direction
-                         if( mod(dirList(n),2) == 0 ) then
-                            if(mod(dir,2) == 0 ) call mpp_error(FATAL, &
-                                 "mpp_get_mosaic_refine_overlap: corner overlap should be unique.")
-                            dirList(n) = dir
-                         end if
-                         if( dirList(n) == 1 .OR. dirList(n) == 5 ) then ! EAST, or WEST
-                            if( isMe(n) .NE. overPtr%isMe(l) .OR. ieMe(n) .NE. overPtr%ieMe(l) ) call mpp_error(FATAL, &
-                                 "mpp_get_mosaic_refine_overlap: mismatch on my index in the overlapping 1" )
-                            jeMe(n) = max(jeMe(n),  overPtr%jeMe(l) )
-                            jsMe(n) = min(jsMe(n),  overPtr%jsMe(l) )
-                         else if(dirList(n) == 3 .OR. dirList(n) == 7 ) then ! SOUTH or NORTH
-                            if( jsMe(n) .NE. overPtr%jsMe(l) .OR. jeMe(n) .NE. overPtr%jeMe(l) ) call mpp_error(FATAL, &
-                                 "mpp_get_mosaic_refine_overlap: mismatch on my index in the overlapping 2" )
-                            ieMe(n) = max(ieMe(n),  overPtr%ieMe(l) )
-                            isMe(n) = min(isMe(n),  overPtr%isMe(l) )
-                         end if
-                         isNbr(n) = min(isNbr(n), overPtr%is(l))
-                         ieNbr(n) = max(ieNbr(n), overPtr%ie(l))
-                         jsNbr(n) = min(jsNbr(n), overPtr%js(l))
-                         jeNbr(n) = max(jeNbr(n), overPtr%je(l))
-                         OverPtr%index(l) = n
-                         new_overlap = .false.
-                         exit
-                      end if
-                   end if
-                end do
-
-                if(new_overlap) then
-                   count = count + 1
-                   if(count > MAXOVERLAP) call mpp_error(FATAL,  &
-                        "mpp_get_mosaic_refine_overlap: number of overlapping is larger than MAXOVERLAP")
-                   tileList(count) = domain%list(pos)%tile_id(overPtr%tileNbr(l))
-                   dirList(count)  = dir
-                   rotation(count) = overPtr%rotation(l)
-                   OverPtr%index(l)= count
-                   isMe(count) = overPtr%isMe(l);    ieMe(count) = overPtr%ieMe(l)
-                   jsMe(count) = overPtr%jsMe(l);    jeMe(count) = overPtr%jeMe(l)
-                   isNbr(count) = overPtr%is(l);     ieNbr(count) = overPtr%ie(l)
-                   jsNbr(count) = overPtr%js(l);     jeNbr(count) = overPtr%je(l)
-                end if
-             end if
-          end do
-       end do
-       if(count > 0) then
-          allocate(update%rSpec(tMe)%isMe(count),  update%rSpec(tMe)%ieMe(count) )
-          allocate(update%rSpec(tMe)%jsMe(count),  update%rSpec(tMe)%jeMe(count) )
-          allocate(update%rSpec(tMe)%isNbr(count), update%rSpec(tMe)%ieNbr(count) )
-          allocate(update%rSpec(tMe)%jsNbr(count), update%rSpec(tMe)%jeNbr(count) )
-          allocate(update%rSpec(tMe)%start(count), update%rSpec(tMe)%end  (count) )
-          allocate(update%rSpec(tMe)%dir  (count), update%rSpec(tMe)%rotation(count) )
-          update%rSpec(tMe)%isMe = isMe(1:count); update%rSpec(tMe)%isNbr = isNbr(1:count)
-          update%rSpec(tMe)%ieMe = ieMe(1:count); update%rSpec(tMe)%ieNbr = ieNbr(1:count)
-          update%rSpec(tMe)%jsMe = jsMe(1:count); update%rSpec(tMe)%jsNbr = jsNbr(1:count)
-          update%rSpec(tMe)%jeMe = jeMe(1:count); update%rSpec(tMe)%jeNbr = jeNbr(1:count)
-          update%rSpec(tMe)%count = count
-          update%rSpec(tMe)%dir   = dirList(1:count)
-          update%rSpec(tMe)%rotation = rotation(1:count)
-          update%rSpec(tMe)%total = sum( (update%rSpec(tMe)%ieNbr-update%rSpec(tMe)%isNbr+1)* &
-               (update%rSpec(tMe)%jeNbr-update%rSpec(tMe)%jsNbr+1)  )
-          iend = 0
-          do n = 1, count
-             istart = iend + 1
-             iend = istart + (ieNbr(n) - isNbr(n) + 1)*(jeNbr(n) - jsNbr(n) + 1) - 1
-             update%rSpec(tMe)%start(n) = istart
-             update%rSpec(tMe)%end(n)   = iend
-          enddo
-       end if
-  end do
-
-end subroutine define_refine_overlap
-
 !##############################################################################
 !--- always fill the contact according to index order.
 subroutine fill_contact(Contact, tile, is1, ie1, js1, je1, is2, ie2, js2, je2, align1, align2, refine1, refine2 )
@@ -6171,10 +5957,9 @@ subroutine set_contact_point(domain, position)
                  isoff1 = jshift; ieoff1 = jshift; jsoff1 = 0;      jeoff1 = 0
               end select
            end select
-           call insert_mosaic_update_overlap(overlap, PtrIn%pe, PtrIn%tileMe(n), PtrIn%tileNbr(n), &
+           call insert_overlap_type(overlap, PtrIn%pe, PtrIn%tileMe(n), PtrIn%tileNbr(n), &
                                              Ptrin%is(n) + isoff1, Ptrin%ie(n) + ieoff1, Ptrin%js(n) + jsoff1, &
-                                             Ptrin%je(n) + jeoff1, PtrIn%dir(n), PtrIn%rotation(n), PtrIn%from_contact(n), &
-                                             PtrIn%is_refined(n) )
+                Ptrin%je(n) + jeoff1, PtrIn%dir(n), PtrIn%rotation(n), PtrIn%from_contact(n))
         end if
      end do ! do n = 1, prtIn%count
      if(overlap%count > 0) then
@@ -6245,51 +6030,10 @@ subroutine set_contact_point(domain, position)
            case ( 8 ) ! NE
               isoff1 = ishift; ieoff1 = ishift; jsoff1 = jshift; jeoff1 = jshift
            end select
-           call insert_mosaic_update_overlap(overlap, PtrIn%pe, PtrIn%tileMe(n), PtrIn%tileNbr(n), &
+           call insert_overlap_type(overlap, PtrIn%pe, PtrIn%tileMe(n), PtrIn%tileNbr(n), &
                                              Ptrin%is(n) + isoff1, Ptrin%ie(n) + ieoff1, Ptrin%js(n) + jsoff1, &
-                                             Ptrin%je(n) + jeoff1, PtrIn%dir(n), PtrIn%rotation(n), PtrIn%from_contact(n), &
-                                             PtrIn%is_refined(n) )
+                Ptrin%je(n) + jeoff1, PtrIn%dir(n), PtrIn%rotation(n), PtrIn%from_contact(n))
            count                     = overlap%count
-           overlap%isMe      (count) = Ptrin%isMe(n) + isoff1
-           overlap%ieMe      (count) = Ptrin%ieMe(n) + ieoff1
-           overlap%jsMe      (count) = Ptrin%jsMe(n) + jsoff1
-           overlap%jeMe      (count) = Ptrin%jeMe(n) + jeoff1
-           !--- figure out neighbor offset when there is refinemnet
-           if( PtrIn%is_refined(n) ) then
-              isoff2 = isoff1; ieoff2 = ieoff1; jsoff2 = jsoff1; jeoff2 = jeoff1
-              if( ptrIn%rotation(n) == NINETY) then
-                 select case ( dir )                                       
-                 case ( 1, 5 ) !S -> E, N -> W
-                    isoff2 = jsoff1; ieoff2 = jeoff1; jsoff2 = isoff1; jeoff2 = ieoff1
-                 case ( 2 ) 
-                    isoff2 = jshift; ieoff2 = jshift; jsoff2 = ishift; jeoff2 = ishift
-                 case ( 4 ) 
-                    isoff2 = jshift; ieoff2 = jshift; jsoff2 = 0;      jeoff2 = 0
-                 case ( 6 ) 
-                    isoff2 = 0;      ieoff2 = 0;      jsoff2 = 0;      jeoff2 = 0
-                 case ( 8 ) 
-                    isoff2 = 0;      ieoff2 = 0;      jsoff2 = ishift; jeoff2 = ishift
-                 end select
-              else if( ptrIn%rotation(n) == MINUS_NINETY) then
-                 select case ( dir )                                       
-                 case ( 3, 7 ) !E -> S, W -> N
-                    isoff2 = jsoff1; ieoff2 = jeoff1; jsoff2 = isoff1; jeoff2 = ieoff1
-                 case ( 2 )
-                    isoff2 = 0;      ieoff2 = 0;      jsoff2 = 0;      jeoff2 = 0 
-                 case ( 4 ) 
-                    isoff2 = 0;      ieoff2 = 0;      jsoff2 = ishift; jeoff2 = ishift
-                 case ( 6 ) 
-                    isoff2 = jshift; ieoff2 = jshift; jsoff2 = ishift; jeoff2 = ishift
-                 case ( 8 ) 
-                    isoff2 = jshift; ieoff2 = jshift; jsoff2 = 0;      jeoff2 = 0
-                 end select
-              end if
-              !--- calculate overlapping
-              overlap%is(count) = Ptrin%is(n) + isoff2
-              overlap%ie(count) = Ptrin%ie(n) + ieoff2
-              overlap%js(count) = Ptrin%js(n) + jsoff2
-              overlap%je(count) = Ptrin%je(n) + jeoff2
-           end if
         end if
      end do ! do n = 1, ptrIn%count
      if(overlap%count > 0) then
@@ -6330,13 +6074,6 @@ subroutine set_contact_point(domain, position)
           "mpp_domains_define.inc(set_contact_point): pos should equal to nrecv")
   endif
 
-
-
-  !--- define the refined overlapping and group the overlapping 
-  !--- between same tiles( in different direction or different pe ). 
-  !--- set up the index position in the return buffer.
-  call define_refine_overlap( domain, position )  
-
   call deallocate_overlap_type(overlap)
 
 end subroutine set_contact_point
@@ -6379,7 +6116,6 @@ nsend = 0
 maxsize = 0
 do m = 1, update%nsend
     do n = 1, update%send(m)%count
-       if( update%send(m)%is_refined(n) ) cycle 
        if( update%send(m)%rotation(n) == ONE_HUNDRED_EIGHTY ) cycle
        if( ( (position == EAST .OR. position == CORNER) .AND. update%send(m)%dir(n) == 1 ) .OR. &
             ( (position == NORTH .OR. position == CORNER) .AND. update%send(m)%dir(n) == 7 ) ) then
@@ -6401,7 +6137,6 @@ nlist = size(domain%list(:))
 pos = 0
 do m = 1, update%nsend
  do n = 1, update%send(m)%count
-    if( update%send(m)%is_refined(n) ) cycle
     if( update%send(m)%rotation(n) == ONE_HUNDRED_EIGHTY ) cycle
     ! comparing east direction on currently pe
     if( (position == EAST .OR. position == CORNER) .AND. update%send(m)%dir(n) == 1 ) then  
@@ -6455,7 +6190,6 @@ nrecv = 0
 maxsize = 0
 do m = 1, update%nrecv
     do n = 1, update%recv(m)%count
-       if( update%recv(m)%is_refined(n) ) cycle 
        if( update%recv(m)%rotation(n) == ONE_HUNDRED_EIGHTY ) cycle
        if( ( (position == EAST .OR. position == CORNER) .AND. update%recv(m)%dir(n) == 1 ) .OR. &
             ( (position == NORTH .OR. position == CORNER) .AND. update%recv(m)%dir(n) == 7 ) ) then
@@ -6476,7 +6210,6 @@ endif
 pos = 0
 do m = 1, update%nrecv
  do n = 1, update%recv(m)%count
-    if( update%recv(m)%is_refined(n) ) cycle
     if( update%recv(m)%rotation(n) == ONE_HUNDRED_EIGHTY ) cycle
     if( (position == EAST .OR. position == CORNER) .AND. update%recv(m)%dir(n) == 1 ) then  
        is = update%recv(m)%is(n) - 1
@@ -6515,23 +6248,38 @@ subroutine set_bound_overlap( domain, position )
   integer,        intent(in)    :: position
   integer                       :: m, n, l, count, dr, tMe, i
   integer, parameter            :: MAXCOUNT = 100
-  integer, dimension(MAXCOUNT)  :: dir, rotation, is, ie, js, je, isMe, ieMe, jsMe, jeMe, tileMe
+  integer, dimension(MAXCOUNT)  :: dir, rotation, is, ie, js, je, tileMe, index
   integer, dimension(size(domain%x(:)), 4)           :: nrecvl
-  integer, dimension(size(domain%x(:)), 4, MAXCOUNT) :: isl, iel, jsl, jel, islMe, ielMe, jslMe, jelMe
+  integer, dimension(size(domain%x(:)), 4, MAXCOUNT) :: isl, iel, jsl, jel
   type(overlap_type),  pointer   :: overlap => NULL()
   type(overlapSpec),   pointer   :: update => NULL()
   type(overlapSpec),   pointer   :: bound => NULL()
   integer                        :: nlist_send, nlist_recv, ishift, jshift
   integer                        :: ism, iem, jsm, jem, nsend, nrecv
   integer                        :: isg, ieg, jsg, jeg, nlist, list
-  integer                        :: isc1, iec1, jsc1, jec1
-  integer                        :: isc2, iec2, jsc2, jec2
+!  integer                        :: isc1, iec1, jsc1, jec1
+!  integer                        :: isc2, iec2, jsc2, jec2
   integer                        :: isd, ied, jsd, jed
+  integer                        :: npes_x, npes_y, ipos, jpos, inbr, jnbr
+  integer                        :: isc, iec, jsc, jec, my_pe
+  integer                        :: pe_south1, pe_south2, pe_west0, pe_west1, pe_west2
+  integer                        :: is_south1, ie_south1, js_south1, je_south1
+  integer                        :: is_south2, ie_south2, js_south2, je_south2
+  integer                        :: is_west0, ie_west0, js_west0, je_west0
+  integer                        :: is_west1, ie_west1, js_west1, je_west1
+  integer                        :: is_west2, ie_west2, js_west2, je_west2
+  logical                        :: x_cyclic, y_cyclic, folded_north
+
+  is_south1=0; ie_south1=0; js_south1=0; je_south1=0
+  is_south2=0; ie_south2=0; js_south2=0; je_south2=0
+  is_west0=0; ie_west0=0; js_west0=0; je_west0=0
+  is_west1=0; ie_west1=0; js_west1=0; je_west1=0
+  is_west2=0; ie_west2=0; js_west2=0; je_west2=0
 
 
   if( position == CENTER .OR. .NOT. domain%symmetry ) return
   call mpp_get_domain_shift(domain, ishift, jshift, position)
-  call mpp_get_global_domain(domain, isg, ieg, jsg, jeg, position=position)
+  call mpp_get_global_domain(domain, isg, ieg, jsg, jeg)
   call mpp_get_memory_domain ( domain, ism, iem, jsm, jem )
 
   select case(position)
@@ -6551,8 +6299,8 @@ subroutine set_bound_overlap( domain, position )
   bound%xbegin = ism; bound%xend = iem + ishift
   bound%ybegin = jsm; bound%yend = jem + jshift
 
-  nlist_send = update%nsend
-  nlist_recv = update%nrecv
+  nlist_send = max(update%nsend,4)
+  nlist_recv = max(update%nrecv,4)
   bound%nsend = nlist_send
   bound%nrecv = nlist_recv
   if(nlist_send >0) then
@@ -6566,58 +6314,147 @@ subroutine set_bound_overlap( domain, position )
   !--- loop over the list of domains to find the boundary overlap for send
   nlist = size(domain%list(:))
 
-  nsend = 0
-  !--- will computing overlap for tripolar grid.
-  if( BTEST(domain%fold,NORTH) ) then
-     !--- currently only set up for west and south boundary.
-    !---south boundary
+  npes_x = size(domain%x(1)%list(:))
+  npes_y = size(domain%y(1)%list(:))
+  x_cyclic = domain%x(1)%cyclic
+  y_cyclic = domain%y(1)%cyclic
+  folded_north = BTEST(domain%fold,NORTH)
+  ipos = domain%x(1)%pos    
+  jpos = domain%y(1)%pos
+  isc  = domain%x(1)%compute%begin; iec = domain%x(1)%compute%end
+  jsc  = domain%y(1)%compute%begin; jec = domain%y(1)%compute%end
+
+  nsend = 0 
+  if(domain%ntiles == 1) then  ! use neighbor processor to configure send and recv
+     ! currently only set up for west and south boundary
+
+     ! south boundary for send
+     pe_south1 = NULL_PE; pe_south2 = NULL_PE
      if( position == NORTH .OR. position == CORNER ) then
-        isc1 = domain%x(1)%compute%begin;      iec1 = domain%x(1)%compute%end +ishift
-        jsc1 = domain%y(1)%compute%end+jshift; jec1 = jsc1
+        inbr = ipos; jnbr = jpos + 1
+        if( jnbr == npes_y .AND. y_cyclic) jnbr = 0
+        if( inbr .GE. 0 .AND. inbr .LT. npes_x .AND. jnbr .GE. 0 .AND. jnbr .LT. npes_y ) then 
+           pe_south1 = domain%pearray(inbr,jnbr)
+           is_south1 = isc + ishift; ie_south1 = iec+ishift
+           js_south1 = jec + jshift; je_south1 = js_south1
+        endif         
      endif
-     !--- west boundary
-     if( position == EAST .OR. position == CORNER ) then
-        isc2 = domain%x(1)%compute%end+ishift; iec2 = isc2
-        jsc2 = domain%y(1)%compute%begin;      jec2 = domain%y(1)%compute%end + jshift
+     !--- send to the southwest processor when position is NORTH
+     if( position == CORNER ) then
+        inbr = ipos + 1; jnbr = jpos + 1
+        if( inbr == npes_x .AND. x_cyclic) inbr = 0
+        if( jnbr == npes_y .AND. y_cyclic) jnbr = 0
+        if( inbr .GE. 0 .AND. inbr .LT. npes_x .AND. jnbr .GE. 0 .AND. jnbr .LT. npes_y ) then 
+           pe_south2 = domain%pearray(inbr,jnbr)
+           is_south2 = iec + ishift; ie_south2 = is_south2
+           js_south2 = jec + jshift; je_south2 = js_south2
+        endif         
      endif
 
-     do list = 0,nlist-1
-        m = mod( domain%pos+list, nlist )
-        count = 0
-
-        !--- south boundary
-        if( position == NORTH .OR. position == CORNER ) then
-           isd = domain%list(m)%x(1)%compute%begin; ied = domain%list(m)%x(1)%compute%end + ishift
-           jsd = domain%list(m)%y(1)%compute%begin; jed = jsd
-           if( isc1 == isd .AND. iec1 == ied .AND. jsc1 == jsd) then
-              count = count + 1          
-              is(count) = isc1; ie(count) = iec1
-              js(count) = jsc1; je(count) = jec1
-              dir(count) = 2
+     !---west boundary for send
+     pe_west0 = NULL_PE; pe_west1 = NULL_PE; pe_west2 = NULL_PE
+     if( position == EAST ) then
+        inbr = ipos+1; jnbr = jpos
+        if( inbr == npes_x .AND. x_cyclic) inbr = 0
+        if( inbr .GE. 0 .AND. inbr .LT. npes_x .AND. jnbr .GE. 0 .AND. jnbr .LT. npes_y ) then 
+           pe_west1 = domain%pearray(inbr,jnbr)
+           is_west1 = iec + ishift; ie_west1 = is_west1
+           js_west1 = jsc + jshift; je_west1 = jec + jshift
+        endif         
+     else if ( position == CORNER ) then  ! possible split into two parts.
+        !--- on the fold.
+        if( folded_north .AND. jec == jeg .AND. ipos .LT. (npes_x-1)/2 ) then
+           inbr = npes_x - ipos - 1; jnbr = jpos
+           if( inbr .GE. 0 .AND. inbr .LT. npes_x .AND. jnbr .GE. 0 .AND. jnbr .LT. npes_y ) then 
+               pe_west0 = domain%pearray(inbr,jnbr)
+               is_west0 = iec+ishift; ie_west0 = is_west0
+               js_west0 = jec+jshift; je_west0 = js_west0
            endif
         endif
 
-        !--- west boundary
-        if( position == EAST .OR. position == CORNER ) then
-           isd = domain%list(m)%x(1)%compute%begin; ied = isd
-           jsd = domain%list(m)%y(1)%compute%begin; jed = domain%list(m)%y(1)%compute%end + jshift
-           !--- cyclic in x-direction is assumed for folded-north
-           if(isd == isg) then
-              isd = ieg; ied = ieg
-           endif
-           if( isc2 == isd .AND. jsc2 == jsd .AND. jec2 == jed ) then
-              count = count + 1          
-              is(count) = isc2; ie(count) = iec2
-              js(count) = jsc2; je(count) = jec2
-              dir(count) = 3
+        if( folded_north .AND. jec == jeg .AND. ipos .GE. npes_x/2 .AND. ipos .LT. (npes_x-1) ) then
+           inbr = ipos+1; jnbr = jpos
+           if( inbr == npes_x .AND. x_cyclic) inbr = 0
+           if( inbr .GE. 0 .AND. inbr .LT. npes_x .AND. jnbr .GE. 0 .AND. jnbr .LT. npes_y ) then 
+              pe_west1 = domain%pearray(inbr,jnbr)
+              is_west1 = iec + ishift; ie_west1 = is_west1
+              js_west1 = jsc + jshift; je_west1 = jec
            endif
+        else
+           inbr = ipos+1; jnbr = jpos
+           if( inbr == npes_x .AND. x_cyclic) inbr = 0
+           if( inbr .GE. 0 .AND. inbr .LT. npes_x .AND. jnbr .GE. 0 .AND. jnbr .LT. npes_y ) then 
+              pe_west1 = domain%pearray(inbr,jnbr)
+              is_west1 = iec + ishift; ie_west1 = is_west1
+              js_west1 = jsc + jshift; je_west1 = jec + jshift
+           endif         
         endif
+     endif 
+     !--- send to the southwest processor when position is NORTH
+     if( position == CORNER ) then
+        inbr = ipos + 1; jnbr = jpos + 1
+        if( inbr == npes_x .AND. x_cyclic) inbr = 0
+        if( jnbr == npes_y .AND. y_cyclic) jnbr = 0
+        if( inbr .GE. 0 .AND. inbr .LT. npes_x .AND. jnbr .GE. 0 .AND. jnbr .LT. npes_y ) then 
+           pe_west2 = domain%pearray(inbr,jnbr)
+           is_west2 = iec + ishift; ie_west2 = is_west2
+           js_west2 = jec + jshift; je_west2 = js_west2
+        endif         
+     endif
+
+!write(1000+mpp_pe(),*)"send south 1", pe_south1, is_south1, ie_south1, js_south1, je_south1
+!write(1000+mpp_pe(),*)"send south 2", pe_south2, is_south2, ie_south2, js_south2, je_south2
+!write(1000+mpp_pe(),*)"send west 0", pe_west0, is_west0, ie_west0, js_west0, je_west0
+!write(1000+mpp_pe(),*)"send west 1", pe_west1, is_west1, ie_west1, js_west1, je_west1
+!write(1000+mpp_pe(),*)"send west 2", pe_west2, is_west2, ie_west2, js_west2, je_west2
+
+
+     do list = 0,nlist-1
+        m = mod( domain%pos+list, nlist )
+        count = 0
+        my_pe = domain%list(m)%pe
+        if(my_pe == pe_south1) then
+           count = count + 1
+           is(count) = is_south1; ie(count) = ie_south1
+           js(count) = js_south1; je(count) = je_south1
+           dir(count) = 2
+           rotation(count) = ZERO
+        endif          
+        if(my_pe == pe_south2) then
+           count = count + 1
+           is(count) = is_south2; ie(count) = ie_south2
+           js(count) = js_south2; je(count) = je_south2
+           dir(count) = 2
+           rotation(count) = ZERO
+        endif          
+
+        if(my_pe == pe_west0) then
+           count = count + 1
+           is(count) = is_west0; ie(count) = ie_west0
+           js(count) = js_west0; je(count) = je_west0
+           dir(count) = 3
+           rotation(count) = ONE_HUNDRED_EIGHTY
+        endif          
+        if(my_pe == pe_west1) then
+           count = count + 1
+           is(count) = is_west1; ie(count) = ie_west1
+           js(count) = js_west1; je(count) = je_west1
+           dir(count) = 3
+           rotation(count) = ZERO
+        endif          
+        if(my_pe == pe_west2) then
+           count = count + 1
+           is(count) = is_west2; ie(count) = ie_west2
+           js(count) = js_west2; je(count) = je_west2
+           dir(count) = 3
+           rotation(count) = ZERO
+        endif          
 
         if(count >0) then
            nsend = nsend + 1
-           if(nsend > nlist_send) call mpp_error(FATAL, "set_bound_overlap: nsend > nlist_send")
+           if(nsend > nlist_send)  call mpp_error(FATAL, "set_bound_overlap: nsend > nlist_send")
            bound%send(nsend)%count = count
-           bound%send(nsend)%pe    = domain%list(m)%pe
+           bound%send(nsend)%pe    = my_pe
            allocate(bound%send(nsend)%is(count),  bound%send(nsend)%ie(count) )
            allocate(bound%send(nsend)%js(count),  bound%send(nsend)%je(count) )
            allocate(bound%send(nsend)%dir(count), bound%send(nsend)%rotation(count) )
@@ -6628,12 +6465,16 @@ subroutine set_bound_overlap( domain, position )
            bound%send(nsend)%je(:)       = je(1:count)
            bound%send(nsend)%dir(:)      = dir(1:count)
            bound%send(nsend)%tileMe(:)   = 1
-           bound%send(nsend)%rotation(:) = ZERO           
+           bound%send(nsend)%rotation(:) = rotation(1:count)     
+!write(1000+mpp_pe(),*) "send:", count, my_pe
+!do i = 1, count
+!   write(1000+mpp_pe(),*) "send index:", is(i), ie(i), js(i), je(i), dir(i), rotation(i)
+!enddo
         endif
      enddo
   else 
      !--- The following did not consider wide halo case.
-     do m = 1, nlist_send
+     do m = 1, update%nsend
         overlap => update%send(m)
         if( overlap%count == 0 ) cycle
         count = 0
@@ -6738,98 +6579,162 @@ subroutine set_bound_overlap( domain, position )
   nrecv       = 0
 
   !--- will computing overlap for tripolar grid.
-  if( BTEST(domain%fold,NORTH) ) then
-     tMe = 1
-     !--- currently only set up for west and south boundary.
-     !---south boundary
+  if( domain%ntiles == 1 ) then
+     ! currently only set up for west and south boundary
+
+     ! south boundary for recv
+     pe_south1 = NULL_PE; pe_south2 = NULL_PE
      if( position == NORTH .OR. position == CORNER ) then
-        isc1 = domain%x(1)%compute%begin; iec1 = domain%x(1)%compute%end +ishift
-        jsc1 = domain%y(1)%compute%begin; jec1 = jsc1
-     endif
-     !--- west boundary
-     if( position == EAST .OR. position == CORNER ) then
-        isc2 = domain%x(1)%compute%begin; iec2 = isc2
-        jsc2 = domain%y(1)%compute%begin; jec2 = domain%y(1)%compute%end + jshift
+        inbr = ipos; jnbr = jpos - 1
+        if( jnbr == -1 .AND. y_cyclic) jnbr = npes_y-1
+        if( inbr .GE. 0 .AND. inbr .LT. npes_x .AND. jnbr .GE. 0 .AND. jnbr .LT. npes_y ) then 
+           pe_south1 = domain%pearray(inbr,jnbr)
+           is_south1 = isc + ishift; ie_south1 = iec+ishift
+           js_south1 = jsc;          je_south1 = js_south1
+        endif         
      endif
 
-     do list = 0,nlist-1
-        m = mod( domain%pos+nlist-list, nlist )
-        count = 0
+     !--- south boudary for recv:  the southwest point when position is NORTH
+     if( position == CORNER ) then
+        inbr = ipos - 1; jnbr = jpos - 1
+        if( inbr == -1 .AND. x_cyclic) inbr = npes_x-1
+        if( jnbr == -1 .AND. y_cyclic) jnbr = npes_y-1
+        if( inbr .GE. 0 .AND. inbr .LT. npes_x .AND. jnbr .GE. 0 .AND. jnbr .LT. npes_y ) then 
+           pe_south2 = domain%pearray(inbr,jnbr)
+           is_south2 = isc; ie_south2 = is_south2
+           js_south2 = jsc; je_south2 = js_south2
+        endif         
+     endif
 
-        !--- south boundary
-        if( position == NORTH .OR. position == CORNER ) then
-           isd = domain%list(m)%x(1)%compute%begin;      ied = domain%list(m)%x(1)%compute%end + ishift
-           jsd = domain%list(m)%y(1)%compute%end+jshift; jed = jsd
-           if( isc1 == isd .AND. iec1 == ied .AND. jsc1 == jsd) then
-              count = count + 1          
-              is(count) = isc1; ie(count) = iec1
-              js(count) = jsc1; je(count) = jec1
-              dir(count) = 2
-              nrecvl(tMe, 2) = nrecvl(tMe,2) + 1
-              islMe(tMe,2,nrecvl(tMe, 2)) = is(count)
-              ielMe(tMe,2,nrecvl(tMe, 2)) = ie(count)
-              jslMe(tMe,2,nrecvl(tMe, 2)) = js(count)
-              jelMe(tMe,2,nrecvl(tMe, 2)) = je(count)
-              isl  (tMe,2,nrecvl(tMe, 2)) = is(count)
-              iel  (tMe,2,nrecvl(tMe, 2)) = ie(count)
-              jsl  (tMe,2,nrecvl(tMe, 2)) = js(count)
-              jel  (tMe,2,nrecvl(tMe, 2)) = je(count)
 
+     !---west boundary for recv
+     pe_west0 = NULL_PE; pe_west1 = NULL_PE; pe_west2 = NULL_PE
+     if( position == EAST ) then
+        inbr = ipos-1; jnbr = jpos
+        if( inbr == -1 .AND. x_cyclic) inbr = npes_x-1
+        if( inbr .GE. 0 .AND. inbr .LT. npes_x .AND. jnbr .GE. 0 .AND. jnbr .LT. npes_y ) then 
+           pe_west1 = domain%pearray(inbr,jnbr)
+           is_west1 = isc;          ie_west1 = is_west1
+           js_west1 = jsc + jshift; je_west1 = jec + jshift
+        endif         
+     else if ( position == CORNER ) then  ! possible split into two parts.
+        !--- on the fold.
+        if( folded_north .AND. jec == jeg .AND. ipos .GT. npes_x/2 ) then
+           inbr = npes_x - ipos - 1; jnbr = jpos
+           if( inbr .GE. 0 .AND. inbr .LT. npes_x .AND. jnbr .GE. 0 .AND. jnbr .LT. npes_y ) then 
+               pe_west0 = domain%pearray(inbr,jnbr)
+               is_west0 = isc;        ie_west0 = is_west0
+               js_west0 = jec+jshift; je_west0 = js_west0
            endif
-        endif
-
-        !--- west boundary
-        if( position == EAST .OR. position == CORNER ) then
-           isd = domain%list(m)%x(1)%compute%end+ishift; ied = isd
-           jsd = domain%list(m)%y(1)%compute%begin;      jed = domain%list(m)%y(1)%compute%end + jshift
-           !--- cyclic in x-direction is assumed for folded-north
-           if(ied == ieg) then
-              isd = isg; ied = isg
+           inbr = ipos-1; jnbr = jpos
+           if( inbr == -1 .AND. x_cyclic) inbr = npes_x-1
+           if( inbr .GE. 0 .AND. inbr .LT. npes_x .AND. jnbr .GE. 0 .AND. jnbr .LT. npes_y ) then 
+              pe_west1 = domain%pearray(inbr,jnbr)
+              is_west1 = isc;          ie_west1 = is_west1
+              js_west1 = jsc + jshift; je_west1 = jec
            endif
-           if( isc2 == isd .AND. jsc2 == jsd .AND. jec2 == jed ) then
-              count = count + 1          
-              is(count) = isc2; ie(count) = iec2
-              js(count) = jsc2; je(count) = jec2
-              dir(count) = 3
-              nrecvl(tMe, 3) = nrecvl(tMe,3) + 1
-              islMe(tMe,3,nrecvl(tMe, 3)) = is(count)
-              ielMe(tMe,3,nrecvl(tMe, 3)) = ie(count)
-              jslMe(tMe,3,nrecvl(tMe, 3)) = js(count)
-              jelMe(tMe,3,nrecvl(tMe, 3)) = je(count)
-              isl  (tMe,3,nrecvl(tMe, 3)) = is(count)
-              iel  (tMe,3,nrecvl(tMe, 3)) = ie(count)
-              jsl  (tMe,3,nrecvl(tMe, 3)) = js(count)
-              jel  (tMe,3,nrecvl(tMe, 3)) = je(count)
+        else
+           inbr = ipos-1; jnbr = jpos
+           if( inbr == -1 .AND. x_cyclic) inbr = npes_x-1
+           if( inbr .GE. 0 .AND. inbr .LT. npes_x .AND. jnbr .GE. 0 .AND. jnbr .LT. npes_y ) then 
+              pe_west1 = domain%pearray(inbr,jnbr)
+              is_west1 = isc;          ie_west1 = is_west1
+              js_west1 = jsc + jshift; je_west1 = jec+jshift
            endif
         endif
+     endif 
+
+     !--- west boundary for recv: the southwest point when position is CORNER
+     if( position == CORNER ) then
+        inbr = ipos - 1; jnbr = jpos - 1
+        if( inbr == -1 .AND. x_cyclic) inbr = npes_x - 1
+        if( jnbr == -1 .AND. y_cyclic) jnbr = npes_y - 1
+        if( inbr .GE. 0 .AND. inbr .LT. npes_x .AND. jnbr .GE. 0 .AND. jnbr .LT. npes_y ) then 
+           pe_west2 = domain%pearray(inbr,jnbr)
+           is_west2 = isc; ie_west2 = is_west2
+           js_west2 = jsc; je_west2 = js_west2
+        endif         
+     endif
+
+!write(1000+mpp_pe(),*)"recv south 1", pe_south1, is_south1, ie_south1, js_south1, je_south1
+!write(1000+mpp_pe(),*)"recv south 2", pe_south2, is_south2, ie_south2, js_south2, je_south2
+!write(1000+mpp_pe(),*)"recv west 0", pe_west0, is_west0, ie_west0, js_west0, je_west0
+!write(1000+mpp_pe(),*)"recv west 1", pe_west1, is_west1, ie_west1, js_west1, je_west1
+!write(1000+mpp_pe(),*)"recv west 2", pe_west2, is_west2, ie_west2, js_west2, je_west2
+     
+     tMe = 1
+     do list = 0,nlist-1
+        m = mod( domain%pos+nlist-list, nlist )
+        count = 0
+        my_pe = domain%list(m)%pe
+        if(my_pe == pe_south1) then
+           count = count + 1
+           is(count) = is_south1; ie(count) = ie_south1
+           js(count) = js_south1; je(count) = je_south1
+           dir(count) = 2
+           rotation(count) = ZERO
+           index(count) = 1 + ishift
+        endif          
+        if(my_pe == pe_south2) then
+           count = count + 1
+           is(count) = is_south2; ie(count) = ie_south2
+           js(count) = js_south2; je(count) = je_south2
+           dir(count) = 2
+           rotation(count) = ZERO
+           index(count) = 1
+        endif          
+        if(my_pe == pe_west0) then
+           count = count + 1
+           is(count) = is_west0; ie(count) = ie_west0
+           js(count) = js_west0; je(count) = je_west0
+           dir(count) = 3
+           rotation(count) = ONE_HUNDRED_EIGHTY
+           index(count) = jec-jsc+1+jshift
+        endif          
+        if(my_pe == pe_west1) then
+           count = count + 1
+           is(count) = is_west1; ie(count) = ie_west1
+           js(count) = js_west1; je(count) = je_west1
+           dir(count) = 3
+           rotation(count) = ZERO
+           index(count) = 1 + jshift
+        endif          
+        if(my_pe == pe_west2) then
+           count = count + 1
+           is(count) = is_west2; ie(count) = ie_west2
+           js(count) = js_west2; je(count) = je_west2
+           dir(count) = 3
+           rotation(count) = ZERO
+           index(count) = 1
+        endif       
 
         if(count >0) then
            nrecv = nrecv + 1
            if(nrecv > nlist_recv) call mpp_error(FATAL, "set_bound_overlap: nrecv > nlist_recv")
            bound%recv(nrecv)%count = count
-           bound%recv(nrecv)%pe    = domain%list(m)%pe
-           allocate(bound%recv(nrecv)%isMe(count),   bound%recv(nrecv)%ieMe(count) )
-           allocate(bound%recv(nrecv)%jsMe(count),   bound%recv(nrecv)%jeMe(count) )
+           bound%recv(nrecv)%pe    = my_pe
            allocate(bound%recv(nrecv)%is(count),     bound%recv(nrecv)%ie(count) )
            allocate(bound%recv(nrecv)%js(count),     bound%recv(nrecv)%je(count) )
            allocate(bound%recv(nrecv)%dir(count),    bound%recv(nrecv)%index(count)  )
            allocate(bound%recv(nrecv)%tileMe(count), bound%recv(nrecv)%rotation(count) )
+           
            bound%recv(nrecv)%is(:)       = is(1:count)
            bound%recv(nrecv)%ie(:)       = ie(1:count)
            bound%recv(nrecv)%js(:)       = js(1:count)
            bound%recv(nrecv)%je(:)       = je(1:count)
-           bound%recv(nrecv)%isMe(:)     = is(1:count)
-           bound%recv(nrecv)%ieMe(:)     = ie(1:count)
-           bound%recv(nrecv)%jsMe(:)     = js(1:count)
-           bound%recv(nrecv)%jeMe(:)     = je(1:count)
-
            bound%recv(nrecv)%dir(:)      = dir(1:count)
            bound%recv(nrecv)%tileMe(:)   = 1
-           bound%recv(nrecv)%rotation(:) = ZERO           
+           bound%recv(nrecv)%rotation(:) = rotation(1:count)    
+           bound%recv(nrecv)%index(:)        = index(1:count)
+!write(1000+mpp_pe(),*) "recv:", count, my_pe
+!do i = 1, count
+!   write(1000+mpp_pe(),*) "recv index:", is(i), ie(i), js(i), je(i), dir(i), rotation(i)
+!enddo
+
         endif
      enddo
   else 
-     do m = 1, nlist_recv
+     do m = 1, update%nrecv
         overlap => update%recv(m)
         if( overlap%count == 0 ) cycle
         count = 0
@@ -6840,36 +6745,13 @@ subroutine set_bound_overlap( domain, position )
               count=count+1
               dir(count) = 1
               rotation(count) = overlap%rotation(n)
-              isMe(count)     = overlap%isMe(n) - 1
-              ieMe(count) = isMe(count)
-              jsMe(count)     = overlap%jsMe(n)
-              jeMe(count)     = overlap%jeMe(n)
               tileMe(count)   = overlap%tileMe(n)
-              if( overlap%is_refined(n)) then
-                 select case( rotation(count) ) 
-                 case( ZERO ) ! W -> E
-                    is(count) = overlap%is(n) - 1
-                    ie(count) = is(count)
-                    js(count) = overlap%js(n)
-                    je(count) = overlap%je(n)
-                 case( NINETY ) ! S -> E
-                    is(count) = overlap%is(n)
-                    ie(count) = overlap%ie(n)
-                    js(count) = overlap%js(n) - 1
-                    je(count) = js(count)
-                 end select
-              else
-                 is(count) = overlap%is(n) - 1
-                 ie(count) = is(count)
-                 js(count) = overlap%js(n)
-                 je(count) = overlap%je(n)
-              end if
+              is(count) = overlap%is(n) - 1
+              ie(count) = is(count)
+              js(count) = overlap%js(n)
+              je(count) = overlap%je(n)
               tMe                        = tileMe(count)
               nrecvl(tMe, 1) = nrecvl(tMe,1) + 1
-              islMe(tMe,1,nrecvl(tMe, 1)) = isMe(count)
-              ielMe(tMe,1,nrecvl(tMe, 1)) = ieMe(count)
-              jslMe(tMe,1,nrecvl(tMe, 1)) = jsMe(count)
-              jelMe(tMe,1,nrecvl(tMe, 1)) = jeMe(count)
               isl  (tMe,1,nrecvl(tMe, 1)) = is  (count)
               iel  (tMe,1,nrecvl(tMe, 1)) = ie  (count)
               jsl  (tMe,1,nrecvl(tMe, 1)) = js  (count)
@@ -6880,36 +6762,13 @@ subroutine set_bound_overlap( domain, position )
               count=count+1
               dir(count) = 2
               rotation(count) = overlap%rotation(n)
-              isMe(count)     = overlap%isMe(n)
-              ieMe(count)     = overlap%ieMe(n)
-              jsMe(count)     = overlap%jeMe(n) + 1
-              jeMe(count) = jsMe(count)
               tileMe(count)   = overlap%tileMe(n)
-              if( overlap%is_refined(n)) then
-                 select case( rotation(count) ) 
-                 case( ZERO ) ! N->S
-                    is(count) = overlap%is(n)
-                    ie(count) = overlap%ie(n)
-                    js(count) = overlap%je(n) + 1
-                    je(count) = js(count)
-                 case( MINUS_NINETY ) ! E->S
-                    is(count) = overlap%ie(n) + 1
-                    ie(count) = is(count)
-                    js(count) = overlap%js(n)
-                    je(count) = overlap%je(n)
-                 end select
-              else
-                 is(count) = overlap%is(n)
-                 ie(count) = overlap%ie(n)
-                 js(count) = overlap%je(n) + 1
-                 je(count) = js(count)
-              end if
+              is(count) = overlap%is(n)
+              ie(count) = overlap%ie(n)
+              js(count) = overlap%je(n) + 1
+              je(count) = js(count)
               tMe                        = tileMe(count)
               nrecvl(tMe, 2) = nrecvl(tMe,2) + 1
-              islMe(tMe,2,nrecvl(tMe, 2)) = isMe(count)
-              ielMe(tMe,2,nrecvl(tMe, 2)) = ieMe(count)
-              jslMe(tMe,2,nrecvl(tMe, 2)) = jsMe(count)
-              jelMe(tMe,2,nrecvl(tMe, 2)) = jeMe(count)
               isl  (tMe,2,nrecvl(tMe, 2)) = is  (count)
               iel  (tMe,2,nrecvl(tMe, 2)) = ie  (count)
               jsl  (tMe,2,nrecvl(tMe, 2)) = js  (count)
@@ -6920,36 +6779,13 @@ subroutine set_bound_overlap( domain, position )
               count=count+1
               dir(count) = 3
               rotation(count) = overlap%rotation(n)
-              isMe(count)     = overlap%ieMe(n) + 1 
-              ieMe(count) = isMe(count)
-              jsMe(count)     = overlap%jsMe(n)
-              jeMe(count)     = overlap%jeMe(n)
               tileMe(count)   = overlap%tileMe(n)
-              if( overlap%is_refined(n)) then
-                 select case( rotation(count) ) 
-                 case( ZERO ) ! E->W
-                    is(count) = overlap%ie(n) + 1
-                    ie(count) = is(count)
-                    js(count) = overlap%js(n)
-                    je(count) = overlap%je(n)
-                 case( NINETY ) ! S -> E
-                    is(count) = overlap%is(n)
-                    ie(count) = overlap%ie(n)
-                    js(count) = overlap%je(n) + 1
-                    je(count) = js(count)
-                 end select
-              else
-                 is(count) = overlap%ie(n) + 1
-                 ie(count) = is(count)
-                 js(count) = overlap%js(n)
-                 je(count) = overlap%je(n)
-              end if
+              is(count) = overlap%ie(n) + 1
+              ie(count) = is(count)
+              js(count) = overlap%js(n)
+              je(count) = overlap%je(n)
               tMe                        = tileMe(count)
               nrecvl(tMe, 3) = nrecvl(tMe,3) + 1
-              islMe(tMe,3,nrecvl(tMe, 3)) = isMe(count)
-              ielMe(tMe,3,nrecvl(tMe, 3)) = ieMe(count)
-              jslMe(tMe,3,nrecvl(tMe, 3)) = jsMe(count)
-              jelMe(tMe,3,nrecvl(tMe, 3)) = jeMe(count)
               isl  (tMe,3,nrecvl(tMe, 3)) = is  (count)
               iel  (tMe,3,nrecvl(tMe, 3)) = ie  (count)
               jsl  (tMe,3,nrecvl(tMe, 3)) = js  (count)
@@ -6960,36 +6796,13 @@ subroutine set_bound_overlap( domain, position )
               count=count+1
               dir(count) = 4
               rotation(count) = overlap%rotation(n)
-              isMe(count)     = overlap%isMe(n)
-              ieMe(count)     = overlap%ieMe(n)
-              jsMe(count)     = overlap%jsMe(n) - 1
-              jeMe(count) = jsMe(count)
               tileMe(count) = overlap%tileMe(n)
-              if( overlap%is_refined(n)) then
-                 select case( rotation(count) ) 
-                 case( ZERO ) ! S->N
-                    is(count) = overlap%is(n)
-                    ie(count) = overlap%ie(n)
-                    js(count) = overlap%js(n) - 1
-                    je(count) = js(count)
-                 case( MINUS_NINETY ) ! W->N
-                    is(count) = overlap%is(n) - 1
-                    ie(count) = is(count)
-                    js(count) = overlap%js(n)
-                    je(count) = overlap%je(n)
-                 end select
-              else
-                 is(count) = overlap%is(n)
-                 ie(count) = overlap%ie(n)
-                 js(count) = overlap%js(n) - 1
-                 je(count) = js(count)
-              end if
+              is(count) = overlap%is(n)
+              ie(count) = overlap%ie(n)
+              js(count) = overlap%js(n) - 1
+              je(count) = js(count)
               tMe                        = tileMe(count)
               nrecvl(tMe, 4) = nrecvl(tMe,4) + 1
-              islMe(tMe,4,nrecvl(tMe, 4)) = isMe(count)
-              ielMe(tMe,4,nrecvl(tMe, 4)) = ieMe(count)
-              jslMe(tMe,4,nrecvl(tMe, 4)) = jsMe(count)
-              jelMe(tMe,4,nrecvl(tMe, 4)) = jeMe(count)
               isl  (tMe,4,nrecvl(tMe, 4)) = is  (count)
               iel  (tMe,4,nrecvl(tMe, 4)) = ie  (count)
               jsl  (tMe,4,nrecvl(tMe, 4)) = js  (count)
@@ -7000,16 +6813,10 @@ subroutine set_bound_overlap( domain, position )
            nrecv = nrecv + 1
            bound%recv(nrecv)%count = count
            bound%recv(nrecv)%pe    = overlap%pe
-           allocate(bound%recv(nrecv)%isMe(count),   bound%recv(nrecv)%ieMe(count) )
-           allocate(bound%recv(nrecv)%jsMe(count),   bound%recv(nrecv)%jeMe(count) )
            allocate(bound%recv(nrecv)%is(count),     bound%recv(nrecv)%ie(count) )
            allocate(bound%recv(nrecv)%js(count),     bound%recv(nrecv)%je(count) )
            allocate(bound%recv(nrecv)%dir(count),    bound%recv(nrecv)%index(count)  )
            allocate(bound%recv(nrecv)%tileMe(count), bound%recv(nrecv)%rotation(count) )
-           bound%recv(nrecv)%isMe(:)     = isMe(1:count)
-           bound%recv(nrecv)%ieMe(:)     = ieMe(1:count)
-           bound%recv(nrecv)%jsMe(:)     = jsMe(1:count)
-           bound%recv(nrecv)%jeMe(:)     = jeMe(1:count)
            bound%recv(nrecv)%is(:)       = is(1:count)
            bound%recv(nrecv)%ie(:)       = ie(1:count)
            bound%recv(nrecv)%js(:)       = js(1:count)
@@ -7019,33 +6826,38 @@ subroutine set_bound_overlap( domain, position )
            bound%recv(nrecv)%rotation(:) = rotation(1:count)
         end if
      end do ! end  do list = 0, nlist
-  endif
-  bound%nrecv = nrecv
-
-  !--- find the boundary index for each contact within the east boundary
-  do m = 1, nrecv
-     do n = 1, bound%recv(m)%count
-        tMe = bound%recv(m)%tileMe(n)
-        dr = bound%recv(m)%dir(n)
-        bound%recv(m)%index(n) = 1
-        do l = 1, nrecvl(tMe,dr)
-           if(dr == 1 .OR. dr == 3) then  ! EAST, WEST
-              if( bound%recv(m)%jsMe(n) > jslMe(tMe, dr, l) ) then
-                 bound%recv(m)%index(n) = bound%recv(m)%index(n)                  + &
-                      max(abs(jel(tMe, dr, l)-jsl(tMe, dr, l)), &
-                      abs(iel(tMe, dr, l)-isl(tMe, dr, l))) + 1 - jshift
-              end if
-           else                             ! South, North
-              if( bound%recv(m)%isMe(n) > islMe(tMe, dr, l) ) then
-                 bound%recv(m)%index(n) = bound%recv(m)%index(n)                  + &
-                      max(abs(jel(tMe, dr, l)-jsl(tMe, dr, l)), &
-                      abs(iel(tMe, dr, l)-isl(tMe, dr, l))) + 1 - ishift
+     !--- find the boundary index for each contact within the east boundary
+     do m = 1, nrecv
+        do n = 1, bound%recv(m)%count
+           tMe = bound%recv(m)%tileMe(n)
+           dr = bound%recv(m)%dir(n)
+           bound%recv(m)%index(n) = 1
+           do l = 1, nrecvl(tMe,dr)
+              if(dr == 1 .OR. dr == 3) then  ! EAST, WEST
+                 if( bound%recv(m)%js(n) > jsl(tMe, dr, l) ) then
+                    if( bound%recv(m)%rotation(n) == ONE_HUNDRED_EIGHTY ) then
+                       bound%recv(m)%index(n) = bound%recv(m)%index(n)                  + &
+                            max(abs(jel(tMe, dr, l)-jsl(tMe, dr, l))+1, &
+                            abs(iel(tMe, dr, l)-isl(tMe, dr, l))+1)
+                    else
+                       bound%recv(m)%index(n) = bound%recv(m)%index(n)                  + &
+                            max(abs(jel(tMe, dr, l)-jsl(tMe, dr, l)), &
+                            abs(iel(tMe, dr, l)-isl(tMe, dr, l))) + 1 - jshift
+                    endif
+                 end if
+              else                             ! South, North
+                 if( bound%recv(m)%is(n) > isl(tMe, dr, l) ) then
+                    bound%recv(m)%index(n) = bound%recv(m)%index(n)                  + &
+                         max(abs(jel(tMe, dr, l)-jsl(tMe, dr, l)), &
+                         abs(iel(tMe, dr, l)-isl(tMe, dr, l))) + 1 - ishift
+                 end if
               end if
-           end if
+           end do
         end do
      end do
-  end do
 
+  endif
+  bound%nrecv = nrecv
 
 
 end subroutine set_bound_overlap
@@ -7908,15 +7720,10 @@ subroutine allocate_update_overlap( overlap, count)
   allocate(overlap%tileMe      (count), overlap%tileNbr (count) )
   allocate(overlap%is          (count), overlap%ie      (count) )
   allocate(overlap%js          (count), overlap%je      (count) )
-  allocate(overlap%isMe        (count), overlap%ieMe    (count) )
-  allocate(overlap%jsMe        (count), overlap%jeMe    (count) )
   allocate(overlap%dir         (count), overlap%rotation(count) )
-  allocate(overlap%is_refined  (count), overlap%index   (count) )
   allocate(overlap%from_contact(count), overlap%msgsize (count) )
-  overlap%is_refined   = .FALSE.
   overlap%rotation     =  ZERO
   overlap%from_contact = .FALSE.
-  overlap%index        = -1   
 
 end subroutine allocate_update_overlap
 
@@ -7949,7 +7756,7 @@ end subroutine allocate_update_overlap
     endif
 
     if(is_overlapped) then
-       if( overlap%pe == NULL_PE ) then
+     if( overlap%count == 0 ) then
           overlap%pe = pe
        else
           if(overlap%pe .NE. pe) call mpp_error(FATAL,  &
@@ -7976,25 +7783,25 @@ end subroutine allocate_update_overlap
   end subroutine insert_update_overlap
 
   !#####################################################################################
-  subroutine insert_mosaic_update_overlap(overlap, pe, tileMe, tileNbr, is, ie, js, je, dir, &
-                                          rotation, from_contact, refined)
+subroutine insert_overlap_type(overlap, pe, tileMe, tileNbr, is, ie, js, je, dir, &
+     rotation, from_contact)
     type(overlap_type), intent(inout) :: overlap
     integer,            intent(in   ) :: tileMe, tileNbr, pe
     integer,            intent(in   ) :: is, ie, js, je
     integer,            intent(in   ) :: dir, rotation
-    logical,            intent(in   ) :: from_contact, refined
+  logical,            intent(in   ) :: from_contact
     integer                           :: count
 
-    if( overlap%pe == NULL_PE ) then
+  if( overlap%count == 0 ) then
        overlap%pe = pe
     else
        if(overlap%pe .NE. pe) call mpp_error(FATAL,  &
-            "mpp_domains_define.inc(insert_mosaic_update_overlap): mismatch on pe")
+          "mpp_domains_define.inc(insert_overlap_type): mismatch on pe")
     endif
     overlap%count = overlap%count+1
     count = overlap%count
     if(count > MAXOVERLAP) call mpp_error(FATAL, &
-         "mpp_domains_define.inc(insert_mosaic_update_overlap): number of overlap is greater than MAXOVERLAP, increase MAXOVERLAP")
+       "mpp_domains_define.inc(insert_overlap_type): number of overlap is greater than MAXOVERLAP, increase MAXOVERLAP")
     overlap%tileMe      (count) = tileMe
     overlap%tileNbr     (count) = tileNbr
     overlap%is          (count) = is
@@ -8004,10 +7811,10 @@ end subroutine allocate_update_overlap
     overlap%dir         (count) = dir
     overlap%rotation    (count) = rotation
     overlap%from_contact(count) = from_contact
-    overlap%is_refined  (count) = refined
     overlap%msgsize     (count) = (ie-is+1)*(je-js+1)
 
-  end subroutine insert_mosaic_update_overlap
+end subroutine insert_overlap_type
+
 
 !#######################################################################
 subroutine deallocate_overlap_type( overlap)
@@ -8025,14 +7832,8 @@ subroutine deallocate_overlap_type( overlap)
   if(ASSOCIATED(overlap%ie)) deallocate(overlap%ie)
   if(ASSOCIATED(overlap%js)) deallocate(overlap%js)
   if(ASSOCIATED(overlap%je)) deallocate(overlap%je)
-  if(ASSOCIATED(overlap%isMe)) deallocate(overlap%isMe)
-  if(ASSOCIATED(overlap%ieMe)) deallocate(overlap%ieMe)
-  if(ASSOCIATED(overlap%jsMe)) deallocate(overlap%jsMe)
-  if(ASSOCIATED(overlap%jeMe)) deallocate(overlap%jeMe)
   if(ASSOCIATED(overlap%dir)) deallocate(overlap%dir)
   if(ASSOCIATED(overlap%rotation)) deallocate(overlap%rotation)
-  if(ASSOCIATED(overlap%is_refined)) deallocate(overlap%is_refined)
-  if(ASSOCIATED(overlap%index)) deallocate(overlap%index)
   if(ASSOCIATED(overlap%from_contact)) deallocate(overlap%from_contact)
   if(ASSOCIATED(overlap%msgsize)) deallocate(overlap%msgsize)
   overlap%count = 0
@@ -8091,14 +7892,8 @@ subroutine add_update_overlap( overlap_out, overlap_in)
      overlap%ie          (1:count_out) = overlap_out%ie          (1:count_out)
      overlap%js          (1:count_out) = overlap_out%js          (1:count_out)
      overlap%je          (1:count_out) = overlap_out%je          (1:count_out)
-     overlap%isMe        (1:count_out) = overlap_out%isMe        (1:count_out)
-     overlap%ieMe        (1:count_out) = overlap_out%ieMe        (1:count_out)
-     overlap%jsMe        (1:count_out) = overlap_out%jsMe        (1:count_out)
-     overlap%jeMe        (1:count_out) = overlap_out%jeMe        (1:count_out)
      overlap%dir         (1:count_out) = overlap_out%dir         (1:count_out)
      overlap%rotation    (1:count_out) = overlap_out%rotation    (1:count_out)
-     overlap%is_refined  (1:count_out) = overlap_out%is_refined  (1:count_out)
-     overlap%index       (1:count_out) = overlap_out%index       (1:count_out)
      overlap%from_contact(1:count_out) = overlap_out%from_contact(1:count_out)
      call deallocate_overlap_type(overlap_out)
      call allocate_update_overlap(overlap_out, count)
@@ -8108,13 +7903,8 @@ subroutine add_update_overlap( overlap_out, overlap_in)
      overlap_out%ie          (1:count_out) = overlap%ie          (1:count_out)
      overlap_out%js          (1:count_out) = overlap%js          (1:count_out)
      overlap_out%je          (1:count_out) = overlap%je          (1:count_out)
-     overlap_out%isMe        (1:count_out) = overlap%isMe        (1:count_out)
-     overlap_out%ieMe        (1:count_out) = overlap%ieMe        (1:count_out)
-     overlap_out%jsMe        (1:count_out) = overlap%jsMe        (1:count_out)
-     overlap_out%jeMe        (1:count_out) = overlap%jeMe        (1:count_out)
      overlap_out%dir         (1:count_out) = overlap%dir         (1:count_out)
      overlap_out%rotation    (1:count_out) = overlap%rotation    (1:count_out)
-     overlap_out%is_refined  (1:count_out) = overlap%is_refined  (1:count_out)
      overlap_out%index       (1:count_out) = overlap%index       (1:count_out)
      overlap_out%from_contact(1:count_out) = overlap%from_contact(1:count_out)        
      overlap_out%msgsize     (1:count_out) = overlap%msgsize     (1:count_out)
@@ -8127,14 +7917,8 @@ subroutine add_update_overlap( overlap_out, overlap_in)
   overlap_out%ie          (count_out+1:count) = overlap_in%ie          (1:count_in)
   overlap_out%js          (count_out+1:count) = overlap_in%js          (1:count_in)
   overlap_out%je          (count_out+1:count) = overlap_in%je          (1:count_in)
-  overlap_out%isMe        (count_out+1:count) = overlap_in%isMe        (1:count_in)
-  overlap_out%ieMe        (count_out+1:count) = overlap_in%ieMe        (1:count_in)
-  overlap_out%jsMe        (count_out+1:count) = overlap_in%jsMe        (1:count_in)
-  overlap_out%jeMe        (count_out+1:count) = overlap_in%jeMe        (1:count_in)
   overlap_out%dir         (count_out+1:count) = overlap_in%dir         (1:count_in)
   overlap_out%rotation    (count_out+1:count) = overlap_in%rotation    (1:count_in)
-  overlap_out%is_refined  (count_out+1:count) = overlap_in%is_refined  (1:count_in)
-  overlap_out%index       (count_out+1:count) = overlap_in%index       (1:count_in)
   overlap_out%from_contact(count_out+1:count) = overlap_in%from_contact(1:count_in)
 
   do n = count_out+1, count
diff --git a/src/shared/mpp/include/mpp_domains_misc.inc b/src/shared/mpp/include/mpp_domains_misc.inc
index 77267e3131..5e3121919d 100644
--- a/src/shared/mpp/include/mpp_domains_misc.inc
+++ b/src/shared/mpp/include/mpp_domains_misc.inc
@@ -117,6 +117,16 @@
           nest_recv_clock     = mpp_clock_id( 'nest recv' )
           nest_unpk_clock     = mpp_clock_id( 'nest unpk' )
           nest_wait_clock     = mpp_clock_id( 'nest wait' )
+          group_pack_clock     = mpp_clock_id( 'group pack' )
+          group_send_clock     = mpp_clock_id( 'group send' )
+          group_recv_clock     = mpp_clock_id( 'group recv' )
+          group_unpk_clock     = mpp_clock_id( 'group unpk' )
+          group_wait_clock     = mpp_clock_id( 'group wait' )
+          nonblock_group_pack_clock     = mpp_clock_id( 'nonblock group pack' )
+          nonblock_group_send_clock     = mpp_clock_id( 'nonblock group send' )
+          nonblock_group_recv_clock     = mpp_clock_id( 'nonblock group recv' )
+          nonblock_group_unpk_clock     = mpp_clock_id( 'nonblock group unpk' )
+          nonblock_group_wait_clock     = mpp_clock_id( 'nonblock group wait' )
       end if
       return
     end subroutine mpp_domains_init
@@ -1731,3 +1741,75 @@ end subroutine init_nonblock_type
 #define MPP_DO_GET_BOUNDARY_3D_V_ mpp_do_get_boundary_r4_3dv
 #include 
 #endif
+
+#undef MPP_TYPE_
+#define MPP_TYPE_ real(DOUBLE_KIND)
+#undef MPI_TYPE_
+#define MPI_TYPE_ MPI_REAL8
+#undef MPP_CREATE_GROUP_UPDATE_2D_
+#define MPP_CREATE_GROUP_UPDATE_2D_ mpp_create_group_update_r8_2d
+#undef MPP_CREATE_GROUP_UPDATE_3D_
+#define MPP_CREATE_GROUP_UPDATE_3D_ mpp_create_group_update_r8_3d
+#undef MPP_CREATE_GROUP_UPDATE_4D_
+#define MPP_CREATE_GROUP_UPDATE_4D_ mpp_create_group_update_r8_4d
+#undef MPP_CREATE_GROUP_UPDATE_2D_V_
+#define MPP_CREATE_GROUP_UPDATE_2D_V_ mpp_create_group_update_r8_2dv
+#undef MPP_CREATE_GROUP_UPDATE_3D_V_
+#define MPP_CREATE_GROUP_UPDATE_3D_V_ mpp_create_group_update_r8_3dv
+#undef MPP_CREATE_GROUP_UPDATE_4D_V_
+#define MPP_CREATE_GROUP_UPDATE_4D_V_ mpp_create_group_update_r8_4dv
+#undef MPP_DO_GROUP_UPDATE_
+#define MPP_DO_GROUP_UPDATE_ mpp_do_group_update_r8
+#undef MPP_START_GROUP_UPDATE_
+#define MPP_START_GROUP_UPDATE_ mpp_start_group_update_r8
+#undef MPP_COMPLETE_GROUP_UPDATE_
+#define MPP_COMPLETE_GROUP_UPDATE_ mpp_complete_group_update_r8
+#undef MPP_RESET_GROUP_UPDATE_FIELD_2D_
+#define MPP_RESET_GROUP_UPDATE_FIELD_2D_ mpp_reset_group_update_field_r8_2d
+#undef MPP_RESET_GROUP_UPDATE_FIELD_3D_
+#define MPP_RESET_GROUP_UPDATE_FIELD_3D_ mpp_reset_group_update_field_r8_3d
+#undef MPP_RESET_GROUP_UPDATE_FIELD_4D_
+#define MPP_RESET_GROUP_UPDATE_FIELD_4D_ mpp_reset_group_update_field_r8_4d
+#undef MPP_RESET_GROUP_UPDATE_FIELD_2D_V_
+#define MPP_RESET_GROUP_UPDATE_FIELD_2D_V_ mpp_reset_group_update_field_r8_2dv
+#undef MPP_RESET_GROUP_UPDATE_FIELD_3D_V_
+#define MPP_RESET_GROUP_UPDATE_FIELD_3D_V_ mpp_reset_group_update_field_r8_3dv
+#undef MPP_RESET_GROUP_UPDATE_FIELD_4D_V_
+#define MPP_RESET_GROUP_UPDATE_FIELD_4D_V_ mpp_reset_group_update_field_r8_4dv
+#include 
+
+#undef MPP_TYPE_
+#define MPP_TYPE_ real(FLOAT_KIND)
+#undef MPI_TYPE_
+#define MPI_TYPE_ MPI_REAL4
+#undef MPP_CREATE_GROUP_UPDATE_2D_
+#define MPP_CREATE_GROUP_UPDATE_2D_ mpp_create_group_update_r4_2d
+#undef MPP_CREATE_GROUP_UPDATE_3D_
+#define MPP_CREATE_GROUP_UPDATE_3D_ mpp_create_group_update_r4_3d
+#undef MPP_CREATE_GROUP_UPDATE_4D_
+#define MPP_CREATE_GROUP_UPDATE_4D_ mpp_create_group_update_r4_4d
+#undef MPP_CREATE_GROUP_UPDATE_2D_V_
+#define MPP_CREATE_GROUP_UPDATE_2D_V_ mpp_create_group_update_r4_2dv
+#undef MPP_CREATE_GROUP_UPDATE_3D_V_
+#define MPP_CREATE_GROUP_UPDATE_3D_V_ mpp_create_group_update_r4_3dv
+#undef MPP_CREATE_GROUP_UPDATE_4D_V_
+#define MPP_CREATE_GROUP_UPDATE_4D_V_ mpp_create_group_update_r4_4dv
+#undef MPP_DO_GROUP_UPDATE_
+#define MPP_DO_GROUP_UPDATE_ mpp_do_group_update_r4
+#undef MPP_START_GROUP_UPDATE_
+#define MPP_START_GROUP_UPDATE_ mpp_start_group_update_r4
+#undef MPP_COMPLETE_GROUP_UPDATE_
+#define MPP_COMPLETE_GROUP_UPDATE_ mpp_complete_group_update_r4
+#undef MPP_RESET_GROUP_UPDATE_FIELD_2D_
+#define MPP_RESET_GROUP_UPDATE_FIELD_2D_ mpp_reset_group_update_field_r4_2d
+#undef MPP_RESET_GROUP_UPDATE_FIELD_3D_
+#define MPP_RESET_GROUP_UPDATE_FIELD_3D_ mpp_reset_group_update_field_r4_3d
+#undef MPP_RESET_GROUP_UPDATE_FIELD_4D_
+#define MPP_RESET_GROUP_UPDATE_FIELD_4D_ mpp_reset_group_update_field_r4_4d
+#undef MPP_RESET_GROUP_UPDATE_FIELD_2D_V_
+#define MPP_RESET_GROUP_UPDATE_FIELD_2D_V_ mpp_reset_group_update_field_r4_2dv
+#undef MPP_RESET_GROUP_UPDATE_FIELD_3D_V_
+#define MPP_RESET_GROUP_UPDATE_FIELD_3D_V_ mpp_reset_group_update_field_r4_3dv
+#undef MPP_RESET_GROUP_UPDATE_FIELD_4D_V_
+#define MPP_RESET_GROUP_UPDATE_FIELD_4D_V_ mpp_reset_group_update_field_r4_4dv
+#include 
diff --git a/src/shared/mpp/include/mpp_domains_reduce.inc b/src/shared/mpp/include/mpp_domains_reduce.inc
index f268075764..673a9b1a99 100644
--- a/src/shared/mpp/include/mpp_domains_reduce.inc
+++ b/src/shared/mpp/include/mpp_domains_reduce.inc
@@ -1,6 +1,6 @@
 
 ! -*-f90-*-
-! $Id: mpp_domains_reduce.inc,v 20.0 2013/12/14 00:24:46 fms Exp $
+! $Id$
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 !                                                                             !
@@ -161,7 +161,7 @@
 !                   MPP_GLOBAL_SUM: global sum of field                       !
 !                                                                             !
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-
+#define DO_EFP_SUM_
 #undef MPP_GLOBAL_SUM_
 #define MPP_GLOBAL_SUM_ mpp_global_sum_r8_2d
 #undef MPP_EXTRA_INDICES_
@@ -194,40 +194,6 @@
 #define MPP_TYPE_ real(DOUBLE_KIND)
 #include 
 
-#ifdef OVERLOAD_C8
-#undef MPP_GLOBAL_SUM_
-#define MPP_GLOBAL_SUM_ mpp_global_sum_c8_2d
-#undef MPP_EXTRA_INDICES_
-#define MPP_EXTRA_INDICES_
-#undef MPP_TYPE_
-#define MPP_TYPE_ complex(DOUBLE_KIND)
-#include 
-
-#undef MPP_GLOBAL_SUM_
-#define MPP_GLOBAL_SUM_ mpp_global_sum_c8_3d
-#undef MPP_EXTRA_INDICES_
-#define MPP_EXTRA_INDICES_ ,:
-#undef MPP_TYPE_
-#define MPP_TYPE_ complex(DOUBLE_KIND)
-#include 
-
-#undef MPP_GLOBAL_SUM_
-#define MPP_GLOBAL_SUM_ mpp_global_sum_c8_4d
-#undef MPP_EXTRA_INDICES_
-#define MPP_EXTRA_INDICES_ ,:,:
-#undef MPP_TYPE_
-#define MPP_TYPE_ complex(DOUBLE_KIND)
-#include 
-
-#undef MPP_GLOBAL_SUM_
-#define MPP_GLOBAL_SUM_ mpp_global_sum_c8_5d
-#undef MPP_EXTRA_INDICES_
-#define MPP_EXTRA_INDICES_ ,:,:,:
-#undef MPP_TYPE_
-#define MPP_TYPE_ complex(DOUBLE_KIND)
-#include 
-#endif
-
 #ifdef OVERLOAD_R4
 #undef MPP_GLOBAL_SUM_
 #define MPP_GLOBAL_SUM_ mpp_global_sum_r4_2d
@@ -262,40 +228,7 @@
 #include 
 #endif
 
-#ifdef OVERLOAD_C4
-#undef MPP_GLOBAL_SUM_
-#define MPP_GLOBAL_SUM_ mpp_global_sum_c4_2d
-#undef MPP_EXTRA_INDICES_
-#define MPP_EXTRA_INDICES_
-#undef MPP_TYPE_
-#define MPP_TYPE_ complex(FLOAT_KIND)
-#include 
-
-#undef MPP_GLOBAL_SUM_
-#define MPP_GLOBAL_SUM_ mpp_global_sum_c4_3d
-#undef MPP_EXTRA_INDICES_
-#define MPP_EXTRA_INDICES_ ,:
-#undef MPP_TYPE_
-#define MPP_TYPE_ complex(FLOAT_KIND)
-#include 
-
-#undef MPP_GLOBAL_SUM_
-#define MPP_GLOBAL_SUM_ mpp_global_sum_c4_4d
-#undef MPP_EXTRA_INDICES_
-#define MPP_EXTRA_INDICES_ ,:,:
-#undef MPP_TYPE_
-#define MPP_TYPE_ complex(FLOAT_KIND)
-#include 
-
-#undef MPP_GLOBAL_SUM_
-#define MPP_GLOBAL_SUM_ mpp_global_sum_c4_5d
-#undef MPP_EXTRA_INDICES_
-#define MPP_EXTRA_INDICES_ ,:,:,:
-#undef MPP_TYPE_
-#define MPP_TYPE_ complex(FLOAT_KIND)
-#include 
-#endif
-
+#undef DO_EFP_SUM_
 #ifndef no_8byte_integers
 #undef MPP_GLOBAL_SUM_
 #define MPP_GLOBAL_SUM_ mpp_global_sum_i8_2d
@@ -361,7 +294,8 @@
 #undef MPP_TYPE_
 #define MPP_TYPE_ integer(INT_KIND)
 #include 
-    
+
+
 !gag
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 !                                                                             !
@@ -401,40 +335,6 @@
 #define MPP_TYPE_ real(DOUBLE_KIND)
 #include 
 
-#ifdef OVERLOAD_C8
-#undef MPP_GLOBAL_SUM_TL_
-#define MPP_GLOBAL_SUM_TL_ mpp_global_sum_tl_c8_2d
-#undef MPP_EXTRA_INDICES_
-#define MPP_EXTRA_INDICES_
-#undef MPP_TYPE_
-#define MPP_TYPE_ complex(DOUBLE_KIND)
-#include 
-
-#undef MPP_GLOBAL_SUM_TL_
-#define MPP_GLOBAL_SUM_TL_ mpp_global_sum_tl_c8_3d
-#undef MPP_EXTRA_INDICES_
-#define MPP_EXTRA_INDICES_ ,:
-#undef MPP_TYPE_
-#define MPP_TYPE_ complex(DOUBLE_KIND)
-#include 
-
-#undef MPP_GLOBAL_SUM_TL_
-#define MPP_GLOBAL_SUM_TL_ mpp_global_sum_tl_c8_4d
-#undef MPP_EXTRA_INDICES_
-#define MPP_EXTRA_INDICES_ ,:,:
-#undef MPP_TYPE_
-#define MPP_TYPE_ complex(DOUBLE_KIND)
-#include 
-
-#undef MPP_GLOBAL_SUM_TL_
-#define MPP_GLOBAL_SUM_TL_ mpp_global_sum_tl_c8_5d
-#undef MPP_EXTRA_INDICES_
-#define MPP_EXTRA_INDICES_ ,:,:,:
-#undef MPP_TYPE_
-#define MPP_TYPE_ complex(DOUBLE_KIND)
-#include 
-#endif
-
 #ifdef OVERLOAD_R4
 #undef MPP_GLOBAL_SUM_TL_
 #define MPP_GLOBAL_SUM_TL_ mpp_global_sum_tl_r4_2d
@@ -469,315 +369,6 @@
 #include 
 #endif
 
-#ifdef OVERLOAD_C4
-#undef MPP_GLOBAL_SUM_TL_
-#define MPP_GLOBAL_SUM_TL_ mpp_global_sum_tl_c4_2d
-#undef MPP_EXTRA_INDICES_
-#define MPP_EXTRA_INDICES_
-#undef MPP_TYPE_
-#define MPP_TYPE_ complex(FLOAT_KIND)
-#include 
-
-#undef MPP_GLOBAL_SUM_TL_
-#define MPP_GLOBAL_SUM_TL_ mpp_global_sum_tl_c4_3d
-#undef MPP_EXTRA_INDICES_
-#define MPP_EXTRA_INDICES_ ,:
-#undef MPP_TYPE_
-#define MPP_TYPE_ complex(FLOAT_KIND)
-#include 
-
-#undef MPP_GLOBAL_SUM_TL_
-#define MPP_GLOBAL_SUM_TL_ mpp_global_sum_tl_c4_4d
-#undef MPP_EXTRA_INDICES_
-#define MPP_EXTRA_INDICES_ ,:,:
-#undef MPP_TYPE_
-#define MPP_TYPE_ complex(FLOAT_KIND)
-#include 
-
-#undef MPP_GLOBAL_SUM_TL_
-#define MPP_GLOBAL_SUM_TL_ mpp_global_sum_tl_c4_5d
-#undef MPP_EXTRA_INDICES_
-#define MPP_EXTRA_INDICES_ ,:,:,:
-#undef MPP_TYPE_
-#define MPP_TYPE_ complex(FLOAT_KIND)
-#include 
-#endif
-
-#ifndef no_8byte_integers
-#undef MPP_GLOBAL_SUM_TL_
-#define MPP_GLOBAL_SUM_TL_ mpp_global_sum_tl_i8_2d
-#undef MPP_EXTRA_INDICES_
-#define MPP_EXTRA_INDICES_
-#undef MPP_TYPE_
-#define MPP_TYPE_ integer(LONG_KIND)
-#include 
-
-#undef MPP_GLOBAL_SUM_TL_
-#define MPP_GLOBAL_SUM_TL_ mpp_global_sum_tl_i8_3d
-#undef MPP_EXTRA_INDICES_
-#define MPP_EXTRA_INDICES_ ,:
-#undef MPP_TYPE_
-#define MPP_TYPE_ integer(LONG_KIND)
-#include 
-
-#undef MPP_GLOBAL_SUM_TL_
-#define MPP_GLOBAL_SUM_TL_ mpp_global_sum_tl_i8_4d
-#undef MPP_EXTRA_INDICES_
-#define MPP_EXTRA_INDICES_ ,:,:
-#undef MPP_TYPE_
-#define MPP_TYPE_ integer(LONG_KIND)
-#include 
-
-#undef MPP_GLOBAL_SUM_TL_
-#define MPP_GLOBAL_SUM_TL_ mpp_global_sum_tl_i8_5d
-#undef MPP_EXTRA_INDICES_
-#define MPP_EXTRA_INDICES_ ,:,:,:
-#undef MPP_TYPE_
-#define MPP_TYPE_ integer(LONG_KIND)
-#include 
-#endif
-
-#undef MPP_GLOBAL_SUM_TL_
-#define MPP_GLOBAL_SUM_TL_ mpp_global_sum_tl_i4_2d
-#undef MPP_EXTRA_INDICES_
-#define MPP_EXTRA_INDICES_
-#undef MPP_TYPE_
-#define MPP_TYPE_ integer(INT_KIND)
-#include 
-
-#undef MPP_GLOBAL_SUM_TL_
-#define MPP_GLOBAL_SUM_TL_ mpp_global_sum_tl_i4_3d
-#undef MPP_EXTRA_INDICES_
-#define MPP_EXTRA_INDICES_ ,:
-#undef MPP_TYPE_
-#define MPP_TYPE_ integer(INT_KIND)
-#include 
-
-#undef MPP_GLOBAL_SUM_TL_
-#define MPP_GLOBAL_SUM_TL_ mpp_global_sum_tl_i4_4d
-#undef MPP_EXTRA_INDICES_
-#define MPP_EXTRA_INDICES_ ,:,:
-#undef MPP_TYPE_
-#define MPP_TYPE_ integer(INT_KIND)
-#include 
-
-#undef MPP_GLOBAL_SUM_TL_
-#define MPP_GLOBAL_SUM_TL_ mpp_global_sum_tl_i4_5d
-#undef MPP_EXTRA_INDICES_
-#define MPP_EXTRA_INDICES_ ,:,:,:
-#undef MPP_TYPE_
-#define MPP_TYPE_ integer(INT_KIND)
-#include 
-!gag
-
-!bnc
-!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-!                                                                             !
-!         MPP_GLOBAL_SUM_AD: global adjoint sum of field                      !
-!                                                                             !
-!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-
-!!$#undef MPP_GLOBAL_SUM_AD_
-!!$#define MPP_GLOBAL_SUM_AD_ mpp_global_sum_ad_r8_2d
-!!$#undef MPP_EXTRA_INDICES_
-!!$#define MPP_EXTRA_INDICES_
-!!$#undef MPP_TYPE_
-!!$#define MPP_TYPE_ real(DOUBLE_KIND)
-!!$#include 
-!!$
-!!$#undef MPP_GLOBAL_SUM_AD_
-!!$#define MPP_GLOBAL_SUM_AD_ mpp_global_sum_ad_r8_3d
-!!$#undef MPP_EXTRA_INDICES_
-!!$#define MPP_EXTRA_INDICES_ ,:
-!!$#undef MPP_TYPE_
-!!$#define MPP_TYPE_ real(DOUBLE_KIND)
-!!$#include 
-!!$
-!!$#undef MPP_GLOBAL_SUM_AD_
-!!$#define MPP_GLOBAL_SUM_AD_ mpp_global_sum_ad_r8_4d
-!!$#undef MPP_EXTRA_INDICES_
-!!$#define MPP_EXTRA_INDICES_ ,:,:
-!!$#undef MPP_TYPE_
-!!$#define MPP_TYPE_ real(DOUBLE_KIND)
-!!$#include 
-!!$
-!!$#undef MPP_GLOBAL_SUM_AD_
-!!$#define MPP_GLOBAL_SUM_AD_ mpp_global_sum_ad_r8_5d
-!!$#undef MPP_EXTRA_INDICES_
-!!$#define MPP_EXTRA_INDICES_ ,:,:,:
-!!$#undef MPP_TYPE_
-!!$#define MPP_TYPE_ real(DOUBLE_KIND)
-!!$#include 
-!!$
-!!$#ifdef OVERLOAD_C8
-!!$#undef MPP_GLOBAL_SUM_AD_
-!!$#define MPP_GLOBAL_SUM_AD_ mpp_global_sum_ad_c8_2d
-!!$#undef MPP_EXTRA_INDICES_
-!!$#define MPP_EXTRA_INDICES_
-!!$#undef MPP_TYPE_
-!!$#define MPP_TYPE_ complex(DOUBLE_KIND)
-!!$#include 
-!!$
-!!$#undef MPP_GLOBAL_SUM_AD_
-!!$#define MPP_GLOBAL_SUM_AD_ mpp_global_sum_ad_c8_3d
-!!$#undef MPP_EXTRA_INDICES_
-!!$#define MPP_EXTRA_INDICES_ ,:
-!!$#undef MPP_TYPE_
-!!$#define MPP_TYPE_ complex(DOUBLE_KIND)
-!!$#include 
-!!$
-!!$#undef MPP_GLOBAL_SUM_AD_
-!!$#define MPP_GLOBAL_SUM_AD_ mpp_global_sum_ad_c8_4d
-!!$#undef MPP_EXTRA_INDICES_
-!!$#define MPP_EXTRA_INDICES_ ,:,:
-!!$#undef MPP_TYPE_
-!!$#define MPP_TYPE_ complex(DOUBLE_KIND)
-!!$#include 
-!!$
-!!$#undef MPP_GLOBAL_SUM_AD_
-!!$#define MPP_GLOBAL_SUM_AD_ mpp_global_sum_ad_c8_5d
-!!$#undef MPP_EXTRA_INDICES_
-!!$#define MPP_EXTRA_INDICES_ ,:,:,:
-!!$#undef MPP_TYPE_
-!!$#define MPP_TYPE_ complex(DOUBLE_KIND)
-!!$#include 
-!!$#endif
-!!$
-!!$#ifdef OVERLOAD_R4
-!!$#undef MPP_GLOBAL_SUM_AD_
-!!$#define MPP_GLOBAL_SUM_AD_ mpp_global_sum_ad_r4_2d
-!!$#undef MPP_EXTRA_INDICES_
-!!$#define MPP_EXTRA_INDICES_
-!!$#undef MPP_TYPE_
-!!$#define MPP_TYPE_ real(FLOAT_KIND)
-!!$#include 
-!!$
-!!$#undef MPP_GLOBAL_SUM_AD_
-!!$#define MPP_GLOBAL_SUM_AD_ mpp_global_sum_ad_r4_3d
-!!$#undef MPP_EXTRA_INDICES_
-!!$#define MPP_EXTRA_INDICES_ ,:
-!!$#undef MPP_TYPE_
-!!$#define MPP_TYPE_ real(FLOAT_KIND)
-!!$#include 
-!!$
-!!$#undef MPP_GLOBAL_SUM_AD_
-!!$#define MPP_GLOBAL_SUM_AD_ mpp_global_sum_ad_r4_4d
-!!$#undef MPP_EXTRA_INDICES_
-!!$#define MPP_EXTRA_INDICES_ ,:,:
-!!$#undef MPP_TYPE_
-!!$#define MPP_TYPE_ real(FLOAT_KIND)
-!!$#include 
-!!$
-!!$#undef MPP_GLOBAL_SUM_AD_
-!!$#define MPP_GLOBAL_SUM_AD_ mpp_global_sum_ad_r4_5d
-!!$#undef MPP_EXTRA_INDICES_
-!!$#define MPP_EXTRA_INDICES_ ,:,:,:
-!!$#undef MPP_TYPE_
-!!$#define MPP_TYPE_ real(FLOAT_KIND)
-!!$#include 
-!!$#endif
-!!$
-!!$#ifdef OVERLOAD_C4
-!!$#undef MPP_GLOBAL_SUM_AD_
-!!$#define MPP_GLOBAL_SUM_AD_ mpp_global_sum_ad_c4_2d
-!!$#undef MPP_EXTRA_INDICES_
-!!$#define MPP_EXTRA_INDICES_
-!!$#undef MPP_TYPE_
-!!$#define MPP_TYPE_ complex(FLOAT_KIND)
-!!$#include 
-!!$
-!!$#undef MPP_GLOBAL_SUM_AD_
-!!$#define MPP_GLOBAL_SUM_AD_ mpp_global_sum_ad_c4_3d
-!!$#undef MPP_EXTRA_INDICES_
-!!$#define MPP_EXTRA_INDICES_ ,:
-!!$#undef MPP_TYPE_
-!!$#define MPP_TYPE_ complex(FLOAT_KIND)
-!!$#include 
-!!$
-!!$#undef MPP_GLOBAL_SUM_AD_
-!!$#define MPP_GLOBAL_SUM_AD_ mpp_global_sum_ad_c4_4d
-!!$#undef MPP_EXTRA_INDICES_
-!!$#define MPP_EXTRA_INDICES_ ,:,:
-!!$#undef MPP_TYPE_
-!!$#define MPP_TYPE_ complex(FLOAT_KIND)
-!!$#include 
-!!$
-!!$#undef MPP_GLOBAL_SUM_AD_
-!!$#define MPP_GLOBAL_SUM_AD_ mpp_global_sum_ad_c4_5d
-!!$#undef MPP_EXTRA_INDICES_
-!!$#define MPP_EXTRA_INDICES_ ,:,:,:
-!!$#undef MPP_TYPE_
-!!$#define MPP_TYPE_ complex(FLOAT_KIND)
-!!$#include 
-!!$#endif
-!!$
-!!$#ifndef no_8byte_integers
-!!$#undef MPP_GLOBAL_SUM_AD_
-!!$#define MPP_GLOBAL_SUM_AD_ mpp_global_sum_ad_i8_2d
-!!$#undef MPP_EXTRA_INDICES_
-!!$#define MPP_EXTRA_INDICES_
-!!$#undef MPP_TYPE_
-!!$#define MPP_TYPE_ integer(LONG_KIND)
-!!$#include 
-!!$
-!!$#undef MPP_GLOBAL_SUM_AD_
-!!$#define MPP_GLOBAL_SUM_AD_ mpp_global_sum_ad_i8_3d
-!!$#undef MPP_EXTRA_INDICES_
-!!$#define MPP_EXTRA_INDICES_ ,:
-!!$#undef MPP_TYPE_
-!!$#define MPP_TYPE_ integer(LONG_KIND)
-!!$#include 
-!!$
-!!$#undef MPP_GLOBAL_SUM_AD_
-!!$#define MPP_GLOBAL_SUM_AD_ mpp_global_sum_ad_i8_4d
-!!$#undef MPP_EXTRA_INDICES_
-!!$#define MPP_EXTRA_INDICES_ ,:,:
-!!$#undef MPP_TYPE_
-!!$#define MPP_TYPE_ integer(LONG_KIND)
-!!$#include 
-!!$
-!!$#undef MPP_GLOBAL_SUM_AD_
-!!$#define MPP_GLOBAL_SUM_AD_ mpp_global_sum_ad_i8_5d
-!!$#undef MPP_EXTRA_INDICES_
-!!$#define MPP_EXTRA_INDICES_ ,:,:,:
-!!$#undef MPP_TYPE_
-!!$#define MPP_TYPE_ integer(LONG_KIND)
-!!$#include 
-!!$#endif
-!!$
-!!$#undef MPP_GLOBAL_SUM_AD_
-!!$#define MPP_GLOBAL_SUM_AD_ mpp_global_sum_ad_i4_2d
-!!$#undef MPP_EXTRA_INDICES_
-!!$#define MPP_EXTRA_INDICES_
-!!$#undef MPP_TYPE_
-!!$#define MPP_TYPE_ integer(INT_KIND)
-!!$#include 
-!!$
-!!$#undef MPP_GLOBAL_SUM_AD_
-!!$#define MPP_GLOBAL_SUM_AD_ mpp_global_sum_ad_i4_3d
-!!$#undef MPP_EXTRA_INDICES_
-!!$#define MPP_EXTRA_INDICES_ ,:
-!!$#undef MPP_TYPE_
-!!$#define MPP_TYPE_ integer(INT_KIND)
-!!$#include 
-!!$
-!!$#undef MPP_GLOBAL_SUM_AD_
-!!$#define MPP_GLOBAL_SUM_AD_ mpp_global_sum_ad_i4_4d
-!!$#undef MPP_EXTRA_INDICES_
-!!$#define MPP_EXTRA_INDICES_ ,:,:
-!!$#undef MPP_TYPE_
-!!$#define MPP_TYPE_ integer(INT_KIND)
-!!$#include 
-!!$
-!!$#undef MPP_GLOBAL_SUM_AD_
-!!$#define MPP_GLOBAL_SUM_AD_ mpp_global_sum_ad_i4_5d
-!!$#undef MPP_EXTRA_INDICES_
-!!$#define MPP_EXTRA_INDICES_ ,:,:,:
-!!$#undef MPP_TYPE_
-!!$#define MPP_TYPE_ integer(INT_KIND)
-!!$#include 
-!!$!bnc
-
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 !                                                                             !
 !              MPP_GLOBAL_FIELD: get global field from domain field           !
diff --git a/src/shared/mpp/include/mpp_domains_util.inc b/src/shared/mpp/include/mpp_domains_util.inc
index 4588ea3baf..30f3daad01 100644
--- a/src/shared/mpp/include/mpp_domains_util.inc
+++ b/src/shared/mpp/include/mpp_domains_util.inc
@@ -1,6 +1,6 @@
 
 ! -*-f90-*-
-! $Id: mpp_domains_util.inc,v 20.0 2013/12/14 00:24:48 fms Exp $
+! $Id$
 
   ! 
   !  
@@ -1019,76 +1019,6 @@ end subroutine mpp_get_domain_shift
 
   end function mpp_domain_is_tile_root_pe
 
-  !#######################################################################
-  !--- the order of overlapping will be consistent with the unpack done in update_domains.
-  function mpp_get_refine_overlap_number(domain, position, whalo, ehalo, shalo, nhalo, tile_count)
-     type(domain2d), intent(inout)        :: domain
-     integer,        intent(in), optional :: position                   ! position of the cell, CENTER/EAST/NORTH/CORNER
-     integer,        intent(in), optional :: whalo, ehalo, shalo, nhalo ! halo size to be updated
-     integer,        intent(in), optional :: tile_count                ! tile number on current pe.
-     integer                              :: mpp_get_refine_overlap_number
-     type(overlapSpec), pointer           :: update => NULL()
-     integer                              :: tMe, whalosz, ehalosz, shalosz, nhalosz, pos
-
-     tMe = 1
-     if(present(tile_count)) tMe = tile_count
-     pos = CENTER;           if(present(position)) pos = position
-     whalosz = domain%whalo; if(present(whalo)) whalosz = whalo
-     ehalosz = domain%ehalo; if(present(ehalo)) ehalosz = ehalo
-     shalosz = domain%shalo; if(present(shalo)) shalosz = shalo
-     nhalosz = domain%nhalo; if(present(nhalo)) nhalosz = nhalo
-     update => search_update_overlap( domain, whalosz, ehalosz, shalosz, nhalosz, pos )
-     mpp_get_refine_overlap_number = update%rSpec(tMe)%count
-     update => NULL()
-
-     return    
- 
-  end function mpp_get_refine_overlap_number
-
-  !#######################################################################
-  ! return overlap information for refined contact.
-  subroutine mpp_get_mosaic_refine_overlap(domain, isMe, ieMe, jsMe, jeMe, isNbr, ieNbr, jsNbr, jeNbr, &
-                                           dir, rotation, position, whalo, ehalo, shalo, nhalo, tile_count)
-     type(domain2d), intent(inout)        :: domain
-     integer, dimension(:), intent(inout) :: isMe,  ieMe,  jsMe,  jeMe  ! index on current tile
-     integer, dimension(:), intent(inout) :: isNbr, ieNbr, jsNbr, jeNbr ! index on neighbor tile
-     integer, dimension(:), intent(inout) :: dir                        ! direction
-     integer, dimension(:), intent(inout) :: rotation                   ! rotation angle
-     integer, optional,     intent(in)    :: position                   ! position of the cell, CENTER/EAST/NORTH/CORNER
-     integer, optional,     intent(in)    :: whalo, ehalo, shalo, nhalo ! halo size to be updated
-     integer, optional,     intent(in)    :: tile_count                ! tile number on current pe.
-     type(overlapSpec), pointer           :: update => NULL()
-     integer                              :: tMe, count, whalosz, ehalosz, shalosz, nhalosz, pos
-
-     tMe = 1
-     if(present(tile_count)) tMe = tile_count
-     pos = CENTER;           if(present(position)) pos = position
-     whalosz = domain%whalo; if(present(whalo)) whalosz = whalo
-     ehalosz = domain%ehalo; if(present(ehalo)) ehalosz = ehalo
-     shalosz = domain%shalo; if(present(shalo)) shalosz = shalo
-     nhalosz = domain%nhalo; if(present(nhalo)) nhalosz = nhalo
-
-     update => search_update_overlap( domain, whalosz, ehalosz, shalosz, nhalosz, pos )
-     count = update%rSpec(tMe)%count
-     if(count>0) then
-        if(size(isMe(:)) < count) call mpp_error(FATAL, &
-           "mpp_domains_util.inc(mpp_get_mosaic_refine_overlap): size of isMe is less than the number of overlap")
-        isMe(1:count)      = update%rSpec(tMe)%isMe
-        ieMe(1:count)      = update%rSpec(tMe)%ieMe
-        jsMe(1:count)      = update%rSpec(tMe)%jsMe
-        jeMe(1:count)      = update%rSpec(tMe)%jeMe
-        isNbr(1:count)     = update%rSpec(tMe)%isNbr
-        ieNbr(1:count)     = update%rSpec(tMe)%ieNbr
-        jsNbr(1:count)     = update%rSpec(tMe)%jsNbr
-        jeNbr(1:count)     = update%rSpec(tMe)%jeNbr
-        dir  (1:count)     = update%rSpec(tMe)%dir
-        rotation(1:count)  = update%rSpec(tMe)%rotation
-     endif
-
-     return
-
-  end subroutine mpp_get_mosaic_refine_overlap
-
   !#########################################################################
   ! return number of processors used on current tile.
   function mpp_get_tile_npes(domain)
@@ -1613,6 +1543,22 @@ end subroutine mpp_get_tile_compute_domains
 
   end function get_rank_unpack
 
+  function get_mesgsize(overlap, do_dir)
+   type(overlap_type), intent(in) :: overlap
+   logical,            intent(in) :: do_dir(:)
+   integer                       :: get_mesgsize
+   integer                       :: n, dir
+
+   get_mesgsize = 0
+   do n = 1, overlap%count
+      dir = overlap%dir(n)
+      if(do_dir(dir)) then
+         get_mesgsize = get_mesgsize + overlap%msgsize(n)
+      end if
+   end do
+
+  end function get_mesgsize
+
   !#############################################################################
   subroutine mpp_set_domain_symmetry(domain, symmetry)
     type(domain2D), intent(inout) :: domain
@@ -1675,3 +1621,480 @@ end subroutine mpp_get_tile_compute_domains
      return
 
   end subroutine mpp_copy_domain2D
+
+  !######################################################################
+  subroutine set_group_update(group, domain)
+     type(mpp_group_update_type), intent(inout) :: group  
+     type(domain2D),              intent(inout) :: domain
+     integer   :: nscalar, nvector, nlist
+     integer   :: nsend, nrecv, nsend_old, nrecv_old
+     integer   :: nsend_s, nsend_x, nsend_y
+     integer   :: nrecv_s, nrecv_x, nrecv_y
+     integer   :: update_buffer_pos, tot_recv_size, tot_send_size
+     integer   :: msgsize_s, msgsize_x, msgsize_y, msgsize
+     logical   :: recv_s(8), send_s(8), recv_v(8), send_v(8)
+     integer   :: ntot, n, l, m, ksize
+     integer   :: i_s, i_x, i_y, rank_s, rank_x, rank_y, rank
+     integer   :: ind_s(3*MAXOVERLAP)
+     integer   :: ind_x(3*MAXOVERLAP)
+     integer   :: ind_y(3*MAXOVERLAP)
+     integer   :: pelist(3*MAXOVERLAP), to_pe_list(3*MAXOVERLAP)
+     integer   :: buffer_pos_recv(3*MAXOVERLAP), buffer_pos_send(3*MAXOVERLAP)
+     integer   :: recv_size(3*MAXOVERLAP), send_size(3*MAXOVERLAP)
+     integer   :: position_x, position_y, npack, nunpack, dir
+     integer   :: pack_buffer_pos, unpack_buffer_pos
+     integer   :: omp_get_num_threads, nthreads
+     character(len=8)            :: text
+     type(overlap_type), pointer :: overPtr => NULL()
+     type(overlapSpec), pointer  :: update_s => NULL()
+     type(overlapSpec), pointer  :: update_x => NULL()
+     type(overlapSpec), pointer  :: update_y => NULL()
+
+     nscalar = group%nscalar
+     nvector = group%nvector
+
+     !--- get the overlap data type
+     select case(group%gridtype)
+     case (AGRID)
+        position_x = CENTER
+        position_y = CENTER
+     case (BGRID_NE, BGRID_SW)
+        position_x = CORNER
+        position_y = CORNER
+     case (CGRID_NE, CGRID_SW)
+        position_x = EAST
+        position_y = NORTH
+     case (DGRID_NE, DGRID_SW)
+        position_x = NORTH
+        position_y = EAST
+     case default
+        call mpp_error(FATAL, "set_group_update: invalid value of gridtype")
+     end select
+     if(nscalar>0) then
+        update_s => search_update_overlap(domain, group%whalo_s, group%ehalo_s, &
+             group%shalo_s, group%nhalo_s, group%position)
+     endif
+     if(nvector>0) then
+        update_x => search_update_overlap(domain, group%whalo_v, group%ehalo_v, &
+             group%shalo_v, group%nhalo_v, position_x)
+        update_y => search_update_overlap(domain, group%whalo_v, group%ehalo_v, &
+             group%shalo_v, group%nhalo_v, position_y)
+     endif
+
+     if(nscalar > 0) then
+        recv_s = group%recv_s
+        send_s = recv_s
+     endif
+     if(nvector > 0) then
+        recv_v = group%recv_v
+        send_v = recv_v
+     end if
+     nlist   = size(domain%list(:))
+     group%initialized = .true.
+     nsend_s = 0; nsend_x = 0; nsend_y = 0
+     nrecv_s = 0; nrecv_x = 0; nrecv_y = 0
+
+     if(nscalar > 0) then
+         !--- This check could not be done because of memory domain
+!        if( group%isize_s .NE. (group%ie_s-group%is_s+1) .OR. group%jsize_s .NE. (group%je_s-group%js_s+1))  &
+!           call mpp_error(FATAL, "set_group_update: mismatch of size of the field and domain memory domain")
+        nsend_s = update_s%nsend
+        nrecv_s = update_s%nrecv
+     endif
+
+     !--- ksize_s must equal ksize_v
+     if(nvector > 0 .AND. nscalar > 0) then
+        if(group%ksize_s .NE. group%ksize_v) then
+           call mpp_error(FATAL, "set_group_update: ksize_s and ksize_v are not equal")
+        endif
+        ksize = group%ksize_s
+     else if (nscalar > 0) then
+        ksize = group%ksize_s
+     else if (nvector > 0) then
+        ksize = group%ksize_v
+     else
+        call mpp_error(FATAL, "set_group_update: nscalar and nvector are all 0")
+     endif
+
+    nthreads = 1
+!$OMP PARALLEL
+!$   nthreads = omp_get_num_threads()
+!$OMP END PARALLEL
+     if( nthreads > nthread_control_loop ) then
+        group%k_loop_inside = .FALSE.
+     else
+        group%k_loop_inside = .TRUE.
+     endif
+
+     if(nvector > 0) then
+        !--- This check could not be done because of memory domain
+!        if( group%isize_x .NE. (group%ie_x-group%is_x+1) .OR. group%jsize_x .NE. (group%je_x-group%js_x+1))  &
+!           call mpp_error(FATAL, "set_group_update: mismatch of size of the fieldx and domain memory domain")
+!        if( group%isize_y .NE. (group%ie_y-group%is_y+1) .OR. group%jsize_y .NE. (group%je_y-group%js_y+1))  &
+!           call mpp_error(FATAL, "set_group_update: mismatch of size of the fieldy and domain memory domain")
+        nsend_x = update_x%nsend
+        nrecv_x = update_x%nrecv
+        nsend_y = update_y%nsend
+        nrecv_y = update_y%nrecv
+     endif
+
+     !figure out message size for each processor.
+     ntot = nrecv_s + nrecv_x + nrecv_y
+     if(ntot > 3*MAXOVERLAP) call mpp_error(FATAL, "set_group_update: ntot is greater than 3*MAXOVERLAP")
+     n = 1
+     i_s = 1
+     i_x = 1
+     i_y = 1
+     ind_s = -1
+     ind_x = -1
+     ind_y = -1
+     nrecv = 0
+     do while(n<=ntot)
+        if( i_s <= nrecv_s ) then
+           rank_s = update_s%recv(i_s)%pe-domain%pe
+           if(rank_s .LE. 0) rank_s = rank_s + nlist
+        else
+           rank_s = -1
+        endif
+        if( i_x <= nrecv_x ) then
+           rank_x = update_x%recv(i_x)%pe-domain%pe
+           if(rank_x .LE. 0) rank_x = rank_x + nlist
+        else
+           rank_x = -1
+        endif
+        if( i_y <= nrecv_y ) then
+           rank_y = update_y%recv(i_y)%pe-domain%pe
+           if(rank_y .LE. 0) rank_y = rank_y + nlist
+        else
+           rank_y = -1
+        endif
+        nrecv = nrecv + 1   
+        rank = maxval((/rank_s, rank_x, rank_y/))
+        if(rank == rank_s) then
+           n = n + 1
+           ind_s(nrecv) = i_s
+           pelist(nrecv) = update_s%recv(i_s)%pe
+           i_s = i_s + 1
+        endif
+        if(rank == rank_x) then
+           n = n + 1
+           ind_x(nrecv) = i_x
+           pelist(nrecv) = update_x%recv(i_x)%pe
+           i_x = i_x + 1
+        endif
+        if(rank == rank_y) then
+           n = n + 1
+           ind_y(nrecv) = i_y
+           pelist(nrecv) = update_y%recv(i_y)%pe
+           i_y = i_y + 1
+        endif
+     enddo
+
+     nrecv_old = nrecv
+     nrecv     = 0
+     update_buffer_pos = 0
+     tot_recv_size = 0
+
+     !--- setup for recv
+     do l = 1, nrecv_old
+        msgsize_s = 0
+        msgsize_x = 0
+        msgsize_y = 0
+        m = ind_s(l)
+        if(m>0) msgsize_s = get_mesgsize(update_s%recv(m), recv_s)*ksize*nscalar
+        m = ind_x(l)
+        if(m>0) msgsize_x = get_mesgsize(update_x%recv(m), recv_v)*ksize*nvector
+        m = ind_y(l)
+        if(m>0) msgsize_y = get_mesgsize(update_y%recv(m), recv_v)*ksize*nvector
+        msgsize = msgsize_s + msgsize_x + msgsize_y
+        if( msgsize.GT.0 )then
+           tot_recv_size = tot_recv_size + msgsize
+           nrecv = nrecv + 1
+           if(nrecv > MAXOVERLAP) then
+              call mpp_error(FATAL, "set_group_update: nrecv is greater than MAXOVERLAP, increase MAXOVERLAP")
+           endif
+           group%from_pe(nrecv) = pelist(l)
+           group%recv_size(nrecv) = msgsize
+           group%buffer_pos_recv(nrecv) = update_buffer_pos
+           update_buffer_pos = update_buffer_pos + msgsize
+        end if
+     end do
+     group%nrecv = nrecv
+
+     !--- setup for unpack
+     nunpack = 0
+     unpack_buffer_pos = 0
+     do l = 1, nrecv_old
+        m = ind_s(l)
+        if(m>0) then
+           overptr => update_s%recv(m)
+           do n = 1, overptr%count
+              dir = overptr%dir(n)
+              if(recv_s(dir)) then
+                 nunpack = nunpack + 1
+                 if(nunpack > MAXOVERLAP) call mpp_error(FATAL,  &
+                   "set_group_update: nunpack is greater than MAXOVERLAP, increase MAXOVERLAP 1")
+                 group%unpack_type(nunpack) = FIELD_S
+                 group%unpack_buffer_pos(nunpack) = unpack_buffer_pos
+                 group%unpack_rotation(nunpack) = overptr%rotation(n) 
+                 group%unpack_is(nunpack) = overptr%is(n)
+                 group%unpack_ie(nunpack) = overptr%ie(n)
+                 group%unpack_js(nunpack) = overptr%js(n)
+                 group%unpack_je(nunpack) = overptr%je(n) 
+                 group%unpack_size(nunpack) = overptr%msgsize(n)*nscalar
+                 unpack_buffer_pos = unpack_buffer_pos + group%unpack_size(nunpack)*ksize
+              end if
+           end do   
+        end if
+
+        m = ind_x(l)
+        if(m>0) then
+           overptr => update_x%recv(m)
+           do n = 1, overptr%count
+              dir = overptr%dir(n)
+              if(recv_v(dir)) then
+                 nunpack = nunpack + 1
+                 if(nunpack > MAXOVERLAP) call mpp_error(FATAL,  &
+                   "set_group_update: nunpack is greater than MAXOVERLAP, increase MAXOVERLAP 2")
+                 group%unpack_type(nunpack) = FIELD_X
+                 group%unpack_buffer_pos(nunpack) = unpack_buffer_pos
+                 group%unpack_rotation(nunpack) = overptr%rotation(n) 
+                 group%unpack_is(nunpack) = overptr%is(n)
+                 group%unpack_ie(nunpack) = overptr%ie(n)
+                 group%unpack_js(nunpack) = overptr%js(n)
+                 group%unpack_je(nunpack) = overptr%je(n) 
+                 group%unpack_size(nunpack) = overptr%msgsize(n)*nvector
+                 unpack_buffer_pos = unpack_buffer_pos + group%unpack_size(nunpack)*ksize
+              end if
+           end do   
+        end if
+
+        m = ind_y(l)
+        if(m>0) then
+           overptr => update_y%recv(m)
+           do n = 1, overptr%count
+              dir = overptr%dir(n)
+              if(recv_v(dir)) then
+                 nunpack = nunpack + 1
+                 if(nunpack > MAXOVERLAP) call mpp_error(FATAL,  &
+                   "set_group_update: nunpack is greater than MAXOVERLAP, increase MAXOVERLAP 3")
+                 group%unpack_type(nunpack) = FIELD_Y
+                 group%unpack_buffer_pos(nunpack) = unpack_buffer_pos
+                 group%unpack_rotation(nunpack) = overptr%rotation(n) 
+                 group%unpack_is(nunpack) = overptr%is(n)
+                 group%unpack_ie(nunpack) = overptr%ie(n)
+                 group%unpack_js(nunpack) = overptr%js(n)
+                 group%unpack_je(nunpack) = overptr%je(n) 
+                 group%unpack_size(nunpack) = overptr%msgsize(n)*nvector
+                 unpack_buffer_pos = unpack_buffer_pos + group%unpack_size(nunpack)*ksize
+              end if
+           end do   
+        end if
+     end do
+     group%nunpack = nunpack
+
+     if(update_buffer_pos .NE. unpack_buffer_pos ) call mpp_error(FATAL,  &
+                   "set_group_update: update_buffer_pos .NE. unpack_buffer_pos")
+
+     !figure out message size for each processor.
+     ntot = nsend_s + nsend_x + nsend_y
+     n = 1
+     i_s = 1
+     i_x = 1
+     i_y = 1
+     ind_s = -1
+     ind_x = -1
+     ind_y = -1
+     nsend = 0
+     do while(n<=ntot)
+        if( i_s <= nsend_s ) then
+           rank_s = update_s%send(i_s)%pe-domain%pe
+           if(rank_s .LT. 0) rank_s = rank_s + nlist
+        else
+           rank_s = nlist+1
+        endif
+        if( i_x <= nsend_x ) then
+           rank_x = update_x%send(i_x)%pe-domain%pe
+           if(rank_x .LT. 0) rank_x = rank_x + nlist
+        else
+           rank_x = nlist+1
+        endif
+        if( i_y <= nsend_y ) then
+           rank_y = update_y%send(i_y)%pe-domain%pe
+           if(rank_y .LT. 0) rank_y = rank_y + nlist
+        else
+           rank_y = nlist+1
+        endif
+        nsend = nsend + 1   
+        rank = minval((/rank_s, rank_x, rank_y/))
+        if(rank == rank_s) then
+           n = n + 1
+           ind_s(nsend) = i_s
+           pelist(nsend) = update_s%send(i_s)%pe
+           i_s = i_s + 1
+        endif
+        if(rank == rank_x) then
+           n = n + 1
+           ind_x(nsend) = i_x
+           pelist(nsend) = update_x%send(i_x)%pe
+           i_x = i_x + 1
+        endif
+        if(rank == rank_y) then
+           n = n + 1
+           ind_y(nsend) = i_y
+           pelist(nsend) = update_y%send(i_y)%pe
+           i_y = i_y + 1
+        endif
+     enddo
+
+     nsend_old = nsend
+     nsend     = 0
+     tot_send_size = 0
+     do l = 1, nsend_old
+        msgsize_s = 0
+        msgsize_x = 0
+        msgsize_y = 0
+        m = ind_s(l)
+        if(m>0) msgsize_s = get_mesgsize(update_s%send(m), send_s)*ksize*nscalar
+        m = ind_x(l)
+        if(m>0) msgsize_x = get_mesgsize(update_x%send(m), send_v)*ksize*nvector
+        m = ind_y(l)
+        if(m>0) msgsize_y = get_mesgsize(update_y%send(m), send_v)*ksize*nvector
+        msgsize = msgsize_s + msgsize_x + msgsize_y
+        if( msgsize.GT.0 )then
+           tot_send_size = tot_send_size + msgsize
+           nsend = nsend + 1
+           if(nsend > MAXOVERLAP) then
+              call mpp_error(FATAL, "set_group_update: nsend is greater than MAXOVERLAP, increase MAXOVERLAP")
+           endif
+           send_size(nsend) = msgsize
+           group%to_pe(nsend) = pelist(l)
+           group%buffer_pos_send(nsend) = update_buffer_pos
+           group%send_size(nsend) = msgsize
+           update_buffer_pos = update_buffer_pos + msgsize
+        end if
+     end do
+     group%nsend = nsend
+
+     !--- setup for pack
+     npack = 0
+     pack_buffer_pos = unpack_buffer_pos
+     do l = 1, nsend_old
+        m = ind_s(l)
+        if(m>0) then
+           overptr => update_s%send(m)
+           do n = 1, overptr%count
+              dir = overptr%dir(n)
+              if(send_s(dir)) then
+                 npack = npack + 1
+                 if(npack > MAXOVERLAP) call mpp_error(FATAL,  &
+                   "set_group_update: npack is greater than MAXOVERLAP, increase MAXOVERLAP 1")
+                 group%pack_type(npack) = FIELD_S
+                 group%pack_buffer_pos(npack) = pack_buffer_pos
+                 group%pack_rotation(npack) = overptr%rotation(n) 
+                 group%pack_is(npack) = overptr%is(n)
+                 group%pack_ie(npack) = overptr%ie(n)
+                 group%pack_js(npack) = overptr%js(n)
+                 group%pack_je(npack) = overptr%je(n) 
+                 group%pack_size(npack) = overptr%msgsize(n)*nscalar
+                 pack_buffer_pos = pack_buffer_pos + group%pack_size(npack)*ksize
+              end if
+           end do   
+        end if
+
+        m = ind_x(l)
+        if(m>0) then
+           overptr => update_x%send(m)
+           do n = 1, overptr%count
+              dir = overptr%dir(n)
+              if(send_v(dir)) then
+                 npack = npack + 1
+                 if(npack > MAXOVERLAP) call mpp_error(FATAL,  &
+                   "set_group_update: npack is greater than MAXOVERLAP, increase MAXOVERLAP 2")
+                 group%pack_type(npack) = FIELD_X
+                 group%pack_buffer_pos(npack) = pack_buffer_pos
+                 group%pack_rotation(npack) = overptr%rotation(n) 
+                 group%pack_is(npack) = overptr%is(n)
+                 group%pack_ie(npack) = overptr%ie(n)
+                 group%pack_js(npack) = overptr%js(n)
+                 group%pack_je(npack) = overptr%je(n) 
+                 group%pack_size(npack) = overptr%msgsize(n)*nvector
+                 pack_buffer_pos = pack_buffer_pos + group%pack_size(npack)*ksize
+              end if
+           end do   
+        end if
+
+        m = ind_y(l)
+        if(m>0) then
+           overptr => update_y%send(m)
+           do n = 1, overptr%count
+              dir = overptr%dir(n)
+              if(send_v(dir)) then
+                 npack = npack + 1
+                 if(npack > MAXOVERLAP) call mpp_error(FATAL,  &
+                   "set_group_update: npack is greater than MAXOVERLAP, increase MAXOVERLAP 3")
+                 group%pack_type(npack) = FIELD_Y
+                 group%pack_buffer_pos(npack) = pack_buffer_pos
+                 group%pack_rotation(npack) = overptr%rotation(n) 
+                 group%pack_is(npack) = overptr%is(n)
+                 group%pack_ie(npack) = overptr%ie(n)
+                 group%pack_js(npack) = overptr%js(n)
+                 group%pack_je(npack) = overptr%je(n) 
+                 group%pack_size(npack) = overptr%msgsize(n)*nvector
+                 pack_buffer_pos = pack_buffer_pos + group%pack_size(npack)*ksize
+              end if
+           end do   
+        end if
+     end do
+     group%npack = npack
+     if(update_buffer_pos .NE. pack_buffer_pos ) call mpp_error(FATAL,  &
+                   "set_group_update: update_buffer_pos .NE. pack_buffer_pos")
+
+     !--- make sure the buffer is large enough
+     mpp_domains_stack_hwm = max( mpp_domains_stack_hwm, tot_recv_size+tot_send_size )
+
+     if( mpp_domains_stack_hwm.GT.mpp_domains_stack_size )then
+        write( text,'(i8)' )mpp_domains_stack_hwm
+        call mpp_error( FATAL, 'set_group_update: mpp_domains_stack overflow, '// &
+                     'call mpp_domains_set_stack_size('//trim(text)//') from all PEs.' )
+     end if
+
+     group%tot_msgsize = tot_recv_size+tot_send_size
+
+end subroutine set_group_update
+
+
+!######################################################################
+  subroutine mpp_clear_group_update(group)
+     type(mpp_group_update_type), intent(inout) :: group
+
+     group%nscalar = 0
+     group%nvector = 0
+     group%nsend   = 0
+     group%nrecv   = 0
+     group%npack   = 0
+     group%nunpack = 0
+     group%initialized = .false.
+
+  end subroutine mpp_clear_group_update
+
+!#####################################################################
+  function mpp_group_update_initialized(group)
+     type(mpp_group_update_type), intent(in) :: group
+     logical :: mpp_group_update_initialized
+
+     mpp_group_update_initialized = group%initialized
+
+  end function mpp_group_update_initialized
+
+!#####################################################################
+  function mpp_group_update_is_set(group)
+     type(mpp_group_update_type), intent(in) :: group
+     logical :: mpp_group_update_is_set
+
+     mpp_group_update_is_set = (group%nscalar > 0 .OR. group%nvector > 0)
+
+  end function mpp_group_update_is_set
+
+
+
diff --git a/src/shared/mpp/include/mpp_gather.h b/src/shared/mpp/include/mpp_gather.h
index 9a199ab198..af2b0b2052 100644
--- a/src/shared/mpp/include/mpp_gather.h
+++ b/src/shared/mpp/include/mpp_gather.h
@@ -1,51 +1,222 @@
-subroutine MPP_GATHER_1D_(sbuf, rbuf)
+subroutine MPP_GATHER_1D_(sbuf, rbuf,pelist)
+! JWD: Did not create mpp_gather_2d because have no requirement for it
+! JWD: See mpp_gather_2dv below
    MPP_TYPE_, dimension(:),    intent(in) :: sbuf
    MPP_TYPE_, dimension(:), intent(inout) :: rbuf
-   integer :: cnt, l, nproc
+   integer,   dimension(:),    intent(in), optional :: pelist(:)
+
+   integer :: cnt, l, nproc, op_root
+   integer, allocatable :: pelist2(:)
+
+
+!  If pelist is provided, the first position must be
+!  the operation root
+   if(PRESENT(pelist))then
+      nproc = size(pelist)
+      allocate(pelist2(nproc))
+      pelist2 = pelist
+   else
+      nproc = mpp_npes()
+      allocate(pelist2(nproc))
+      pelist2 = (/ (l, l=root_pe, nproc-1+root_pe) /)
+   endif
+   op_root = pelist2(1)
 
-   nproc = mpp_npes()
    cnt = size(sbuf(:))
-   if(size(rbuf(:)) .NE. cnt*nproc) call mpp_error(FATAL, &
-          "MPP_GATHER_1D_: size(rbuf) should equal to npes*size(sbuf) ")
+   if(size(rbuf(:)) < cnt*nproc) call mpp_error(FATAL, &
+          "MPP_GATHER_1D_: size(rbuf) must be at least npes*size(sbuf) ")
 
    !--- pre-post receiving
-   if(pe == root_pe ) then
+   if(pe == op_root) then
       rbuf(1:cnt) = sbuf
-      do l = 1, nproc-1
-         call mpp_recv(rbuf(l*cnt+1), glen=cnt, from_pe=root_pe+l, block=.FALSE., tag=COMM_TAG_1 )
+      do l = 2, nproc
+         call mpp_recv(rbuf((l-1)*cnt+1), glen=cnt, from_pe=pelist2(l), block=.FALSE., tag=COMM_TAG_1 )
       enddo
    else
-      call mpp_send(sbuf(1), plen=cnt, to_pe=root_pe, tag=COMM_TAG_1)
+      call mpp_send(sbuf(1), plen=cnt, to_pe=op_root, tag=COMM_TAG_1)
    endif
 
    call mpp_sync_self(check=EVENT_RECV)
    call mpp_sync_self()
-
+   deallocate(pelist2)
 end subroutine MPP_GATHER_1D_
 
-
-subroutine MPP_GATHER_1DV_(sbuf, ssize, rbuf, rsize)
+subroutine MPP_GATHER_1DV_(sbuf, ssize, rbuf, rsize, pelist)
    MPP_TYPE_, dimension(:),    intent(in) :: sbuf
    MPP_TYPE_, dimension(:), intent(inout) :: rbuf
    integer,                    intent(in) :: ssize
-   integer,   dimension(:),    intent(in) :: rsize 
-   integer :: cnt, l, nproc, pos
+   integer,   dimension(:),    intent(in) :: rsize
+   integer,   dimension(:),    intent(in), optional :: pelist(:)
+
+   integer :: cnt, l, nproc, pos, op_root
+   integer, allocatable :: pelist2(:)
+
+!  If pelist is provided, the first position must be
+!  the operation root
+   if(PRESENT(pelist))then
+      nproc = size(pelist)
+      allocate(pelist2(nproc)) 
+      pelist2 = pelist
+   else
+      nproc = mpp_npes()
+      allocate(pelist2(nproc))        
+      pelist2 = (/ (l, l=0+root_pe, nproc-1+root_pe) /)
+   endif
+   op_root = pelist2(1)
 
-   nproc = mpp_npes()
 
    !--- pre-post receiving
-   if(pe == root_pe ) then
-      rbuf(1:ssize) = sbuf
+   if(pe == op_root) then
+      rbuf(1:ssize) = sbuf(:)
       pos = ssize
-      do l = 1, nproc-1
-         call mpp_recv(rbuf(pos+1), glen=rsize(l+1), from_pe=root_pe+l, block=.FALSE., tag=COMM_TAG_2 )
-         pos = pos + rsize(l+1)
+      do l = 2, nproc
+         if(rsize(l) >0) then
+            call mpp_recv(rbuf(pos+1), glen=rsize(l), from_pe=pelist2(l), block=.FALSE., tag=COMM_TAG_2 )
+            pos = pos + rsize(l)
+         endif
       enddo
    else
-      call mpp_send(sbuf(1), plen=ssize, to_pe=root_pe, tag=COMM_TAG_2)
+      if(ssize>0) call mpp_send(sbuf(1), plen=ssize, to_pe=op_root, tag=COMM_TAG_2)
    endif
 
    call mpp_sync_self(check=EVENT_RECV)
    call mpp_sync_self()
-
+   deallocate(pelist2)
 end subroutine MPP_GATHER_1DV_
+
+
+subroutine MPP_GATHER_PELIST_2D_(is, ie, js, je, pelist, array_seg, data, is_root_pe, &
+                                 ishift, jshift)
+   integer,                           intent(in)    :: is, ie, js, je
+   integer,   dimension(:),           intent(in)    :: pelist
+   MPP_TYPE_, dimension(is:ie,js:je), intent(in)    :: array_seg
+   MPP_TYPE_, dimension(:,:),         intent(inout) :: data
+   logical,                           intent(in)    :: is_root_pe
+   integer,   optional,               intent(in)    :: ishift, jshift
+
+   MPP_TYPE_ ::  arr3D(size(array_seg,1),size(array_seg,2),1)
+   MPP_TYPE_ :: data3D(size(     data,1),size(     data,2),1)
+   pointer( aptr,  arr3D )
+   pointer( dptr, data3D )
+   aptr = LOC(array_seg)
+   dptr = LOC(     data)
+
+   call mpp_gather(is, ie, js, je, 1, pelist, arr3D, data3D, is_root_pe, &
+                   ishift, jshift)
+   return
+
+end subroutine MPP_GATHER_PELIST_2D_
+
+
+subroutine MPP_GATHER_PELIST_3D_(is, ie, js, je, nk, pelist, array_seg, data, is_root_pe, &
+                                 ishift, jshift)
+   integer,                                intent(in)    :: is, ie, js, je, nk
+   integer,   dimension(:),                intent(in)    :: pelist
+   MPP_TYPE_, dimension(is:ie,js:je,1:nk), intent(in)    :: array_seg
+   MPP_TYPE_, dimension(:,:,:),            intent(inout) :: data
+   logical,                                intent(in)    :: is_root_pe
+   integer,   optional,                    intent(in)    :: ishift, jshift
+
+   integer :: i, msgsize, root_pe, root_pe_test
+   integer :: i1, i2, j1, j2, ioff, joff
+   integer :: my_ind(4), gind(4,size(pelist))
+   type array3D
+     MPP_TYPE_, dimension(:,:,:), allocatable :: data
+   endtype array3D
+   type(array3d), dimension(size(pelist)) :: temp
+
+   if (.not.ANY(mpp_pe().eq.pelist(:))) return
+
+   if (is_root_pe) then
+     root_pe = mpp_pe()
+     root_pe_test = 999
+     if (.not.ANY(pelist(:).eq.root_pe)) call mpp_error(FATAL, &
+                "fms_io(mpp_gather_pelist): root_pe not a member of pelist")
+   else
+     root_pe = 0
+     root_pe_test = -999
+   endif
+! need this check in case MPI-rank 0 is a member of the pelist
+   call mpp_max(root_pe_test, pelist)
+   if (root_pe_test.lt.0) call mpp_error(FATAL, &
+                "fms_io(mpp_gather_pelist): root_pe not specified or not a member of the pelist")
+! need to make sure only one root_pe has been specified
+   call mpp_sum(root_pe, pelist)
+   if ((is_root_pe) .and. (mpp_pe().ne.root_pe)) call mpp_error(FATAL, &
+                "fms_io(mpp_gather_pelist): too many root_pes specified")
+
+
+   ioff=0
+   joff=0
+   if (present(ishift)) ioff=ishift
+   if (present(jshift)) joff=jshift
+
+   my_ind(1) = is
+   my_ind(2) = ie
+   my_ind(3) = js
+   my_ind(4) = je
+
+! gather indices into global index on root_pe
+   if (is_root_pe) then
+     do i = 1, size(pelist)
+! root_pe data copy - no send to self
+       if (pelist(i).eq.root_pe) then
+         gind(:,i) = my_ind(:)
+       else
+         call mpp_recv(gind(:,i:i), 4, pelist(i), .FALSE., COMM_TAG_1)
+       endif
+     enddo
+     call mpp_sync_self(check=EVENT_RECV)
+     gind(1,:)=gind(1,:)+ioff
+     gind(2,:)=gind(2,:)+ioff
+     gind(3,:)=gind(3,:)+joff
+     gind(4,:)=gind(4,:)+joff
+! check indices to make sure they are within the range of "data"
+     if ((minval(gind).lt.1) .OR. (maxval(gind(1:2,:)).gt.size(data,1)) .OR. (maxval(gind(3:4,:)).gt.size(data,2))) &
+         call mpp_error(FATAL,"fms_io(mpp_gather_pelist): specified indices (with shift) are outside of the & 
+                        &range of the receiving array")
+   else
+! non root_pe's send indices to root_pe
+     call mpp_send(my_ind(:), 4, root_pe, COMM_TAG_1)
+     call mpp_sync_self(check=EVENT_SEND)
+   endif
+
+!  gather segments into data based on indices
+   if (is_root_pe) then
+     do i = 1, size(pelist)
+       if (pelist(i).ne.root_pe) then    ! no send to self
+         i1 = gind(1,i)
+         i2 = gind(2,i)
+         j1 = gind(3,i)
+         j2 = gind(4,i)
+         msgsize = (i2-i1+1)*(j2-j1+1)*nk
+         allocate(temp(i)%data(i1:i2,j1:j2,1:nk))
+         call mpp_recv(temp(i)%data(i1:i2,j1:j2,1:nk), msgsize, pelist(i), .FALSE., COMM_TAG_2)
+       endif
+     enddo
+     call mpp_sync_self(check=EVENT_RECV)
+!  unbuffer/copy the data into the return array
+     do i = 1, size(pelist)
+       if (pelist(i).eq.root_pe) then
+!        data copy - no send to self
+         data(is+ioff:ie+ioff,js+joff:je+joff,1:nk) = array_seg(is:ie,js:je,1:nk)
+       else
+         i1 = gind(1,i)
+         i2 = gind(2,i)
+         j1 = gind(3,i)
+         j2 = gind(4,i)
+         data(i1:i2,j1:j2,1:nk)=temp(i)%data(i1:i2,j1:j2,1:nk)
+         deallocate(temp(i)%data)
+       endif
+     enddo
+   else
+!    non root_pe's send data to root_pe
+     msgsize = (my_ind(2)-my_ind(1)+1) * (my_ind(4)-my_ind(3)+1) * nk
+     call mpp_send(array_seg, msgsize, root_pe, COMM_TAG_2)
+     call mpp_sync_self(check=EVENT_SEND)
+   endif
+
+   call mpp_sync_self()
+   return
+
+end subroutine MPP_GATHER_PELIST_3D_
diff --git a/src/shared/mpp/include/mpp_get_boundary.h b/src/shared/mpp/include/mpp_get_boundary.h
index 60be0228d9..02d2d7fae6 100644
--- a/src/shared/mpp/include/mpp_get_boundary.h
+++ b/src/shared/mpp/include/mpp_get_boundary.h
@@ -497,7 +497,7 @@ subroutine MPP_GET_BOUNDARY_2D_V_(fieldx, fieldy, domain, ebufferx, sbufferx, wb
      if(ASSOCIATED(boundx) ) then
         call mpp_do_get_boundary(f_addrsx(1:l_size,1:ntile), f_addrsy(1:l_size,1:ntile), domain, boundx, boundy, &
              b_addrsx(:,1:l_size,1:ntile), b_addrsy(:,1:l_size,1:ntile), bsizex, &
-             bsizey, ksize, d_type, update_flags)
+             bsizey, ksize, d_type, update_flags, grid_offset_type)
      endif
      l_size=0; f_addrsx=-9999; f_addrsy=-9999; bsizex=0; bsizey=0; 
      b_addrsx=-9999; b_addrsy=-9999; isize=0;  jsize=0;  ksize=0
@@ -721,7 +721,7 @@ subroutine MPP_GET_BOUNDARY_3D_V_(fieldx, fieldy, domain, ebufferx, sbufferx, wb
      if(ASSOCIATED(boundx) ) then
         call mpp_do_get_boundary(f_addrsx(1:l_size,1:ntile), f_addrsy(1:l_size,1:ntile), domain, boundx, boundy, &
              b_addrsx(:,1:l_size,1:ntile), b_addrsy(:,1:l_size,1:ntile), bsizex, &
-             bsizey, ksize, d_type, update_flags)
+             bsizey, ksize, d_type, update_flags, grid_offset_type)
      endif
      l_size=0; f_addrsx=-9999; f_addrsy=-9999; bsizex=0; bsizey=0; 
      b_addrsx=-9999; b_addrsy=-9999; isize=0;  jsize=0;  ksize=0
diff --git a/src/shared/mpp/include/mpp_global_sum.h b/src/shared/mpp/include/mpp_global_sum.h
index c1b9954fa2..7684fe50b8 100644
--- a/src/shared/mpp/include/mpp_global_sum.h
+++ b/src/shared/mpp/include/mpp_global_sum.h
@@ -1,10 +1,11 @@
-  function MPP_GLOBAL_SUM_( domain, field, flags, position, tile_count )
+  function MPP_GLOBAL_SUM_( domain, field, flags, position, tile_count, overflow_check)
     MPP_TYPE_ :: MPP_GLOBAL_SUM_
     type(domain2D), intent(in) :: domain
     MPP_TYPE_, intent(in) :: field(:,: MPP_EXTRA_INDICES_ )
     integer, intent(in), optional :: flags
     integer, intent(in), optional :: position
     integer, intent(in), optional :: tile_count
+    logical, intent(in), optional :: overflow_check
 
     MPP_TYPE_, dimension(:,:),       allocatable :: field2D
     MPP_TYPE_, dimension(:,:),       allocatable :: global2D
@@ -99,11 +100,32 @@
              MPP_GLOBAL_SUM_ = sum(gsum(1:domain%ntiles))
           end if
        end if
+    else if ( global_flag == BITWISE_EFP_SUM )then
+#ifdef DO_EFP_SUM_
+       !this is bitwise across different PE counts using EFP sum
+       if( ntile > 1 ) then 
+          call mpp_error( FATAL, 'MPP_GLOBAL_SUM_: multiple tile per pe is not supported for BITWISE_EFP_SUM')
+       endif
+       allocate( field2D (isc:iec,jsc:jec) )
+       do j = jsc, jec
+          do i = isc, iec
+             field2D(i,j) = sum( field(i+ioff:i+ioff,j+joff:j+joff MPP_EXTRA_INDICES_) )
+          end do
+       end do
+       !--- using efp sum.
+       if(efp_sum_overflow_check) then
+          MPP_GLOBAL_SUM_ = mpp_reproducing_sum(field2D, overflow_check=.true.)
+       else
+          MPP_GLOBAL_SUM_ = mpp_reproducing_sum(field2D, overflow_check=overflow_check)
+       endif
+#else
+        call mpp_error( FATAL, 'MPP_GLOBAL_SUM_: BITWISE_EFP_SUM is only implemented for real number, contact developer') 
+#endif
     else  !this is not bitwise-exact across different PE counts
        ioffset = domain%x(tile)%loffset*ishift; joffset = domain%y(tile)%loffset*jshift
        mygsum(tile) = sum( field(is+ioff:ie+ioff+ioffset, js+joff:je+joff+joffset MPP_EXTRA_INDICES_) )
        if(tile == ntile) then
-          MPP_GLOBAL_SUM_ = sum(mygsum)
+          MPP_GLOBAL_SUM_ = sum(mygsum(1:ntile))
           call mpp_sum( MPP_GLOBAL_SUM_, domain%list(:)%pe )
        end if
     end if
diff --git a/src/shared/mpp/include/mpp_group_update.h b/src/shared/mpp/include/mpp_group_update.h
new file mode 100644
index 0000000000..df4518e608
--- /dev/null
+++ b/src/shared/mpp/include/mpp_group_update.h
@@ -0,0 +1,973 @@
+! -*-f90-*-
+subroutine MPP_CREATE_GROUP_UPDATE_2D_(group, field, domain, flags, position, &
+     whalo, ehalo, shalo, nhalo)
+  type(mpp_group_update_type), intent(inout) :: group
+  MPP_TYPE_,            intent(inout)        :: field(:,:)
+  type(domain2D),       intent(inout)        :: domain  
+  integer,              intent(in), optional :: flags
+  integer,              intent(in), optional :: position
+  integer,              intent(in), optional :: whalo, ehalo, shalo, nhalo
+
+  MPP_TYPE_ :: field3D(size(field,1),size(field,2),1)
+  pointer( ptr, field3D )
+  ptr = LOC(field)
+
+  call mpp_create_group_update(group, field3D, domain, flags, position, whalo, ehalo, shalo, nhalo)
+
+  return
+
+end subroutine MPP_CREATE_GROUP_UPDATE_2D_
+
+subroutine MPP_CREATE_GROUP_UPDATE_3D_(group, field, domain, flags, position, whalo, ehalo, shalo, nhalo)
+  type(mpp_group_update_type), intent(inout) :: group
+  MPP_TYPE_,        intent(inout)        :: field(:,:,:)
+  type(domain2D),   intent(inout)        :: domain  
+  integer,          intent(in), optional :: flags
+  integer,          intent(in), optional :: position
+  integer,          intent(in), optional :: whalo, ehalo, shalo, nhalo ! specify halo region to be updated.
+
+  integer          :: update_position, update_whalo, update_ehalo, update_shalo, update_nhalo
+  integer          :: update_flags, isize, jsize, ksize
+  integer          :: nscalar
+  character(len=3) :: text
+  logical          :: set_mismatch, update_edge_only
+  logical          :: recv(8)
+
+  if(group%initialized) then
+      call mpp_error(FATAL, "MPP_CREATE_GROUP_UPDATE_3D: group is already initialized")
+  endif
+
+  if(present(whalo)) then
+     update_whalo = whalo
+     if(abs(update_whalo) > domain%whalo ) call mpp_error(FATAL, "MPP_CREATE_GROUP_UPDATE: "// &
+          "optional argument whalo should not be larger than the whalo when define domain.")
+  else
+     update_whalo = domain%whalo
+  end if
+  if(present(ehalo)) then
+     update_ehalo = ehalo
+     if(abs(update_ehalo) > domain%ehalo ) call mpp_error(FATAL, "MPP_CREATE_GROUP_UPDATE: "// &
+          "optional argument ehalo should not be larger than the ehalo when define domain.")
+  else
+     update_ehalo = domain%ehalo
+  end if
+  if(present(shalo)) then
+     update_shalo = shalo
+     if(abs(update_shalo) > domain%shalo ) call mpp_error(FATAL, "MPP_CREATE_GROUP_UPDATE: "// &
+          "optional argument shalo should not be larger than the shalo when define domain.")
+  else
+     update_shalo = domain%shalo
+  end if
+  if(present(nhalo)) then
+     update_nhalo = nhalo
+     if(abs(update_nhalo) > domain%nhalo ) call mpp_error(FATAL, "MPP_CREATE_GROUP_UPDATE: "// &
+          "optional argument nhalo should not be larger than the nhalo when define domain.")
+  else
+     update_nhalo = domain%nhalo
+  end if
+  update_position = CENTER
+  !--- when there is NINETY or MINUS_NINETY rotation for some contact, the salar data can not be on E or N-cell,
+  if(present(position)) then
+     update_position = position 
+     if(domain%rotated_ninety .AND. ( position == EAST .OR. position == NORTH ) )  &
+          call mpp_error(FATAL, 'MPP_CREATE_GROUP_UPDATE_3D: hen there is NINETY or MINUS_NINETY rotation, ' // &
+          'can not use scalar version update_domain for data on E or N-cell' )
+  end if
+
+  if( domain%max_ntile_pe > 1 ) then
+     call mpp_error(FATAL,'MPP_CREATE_GROUP_UPDATE: do not support multiple tile per processor')
+  endif
+
+  update_flags = XUPDATE+YUPDATE 
+  if(present(flags)) update_flags = flags
+
+  group%nscalar = group%nscalar + 1
+  nscalar = group%nscalar
+  if( nscalar > MAX_DOMAIN_FIELDS)then
+     write( text,'(i2)' ) MAX_DOMAIN_FIELDS
+     call mpp_error(FATAL,'MPP_CREATE_GROUP_UPDATE: MAX_DOMAIN_FIELDS='//text//' exceeded for group update.' )
+  endif
+
+  isize = size(field,1); jsize=size(field,2); ksize = size(field,3)
+
+  group%addrs_s(nscalar) = LOC(field)
+  if( group%nscalar == 1 ) then
+     group%flags_s  = update_flags
+     group%whalo_s  = update_whalo
+     group%ehalo_s  = update_ehalo
+     group%shalo_s  = update_shalo
+     group%nhalo_s  = update_nhalo
+     group%position = update_position
+     group%isize_s  = isize
+     group%jsize_s  = jsize
+     group%ksize_s  = ksize
+     call mpp_get_memory_domain(domain, group%is_s, group%ie_s, group%js_s, group%je_s, position=position)
+
+     update_edge_only = BTEST(update_flags, EDGEONLY)
+     recv(1) = BTEST(update_flags,EAST)
+     recv(3) = BTEST(update_flags,SOUTH)
+     recv(5) = BTEST(update_flags,WEST)
+     recv(7) = BTEST(update_flags,NORTH)
+     if( update_edge_only ) then
+        recv(2) = .false.
+        recv(4) = .false.
+        recv(6) = .false.
+        recv(8) = .false.
+        if( .NOT. (recv(1) .OR. recv(3) .OR. recv(5) .OR. recv(7)) ) then
+           recv(1) = .true.
+           recv(3) = .true.
+           recv(5) = .true.
+           recv(7) = .true.
+        endif
+     else
+        recv(2) = recv(1) .AND. recv(3)
+        recv(4) = recv(3) .AND. recv(5)
+        recv(6) = recv(5) .AND. recv(7)
+        recv(8) = recv(7) .AND. recv(1)
+     endif
+     group%recv_s = recv
+  else
+     set_mismatch = .false.
+     set_mismatch = set_mismatch .OR. (group%flags_s  .NE. update_flags)
+     set_mismatch = set_mismatch .OR. (group%whalo_s  .NE. update_whalo)
+     set_mismatch = set_mismatch .OR. (group%ehalo_s  .NE. update_ehalo)
+     set_mismatch = set_mismatch .OR. (group%shalo_s  .NE. update_shalo)
+     set_mismatch = set_mismatch .OR. (group%nhalo_s  .NE. update_nhalo)
+     set_mismatch = set_mismatch .OR. (group%position .NE. update_position)
+     set_mismatch = set_mismatch .OR. (group%isize_s  .NE. isize)
+     set_mismatch = set_mismatch .OR. (group%jsize_s  .NE. jsize)
+     set_mismatch = set_mismatch .OR. (group%ksize_s  .NE. ksize)
+
+     if(set_mismatch)then
+        write( text,'(i2)' ) nscalar
+        call mpp_error(FATAL,'MPP_CREATE_GROUP_UPDATE_3D: Incompatible field at count '//text//' for group update.' )
+     endif
+  endif
+
+  return
+
+end subroutine MPP_CREATE_GROUP_UPDATE_3D_
+
+
+subroutine MPP_CREATE_GROUP_UPDATE_4D_(group, field, domain, flags, position, &
+     whalo, ehalo, shalo, nhalo)
+  type(mpp_group_update_type), intent(inout) :: group
+  MPP_TYPE_,            intent(inout)        :: field(:,:,:,:)
+  type(domain2D),       intent(inout)        :: domain  
+  integer,              intent(in), optional :: flags
+  integer,              intent(in), optional :: position
+  integer,              intent(in), optional :: whalo, ehalo, shalo, nhalo
+
+  MPP_TYPE_ :: field3D(size(field,1),size(field,2),size(field,3)*size(field,4))
+  pointer( ptr, field3D )
+  ptr = LOC(field)
+
+  call mpp_create_group_update(group, field3D, domain, flags, position, whalo, ehalo, shalo, nhalo)
+
+  return
+
+end subroutine MPP_CREATE_GROUP_UPDATE_4D_
+
+subroutine MPP_CREATE_GROUP_UPDATE_2D_V_( group, fieldx, fieldy, domain, flags, gridtype, &
+                                          whalo, ehalo, shalo, nhalo)
+
+  type(mpp_group_update_type), intent(inout) :: group
+  MPP_TYPE_,        intent(inout)            :: fieldx(:,:), fieldy(:,:)
+  type(domain2D),   intent(inout)            :: domain
+  integer,          intent(in),     optional :: flags, gridtype
+  integer,          intent(in),     optional :: whalo, ehalo, shalo, nhalo
+  MPP_TYPE_ :: field3Dx(size(fieldx,1),size(fieldx,2),1)
+  MPP_TYPE_ :: field3Dy(size(fieldy,1),size(fieldy,2),1)
+  pointer( ptrx, field3Dx )
+  pointer( ptry, field3Dy )
+  ptrx = LOC(fieldx)
+  ptry = LOC(fieldy)
+
+  call  mpp_create_group_update(group, field3Dx, field3Dy, domain, flags, gridtype, &
+                                          whalo, ehalo, shalo, nhalo)
+
+  return
+
+end subroutine MPP_CREATE_GROUP_UPDATE_2D_V_
+
+
+
+subroutine MPP_CREATE_GROUP_UPDATE_3D_V_( group, fieldx, fieldy, domain, flags, gridtype, &
+                                          whalo, ehalo, shalo, nhalo)
+  type(mpp_group_update_type), intent(inout) :: group
+  MPP_TYPE_,        intent(inout)        :: fieldx(:,:,:), fieldy(:,:,:)
+  type(domain2D),   intent(inout)        :: domain
+  integer,          intent(in), optional :: flags, gridtype
+  integer,          intent(in), optional :: whalo, ehalo, shalo, nhalo
+
+  integer          :: update_whalo, update_ehalo, update_shalo, update_nhalo
+  integer          :: update_flags, isize_x, jsize_x, ksize_x, isize_y, jsize_y, ksize_y
+  integer          :: nvector, update_gridtype, position_x, position_y
+  character(len=3) :: text
+  logical          :: set_mismatch, update_edge_only
+  logical          :: recv(8)
+
+
+  if(group%initialized) then
+     call mpp_error(FATAL, "MPP_CREATE_GROUP_UPDATE_V: group is already initialized")
+  endif
+
+  if(present(whalo)) then
+     update_whalo = whalo
+     if(abs(update_whalo) > domain%whalo ) call mpp_error(FATAL, "MPP_CREATE_GROUP_UPDATE_V: "// &
+          "optional argument whalo should not be larger than the whalo when define domain.")
+  else
+     update_whalo = domain%whalo
+  end if
+  if(present(ehalo)) then
+     update_ehalo = ehalo
+     if(abs(update_ehalo) > domain%ehalo ) call mpp_error(FATAL, "MPP_CREATE_GROUP_UPDATE_V: "// &
+          "optional argument ehalo should not be larger than the ehalo when define domain.")
+  else
+     update_ehalo = domain%ehalo
+  end if
+  if(present(shalo)) then
+     update_shalo = shalo
+     if(abs(update_shalo) > domain%shalo ) call mpp_error(FATAL, "MPP_CREATE_GROUP_UPDATE_V: "// &
+          "optional argument shalo should not be larger than the shalo when define domain.")
+  else
+     update_shalo = domain%shalo
+  end if
+  if(present(nhalo)) then
+     update_nhalo = nhalo
+     if(abs(update_nhalo) > domain%nhalo ) call mpp_error(FATAL, "MPP_CREATE_GROUP_UPDATE_V: "// &
+          "optional argument nhalo should not be larger than the nhalo when define domain.")
+  else
+     update_nhalo = domain%nhalo
+  end if
+
+  update_gridtype = AGRID
+  if(PRESENT(gridtype)) update_gridtype = gridtype
+
+  if( domain%max_ntile_pe > 1 ) then
+     call mpp_error(FATAL,'MPP_CREATE_GROUP_UPDATE_V: do not support multiple tile per processor')
+  endif
+
+  update_flags = XUPDATE+YUPDATE   !default
+  if( PRESENT(flags) )update_flags = flags
+
+  group%nvector = group%nvector + 1
+  nvector = group%nvector
+  if( nvector > MAX_DOMAIN_FIELDS)then
+     write( text,'(i2)' ) MAX_DOMAIN_FIELDS
+     call mpp_error(FATAL,'MPP_CREATE_GROUP_UPDATE_V: MAX_DOMAIN_FIELDS='//text//' exceeded for group update.' )
+  endif
+
+  isize_x = size(fieldx,1); jsize_x = size(fieldx,2); ksize_x = size(fieldx,3) 
+  isize_y = size(fieldy,1); jsize_y = size(fieldy,2); ksize_y = size(fieldy,3) 
+  
+  if(ksize_x .NE. ksize_y) call mpp_error(FATAL,  &
+           'MPP_CREATE_GROUP_UPDATE_V: mismatch of ksize between fieldx and fieldy')
+
+  group%addrs_x(nvector) = LOC(fieldx)
+  group%addrs_y(nvector) = LOC(fieldy)
+
+  if( group%nvector == 1 ) then
+     group%flags_v  = update_flags
+     group%whalo_v  = update_whalo
+     group%ehalo_v  = update_ehalo
+     group%shalo_v  = update_shalo
+     group%nhalo_v  = update_nhalo
+     group%gridtype = update_gridtype
+     group%isize_x  = isize_x
+     group%jsize_x  = jsize_x
+     group%isize_y  = isize_y
+     group%jsize_y  = jsize_y
+     group%ksize_v  = ksize_x
+     update_edge_only = BTEST(update_flags, EDGEONLY)
+     recv(1) = BTEST(update_flags,EAST)
+     recv(3) = BTEST(update_flags,SOUTH)
+     recv(5) = BTEST(update_flags,WEST)
+     recv(7) = BTEST(update_flags,NORTH)
+     if( update_edge_only ) then
+        recv(2) = .false.
+        recv(4) = .false.
+        recv(6) = .false.
+        recv(8) = .false.
+        if( .NOT. (recv(1) .OR. recv(3) .OR. recv(5) .OR. recv(7)) ) then
+           recv(1) = .true.
+           recv(3) = .true.
+           recv(5) = .true.
+           recv(7) = .true.
+        endif
+     else
+        recv(2) = recv(1) .AND. recv(3)
+        recv(4) = recv(3) .AND. recv(5)
+        recv(6) = recv(5) .AND. recv(7)
+        recv(8) = recv(7) .AND. recv(1)
+     endif
+     group%recv_v = recv
+     select case(group%gridtype)
+     case (AGRID)
+        position_x = CENTER
+        position_y = CENTER
+     case (BGRID_NE, BGRID_SW)
+        position_x = CORNER
+        position_y = CORNER
+     case (CGRID_NE, CGRID_SW)
+        position_x = EAST
+        position_y = NORTH
+     case (DGRID_NE, DGRID_SW)
+        position_x = NORTH
+        position_y = EAST
+     case default
+        call mpp_error(FATAL, "mpp_CREATE_GROUP_UPDATE_V: invalid value of gridtype")
+     end select
+
+     call mpp_get_memory_domain(domain, group%is_x, group%ie_x, group%js_x, group%je_x, position=position_x)
+     call mpp_get_memory_domain(domain, group%is_y, group%ie_y, group%js_y, group%je_y, position=position_y)
+  else
+     set_mismatch = .false.
+     set_mismatch = set_mismatch .OR. (group%flags_v  .NE. update_flags)
+     set_mismatch = set_mismatch .OR. (group%whalo_v  .NE. update_whalo)
+     set_mismatch = set_mismatch .OR. (group%ehalo_v  .NE. update_ehalo)
+     set_mismatch = set_mismatch .OR. (group%shalo_v  .NE. update_shalo)
+     set_mismatch = set_mismatch .OR. (group%nhalo_v  .NE. update_nhalo)
+     set_mismatch = set_mismatch .OR. (group%gridtype .NE. update_gridtype)
+     set_mismatch = set_mismatch .OR. (group%isize_x  .NE. isize_x)
+     set_mismatch = set_mismatch .OR. (group%jsize_x  .NE. jsize_x)
+     set_mismatch = set_mismatch .OR. (group%isize_y  .NE. isize_y)
+     set_mismatch = set_mismatch .OR. (group%jsize_y  .NE. jsize_y)
+     set_mismatch = set_mismatch .OR. (group%ksize_v  .NE. ksize_x)
+
+     if(set_mismatch)then
+        write( text,'(i2)' ) nvector
+        call mpp_error(FATAL,'MPP_CREATE_GROUP_UPDATE_V: Incompatible field at count '//text//' for group update.' )
+     endif
+  endif
+
+  return
+
+end subroutine MPP_CREATE_GROUP_UPDATE_3D_V_
+
+subroutine MPP_CREATE_GROUP_UPDATE_4D_V_( group, fieldx, fieldy, domain, flags, gridtype, &
+                                          whalo, ehalo, shalo, nhalo)
+
+  type(mpp_group_update_type), intent(inout) :: group
+  MPP_TYPE_,        intent(inout)            :: fieldx(:,:,:,:), fieldy(:,:,:,:)
+  type(domain2D),   intent(inout)            :: domain
+  integer,          intent(in),     optional :: flags, gridtype
+  integer,          intent(in),     optional :: whalo, ehalo, shalo, nhalo
+  MPP_TYPE_ :: field3Dx(size(fieldx,1),size(fieldx,2),size(fieldx,3)*size(fieldx,4))
+  MPP_TYPE_ :: field3Dy(size(fieldy,1),size(fieldy,2),size(fieldy,3)*size(fieldy,4))
+  pointer( ptrx, field3Dx )
+  pointer( ptry, field3Dy )
+  ptrx = LOC(fieldx)
+  ptry = LOC(fieldy)
+
+  call  mpp_create_group_update(group, field3Dx, field3Dy, domain, flags, gridtype, &
+                                          whalo, ehalo, shalo, nhalo)
+
+  return
+
+end subroutine MPP_CREATE_GROUP_UPDATE_4D_V_
+
+
+subroutine MPP_DO_GROUP_UPDATE_(group, domain, d_type)
+  type(mpp_group_update_type), intent(inout) :: group
+  type(domain2D),              intent(inout) :: domain  
+  MPP_TYPE_,                   intent(in)    :: d_type
+
+  integer   :: nscalar, nvector, nlist
+  logical   :: recv_v(8)
+  integer   :: nsend, nrecv, flags_v
+  integer   :: msgsize
+  integer   :: from_pe, to_pe, buffer_pos, pos
+  integer   :: ksize, is, ie, js, je
+  integer   :: n, l, m, i, j, k, buffer_start_pos, nk
+  integer   :: shift, gridtype, midpoint
+  integer   :: npack, nunpack, rotation
+  character(len=8)            :: text
+
+  MPP_TYPE_ :: buffer(mpp_domains_stack_size)
+  MPP_TYPE_ :: field (group%is_s:group%ie_s,group%js_s:group%je_s, group%ksize_s)
+  MPP_TYPE_ :: fieldx(group%is_x:group%ie_x,group%js_x:group%je_x, group%ksize_v)
+  MPP_TYPE_ :: fieldy(group%is_y:group%ie_y,group%js_y:group%je_y, group%ksize_v)
+  pointer(ptr, buffer )
+  pointer(ptr_field, field)
+  pointer(ptr_fieldx, fieldx)
+  pointer(ptr_fieldy, fieldy)
+
+  nscalar = group%nscalar
+  nvector = group%nvector
+  nlist   = size(domain%list(:))
+  gridtype = group%gridtype
+
+  !--- ksize_s must equal ksize_v 
+  if(nvector > 0 .AND. nscalar > 0) then
+     if(group%ksize_s .NE. group%ksize_v) then
+        call mpp_error(FATAL, "MPP_DO_GROUP_UPDATE: ksize_s and ksize_v are not equal")
+     endif
+     ksize = group%ksize_s
+  else if (nscalar > 0) then
+     ksize = group%ksize_s
+  else if (nvector > 0) then
+     ksize = group%ksize_v
+  else
+     call mpp_error(FATAL, "MPP_DO_GROUP_UPDATE: nscalar and nvector are all 0")
+  endif
+  if(nvector > 0) recv_v = group%recv_v
+
+  ptr = LOC(mpp_domains_stack)
+
+  !--- set reset_index_s and reset_index_v to 0 
+  group%reset_index_s = 0
+  group%reset_index_v = 0
+
+  if(.not. group%initialized) call set_group_update(group,domain)
+
+  nrecv = group%nrecv
+  nsend = group%nsend
+
+  !---pre-post receive.
+  call mpp_clock_begin(group_recv_clock)
+  do m = 1, nrecv
+     msgsize = group%recv_size(m)
+     from_pe = group%from_pe(m)
+     if( msgsize .GT. 0 )then
+        buffer_pos = group%buffer_pos_recv(m)
+        call mpp_recv( buffer(buffer_pos+1), glen=msgsize, from_pe=from_pe, block=.false., &
+             tag=COMM_TAG_1)
+     end if
+  end do
+
+ !pack the data
+  call mpp_clock_end(group_recv_clock)
+
+  flags_v = group%flags_v
+  npack = group%npack
+
+  call mpp_clock_begin(group_pack_clock)
+  !pack the data
+  buffer_start_pos = 0
+#include 
+  call mpp_clock_end(group_pack_clock)
+
+  call mpp_clock_begin(group_send_clock)
+  do n = 1, nsend  
+     msgsize = group%send_size(n)
+     if( msgsize .GT. 0 )then
+        buffer_pos = group%buffer_pos_send(n)
+        to_pe = group%to_pe(n)
+        call mpp_send( buffer(buffer_pos+1), plen=msgsize, to_pe=to_pe, tag=COMM_TAG_1)
+     endif
+  enddo
+  call mpp_clock_end(group_send_clock)
+
+  if(nrecv>0) then
+     call mpp_clock_begin(group_wait_clock)
+     call mpp_sync_self(check=EVENT_RECV)
+     call mpp_clock_end(group_wait_clock)
+  endif
+
+  !---unpack the buffer
+  nunpack = group%nunpack
+  call mpp_clock_begin(group_unpk_clock)
+#include 
+  call mpp_clock_end(group_unpk_clock)
+
+  ! ---northern boundary fold
+  shift = 0
+  if(domain%symmetry) shift = 1
+  if( nvector >0 .AND. BTEST(domain%fold,NORTH) .AND. (.NOT.BTEST(flags_v,SCALAR_BIT)) )then
+     j = domain%y(1)%global%end+shift
+     if( domain%y(1)%data%begin.LE.j .AND. j.LE.domain%y(1)%data%end+shift )then !fold is within domain
+        !poles set to 0: BGRID only
+        if( gridtype.EQ.BGRID_NE )then
+           midpoint = (domain%x(1)%global%begin+domain%x(1)%global%end-1+shift)/2
+           j  = domain%y(1)%global%end+shift
+           is = domain%x(1)%global%begin; ie = domain%x(1)%global%end+shift
+           if( .NOT. domain%symmetry ) is = is - 1
+           do i = is ,ie, midpoint
+              if( domain%x(1)%data%begin.LE.i .AND. i.LE. domain%x(1)%data%end+shift )then
+                 do l=1,nvector
+                    ptr_fieldx = group%addrs_x(l)
+                    ptr_fieldy = group%addrs_y(l)
+                    do k = 1,ksize
+                       fieldx(i,j,k) = 0.
+                       fieldy(i,j,k) = 0.
+                    end do
+                 end do
+              end if
+           end do
+        endif
+        ! the following code code block correct an error where the data in your halo coming from
+        ! other half may have the wrong sign
+        !off west edge, when update north or west direction
+        j = domain%y(1)%global%end+shift
+        if ( recv_v(7) .OR. recv_v(5) ) then
+           select case(gridtype)
+           case(BGRID_NE)
+              if(domain%symmetry) then
+                 is = domain%x(1)%global%begin
+              else
+                 is = domain%x(1)%global%begin - 1
+              end if
+              if( is.GT.domain%x(1)%data%begin )then
+
+                 if( 2*is-domain%x(1)%data%begin.GT.domain%x(1)%data%end+shift ) &
+                      call mpp_error( FATAL, 'MPP_DO_UPDATE_V: folded-north BGRID_NE west edge ubound error.' )
+                 do l=1,nvector
+                    ptr_fieldx = group%addrs_x(l)
+                    ptr_fieldy = group%addrs_y(l)
+                    do k = 1,ksize
+                       do i = domain%x(1)%data%begin,is-1
+                          fieldx(i,j,k) = fieldx(2*is-i,j,k)
+                          fieldy(i,j,k) = fieldy(2*is-i,j,k)
+                       end do
+                    end do
+                 end do
+              end if
+           case(CGRID_NE)
+              is = domain%x(1)%global%begin
+              if( is.GT.domain%x(1)%data%begin )then
+                 if( 2*is-domain%x(1)%data%begin-1.GT.domain%x(1)%data%end ) &
+                      call mpp_error( FATAL, 'MPP_DO_UPDATE_V: folded-north CGRID_NE west edge ubound error.' )
+                 do l=1,nvector
+                    ptr_fieldy = group%addrs_y(l)
+                    do k = 1,ksize
+                       do i = domain%x(1)%data%begin,is-1
+                          fieldy(i,j,k) = fieldy(2*is-i-1,j,k)
+                       end do
+                    end do
+                 end do
+              end if
+           end select
+        end if
+        !off east edge
+        is = domain%x(1)%global%end
+        if(domain%x(1)%cyclic .AND. is.LT.domain%x(1)%data%end )then
+           ie = domain%x(1)%data%end
+           is = is + 1
+           select case(gridtype)
+           case(BGRID_NE)
+              is = is + shift
+              ie = ie + shift
+              do l=1,nvector
+                 ptr_fieldx = group%addrs_x(l)
+                 ptr_fieldy = group%addrs_y(l)
+                 do k = 1,ksize
+                    do i = is,ie
+                       fieldx(i,j,k) = -fieldx(i,j,k)
+                       fieldy(i,j,k) = -fieldy(i,j,k)
+                    end do
+                 end do
+              end do
+           case(CGRID_NE)
+              do l=1,nvector
+                 ptr_fieldy = group%addrs_y(l)
+                 do k = 1,ksize
+                    do i = is, ie
+                       fieldy(i,j,k) = -fieldy(i,j,k)
+                    end do
+                 end do
+              end do
+           end select
+        end if
+     end if
+  else if( BTEST(domain%fold,SOUTH) .OR. BTEST(domain%fold,WEST) .OR. BTEST(domain%fold,EAST) ) then
+     call mpp_error(FATAL, "MPP_DO_GROUP_UPDATE: this interface does not support folded_south, " // &
+          "folded_west of folded_east, contact developer")
+  endif
+
+  if(nsend>0) then
+     call mpp_clock_begin(group_wait_clock)
+     call mpp_sync_self( )
+     call mpp_clock_end(group_wait_clock)
+  endif
+
+end subroutine MPP_DO_GROUP_UPDATE_
+
+
+subroutine MPP_START_GROUP_UPDATE_(group, domain, d_type, reuse_buffer)
+  type(mpp_group_update_type), intent(inout) :: group
+  type(domain2D),              intent(inout) :: domain  
+  MPP_TYPE_,                   intent(in)    :: d_type
+  logical,  optional,          intent(in)    :: reuse_buffer
+
+  integer   :: nscalar, nvector
+  integer   :: nsend, nrecv, flags_v
+  integer   :: msgsize, npack, rotation
+  integer   :: from_pe, to_pe, buffer_pos, pos
+  integer   :: ksize, is, ie, js, je
+  integer   :: n, l, m, i, j, k, buffer_start_pos, nk
+  logical   :: reuse_buf_pos
+  character(len=8)            :: text
+
+  MPP_TYPE_ :: buffer(size(mpp_domains_stack_nonblock(:)))
+  MPP_TYPE_ :: field (group%is_s:group%ie_s,group%js_s:group%je_s, group%ksize_s)
+  MPP_TYPE_ :: fieldx(group%is_x:group%ie_x,group%js_x:group%je_x, group%ksize_v)
+  MPP_TYPE_ :: fieldy(group%is_y:group%ie_y,group%js_y:group%je_y, group%ksize_v)
+  pointer( ptr, buffer )
+  pointer(ptr_field, field)
+  pointer(ptr_fieldx, fieldx)
+  pointer(ptr_fieldy, fieldy)
+
+  nscalar = group%nscalar
+  nvector = group%nvector
+
+  if(nscalar>0) then
+     ksize = group%ksize_s
+  else
+     ksize = group%ksize_v
+  endif
+
+  !--- set reset_index_s and reset_index_v to 0 
+  group%reset_index_s = 0
+  group%reset_index_v = 0
+
+  reuse_buf_pos = .FALSE.
+  if (PRESENT(reuse_buffer)) reuse_buf_pos = reuse_buffer
+
+  if (.not. group%initialized) then
+    call set_group_update(group,domain)
+  endif
+
+  if (.not. reuse_buf_pos) then
+     group%buffer_start_pos = nonblock_group_buffer_pos
+     nonblock_group_buffer_pos = nonblock_group_buffer_pos + group%tot_msgsize
+     mpp_domains_stack_hwm = nonblock_group_buffer_pos + 1    
+     if( mpp_domains_stack_hwm .GT. mpp_domains_stack_size )then
+        write( text,'(i8)' )mpp_domains_stack_hwm
+        call mpp_error( FATAL, 'set_group_update: mpp_domains_stack overflow, '// &
+                     'call mpp_domains_set_stack_size('//trim(text)//') from all PEs.' )
+     end if
+
+  else if( group%buffer_start_pos < 0 ) then
+     call mpp_error(FATAL, "MPP_START_GROUP_UPDATE: group%buffer_start_pos is not set")
+  endif
+
+  nrecv = group%nrecv
+  nsend = group%nsend
+
+  ptr = LOC(mpp_domains_stack_nonblock)
+
+  ! Make sure it is not in the middle of the old version of non-blocking halo update.
+  if(num_update>0) call mpp_error(FATAL, "MPP_START_GROUP_UPDATE: can not be called in the middle of "// &
+              "mpp_start_update_domains/mpp_complete_update_domains call")
+
+  num_nonblock_group_update = num_nonblock_group_update + 1
+
+  !---pre-post receive.
+  call mpp_clock_begin(nonblock_group_recv_clock)
+  do m = 1, nrecv
+     msgsize = group%recv_size(m)
+     from_pe = group%from_pe(m)
+     if( msgsize .GT. 0 )then
+        buffer_pos = group%buffer_pos_recv(m) + group%buffer_start_pos
+        call mpp_recv( buffer(buffer_pos+1), glen=msgsize, from_pe=from_pe, block=.false., &
+             tag=COMM_TAG_1, request=group%request_recv(m))
+#ifdef use_libMPI
+        group%type_recv(m) = MPI_TYPE_
+#endif
+     end if
+  end do
+  call mpp_clock_end(nonblock_group_recv_clock)
+
+  flags_v = group%flags_v
+
+  !pack the data
+  call mpp_clock_begin(nonblock_group_pack_clock)
+  npack = group%npack
+  buffer_start_pos = group%buffer_start_pos
+#include 
+  call mpp_clock_end(nonblock_group_pack_clock)
+
+  call mpp_clock_begin(nonblock_group_send_clock)
+  do n = 1, nsend  
+     msgsize = group%send_size(n)
+     if( msgsize .GT. 0 )then
+        buffer_pos = group%buffer_pos_send(n) + group%buffer_start_pos
+        to_pe = group%to_pe(n)
+        call mpp_send( buffer(buffer_pos+1), plen=msgsize, to_pe=to_pe, tag=COMM_TAG_1, &
+                       request=group%request_send(n))
+     endif
+  enddo
+  call mpp_clock_end(nonblock_group_send_clock)
+
+end subroutine MPP_START_GROUP_UPDATE_
+
+subroutine MPP_COMPLETE_GROUP_UPDATE_(group, domain, d_type)
+  type(mpp_group_update_type), intent(inout) :: group
+  type(domain2D),              intent(inout) :: domain  
+  MPP_TYPE_,                   intent(in)    :: d_type
+
+  integer   :: nsend, nrecv, nscalar, nvector
+  integer   :: k, buffer_pos, msgsize, pos, m, n, l
+  integer   :: is, ie, js, je, dir, ksize, i, j
+  integer   :: shift, gridtype, midpoint, flags_v
+  integer   :: nunpack, rotation, buffer_start_pos, nk
+  logical   :: recv_v(8)
+  MPP_TYPE_ :: buffer(size(mpp_domains_stack_nonblock(:)))
+  MPP_TYPE_ :: field (group%is_s:group%ie_s,group%js_s:group%je_s, group%ksize_s)
+  MPP_TYPE_ :: fieldx(group%is_x:group%ie_x,group%js_x:group%je_x, group%ksize_v)
+  MPP_TYPE_ :: fieldy(group%is_y:group%ie_y,group%js_y:group%je_y, group%ksize_v)
+  pointer(ptr, buffer )
+  pointer(ptr_field, field)
+  pointer(ptr_fieldx, fieldx)
+  pointer(ptr_fieldy, fieldy)
+
+  gridtype = group%gridtype
+  flags_v = group%flags_v
+  nscalar = group%nscalar
+  nvector = group%nvector
+  nrecv = group%nrecv
+  nsend = group%nsend
+  if(nscalar>0) then
+     ksize = group%ksize_s
+  else
+     ksize = group%ksize_v
+  endif
+  if(nvector > 0) recv_v = group%recv_v
+  ptr = LOC(mpp_domains_stack_nonblock)
+
+  if(num_nonblock_group_update < 1) call mpp_error(FATAL, &
+    'mpp_start_group_update must be called before calling mpp_end_group_update')  
+  num_nonblock_group_update = num_nonblock_group_update - 1
+  complete_group_update_on = .true.
+
+  if(nrecv>0) then
+     call mpp_clock_begin(nonblock_group_wait_clock)
+     call mpp_sync_self(check=EVENT_RECV, request=group%request_recv(1:nrecv), &
+                        msg_size=group%recv_size(1:nrecv), msg_type=group%type_recv(1:nrecv))
+     call mpp_clock_end(nonblock_group_wait_clock)
+  endif
+
+  !---unpack the buffer
+  nunpack = group%nunpack
+
+  call mpp_clock_begin(nonblock_group_unpk_clock)
+  buffer_start_pos = group%buffer_start_pos
+#include 
+  call mpp_clock_end(nonblock_group_unpk_clock)
+
+  ! ---northern boundary fold
+  shift = 0
+  if(domain%symmetry) shift = 1
+  if( nvector >0 .AND. BTEST(domain%fold,NORTH) .AND. (.NOT.BTEST(flags_v,SCALAR_BIT)) )then
+     j = domain%y(1)%global%end+shift
+     if( domain%y(1)%data%begin.LE.j .AND. j.LE.domain%y(1)%data%end+shift )then !fold is within domain
+        !poles set to 0: BGRID only
+        if( gridtype.EQ.BGRID_NE )then
+           midpoint = (domain%x(1)%global%begin+domain%x(1)%global%end-1+shift)/2
+           j  = domain%y(1)%global%end+shift
+           is = domain%x(1)%global%begin; ie = domain%x(1)%global%end+shift
+           if( .NOT. domain%symmetry ) is = is - 1
+           do i = is ,ie, midpoint
+              if( domain%x(1)%data%begin.LE.i .AND. i.LE. domain%x(1)%data%end+shift )then
+                 do l=1,nvector
+                    ptr_fieldx = group%addrs_x(l)
+                    ptr_fieldy = group%addrs_y(l)
+                    do k = 1,ksize
+                       fieldx(i,j,k) = 0.
+                       fieldy(i,j,k) = 0.
+                    end do
+                 end do
+              end if
+           end do
+        endif
+        ! the following code code block correct an error where the data in your halo coming from
+        ! other half may have the wrong sign
+        !off west edge, when update north or west direction
+        j = domain%y(1)%global%end+shift
+        if ( recv_v(7) .OR. recv_v(5) ) then
+           select case(gridtype)
+           case(BGRID_NE)
+              if(domain%symmetry) then
+                 is = domain%x(1)%global%begin
+              else
+                 is = domain%x(1)%global%begin - 1
+              end if
+              if( is.GT.domain%x(1)%data%begin )then
+
+                 if( 2*is-domain%x(1)%data%begin.GT.domain%x(1)%data%end+shift ) &
+                      call mpp_error( FATAL, 'MPP_DO_UPDATE_V: folded-north BGRID_NE west edge ubound error.' )
+                 do l=1,nvector
+                    ptr_fieldx = group%addrs_x(l)
+                    ptr_fieldy = group%addrs_y(l)
+                    do k = 1,ksize
+                       do i = domain%x(1)%data%begin,is-1
+                          fieldx(i,j,k) = fieldx(2*is-i,j,k)
+                          fieldy(i,j,k) = fieldy(2*is-i,j,k)
+                       end do
+                    end do
+                 end do
+              end if
+           case(CGRID_NE)
+              is = domain%x(1)%global%begin
+              if( is.GT.domain%x(1)%data%begin )then
+                 if( 2*is-domain%x(1)%data%begin-1.GT.domain%x(1)%data%end ) &
+                      call mpp_error( FATAL, 'MPP_DO_UPDATE_V: folded-north CGRID_NE west edge ubound error.' )
+                 do l=1,nvector
+                    ptr_fieldy = group%addrs_y(l)
+                    do k = 1,ksize
+                       do i = domain%x(1)%data%begin,is-1
+                          fieldy(i,j,k) = fieldy(2*is-i-1,j,k)
+                       end do
+                    end do
+                 end do
+              end if
+           end select
+        end if
+        !off east edge
+        is = domain%x(1)%global%end
+        if(domain%x(1)%cyclic .AND. is.LT.domain%x(1)%data%end )then
+           ie = domain%x(1)%data%end
+           is = is + 1
+           select case(gridtype)
+           case(BGRID_NE)
+              is = is + shift
+              ie = ie + shift
+              do l=1,nvector
+                 ptr_fieldx = group%addrs_x(l)
+                 ptr_fieldy = group%addrs_y(l)
+                 do k = 1,ksize
+                    do i = is,ie
+                       fieldx(i,j,k) = -fieldx(i,j,k)
+                       fieldy(i,j,k) = -fieldy(i,j,k)
+                    end do
+                 end do
+              end do
+           case(CGRID_NE)
+              do l=1,nvector
+                 ptr_fieldy = group%addrs_y(l)
+                 do k = 1,ksize
+                    do i = is, ie
+                       fieldy(i,j,k) = -fieldy(i,j,k)
+                    end do
+                 end do
+              end do
+           end select
+        end if
+     end if
+  else if( BTEST(domain%fold,SOUTH) .OR. BTEST(domain%fold,WEST) .OR. BTEST(domain%fold,EAST) ) then
+     call mpp_error(FATAL, "MPP_COMPLETE_GROUP_UPDATE: this interface does not support folded_south, " // &
+          "folded_west of folded_east, contact developer")
+  endif
+
+  if(nsend>0) then
+     call mpp_clock_begin(nonblock_group_wait_clock)
+     call mpp_sync_self(check=EVENT_SEND, request=group%request_send(1:nsend) )
+     call mpp_clock_end(nonblock_group_wait_clock)
+  endif
+
+  if( num_nonblock_group_update == 0) then
+     nonblock_group_buffer_pos   = 0
+  endif
+
+end subroutine MPP_COMPLETE_GROUP_UPDATE_
+
+subroutine MPP_RESET_GROUP_UPDATE_FIELD_2D_(group, field)
+  type(mpp_group_update_type), intent(inout) :: group
+  MPP_TYPE_,                   intent(in)    :: field(:,:)
+
+  group%reset_index_s = group%reset_index_s + 1
+
+  if(group%reset_index_s > group%nscalar) &
+     call mpp_error(FATAL, "MPP_RESET_GROUP_UPDATE_FIELD_2D_: group%reset_index_s > group%nscalar")
+  if(size(field,1) .NE. group%isize_s .OR. size(field,2) .NE. group%jsize_s .OR. group%ksize_s .NE. 1) &
+     call mpp_error(FATAL, "MPP_RESET_GROUP_UPDATE_FIELD_2D_: size of field does not match the size stored in group") 
+
+  group%addrs_s(group%reset_index_s) = LOC(field)
+
+end subroutine MPP_RESET_GROUP_UPDATE_FIELD_2D_
+
+subroutine MPP_RESET_GROUP_UPDATE_FIELD_3D_(group, field)
+  type(mpp_group_update_type), intent(inout) :: group
+  MPP_TYPE_,                   intent(in)    :: field(:,:,:)
+
+  group%reset_index_s = group%reset_index_s + 1
+
+  if(group%reset_index_s > group%nscalar) &
+     call mpp_error(FATAL, "MPP_RESET_GROUP_UPDATE_FIELD_3D_: group%reset_index_s > group%nscalar")
+  if(size(field,1) .NE. group%isize_s .OR. size(field,2) .NE. group%jsize_s .OR. size(field,3) .NE. group%ksize_s) &
+     call mpp_error(FATAL, "MPP_RESET_GROUP_UPDATE_FIELD_3D_: size of field does not match the size stored in group") 
+
+  group%addrs_s(group%reset_index_s) = LOC(field)
+
+end subroutine MPP_RESET_GROUP_UPDATE_FIELD_3D_
+
+subroutine MPP_RESET_GROUP_UPDATE_FIELD_4D_(group, field)
+  type(mpp_group_update_type), intent(inout) :: group
+  MPP_TYPE_,                   intent(in)    :: field(:,:,:,:)
+
+  group%reset_index_s = group%reset_index_s + 1
+
+  if(group%reset_index_s > group%nscalar) &
+     call mpp_error(FATAL, "MPP_RESET_GROUP_UPDATE_FIELD_4D_: group%reset_index_s > group%nscalar")
+  if(size(field,1) .NE. group%isize_s .OR. size(field,2) .NE. group%jsize_s .OR. &
+              size(field,3)*size(field,4) .NE. group%ksize_s) &
+     call mpp_error(FATAL, "MPP_RESET_GROUP_UPDATE_FIELD_4D_: size of field does not match the size stored in group") 
+
+  group%addrs_s(group%reset_index_s) = LOC(field)
+
+end subroutine MPP_RESET_GROUP_UPDATE_FIELD_4D_
+
+
+subroutine MPP_RESET_GROUP_UPDATE_FIELD_2D_V_(group, fieldx, fieldy)
+  type(mpp_group_update_type), intent(inout) :: group
+  MPP_TYPE_,                   intent(in)    :: fieldx(:,:), fieldy(:,:)
+  integer :: indx
+
+  group%reset_index_v = group%reset_index_v + 1
+
+  if(group%reset_index_v > group%nvector) &
+     call mpp_error(FATAL, "MPP_RESET_GROUP_UPDATE_FIELD_2D_V_: group%reset_index_v > group%nvector")
+  if(size(fieldx,1) .NE. group%isize_x .OR. size(fieldx,2) .NE. group%jsize_x .OR. group%ksize_v .NE. 1) &
+     call mpp_error(FATAL, "MPP_RESET_GROUP_UPDATE_FIELD_2D_V_: size of fieldx does not match the size stored in group") 
+  if(size(fieldy,1) .NE. group%isize_y .OR. size(fieldy,2) .NE. group%jsize_y ) &
+     call mpp_error(FATAL, "MPP_RESET_GROUP_UPDATE_FIELD_2D_V_: size of fieldy does not match the size stored in group") 
+
+  group%addrs_x(group%reset_index_v) = LOC(fieldx)
+  group%addrs_y(group%reset_index_v) = LOC(fieldy)
+
+end subroutine MPP_RESET_GROUP_UPDATE_FIELD_2D_V_
+
+
+subroutine MPP_RESET_GROUP_UPDATE_FIELD_3D_V_(group, fieldx, fieldy)
+  type(mpp_group_update_type), intent(inout) :: group
+  MPP_TYPE_,                   intent(in)    :: fieldx(:,:,:), fieldy(:,:,:)
+  integer :: indx
+
+  group%reset_index_v = group%reset_index_v + 1
+
+  if(group%reset_index_v > group%nvector) &
+     call mpp_error(FATAL, "MPP_RESET_GROUP_UPDATE_FIELD_3D_V_: group%reset_index_v > group%nvector")
+  if(size(fieldx,1) .NE. group%isize_x .OR. size(fieldx,2) .NE. group%jsize_x .OR. size(fieldx,3) .NE. group%ksize_v) &
+     call mpp_error(FATAL, "MPP_RESET_GROUP_UPDATE_FIELD_3D_V_: size of fieldx does not match the size stored in group") 
+  if(size(fieldy,1) .NE. group%isize_y .OR. size(fieldy,2) .NE. group%jsize_y .OR. size(fieldy,3) .NE. group%ksize_v) &
+     call mpp_error(FATAL, "MPP_RESET_GROUP_UPDATE_FIELD_3D_V_: size of fieldy does not match the size stored in group") 
+
+  group%addrs_x(group%reset_index_v) = LOC(fieldx)
+  group%addrs_y(group%reset_index_v) = LOC(fieldy)
+
+end subroutine MPP_RESET_GROUP_UPDATE_FIELD_3D_V_
+
+
+subroutine MPP_RESET_GROUP_UPDATE_FIELD_4D_V_(group, fieldx, fieldy)
+  type(mpp_group_update_type), intent(inout) :: group
+  MPP_TYPE_,                   intent(in)    :: fieldx(:,:,:,:), fieldy(:,:,:,:)
+  integer :: indx
+
+  group%reset_index_v = group%reset_index_v + 1
+
+  if(group%reset_index_v > group%nvector) &
+     call mpp_error(FATAL, "MPP_RESET_GROUP_UPDATE_FIELD_4D_V_: group%reset_index_v > group%nvector")
+  if(size(fieldx,1) .NE. group%isize_x .OR. size(fieldx,2) .NE. group%jsize_x .OR. &
+              size(fieldx,3)*size(fieldx,4) .NE. group%ksize_v) &
+     call mpp_error(FATAL, "MPP_RESET_GROUP_UPDATE_FIELD_4D_V_: size of fieldx does not match the size stored in group") 
+  if(size(fieldy,1) .NE. group%isize_y .OR. size(fieldy,2) .NE. group%jsize_y .OR. &
+              size(fieldy,3)*size(fieldy,4) .NE. group%ksize_v) &
+     call mpp_error(FATAL, "MPP_RESET_GROUP_UPDATE_FIELD_4D_V_: size of fieldy does not match the size stored in group") 
+
+  group%addrs_x(group%reset_index_v) = LOC(fieldx)
+  group%addrs_y(group%reset_index_v) = LOC(fieldy)
+
+end subroutine MPP_RESET_GROUP_UPDATE_FIELD_4D_V_
+
+
diff --git a/src/shared/mpp/include/mpp_io_connect.inc b/src/shared/mpp/include/mpp_io_connect.inc
index 0a05aedf1e..5db528c014 100644
--- a/src/shared/mpp/include/mpp_io_connect.inc
+++ b/src/shared/mpp/include/mpp_io_connect.inc
@@ -1,6 +1,6 @@
 
 ! -*-f90-*-
-! $Id: mpp_io_connect.inc,v 19.0 2012/01/06 22:03:20 fms Exp $
+! $Id$
 
 ! 
 
@@ -176,10 +176,11 @@
 
       character(len=16)  :: act, acc, for, pos
       character(len=128) :: mesg
+      character(len=256) :: text2
       integer :: action_flag, form_flag, access_flag, threading_flag, fileset_flag, length
       integer :: nfiles, tile_id(1), io_layout(2)
       logical :: exists, on_root_pe, dist_file
-      logical :: write_on_this_pe, io_domain_exist
+      logical :: write_on_this_pe, read_on_this_pe, io_domain_exist
       integer :: ios, nc_pos     !position of .nc in file name
       type(axistype) :: unlim    !used by netCDF with mpp_append
       type(domain2d), pointer :: io_domain=>NULL()
@@ -225,20 +226,21 @@
          if(associated(io_domain)) io_domain_exist = .true.
       endif
 
-      !--- when io_domain_exist, will ignore the threading_flag and fileset_flag.
-      if(io_domain_exist .AND. PRESENT(fileset) ) then
-         call mpp_error(NOTE, "mpp_io_connect.inc(mpp_open): io_domain exists for domain "// &
-            trim(mpp_get_domain_name(domain))//", optional argument fileset will be ignored")
-      endif
-
       write_on_this_pe = .true.
-      if( threading_flag.EQ.MPP_SINGLE .AND. .NOT. on_root_pe ) write_on_this_pe = .false.
+      read_on_this_pe = .true.
+      if( threading_flag.EQ.MPP_SINGLE .AND. .NOT.on_root_pe ) then
+         write_on_this_pe = .false.
+         read_on_this_pe = .false.
+      endif
       if(form_flag == MPP_NETCDF .AND. action_flag .NE. MPP_RDONLY) then
          if(fileset_flag .EQ.MPP_SINGLE .AND.  threading_flag.EQ.MPP_MULTI) then
             call mpp_error(FATAL, "mpp_io_connect.inc(mpp_open): multiple thread and single "// &
                 "file writing/appending is not supported for netCDF file")
          endif
-         if( fileset_flag  .EQ.MPP_SINGLE .AND. .NOT. on_root_pe ) write_on_this_pe = .false.
+         if( fileset_flag.EQ.MPP_SINGLE .AND. .NOT.on_root_pe ) then
+            write_on_this_pe = .false.
+            read_on_this_pe = .false.
+         endif   
       endif   
    
       if( io_domain_exist) then
@@ -272,6 +274,7 @@
       end if
       mpp_file(unit)%valid            = .true.
       mpp_file(unit)%write_on_this_pe = write_on_this_pe
+      mpp_file(unit)%read_on_this_pe  = read_on_this_pe
       mpp_file(unit)%io_domain_exist  = io_domain_exist
       if( PRESENT(domain) ) then
          allocate(mpp_file(unit)%domain)
@@ -291,10 +294,18 @@
             fileset_flag = MPP_MULTI
             threading_flag = MPP_MULTI
             tile_id = mpp_get_tile_id(io_domain)
-            if(mpp_npes() > 10000) then 
-               write( text,'(a,i6.6)' )trim(text)//'.', tile_id(1)
-            else
-               write( text,'(a,i4.4)' )trim(text)//'.', tile_id(1)
+            text2 = trim(text)
+            if(tile_id(1) .GE. 10000) call mpp_error(FATAL, 'mpp_open: tile_id should be less than 10000 when io_domain exist')
+            write( text,'(a,i4.4)' )trim(text)//'.', tile_id(1)
+            if( action_flag == MPP_RDONLY ) then
+               inquire(file=trim(text),EXIST=exists)
+               if(.not. exists) then
+                  write( text2,'(a,i6.6)' )trim(text2)//'.', tile_id(1)
+                  inquire(file=trim(text2),EXIST=exists)
+                  if(.not.exists) call mpp_error(FATAL, 'mpp_open: neither '// &
+                                  trim(text)//' nor '//trim(text2)//' exist and io domain exist')
+                  text = trim(text2)
+               endif
             endif
          else
             fileset_flag   = MPP_SINGLE
@@ -502,7 +513,7 @@
               if( verbose )print '(a,i6,i16,i4)', 'MPP_OPEN: append to existing netCDF file: pe, ncid, time_axis_id=',&
                    pe, mpp_file(unit)%ncid, mpp_file(unit)%id
               mpp_file(unit)%format=form_flag ! need this for mpp_read
-              call mpp_read_meta(unit)
+!              call mpp_read_meta(unit)
           else if( action_flag.EQ.MPP_RDONLY )then
                inquire(file=trim(mpp_file(unit)%name),EXIST=exists)
               if (.NOT.exists) call mpp_error(FATAL,'MPP_OPEN:'&
@@ -542,7 +553,7 @@
               if( verbose )print '(a,i6,i16,i4)', 'MPP_OPEN: append to existing netCDF file: pe, ncid, time_axis_id=',&
                    pe, mpp_file(unit)%ncid, mpp_file(unit)%id
               mpp_file(unit)%format=form_flag ! need this for mpp_read
-              call mpp_read_meta(unit)
+!              call mpp_read_meta(unit)
           else if( action_flag.EQ.MPP_RDONLY )then
                inquire(file=trim(mpp_file(unit)%name),EXIST=exists)
               if (.NOT.exists) call mpp_error(FATAL,'MPP_OPEN:'&
@@ -582,7 +593,7 @@
               if( verbose )print '(a,i6,i16,i4)', 'MPP_OPEN: append to existing netCDF file: pe, ncid, time_axis_id=',&
                    pe, mpp_file(unit)%ncid, mpp_file(unit)%id
               mpp_file(unit)%format=form_flag ! need this for mpp_read
-              call mpp_read_meta(unit)
+!              call mpp_read_meta(unit)
           else if( action_flag.EQ.MPP_RDONLY )then
                inquire(file=trim(mpp_file(unit)%name),EXIST=exists)
               if (.NOT.exists) call mpp_error(FATAL,'MPP_OPEN:'&
diff --git a/src/shared/mpp/include/mpp_io_misc.inc b/src/shared/mpp/include/mpp_io_misc.inc
index e2e6bf7dfc..aef7377768 100644
--- a/src/shared/mpp/include/mpp_io_misc.inc
+++ b/src/shared/mpp/include/mpp_io_misc.inc
@@ -1,6 +1,6 @@
 
 ! -*-f90-*-
-! $Id: mpp_io_misc.inc,v 20.0 2013/12/14 00:26:36 fms Exp $
+! $Id$
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 !                                                                      !
@@ -96,11 +96,12 @@
       default_field%type = -1
       default_field%natt = -1
       default_field%ndim = -1
+      default_field%checksum = 0
 !largest possible 4-byte reals
       default_field%min = -huge(1._4)
       default_field%max =  huge(1._4)
-      default_field%missing = -1e36
-      default_field%fill = -1e36
+      default_field%missing = MPP_FILL_DOUBLE ! now using netcdf:NF_FILL_DOUBLE instead of -1e36
+      default_field%fill = MPP_FILL_DOUBLE ! now using netcdf:NF_FILL_DOUBLE instead of -1e36
       default_field%scale = 1.0
       default_field%add = 0.0
       default_field%pack = 1
@@ -110,6 +111,7 @@
       default_axis%units = 'nounits'
       default_axis%longname = 'noname'
       default_axis%cartesian = 'none'
+      default_axis%compressed = 'unspecified'
       default_axis%calendar = 'unspecified'
       default_axis%sense = 0
       default_axis%len = -1
diff --git a/src/shared/mpp/include/mpp_io_read.inc b/src/shared/mpp/include/mpp_io_read.inc
index 770faa5eeb..1218965e8b 100644
--- a/src/shared/mpp/include/mpp_io_read.inc
+++ b/src/shared/mpp/include/mpp_io_read.inc
@@ -1,6 +1,6 @@
 
 ! -*-f90-*-
-! $Id: mpp_io_read.inc,v 20.0 2013/12/14 00:26:38 fms Exp $
+! $Id$
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 !                                                                      !
@@ -12,10 +12,22 @@
 #define MPP_READ_2DDECOMP_2D_ mpp_read_2ddecomp_r2d
 #undef MPP_READ_2DDECOMP_3D_
 #define MPP_READ_2DDECOMP_3D_ mpp_read_2ddecomp_r3d
+#undef MPP_READ_2DDECOMP_4D_
+#define MPP_READ_2DDECOMP_4D_ mpp_read_2ddecomp_r4d
 #undef MPP_TYPE_
 #define MPP_TYPE_ real
 #include 
 
+#undef MPP_READ_COMPRESSED_1D_
+#define MPP_READ_COMPRESSED_1D_ mpp_read_compressed_r1d
+#undef MPP_READ_COMPRESSED_2D_
+#define MPP_READ_COMPRESSED_2D_ mpp_read_compressed_r2d
+#undef MPP_TYPE_
+#define MPP_TYPE_ real
+#include 
+
+#include 
+
     subroutine read_record_core(unit, field, nwords, data, start, axsiz)
       integer,         intent(in)    :: unit
       type(fieldtype), intent(in)    :: field
@@ -113,7 +125,7 @@
     end subroutine read_record_core
 
 
-    subroutine read_record( unit, field, nwords, data, time_level, domain, position, tile_count )
+    subroutine read_record( unit, field, nwords, data, time_level, domain, position, tile_count, start_in, axsiz_in )
 !routine that is finally called by all mpp_read routines to perform the read
 !a non-netCDF record contains:
 !      field ID
@@ -136,6 +148,7 @@
       integer,        intent(in),    optional :: time_level
       type(domain2D), intent(in),    optional :: domain
       integer,        intent(in),    optional :: position, tile_count
+      integer,        intent(in),    optional :: start_in(:), axsiz_in(:)
       integer, dimension(size(field%axes(:))) :: start, axsiz
 
       integer :: tlevel !,subdomain(4)
@@ -152,14 +165,25 @@
 
       if( .NOT.module_is_initialized )call mpp_error( FATAL, 'READ_RECORD: must first call mpp_io_init.' )
       if( .NOT.mpp_file(unit)%opened )call mpp_error( FATAL, 'READ_RECORD: invalid unit number.' )
-      if( mpp_file(unit)%threading.EQ.MPP_SINGLE .AND. pe.NE.mpp_root_pe() )return
+      if( .NOT.mpp_file(unit)%read_on_this_pe )return
 
       if( .NOT.mpp_file(unit)%initialized ) call mpp_error( FATAL, 'MPP_READ: must first call mpp_read_meta.' )
+      if( mpp_file(unit)%format .NE. MPP_NETCDF ) call mpp_error( FATAL, 'Currently dont support non-NetCDF mpp read' )
 
-      if( verbose )print '(a,2i3,2i5)', 'MPP_READ: PE, unit, %id, %time_level =',&
-           pe, unit, mpp_file(unit)%id, tlevel
+      if (.not.PRESENT(time_level)) then
+          tlevel = 0
+      else
+          tlevel = time_level
+      endif
 
-      if( mpp_file(unit)%format.EQ.MPP_NETCDF )then
+      if( verbose )print '(a,2i6,2i5)', 'MPP_READ: PE, unit, %id, %time_level =',&
+           pe, unit, mpp_file(unit)%id, tlevel
+      if( PRESENT(start_in) .AND. PRESENT(axsiz_in) ) then
+         if(size(start(:)) > size(start_in(:)) )call mpp_error( FATAL, 'MPP_READ: size(start_in) < size(start)')
+         if(size(axsiz(:)) > size(axsiz_in(:)) )call mpp_error( FATAL, 'MPP_READ: size(axsiz_in) < size(axsiz)')
+         start(:) = start_in(1:size(start(:)))
+         axsiz(:) = axsiz_in(1:size(axsiz(:)))
+      else
 !define netCDF data block to be read:
 !  time axis: START = time level
 !             AXSIZ = 1
@@ -216,19 +240,29 @@
                  io_domain => NULL()
               end if
           end if  
-              
-          if( verbose )print '(a,2i6,i6,12i4)', 'READ_RECORD: PE, unit, nwords, start, axsiz=', pe, unit, nwords, start, axsiz
+      endif        
+      if( verbose )print '(a,2i6,i6,12i4)', 'READ_RECORD: PE, unit, nwords, start, axsiz=', pe, unit, nwords, start, axsiz
 
-          call read_record_core(unit, field, nwords, data, start, axsiz)
-      else                      !non-netCDF
-!subdomain contains (/is,ie,js,je/)
-          call mpp_error( FATAL, 'Currently dont support non-NetCDF mpp read' )
-          
-      end if
+      call read_record_core(unit, field, nwords, data, start, axsiz)
 
       return
     end subroutine read_record
 
+! 
+!   
+!   
+!   
+!   
+! 
+    subroutine mpp_read_r4D( unit, field, data, tindex)
+      integer, intent(in) :: unit
+      type(fieldtype), intent(in) :: field
+      real, intent(inout) :: data(:,:,:,:)
+      integer, intent(in), optional :: tindex
+
+      call read_record( unit, field, size(data(:,:,:,:)), data, tindex )
+    end subroutine mpp_read_r4D
+
 
 ! 
 !   
@@ -339,7 +373,7 @@
       lev = 1
       if(present(level)) lev = level
 
-      if( verbose )print '(a,2i3,2i5)', 'MPP_READ: PE, unit, %id, level =', pe, unit, mpp_file(unit)%id, lev
+      if( verbose )print '(a,2i6,2i5)', 'MPP_READ: PE, unit, %id, level =', pe, unit, mpp_file(unit)%id, lev
 
       if( mpp_file(unit)%format.EQ.MPP_NETCDF )then
          start = 1
@@ -428,8 +462,8 @@
       integer :: error, i, j, istat, check_exist
       integer         :: type, nvdims, nvatts, dimid
       integer, allocatable, dimension(:) :: dimids
-      character(len=128) :: name, attname, unlimname, attval
-      logical :: isdim
+      character(len=128) :: name, attname, unlimname, attval, bounds_name
+      logical :: isdim, found_bounds
       integer(LONG_KIND) :: checksumf
       character(len=64)  :: checksum_char
       integer :: num_checksumf, last, is, k
@@ -438,7 +472,7 @@
       integer(INT_KIND), allocatable :: ivals(:)
       real(FLOAT_KIND), allocatable  :: rvals(:)
       real(DOUBLE_KIND), allocatable :: r8vals(:)
-      
+
 #ifdef use_netCDF
 
       if( mpp_file(unit)%format.EQ.MPP_NETCDF )then
@@ -542,6 +576,8 @@
                  call netcdf_err( error, mpp_file(unit), attr=mpp_file(unit)%att(i) )
                  if( verbose .and. pe == 0 )print *, 'GLOBAL ATT ',trim(name),' ',ivals(1:len)
                  mpp_file(unit)%Att(i)%fatt(1:len)=ivals(1:len)
+                 if(lowercase(trim(name)) == 'time_axis' .and. ivals(1)==0) &
+                        mpp_file(unit)%time_level = -1 ! This file is an unlimited axis restart
                  deallocate(ivals)
               case (NF_FLOAT)
                  allocate(mpp_file(unit)%Att(i)%fatt(len), STAT=istat)
@@ -697,8 +733,13 @@
                           call mpp_error(FATAL, "mpp_io_mod(mpp_read_meta): Unable to allocate temporary array rvals.  STAT = "&
                                & //trim(text))
                        end if
-                       error = NF_GET_VAR_REAL(ncid,i,rvals)
-                       call netcdf_err( error, mpp_file(unit), mpp_file(unit)%Axis(dimid) )
+                       !z1l read from root pe and broadcast to other processor. 
+                       !In the future we will modify the code if there is performance issue for very high MPI ranks.
+                       if(mpp_pe()==mpp_root_pe()) then
+                          error = NF_GET_VAR_REAL(ncid,i,rvals)
+                          call netcdf_err( error, mpp_file(unit), mpp_file(unit)%Axis(dimid) )
+                       endif
+                       call mpp_broadcast(rvals, len, mpp_root_pe())
                        mpp_file(unit)%time_values(1:len) = rvals(1:len)
                        deallocate(rvals)
                     case (NF_DOUBLE)
@@ -708,8 +749,13 @@
                           call mpp_error(FATAL, "mpp_io_mod(mpp_read_meta): Unable to allocate temporary array r8vals.  STAT = "&
                             & //trim(text))
                        end if
-                       error = NF_GET_VAR_DOUBLE(ncid,i,r8vals)
-                       call netcdf_err( error, mpp_file(unit), mpp_file(unit)%Axis(dimid) )
+                       !z1l read from root pe and broadcast to other processor. 
+                       !In the future we will modify the code if there is performance issue for very high MPI ranks.
+                       if(mpp_pe()==mpp_root_pe()) then
+                         error = NF_GET_VAR_DOUBLE(ncid,i,r8vals)
+                         call netcdf_err( error, mpp_file(unit), mpp_file(unit)%Axis(dimid) )
+                       endif
+                       call mpp_broadcast(r8vals, len, mpp_root_pe())
                        mpp_file(unit)%time_values(1:len) = r8vals(1:len)
                        deallocate(r8vals)
                     case default
@@ -869,6 +915,106 @@
               enddo
            endif
         enddo 
+
+        ! assign axis bounds
+        do j = 1, mpp_file(unit)%ndim
+           if(.not. associated(mpp_file(unit)%Axis(j)%data)) cycle
+           len = size(mpp_file(unit)%Axis(j)%data(:))
+           allocate(mpp_file(unit)%Axis(j)%data_bounds(len+1))
+           mpp_file(unit)%Axis(j)%name_bounds = 'none'
+           bounds_name = 'none'
+           found_bounds = .false.
+           do i = 1, mpp_file(unit)%Axis(j)%natt
+              if(trim(mpp_file(unit)%Axis(j)%Att(i)%name) == 'bounds' .OR. &
+                   trim(mpp_file(unit)%Axis(j)%Att(i)%name) == 'edges' ) then
+                 bounds_name = mpp_file(unit)%Axis(j)%Att(i)%catt
+                 found_bounds = .true.
+                 exit 
+              endif
+           enddo
+           !-- loop through all the fields to locate bounds_name
+           if( found_bounds ) then
+              found_bounds = .false.
+              do i = 1, mpp_file(unit)%ndim
+                 if(.not. associated(mpp_file(unit)%Axis(i)%data)) cycle
+                 if(trim(mpp_file(unit)%Axis(i)%name) == trim(bounds_name)) then
+                    found_bounds = .true.
+                    if(size(mpp_file(unit)%Axis(i)%data(:)) .NE. len+1) &
+                         call mpp_error(FATAL, "mpp_read_meta: improperly size bounds for field "// &
+                            trim(bounds_name)//" in file "// trim(mpp_file(unit)%name) )
+                    mpp_file(unit)%Axis(j)%data_bounds(:) = mpp_file(unit)%Axis(i)%data(:)
+                    exit
+                 endif
+              enddo
+              if( .not. found_bounds ) then
+                 do i=1, nvar_total
+                    error=NF_INQ_VAR(ncid,i,name,type,nvdims,dimids,nvatts);call netcdf_err( error, mpp_file(unit) )
+                    if(trim(name) == trim(bounds_name)) then
+                       found_bounds = .true.
+                       if(nvdims .NE. 2) &
+                            call mpp_error(FATAL, "mpp_read_meta: field "//trim(bounds_name)//" in file "//&
+                              trim(mpp_file(unit)%name)//" must be 2-D field")
+                       if(mpp_file(unit)%Axis(dimids(1))%len .NE. 2) &
+                            call mpp_error(FATAL, "mpp_read_meta: first dimension size of field "// &
+                            trim(mpp_file(unit)%Var(i)%name)//" from file "//trim(mpp_file(unit)%name)// &
+                            " must be 2")
+                       if(mpp_file(unit)%Axis(dimids(2))%len .NE. len) &
+                            call mpp_error(FATAL, "mpp_read_meta: second dimension size of field "// &
+                            trim(mpp_file(unit)%Var(i)%name)//" from file "//trim(mpp_file(unit)%name)// &
+                            " is not correct")
+                       select case (type)
+                       case (NF_INT)
+                          allocate(ivals(2*len), STAT=istat)
+                          if ( istat .ne. 0 ) then
+                             write(text,'(A)') istat
+                             call mpp_error(FATAL, "mpp_io_mod(mpp_read_meta): Unable to allocate array ivals."//&
+                                  "  STAT = "//trim(text))
+                          end if            
+                          error = NF_GET_VAR_INT(ncid,i,ivals)
+                          call netcdf_err( error, mpp_file(unit), string=" Field="//trim(bounds_name) )
+                          mpp_file(unit)%Axis(j)%data_bounds(1:len) =ivals(1:(2*len-1):2)
+                          mpp_file(unit)%Axis(j)%data_bounds(len+1) = ivals(2*len)
+                          deallocate(ivals)
+                       case (NF_FLOAT)
+                          allocate(rvals(2*len), STAT=istat)
+                          if ( istat .ne. 0 ) then
+                             write(text,'(A)') istat
+                             call mpp_error(FATAL, "mpp_io_mod(mpp_read_meta): Unable to allocate array rvals. "// &
+                                  "  STAT = "//trim(text))
+                          end if            
+                          error = NF_GET_VAR_REAL(ncid,i,rvals)
+                          call netcdf_err( error, mpp_file(unit), string=" Field="//trim(bounds_name) )
+                          mpp_file(unit)%Axis(j)%data_bounds(1:len) =rvals(1:(2*len-1):2)
+                          mpp_file(unit)%Axis(j)%data_bounds(len+1) = rvals(2*len)
+                          deallocate(rvals)
+                       case (NF_DOUBLE)
+                          allocate(r8vals(2*len), STAT=istat)
+                          if ( istat .ne. 0 ) then
+                             write(text,'(A)') istat
+                             call mpp_error(FATAL, "mpp_io_mod(mpp_read_meta): Unable to allocate array r8vals. "//&
+                                  "  STAT = "//trim(text))
+                          end if            
+                          error = NF_GET_VAR_DOUBLE(ncid,i,r8vals)
+                          call netcdf_err( error, mpp_file(unit), string=" Field="//trim(bounds_name) )
+                          mpp_file(unit)%Axis(j)%data_bounds(1:len) =r8vals(1:(2*len-1):2)
+                          mpp_file(unit)%Axis(j)%data_bounds(len+1) = r8vals(2*len)
+                          deallocate(r8vals)
+                       case default
+                          call mpp_error( FATAL, 'mpp_io_mod(mpp_read_meta): Invalid data type for dimension' )
+                       end select
+                       exit
+                    endif
+                 enddo
+              endif
+           endif
+           if (found_bounds) then
+              mpp_file(unit)%Axis(j)%name_bounds = trim(bounds_name)
+           else
+              deallocate(mpp_file(unit)%Axis(j)%data_bounds)
+              mpp_file(unit)%Axis(j)%data_bounds =>NULL()
+           endif
+        enddo
+
 ! assign variable info
         nv = 0
         do i=1, nvar_total
@@ -912,7 +1058,7 @@
               allocate(mpp_file(unit)%Var(nv)%size(nvdims))
               
               do j=1,nvdims
-                 if( dimids(j).eq.mpp_file(unit)%recdimid )then
+                 if(dimids(j).eq.mpp_file(unit)%recdimid  .and. mpp_file(unit)%time_level/=-1)then
                     mpp_file(unit)%Var(nv)%time_axis_index = dimids(j)
                     mpp_file(unit)%Var(nv)%size(j)=1    ! dimid length set to 1 here for consistency w/ mpp_write
                  else
@@ -1033,8 +1179,12 @@
                        mpp_file(unit)%Var(nv)%missing=mpp_file(unit)%Var(nv)%Att(j)%fatt(1)
                     case('missing_value')
                        mpp_file(unit)%Var(nv)%missing=mpp_file(unit)%Var(nv)%Att(j)%fatt(1)
+                    case('_FillValue')                       
+                       mpp_file(unit)%Var(nv)%fill=mpp_file(unit)%Var(nv)%Att(j)%fatt(1)
                     case('add_offset')
                        mpp_file(unit)%Var(nv)%add=mpp_file(unit)%Var(nv)%Att(j)%fatt(1)
+                    case('packing')
+                       mpp_file(unit)%Var(nv)%pack=mpp_file(unit)%Var(nv)%Att(j)%fatt(1)
                     case('valid_range')
                        mpp_file(unit)%Var(nv)%min=mpp_file(unit)%Var(nv)%Att(j)%fatt(1)
                        mpp_file(unit)%Var(nv)%max=mpp_file(unit)%Var(nv)%Att(j)%fatt(2)
@@ -1063,7 +1213,7 @@
       else 
         call mpp_error( FATAL,  'MPP READ CURRENTLY DOES NOT SUPPORT NON-NETCDF' )
       endif
-        
+
       mpp_file(unit)%initialized = .TRUE.
 #else 
       call mpp_error( FATAL, 'MPP_READ currently requires use_netCDF option' )
diff --git a/src/shared/mpp/include/mpp_io_util.inc b/src/shared/mpp/include/mpp_io_util.inc
index 7894248d4a..2031d7efab 100644
--- a/src/shared/mpp/include/mpp_io_util.inc
+++ b/src/shared/mpp/include/mpp_io_util.inc
@@ -1,6 +1,6 @@
 
 ! -*-f90-*-
-! $Id: mpp_io_util.inc,v 20.0 2013/12/14 00:26:40 fms Exp $
+! $Id$
 
 
   !#####################################################################
@@ -148,11 +148,13 @@
    end subroutine mpp_get_field_atts
 
   !#####################################################################
-   subroutine mpp_get_axis_atts( axis, name, units, longname, cartesian, calendar, sense, len, natts, atts )
+   subroutine mpp_get_axis_atts( axis, name, units, longname, cartesian, &
+                                 calendar, sense, len, natts, atts, compressed )
 
      type(axistype), intent(in) :: axis
      character(len=*), intent(out) , optional :: name, units
-     character(len=*), intent(out), optional :: longname, cartesian,calendar
+     character(len=*), intent(out), optional :: longname, cartesian
+     character(len=*), intent(out), optional :: compressed, calendar
      integer,intent(out), optional :: sense, len , natts
      type(atttype), intent(inout), optional, dimension(:) :: atts
 
@@ -162,6 +164,7 @@
      if (PRESENT(units)) units = axis%units
      if (PRESENT(longname)) longname = axis%longname
      if (PRESENT(cartesian)) cartesian = axis%cartesian
+     if (PRESENT(compressed)) compressed = axis%compressed
      if (PRESENT(calendar)) calendar = axis%calendar
      if (PRESENT(sense)) sense = axis%sense
      if (PRESENT(len)) len = axis%len
@@ -249,6 +252,34 @@
    end subroutine mpp_get_axes
 
   !#####################################################################
+   function mpp_get_dimension_length(unit, dimname, found)
+     integer,           intent(in)  :: unit
+     character(len=*),  intent(in)  :: dimname
+     logical, optional, intent(out) :: found
+     integer                        :: mpp_get_dimension_length
+     logical                        :: found_dim
+     integer                        :: i
+
+
+     if( .NOT.module_is_initialized ) &
+       call mpp_error( FATAL, 'mpp_get_dimension_length: must first call mpp_io_init.' )
+     if( .NOT.mpp_file(unit)%opened )&
+       call mpp_error( FATAL, 'mpp_get_dimension_length: invalid unit number, file '//trim(mpp_file(unit)%name))
+     found_dim = .false.
+     mpp_get_dimension_length = -1
+     do i = 1, mpp_file(unit)%ndim
+        if(trim(dimname) == trim(mpp_file(unit)%Axis(i)%name)) then
+          mpp_get_dimension_length = mpp_file(unit)%Axis(i)%len
+          found_dim = .true.
+          exit
+        endif
+     enddo
+
+     if(present(found)) found = found_dim
+
+   end function mpp_get_dimension_length
+
+   !#####################################################################
     subroutine mpp_get_time_axis( unit, time_axis )
       integer, intent(in) :: unit
       type(axistype), intent(inout) :: time_axis
@@ -265,7 +296,7 @@
   !####################################################################
   function mpp_get_default_calendar( )
      character(len=len(default_axis%calendar)) :: mpp_get_default_calendar
-    
+
      mpp_get_default_calendar = default_axis%calendar
 
   end function mpp_get_default_calendar
@@ -410,6 +441,25 @@
      return
    end function mpp_get_axis_length
 
+  !#####################################################################
+  function mpp_get_axis_bounds(axis, data, name)
+     type(axistype), intent(in) :: axis
+     real, dimension(:), intent(out) :: data
+     character(len=*), optional, intent(out) :: name
+     logical                         :: mpp_get_axis_bounds
+
+     if (size(data(:)).lt.axis%len+1)&
+          call mpp_error(FATAL,'MPP_GET_AXIS_BOUNDS: data array not large enough, axis '//trim(axis%name))
+     if (.NOT.ASSOCIATED(axis%data_bounds)) then
+        mpp_get_axis_bounds = .false.
+     else
+        mpp_get_axis_bounds = .true.
+        data(1:axis%len+1) = axis%data_bounds(:)
+     endif
+     if(present(name)) name = trim(axis%name_bounds)
+
+     return
+  end function mpp_get_axis_bounds
 
   !#####################################################################
    subroutine mpp_get_axis_data( axis, data )
@@ -426,7 +476,7 @@
      else
         data(1:axis%len) = axis%data
      endif
- 
+
      return
    end subroutine mpp_get_axis_data
 
@@ -589,7 +639,7 @@
         v%min = f%att(imissing)%fatt(1)*f%scale + f%add
      else if (ifill>0) then
      !z1l ifdef is added in to be able to compile without using use_netCDF.
-#ifdef use_netCDF  
+#ifdef use_netCDF
         ! define min and max according to _FillValue
         if(f%att(ifill)%fatt(1)>0) then
             ! if _FillValue is positive, then it defines valid maximum
@@ -675,7 +725,7 @@
    function mpp_get_att_name(att)
       type(atttype),    intent(in) :: att
       character(len=len(att%name)) :: mpp_get_att_name
-     
+
       mpp_get_att_name = att%name
       return
 
@@ -687,7 +737,7 @@
    function mpp_get_att_type(att)
       type(atttype), intent(in) :: att
       integer                   :: mpp_get_att_type
-     
+
       mpp_get_att_type = att%type
       return
 
@@ -699,7 +749,7 @@
    function mpp_get_att_length(att)
       type(atttype), intent(in) :: att
       integer                   :: mpp_get_att_length
-     
+
       mpp_get_att_length = att%len
 
       return
@@ -712,7 +762,7 @@
    function mpp_get_att_char(att)
       type(atttype), intent(in) :: att
       character(len=att%len)    :: mpp_get_att_char
-     
+
       mpp_get_att_char = att%catt
       return
 
@@ -724,7 +774,7 @@
    function mpp_get_att_real(att)
       type(atttype), intent(in)          :: att
       real, dimension(size(att%fatt(:))) :: mpp_get_att_real
-     
+
       mpp_get_att_real = att%fatt
       return
 
@@ -756,7 +806,7 @@
    ! return the  file name of corresponding unit
    function mpp_get_file_name(unit)
       integer,                  intent(in) :: unit
-      character(len=len(mpp_file(1)%name)) :: mpp_get_file_name      
+      character(len=len(mpp_file(1)%name)) :: mpp_get_file_name
 
       mpp_get_file_name = mpp_file(unit)%name
       return
@@ -767,7 +817,7 @@
    ! return if certain file with unit is opened or not
    function mpp_file_is_opened(unit)
       integer,  intent(in) :: unit
-      logical              :: mpp_file_is_opened      
+      logical              :: mpp_file_is_opened
 
       mpp_file_is_opened = mpp_file(unit)%opened
       return
@@ -777,9 +827,9 @@
    !####################################################################
    ! return the attribute value of given field name
    subroutine mpp_get_field_att_text(unit, fieldname, attname, attvalue)
-     integer,           intent(in) :: unit  
+     integer,           intent(in) :: unit
      character(len=*),  intent(in) :: fieldname, attname
-     character(len=*), intent(out) :: attvalue 
+     character(len=*), intent(out) :: attvalue
      logical                       :: found_field,  found_att
      integer                       :: i, j, length
 
@@ -820,15 +870,69 @@
       mpp_io_clock_on = io_clocks_on
       return
 
-   end function mpp_io_clock_on   
+   end function mpp_io_clock_on
 
 
    function mpp_attribute_exist(field,name)
-      integer                      :: mpp_attribute_exist
+      logical                      :: mpp_attribute_exist
       type(fieldtype),  intent(in) :: field ! The field that you are searching for the attribute.
       character(len=*), intent(in) :: name ! name of the attributes
 
-      mpp_attribute_exist = mpp_find_att(field%Att(:),name)
-      
+      if(field%natt > 0) then
+         mpp_attribute_exist = ( mpp_find_att(field%Att(:),name) > 0 )
+      else
+         mpp_attribute_exist = .false.
+      endif
+
    end function mpp_attribute_exist
-   
+
+!#######################################################################
+subroutine mpp_dist_io_pelist(ssize,pelist)
+  integer,              intent(in)  :: ssize   ! Stripe size for dist read
+  integer, allocatable, intent(out) :: pelist(:)
+  integer :: i, lsize, ioroot
+  logical :: is_ioroot=.false.
+
+  ! Did you make a mistake?
+  if(ssize < 1) call mpp_error(FATAL,'mpp_dist_io_pelist: I/O stripe size < 1')
+
+  is_ioroot = mpp_is_dist_ioroot(ssize,ioroot=ioroot,lsize=lsize)
+
+  ! Did I make a mistake?
+  if(lsize < 1) call mpp_error(FATAL,'mpp_dist_io_pelist: size of pelist < 1')
+
+  allocate(pelist(lsize))
+  do i=1,lsize
+    pelist(i) = ioroot + i - 1
+  enddo
+end subroutine mpp_dist_io_pelist
+
+!#######################################################################
+logical function mpp_is_dist_ioroot(ssize,ioroot,lsize)
+  integer, intent(in)  :: ssize   ! Dist io set size
+  integer, intent(out), optional :: ioroot, lsize
+  integer :: pe, npes, mypos, maxpe, d_ioroot, d_lsize, last_ioroot
+  integer :: rootpe
+
+  if(ssize < 1) call mpp_error(FATAL,'mpp_is_dist_ioroot: I/O stripe size < 1')
+
+  mpp_is_dist_ioroot = .false.
+  rootpe = mpp_root_pe()
+  d_lsize = ssize
+  pe = mpp_pe()
+  mypos = modulo(pe-rootpe,ssize)  ! Which PE am I in the io group?
+  d_ioroot = pe - mypos            ! What is the io root for the group?
+  npes = mpp_npes()
+  maxpe = min(d_ioroot+ssize,npes+rootpe) - 1  ! Handle end case
+  d_lsize = maxpe - d_ioroot + 1
+  if(mod(npes,ssize) == 1)then  ! Ensure there are no sets with 1 member
+    last_ioroot = (npes-1) - ssize
+    if(pe >= last_ioroot) then
+       d_ioroot = last_ioroot
+       d_lsize = ssize + 1
+    endif
+  endif
+  if(pe == d_ioroot) mpp_is_dist_ioroot = .true.
+  if(PRESENT(ioroot)) ioroot = d_ioroot
+  if(PRESENT(lsize)) lsize = d_lsize
+end function mpp_is_dist_ioroot
diff --git a/src/shared/mpp/include/mpp_io_write.inc b/src/shared/mpp/include/mpp_io_write.inc
index 5fcf08e81d..cf13208df0 100644
--- a/src/shared/mpp/include/mpp_io_write.inc
+++ b/src/shared/mpp/include/mpp_io_write.inc
@@ -1,6 +1,6 @@
 
 ! -*-f90-*-
-! $Id: mpp_io_write.inc,v 20.0 2013/12/14 00:26:42 fms Exp $
+! $Id$
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 !                                                                            !
@@ -162,12 +162,13 @@
       return
     end subroutine mpp_write_meta_global_scalar_r
       
-    subroutine mpp_write_meta_global_scalar_i( unit, name, ival )
+    subroutine mpp_write_meta_global_scalar_i( unit, name, ival, pack )
       integer, intent(in) :: unit
       character(len=*), intent(in) :: name
       integer, intent(in) :: ival
+      integer, intent(in), optional :: pack
       
-      call mpp_write_meta_global( unit, name, ival=(/ival/) )
+      call mpp_write_meta_global( unit, name, ival=(/ival/), pack=pack )
       return
     end subroutine mpp_write_meta_global_scalar_i
       
@@ -213,12 +214,13 @@
       return
     end subroutine mpp_write_meta_scalar_r
       
-    subroutine mpp_write_meta_scalar_i( unit, id, name, ival )
+    subroutine mpp_write_meta_scalar_i( unit, id, name, ival,pack )
       integer, intent(in) :: unit, id
       character(len=*), intent(in) :: name
       integer, intent(in) :: ival
+      integer, intent(in), optional :: pack
       
-      call mpp_write_meta( unit, id, name, ival=(/ival/) )
+      call mpp_write_meta( unit, id, name, ival=(/ival/),pack=pack )
       return
     end subroutine mpp_write_meta_scalar_i
       
@@ -242,8 +244,98 @@
       endif
 #endif
     end subroutine mpp_write_axis_data
-    
-    subroutine mpp_write_meta_axis( unit, axis, name, units, longname, cartesian, sense, domain, data, min, compressed)
+
+    subroutine mpp_def_dim_nodata(unit,name,size)
+      integer, intent(in) :: unit
+      character(len=*), intent(in) :: name
+      integer, intent(in) :: size
+      integer :: error,did
+
+    ! This routine assumes the file is in define mode
+      if(.NOT. mpp_file(unit)%write_on_this_pe) return
+#ifdef use_netCDF
+      error = NF_DEF_DIM(mpp_file(unit)%ncid,name,size,did)
+      call netcdf_err(error, mpp_file(unit),string='Axis='//trim(name))
+#endif
+    end subroutine mpp_def_dim_nodata
+
+    subroutine mpp_def_dim_int(unit,name,dsize,longname,data)
+      integer, intent(in) :: unit
+      character(len=*), intent(in) :: name
+      integer, intent(in) :: dsize
+      character(len=*), intent(in) :: longname
+      integer, intent(in) :: data(:)
+      integer :: error,did,id
+
+    ! This routine assumes the file is in define mode
+#ifdef use_netCDF
+      if(.NOT. mpp_file(unit)%write_on_this_pe) return
+      error = NF_DEF_DIM(mpp_file(unit)%ncid,name,dsize,did)
+      call netcdf_err(error, mpp_file(unit),string='Axis='//trim(name))
+
+    ! Write dimension data.
+      error = NF_DEF_VAR( mpp_file(unit)%ncid, name, NF_INT, 1, did, id )
+      call netcdf_err( error, mpp_file(unit), string=' axis varable '//trim(name))
+
+      error = NF_PUT_ATT_TEXT( mpp_file(unit)%ncid, id, 'long_name', len_trim(longname), longname )
+      call netcdf_err( error, mpp_file(unit), string=' Attribute=long_name' )
+
+      if( mpp_file(unit)%action.EQ.MPP_WRONLY )then
+         if(header_buffer_val>0) then
+            error = NF__ENDDEF(mpp_file(unit)%ncid,header_buffer_val,4,0,4)
+         else
+            error = NF_ENDDEF(mpp_file(unit)%ncid)
+         endif
+      endif
+      call netcdf_err( error, mpp_file(unit), string=' subroutine mpp_def_dim')
+      error = NF_PUT_VARA_INT ( mpp_file(unit)%ncid, id, 1, size(data), data )
+      call netcdf_err( error, mpp_file(unit), string=' axis varable '//trim(name))
+      error = NF_REDEF(mpp_file(unit)%ncid)
+      call netcdf_err( error, mpp_file(unit), string=' subroutine mpp_def_dim')
+#endif
+    return
+  end subroutine mpp_def_dim_int
+
+    subroutine mpp_def_dim_real(unit,name,dsize,longname,data)
+      integer, intent(in) :: unit
+      character(len=*), intent(in) :: name
+      integer, intent(in) :: dsize
+      character(len=*), intent(in) :: longname
+      real, intent(in) :: data(:)
+      integer :: error,did,id
+
+    ! This routine assumes the file is in define mode
+#ifdef use_netCDF
+      if(.NOT. mpp_file(unit)%write_on_this_pe) return
+      error = NF_DEF_DIM(mpp_file(unit)%ncid,name,dsize,did)
+      call netcdf_err(error, mpp_file(unit),string='Axis='//trim(name))
+
+    ! Write dimension data.
+      error = NF_DEF_VAR( mpp_file(unit)%ncid, name, NF_INT, 1, did, id )
+      call netcdf_err( error, mpp_file(unit), string=' axis varable '//trim(name))
+
+      error = NF_PUT_ATT_TEXT( mpp_file(unit)%ncid, id, 'long_name', len_trim(longname), longname )
+      call netcdf_err( error, mpp_file(unit), string=' Attribute=long_name' )
+
+      if( mpp_file(unit)%action.EQ.MPP_WRONLY )then
+         if(header_buffer_val>0) then
+            error = NF__ENDDEF(mpp_file(unit)%ncid,header_buffer_val,4,0,4)
+         else
+            error = NF_ENDDEF(mpp_file(unit)%ncid)
+         endif
+      endif
+      call netcdf_err( error, mpp_file(unit), string=' subroutine mpp_def_dim')
+      error = NF_PUT_VARA_INT ( mpp_file(unit)%ncid, id, 1, size(data), data )
+      call netcdf_err( error, mpp_file(unit), string=' axis varable '//trim(name))
+      error = NF_REDEF(mpp_file(unit)%ncid)
+      call netcdf_err( error, mpp_file(unit), string=' subroutine mpp_def_dim')
+#endif
+    return
+  end subroutine mpp_def_dim_real
+
+
+
+    subroutine mpp_write_meta_axis_r1d( unit, axis, name, units, longname, cartesian, sense, domain, data, min, calendar)
 !load the values in an axistype (still need to call mpp_write)
 !write metadata attributes for axis
 !it is declared intent(inout) so you can nullify pointers in the incoming object if needed
@@ -256,7 +348,7 @@
       type(domain1D),   intent(in), optional :: domain
       real,             intent(in), optional :: data(:)
       real,             intent(in), optional :: min
-      character(len=*), intent(in), optional :: compressed
+      character(len=*), intent(in), optional :: calendar
 
       integer :: is, ie, isg, ieg
       integer :: istat      
@@ -314,8 +406,8 @@
       axis%name     = name
       axis%units    = units
       axis%longname = longname
+      if( PRESENT(calendar)  ) axis%calendar  = calendar
       if( PRESENT(sense)     ) axis%sense     = sense
-      if( PRESENT(compressed)) axis%compressed     = trim(compressed)
       if( PRESENT(data) )then
          if( mpp_file(unit)%fileset.EQ.MPP_MULTI .AND. domain_exist ) then
             axis%len = ie - is + 1
@@ -379,6 +471,10 @@
 !write axis attributes
       call mpp_write_meta( unit, axis%id, 'long_name', cval=axis%longname) ; axis%natt = axis%natt + 1
       call mpp_write_meta( unit, axis%id, 'units',     cval=axis%units) ; axis%natt = axis%natt + 1
+      if( PRESENT(calendar) ) then
+        call mpp_write_meta( unit, axis%id, 'calendar', cval=axis%calendar)
+        axis%natt = axis%natt + 1
+      endif 
       if( PRESENT(cartesian) ) then
         call mpp_write_meta( unit, axis%id, 'cartesian_axis', cval=axis%cartesian)
         axis%natt = axis%natt + 1
@@ -390,13 +486,10 @@
           else if( sense.EQ.1 )then
               call mpp_write_meta( unit, axis%id, 'positive', cval='up')
               axis%natt = axis%natt + 1
+          else
+             ! silently ignore values of sense other than +/-1.
           end if
-!silently ignore values of sense other than +/-1.
       end if
-      if( PRESENT(compressed) ) then
-        call mpp_write_meta( unit, axis%id, 'compress', cval=axis%compressed)
-        axis%natt = axis%natt + 1
-      endif 
       if( PRESENT(min) ) then
         call mpp_write_meta( unit, axis%id, 'valid_min', rval=min)
         axis%natt = axis%natt + 1
@@ -405,16 +498,143 @@
           call mpp_write_meta( unit, axis%id, 'domain_decomposition', ival=(/isg,ieg,is,ie/))
           axis%natt = axis%natt + 1
       end if
-      if( verbose )print '(a,2i3,x,a,2i3)', 'MPP_WRITE_META: Wrote axis metadata, pe, unit, axis%name, axis%id, axis%did=', &
+      if( verbose )print '(a,2i6,x,a,2i3)', 'MPP_WRITE_META: Wrote axis metadata, pe, unit, axis%name, axis%id, axis%did=', &
            pe, unit, trim(axis%name), axis%id, axis%did
       
       mpp_file(unit)%ndim = max(1,mpp_file(unit)%ndim + 1)
 
+!      call mpp_clock_end(mpp_write_clock)           
+      return
+    end subroutine mpp_write_meta_axis_r1d
+
+    subroutine mpp_write_meta_axis_i1d(unit, axis, name, units, longname, data, min, compressed)
+!load the values in an axistype (still need to call mpp_write)
+!write metadata attributes for axis
+!it is declared intent(inout) so you can nullify pointers in the incoming object if needed
+!the f90 standard doesn't guarantee that intent(out) on a type guarantees that its pointer components will be unassociated
+      integer,          intent(in)           :: unit
+      type(axistype),   intent(inout)        :: axis
+      character(len=*), intent(in)           :: name, units, longname
+      integer,          intent(in)           :: data(:)
+      integer,          intent(in), optional :: min
+      character(len=*), intent(in), optional :: compressed
+
+      integer :: istat      
+      logical :: domain_exist
+      type(domain2d), pointer :: io_domain => NULL()
+
+!      call mpp_clock_begin(mpp_write_clock)
+      if( .NOT.module_is_initialized    )call mpp_error( FATAL, 'MPP_WRITE_META_I1D: must first call mpp_io_init.' )
+      if( .NOT. mpp_file(unit)%write_on_this_pe) then
+!         call mpp_clock_end(mpp_write_clock)
+         return
+      endif
+      if( .NOT.mpp_file(unit)%opened )call mpp_error( FATAL, 'MPP_WRITE_META: invalid unit number.' )
+      if( mpp_file(unit)%initialized ) &
+           call mpp_error( FATAL, 'MPP_WRITE_META_I1D: cannot write metadata to file after an mpp_write.' )
+
+!pre-existing pointers need to be nullified
+      if( ASSOCIATED(axis%idata) ) then
+         DEALLOCATE(axis%idata, stat=istat)
+      endif
+!load axistype
+      axis%name     = name
+      axis%units    = units
+      axis%longname = longname
+      if( PRESENT(compressed)) axis%compressed = trim(compressed)
+      axis%len = size(data(:))
+      allocate(axis%idata(axis%len))
+       axis%idata = data
+!write metadata
+      if( mpp_file(unit)%format.EQ.MPP_NETCDF )then
+          error = NF_DEF_DIM( mpp_file(unit)%ncid, axis%name, axis%len, axis%did )
+          call netcdf_err( error, mpp_file(unit), axis )
+          error = NF_DEF_VAR( mpp_file(unit)%ncid, axis%name, NF_INT, 1, axis%did, axis%id )
+          call netcdf_err( error, mpp_file(unit), axis )
+      else
+          call mpp_error( FATAL, 'MPP_WRITE_META_AXIS_I1D: Only netCDF format is currently supported.' )
+      end if  
+!write axis attributes
+      call mpp_write_meta( unit, axis%id, 'long_name', cval=axis%longname) ; axis%natt = axis%natt + 1
+      call mpp_write_meta( unit, axis%id, 'units',     cval=axis%units) ; axis%natt = axis%natt + 1
+      if( PRESENT(compressed) ) then
+        call mpp_write_meta( unit, axis%id, 'compress', cval=axis%compressed)
+        axis%natt = axis%natt + 1
+      endif 
+      if( PRESENT(min) ) then
+        call mpp_write_meta( unit, axis%id, 'valid_min', ival=min)
+        axis%natt = axis%natt + 1
+      endif 
+      if( verbose )print '(a,2i6,x,a,2i3)', 'MPP_WRITE_META: Wrote axis metadata, pe, unit, axis%name, axis%id, axis%did=', &
+           pe, unit, trim(axis%name), axis%id, axis%did
+      
+      mpp_file(unit)%ndim = max(1,mpp_file(unit)%ndim + 1)
 
 !      call mpp_clock_end(mpp_write_clock)           
       return
-    end subroutine mpp_write_meta_axis
+    end subroutine mpp_write_meta_axis_i1d
+
+
+    subroutine mpp_write_meta_axis_unlimited(unit, axis, name, data, unlimited, units, longname)
+!load the values in an axistype (still need to call mpp_write)
+!write metadata attributes for axis
+!it is declared intent(inout) so you can nullify pointers in the incoming object if needed
+!the f90 standard doesn't guarantee that intent(out) on a type guarantees that its pointer components will be unassociated
+      integer,          intent(in)           :: unit
+      type(axistype),   intent(inout)        :: axis
+      character(len=*), intent(in)           :: name
+      integer,          intent(in)           :: data  ! Number of elements to be written
+      logical,          intent(in)           :: unlimited  ! Provides unique arg signature
+      character(len=*), intent(in), optional :: units, longname
+
+      integer :: istat      
+      logical :: domain_exist
+      type(domain2d), pointer :: io_domain => NULL()
+
+!      call mpp_clock_begin(mpp_write_clock)
+      if( .NOT.module_is_initialized    )call mpp_error( FATAL, 'MPP_WRITE_META_I1D: must first call mpp_io_init.' )
+      if( .NOT. mpp_file(unit)%write_on_this_pe) then
+!         call mpp_clock_end(mpp_write_clock)
+         return
+      endif
+      if( .NOT.mpp_file(unit)%opened )call mpp_error( FATAL, 'MPP_WRITE_META: invalid unit number.' )
+      if( mpp_file(unit)%initialized ) &
+           call mpp_error( FATAL, 'MPP_WRITE_META_I1D: cannot write metadata to file after an mpp_write.' )
+
+!load axistype
+      axis%name     = name
+      if(present(units)) axis%units    = units
+      if(present(longname)) axis%longname = longname
+      axis%len = 1
+      allocate(axis%idata(1))
+      axis%idata = data
+!write metadata
+      if( mpp_file(unit)%format.EQ.MPP_NETCDF )then
+          error = NF_DEF_DIM( mpp_file(unit)%ncid, axis%name, NF_UNLIMITED, axis%did )
+          call netcdf_err( error, mpp_file(unit), axis )
+          error = NF_DEF_VAR( mpp_file(unit)%ncid, axis%name, NF_INT, 0, axis%did, axis%id )
+          call netcdf_err( error, mpp_file(unit), axis )
+      else
+          call mpp_error( FATAL, 'MPP_WRITE_META_AXIS_UNLIMITED: Only netCDF format is currently supported.' )
+      end if  
+!write axis attributes
+      if(present(longname)) then
+         call mpp_write_meta(unit,axis%id,'long_name',cval=axis%longname); axis%natt=axis%natt+1
+      endif
+      if(present(units)) then
+         call mpp_write_meta(unit,axis%id,'units',     cval=axis%units); axis%natt=axis%natt+1
+      endif
+      if( verbose )print '(a,2i6,x,a,2i3)', &
+          'MPP_WRITE_META_UNLIMITED: Wrote axis metadata, pe, unit, axis%name, axis%id, axis%did=', &
+           pe, unit, trim(axis%name), axis%id, axis%did
       
+      mpp_file(unit)%ndim = max(1,mpp_file(unit)%ndim + 1)
+
+!      call mpp_clock_end(mpp_write_clock)           
+      return
+    end subroutine mpp_write_meta_axis_unlimited
+
+
     subroutine mpp_write_meta_field( unit, field, axes, name, units, longname,&
          min, max, missing, fill, scale, add, pack, time_method,standard_name, checksum)
 !define field: must have already called mpp_write_meta(axis) for each axis
@@ -475,9 +695,8 @@
 !           call mpp_error( FATAL, 'MPP_WRITE_META: cannot write metadata to file after an mpp_write.' )
            
 !pre-existing pointers need to be nullified
-      if( ASSOCIATED(field%axes) ) then
-         DEALLOCATE(field%axes, stat=istat)
-      endif
+      if( ASSOCIATED(field%axes) ) DEALLOCATE(field%axes, stat=istat)
+      if( ASSOCIATED(field%size) ) DEALLOCATE(field%size, stat=istat)
 !fill in field metadata
       field%name = name
       field%units = units
@@ -541,6 +760,7 @@
                   call mpp_error( FATAL, 'MPP_WRITE_META_FIELD: only legal packing values are 1,2,4,8.' )
           end select
           call netcdf_err( error, mpp_file(unit), field=field )
+          deallocate(axis_id)
 #ifndef use_netCDF3
           if(shuffle .NE. 0 .OR. deflate .NE. 0) then
              error = NF_DEF_VAR_DEFLATE(mpp_file(unit)%ncid, field%id, shuffle, deflate, deflate_level)
@@ -598,6 +818,22 @@
       if ( present(fill) ) then
          if( field%pack.EQ.1 .OR. field%pack.EQ.2 )then
             call mpp_write_meta( unit, field%id, '_FillValue', rval=field%fill, pack=pack )
+         else if (field%pack==0) then ! some safety checks for integer fills
+            if ( present(scale).OR.present(add) ) then
+               call mpp_error(FATAL,"add,scale not currently implimented for pack=0 int handling, try reals instead.")
+            else
+               if (KIND(field%fill)==DOUBLE_KIND) then
+                  !! check if type safe cast
+                  if ( field%fill /= TRANSFER( IBITS( TRANSFER(field%fill,INT(0,DOUBLE_KIND)) ,0 , BIT_SIZE(INT(0,INT_KIND))) , field%fill ) ) then
+                     call mpp_error(FATAL,"mpp_io(mpp_io_write.inc): For pack=0, fill and data should both be safely castable"// &
+                          "to NF_INT type, _FillValue was actually using more precision than INT_KIND, cannot safely cast.")
+                  end if  
+               else if (KIND(field%fill)/=INT_KIND) then
+                  call mpp_error(FATAL,"mpp_io(mpp_io_write.inc): Unexpected KIND of fill, use either 4, or 8 byte KIND.")
+               end if
+               !! then safe to write ival _FillValue and pack
+               call mpp_write_meta( unit, field%id, '_FillValue', ival=TRANSFER( field%fill, INT(0,INT_KIND)), pack=pack )
+            end if                     
          else
             a = nint((fill-add)/scale)
             call mpp_write_meta( unit, field%id, '_FillValue', rval=a, pack=pack )
@@ -624,7 +860,7 @@
       if ( PRESENT(standard_name)) &
            call mpp_write_meta(unit,field%id,'standard_name ', cval=field%standard_name) 
 
-      if( verbose )print '(a,2i3,x,a,i3)', 'MPP_WRITE_META: Wrote field metadata: pe, unit, field%name, field%id=', &
+      if( verbose )print '(a,2i6,x,a,i3)', 'MPP_WRITE_META: Wrote field metadata: pe, unit, field%name, field%id=', &
            pe, unit, trim(field%name), field%id
           
 !      call mpp_clock_end(mpp_write_clock) 
@@ -660,7 +896,7 @@
               write(unit)trim(text)//char(10)
           else                  !MPP_DIRECT
               write( unit,rec=mpp_file(unit)%record )trim(text)//char(10)
-              if( verbose )print '(a,i3,a,i3)', 'WRITE_ATTRIBUTE: PE=', pe, ' wrote record ', mpp_file(unit)%record
+              if( verbose )print '(a,i6,a,i3)', 'WRITE_ATTRIBUTE: PE=', pe, ' wrote record ', mpp_file(unit)%record
               mpp_file(unit)%record = mpp_file(unit)%record + 1
           end if
       end if  
@@ -679,9 +915,20 @@
       integer, allocatable :: rval_i(:)
 #ifdef use_netCDF
       if( PRESENT(rval) )then
-!pack is only meaningful for FP numbers
+!pack was only meaningful for FP numbers, but is now extended by the ival branch of this routine
           if( PRESENT(pack) )then
-              if( pack.EQ.1 )then
+             if( pack== 0 ) then !! here be dragons, use ival branch!...
+                if( KIND(rval).EQ.DOUBLE_KIND )then
+                   call mpp_error( FATAL, &
+                        'WRITE_ATTRIBUTE_NETCDF: attempting to write internal NF_INT, currently int32, as double.' )
+                   error = NF_PUT_ATT_DOUBLE( mpp_file(unit)%ncid, id, name, NF_DOUBLE, size(rval(:)), rval )
+                else if( KIND(rval).EQ.FLOAT_KIND )then
+                   call mpp_error( FATAL, &
+                        'WRITE_ATTRIBUTE_NETCDF: attempting to write internal NF_INT, currently int32, as float.' )
+                   error = NF_PUT_ATT_REAL  ( mpp_file(unit)%ncid, id, name, NF_DOUBLE, size(rval(:)), rval )
+                end if
+                call netcdf_err( error, mpp_file(unit), string=' Attribute='//name )
+             else if( pack.EQ.1 )then
                   if( KIND(rval).EQ.DOUBLE_KIND )then
                       error = NF_PUT_ATT_DOUBLE( mpp_file(unit)%ncid, id, name, NF_DOUBLE, size(rval(:)), rval )
                   else if( KIND(rval).EQ.FLOAT_KIND )then
@@ -730,8 +977,20 @@
               call netcdf_err( error, mpp_file(unit), string=' Attribute='//name )
           end if
       else if( PRESENT(ival) )then
+          if( PRESENT(pack) ) then
+             if (pack ==0) then
+                if (KIND(ival).EQ.LONG_KIND ) then
+                   call mpp_error(FATAL,'only use NF_INTs with pack=0 for now')
+                end if
+                error = NF_PUT_ATT_INT( mpp_file(unit)%ncid, id, name, NF_INT, size(ival(:)), ival ) !!XXX int32_t..
+                call netcdf_err( error, mpp_file(unit), string=' Attribute='//name )
+             else
+                call mpp_error( FATAL, 'WRITE_ATTRIBUTE_NETCDF: only implimented ints when pack=0, else use reals.' )
+             endif
+          else
           error = NF_PUT_ATT_INT ( mpp_file(unit)%ncid, id, name, NF_INT, size(ival(:)), ival )
           call netcdf_err( error, mpp_file(unit), string=' Attribute='//name )
+          end if
       else if( present(cval) )then
           error = NF_PUT_ATT_TEXT( mpp_file(unit)%ncid, id, name, len_trim(cval), cval )
           call netcdf_err( error, mpp_file(unit), string=' Attribute='//name )
@@ -802,6 +1061,20 @@
 #define MPP_TYPE_ real
 #include 
 
+#undef MPP_WRITE_COMPRESSED_1D_
+#define MPP_WRITE_COMPRESSED_1D_ mpp_write_compressed_r1d
+#undef MPP_WRITE_COMPRESSED_2D_
+#define MPP_WRITE_COMPRESSED_2D_ mpp_write_compressed_r2d
+#undef MPP_TYPE_
+#define MPP_TYPE_ real
+#include 
+
+#undef MPP_WRITE_UNLIMITED_AXIS_1D_
+#define MPP_WRITE_UNLIMITED_AXIS_1D_ mpp_write_unlimited_axis_r1d
+#undef MPP_TYPE_
+#define MPP_TYPE_ real
+#include 
+
 #undef MPP_WRITE_
 #define MPP_WRITE_ mpp_write_r0D
 #undef MPP_TYPE_
@@ -876,11 +1149,22 @@
       field%longname = axis%longname
       field%units = axis%units
 
-      allocate( field%axes(1)%data(size(axis%data) ))
-      field%axes(1)%data = axis%data
-      
+      if(ASSOCIATED(axis%data))then
+        allocate( field%axes(1)%data(size(axis%data) ))
+        field%axes(1)%data = axis%data
+        call write_record( unit, field, axis%len, axis%data )
+      elseif(ASSOCIATED(axis%idata))then
+        allocate( field%axes(1)%data(size(axis%idata) ))
+        field%axes(1)%data = REAL(axis%idata)
+        field%pack=4
+        call write_record( unit, field, axis%len, REAL(axis%idata) )
+      else
+        call mpp_error( FATAL, 'MPP_WRITE_AXIS: No data associated with axis.' )
+      endif
+
+      deallocate(field%axes(1)%data)
+      deallocate(field%axes,field%size)
 
-      call write_record( unit, field, axis%len, axis%data )
       call mpp_clock_end(mpp_write_clock)
       return
     end subroutine mpp_write_axis
@@ -946,7 +1230,7 @@
               call mpp_write_meta( unit, 'END', cval='metadata' )
           end if
           mpp_file(unit)%initialized = .TRUE.
-          if( verbose )print '(a,i3,a)', 'MPP_WRITE: PE=', pe, ' initialized file '//trim(mpp_file(unit)%name)//'.'
+          if( verbose )print '(a,i6,a)', 'MPP_WRITE: PE=', pe, ' initialized file '//trim(mpp_file(unit)%name)//'.'
       end if
           
 !initialize time: by default assume NULLTIME
@@ -960,7 +1244,7 @@
           mpp_file(unit)%time = time
           newtime = .TRUE.
       end if
-      if( verbose )print '(a,2i3,2i5,es13.5)', 'MPP_WRITE: PE, unit, %id, %time_level, %time=',&
+      if( verbose )print '(a,2i6,2i5,es13.5)', 'MPP_WRITE: PE, unit, %id, %time_level, %time=',&
            pe, unit, mpp_file(unit)%id, mpp_file(unit)%time_level, mpp_file(unit)%time
            
       if( mpp_file(unit)%format.EQ.MPP_NETCDF )then
@@ -1004,7 +1288,10 @@
                   error = NF_PUT_VAR1_REAL  ( mpp_file(unit)%ncid, mpp_file(unit)%id, mpp_file(unit)%time_level, time )
               end if
           end if  
-          if( field%pack.LE.2 )then
+          if( field%pack == 0 )then
+              packed_data = CEILING(data)
+              error = NF_PUT_VARA_INT   ( mpp_file(unit)%ncid, field%id, start, axsiz, packed_data )
+          elseif( field%pack.GT.0 .and. field%pack.LE.2 )then
               if( KIND(data).EQ.DOUBLE_KIND )then
                   error = NF_PUT_VARA_DOUBLE( mpp_file(unit)%ncid, field%id, start, axsiz, data )
               else if( KIND(data).EQ.FLOAT_KIND )then
@@ -1051,7 +1338,7 @@
 #else                 
                   write( unit, rec=mpp_file(unit)%record )field%id, subdomain, time_level, time, data
 #endif            
-                  if( debug )print '(a,i3,a,i3)', 'MPP_WRITE: PE=', pe, ' wrote record ', mpp_file(unit)%record
+                  if( debug )print '(a,i6,a,i6)', 'MPP_WRITE: PE=', pe, ' wrote record ', mpp_file(unit)%record
               end if
           end if  
       end if  
@@ -1215,7 +1502,7 @@
       if( mpp_file(unit)%threading.EQ.MPP_MULTI .AND. mpp_file(unit)%fileset.EQ.MPP_MULTI .AND. axis%domain.NE.NULL_DOMAIN1D )then
           call mpp_write_meta( unit, axis%id, 'domain_decomposition', ival=(/isg,ieg,is,ie/) )
       end if
-      if( verbose )print '(a,2i3,x,a,2i3)', 'MPP_WRITE_META: Wrote axis metadata, pe, unit, axis%name, axis%id, axis%did=', &
+      if( verbose )print '(a,2i6,x,a,2i3)', 'MPP_WRITE_META: Wrote axis metadata, pe, unit, axis%name, axis%id, axis%did=', &
            pe, unit, trim(axis%name), axis%id, axis%did
 #else      
       call mpp_error( FATAL, 'MPP_READ currently requires use_netCDF option' )
@@ -1295,6 +1582,7 @@
               case default
                   call mpp_error( FATAL, 'MPP_WRITE_META_FIELD: only legal packing values are 1,2,4,8.' )
           end select
+          deallocate( axis_id )
 #endif            
       else
           varnum = varnum + 1
@@ -1355,7 +1643,7 @@
           if( field%scale.NE.default_field%scale )call mpp_write_meta( unit, field%id, 'scale_factor',  rval=field%scale )
           if( field%add.NE.default_field%add   )call mpp_write_meta( unit, field%id, 'add_offset',    rval=field%add   )
       end if
-      if( verbose )print '(a,2i3,x,a,i3)', 'MPP_WRITE_META: Wrote field metadata: pe, unit, field%name, field%id=', &
+      if( verbose )print '(a,2i6,x,a,i3)', 'MPP_WRITE_META: Wrote field metadata: pe, unit, field%name, field%id=', &
            pe, unit, trim(field%name), field%id
 
 !      call mpp_clock_end(mpp_write_clock)           
diff --git a/src/shared/mpp/include/mpp_read_2Ddecomp.h b/src/shared/mpp/include/mpp_read_2Ddecomp.h
index 65964344e5..f76a8f1c23 100644
--- a/src/shared/mpp/include/mpp_read_2Ddecomp.h
+++ b/src/shared/mpp/include/mpp_read_2Ddecomp.h
@@ -132,3 +132,18 @@
 
       return
     end subroutine MPP_READ_2DDECOMP_3D_
+
+
+    subroutine MPP_READ_2DDECOMP_4D_( unit, field, domain, data, tindex, tile_count )
+      integer, intent(in) :: unit
+      type(fieldtype), intent(in) :: field
+      type(domain2D), intent(in) :: domain
+      MPP_TYPE_, intent(inout) :: data(:,:,:,:)
+      integer, intent(in), optional :: tindex, tile_count
+      MPP_TYPE_ :: data3D(size(data,1),size(data,2),size(data,3)*size(data,4))
+      pointer( ptr, data3D )
+      ptr = LOC(data)
+      call mpp_read( unit, field, domain, data3D, tindex, tile_count)
+      return
+    end subroutine MPP_READ_2DDECOMP_4D_
+
diff --git a/src/shared/mpp/include/mpp_read_compressed.h b/src/shared/mpp/include/mpp_read_compressed.h
new file mode 100644
index 0000000000..d32c0f5f57
--- /dev/null
+++ b/src/shared/mpp/include/mpp_read_compressed.h
@@ -0,0 +1,93 @@
+    subroutine MPP_READ_COMPRESSED_1D_(unit, field, domain, data, tindex)
+      integer, intent(in) :: unit 
+      type(fieldtype), intent(in) :: field
+      type(domain2D),  intent(in) :: domain
+      MPP_TYPE_, intent(inout) :: data(:)
+      integer,           intent(in), optional :: tindex
+
+      MPP_TYPE_ :: data2D(size(data,1),1)
+      pointer( ptr, data2D )
+      ptr = LOC(data)
+
+      call mpp_read(unit, field, domain, data2D, tindex)
+      return
+    end subroutine MPP_READ_COMPRESSED_1D_
+
+    subroutine MPP_READ_COMPRESSED_2D_(unit, field, domain, data, tindex, start, nread, threading)
+      integer,           intent(in)           :: unit
+      type(fieldtype),   intent(in)           :: field
+      type(domain2D),    intent(in)           :: domain
+      MPP_TYPE_,         intent(inout)        :: data(:,:)
+      integer,           intent(in), optional :: tindex
+      integer,           intent(in), optional :: start(:), nread(:)
+      integer,           intent(in), optional :: threading
+
+      integer, allocatable :: pelist(:)
+      integer :: npes, p, threading_flag
+      type(domain2d), pointer :: io_domain=>NULL()
+      logical :: compute_chksum
+      integer(LONG_KIND) ::chk
+
+      call mpp_clock_begin(mpp_read_clock)
+
+      data = 0 !! zero out data so other tiles do not contribute junk to chksum
+
+      if( .NOT.module_is_initialized )call mpp_error( FATAL, 'MPP_READ_COMPRESSED_2D_: must first call mpp_io_init.' )
+      if( .NOT.mpp_file(unit)%valid )call mpp_error( FATAL, 'MPP_READ_COMPRESSED_2D_: invalid unit number.' )
+
+      threading_flag = MPP_SINGLE
+      if( PRESENT(threading) )threading_flag = threading
+      if( threading_flag == MPP_MULTI ) then
+         call read_record(unit,field,size(data(:,:)),data,tindex,start_in=start, axsiz_in=nread)
+      else if( threading_flag == MPP_SINGLE ) then
+
+         io_domain=>mpp_get_io_domain(domain) 
+         if(.not. ASSOCIATED(io_domain)) call mpp_error( FATAL, 'MPP_READ_COMPRESSED_2D_: io_domain must be defined.' )
+         npes = mpp_get_domain_npes(io_domain)
+         allocate(pelist(npes))
+         call mpp_get_pelist(io_domain,pelist)
+         
+         if(mpp_pe() == pelist(1)) call read_record(unit,field,size(data(:,:)),data,tindex,start_in=start, axsiz_in=nread)
+
+         !--- z1l replace mpp_broadcast with mpp_send/mpp_recv to avoid hang in calling MPI_COMM_CREATE
+         !---     because size(pelist) might be different for different rank.
+         !--- prepost receive
+         if( mpp_pe() == pelist(1) ) then
+            do p = 2, npes
+               call mpp_send(data(1,1), plen=size(data(:,:)), to_pe=pelist(p), tag=COMM_TAG_1)
+            enddo
+            call mpp_sync_self()
+         else
+            call mpp_recv(data(1,1), glen=size(data(:,:)), from_pe=pelist(1), block=.false., tag=COMM_TAG_1)
+            call mpp_sync_self(check=EVENT_RECV)
+         endif
+
+         deallocate(pelist)
+      else
+         call mpp_error( FATAL, 'MPP_READ_COMPRESSED_2D_: threading should be MPP_SINGLE or MPP_MULTI')
+      endif
+
+      compute_chksum = .FALSE.
+      if (ANY(field%checksum /= default_field%checksum) ) compute_chksum = .TRUE.
+
+      if (compute_chksum) then 
+         if (field%type==NF_INT) then 
+            if (CEILING(field%fill) /= MPP_FILL_INT ) then
+               call mpp_error(NOTE,"During mpp_io(read_compressed) int field "//trim(field%name)// & 
+							 " found integer fill /= MPP_FILL_INT. Confirm this is what you want.")
+               chk = mpp_chksum( ceiling(data), mask_val=field%fill)  
+            else
+               chk = mpp_chksum( ceiling(data), mask_val=CEILING(field%fill) )  
+            end if    
+         else !!real
+            chk = mpp_chksum(data,mask_val=field%fill)
+         end if
+         !!compare
+         if (mpp_pe()==mpp_root_pe()) print '(A,Z16)', "mpp_read_compressed chksum: "//trim(field%name)//" = ",  chk
+         !! discuss making fatal after testing/review to match other routines.
+         if (chk /= field%checksum(1) ) call mpp_error(NOTE,"mpp_read_compressed chksum: "//trim(field%name)//" failed!")
+      end if
+
+      call mpp_clock_end(mpp_read_clock)
+      return
+    end subroutine MPP_READ_COMPRESSED_2D_
diff --git a/src/shared/mpp/include/mpp_read_distributed_ascii.h b/src/shared/mpp/include/mpp_read_distributed_ascii.h
new file mode 100644
index 0000000000..8334d3bcae
--- /dev/null
+++ b/src/shared/mpp/include/mpp_read_distributed_ascii.h
@@ -0,0 +1,29 @@
+subroutine MPP_READ_DISTRIBUTED_ASCII_1D_ (unit,fmt,ssize,data,iostat)
+  integer, intent(in)               :: unit
+  character(*), intent(in)          :: fmt
+  integer, intent(in)               :: ssize
+  MPP_TYPE_, dimension(:), intent(inout) :: data
+  integer, intent(out)              :: iostat
+
+  integer, allocatable :: pelist(:)
+  logical              :: is_ioroot=.false.
+
+  if(.not.module_is_initialized) call mpp_error(FATAL,'MPP_READ_DISTRIBUTED_ASCII_1D_:  module not initialized')
+
+  iostat = 0
+  call mpp_dist_io_pelist(ssize,pelist)  ! ALLOCATE and create pelist if size of group > 1
+  if(.not. ALLOCATED(pelist)) &
+           call mpp_error(FATAL,'MPP_READ_DISTRIBUTED_ASCII_1D_:: pelist allocation failed')
+  is_ioroot = mpp_is_dist_ioroot(ssize)
+  if(is_ioroot) then
+    if(trim(fmt)=='*')then
+       read(unit,*,iostat=iostat) data
+    else
+       read(unit,fmt=trim(fmt),iostat=iostat) data
+    endif
+    if(iostat /= 0) return  ! Calling routine must handle error
+  endif
+
+  call mpp_broadcast(data,size(data),pelist(1),pelist)
+  deallocate(pelist)  ! Don't forget to deallocate pelist
+end subroutine MPP_READ_DISTRIBUTED_ASCII_1D_
diff --git a/src/shared/mpp/include/mpp_read_distributed_ascii.inc b/src/shared/mpp/include/mpp_read_distributed_ascii.inc
new file mode 100644
index 0000000000..7eccf03b65
--- /dev/null
+++ b/src/shared/mpp/include/mpp_read_distributed_ascii.inc
@@ -0,0 +1,42 @@
+#undef MPP_READ_DISTRIBUTED_ASCII_1D_
+#define MPP_READ_DISTRIBUTED_ASCII_1D_ mpp_read_distributed_ascii_r1D
+#undef MPP_TYPE_
+#define MPP_TYPE_ real
+#include 
+
+#undef MPP_READ_DISTRIBUTED_ASCII_1D_
+#define MPP_READ_DISTRIBUTED_ASCII_1D_ mpp_read_distributed_ascii_i1D
+#undef MPP_TYPE_
+#define MPP_TYPE_ integer
+#include 
+
+subroutine mpp_read_distributed_ascii_a1D(unit,fmt,ssize,data,iostat)
+  integer, intent(in)               :: unit
+  character(*), intent(in)          :: fmt
+  integer, intent(in)               :: ssize
+  character(len=*), dimension(:), intent(inout) :: data
+  integer, intent(out)              :: iostat
+
+  integer, allocatable :: pelist(:)
+  logical              :: is_ioroot=.false.
+
+
+  if(.not.module_is_initialized) call mpp_error(FATAL,'mpp_read_distributed_ascii_a1D:  module not initialized')
+
+  iostat = 0
+  call mpp_dist_io_pelist(ssize,pelist)
+  if(.not. ALLOCATED(pelist)) &
+           call mpp_error(FATAL,'mpp_read_distributed_ascii_a1D: pelist allocation failed')
+  is_ioroot = mpp_is_dist_ioroot(ssize)
+  if(is_ioroot) then
+    if(trim(fmt)=='*')then
+       read(unit,*,iostat=iostat) data
+    else
+       read(unit,fmt=trim(fmt),iostat=iostat) data
+    endif
+    if(iostat /= 0) return  ! Calling routine must handle error
+  endif
+
+  call mpp_broadcast(data,len(data(1)),pelist(1),pelist)
+  deallocate(pelist)  ! Don't forget to deallocate pelist
+end subroutine mpp_read_distributed_ascii_a1D
diff --git a/src/shared/mpp/include/mpp_reduce_mpi.h b/src/shared/mpp/include/mpp_reduce_mpi.h
index 2f79767566..3fe50c8567 100644
--- a/src/shared/mpp/include/mpp_reduce_mpi.h
+++ b/src/shared/mpp/include/mpp_reduce_mpi.h
@@ -9,10 +9,10 @@
       if( .NOT.module_is_initialized )call mpp_error( FATAL, 'MPP_REDUCE: You must first call mpp_init.' )
       n = get_peset(pelist); if( peset(n)%count.EQ.1 )return
 
-      if( current_clock.NE.0 )call SYSTEM_CLOCK(start_tick)
+      if( debug .and. (current_clock.NE.0) )call SYSTEM_CLOCK(start_tick)
       if( verbose )call mpp_error( NOTE, 'MPP_REDUCE_: using MPI_ALLREDUCE...' )
       call MPI_ALLREDUCE( a, work, 1, MPI_TYPE_, MPI_REDUCE_, peset(n)%id, error )
       a = work
-      if( current_clock.NE.0 )call increment_current_clock( EVENT_ALLREDUCE, MPP_TYPE_BYTELEN_ )
+      if( debug .and. (current_clock.NE.0) )call increment_current_clock( EVENT_ALLREDUCE, MPP_TYPE_BYTELEN_ )
       return
     end subroutine MPP_REDUCE_
diff --git a/src/shared/mpp/include/mpp_reduce_sma.h b/src/shared/mpp/include/mpp_reduce_sma.h
index 595746dc85..a6cfa992c8 100644
--- a/src/shared/mpp/include/mpp_reduce_sma.h
+++ b/src/shared/mpp/include/mpp_reduce_sma.h
@@ -13,7 +13,7 @@
       if( .NOT.module_is_initialized )call mpp_error( FATAL, 'MPP_REDUCE: You must first call mpp_init.' )
       n = get_peset(pelist); if( peset(n)%count.EQ.1 )return
 
-      if( current_clock.NE.0 )call SYSTEM_CLOCK(start_tick)
+      if( debug .and. (current_clock.NE.0) )call SYSTEM_CLOCK(start_tick)
 !allocate space from the stack for pwrk and b
       ptr = LOC(mpp_stack)
       words = size(work(:))*size(transfer(work(1),word))
@@ -27,6 +27,6 @@
       call SHMEM_REDUCE_( work, work, 1, peset(n)%start, peset(n)%log2stride, peset(n)%count, work(2), sync )
       call mpp_sync(pelist)
       a = work(1)
-      if( current_clock.NE.0 )call increment_current_clock( EVENT_ALLREDUCE, MPP_TYPE_BYTELEN_ )
+      if( debug .and. (current_clock.NE.0) )call increment_current_clock( EVENT_ALLREDUCE, MPP_TYPE_BYTELEN_ )
       return
     end subroutine MPP_REDUCE_
diff --git a/src/shared/mpp/include/mpp_scatter.h b/src/shared/mpp/include/mpp_scatter.h
new file mode 100644
index 0000000000..f3d73dcc45
--- /dev/null
+++ b/src/shared/mpp/include/mpp_scatter.h
@@ -0,0 +1,131 @@
+subroutine MPP_SCATTER_PELIST_2D_(is, ie, js, je, pelist, array_seg, data, is_root_pe, &
+                                  ishift, jshift)
+   integer,                           intent(in)    :: is, ie, js, je
+   integer,   dimension(:),           intent(in)    :: pelist
+   MPP_TYPE_, dimension(is:ie,js:je), intent(inout)    :: array_seg
+   MPP_TYPE_, dimension(:,:),         intent(in) :: data
+   logical,                           intent(in)    :: is_root_pe
+   integer,   optional,               intent(in)    :: ishift, jshift
+
+   MPP_TYPE_ ::  arr3D(size(array_seg,1),size(array_seg,2),1)
+   MPP_TYPE_ :: data3D(size(     data,1),size(     data,2),1)
+   pointer( aptr,  arr3D )
+   pointer( dptr, data3D )
+   aptr = LOC(array_seg)
+   dptr = LOC(     data)
+
+   call mpp_scatter(is, ie, js, je, 1, pelist, arr3D, data3D, is_root_pe, &
+                    ishift, jshift)
+   return
+
+end subroutine MPP_SCATTER_PELIST_2D_
+
+
+subroutine MPP_SCATTER_PELIST_3D_(is, ie, js, je, nk, pelist, array_seg, data, is_root_pe, &
+                                  ishift, jshift)
+   integer,                                intent(in)    :: is, ie, js, je, nk
+   integer,   dimension(:),                intent(in)    :: pelist
+   MPP_TYPE_, dimension(is:ie,js:je,1:nk), intent(inout)    :: array_seg
+   MPP_TYPE_, dimension(:,:,:),            intent(in) :: data
+   logical,                                intent(in)    :: is_root_pe
+   integer,   optional,                    intent(in)    :: ishift, jshift
+
+   integer :: i, msgsize, root_pe, root_pe_test
+   integer :: i1, i2, j1, j2, ioff, joff
+   integer :: my_ind(4), gind(4,size(pelist))
+   type array3D
+     MPP_TYPE_, dimension(:,:,:), allocatable :: data
+   endtype array3D
+   type(array3d), dimension(size(pelist)) :: temp
+
+   if (.not.ANY(mpp_pe().eq.pelist(:))) return
+
+   if (is_root_pe) then
+     root_pe = mpp_pe()
+     root_pe_test = 999
+     if (.not.ANY(pelist(:).eq.root_pe)) call mpp_error(FATAL, &
+                "fms_io(mpp_scatter_pelist): root_pe not a member of pelist")
+   else
+     root_pe = 0
+     root_pe_test = -999
+   endif
+! need this check in case MPI-rank 0 is a member of the pelist
+   call mpp_max(root_pe_test, pelist)
+   if (root_pe_test.lt.0) call mpp_error(FATAL, &
+                "fms_io(mpp_scatter_pelist): root_pe not specified or not a member of the pelist")
+! need to make sure only one root_pe has been specified
+   call mpp_sum(root_pe, pelist)
+   if ((is_root_pe) .and. (mpp_pe().ne.root_pe)) call mpp_error(FATAL, &
+                "fms_io(mpp_scatter_pelist): too many root_pes specified")
+
+
+   ioff=0
+   joff=0
+   if (present(ishift)) ioff=ishift
+   if (present(jshift)) joff=jshift
+
+   my_ind(1) = is
+   my_ind(2) = ie
+   my_ind(3) = js
+   my_ind(4) = je
+
+! scatter indices into global index on root_pe
+   if (is_root_pe) then
+     do i = 1, size(pelist)
+! root_pe data copy - no send to self
+       if (pelist(i).eq.root_pe) then
+         gind(:,i) = my_ind(:)
+       else
+         call mpp_recv(gind(:,i:i), 4, pelist(i), .FALSE., COMM_TAG_1)
+       endif
+     enddo
+     call mpp_sync_self(check=EVENT_RECV)
+     gind(1,:)=gind(1,:)+ioff
+     gind(2,:)=gind(2,:)+ioff
+     gind(3,:)=gind(3,:)+joff
+     gind(4,:)=gind(4,:)+joff
+! check indices to make sure they are within the range of "data"
+     if ((minval(gind).lt.1) .OR. (maxval(gind(1:2,:)).gt.size(data,1)) .OR. (maxval(gind(3:4,:)).gt.size(data,2))) &
+         call mpp_error(FATAL,"fms_io(mpp_scatter_pelist): specified indices (with shift) are outside of the & 
+                        &range of the receiving array")
+   else
+! non root_pe's send indices to root_pe
+     call mpp_send(my_ind(:), 4, root_pe, COMM_TAG_1)
+     call mpp_sync_self(check=EVENT_SEND)
+   endif
+
+!  scatter segments into data based on indices
+   if (is_root_pe) then
+     do i = 1, size(pelist)
+       if (pelist(i).ne.root_pe) then    ! no send to self
+         i1 = gind(1,i)
+         i2 = gind(2,i)
+         j1 = gind(3,i)
+         j2 = gind(4,i)
+         msgsize = (i2-i1+1)*(j2-j1+1)*nk
+! allocate and copy data into a contiguous memory space
+         allocate(temp(i)%data(i1:i2,j1:j2,1:nk))
+         temp(i)%data(i1:i2,j1:j2,1:nk)=data(i1:i2,j1:j2,1:nk)
+         call mpp_send(temp(i)%data, msgsize, pelist(i), COMM_TAG_2)
+       else
+!        data copy - no send to self
+         array_seg(is:ie,js:je,1:nk) = data(is+ioff:ie+ioff,js+joff:je+joff,1:nk)
+       endif
+     enddo
+     call mpp_sync_self(check=EVENT_SEND)
+! deallocate the temporary array used for the send
+     do i = 1, size(pelist)
+       if (allocated(temp(i)%data)) deallocate(temp(i)%data)
+     enddo
+   else
+!    non root_pe's recv data from root_pe
+     msgsize = (my_ind(2)-my_ind(1)+1) * (my_ind(4)-my_ind(3)+1) * nk
+     call mpp_recv(array_seg, msgsize, root_pe, .FALSE., COMM_TAG_2)
+     call mpp_sync_self(check=EVENT_RECV)
+   endif
+
+   call mpp_sync_self()
+
+   return
+
+end subroutine MPP_SCATTER_PELIST_3D_
diff --git a/src/shared/mpp/include/mpp_sum.inc b/src/shared/mpp/include/mpp_sum.inc
index 35037d3f3b..bb6aa7b4b1 100644
--- a/src/shared/mpp/include/mpp_sum.inc
+++ b/src/shared/mpp/include/mpp_sum.inc
@@ -1,6 +1,6 @@
 
 ! -*-f90-*-
-! $Id: mpp_sum.inc,v 18.0 2010/03/02 23:57:50 fms Exp $
+! $Id$
 
 !#######################################################################
 
diff --git a/src/shared/mpp/include/mpp_sum_mpi.h b/src/shared/mpp/include/mpp_sum_mpi.h
index 7574e1c1af..4047492194 100644
--- a/src/shared/mpp/include/mpp_sum_mpi.h
+++ b/src/shared/mpp/include/mpp_sum_mpi.h
@@ -12,7 +12,7 @@
       if( .NOT.module_is_initialized )call mpp_error( FATAL, 'MPP_SUM: You must first call mpp_init.' )
       n = get_peset(pelist); if( peset(n)%count.EQ.1 )return
 
-      if( current_clock.NE.0 )call SYSTEM_CLOCK(start_tick)
+      if( debug .and. (current_clock.NE.0) )call SYSTEM_CLOCK(start_tick)
       if( verbose )call mpp_error( NOTE, 'MPP_SUM: using MPI_ALLREDUCE...' )
       if( debug ) then
           errunit = stderr()
@@ -20,7 +20,7 @@
       endif
       call MPI_ALLREDUCE( a, work, length, MPI_TYPE_, MPI_SUM, peset(n)%id, error )
       a(1:length) = work(1:length)
-      if( current_clock.NE.0 )call increment_current_clock( EVENT_ALLREDUCE, length*MPP_TYPE_BYTELEN_ )
+      if( debug .and. (current_clock.NE.0) )call increment_current_clock( EVENT_ALLREDUCE, length*MPP_TYPE_BYTELEN_ )
       return
     end subroutine MPP_SUM_
 
diff --git a/src/shared/mpp/include/mpp_sum_sma.h b/src/shared/mpp/include/mpp_sum_sma.h
index 28fc24957c..53396e0bf3 100644
--- a/src/shared/mpp/include/mpp_sum_sma.h
+++ b/src/shared/mpp/include/mpp_sum_sma.h
@@ -16,7 +16,7 @@
       if( .NOT.module_is_initialized )call mpp_error( FATAL, 'MPP_SUM: You must first call mpp_init.' )
       n = get_peset(pelist); if( peset(n)%count.EQ.1 )return
 
-      if( current_clock.NE.0 )call SYSTEM_CLOCK(start_tick)
+      if( debug .and. (current_clock.NE.0) )call SYSTEM_CLOCK(start_tick)
 !allocate space from the stack for pwrk and b
       ptr = LOC(mpp_stack)
       words = size(work(:))*size(transfer(work(1),word))
@@ -29,7 +29,7 @@
       call mpp_sync(pelist)
       call SHMEM_SUM_( work, work, length, peset(n)%start, peset(n)%log2stride, peset(n)%count, work(length+1), sync )
       a(1:length) = work(1:length)
-      if( current_clock.NE.0 )call increment_current_clock( EVENT_ALLREDUCE, length*MPP_TYPE_BYTELEN_ )
+      if( debug .and. (current_clock.NE.0) )call increment_current_clock( EVENT_ALLREDUCE, length*MPP_TYPE_BYTELEN_ )
       return
     end subroutine MPP_SUM_
 
diff --git a/src/shared/mpp/include/mpp_transmit.inc b/src/shared/mpp/include/mpp_transmit.inc
index de2c45a7c6..82f7073527 100644
--- a/src/shared/mpp/include/mpp_transmit.inc
+++ b/src/shared/mpp/include/mpp_transmit.inc
@@ -1,6 +1,6 @@
 
 ! -*-f90-*-
-! $Id: mpp_transmit.inc,v 19.0 2012/01/06 22:04:03 fms Exp $
+! $Id$
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 !                                                                             !
diff --git a/src/shared/mpp/include/mpp_transmit_mpi.h b/src/shared/mpp/include/mpp_transmit_mpi.h
index 4dcaa9184d..1f9009df6b 100644
--- a/src/shared/mpp/include/mpp_transmit_mpi.h
+++ b/src/shared/mpp/include/mpp_transmit_mpi.h
@@ -44,7 +44,7 @@
       out_unit = stdout()
       if( debug )then
           call SYSTEM_CLOCK(tick)
-          write( out_unit,'(a,i18,a,i5,a,2i5,2i8)' )&
+          write( out_unit,'(a,i18,a,i6,a,2i6,2i8)' )&
                'T=',tick, ' PE=',pe, ' MPP_TRANSMIT begin: to_pe, from_pe, put_len, get_len=', to_pe, from_pe, put_len, get_len
       end if
 
@@ -54,7 +54,7 @@
 !do put first and then get
       if( to_pe.GE.0 .AND. to_pe.LT.npes )then
 !use non-blocking sends
-          if( current_clock.NE.0 )call SYSTEM_CLOCK(start_tick)
+          if( debug .and. (current_clock.NE.0) )call SYSTEM_CLOCK(start_tick)
 !z1l: truly non-blocking send.
 !          if( request(to_pe).NE.MPI_REQUEST_NULL )then !only one message from pe->to_pe in queue 
 !              if( debug )write( stderr(),* )'PE waiting for sending', pe, to_pe
@@ -68,7 +68,7 @@
                 "MPP_TRANSMIT: cur_send_request is greater than max_request, increase mpp_nml request_multiply")
              call MPI_ISEND( put_data, put_len, MPI_TYPE_, to_pe, comm_tag, mpp_comm_private, request_send(cur_send_request), error)
           endif
-          if( current_clock.NE.0 )call increment_current_clock( EVENT_SEND, put_len*MPP_TYPE_BYTELEN_ )
+          if( debug .and. (current_clock.NE.0) )call increment_current_clock( EVENT_SEND, put_len*MPP_TYPE_BYTELEN_ )
       else if( to_pe.EQ.ALL_PES )then !this is a broadcast from from_pe
           if( from_pe.LT.0 .OR. from_pe.GE.npes )call mpp_error( FATAL, 'MPP_TRANSMIT: broadcasting from invalid PE.' )
           if( put_len.GT.get_len )call mpp_error( FATAL, 'MPP_TRANSMIT: size mismatch between put_data and get_data.' )
@@ -92,7 +92,7 @@
 !do the get: for libSMA, a get means do a wait to ensure put on remote PE is complete
       if( from_pe.GE.0 .AND. from_pe.LT.npes )then
 !receive from from_pe
-          if( current_clock.NE.0 )call SYSTEM_CLOCK(start_tick)
+          if( debug .and. (current_clock.NE.0) )call SYSTEM_CLOCK(start_tick)
           if( block_comm ) then
              call MPI_RECV( get_data, get_len, MPI_TYPE_, from_pe, comm_tag, mpp_comm_private, stat, error )
              call MPI_GET_COUNT( stat, MPI_TYPE_, rsize, error)
@@ -117,12 +117,12 @@
                 type_recv(cur_recv_request) = MPI_TYPE_
              endif
           endif
-          if( current_clock.NE.0 )call increment_current_clock( EVENT_RECV, get_len*MPP_TYPE_BYTELEN_ )
+          if( debug .and. (current_clock.NE.0) )call increment_current_clock( EVENT_RECV, get_len*MPP_TYPE_BYTELEN_ )
       else if( from_pe.EQ.ANY_PE )then
 !receive from MPI_ANY_SOURCE
-          if( current_clock.NE.0 )call SYSTEM_CLOCK(start_tick)
+          if( debug .and. (current_clock.NE.0) )call SYSTEM_CLOCK(start_tick)
           call MPI_RECV( get_data, get_len, MPI_TYPE_, MPI_ANY_SOURCE, comm_tag, mpp_comm_private, stat, error )
-          if( current_clock.NE.0 )call increment_current_clock( EVENT_RECV, get_len*MPP_TYPE_BYTELEN_ )
+          if( debug .and. (current_clock.NE.0) )call increment_current_clock( EVENT_RECV, get_len*MPP_TYPE_BYTELEN_ )
       else if( from_pe.EQ.ALL_PES )then
           call mpp_error( FATAL, 'MPP_TRANSMIT: from_pe=ALL_PES has ambiguous meaning, and hence is not implemented.' )
       else if( from_pe.NE.NULL_PE )then !only remaining valid choice is NULL_PE
@@ -131,7 +131,7 @@
 
       if( debug )then
           call SYSTEM_CLOCK(tick)
-          write( out_unit,'(a,i18,a,i5,a,2i5,2i8)' )&
+          write( out_unit,'(a,i18,a,i6,a,2i6,2i8)' )&
                'T=',tick, ' PE=',pe, ' MPP_TRANSMIT end: to_pe, from_pe, put_len, get_len=', to_pe, from_pe, put_len, get_len
       end if
       return
@@ -158,14 +158,14 @@
       out_unit = stdout()
       if( debug )then
           call SYSTEM_CLOCK(tick)
-          write( out_unit,'(a,i18,a,i5,a,2i5,2i8)' )&
+          write( out_unit,'(a,i18,a,i6,a,2i6,2i8)' )&
                'T=',tick, ' PE=',pe, ' MPP_BROADCAST begin: from_pe, length=', from_pe, length
       end if
 
       if( .NOT.ANY(from_pe.EQ.peset(current_peset_num)%list) ) &
            call mpp_error( FATAL, 'MPP_BROADCAST: broadcasting from invalid PE.' )
 
-      if( current_clock.NE.0 )call SYSTEM_CLOCK(start_tick)
+      if( debug .and. (current_clock.NE.0) )call SYSTEM_CLOCK(start_tick)
  ! find the rank of from_pe in the pelist.     
       do i = 1, mpp_npes()
          if(peset(n)%list(i) == from_pe) then
@@ -174,7 +174,7 @@
          endif
       enddo
       if( mpp_npes().GT.1 )call MPI_BCAST( data, length, MPI_TYPE_, from_rank, peset(n)%id, error )
-      if( current_clock.NE.0 )call increment_current_clock( EVENT_BROADCAST, length*MPP_TYPE_BYTELEN_ )
+      if( debug .and. (current_clock.NE.0) )call increment_current_clock( EVENT_BROADCAST, length*MPP_TYPE_BYTELEN_ )
       return
     end subroutine MPP_BROADCAST_
 
diff --git a/src/shared/mpp/include/mpp_transmit_sma.h b/src/shared/mpp/include/mpp_transmit_sma.h
index 7b2acce659..5cb9a92f0a 100644
--- a/src/shared/mpp/include/mpp_transmit_sma.h
+++ b/src/shared/mpp/include/mpp_transmit_sma.h
@@ -62,20 +62,20 @@
 #ifdef _CRAYT90
           call SHMEM_UDCFLUSH !invalidate data cache
 #endif
-          if( current_clock.NE.0 )call SYSTEM_CLOCK(start_tick)
+          if( debug .and. (current_clock.NE.0) )call SYSTEM_CLOCK(start_tick)
           call SHMEM_INT8_WAIT( status(to_pe), MPP_WAIT )
           status(to_pe) = MPP_WAIT !prohibit puts to to_pe until it has retrieved this message
-          if( current_clock.NE.0 )call increment_current_clock(EVENT_WAIT)
+          if( debug .and. (current_clock.NE.0) )call increment_current_clock(EVENT_WAIT)
 #ifdef __ia64
           data_loc = shmem_ptr(put_data,pe)
 !          write(0,*)'pe, data_loc, loc(put_data)=', pe, data_loc, loc(put_data)
 #else
           data_loc = LOC(put_data)
 #endif
-          if( current_clock.NE.0 )call SYSTEM_CLOCK(start_tick)
+          if( debug .and. (current_clock.NE.0) )call SYSTEM_CLOCK(start_tick)
           call SHMEM_INTEGER_PUT( mpp_from_pe, pe, 1, to_pe )
           call SHMEM_PUT8( remote_data_loc(pe), data_loc, 1, to_pe )
-          if( current_clock.NE.0 )call increment_current_clock( EVENT_SEND, put_len*MPP_TYPE_BYTELEN_ )
+          if( debug .and. (current_clock.NE.0) )call increment_current_clock( EVENT_SEND, put_len*MPP_TYPE_BYTELEN_ )
       else if( to_pe.EQ.ALL_PES )then !this is a broadcast from from_pe
           if( from_pe.LT.0 .OR. from_pe.GE.npes )call mpp_error( FATAL, 'MPP_TRANSMIT: broadcasting from invalid PE.' )
           if( put_len.GT.get_len )call mpp_error( FATAL, 'MPP_TRANSMIT: size mismatch between put_data and get_data.' )
@@ -92,10 +92,10 @@
 
       else if( to_pe.EQ.ANY_PE )then !we don't have a destination to do puts to, so only do gets
           if( from_pe.LT.0 .OR. from_pe.GE.npes )call mpp_error( FATAL, 'MPP_TRANSMIT: invalid from_pe along with to_pe=ANY_PE.' )
-          if( current_clock.NE.0 )call SYSTEM_CLOCK(start_tick)
+          if( debug .and. (current_clock.NE.0) )call SYSTEM_CLOCK(start_tick)
           call SHMEM_GET_( get_data, put_data, get_len, from_pe )
           call SHMEM_PUT8( status(pe), MPP_READY, 1, from_pe ) !tell from_pe that you have retrieved this message
-          if( current_clock.NE.0 )call increment_current_clock( EVENT_RECV, get_len*MPP_TYPE_BYTELEN_ )
+          if( debug .and. (current_clock.NE.0) )call increment_current_clock( EVENT_RECV, get_len*MPP_TYPE_BYTELEN_ )
           return
       else if( to_pe.NE.NULL_PE )then  !no other valid cases except NULL_PE
           call mpp_error( FATAL, 'MPP_TRANSMIT: invalid to_pe.' )
@@ -106,14 +106,14 @@
 #ifdef _CRAYT90
           call SHMEM_UDCFLUSH !invalidate data cache
 #endif
-          if( current_clock.NE.0 )call SYSTEM_CLOCK(start_tick)
+          if( debug .and. (current_clock.NE.0) )call SYSTEM_CLOCK(start_tick)
           if( debug )write( errunit,* )'pe, from_pe, remote_data_loc(from_pe)=', pe, from_pe, remote_data_loc(from_pe)
           call SHMEM_INT8_WAIT( remote_data_loc(from_pe), MPP_WAIT )
-          if( current_clock.NE.0 )call increment_current_clock(EVENT_WAIT)
+          if( debug .and. (current_clock.NE.0) )call increment_current_clock(EVENT_WAIT)
           ptr_remote_data = remote_data_loc(from_pe)
           remote_data_loc(from_pe) = MPP_WAIT !reset
 !          call SHMEM_PUT8( status(pe), MPP_READY, 1, from_pe ) !tell from_pe we have retrieved the location
-          if( current_clock.NE.0 )call SYSTEM_CLOCK(start_tick)
+          if( debug .and. (current_clock.NE.0) )call SYSTEM_CLOCK(start_tick)
 #if defined(CRAYPVP) || defined(sgi_mipspro) || defined(__ia64)
 !since we have the pointer to remote data, just retrieve it with a simple copy
           if( LOC(get_data).NE.LOC(remote_data) )then
@@ -128,7 +128,7 @@
           call SHMEM_GET_( get_data, remote_data, get_len, from_pe )
 #endif
           call SHMEM_PUT8( status(pe), MPP_READY, 1, from_pe ) !tell from_pe we have retrieved the location
-          if( current_clock.NE.0 )call increment_current_clock( EVENT_RECV, get_len*MPP_TYPE_BYTELEN_ )
+          if( debug .and. (current_clock.NE.0) )call increment_current_clock( EVENT_RECV, get_len*MPP_TYPE_BYTELEN_ )
 
       else if( from_pe.EQ.ANY_PE )then
 #ifdef _CRAYT90
@@ -137,15 +137,15 @@
 !since we don't know which PE is sending us data, we wait for remote PE to send us its ID
 !this is only required for !CRAYPVP  && !sgi_mipspro, but is done there too, so that we can send put_is_done back.
           call shmem_integer_wait( mpp_from_pe, ANY_PE )
-          if( current_clock.NE.0 )call SYSTEM_CLOCK(start_tick)
+          if( debug .and. (current_clock.NE.0) )call SYSTEM_CLOCK(start_tick)
           call SHMEM_INT8_WAIT( remote_data_loc(mpp_from_pe), MPP_WAIT )
-          if( current_clock.NE.0 )call increment_current_clock(EVENT_WAIT)
+          if( debug .and. (current_clock.NE.0) )call increment_current_clock(EVENT_WAIT)
           ptr_remote_data = remote_data_loc(mpp_from_pe)
           remote_data_loc(mpp_from_pe) = MPP_WAIT !reset
           call SHMEM_PUT8( status(pe), MPP_READY, 1, mpp_from_pe ) !tell mpp_from_pe we have retrieved the location
 #if defined(CRAYPVP) || defined(sgi_mipspro) || defined(__ia64)
 !since we have the pointer to remote data, just retrieve it with a simple copy
-          if( current_clock.NE.0 )call SYSTEM_CLOCK(start_tick)
+          if( debug .and. (current_clock.NE.0) )call SYSTEM_CLOCK(start_tick)
           if( LOC(get_data).NE.LOC(remote_data) )then
 !dir$ IVDEP
               do i = 1,get_len
@@ -155,7 +155,7 @@
 #else
           call SHMEM_GET_( get_data, remote_data, get_len, mpp_from_pe )
 #endif
-          if( current_clock.NE.0 )call increment_current_clock( EVENT_RECV, get_len*MPP_TYPE_BYTELEN_ )
+          if( debug .and. (current_clock.NE.0) )call increment_current_clock( EVENT_RECV, get_len*MPP_TYPE_BYTELEN_ )
           mpp_from_pe = ANY_PE   !reset
       else if( from_pe.EQ.ALL_PES )then
           call mpp_error( FATAL, 'MPP_TRANSMIT: from_pe=ALL_PES has ambiguous meaning, and hence is not implemented.' )
@@ -208,7 +208,7 @@
       if( .NOT.ANY(from_pe.EQ.peset(current_peset_num)%list) ) &
            call mpp_error( FATAL, 'MPP_BROADCAST: broadcasting from invalid PE.' )
 
-      if( current_clock.NE.0 )call SYSTEM_CLOCK(start_tick)
+      if( debug .and. (current_clock.NE.0) )call SYSTEM_CLOCK(start_tick)
       ptr = LOC(mpp_stack)
       words = size(bdata(:))*size(transfer(bdata(1),word))
       if( words.GT.mpp_stack_size )then
@@ -232,7 +232,7 @@
              data(i) = bdata(i)
           end do
       end if
-      if( current_clock.NE.0 )call increment_current_clock( EVENT_BROADCAST, length*MPP_TYPE_BYTELEN_ )
+      if( debug .and. (current_clock.NE.0) )call increment_current_clock( EVENT_BROADCAST, length*MPP_TYPE_BYTELEN_ )
       return
     end subroutine MPP_BROADCAST_
 
diff --git a/src/shared/mpp/include/mpp_update_domains2D.h b/src/shared/mpp/include/mpp_update_domains2D.h
index a03ac2b203..521706e22d 100644
--- a/src/shared/mpp/include/mpp_update_domains2D.h
+++ b/src/shared/mpp/include/mpp_update_domains2D.h
@@ -1,6 +1,6 @@
 ! -*-f90-*-
     subroutine MPP_UPDATE_DOMAINS_2D_( field, domain, flags, complete, position, &
-                                       whalo, ehalo, shalo, nhalo, name, tile_count, buffer)
+                                       whalo, ehalo, shalo, nhalo, name, tile_count)
 !updates data domain of 2D field whose computational domains have been computed
       MPP_TYPE_,        intent(inout)        :: field(:,:)
       type(domain2D),   intent(inout)        :: domain  
@@ -10,18 +10,17 @@
       integer,          intent(in), optional :: whalo, ehalo, shalo, nhalo
       character(len=*), intent(in), optional :: name
       integer,          intent(in), optional :: tile_count
-      MPP_TYPE_,     intent(inout), optional :: buffer(:)
 
       MPP_TYPE_ :: field3D(size(field,1),size(field,2),1)
       pointer( ptr, field3D )
       ptr = LOC(field)
       call mpp_update_domains( field3D, domain, flags, complete, position, &
-                               whalo, ehalo, shalo, nhalo, name, tile_count, buffer )
+                               whalo, ehalo, shalo, nhalo, name, tile_count )
       return
     end subroutine MPP_UPDATE_DOMAINS_2D_
 
     subroutine MPP_UPDATE_DOMAINS_3D_( field, domain, flags, complete, position, &
-                                       whalo, ehalo, shalo, nhalo, name, tile_count, buffer )
+                                       whalo, ehalo, shalo, nhalo, name, tile_count)
 !updates data domain of 3D field whose computational domains have been computed
       MPP_TYPE_,        intent(inout)        :: field(:,:,:)
       type(domain2D),   intent(inout)        :: domain  
@@ -31,17 +30,15 @@
       integer,          intent(in), optional :: whalo, ehalo, shalo, nhalo ! specify halo region to be updated.
       character(len=*), intent(in), optional :: name
       integer,          intent(in), optional :: tile_count
-      MPP_TYPE_,     intent(inout), optional :: buffer(:)
 
       integer                 :: update_position, update_whalo, update_ehalo, update_shalo, update_nhalo, ntile
 
       integer(LONG_KIND),dimension(MAX_DOMAIN_FIELDS, MAX_TILES),save :: f_addrs=-9999
-      integer(LONG_KIND),dimension(MAX_DOMAIN_FIELDS, MAX_TILES),save :: b_addrs=-9999
-      integer          :: tile, buffer_size, max_ntile
+      integer          :: tile, max_ntile
       character(len=3) :: text
       logical          :: set_mismatch, is_complete
       logical          :: do_update
-      integer, save    :: isize=0, jsize=0, ke=0, l_size=0, bsize=0, list=0
+      integer, save    :: isize=0, jsize=0, ke=0, l_size=0, list=0
       integer, save    :: pos, whalosz, ehalosz, shalosz, nhalosz
       MPP_TYPE_        :: d_type
       type(overlapSpec), pointer :: update => NULL()
@@ -107,17 +104,11 @@
          call mpp_error(FATAL,'MPP_UPDATE_3D: MAX_DOMAIN_FIELDS='//text//' exceeded for group update.' )
       endif
       f_addrs(list, tile) = LOC(field)
-      buffer_size = 0
-      if(present(buffer)) then
-         buffer_size = size(buffer(:))
-         b_addrs(list, tile) = LOC(buffer)
-      end if
       update_position = CENTER
       if(present(position)) update_position = position
       if(list == 1 .AND. tile == 1 )then
          isize=size(field,1); jsize=size(field,2); ke = size(field,3); pos = update_position
          whalosz = update_whalo; ehalosz = update_ehalo; shalosz = update_shalo; nhalosz = update_nhalo
-         bsize = buffer_size
       else
          set_mismatch = .false.
          set_mismatch = set_mismatch .OR. (isize /= size(field,1))
@@ -128,7 +119,6 @@
          set_mismatch = set_mismatch .OR. (update_ehalo /= ehalosz)
          set_mismatch = set_mismatch .OR. (update_shalo /= shalosz)
          set_mismatch = set_mismatch .OR. (update_nhalo /= nhalosz)
-         set_mismatch = set_mismatch .OR. (buffer_size  /= bsize)
          if(set_mismatch)then
             write( text,'(i2)' ) list
             call mpp_error(FATAL,'MPP_UPDATE_3D: Incompatible field at count '//text//' for group update.' )
@@ -152,21 +142,21 @@
             !                    b_addrs(1:l_size,1:ntile), bsize, flags)
 
             if ( PRESENT ( flags ) ) then
-                call mpp_do_update( f_addrs(1:l_size,1:ntile), domain, update, d_type, ke, b_addrs(1:l_size,1:ntile), bsize, flags )
+                call mpp_do_update( f_addrs(1:l_size,1:ntile), domain, update, d_type, ke, flags )
             else
-                call mpp_do_update( f_addrs(1:l_size,1:ntile), domain, update, d_type, ke, b_addrs(1:l_size,1:ntile), bsize )
+                call mpp_do_update( f_addrs(1:l_size,1:ntile), domain, update, d_type, ke )
             endif    
                 
 
          end if
-         l_size=0; f_addrs=-9999; bsize=0; b_addrs=-9999; isize=0;  jsize=0;  ke=0
+         l_size=0; f_addrs=-9999; isize=0;  jsize=0;  ke=0
       endif
       return
 
     end subroutine MPP_UPDATE_DOMAINS_3D_
 
     subroutine MPP_UPDATE_DOMAINS_4D_( field, domain, flags, complete, position, &
-                                       whalo, ehalo, shalo, nhalo, name, tile_count, buffer )
+                                       whalo, ehalo, shalo, nhalo, name, tile_count )
 !updates data domain of 4D field whose computational domains have been computed
       MPP_TYPE_,        intent(inout)        :: field(:,:,:,:)
       type(domain2D),   intent(inout)        :: domain  
@@ -176,18 +166,17 @@
       integer,          intent(in), optional :: whalo, ehalo, shalo, nhalo
       character(len=*), intent(in), optional :: name
       integer,          intent(in), optional :: tile_count
-      MPP_TYPE_,     intent(inout), optional :: buffer(:)
 
       MPP_TYPE_ :: field3D(size(field,1),size(field,2),size(field,3)*size(field,4))
       pointer( ptr, field3D )
       ptr = LOC(field)
       call mpp_update_domains( field3D, domain, flags, complete, position, &
-                               whalo, ehalo, shalo, nhalo, name, tile_count, buffer)
+                               whalo, ehalo, shalo, nhalo, name, tile_count)
       return
     end subroutine MPP_UPDATE_DOMAINS_4D_
 
     subroutine MPP_UPDATE_DOMAINS_5D_( field, domain, flags, complete, position, &
-                                       whalo, ehalo, shalo, nhalo, name, tile_count, buffer )
+                                       whalo, ehalo, shalo, nhalo, name, tile_count )
 !updates data domain of 5D field whose computational domains have been computed
       MPP_TYPE_,        intent(inout)        :: field(:,:,:,:,:)
       type(domain2D),   intent(inout)        :: domain  
@@ -197,14 +186,13 @@
       integer,          intent(in), optional :: whalo, ehalo, shalo, nhalo
       character(len=*), intent(in), optional :: name
       integer,          intent(in), optional :: tile_count
-      MPP_TYPE_,     intent(inout), optional :: buffer(:)
 
       MPP_TYPE_ :: field3D(size(field,1),size(field,2),size(field,3)*size(field,4)*size(field,5))
 
       pointer( ptr, field3D )
       ptr = LOC(field)
       call mpp_update_domains( field3D, domain, flags, complete, position, &
-                               whalo, ehalo, shalo, nhalo, name, tile_count, buffer )
+                               whalo, ehalo, shalo, nhalo, name, tile_count )
       return
     end subroutine MPP_UPDATE_DOMAINS_5D_
 
@@ -220,8 +208,11 @@
       type(DomainCommunicator2D),pointer,optional :: dc_handle
       pointer( ptr_in,  field3D_in  )
       pointer( ptr_out, field3D_out )
-      ptr_in  = LOC(field_in )
-      ptr_out = LOC(field_out)
+
+      ptr_in = 0
+      ptr_out = 0
+      if(domain_in%initialized) ptr_in  = LOC(field_in )
+      if(domain_out%initialized) ptr_out = LOC(field_out)
       call mpp_redistribute( domain_in, field3D_in, domain_out, field3D_out, complete, free, list_size, dc_handle, position )
 
       return
@@ -334,8 +325,11 @@
       type(DomainCommunicator2D),pointer,optional :: dc_handle
       pointer( ptr_in,  field3D_in  )
       pointer( ptr_out, field3D_out )
-      ptr_in  = LOC(field_in )
-      ptr_out = LOC(field_out)
+
+      ptr_in = 0
+      ptr_out = 0
+      if(domain_in%initialized) ptr_in  = LOC(field_in )
+      if(domain_out%initialized) ptr_out = LOC(field_out)
       call mpp_redistribute( domain_in, field3D_in, domain_out, field3D_out, complete, free, list_size, dc_handle, position  )
 
       return
@@ -354,8 +348,11 @@
       type(DomainCommunicator2D),pointer,optional :: dc_handle
       pointer( ptr_in,  field3D_in  )
       pointer( ptr_out, field3D_out )
-      ptr_in  = LOC(field_in )
-      ptr_out = LOC(field_out)
+
+      ptr_in = 0
+      ptr_out = 0
+      if(domain_in%initialized) ptr_in  = LOC(field_in )
+      if(domain_out%initialized) ptr_out = LOC(field_out)
       call mpp_redistribute( domain_in, field3D_in, domain_out, field3D_out, complete, free, list_size, dc_handle, position  )
 
       return
@@ -366,7 +363,7 @@
 !VECTOR_FIELD_ is set to false for MPP_TYPE_ integer.
 !vector fields
     subroutine MPP_UPDATE_DOMAINS_2D_V_( fieldx, fieldy, domain, flags, gridtype, complete, &
-                                         whalo, ehalo, shalo, nhalo, name, tile_count, bufferx, buffery)
+                                         whalo, ehalo, shalo, nhalo, name, tile_count)
 !updates data domain of 2D field whose computational domains have been computed
       MPP_TYPE_,        intent(inout)        :: fieldx(:,:), fieldy(:,:)
       type(domain2D),   intent(inout)        :: domain
@@ -375,7 +372,6 @@
       integer,          intent(in), optional :: whalo, ehalo, shalo, nhalo
       character(len=*), intent(in), optional :: name
       integer,          intent(in), optional :: tile_count
-      MPP_TYPE_,     intent(inout), optional :: bufferx(:), buffery(:)
 
       MPP_TYPE_ :: field3Dx(size(fieldx,1),size(fieldx,2),1)
       MPP_TYPE_ :: field3Dy(size(fieldy,1),size(fieldy,2),1)
@@ -384,13 +380,13 @@
       ptrx = LOC(fieldx)
       ptry = LOC(fieldy)
       call mpp_update_domains( field3Dx, field3Dy, domain, flags, gridtype, complete, &
-                               whalo, ehalo, shalo, nhalo, name, tile_count, bufferx, buffery )
+                               whalo, ehalo, shalo, nhalo, name, tile_count)
       return
     end subroutine MPP_UPDATE_DOMAINS_2D_V_
 
 
     subroutine MPP_UPDATE_DOMAINS_3D_V_( fieldx, fieldy, domain, flags, gridtype, complete, &
-                                         whalo, ehalo, shalo, nhalo, name, tile_count, bufferx, buffery )
+                                         whalo, ehalo, shalo, nhalo, name, tile_count)
 !updates data domain of 3D field whose computational domains have been computed
       MPP_TYPE_,        intent(inout)        :: fieldx(:,:,:), fieldy(:,:,:)
       type(domain2D),   intent(inout)        :: domain
@@ -399,19 +395,16 @@
       integer,          intent(in), optional :: whalo, ehalo, shalo, nhalo
       character(len=*), intent(in), optional :: name
       integer,          intent(in), optional :: tile_count
-      MPP_TYPE_,     intent(inout), optional :: bufferx(:), buffery(:)
 
       integer                                :: update_whalo, update_ehalo, update_shalo, update_nhalo, ntile    
       integer                                :: grid_offset_type
       logical                                :: exchange_uv
         
       integer(LONG_KIND),dimension(MAX_DOMAIN_FIELDS, MAX_TILES),save :: f_addrsx=-9999, f_addrsy=-9999
-      integer(LONG_KIND),dimension(MAX_DOMAIN_FIELDS, MAX_TILES),save :: b_addrsx=-9999, b_addrsy=-9999
       logical          :: do_update, is_complete
       integer, save    :: isize(2)=0,jsize(2)=0,ke=0,l_size=0, offset_type=0, list=0
       integer, save    :: whalosz, ehalosz, shalosz, nhalosz
-      integer, save    :: bsizex=1, bsizey=1
-      integer          :: bufferx_size, buffery_size, tile, max_ntile
+      integer          :: tile, max_ntile
       integer          :: position_x, position_y
       logical          :: set_mismatch
       character(len=3) :: text
@@ -490,20 +483,12 @@
 
       f_addrsx(list, tile) = LOC(fieldx)
       f_addrsy(list, tile) = LOC(fieldy)
-      bufferx_size = 1; buffery_size = 1
-      if(present(bufferx)) then
-         bufferx_size = size(bufferx(:))
-         buffery_size = size(buffery(:))
-         b_addrsx(list, tile) = LOC(bufferx)
-         b_addrsy(list, tile) = LOC(buffery)
-      end if
 
       if(list == 1 .AND. tile == 1)then
          isize(1)=size(fieldx,1); jsize(1)=size(fieldx,2); ke = size(fieldx,3)
          isize(2)=size(fieldy,1); jsize(2)=size(fieldy,2)
          offset_type = grid_offset_type
          whalosz = update_whalo; ehalosz = update_ehalo; shalosz = update_shalo; nhalosz = update_nhalo
-         bsizex = bufferx_size; bsizey = buffery_size
       else
          set_mismatch = .false.
          set_mismatch = set_mismatch .OR. (isize(1) /= size(fieldx,1))
@@ -517,8 +502,6 @@
          set_mismatch = set_mismatch .OR. (update_ehalo /= ehalosz)
          set_mismatch = set_mismatch .OR. (update_shalo /= shalosz)
          set_mismatch = set_mismatch .OR. (update_nhalo /= nhalosz)
-         set_mismatch = set_mismatch .OR. (bufferx_size  /= bsizex)
-         set_mismatch = set_mismatch .OR. (buffery_size  /= bsizey)
          if(set_mismatch)then
             write( text,'(i2)' ) list
             call mpp_error(FATAL,'MPP_UPDATE_3D_V: Incompatible field at count '//text//' for group vector update.' )
@@ -561,16 +544,13 @@
             updatey => search_update_overlap(domain, update_whalo, update_ehalo, update_shalo, update_nhalo, position_y)
             if(exchange_uv) then
                call mpp_do_update(f_addrsx(1:l_size,1:ntile),f_addrsy(1:l_size,1:ntile), domain, updatey, updatex, &
-                    d_type,ke, b_addrsx(1:l_size,1:ntile), b_addrsy(1:l_size,1:ntile),                             &
-                    bsizex, bsizey, grid_offset_type, flags)
+                    d_type,ke, grid_offset_type, flags)
             else
                call mpp_do_update(f_addrsx(1:l_size,1:ntile),f_addrsy(1:l_size,1:ntile), domain, updatex, updatey, &
-                    d_type,ke, b_addrsx(1:l_size,1:ntile), b_addrsy(1:l_size,1:ntile),                             &
-                    bsizex, bsizey, grid_offset_type, flags)
+                    d_type,ke,grid_offset_type, flags)
             end if
          end if
          l_size=0; f_addrsx=-9999; f_addrsy=-9999; isize=0;  jsize=0;  ke=0
-         bsizex=1; b_addrsx=-9999; bsizey=1; b_addrsy=-9999;
       end if
 
       return
@@ -578,7 +558,7 @@
 
 
     subroutine MPP_UPDATE_DOMAINS_4D_V_( fieldx, fieldy, domain, flags, gridtype, complete, &
-                                         whalo, ehalo, shalo, nhalo, name, tile_count, bufferx, buffery )
+                                         whalo, ehalo, shalo, nhalo, name, tile_count )
 !updates data domain of 4D field whose computational domains have been computed
       MPP_TYPE_,        intent(inout)        :: fieldx(:,:,:,:), fieldy(:,:,:,:)
       type(domain2D),   intent(inout)        :: domain
@@ -587,7 +567,6 @@
       integer,          intent(in), optional :: whalo, ehalo, shalo, nhalo
       character(len=*), intent(in), optional :: name
       integer,          intent(in), optional :: tile_count
-      MPP_TYPE_,     intent(inout), optional :: bufferx(:), buffery(:)
 
       MPP_TYPE_ :: field3Dx(size(fieldx,1),size(fieldx,2),size(fieldx,3)*size(fieldx,4))
       MPP_TYPE_ :: field3Dy(size(fieldy,1),size(fieldy,2),size(fieldy,3)*size(fieldy,4))
@@ -597,12 +576,12 @@
       ptrx = LOC(fieldx)
       ptry = LOC(fieldy)
       call mpp_update_domains( field3Dx, field3Dy, domain, flags, gridtype, complete, &
-                               whalo, ehalo, shalo, nhalo, name, tile_count, bufferx, buffery )
+                               whalo, ehalo, shalo, nhalo, name, tile_count)
       return
     end subroutine MPP_UPDATE_DOMAINS_4D_V_
 
     subroutine MPP_UPDATE_DOMAINS_5D_V_( fieldx, fieldy, domain, flags, gridtype, complete, &
-                                         whalo, ehalo, shalo, nhalo, name, tile_count, bufferx, buffery )
+                                         whalo, ehalo, shalo, nhalo, name, tile_count )
 !updates data domain of 5D field whose computational domains have been computed
       MPP_TYPE_,        intent(inout)        :: fieldx(:,:,:,:,:), fieldy(:,:,:,:,:)
       type(domain2D),   intent(inout)        :: domain
@@ -611,7 +590,6 @@
       integer,          intent(in), optional :: whalo, ehalo, shalo, nhalo
       character(len=*), intent(in), optional :: name
       integer,          intent(in), optional :: tile_count
-      MPP_TYPE_,     intent(inout), optional :: bufferx(:), buffery(:)
 
       MPP_TYPE_ :: field3Dx(size(fieldx,1),size(fieldx,2),size(fieldx,3)*size(fieldx,4)*size(fieldx,5))
       MPP_TYPE_ :: field3Dy(size(fieldy,1),size(fieldy,2),size(fieldy,3)*size(fieldy,4)*size(fieldy,5))
@@ -620,7 +598,7 @@
       ptrx = LOC(fieldx)
       ptry = LOC(fieldy)
       call mpp_update_domains( field3Dx, field3Dy, domain, flags, gridtype, complete, &
-                               whalo, ehalo, shalo, nhalo, name, tile_count, bufferx, buffery )
+                               whalo, ehalo, shalo, nhalo, name, tile_count)
 
       return
     end subroutine MPP_UPDATE_DOMAINS_5D_V_
diff --git a/src/shared/mpp/include/mpp_update_domains2D_nonblock.h b/src/shared/mpp/include/mpp_update_domains2D_nonblock.h
index 19c2d0736b..ece7fd0abd 100644
--- a/src/shared/mpp/include/mpp_update_domains2D_nonblock.h
+++ b/src/shared/mpp/include/mpp_update_domains2D_nonblock.h
@@ -159,8 +159,10 @@ function MPP_START_UPDATE_DOMAINS_3D_( field, domain, flags, position, &
   end if
   
   if(do_update) then
-     num_update = num_update + 1
+    if(num_nonblock_group_update>0) call mpp_error(FATAL, "MPP_START_UPDATE_DOMAINS: "// &
+          " can not be called in the middle of mpp_start_group_update/mpp_complete_group_update call")
 
+     num_update = num_update + 1
      if( PRESENT(update_id) ) then
         if( update_id < 1 .OR. update_id > MAX_NONBLOCK_UPDATE ) then
            write( text,'(a,i8,a,i8)' ) 'optional argument update_id =', update_id, &
@@ -267,7 +269,7 @@ end function MPP_START_UPDATE_DOMAINS_5D_
 
 !##################################################################################
 subroutine MPP_COMPLETE_UPDATE_DOMAINS_2D_( id_update, field, domain, flags, position, &
-                                            whalo, ehalo, shalo, nhalo, name, tile_count, buffer, complete )
+                                            whalo, ehalo, shalo, nhalo, name, tile_count, complete )
   integer,          intent(in)           :: id_update
   type(domain2D),   intent(inout)        :: domain  
   MPP_TYPE_,        intent(inout)        :: field(:,:)
@@ -277,19 +279,18 @@ subroutine MPP_COMPLETE_UPDATE_DOMAINS_2D_( id_update, field, domain, flags, pos
   character(len=*), intent(in), optional :: name
   integer,          intent(in), optional :: tile_count
   logical,          intent(in), optional :: complete
-  MPP_TYPE_,     intent(inout), optional :: buffer(:)
 
   MPP_TYPE_ :: field3D(size(field,1),size(field,2),1)
   pointer( ptr, field3D )
   ptr = LOC(field)
   call mpp_complete_update_domains(id_update, field3D, domain, flags, position, &
-                                   whalo, ehalo, shalo, nhalo, name, tile_count, buffer, complete )
+                                   whalo, ehalo, shalo, nhalo, name, tile_count, complete )
 
 end subroutine MPP_COMPLETE_UPDATE_DOMAINS_2D_
 
 !##################################################################################
 subroutine MPP_COMPLETE_UPDATE_DOMAINS_3D_( id_update, field, domain, flags, position, &
-                                            whalo, ehalo, shalo, nhalo, name, tile_count, buffer, complete )
+                                            whalo, ehalo, shalo, nhalo, name, tile_count, complete )
   integer,          intent(in)           :: id_update
   type(domain2D),   intent(inout)        :: domain  
   MPP_TYPE_,        intent(inout)        :: field(domain%x(1)%data%begin:,domain%y(1)%data%begin:,:)
@@ -299,7 +300,6 @@ subroutine MPP_COMPLETE_UPDATE_DOMAINS_3D_( id_update, field, domain, flags, pos
   character(len=*), intent(in), optional :: name
   integer,          intent(in), optional :: tile_count
   logical,          intent(in), optional :: complete
-  MPP_TYPE_,     intent(inout), optional :: buffer(:)
 
 
   integer                    :: update_whalo, update_ehalo, update_shalo, update_nhalo
@@ -308,14 +308,12 @@ subroutine MPP_COMPLETE_UPDATE_DOMAINS_3D_( id_update, field, domain, flags, pos
   integer                    :: tile, max_ntile, ntile, n
   logical                    :: is_complete
   logical                    :: do_update
-  integer                    :: ke_max, buffer_size
-  integer, save              :: list=0, bsize=0, l_size=0
+  integer                    :: ke_max
+  integer, save              :: list=0, l_size=0
   integer, save              :: ke_list(MAX_DOMAIN_FIELDS, MAX_TILES)=0
   integer(LONG_KIND), save   :: f_addrs(MAX_DOMAIN_FIELDS, MAX_TILES)=-9999
-  integer(LONG_KIND), save   :: b_addrs(MAX_DOMAIN_FIELDS, MAX_TILES)=-9999
   character(len=128)         :: text
-
-      MPP_TYPE_        :: d_type
+  MPP_TYPE_        :: d_type
 
   if(present(whalo)) then
      update_whalo = whalo
@@ -383,14 +381,7 @@ subroutine MPP_COMPLETE_UPDATE_DOMAINS_3D_( id_update, field, domain, flags, pos
      endif
   endif
   
-  buffer_size = 0
-  if(present(buffer)) then
-     buffer_size = size(buffer(:))
-     b_addrs(list, tile) = LOC(buffer)
-  end if
-
   ke_list(list,tile) = size(field,3)
-  bsize = max(bsize, buffer_size)
 
   !check to make sure the consistency of halo size, position and flags.
   if( nonblock_data(id_update)%update_flags .NE. update_flags ) call mpp_error(FATAL, "MPP_COMPLETE_UPDATE_DOMAINS_3D: "// &
@@ -419,11 +410,11 @@ subroutine MPP_COMPLETE_UPDATE_DOMAINS_3D_( id_update, field, domain, flags, pos
         update => search_update_overlap(domain, update_whalo, update_ehalo, update_shalo, update_nhalo, update_position)
         ke_max = maxval(ke_list(1:l_size,1:ntile))
         call mpp_complete_do_update(id_update, f_addrs(1:l_size,1:ntile), domain, update, d_type, &
-                                    ke_max, ke_list(1:l_size,1:ntile), b_addrs(1:l_size,1:ntile), bsize, update_flags) 
+                                    ke_max, ke_list(1:l_size,1:ntile), update_flags) 
      endif
      nonblock_data(id_update)%nfields = 0
      nonblock_data(id_update)%field_addrs(1:l_size) = 0
-     l_size=0; f_addrs=-9999; bsize=0; b_addrs=-9999; ke_list=0
+     l_size=0; f_addrs=-9999; ke_list=0
      !--- For the last call of mpp_complete_update_domains
      !--- reset everything to init state
      if( num_update == 0) then
@@ -439,7 +430,7 @@ end subroutine MPP_COMPLETE_UPDATE_DOMAINS_3D_
 
 !##################################################################################
 subroutine MPP_COMPLETE_UPDATE_DOMAINS_4D_( id_update, field, domain, flags, position, &
-                                            whalo, ehalo, shalo, nhalo, name, tile_count, buffer, complete )
+                                            whalo, ehalo, shalo, nhalo, name, tile_count, complete )
   integer,          intent(in)           :: id_update
   type(domain2D),   intent(inout)        :: domain  
   MPP_TYPE_,        intent(inout)        :: field(:,:,:,:)
@@ -449,19 +440,18 @@ subroutine MPP_COMPLETE_UPDATE_DOMAINS_4D_( id_update, field, domain, flags, pos
   character(len=*), intent(in), optional :: name
   integer,          intent(in), optional :: tile_count
   logical,          intent(in), optional :: complete
-  MPP_TYPE_,     intent(inout), optional :: buffer(:)
 
   MPP_TYPE_ :: field3D(size(field,1),size(field,2),size(field,3)*size(field,4))
   pointer( ptr, field3D )
   ptr = LOC(field)
   call mpp_complete_update_domains(id_update, field3D, domain, flags, position, &
-                                   whalo, ehalo, shalo, nhalo, name, tile_count, buffer, complete )
+                                   whalo, ehalo, shalo, nhalo, name, tile_count, complete )
 
 end subroutine MPP_COMPLETE_UPDATE_DOMAINS_4D_
 
 !##################################################################################
 subroutine MPP_COMPLETE_UPDATE_DOMAINS_5D_( id_update, field, domain, flags, position, &
-                                            whalo, ehalo, shalo, nhalo, name, tile_count, buffer, complete )
+                                            whalo, ehalo, shalo, nhalo, name, tile_count, complete )
   integer,          intent(in)           :: id_update
   type(domain2D),   intent(inout)        :: domain  
   MPP_TYPE_,        intent(inout)        :: field(:,:,:,:,:)
@@ -471,13 +461,12 @@ subroutine MPP_COMPLETE_UPDATE_DOMAINS_5D_( id_update, field, domain, flags, pos
   character(len=*), intent(in), optional :: name
   integer,          intent(in), optional :: tile_count
   logical,          intent(in), optional :: complete
-  MPP_TYPE_,     intent(inout), optional :: buffer(:)
 
   MPP_TYPE_ :: field3D(size(field,1),size(field,2),size(field,3)*size(field,4)*size(field,5))
   pointer( ptr, field3D )
   ptr = LOC(field)
   call mpp_complete_update_domains(id_update, field3D, domain, flags, position, &
-                                   whalo, ehalo, shalo, nhalo, name, tile_count, buffer, complete )
+                                   whalo, ehalo, shalo, nhalo, name, tile_count, complete )
 
 end subroutine MPP_COMPLETE_UPDATE_DOMAINS_5D_
 
@@ -656,6 +645,8 @@ function MPP_START_UPDATE_DOMAINS_3D_V_( fieldx, fieldy, domain, flags, gridtype
      list = 0
   end if
   if(do_update)then
+    if(num_nonblock_group_update>0) call mpp_error(FATAL, "MPP_START_UPDATE_DOMAINS_V: "// &       
+         " can not be called in the middle of mpp_start_group_update/mpp_complete_group_update call")
      num_update = num_update + 1
      if( PRESENT(update_id) ) then
         reuse_id_update = .true.
@@ -715,7 +706,7 @@ function MPP_START_UPDATE_DOMAINS_3D_V_( fieldx, fieldy, domain, flags, gridtype
            position_x = EAST
            position_y = NORTH
         case default
-           call mpp_error(FATAL, "mpp_update_domains2D.h: invalid value of grid_offset_type")
+           call mpp_error(FATAL, "mpp_update_domains2D_nonblock.h: invalid value of grid_offset_type")
         end select
         updatex => search_update_overlap(domain, update_whalo, update_ehalo, update_shalo, update_nhalo, position_x)
         updatey => search_update_overlap(domain, update_whalo, update_ehalo, update_shalo, update_nhalo, position_y)
@@ -798,7 +789,7 @@ end function MPP_START_UPDATE_DOMAINS_5D_V_
 
 !####################################################################################
 subroutine MPP_COMPLETE_UPDATE_DOMAINS_2D_V_( id_update, fieldx, fieldy, domain, flags, gridtype, &
-     whalo, ehalo, shalo, nhalo, name, tile_count, bufferx, buffery, complete )
+     whalo, ehalo, shalo, nhalo, name, tile_count, complete )
   !updates data domain of 3D field whose computational domains have been computed
   integer,          intent(in)           :: id_update
   MPP_TYPE_,        intent(inout)        :: fieldx(:,:), fieldy(:,:)
@@ -809,7 +800,6 @@ subroutine MPP_COMPLETE_UPDATE_DOMAINS_2D_V_( id_update, fieldx, fieldy, domain,
   integer,          intent(in), optional :: tile_count
   logical,          intent(in), optional :: complete
 
-  MPP_TYPE_,     intent(inout), optional :: bufferx(:), buffery(:)
   MPP_TYPE_ :: field3Dx(size(fieldx,1),size(fieldx,2),1)
   MPP_TYPE_ :: field3Dy(size(fieldy,1),size(fieldy,2),1)
   pointer( ptrx, field3Dx )
@@ -818,7 +808,7 @@ subroutine MPP_COMPLETE_UPDATE_DOMAINS_2D_V_( id_update, fieldx, fieldy, domain,
   ptry = LOC(fieldy)
 
   call mpp_complete_update_domains(id_update, field3Dx, field3Dy, domain, flags, gridtype, &
-     whalo, ehalo, shalo, nhalo, name, tile_count, bufferx, buffery, complete )
+     whalo, ehalo, shalo, nhalo, name, tile_count, complete )
 
   return
 
@@ -826,7 +816,7 @@ end subroutine MPP_COMPLETE_UPDATE_DOMAINS_2D_V_
 
 !####################################################################################
 subroutine MPP_COMPLETE_UPDATE_DOMAINS_3D_V_( id_update, fieldx, fieldy, domain, flags, gridtype, &
-     whalo, ehalo, shalo, nhalo, name, tile_count, bufferx, buffery, complete )
+     whalo, ehalo, shalo, nhalo, name, tile_count, complete )
   !updates data domain of 3D field whose computational domains have been computed
   integer,          intent(in)           :: id_update
   MPP_TYPE_,        intent(inout)        :: fieldx(:,:,:), fieldy(:,:,:)
@@ -835,22 +825,18 @@ subroutine MPP_COMPLETE_UPDATE_DOMAINS_3D_V_( id_update, fieldx, fieldy, domain,
   integer,          intent(in), optional :: whalo, ehalo, shalo, nhalo
   character(len=*), intent(in), optional :: name
   integer,          intent(in), optional :: tile_count
-  MPP_TYPE_,     intent(inout), optional :: bufferx(:), buffery(:)
   logical,          intent(in), optional :: complete
 
   integer                     :: update_whalo, update_ehalo, update_shalo, update_nhalo
   integer                     :: grid_offset_type, position_x, position_y, update_flags
   logical                     :: do_update, is_complete
   integer                     :: ntile, max_ntile, tile, ke_max, n
-  integer                     :: bufferx_size, buffery_size
   logical                     :: exchange_uv
   character(len=128)          :: text
-  integer,            save    :: l_size=0, list=0, bsizex=1, bsizey=1
+  integer,            save    :: l_size=0, list=0
   integer,            save    :: ke_list (MAX_DOMAIN_FIELDS, MAX_TILES)=0
   integer(LONG_KIND), save    :: f_addrsx(MAX_DOMAIN_FIELDS, MAX_TILES)=-9999
   integer(LONG_KIND), save    :: f_addrsy(MAX_DOMAIN_FIELDS, MAX_TILES)=-9999
-  integer(LONG_KIND) ,save    :: b_addrsx(MAX_DOMAIN_FIELDS, MAX_TILES)=-9999
-  integer(LONG_KIND) ,save    :: b_addrsy(MAX_DOMAIN_FIELDS, MAX_TILES)=-9999
   type(overlapSpec),  pointer :: updatex => NULL()
   type(overlapSpec),  pointer :: updatey => NULL()
   MPP_TYPE_                   :: d_type
@@ -941,17 +927,7 @@ subroutine MPP_COMPLETE_UPDATE_DOMAINS_3D_V_( id_update, fieldx, fieldy, domain,
      endif
   endif
 
-  bufferx_size = 1; buffery_size = 1
-  if(present(bufferx)) then
-     bufferx_size = size(bufferx(:))
-     buffery_size = size(buffery(:))
-     b_addrsx(list, tile) = LOC(bufferx)
-     b_addrsy(list, tile) = LOC(buffery)
-  end if
-
   ke_list(list, tile) = size(fieldx,3)
-  bsizex = max(bsizex, bufferx_size)
-  bsizey = max(bsizey, buffery_size)
 
   if(is_complete) then
      l_size = list
@@ -991,12 +967,10 @@ subroutine MPP_COMPLETE_UPDATE_DOMAINS_3D_V_( id_update, fieldx, fieldy, domain,
         if(exchange_uv) then
            call mpp_complete_do_update(id_update, f_addrsx(1:l_size,1:ntile), f_addrsy(1:l_size,1:ntile), domain, &
                                        updatey, updatex, d_type, ke_max, ke_list(1:l_size,1:ntile),               &
-                                       b_addrsx(1:l_size,1:ntile), b_addrsy(1:l_size,1:ntile), bsizex, bsizey,    &
                                        grid_offset_type, update_flags)
         else
            call mpp_complete_do_update(id_update, f_addrsx(1:l_size,1:ntile), f_addrsy(1:l_size,1:ntile), domain, &
                                        updatex, updatey, d_type, ke_max, ke_list(1:l_size,1:ntile),               &
-                                       b_addrsx(1:l_size,1:ntile), b_addrsy(1:l_size,1:ntile), bsizex, bsizey,    &
                                        grid_offset_type, update_flags)
         endif
      endif
@@ -1004,7 +978,6 @@ subroutine MPP_COMPLETE_UPDATE_DOMAINS_3D_V_( id_update, fieldx, fieldy, domain,
      nonblock_data(id_update)%field_addrs(1:l_size) = 0
      nonblock_data(id_update)%field_addrs2(1:l_size) = 0
      l_size=0; f_addrsx=-9999; f_addrsy=-9999; ke_list=0
-     bsizex=1; b_addrsx=-9999; bsizey=1; b_addrsy=-9999
      !--- For the last call of mpp_complete_update_domains
      !--- reset everything to init state
      if( num_update == 0) then
@@ -1021,7 +994,7 @@ end subroutine MPP_COMPLETE_UPDATE_DOMAINS_3D_V_
 
 !####################################################################################
 subroutine MPP_COMPLETE_UPDATE_DOMAINS_4D_V_( id_update, fieldx, fieldy, domain, flags, gridtype, &
-     whalo, ehalo, shalo, nhalo, name, tile_count, bufferx, buffery, complete )
+     whalo, ehalo, shalo, nhalo, name, tile_count, complete )
   !updates data domain of 3D field whose computational domains have been computed
   integer,          intent(in)           :: id_update
   MPP_TYPE_,        intent(inout)        :: fieldx(:,:,:,:), fieldy(:,:,:,:)
@@ -1032,7 +1005,6 @@ subroutine MPP_COMPLETE_UPDATE_DOMAINS_4D_V_( id_update, fieldx, fieldy, domain,
   integer,          intent(in), optional :: tile_count
   logical,          intent(in), optional :: complete
 
-  MPP_TYPE_,     intent(inout), optional :: bufferx(:), buffery(:)
   MPP_TYPE_ :: field3Dx(size(fieldx,1),size(fieldx,2),size(fieldx,3)*size(fieldx,4))
   MPP_TYPE_ :: field3Dy(size(fieldy,1),size(fieldy,2),size(fieldy,3)*size(fieldy,4))
   pointer( ptrx, field3Dx )
@@ -1041,7 +1013,7 @@ subroutine MPP_COMPLETE_UPDATE_DOMAINS_4D_V_( id_update, fieldx, fieldy, domain,
   ptry = LOC(fieldy)
 
   call mpp_complete_update_domains(id_update, field3Dx, field3Dy, domain, flags, gridtype, &
-     whalo, ehalo, shalo, nhalo, name, tile_count, bufferx, buffery, complete )
+     whalo, ehalo, shalo, nhalo, name, tile_count, complete )
 
   return
 
@@ -1049,7 +1021,7 @@ end subroutine MPP_COMPLETE_UPDATE_DOMAINS_4D_V_
 
 !####################################################################################
 subroutine MPP_COMPLETE_UPDATE_DOMAINS_5D_V_( id_update, fieldx, fieldy, domain, flags, gridtype, &
-     whalo, ehalo, shalo, nhalo, name, tile_count, bufferx, buffery, complete )
+     whalo, ehalo, shalo, nhalo, name, tile_count, complete )
   !updates data domain of 3D field whose computational domains have been computed
   integer,          intent(in)           :: id_update
   MPP_TYPE_,        intent(inout)        :: fieldx(:,:,:,:,:), fieldy(:,:,:,:,:)
@@ -1060,7 +1032,6 @@ subroutine MPP_COMPLETE_UPDATE_DOMAINS_5D_V_( id_update, fieldx, fieldy, domain,
   integer,          intent(in), optional :: tile_count
   logical,          intent(in), optional :: complete
 
-  MPP_TYPE_,     intent(inout), optional :: bufferx(:), buffery(:)
   MPP_TYPE_ :: field3Dx(size(fieldx,1),size(fieldx,2),size(fieldx,3)*size(fieldx,4)*size(fieldx,5))
   MPP_TYPE_ :: field3Dy(size(fieldy,1),size(fieldy,2),size(fieldy,3)*size(fieldy,4)*size(fieldy,5))
   pointer( ptrx, field3Dx )
@@ -1069,7 +1040,7 @@ subroutine MPP_COMPLETE_UPDATE_DOMAINS_5D_V_( id_update, fieldx, fieldy, domain,
   ptry = LOC(fieldy)
 
   call mpp_complete_update_domains(id_update, field3Dx, field3Dy, domain, flags, gridtype, &
-     whalo, ehalo, shalo, nhalo, name, tile_count, bufferx, buffery, complete )
+     whalo, ehalo, shalo, nhalo, name, tile_count, complete )
 
   return
 
diff --git a/src/shared/mpp/include/mpp_util.inc b/src/shared/mpp/include/mpp_util.inc
index 290ecf1ac3..7230f989d1 100644
--- a/src/shared/mpp/include/mpp_util.inc
+++ b/src/shared/mpp/include/mpp_util.inc
@@ -1,6 +1,6 @@
 
 ! -*-f90-*-
-! $Id: mpp_util.inc,v 20.0 2013/12/14 00:27:35 fms Exp $
+! $Id$
 
 
 #ifdef use_libSMA
@@ -816,10 +816,12 @@ end function rarray_to_char
        call mpp_sync()
     end if
 
-    num_clock_ids = num_clock_ids+1
-    if(num_clock_ids > MAX_CLOCKS)call mpp_error(FATAL,'MPP_CLOCK_BEGIN: max num previous_clock exceeded.' )
-    previous_clock(num_clock_ids) = current_clock
-    current_clock = id
+    if (debug) then
+      num_clock_ids = num_clock_ids+1
+      if(num_clock_ids > MAX_CLOCKS)call mpp_error(FATAL,'MPP_CLOCK_BEGIN: max num previous_clock exceeded.' )
+      previous_clock(num_clock_ids) = current_clock
+      current_clock = id
+    endif
     call SYSTEM_CLOCK( clocks(id)%tick )
     clocks(id)%is_on = .true.
 !$OMP END MASTER
@@ -851,11 +853,11 @@ end function rarray_to_char
        call mpp_error( WARNING, 'MPP_CLOCK_END: Clock rollover, assumed single roll.' )
     end if
     clocks(id)%total_ticks = clocks(id)%total_ticks + delta
-    if(num_clock_ids < 1)then
-       call mpp_error(FATAL,'MPP_CLOCK_END: min num previous_clock < 1.' )
+    if (debug) then
+      if(num_clock_ids < 1) call mpp_error(NOTE,'MPP_CLOCK_END: min num previous_clock < 1.' )
+      current_clock = previous_clock(num_clock_ids)
+      num_clock_ids = num_clock_ids-1
     endif
-    current_clock = previous_clock(num_clock_ids)
-    num_clock_ids = num_clock_ids-1
     clocks(id)%is_on = .false.
 !$OMP END MASTER
     return
@@ -885,7 +887,7 @@ end function rarray_to_char
     integer                       :: errunit
 
     if( .not. mpp_record_timing_data )return
-    if( current_clock.EQ.0 )return
+    if( .not.debug .or. (current_clock.EQ.0) )return
     if( current_clock.LT.0 .OR. current_clock.GT.clock_num )call mpp_error( FATAL, 'MPP_CLOCK_BEGIN: invalid current_clock.' )
     if( .NOT.clocks(current_clock)%detailed )return
     call SYSTEM_CLOCK(end_tick)
@@ -1259,7 +1261,7 @@ end function rarray_to_char
 
 !-----------------------------------------------------------------------
 !
-! AUTHOR: Rusty Benson (rusty.benson)
+! AUTHOR: Rusty Benson (rusty.benson@noaa.gov)
 !
 !
 ! THESE LINES MUST BE PRESENT IN MPP.F90
@@ -1284,8 +1286,8 @@ end function rarray_to_char
   subroutine read_input_nml()
 
 ! version and tagging information
-    character(len=128) :: version = '$Id: mpp_util.inc,v 20.0 2013/12/14 00:27:35 fms Exp $'
-    character(len=128) :: tagname = '$Name: tikal $' 
+    character(len=128) :: version = '$Id$'
+    character(len=128) :: tagname = '$Name$' 
 
 ! private variables
     integer :: log_unit
@@ -1388,8 +1390,8 @@ end function rarray_to_char
 
   !-----------------------------------------------------------------------
   !
-  ! AUTHOR: Rusty Benson , 
-  !         Seth Underwood 
+  ! AUTHOR: Rusty Benson , 
+  !         Seth Underwood 
   !
   !-----------------------------------------------------------------------
   ! subroutine READ_ASCII_FILE
@@ -1415,8 +1417,8 @@ end function rarray_to_char
     integer, intent(in), optional,   dimension(:) :: PELIST
 
     ! version and tagging information
-    character(len=*), parameter :: VERSION = '$Id: mpp_util.inc,v 20.0 2013/12/14 00:27:35 fms Exp $'
-    character(len=*), parameter :: TAGNAME = '$Name: tikal $' 
+    character(len=*), parameter :: VERSION = '$Id$'
+    character(len=*), parameter :: TAGNAME = '$Name$' 
 
     character(len=5) :: text
     logical :: file_exist
diff --git a/src/shared/mpp/include/mpp_util_mpi.inc b/src/shared/mpp/include/mpp_util_mpi.inc
index 758e413052..0e8e2b9c24 100644
--- a/src/shared/mpp/include/mpp_util_mpi.inc
+++ b/src/shared/mpp/include/mpp_util_mpi.inc
@@ -1,5 +1,5 @@
 ! -*-f90-*-
-! $Id: mpp_util_mpi.inc,v 20.0 2013/12/14 00:27:37 fms Exp $
+! $Id$
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 !                                                                             !
@@ -29,7 +29,7 @@ subroutine mpp_error_basic( errortype, errormsg )
      text = 'WARNING: non-existent errortype (must be NOTE|WARNING|FATAL)'
   end select
 
-  if( npes.GT.1 )write( text,'(a,i5)' )trim(text)//' from PE', pe   !this is the mpp part
+  if( npes.GT.1 )write( text,'(a,i6)' )trim(text)//' from PE', pe   !this is the mpp part
   if( PRESENT(errormsg) )text = trim(text)//': '//trim(errormsg)
 
   out_unit = stdout()
@@ -144,7 +144,7 @@ subroutine mpp_sync( pelist, do_self )
 
   n = get_peset(pelist); if( peset(n)%count.EQ.1 )return
 
-  if( current_clock.NE.0 )call SYSTEM_CLOCK(start_tick)
+  if( debug .and. (current_clock.NE.0) )call SYSTEM_CLOCK(start_tick)
 #ifdef use_MPI_SMA
   if( n.EQ.world_peset_num )then
      call SHMEM_BARRIER_ALL() !special call is faster
@@ -155,7 +155,7 @@ subroutine mpp_sync( pelist, do_self )
   call MPI_BARRIER( peset(n)%id, error )
 #endif
 
-  if( current_clock.NE.0 )call increment_current_clock(EVENT_WAIT)
+  if( debug .and. (current_clock.NE.0) )call increment_current_clock(EVENT_WAIT)
 
   return
 end subroutine mpp_sync
@@ -173,7 +173,7 @@ subroutine mpp_sync_self( pelist, check, request, msg_size, msg_type)
 
   integer                       :: i, m, n, stride, my_check, rsize
 
-  if( current_clock.NE.0 )call SYSTEM_CLOCK(start_tick)
+  if( debug .and. (current_clock.NE.0) )call SYSTEM_CLOCK(start_tick)
   my_check = EVENT_SEND
   if(present(check)) my_check = check
   if( my_check .NE. EVENT_SEND .AND. my_check .NE. EVENT_RECV ) then
@@ -230,7 +230,7 @@ subroutine mpp_sync_self( pelist, check, request, msg_size, msg_type)
         cur_recv_request = 0
      end select 
   endif
-  if( current_clock.NE.0 )call increment_current_clock(EVENT_WAIT)
+  if( debug .and. (current_clock.NE.0) )call increment_current_clock(EVENT_WAIT)
   return
 end subroutine mpp_sync_self
 
diff --git a/src/shared/mpp/include/mpp_util_nocomm.inc b/src/shared/mpp/include/mpp_util_nocomm.inc
index d8a81ee407..289cf4cb09 100644
--- a/src/shared/mpp/include/mpp_util_nocomm.inc
+++ b/src/shared/mpp/include/mpp_util_nocomm.inc
@@ -1,6 +1,6 @@
 
 ! -*-f90-*-
-! $Id: mpp_util_nocomm.inc,v 20.0 2013/12/14 00:27:39 fms Exp $
+! $Id$
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 !                                                                             !
diff --git a/src/shared/mpp/include/mpp_util_sma.inc b/src/shared/mpp/include/mpp_util_sma.inc
index addfd6b3db..125c56615d 100644
--- a/src/shared/mpp/include/mpp_util_sma.inc
+++ b/src/shared/mpp/include/mpp_util_sma.inc
@@ -1,6 +1,6 @@
 
 ! -*-f90-*-
-! $Id: mpp_util_sma.inc,v 20.0 2013/12/14 00:27:41 fms Exp $
+! $Id$
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 !                                                                             !
diff --git a/src/shared/mpp/include/mpp_write.h b/src/shared/mpp/include/mpp_write.h
index 79a2d9c08e..8c44e2a455 100644
--- a/src/shared/mpp/include/mpp_write.h
+++ b/src/shared/mpp/include/mpp_write.h
@@ -1,4 +1,4 @@
-    subroutine MPP_WRITE_( unit, field, data, tstamp)
+subroutine MPP_WRITE_( unit, field, data, tstamp)
       use mpp_parameter_mod, only : NULLUNIT
       integer, intent(in) :: unit
       type(fieldtype), intent(in) :: field
@@ -8,4 +8,4 @@
       if (unit == NULLUNIT) return
       MPP_WRITE_RECORD_
       return
-    end subroutine MPP_WRITE_
+    end subroutine MPP_WRITE_
\ No newline at end of file
diff --git a/src/shared/mpp/include/mpp_write_compressed.h b/src/shared/mpp/include/mpp_write_compressed.h
new file mode 100644
index 0000000000..51dbfa14e1
--- /dev/null
+++ b/src/shared/mpp/include/mpp_write_compressed.h
@@ -0,0 +1,101 @@
+    subroutine MPP_WRITE_COMPRESSED_1D_(unit, field, domain, data, nelems_io, tstamp, default_data)
+      integer, intent(in) :: unit
+      type(fieldtype), intent(inout) :: field
+      type(domain2D), intent(inout) :: domain
+      MPP_TYPE_, intent(inout) :: data(:)
+      integer, intent(in) :: nelems_io(:)  ! number of compressed elements
+      real,              intent(in), optional :: tstamp
+      MPP_TYPE_,         intent(in), optional :: default_data
+
+      MPP_TYPE_ :: data2D(size(data,1),1)
+      pointer( ptr, data2D )
+      ptr = LOC(data)
+
+      call mpp_write_compressed(unit, field, domain, data2D, nelems_io, tstamp, default_data)
+      return
+    end subroutine MPP_WRITE_COMPRESSED_1D_
+
+    subroutine MPP_WRITE_COMPRESSED_2D_(unit, field, domain, data, nelems_io, tstamp, default_data)
+      integer,           intent(in)           :: unit
+      type(fieldtype),   intent(inout)        :: field
+      type(domain2D),    intent(inout)        :: domain
+      MPP_TYPE_,         intent(inout)        :: data(:,:)
+      integer,           intent(in)           :: nelems_io(:)  ! number of compressed elements from each
+                                                               ! member of the io_domain. It MUST have the
+                                                               ! same order as the io_domain pelist.
+      real,              intent(in), optional :: tstamp
+      MPP_TYPE_,         intent(in), optional :: default_data
+
+!cdata is used to store the io-domain compressed data
+      MPP_TYPE_, allocatable, dimension(:,:) :: cdata
+      MPP_TYPE_, allocatable, dimension(:,:) :: sbuff,rbuff
+      MPP_TYPE_                            :: fill
+
+      MPP_TYPE_ :: sbuff1D(size(data))
+      MPP_TYPE_ :: rbuff1D(size(data,2)*sum(nelems_io(:)))
+      pointer(sptr,sbuff1D); pointer(rptr,rbuff1D)
+      integer, allocatable :: pelist(:)
+      integer, allocatable :: nz_gather(:)
+      integer :: i,j,nz,nelems,mynelems,idx,npes
+      type(domain2d), pointer :: io_domain=>NULL()
+
+      call mpp_clock_begin(mpp_write_clock)
+
+      if( .NOT.module_is_initialized )call mpp_error( FATAL, 'MPP_WRITE_COMPRESSED_2D_: must first call mpp_io_init.' )
+      if( .NOT.mpp_file(unit)%valid )call mpp_error( FATAL, 'MPP_WRITE_COMPRESSED_2D_: invalid unit number.' )
+
+      fill = 0
+      if(PRESENT(default_data)) fill = default_data
+
+      io_domain=>mpp_get_io_domain(domain) 
+      if (.not. ASSOCIATED(io_domain)) call mpp_error( FATAL, 'MPP_WRITE_COMPRESSED_2D_: io_domain must be defined.' )
+      npes = mpp_get_domain_npes(io_domain)
+      allocate(pelist(npes))
+      call mpp_get_pelist(io_domain,pelist)
+
+      mynelems = size(data,1)
+      nz = size(data,2)
+      nelems = sum(nelems_io(:))
+      ! Check that nz is consistent across all PEs in io_domain
+      allocate(nz_gather(npes))
+      call mpp_gather((/nz/), nz_gather, pelist)
+      if ( mpp_file(unit)%write_on_this_pe.and.maxloc(nz_gather,1).ne.minloc(nz_gather,1) ) then
+         call mpp_error( FATAL, 'MPP_WRITE_COMPRESSED_2D_: size(data,2) must be consistent across all PEs in io_domain' )
+      end if
+      deallocate(nz_gather)
+
+      if(mpp_file(unit)%write_on_this_pe ) allocate(rbuff(nz,nelems))
+      allocate(sbuff(nz,mynelems))
+
+      ! this matrix inversion makes for easy gather to the IO root
+      ! and a clear, concise unpack
+      do j=1,mynelems
+        do i=1,nz
+          sbuff(i,j) = data(j,i) 
+      enddo; enddo
+
+   !  Note that the gatherV implied here is asymmetric; only root needs to know the vector of recv size
+      sptr = LOC(sbuff)
+      rptr = 0
+      if(mpp_file(unit)%write_on_this_pe ) rptr = LOC(rbuff)
+      call mpp_gather(sbuff1D,size(sbuff),rbuff1D,nz*nelems_io(:),pelist)
+
+      if(mpp_file(unit)%write_on_this_pe ) then
+         allocate(cdata(nelems,nz))
+         cdata = fill
+         do j=1,nz
+           do i=1,nelems
+             cdata(i,j) = rbuff(j,i)
+         enddo; enddo
+         ! cludge for now; need resizing accessor
+         field%size(1) = nelems
+         call write_record( unit, field, nelems*nz, cdata, tstamp)
+         deallocate(rbuff,cdata)
+      endif
+
+      deallocate(sbuff,pelist)
+
+      call mpp_clock_end(mpp_write_clock)
+
+      return
+    end subroutine MPP_WRITE_COMPRESSED_2D_
diff --git a/src/shared/mpp/include/mpp_write_unlimited_axis.h b/src/shared/mpp/include/mpp_write_unlimited_axis.h
new file mode 100644
index 0000000000..5b4026fc54
--- /dev/null
+++ b/src/shared/mpp/include/mpp_write_unlimited_axis.h
@@ -0,0 +1,41 @@
+    subroutine MPP_WRITE_UNLIMITED_AXIS_1D_(unit,field,domain,data,nelems_io)
+      integer,           intent(in)           :: unit
+      type(fieldtype),   intent(inout)        :: field
+      type(domain2D),    intent(inout)        :: domain
+      MPP_TYPE_,         intent(inout)        :: data(:)
+      integer,           intent(in)           :: nelems_io(:)  ! number of compressed elements from each
+                                                               ! member of the io_domain. It MUST have the
+                                                               ! same order as the io_domain pelist.
+      integer, allocatable :: pelist(:)
+      integer :: i,j,nelems,npes
+      type(domain2d), pointer :: io_domain=>NULL()
+      MPP_TYPE_, allocatable, dimension(:) :: rbuff
+
+      call mpp_clock_begin(mpp_write_clock)
+
+      if( .NOT.module_is_initialized )call mpp_error( FATAL, 'MPP_WRITE_UNLIMITED_AXIS_1D_: must first call mpp_io_init.' )
+      if( .NOT.mpp_file(unit)%valid )call mpp_error( FATAL, 'MPP_WRITE_UNLIMITED_AXIS_1D_: invalid unit number.' )
+
+      io_domain=>mpp_get_io_domain(domain) 
+      if (.not. ASSOCIATED(io_domain)) call mpp_error( FATAL, 'MPP_WRITE_UNLIMITED_AXIS_1D_: io_domain must be defined.' )
+      npes = mpp_get_domain_npes(io_domain)
+      allocate(pelist(npes))
+      call mpp_get_pelist(io_domain,pelist)
+
+      nelems = sum(nelems_io(:))
+
+      if(mpp_file(unit)%write_on_this_pe) allocate(rbuff(nelems))
+
+   !  Note that the gatherV implied here is asymmetric; only root needs to know the vector of recv size
+      call mpp_gather(data,size(data),rbuff,nelems_io(:),pelist)
+
+      if(mpp_file(unit)%write_on_this_pe) then
+         field%size(1) = nelems  ! Correct the field size now that we have all the data
+         call write_record(unit, field, nelems, rbuff)
+         deallocate(rbuff)
+      endif
+      deallocate(pelist)
+
+      call mpp_clock_end(mpp_write_clock)
+      return
+    end subroutine MPP_WRITE_UNLIMITED_AXIS_1D_
diff --git a/src/shared/mpp/mpp.F90 b/src/shared/mpp/mpp.F90
index f21bf39435..26bb9ed54e 100644
--- a/src/shared/mpp/mpp.F90
+++ b/src/shared/mpp/mpp.F90
@@ -1,7 +1,7 @@
 !-----------------------------------------------------------------------
 !                 Communication for message-passing codes
 !
-! AUTHOR: V. Balaji (V.Balaji)
+! AUTHOR: V. Balaji (V.Balaji@noaa.gov)
 !         SGI/GFDL Princeton University
 !
 ! This program is free software; you can redistribute it and/or modify
@@ -21,12 +21,14 @@
 module mpp_mod
 !a generalized communication package for use with shmem and MPI
 !will add: co_array_fortran, MPI2
-!Balaji (V.Balaji) 11 May 1998
+!Balaji (V.Balaji@noaa.gov) 11 May 1998
 
-! 
+! 
 !   V. Balaji
 ! 
 
+! 
+! 
 
 ! 
 !   mpp_mod, is a set of simple calls to provide a uniform interface
@@ -163,8 +165,8 @@ module mpp_mod
   use mpp_parameter_mod, only : CLOCK_MODULE, CLOCK_ROUTINE, CLOCK_LOOP, CLOCK_INFRA
   use mpp_parameter_mod, only : MAX_EVENTS, MAX_BINS, MAX_EVENT_TYPES, MAX_CLOCKS
   use mpp_parameter_mod, only : MAXPES, EVENT_WAIT, EVENT_ALLREDUCE, EVENT_BROADCAST
-  use mpp_parameter_mod, only : EVENT_RECV, EVENT_SEND, MPP_READY, MPP_WAIT
   use mpp_parameter_mod, only : EVENT_ALLTOALL
+  use mpp_parameter_mod, only : EVENT_RECV, EVENT_SEND, MPP_READY, MPP_WAIT
   use mpp_parameter_mod, only : mpp_parameter_version=>version, mpp_parameter_tagname=>tagname
   use mpp_parameter_mod, only : DEFAULT_TAG
   use mpp_parameter_mod, only : COMM_TAG_1,  COMM_TAG_2,  COMM_TAG_3,  COMM_TAG_4
@@ -172,6 +174,7 @@ module mpp_mod
   use mpp_parameter_mod, only : COMM_TAG_9,  COMM_TAG_10, COMM_TAG_11, COMM_TAG_12
   use mpp_parameter_mod, only : COMM_TAG_13, COMM_TAG_14, COMM_TAG_15, COMM_TAG_16
   use mpp_parameter_mod, only : COMM_TAG_17, COMM_TAG_18, COMM_TAG_19, COMM_TAG_20
+  use mpp_parameter_mod, only : MPP_FILL_INT,MPP_FILL_DOUBLE
   use mpp_data_mod,      only : stat, mpp_stack, ptr_stack, status, ptr_status, sync, ptr_sync  
   use mpp_data_mod,      only : mpp_from_pe, ptr_from, remote_data_loc, ptr_remote
   use mpp_data_mod,      only : mpp_data_version=>version, mpp_data_tagname=>tagname
@@ -198,7 +201,7 @@ module mpp_mod
   public :: COMM_TAG_9,  COMM_TAG_10, COMM_TAG_11, COMM_TAG_12
   public :: COMM_TAG_13, COMM_TAG_14, COMM_TAG_15, COMM_TAG_16
   public :: COMM_TAG_17, COMM_TAG_18, COMM_TAG_19, COMM_TAG_20
-
+  public :: MPP_FILL_INT,MPP_FILL_DOUBLE
 
   !--- public data from mpp_data_mod ------------------------------
 !  public :: request
@@ -216,8 +219,7 @@ module mpp_mod
   !--- public interface from mpp_comm.h ------------------------------
   public :: mpp_chksum, mpp_max, mpp_min, mpp_sum, mpp_transmit, mpp_send, mpp_recv
   public :: mpp_broadcast, mpp_malloc, mpp_init, mpp_exit
-  public :: mpp_gather
-  public :: mpp_alltoall
+  public :: mpp_gather, mpp_scatter, mpp_alltoall
 #ifdef use_MPI_GSM
   public :: mpp_gsm_malloc, mpp_gsm_free
 #endif
@@ -663,8 +665,28 @@ module mpp_mod
      module procedure mpp_gather_int4_1dv
      module procedure mpp_gather_real4_1dv
      module procedure mpp_gather_real8_1dv
+     module procedure mpp_gather_pelist_int4_2d
+     module procedure mpp_gather_pelist_int4_3d
+     module procedure mpp_gather_pelist_real4_2d
+     module procedure mpp_gather_pelist_real4_3d
+     module procedure mpp_gather_pelist_real8_2d
+     module procedure mpp_gather_pelist_real8_3d
   end interface
 
+  !#####################################################################
+  ! 
+  !  
+  !    gather information onto root pe.
+  !  
+  ! 
+  interface mpp_scatter
+     module procedure mpp_scatter_pelist_int4_2d
+     module procedure mpp_scatter_pelist_int4_3d
+     module procedure mpp_scatter_pelist_real4_2d
+     module procedure mpp_scatter_pelist_real4_3d
+     module procedure mpp_scatter_pelist_real8_2d
+     module procedure mpp_scatter_pelist_real8_3d
+  end interface
 
   !#####################################################################
   ! 
@@ -684,6 +706,7 @@ module mpp_mod
      module procedure mpp_alltoall_real8_v
   end interface
 
+
   !#####################################################################
 
   ! 
@@ -1044,6 +1067,7 @@ module mpp_mod
   !     Integer checksums on FP data use the F90 TRANSFER()
   !     intrinsic.
   !
+  !     The serial checksum module is superseded
   !     by this function, and is no longer being actively maintained. This
   !     provides identical results on a single-processor job, and to perform
   !     serial checksums on a single processor of a parallel job, you only
@@ -1078,11 +1102,24 @@ module mpp_mod
      module procedure mpp_chksum_i8_2d
      module procedure mpp_chksum_i8_3d
      module procedure mpp_chksum_i8_4d
+     module procedure mpp_chksum_i8_5d
+     module procedure mpp_chksum_i8_1d_rmask
+     module procedure mpp_chksum_i8_2d_rmask
+     module procedure mpp_chksum_i8_3d_rmask
+     module procedure mpp_chksum_i8_4d_rmask
+     module procedure mpp_chksum_i8_5d_rmask
+
 #endif
      module procedure mpp_chksum_i4_1d
      module procedure mpp_chksum_i4_2d
      module procedure mpp_chksum_i4_3d
      module procedure mpp_chksum_i4_4d
+     module procedure mpp_chksum_i4_5d
+     module procedure mpp_chksum_i4_1d_rmask
+     module procedure mpp_chksum_i4_2d_rmask
+     module procedure mpp_chksum_i4_3d_rmask
+     module procedure mpp_chksum_i4_4d_rmask
+     module procedure mpp_chksum_i4_5d_rmask
      module procedure mpp_chksum_r8_0d
      module procedure mpp_chksum_r8_1d
      module procedure mpp_chksum_r8_2d
@@ -1207,7 +1244,7 @@ module mpp_mod
   character(len=128), public :: version= &
        '$Id mpp.F90 $'
   character(len=128), public :: tagname= &
-       '$Name: tikal $'
+       '$Name$'
 
   integer, parameter :: MAX_REQUEST_MIN  = 10000
   integer            :: request_multiply = 20
diff --git a/src/shared/mpp/mpp_data.F90 b/src/shared/mpp/mpp_data.F90
index cb3655c07c..6295467d51 100644
--- a/src/shared/mpp/mpp_data.F90
+++ b/src/shared/mpp/mpp_data.F90
@@ -13,7 +13,7 @@ module mpp_data_mod
   character(len=128), public :: version= &
        '$Id mpp_data.F90 $'
   character(len=128), public :: tagname= &
-       '$Name: tikal $'
+       '$Name$'
 
 #if defined(use_libSMA) || defined(use_MPI_SMA)
 #include 
diff --git a/src/shared/mpp/mpp_domains.F90 b/src/shared/mpp/mpp_domains.F90
index b3f063d3b6..6b9c8330e6 100644
--- a/src/shared/mpp/mpp_domains.F90
+++ b/src/shared/mpp/mpp_domains.F90
@@ -19,12 +19,14 @@
 !           675 Mass Ave, Cambridge, MA 02139, USA.  
 !-----------------------------------------------------------------------
 
-! 
+! 
 !   V. Balaji
 ! 
-! 
+! 
 !   Zhi Liang
 ! 
+! 
+! 
 
 ! 
 !   mpp_domains_mod is a set of simple calls for domain
@@ -120,7 +122,7 @@ module mpp_domains_mod
   use mpp_parameter_mod,      only : FOLD_WEST_EDGE, FOLD_EAST_EDGE, FOLD_SOUTH_EDGE, FOLD_NORTH_EDGE
   use mpp_parameter_mod,      only : WUPDATE, EUPDATE, SUPDATE, NUPDATE, XUPDATE, YUPDATE
   use mpp_parameter_mod,      only : NON_BITWISE_EXACT_SUM, BITWISE_EXACT_SUM, MPP_DOMAIN_TIME
-  use mpp_parameter_mod,      only : CENTER, CORNER, SCALAR_PAIR, SCALAR_BIT
+  use mpp_parameter_mod,      only : CENTER, CORNER, SCALAR_PAIR, SCALAR_BIT, BITWISE_EFP_SUM
   use mpp_parameter_mod,      only : NORTH, NORTH_EAST, EAST, SOUTH_EAST
   use mpp_parameter_mod,      only : SOUTH, SOUTH_WEST, WEST, NORTH_WEST
   use mpp_parameter_mod,      only : MAX_DOMAIN_FIELDS, NULL_PE, DOMAIN_ID_BASE
@@ -137,7 +139,8 @@ module mpp_domains_mod
   use mpp_mod,                only : input_nml_file
   use mpp_mod,                only : COMM_TAG_1, COMM_TAG_2, COMM_TAG_3, COMM_TAG_4
   use mpp_memutils_mod,       only : mpp_memuse_begin, mpp_memuse_end
-  use mpp_pset_mod, only: mpp_pset_init
+  use mpp_pset_mod,           only : mpp_pset_init
+  use mpp_efp_mod,            only : mpp_reproducing_sum
   implicit none
   private
 
@@ -150,7 +153,7 @@ module mpp_domains_mod
   public :: GLOBAL_DATA_DOMAIN, CYCLIC_GLOBAL_DOMAIN, BGRID_NE, BGRID_SW, CGRID_NE, CGRID_SW
   public :: DGRID_NE, DGRID_SW, FOLD_WEST_EDGE, FOLD_EAST_EDGE, FOLD_SOUTH_EDGE, FOLD_NORTH_EDGE
   public :: WUPDATE, EUPDATE, SUPDATE, NUPDATE, XUPDATE, YUPDATE
-  public :: NON_BITWISE_EXACT_SUM, BITWISE_EXACT_SUM, MPP_DOMAIN_TIME
+  public :: NON_BITWISE_EXACT_SUM, BITWISE_EXACT_SUM, MPP_DOMAIN_TIME, BITWISE_EFP_SUM
   public :: CENTER, CORNER, SCALAR_PAIR
   public :: NORTH, NORTH_EAST, EAST, SOUTH_EAST
   public :: SOUTH, SOUTH_WEST, WEST, NORTH_WEST
@@ -161,7 +164,7 @@ module mpp_domains_mod
   public :: NULL_DOMAIN1D, NULL_DOMAIN2D
 
   public :: domain_axis_spec, domain1D, domain2D, DomainCommunicator2D
-  public :: nest_domain_type
+  public :: nest_domain_type, mpp_group_update_type
 
   !--- public interface from mpp_domains_util.h
   public :: mpp_domains_set_stack_size, mpp_get_compute_domain, mpp_get_compute_domains
@@ -172,7 +175,6 @@ module mpp_domains_mod
   public :: mpp_set_compute_domain, mpp_set_data_domain, mpp_set_global_domain
   public :: mpp_get_memory_domain, mpp_get_domain_shift, mpp_domain_is_tile_root_pe
   public :: mpp_get_tile_id, mpp_get_domain_extents, mpp_get_current_ntile, mpp_get_ntile_count
-  public :: mpp_get_refine_overlap_number, mpp_get_mosaic_refine_overlap
   public :: mpp_get_tile_list
   public :: mpp_get_tile_npes, mpp_get_domain_root_pe, mpp_get_tile_pelist, mpp_get_tile_compute_domains
   public :: mpp_get_num_overlap, mpp_get_overlap
@@ -181,6 +183,8 @@ module mpp_domains_mod
   public :: mpp_copy_domain, mpp_set_domain_symmetry
   public :: mpp_get_update_pelist, mpp_get_update_size
   public :: mpp_get_domain_npes, mpp_get_domain_pelist
+  public :: mpp_clear_group_update
+  public :: mpp_group_update_initialized, mpp_group_update_is_set
 
   !--- public interface from mpp_domains_reduce.h
   public :: mpp_global_field, mpp_global_max, mpp_global_min, mpp_global_sum
@@ -189,21 +193,28 @@ module mpp_domains_mod
   public :: mpp_broadcast_domain, mpp_domains_init, mpp_domains_exit, mpp_redistribute
   public :: mpp_update_domains, mpp_check_field
   public :: mpp_start_update_domains, mpp_complete_update_domains
+  public :: mpp_create_group_update, mpp_do_group_update
+  public :: mpp_start_group_update, mpp_complete_group_update
+  public :: mpp_reset_group_update_field
   public :: mpp_update_nest_fine, mpp_update_nest_coarse
-!  public :: mpp_update_domains_ad   ! bnc
   public :: mpp_get_boundary
   !--- public interface from mpp_domains_define.h
   public :: mpp_define_layout, mpp_define_domains, mpp_modify_domain, mpp_define_mosaic
   public :: mpp_define_mosaic_pelist, mpp_define_null_domain, mpp_mosaic_defined
   public :: mpp_define_io_domain, mpp_deallocate_domain
-  public :: mpp_compute_extent
+  public :: mpp_compute_extent, mpp_compute_block_extent
 
   !--- public interface from mpp_define_domains.inc
   public :: mpp_define_nest_domains, mpp_get_C2F_index, mpp_get_F2C_index
 
   integer, parameter :: NAME_LENGTH = 64
   integer, parameter :: MAXLIST = 24
- 
+  integer, parameter :: MAXOVERLAP = 100
+  integer, parameter :: FIELD_S = 0
+  integer, parameter :: FIELD_X = 1
+  integer, parameter :: FIELD_Y = 2
+
+
   !--- data types used mpp_domains_mod.
   type domain_axis_spec        !type used to specify index limits along an axis of a domain
      private
@@ -250,13 +261,8 @@ module mpp_domains_mod
      integer,         pointer :: ie(:)           => NULL() ! ending   i-index 
      integer,         pointer :: js(:)           => NULL() ! starting j-index 
      integer,         pointer :: je(:)           => NULL() ! ending   j-index 
-     integer,         pointer :: isMe(:)         => NULL() ! starting i-index of my tile on current pe
-     integer,         pointer :: ieMe(:)         => NULL() ! ending   i-index of my tile on current pe
-     integer,         pointer :: jsMe(:)         => NULL() ! starting j-index of my tile on current pe
-     integer,         pointer :: jeMe(:)         => NULL() ! ending   j-index of my tile on current pe
      integer,         pointer :: dir(:)          => NULL() ! direction ( value 1,2,3,4 = E,S,W,N)
      integer,         pointer :: rotation(:)     => NULL() ! rotation angle.
-     logical,         pointer :: is_refined(:)   => NULL() ! indicate if the overlap is refined or not.
      integer,         pointer :: index(:)        => NULL() ! for refinement
      logical,         pointer :: from_contact(:) => NULL() ! indicate if the overlap is computed from define_contact_overlap
   end type overlap_type
@@ -269,7 +275,6 @@ module mpp_domains_mod
      integer                     :: sendsize, recvsize
      type(overlap_type), pointer :: send(:) => NULL()
      type(overlap_type), pointer :: recv(:) => NULL()
-     type(refineSpec),   pointer :: rSpec(:)=> NULL()
      type(overlapSpec),  pointer :: next
   end type overlapSpec
 
@@ -277,24 +282,6 @@ module mpp_domains_mod
      integer :: xbegin, xend, ybegin, yend
   end type tile_type
 
-  type refineSpec
-     private
-     integer          :: count                 ! number of ovrelapping
-     integer          :: total                 ! total number of points to be saved in buffer.
-     integer, pointer :: isMe(:)     => NULL() ! starting i-index on current pe and tile.
-     integer, pointer :: ieMe(:)     => NULL() ! ending i-index on current pe and tile.
-     integer, pointer :: jsMe(:)     => NULL() ! starting j-index on current pe and tile.
-     integer, pointer :: jeMe(:)     => NULL() ! ending j-index on current pe and tile.
-     integer, pointer :: isNbr(:)    => NULL() ! starting i-index on neighbor pe or tile
-     integer, pointer :: ieNbr(:)    => NULL() ! ending i-index on neighbor pe or tile
-     integer, pointer :: jsNbr(:)    => NULL() ! starting j-index on neighbor pe or tile
-     integer, pointer :: jeNbr(:)    => NULL() ! ending j-index on neighbor pe or tile
-     integer, pointer :: start(:)    => NULL() ! starting index in the buffer
-     integer, pointer :: end(:)      => NULL() ! ending index in the buffer
-     integer, pointer :: dir(:)      => NULL() ! direction 
-     integer, pointer :: rotation(:) => NULL() ! rotation angle.
-  end type refineSpec
-
 !domaintypes of higher rank can be constructed from type domain1D
 !typically we only need 1 and 2D, but could need higher (e.g 3D LES)
 !some elements are repeated below if they are needed once per domain, not once per axis
@@ -466,6 +453,60 @@ module mpp_domains_mod
      integer(LONG_KIND)              :: field_addrs2(MAX_DOMAIN_FIELDS)
      integer                         :: nfields 
   end type nonblock_type
+
+  type mpp_group_update_type
+     private
+     logical            :: initialized = .FALSE.
+     logical            :: k_loop_inside = .TRUE.
+     integer            :: nscalar = 0
+     integer            :: nvector = 0
+     integer            :: flags_s=0, flags_v=0
+     integer            :: whalo_s=0, ehalo_s=0, shalo_s=0, nhalo_s=0
+     integer            :: isize_s=0, jsize_s=0, ksize_s=1
+     integer            :: whalo_v=0, ehalo_v=0, shalo_v=0, nhalo_v=0
+     integer            :: isize_x=0, jsize_x=0, ksize_v=1
+     integer            :: isize_y=0, jsize_y=0
+     integer            :: position=0, gridtype=0
+     logical            :: recv_s(8), recv_v(8)
+     integer            :: is_s=0, ie_s=0, js_s=0, je_s=0
+     integer            :: is_x=0, ie_x=0, js_x=0, je_x=0
+     integer            :: is_y=0, ie_y=0, js_y=0, je_y=0
+     integer            :: nrecv=0, nsend=0
+     integer            :: npack=0, nunpack=0
+     integer            :: reset_index_s = 0
+     integer            :: reset_index_v = 0
+     integer            :: tot_msgsize = 0
+     integer            :: from_pe(MAXOVERLAP)
+     integer            :: to_pe(MAXOVERLAP)
+     integer            :: recv_size(MAXOVERLAP)
+     integer            :: send_size(MAXOVERLAP)
+     integer            :: buffer_pos_recv(MAXOVERLAP)
+     integer            :: buffer_pos_send(MAXOVERLAP)
+     integer            :: pack_type(MAXOVERLAP)
+     integer            :: pack_buffer_pos(MAXOVERLAP)
+     integer            :: pack_rotation(MAXOVERLAP)
+     integer            :: pack_size(MAXOVERLAP)
+     integer            :: pack_is(MAXOVERLAP)
+     integer            :: pack_ie(MAXOVERLAP)
+     integer            :: pack_js(MAXOVERLAP)
+     integer            :: pack_je(MAXOVERLAP)
+     integer            :: unpack_type(MAXOVERLAP)
+     integer            :: unpack_buffer_pos(MAXOVERLAP)
+     integer            :: unpack_rotation(MAXOVERLAP)
+     integer            :: unpack_size(MAXOVERLAP)
+     integer            :: unpack_is(MAXOVERLAP)
+     integer            :: unpack_ie(MAXOVERLAP)
+     integer            :: unpack_js(MAXOVERLAP)
+     integer            :: unpack_je(MAXOVERLAP)
+     integer(LONG_KIND) :: addrs_s(MAX_DOMAIN_FIELDS)
+     integer(LONG_KIND) :: addrs_x(MAX_DOMAIN_FIELDS)
+     integer(LONG_KIND) :: addrs_y(MAX_DOMAIN_FIELDS)
+     integer            :: buffer_start_pos = -1
+     integer            :: request_send(MAX_REQUEST)
+     integer            :: request_recv(MAX_REQUEST)
+     integer            :: type_recv(MAX_REQUEST)
+  end type mpp_group_update_type
+
 !#######################################################################
 
 !***********************************************************************
@@ -484,13 +525,16 @@ module mpp_domains_mod
   type(domain2D),save  :: NULL_DOMAIN2D
   integer              :: current_id_update = 0
   integer                         :: num_update = 0
+  integer                         :: num_nonblock_group_update = 0
   integer                         :: nonblock_buffer_pos = 0
+  integer                         :: nonblock_group_buffer_pos = 0
   logical                         :: start_update = .true.
   logical                         :: complete_update = .false.
   type(nonblock_type), allocatable :: nonblock_data(:)
   integer, parameter              :: MAX_NONBLOCK_UPDATE = 100
 
-
+  integer                         :: group_update_buffer_pos = 0
+  logical                         :: complete_group_update_on = .false.
   !-------- The following variables are used in mpp_domains_comm.h
   
   integer, parameter :: MAX_ADDRS=512
@@ -530,8 +574,6 @@ module mpp_domains_mod
   !     integer(LONG_KIND), parameter :: KE_BASE=2**48
   integer(LONG_KIND), parameter :: KE_BASE=Z'0001000000000000'  ! Workaround for 64bit int init problem
 
-  integer, parameter :: MAXOVERLAP = 100 
-
   integer(LONG_KIND) :: domain_cnt=0
 
   !--- the following variables are used in mpp_domains_misc.h
@@ -542,6 +584,9 @@ module mpp_domains_mod
   integer :: wait_clock_nonblock=0  
   integer :: nest_send_clock=0, nest_recv_clock=0, nest_unpk_clock=0
   integer :: nest_wait_clock=0, nest_pack_clock=0
+  integer :: group_recv_clock=0, group_send_clock=0, group_pack_clock=0, group_unpk_clock=0, group_wait_clock=0
+  integer :: nonblock_group_recv_clock=0, nonblock_group_send_clock=0, nonblock_group_pack_clock=0
+  integer :: nonblock_group_unpk_clock=0, nonblock_group_wait_clock=0
 
   !--- namelist interface
 ! 
@@ -553,10 +598,21 @@ module mpp_domains_mod
 !     processor/tile when updating doamin for symmetric domain and check the consistency on the north
 !     folded edge. 
 !   
+!   
+!     Set true to always do overflow_check when doing EFP bitwise mpp_global_sum. 
+!   
+!   
+!     Determine the loop order for packing and unpacking. When number of threads is greater than nthread_control_loop,
+!     k-loop will be moved outside and combined with number of pack and unpack. When number of threads is less 
+!     than or equal to nthread_control_loop, k-loop is moved inside but still outside of j,i loop.
+!   
 ! 
   character(len=32) :: debug_update_domain = "none"
   logical           :: debug_message_passing = .false.
-  namelist /mpp_domains_nml/ debug_update_domain, domain_clocks_on, debug_message_passing
+  integer           :: nthread_control_loop = 4
+  logical           :: efp_sum_overflow_check = .false.
+  namelist /mpp_domains_nml/ debug_update_domain, domain_clocks_on, debug_message_passing, nthread_control_loop, &
+                             efp_sum_overflow_check
 
   !***********************************************************************
 
@@ -1234,6 +1290,52 @@ module mpp_domains_mod
      module procedure mpp_complete_do_update_i4_3d
   end interface
 
+
+  interface mpp_create_group_update
+     module procedure mpp_create_group_update_r4_2d
+     module procedure mpp_create_group_update_r4_3d
+     module procedure mpp_create_group_update_r4_4d
+     module procedure mpp_create_group_update_r4_2dv
+     module procedure mpp_create_group_update_r4_3dv
+     module procedure mpp_create_group_update_r4_4dv
+     module procedure mpp_create_group_update_r8_2d
+     module procedure mpp_create_group_update_r8_3d
+     module procedure mpp_create_group_update_r8_4d
+     module procedure mpp_create_group_update_r8_2dv
+     module procedure mpp_create_group_update_r8_3dv
+     module procedure mpp_create_group_update_r8_4dv
+  end interface mpp_create_group_update
+
+  interface mpp_do_group_update
+     module procedure mpp_do_group_update_r4
+     module procedure mpp_do_group_update_r8
+  end interface mpp_do_group_update
+
+  interface mpp_start_group_update
+     module procedure mpp_start_group_update_r4
+     module procedure mpp_start_group_update_r8
+  end interface mpp_start_group_update
+
+  interface mpp_complete_group_update
+     module procedure mpp_complete_group_update_r4
+     module procedure mpp_complete_group_update_r8
+  end interface mpp_complete_group_update
+
+  interface mpp_reset_group_update_field
+     module procedure mpp_reset_group_update_field_r4_2d
+     module procedure mpp_reset_group_update_field_r4_3d
+     module procedure mpp_reset_group_update_field_r4_4d
+     module procedure mpp_reset_group_update_field_r4_2dv
+     module procedure mpp_reset_group_update_field_r4_3dv
+     module procedure mpp_reset_group_update_field_r4_4dv
+     module procedure mpp_reset_group_update_field_r8_2d
+     module procedure mpp_reset_group_update_field_r8_3d
+     module procedure mpp_reset_group_update_field_r8_4d
+     module procedure mpp_reset_group_update_field_r8_2dv
+     module procedure mpp_reset_group_update_field_r8_3dv
+     module procedure mpp_reset_group_update_field_r8_4dv
+  end interface mpp_reset_group_update_field
+
   ! 
   !   
   !     Set up a domain to pass data between coarse and fine grid of nested model.
@@ -1583,58 +1685,6 @@ module mpp_domains_mod
 end interface
 
 
-!--------------------------------------------------------------
-!bnc: for adjoint update
-!--------------------------------------------------------------
-!!$  interface mpp_update_domains_ad
-!!$     module procedure mpp_update_domain2D_ad_r8_2d
-!!$     module procedure mpp_update_domain2D_ad_r8_3d
-!!$     module procedure mpp_update_domain2D_ad_r8_4d
-!!$     module procedure mpp_update_domain2D_ad_r8_5d
-!!$     module procedure mpp_update_domain2D_ad_r8_2dv
-!!$     module procedure mpp_update_domain2D_ad_r8_3dv
-!!$     module procedure mpp_update_domain2D_ad_r8_4dv
-!!$     module procedure mpp_update_domain2D_ad_r8_5dv
-!!$#ifdef OVERLOAD_C8
-!!$     module procedure mpp_update_domain2D_ad_c8_2d
-!!$     module procedure mpp_update_domain2D_ad_c8_3d
-!!$     module procedure mpp_update_domain2D_ad_c8_4d
-!!$     module procedure mpp_update_domain2D_ad_c8_5d
-!!$#endif
-!!$#ifndef no_8byte_integers
-!!$     module procedure mpp_update_domain2D_ad_i8_2d
-!!$     module procedure mpp_update_domain2D_ad_i8_3d
-!!$     module procedure mpp_update_domain2D_ad_i8_4d
-!!$     module procedure mpp_update_domain2D_ad_i8_5d
-!!$     module procedure mpp_update_domain2D_ad_l8_2d
-!!$     module procedure mpp_update_domain2D_ad_l8_3d
-!!$     module procedure mpp_update_domain2D_ad_l8_4d
-!!$     module procedure mpp_update_domain2D_ad_l8_5d
-!!$#endif
-!!$#ifdef OVERLOAD_R4
-!!$     module procedure mpp_update_domain2D_ad_r4_2d
-!!$     module procedure mpp_update_domain2D_ad_r4_3d
-!!$     module procedure mpp_update_domain2D_ad_r4_4d
-!!$     module procedure mpp_update_domain2D_ad_r4_5d
-!!$     module procedure mpp_update_domain2D_ad_r4_2dv
-!!$     module procedure mpp_update_domain2D_ad_r4_3dv
-!!$     module procedure mpp_update_domain2D_ad_r4_4dv
-!!$     module procedure mpp_update_domain2D_ad_r4_5dv
-!!$#endif
-!!$#ifdef OVERLOAD_C4
-!!$     module procedure mpp_update_domain2D_ad_c4_2d
-!!$     module procedure mpp_update_domain2D_ad_c4_3d
-!!$     module procedure mpp_update_domain2D_ad_c4_4d
-!!$     module procedure mpp_update_domain2D_ad_c4_5d
-!!$#endif
-!!$     module procedure mpp_update_domain2D_ad_i4_2d
-!!$     module procedure mpp_update_domain2D_ad_i4_3d
-!!$     module procedure mpp_update_domain2D_ad_i4_4d
-!!$     module procedure mpp_update_domain2D_ad_i4_5d
-!!$  end interface
-!bnc
-
-
   interface mpp_do_update
      module procedure mpp_do_update_r8_3d
      module procedure mpp_do_update_r8_3dv
@@ -1674,25 +1724,6 @@ module mpp_domains_mod
   end interface
 
 
-!-------------------------------------------------------
-!bnc  for adjoint do_update
-!-------------------------------------------------------
-!!$  interface mpp_do_update_ad
-!!$     module procedure mpp_do_update_ad_r8_3d
-!!$     module procedure mpp_do_update_ad_r8_3dv
-!!$#ifdef OVERLOAD_C8
-!!$     module procedure mpp_do_update_ad_c8_3d
-!!$#endif
-!!$#ifndef no_8byte_integers
-!!$     module procedure mpp_do_update_ad_i8_3d
-!!$#endif
-!!$#ifdef OVERLOAD_R4
-!!$     module procedure mpp_do_update_ad_r4_3d
-!!$     module procedure mpp_do_update_ad_r4_3dv
-!!$#endif
-!!$#ifdef OVERLOAD_C4
-!!$     module procedure mpp_do_update_ad_c4_3d
-!!$#endif
 !!$     module procedure mpp_do_update_ad_i4_3d
 !!$  end interface
 !bnc
@@ -2082,24 +2113,12 @@ module mpp_domains_mod
      module procedure mpp_global_sum_r8_3d
      module procedure mpp_global_sum_r8_4d
      module procedure mpp_global_sum_r8_5d
-#ifdef OVERLOAD_C8
-     module procedure mpp_global_sum_c8_2d
-     module procedure mpp_global_sum_c8_3d
-     module procedure mpp_global_sum_c8_4d
-     module procedure mpp_global_sum_c8_5d
-#endif
 #ifdef OVERLOAD_R4
      module procedure mpp_global_sum_r4_2d
      module procedure mpp_global_sum_r4_3d
      module procedure mpp_global_sum_r4_4d
      module procedure mpp_global_sum_r4_5d
 #endif
-#ifdef OVERLOAD_C4
-     module procedure mpp_global_sum_c4_2d
-     module procedure mpp_global_sum_c4_3d
-     module procedure mpp_global_sum_c4_4d
-     module procedure mpp_global_sum_c4_5d
-#endif
 #ifndef no_8byte_integers
      module procedure mpp_global_sum_i8_2d
      module procedure mpp_global_sum_i8_3d
@@ -2118,74 +2137,15 @@ module mpp_domains_mod
      module procedure mpp_global_sum_tl_r8_3d
      module procedure mpp_global_sum_tl_r8_4d
      module procedure mpp_global_sum_tl_r8_5d
-#ifdef OVERLOAD_C8
-     module procedure mpp_global_sum_tl_c8_2d
-     module procedure mpp_global_sum_tl_c8_3d
-     module procedure mpp_global_sum_tl_c8_4d
-     module procedure mpp_global_sum_tl_c8_5d
-#endif
 #ifdef OVERLOAD_R4
      module procedure mpp_global_sum_tl_r4_2d
      module procedure mpp_global_sum_tl_r4_3d
      module procedure mpp_global_sum_tl_r4_4d
      module procedure mpp_global_sum_tl_r4_5d
 #endif
-#ifdef OVERLOAD_C4
-     module procedure mpp_global_sum_tl_c4_2d
-     module procedure mpp_global_sum_tl_c4_3d
-     module procedure mpp_global_sum_tl_c4_4d
-     module procedure mpp_global_sum_tl_c4_5d
-#endif
-#ifndef no_8byte_integers
-     module procedure mpp_global_sum_tl_i8_2d
-     module procedure mpp_global_sum_tl_i8_3d
-     module procedure mpp_global_sum_tl_i8_4d
-     module procedure mpp_global_sum_tl_i8_5d
-#endif
-     module procedure mpp_global_sum_tl_i4_2d
-     module procedure mpp_global_sum_tl_i4_3d
-     module procedure mpp_global_sum_tl_i4_4d
-     module procedure mpp_global_sum_tl_i4_5d
   end interface
 !gag
 
-!bnc
-!!$  interface mpp_global_sum_ad
-!!$     module procedure mpp_global_sum_ad_r8_2d
-!!$     module procedure mpp_global_sum_ad_r8_3d
-!!$     module procedure mpp_global_sum_ad_r8_4d
-!!$     module procedure mpp_global_sum_ad_r8_5d
-!!$#ifdef OVERLOAD_C8
-!!$     module procedure mpp_global_sum_ad_c8_2d
-!!$     module procedure mpp_global_sum_ad_c8_3d
-!!$     module procedure mpp_global_sum_ad_c8_4d
-!!$     module procedure mpp_global_sum_ad_c8_5d
-!!$#endif
-!!$#ifdef OVERLOAD_R4
-!!$     module procedure mpp_global_sum_ad_r4_2d
-!!$     module procedure mpp_global_sum_ad_r4_3d
-!!$     module procedure mpp_global_sum_ad_r4_4d
-!!$     module procedure mpp_global_sum_ad_r4_5d
-!!$#endif
-!!$#ifdef OVERLOAD_C4
-!!$     module procedure mpp_global_sum_ad_c4_2d
-!!$     module procedure mpp_global_sum_ad_c4_3d
-!!$     module procedure mpp_global_sum_ad_c4_4d
-!!$     module procedure mpp_global_sum_ad_c4_5d
-!!$#endif
-!!$#ifndef no_8byte_integers
-!!$     module procedure mpp_global_sum_ad_i8_2d
-!!$     module procedure mpp_global_sum_ad_i8_3d
-!!$     module procedure mpp_global_sum_ad_i8_4d
-!!$     module procedure mpp_global_sum_ad_i8_5d
-!!$#endif
-!!$     module procedure mpp_global_sum_ad_i4_2d
-!!$     module procedure mpp_global_sum_ad_i4_3d
-!!$     module procedure mpp_global_sum_ad_i4_4d
-!!$     module procedure mpp_global_sum_ad_i4_5d
-!!$  end interface
-!bnc
-
 !***********************************************************************
 !
 !            public interface from mpp_domain_util.h
@@ -2463,9 +2423,9 @@ module mpp_domains_mod
 
   !--- version information variables
   character(len=128), public :: version= &
-       '$Id: mpp_domains.F90,v 20.0 2013/12/14 00:22:42 fms Exp $'
+       '$Id$'
   character(len=128), public :: tagname= &
-       '$Name: tikal $'
+       '$Name$'
 
 
 contains
diff --git a/src/shared/mpp/mpp_efp.F90 b/src/shared/mpp/mpp_efp.F90
new file mode 100644
index 0000000000..85c46d30e7
--- /dev/null
+++ b/src/shared/mpp/mpp_efp.F90
@@ -0,0 +1,665 @@
+module mpp_efp_mod
+#include 
+
+use mpp_mod, only : mpp_error, FATAL, WARNING, NOTE
+use mpp_mod, only : mpp_pe, mpp_root_pe, mpp_npes
+use mpp_mod, only : mpp_sum
+
+implicit none ; private
+
+public :: mpp_reproducing_sum, mpp_efp_list_sum_across_PEs
+public :: mpp_efp_plus, mpp_efp_minus, mpp_efp_to_real, mpp_real_to_efp, mpp_efp_real_diff
+public :: operator(+), operator(-), assignment(=)
+public :: mpp_query_efp_overflow_error, mpp_reset_efp_overlow_error
+
+!   This module provides interfaces to the non-domain-oriented communication
+! subroutines.
+integer, parameter :: NUMBIT = 46  ! number of bits used in the 64-bit signed integer representation.
+integer, parameter :: NUMINT = 6   ! The number of long integers to use to represent
+                                   ! a real number.
+
+integer(LONG_KIND), parameter :: prec=2_8**NUMBIT ! The precision of each integer.
+real(DOUBLE_KIND), parameter :: r_prec=2.0**NUMBIT  ! A real version of prec.
+real(DOUBLE_KIND), parameter :: I_prec=1.0/(2.0**NUMBIT) ! The inverse of prec.
+integer, parameter :: max_count_prec=2**(63-NUMBIT)-1
+                              ! The number of values that can be added together
+                              ! with the current value of prec before there will
+                              ! be roundoff problems.
+
+real(DOUBLE_KIND), parameter, dimension(NUMINT) :: &
+  pr = (/ r_prec**2, r_prec, 1.0, 1.0/r_prec, 1.0/r_prec**2, 1.0/r_prec**3 /)
+real(DOUBLE_KIND), parameter, dimension(NUMINT) :: &
+  I_pr = (/ 1.0/r_prec**2, 1.0/r_prec, 1.0, r_prec, r_prec**2, r_prec**3 /)
+
+logical :: overflow_error = .false., NaN_error = .false.
+logical :: debug = .false.    ! Making this true enables debugging output.
+
+interface mpp_reproducing_sum
+  module procedure mpp_reproducing_sum_r8_2d
+  module procedure mpp_reproducing_sum_r8_3d
+  module procedure mpp_reproducing_sum_r4_2d
+end interface mpp_reproducing_sum
+
+! The Extended Fixed Point (mpp_efp) type provides a public interface for doing
+! sums and taking differences with this type.
+type, public :: mpp_efp_type ; private
+  integer(kind=8), dimension(NUMINT) :: v
+end type mpp_efp_type
+
+interface operator (+); module procedure mpp_efp_plus  ; end interface
+interface operator (-); module procedure mpp_efp_minus ; end interface
+interface assignment(=); module procedure mpp_efp_assign ; end interface
+
+contains
+
+function mpp_reproducing_sum_r8_2d(array, isr, ier, jsr, jer, EFP_sum, reproducing, &
+                            overflow_check, err) result(sum)
+  real(DOUBLE_KIND), dimension(:,:), intent(in) :: array
+  integer,        optional,          intent(in) :: isr, ier, jsr, jer
+  type(mpp_efp_type), optional,     intent(out) :: EFP_sum
+  logical,        optional,          intent(in) :: reproducing
+  logical,        optional,          intent(in) :: overflow_check
+  integer,        optional,         intent(out) :: err
+  real(DOUBLE_KIND)                             :: sum  ! Result
+
+  !   This subroutine uses a conversion to an integer representation
+  ! of real numbers to give order-invariant sums that will reproduce
+  ! across PE count.  This idea comes from R. Hallberg and A. Adcroft.
+
+  integer(LONG_KIND), dimension(NUMINT)  :: ints_sum
+  integer(LONG_KIND) :: ival, prec_error
+  real(DOUBLE_KIND)  :: rsum(1), rs
+  real(DOUBLE_KIND)  :: max_mag_term
+  logical :: repro, over_check
+  character(len=256) :: mesg
+  integer :: i, j, n, is, ie, js, je, sgn
+
+  if (mpp_npes() > max_count_prec) call mpp_error(FATAL, &
+    "mpp_reproducing_sum: Too many processors are being used for the value of "//&
+    "prec.  Reduce prec to (2^63-1)/mpp_npes.")
+
+  prec_error = (2_8**62 + (2_8**62 - 1)) / mpp_npes()
+
+  is = 1 ; ie = size(array,1) ; js = 1 ; je = size(array,2 )
+  if (present(isr)) then
+    if (isr < is) call mpp_error(FATAL, &
+      "Value of isr too small in mpp_reproducing_sum_2d.")
+    is = isr
+  endif
+  if (present(ier)) then
+    if (ier > ie) call mpp_error(FATAL, &
+      "Value of ier too large in mpp_reproducing_sum_2d.")
+    ie = ier
+  endif
+  if (present(jsr)) then
+    if (jsr < js) call mpp_error(FATAL, &
+      "Value of jsr too small in mpp_reproducing_sum_2d.")
+    js = jsr
+  endif
+  if (present(jer)) then
+    if (jer > je) call mpp_error(FATAL, &
+      "Value of jer too large in mpp_reproducing_sum_2d.")
+    je = jer
+  endif
+
+  repro = .true. ; if (present(reproducing)) repro = reproducing
+  over_check = .true. ; if (present(overflow_check)) over_check = overflow_check
+
+  if (repro) then
+    overflow_error = .false. ; NaN_error = .false. ; max_mag_term = 0.0
+    ints_sum(:) = 0
+    if (over_check) then
+      if ((je+1-js)*(ie+1-is) < max_count_prec) then
+        do j=js,je ; do i=is,ie
+          call increment_ints_faster(ints_sum, array(i,j), max_mag_term);
+        enddo ; enddo
+        call carry_overflow(ints_sum, prec_error)
+      elseif ((ie+1-is) < max_count_prec) then
+        do j=js,je
+          do i=is,ie
+            call increment_ints_faster(ints_sum, array(i,j), max_mag_term);
+          enddo
+          call carry_overflow(ints_sum, prec_error)
+        enddo
+      else
+        do j=js,je ; do i=is,ie
+          call increment_ints(ints_sum, real_to_ints(array(i,j), prec_error), &
+                              prec_error);
+        enddo ; enddo
+      endif
+    else
+      do j=js,je ; do i=is,ie
+        sgn = 1 ; if (array(i,j)<0.0) sgn = -1
+        rs = abs(array(i,j))
+        do n=1,NUMINT
+          ival = int(rs*I_pr(n), 8)
+          rs = rs - ival*pr(n)
+          ints_sum(n) = ints_sum(n) + sgn*ival
+        enddo
+      enddo ; enddo
+      call carry_overflow(ints_sum, prec_error)
+    endif
+
+    if (present(err)) then
+      err = 0
+      if (overflow_error) &
+        err = err+2
+      if (NaN_error) &
+        err = err+4
+      if (err > 0) then ; do n=1,NUMINT ; ints_sum(n) = 0 ; enddo ; endif
+    else
+      if (NaN_error) then
+        call mpp_error(FATAL, "NaN in input field of mpp_reproducing_sum(_2d).")
+      endif
+      if (abs(max_mag_term) >= prec_error*pr(1)) then
+        write(mesg, '(ES13.5)') max_mag_term
+        call mpp_error(FATAL,"Overflow in mpp_reproducing_sum(_2d) conversion of "//trim(mesg))
+      endif
+      if (overflow_error) then
+        call mpp_error(FATAL, "Overflow in mpp_reproducing_sum(_2d).")
+      endif
+    endif
+
+    call mpp_sum(ints_sum, NUMINT)
+
+    call regularize_ints(ints_sum)
+    sum = ints_to_real(ints_sum)
+  else
+    rsum(1) = 0.0
+    do j=js,je ; do i=is,ie
+      rsum(1) = rsum(1) + array(i,j);
+    enddo ; enddo
+    call mpp_sum(rsum,1)
+    sum = rsum(1)
+
+    if (present(err)) then ; err = 0 ; endif
+
+    if (debug .or. present(EFP_sum)) then
+      overflow_error = .false.
+      ints_sum = real_to_ints(sum, prec_error, overflow_error)
+      if (overflow_error) then
+        if (present(err)) then
+          err = err + 2
+        else
+          write(mesg, '(ES13.5)') sum
+          call mpp_error(FATAL,"Repro_sum_2d: Overflow in real_to_ints conversion of "//trim(mesg))
+        endif
+      endif
+    endif
+  endif
+
+  if (present(EFP_sum)) EFP_sum%v(:) = ints_sum(:)
+
+  if (debug) then
+    write(mesg,'("2d RS: ", ES24.16, 6 Z17.16)') sum, ints_sum(1:NUMINT)
+    if(mpp_pe() == mpp_root_pe()) call mpp_error(NOTE, mesg)
+  endif
+
+end function mpp_reproducing_sum_r8_2d
+
+function mpp_reproducing_sum_r4_2d(array, isr, ier, jsr, jer, EFP_sum, reproducing, &
+                            overflow_check, err) result(sum)
+  real(FLOAT_KIND), dimension(:,:), intent(in) :: array
+  integer,        optional,          intent(in) :: isr, ier, jsr, jer
+  type(mpp_efp_type), optional,     intent(out) :: EFP_sum
+  logical,        optional,          intent(in) :: reproducing
+  logical,        optional,          intent(in) :: overflow_check
+  integer,        optional,         intent(out) :: err
+  real(FLOAT_KIND)                             :: sum  ! Result
+
+  real(DOUBLE_KIND) :: array_r8(size(array,1), size(array,2))
+
+  array_r8 = array
+  
+  sum = mpp_reproducing_sum_r8_2d(array_r8, isr, ier, jsr, jer, EFP_sum, reproducing, &
+                            overflow_check, err)
+
+  return
+
+end function mpp_reproducing_sum_r4_2d
+
+
+function mpp_reproducing_sum_r8_3d(array, isr, ier, jsr, jer, sums, EFP_sum, err) &
+                            result(sum)
+  real(DOUBLE_KIND), dimension(:,:,:),        intent(in) :: array
+  integer,    optional,                       intent(in) :: isr, ier, jsr, jer
+  real(DOUBLE_KIND), dimension(:), optional, intent(out) :: sums
+  type(mpp_efp_type),     optional,          intent(out) :: EFP_sum
+  integer,            optional,              intent(out) :: err
+  real(DOUBLE_KIND)                                      :: sum  ! Result
+
+  !   This subroutine uses a conversion to an integer representation
+  ! of real numbers to give order-invariant sums that will reproduce
+  ! across PE count.  This idea comes from R. Hallberg and A. Adcroft.
+
+  real(DOUBLE_KIND)    :: max_mag_term
+  integer(LONG_KIND), dimension(NUMINT)  :: ints_sum
+  integer(LONG_KIND), dimension(NUMINT,size(array,3))  :: ints_sums
+  integer(LONG_KIND) :: prec_error
+  character(len=256) :: mesg
+  integer :: i, j, k, is, ie, js, je, ke, isz, jsz, n
+
+  if (mpp_npes() > max_count_prec) call mpp_error(FATAL, &
+    "mpp_reproducing_sum: Too many processors are being used for the value of "//&
+    "prec.  Reduce prec to (2^63-1)/mpp_npes.")
+
+  prec_error = (2_8**62 + (2_8**62 - 1)) / mpp_npes()
+  max_mag_term = 0.0
+
+  is = 1 ; ie = size(array,1) ; js = 1 ; je = size(array,2) ; ke = size(array,3)
+  if (present(isr)) then
+    if (isr < is) call mpp_error(FATAL, &
+      "Value of isr too small in mpp_reproducing_sum(_3d).")
+    is = isr
+  endif
+  if (present(ier)) then
+    if (ier > ie) call mpp_error(FATAL, &
+      "Value of ier too large in mpp_reproducing_sum(_3d).")
+    ie = ier
+  endif
+  if (present(jsr)) then
+    if (jsr < js) call mpp_error(FATAL, &
+      "Value of jsr too small in mpp_reproducing_sum(_3d).")
+    js = jsr
+  endif
+  if (present(jer)) then
+    if (jer > je) call mpp_error(FATAL, &
+      "Value of jer too large in mpp_reproducing_sum(_3d).")
+    je = jer
+  endif
+  jsz = je+1-js; isz = ie+1-is
+
+  if (present(sums)) then
+    if (size(sums) > ke) call mpp_error(FATAL, "Sums is smaller than "//&
+      "the vertical extent of array in mpp_reproducing_sum(_3d).")
+    ints_sums(:,:) = 0
+    overflow_error = .false. ; NaN_error = .false. ; max_mag_term = 0.0
+    if (jsz*isz < max_count_prec) then
+      do k=1,ke
+        do j=js,je ; do i=is,ie
+          call increment_ints_faster(ints_sums(:,k), array(i,j,k), max_mag_term);
+        enddo ; enddo
+        call carry_overflow(ints_sums(:,k), prec_error)
+      enddo
+    elseif (isz < max_count_prec) then
+      do k=1,ke ; do j=js,je
+        do i=is,ie
+          call increment_ints_faster(ints_sums(:,k), array(i,j,k), max_mag_term);
+        enddo
+        call carry_overflow(ints_sums(:,k), prec_error)
+      enddo ; enddo
+    else
+      do k=1,ke ; do j=js,je ; do i=is,ie
+        call increment_ints(ints_sums(:,k), &
+                            real_to_ints(array(i,j,k), prec_error), prec_error);
+      enddo ; enddo ; enddo
+    endif
+    if (present(err)) then
+      err = 0
+      if (abs(max_mag_term) >= prec_error*pr(1)) err = err+1
+      if (overflow_error) err = err+2
+      if (NaN_error) err = err+2
+      if (err > 0) then ; do k=1,ke ; do n=1,NUMINT ; ints_sums(n,k) = 0 ; enddo ; enddo ; endif
+    else
+      if (NaN_error) call mpp_error(FATAL, "NaN in input field of mpp_reproducing_sum(_3d).")
+      if (abs(max_mag_term) >= prec_error*pr(1)) then
+        write(mesg, '(ES13.5)') max_mag_term
+        call mpp_error(FATAL,"Overflow in mpp_reproducing_sum(_3d) conversion of "//trim(mesg))
+      endif
+      if (overflow_error) call mpp_error(FATAL, "Overflow in mpp_reproducing_sum(_3d).")
+    endif
+
+    call mpp_sum(ints_sums(:,1:ke), NUMINT*ke)
+
+    sum = 0.0
+    do k=1,ke
+      call regularize_ints(ints_sums(:,k))
+      sums(k) = ints_to_real(ints_sums(:,k))
+      sum = sum + sums(k)
+    enddo
+
+    if (present(EFP_sum)) then
+      EFP_sum%v(:) = 0
+      do k=1,ke ; call increment_ints(EFP_sum%v(:), ints_sums(:,k)) ; enddo
+    endif
+
+    if (debug) then
+      do n=1,NUMINT ; ints_sum(n) = 0 ; enddo
+      do k=1,ke ; do n=1,NUMINT ; ints_sum(n) = ints_sum(n) + ints_sums(n,k) ; enddo ; enddo
+      write(mesg,'("3D RS: ", ES24.16, 6 Z17.16)') sum, ints_sum(1:NUMINT)
+      if(mpp_pe()==mpp_root_pe()) call mpp_error(NOTE, mesg)
+    endif
+  else
+    ints_sum(:) = 0
+    overflow_error = .false. ; NaN_error = .false. ; max_mag_term = 0.0
+    if (jsz*isz < max_count_prec) then
+      do k=1,ke
+        do j=js,je ; do i=is,ie
+          call increment_ints_faster(ints_sum, array(i,j,k), max_mag_term);
+        enddo ; enddo
+        call carry_overflow(ints_sum, prec_error)
+      enddo
+    elseif (isz < max_count_prec) then
+      do k=1,ke ; do j=js,je
+        do i=is,ie
+          call increment_ints_faster(ints_sum, array(i,j,k), max_mag_term);
+        enddo
+        call carry_overflow(ints_sum, prec_error)
+      enddo ; enddo
+    else
+      do k=1,ke ; do j=js,je ; do i=is,ie
+        call increment_ints(ints_sum, real_to_ints(array(i,j,k), prec_error), &
+                            prec_error);
+      enddo ; enddo ; enddo
+    endif
+    if (present(err)) then
+      err = 0
+      if (abs(max_mag_term) >= prec_error*pr(1)) err = err+1
+      if (overflow_error) err = err+2
+      if (NaN_error) err = err+2
+      if (err > 0) then ; do n=1,NUMINT ; ints_sum(n) = 0 ; enddo ; endif
+    else
+      if (NaN_error) call mpp_error(FATAL, "NaN in input field of mpp_reproducing_sum(_3d).")
+      if (abs(max_mag_term) >= prec_error*pr(1)) then
+        write(mesg, '(ES13.5)') max_mag_term
+        call mpp_error(FATAL,"Overflow in mpp_reproducing_sum(_3d) conversion of "//trim(mesg))
+      endif
+      if (overflow_error) call mpp_error(FATAL, "Overflow in mpp_reproducing_sum(_3d).")
+    endif
+
+    call mpp_sum(ints_sum, NUMINT)
+
+    call regularize_ints(ints_sum)
+    sum = ints_to_real(ints_sum)
+
+    if (present(EFP_sum)) EFP_sum%v(:) = ints_sum(:)
+
+    if (debug) then
+      write(mesg,'("3d RS: ", ES24.16, 6 Z17.16)') sum, ints_sum(1:NUMINT)
+      if(mpp_pe()==mpp_root_pe()) call mpp_error(NOTE, mesg)
+    endif
+  endif
+
+end function mpp_reproducing_sum_r8_3d
+
+function real_to_ints(r, prec_error, overflow) result(ints)
+  real(DOUBLE_KIND),            intent(in) :: r
+  integer(LONG_KIND), optional, intent(in) :: prec_error
+  logical,   optional,       intent(inout) :: overflow
+  integer(LONG_KIND),    dimension(NUMINT) :: ints
+  !   This subroutine converts a real number to an equivalent representation
+  ! using several long integers.
+
+  real(DOUBLE_KIND) :: rs
+  character(len=80) :: mesg
+  integer(LONG_KIND) :: ival, prec_err
+  integer :: sgn, i
+
+  prec_err = prec ; if (present(prec_error)) prec_err = prec_error
+  ints(:) = 0_8
+  if ((r >= 1e30) .eqv. (r < 1e30)) then ; NaN_error = .true. ; return ; endif
+
+  sgn = 1 ; if (r<0.0) sgn = -1
+  rs = abs(r)
+
+  if (present(overflow)) then
+    if (.not.(rs < prec_err*pr(1))) overflow = .true.
+    if ((r >= 1e30) .eqv. (r < 1e30)) overflow = .true.
+  elseif (.not.(rs < prec_err*pr(1))) then
+    write(mesg, '(ES13.5)') r
+    call mpp_error(FATAL,"Overflow in real_to_ints conversion of "//trim(mesg))
+  endif
+
+  do i=1,NUMINT
+    ival = int(rs*I_pr(i), 8)
+    rs = rs - ival*pr(i)
+    ints(i) = sgn*ival
+  enddo
+
+end function real_to_ints
+
+function ints_to_real(ints) result(r)
+  integer(LONG_KIND), dimension(NUMINT), intent(in) :: ints
+  real(DOUBLE_KIND) :: r
+  ! This subroutine reverses the conversion in real_to_ints.
+
+  integer :: i
+
+  r = 0.0
+  do i=1,NUMINT ; r = r + pr(i)*ints(i) ; enddo
+end function ints_to_real
+
+subroutine increment_ints(int_sum, int2, prec_error)
+  integer(LONG_KIND), dimension(NUMINT), intent(inout) :: int_sum
+  integer(LONG_KIND), dimension(NUMINT), intent(in)    :: int2
+  integer(LONG_KIND), optional,      intent(in)    :: prec_error
+
+  ! This subroutine increments a number with another, both using the integer
+  ! representation in real_to_ints.
+  integer :: i
+
+  do i=NUMINT,2,-1
+    int_sum(i) = int_sum(i) + int2(i)
+    ! Carry the local overflow.
+    if (int_sum(i) > prec) then
+      int_sum(i) = int_sum(i) - prec
+      int_sum(i-1) = int_sum(i-1) + 1
+    elseif (int_sum(i) < -prec) then
+      int_sum(i) = int_sum(i) + prec
+      int_sum(i-1) = int_sum(i-1) - 1
+    endif
+  enddo
+  int_sum(1) = int_sum(1) + int2(1)
+  if (present(prec_error)) then
+    if (abs(int_sum(1)) > prec_error) overflow_error = .true.
+  else
+    if (abs(int_sum(1)) > prec) overflow_error = .true.
+  endif
+
+end subroutine increment_ints
+
+subroutine increment_ints_faster(int_sum, r, max_mag_term)
+  integer(LONG_KIND), dimension(NUMINT), intent(inout) :: int_sum
+  real(DOUBLE_KIND),                        intent(in) :: r
+  real(DOUBLE_KIND),                     intent(inout) :: max_mag_term
+
+  ! This subroutine increments a number with another, both using the integer
+  ! representation in real_to_ints, but without doing any carrying of overflow.
+  ! The entire operation is embedded in a single call for greater speed.
+  real(DOUBLE_KIND) :: rs
+  integer(LONG_KIND) :: ival
+  integer :: sgn, i
+
+  if ((r >= 1e30) .eqv. (r < 1e30)) then ; NaN_error = .true. ; return ; endif
+  sgn = 1 ; if (r<0.0) sgn = -1
+  rs = abs(r)
+  if (rs > abs(max_mag_term)) max_mag_term = r
+
+  do i=1,NUMINT
+    ival = int(rs*I_pr(i), 8)
+    rs = rs - ival*pr(i)
+    int_sum(i) = int_sum(i) + sgn*ival
+  enddo
+
+end subroutine increment_ints_faster
+
+subroutine carry_overflow(int_sum, prec_error)
+  integer(LONG_KIND), dimension(NUMINT), intent(inout) :: int_sum
+  integer(LONG_KIND),                intent(in)    :: prec_error
+
+  ! This subroutine handles carrying of the overflow.
+  integer :: i, num_carry
+
+  do i=NUMINT,2,-1 ; if (abs(int_sum(i)) > prec) then
+    num_carry = int(int_sum(i) * I_prec)
+    int_sum(i) = int_sum(i) - num_carry*prec
+    int_sum(i-1) = int_sum(i-1) + num_carry
+  endif ; enddo
+  if (abs(int_sum(1)) > prec_error) then
+    overflow_error = .true.
+  endif
+
+end subroutine carry_overflow
+
+subroutine regularize_ints(int_sum)
+  integer(LONG_KIND), dimension(NUMINT), intent(inout) :: int_sum
+
+  ! This subroutine carries the overflow, and then makes sure that
+  ! all integers are of the same sign as the overall value.
+  logical :: positive
+  integer :: i, num_carry
+
+  do i=NUMINT,2,-1 ; if (abs(int_sum(i)) > prec) then
+    num_carry = int(int_sum(i) * I_prec)
+    int_sum(i) = int_sum(i) - num_carry*prec
+    int_sum(i-1) = int_sum(i-1) + num_carry
+  endif ; enddo
+
+  ! Determine the sign of the final number.
+  positive = .true.
+  do i=1,NUMINT
+    if (abs(int_sum(i)) > 0) then
+      if (int_sum(i) < 0) positive = .false.
+      exit
+    endif
+  enddo
+
+  if (positive) then
+    do i=NUMINT,2,-1 ; if (int_sum(i) < 0) then
+      int_sum(i) = int_sum(i) + prec
+      int_sum(i-1) = int_sum(i-1) - 1
+    endif ; enddo
+  else
+    do i=NUMINT,2,-1 ; if (int_sum(i) > 0) then
+      int_sum(i) = int_sum(i) - prec
+      int_sum(i-1) = int_sum(i-1) + 1
+    endif ; enddo
+  endif
+
+end subroutine regularize_ints
+
+function mpp_query_efp_overflow_error()
+  logical :: mpp_query_efp_overflow_error
+  mpp_query_efp_overflow_error = overflow_error
+end function mpp_query_efp_overflow_error
+
+subroutine mpp_reset_efp_overlow_error()
+  overflow_error = .false.
+end subroutine mpp_reset_efp_overlow_error
+
+function mpp_efp_plus(EFP1, EFP2)
+  type(mpp_efp_type)             :: mpp_efp_plus
+  type(mpp_efp_type), intent(in) :: EFP1, EFP2
+
+  mpp_efp_plus = EFP1
+
+  call increment_ints(mpp_efp_plus%v(:), EFP2%v(:))
+end function mpp_efp_plus
+
+function mpp_efp_minus(EFP1, EFP2)
+  type(mpp_efp_type)             :: mpp_efp_minus
+  type(mpp_efp_type), intent(in) :: EFP1, EFP2
+  integer :: i
+
+  do i=1,NUMINT ; mpp_efp_minus%v(i) = -1*EFP2%v(i) ; enddo
+
+  call increment_ints(mpp_efp_minus%v(:), EFP1%v(:))
+end function mpp_efp_minus
+
+subroutine mpp_efp_assign(EFP1, EFP2)
+  type(mpp_efp_type), intent(out) :: EFP1
+  type(mpp_efp_type), intent(in)  :: EFP2
+  integer i
+  ! This subroutine assigns all components of the extended fixed point type
+  ! variable on the RHS (EFP2) to the components of the variable on the LHS
+  ! (EFP1).
+
+  do i=1,NUMINT ; EFP1%v(i) = EFP2%v(i) ; enddo
+end subroutine mpp_efp_assign
+
+function mpp_efp_to_real(EFP1)
+  type(mpp_efp_type), intent(inout) :: EFP1
+  real(DOUBLE_KIND) :: mpp_efp_to_real
+
+  call regularize_ints(EFP1%v)
+  mpp_efp_to_real = ints_to_real(EFP1%v)
+end function mpp_efp_to_real
+
+function mpp_efp_real_diff(EFP1, EFP2)
+  type(mpp_efp_type), intent(in) :: EFP1, EFP2
+  real(DOUBLE_KIND) :: mpp_efp_real_diff
+
+  type(mpp_efp_type)             :: EFP_diff
+
+  EFP_diff = EFP1 - EFP2
+  mpp_efp_real_diff = mpp_efp_to_real(EFP_diff)
+
+end function mpp_efp_real_diff
+
+function mpp_real_to_efp(val, overflow)
+  real(DOUBLE_KIND),    intent(in) :: val
+  logical, optional, intent(inout) :: overflow
+  type(mpp_efp_type)               :: mpp_real_to_efp
+
+  logical :: over
+  character(len=80) :: mesg
+
+  if (present(overflow)) then
+    mpp_real_to_efp%v(:) = real_to_ints(val, overflow=overflow)
+  else
+    over = .false.
+    mpp_real_to_efp%v(:) = real_to_ints(val, overflow=over)
+    if (over) then
+      write(mesg, '(ES13.5)') val
+      call mpp_error(FATAL,"Overflow in mpp_real_to_efp conversion of "//trim(mesg))
+    endif
+  endif
+
+end function mpp_real_to_efp
+
+subroutine mpp_efp_list_sum_across_PEs(EFPs, nval, errors)
+  type(mpp_efp_type), dimension(:), intent(inout) :: EFPs
+  integer, intent(in) :: nval
+  logical, dimension(:), optional, intent(out) :: errors
+
+  !   This subroutine does a sum across PEs of a list of EFP variables,
+  ! returning the sums in place, with all overflows carried.
+
+  integer(LONG_KIND), dimension(NUMINT,nval) :: ints
+  integer(LONG_KIND) :: prec_error
+  logical :: error_found
+  character(len=256) :: mesg
+  integer :: i, n
+
+  if (mpp_npes() > max_count_prec) call mpp_error(FATAL, &
+    "mpp_efp_list_sum_across_PEs: Too many processors are being used for the value of "//&
+    "prec.  Reduce prec to (2^63-1)/mpp_npes.")
+
+  prec_error = (2_8**62 + (2_8**62 - 1)) / mpp_npes()
+  ! overflow_error is an overflow error flag for the whole module.
+  overflow_error = .false. ; error_found = .false.
+
+  do i=1,nval ; do n=1,NUMINT ; ints(n,i) = EFPs(i)%v(n) ; enddo ; enddo
+
+  call mpp_sum(ints(:,:), NUMINT*nval)
+
+  if (present(errors)) errors(:) = .false.
+  do i=1,nval
+    overflow_error = .false.
+    call carry_overflow(ints(:,i), prec_error)
+    do n=1,NUMINT ; EFPs(i)%v(n) = ints(n,i) ; enddo
+    if (present(errors)) errors(i) = overflow_error
+    if (overflow_error) then
+      write (mesg,'("mpp_efp_list_sum_across_PEs error at ",i6," val was ",ES12.6, ", prec_error = ",ES12.6)') &
+             i, mpp_efp_to_real(EFPs(i)), real(prec_error)
+      if(mpp_pe()==mpp_root_pe()) call mpp_error(WARNING, mesg)
+    endif
+    error_found = error_found .or. overflow_error
+  enddo
+  if (error_found .and. .not.(present(errors))) then
+    call mpp_error(FATAL, "Overflow in mpp_efp_list_sum_across_PEs.")
+  endif
+
+end subroutine mpp_efp_list_sum_across_PEs
+
+end module mpp_efp_mod
diff --git a/src/shared/mpp/mpp_io.F90 b/src/shared/mpp/mpp_io.F90
index 4c02397b87..b234df8417 100644
--- a/src/shared/mpp/mpp_io.F90
+++ b/src/shared/mpp/mpp_io.F90
@@ -19,10 +19,12 @@
 !           675 Mass Ave, Cambridge, MA 02139, USA.  
 !-----------------------------------------------------------------------
 
-! 
+! 
 !   V. Balaji
 ! 
 
+! 
+! 
 
 ! 
 !   mpp_io_mod, is a set of simple calls for parallel I/O on
@@ -317,15 +319,18 @@ module mpp_io_mod
 use mpp_mod,            only : mpp_pe, mpp_root_pe, mpp_npes, lowercase, mpp_transmit, mpp_sync_self
 use mpp_mod,            only : mpp_init, mpp_sync, mpp_clock_id, mpp_clock_begin, mpp_clock_end
 use mpp_mod,            only : MPP_CLOCK_SYNC, MPP_CLOCK_DETAILED, CLOCK_ROUTINE
-use mpp_mod,            only : input_nml_file
+use mpp_mod,            only : input_nml_file, mpp_gather, mpp_broadcast
+use mpp_mod,            only : mpp_send, mpp_recv, mpp_sync_self, EVENT_RECV, COMM_TAG_1
 use mpp_domains_mod,    only : domain1d, domain2d, NULL_DOMAIN1D, mpp_domains_init
 use mpp_domains_mod,    only : mpp_get_global_domain, mpp_get_compute_domain
-use mpp_domains_mod,    only :  mpp_get_data_domain, mpp_get_memory_domain
+use mpp_domains_mod,    only : mpp_get_data_domain, mpp_get_memory_domain, mpp_get_pelist
 use mpp_domains_mod,    only : mpp_update_domains, mpp_global_field, mpp_domain_is_symmetry
 use mpp_domains_mod,    only : operator( .NE. ), mpp_get_domain_shift
 use mpp_domains_mod,    only : mpp_get_io_domain, mpp_domain_is_tile_root_pe, mpp_get_domain_tile_root_pe
 use mpp_domains_mod,    only : mpp_get_tile_id, mpp_get_tile_npes, mpp_get_io_domain_layout
 use mpp_domains_mod,    only : mpp_get_domain_name, mpp_get_domain_npes
+use mpp_parameter_mod,  only : MPP_FILL_DOUBLE,MPP_FILL_INT
+use mpp_mod,            only : mpp_chksum
 
 implicit none
 private
@@ -353,18 +358,20 @@ module mpp_io_mod
   public :: mpp_io_set_stack_size, mpp_get_field_index, mpp_get_axis_index
   public :: mpp_get_field_name, mpp_get_att_value, mpp_get_att_length
   public :: mpp_get_att_type, mpp_get_att_name, mpp_get_att_real, mpp_get_att_char
-  public :: mpp_get_att_real_scalar, mpp_get_axis_length
+  public :: mpp_get_att_real_scalar, mpp_get_axis_length, mpp_is_dist_ioroot
   public :: mpp_get_file_name, mpp_file_is_opened, mpp_attribute_exist 
   public :: mpp_io_clock_on, mpp_get_time_axis, mpp_get_default_calendar
+  public :: mpp_get_dimension_length, mpp_get_axis_bounds
 
   !--- public interface from mpp_io_misc.h ----------------------
   public :: mpp_io_init, mpp_io_exit, netcdf_err, mpp_flush
 
   !--- public interface from mpp_io_write.h ---------------------
-  public :: mpp_write, mpp_write_meta, mpp_copy_meta, mpp_modify_meta, mpp_write_axis_data
+  public :: mpp_write, mpp_write_meta, mpp_copy_meta, mpp_modify_meta, mpp_write_axis_data, mpp_def_dim
 
   !--- public interface from mpp_io_read.h ---------------------
   public :: mpp_read, mpp_read_meta, mpp_get_tavg_info
+  public :: mpp_read_compressed, mpp_write_compressed, mpp_read_distributed_ascii, mpp_write_unlimited_axis
 
   !--- public interface from mpp_io_switch.h ---------------------
   public :: mpp_open, mpp_close
@@ -384,6 +391,7 @@ module mpp_io_mod
   type :: axistype
      private
      character(len=128) :: name
+     character(len=128) :: name_bounds
      character(len=128) :: units
      character(len=256) :: longname
      character(len=8)   :: cartesian
@@ -392,6 +400,8 @@ module mpp_io_mod
      integer            :: sense, len          !+/-1, depth or height?
      type(domain1D)     :: domain              !if pointer is associated, it is a distributed data axis
      real, pointer      :: data(:) =>NULL()    !axis values (not used if time axis)
+     real, pointer      :: data_bounds(:) =>NULL()    !axis bounds values
+     integer, pointer   :: idata(:) =>NULL()   !compressed axis valuesi
      integer            :: id, did, type, natt !id is the "variable ID", did is the "dimension ID": 
                                                !netCDF requires 2 IDs for axes
      integer            :: shift               !normally is 0. when domain is symmetry, its value maybe 1.
@@ -432,6 +442,7 @@ module mpp_io_mod
      real(DOUBLE_KIND)  :: time
      logical            :: valid
      logical            :: write_on_this_pe   ! indicate if will write out from this pe
+     logical            :: read_on_this_pe    ! indicate if will read from this pe
      logical            :: io_domain_exist    ! indicate if io_domain exist or not.
      integer            :: id       !variable ID of time axis associated with file (only one time axis per file)
      integer            :: recdimid !dim ID of time axis associated with file (only one time axis per file)
@@ -546,15 +557,94 @@ module mpp_io_mod
   interface mpp_read
      module procedure mpp_read_2ddecomp_r2d
      module procedure mpp_read_2ddecomp_r3d
+     module procedure mpp_read_2ddecomp_r4d
      module procedure mpp_read_r0D
      module procedure mpp_read_r1D
      module procedure mpp_read_r2D
      module procedure mpp_read_r3D
+     module procedure mpp_read_r4D
      module procedure mpp_read_text
      module procedure mpp_read_region_r2D
      module procedure mpp_read_region_r3D
   end interface
 
+!***********************************************************************
+!
+!      public interfaces from mpp_io_read_distributed_ascii.inc
+!
+!***********************************************************************
+! 
+!   
+!     Read from an opened, ascii file, translating data to the desired format
+!   
+!   
+!     These routines are part of the mpp_read family. It is intended to 
+!     provide a general purpose, distributed, list directed read
+!  
+!   
+!  
+!  
+!  
+!  
+!  
+!  
+!  
+!     mpp_read_distributed_ascii
+!     The stripe size must be greater than or equal to 1. The stripe
+!     size does not have to be a common denominator of the number of
+!     MPI ranks.
+!  
+! 
+  interface mpp_read_distributed_ascii
+     module procedure mpp_read_distributed_ascii_r1d
+     module procedure mpp_read_distributed_ascii_i1d
+     module procedure mpp_read_distributed_ascii_a1d
+  end interface
+
+
+!***********************************************************************
+!
+!      public interfaces from mpp_io_read_compressed.h
+!
+!***********************************************************************
+! 
+!   
+!     Read from an opened, sparse data, compressed file (e.g. land_model)
+!   
+!   
+!     These routines are similar to mpp_read except that they are designed
+!     to handle sparse, compressed vectors of data such as from the
+!     land model. Currently, the sparse vector may vary in z. Hence
+!     the need for the rank 2 treatment.
+!  
+!   
+!  
+!  
+!  
+!  
+!  
+!     time_index is an optional argument. It is to be omitted if the
+!     field was defined not to be a function of time. Results are
+!     unpredictable if the argument is supplied for a time- independent
+!     field, or omitted for a time-dependent field.
+!  
+!  
+!     mpp_read_meta must be called prior to calling 
+!     mpp_read_compressed.
+!     Since in general, the vector is distributed across the io-domain
+!     The read expects the io_domain to be defined.
+!  
+! 
+  interface mpp_read_compressed
+     module procedure mpp_read_compressed_r1d
+     module procedure mpp_read_compressed_r2d
+  end interface mpp_read_compressed
+
+
 !***********************************************************************
 !
 !    public interface from mpp_io_write.h
@@ -669,7 +759,9 @@ module mpp_io_mod
      module procedure mpp_write_meta_var
      module procedure mpp_write_meta_scalar_r
      module procedure mpp_write_meta_scalar_i
-     module procedure mpp_write_meta_axis
+     module procedure mpp_write_meta_axis_r1d
+     module procedure mpp_write_meta_axis_i1d
+     module procedure mpp_write_meta_axis_unlimited
      module procedure mpp_write_meta_field
      module procedure mpp_write_meta_global
      module procedure mpp_write_meta_global_scalar_r
@@ -781,6 +873,109 @@ module mpp_io_mod
      module procedure mpp_write_axis
   end interface
 
+
+!***********************************************************************
+! 
+!   
+!     Write to an opened, sparse data, compressed file (e.g. land_model)
+!   
+!   
+!     These routines are similar to mpp_write except that they are 
+!     designed to handle sparse, compressed vectors of data such 
+!     as from the land model. Currently, the sparse vector may vary in z.
+!     Hence the need for the rank 2 treatment.
+!  
+!   
+!  
+!  
+!  
+!  
+!  
+!    nelems is a vector containing the number of elements expected
+!    from each member of the io_domain. It MUST have the same order as
+!    the io_domain pelist.
+!  
+!  
+!    tstamp is an optional argument. It is to
+!    be omitted if the field was defined not to be a function of time.
+!    Results are unpredictable if the argument is supplied for a time-
+!    independent field, or omitted for a time-dependent field. Repeated
+!    writes of a time-independent field are also not recommended. One
+!    time level of one field is written per call. tstamp must be an 8-byte
+!    real, even if the default real type is 4-byte.
+!  
+!  
+!  
+!     mpp_write_meta must be called prior to calling 
+!     mpp_write_compressed.
+!     Since in general, the vector is distributed across the io-domain
+!     The write expects the io_domain to be defined.
+!  
+! 
+  interface mpp_write_compressed
+     module procedure mpp_write_compressed_r1d
+     module procedure mpp_write_compressed_r2d
+  end interface mpp_write_compressed
+
+!***********************************************************************
+! 
+!   
+!     Write to an opened file along the unlimited axis (e.g. icebergs)
+!   
+!   
+!     These routines are similar to mpp_write except that they are 
+!     designed to handle data written along the unlimited axis that
+!     is not time (e.g. icebergs).
+!  
+!   
+!  
+!  
+!  
+!  
+!  
+!    nelems is a vector containing the number of elements expected
+!    from each member of the io_domain. It MUST have the same order as
+!    the io_domain pelist.
+!  
+!  
+!     mpp_write_meta must be called prior to calling 
+!     mpp_write_unlimited_axis.
+!     Since in general, the vector is distributed across the io-domain
+!     The write expects the io_domain to be defined.
+!  
+! 
+  interface mpp_write_unlimited_axis
+     module procedure mpp_write_unlimited_axis_r1d
+  end interface mpp_write_unlimited_axis
+
+
+!***********************************************************************
+! 
+!   
+!     Define an dimension variable
+!   
+!   
+!     Similar to the mpp_write_meta routines, but simply defines the
+!     a dimension variable with the optional attributes
+!  
+!   
+!  
+!  
+!  
+!  
+! 
+  interface mpp_def_dim
+     module procedure mpp_def_dim_nodata
+     module procedure mpp_def_dim_int
+     module procedure mpp_def_dim_real
+  end interface mpp_def_dim
+
 !***********************************************************************
 !
 !            module variables
@@ -820,9 +1015,9 @@ module mpp_io_mod
   integer :: pack_size ! = 1 when compiling with -r8 and = 2 when compiling with -r4.
 
   character(len=128) :: version= &
-       '$Id: mpp_io.F90,v 20.0 2013/12/14 00:23:45 fms Exp $'
+       '$Id$'
   character(len=128) :: tagname= &
-       '$Name: tikal $'
+       '$Name$'
 
 contains
 
diff --git a/src/shared/mpp/mpp_parameter.F90 b/src/shared/mpp/mpp_parameter.F90
index 85f6deffe5..1fc59f5cb1 100644
--- a/src/shared/mpp/mpp_parameter.F90
+++ b/src/shared/mpp/mpp_parameter.F90
@@ -7,7 +7,7 @@ module mpp_parameter_mod
   character(len=128), public :: version= &
        '$Id mpp_parameter.F90 $'
   character(len=128), public :: tagname= &
-       '$Name: tikal $'
+       '$Name$'
 
   !--- public paramters which is used by mpp_mod and its components. 
   !--- All othere modules should import these parameters from mpp_mod. 
@@ -23,14 +23,14 @@ module mpp_parameter_mod
   public :: COMM_TAG_9,  COMM_TAG_10, COMM_TAG_11, COMM_TAG_12
   public :: COMM_TAG_13, COMM_TAG_14, COMM_TAG_15, COMM_TAG_16
   public :: COMM_TAG_17, COMM_TAG_18, COMM_TAG_19, COMM_TAG_20
-
+  public :: MPP_FILL_INT,MPP_FILL_DOUBLE
 
   !--- public paramters which is used by mpp_domains_mod and its components. 
   !--- All othere modules should import these parameters from mpp_domains_mod. 
   public :: GLOBAL_DATA_DOMAIN, CYCLIC_GLOBAL_DOMAIN, BGRID_NE, BGRID_SW, CGRID_NE, CGRID_SW
   public :: DGRID_NE, DGRID_SW, FOLD_WEST_EDGE, FOLD_EAST_EDGE, FOLD_SOUTH_EDGE, FOLD_NORTH_EDGE
   public :: WUPDATE, EUPDATE, SUPDATE, NUPDATE, XUPDATE, YUPDATE, BITWISE_EXACT_SUM, NON_BITWISE_EXACT_SUM
-  public :: MPP_DOMAIN_TIME, WEST, EAST, SOUTH, NORTH, SCALAR_BIT, SCALAR_PAIR
+  public :: MPP_DOMAIN_TIME, WEST, EAST, SOUTH, NORTH, SCALAR_BIT, SCALAR_PAIR, BITWISE_EFP_SUM
   public :: NORTH_EAST, SOUTH_EAST, SOUTH_WEST, NORTH_WEST
   public :: AGRID, GLOBAL, CYCLIC, DOMAIN_ID_BASE, CENTER, CORNER
   public :: MAX_DOMAIN_FIELDS, MAX_TILES
@@ -54,6 +54,10 @@ module mpp_parameter_mod
   integer, parameter :: EVENT_ALLTOALL=6
   integer, parameter :: MPP_CLOCK_SYNC=1, MPP_CLOCK_DETAILED=2
   integer            :: DEFAULT_TAG = 1
+  !--- implimented to centralize _FILL_ values for land_model.F90 into mpp_mod
+  !------- instead of multiple includes of netcdf.inc and manual assignments
+  integer, parameter :: MPP_FILL_INT =-2147483647               !NF_FILL_INT
+  real,    parameter :: MPP_FILL_DOUBLE= 9.9692099683868690e+36 !NF_FILL_DOUBLE
   !--- predefined clock granularities, but you can use any integer
   !--- using CLOCK_LOOP and above may distort coarser-grain measurements
   integer, parameter :: CLOCK_COMPONENT=1      !component level, e.g model, exchange
@@ -96,6 +100,7 @@ module mpp_parameter_mod
   integer(LONG_KIND), parameter :: DOMAIN_ID_BASE=Z'0000000100000000' ! Workaround for 64bit init problem
   integer, parameter :: NON_BITWISE_EXACT_SUM=0
   integer, parameter :: BITWISE_EXACT_SUM=1
+  integer, parameter :: BITWISE_EFP_SUM=2
   integer, parameter :: MPP_DOMAIN_TIME=MPP_DEBUG+1
   integer, parameter :: MAX_DOMAIN_FIELDS=100
   integer, parameter :: MAX_TILES=100
diff --git a/src/shared/mpp/mpp_pset.F90 b/src/shared/mpp/mpp_pset.F90
index 02914d55f0..c197366a49 100644
--- a/src/shared/mpp/mpp_pset.F90
+++ b/src/shared/mpp/mpp_pset.F90
@@ -1,7 +1,7 @@
 ! module within MPP for handling PSETs:
 ! PSET: Persistent Shared-memory Execution Thread
 !
-! AUTHOR: V. Balaji (v.balaji)
+! AUTHOR: V. Balaji (v.balaji@noaa.gov)
 ! DATE: 2006-01-15
 #include 
 #ifdef test_mpp_pset
diff --git a/src/shared/mpp/mpp_utilities.F90 b/src/shared/mpp/mpp_utilities.F90
index e78d348754..001513e1fc 100644
--- a/src/shared/mpp/mpp_utilities.F90
+++ b/src/shared/mpp/mpp_utilities.F90
@@ -1,8 +1,8 @@
 module mpp_utilities_mod
 
 !-----------------------------------------------------------------------
-  character(len=128) :: version = '$Id: mpp_utilities.F90,v 17.0 2009/07/21 03:21:23 fms Exp $'
-  character(len=128) :: tag = '$Name: tikal $'
+  character(len=128) :: version = '$Id$'
+  character(len=128) :: tag = '$Name$'
 !-----------------------------------------------------------------------
 
   public :: mpp_array_global_min_max
@@ -24,9 +24,9 @@ module mpp_utilities_mod
 ! Vectorized using maxloc() and minloc() intrinsic functions by 
 ! Russell.Fiedler@csiro.au (May 2005).
 !
-! Modified by Zhi.Liang (July 2005)
+! Modified by Zhi.Liang@noaa.gov (July 2005)
 !          
-! Modified by Niki.Zadeh (Feb. 2009)
+! Modified by Niki.Zadeh@noaa.gov (Feb. 2009)
 !
 ! 
 !
@@ -73,14 +73,14 @@ subroutine mpp_array_global_min_max(in_array, tmask,isd,jsd,isc,iec,jsc,jec,nk,
   end if
 
   ! use "fudge" to distinguish processors when tracer extreme is independent of processor
-  fudge = 1.0 + 1.e-12*mpp_pe() 
+  fudge = 1.0 + 1.e-12*real(mpp_pe() )
   tmax = tmax*fudge
   tmin = tmin*fudge
   if(tmax == 0.0) then 
-    tmax = tmax + 1.e-12*mpp_pe() 
+    tmax = tmax + 1.e-12*real(mpp_pe() )
   endif 
   if(tmin == 0.0) then 
-    tmin = tmin + 1.e-12*mpp_pe() 
+    tmin = tmin + 1.e-12*real(mpp_pe() )
   endif 
   
 
@@ -101,8 +101,8 @@ subroutine mpp_array_global_min_max(in_array, tmask,isd,jsd,isc,iec,jsc,jec,nk,
   !
   !mpp_max trick
   !-999 on all current PE's
-  xgmax=-999; ygmax=-999; zgmax=-999
-  xgmin=-999; ygmin=-999; zgmin=-999
+  xgmax=-999.; ygmax=-999.; zgmax=-999.
+  xgmin=-999.; ygmin=-999.; zgmin=-999.
 
 
   !except when
diff --git a/src/shared/mpp/test_mpp.F90 b/src/shared/mpp/test_mpp.F90
index ce34fb2e27..f182276499 100644
--- a/src/shared/mpp/test_mpp.F90
+++ b/src/shared/mpp/test_mpp.F90
@@ -14,7 +14,8 @@ program test   !test various aspects of mpp_mod
   use mpp_mod, only : mpp_clock_id, mpp_clock_begin, mpp_clock_end, mpp_sync, mpp_malloc
   use mpp_mod, only : mpp_declare_pelist, mpp_set_current_pelist, mpp_set_stack_size
   use mpp_mod, only : mpp_broadcast, mpp_transmit, mpp_sum, mpp_max, mpp_chksum, ALL_PES
-  use mpp_mod, only : mpp_error, FATAL, mpp_sync_self
+  use mpp_mod, only : mpp_gather, mpp_error, FATAL, mpp_sync_self
+  use mpp_io_mod, only: mpp_io_init, mpp_flush
 #ifdef use_MPI_GSM
   use mpp_mod, only : mpp_gsm_malloc, mpp_gsm_free
 #endif
@@ -36,16 +37,24 @@ program test   !test various aspects of mpp_mod
   real                            :: dt
 
   call mpp_init()
+  call mpp_io_init()
   call mpp_set_stack_size(3145746)
   pe = mpp_pe()
   npes = mpp_npes()
   root = mpp_root_pe()
 
   out_unit = stdout()
+  call test_gather(npes,pe,root,out_unit)
+  call test_gatherV(npes,pe,root,out_unit)
+  call test_gather2DV(npes,pe,root,out_unit)
+
+  if(.false.) then
+
   ! first test broadcast
   call test_broadcast()
 
   call SYSTEM_CLOCK( count_rate=ticks_per_sec )
+
   allocate( a(n), b(n) )
   id = mpp_clock_id( 'Random number' )
   call mpp_clock_begin(id)
@@ -210,6 +219,8 @@ program test   !test various aspects of mpp_mod
   end if
 #endif
 #endif
+  endif  ! if(.false.)
+
   call mpp_exit()
 
 contains
@@ -255,6 +266,247 @@ subroutine test_broadcast()
 
   end subroutine test_broadcast
 
+  subroutine test_gather(npes,pe,root,out_unit)
+     integer, intent(in) :: npes,pe,root,out_unit
+
+     integer :: pelist(npes)
+     integer :: i
+     real :: rdata(npes)
+     real :: val
+
+     if(npes < 3)then
+       write(out_unit,*) "Minimum of 3 ranks required. Not testing gather; too few ranks."
+       return
+     endif
+     write(out_unit,*)
+
+     val = pe
+     rdata = -1.0
+     do i=1,npes
+       pelist(i) = i-1
+     enddo
+
+     call mpp_gather((/val/),rdata)
+     if(pe == root)then 
+       do i=1,npes
+        if(INT(rdata(i)) /= pelist(i))then
+           write(6,*) "Gathered data ",INT(rdata(i)), " NE reference ",pelist(i), "at i=",i
+           call mpp_error(FATAL, "Test gather uniform vector with global pelist failed")
+        endif
+       enddo
+     endif
+
+     call mpp_sync()
+     write(out_unit,*) "Test gather uniform vector with global pelist successful"
+
+     rdata = -1.0
+     if(ANY(pe == pelist(2:npes)))call mpp_gather((/val/),rdata(2:npes),pelist(2:npes))
+     if(pe == pelist(2))then
+       do i=2,npes
+        if(INT(rdata(i)) /= pelist(i))then
+           write(6,*) "Gathered data ",INT(rdata(i)), " NE reference ",pelist(i), "at i=",i
+           call mpp_error(FATAL, "Test gather uniform vector with reduced pelist failed")
+        endif
+       enddo
+     endif
+     call mpp_sync()
+     write(out_unit,*) "Test gather uniform vector with reduced pelist successful"
+
+  end subroutine test_gather
+
+
+  subroutine test_gatherV(npes,pe,root,out_unit)
+  implicit none
+     integer, intent(in) :: npes,pe,root,out_unit
+
+     integer :: pelist(npes),rsize(npes)
+     integer :: i,j,k,dsize,ssize
+     real,allocatable :: sdata(:), rdata(:), ref(:)
+
+     if(npes < 3)then
+       write(out_unit,*) "Minimum of 3 ranks required. Not testing gatherV; too few ranks."
+       return
+     elseif(npes > 9999)then
+       write(out_unit,*) "Maximum of 9999 ranks supported. Not testing gatherV; too many ranks."
+       return
+     endif
+     write(out_unit,*)
+
+     ssize = pe+1
+     allocate(sdata(ssize))
+     do i=1,ssize
+       sdata(i) = pe + 0.0001*i
+     enddo
+     do i=1,npes
+       pelist(i) = i-1
+       rsize(i) = i
+     enddo
+
+     dsize = sum(rsize)
+     allocate(rdata(dsize),ref(dsize))
+     rdata = -1.0
+     k=1
+     do j=1,npes
+       do i=1,rsize(j)
+          ref(k) = pelist(j) + 0.0001*i
+          k = k+1
+     enddo;enddo
+
+     call mpp_gather(sdata,ssize,rdata,rsize)
+ 
+     if(pe == root)then
+       k = 1
+       do j=1,npes
+         do i=1,rsize(j)
+           if(rdata(k) /= ref(k))then
+              write(6,*) "Gathered data ",rdata(k), " NE reference ",ref(k), "at k=",k
+              call mpp_error(FATAL, "Test gatherV global pelist failed")
+           endif
+           k = k+1
+       enddo;enddo
+     endif
+
+     call mpp_sync()
+     write(out_unit,*) "Test gatherV with global pelist successful"
+
+     rdata = -1.0
+     ref(1) = -1.0
+
+     if(ANY(pe == pelist(2:npes)))call mpp_gather(sdata,ssize,rdata(2:),rsize(2:),pelist(2:npes))
+
+     if(pe == pelist(2))then
+       k = 1
+       do j=1,npes
+         do i=1,rsize(j)
+           if(rdata(k) /= ref(k))then
+              write(6,*) "Gathered data ",rdata(k), " NE reference ",ref(k), "at k=",k
+              call mpp_error(FATAL, "Test gatherV with reduced pelist failed")
+           endif
+           k = k+1
+       enddo;enddo
+     endif
+     call mpp_sync()
+
+     write(out_unit,*) "Test gatherV with reduced pelist successful"
+     deallocate(sdata,rdata,ref)
+  end subroutine test_gatherV
+
+subroutine test_gather2DV(npes,pe,root,out_unit)
+  implicit none
+     integer, intent(in) :: npes,pe,root,out_unit
+
+     integer :: pelist(npes),rsize(npes)
+     integer :: pelist2(npes),rsize2(npes)
+     integer :: i,j,k,l,nz,ssize,nelems
+     real,allocatable,dimension(:,:) :: data, cdata, sbuff,rbuff
+     real,allocatable :: ref(:,:)
+     integer, parameter :: KSIZE=10
+
+     real :: sbuff1D(size(sbuff))
+     real :: rbuff1D(size(rbuff))
+     pointer(sptr,sbuff1D); pointer(rptr,rbuff1D)
+
+
+     if(npes < 3)then
+       write(out_unit,*) "Minimum of 3 ranks required. Not testing gather2DV; too few ranks."
+       return
+     elseif(npes > 9999)then
+       write(out_unit,*) "Maximum of 9999 ranks supported. Not testing gather2DV; too many ranks."
+       return
+     endif
+     write(out_unit,*)
+
+     ssize = pe+1
+     allocate(data(ssize,KSIZE))
+     do k=1,KSIZE; do i=1,ssize
+       data(i,k) = 10000.0*k + pe + 0.0001*i
+     enddo; enddo
+     do i=1,npes
+       pelist(i) = i-1
+       rsize(i) = i
+     enddo
+
+     nz = KSIZE
+     nelems = sum(rsize(:))
+
+     allocate(rbuff(nz,nelems)); rbuff = -1.0
+     allocate(ref(nelems,nz),cdata(nelems,nz))
+     ref = 0.0; cdata = 0.0
+     if(pe == root)then
+       do k=1,KSIZE
+       l=1
+       do j=1,npes
+         do i=1,rsize(j)
+            ref(l,k) = 10000.0*k + pelist(j) + 0.0001*i
+            l = l+1
+       enddo; enddo;enddo
+     endif
+     allocate(sbuff(nz,ssize))
+     ! this matrix inversion makes for easy gather to the IO root
+     ! and a clear, concise unpack
+     do j=1,ssize
+       do i=1,nz
+         sbuff(i,j) = data(j,i)
+     enddo; enddo
+
+  !  Note that the gatherV implied here is asymmetric; only root needs to know the vector of recv size
+     sptr = LOC(sbuff); rptr = LOC(rbuff)
+     call mpp_gather(sbuff1D,size(sbuff),rbuff1D,nz*rsize(:))
+
+     if(pe == root)then
+        do j=1,nz
+           do i=1,nelems
+             cdata(i,j) = rbuff(j,i)
+        enddo; enddo
+        do j=1,nz
+           do i=1,nelems
+            if(cdata(i,j) /= ref(i,j))then
+               write(6,*) "Gathered data ",cdata(i,j), " NE reference ",ref(i,j), "at i,j=",i,j
+               call mpp_error(FATAL, "Test gather2DV global pelist failed")
+            endif
+       enddo;enddo
+     endif
+
+     call mpp_sync()
+     write(out_unit,*) "Test gather2DV with global pelist successful"
+
+     do i=1,npes
+       pelist2(i) = pelist(npes-i+1) 
+       rsize2(i) = rsize(npes-i+1)
+     enddo
+
+     rbuff = -1.0
+     ref = 0.0; cdata = 0.0
+     if(pe == pelist2(1))then
+       do k=1,KSIZE
+       l=1
+       do j=1,npes
+         do i=1,rsize2(j)
+            ref(l,k) = 10000.0*k + pelist2(j) + 0.0001*i
+            l = l+1
+       enddo; enddo;enddo
+     endif
+
+     call mpp_gather(sbuff1D,size(sbuff),rbuff1D,nz*rsize2(:),pelist2)
+
+     if(pe == pelist2(1))then
+        do j=1,nz
+           do i=1,nelems
+             cdata(i,j) = rbuff(j,i)
+        enddo; enddo
+        do j=1,nz
+           do i=1,nelems
+            if(cdata(i,j) /= ref(i,j))then
+               write(6,*) "Gathered data ",cdata(i,j), " NE reference ",ref(i,j), "at i,j=",i,j
+               call mpp_error(FATAL, "Test gather2DV with reversed pelist failed")
+            endif
+       enddo;enddo
+     endif
+     call mpp_sync()
+     write(out_unit,*) "Test gather2DV with reversed pelist successful"
+     deallocate(data,sbuff,rbuff,cdata,ref)
+  end subroutine test_gather2DV
+
   subroutine test_shared_pointers(locd,n)
     integer(LONG_KIND), intent(in) :: locd
     integer :: n
diff --git a/src/shared/mpp/test_mpp_domains.F90 b/src/shared/mpp/test_mpp_domains.F90
index 4a4adde0bf..6e9ad7706e 100644
--- a/src/shared/mpp/test_mpp_domains.F90
+++ b/src/shared/mpp/test_mpp_domains.F90
@@ -10,7 +10,7 @@ program test
   use mpp_domains_mod, only : GLOBAL_DATA_DOMAIN, BITWISE_EXACT_SUM, BGRID_NE, CGRID_NE, DGRID_NE
   use mpp_domains_mod, only : FOLD_SOUTH_EDGE, FOLD_NORTH_EDGE, FOLD_WEST_EDGE, FOLD_EAST_EDGE
   use mpp_domains_mod, only : MPP_DOMAIN_TIME, CYCLIC_GLOBAL_DOMAIN, NUPDATE,EUPDATE, XUPDATE, YUPDATE, SCALAR_PAIR
-  use mpp_domains_mod, only : domain1D, domain2D, DomainCommunicator2D
+  use mpp_domains_mod, only : domain1D, domain2D, DomainCommunicator2D, BITWISE_EFP_SUM
   use mpp_domains_mod, only : mpp_get_compute_domain, mpp_get_data_domain, mpp_domains_set_stack_size
   use mpp_domains_mod, only : mpp_global_field, mpp_global_sum, mpp_global_max, mpp_global_min
   use mpp_domains_mod, only : mpp_domains_init, mpp_domains_exit, mpp_broadcast_domain
@@ -19,13 +19,16 @@ program test
   use mpp_domains_mod, only : mpp_get_neighbor_pe, mpp_define_mosaic, mpp_nullify_domain_list
   use mpp_domains_mod, only : NORTH, NORTH_EAST, EAST, SOUTH_EAST, CORNER, CENTER
   use mpp_domains_mod, only : SOUTH, SOUTH_WEST, WEST, NORTH_WEST, mpp_define_mosaic_pelist
-  use mpp_domains_mod, only : mpp_get_refine_overlap_number, mpp_get_mosaic_refine_overlap
   use mpp_domains_mod, only : mpp_get_global_domain, ZERO, NINETY, MINUS_NINETY
   use mpp_domains_mod, only : mpp_get_boundary, mpp_start_update_domains, mpp_complete_update_domains
   use mpp_domains_mod, only : mpp_define_nest_domains, nest_domain_type
   use mpp_domains_mod, only : mpp_get_C2F_index, mpp_update_nest_fine
   use mpp_domains_mod, only : mpp_get_F2C_index, mpp_update_nest_coarse
   use mpp_domains_mod, only : mpp_get_domain_shift, EDGEUPDATE, mpp_deallocate_domain
+  use mpp_domains_mod, only : mpp_group_update_type, mpp_create_group_update
+  use mpp_domains_mod, only : mpp_do_group_update, mpp_clear_group_update
+  use mpp_domains_mod, only : mpp_start_group_update, mpp_complete_group_update
+  use mpp_domains_mod, only : WUPDATE, SUPDATE
   use mpp_memutils_mod, only : mpp_memuse_begin, mpp_memuse_end
 
   implicit none
@@ -47,12 +50,15 @@ program test
   logical :: test_interface = .true.
   logical :: test_nest_domain = .false.
   logical :: test_edge_update = .false.
+  logical :: test_group = .false.
   logical :: test_cubic_grid_redistribute = .false.
   logical :: check_parallel = .FALSE.  ! when check_parallel set to false,
   logical :: test_get_nbr = .FALSE.
   logical :: test_boundary = .false.
+  logical :: test_global_sum = .false.
   integer :: ensemble_size
   integer :: layout_cubic(2) = (/0,0/)
+  integer :: layout_tripolar(2) = (/0,0/)
   integer :: layout_ensemble(2) = (/0,0/)
   logical :: do_sleep = .false.
   integer :: num_iter = 1
@@ -77,7 +83,8 @@ program test
                                jstart_fine, jend_fine, istart_coarse, iend_coarse, jstart_coarse, &
                                jend_coarse, extra_halo, npes_fine, npes_coarse, mix_2D_3D, test_get_nbr, &
                                test_edge_update, test_cubic_grid_redistribute, ensemble_size, &
-                               layout_cubic, layout_ensemble, nthreads, test_boundary
+                               layout_cubic, layout_ensemble, nthreads, test_boundary, &
+                               layout_tripolar, test_group, test_global_sum
   integer :: i, j, k
   integer :: layout(2)
   integer :: id
@@ -168,16 +175,26 @@ program test
       call update_domains_performance('Cubic-Grid')
   endif
 
+  if( test_global_sum ) then
+      call test_mpp_global_sum('Folded-north')
+  endif
+
   if( test_cubic_grid_redistribute ) then
      call cubic_grid_redistribute()
   endif
 
   if(test_boundary) then
+      call test_get_boundary('torus')
       call test_get_boundary('Four-Tile')
       call test_get_boundary('Cubic-Grid')
       call test_get_boundary('Folded-north')
   endif
 
+  if( test_group) then
+     call test_group_update( 'Folded-north' )
+     call test_group_update( 'Cubic-Grid' )
+  endif
+
   if( test_interface ) then
       call test_modify_domain()
 !!$      call test_cyclic_offset('x_cyclic_offset')
@@ -193,9 +210,6 @@ program test
       call test_uniform_mosaic('Four-Tile')
       call test_uniform_mosaic('Cubic-Grid') ! 6 tiles.
       call test_nonuniform_mosaic('Five-Tile')
-      call test_refined_mosaic('Refined-Four-Tile')
-      call test_refined_mosaic('Refined-Symmetric-Four-Tile')
-      call test_refined_mosaic('Refined-Cubic-Grid')
 
       call test_halo_update( 'Simple' ) !includes global field, global sum tests
       call test_halo_update( 'Cyclic' )
@@ -686,7 +700,7 @@ subroutine cubic_grid_redistribute
               enddo
            enddo
         enddo
-       write(mesg,'(a,i)') "cubic_grid redistribute from ensemble", n
+       write(mesg,'(a,i4)') "cubic_grid redistribute from ensemble", n
         call compare_checksums( x(isc:iec,jsc:jec,:,n), y(isc:iec,jsc:jec,:), trim(mesg) )
      enddo
     
@@ -718,7 +732,7 @@ subroutine cubic_grid_redistribute
               enddo
            enddo
         endif
-        write(mesg,'(a,i)') "cubic_grid redistribute to ensemble", n
+        write(mesg,'(a,i4)') "cubic_grid redistribute to ensemble", n
         call compare_checksums( x_ens(isc_ens:iec_ens,jsc_ens:jec_ens,:), y(isc_ens:iec_ens,jsc_ens:jec_ens,:), trim(mesg) )
      enddo
 
@@ -2147,1313 +2161,1252 @@ subroutine update_domains_performance( type )
 
   end subroutine update_domains_performance
 
-  !#################################################################################
 
-  subroutine fill_halo_zero(data, whalo, ehalo, shalo, nhalo, xshift, yshift, isc, iec, jsc, jec, isd, ied, jsd, jed)
-    integer,                         intent(in) :: isc, iec, jsc, jec, isd, ied, jsd, jed
-    integer,                         intent(in) :: whalo, ehalo, shalo, nhalo, xshift, yshift
-    real, dimension(isd:,jsd:,:), intent(inout) :: data
+  !###############################################################
+  subroutine test_mpp_global_sum( type )
+    character(len=*), intent(in) :: type
 
-    if(whalo >=0) then
-       data(iec+ehalo+1+xshift:ied+xshift,jsd:jed+yshift,:) = 0
-       data(isd:isc-whalo-1,jsd:jed+yshift,:) = 0
-    else
-       data(iec+1+xshift:iec-ehalo+xshift,jsc+shalo:jec-nhalo+yshift,:) = 0
-       data(isc+whalo:isc-1,jsc+shalo:jec-nhalo+yshift,:) = 0
-    end if
+    type(domain2D) :: domain
+    integer        :: num_contact, ntiles, npes_per_tile
+    integer        :: i, j, k, l, n, shift
+    integer        :: isc, iec, jsc, jec, isd, ied, jsd, jed
+    integer        :: ism, iem, jsm, jem
 
-    if(shalo>=0) then
-       data(isd:ied+xshift, jec+nhalo+1+yshift:jed+yshift,:) = 0
-       data(isd:ied+xshift, jsd:jsc-shalo-1,:) = 0
-    else
-       data(isc+whalo:iec-ehalo+xshift,jec+1+yshift:jec-nhalo+yshift,:) = 0
-       data(isc+whalo:iec-ehalo+xshift,jsc+shalo:jsc-1,:) = 0
-    end if
+    integer, allocatable, dimension(:)       :: pe_start, pe_end, tile1, tile2
+    integer, allocatable, dimension(:)       :: istart1, iend1, jstart1, jend1
+    integer, allocatable, dimension(:)       :: istart2, iend2, jstart2, jend2
+    integer, allocatable, dimension(:,:)     :: layout2D, global_indices
+    real,    allocatable, dimension(:,:,:)   :: data_3D
+    real,    allocatable, dimension(:,:)     :: data_2D
 
-  end subroutine fill_halo_zero
+    integer(kind=8)    :: mold
+    logical            :: folded_north, cubic_grid 
+    character(len=3)   :: text
+    integer            :: nx_save, ny_save
+    integer            :: id1, id2, id3, id4
+    real               :: gsum1, gsum2, gsum3, gsum4
 
-  !##############################################################################
-  ! this routine fill the halo points for the regular mosaic. 
-  subroutine fill_regular_mosaic_halo(data, data_all, te, tse, ts, tsw, tw, tnw, tn, tne)
-    real, dimension(1-whalo:,1-shalo:,:), intent(inout) :: data
-    real, dimension(:,:,:,:),             intent(in)    :: data_all
-    integer,                              intent(in)    :: te, tse, ts, tsw, tw, tnw, tn, tne
+    folded_north       = .false.
+    cubic_grid         = .false.
 
-       data(nx+1:nx+ehalo, 1:ny,          :) = data_all(1:ehalo,       1:ny,          :, te) ! east
-       data(1:nx,          1-shalo:0,     :) = data_all(1:nx,          ny-shalo+1:ny, :, ts) ! south 
-       data(1-whalo:0,     1:ny,          :) = data_all(nx-whalo+1:nx, 1:ny,          :, tw) ! west
-       data(1:nx,          ny+1:ny+nhalo, :) = data_all(1:nx,          1:nhalo,       :, tn) ! north  
-       data(nx+1:nx+ehalo, 1-shalo:0,     :) = data_all(1:ehalo,       ny-shalo+1:ny, :,tse) ! southeast
-       data(1-whalo:0,     1-shalo:0,     :) = data_all(nx-whalo+1:nx, ny-shalo+1:ny, :,tsw) ! southwest
-       data(nx+1:nx+ehalo, ny+1:ny+nhalo, :) = data_all(1:ehalo,       1:nhalo,       :,tnw) ! northeast
-       data(1-whalo:0,     ny+1:ny+nhalo, :) = data_all(nx-whalo+1:nx, 1:nhalo,       :,tne) ! northwest    
+    nx_save = nx
+    ny_save = ny
+    !--- check the type
+    select case(type)
+    case ( 'Folded-north' )
+       ntiles = 1
+       shift = 0
+       num_contact = 2
+       folded_north = .true.
+       npes_per_tile = npes
+       if(layout_tripolar(1)*layout_tripolar(2) == npes ) then
+          layout = layout_tripolar
+       else
+          call mpp_define_layout( (/1,nx,1,ny/), npes_per_tile, layout )
+       endif      
+    case ( 'Cubic-Grid' )
+       if( nx_cubic == 0 ) then
+          call mpp_error(NOTE,'test_group_update: for Cubic_grid mosaic, nx_cubic is zero, '//&
+               'No test is done for Cubic-Grid mosaic. ' )
+          return
+       endif
+       if( nx_cubic .NE. ny_cubic ) then
+          call mpp_error(NOTE,'test_group_update: for Cubic_grid mosaic, nx_cubic does not equal ny_cubic, '//&
+               'No test is done for Cubic-Grid mosaic. ' )
+          return
+       endif
+       shift = 1
+       nx = nx_cubic
+       ny = ny_cubic
+       ntiles = 6
+       num_contact = 12
+       cubic_grid = .true.
+       if( mod(npes, ntiles) == 0 ) then
+          npes_per_tile = npes/ntiles
+          write(outunit,*)'NOTE from test_mpp_global_sum ==> For Mosaic "', trim(type), &
+               '", each tile will be distributed over ', npes_per_tile, ' processors.'
+       else
+          call mpp_error(NOTE,'test_group_update: npes should be multiple of ntiles No test is done for '//trim(type))
+          return
+       endif
+       if(layout_cubic(1)*layout_cubic(2) == npes_per_tile) then
+          layout = layout_cubic
+       else
+          call mpp_define_layout( (/1,nx,1,ny/), npes_per_tile, layout )
+       endif
+    case default
+       call mpp_error(FATAL, 'test_mpp_global_sum: no such test: '//type)
+    end select
 
+    allocate(layout2D(2,ntiles), global_indices(4,ntiles), pe_start(ntiles), pe_end(ntiles) )
+    do n = 1, ntiles
+       pe_start(n) = (n-1)*npes_per_tile
+       pe_end(n)   = n*npes_per_tile-1
+    end do
 
+    do n = 1, ntiles
+       global_indices(:,n) = (/1,nx,1,ny/)
+       layout2D(:,n)         = layout
+    end do
 
-  end subroutine fill_regular_mosaic_halo
+    allocate(tile1(num_contact), tile2(num_contact) )
+    allocate(istart1(num_contact), iend1(num_contact), jstart1(num_contact), jend1(num_contact) ) 
+    allocate(istart2(num_contact), iend2(num_contact), jstart2(num_contact), jend2(num_contact) ) 
 
-  !################################################################################
-  subroutine fill_folded_north_halo(data, ioff, joff, ishift, jshift, sign)
-    real, dimension(1-whalo:,1-shalo:,:), intent(inout) :: data
-    integer,                              intent(in   ) :: ioff, joff, ishift, jshift, sign    
-    integer  :: nxp, nyp, m1, m2
+    !--- define domain
+    if(folded_north) then
+       !--- Contact line 1, between tile 1 (EAST) and tile 1 (WEST)  --- cyclic
+       tile1(1) = 1; tile2(1) = 1
+       istart1(1) = nx; iend1(1) = nx; jstart1(1) = 1;  jend1(1) = ny
+       istart2(1) = 1;  iend2(1) = 1;  jstart2(1) = 1;  jend2(1) = ny
+       !--- Contact line 2, between tile 1 (NORTH) and tile 1 (NORTH)  --- folded-north-edge
+       tile1(2) = 1; tile2(2) = 1
+       istart1(2) = 1;  iend1(2) = nx/2;   jstart1(2) = ny;  jend1(2) = ny
+       istart2(2) = nx; iend2(2) = nx/2+1; jstart2(2) = ny;  jend2(2) = ny
+       call mpp_define_mosaic(global_indices, layout2D, domain, ntiles, num_contact, tile1, tile2, &
+            istart1, iend1, jstart1, jend1, istart2, iend2, jstart2, jend2,      &
+            pe_start, pe_end, whalo=whalo, ehalo=ehalo, shalo=shalo, nhalo=nhalo, &
+            name = type, symmetry = .false.  )
+    else if( cubic_grid ) then
+       call define_cubic_mosaic(type, domain, (/nx,nx,nx,nx,nx,nx/), (/ny,ny,ny,ny,ny,ny/), &
+            global_indices, layout2D, pe_start, pe_end )
+    endif
 
-    nxp = nx+ishift
-    nyp = ny+jshift
-    m1 = ishift - ioff
-    m2 = 2*ishift - ioff
+    !--- setup data
+    call mpp_get_compute_domain( domain, isc, iec, jsc, jec )
+    call mpp_get_data_domain   ( domain, isd, ied, jsd, jed )
 
-    data(1-whalo:0,                  1:nyp,:) =      data(nx-whalo+1:nx,        1:ny+jshift,:) ! west
-    data(nx+1:nx+ehalo+ishift,       1:nyp,:) =      data(1:ehalo+ishift,       1:ny+jshift,:) ! east
-    if(m1 .GE. 1-whalo) data(1-whalo:m1,  nyp+1:nyp+nhalo,:) = sign*data(whalo+m2:1+ishift:-1, nyp-joff:nyp-nhalo-joff+1:-1,:)
-    data(m1+1:nx+m2,       nyp+1:nyp+nhalo,:) = sign*data(nx+ishift:1:-1,       nyp-joff:nyp-nhalo-joff+1:-1,:)
-    data(nx+m2+1:nxp+ehalo,nyp+1:nyp+nhalo,:) = sign*data(nx:nx-ehalo+m1+1:-1,  nyp-joff:nyp-nhalo-joff+1:-1,:)
+    allocate(data_2d(isd:ied,jsd:jed))
+    allocate(data_3d(isd:ied,jsd:jed,nz))
 
-  end subroutine fill_folded_north_halo
+    do k = 1, nz
+       do j = jsd, jed
+          do i = isd, ied
+             data_3d(i,j,k) =  k*1e3 + i + j*1e-3      
+          enddo
+       enddo
+    enddo
 
-  !################################################################################
-  subroutine fill_folded_south_halo(data, ioff, joff, ishift, jshift, sign)
-    real, dimension(1-whalo:,1-shalo:,:), intent(inout) :: data
-    integer,                              intent(in   ) :: ioff, joff, ishift, jshift, sign    
-    integer  :: nxp, nyp, m1, m2
+    do j = jsd, jed
+       do i = isd, ied
+          data_2d(i,j) =  i*1e3 + j*1e-3      
+       enddo
+    enddo
 
-    nxp = nx+ishift
-    nyp = ny+jshift
-    m1 = ishift - ioff
-    m2 = 2*ishift - ioff
+    id1 = mpp_clock_id( type//' bitwise sum 3D', flags=MPP_CLOCK_SYNC )
+    id2 = mpp_clock_id( type//' EFP sum 3D', flags=MPP_CLOCK_SYNC )
+    id3 = mpp_clock_id( type//' EFP sum 3D check', flags=MPP_CLOCK_SYNC )
+    id4 = mpp_clock_id( type//' non-bitwise sum 3D', flags=MPP_CLOCK_SYNC )
+ 
+    call mpp_clock_begin(id1)
+    do n = 1, num_iter
+       gsum1 = mpp_global_sum(domain, data_3d, flags=BITWISE_EXACT_SUM)
+    enddo
+    call mpp_clock_end(id1)
 
+    call mpp_clock_begin(id2)
+    do n = 1, num_iter
+       gsum2 = mpp_global_sum(domain, data_3d, flags=BITWISE_EFP_SUM)
+    enddo
+    call mpp_clock_end(id2)
 
-    data(1-whalo:0,                  1:nyp,:) =      data(nx-whalo+1:nx,        1:nyp,:) ! west
-    data(nx+1:nx+ehalo+ishift,       1:nyp,:) =      data(1:ehalo+ishift,       1:nyp,:) ! east
-    if(m1 .GE. 1-whalo)data(1-whalo:m1, 1-shalo:0,:) = sign*data(whalo+m2:1+ishift:-1, shalo+jshift:1+jshift:-1,:)
-    data(m1+1:nx+m2,       1-shalo:0,:) = sign*data(nxp:1:-1,             shalo+jshift:1+jshift:-1,:)
-    data(nx+m2+1:nxp+ehalo,1-shalo:0,:) = sign*data(nx:nx-ehalo+m1+1:-1,  shalo+jshift:1+jshift:-1,:)
+    call mpp_clock_begin(id3)
+    do n = 1, num_iter
+       gsum3 = mpp_global_sum(domain, data_3d, flags=BITWISE_EFP_SUM, overflow_check=.true. )
+    enddo
+    call mpp_clock_end(id3)
 
-  end subroutine fill_folded_south_halo
+    call mpp_clock_begin(id4)
+    do n = 1, num_iter
+       gsum4= mpp_global_sum(domain, data_3d)
+    enddo
+    call mpp_clock_end(id4)
+ 
+    write(outunit, *) " ********************************************************************************"
+    write(outunit, *) " global sum for "//type//' bitwise exact sum 3D = ', gsum1
+    write(outunit, *) " global sum for "//type//' bitwise EFP sum 3D = ', gsum2
+    write(outunit, *) " global sum for "//type//' bitwise EFP sum 3D with overflow_check = ', gsum3
+    write(outunit, *) " global sum for "//type//' non-bitwise sum 3D = ', gsum4
+    write(outunit, *) " "
+    write(outunit, *) " chksum for "//type//' bitwise exact sum 3D = ', transfer(gsum1, mold)
+    write(outunit, *) " chksum for "//type//' bitwise EFP sum 3D = ', transfer(gsum2, mold)
+    write(outunit, *) " chksum for "//type//' bitwise EFP sum 3D with overflow_check = ', transfer(gsum3, mold)
+    write(outunit, *) " chksum for "//type//' non-bitwise sum 3D = ', transfer(gsum4, mold)
+    write(outunit, *) " ********************************************************************************"
+
+    id1 = mpp_clock_id( type//' bitwise sum 2D', flags=MPP_CLOCK_SYNC )
+    id2 = mpp_clock_id( type//' EFP sum 2D', flags=MPP_CLOCK_SYNC )
+    id3 = mpp_clock_id( type//' EFP sum 2D check', flags=MPP_CLOCK_SYNC )
+    id4 = mpp_clock_id( type//' non-bitwise sum 2D', flags=MPP_CLOCK_SYNC )
 
-  !################################################################################
-  subroutine fill_folded_west_halo(data, ioff, joff, ishift, jshift, sign)
-    real, dimension(1-whalo:,1-shalo:,:), intent(inout) :: data
-    integer,                              intent(in   ) :: ioff, joff, ishift, jshift, sign    
-    integer  :: nxp, nyp, m1, m2
+    call mpp_clock_begin(id1)
+    do n = 1, num_iter
+       gsum1 = mpp_global_sum(domain, data_2d, flags=BITWISE_EXACT_SUM)
+    enddo
+    call mpp_clock_end(id1)
 
-    nxp = nx+ishift
-    nyp = ny+jshift
-    m1 = jshift - joff
-    m2 = 2*jshift - joff
+    call mpp_clock_begin(id2)
+    do n = 1, num_iter
+       gsum2 = mpp_global_sum(domain, data_2d, flags=BITWISE_EFP_SUM)
+    enddo
+    call mpp_clock_end(id2)
 
-    data(1:nxp, 1-shalo:0, :)      = data(1:nxp, ny-shalo+1:ny, :) ! south
-    data(1:nxp, ny+1:nyp+nhalo, :) = data(1:nxp, 1:nhalo+jshift,:) ! north
-    if(m1 .GE. 1-shalo) data(1-whalo:0, 1-shalo:m1, :) = sign*data(whalo+ishift:1+ishift:-1, shalo+m2:1+jshift:-1,:)
-    data(1-whalo:0, m1+1:ny+m2, :) = sign*data(whalo+ishift:1+ishift:-1, nyp:1:-1, :)
-    data(1-whalo:0, ny+m2+1:nyp+nhalo,:) = sign*data(whalo+ishift:1+ishift:-1, ny:ny-nhalo+m1+1:-1,:)
+    call mpp_clock_begin(id3)
+    do n = 1, num_iter
+       gsum3 = mpp_global_sum(domain, data_2d, flags=BITWISE_EFP_SUM, overflow_check=.true. )
+    enddo
+    call mpp_clock_end(id3)
 
-  end subroutine fill_folded_west_halo
+    call mpp_clock_begin(id4)
+    do n = 1, num_iter
+       gsum4= mpp_global_sum(domain, data_2d)
+    enddo
+    call mpp_clock_end(id4)
+  
+    write(outunit, *) " ********************************************************************************"
+    write(outunit, *) " global sum for "//type//' bitwise exact sum 2D = ', gsum1
+    write(outunit, *) " global sum for "//type//' bitwise EFP sum 2D = ', gsum2
+    write(outunit, *) " global sum for "//type//' bitwise EFP sum 2D with overflow_check = ', gsum3
+    write(outunit, *) " global sum for "//type//' non-bitwise sum 2D = ', gsum4
+    write(outunit, *) " "
+    write(outunit, *) " chksum for "//type//' bitwise exact sum 2D = ', transfer(gsum1, mold)
+    write(outunit, *) " chksum for "//type//' bitwise EFP sum 2D = ', transfer(gsum2, mold)
+    write(outunit, *) " chksum for "//type//' bitwise EFP sum 2D with overflow_check = ', transfer(gsum3, mold)
+    write(outunit, *) " chksum for "//type//' non-bitwise sum 2D = ', transfer(gsum4, mold)
+    write(outunit, *) " ********************************************************************************"
 
-  !################################################################################
-  subroutine fill_folded_east_halo(data, ioff, joff, ishift, jshift, sign)
-    real, dimension(1-whalo:,1-shalo:,:), intent(inout) :: data
-    integer,                              intent(in   ) :: ioff, joff, ishift, jshift, sign    
-    integer  :: nxp, nyp, m1, m2
 
-    nxp = nx+ishift
-    nyp = ny+jshift
-    m1 = jshift - joff
-    m2 = 2*jshift - joff
 
-    data(1:nxp, 1-shalo:0, :)      = data(1:nxp, ny-shalo+1:ny, :) ! south
-    data(1:nxp, ny+1:nyp+nhalo, :) = data(1:nxp, 1:nhalo+jshift,:) ! north
-    if(m1 .GE. 1-shalo) data(nxp+1:nxp+ehalo, 1-shalo:m1, :) = sign*data(nxp-ioff:nxp-ehalo-ioff+1:-1, shalo+m2:1+jshift:-1,:)
-    data(nxp+1:nxp+ehalo, m1+1:ny+m2, :) = sign*data(nxp-ioff:nxp-ehalo-ioff+1:-1, nyp:1:-1, :)
-    data(nxp+1:nxp+ehalo, ny+m2+1:nyp+nhalo,:) = sign*data(nxp-ioff:nxp-ehalo-ioff+1:-1, ny:ny-nhalo+m1+1:-1,:)
+    nx = nx_save
+    ny = ny_save
 
-  end subroutine fill_folded_east_halo
+  end subroutine test_mpp_global_sum
 
-  !################################################################################
-  subroutine fill_four_tile_bound(data_all, is, ie, js, je, ioff, joff, tile, &
-                                   ebound, sbound, wbound, nbound )
-    real, dimension(:,:,:,:),       intent(in)    :: data_all
-    integer,                        intent(in)    :: is, ie, js, je
-    integer,                        intent(in)    :: tile, ioff, joff
-    real, dimension(:,:), optional, intent(inout) :: ebound, sbound, wbound, nbound
-    integer                                       :: tw, te, ts, tn
+  !###############################################################
+  subroutine test_group_update( type )
+    character(len=*), intent(in) :: type
 
-    if(tile == 1 .OR. tile == 3) te = tile + 1
-    if(tile == 2 .OR. tile == 4) te = tile - 1
-    if(tile == 1 .OR. tile == 2) ts = tile + 2
-    if(tile == 3 .OR. tile == 4) ts = tile - 2
-    tw = te;   tn = ts
-    if(present(ebound)) then
-       if( ie == nx ) then
-          ebound(:,:) = data_all(1, js:je+joff, :, te)
-       else
-          ebound(:,:) = data_all(ie+ioff, js:je+joff, :, tile)
-       end if
-    end if
+    type(domain2D) :: domain
+    integer        :: num_contact, ntiles, npes_per_tile
+    integer        :: i, j, k, l, n, shift
+    integer        :: isc, iec, jsc, jec, isd, ied, jsd, jed
+    integer        :: ism, iem, jsm, jem
 
-    if(present(wbound)) then
-       if( is == 1 ) then
-          wbound(:,:) = data_all(nx+ioff, js:je+joff, :, tw)
-       else
-          wbound(:,:) = data_all(is, js:je+joff, :, tile)
-       end if
-    end if
+    integer, allocatable, dimension(:)       :: pe_start, pe_end, tile1, tile2
+    integer, allocatable, dimension(:)       :: istart1, iend1, jstart1, jend1
+    integer, allocatable, dimension(:)       :: istart2, iend2, jstart2, jend2
+    integer, allocatable, dimension(:,:)     :: layout2D, global_indices
+    real,    allocatable, dimension(:,:,:,:) :: x1, y1, x2, y2
+    real,    allocatable, dimension(:,:,:,:) :: a1, a2
+    real,    allocatable, dimension(:,:,:)   :: base
+    integer            :: id1, id2, id3   
+    logical            :: folded_north
+    logical            :: cubic_grid
+    character(len=3)   :: text
+    integer            :: nx_save, ny_save
+    type(mpp_group_update_type) :: group_update
+    type(mpp_group_update_type), allocatable :: update_list(:)
 
-    if(present(sbound)) then
-       if( js == 1 ) then
-          sbound(:,:) = data_all(is:ie+ioff, ny+joff, :, ts)
-       else
-          sbound(:,:) = data_all(is:ie+ioff, js, :, tile)
-       end if
-    end if
+    folded_north       = .false.
+    cubic_grid         = .false.
 
-    if(present(nbound)) then
-       if( je == ny ) then
-          nbound(:,:) = data_all(is:ie+ioff, 1, :, tn)
+    nx_save = nx
+    ny_save = ny
+    !--- check the type
+    select case(type)
+    case ( 'Folded-north' )
+       ntiles = 1
+       shift = 0
+       num_contact = 2
+       folded_north = .true.
+       npes_per_tile = npes
+       if(layout_tripolar(1)*layout_tripolar(2) == npes ) then
+          layout = layout_tripolar
        else
-          nbound(:,:) = data_all(is:ie+ioff, je+joff, :, tile)
-       end if
-    end if
+          call mpp_define_layout( (/1,nx,1,ny/), npes_per_tile, layout )
+       endif      
+    case ( 'Cubic-Grid' )
+       if( nx_cubic == 0 ) then
+          call mpp_error(NOTE,'test_group_update: for Cubic_grid mosaic, nx_cubic is zero, '//&
+               'No test is done for Cubic-Grid mosaic. ' )
+          return
+       endif
+       if( nx_cubic .NE. ny_cubic ) then
+          call mpp_error(NOTE,'test_group_update: for Cubic_grid mosaic, nx_cubic does not equal ny_cubic, '//&
+               'No test is done for Cubic-Grid mosaic. ' )
+          return
+       endif
+       shift = 1
+       nx = nx_cubic
+       ny = ny_cubic
+       ntiles = 6
+       num_contact = 12
+       cubic_grid = .true.
+       if( mod(npes, ntiles) == 0 ) then
+          npes_per_tile = npes/ntiles
+          write(outunit,*)'NOTE from update_domains_performance ==> For Mosaic "', trim(type), &
+               '", each tile will be distributed over ', npes_per_tile, ' processors.'
+       else
+          call mpp_error(NOTE,'test_group_update: npes should be multiple of ntiles No test is done for '//trim(type))
+          return
+       endif
+       if(layout_cubic(1)*layout_cubic(2) == npes_per_tile) then
+          layout = layout_cubic
+       else
+          call mpp_define_layout( (/1,nx,1,ny/), npes_per_tile, layout )
+       endif
+    case default
+       call mpp_error(FATAL, 'test_group_update: no such test: '//type)
+    end select
 
-    return
+    allocate(layout2D(2,ntiles), global_indices(4,ntiles), pe_start(ntiles), pe_end(ntiles) )
+    do n = 1, ntiles
+       pe_start(n) = (n-1)*npes_per_tile
+       pe_end(n)   = n*npes_per_tile-1
+    end do
 
-  end subroutine fill_four_tile_bound
+    do n = 1, ntiles
+       global_indices(:,n) = (/1,nx,1,ny/)
+       layout2D(:,n)         = layout
+    end do
 
+    allocate(tile1(num_contact), tile2(num_contact) )
+    allocate(istart1(num_contact), iend1(num_contact), jstart1(num_contact), jend1(num_contact) ) 
+    allocate(istart2(num_contact), iend2(num_contact), jstart2(num_contact), jend2(num_contact) ) 
 
-  !################################################################################
-  subroutine fill_folded_north_bound(data_all, is, ie, js, je, ioff, joff, tile, &
-                                     sbound, wbound)
-    real, dimension(:,:,:),       intent(in)    :: data_all
-    integer,                        intent(in)    :: is, ie, js, je
-    integer,                        intent(in)    :: tile, ioff, joff
-    real, dimension(:,:), optional, intent(inout) :: sbound, wbound
-    integer                                       :: tw, te, ts, tn
+    !--- define domain
+    if(folded_north) then
+       !--- Contact line 1, between tile 1 (EAST) and tile 1 (WEST)  --- cyclic
+       tile1(1) = 1; tile2(1) = 1
+       istart1(1) = nx; iend1(1) = nx; jstart1(1) = 1;  jend1(1) = ny
+       istart2(1) = 1;  iend2(1) = 1;  jstart2(1) = 1;  jend2(1) = ny
+       !--- Contact line 2, between tile 1 (NORTH) and tile 1 (NORTH)  --- folded-north-edge
+       tile1(2) = 1; tile2(2) = 1
+       istart1(2) = 1;  iend1(2) = nx/2;   jstart1(2) = ny;  jend1(2) = ny
+       istart2(2) = nx; iend2(2) = nx/2+1; jstart2(2) = ny;  jend2(2) = ny
+       call mpp_define_mosaic(global_indices, layout2D, domain, ntiles, num_contact, tile1, tile2, &
+            istart1, iend1, jstart1, jend1, istart2, iend2, jstart2, jend2,      &
+            pe_start, pe_end, whalo=whalo, ehalo=ehalo, shalo=shalo, nhalo=nhalo, &
+            name = type, symmetry = .false.  )
+    else if( cubic_grid ) then
+       call define_cubic_mosaic(type, domain, (/nx,nx,nx,nx,nx,nx/), (/ny,ny,ny,ny,ny,ny/), &
+            global_indices, layout2D, pe_start, pe_end )
+    endif
 
-    if(tile .NE. 1) call mpp_error(FATAL, "fill_folded_north_bound: tile must be 1")
+    !--- setup data
+    call mpp_get_compute_domain( domain, isc, iec, jsc, jec )
+    call mpp_get_data_domain   ( domain, isd, ied, jsd, jed )
+    call mpp_get_memory_domain   ( domain, ism, iem, jsm, jem )
 
-    if(present(wbound)) then
-       if( is == 1 ) then
-          wbound(:,:) = data_all(nx+ioff, js:je+joff, :)
-       else
-          wbound(:,:) = data_all(is, js:je+joff, :)
-       end if
-    end if
+    if(num_fields<1) then
+       call mpp_error(FATAL, "test_mpp_domains: num_fields must be a positive integer")
+    endif
 
-    if(present(sbound)) then
-       if( js == 1 ) then
-          sbound(:,:) = 0
-       else
-          sbound(:,:) = data_all(is:ie+ioff, js, :)
-       end if
-    end if
+    allocate(update_list(num_fields))
+ 
+    id1 = mpp_clock_id( type//' group 2D', flags=MPP_CLOCK_SYNC )
+    id2 = mpp_clock_id( type//' non-group 2D', flags=MPP_CLOCK_SYNC )
+    id3 = mpp_clock_id( type//' non-block group 2D', flags=MPP_CLOCK_SYNC )
+
+    allocate( a1(ism:iem,      jsm:jem,       nz, num_fields) )  
+    allocate( x1(ism:iem+shift,jsm:jem,       nz, num_fields) ) 
+    allocate( y1(ism:iem,      jsm:jem+shift, nz, num_fields) ) 
+    allocate( a2(ism:iem,      jsm:jem,       nz, num_fields) )
+    allocate( x2(ism:iem+shift,jsm:jem,       nz, num_fields) )
+    allocate( y2(ism:iem,      jsm:jem+shift, nz, num_fields) )
+    allocate( base(isc:iec+shift,jsc:jec+shift,nz) )
+    a1 = 0; x1 = 0; y1 = 0
+
+    base = 0
+    do k = 1,nz
+       do j = jsc, jec+shift
+          do i = isc, iec+shift
+             base(i,j,k) = k + i*1e-3 + j*1e-6
+          end do
+       end do
+    end do
 
-    return
+    !--- Test for partial direction update
+    do l =1, num_fields
+       call mpp_create_group_update(group_update, a1(:,:,:,l), domain, flags=WUPDATE+SUPDATE)
+    end do    
 
-  end subroutine fill_folded_north_bound
+    do l = 1, num_fields
+       a1(isc:iec,jsc:jec,:,l) = base(isc:iec,jsc:jec,:) + l*1e3
+       do k=1,nz
+          do i=isc-1,iec+1
+             a1(i,jsc-1,k,l) = 999;
+             a1(i,jec+1,k,l) = 999;
+          enddo
+          do j=jsc,jec
+             a1(isc-1,j,k,l) = 999
+             a1(iec+1,j,k,l) = 999
+          enddo
+       enddo
+    enddo
+    
+    a2 = a1
+    call mpp_do_group_update(group_update, domain, a1(isc,jsc,1,1))
 
-  !################################################################################
-  subroutine fill_cubic_grid_bound(data1_all, data2_all, is, ie, js, je, ioff, joff, tile, sign1, sign2, &
-                                   ebound, sbound, wbound, nbound )
-    real, dimension(:,:,:,:),       intent(in)    :: data1_all, data2_all
-    integer,                        intent(in)    :: is, ie, js, je
-    integer,                        intent(in)    :: tile, ioff, joff, sign1, sign2
-    real, dimension(:,:), optional, intent(inout) :: ebound, sbound, wbound, nbound
-    integer                                       :: tw, te, ts, tn
+    do l = 1, num_fields
+       call mpp_update_domains( a2(:,:,:,l), domain, flags=WUPDATE+SUPDATE, complete=l==num_fields )
+    enddo
 
-    if(mod(tile,2) == 0) then ! tile 2, 4, 6
-       tw = tile - 1; te = tile + 2; ts = tile - 2; tn = tile + 1
-       if(te > 6 ) te = te - 6
-       if(ts < 1 ) ts = ts + 6
-       if(tn > 6 ) tn = tn - 6
-       !--- East bound
-       if(present(ebound)) then
-          if(ie == nx) then                
-             ebound(:,:) = sign1*data2_all(nx+joff-js+1:nx-je+1:-1,1,:,te)
-          else
-             ebound(:,:) = data1_all(ie+ioff, js:je+joff, :,tile)
-          end if
-       end if
-       !--- South bound
-       if(present(sbound)) then
-          if(js == 1) then                
-             sbound(:,:) = sign2*data2_all(nx+joff, ny+ioff-is+1:ny-ie+1:-1,:,ts)
-          else
-             sbound(:,:) = data1_all(is:ie+ioff, js, :,tile)
-          end if
-       end if
+    do l = 1, num_fields
+       write(text, '(i3.3)') l
+       call compare_checksums(a1(isd:ied,jsd:jed,:,l),a2(isd:ied,jsd:jed,:,l),type//' CENTER South West '//text)
+    enddo
 
-       !--- West bound
-       if(present(wbound)) then
-          if(is == 1) then
-             wbound(:,:) = data1_all(nx+ioff, js:je+joff,:,tw)
-          else
-             wbound(:,:) = data1_all(is, js:je+joff,:,tile)
-          end if
-       end if
+    call mpp_clear_group_update(group_update)
 
-       !--- north bound
-       if(present(nbound)) then
-          if(je == ny) then                
-             nbound(:,:) = data1_all(is:ie+ioff, 1,:,tn)
-          else
-             nbound(:,:) = data1_all(is:ie+ioff, je+joff, :,tile)
-          end if
-       end if
-    else ! tile 1, 3, 5
-       tw = tile - 2; te = tile + 1; ts = tile - 1; tn = tile + 2
-       if(tw < 1 ) tw = tw + 6
-       if(ts < 1 ) ts = ts + 6
-       if(tn > 6 ) tn = tn - 6
-       !--- East bound
-       if(present(ebound)) then
-          if(ie == nx) then                
-             ebound(:,:) = data1_all(1, js:je+joff, :,te) 
-          else
-             ebound(:,:) = data1_all(ie+ioff, js:je+joff, :,tile)
-          end if
-       end if
-       !--- South bound
-       if(present(sbound)) then
-          if(js == 1) then                
-             sbound(:,:) = data1_all(is:ie+ioff,ny+joff,:,ts)
-          else
-             sbound(:,:) = data1_all(is:ie+ioff, js, :,tile)
-          end if
-       end if
+    !--- Test for DGRID update
+    if(type == 'Cubic-Grid' ) then
+       x1 = 0; y1 = 0
+       do l =1, num_fields
+          call mpp_create_group_update(group_update, x1(:,:,:,l), y1(:,:,:,l), domain, gridtype=DGRID_NE)
+       end do
 
-       !--- West bound
-       if(present(wbound)) then
-          if(is == 1) then
-             wbound(:,:) = sign1*data2_all(nx+joff-js+1:nx-je+1:-1,ny+ioff,:,tw)
-          else
-             wbound(:,:) = data1_all(is, js:je+joff,:,tile)
-          end if
-       end if
+       do l = 1, num_fields
+          y1(isc:iec+shift,jsc:jec,      :,l) = base(isc:iec+shift,jsc:jec,      :) + l*1e3 + 1e6
+          x1(isc:iec,      jsc:jec+shift,:,l) = base(isc:iec,      jsc:jec+shift,:) + l*1e3 + 2*1e6
+       enddo
+       x2 = x1; y2 = y1
+       call mpp_start_group_update(group_update, domain, x1(isc,jsc,1,1))
+       call mpp_complete_group_update(group_update, domain, x1(isc,jsc,1,1))
 
-       !--- north bound
-       if(present(nbound)) then
-          if(je == ny) then                
-             nbound(:,:) = sign2*data2_all(1, ny+ioff-is+1:ny-ie+1:-1,:,tn)
-          else
-             nbound(:,:) = data1_all(is:ie+ioff, je+joff, :,tile)
-          end if
-       end if
+       do l = 1, num_fields
+          call mpp_update_domains( x2(:,:,:,l), y2(:,:,:,l), domain, gridtype=DGRID_NE, complete=l==num_fields )
+       enddo
 
-    end if
+    !--- compare checksum
+       do l = 1, num_fields
+          write(text, '(i3.3)') l
+          call compare_checksums(x1(isd:ied+shift,jsd:jed,      :,l),x2(isd:ied+shift,jsd:jed,      :,l),type//' DGRID X'//text)
+          call compare_checksums(y1(isd:ied,      jsd:jed+shift,:,l),y2(isd:ied,      jsd:jed+shift,:,l),type//' DGRID Y'//text)
+       enddo
 
-  end subroutine fill_cubic_grid_bound
+       call mpp_clear_group_update(group_update)
+    endif
+    !--- Test for CGRID
+    a1 = 0; x1 = 0; y1 = 0
+    do l =1, num_fields
+       call mpp_create_group_update(group_update, a1(:,:,:,l), domain)
+       call mpp_create_group_update(group_update, x1(:,:,:,l), y1(:,:,:,l), domain, gridtype=CGRID_NE)
+    end do
 
-  !##############################################################################
-  ! this routine fill the halo points for the cubic grid. ioff and joff is used to distinguish
-  ! T, C, E, or N-cell
-  subroutine fill_cubic_grid_halo(data, data1_all, data2_all, tile, ioff, joff, sign1, sign2)
-    real, dimension(1-whalo:,1-shalo:,:), intent(inout) :: data
-    real, dimension(:,:,:,:),             intent(in)    :: data1_all, data2_all
-    integer,                              intent(in)    :: tile, ioff, joff, sign1, sign2 
-    integer                                             :: lw, le, ls, ln
+    do n = 1, num_iter
+       a1 = 0; x1 = 0; y1 = 0
+       do l = 1, num_fields
+          a1(isc:iec,      jsc:jec,      :,l) = base(isc:iec,      jsc:jec,      :) + l*1e3
+          x1(isc:iec+shift,jsc:jec,      :,l) = base(isc:iec+shift,jsc:jec,      :) + l*1e3 + 1e6
+          y1(isc:iec,      jsc:jec+shift,:,l) = base(isc:iec,      jsc:jec+shift,:) + l*1e3 + 2*1e6
+       enddo
+       a2 = a1; x2 = x1; y2 = y1
+       call mpp_clock_begin(id1)
+       call mpp_do_group_update(group_update, domain, a1(isc,jsc,1,1))
+       call mpp_clock_end  (id1)
+
+       call mpp_clock_begin(id2)
+       do l = 1, num_fields
+          call mpp_update_domains( a2(:,:,:,l), domain, complete=l==num_fields )
+       enddo
+       do l = 1, num_fields
+          call mpp_update_domains( x2(:,:,:,l), y2(:,:,:,l), domain, gridtype=CGRID_NE, complete=l==num_fields )
+       enddo
+       call mpp_clock_end(id2)
+
+       !--- compare checksum
+       do l = 1, num_fields
+          write(text, '(i3.3)') l
+          call compare_checksums(a1(isd:ied,      jsd:jed,      :,l),a2(isd:ied,      jsd:jed,      :,l),type//' CENTER '//text)
+          call compare_checksums(x1(isd:ied+shift,jsd:jed,      :,l),x2(isd:ied+shift,jsd:jed,      :,l),type//' CGRID X'//text)
+          call compare_checksums(y1(isd:ied,      jsd:jed+shift,:,l),y2(isd:ied,      jsd:jed+shift,:,l),type//' CGRID Y'//text)
+       enddo
+       a1 = 0; x1 = 0; y1 = 0
+       do l = 1, num_fields
+          a1(isc:iec,      jsc:jec,      :,l) = base(isc:iec,      jsc:jec,      :) + l*1e3
+          x1(isc:iec+shift,jsc:jec,      :,l) = base(isc:iec+shift,jsc:jec,      :) + l*1e3 + 1e6
+          y1(isc:iec,      jsc:jec+shift,:,l) = base(isc:iec,      jsc:jec+shift,:) + l*1e3 + 2*1e6
+       enddo
+       call mpp_clock_begin(id3)
+       call mpp_start_group_update(group_update, domain, a1(isc,jsc,1,1))
+       call mpp_complete_group_update(group_update, domain, a1(isc,jsc,1,1))
+       call mpp_clock_end  (id3)
+       !--- compare checksum
+       do l = 1, num_fields
+          write(text, '(i3.3)') l
+          call compare_checksums(a1(isd:ied,      jsd:jed,      :,l),a2(isd:ied,      jsd:jed,      :,l), &
+                                 type//' nonblock CENTER '//text)
+          call compare_checksums(x1(isd:ied+shift,jsd:jed,      :,l),x2(isd:ied+shift,jsd:jed,      :,l), &
+                                 type//' nonblock CGRID X'//text)
+          call compare_checksums(y1(isd:ied,      jsd:jed+shift,:,l),y2(isd:ied,      jsd:jed+shift,:,l), &
+                                 type//' nonblock CGRID Y'//text)
+       enddo
 
-    if(mod(tile,2) == 0) then ! tile 2, 4, 6
-       lw = tile - 1; le = tile + 2; ls = tile - 2; ln = tile + 1
-       if(le > 6 ) le = le - 6
-       if(ls < 1 ) ls = ls + 6
-       if(ln > 6 ) ln = ln - 6
-       data(1-whalo:0, 1:ny+joff, :) = data1_all(nx-whalo+1:nx, 1:ny+joff, :, lw) ! west 
-       do i = 1, ehalo 
-          data(nx+i+ioff, 1:ny+joff, :)    = sign1*data2_all(nx+joff:1:-1, i+ioff, :, le) ! east 
-       end do
-       do i = 1, shalo 
-          data(1:nx+ioff, 1-i, :)     = sign2*data2_all(nx-i+1, ny+ioff:1:-1, :, ls) ! south 
-       end do
-       data(1:nx+ioff, ny+1+joff:ny+nhalo+joff, :) = data1_all(1:nx+ioff, 1+joff:nhalo+joff, :, ln) ! north
-    else ! tile 1, 3, 5
-       lw = tile - 2; le = tile + 1; ls = tile - 1; ln = tile + 2
-       if(lw < 1 ) lw = lw + 6
-       if(ls < 1 ) ls = ls + 6
-       if(ln > 6 ) ln = ln - 6
-       do i = 1, whalo 
-          data(1-i, 1:ny+joff, :)     = sign1*data2_all(nx+joff:1:-1, ny-i+1, :, lw) ! west 
-       end do
-       data(nx+1+ioff:nx+ehalo+ioff, 1:ny+joff, :) = data1_all(1+ioff:ehalo+ioff, 1:ny+joff, :, le) ! east 
-       data(1:nx+ioff, 1-shalo:0, :)     = data1_all(1:nx+ioff, ny-shalo+1:ny, :, ls) ! south 
-       do i = 1, nhalo 
-          data(1:nx+ioff, ny+i+joff, :)    = sign2*data2_all(i+joff, ny+ioff:1:-1, :, ln) ! north 
+    enddo
+
+    call mpp_clear_group_update(group_update)
+
+    !--- The following is to test overlapping start and complete
+    if( num_fields > 1 ) then
+       do l =1, num_fields
+          call mpp_create_group_update(update_list(l), a1(:,:,:,l), domain)
+          call mpp_create_group_update(update_list(l), x1(:,:,:,l), y1(:,:,:,l), domain, gridtype=CGRID_NE)
        end do
-    end if
 
-  end subroutine fill_cubic_grid_halo
-    
-   !#####################################################################
-  subroutine test_nonuniform_mosaic( type )
-    character(len=*), intent(in) :: type
+       do n = 1, num_iter
 
-    type(domain2D)               :: domain
-    integer                      :: num_contact, ntiles, ntile_per_pe
-    integer                      :: i, j, k, n, nxm, nym, ni, nj, shift
-    integer                      :: ism, iem, jsm, jem, isc, iec, jsc, jec
-    integer                      :: isd, ied, jsd, jed
-    integer                      :: indices(4), msize(2)
-    character(len=128)           :: type2
+          a1 = 0; x1 = 0; y1 = 0
+          do l = 1, num_fields
+             a1(isc:iec,      jsc:jec,      :,l) = base(isc:iec,      jsc:jec,      :) + l*1e3
+             x1(isc:iec+shift,jsc:jec,      :,l) = base(isc:iec+shift,jsc:jec,      :) + l*1e3 + 1e6
+             y1(isc:iec,      jsc:jec+shift,:,l) = base(isc:iec,      jsc:jec+shift,:) + l*1e3 + 2*1e6
+          enddo
+          do l = 1, num_fields-1
+             call mpp_start_group_update(update_list(l), domain, a1(isc,jsc,1,1))
+          enddo
 
-    integer, allocatable, dimension(:)       :: tile
-    integer, allocatable, dimension(:)       :: pe_start, pe_end, tile1, tile2
-    integer, allocatable, dimension(:)       :: istart1, iend1, jstart1, jend1
-    integer, allocatable, dimension(:)       :: istart2, iend2, jstart2, jend2
-    integer, allocatable, dimension(:,:)     :: layout2D, global_indices
-    real,    allocatable, dimension(:,:,:,:) :: global1_all, global2_all
-    real,    allocatable, dimension(:,:,:,:) :: global1, global2, x, y  
+          call mpp_complete_group_update(update_list(1), domain, a1(isc,jsc,1,1))
+          call mpp_start_group_update(update_list(num_fields), domain, a1(isc,jsc,1,1))
+          do l = 2, num_fields
+             call mpp_complete_group_update(update_list(l), domain, a1(isc,jsc,1,1))
+          enddo
+          !--- compare checksum
+          do l = 1, num_fields
+             write(text, '(i3.3)') l
+             call compare_checksums(a1(isd:ied,      jsd:jed,      :,l),a2(isd:ied,      jsd:jed,      :,l), &
+                                    type//' multiple nonblock CENTER '//text)
+             call compare_checksums(x1(isd:ied+shift,jsd:jed,      :,l),x2(isd:ied+shift,jsd:jed,      :,l), &
+                                    type//' multiple nonblock CGRID X'//text)
+             call compare_checksums(y1(isd:ied,      jsd:jed+shift,:,l),y2(isd:ied,      jsd:jed+shift,:,l), &
+                                    type//' multiple nonblock CGRID Y'//text)
+          enddo
 
-    shift = 0
-    select case(type)
-    case('Five-Tile') ! one tile will run on pe 0 and other four tiles will run on pe 1
-       shift = 1      ! one extra point for symmetry domain
-       ntiles = 5     ! tile 1 with resolution 2*nx and 2*ny and the tiles are nx and ny.
-       num_contact = 11
-       if(npes .NE. 2) then
-          call mpp_error(NOTE,'TEST_MPP_DOMAINS: Five-Tile mosaic will not be tested because npes is not 2')
-          return
-       end if 
-       nxm = 2*nx; nym = 2*ny
-       layout = 1
-       if( pe == 0) then
-          ntile_per_pe = 1
-          allocate(tile(ntile_per_pe))
-          tile = 1
-          indices = (/1,2*nx,1,2*ny/)
-          ni = 2*nx; nj = 2*ny
-       else
-          ntile_per_pe = 4
-          allocate(tile(ntile_per_pe))
-          do n = 1, ntile_per_pe
-             tile(n) = n + 1
-          end do
-          indices = (/1,nx,1,ny/)
-          ni = nx; nj = ny
-       end if
-       allocate(pe_start(ntiles), pe_end(ntiles) )
-       pe_start(1) = 0; pe_start(2:) = 1
-       pe_end = pe_start
-    case default
-       call mpp_error(FATAL, 'TEST_MPP_DOMAINS: no such test: '//type)
-    end select
+       enddo
+    endif
 
-    allocate(layout2D(2,ntiles), global_indices(4,ntiles) )
+    do l =1, num_fields
+      call mpp_clear_group_update(update_list(l))
+    enddo
+    deallocate(update_list)
 
-    do n = 1, ntiles
-       if(n==1) then
-          global_indices(:,n) = (/1,2*nx,1,2*ny/)
-       else
-          global_indices(:,n) = (/1,nx,1,ny/)
-       endif  
-!       global_indices(:,n) = indices
-       layout2D(:,n)       = layout
-    end do
+    !--- test scalar 4-D variable
+    call mpp_create_group_update(group_update, a1(:,:,:,:), domain)
 
-    allocate(tile1(num_contact), tile2(num_contact) )
-    allocate(istart1(num_contact), iend1(num_contact), jstart1(num_contact), jend1(num_contact) ) 
-    allocate(istart2(num_contact), iend2(num_contact), jstart2(num_contact), jend2(num_contact) ) 
+    a1 = 0; x1 = 0; y1 = 0
+    do l = 1, num_fields
+       a1(isc:iec,      jsc:jec,      :,l) = base(isc:iec,      jsc:jec,      :) + l*1e3
+    enddo
+    a2 = a1; x2 = x1; y2 = y1
+    call mpp_clock_begin(id1)
+    call mpp_do_group_update(group_update, domain, a1(isc,jsc,1,1))
+    call mpp_clock_end  (id1)
 
-    !--- define domain
-    select case(type)
-    case( 'Five-Tile' )
-       !--- Contact line 1, between tile 1 (EAST) and tile 2 (WEST)
-       tile1(1) = 1; tile2(1) = 2
-       istart1(1) = 2*nx; iend1(1) = 2*nx; jstart1(1) = 1;  jend1(1) = ny
-       istart2(1) = 1;    iend2(1) = 1;    jstart2(1) = 1;  jend2(1) = ny
-       !--- Contact line 2, between tile 1 (EAST) and tile 4 (WEST)
-       tile1(2) = 1; tile2(2) = 4
-       istart1(2) = 2*nx; iend1(2) = 2*nx; jstart1(2) = ny+1; jend1(2) = 2*ny
-       istart2(2) = 1;    iend2(2) = 1;    jstart2(2) = 1;    jend2(2) = ny
-       !--- Contact line 3, between tile 1 (SOUTH) and tile 1 (NORTH)
-       tile1(3) = 1; tile2(3) = 1
-       istart1(3) = 1; iend1(3) = 2*nx; jstart1(3) = 1;    jend1(3) = 1
-       istart2(3) = 1; iend2(3) = 2*nx; jstart2(3) = 2*ny; jend2(3) = 2*ny
-       !--- Contact line 4, between tile 1 (WEST) and tile 3 (EAST)
-       tile1(4) = 1; tile2(4) = 3
-       istart1(4) = 1;  iend1(4) = 1;  jstart1(4) = 1;  jend1(4) = ny
-       istart2(4) = nx; iend2(4) = nx; jstart2(4) = 1;  jend2(4) = ny
-       !--- Contact line 5, between tile 1 (WEST) and tile 5 (EAST)
-       tile1(5) = 1; tile2(5) = 5
-       istart1(5) = 1;  iend1(5) = 1;  jstart1(5) = ny+1;  jend1(5) = 2*ny
-       istart2(5) = nx; iend2(5) = nx; jstart2(5) = 1;     jend2(5) = ny
-       !--- Contact line 6, between tile 2 (EAST) and tile 3 (WEST)
-       tile1(6) = 2; tile2(6) = 3
-       istart1(6) = nx; iend1(6) = nx; jstart1(6) = 1;  jend1(6) = ny
-       istart2(6) = 1;  iend2(6) = 1;  jstart2(6) = 1;  jend2(6) = ny       
-       !--- Contact line 7, between tile 2 (SOUTH) and tile 4 (NORTH)  --- cyclic
-       tile1(7) = 2; tile2(7) = 4
-       istart1(7) = 1;  iend1(7) = nx; jstart1(7) = 1;   jend1(7) = 1
-       istart2(7) = 1;  iend2(7) = nx; jstart2(7) = ny;  jend2(7) = ny
-       !--- Contact line 8, between tile 2 (NORTH) and tile 4 (SOUTH) 
-       tile1(8) = 2; tile2(8) = 4
-       istart1(8) = 1;  iend1(8) = nx; jstart1(8) = ny;  jend1(8) = ny
-       istart2(8) = 1;  iend2(8) = nx; jstart2(8) = 1;   jend2(8) = 1
-       !--- Contact line 9, between tile 3 (SOUTH) and tile 5 (NORTH)  --- cyclic
-       tile1(9) = 3; tile2(9) = 5
-       istart1(9) = 1;  iend1(9) = nx; jstart1(9) = 1;   jend1(9) = 1
-       istart2(9) = 1;  iend2(9) = nx; jstart2(9) = ny;  jend2(9) = ny
-       !--- Contact line 10, between tile 3 (NORTH) and tile 5 (SOUTH) 
-       tile1(10) = 3; tile2(10) = 5
-       istart1(10) = 1;  iend1(10) = nx; jstart1(10) = ny;  jend1(10) = ny
-       istart2(10) = 1;  iend2(10) = nx; jstart2(10) = 1;   jend2(10) = 1
-       !--- Contact line 11, between tile 4 (EAST) and tile 5 (WEST)
-       tile1(11) = 4; tile2(11) = 5
-       istart1(11) = nx; iend1(11) = nx; jstart1(11) = 1;  jend1(11) = ny
-       istart2(11) = 1;  iend2(11) = 1;  jstart2(11) = 1;  jend2(11) = ny  
-       msize(1) = 2*nx + whalo + ehalo
-       msize(2) = 2*ny + shalo + nhalo
-       call mpp_define_mosaic(global_indices, layout2D, domain, ntiles, num_contact, tile1, tile2, &
-            istart1, iend1, jstart1, jend1, istart2, iend2, jstart2, jend2,      &
-            pe_start, pe_end, whalo=whalo, ehalo=ehalo, shalo=shalo, nhalo=nhalo,       &
-            name = type, memory_size = msize, symmetry = .true.  )
-    end select
-    
-    !--- setup data
-    allocate(global1_all(1:nxm,1:nym,nz, ntiles) )  
-    allocate(global1(1-whalo:ni+ehalo,1-shalo:nj+nhalo,nz, ntile_per_pe) )   
-    do n = 1, ntiles
-       do k = 1, nz
-          do j = 1, nym
-             do i = 1, nxm
-                global1_all(i,j,k,n) = n + i*1.0e-3 + j*1.0e-6 + k*1.0e-9
-             end do
-          end do
-       end do
-    end do
+    call mpp_clock_begin(id2)
+    call mpp_update_domains( a2(:,:,:,:), domain )
+    call mpp_clock_end(id2)
 
-    do n = 1, ntile_per_pe
-       global1(1:ni,1:nj,:,n) = global1_all(1:ni,1:nj,:,tile(n))
-    end do
+    !--- compare checksum
+    do l = 1, num_fields
+       write(text, '(i3.3)') l
+       call compare_checksums(a1(isd:ied, jsd:jed, :,l),a2(isd:ied, jsd:jed, :,l),type//' 4D CENTER '//text)
+    enddo
 
-    call mpp_get_compute_domain( domain, isc, iec, jsc, jec )
-    call mpp_get_data_domain   ( domain, isd, ied, jsd, jed )
-    call mpp_get_memory_domain   ( domain, ism, iem, jsm, jem )
+    a1 = 0
+    do l = 1, num_fields
+       a1(isc:iec,      jsc:jec,      :,l) = base(isc:iec,      jsc:jec,      :) + l*1e3
+    enddo
+    call mpp_clock_begin(id3)
+    call mpp_start_group_update(group_update, domain, a1(isc,jsc,1,1))
+    call mpp_complete_group_update(group_update, domain, a1(isc,jsc,1,1))
+    call mpp_clock_end  (id3)
 
-    allocate( x (ism:iem,jsm:jem,nz, ntile_per_pe) )
-    x = 0.
-    x(isc:iec,jsc:jec,:,:) = global1(isc:iec,jsc:jec,:,:)
+    !--- compare checksum
+    do l = 1, num_fields
+       write(text, '(i3.3)') l
+       call compare_checksums(a1(isd:ied, jsd:jed, :,l),a2(isd:ied, jsd:jed, :,l), &
+                              type//' nonblock 4D CENTER '//text)
+    enddo
 
-    !--- fill up the value at halo points
-    do n = 1, ntile_per_pe
-       call fill_five_tile_halo(global1(:,:,:,n), global1_all, tile(n), 0, 0 )
-    end do
 
-    ! full update
-    id = mpp_clock_id( type, flags=MPP_CLOCK_SYNC+MPP_CLOCK_DETAILED )
-    call mpp_clock_begin(id)
-    do n = 1, ntile_per_pe
-       call mpp_update_domains( x(:,:,:,n), domain, tile_count = n )
-    end do
-    call mpp_clock_end(id)
 
-   do n = 1, ntile_per_pe
-      write(type2, *)type, " at tile_count = ",n
-      call compare_checksums( x(isd:ied,jsd:jed,:,n), global1(isd:ied,jsd:jed,:,n), trim(type2) )
-   end do
+    !--- test for BGRID.
+    deallocate(a1, x1, y1)
+    deallocate(a2, x2, y2)
+    call mpp_clear_group_update(group_update)
 
-   deallocate(global1_all, global1, x)
+    allocate( a1(ism:iem+shift,jsm:jem+shift, nz, num_fields) )  
+    allocate( x1(ism:iem+shift,jsm:jem+shift, nz, num_fields) )  
+    allocate( y1(ism:iem+shift,jsm:jem+shift, nz, num_fields) )  
+    allocate( a2(ism:iem+shift,jsm:jem+shift, nz, num_fields) )  
+    allocate( x2(ism:iem+shift,jsm:jem+shift, nz, num_fields) )  
+    allocate( y2(ism:iem+shift,jsm:jem+shift, nz, num_fields) )  
 
-    !------------------------------------------------------------------
-    !  vector update : BGRID_NE, one extra point in each direction for Five-Tile
-    !------------------------------------------------------------------
-    !--- setup data
-    allocate(global1_all(nxm+shift,nym+shift,nz, ntiles), global2_all(nxm+shift,nym+shift,nz, ntiles) )  
-    allocate(global1(1-whalo:ni+ehalo+shift,1-shalo:nj+nhalo+shift,nz, ntile_per_pe) )  
-    allocate(global2(1-whalo:ni+ehalo+shift,1-shalo:nj+nhalo+shift,nz, ntile_per_pe) )   
-    do n = 1, ntiles
-       do k = 1, nz
-          do j = 1, nym+shift
-             do i = 1, nxm+shift
-                global1_all(i,j,k,n) = 1.0e3 + n + i*1.0e-3 + j*1.0e-6 + k*1.0e-9
-                global2_all(i,j,k,n) = 2.0e3 + n + i*1.0e-3 + j*1.0e-6 + k*1.0e-9
-             end do
-          end do
-       end do
+    do l =1, num_fields
+       call mpp_create_group_update(group_update, a1(:,:,:,l), domain, position=CORNER)
+       call mpp_create_group_update(group_update, x1(:,:,:,l), y1(:,:,:,l), domain, gridtype=BGRID_NE)
     end do
 
-    !------------------------------------------------------------------------
-    ! --- make sure consisency on the boundary for Five-Tile mosaic
-    ! --- east boundary will take the value of neighbor tile west,
-    ! --- north boundary will take the value of neighbor tile south.
-    !------------------------------------------------------------------------
-    if(type == 'Five-Tile') then
-       global1_all(nxm+1,    1:ny,:,1) = global1_all(1,    1:ny,:,2)  ! east
-       global1_all(nxm+1,ny+1:nym,:,1) = global1_all(1,    1:ny,:,4)  ! east
-       global1_all(1:nxm+1, nym+1,:,1) = global1_all(1:nxm+1, 1,:,1)  ! north
-       global1_all(nx+1,     1:ny,:,2) = global1_all(1,    1:ny,:,3)  ! east
-       global1_all(1:nx+1,   ny+1,:,2) = global1_all(1:nx+1,  1,:,4)  ! north
-       global1_all(nx+1,     1:ny,:,3) = global1_all(1,    1:ny,:,1)  ! east
-       global1_all(1:nx+1,   ny+1,:,3) = global1_all(1:nx+1,  1,:,5)  ! north
-       global1_all(nx+1,     1:ny,:,4) = global1_all(1,    1:ny,:,5)  ! east
-       global1_all(1:nx+1,   ny+1,:,4) = global1_all(1:nx+1,  1,:,2)  ! north
-       global1_all(nx+1,     1:ny,:,5) = global1_all(1,ny+1:nym,:,1)  ! east
-       global1_all(1:nx+1,   ny+1,:,5) = global1_all(1:nx+1,  1,:,3)  ! north
-       global1_all(nx+1,     ny+1,:,2) = global1_all(1,       1,:,5)  ! northeast 
-       global1_all(nx+1,     ny+1,:,3) = global1_all(1,    ny+1,:,1)  ! northeast 
-       global2_all(nxm+1,    1:ny,:,1) = global2_all(1,    1:ny,:,2)  ! east
-       global2_all(nxm+1,ny+1:nym,:,1) = global2_all(1,    1:ny,:,4)  ! east
-       global2_all(1:nxm+1, nym+1,:,1) = global2_all(1:nxm+1, 1,:,1)  ! north
-       global2_all(nx+1,     1:ny,:,2) = global2_all(1,    1:ny,:,3)  ! east
-       global2_all(1:nx+1,   ny+1,:,2) = global2_all(1:nx+1,  1,:,4)  ! north
-       global2_all(nx+1,     1:ny,:,3) = global2_all(1,    1:ny,:,1)  ! east
-       global2_all(1:nx+1,   ny+1,:,3) = global2_all(1:nx+1,  1,:,5)  ! north
-       global2_all(nx+1,     1:ny,:,4) = global2_all(1,    1:ny,:,5)  ! east
-       global2_all(1:nx+1,   ny+1,:,4) = global2_all(1:nx+1,  1,:,2)  ! north
-       global2_all(nx+1,     1:ny,:,5) = global2_all(1,ny+1:nym,:,1)  ! east
-       global2_all(1:nx+1,   ny+1,:,5) = global2_all(1:nx+1,  1,:,3)  ! north
-       global2_all(nx+1,     ny+1,:,2) = global2_all(1,       1,:,5)  ! northeast 
-       global2_all(nx+1,     ny+1,:,3) = global2_all(1,    ny+1,:,1)  ! northeast 
-    end if
+    do n = 1, num_iter
+       a1 = 0; x1 = 0; y1 = 0
+       do l = 1, num_fields
+          a1(isc:iec+shift,jsc:jec+shift,:,l) = base(isc:iec+shift,jsc:jec+shift,:) + l*1e3
+          x1(isc:iec+shift,jsc:jec+shift,:,l) = base(isc:iec+shift,jsc:jec+shift,:) + l*1e3 + 1e6
+          y1(isc:iec+shift,jsc:jec+shift,:,l) = base(isc:iec+shift,jsc:jec+shift,:) + l*1e3 + 2*1e6
+       enddo
+       a2 = a1; x2 = x1; y2 = y1
+       call mpp_clock_begin(id1)
+       call mpp_do_group_update(group_update, domain, a1(isc,jsc,1,1))
+       call mpp_clock_end  (id1)
+
+       call mpp_clock_begin(id2)
+       do l = 1, num_fields
+          call mpp_update_domains( a2(:,:,:,l), domain, position=CORNER, complete=l==num_fields )
+       enddo
+       do l = 1, num_fields
+          call mpp_update_domains( x2(:,:,:,l), y2(:,:,:,l), domain, gridtype=BGRID_NE, complete=l==num_fields )
+       enddo
+       call mpp_clock_end(id2)
+
+       !--- compare checksum
+       do l = 1, num_fields
+          write(text, '(i3.3)') l
+          call compare_checksums(a1(isd:ied+shift,jsd:jed+shift,:,l),a2(isd:ied+shift,jsd:jed+shift,:,l),type//' CORNER '//text)
+          call compare_checksums(x1(isd:ied+shift,jsd:jed+shift,:,l),x2(isd:ied+shift,jsd:jed+shift,:,l),type//' BGRID X'//text)
+          call compare_checksums(y1(isd:ied+shift,jsd:jed+shift,:,l),y2(isd:ied+shift,jsd:jed+shift,:,l),type//' BGRID Y'//text)
+       enddo
 
-    do n = 1, ntile_per_pe
-       global1(1:ni+shift,1:nj+shift,:,n) = global1_all(1:ni+shift,1:nj+shift,:,tile(n))
-       global2(1:ni+shift,1:nj+shift,:,n) = global2_all(1:ni+shift,1:nj+shift,:,tile(n))
-    end do
+       a1 = 0; x1 = 0; y1 = 0
+       do l = 1, num_fields
+          a1(isc:iec+shift,jsc:jec+shift,:,l) = base(isc:iec+shift,jsc:jec+shift,:) + l*1e3
+          x1(isc:iec+shift,jsc:jec+shift,:,l) = base(isc:iec+shift,jsc:jec+shift,:) + l*1e3 + 1e6
+          y1(isc:iec+shift,jsc:jec+shift,:,l) = base(isc:iec+shift,jsc:jec+shift,:) + l*1e3 + 2*1e6
+       enddo
+       call mpp_clock_begin(id3)
+       call mpp_start_group_update(group_update, domain, a1(isc,jsc,1,1))
+       call mpp_complete_group_update(group_update, domain, a1(isc,jsc,1,1))
+       call mpp_clock_end  (id3)
+       !--- compare checksum
+       do l = 1, num_fields
+          write(text, '(i3.3)') l
+          call compare_checksums(a1(isd:ied+shift,jsd:jed+shift,:,l),a2(isd:ied+shift,jsd:jed+shift,:,l), &
+                                 type//' nonblockCORNER '//text)
+          call compare_checksums(x1(isd:ied+shift,jsd:jed+shift,:,l),x2(isd:ied+shift,jsd:jed+shift,:,l), &
+                                 type//' nonblock BGRID X'//text)
+          call compare_checksums(y1(isd:ied+shift,jsd:jed+shift,:,l),y2(isd:ied+shift,jsd:jed+shift,:,l), &
+                                 type//' nonblock BGRID Y'//text)
+       enddo
 
-    allocate( x (ism:iem+shift,jsm:jem+shift,nz,ntile_per_pe) )
-    allocate( y (ism:iem+shift,jsm:jem+shift,nz,ntile_per_pe) )
+    enddo 
 
-    x = 0.; y = 0
-    x (isc:iec+shift,jsc:jec+shift,:,:) = global1(isc:iec+shift,jsc:jec+shift,:,:)
-    y (isc:iec+shift,jsc:jec+shift,:,:) = global2(isc:iec+shift,jsc:jec+shift,:,:)
+    call mpp_clear_group_update(group_update)
 
-    !-----------------------------------------------------------------------
-    !                   fill up the value at halo points.     
-    !-----------------------------------------------------------------------
-    do n = 1, ntile_per_pe
-       call fill_five_tile_halo(global1(:,:,:,n), global1_all, tile(n), shift, shift)
-       call fill_five_tile_halo(global2(:,:,:,n), global2_all, tile(n), shift, shift)
-    end do
 
-    id = mpp_clock_id( type//' BGRID_NE', flags=MPP_CLOCK_SYNC+MPP_CLOCK_DETAILED )
-    call mpp_clock_begin(id)
-    do n = 1, ntile_per_pe
-       call mpp_update_domains( x(:,:,:,n), y(:,:,:,n), domain, gridtype=BGRID_NE, tile_count = n )
-    end do
-    call mpp_clock_end(id)
 
-   do n = 1, ntile_per_pe
-      write(type2, *)type, " at tile_count = ",n
-      call compare_checksums( x(isd:ied+shift,jsd:jed+shift,:,n), global1(isd:ied+shift,jsd:jed+shift,:,n), &
-                              trim(type2)//' BGRID_NE X')
-      call compare_checksums( y(isd:ied+shift,jsd:jed+shift,:,n), global2(isd:ied+shift,jsd:jed+shift,:,n), &
-                              trim(type2)//' BGRID_NE Y')
-   end do
+   
+    deallocate(pe_start, pe_end, tile1, tile2)
+    deallocate(istart1, iend1, jstart1, jend1)
+    deallocate(istart2, iend2, jstart2, jend2)
+    deallocate(layout2D, global_indices)
 
-   deallocate(global1_all, global2_all, global1, global2, x, y)
+    deallocate(a1, x1, y1)
+    deallocate(a2, x2, y2)
+    deallocate(base)
+    call mpp_deallocate_domain(domain)
 
-    !------------------------------------------------------------------
-    !  vector update : CGRID_NE
-    !------------------------------------------------------------------
-    !--- setup data
-    allocate(global1_all(nxm+shift,nym,nz, ntiles), global2_all(nxm,nym+shift,nz, ntiles) )  
-    allocate(global1(1-whalo:ni+ehalo+shift, 1-shalo:nj+nhalo,       nz, ntile_per_pe) )  
-    allocate(global2(1-whalo:ni+ehalo,       1-shalo:nj+nhalo+shift, nz, ntile_per_pe) )   
-    do n = 1, ntiles
-       do k = 1, nz
-          do j = 1, nym
-             do i = 1, nxm+shift
-                global1_all(i,j,k,n) = 1.0e3 + n + i*1.0e-3 + j*1.0e-6 + k*1.0e-9
-             end do
-          end do
-          do j = 1, nym+shift
-             do i = 1, nxm
-                global2_all(i,j,k,n) = 2.0e3 + n + i*1.0e-3 + j*1.0e-6 + k*1.0e-9
-             end do
-          end do
-       end do
-    end do
+end subroutine test_group_update
 
-    !------------------------------------------------------------------------
-    ! --- make sure consisency on the boundary for Five-Tile mosaic
-    ! --- east boundary will take the value of neighbor tile west,
-    ! --- north boundary will take the value of neighbor tile south.
-    !------------------------------------------------------------------------
-    if(type == 'Five-Tile') then
-       global1_all(nxm+1,    1:ny,:,1) = global1_all(1,    1:ny,:,2)  ! east
-       global1_all(nxm+1,ny+1:nym,:,1) = global1_all(1,    1:ny,:,4)  ! east
-       global1_all(nx+1,     1:ny,:,2) = global1_all(1,    1:ny,:,3)  ! east
-       global1_all(nx+1,     1:ny,:,3) = global1_all(1,    1:ny,:,1)  ! east
-       global1_all(nx+1,     1:ny,:,4) = global1_all(1,    1:ny,:,5)  ! east
-       global1_all(nx+1,     1:ny,:,5) = global1_all(1,ny+1:nym,:,1)  ! east
-       global2_all(1:nxm,   nym+1,:,1) = global2_all(1:nxm,   1,:,1)  ! north
-       global2_all(1:nx,     ny+1,:,2) = global2_all(1:nx,    1,:,4)  ! north
-       global2_all(1:nx,     ny+1,:,3) = global2_all(1:nx,    1,:,5)  ! north
-       global2_all(1:nx,     ny+1,:,4) = global2_all(1:nx,    1,:,2)  ! north
-       global2_all(1:nx,     ny+1,:,5) = global2_all(1:nx,    1,:,3)  ! north
+
+  !#################################################################################
+
+  subroutine fill_halo_zero(data, whalo, ehalo, shalo, nhalo, xshift, yshift, isc, iec, jsc, jec, isd, ied, jsd, jed)
+    integer,                         intent(in) :: isc, iec, jsc, jec, isd, ied, jsd, jed
+    integer,                         intent(in) :: whalo, ehalo, shalo, nhalo, xshift, yshift
+    real, dimension(isd:,jsd:,:), intent(inout) :: data
+
+    if(whalo >=0) then
+       data(iec+ehalo+1+xshift:ied+xshift,jsd:jed+yshift,:) = 0
+       data(isd:isc-whalo-1,jsd:jed+yshift,:) = 0
+    else
+       data(iec+1+xshift:iec-ehalo+xshift,jsc+shalo:jec-nhalo+yshift,:) = 0
+       data(isc+whalo:isc-1,jsc+shalo:jec-nhalo+yshift,:) = 0
     end if
 
-    do n = 1, ntile_per_pe
-       global1(1:ni+shift,      1:nj,:,n) = global1_all(1:ni+shift,      1:nj,:,tile(n))
-       global2(1:ni,      1:nj+shift,:,n) = global2_all(1:ni,      1:nj+shift,:,tile(n))
-    end do
+    if(shalo>=0) then
+       data(isd:ied+xshift, jec+nhalo+1+yshift:jed+yshift,:) = 0
+       data(isd:ied+xshift, jsd:jsc-shalo-1,:) = 0
+    else
+       data(isc+whalo:iec-ehalo+xshift,jec+1+yshift:jec-nhalo+yshift,:) = 0
+       data(isc+whalo:iec-ehalo+xshift,jsc+shalo:jsc-1,:) = 0
+    end if
 
-    allocate( x (ism:iem+shift,      jsm:jem,nz,ntile_per_pe) )
-    allocate( y (ism:iem,      jsm:jem+shift,nz,ntile_per_pe) )
+  end subroutine fill_halo_zero
 
-    x = 0.; y = 0
-    x (isc:iec+shift,      jsc:jec,:,:) = global1(isc:iec+shift,      jsc:jec,:,:)
-    y (isc:iec,      jsc:jec+shift,:,:) = global2(isc:iec,      jsc:jec+shift,:,:)
+  !##############################################################################
+  ! this routine fill the halo points for the regular mosaic. 
+  subroutine fill_regular_mosaic_halo(data, data_all, te, tse, ts, tsw, tw, tnw, tn, tne)
+    real, dimension(1-whalo:,1-shalo:,:), intent(inout) :: data
+    real, dimension(:,:,:,:),             intent(in)    :: data_all
+    integer,                              intent(in)    :: te, tse, ts, tsw, tw, tnw, tn, tne
 
-    !-----------------------------------------------------------------------
-    !                   fill up the value at halo points.     
-    !-----------------------------------------------------------------------
-    do n = 1, ntile_per_pe
-       call fill_five_tile_halo(global1(:,:,:,n), global1_all, tile(n), shift, 0)
-       call fill_five_tile_halo(global2(:,:,:,n), global2_all, tile(n), 0, shift)
-    end do
+       data(nx+1:nx+ehalo, 1:ny,          :) = data_all(1:ehalo,       1:ny,          :, te) ! east
+       data(1:nx,          1-shalo:0,     :) = data_all(1:nx,          ny-shalo+1:ny, :, ts) ! south 
+       data(1-whalo:0,     1:ny,          :) = data_all(nx-whalo+1:nx, 1:ny,          :, tw) ! west
+       data(1:nx,          ny+1:ny+nhalo, :) = data_all(1:nx,          1:nhalo,       :, tn) ! north  
+       data(nx+1:nx+ehalo, 1-shalo:0,     :) = data_all(1:ehalo,       ny-shalo+1:ny, :,tse) ! southeast
+       data(1-whalo:0,     1-shalo:0,     :) = data_all(nx-whalo+1:nx, ny-shalo+1:ny, :,tsw) ! southwest
+       data(nx+1:nx+ehalo, ny+1:ny+nhalo, :) = data_all(1:ehalo,       1:nhalo,       :,tnw) ! northeast
+       data(1-whalo:0,     ny+1:ny+nhalo, :) = data_all(nx-whalo+1:nx, 1:nhalo,       :,tne) ! northwest    
 
-    id = mpp_clock_id( type//' CGRID_NE', flags=MPP_CLOCK_SYNC+MPP_CLOCK_DETAILED )
-    call mpp_clock_begin(id)
-    do n = 1, ntile_per_pe
-       call mpp_update_domains( x(:,:,:,n), y(:,:,:,n), domain, gridtype=CGRID_NE, tile_count = n )
-    end do
-    call mpp_clock_end(id)
 
-   do n = 1, ntile_per_pe
-      write(type2, *)type, " at tile_count = ",n
-      call compare_checksums( x(isd:ied+shift,jsd:jed,:,n), global1(isd:ied+shift,jsd:jed,:,n), &
-                              trim(type2)//' CGRID_NE X')
-      call compare_checksums( y(isd:ied,jsd:jed+shift,:,n), global2(isd:ied,jsd:jed+shift,:,n), &
-                              trim(type2)//' CGRID_NE Y')
-   end do
 
-   deallocate(global1_all, global2_all, global1, global2, x, y)
+  end subroutine fill_regular_mosaic_halo
 
-  end subroutine test_nonuniform_mosaic
+  !################################################################################
+  subroutine fill_folded_north_halo(data, ioff, joff, ishift, jshift, sign)
+    real, dimension(1-whalo:,1-shalo:,:), intent(inout) :: data
+    integer,                              intent(in   ) :: ioff, joff, ishift, jshift, sign    
+    integer  :: nxp, nyp, m1, m2
 
-  subroutine fill_five_tile_halo(data, data_all, tile, ioff, joff)
+    nxp = nx+ishift
+    nyp = ny+jshift
+    m1 = ishift - ioff
+    m2 = 2*ishift - ioff
+
+    data(1-whalo:0,                  1:nyp,:) =      data(nx-whalo+1:nx,        1:ny+jshift,:) ! west
+    data(nx+1:nx+ehalo+ishift,       1:nyp,:) =      data(1:ehalo+ishift,       1:ny+jshift,:) ! east
+    if(m1 .GE. 1-whalo) data(1-whalo:m1,  nyp+1:nyp+nhalo,:) = sign*data(whalo+m2:1+ishift:-1, nyp-joff:nyp-nhalo-joff+1:-1,:)
+    data(m1+1:nx+m2,       nyp+1:nyp+nhalo,:) = sign*data(nx+ishift:1:-1,       nyp-joff:nyp-nhalo-joff+1:-1,:)
+    data(nx+m2+1:nxp+ehalo,nyp+1:nyp+nhalo,:) = sign*data(nx:nx-ehalo+m1+1:-1,  nyp-joff:nyp-nhalo-joff+1:-1,:)
+
+  end subroutine fill_folded_north_halo
+
+  !################################################################################
+  subroutine fill_folded_south_halo(data, ioff, joff, ishift, jshift, sign)
     real, dimension(1-whalo:,1-shalo:,:), intent(inout) :: data
-    real, dimension(:,:,:,:),             intent(in)    :: data_all
-    integer,                              intent(in)    :: tile, ioff, joff
-    integer                                             :: nxm, nym
+    integer,                              intent(in   ) :: ioff, joff, ishift, jshift, sign    
+    integer  :: nxp, nyp, m1, m2
 
-    nxm = 2*nx; nym = 2*ny
+    nxp = nx+ishift
+    nyp = ny+jshift
+    m1 = ishift - ioff
+    m2 = 2*ishift - ioff
 
-    select case(tile)
-    case(1)
-       data(nxm+1+ioff:nxm+ehalo+ioff,                     1:ny,:) = data_all(1+ioff:ehalo+ioff,              1:ny,:,2) ! east
-       data(nxm+1+ioff:nxm+ehalo+ioff,            ny+1:nym+joff,:) = data_all(1+ioff:ehalo+ioff,         1:ny+joff,:,4) ! east
-       data(1-whalo:0,                                     1:ny,:) = data_all(nx-whalo+1:nx,                  1:ny,:,3) ! west
-       data(1-whalo:0,                            ny+1:nym+joff,:) = data_all(nx-whalo+1:nx,             1:ny+joff,:,5) ! west
-       data(1:nxm+ioff,                               1-shalo:0,:) = data_all(1:nxm+ioff,          nym-shalo+1:nym,:,1) ! south
-       data(1:nxm+ioff,               nym+1+joff:nym+nhalo+joff,:) = data_all(1:nxm+ioff,        1+joff:nhalo+joff,:,1) ! north
-       data(nxm+1+ioff:nxm+ehalo+ioff,                1-shalo:0,:) = data_all(1+ioff:ehalo+ioff,     ny-shalo+1:ny,:,4) ! southeast
-       data(1-whalo:0,                                1-shalo:0,:) = data_all(nx-whalo+1:nx,         ny-shalo+1:ny,:,5) ! southwest
-       data(nxm+1+ioff:nxm+ehalo+ioff,nym+1+joff:nym+nhalo+joff,:) = data_all(1+ioff:ehalo+ioff, 1+joff:nhalo+joff,:,2) ! northeast
-       data(1-whalo:0,                nym+1+joff:nym+nhalo+joff,:) = data_all(nx-whalo+1:nx,     1+joff:nhalo+joff,:,3) ! northwest
-    case(2)
-       data(nx+1+ioff:nx+ehalo+ioff,              1:ny+joff,:) = data_all(1+ioff:ehalo+ioff,              1:ny+joff,:,3) ! east
-       data(1-whalo:0,                            1:ny+joff,:) = data_all(nxm-whalo+1:nxm,                1:ny+joff,:,1) ! west
-       data(1:nx+ioff,                            1-shalo:0,:) = data_all(1:nx+ioff,                  ny-shalo+1:ny,:,4) ! south 
-       data(1:nx+ioff,              ny+1+joff:ny+nhalo+joff,:) = data_all(1:nx+ioff,              1+joff:nhalo+joff,:,4) ! north
-       data(nx+1+ioff:nx+ehalo+ioff,              1-shalo:0,:) = data_all(1+ioff:ehalo+ioff,          ny-shalo+1:ny,:,5) ! southeast
-       data(1-whalo:0,                            1-shalo:0,:) = data_all(nxm-whalo+1:nxm,          nym-shalo+1:nym,:,1) ! southwest
-       data(nx+1+ioff:nx+ehalo+ioff,ny+1+joff:ny+nhalo+joff,:) = data_all(1+ioff:ehalo+ioff,      1+joff:nhalo+joff,:,5) ! northeast
-       data(1-whalo:0,              ny+1+joff:ny+nhalo+joff,:) = data_all(nxm-whalo+1:nxm,  ny+1+joff:ny+nhalo+joff,:,1) ! northwest
-    case(3)
-       data(nx+1+ioff:nx+ehalo+ioff,              1:ny+joff,:) = data_all(1+ioff:ehalo+ioff,              1:ny+joff,:,1) ! east
-       data(1-whalo:0,                            1:ny+joff,:) = data_all(nx-whalo+1:nx,                  1:ny+joff,:,2) ! west
-       data(1:nx+ioff,                            1-shalo:0,:) = data_all(1:nx+ioff,                  ny-shalo+1:ny,:,5) ! south 
-       data(1:nx+ioff,              ny+1+joff:ny+nhalo+joff,:) = data_all(1:nx+ioff,              1+joff:nhalo+joff,:,5) ! north
-       data(nx+1+ioff:nx+ehalo+ioff,              1-shalo:0,:) = data_all(1+ioff:ehalo+ioff,        nym-shalo+1:nym,:,1) ! southeast
-       data(1-whalo:0,                            1-shalo:0,:) = data_all(nx-whalo+1:nx,              ny-shalo+1:ny,:,4) ! southwest
-       data(nx+1+ioff:nx+ehalo+ioff,ny+1+joff:ny+nhalo+joff,:) = data_all(1+ioff:ehalo+ioff,ny+1+joff:ny+nhalo+joff,:,1) ! northeast
-       data(1-whalo:0,              ny+1+joff:ny+nhalo+joff,:) = data_all(nx-whalo+1:nx,          1+joff:nhalo+joff,:,4) ! northwest
-    case(4)
-       data(nx+1+ioff:nx+ehalo+ioff,              1:ny+joff,:) = data_all(1+ioff:ehalo+ioff,        1:ny+joff,:,5) ! east
-       data(1-whalo:0,                            1:ny+joff,:) = data_all(nxm-whalo+1:nxm,     ny+1:2*ny+joff,:,1) ! west
-       data(1:nx+ioff,                            1-shalo:0,:) = data_all(1:nx+ioff,            ny-shalo+1:ny,:,2) ! south 
-       data(1:nx+ioff,              ny+1+joff:ny+nhalo+joff,:) = data_all(1:nx+ioff,        1+joff:nhalo+joff,:,2) ! north
-       data(nx+1+ioff:nx+ehalo+ioff,              1-shalo:0,:) = data_all(1+ioff:ehalo+ioff,    ny-shalo+1:ny,:,3) ! southeast
-       data(1-whalo:0,                            1-shalo:0,:) = data_all(nxm-whalo+1:nxm,      ny-shalo+1:ny,:,1) ! southwest
-       data(nx+1+ioff:nx+ehalo+ioff,ny+1+joff:ny+nhalo+joff,:) = data_all(1+ioff:ehalo+ioff,1+joff:nhalo+joff,:,3) ! northeast
-       data(1-whalo:0,              ny+1+joff:ny+nhalo+joff,:) = data_all(nxm-whalo+1:nxm,  1+joff:nhalo+joff,:,1) ! northwest
-    case(5)
-       data(nx+1+ioff:nx+ehalo+ioff,            1:  ny+joff,:) = data_all(1+ioff:ehalo+ioff,   ny+1:2*ny+joff,:,1) ! east
-       data(1-whalo:0,                            1:ny+joff,:) = data_all(nx-whalo+1:nx,            1:ny+joff,:,4) ! west
-       data(1:nx+ioff,                            1-shalo:0,:) = data_all(1:nx+ioff,            ny-shalo+1:ny,:,3) ! south 
-       data(1:nx+ioff,              ny+1+joff:ny+nhalo+joff,:) = data_all(1:nx+ioff,        1+joff:nhalo+joff,:,3) ! north
-       data(nx+1+ioff:nx+ehalo+ioff,              1-shalo:0,:) = data_all(1+ioff:ehalo+ioff,    ny-shalo+1:ny,:,1) ! southeast
-       data(1-whalo:0,                            1-shalo:0,:) = data_all(nx-whalo+1:nx,        ny-shalo+1:ny,:,2) ! southwest
-       data(nx+1+ioff:nx+ehalo+ioff,ny+1+joff:ny+nhalo+joff,:) = data_all(1+ioff:ehalo+ioff,1+joff:nhalo+joff,:,1) ! northeast
-       data(1-whalo:0,              ny+1+joff:ny+nhalo+joff,:) = data_all(nx-whalo+1:nx,    1+joff:nhalo+joff,:,2) ! northwest
-    end select
 
-  end subroutine fill_five_tile_halo
+    data(1-whalo:0,                  1:nyp,:) =      data(nx-whalo+1:nx,        1:nyp,:) ! west
+    data(nx+1:nx+ehalo+ishift,       1:nyp,:) =      data(1:ehalo+ishift,       1:nyp,:) ! east
+    if(m1 .GE. 1-whalo)data(1-whalo:m1, 1-shalo:0,:) = sign*data(whalo+m2:1+ishift:-1, shalo+jshift:1+jshift:-1,:)
+    data(m1+1:nx+m2,       1-shalo:0,:) = sign*data(nxp:1:-1,             shalo+jshift:1+jshift:-1,:)
+    data(nx+m2+1:nxp+ehalo,1-shalo:0,:) = sign*data(nx:nx-ehalo+m1+1:-1,  shalo+jshift:1+jshift:-1,:)
+
+  end subroutine fill_folded_south_halo
+
+  !################################################################################
+  subroutine fill_folded_west_halo(data, ioff, joff, ishift, jshift, sign)
+    real, dimension(1-whalo:,1-shalo:,:), intent(inout) :: data
+    integer,                              intent(in   ) :: ioff, joff, ishift, jshift, sign    
+    integer  :: nxp, nyp, m1, m2
+
+    nxp = nx+ishift
+    nyp = ny+jshift
+    m1 = jshift - joff
+    m2 = 2*jshift - joff
+
+    data(1:nxp, 1-shalo:0, :)      = data(1:nxp, ny-shalo+1:ny, :) ! south
+    data(1:nxp, ny+1:nyp+nhalo, :) = data(1:nxp, 1:nhalo+jshift,:) ! north
+    if(m1 .GE. 1-shalo) data(1-whalo:0, 1-shalo:m1, :) = sign*data(whalo+ishift:1+ishift:-1, shalo+m2:1+jshift:-1,:)
+    data(1-whalo:0, m1+1:ny+m2, :) = sign*data(whalo+ishift:1+ishift:-1, nyp:1:-1, :)
+    data(1-whalo:0, ny+m2+1:nyp+nhalo,:) = sign*data(whalo+ishift:1+ishift:-1, ny:ny-nhalo+m1+1:-1,:)
+
+  end subroutine fill_folded_west_halo
+
+  !################################################################################
+  subroutine fill_folded_east_halo(data, ioff, joff, ishift, jshift, sign)
+    real, dimension(1-whalo:,1-shalo:,:), intent(inout) :: data
+    integer,                              intent(in   ) :: ioff, joff, ishift, jshift, sign    
+    integer  :: nxp, nyp, m1, m2
+
+    nxp = nx+ishift
+    nyp = ny+jshift
+    m1 = jshift - joff
+    m2 = 2*jshift - joff
+
+    data(1:nxp, 1-shalo:0, :)      = data(1:nxp, ny-shalo+1:ny, :) ! south
+    data(1:nxp, ny+1:nyp+nhalo, :) = data(1:nxp, 1:nhalo+jshift,:) ! north
+    if(m1 .GE. 1-shalo) data(nxp+1:nxp+ehalo, 1-shalo:m1, :) = sign*data(nxp-ioff:nxp-ehalo-ioff+1:-1, shalo+m2:1+jshift:-1,:)
+    data(nxp+1:nxp+ehalo, m1+1:ny+m2, :) = sign*data(nxp-ioff:nxp-ehalo-ioff+1:-1, nyp:1:-1, :)
+    data(nxp+1:nxp+ehalo, ny+m2+1:nyp+nhalo,:) = sign*data(nxp-ioff:nxp-ehalo-ioff+1:-1, ny:ny-nhalo+m1+1:-1,:)
+
+  end subroutine fill_folded_east_halo
+
+  !################################################################################
+  subroutine fill_four_tile_bound(data_all, is, ie, js, je, ioff, joff, tile, &
+                                   ebound, sbound, wbound, nbound )
+    real, dimension(:,:,:,:),       intent(in)    :: data_all
+    integer,                        intent(in)    :: is, ie, js, je
+    integer,                        intent(in)    :: tile, ioff, joff
+    real, dimension(:,:), optional, intent(inout) :: ebound, sbound, wbound, nbound
+    integer                                       :: tw, te, ts, tn
+
+    if(tile == 1 .OR. tile == 3) te = tile + 1
+    if(tile == 2 .OR. tile == 4) te = tile - 1
+    if(tile == 1 .OR. tile == 2) ts = tile + 2
+    if(tile == 3 .OR. tile == 4) ts = tile - 2
+    tw = te;   tn = ts
+    if(present(ebound)) then
+       if( ie == nx ) then
+          ebound(:,:) = data_all(1, js:je+joff, :, te)
+       else
+          ebound(:,:) = data_all(ie+ioff, js:je+joff, :, tile)
+       end if
+    end if
+
+    if(present(wbound)) then
+       if( is == 1 ) then
+          wbound(:,:) = data_all(nx+ioff, js:je+joff, :, tw)
+       else
+          wbound(:,:) = data_all(is, js:je+joff, :, tile)
+       end if
+    end if
+
+    if(present(sbound)) then
+       if( js == 1 ) then
+          sbound(:,:) = data_all(is:ie+ioff, ny+joff, :, ts)
+       else
+          sbound(:,:) = data_all(is:ie+ioff, js, :, tile)
+       end if
+    end if
+
+    if(present(nbound)) then
+       if( je == ny ) then
+          nbound(:,:) = data_all(is:ie+ioff, 1, :, tn)
+       else
+          nbound(:,:) = data_all(is:ie+ioff, je+joff, :, tile)
+       end if
+    end if
+
+    return
+
+  end subroutine fill_four_tile_bound
+
+
+  !################################################################################
+  subroutine fill_torus_bound(data_all, is, ie, js, je, ioff, joff, tile, &
+                                     sbound, wbound)
+    real, dimension(:,:,:),       intent(in)    :: data_all
+    integer,                        intent(in)    :: is, ie, js, je
+    integer,                        intent(in)    :: tile, ioff, joff
+    real, dimension(:,:), optional, intent(inout) :: sbound, wbound
+    integer                                       :: tw, te, ts, tn
+    integer                                       :: js1, js2, is1, is2
+
+    if(tile .NE. 1) call mpp_error(FATAL, "fill_torus_bound: tile must be 1")
+
+    js2 = js
+    js1 = 1
+    if( js == 1 .AND. joff==1 )  then
+      js1 = 2
+      js2 = js+1
+    endif
+    is2 = is
+    is1 = 1
+    if( is == 1 .AND. ioff==1 )  then
+      is1 = 2
+      is2 = is+1
+    endif    
+
+    if(present(wbound)) then
+       if(ioff .NE. 1) call mpp_error(FATAL, "fill_torus_bound: ioff must be 1 when wbound present")
+       if( is == 1 ) then
+          wbound(js1:,:) = data_all(nx+ioff, js2:je+joff, :)
+       else
+          wbound(js1:,:) = data_all(is, js2:je+joff, :)
+       end if
+       if(js1 == 2) then
+          if( is == 1 ) then
+             wbound(1,:) = data_all(nx+1, ny+1, :) 
+          else
+             wbound(1,:) = data_all(is, ny+1, :) 
+          endif
+       endif
+    end if
+
+    if(present(sbound)) then
+       if(joff .NE. 1) call mpp_error(FATAL, "fill_torus_bound: joff must be 1 when sbound present")
+       if( js == 1 ) then
+          sbound(is1:,:) = data_all(is2:ie+ioff, ny+joff, :)
+       else
+          sbound(is1:,:) = data_all(is2:ie+ioff, js, :)
+       end if
+       if(is1 == 2) then
+          if( js == 1 ) then
+             sbound(1,:) = data_all(nx+1, ny+1, :) 
+          else
+             sbound(1,:) = data_all(nx+1, js, :) 
+          endif
+       endif
+    end if
+
+    return
+
+  end subroutine fill_torus_bound
+
+  !################################################################################
+  subroutine fill_folded_north_bound(data_all, is, ie, js, je, ioff, joff, tile, &
+                                     sbound, wbound)
+    real, dimension(:,:,:),       intent(in)    :: data_all
+    integer,                        intent(in)    :: is, ie, js, je
+    integer,                        intent(in)    :: tile, ioff, joff
+    real, dimension(:,:), optional, intent(inout) :: sbound, wbound
+    integer                                       :: tw, te, ts, tn
+    integer                                       :: js1, js2
+
+    if(tile .NE. 1) call mpp_error(FATAL, "fill_folded_north_bound: tile must be 1")
+
+    js2 = js
+    js1 = 1
+    if( js == 1 .AND. joff==1 )  then
+      js1 = 2
+      js2 = js+1
+    endif
+
+    if(present(wbound)) then
+       if( is == 1 ) then
+          wbound(js1:,:) = data_all(nx+ioff, js2:je+joff, :)
+       else
+          wbound(js1:,:) = data_all(is, js2:je+joff, :)
+       end if
+    end if
+
+    if(present(sbound)) then
+       if( js == 1 ) then
+          sbound(:,:) = 0
+       else
+          if( is == 1 .AND. ioff == 1 ) then
+             sbound(1,:)  = data_all(nx+1, js, :)
+             sbound(2:,:) = data_all(is+1:ie+ioff, js, :)
+          else
+             sbound(:,:) = data_all(is:ie+ioff, js, :)          
+          endif
+       end if
+    end if
+
+    return
+
+  end subroutine fill_folded_north_bound
+
+  !################################################################################
+  subroutine fill_cubic_grid_bound(data1_all, data2_all, is, ie, js, je, ioff, joff, tile, sign1, sign2, &
+                                   ebound, sbound, wbound, nbound )
+    real, dimension(:,:,:,:),       intent(in)    :: data1_all, data2_all
+    integer,                        intent(in)    :: is, ie, js, je
+    integer,                        intent(in)    :: tile, ioff, joff, sign1, sign2
+    real, dimension(:,:), optional, intent(inout) :: ebound, sbound, wbound, nbound
+    integer                                       :: tw, te, ts, tn
+
+    if(mod(tile,2) == 0) then ! tile 2, 4, 6
+       tw = tile - 1; te = tile + 2; ts = tile - 2; tn = tile + 1
+       if(te > 6 ) te = te - 6
+       if(ts < 1 ) ts = ts + 6
+       if(tn > 6 ) tn = tn - 6
+       !--- East bound
+       if(present(ebound)) then
+          if(ie == nx) then                
+             ebound(:,:) = sign1*data2_all(nx+joff-js+1:nx-je+1:-1,1,:,te)
+          else
+             ebound(:,:) = data1_all(ie+ioff, js:je+joff, :,tile)
+          end if
+       end if
+       !--- South bound
+       if(present(sbound)) then
+          if(js == 1) then                
+             sbound(:,:) = sign2*data2_all(nx+joff, ny+ioff-is+1:ny-ie+1:-1,:,ts)
+          else
+             sbound(:,:) = data1_all(is:ie+ioff, js, :,tile)
+          end if
+       end if
+
+       !--- West bound
+       if(present(wbound)) then
+          if(is == 1) then
+             wbound(:,:) = data1_all(nx+ioff, js:je+joff,:,tw)
+          else
+             wbound(:,:) = data1_all(is, js:je+joff,:,tile)
+          end if
+       end if
+
+       !--- north bound
+       if(present(nbound)) then
+          if(je == ny) then                
+             nbound(:,:) = data1_all(is:ie+ioff, 1,:,tn)
+          else
+             nbound(:,:) = data1_all(is:ie+ioff, je+joff, :,tile)
+          end if
+       end if
+    else ! tile 1, 3, 5
+       tw = tile - 2; te = tile + 1; ts = tile - 1; tn = tile + 2
+       if(tw < 1 ) tw = tw + 6
+       if(ts < 1 ) ts = ts + 6
+       if(tn > 6 ) tn = tn - 6
+       !--- East bound
+       if(present(ebound)) then
+          if(ie == nx) then                
+             ebound(:,:) = data1_all(1, js:je+joff, :,te) 
+          else
+             ebound(:,:) = data1_all(ie+ioff, js:je+joff, :,tile)
+          end if
+       end if
+       !--- South bound
+       if(present(sbound)) then
+          if(js == 1) then                
+             sbound(:,:) = data1_all(is:ie+ioff,ny+joff,:,ts)
+          else
+             sbound(:,:) = data1_all(is:ie+ioff, js, :,tile)
+          end if
+       end if
+
+       !--- West bound
+       if(present(wbound)) then
+          if(is == 1) then
+             wbound(:,:) = sign1*data2_all(nx+joff-js+1:nx-je+1:-1,ny+ioff,:,tw)
+          else
+             wbound(:,:) = data1_all(is, js:je+joff,:,tile)
+          end if
+       end if
+
+       !--- north bound
+       if(present(nbound)) then
+          if(je == ny) then                
+             nbound(:,:) = sign2*data2_all(1, ny+ioff-is+1:ny-ie+1:-1,:,tn)
+          else
+             nbound(:,:) = data1_all(is:ie+ioff, je+joff, :,tile)
+          end if
+       end if
+
+    end if
+
+  end subroutine fill_cubic_grid_bound
 
   !##############################################################################
-  !--- The following is used to test the refined mosaic. Three cases will be tested, 
-  !--- non-symmetric regular mosaic, symmetric regular mosaic cubic grid. The regular mosaic 
-  !--- contains 4 tiles. East of tile 1 connected with West of tile 2 (refine = 3)
-  !--- and vice verse; East of tile 3 connected with West of tile 4 (refine = 2)
-  !--- and vice verse; North of tile 1 connected with South of tile 3 (refine = 2)
-  !--- and vice verse; North of tile 2 connected with South of tile 4 (refine = 1)
-  !--- and vice verse; So there ar total 8 contacts. 
-  subroutine test_refined_mosaic(type)
+  ! this routine fill the halo points for the cubic grid. ioff and joff is used to distinguish
+  ! T, C, E, or N-cell
+  subroutine fill_cubic_grid_halo(data, data1_all, data2_all, tile, ioff, joff, sign1, sign2)
+    real, dimension(1-whalo:,1-shalo:,:), intent(inout) :: data
+    real, dimension(:,:,:,:),             intent(in)    :: data1_all, data2_all
+    integer,                              intent(in)    :: tile, ioff, joff, sign1, sign2 
+    integer                                             :: lw, le, ls, ln
+
+    if(mod(tile,2) == 0) then ! tile 2, 4, 6
+       lw = tile - 1; le = tile + 2; ls = tile - 2; ln = tile + 1
+       if(le > 6 ) le = le - 6
+       if(ls < 1 ) ls = ls + 6
+       if(ln > 6 ) ln = ln - 6
+       data(1-whalo:0, 1:ny+joff, :) = data1_all(nx-whalo+1:nx, 1:ny+joff, :, lw) ! west 
+       do i = 1, ehalo 
+          data(nx+i+ioff, 1:ny+joff, :)    = sign1*data2_all(nx+joff:1:-1, i+ioff, :, le) ! east 
+       end do
+       do i = 1, shalo 
+          data(1:nx+ioff, 1-i, :)     = sign2*data2_all(nx-i+1, ny+ioff:1:-1, :, ls) ! south 
+       end do
+       data(1:nx+ioff, ny+1+joff:ny+nhalo+joff, :) = data1_all(1:nx+ioff, 1+joff:nhalo+joff, :, ln) ! north
+    else ! tile 1, 3, 5
+       lw = tile - 2; le = tile + 1; ls = tile - 1; ln = tile + 2
+       if(lw < 1 ) lw = lw + 6
+       if(ls < 1 ) ls = ls + 6
+       if(ln > 6 ) ln = ln - 6
+       do i = 1, whalo 
+          data(1-i, 1:ny+joff, :)     = sign1*data2_all(nx+joff:1:-1, ny-i+1, :, lw) ! west 
+       end do
+       data(nx+1+ioff:nx+ehalo+ioff, 1:ny+joff, :) = data1_all(1+ioff:ehalo+ioff, 1:ny+joff, :, le) ! east 
+       data(1:nx+ioff, 1-shalo:0, :)     = data1_all(1:nx+ioff, ny-shalo+1:ny, :, ls) ! south 
+       do i = 1, nhalo 
+          data(1:nx+ioff, ny+i+joff, :)    = sign2*data2_all(i+joff, ny+ioff:1:-1, :, ln) ! north 
+       end do
+    end if
+
+  end subroutine fill_cubic_grid_halo
+    
+   !#####################################################################
+  subroutine test_nonuniform_mosaic( type )
     character(len=*), intent(in) :: type
 
-    type(domain2D)                           :: domain
-    integer,              dimension(4)       :: isMe1, ieMe1, jsMe1, jeMe1
-    integer,              dimension(4)       :: isNb1, ieNb1, jsNb1, jeNb1
-    integer,              dimension(4)       :: isMe2, ieMe2, jsMe2, jeMe2
-    integer,              dimension(4)       :: isNb2, ieNb2, jsNb2, jeNb2
-    integer,              dimension(4)       :: rotation1, rotation2, dirMe1, dirMe2
-    integer, allocatable, dimension(:,:)     :: from_tile1, from_tile2
-    integer                                  :: ntiles, num_contact, npes_on_tile
-    integer                                  :: totpoints, maxtotal, pos, ntiles_on_pe
-    integer                                  :: tNb, nimax, njmax, avgpoints
-    integer                                  :: n, m, l, te, tse, ts, tsw, tw, tnw, tn, tne, nn
-    integer                                  :: noverlap1, noverlap2, total1, total2
-    integer                                  :: isc, iec, jsc, jec, isg, ieg, jsg, jeg
-    integer                                  :: ism, iem, jsm, jem, isd, ied, jsd, jed
-    integer, allocatable, dimension(:)       :: tiles, ni, nj
+    type(domain2D)               :: domain
+    integer                      :: num_contact, ntiles, ntile_per_pe
+    integer                      :: i, j, k, n, nxm, nym, ni, nj, shift
+    integer                      :: ism, iem, jsm, jem, isc, iec, jsc, jec
+    integer                      :: isd, ied, jsd, jed
+    integer                      :: indices(4), msize(2)
+    character(len=128)           :: type2
+
+    integer, allocatable, dimension(:)       :: tile
     integer, allocatable, dimension(:)       :: pe_start, pe_end, tile1, tile2
     integer, allocatable, dimension(:)       :: istart1, iend1, jstart1, jend1
     integer, allocatable, dimension(:)       :: istart2, iend2, jstart2, jend2
-    integer, allocatable, dimension(:,:)     :: layout, global_indices
-    real,    allocatable, dimension(:,:,:,:) :: global_all, global1, global2
-    real,    allocatable, dimension(:,:,:,:) :: x, y, x1, y1, x2, y2
+    integer, allocatable, dimension(:,:)     :: layout2D, global_indices
     real,    allocatable, dimension(:,:,:,:) :: global1_all, global2_all
-    real,    allocatable, dimension(:,:)     :: buffer, buffer1, buffer2, bufferx, buffery
-    real,    allocatable, dimension(:,:)     :: bufferx1, buffery1, bufferx2, buffery2
-    integer                                  :: shift
-    character(len=128)                       :: type2
-    logical                                  :: found
+    real,    allocatable, dimension(:,:,:,:) :: global1, global2, x, y  
 
-    !--- check the type
+    shift = 0
     select case(type)
-    case ("Refined-Four-Tile", "Refined-Symmetric-Four-Tile" )
-       ntiles = 4
-       allocate(ni(ntiles), nj(ntiles))
-       ! "Four-Tile" test case will only run on one pe or multiple 0f 8 ( balanced).
-       if( npes .NE. 1 .AND. npes .NE. 8 .AND. npes .NE. 16 .AND. npes .NE. 32) then
-          call mpp_error(NOTE,'TEST_MPP_DOMAINS(test_refined_mosaic: ' // &
-                  type// ' mosaic will not be tested because npes is not 1, 8, 16 or 32')
-          return
-       end if            
-       ni(1) =   nx; nj(1) =   ny
-       ni(2) =   nx; nj(2) = 3*ny
-       ni(3) = 2*nx; nj(3) =   ny
-       ni(4) =   nx; nj(4) = 2*ny
-       num_contact = 8   
-    case ("Refined-Cubic-Grid")
-       ntiles = 6; num_contact = 12
-       allocate(ni(ntiles), nj(ntiles))
-       ! "Cubic-Grid" will be tested only when nx = ny
-       if( nx /= ny ) then
-           call mpp_error(NOTE,'TEST_MPP_DOMAINS(test_refined_mosaic: ' // &
-           type//' will not be tested because nx is not equal to ny' )
-           return
-       end if
-       ! "Cubic-Grid" test case will only run on one pe or multiple 0f 16 pes ( balanced).
-       if( npes .NE. 1 .AND. mod(npes,16) .NE. 0) then
-          call mpp_error(NOTE,'TEST_MPP_DOMAINS(test_refined_mosaic: ' // &
-                  type//' will not be tested because npes is not 1 and can not be divided by 16')
+    case('Five-Tile') ! one tile will run on pe 0 and other four tiles will run on pe 1
+       shift = 1      ! one extra point for symmetry domain
+       ntiles = 5     ! tile 1 with resolution 2*nx and 2*ny and the tiles are nx and ny.
+       num_contact = 11
+       if(npes .NE. 2) then
+          call mpp_error(NOTE,'TEST_MPP_DOMAINS: Five-Tile mosaic will not be tested because npes is not 2')
           return
+       end if 
+       nxm = 2*nx; nym = 2*ny
+       layout = 1
+       if( pe == 0) then
+          ntile_per_pe = 1
+          allocate(tile(ntile_per_pe))
+          tile = 1
+          indices = (/1,2*nx,1,2*ny/)
+          ni = 2*nx; nj = 2*ny
+       else
+          ntile_per_pe = 4
+          allocate(tile(ntile_per_pe))
+          do n = 1, ntile_per_pe
+             tile(n) = n + 1
+          end do
+          indices = (/1,nx,1,ny/)
+          ni = nx; nj = ny
        end if
-       ni(1) =   nx; nj(1) =   ny
-       ni(2) = 2*nx; nj(2) = 3*ny
-       ni(3) = 2*nx; nj(3) =   ny
-       ni(4) =   nx; nj(4) = 3*ny
-       ni(5) = 2*nx; nj(5) =   ny
-       ni(6) =   nx; nj(6) = 2*ny
+       allocate(pe_start(ntiles), pe_end(ntiles) )
+       pe_start(1) = 0; pe_start(2:) = 1
+       pe_end = pe_start
     case default
-       call mpp_error(FATAL, 'TEST_MPP_DOMAINS(test_refined_mosaic): no such test: '//type)
+       call mpp_error(FATAL, 'TEST_MPP_DOMAINS: no such test: '//type)
     end select
 
-    allocate(layout(2,ntiles), global_indices(4,ntiles), pe_start(ntiles), pe_end(ntiles) )
-    totpoints = sum(ni*nj)
-    if(mod(totpoints, npes) .NE. 0) call mpp_error(FATAL,    &
-        "TEST_MPP_DOMAINS(test_refined_mosaic): totpoints can not be divided by npes")
-    avgpoints = totpoints/npes
-    layout = 1
-    pe_start = 0; pe_end = 0; pos = 0
+    allocate(layout2D(2,ntiles), global_indices(4,ntiles) )
 
     do n = 1, ntiles
-       global_indices(:,n) = (/1, ni(n), 1, nj(n)/)
-       if(npes > 1) then  ! no sharing processor between tiles
-          if( mod(ni(n)*nj(n), avgpoints) .NE. 0) call mpp_error(FATAL, &
-               'TEST_MPP_DOMAINS(test_refined_mosaic): number of points should be divided by average of points in each pe')
-          npes_on_tile = ni(n)*nj(n)/avgpoints
-          call mpp_define_layout( (/1,ni(n),1,nj(n)/), npes_on_tile, layout(:,n) )
-          pe_start(n) = pos
-          pe_end(n)   = pos + npes_on_tile - 1       
-          pos         = pos + npes_on_tile
-       end if
+       if(n==1) then
+          global_indices(:,n) = (/1,2*nx,1,2*ny/)
+       else
+          global_indices(:,n) = (/1,nx,1,ny/)
+       endif  
+!       global_indices(:,n) = indices
+       layout2D(:,n)       = layout
     end do
 
-    ntiles_on_pe = 1
-    if(npes == 1) then
-       ntiles_on_pe = ntiles
-       allocate(tiles(ntiles_on_pe))
-       tiles = (/ (i, i=1,ntiles) /)
-    else
-       ntiles_on_pe = 1
-       allocate(tiles(ntiles_on_pe))
-       do n = 1, ntiles
-          if( pe .GE. pe_start(n) .AND. pe .LE. pe_end(n) ) tiles = n
-       end do
-    end if
-
     allocate(tile1(num_contact), tile2(num_contact) )
     allocate(istart1(num_contact), iend1(num_contact), jstart1(num_contact), jend1(num_contact) ) 
     allocate(istart2(num_contact), iend2(num_contact), jstart2(num_contact), jend2(num_contact) ) 
 
     !--- define domain
     select case(type)
-    case( 'Refined-Four-Tile', 'Refined-Symmetric-Four-Tile' )
-       call define_fourtile_mosaic(type, domain, ni, nj, global_indices, layout, pe_start, pe_end, &
-                                   type == 'Refined-Symmetric-Four-Tile'   )
-    case( 'Refined-Cubic-Grid' )
-       call define_cubic_mosaic(type, domain, ni, nj, global_indices, layout, pe_start, pe_end )
+    case( 'Five-Tile' )
+       !--- Contact line 1, between tile 1 (EAST) and tile 2 (WEST)
+       tile1(1) = 1; tile2(1) = 2
+       istart1(1) = 2*nx; iend1(1) = 2*nx; jstart1(1) = 1;  jend1(1) = ny
+       istart2(1) = 1;    iend2(1) = 1;    jstart2(1) = 1;  jend2(1) = ny
+       !--- Contact line 2, between tile 1 (EAST) and tile 4 (WEST)
+       tile1(2) = 1; tile2(2) = 4
+       istart1(2) = 2*nx; iend1(2) = 2*nx; jstart1(2) = ny+1; jend1(2) = 2*ny
+       istart2(2) = 1;    iend2(2) = 1;    jstart2(2) = 1;    jend2(2) = ny
+       !--- Contact line 3, between tile 1 (SOUTH) and tile 1 (NORTH)
+       tile1(3) = 1; tile2(3) = 1
+       istart1(3) = 1; iend1(3) = 2*nx; jstart1(3) = 1;    jend1(3) = 1
+       istart2(3) = 1; iend2(3) = 2*nx; jstart2(3) = 2*ny; jend2(3) = 2*ny
+       !--- Contact line 4, between tile 1 (WEST) and tile 3 (EAST)
+       tile1(4) = 1; tile2(4) = 3
+       istart1(4) = 1;  iend1(4) = 1;  jstart1(4) = 1;  jend1(4) = ny
+       istart2(4) = nx; iend2(4) = nx; jstart2(4) = 1;  jend2(4) = ny
+       !--- Contact line 5, between tile 1 (WEST) and tile 5 (EAST)
+       tile1(5) = 1; tile2(5) = 5
+       istart1(5) = 1;  iend1(5) = 1;  jstart1(5) = ny+1;  jend1(5) = 2*ny
+       istart2(5) = nx; iend2(5) = nx; jstart2(5) = 1;     jend2(5) = ny
+       !--- Contact line 6, between tile 2 (EAST) and tile 3 (WEST)
+       tile1(6) = 2; tile2(6) = 3
+       istart1(6) = nx; iend1(6) = nx; jstart1(6) = 1;  jend1(6) = ny
+       istart2(6) = 1;  iend2(6) = 1;  jstart2(6) = 1;  jend2(6) = ny       
+       !--- Contact line 7, between tile 2 (SOUTH) and tile 4 (NORTH)  --- cyclic
+       tile1(7) = 2; tile2(7) = 4
+       istart1(7) = 1;  iend1(7) = nx; jstart1(7) = 1;   jend1(7) = 1
+       istart2(7) = 1;  iend2(7) = nx; jstart2(7) = ny;  jend2(7) = ny
+       !--- Contact line 8, between tile 2 (NORTH) and tile 4 (SOUTH) 
+       tile1(8) = 2; tile2(8) = 4
+       istart1(8) = 1;  iend1(8) = nx; jstart1(8) = ny;  jend1(8) = ny
+       istart2(8) = 1;  iend2(8) = nx; jstart2(8) = 1;   jend2(8) = 1
+       !--- Contact line 9, between tile 3 (SOUTH) and tile 5 (NORTH)  --- cyclic
+       tile1(9) = 3; tile2(9) = 5
+       istart1(9) = 1;  iend1(9) = nx; jstart1(9) = 1;   jend1(9) = 1
+       istart2(9) = 1;  iend2(9) = nx; jstart2(9) = ny;  jend2(9) = ny
+       !--- Contact line 10, between tile 3 (NORTH) and tile 5 (SOUTH) 
+       tile1(10) = 3; tile2(10) = 5
+       istart1(10) = 1;  iend1(10) = nx; jstart1(10) = ny;  jend1(10) = ny
+       istart2(10) = 1;  iend2(10) = nx; jstart2(10) = 1;   jend2(10) = 1
+       !--- Contact line 11, between tile 4 (EAST) and tile 5 (WEST)
+       tile1(11) = 4; tile2(11) = 5
+       istart1(11) = nx; iend1(11) = nx; jstart1(11) = 1;  jend1(11) = ny
+       istart2(11) = 1;  iend2(11) = 1;  jstart2(11) = 1;  jend2(11) = ny  
+       msize(1) = 2*nx + whalo + ehalo
+       msize(2) = 2*ny + shalo + nhalo
+       call mpp_define_mosaic(global_indices, layout2D, domain, ntiles, num_contact, tile1, tile2, &
+            istart1, iend1, jstart1, jend1, istart2, iend2, jstart2, jend2,      &
+            pe_start, pe_end, whalo=whalo, ehalo=ehalo, shalo=shalo, nhalo=nhalo,       &
+            name = type, memory_size = msize, symmetry = .true.  )
     end select
-
-    !--- first test mpp_get_mosaic_refine_overlap
-    maxtotal = 0
-    allocate(from_tile1(4, ntiles_on_pe), from_tile2(4, ntiles_on_pe))
-    do n = 1, ntiles_on_pe
-       rotation2 = ZERO
-       noverlap1 = mpp_get_refine_overlap_number(domain, tile_count=n)
-       call mpp_get_mosaic_refine_overlap(domain, isMe1, ieMe1, jsMe1, jeMe1, isNb1, ieNb1, jsNb1, jeNb1, &
-                                          dirMe1, rotation1, tile_count = n)
-       total1 = sum( (ieNb1(1:noverlap1)-isNb1(1:noverlap1)+1) * (jeNb1(1:noverlap1)-jsNb1(1:noverlap1)+1)  )
-
-       !--- the following will figure out the overlapping
-       call mpp_get_compute_domain(domain, isc, iec, jsc, jec, tile_count=n)
-       call mpp_get_global_domain(domain, isg, ieg, jsg, jeg, tile_count=n)
-       noverlap2 = 0; total2 = 0
-       select case ( type )
-       case ( 'Refined-Four-Tile', 'Refined-Symmetric-Four-Tile' )
-          if( iec == ieg ) then   ! --- EAST
-             noverlap2 = noverlap2 + 1
-             if( mod(tiles(n),2) == 1) then ! tile 1, 3    
-                tNb = tiles(n) + 1
-             else                           ! tile 2, 4
-                tNb = tiles(n) - 1
-             endif
-             from_tile2(noverlap2,n) = tNb
-             dirMe2(noverlap2) = 1
-             isMe2(noverlap2) = iec + 1;    ieMe2(noverlap2) = iec + ehalo
-             jsMe2(noverlap2) = max(jsg,jsc - shalo); jeMe2(noverlap2) = min(jeg, jec+nhalo)
-             isNb2(noverlap2) = 1;          ieNb2(noverlap2) = ehalo
-             select case(tiles(n))
-             case(1)
-                jsNb2(noverlap2) = (jsMe2(noverlap2)-1)*3+1;  jeNb2(noverlap2) = jeMe2(noverlap2)*3
-             case(2)
-                jsNb2(noverlap2) = (jsMe2(noverlap2)-1)/3+1; jeNb2(noverlap2) = ceiling(jeMe2(noverlap2)/3.)
-             case(3)
-                jsNb2(noverlap2) = (jsMe2(noverlap2)-1)*2+1;  jeNb2(noverlap2) = jeMe2(noverlap2)*2
-             case(4)
-                jsNb2(noverlap2) = (jsMe2(noverlap2)-1)/2+1; jeNb2(noverlap2) = ceiling(jeMe2(noverlap2)/2.)
-             end select
-             total2 = total2 + (ieNb2(noverlap2) - isNb2(noverlap2) + 1) * (jeNb2(noverlap2) - jsNb2(noverlap2) + 1)
-          end if
-
-          if( jsc == jsg .AND. mod(tiles(n),2) == 1 ) then   ! --- SOUTH (only tile 1 and 3)
-             noverlap2 = noverlap2 + 1
-             tNb = mod(tiles(n)+2, ntiles)
-             from_tile2(noverlap2,n) = tNb
-             dirMe2(noverlap2) = 3
-             isMe2(noverlap2) = max(isg,isc-whalo); ieMe2(noverlap2) = min(ieg, iec+ehalo)
-             jsMe2(noverlap2) = jsc - shalo;        jeMe2(noverlap2) = jsc - 1
-             if(tiles(n) == 1) then  ! refinement is 2
-                isNb2(noverlap2) = (isMe2(noverlap2)-1)*2+1;    ieNb2(noverlap2) = ieMe2(noverlap2)*2
-             else                   ! refinement is 2
-                isNb2(noverlap2) = (isMe2(noverlap2)-1)/2+1;    ieNb2(noverlap2) = ceiling(ieMe2(noverlap2)/2.)
-             end if
-             jsNb2(noverlap2) = nj(tNb) - shalo + 1; jeNb2(noverlap2) = nj(tNb)
-             total2 = total2 + (ieNb2(noverlap2) - isNb2(noverlap2) + 1) * (jeNb2(noverlap2) - jsNb2(noverlap2) + 1)
-          end if
-
-          if( isc == isg ) then   ! --- WEST
-             noverlap2 = noverlap2 + 1
-             if( mod(tiles(n),2) == 1) then ! tile 1, 3    
-                tNb = tiles(n) + 1
-             else                           ! tile 2, 4
-                tNb = tiles(n) - 1
-             endif
-             from_tile2(noverlap2,n) = tNb
-             dirMe2(noverlap2) = 5
-             isMe2(noverlap2) = isc - whalo;          ieMe2(noverlap2) = isc - 1
-             jsMe2(noverlap2) = max(jsg,jsc - shalo); jeMe2(noverlap2) = min(jeg, jec+nhalo)
-             isNb2(noverlap2) = ni(tNb) - whalo + 1; ieNb2(noverlap2) = ni(tNb)
-             select case(tiles(n))
-             case(1)
-                jsNb2(noverlap2) = (jsMe2(noverlap2)-1)*3+1;  jeNb2(noverlap2) = jeMe2(noverlap2)*3
-             case(2)
-                jsNb2(noverlap2) = (jsMe2(noverlap2)-1)/3+1; jeNb2(noverlap2) = ceiling(jeMe2(noverlap2)/3.)
-             case(3)
-                jsNb2(noverlap2) = (jsMe2(noverlap2)-1)*2+1;  jeNb2(noverlap2) = jeMe2(noverlap2)*2
-             case(4)
-                jsNb2(noverlap2) = (jsMe2(noverlap2)-1)/2+1; jeNb2(noverlap2) = ceiling(jeMe2(noverlap2)/2.)
-             end select
-             total2 = total2 + (ieNb2(noverlap2) - isNb2(noverlap2) + 1) * (jeNb2(noverlap2) - jsNb2(noverlap2) + 1)
-          end if
-
-          if( jec == jeg .AND. mod(tiles(n),2) == 1 ) then   ! --- NORTH (only tile 1 and 3)
-             noverlap2 = noverlap2 + 1
-             tNb = mod(tiles(n)+2, ntiles)
-             from_tile2(noverlap2,n) = tNb
-             dirMe2(noverlap2) = 7
-             isMe2(noverlap2) = max(isg,isc-whalo); ieMe2(noverlap2) = min(ieg, iec+ehalo)
-             jsMe2(noverlap2) = jec + 1;        jeMe2(noverlap2) = jec + nhalo
-             if(tiles(n) == 1) then  ! refinement is 2
-                isNb2(noverlap2) = (isMe2(noverlap2)-1)*2+1;    ieNb2(noverlap2) = ieMe2(noverlap2)*2
-             else                   ! refinement is 2
-                isNb2(noverlap2) = (isMe2(noverlap2)-1)/2+1;    ieNb2(noverlap2) = ceiling(ieMe2(noverlap2)/2.)
-             end if
-             jsNb2(noverlap2) = 1;          jeNb2(noverlap2) = nhalo
-             total2 = total2 + (ieNb2(noverlap2) - isNb2(noverlap2) + 1) * (jeNb2(noverlap2) - jsNb2(noverlap2) + 1)
-          end if
-
-       case ( 'Refined-Cubic-Grid' )
-          select case( tiles(n) )
-          case ( 1 )  ! possible refined overlap will be at EAST, WEST
-             if( iec == ieg ) then   ! --- EAST
-                noverlap2 = noverlap2 + 1
-                tNb = 2
-                from_tile2(noverlap2,n) = tNb
-                dirMe2(noverlap2) = 1
-                isMe2(noverlap2) = iec + 1;                  ieMe2(noverlap2) = iec + ehalo
-                jsMe2(noverlap2) = max(jsg,jsc - shalo);     jeMe2(noverlap2) = min(jeg, jec+nhalo)
-                isNb2(noverlap2) = 1;                        ieNb2(noverlap2) = ehalo
-                jsNb2(noverlap2) = (jsMe2(noverlap2)-1)*3+1; jeNb2(noverlap2) = jeMe2(noverlap2)*3
-                total2 = total2 + (ieNb2(noverlap2) - isNb2(noverlap2) + 1) * (jeNb2(noverlap2) - jsNb2(noverlap2) + 1) 
-             end if
-             if( isc == isg ) then   ! --- WEST
-                noverlap2 = noverlap2 + 1
-                tNb = 5
-                from_tile2(noverlap2,n) = tNb
-                dirMe2(noverlap2) = 5
-                isMe2(noverlap2) = isc - whalo;                  ieMe2(noverlap2) = isc - 1
-                jsMe2(noverlap2) = max(jsg,jsc - shalo);         jeMe2(noverlap2) = min(jeg, jec+nhalo)
-                isNb2(noverlap2) = (nj(1)-jeMe2(noverlap2))*2+1; ieNb2(noverlap2) = (nj(1)-jsMe2(noverlap2)+1)*2
-                jsNb2(noverlap2) = nj(tNb) - whalo + 1;          jeNb2(noverlap2) = nj(tNb)
-                total2 = total2 + (ieNb2(noverlap2) - isNb2(noverlap2) + 1) * (jeNb2(noverlap2) - jsNb2(noverlap2) + 1)
-                rotation2(noverlap2) = NINETY
-             end if
-          case ( 2 )  ! possible refined overlap will be at EAST, WEST
-             if( iec == ieg ) then   ! --- EAST
-                noverlap2 = noverlap2 + 1
-                tNb = 4
-                from_tile2(noverlap2,n) = tNb
-                dirMe2(noverlap2) = 1
-                isMe2(noverlap2) = iec + 1;                      ieMe2(noverlap2) = iec + ehalo
-                jsMe2(noverlap2) = max(jsg,jsc - shalo);         jeMe2(noverlap2) = min(jeg, jec+nhalo)
-                isNb2(noverlap2) = (nj(2)-jeMe2(noverlap2))/3+1; ieNb2(noverlap2) = ceiling((nj(2)-jsMe2(noverlap2)+1)/3.)
-                jsNb2(noverlap2) = 1;                            jeNb2(noverlap2) = ehalo
-                total2 = total2 + (ieNb2(noverlap2) - isNb2(noverlap2) + 1) * (jeNb2(noverlap2) - jsNb2(noverlap2) + 1)
-                rotation2(noverlap2) = NINETY       
-             end if
-             if( isc == isg ) then   ! --- WEST
-                noverlap2 = noverlap2 + 1
-                tNb = 1
-                from_tile2(noverlap2,n) = tNb
-                dirMe2(noverlap2) = 5
-                isMe2(noverlap2) = isc - whalo;              ieMe2(noverlap2) = isc - 1
-                jsMe2(noverlap2) = max(jsg,jsc - shalo);     jeMe2(noverlap2) = min(jeg, jec+nhalo)
-                isNb2(noverlap2) = ni(tNb) - whalo + 1;      ieNb2(noverlap2) = ni(tNb)
-                jsNb2(noverlap2) = (jsMe2(noverlap2)-1)/3+1; jeNb2(noverlap2) = ceiling(jeMe2(noverlap2)/3.)
-                total2 = total2 + (ieNb2(noverlap2) - isNb2(noverlap2) + 1) * (jeNb2(noverlap2) - jsNb2(noverlap2) + 1)
-             end if
-          case ( 3 )  ! possible refined overlap will be at EAST, NORTH          
-             if( iec == ieg ) then   ! --- EAST
-                noverlap2 = noverlap2 + 1
-                tNb = 4
-                from_tile2(noverlap2,n) = tNb
-                dirMe2(noverlap2) = 1
-                isMe2(noverlap2) = iec + 1;                  ieMe2(noverlap2) = iec + ehalo
-                jsMe2(noverlap2) = max(jsg,jsc - shalo);     jeMe2(noverlap2) = min(jeg, jec+nhalo)
-                isNb2(noverlap2) = 1;                        ieNb2(noverlap2) = ehalo
-                jsNb2(noverlap2) = (jsMe2(noverlap2)-1)*3+1; jeNb2(noverlap2) = jeMe2(noverlap2)*3
-                total2 = total2 + (ieNb2(noverlap2) - isNb2(noverlap2) + 1) * (jeNb2(noverlap2) - jsNb2(noverlap2) + 1) 
-             end if
-             if( jec == jeg ) then   ! --- NORTH
-                noverlap2 = noverlap2 + 1
-                tNb = 5
-                from_tile2(noverlap2,n) = tNb
-                dirMe2(noverlap2) = 7
-                isMe2(noverlap2) = max(isg,isc - whalo);        ieMe2(noverlap2) = min(ieg, iec+ehalo)
-                jsMe2(noverlap2) = jec + 1;                     jeMe2(noverlap2) = jec + nhalo
-                isNb2(noverlap2) = 1;                           ieNb2(noverlap2) = nhalo
-                jsNb2(noverlap2) = (ni(3)-ieMe2(noverlap2))/2+1; jeNb2(noverlap2) = ceiling((ni(3)-isMe2(noverlap2)+1)/2.)
-                total2 = total2 + (ieNb2(noverlap2) - isNb2(noverlap2) + 1) * (jeNb2(noverlap2) - jsNb2(noverlap2) + 1) 
-                rotation2(noverlap2) = MINUS_NINETY
-             end if
-          case ( 4 )  ! possible refined overlap will be at NORTH, EAST, SOUTH, WEST
-             if( jec == jeg ) then   ! --- NORTH
-                noverlap2 = noverlap2 + 1
-                tNb = 5
-                from_tile2(noverlap2,n) = tNb
-                dirMe2(noverlap2) = 7
-                isMe2(noverlap2) = max(isg,isc - whalo);        ieMe2(noverlap2) = min(ieg, iec+ehalo)
-                jsMe2(noverlap2) = jec + 1;                     jeMe2(noverlap2) = jec + nhalo
-                isNb2(noverlap2) = (isMe2(noverlap2)-1)*2+1;    ieNb2(noverlap2) = ieMe2(noverlap2)*2
-                jsNb2(noverlap2) = 1;                           jeNb2(noverlap2) = nhalo
-                total2 = total2 + (ieNb2(noverlap2) - isNb2(noverlap2) + 1) * (jeNb2(noverlap2) - jsNb2(noverlap2) + 1) 
-             end if
-             if( iec == ieg ) then   ! --- EAST
-                noverlap2 = noverlap2 + 1
-                tNb = 6
-                from_tile2(noverlap2,n) = tNb
-                dirMe2(noverlap2) = 1
-                isMe2(noverlap2) = iec + 1;                        ieMe2(noverlap2) = iec + ehalo
-                jsMe2(noverlap2) = max(jsg,jsc - shalo);           jeMe2(noverlap2) = min(jeg, jec+nhalo)
-                isNb2(noverlap2) = (nj(4)-jeMe2(noverlap2)-1)/3+1; ieNb2(noverlap2) = ceiling((nj(4)-jsMe2(noverlap2)+1)/3.)
-                jsNb2(noverlap2) = 1;                              jeNb2(noverlap2) = ehalo
-                total2 = total2 + (ieNb2(noverlap2) - isNb2(noverlap2) + 1) * (jeNb2(noverlap2) - jsNb2(noverlap2) + 1)
-                rotation2(noverlap2) = NINETY
-             end if
-             if( jsc == jsg ) then   ! --- SOUTH
-                noverlap2 = noverlap2 + 1
-                tNb = 2
-                from_tile2(noverlap2,n) = tNb
-                dirMe2(noverlap2) = 3
-                isMe2(noverlap2) = max(isg,isc - whalo);        ieMe2(noverlap2) = min(ieg, iec+ehalo)
-                jsMe2(noverlap2) = jsc - shalo;                 jeMe2(noverlap2) = jsc  - 1
-                isNb2(noverlap2) = ni(tNb) - shalo + 1;         ieNb2(noverlap2) = ni(tNb)
-                jsNb2(noverlap2) = (ni(4)-ieMe2(noverlap2))*3+1; jeNb2(noverlap2) = (ni(4)-isMe2(noverlap2)+1)*3
-                total2 = total2 + (ieNb2(noverlap2) - isNb2(noverlap2) + 1) * (jeNb2(noverlap2) - jsNb2(noverlap2) + 1) 
-                rotation2(noverlap2) = MINUS_NINETY
-             end if
-             if( isc == isg ) then   ! --- WEST
-                noverlap2 = noverlap2 + 1
-                tNb = 3
-                from_tile2(noverlap2,n) = tNb
-                dirMe2(noverlap2) = 5
-                isMe2(noverlap2) = isc - whalo;              ieMe2(noverlap2) = isc - 1
-                jsMe2(noverlap2) = max(jsg,jsc - shalo);     jeMe2(noverlap2) = min(jeg, jec+nhalo)
-                isNb2(noverlap2) = ni(tNb) - whalo + 1;      ieNb2(noverlap2) = ni(tNb)
-                jsNb2(noverlap2) = (jsMe2(noverlap2)-1)/3+1; jeNb2(noverlap2) = ceiling(jeMe2(noverlap2)/3.)
-                total2 = total2 + (ieNb2(noverlap2) - isNb2(noverlap2) + 1) * (jeNb2(noverlap2) - jsNb2(noverlap2) + 1)
-             end if
-
-         case ( 5 )  ! possible refined overlap will be at EAST, NORTH, WEST, SOUTH       
-             if( iec == ieg ) then   ! --- EAST
-                noverlap2 = noverlap2 + 1
-                tNb = 6
-                from_tile2(noverlap2,n) = tNb
-                dirMe2(noverlap2) = 1
-                isMe2(noverlap2) = iec + 1;                  ieMe2(noverlap2) = iec + ehalo
-                jsMe2(noverlap2) = max(jsg,jsc - shalo);     jeMe2(noverlap2) = min(jeg, jec+nhalo)
-                isNb2(noverlap2) = 1;                        ieNb2(noverlap2) = ehalo
-                jsNb2(noverlap2) = (jsMe2(noverlap2)-1)*2+1; jeNb2(noverlap2) = jeMe2(noverlap2)*2
-                total2 = total2 + (ieNb2(noverlap2) - isNb2(noverlap2) + 1) * (jeNb2(noverlap2) - jsNb2(noverlap2) + 1) 
-             end if
-             if( jec == jeg ) then   ! --- NORTH
-                noverlap2 = noverlap2 + 1
-                tNb = 1
-                from_tile2(noverlap2,n) = tNb
-                dirMe2(noverlap2) = 7
-                isMe2(noverlap2) = max(isg,isc - whalo);         ieMe2(noverlap2) = min(ieg, iec+ehalo)
-                jsMe2(noverlap2) = jec + 1;                      jeMe2(noverlap2) = jec + nhalo
-                isNb2(noverlap2) = 1;                            ieNb2(noverlap2) = nhalo
-                jsNb2(noverlap2) = (ni(5)-ieMe2(noverlap2))/2+1; jeNb2(noverlap2) = ceiling((ni(5)-isMe2(noverlap2)+1)/2.)
-                total2 = total2 + (ieNb2(noverlap2) - isNb2(noverlap2) + 1) * (jeNb2(noverlap2) - jsNb2(noverlap2) + 1)
-                rotation2(noverlap2) = MINUS_NINETY 
-             end if
-             if( isc == isg ) then   ! --- WEST
-                noverlap2 = noverlap2 + 1
-                tNb = 3
-                from_tile2(noverlap2,n) = tNb
-                dirMe2(noverlap2) = 5
-                isMe2(noverlap2) = isc - whalo;                  ieMe2(noverlap2) = isc - 1
-                jsMe2(noverlap2) = max(jsg,jsc - shalo);         jeMe2(noverlap2) = min(jeg, jec+nhalo)
-                isNb2(noverlap2) = (nj(5)-jeMe2(noverlap2))*2+1; ieNb2(noverlap2) = (nj(5)-jsMe2(noverlap2)+1)*2
-                jsNb2(noverlap2) = nj(tNb) - whalo + 1;          jeNb2(noverlap2) = nj(tNb)
-                total2 = total2 + (ieNb2(noverlap2) - isNb2(noverlap2) + 1) * (jeNb2(noverlap2) - jsNb2(noverlap2) + 1)
-                rotation2(noverlap2) = NINETY
-             end if
-             if( jsc == jsg ) then   ! --- SOUTH
-                noverlap2 = noverlap2 + 1
-                tNb = 4
-                from_tile2(noverlap2,n) = tNb
-                dirMe2(noverlap2) = 3
-                isMe2(noverlap2) = max(isg,isc - whalo);        ieMe2(noverlap2) = min(ieg, iec+ehalo)
-                jsMe2(noverlap2) = jsc - shalo;                 jeMe2(noverlap2) = jsc  - 1
-                isNb2(noverlap2) = (isMe2(noverlap2)-1)/2+1;    ieNb2(noverlap2) = ceiling(ieMe2(noverlap2)/2.)
-                jsNb2(noverlap2) = nj(tNb) - shalo + 1;         jeNb2(noverlap2) = nj(tNb)
-                total2 = total2 + (ieNb2(noverlap2) - isNb2(noverlap2) + 1) * (jeNb2(noverlap2) - jsNb2(noverlap2) + 1) 
-             end if
-
-
-          case ( 6 )  ! possible refined overlap will be at SOUTH, WEST
-             if( jsc == jsg ) then   ! --- SOUTH
-                noverlap2 = noverlap2 + 1
-                tNb = 4
-                from_tile2(noverlap2,n) = tNb
-                dirMe2(noverlap2) = 3
-                isMe2(noverlap2) = max(isg,isc - whalo);        ieMe2(noverlap2) = min(ieg, iec+ehalo)
-                jsMe2(noverlap2) = jsc - shalo;                 jeMe2(noverlap2) = jsc  - 1
-                isNb2(noverlap2) = ni(tNb) - shalo + 1;         ieNb2(noverlap2) = ni(tNb)
-                jsNb2(noverlap2) = (ni(6)-ieMe2(noverlap2))*3+1; jeNb2(noverlap2) = (ni(6)-isMe2(noverlap2)+1)*3
-                total2 = total2 + (ieNb2(noverlap2) - isNb2(noverlap2) + 1) * (jeNb2(noverlap2) - jsNb2(noverlap2) + 1) 
-                rotation2(noverlap2) = MINUS_NINETY
-             end if
-             if( isc == isg ) then   ! --- WEST
-                noverlap2 = noverlap2 + 1
-                tNb = 5
-                from_tile2(noverlap2,n) = tNb
-                dirMe2(noverlap2) = 5
-                isMe2(noverlap2) = isc - whalo;              ieMe2(noverlap2) = isc - 1
-                jsMe2(noverlap2) = max(jsg,jsc - shalo);     jeMe2(noverlap2) = min(jeg, jec+nhalo)
-                isNb2(noverlap2) = ni(tNb) - whalo + 1;      ieNb2(noverlap2) = ni(tNb)
-                jsNb2(noverlap2) = (jsMe2(noverlap2)-1)/2+1; jeNb2(noverlap2) = ceiling(jeMe2(noverlap2)/2.)
-                total2 = total2 + (ieNb2(noverlap2) - isNb2(noverlap2) + 1) * (jeNb2(noverlap2) - jsNb2(noverlap2) + 1)
-             end if
-          end select
-       end select
-
-       if(total1 .NE. total2) call mpp_error(FATAL, "test_mpp_domains: mismatch on total number of points")
-       !--- we add one extra point in each direction for the consideration of symmetric domain.
-       total2 = sum( (ieNb2(1:noverlap2) - isNb2(1:noverlap2) + 2) * (jeNb2(1:noverlap2) - jsNb2(1:noverlap2) + 2) )
-       maxtotal = max(maxtotal, total2)
-       !--- comparing
-       if( noverlap1 .NE. noverlap2 ) call mpp_error(FATAL, "test_mpp_domains: mismatch on number of overlapping region")
-       do m = 1, noverlap1
-          found = .false.
-          do l = 1, noverlap2
-             if(dirMe1(m) == dirMe2(l)) then
-                found = .true.
-                exit
-             endif
-          enddo
-          from_tile1(m,n) = from_tile2(l,n)
-          if(.not. found) call mpp_error(FATAL, "test_mpp_domains: mismatch on direction")
-          if( (isMe1(m) .NE. isMe2(l)) .OR. (ieMe1(m) .NE. ieMe2(l))        &
-              .OR. (jsMe1(m) .NE. jsMe2(l)) .OR. (jeMe1(m) .NE. jeMe2(l)) ) &
-              call mpp_error(FATAL, "test_mpp_domains: mismatch on myself overlapping index")
-          if( (isNb1(m) .NE. isNb2(l)) .OR. (ieNb1(m) .NE. ieNb2(l))        &
-              .OR. (jsNb1(m) .NE. jsNb2(l)) .OR. (jeNb1(m) .NE. jeNb2(l)) ) &
-              call mpp_error(FATAL, "test_mpp_domains: mismatch on neighbor overlapping index") 
-          if(rotation1(m) .NE. rotation2(l)) call mpp_error(FATAL, "test_mpp_domains: mismatch on rotation angle");
-       end do
-    end do
-
+    
     !--- setup data
-    nimax = maxval(ni); njmax = maxval(nj)
-    allocate(global_all(1:nimax,1:njmax,nz,ntiles) )
-    allocate(global2(1-whalo:nimax+ehalo,1-shalo:njmax+nhalo,nz, ntiles_on_pe) ) 
-    global2 = 0
+    allocate(global1_all(1:nxm,1:nym,nz, ntiles) )  
+    allocate(global1(1-whalo:ni+ehalo,1-shalo:nj+nhalo,nz, ntile_per_pe) )   
     do n = 1, ntiles
        do k = 1, nz
-          do j = 1, njmax
-             do i = 1, nimax
-                global_all(i,j,k,n) = n + i*1.0e-3 + j*1.0e-6 + k*1.0e-9
+          do j = 1, nym
+             do i = 1, nxm
+                global1_all(i,j,k,n) = n + i*1.0e-3 + j*1.0e-6 + k*1.0e-9
              end do
           end do
        end do
     end do
 
-    do n = 1, ntiles_on_pe
-       nn = tiles(n)
-       global2(1:ni(nn),1:nj(nn),:,n) = global_all(1:ni(nn),1:nj(nn),:,nn)
+    do n = 1, ntile_per_pe
+       global1(1:ni,1:nj,:,n) = global1_all(1:ni,1:nj,:,tile(n))
     end do
 
+    call mpp_get_compute_domain( domain, isc, iec, jsc, jec )
+    call mpp_get_data_domain   ( domain, isd, ied, jsd, jed )
     call mpp_get_memory_domain   ( domain, ism, iem, jsm, jem )
-    allocate( x (ism:iem,jsm:jem,nz, ntiles_on_pe) )
-    allocate( x1(ism:iem,jsm:jem,nz, ntiles_on_pe) )
-    allocate( x2(ism:iem,jsm:jem,nz, ntiles_on_pe) )
-    x = 0
-
-    do n = 1, ntiles_on_pe
-       call mpp_get_compute_domain( domain, isc, iec, jsc, jec, tile_count=n  )
-       call mpp_get_data_domain   ( domain, isd, ied, jsd, jed, tile_count=n )
-       x(isc:iec,jsc:jec,:,:) = global2(isc:iec,jsc:jec,:,:)
-    end do
-    x1 = x; x2 = x
 
-    allocate(buffer (maxtotal*nz, ntiles_on_pe))
-    allocate(buffer1(maxtotal*nz, ntiles_on_pe))
-    allocate(buffer2(maxtotal*nz, ntiles_on_pe))
+    allocate( x (ism:iem,jsm:jem,nz, ntile_per_pe) )
+    x = 0.
+    x(isc:iec,jsc:jec,:,:) = global1(isc:iec,jsc:jec,:,:)
 
-    !--- call mpp_update_domains to update domain 
-    do n = 1, ntiles_on_pe
-       call mpp_update_domains( x(:,:,:,n), domain, buffer=buffer(:,n), tile_count=n )
+    !--- fill up the value at halo points
+    do n = 1, ntile_per_pe
+       call fill_five_tile_halo(global1(:,:,:,n), global1_all, tile(n), 0, 0 )
     end do
 
-    !--- multiple varaibles update
-    do n = 1, ntiles_on_pe
-       call mpp_update_domains( x1(:,:,:,n), domain, buffer=buffer1(:,n), complete=.false., tile_count=n )
-       call mpp_update_domains( x2(:,:,:,n), domain, buffer=buffer2(:,n), complete=.true., tile_count=n )
+    ! full update
+    id = mpp_clock_id( type, flags=MPP_CLOCK_SYNC+MPP_CLOCK_DETAILED )
+    call mpp_clock_begin(id)
+    do n = 1, ntile_per_pe
+       call mpp_update_domains( x(:,:,:,n), domain, tile_count = n )
     end do
+    call mpp_clock_end(id)
 
-    !--- fill up the value at halo points and compare the value at buffer.
-    do n = 1, ntiles_on_pe  
-
-       !--- comparing the buffer.
-       noverlap1 = mpp_get_refine_overlap_number(domain, tile_count=n)
-       call mpp_get_mosaic_refine_overlap(domain, isMe1, ieMe1, jsMe1, jeMe1, isNb1, ieNb1, jsNb1, jeNb1, &
-                                          dirMe1, rotation1, tile_count = n)             
-       pos = 0
-       do m = 1, noverlap1       
-          do k = 1, nz
-             do j = jsNb1(m), jeNb1(m)
-                do i = isNb1(m), ieNb1(m)
-                   pos = pos + 1
-                   if(global_all(i,j,k,from_tile1(m,n)) .NE. buffer(pos,n) ) then
-                      write(stdunit, 111) 'x', type, mpp_pe(), i, j, k, buffer(pos,n), global_all(i,j,k,from_tile1(m,n)) 
-                      call mpp_error(FATAL, "test_refined_mosaic: mismatch between buffer data and actual data for "//type )
-                   end if
-                   if(global_all(i,j,k,from_tile1(m,n)) .NE. buffer1(pos,n) ) then
-                      write(stdunit, 111) 'x1', type, mpp_pe(), i, j, k, buffer1(pos,n), global_all(i,j,k,from_tile1(m,n)) 
-                      call mpp_error(FATAL, "test_refined_mosaic: mismatch between buffer data and actual data for "//type )
-                   end if
-                   if(global_all(i,j,k,from_tile1(m,n)) .NE. buffer2(pos,n) ) then
-                      write(stdunit, 111) 'x2', type, mpp_pe(), i, j, k, buffer2(pos,n), global_all(i,j,k,from_tile1(m,n)) 
-                      call mpp_error(FATAL, "test_refined_mosaic: mismatch between buffer data and actual data for "//type )
-                   end if
-                end do
-             end do
-          end do
-       end do
+   do n = 1, ntile_per_pe
+      write(type2, *)type, " at tile_count = ",n
+      call compare_checksums( x(isd:ied,jsd:jed,:,n), global1(isd:ied,jsd:jed,:,n), trim(type2) )
+   end do
 
-       !--- fill the halo and compare
-       select case(type)
-       case('Refined-Four-Tile', 'Refined-Symmetric-Four-Tile')
-          te = 0; ts = 0; tn = 0; tw = 0
-          select case(tiles(n))       
-          case(1)
-             tsw = 4 
-          case(2)
-             tsw = 3; tn = 4; ts = 4
-          case(3)
-             tsw = 2
-          case(4)
-             tsw = 1; ts = 2; tn = 2
-          end select
-          tse = tsw; tnw = tsw; tne = tsw
-          call fill_regular_refinement_halo( global2(:,:,:,n), global_all, ni, nj, tiles(n), &
-                                             te, tse, ts, tsw, tw, tnw, tn, tne, 0, 0)
-       case('Refined-Cubic-Grid')       
-          call fill_cubicgrid_refined_halo(global2(:,:,:,n), global_all, global_all, ni, nj, tiles(n), 0, 0, 1, 1 )
-       end select
-       call mpp_get_data_domain   ( domain, isd, ied, jsd, jed )
-       type2 = type
-       if(ntiles_on_pe>1) write(type2, *)trim(type2), " at tile_count = ", tiles(n)
-       call compare_checksums( x (isd:ied,jsd:jed,:,n), global2(isd:ied,jsd:jed,:,n), trim(type2)//' X' )
-       call compare_checksums( x1(isd:ied,jsd:jed,:,n), global2(isd:ied,jsd:jed,:,n), trim(type2)//' X1' )
-       call compare_checksums( x2(isd:ied,jsd:jed,:,n), global2(isd:ied,jsd:jed,:,n), trim(type2)//' X2' )
-    end do  
+   deallocate(global1_all, global1, x)
 
-    deallocate(global2, global_all, x, x1, x2)
     !------------------------------------------------------------------
-    !              vector update : BGRID_NE, one extra point in each direction for cubic-grid
+    !  vector update : BGRID_NE, one extra point in each direction for Five-Tile
     !------------------------------------------------------------------
     !--- setup data
-    shift = 0
-    if( type == 'Refined-Symmetric-Four-Tile' .OR. type == 'Refined-Cubic-Grid' ) shift = 1
-
-    nimax = maxval(ni) + shift; njmax = maxval(nj) + shift
-    allocate(global1_all(1:nimax,1:njmax,nz,ntiles) )
-    allocate(global2_all(1:nimax,1:njmax,nz,ntiles) )
-    allocate(global1(1-whalo:nimax+ehalo,1-shalo:njmax+nhalo,nz, ntiles_on_pe) ) 
-    allocate(global2(1-whalo:nimax+ehalo,1-shalo:njmax+nhalo,nz, ntiles_on_pe) ) 
-    global1 = 0; global2 = 0
+    allocate(global1_all(nxm+shift,nym+shift,nz, ntiles), global2_all(nxm+shift,nym+shift,nz, ntiles) )  
+    allocate(global1(1-whalo:ni+ehalo+shift,1-shalo:nj+nhalo+shift,nz, ntile_per_pe) )  
+    allocate(global2(1-whalo:ni+ehalo+shift,1-shalo:nj+nhalo+shift,nz, ntile_per_pe) )   
     do n = 1, ntiles
        do k = 1, nz
-          do j = 1, njmax
-             do i = 1, nimax
+          do j = 1, nym+shift
+             do i = 1, nxm+shift
                 global1_all(i,j,k,n) = 1.0e3 + n + i*1.0e-3 + j*1.0e-6 + k*1.0e-9
                 global2_all(i,j,k,n) = 2.0e3 + n + i*1.0e-3 + j*1.0e-6 + k*1.0e-9
              end do
@@ -3461,489 +3414,222 @@ subroutine test_refined_mosaic(type)
        end do
     end do
 
-    !--- make sure consistency on the no-refinement boundary  for symmetric domain.
-    !--- For "Symmetric four tile" mosaic, north of tile 2 and 4 need to be filled.
-    !--- For "Cubic-Grid", The following need to be filled: north of tile 1,
-    !--- north of tile 2, east and north of tile 6. The velocity at the corner point will be 0.
-    select case( type ) 
-    case ( 'Refined-Symmetric-Four-Tile' ) 
-       global1_all(1:ni(2)+1,nj(2)+1,:,2) = global1_all(1:ni(2)+1,1,:,4)
-       global2_all(1:ni(2)+1,nj(2)+1,:,2) = global2_all(1:ni(2)+1,1,:,4)
-       global1_all(1:ni(4)+1,nj(4)+1,:,4) = global1_all(1:ni(4)+1,1,:,2)
-       global2_all(1:ni(4)+1,nj(4)+1,:,4) = global2_all(1:ni(4)+1,1,:,2)
-    case ( 'Refined-Cubic-Grid' )
-       global1_all(1:ni(1)+1,nj(1)+1,:,1) = -global2_all(1,nj(3)+1:1:-1,:,3) ! north
-       global2_all(1:ni(1)+1,nj(1)+1,:,1) =  global1_all(1,nj(3)+1:1:-1,:,3) ! north
-       global1_all(1:ni(2)+1,nj(2)+1,:,2) =  global1_all(1:ni(3)+1,1,   :,3) ! north
-       global2_all(1:ni(2)+1,nj(2)+1,:,2) =  global2_all(1:ni(3)+1,1,   :,3) ! north
-       global1_all(1:ni(6)+1,nj(6)+1,:,6) =  global1_all(1:ni(1)+1,1,   :,1) ! north
-       global2_all(1:ni(6)+1,nj(6)+1,:,6) =  global2_all(1:ni(1)+1,1,   :,1) ! north
-       global1_all(ni(6)+1,1:nj(6)+1,:,6) =  global2_all(ni(2)+1:1:-1,1,:,2)  ! east 
-       global2_all(ni(6)+1,1:nj(6)+1,:,6) = -global1_all(ni(2)+1:1:-1,1,:,2)  ! east 
-       do n = 1, ntiles
-          global1_all(1,      1,:,n) = 0;  global1_all(1,      nj(n)+1,:,n) = 0;
-          global1_all(ni(n)+1,1,:,n) = 0;  global1_all(ni(n)+1,nj(n)+1,:,n) = 0;
-          global2_all(1,      1,:,n) = 0;  global2_all(1,      nj(n)+1,:,n) = 0;
-          global2_all(ni(n)+1,1,:,n) = 0;  global2_all(ni(n)+1,nj(n)+1,:,n) = 0;
-       end do
-    end select
+    !------------------------------------------------------------------------
+    ! --- make sure consisency on the boundary for Five-Tile mosaic
+    ! --- east boundary will take the value of neighbor tile west,
+    ! --- north boundary will take the value of neighbor tile south.
+    !------------------------------------------------------------------------
+    if(type == 'Five-Tile') then
+       global1_all(nxm+1,    1:ny,:,1) = global1_all(1,    1:ny,:,2)  ! east
+       global1_all(nxm+1,ny+1:nym,:,1) = global1_all(1,    1:ny,:,4)  ! east
+       global1_all(1:nxm+1, nym+1,:,1) = global1_all(1:nxm+1, 1,:,1)  ! north
+       global1_all(nx+1,     1:ny,:,2) = global1_all(1,    1:ny,:,3)  ! east
+       global1_all(1:nx+1,   ny+1,:,2) = global1_all(1:nx+1,  1,:,4)  ! north
+       global1_all(nx+1,     1:ny,:,3) = global1_all(1,    1:ny,:,1)  ! east
+       global1_all(1:nx+1,   ny+1,:,3) = global1_all(1:nx+1,  1,:,5)  ! north
+       global1_all(nx+1,     1:ny,:,4) = global1_all(1,    1:ny,:,5)  ! east
+       global1_all(1:nx+1,   ny+1,:,4) = global1_all(1:nx+1,  1,:,2)  ! north
+       global1_all(nx+1,     1:ny,:,5) = global1_all(1,ny+1:nym,:,1)  ! east
+       global1_all(1:nx+1,   ny+1,:,5) = global1_all(1:nx+1,  1,:,3)  ! north
+       global1_all(nx+1,     ny+1,:,2) = global1_all(1,       1,:,5)  ! northeast 
+       global1_all(nx+1,     ny+1,:,3) = global1_all(1,    ny+1,:,1)  ! northeast 
+       global2_all(nxm+1,    1:ny,:,1) = global2_all(1,    1:ny,:,2)  ! east
+       global2_all(nxm+1,ny+1:nym,:,1) = global2_all(1,    1:ny,:,4)  ! east
+       global2_all(1:nxm+1, nym+1,:,1) = global2_all(1:nxm+1, 1,:,1)  ! north
+       global2_all(nx+1,     1:ny,:,2) = global2_all(1,    1:ny,:,3)  ! east
+       global2_all(1:nx+1,   ny+1,:,2) = global2_all(1:nx+1,  1,:,4)  ! north
+       global2_all(nx+1,     1:ny,:,3) = global2_all(1,    1:ny,:,1)  ! east
+       global2_all(1:nx+1,   ny+1,:,3) = global2_all(1:nx+1,  1,:,5)  ! north
+       global2_all(nx+1,     1:ny,:,4) = global2_all(1,    1:ny,:,5)  ! east
+       global2_all(1:nx+1,   ny+1,:,4) = global2_all(1:nx+1,  1,:,2)  ! north
+       global2_all(nx+1,     1:ny,:,5) = global2_all(1,ny+1:nym,:,1)  ! east
+       global2_all(1:nx+1,   ny+1,:,5) = global2_all(1:nx+1,  1,:,3)  ! north
+       global2_all(nx+1,     ny+1,:,2) = global2_all(1,       1,:,5)  ! northeast 
+       global2_all(nx+1,     ny+1,:,3) = global2_all(1,    ny+1,:,1)  ! northeast 
+    end if
 
-    do n = 1, ntiles_on_pe
-       nn = tiles(n)
-       global1(1:ni(nn)+shift,1:nj(nn)+shift,:,n) = global1_all(1:ni(nn)+shift,1:nj(nn)+shift,:,nn)
-       global2(1:ni(nn)+shift,1:nj(nn)+shift,:,n) = global2_all(1:ni(nn)+shift,1:nj(nn)+shift,:,nn)
+    do n = 1, ntile_per_pe
+       global1(1:ni+shift,1:nj+shift,:,n) = global1_all(1:ni+shift,1:nj+shift,:,tile(n))
+       global2(1:ni+shift,1:nj+shift,:,n) = global2_all(1:ni+shift,1:nj+shift,:,tile(n))
     end do
 
-    call mpp_get_memory_domain   ( domain, ism, iem, jsm, jem )
-    allocate( x  (ism:iem+shift,jsm:jem+shift,nz, ntiles_on_pe) )
-    allocate( y  (ism:iem+shift,jsm:jem+shift,nz, ntiles_on_pe) )
-    allocate( x1 (ism:iem+shift,jsm:jem+shift,nz, ntiles_on_pe) )
-    allocate( y1 (ism:iem+shift,jsm:jem+shift,nz, ntiles_on_pe) )
-    allocate( x2 (ism:iem+shift,jsm:jem+shift,nz, ntiles_on_pe) )
-    allocate( y2 (ism:iem+shift,jsm:jem+shift,nz, ntiles_on_pe) )
-    x = 0; y = 0
-
-    do n = 1, ntiles_on_pe
-       call mpp_get_compute_domain( domain, isc, iec, jsc, jec, tile_count=n )
-       call mpp_get_data_domain   ( domain, isd, ied, jsd, jed, tile_count=n )
-       x(isc:iec+shift,jsc:jec+shift,:,:) = global1(isc:iec+shift,jsc:jec+shift,:,:)
-       y(isc:iec+shift,jsc:jec+shift,:,:) = global2(isc:iec+shift,jsc:jec+shift,:,:)
-    end do
-    x1 = x; x2 =x; y1 = y; y2 = y
+    allocate( x (ism:iem+shift,jsm:jem+shift,nz,ntile_per_pe) )
+    allocate( y (ism:iem+shift,jsm:jem+shift,nz,ntile_per_pe) )
 
-    allocate(bufferx(maxtotal*nz, ntiles_on_pe),  buffery(maxtotal*nz, ntiles_on_pe) )
-    allocate(bufferx1(maxtotal*nz, ntiles_on_pe), buffery1(maxtotal*nz, ntiles_on_pe) )
-    allocate(bufferx2(maxtotal*nz, ntiles_on_pe), buffery2(maxtotal*nz, ntiles_on_pe) )
+    x = 0.; y = 0
+    x (isc:iec+shift,jsc:jec+shift,:,:) = global1(isc:iec+shift,jsc:jec+shift,:,:)
+    y (isc:iec+shift,jsc:jec+shift,:,:) = global2(isc:iec+shift,jsc:jec+shift,:,:)
 
-    !--- call mpp_update_domains to update domain 
-    do n = 1, ntiles_on_pe
-       call mpp_update_domains( x(:,:,:,n), y(:,:,:,n), domain, gridtype=BGRID_NE, &
-                                bufferx=bufferx(:,n), buffery=buffery(:,n), tile_count=n )
+    !-----------------------------------------------------------------------
+    !                   fill up the value at halo points.     
+    !-----------------------------------------------------------------------
+    do n = 1, ntile_per_pe
+       call fill_five_tile_halo(global1(:,:,:,n), global1_all, tile(n), shift, shift)
+       call fill_five_tile_halo(global2(:,:,:,n), global2_all, tile(n), shift, shift)
     end do
 
-    !--- multiple update
-    do n = 1, ntiles_on_pe
-       call mpp_update_domains( x1(:,:,:,n), y1(:,:,:,n), domain, gridtype=BGRID_NE, &
-                                bufferx=bufferx1(:,n), buffery=buffery1(:,n), complete=.false., tile_count=n )
-       call mpp_update_domains( x2(:,:,:,n), y2(:,:,:,n), domain, gridtype=BGRID_NE, &
-                                bufferx=bufferx2(:,n), buffery=buffery2(:,n), complete=.true., tile_count=n )
+    id = mpp_clock_id( type//' BGRID_NE', flags=MPP_CLOCK_SYNC+MPP_CLOCK_DETAILED )
+    call mpp_clock_begin(id)
+    do n = 1, ntile_per_pe
+       call mpp_update_domains( x(:,:,:,n), y(:,:,:,n), domain, gridtype=BGRID_NE, tile_count = n )
     end do
+    call mpp_clock_end(id)
 
-    !--- fill up the value at halo points and compare the value at buffer.
-    do n = 1, ntiles_on_pe  
-       !--- comparing the buffer.
-       noverlap1 = mpp_get_refine_overlap_number(domain, tile_count=n, position = CORNER)
-       call mpp_get_mosaic_refine_overlap(domain, isMe1, ieMe1, jsMe1, jeMe1, isNb1, ieNb1, jsNb1, jeNb1, &
-               dirMe1, rotation1, tile_count = n, position = CORNER)             
-       pos = 0
-
-       do m = 1, noverlap1  
-          select case( rotation1(m) )
-          case (ZERO)     
-             do k = 1, nz
-                do j = jsNb1(m), jeNb1(m)
-                   do i = isNb1(m), ieNb1(m)
-                      pos = pos + 1
-                      if(global1_all(i,j,k,from_tile1(m,n)) .NE. bufferx(pos,n) ) then
-                         write(stdunit,111)'x','BGRID '//type,mpp_pe(),i,j,k,bufferx(pos,n),global1_all(i,j,k,from_tile1(m,n))
-                         call mpp_error(FATAL,"test_refined_mosaic: mismatch between buffer and actual data: BGRID "//type//" X")
-                      end if
-                      if(global2_all(i,j,k,from_tile1(m,n)) .NE. buffery(pos,n) ) then
-                         write(stdunit,111)'y','BGRID '//type,mpp_pe(),i,j,k,buffery(pos,n),global2_all(i,j,k,from_tile1(m,n))
-                         call mpp_error(FATAL,"test_refined_mosaic: mismatch between buffer and actual data: BGRID "//type//" Y")
-                      end if
-                      if(global1_all(i,j,k,from_tile1(m,n)) .NE. bufferx1(pos,n) ) then
-                         write(stdunit,111)'x1','BGRID '//type,mpp_pe(),i,j,k,bufferx1(pos,n),global1_all(i,j,k,from_tile1(m,n))
-                         call mpp_error(FATAL,"test_refined_mosaic: mismatch between buffer and actual data: BGRID "//type//" X1")
-                      end if
-                      if(global2_all(i,j,k,from_tile1(m,n)) .NE. buffery1(pos,n) ) then
-                         write(stdunit,111)'y1','BGRID '//type,mpp_pe(),i,j,k,buffery1(pos,n),global2_all(i,j,k,from_tile1(m,n))
-                         call mpp_error(FATAL,"test_refined_mosaic: mismatch between buffer and actual data: BGRID "//type//" Y1")
-                      end if
-                      if(global1_all(i,j,k,from_tile1(m,n)) .NE. bufferx2(pos,n) ) then
-                         write(stdunit,111)'x2','BGRID '//type,mpp_pe(),i,j,k,bufferx2(pos,n),global1_all(i,j,k,from_tile1(m,n))
-                         call mpp_error(FATAL,"test_refined_mosaic: mismatch between buffer and actual data: BGRID "//type//" X2")
-                      end if
-                      if(global2_all(i,j,k,from_tile1(m,n)) .NE. buffery2(pos,n) ) then
-                         write(stdunit,111)'y2','BGRID '//type,mpp_pe(),i,j,k,buffery2(pos,n),global2_all(i,j,k,from_tile1(m,n))
-                         call mpp_error(FATAL,"test_refined_mosaic: mismatch between buffer and actual data: BGRID "//type//" Y2")
-                      end if
-                   end do
-                end do
-             end do
-          case (NINETY) ! S->E, N->W, u->-v, v->u    
-             do k = 1, nz
-                do j = jsNb1(m), jeNb1(m)
-                   do i = isNb1(m), ieNb1(m)
-                      pos = pos + 1
-                      if(global2_all(i,j,k,from_tile1(m,n)) .NE. bufferx(pos,n) ) then
-                         write(stdunit,111)'x','BGRID '//type,mpp_pe(),i,j,k,bufferx(pos,n),global2_all(i,j,k,from_tile1(m,n))
-                         call mpp_error(FATAL,"test_refined_mosaic: mismatch between buffer and actual data: BGRID "//type//" X")
-                      end if
-                      if(-global1_all(i,j,k,from_tile1(m,n)) .NE. buffery(pos,n) ) then
-                         write(stdunit,111)'y','BGRID '//type,mpp_pe(),i,j,k,buffery(pos,n),-global1_all(i,j,k,from_tile1(m,n))
-                         call mpp_error(FATAL,"test_refined_mosaic: mismatch between buffer and actual data: BGRID "//type//" Y")
-                      end if
-                      if(global2_all(i,j,k,from_tile1(m,n)) .NE. bufferx1(pos,n) ) then
-                         write(stdunit,111)'x1','BGRID '//type,mpp_pe(),i,j,k,bufferx1(pos,n),global2_all(i,j,k,from_tile1(m,n))
-                         call mpp_error(FATAL,"test_refined_mosaic: mismatch between buffer and actual data: BGRID "//type//" X1")
-                      end if
-                      if(-global1_all(i,j,k,from_tile1(m,n)) .NE. buffery1(pos,n) ) then
-                         write(stdunit,111)'y1','BGRID '//type,mpp_pe(),i,j,k,buffery1(pos,n),-global1_all(i,j,k,from_tile1(m,n))
-                         call mpp_error(FATAL,"test_refined_mosaic: mismatch between buffer and actual data: BGRID "//type//" Y1")
-                      end if
-                      if(global2_all(i,j,k,from_tile1(m,n)) .NE. bufferx2(pos,n) ) then
-                         write(stdunit,111)'x2','BGRID '//type,mpp_pe(),i,j,k,bufferx2(pos,n),global2_all(i,j,k,from_tile1(m,n))
-                         call mpp_error(FATAL,"test_refined_mosaic: mismatch between buffer and actual data: BGRID "//type//" X2")
-                      end if
-                      if(-global1_all(i,j,k,from_tile1(m,n)) .NE. buffery2(pos,n) ) then
-                         write(stdunit,111)'y2','BGRID '//type,mpp_pe(),i,j,k,buffery2(pos,n),-global1_all(i,j,k,from_tile1(m,n))
-                         call mpp_error(FATAL,"test_refined_mosaic: mismatch between buffer and actual data: BGRID "//type//" Y2")
-                      end if
-                   end do
-                end do
-             end do
-          case (MINUS_NINETY) ! S->E, N->W, u->-v, v->u    
-             do k = 1, nz
-                do j = jsNb1(m), jeNb1(m)
-                   do i = isNb1(m), ieNb1(m)
-                      pos = pos + 1
-                      if(-global2_all(i,j,k,from_tile1(m,n)) .NE. bufferx(pos,n) ) then
-                         write(stdunit,111)'x','BGRID '//type,mpp_pe(),i,j,k,bufferx(pos,n),-global2_all(i,j,k,from_tile1(m,n))
-                         call mpp_error(FATAL,"test_refined_mosaic: mismatch between buffer and actual data: BGRID "//type//" X")
-                      end if
-                      if(global1_all(i,j,k,from_tile1(m,n)) .NE. buffery(pos,n) ) then
-                         write(stdunit,111)'y','BGRID '//type,mpp_pe(),i,j,k,buffery(pos,n),global1_all(i,j,k,from_tile1(m,n))
-                         call mpp_error(FATAL,"test_refined_mosaic: mismatch between buffer and actual data: BGRID "//type//" Y")
-                      end if
-                      if(-global2_all(i,j,k,from_tile1(m,n)) .NE. bufferx1(pos,n) ) then
-                         write(stdunit,111)'x1','BGRID '//type,mpp_pe(),i,j,k,bufferx1(pos,n),-global2_all(i,j,k,from_tile1(m,n))
-                         call mpp_error(FATAL,"test_refined_mosaic: mismatch between buffer and actual data: BGRID "//type//" X1")
-                      end if
-                      if(global1_all(i,j,k,from_tile1(m,n)) .NE. buffery1(pos,n) ) then
-                         write(stdunit,111)'y1','BGRID '//type,mpp_pe(),i,j,k,buffery1(pos,n),global1_all(i,j,k,from_tile1(m,n))
-                         call mpp_error(FATAL,"test_refined_mosaic: mismatch between buffer and actual data: BGRID "//type//" Y1")
-                      end if
-                      if(-global2_all(i,j,k,from_tile1(m,n)) .NE. bufferx2(pos,n) ) then
-                         write(stdunit,111)'x2','BGRID '//type,mpp_pe(),i,j,k,bufferx2(pos,n),-global2_all(i,j,k,from_tile1(m,n))
-                         call mpp_error(FATAL,"test_refined_mosaic: mismatch between buffer and actual data: BGRID "//type//" X2")
-                      end if
-                      if(global1_all(i,j,k,from_tile1(m,n)) .NE. buffery2(pos,n) ) then
-                         write(stdunit,111)'y2','BGRID '//type,mpp_pe(),i,j,k,buffery2(pos,n),global1_all(i,j,k,from_tile1(m,n))
-                         call mpp_error(FATAL,"test_refined_mosaic: mismatch between buffer and actual data: BGRID "//type//" Y2")
-                      end if
-                   end do
-                end do
-             end do
-          end select
-       end do
-
-       !--- fill the halo data and compare
-       select case(type)
-       case('Refined-Four-Tile', 'Refined-Symmetric-Four-Tile')
-          te = 0; ts = 0; tn = 0; tw = 0
-          select case(tiles(n))       
-          case(1)
-             tsw = 4 
-          case(2)
-             tsw = 3; tn = 4; ts = 4
-          case(3)
-             tsw = 2
-          case(4)
-             tsw = 1; ts = 2; tn = 2
-          end select
-          tse = tsw; tnw = tsw; tne = tsw
-          call fill_regular_refinement_halo( global1(:,:,:,n), global1_all, ni, nj, tiles(n), &
-               te, tse, ts, tsw, tw, tnw, tn, tne, shift, shift )
-          call fill_regular_refinement_halo( global2(:,:,:,n), global2_all, ni, nj, tiles(n), &
-               te, tse, ts, tsw, tw, tnw, tn, tne, shift, shift )
-       case('Refined-Cubic-Grid')       
-          call fill_cubicgrid_refined_halo(global1(:,:,:,n), global1_all, global2_all, ni, nj, tiles(n), 1, 1, 1, -1 )
-          call fill_cubicgrid_refined_halo(global2(:,:,:,n), global2_all, global1_all, ni, nj, tiles(n), 1, 1, -1, 1 )
-       end select
+   do n = 1, ntile_per_pe
+      write(type2, *)type, " at tile_count = ",n
+      call compare_checksums( x(isd:ied+shift,jsd:jed+shift,:,n), global1(isd:ied+shift,jsd:jed+shift,:,n), &
+                              trim(type2)//' BGRID_NE X')
+      call compare_checksums( y(isd:ied+shift,jsd:jed+shift,:,n), global2(isd:ied+shift,jsd:jed+shift,:,n), &
+                              trim(type2)//' BGRID_NE Y')
+   end do
 
-       call mpp_get_data_domain   ( domain, isd, ied, jsd, jed )
-       write(type2, *)"BGRID ", type
-       if(ntiles_on_pe>1) write(type2, *)trim(type2), " at tile_count = ", tiles(n)
-       call compare_checksums( x (isd:ied+shift,jsd:jed+shift,:,n), global1(isd:ied+shift,jsd:jed+shift,:,n), trim(type2)//' X' )
-       call compare_checksums( x1(isd:ied+shift,jsd:jed+shift,:,n), global1(isd:ied+shift,jsd:jed+shift,:,n), trim(type2)//' X1')
-       call compare_checksums( x2(isd:ied+shift,jsd:jed+shift,:,n), global1(isd:ied+shift,jsd:jed+shift,:,n), trim(type2)//' X2')
-       write(type2, *)"BGRID ", type
-       if(ntiles_on_pe>1) write(type2, *)trim(type2), " at tile_count = ", tiles(n)
-       call compare_checksums( y (isd:ied+shift,jsd:jed+shift,:,n), global2(isd:ied+shift,jsd:jed+shift,:,n), trim(type2)//' Y' )
-       call compare_checksums( y1(isd:ied+shift,jsd:jed+shift,:,n), global2(isd:ied+shift,jsd:jed+shift,:,n), trim(type2)//' Y1')
-       call compare_checksums( y2(isd:ied+shift,jsd:jed+shift,:,n), global2(isd:ied+shift,jsd:jed+shift,:,n), trim(type2)//' Y2')
-    end do  
+   deallocate(global1_all, global2_all, global1, global2, x, y)
 
-    deallocate(global1_all, global2_all, global1, global2, x, y, x1, x2, y1, y2 )
     !------------------------------------------------------------------
-    !              vector update : CGRID_NE, one extra point may needed to symmetric domain
+    !  vector update : CGRID_NE
     !------------------------------------------------------------------
     !--- setup data
-
-    nimax = maxval(ni); njmax = maxval(nj)
-    allocate(global1_all(1:nimax+shift,1:njmax,nz,ntiles) )
-    allocate(global2_all(1:nimax,1:njmax+shift,nz,ntiles) )
-    allocate(global1(1-whalo:nimax+ehalo+shift,1-shalo:njmax+nhalo,nz, ntiles_on_pe) ) 
-    allocate(global2(1-whalo:nimax+ehalo,1-shalo:njmax+nhalo+shift,nz, ntiles_on_pe) ) 
-    global1 = 0; global2 = 0
+    allocate(global1_all(nxm+shift,nym,nz, ntiles), global2_all(nxm,nym+shift,nz, ntiles) )  
+    allocate(global1(1-whalo:ni+ehalo+shift, 1-shalo:nj+nhalo,       nz, ntile_per_pe) )  
+    allocate(global2(1-whalo:ni+ehalo,       1-shalo:nj+nhalo+shift, nz, ntile_per_pe) )   
     do n = 1, ntiles
        do k = 1, nz
-          do j = 1, njmax
-             do i = 1, nimax + shift
+          do j = 1, nym
+             do i = 1, nxm+shift
                 global1_all(i,j,k,n) = 1.0e3 + n + i*1.0e-3 + j*1.0e-6 + k*1.0e-9
              end do
           end do
-          do j = 1, njmax + shift
-             do i = 1, nimax
+          do j = 1, nym+shift
+             do i = 1, nxm
                 global2_all(i,j,k,n) = 2.0e3 + n + i*1.0e-3 + j*1.0e-6 + k*1.0e-9
              end do
           end do
        end do
     end do
 
-    !--- make sure consistency on the no-refinement boundary  for symmetric domain.
-    !--- For "Symmetric four tile" mosaic, north of tile 2 and 4 need to be filled.
-    !--- For "Cubic-Grid", The following need to be filled: north of tile 1,
-    !--- north of tile 2, east and north of tile 6. The velocity at the corner point will be 0.
-    select case( type ) 
-    case ('Refined-Symmetric-Four-Tile' ) 
-       global2_all(1:ni(2),nj(2)+1,:,2) = global2_all(1:ni(2),1,:,4)
-       global2_all(1:ni(4),nj(4)+1,:,4) = global2_all(1:ni(4),1,:,2)
-    case ('Refined-Cubic-Grid' )
-       global2_all(1:ni(1),nj(1)+1,:,1) =  global1_all(1,nj(3):1:-1,:,3) ! north
-       global2_all(1:ni(2),nj(2)+1,:,2) =  global2_all(1:ni(3),1,   :,3) ! north
-       global2_all(1:ni(6),nj(6)+1,:,6) =  global2_all(1:ni(1),1,   :,1) ! north
-       global1_all(ni(6)+1,1:nj(6),:,6) =  global2_all(ni(2):1:-1,1,:,2)  ! east 
-    end select
+    !------------------------------------------------------------------------
+    ! --- make sure consisency on the boundary for Five-Tile mosaic
+    ! --- east boundary will take the value of neighbor tile west,
+    ! --- north boundary will take the value of neighbor tile south.
+    !------------------------------------------------------------------------
+    if(type == 'Five-Tile') then
+       global1_all(nxm+1,    1:ny,:,1) = global1_all(1,    1:ny,:,2)  ! east
+       global1_all(nxm+1,ny+1:nym,:,1) = global1_all(1,    1:ny,:,4)  ! east
+       global1_all(nx+1,     1:ny,:,2) = global1_all(1,    1:ny,:,3)  ! east
+       global1_all(nx+1,     1:ny,:,3) = global1_all(1,    1:ny,:,1)  ! east
+       global1_all(nx+1,     1:ny,:,4) = global1_all(1,    1:ny,:,5)  ! east
+       global1_all(nx+1,     1:ny,:,5) = global1_all(1,ny+1:nym,:,1)  ! east
+       global2_all(1:nxm,   nym+1,:,1) = global2_all(1:nxm,   1,:,1)  ! north
+       global2_all(1:nx,     ny+1,:,2) = global2_all(1:nx,    1,:,4)  ! north
+       global2_all(1:nx,     ny+1,:,3) = global2_all(1:nx,    1,:,5)  ! north
+       global2_all(1:nx,     ny+1,:,4) = global2_all(1:nx,    1,:,2)  ! north
+       global2_all(1:nx,     ny+1,:,5) = global2_all(1:nx,    1,:,3)  ! north
+    end if
 
-    do n = 1, ntiles_on_pe
-       nn = tiles(n)
-       global1(1:ni(nn)+shift,1:nj(nn),:,n) = global1_all(1:ni(nn)+shift,1:nj(nn),:,nn)
-       global2(1:ni(nn),1:nj(nn)+shift,:,n) = global2_all(1:ni(nn),1:nj(nn)+shift,:,nn)
+    do n = 1, ntile_per_pe
+       global1(1:ni+shift,      1:nj,:,n) = global1_all(1:ni+shift,      1:nj,:,tile(n))
+       global2(1:ni,      1:nj+shift,:,n) = global2_all(1:ni,      1:nj+shift,:,tile(n))
     end do
 
-    call mpp_get_memory_domain   ( domain, ism, iem, jsm, jem )
-    allocate( x  (ism:iem+shift,jsm:jem,nz, ntiles_on_pe) )
-    allocate( y  (ism:iem,jsm:jem+shift,nz, ntiles_on_pe) )
-    allocate( x1 (ism:iem+shift,jsm:jem,nz, ntiles_on_pe) )
-    allocate( y1 (ism:iem,jsm:jem+shift,nz, ntiles_on_pe) )
-    allocate( x2 (ism:iem+shift,jsm:jem,nz, ntiles_on_pe) )
-    allocate( y2 (ism:iem,jsm:jem+shift,nz, ntiles_on_pe) )
-    x = 0; y = 0
-    bufferx  = 0; buffery = 0 
-    bufferx1 = 0; buffery1 = 0
-    bufferx2 = 0; buffery2 = 0
-    do n = 1, ntiles_on_pe
-       call mpp_get_compute_domain( domain, isc, iec, jsc, jec, tile_count=n )
-       call mpp_get_data_domain   ( domain, isd, ied, jsd, jed, tile_count=n )
-       x(isc:iec+shift,jsc:jec,:,:) = global1(isc:iec+shift,jsc:jec,:,:)
-       y(isc:iec,jsc:jec+shift,:,:) = global2(isc:iec,jsc:jec+shift,:,:)
-    end do
-    x1 = x; x2 =x; y1 = y; y2 = y   
+    allocate( x (ism:iem+shift,      jsm:jem,nz,ntile_per_pe) )
+    allocate( y (ism:iem,      jsm:jem+shift,nz,ntile_per_pe) )
 
-    !--- call mpp_update_domains to update domain 
-    do n = 1, ntiles_on_pe
-       call mpp_update_domains( x(:,:,:,n), y(:,:,:,n), domain, gridtype=CGRID_NE, &
-                                bufferx=bufferx(:,n), buffery=buffery(:,n), tile_count=n )
+    x = 0.; y = 0
+    x (isc:iec+shift,      jsc:jec,:,:) = global1(isc:iec+shift,      jsc:jec,:,:)
+    y (isc:iec,      jsc:jec+shift,:,:) = global2(isc:iec,      jsc:jec+shift,:,:)
+
+    !-----------------------------------------------------------------------
+    !                   fill up the value at halo points.     
+    !-----------------------------------------------------------------------
+    do n = 1, ntile_per_pe
+       call fill_five_tile_halo(global1(:,:,:,n), global1_all, tile(n), shift, 0)
+       call fill_five_tile_halo(global2(:,:,:,n), global2_all, tile(n), 0, shift)
     end do
 
-    !--- multiple update
-    do n = 1, ntiles_on_pe
-       call mpp_update_domains( x1(:,:,:,n), y1(:,:,:,n), domain, gridtype=CGRID_NE, &
-                                bufferx=bufferx1(:,n), buffery=buffery1(:,n), complete=.false., tile_count=n )
-       call mpp_update_domains( x2(:,:,:,n), y2(:,:,:,n), domain, gridtype=CGRID_NE, &
-                                bufferx=bufferx2(:,n), buffery=buffery2(:,n), complete=.true., tile_count=n )
+    id = mpp_clock_id( type//' CGRID_NE', flags=MPP_CLOCK_SYNC+MPP_CLOCK_DETAILED )
+    call mpp_clock_begin(id)
+    do n = 1, ntile_per_pe
+       call mpp_update_domains( x(:,:,:,n), y(:,:,:,n), domain, gridtype=CGRID_NE, tile_count = n )
     end do
+    call mpp_clock_end(id)
 
-    !--- fill up the value at halo points and compare the value at buffer.
-    do n = 1, ntiles_on_pe  
-
-       !--- comparing the buffer.
-       noverlap1 = mpp_get_refine_overlap_number(domain, tile_count=n)
-       call mpp_get_mosaic_refine_overlap(domain, isMe1, ieMe1, jsMe1, jeMe1, isNb1, ieNb1, jsNb1, jeNb1, &
-                                          dirMe1, rotation1, tile_count = n, position = EAST)             
-       pos = 0
-       do m = 1, noverlap1 
-          select case( rotation1(m) )
-          case (ZERO)     
-             do k = 1, nz
-                do j = jsNb1(m), jeNb1(m)
-                   do i = isNb1(m), ieNb1(m)
-                      pos = pos + 1
-                      if(global1_all(i,j,k,from_tile1(m,n)) .NE. bufferx(pos,n) ) then
-                         write(stdunit,111)'x','CGRID '//type,mpp_pe(),i,j,k,bufferx(pos,n),global1_all(i,j,k,from_tile1(m,n))
-                         call mpp_error(FATAL,"test_refined_mosaic: mismatch between buffer and actual data: CGRID "//type//" X")
-                      end if
-                      if(global1_all(i,j,k,from_tile1(m,n)) .NE. bufferx1(pos,n) ) then
-                         write(stdunit,111)'x1','CGRID '//type,mpp_pe(),i,j,k,bufferx1(pos,n),global1_all(i,j,k,from_tile1(m,n))
-                         call mpp_error(FATAL,"test_refined_mosaic: mismatch between buffer and actual data: CGRID "//type//" X1")
-                      end if
-                      if(global1_all(i,j,k,from_tile1(m,n)) .NE. bufferx2(pos,n) ) then
-                         write(stdunit,111)'x2','CGRID '//type,mpp_pe(),i,j,k,bufferx2(pos,n),global1_all(i,j,k,from_tile1(m,n))
-                         call mpp_error(FATAL,"test_refined_mosaic: mismatch between buffer and actual data: CGRID "//type//" X2")
-                      end if
-                   end do
-                end do
-             end do
-          case (NINETY) ! S->E, N->W, u->-v, v->u    
-             do k = 1, nz
-                do j = jsNb1(m), jeNb1(m)
-                   do i = isNb1(m), ieNb1(m)
-                      pos = pos + 1
-                      if(global2_all(i,j,k,from_tile1(m,n)) .NE. bufferx(pos,n) ) then
-                         write(stdunit,111)'x','CGRID '//type,mpp_pe(),i,j,k,bufferx(pos,n),global2_all(i,j,k,from_tile1(m,n))
-                         call mpp_error(FATAL,"test_refined_mosaic: mismatch between buffer and actual data: CGRID "//type//" X")
-                      end if
-                      if(global2_all(i,j,k,from_tile1(m,n)) .NE. bufferx1(pos,n) ) then
-                         write(stdunit,111)'x1','CGRID '//type,mpp_pe(),i,j,k,bufferx1(pos,n),global2_all(i,j,k,from_tile1(m,n))
-                         call mpp_error(FATAL,"test_refined_mosaic: mismatch between buffer and actual data: CGRID "//type//" X1")
-                      end if
-                      if(global2_all(i,j,k,from_tile1(m,n)) .NE. bufferx2(pos,n) ) then
-                         write(stdunit,111)'x2','CGRID '//type,mpp_pe(),i,j,k,bufferx2(pos,n),global2_all(i,j,k,from_tile1(m,n))
-                         call mpp_error(FATAL,"test_refined_mosaic: mismatch between buffer and actual data: CGRID "//type//" X2")
-                      end if
-                   end do
-                end do
-             end do
-          case (MINUS_NINETY) ! S->E, N->W, u->-v, v->u    
-             do k = 1, nz
-                do j = jsNb1(m), jeNb1(m)
-                   do i = isNb1(m), ieNb1(m)
-                      pos = pos + 1
-                      if(-global2_all(i,j,k,from_tile1(m,n)) .NE. bufferx(pos,n) ) then
-                         write(stdunit,111)'x','CGRID '//type,mpp_pe(),i,j,k,bufferx(pos,n),-global2_all(i,j,k,from_tile1(m,n))
-                         call mpp_error(FATAL,"test_refined_mosaic: mismatch between buffer and actual data: CGRID "//type//" X")
-                      end if
-                      if(-global2_all(i,j,k,from_tile1(m,n)) .NE. bufferx1(pos,n) ) then
-                         write(stdunit,111)'x1','CGRID '//type,mpp_pe(),i,j,k,bufferx1(pos,n),-global2_all(i,j,k,from_tile1(m,n))
-                         call mpp_error(FATAL,"test_refined_mosaic: mismatch between buffer and actual data: CGRID "//type//" X1")
-                      end if
-                      if(-global2_all(i,j,k,from_tile1(m,n)) .NE. bufferx2(pos,n) ) then
-                         write(stdunit,111)'x2','CGRID '//type,mpp_pe(),i,j,k,bufferx2(pos,n),-global2_all(i,j,k,from_tile1(m,n))
-                         call mpp_error(FATAL,"test_refined_mosaic: mismatch between buffer and actual data: CGRID "//type//" X2")
-                      end if
-                   end do
-                end do
-             end do
-          end select
-       end do
+   do n = 1, ntile_per_pe
+      write(type2, *)type, " at tile_count = ",n
+      call compare_checksums( x(isd:ied+shift,jsd:jed,:,n), global1(isd:ied+shift,jsd:jed,:,n), &
+                              trim(type2)//' CGRID_NE X')
+      call compare_checksums( y(isd:ied,jsd:jed+shift,:,n), global2(isd:ied,jsd:jed+shift,:,n), &
+                              trim(type2)//' CGRID_NE Y')
+   end do
 
-       call mpp_get_mosaic_refine_overlap(domain, isMe1, ieMe1, jsMe1, jeMe1, isNb1, ieNb1, jsNb1, jeNb1, &
-            dirMe1, rotation1, tile_count = n, position = NORTH) 
-       pos = 0
-       do m = 1, noverlap1
-          select case( rotation1(m) )
-          case (ZERO)               
-             do k = 1, nz  
-                do j = jsNb1(m), jeNb1(m)
-                   do i = isNb1(m), ieNb1(m)
-                      pos = pos + 1
-                      if(global2_all(i,j,k,from_tile1(m,n)) .NE. buffery(pos,n) ) then
-                         write(stdunit,111)'y','CGRID '//type,mpp_pe(),i,j,k,buffery(pos,n),global2_all(i,j,k,from_tile1(m,n))
-                         call mpp_error(FATAL,"test_refined_mosaic: mismatch between buffer and actual data: CGRID "//type//" Y")
-                      end if
-                      if(global2_all(i,j,k,from_tile1(m,n)) .NE. buffery1(pos,n) ) then
-                         write(stdunit,111)'y1','CGRID '//type,mpp_pe(),i,j,k,buffery1(pos,n),global2_all(i,j,k,from_tile1(m,n))
-                         call mpp_error(FATAL,"test_refined_mosaic: mismatch between buffer and actual data: CGRID "//type//" Y1")
-                      end if
-                      if(global2_all(i,j,k,from_tile1(m,n)) .NE. buffery2(pos,n) ) then
-                         write(stdunit,111)'y2','CGRID '//type,mpp_pe(),i,j,k,buffery2(pos,n),global2_all(i,j,k,from_tile1(m,n))
-                         call mpp_error(FATAL,"test_refined_mosaic: mismatch between buffer and actual data: CGRID "//type//" Y2")
-                      end if
-                   end do
-                end do
-             end do
-          case (NINETY) ! S->E, N->W, u->-v, v->u    
-             do k = 1, nz
-                do j = jsNb1(m), jeNb1(m)
-                   do i = isNb1(m), ieNb1(m)
-                      pos = pos + 1
-                      if(-global1_all(i,j,k,from_tile1(m,n)) .NE. buffery(pos,n) ) then
-                         write(stdunit,111)'y','CGRID '//type,mpp_pe(),i,j,k,buffery(pos,n),-global1_all(i,j,k,from_tile1(m,n))
-                         call mpp_error(FATAL,"test_refined_mosaic: mismatch between buffer and actual data: CGRID "//type//" Y")
-                      end if
-                      if(-global1_all(i,j,k,from_tile1(m,n)) .NE. buffery1(pos,n) ) then
-                         write(stdunit,111)'y1','CGRID '//type,mpp_pe(),i,j,k,buffery1(pos,n),-global1_all(i,j,k,from_tile1(m,n))
-                         call mpp_error(FATAL,"test_refined_mosaic: mismatch between buffer and actual data: CGRID "//type//" Y1")
-                      end if
-                      if(-global1_all(i,j,k,from_tile1(m,n)) .NE. buffery2(pos,n) ) then
-                         write(stdunit,111)'y2','CGRID '//type,mpp_pe(),i,j,k,buffery2(pos,n),-global1_all(i,j,k,from_tile1(m,n))
-                         call mpp_error(FATAL,"test_refined_mosaic: mismatch between buffer and actual data: CGRID "//type//" Y2")
-                      end if
-                   end do
-                end do
-             end do
-          case (MINUS_NINETY) ! S->E, N->W, u->-v, v->u    
-             do k = 1, nz
-                do j = jsNb1(m), jeNb1(m)
-                   do i = isNb1(m), ieNb1(m)
-                      pos = pos + 1
-                      if(global1_all(i,j,k,from_tile1(m,n)) .NE. buffery(pos,n) ) then
-                         write(stdunit,111)'y','CGRID '//type,mpp_pe(),i,j,k,buffery(pos,n),global1_all(i,j,k,from_tile1(m,n))
-                         call mpp_error(FATAL,"test_refined_mosaic: mismatch between buffer and actual data: CGRID "//type//" Y")
-                      end if
-                      if(global1_all(i,j,k,from_tile1(m,n)) .NE. buffery1(pos,n) ) then
-                         write(stdunit,111)'y1','CGRID '//type,mpp_pe(),i,j,k,buffery1(pos,n),global1_all(i,j,k,from_tile1(m,n))
-                         call mpp_error(FATAL,"test_refined_mosaic: mismatch between buffer and actual data: CGRID "//type//" Y1")
-                      end if
-                      if(global1_all(i,j,k,from_tile1(m,n)) .NE. buffery2(pos,n) ) then
-                         write(stdunit,111)'y2','CGRID '//type,mpp_pe(),i,j,k,buffery2(pos,n),global1_all(i,j,k,from_tile1(m,n))
-                         call mpp_error(FATAL,"test_refined_mosaic: mismatch between buffer and actual data: CGRID "//type//" Y2")
-                      end if
-                   end do
-                end do
-             end do
-          end select
-       end do
-       !--- fill the halo data and compare
-       select case(type)
-       case('Refined-Four-Tile', 'Refined-Symmetric-Four-Tile')
-          te = 0; ts = 0; tn = 0; tw = 0
-          select case(tiles(n))       
-          case(1)
-             tsw = 4 
-          case(2)
-             tsw = 3; tn = 4; ts = 4
-          case(3)
-             tsw = 2
-          case(4)
-             tsw = 1; ts = 2; tn = 2
-          end select
-          tse = tsw; tnw = tsw; tne = tsw
-          call fill_regular_refinement_halo( global1(:,:,:,n), global1_all, ni, nj, tiles(n), &
-               te, tse, ts, tsw, tw, tnw, tn, tne, shift, 0 )
-          call fill_regular_refinement_halo( global2(:,:,:,n), global2_all, ni, nj, tiles(n), &
-               te, tse, ts, tsw, tw, tnw, tn, tne, 0, shift )
-       case('Refined-Cubic-Grid')       
-          call fill_cubicgrid_refined_halo(global1(:,:,:,n), global1_all, global2_all, ni, nj, tiles(n), 1, 0, 1, -1 )
-          call fill_cubicgrid_refined_halo(global2(:,:,:,n), global2_all, global1_all, ni, nj, tiles(n), 0, 1, -1, 1 )
-       end select
+   deallocate(global1_all, global2_all, global1, global2, x, y)
 
-       call mpp_get_data_domain   ( domain, isd, ied, jsd, jed )
-       write(type2, *)"CGRID ", type
-       if(ntiles_on_pe>1) write(type2, *)trim(type2), " at tile_count = ", tiles(n)
-       call compare_checksums( x (isd:ied+shift,jsd:jed,:,n), global1(isd:ied+shift,jsd:jed,:,n), trim(type2)//' X' )
-       call compare_checksums( x1(isd:ied+shift,jsd:jed,:,n), global1(isd:ied+shift,jsd:jed,:,n), trim(type2)//' X1')
-       call compare_checksums( x2(isd:ied+shift,jsd:jed,:,n), global1(isd:ied+shift,jsd:jed,:,n), trim(type2)//' X2')
-       write(type2, *)"CGRID ", type
-       if(ntiles_on_pe>1) write(type2, *)trim(type2), " at tile_count = ", tiles(n)
-       call compare_checksums( y (isd:ied,jsd:jed+shift,:,n), global2(isd:ied,jsd:jed+shift,:,n), trim(type2)//' Y' )
-       call compare_checksums( y1(isd:ied,jsd:jed+shift,:,n), global2(isd:ied,jsd:jed+shift,:,n), trim(type2)//' Y1' )
-       call compare_checksums( y2(isd:ied,jsd:jed+shift,:,n), global2(isd:ied,jsd:jed+shift,:,n), trim(type2)//' Y2' )
-    end do  
+  end subroutine test_nonuniform_mosaic
+
+  subroutine fill_five_tile_halo(data, data_all, tile, ioff, joff)
+    real, dimension(1-whalo:,1-shalo:,:), intent(inout) :: data
+    real, dimension(:,:,:,:),             intent(in)    :: data_all
+    integer,                              intent(in)    :: tile, ioff, joff
+    integer                                             :: nxm, nym
 
-  111 format('For variable ', a, ', type = ', a, ', at pe = ', i3, ', at neighbor point (',i3,',',i3,',',i3, &
-             '), failed value = ', f14.9, ', but the value should be ', f14.9 )
+    nxm = 2*nx; nym = 2*ny
 
+    select case(tile)
+    case(1)
+       data(nxm+1+ioff:nxm+ehalo+ioff,                     1:ny,:) = data_all(1+ioff:ehalo+ioff,              1:ny,:,2) ! east
+       data(nxm+1+ioff:nxm+ehalo+ioff,            ny+1:nym+joff,:) = data_all(1+ioff:ehalo+ioff,         1:ny+joff,:,4) ! east
+       data(1-whalo:0,                                     1:ny,:) = data_all(nx-whalo+1:nx,                  1:ny,:,3) ! west
+       data(1-whalo:0,                            ny+1:nym+joff,:) = data_all(nx-whalo+1:nx,             1:ny+joff,:,5) ! west
+       data(1:nxm+ioff,                               1-shalo:0,:) = data_all(1:nxm+ioff,          nym-shalo+1:nym,:,1) ! south
+       data(1:nxm+ioff,               nym+1+joff:nym+nhalo+joff,:) = data_all(1:nxm+ioff,        1+joff:nhalo+joff,:,1) ! north
+       data(nxm+1+ioff:nxm+ehalo+ioff,                1-shalo:0,:) = data_all(1+ioff:ehalo+ioff,     ny-shalo+1:ny,:,4) ! southeast
+       data(1-whalo:0,                                1-shalo:0,:) = data_all(nx-whalo+1:nx,         ny-shalo+1:ny,:,5) ! southwest
+       data(nxm+1+ioff:nxm+ehalo+ioff,nym+1+joff:nym+nhalo+joff,:) = data_all(1+ioff:ehalo+ioff, 1+joff:nhalo+joff,:,2) ! northeast
+       data(1-whalo:0,                nym+1+joff:nym+nhalo+joff,:) = data_all(nx-whalo+1:nx,     1+joff:nhalo+joff,:,3) ! northwest
+    case(2)
+       data(nx+1+ioff:nx+ehalo+ioff,              1:ny+joff,:) = data_all(1+ioff:ehalo+ioff,              1:ny+joff,:,3) ! east
+       data(1-whalo:0,                            1:ny+joff,:) = data_all(nxm-whalo+1:nxm,                1:ny+joff,:,1) ! west
+       data(1:nx+ioff,                            1-shalo:0,:) = data_all(1:nx+ioff,                  ny-shalo+1:ny,:,4) ! south 
+       data(1:nx+ioff,              ny+1+joff:ny+nhalo+joff,:) = data_all(1:nx+ioff,              1+joff:nhalo+joff,:,4) ! north
+       data(nx+1+ioff:nx+ehalo+ioff,              1-shalo:0,:) = data_all(1+ioff:ehalo+ioff,          ny-shalo+1:ny,:,5) ! southeast
+       data(1-whalo:0,                            1-shalo:0,:) = data_all(nxm-whalo+1:nxm,          nym-shalo+1:nym,:,1) ! southwest
+       data(nx+1+ioff:nx+ehalo+ioff,ny+1+joff:ny+nhalo+joff,:) = data_all(1+ioff:ehalo+ioff,      1+joff:nhalo+joff,:,5) ! northeast
+       data(1-whalo:0,              ny+1+joff:ny+nhalo+joff,:) = data_all(nxm-whalo+1:nxm,  ny+1+joff:ny+nhalo+joff,:,1) ! northwest
+    case(3)
+       data(nx+1+ioff:nx+ehalo+ioff,              1:ny+joff,:) = data_all(1+ioff:ehalo+ioff,              1:ny+joff,:,1) ! east
+       data(1-whalo:0,                            1:ny+joff,:) = data_all(nx-whalo+1:nx,                  1:ny+joff,:,2) ! west
+       data(1:nx+ioff,                            1-shalo:0,:) = data_all(1:nx+ioff,                  ny-shalo+1:ny,:,5) ! south 
+       data(1:nx+ioff,              ny+1+joff:ny+nhalo+joff,:) = data_all(1:nx+ioff,              1+joff:nhalo+joff,:,5) ! north
+       data(nx+1+ioff:nx+ehalo+ioff,              1-shalo:0,:) = data_all(1+ioff:ehalo+ioff,        nym-shalo+1:nym,:,1) ! southeast
+       data(1-whalo:0,                            1-shalo:0,:) = data_all(nx-whalo+1:nx,              ny-shalo+1:ny,:,4) ! southwest
+       data(nx+1+ioff:nx+ehalo+ioff,ny+1+joff:ny+nhalo+joff,:) = data_all(1+ioff:ehalo+ioff,ny+1+joff:ny+nhalo+joff,:,1) ! northeast
+       data(1-whalo:0,              ny+1+joff:ny+nhalo+joff,:) = data_all(nx-whalo+1:nx,          1+joff:nhalo+joff,:,4) ! northwest
+    case(4)
+       data(nx+1+ioff:nx+ehalo+ioff,              1:ny+joff,:) = data_all(1+ioff:ehalo+ioff,        1:ny+joff,:,5) ! east
+       data(1-whalo:0,                            1:ny+joff,:) = data_all(nxm-whalo+1:nxm,     ny+1:2*ny+joff,:,1) ! west
+       data(1:nx+ioff,                            1-shalo:0,:) = data_all(1:nx+ioff,            ny-shalo+1:ny,:,2) ! south 
+       data(1:nx+ioff,              ny+1+joff:ny+nhalo+joff,:) = data_all(1:nx+ioff,        1+joff:nhalo+joff,:,2) ! north
+       data(nx+1+ioff:nx+ehalo+ioff,              1-shalo:0,:) = data_all(1+ioff:ehalo+ioff,    ny-shalo+1:ny,:,3) ! southeast
+       data(1-whalo:0,                            1-shalo:0,:) = data_all(nxm-whalo+1:nxm,      ny-shalo+1:ny,:,1) ! southwest
+       data(nx+1+ioff:nx+ehalo+ioff,ny+1+joff:ny+nhalo+joff,:) = data_all(1+ioff:ehalo+ioff,1+joff:nhalo+joff,:,3) ! northeast
+       data(1-whalo:0,              ny+1+joff:ny+nhalo+joff,:) = data_all(nxm-whalo+1:nxm,  1+joff:nhalo+joff,:,1) ! northwest
+    case(5)
+       data(nx+1+ioff:nx+ehalo+ioff,            1:  ny+joff,:) = data_all(1+ioff:ehalo+ioff,   ny+1:2*ny+joff,:,1) ! east
+       data(1-whalo:0,                            1:ny+joff,:) = data_all(nx-whalo+1:nx,            1:ny+joff,:,4) ! west
+       data(1:nx+ioff,                            1-shalo:0,:) = data_all(1:nx+ioff,            ny-shalo+1:ny,:,3) ! south 
+       data(1:nx+ioff,              ny+1+joff:ny+nhalo+joff,:) = data_all(1:nx+ioff,        1+joff:nhalo+joff,:,3) ! north
+       data(nx+1+ioff:nx+ehalo+ioff,              1-shalo:0,:) = data_all(1+ioff:ehalo+ioff,    ny-shalo+1:ny,:,1) ! southeast
+       data(1-whalo:0,                            1-shalo:0,:) = data_all(nx-whalo+1:nx,        ny-shalo+1:ny,:,2) ! southwest
+       data(nx+1+ioff:nx+ehalo+ioff,ny+1+joff:ny+nhalo+joff,:) = data_all(1+ioff:ehalo+ioff,1+joff:nhalo+joff,:,1) ! northeast
+       data(1-whalo:0,              ny+1+joff:ny+nhalo+joff,:) = data_all(nx-whalo+1:nx,    1+joff:nhalo+joff,:,2) ! northwest
+    end select
 
-  end subroutine test_refined_mosaic
+  end subroutine fill_five_tile_halo
 
   !#######################################################################################
   subroutine test_get_boundary(type)
      character(len=*), intent(in)  :: type
 
-     type(domain2D)       :: domain
+     type(domain2D)       :: domain, domain_nonsym
      integer              :: ntiles, num_contact, npes_per_tile, ntile_per_pe, layout(2)
      integer              :: n, l, isc, iec, jsc, jec, ism, iem, jsm, jem
      integer, allocatable, dimension(:)       :: tile, ni, nj, pe_start, pe_end
@@ -3963,7 +3649,9 @@ subroutine test_get_boundary(type)
      real,    allocatable, dimension(:,:,:,:) :: global_all, global1_all, global2_all
      real,    allocatable, dimension(:,:,:,:) :: global, global1, global2
      real,    allocatable, dimension(:,:,:,:) :: x, x1, x2, y, y1, y2
+     real,    allocatable, dimension(:,:)     :: u_nonsym, v_nonsym
      logical    :: folded_north = .false.
+     logical    :: is_torus = .false.
      integer    :: nx_save, ny_save
 
      nx_save    = nx
@@ -3982,6 +3670,9 @@ subroutine test_get_boundary(type)
     case ( 'Folded-north' )
        folded_north = .true.
        ntiles = 1       
+    case ( 'torus' )
+       is_torus = .true.
+       ntiles = 1
     case default
        call mpp_error(FATAL, 'TEST_MPP_DOMAINS: no such test: '//type)
     end select
@@ -4036,7 +3727,14 @@ subroutine test_get_boundary(type)
                           xflags=CYCLIC_GLOBAL_DOMAIN, yflags=FOLD_NORTH_EDGE, &
                           whalo=whalo, ehalo=ehalo, shalo=shalo, nhalo=nhalo, &
                           symmetry=.true., name='tripolar' )
-
+        call mpp_define_domains((/1,nx,1,ny/), layout, domain_nonsym, &
+                          xflags=CYCLIC_GLOBAL_DOMAIN, yflags=FOLD_NORTH_EDGE, &
+                          whalo=whalo, ehalo=ehalo, shalo=shalo, nhalo=nhalo, &
+                          symmetry=.false., name='tripolar' )        
+     case("torus")
+        call mpp_define_domains( (/1,nx,1,ny/), layout, domain, whalo=whalo, ehalo=ehalo, &
+                                 shalo=shalo, nhalo=nhalo, xflags=CYCLIC_GLOBAL_DOMAIN,   &
+                                 yflags=CYCLIC_GLOBAL_DOMAIN, symmetry=.true., name=type)        
      end select
 
     !--- Test the get_boundary of the data at C-cell center. 
@@ -4067,21 +3765,21 @@ subroutine test_get_boundary(type)
     x1 = x; x2 = x*10
 
     !--- buffer allocation
-    allocate(ebuffer(jec-jsc+2, nz, ntile_per_pe), wbuffer(jec-jsc+2, nz, ntile_per_pe))
-    allocate(sbuffer(iec-isc+2, nz, ntile_per_pe), nbuffer(iec-isc+2, nz, ntile_per_pe))
-    allocate(ebuffer1(jec-jsc+2, nz, ntile_per_pe), wbuffer1(jec-jsc+2, nz, ntile_per_pe))
-    allocate(sbuffer1(iec-isc+2, nz, ntile_per_pe), nbuffer1(iec-isc+2, nz, ntile_per_pe))
-    allocate(ebuffer2(jec-jsc+2, nz, ntile_per_pe), wbuffer2(jec-jsc+2, nz, ntile_per_pe))
-    allocate(sbuffer2(iec-isc+2, nz, ntile_per_pe), nbuffer2(iec-isc+2, nz, ntile_per_pe))
-    allocate(ebound(jec-jsc+2, nz, ntile_per_pe), wbound(jec-jsc+2, nz, ntile_per_pe))
-    allocate(sbound(iec-isc+2, nz, ntile_per_pe), nbound(iec-isc+2, nz, ntile_per_pe))
+    allocate(ebuffer(jsc:jec+1, nz, ntile_per_pe), wbuffer(jsc:jec+1, nz, ntile_per_pe))
+    allocate(sbuffer(isc:iec+1, nz, ntile_per_pe), nbuffer(isc:iec+1, nz, ntile_per_pe))
+    allocate(ebuffer1(jsc:jec+1, nz, ntile_per_pe), wbuffer1(jsc:jec+1, nz, ntile_per_pe))
+    allocate(sbuffer1(isc:iec+1, nz, ntile_per_pe), nbuffer1(isc:iec+1, nz, ntile_per_pe))
+    allocate(ebuffer2(jsc:jec+1, nz, ntile_per_pe), wbuffer2(jsc:jec+1, nz, ntile_per_pe))
+    allocate(sbuffer2(isc:iec+1, nz, ntile_per_pe), nbuffer2(isc:iec+1, nz, ntile_per_pe))
+    allocate(ebound(jsc:jec+1, nz, ntile_per_pe), wbound(jsc:jec+1, nz, ntile_per_pe))
+    allocate(sbound(isc:iec+1, nz, ntile_per_pe), nbound(isc:iec+1, nz, ntile_per_pe))
     ebound  = 0; ebuffer = 0; ebuffer1 = 0; ebuffer2 = 0
     sbound  = 0; sbuffer = 0; sbuffer1 = 0; sbuffer2 = 0
     wbound  = 0; wbuffer = 0; wbuffer1 = 0; wbuffer2 = 0
     nbound  = 0; nbuffer = 0; nbuffer1 = 0; nbuffer2 = 0
 
     do n = 1, ntile_per_pe 
-       if(folded_north) then
+       if(folded_north .or. is_torus ) then
           call mpp_get_boundary(x(:,:,:,n), domain, sbuffer=sbuffer(:,:,n), wbuffer=wbuffer(:,:,n), &
                                 position=CORNER, tile_count=n  )
        else
@@ -4092,7 +3790,7 @@ subroutine test_get_boundary(type)
 
     !--- multiple variable 
     do n = 1, ntile_per_pe 
-       if(folded_north) then
+       if(folded_north .or. is_torus) then
           call mpp_get_boundary(x1(:,:,:,n), domain, sbuffer=sbuffer1(:,:,n), wbuffer=wbuffer1(:,:,n), &
                position=CORNER, tile_count=n, complete = .false.  )
           call mpp_get_boundary(x2(:,:,:,n), domain, sbuffer=sbuffer2(:,:,n), wbuffer=wbuffer2(:,:,n), &
@@ -4118,13 +3816,20 @@ subroutine test_get_boundary(type)
                tile(n), 1, 1, ebound(:,:,n), sbound(:,:,n), wbound(:,:,n), nbound(:,:,n) )  
        end do
     case("Folded-north")
+       !---- folded line update
+       global_all(nx/2+2:nx, ny+1,:,1) = global_all(nx/2:2:-1, ny+1,:,1)
        do n = 1, ntile_per_pe
           call fill_folded_north_bound(global_all(:,:,:,1), isc, iec, jsc, jec, 1, 1, &
                tile(n), sbound(:,:,n), wbound(:,:,n) )  
        end do       
+    case("torus")
+       do n = 1, ntile_per_pe
+          call fill_torus_bound(global_all(:,:,:,1), isc, iec, jsc, jec, 1, 1, &
+               tile(n), sbound(:,:,n), wbound(:,:,n) )  
+       end do                    
     end select
 
-    if(.not. folded_north) then
+    if(.not. folded_north .AND. .not. is_torus) then
        call compare_checksums( ebound, ebuffer(:,:,:),  "east bound of "//trim(type) )
        call compare_checksums( nbound, nbuffer(:,:,:),  "north bound of "//trim(type) ) 
        call compare_checksums( ebound, ebuffer1(:,:,:),  "east bound of "//trim(type)//" X1" ) 
@@ -4177,28 +3882,33 @@ subroutine test_get_boundary(type)
     allocate( y1(ism:iem+1,jsm:jem+1,nz, ntile_per_pe) )
     allocate( y2(ism:iem+1,jsm:jem+1,nz, ntile_per_pe) )
     x = 0.; y = 0
-    x(isc:iec+1,jsc:jec+1,:,:) = global1(isc:iec+1,jsc:jec+1,:,:)
-    y(isc:iec+1,jsc:jec+1,:,:) = global2(isc:iec+1,jsc:jec+1,:,:)
+    if( trim(type) == "Folded-north" ) then
+       x(isc+1:iec+1,jsc+1:jec+1,:,:) = global1(isc+1:iec+1,jsc+1:jec+1,:,:)
+       y(isc+1:iec+1,jsc+1:jec+1,:,:) = global2(isc+1:iec+1,jsc+1:jec+1,:,:)
+    else
+       x(isc:iec+1,jsc:jec+1,:,:) = global1(isc:iec+1,jsc:jec+1,:,:)
+       y(isc:iec+1,jsc:jec+1,:,:) = global2(isc:iec+1,jsc:jec+1,:,:)
+    endif
     x1 = x; x2 = x*10
     y1 = y; y2 = y*10
 
     !--- buffer allocation
-    allocate(ebufferx(jec-jsc+2, nz, ntile_per_pe), wbufferx(jec-jsc+2, nz, ntile_per_pe))
-    allocate(sbufferx(iec-isc+2, nz, ntile_per_pe), nbufferx(iec-isc+2, nz, ntile_per_pe))
-    allocate(ebufferx1(jec-jsc+2, nz, ntile_per_pe), wbufferx1(jec-jsc+2, nz, ntile_per_pe))
-    allocate(sbufferx1(iec-isc+2, nz, ntile_per_pe), nbufferx1(iec-isc+2, nz, ntile_per_pe))
-    allocate(ebufferx2(jec-jsc+2, nz, ntile_per_pe), wbufferx2(jec-jsc+2, nz, ntile_per_pe))
-    allocate(sbufferx2(iec-isc+2, nz, ntile_per_pe), nbufferx2(iec-isc+2, nz, ntile_per_pe))
-    allocate(eboundx(jec-jsc+2, nz, ntile_per_pe), wboundx(jec-jsc+2, nz, ntile_per_pe))
-    allocate(sboundx(iec-isc+2, nz, ntile_per_pe), nboundx(iec-isc+2, nz, ntile_per_pe))
-    allocate(ebuffery(jec-jsc+2, nz, ntile_per_pe), wbuffery(jec-jsc+2, nz, ntile_per_pe))
-    allocate(sbuffery(iec-isc+2, nz, ntile_per_pe), nbuffery(iec-isc+2, nz, ntile_per_pe))
-    allocate(ebuffery1(jec-jsc+2, nz, ntile_per_pe), wbuffery1(jec-jsc+2, nz, ntile_per_pe))
-    allocate(sbuffery1(iec-isc+2, nz, ntile_per_pe), nbuffery1(iec-isc+2, nz, ntile_per_pe))
-    allocate(ebuffery2(jec-jsc+2, nz, ntile_per_pe), wbuffery2(jec-jsc+2, nz, ntile_per_pe))
-    allocate(sbuffery2(iec-isc+2, nz, ntile_per_pe), nbuffery2(iec-isc+2, nz, ntile_per_pe))
-    allocate(eboundy(jec-jsc+2, nz, ntile_per_pe), wboundy(jec-jsc+2, nz, ntile_per_pe))
-    allocate(sboundy(iec-isc+2, nz, ntile_per_pe), nboundy(iec-isc+2, nz, ntile_per_pe))
+    allocate(ebufferx(jsc:jec+1, nz, ntile_per_pe), wbufferx(jsc:jec+1, nz, ntile_per_pe))
+    allocate(sbufferx(isc:iec+1, nz, ntile_per_pe), nbufferx(isc:iec+1, nz, ntile_per_pe))
+    allocate(ebufferx1(jsc:jec+1, nz, ntile_per_pe), wbufferx1(jsc:jec+1, nz, ntile_per_pe))
+    allocate(sbufferx1(isc:iec+1, nz, ntile_per_pe), nbufferx1(isc:iec+1, nz, ntile_per_pe))
+    allocate(ebufferx2(jsc:jec+1, nz, ntile_per_pe), wbufferx2(jsc:jec+1, nz, ntile_per_pe))
+    allocate(sbufferx2(isc:iec+1, nz, ntile_per_pe), nbufferx2(isc:iec+1, nz, ntile_per_pe))
+    allocate(eboundx(jsc:jec+1, nz, ntile_per_pe), wboundx(jsc:jec+1, nz, ntile_per_pe))
+    allocate(sboundx(isc:iec+1, nz, ntile_per_pe), nboundx(isc:iec+1, nz, ntile_per_pe))
+    allocate(ebuffery(jsc:jec+1, nz, ntile_per_pe), wbuffery(jsc:jec+1, nz, ntile_per_pe))
+    allocate(sbuffery(isc:iec+1, nz, ntile_per_pe), nbuffery(isc:iec+1, nz, ntile_per_pe))
+    allocate(ebuffery1(jsc:jec+1, nz, ntile_per_pe), wbuffery1(jsc:jec+1, nz, ntile_per_pe))
+    allocate(sbuffery1(isc:iec+1, nz, ntile_per_pe), nbuffery1(isc:iec+1, nz, ntile_per_pe))
+    allocate(ebuffery2(jsc:jec+1, nz, ntile_per_pe), wbuffery2(jsc:jec+1, nz, ntile_per_pe))
+    allocate(sbuffery2(isc:iec+1, nz, ntile_per_pe), nbuffery2(isc:iec+1, nz, ntile_per_pe))
+    allocate(eboundy(jsc:jec+1, nz, ntile_per_pe), wboundy(jsc:jec+1, nz, ntile_per_pe))
+    allocate(sboundy(isc:iec+1, nz, ntile_per_pe), nboundy(isc:iec+1, nz, ntile_per_pe))
     eboundx  = 0; ebufferx = 0; ebufferx1 = 0; ebufferx2 = 0
     sboundx  = 0; sbufferx = 0; sbufferx1 = 0; sbufferx2 = 0
     wboundx  = 0; wbufferx = 0; wbufferx1 = 0; wbufferx2 = 0
@@ -4210,7 +3920,7 @@ subroutine test_get_boundary(type)
 
 
     do n = 1, ntile_per_pe 
-       if(folded_north) then
+       if(folded_north .or. is_torus) then
           call mpp_get_boundary(x(:,:,:,n), y(:,:,:,n), domain, sbufferx=sbufferx(:,:,n), wbufferx=wbufferx(:,:,n), &
                sbuffery=sbuffery(:,:,n), wbuffery=wbuffery(:,:,n), gridtype=BGRID_NE, tile_count=n, flags = SCALAR_PAIR  )
        else
@@ -4222,7 +3932,7 @@ subroutine test_get_boundary(type)
     end do
 
     do n = 1, ntile_per_pe 
-       if(folded_north) then
+       if(folded_north .or. is_torus) then
           call mpp_get_boundary(x1(:,:,:,n), y1(:,:,:,n), domain, sbufferx=sbufferx1(:,:,n), wbufferx=wbufferx1(:,:,n), &
                sbuffery=sbuffery1(:,:,n), wbuffery=wbuffery1(:,:,n),                           &
                gridtype=BGRID_NE, tile_count=n, flags = SCALAR_PAIR, complete = .false.  )
@@ -4258,15 +3968,24 @@ subroutine test_get_boundary(type)
                tile(n), 1, 1, eboundy(:,:,n), sboundy(:,:,n), wboundy(:,:,n), nboundy(:,:,n) )  
        end do
     case("Folded-north")
+       global1_all(nx/2+2:nx, ny+1,:,1) = global1_all(nx/2:2:-1, ny+1,:,1)
+       global2_all(nx/2+2:nx, ny+1,:,1) = global2_all(nx/2:2:-1, ny+1,:,1)
        do n = 1, ntile_per_pe
           call fill_folded_north_bound(global1_all(:,:,:,1), isc, iec, jsc, jec, 1, 1, &
                tile(n), sboundx(:,:,n), wboundx(:,:,n) )
           call fill_folded_north_bound(global2_all(:,:,:,1), isc, iec, jsc, jec, 1, 1, &
                tile(n), sboundy(:,:,n), wboundy(:,:,n) )  
        end do       
+    case("torus")
+       do n = 1, ntile_per_pe
+          call fill_torus_bound(global1_all(:,:,:,1), isc, iec, jsc, jec, 1, 1, &
+               tile(n), sboundx(:,:,n), wboundx(:,:,n) )
+          call fill_torus_bound(global2_all(:,:,:,1), isc, iec, jsc, jec, 1, 1, &
+               tile(n), sboundy(:,:,n), wboundy(:,:,n) )  
+       end do       
     end select
 
-    if(.not. folded_north) then
+    if(.not. folded_north .AND. .not. is_torus ) then
        call compare_checksums( eboundx, ebufferx(:,:,:),   "east bound of SCALAR_PAIR BGRID " //trim(type)//" X" )
        call compare_checksums( nboundx, nbufferx(:,:,:),   "north bound of SCALAR_PAIR BGRID "//trim(type)//" X" )
        call compare_checksums( eboundy, ebuffery(:,:,:),   "east bound of SCALAR_PAIR BGRID " //trim(type)//" Y" )
@@ -4308,9 +4027,16 @@ subroutine test_get_boundary(type)
           call fill_folded_north_bound(global2_all(:,:,:,1)*10, isc, iec, jsc, jec, 1, 1, &
                tile(n), sboundy(:,:,n), wboundy(:,:,n) )  
        end do       
+    case("torus")
+       do n = 1, ntile_per_pe
+          call fill_torus_bound(global1_all(:,:,:,1)*10, isc, iec, jsc, jec, 1, 1, &
+               tile(n), sboundx(:,:,n), wboundx(:,:,n) )
+          call fill_torus_bound(global2_all(:,:,:,1)*10, isc, iec, jsc, jec, 1, 1, &
+               tile(n), sboundy(:,:,n), wboundy(:,:,n) )  
+       end do       
     end select
 
-    if(.not. folded_north) then
+    if(.not. folded_north .AND. .not. is_torus ) then
        call compare_checksums( eboundx, ebufferx2(:,:,:),  "east bound of SCALAR_PAIR BGRID " //trim(type)//" X2" )
        call compare_checksums( nboundx, nbufferx2(:,:,:),  "north bound of SCALAR_PAIR BGRID "//trim(type)//" X2" )
        call compare_checksums( eboundy, ebuffery2(:,:,:),  "east bound of SCALAR_PAIR BGRID " //trim(type)//" Y2" )
@@ -4326,6 +4052,17 @@ subroutine test_get_boundary(type)
     !             Test 2-D Vector BGRID
     !
     !-------------------------------------------------------------------------------------------
+    do l = 1, ntiles
+       do k = 1, nz
+          do j = 1, ny+1
+             do i = 1, nx+1
+                global1_all(i,j,k,l) = 1.0e3 + l + i*1.0e-3 + j*1.0e-6 + k*1.0e-9
+                global2_all(i,j,k,l) = 2.0e3 + l + i*1.0e-3 + j*1.0e-6 + k*1.0e-9
+             end do
+          end do
+       end do
+    end do
+	
     x = 0.; y = 0
     eboundx  = 0; ebufferx = 0; ebufferx1 = 0; ebufferx2 = 0
     sboundx  = 0; sbufferx = 0; sbufferx1 = 0; sbufferx2 = 0
@@ -4340,9 +4077,10 @@ subroutine test_get_boundary(type)
     y(isc:iec+1,jsc:jec+1,1,:) = global2(isc:iec+1,jsc:jec+1,1,:)
 
     do n = 1, ntile_per_pe 
-       if(folded_north) then
+       if(folded_north .or. is_torus ) then
           call mpp_get_boundary(x(:,:,1,n), y(:,:,1,n), domain, sbufferx=sbufferx(:,1,n), wbufferx=wbufferx(:,1,n), &
-               sbuffery=sbuffery(:,1,n), wbuffery=wbuffery(:,1,n), gridtype=BGRID_NE, tile_count=n)
+               sbuffery=sbuffery(:,1,n), wbuffery=wbuffery(:,1,n), gridtype=BGRID_NE, tile_count=n) 
+
        else
           call mpp_get_boundary(x(:,:,1,n), y(:,:,1,n), domain, ebufferx=ebufferx(:,1,n), sbufferx=sbufferx(:,1,n), &
                wbufferx=wbufferx(:,1,n), nbufferx=nbufferx(:,1,n), ebuffery=ebuffery(:,1,n),       &
@@ -4351,6 +4089,53 @@ subroutine test_get_boundary(type)
        endif
     end do
 
+    if(folded_north) then
+       allocate(u_nonsym(ism:iem,jsm:jem), v_nonsym(ism:iem,jsm:jem))
+       u_nonsym = 0.0; v_nonsym = 0.0
+       u_nonsym(isc:iec,jsc:jec) = global1(isc+1:iec+1,jsc+1:jec+1,1,1)
+       v_nonsym(isc:iec,jsc:jec) = global2(isc+1:iec+1,jsc+1:jec+1,1,1)
+       call mpp_update_domains(u_nonsym, v_nonsym, domain_nonsym, gridtype=BGRID_NE)
+       !--- comparing boundary data
+       do i = isc,iec+1
+          if(i==1) cycle
+          if(sbufferx(i,1,1) .NE. u_nonsym(i-1,jsc-1)) then
+             print*,"pe ", mpp_pe(), i, jsc-1, sbufferx(i,1,1), u_nonsym(i-1,jsc-1)
+             call mpp_error(FATAL, "test_get_boundary: mismatch of sbufferx")
+          endif
+       enddo
+       call mpp_error(NOTE,"test_get_boundary: reproduce non-symmetric halo update for sbufferx")
+
+       do i = isc,iec+1
+          if(i==1) cycle
+          if(sbuffery(i,1,1) .NE. v_nonsym(i-1,jsc-1)) then
+             print*,"pe ", mpp_pe(), i, jsc-1, sbufferx(i,1,1), v_nonsym(i-1,jsc-1)
+             call mpp_error(FATAL, "test_get_boundary: mismatch of sbuffery")
+          endif
+       enddo
+      call mpp_error(NOTE,"test_get_boundary: reproduce non-symmetric halo update for sbuffery")
+
+       do j = jsc,jec+1
+          if(j == 1) cycle
+          if(wbufferx(j,1,1) .NE. u_nonsym(isc-1,j-1)) then
+             print*,"pe ", mpp_pe(), isc-1, j, wbufferx(j,1,1), u_nonsym(isc-1,j-1)
+             call mpp_error(FATAL, "test_get_boundary: mismatch of wbufferx")
+          endif
+       enddo
+      call mpp_error(NOTE,"test_get_boundary: reproduce non-symmetric halo update for wbufferx")
+
+       do j = jsc,jec+1
+          if(j==1) cycle
+          if(wbuffery(j,1,1) .NE. v_nonsym(isc-1,j-1)) then
+             print*,"pe ", mpp_pe(), isc-1, j, wbuffery(j,1,1), v_nonsym(isc-1,j-1)
+             call mpp_error(FATAL, "test_get_boundary: mismatch of wbuffery")
+          endif
+       enddo
+       call mpp_error(NOTE,"test_get_boundary: reproduce non-symmetric halo update for wbuffery")
+
+       deallocate(u_nonsym, v_nonsym)
+
+    endif
+
     !--- compare the buffer.
     select case(type)
     case("Four-Tile")
@@ -4368,15 +4153,39 @@ subroutine test_get_boundary(type)
                tile(n), -1, 1, eboundy(:,:,n), sboundy(:,:,n), wboundy(:,:,n), nboundy(:,:,n) )  
        end do
     case("Folded-north")
+       global1_all(nx/2+2:nx, ny+1,:,1) = -global1_all(nx/2:2:-1, ny+1,:,1)
+       global2_all(nx/2+2:nx, ny+1,:,1) = -global2_all(nx/2:2:-1, ny+1,:,1)
+       global1_all(1, ny+1,:,1) = 0
+       global2_all(1, ny+1,:,1) = 0
+       global1_all(nx/2+1, ny+1,:,1) = 0
+       global2_all(nx/2+1, ny+1,:,1) = 0
+       global1_all(nx+1, ny+1,:,1) = 0
+       global2_all(nx+1, ny+1,:,1) = 0
+
+
        do n = 1, ntile_per_pe
           call fill_folded_north_bound(global1_all(:,:,:,1), isc, iec, jsc, jec, 1, 1, &
                tile(n), sboundx(:,:,n), wboundx(:,:,n) )
           call fill_folded_north_bound(global2_all(:,:,:,1), isc, iec, jsc, jec, 1, 1, &
                tile(n), sboundy(:,:,n), wboundy(:,:,n) )  
+          ! set wboundx and wbouny to zero at pole (i=1, nx/2+1, nx+1)
+!          if( jec == ny ) then
+!             if( isc == 1 .OR. isc == nx/2+1 .OR. isc == nx+1 ) then
+!                wboundx(jec+1,:,n) = 0
+!                wboundy(jec+1,:,n) = 0
+!             endif
+!         endif
        end do       
+    case("torus")
+       do n = 1, ntile_per_pe
+          call fill_torus_bound(global1_all(:,:,:,1), isc, iec, jsc, jec, 1, 1, &
+               tile(n), sboundx(:,:,n), wboundx(:,:,n) )
+          call fill_torus_bound(global2_all(:,:,:,1), isc, iec, jsc, jec, 1, 1, &
+               tile(n), sboundy(:,:,n), wboundy(:,:,n) )  
+       enddo
     end select
 
-    if(.not. folded_north) then
+    if(.not. folded_north .AND. .not. is_torus ) then
        call compare_checksums( eboundx(:,1:1,:), ebufferx(:,1:1,:),   "east bound of 2-D BGRID " //trim(type)//" X" )
        call compare_checksums( nboundx(:,1:1,:), nbufferx(:,1:1,:),   "north bound of 2-D BGRID "//trim(type)//" X" )
        call compare_checksums( eboundy(:,1:1,:), ebuffery(:,1:1,:),   "east bound of 2-D BGRID " //trim(type)//" Y" )
@@ -4463,7 +4272,7 @@ subroutine test_get_boundary(type)
 
 
     do n = 1, ntile_per_pe 
-       if(folded_north) then
+       if(folded_north .or. is_torus) then
           call mpp_get_boundary(x(:,:,:,n), y(:,:,:,n), domain, wbufferx=wbufferx(:,:,n), &
                sbuffery=sbuffery(:,:,n), gridtype=CGRID_NE, tile_count=n  )
        else
@@ -4473,7 +4282,7 @@ subroutine test_get_boundary(type)
     end do
 
     do n = 1, ntile_per_pe 
-       if( folded_north ) then
+       if( folded_north .or. is_torus ) then
           call mpp_get_boundary(x1(:,:,:,n), y1(:,:,:,n), domain, wbufferx=wbufferx1(:,:,n), &
                sbuffery=sbuffery1(:,:,n), gridtype=CGRID_NE, tile_count=n,  &
                complete = .false.  )
@@ -4508,14 +4317,21 @@ subroutine test_get_boundary(type)
        end do
     case("Folded-north")
        do n = 1, ntile_per_pe
-          call fill_folded_north_bound(global1_all(:,:,:,1), isc, iec, jsc, jec, 1, 1, &
+          call fill_folded_north_bound(global1_all(:,:,:,1), isc, iec, jsc, jec, 1, 0, &
                tile(n), wbound=wboundx(:,:,n) )
-          call fill_folded_north_bound(global2_all(:,:,:,1), isc, iec, jsc, jec, 1, 1, &
+          call fill_folded_north_bound(global2_all(:,:,:,1), isc, iec, jsc, jec, 0, 1, &
+               tile(n), sbound=sboundy(:,:,n) )  
+       end do       
+    case("torus")
+       do n = 1, ntile_per_pe
+          call fill_torus_bound(global1_all(:,:,:,1), isc, iec, jsc, jec, 1, 0, &
+               tile(n), wbound=wboundx(:,:,n) )
+          call fill_torus_bound(global2_all(:,:,:,1), isc, iec, jsc, jec, 0, 1, &
                tile(n), sbound=sboundy(:,:,n) )  
        end do       
     end select
 
-    if(.not. folded_north) then
+    if(.not. folded_north .and. .not. is_torus ) then
        call compare_checksums( eboundx, ebufferx(:,:,:),   "east bound of CGRID " //trim(type)//" X" )
        call compare_checksums( nboundy, nbuffery(:,:,:),   "north bound of CGRID "//trim(type)//" Y" )
        call compare_checksums( eboundx, ebufferx1(:,:,:),  "east bound of CGRID " //trim(type)//" X1" )
@@ -4543,14 +4359,21 @@ subroutine test_get_boundary(type)
        end do
     case("Folded-north")
        do n = 1, ntile_per_pe
-          call fill_folded_north_bound(global1_all(:,:,:,1)*10, isc, iec, jsc, jec, 1, 1, &
+          call fill_folded_north_bound(global1_all(:,:,:,1)*10, isc, iec, jsc, jec, 1, 0, &
                tile(n), wbound=wboundx(:,:,n) )
-          call fill_folded_north_bound(global2_all(:,:,:,1)*10, isc, iec, jsc, jec, 1, 1, &
+          call fill_folded_north_bound(global2_all(:,:,:,1)*10, isc, iec, jsc, jec, 0, 1, &
+               tile(n), sbound=sboundy(:,:,n) )  
+       end do       
+    case("torus")
+       do n = 1, ntile_per_pe
+          call fill_torus_bound(global1_all(:,:,:,1)*10, isc, iec, jsc, jec, 1, 0, &
+               tile(n), wbound=wboundx(:,:,n) )
+          call fill_torus_bound(global2_all(:,:,:,1)*10, isc, iec, jsc, jec, 0, 1, &
                tile(n), sbound=sboundy(:,:,n) )  
        end do       
     end select
 
-    if(.not. folded_north) then
+    if(.not. folded_north .and. .not. is_torus ) then
        call compare_checksums( eboundx, ebufferx2(:,:,:),  "east bound of CGRID " //trim(type)//" X2" )
        call compare_checksums( nboundy, nbuffery2(:,:,:),  "north bound of CGRID "//trim(type)//" Y2" )
     endif
diff --git a/src/shared/mpp/test_mpp_io.F90 b/src/shared/mpp/test_mpp_io.F90
index 6579ea0e15..7774f2e97e 100644
--- a/src/shared/mpp/test_mpp_io.F90
+++ b/src/shared/mpp/test_mpp_io.F90
@@ -16,6 +16,10 @@ program test
   use mpp_io_mod,      only : mpp_get_info, mpp_get_axes, mpp_get_fields, mpp_get_times
   use mpp_io_mod,      only : mpp_read, mpp_io_exit
 
+#ifdef INTERNAL_FILE_NML
+  USE mpp_mod, ONLY: input_nml_file
+#endif
+
   implicit none
 
 #ifdef use_netCDF
@@ -61,7 +65,7 @@ program test
   npes = mpp_npes()
 
 #ifdef INTERNAL_FILE_NML
-  read (input_nml_file, test_mpp_io_nml, status=io_status)
+  read (input_nml_file, test_mpp_io_nml, iostat=io_status)
 #else
   do
      inquire( unit=unit, opened=opened )
diff --git a/src/shared/oda_tools/oda_core.F90 b/src/shared/oda_tools/oda_core.F90
index 335c1999d2..2be3564dac 100644
--- a/src/shared/oda_tools/oda_core.F90
+++ b/src/shared/oda_tools/oda_core.F90
@@ -1,6 +1,6 @@
 module oda_core_mod
 !
-! Matthew Harrison
+! Matthew Harrison
 !
 !
 !
@@ -141,7 +141,7 @@ module oda_core_mod
 
   integer, allocatable, dimension(:) :: nprof_in_comp_domain
   
-  namelist /oda_core_nml/ max_misfit, add_tidal_aliasing, min_obs_err_t,&
+  namelist /oda_core_nml/ max_misfit,add_tidal_aliasing,min_obs_err_t,&
                           min_obs_err_s, eta_tide_const, debug, max_profiles,&
                           max_sfc_obs, temp_obs_rmse, salt_obs_rmse, ndebug
                           
@@ -643,8 +643,8 @@ subroutine open_profile_dataset(filename, localize)
 ! NOTE: this only works for lat/lon grids. Lower left indices.           
 !       
            
-           ri0 = frac_index(lon, x_grid(isg-1:ieg+1)) - 1 
-           rj0 = frac_index(lat, y_grid(jsg-1:jeg+1)) - 1
+           ri0 = frac_index(lon, x_grid(isg-1:ieg+1)) - 1.
+           rj0 = frac_index(lat, y_grid(jsg-1:jeg+1)) - 1.
            i0 = floor(ri0)
            j0 = floor(rj0)
            Profiles(num_profiles)%i_index = ri0
@@ -1147,14 +1147,14 @@ subroutine backward_obs_profile(Obs,model_t,model_s)
     end do
 
 
-    where(sum_wgt > 0)
+    where(sum_wgt > 0.0)
         model_t = model_t /sum_wgt
     elsewhere
         model_t = 0.0
     end where
 
     if (PRESENT(model_s)) then
-        where(sum_wgt > 0)
+        where(sum_wgt > 0.0)
             model_s = model_s /sum_wgt
         elsewhere
             model_s = 0.0
@@ -1366,7 +1366,7 @@ subroutine mult_obs_I_mse_profile(Obs)
     
     do n=1,size(Obs)
        do k = 1, Obs(n)%levels
-          Ims = 1/Obs(n)%ms_t(k)
+          Ims = 1./Obs(n)%ms_t(k)
           if (Obs(n)%flag_t(k) .eq. 0) Obs(n)%data_t(k) = Ims*Obs(n)%data_t(k)
           if (ASSOCIATED(Obs(n)%data_s)) then
               Ims = 1/Obs(n)%ms_s(k)
@@ -1705,7 +1705,7 @@ subroutine create_ideal_profiles(localize)
            allocate(profiles(num_profiles)%flag_s(nlevels))
            profiles(num_profiles)%flag_s= 0
 
-           profiles(num_profiles)%probe = 0
+           profiles(num_profiles)%probe = 0.
            profiles(num_profiles)%levels = nlevels
            profiles(num_profiles)%lat = lat(i)
            profiles(num_profiles)%lon = lon(i)
@@ -1719,8 +1719,8 @@ subroutine create_ideal_profiles(localize)
 ! calculate interpolation coefficients (make sure to account for grid offsets here!)
 ! note that this only works for lat/lon grids
            
-           ri0 = frac_index(lon(i), x_grid(isg-1:ieg+1)) - 1 
-           rj0 = frac_index(lat(i), y_grid(jsg-1:jeg+1)) - 1
+           ri0 = frac_index(lon(i), x_grid(isg-1:ieg+1)) - 1.
+           rj0 = frac_index(lat(i), y_grid(jsg-1:jeg+1)) - 1.
            i0 = floor(ri0)
            j0 = floor(rj0)
            Profiles(num_profiles)%i_index = ri0
@@ -1818,13 +1818,13 @@ subroutine nullify_obs_prof(profile)
 
 
     profile%nvar = 0
-    profile%project=0
-    profile%probe=0
-    profile%ref_inst=0
+    profile%project=0.
+    profile%probe=0.
+    profile%ref_inst=0.
     profile%wod_cast_num=0
-    profile%fix_depth=0
-    profile%ocn_vehicle=0
-    profile%database_id=0
+    profile%fix_depth=0.
+    profile%ocn_vehicle=0.
+    profile%database_id=0.
     profile%levels=0
     profile%profile_flag=-1
     profile%profile_flag_s=-1
diff --git a/src/shared/oda_tools/oda_core_ecda.F90 b/src/shared/oda_tools/oda_core_ecda.F90
index 23225f8119..c4b4989adc 100644
--- a/src/shared/oda_tools/oda_core_ecda.F90
+++ b/src/shared/oda_tools/oda_core_ecda.F90
@@ -669,37 +669,40 @@ subroutine open_profile_dataset(filename, time_start, time_end, obs_variable, lo
        if ( profile_time > time_start .and. profile_time < time_end ) data_in_period = .true.
        if ( (data_in_period .and. data_is_local) .and. (.NOT.localize_data) ) then ! localize
 
-          if (isd_filt >= 1 .and. ied_filt <= ieg) then
-            if (lon >= x_grid(isd_filt,jsd_flt0) .and.&
-               & lon <= x_grid(ied_filt-1,jsd_flt0) .and.&
-               & lat >= y_grid(isd_filt,jsd_flt0) .and.&
-               & lat <=  y_grid(ied_filt-1,jed_flt0-1)) then
+          if (isd_filt < 1 .and. ied_filt > ieg) then
+             ! filter domain is a full x band
+             if (lon >= x_grid(1,jsd_flt0) .and. lon <= x_grid(ieg-1,jsd_flt0) .and.&
+                  lat >= y_grid(1,jsd_flt0) .and. lat <= y_grid(ieg-1,jsd_flt0)) then
                 prof_in_filt_domain = .true.
-            end if
-          end if
-          if (isd_filt < 1) then
-            isd_flt0 = isd_filt + ieg
-            if ( lon >= x_grid(1,jsd_flt0) .and. lon <= x_grid(ied_filt-1,jsd_flt0) .and.&
-                  & lat >= y_grid(1,jsd_flt0) .and. lat <=  y_grid(ied_filt-1,jed_flt0-1) ) then
+             end if
+          else if (isd_filt >= 1 .and. ied_filt <= ieg) then
+             ! Interior filter domain
+            if (lon >= x_grid(isd_filt,jsd_flt0) .and. lon <= x_grid(ied_filt-1,jsd_flt0) .and.&
+               & lat >= y_grid(isd_filt,jsd_flt0) .and. lat <=  y_grid(ied_filt-1,jed_flt0-1)) then
                 prof_in_filt_domain = .true.
             end if
-            if ( lon >= x_grid(isd_flt0,jsd_flt0) .and. lon <= x_grid(ieg-1,jsd_flt0) .and.&
-                  & lat >= y_grid(isd_flt0,jsd_flt0) .and. lat <=  y_grid(ieg-1,jed_flt0-1) ) then
+          else if (isd_filt < 1 .and. ied_filt <= ieg) then
+             ! lhs filter domain
+             isd_flt0 = isd_filt + ieg
+             if ((lon >= x_grid(1,jsd_flt0) .and. lon <= x_grid(ied_filt-1,jsd_flt0) .and.&
+                  & lat >= y_grid(1,jsd_flt0) .and. lat <=  y_grid(ied_filt-1,jed_flt0-1)).or.&
+                  & (lon >= x_grid(isd_flt0,jsd_flt0) .and. lon <= x_grid(ieg-1,jsd_flt0) .and.&
+                  & lat >= y_grid(isd_flt0,jsd_flt0) .and. lat <=  y_grid(ieg-1,jed_flt0-1))) then
                 prof_in_filt_domain = .true.
-            end if
-          end if
-          if (ied_filt > ieg) then
-            ied_flt0 = ied_filt - ieg
-            if ( lon >= x_grid(isd_filt,jsd_flt0) .and. lon <= x_grid(ieg-1,jsd_flt0) .and.&
+             end if
+          else if (isd_filt >= 1 .and. ied_filt > ieg) then
+             ! rhs filter domain
+             ied_flt0 = ied_filt - ieg
+             if ( lon >= x_grid(isd_filt,jsd_flt0) .and. lon <= x_grid(ieg-1,jsd_flt0) .and.&
                   & lat >= y_grid(isd_filt,jsd_flt0) .and. lat <=  y_grid(ieg-1,jed_flt0-1) ) then
                 prof_in_filt_domain = .true.
-            end if
-            if (ied_flt0-1 > 1) then
-            if ( lon >= x_grid(1,jsd_flt0) .and. lon <= x_grid(ied_flt0-1,jsd_flt0) .and.&
-                  & lat >= y_grid(1,jsd_flt0) .and. lat <=  y_grid(ied_flt0-1,jed_flt0-1) ) then
-                prof_in_filt_domain = .true.
-            end if
-            end if
+             end if
+             if (ied_flt0-1 > 1) then
+                if ( lon >= x_grid(1,jsd_flt0) .and. lon <= x_grid(ied_flt0-1,jsd_flt0) .and.&
+                     & lat >= y_grid(1,jsd_flt0) .and. lat <=  y_grid(ied_flt0-1,jed_flt0-1) ) then
+                   prof_in_filt_domain = .true.
+                end if
+             end if
           end if
 
           if ( var_id == TEMP_ID .and. flag_t == 0.0 ) then
@@ -939,7 +942,7 @@ subroutine open_profile_dataset(filename, time_start, time_end, obs_variable, lo
              if (i0 < 1 .or. j0 < 1) then
                 Profiles(num_profiles)%accepted = .false.
              end if
-             if( i0 < isd_filt .or. i0 > ied_filt .or. j0 < jsd_filt .or. j0 > jed_filt ) then
+             if( i0 < isd_filt .or. i0 >= ied_filt .or. j0 < jsd_filt .or. j0 >= jed_filt ) then
                 Profiles(num_profiles)%accepted = .false.
              end if
 
@@ -1288,37 +1291,40 @@ subroutine open_profile_dataset_argo(filename, time_start, time_end, obs_variabl
        if ( profile_time > time_start .and. profile_time < time_end ) data_in_period = .true.
        if ( (data_in_period .and. data_is_local) .and. (.NOT.localize_data) ) then
 
-          if (isd_filt >= 1 .and. ied_filt <= ieg) then
-            if (lon >= x_grid(isd_filt,jsd_flt0) .and.&
-               & lon <= x_grid(ied_filt-1,jsd_flt0) .and.&
-               & lat >= y_grid(isd_filt,jsd_flt0) .and.&
-               & lat <=  y_grid(ied_filt-1,jed_flt0-1)) then
+          if (isd_filt < 1 .and. ied_filt > ieg) then
+             ! filter domain is a full x band
+             if (lon >= x_grid(1,jsd_flt0) .and. lon <= x_grid(ieg-1,jsd_flt0) .and.&
+                  lat >= y_grid(1,jsd_flt0) .and. lat <= y_grid(ieg-1,jsd_flt0)) then
                 prof_in_filt_domain = .true.
-            end if
-          end if
-          if (isd_filt < 1) then
-            isd_flt0 = isd_filt + ieg
-            if ( lon >= x_grid(1,jsd_flt0) .and. lon <= x_grid(ied_filt-1,jsd_flt0) .and.&
-                  & lat >= y_grid(1,jsd_flt0) .and. lat <=  y_grid(ied_filt-1,jed_flt0-1) ) then
+             end if
+          else if (isd_filt >= 1 .and. ied_filt <= ieg) then
+             ! Interior filter domain
+            if (lon >= x_grid(isd_filt,jsd_flt0) .and. lon <= x_grid(ied_filt-1,jsd_flt0) .and.&
+               & lat >= y_grid(isd_filt,jsd_flt0) .and. lat <=  y_grid(ied_filt-1,jed_flt0-1)) then
                 prof_in_filt_domain = .true.
             end if
-            if ( lon >= x_grid(isd_flt0,jsd_flt0) .and. lon <= x_grid(ieg-1,jsd_flt0) .and.&
-                  & lat >= y_grid(isd_flt0,jsd_flt0) .and. lat <=  y_grid(ieg-1,jed_flt0-1) ) then
+          else if (isd_filt < 1 .and. ied_filt <= ieg) then
+             ! lhs filter domain
+             isd_flt0 = isd_filt + ieg
+             if ((lon >= x_grid(1,jsd_flt0) .and. lon <= x_grid(ied_filt-1,jsd_flt0) .and.&
+                  & lat >= y_grid(1,jsd_flt0) .and. lat <=  y_grid(ied_filt-1,jed_flt0-1)).or.&
+                  & (lon >= x_grid(isd_flt0,jsd_flt0) .and. lon <= x_grid(ieg-1,jsd_flt0) .and.&
+                  & lat >= y_grid(isd_flt0,jsd_flt0) .and. lat <=  y_grid(ieg-1,jed_flt0-1))) then
                 prof_in_filt_domain = .true.
-            end if
-          end if
-          if (ied_filt > ieg) then
-            ied_flt0 = ied_filt - ieg
-            if ( lon >= x_grid(isd_filt,jsd_flt0) .and. lon <= x_grid(ieg-1,jsd_flt0) .and.&
+             end if
+          else if (isd_filt >= 1 .and. ied_filt > ieg) then
+             ! rhs filter domain
+             ied_flt0 = ied_filt - ieg
+             if ( lon >= x_grid(isd_filt,jsd_flt0) .and. lon <= x_grid(ieg-1,jsd_flt0) .and.&
                   & lat >= y_grid(isd_filt,jsd_flt0) .and. lat <=  y_grid(ieg-1,jed_flt0-1) ) then
                 prof_in_filt_domain = .true.
-            end if
-            if (ied_flt0-1 > 1) then
-            if ( lon >= x_grid(1,jsd_flt0) .and. lon <= x_grid(ied_flt0-1,jsd_flt0) .and.&
-                  & lat >= y_grid(1,jsd_flt0) .and. lat <=  y_grid(ied_flt0-1,jed_flt0-1) ) then
-                prof_in_filt_domain = .true.
-            end if
-            end if
+             end if
+             if (ied_flt0-1 > 1) then
+                if ( lon >= x_grid(1,jsd_flt0) .and. lon <= x_grid(ied_flt0-1,jsd_flt0) .and.&
+                     & lat >= y_grid(1,jsd_flt0) .and. lat <=  y_grid(ied_flt0-1,jed_flt0-1) ) then
+                   prof_in_filt_domain = .true.
+                end if
+             end if
           end if
 
           if ( var_id == TEMP_ID ) then
@@ -1543,7 +1549,7 @@ subroutine open_profile_dataset_argo(filename, time_start, time_end, obs_variabl
            if ( i0 < 1 .or. j0 < 1 ) then
               Profiles(num_profiles)%accepted = .false.
            end if
-           if ( i0 < isd_filt .or. i0 > ied_filt .or. j0 < jsd_filt .or. j0 > jed_filt ) then
+           if ( i0 < isd_filt .or. i0 >= ied_filt .or. j0 < jsd_filt .or. j0 >= jed_filt ) then
               Profiles(num_profiles)%accepted = .false.
            end if
 
@@ -2185,37 +2191,40 @@ subroutine open_profile_dataset_woa05t(filename, obs_variable, localize)
           if ( abs(lat) >= 20.0 .and. (mod(i,4) /= 0 .or. mod(j,4) /= 0) ) data_is_local = .false.
           if ( abs(lat) >= 60.0 .and. (mod(i,6) /= 0 .or. mod(j,6) /= 0) ) data_is_local = .false.
 
-          if (isd_filt >= 1 .and. ied_filt <= ieg) then
-            if (lon >= x_grid(isd_filt,jsd_flt0) .and.&
-               & lon <= x_grid(ied_filt-1,jsd_flt0) .and.&
-               & lat >= y_grid(isd_filt,jsd_flt0) .and.&
-               & lat <=  y_grid(ied_filt-1,jed_flt0-1)) then
+          if (isd_filt < 1 .and. ied_filt > ieg) then
+             ! filter domain is a full x band
+             if (lon >= x_grid(1,jsd_flt0) .and. lon <= x_grid(ieg-1,jsd_flt0) .and.&
+                  lat >= y_grid(1,jsd_flt0) .and. lat <= y_grid(ieg-1,jsd_flt0)) then
                 prof_in_filt_domain = .true.
-            end if
-          end if
-          if (isd_filt < 1) then
-            isd_flt0 = isd_filt + ieg
-            if ( lon >= x_grid(1,jsd_flt0) .and. lon <= x_grid(ied_filt-1,jsd_flt0) .and.&
-                  & lat >= y_grid(1,jsd_flt0) .and. lat <=  y_grid(ied_filt-1,jed_flt0-1) ) then
+             end if
+          else if (isd_filt >= 1 .and. ied_filt <= ieg) then
+             ! Interior filter domain
+            if (lon >= x_grid(isd_filt,jsd_flt0) .and. lon <= x_grid(ied_filt-1,jsd_flt0) .and.&
+               & lat >= y_grid(isd_filt,jsd_flt0) .and. lat <=  y_grid(ied_filt-1,jed_flt0-1)) then
                 prof_in_filt_domain = .true.
             end if
-            if ( lon >= x_grid(isd_flt0,jsd_flt0) .and. lon <= x_grid(ieg-1,jsd_flt0) .and.&
-                  & lat >= y_grid(isd_flt0,jsd_flt0) .and. lat <=  y_grid(ieg-1,jed_flt0-1) ) then
+          else if (isd_filt < 1 .and. ied_filt <= ieg) then
+             ! lhs filter domain
+             isd_flt0 = isd_filt + ieg
+             if ((lon >= x_grid(1,jsd_flt0) .and. lon <= x_grid(ied_filt-1,jsd_flt0) .and.&
+                  & lat >= y_grid(1,jsd_flt0) .and. lat <=  y_grid(ied_filt-1,jed_flt0-1)).or.&
+                  & (lon >= x_grid(isd_flt0,jsd_flt0) .and. lon <= x_grid(ieg-1,jsd_flt0) .and.&
+                  & lat >= y_grid(isd_flt0,jsd_flt0) .and. lat <=  y_grid(ieg-1,jed_flt0-1))) then
                 prof_in_filt_domain = .true.
-            end if
-          end if
-          if (ied_filt > ieg) then
-            ied_flt0 = ied_filt - ieg
-            if ( lon >= x_grid(isd_filt,jsd_flt0) .and. lon <= x_grid(ieg-1,jsd_flt0) .and.&
+             end if
+          else if (isd_filt >= 1 .and. ied_filt > ieg) then
+             ! rhs filter domain
+             ied_flt0 = ied_filt - ieg
+             if ( lon >= x_grid(isd_filt,jsd_flt0) .and. lon <= x_grid(ieg-1,jsd_flt0) .and.&
                   & lat >= y_grid(isd_filt,jsd_flt0) .and. lat <=  y_grid(ieg-1,jed_flt0-1) ) then
                 prof_in_filt_domain = .true.
-            end if
-            if (ied_flt0-1 > 1) then
-            if ( lon >= x_grid(1,jsd_flt0) .and. lon <= x_grid(ied_flt0-1,jsd_flt0) .and.&
-                  & lat >= y_grid(1,jsd_flt0) .and. lat <=  y_grid(ied_flt0-1,jed_flt0-1) ) then
-                prof_in_filt_domain = .true.
-            end if
-            end if
+             end if
+             if (ied_flt0-1 > 1) then
+                if ( lon >= x_grid(1,jsd_flt0) .and. lon <= x_grid(ied_flt0-1,jsd_flt0) .and.&
+                     & lat >= y_grid(1,jsd_flt0) .and. lat <=  y_grid(ied_flt0-1,jed_flt0-1) ) then
+                   prof_in_filt_domain = .true.
+                end if
+             end if
           end if
 
           if ( data_is_local .and. (.NOT.localize_data) ) then ! global index
@@ -2334,7 +2343,7 @@ subroutine open_profile_dataset_woa05t(filename, obs_variable, localize)
              if ( i0 < 1 .or. j0 < 1 ) then
                 Profiles(num_profiles)%accepted = .false.
              end if
-             if ( i0 < isd_filt .or. i0 > ied_filt .or. j0 < jsd_filt .or. j0 > jed_filt ) then
+             if ( i0 < isd_filt .or. i0 >= ied_filt .or. j0 < jsd_filt .or. j0 >= jed_filt ) then
                 Profiles(num_profiles)%accepted = .false.
              end if
 
@@ -2540,37 +2549,40 @@ subroutine open_profile_dataset_woa05s(filename, obs_variable, localize)
           if ( abs(lat) >= 20.0 .and. (mod(i,4) /= 0 .or. mod(j,4) /= 0) ) data_is_local = .false.
           if ( abs(lat) >= 60.0 .and. (mod(i,6) /= 0 .or. mod(j,6) /= 0) ) data_is_local = .false.
 
-          if (isd_filt >= 1 .and. ied_filt <= ieg) then
-            if (lon >= x_grid(isd_filt,jsd_flt0) .and.&
-               & lon <= x_grid(ied_filt-1,jsd_flt0) .and.&
-               & lat >= y_grid(isd_filt,jsd_flt0) .and.&
-               & lat <=  y_grid(ied_filt-1,jed_flt0-1)) then
+          if (isd_filt < 1 .and. ied_filt > ieg) then
+             ! filter domain is a full x band
+             if (lon >= x_grid(1,jsd_flt0) .and. lon <= x_grid(ieg-1,jsd_flt0) .and.&
+                  lat >= y_grid(1,jsd_flt0) .and. lat <= y_grid(ieg-1,jsd_flt0)) then
                 prof_in_filt_domain = .true.
-            end if
-          end if
-          if (isd_filt < 1) then
-            isd_flt0 = isd_filt + ieg
-            if ( lon >= x_grid(1,jsd_flt0) .and. lon <= x_grid(ied_filt-1,jsd_flt0) .and.&
-                  & lat >= y_grid(1,jsd_flt0) .and. lat <=  y_grid(ied_filt-1,jed_flt0-1) ) then
+             end if
+          else if (isd_filt >= 1 .and. ied_filt <= ieg) then
+             ! Interior filter domain
+            if (lon >= x_grid(isd_filt,jsd_flt0) .and. lon <= x_grid(ied_filt-1,jsd_flt0) .and.&
+               & lat >= y_grid(isd_filt,jsd_flt0) .and. lat <=  y_grid(ied_filt-1,jed_flt0-1)) then
                 prof_in_filt_domain = .true.
             end if
-            if ( lon >= x_grid(isd_flt0,jsd_flt0) .and. lon <= x_grid(ieg-1,jsd_flt0) .and.&
-                  & lat >= y_grid(isd_flt0,jsd_flt0) .and. lat <=  y_grid(ieg-1,jed_flt0-1) ) then
+          else if (isd_filt < 1 .and. ied_filt <= ieg) then
+             ! lhs filter domain
+             isd_flt0 = isd_filt + ieg
+             if ((lon >= x_grid(1,jsd_flt0) .and. lon <= x_grid(ied_filt-1,jsd_flt0) .and.&
+                  & lat >= y_grid(1,jsd_flt0) .and. lat <=  y_grid(ied_filt-1,jed_flt0-1)).or.&
+                  & (lon >= x_grid(isd_flt0,jsd_flt0) .and. lon <= x_grid(ieg-1,jsd_flt0) .and.&
+                  & lat >= y_grid(isd_flt0,jsd_flt0) .and. lat <=  y_grid(ieg-1,jed_flt0-1))) then
                 prof_in_filt_domain = .true.
-            end if
-          end if
-          if (ied_filt > ieg) then
-            ied_flt0 = ied_filt - ieg
-            if ( lon >= x_grid(isd_filt,jsd_flt0) .and. lon <= x_grid(ieg-1,jsd_flt0) .and.&
+             end if
+          else if (isd_filt >= 1 .and. ied_filt > ieg) then
+             ! rhs filter domain
+             ied_flt0 = ied_filt - ieg
+             if ( lon >= x_grid(isd_filt,jsd_flt0) .and. lon <= x_grid(ieg-1,jsd_flt0) .and.&
                   & lat >= y_grid(isd_filt,jsd_flt0) .and. lat <=  y_grid(ieg-1,jed_flt0-1) ) then
                 prof_in_filt_domain = .true.
-            end if
-            if (ied_flt0-1 > 1) then
-            if ( lon >= x_grid(1,jsd_flt0) .and. lon <= x_grid(ied_flt0-1,jsd_flt0) .and.&
-                  & lat >= y_grid(1,jsd_flt0) .and. lat <=  y_grid(ied_flt0-1,jed_flt0-1) ) then
-                prof_in_filt_domain = .true.
-            end if
-            end if
+             end if
+             if (ied_flt0-1 > 1) then
+                if ( lon >= x_grid(1,jsd_flt0) .and. lon <= x_grid(ied_flt0-1,jsd_flt0) .and.&
+                     & lat >= y_grid(1,jsd_flt0) .and. lat <=  y_grid(ied_flt0-1,jed_flt0-1) ) then
+                   prof_in_filt_domain = .true.
+                end if
+             end if
           end if
 
           if ( data_is_local .and. (.NOT.localize_data) ) then ! global index
@@ -2689,7 +2701,7 @@ subroutine open_profile_dataset_woa05s(filename, obs_variable, localize)
              if ( i0 < 1 .or. j0 < 1 ) then
                 Profiles(num_profiles)%accepted = .false.
              end if
-             if ( i0 < isd_filt .or. i0 > ied_filt .or. j0 < jsd_filt .or. j0 > jed_filt ) then
+             if ( i0 < isd_filt .or. i0 >= ied_filt .or. j0 < jsd_filt .or. j0 >= jed_filt ) then
                 Profiles(num_profiles)%accepted = .false.
              end if
 
diff --git a/src/shared/oda_tools/oda_types.F90 b/src/shared/oda_tools/oda_types.F90
index af400d6123..6438f1aa96 100644
--- a/src/shared/oda_tools/oda_types.F90
+++ b/src/shared/oda_tools/oda_types.F90
@@ -9,22 +9,22 @@ module oda_types_mod
 
 !============================================================
 ! This module contains type declarations and default values
-! for oda modules.
+! for oda modules.  
 !============================================================
-
-! Contact: Matthew.Harrison
+  
+! Contact: Matthew.Harrison@gfdl.noaa.gov
 
   use time_manager_mod, only : time_type, set_time
   use mpp_mod, only : stdout
   use mpp_domains_mod, only : domain2d
-
+  
   implicit none
 
   private
 
-  integer, parameter, public :: MAX_LEVELS_FILE = MAX_LEVS_FILE_ !< Controls record length for optimal storage
+  integer, parameter, public :: MAX_LEVELS_FILE = MAX_LEVS_FILE_ !< Controls record length for optimal storage  
   integer, parameter, public :: MAX_NEIGHBORS = 100 !< Maximum number of neighbors for QC or analysis for profiles
-  integer, parameter, public :: MAX_LINKS = MAX_LINKS_ !< Maximum number of records per profile for storage for profiles
+  integer, parameter, public :: MAX_LINKS = MAX_LINKS_ !< Maximum number of records per profile for storage for profiles  
 
   ! Additional Pramaeters needed for snz's ECDA
   integer, parameter, public :: DROP_PROFILER = 10
@@ -46,7 +46,7 @@ module oda_types_mod
   integer, save, public :: TEMP_ID = 1
   integer, save, public :: SALT_ID = 2
 
-  ! List of variables for ODA
+  ! List of variables for ODA   
 #ifndef ENABLE_ECDA
   real, parameter, public :: MISSING_VALUE = -1.e20
 #else
@@ -57,7 +57,7 @@ module oda_types_mod
   type, public :: forward_model_type
      real, dimension(:,:,:,:), pointer :: wgt ! interpolation weights
   end type forward_model_type
-
+  
   type, public :: ocean_profile_type
      integer :: variable !< variable ids are defined by the ocean_types module (e.g. TEMP_ID, SALT_ID)
      integer :: inst_type !< instrument types are defined by platform class (e.g. MOORING, DROP, etc.) and instrument type (XBT, CDT, etc.)
@@ -71,31 +71,24 @@ module oda_types_mod
      real    :: database_id
      integer :: levels
      integer :: profile_flag ! an overall flag for the profile
-     integer :: profile_flag_s ! an overall flag for the profile salinity
+     integer :: profile_flag_s ! an overall flag for the profile salinity     
      real :: lat, lon
      logical :: accepted
      integer :: nlinks
      type(ocean_profile_type), pointer, dimension(:) :: next ! Large profiles are stored as linked list.
      integer, dimension(MAX_NEIGHBORS) :: nbr_index
-     real, dimension(MAX_NEIGHBORS) :: nbr_dist ! distance in radians
+     real, dimension(MAX_NEIGHBORS) :: nbr_dist ! distance in radians 
      real, dimension(:), pointer :: depth, data_t, data_s
      real, dimension(:), pointer :: data
      integer, dimension(:), pointer :: flag_t
      integer, dimension(:), pointer :: flag_s ! level-by-level flags for salinity
-#if !(defined(ENABLE_ECDA)||defined(__GFORTRAN__)||defined(__FUJITSU))
-     ! this #if needed due to GNU not doing a logical cast properly
-     ! Will be removed in a later patch to only allow logical flag.
-     integer, dimension(:), pointer :: flag
-#else
-     !::sdu:: For now ECDA use flag as a logical, will likely change in future releases.
      logical, dimension(:), pointer :: flag
-#endif
      real    :: temp_err, salt_err ! measurement error
      real, dimension(:), pointer :: ms_t ! ms temperature by level
-     real, dimension(:), pointer :: ms_s ! ms salinity by level
+     real, dimension(:), pointer :: ms_s ! ms salinity by level  
      real, dimension(:), pointer :: ms_inv
      real, dimension(:), pointer :: ms
-     type(time_type) :: time
+     type(time_type) :: time 
      integer         :: yyyy
      integer         :: mmdd
      type(time_type), pointer :: Model_time ! each profile can be associated with a first-guess field with an associated time and grid
@@ -121,7 +114,7 @@ module oda_types_mod
      real, pointer, dimension(:,:) :: data2 => NULL()
      real, dimension(:,:), pointer :: ms2 => NULL()
      real, dimension(:,:), pointer :: i_index2=>NULL(), j_index2=>NULL() ! model indices
-     real :: k_index
+     real :: k_index          
      type(forward_model_type) :: Forward_model
      type(time_type) :: time
      integer :: yyyy
@@ -129,7 +122,7 @@ module oda_types_mod
      character(len=8) :: wmo_id
      type(time_type), pointer :: Model_time => NULL()
      type(grid_type), pointer :: Model_grid => NULL()
-     ! positive difference between current model time
+     ! positive difference between current model time 
      ! and observation time
      type(time_type) :: tdiff
   end type ocean_surface_type
@@ -143,7 +136,7 @@ module oda_types_mod
      real, pointer, dimension(:,:) :: lw_flux => NULL()
      real, pointer, dimension(:,:) :: sw_flux_vis_dir => NULL()
      real, pointer, dimension(:,:) :: sw_flux_vis_dif => NULL()
-     real, pointer, dimension(:,:) :: sw_flux_nir_dir => NULL()
+     real, pointer, dimension(:,:) :: sw_flux_nir_dir => NULL() 
      real, pointer, dimension(:,:) :: sw_flux_nir_dif => NULL()
   end type da_flux_type
 
@@ -188,7 +181,7 @@ module oda_types_mod
      type(grid_type), pointer :: grid => NULL()
      real, pointer, dimension(:,:) :: ex=>NULL(), vr=>NULL()
   end type field_dist_type_2d
-
+     
   type, public :: ocean_dist_type
      type(field_dist_type_3d) :: temp,salt,u,v
      type(field_dist_type_2d) :: eta
@@ -199,11 +192,11 @@ module oda_types_mod
   end type obs_clim_type
 
   public init_obs
-
+  
   interface init_obs
      module procedure init_obs_profile
   end interface
-
+  
   contains
 
     subroutine init_obs_profile(profile)
@@ -238,7 +231,7 @@ subroutine init_obs_profile(profile)
       profile%salt_err = -1.0
       profile%time = set_time(0,0)
       profile%yyyy = 0
-      profile%mmdd = 0
+      profile%mmdd = 0      
       if (associated(profile%model_time)) deallocate(profile%model_time)
       if (associated(profile%model_grid)) deallocate(profile%model_grid)
       profile%i_index = -1
@@ -247,7 +240,7 @@ subroutine init_obs_profile(profile)
       profile%tdiff = set_time(0,0)
 
       return
-
+      
     end subroutine init_obs_profile
-
+    
 end module oda_types_mod
diff --git a/src/shared/sat_vapor_pres/sat_vapor_pres.F90 b/src/shared/sat_vapor_pres/sat_vapor_pres.F90
index 7e41d7dac0..83accd43ba 100644
--- a/src/shared/sat_vapor_pres/sat_vapor_pres.F90
+++ b/src/shared/sat_vapor_pres/sat_vapor_pres.F90
@@ -71,10 +71,11 @@ module sat_vapor_pres_mod
 !
 !-----------------------------------------------------------------------
 
-! 
+! 
 !   Bruce Wyman
 ! 
 
+! 
 
 ! 
 !   Routines for determining the saturation vapor pressure 
@@ -491,8 +492,8 @@ module sat_vapor_pres_mod
 !-----------------------------------------------------------------------
 !  cvs version and tag name
 
- character(len=128) :: version = '$Id: sat_vapor_pres.F90,v 20.0 2013/12/14 00:27:58 fms Exp $'
- character(len=128) :: tagname = '$Name: tikal $'
+ character(len=128) :: version = '$Id$'
+ character(len=128) :: tagname = '$Name$'
 
  logical :: module_is_initialized = .false.
 
@@ -523,7 +524,7 @@ module sat_vapor_pres_mod
  logical :: construct_table_wrt_liq = .false.
  logical :: construct_table_wrt_liq_and_ice = .false.
 
- namelist /sat_vapor_pres_nml/ show_bad_value_count_by_slice, show_all_bad_values, &
+ namelist / sat_vapor_pres_nml / show_bad_value_count_by_slice, show_all_bad_values, &
                                  use_exact_qs, do_simple, &
                                  construct_table_wrt_liq, &
                                  construct_table_wrt_liq_and_ice
@@ -2220,7 +2221,7 @@ subroutine sat_vapor_pres_init(err_msg)
 #endif
 
 ! write version number and namelist to log file
-  call write_version_number()
+  call write_version_number (version, tagname)
   unit = stdlog()
   if (mpp_pe() == mpp_root_pe()) write (unit, nml=sat_vapor_pres_nml)
 
diff --git a/src/shared/sat_vapor_pres/sat_vapor_pres_k.F90 b/src/shared/sat_vapor_pres/sat_vapor_pres_k.F90
index 7c6dab864a..b7daf62812 100644
--- a/src/shared/sat_vapor_pres/sat_vapor_pres_k.F90
+++ b/src/shared/sat_vapor_pres/sat_vapor_pres_k.F90
@@ -28,8 +28,8 @@ module sat_vapor_pres_k_mod
  implicit none
  private
 
- character(len=128), parameter :: version = '$Id: sat_vapor_pres_k.F90,v 18.0 2010/03/02 23:58:26 fms Exp $'
- character(len=128), parameter :: tagname = '$Name: tikal $'
+ character(len=128), parameter :: version = '$Id$'
+ character(len=128), parameter :: tagname = '$Name$'
 
  public :: sat_vapor_pres_init_k
  public :: lookup_es_k
@@ -191,7 +191,7 @@ subroutine sat_vapor_pres_init_k(table_size, tcmin, tcmax, TFREEZE, HLV, RVGAS,
    endif
 
       table_siz = table_size
-      dtres = (tcmax - tcmin)/(table_size-1)
+      dtres = (tcmax - tcmin)/real(table_size-1)
       tminl = real(tcmin)+TFREEZE  ! minimum valid temp in table
       dtinvl = 1./dtres
       tepsl = .5*dtres
@@ -203,7 +203,7 @@ subroutine sat_vapor_pres_init_k(table_size, tcmin, tcmax, TFREEZE, HLV, RVGAS,
 ! To be able to compute tables for any temperature range and resolution,
 ! and at the same time exactly reproduce answers from memphis revision,
 ! it is necessary to compute ftact differently than it is in memphis.
-      tfact = 5*dtinvl
+      tfact = 5.0*dtinvl
 
       hdtinv = dtinvl*0.5
 
@@ -337,9 +337,9 @@ function compute_es_k(tem, TFREEZE) result (es)
 !  see smithsonian meteorological tables page 350.
 
      if (tem(i) > -20.+TBASI) then
-         x = -7.90298*(TBASW/tem(i)-1) + 5.02808*log10(TBASW/tem(i)) &
-             -1.3816e-07*(10**((1-tem(i)/TBASW)*11.344)-1)        &
-             +8.1328e-03*(10**((TBASW/tem(i)-1)*(-3.49149))-1)    &
+         x = -7.90298*(TBASW/tem(i)-1.0) + 5.02808*log10(TBASW/tem(i)) &
+             -1.3816e-07*(10.0**((1.0-tem(i)/TBASW)*11.344)-1.0)        &
+             +8.1328e-03*(10.0**((TBASW/tem(i)-1.0)*(-3.49149))-1.0)    &
              +log10(ESBASW)
          esh2o = 10.**(x)
      else
@@ -379,9 +379,9 @@ function compute_es_liq_k(tem, TFREEZE) result (es)
 !  values over 100 c may not be valid
 !  see smithsonian meteorological tables page 350.
 
-         x = -7.90298*(TBASW/tem(i)-1) + 5.02808*log10(TBASW/tem(i)) &
-             -1.3816e-07*(10**((1-tem(i)/TBASW)*11.344)-1)        &
-             +8.1328e-03*(10**((TBASW/tem(i)-1)*(-3.49149))-1)    &
+         x = -7.90298*(TBASW/tem(i)-1.0) + 5.02808*log10(TBASW/tem(i)) &
+             -1.3816e-07*(10.0**((1.0-tem(i)/TBASW)*11.344)-1.0)        &
+             +8.1328e-03*(10.0**((TBASW/tem(i)-1.0)*(-3.49149))-1.0)    &
              +log10(ESBASW)
          esh2o = 10.**(x)
 
@@ -421,9 +421,9 @@ function compute_es_liq_ice_k(tem, TFREEZE) result (es)
 !  values over 100 c may not be valid
 !  see smithsonian meteorological tables page 350.
 
-         x = -7.90298*(TBASW/tem(i)-1) + 5.02808*log10(TBASW/tem(i)) &
-             -1.3816e-07*(10**((1-tem(i)/TBASW)*11.344)-1)        &
-             +8.1328e-03*(10**((TBASW/tem(i)-1)*(-3.49149))-1)    &
+         x = -7.90298*(TBASW/tem(i)-1.0) + 5.02808*log10(TBASW/tem(i)) &
+             -1.3816e-07*(10.0**((1.0-tem(i)/TBASW)*11.344)-1.0)        &
+             +8.1328e-03*(10.0**((TBASW/tem(i)-1.0)*(-3.49149))-1.0)    &
              +log10(ESBASW)
          es(i) = 10.**(x)
      endif
diff --git a/src/shared/station_data/station_data.F90 b/src/shared/station_data/station_data.F90
index a21902e6cd..ecb6bfefb2 100644
--- a/src/shared/station_data/station_data.F90
+++ b/src/shared/station_data/station_data.F90
@@ -1,5 +1,5 @@
 module station_data_mod 
-! 
+! 
 !   Giang Nong
 ! 
 ! 
@@ -288,7 +288,7 @@ subroutine station_data_init()
 94  continue
     call close_file(iunit)
     call check_duplicate_output_fields
-    call write_version_number()
+    call write_version_number (version, tagname)
     module_is_initialized = .true.
     return
 99  continue
diff --git a/src/shared/time_interp/time_interp.F90 b/src/shared/time_interp/time_interp.F90
index 4d080bb909..5c0ec0c2f3 100644
--- a/src/shared/time_interp/time_interp.F90
+++ b/src/shared/time_interp/time_interp.F90
@@ -1,10 +1,11 @@
 
 module time_interp_mod
 
-! 
+! 
 !   Bruce Wyman
 ! 
 
+! 
 
 ! 
 !   Computes a weight and dates/indices for linearly interpolating between two dates.
@@ -195,13 +196,13 @@ module time_interp_mod
    integer :: yrmod, momod, dymod
    logical :: mod_leapyear
 
-   character(len=128) :: version='$Id: time_interp.F90,v 20.0 2013/12/14 00:28:05 fms Exp $'
-   character(len=128) :: tagname='$Name: tikal $'
+   character(len=128) :: version='$Id$'
+   character(len=128) :: tagname='$Name$'
 
    logical :: module_is_initialized=.FALSE.
    logical :: perthlike_behavior=.FALSE.
 
-   namelist /time_interp_nml/ perthlike_behavior
+   namelist / time_interp_nml / perthlike_behavior
 
 contains
 
@@ -224,7 +225,7 @@ subroutine time_interp_init()
    20 call close_file (namelist_unit)
 #endif
 
-   call write_version_number()
+   call write_version_number( version, tagname )
    logunit = stdlog()
    write(logunit,time_interp_nml)
 
@@ -910,7 +911,7 @@ program test_time_interp
 
  integer :: nmin, nmax
 
- namelist /test_time_interp_nml/ timelist_len
+ namelist / test_time_interp_nml / timelist_len
 
  call fms_init
  outunit = stdout()
diff --git a/src/shared/time_interp/time_interp_external.F90 b/src/shared/time_interp/time_interp_external.F90
index 6156af3c3e..fe088b91c5 100644
--- a/src/shared/time_interp/time_interp_external.F90
+++ b/src/shared/time_interp/time_interp_external.F90
@@ -2,7 +2,7 @@
 
 module time_interp_external_mod
 !
-!M.J. Harrison
+!M.J. Harrison
 !
 !Harper Simmons
 !
@@ -50,8 +50,8 @@ module time_interp_external_mod
   private
 
   character(len=128), private :: version= &
-   'CVS $Id: time_interp_external.F90,v 20.0 2013/12/14 00:28:08 fms Exp $'
-  character(len=128), private :: tagname='Tag $Name: tikal $'
+   'CVS $Id$'
+  character(len=128), private :: tagname='Tag $Name$'
 
   integer, parameter, public  :: NO_REGION=0, INSIDE_REGION=1, OUTSIDE_REGION=2
   integer, parameter, private :: modulo_year= 0001
diff --git a/src/shared/time_manager/get_cal_time.F90 b/src/shared/time_manager/get_cal_time.F90
index aabe5952df..7ac082fd32 100644
--- a/src/shared/time_manager/get_cal_time.F90
+++ b/src/shared/time_manager/get_cal_time.F90
@@ -1,6 +1,6 @@
 module get_cal_time_mod
 
-!   
+!   
 !     fms
 !   
 !   
@@ -36,11 +36,11 @@ module get_cal_time_mod
 
 logical :: allow_calendar_conversion=.true.
 
-namelist /get_cal_time_nml/ allow_calendar_conversion
+namelist / get_cal_time_nml / allow_calendar_conversion
 ! 
 
-character(len=128) :: version='$Id: get_cal_time.F90,v 20.0 2013/12/14 00:28:11 fms Exp $'
-character(len=128) :: tagname='$Name: tikal $'
+character(len=128) :: version='$Id$'
+character(len=128) :: tagname='$Name$'
 
 contains
 !------------------------------------------------------------------------
@@ -178,7 +178,7 @@ function get_cal_time(time_increment, units, calendar, permit_calendar_conversio
   20 call close_file (namelist_unit)
 #endif
 
-  call write_version_number()
+  call write_version_number (version, tagname)
   logunit = stdlog()
   if(mpp_pe() == mpp_root_pe()) write (logunit, nml=get_cal_time_nml)
   module_is_initialized = .true.
diff --git a/src/shared/time_manager/time_manager.F90 b/src/shared/time_manager/time_manager.F90
index 35d81bbfea..3cf84431aa 100644
--- a/src/shared/time_manager/time_manager.F90
+++ b/src/shared/time_manager/time_manager.F90
@@ -1,9 +1,10 @@
 module time_manager_mod
 
-! 
+! 
 !   fms
 ! 
 
+! 
 
 ! 
 !   A software package that provides a set of simple interfaces for
@@ -179,8 +180,8 @@ module time_manager_mod
 
 !======================================================================
 
-character(len=128) :: version='$Id: time_manager.F90,v 20.0 2013/12/14 00:28:14 fms Exp $'
-character(len=128) :: tagname='$Name: tikal $'
+character(len=128) :: version='$Id$'
+character(len=128) :: tagname='$Name$'
 logical :: module_is_initialized = .false.
 
 !======================================================================
@@ -3089,7 +3090,7 @@ subroutine time_manager_init ( )
 
   if (module_is_initialized) return  ! silent return if already called
 
-  call write_version_number()
+  call write_version_number (version, tagname)
   module_is_initialized = .true.
 
 end subroutine time_manager_init
diff --git a/src/shared/topography/gaussian_topog.F90 b/src/shared/topography/gaussian_topog.F90
index 9f8c0f0bf2..77951e002d 100644
--- a/src/shared/topography/gaussian_topog.F90
+++ b/src/shared/topography/gaussian_topog.F90
@@ -1,10 +1,11 @@
 
 module gaussian_topog_mod
 
-! 
+! 
 !   Bruce Wyman
 ! 
 
+! 
 
 ! 
 !   Routines for creating Gaussian-shaped land surface topography
@@ -72,8 +73,8 @@ module gaussian_topog_mod
 
 !-----------------------------------------------------------------------
 
-character(len=128) :: version = '$Id: gaussian_topog.F90,v 20.0 2013/12/14 00:28:17 fms Exp $'
-character(len=128) :: tagname = '$Name: tikal $'
+character(len=128) :: version = '$Id$'
+character(len=128) :: tagname = '$Name$'
 
 logical :: do_nml = .true.
 logical :: module_is_initialized = .FALSE.
@@ -119,7 +120,7 @@ subroutine gaussian_topog_init ( lon, lat, zsurf )
 integer :: n
 
   if (.not.module_is_initialized) then
-     call write_version_number()
+     call write_version_number( version, tagname )
   endif
 
   if(any(shape(zsurf) /= (/size(lon(:)),size(lat(:))/))) then
diff --git a/src/shared/topography/topography.F90 b/src/shared/topography/topography.F90
index c3f5d9a242..b492f8b1cf 100644
--- a/src/shared/topography/topography.F90
+++ b/src/shared/topography/topography.F90
@@ -1,10 +1,11 @@
 
 module topography_mod
 
-! 
+! 
 !   Bruce Wyman
 ! 
 
+! 
 
 ! 
 !   Routines for creating land surface topography fields and land-water masks
@@ -114,8 +115,8 @@ module topography_mod
 
 !-----------------------------------------------------------------------
 
- character(len=128) :: version = '$Id: topography.F90,v 20.0 2013/12/14 00:28:20 fms Exp $'
- character(len=128) :: tagname = '$Name: tikal $'
+ character(len=128) :: version = '$Id$'
+ character(len=128) :: tagname = '$Name$'
 
  logical :: module_is_initialized = .FALSE.
 
@@ -129,7 +130,7 @@ subroutine topography_init ()
 
      if ( module_is_initialized ) return
 
-     call write_version_number()
+     call write_version_number (version,tagname)
      call read_namelist
      module_is_initialized = .TRUE.
 
diff --git a/src/shared/tracer_manager/tracer_manager.F90 b/src/shared/tracer_manager/tracer_manager.F90
index 8858127f65..a4304fb1a0 100644
--- a/src/shared/tracer_manager/tracer_manager.F90
+++ b/src/shared/tracer_manager/tracer_manager.F90
@@ -1,20 +1,21 @@
 module tracer_manager_mod
-! 
+! 
 !   William Cooke
 ! 
 
-! 
+! 
 !   Matt Harrison
 ! 
 
-! 
+! 
 !   Bruce Wyman
 ! 
 
-! 
+! 
 !   Peter Phillipps
 ! 
 
+! 
 
 ! 
 !   Code to manage the simple addition of tracers to the FMS code.
@@ -141,8 +142,8 @@ module tracer_manager_mod
 type(tracer_type), save  :: tracers(MAX_TRACER_FIELDS)
 type(inst_type)  , save  :: instantiations(MAX_TRACER_FIELDS)
 
-character(len=128) :: version = '$Id: tracer_manager.F90,v 20.0 2013/12/14 00:28:23 fms Exp $'
-character(len=128) :: tagname = '$Name: tikal $'
+character(len=128) :: version = '$Id$'
+character(len=128) :: tagname = '$Name$'
 logical            :: module_is_initialized = .false.
 
 logical            :: verbose_local
@@ -171,7 +172,7 @@ subroutine tracer_manager_init
   if(module_is_initialized) return
   module_is_initialized = .TRUE.
 
-  call write_version_number()
+  call write_version_number (version, tagname)
   call field_manager_init()
   TRACER_ARRAY = NOTRACER
   do model=1,NUM_MODELS 
diff --git a/src/shared/tridiagonal/tridiagonal.F90 b/src/shared/tridiagonal/tridiagonal.F90
index 069ac2cd14..e177044083 100644
--- a/src/shared/tridiagonal/tridiagonal.F90
+++ b/src/shared/tridiagonal/tridiagonal.F90
@@ -1,12 +1,13 @@
 module tridiagonal_mod
 
-! 
+! 
 !    Isaac Held 
 ! 
-! 
+! 
 !    Bruce Wyman
 ! 
 
+! 
 
 ! 
 !   Solves the tridiagonal system of equations.
@@ -120,7 +121,7 @@ subroutine tri_invert(x,d,a,b,c)
   a(:,:,size(x,3)) = 0.0
 
   do  k= 2,size(x,3)
-    g(:,:,k) = 1/(b(:,:,k)+c(:,:,k)*e(:,:,k-1))
+    g(:,:,k) = 1.0/(b(:,:,k)+c(:,:,k)*e(:,:,k-1))
     e(:,:,k) = - a(:,:,k)*g(:,:,k)
   end do
   cc = c