solved merge conflict in vic_store.c

ci/image.travis

@@ -81,6 +81,10 @@ function vic_script {
     $DRIVER_EXE -v
     $DRIVER_EXE -o
 
+    # Set the number of OpenMP threads to use
+    # https://docs.travis-ci.com/user/languages/c/#OpenMP-projects 
+    export OMP_NUM_THREADS=4
+
     # Run test package
     ./tests/run_tests.py unit examples system \
         --image=${DRIVER_EXE} \

docs/Development/ReleaseNotes.md

@@ -25,16 +25,91 @@ To check which release of VIC you are running:
 
 	Drainage from upper layer to adjacent lower layer is calculated according to Brook & Corey curve (where drainage rate is a function of upper-layer soil moisture). In previous versions, a simple numerical solution is applied which uses the timestep-beginning upper-layer soil moisture to calculate drainage rate, and assume this constant rate over the entire timestep. This can cause unreasonably large drainage if the curve has a steep shape and when soil moisture is high. Now, the current version uses exact integral (instead of numerical solution) for layer drainage calculation.
 
-2. Fixes for the CESM driver ([GH#642](https://github.com/UW-Hydro/VIC/pull/642))
+2. Fixes for the CESM driver
 
-	1. Using correct fill value datatypes in MPI Gather steps 
-	2. Updated state file name time step to be period-ending rather than period-beginning
-	3. Set the state file name to the RASM case ID
-	4. Removed decimal point for missing values for unsigned integers
-	5. Create dummy forcings when initializing the model (so that there is forcing data for the first time step)
-	6. Changed pressure units from kPa to Pa
-	7. Fixed bug that prevented using the correct local domain grid cells in `cesm_put_data.c`
-	8. Changed reference temperature units from Celsius to Kelvin in `cesm_put_data.c`
+	[GH#642](https://github.com/UW-Hydro/VIC/pull/642)
+
+	    1. Using correct fill value datatypes in MPI Gather steps
+	    2. Updated state file name time step to be period-ending rather than period-beginning
+	    3. Set the state file name to the RASM case ID
+	    4. Removed decimal point for missing values for unsigned integers
+	    5. Create dummy forcings when initializing the model (so that there is forcing data for the first time step)
+	    6. Changed pressure units from kPa to Pa
+	    7. Fixed bug that prevented using the correct local domain grid cells in `cesm_put_data.c`
+	    8. Changed reference temperature units from Celsius to Kelvin in `cesm_put_data.c`
+
+	[GH#695](https://github.com/UW-Hydro/VIC/pull/695)
+
+	    1. Fix sign for latent heat fluxes passed from VIC to the coupler
+	    2. Fix sign for longwave radiation passed from VIC to the coupler
+
+	[GH#696](https://github.com/UW-Hydro/VIC/pull/696)
+
+	    Changes names of CESM driver functions `trim` and `advance_time` to `trimstr` and `advance_vic_time`, respectively, to avoid conflicts with WRF functions with the same names when compiling RFR case.
+
+    [GH#702] (https://github.com/UW-Hydro/VIC/pull/702)
+
+        Fixes Julian day for the first timestep in the dmy struct for the CESM driver.
+
+    [GH#710] (https://github.com/UW-Hydro/VIC/pull/710)
+
+	    Refactor the cesm_put_data.c routine in the CESM driver to use values from out_data directly, rather than computing them separately in cesm_put_data.c.
+
+    [GH#716] (https://github.com/UW-Hydro/VIC/pull/716)
+
+        Fixes initialization of coupler fields and calculates temperature and upwelling longwave to pass to WRF during initialization.
+
+    [GH#718] (https://github.com/UW-Hydro/VIC/pull/718)
+
+        Updates the cesm_put_data.c routine in the CESM driver to pass gridcell-averaged albedo to the coupler.
+
+    [GH#726] (https://github.com/UW-Hydro/VIC/pull/726)
+
+        Updates the cesm_put_data.c routine in the CESM driver to include the correct units for evap passed to the coupler.
+
+    [GH#732] (https://github.com/UW-Hydro/VIC/pull/732)
+
+        Updates the cesm_put_data.c routine in the CESM driver to include the correct units for sensible heat flux and updates the rofliq calculation to be correct (previously only OUT_BASEFLOW was being divided by global_param.dt).
+
+    [GH#734] (https://github.com/UW-Hydro/VIC/pull/734)
+
+        Updates the cesm_put_data.c routine in the CESM driver to include the correct signs for turbulent heat fluxes and evaporation. Previously we had switched the signs to agree with the image driver and they should instead be in accordance with the sign conventions for coupled models, which differ from those of land surface models. Also, eliminate populating the `l2x_Sl_ram1` field with aero_resist to agree with the VIC 4 implementation in RASM.
+
+    [GH#739] (https://github.com/UW-Hydro/VIC/pull/739)
+
+        Updates the cesm_put_data.c routine in the CESM driver to include the correct signs for the wind stresses and fixes a bug in calculating friction velocity (previously it was missing a square root).
+
+    [GH#744](https://github.com/UW-Hydro/VIC/pull/744)
+
+        Updates the cesm_interface_c.c routine to include missing timers and the VIC RUN timer in the CESM driver.
+
+    [GH#746](https://github.com/UW-Hydro/VIC/pull/746)
+
+        Updates the cesm_interface_c.c routine in the CESM driver to populate the nrecs, endyear, endmonth and endday fields in the global_param struct to make them available to vic_finalize for timing tables (specifically the secs/day columns).  
+
+3. Speed up NetCDF operations in the image/CESM drivers ([GH#684](https://github.com/UW-Hydro/VIC/pull/684))
+
+    These changes speed up image driver initialization, forcing reads, and history writes by only opening and closing each input netCDF file once.
+
+4. Added two new timers to measure time in I/O operations ([GH#703](https://github.com/UW-Hydro/VIC/pull/703))
+
+    These two timers count the CPU and WALL time spent in ``vic_force`` and ``vic_write``. The accumulated time from these timers is printed out at the end of each simulation in the timing table. See also [GH#442](https://github.com/UW-Hydro/VIC/pull/442).
+
+5. Added gridcell-averaged albedo (STATE_AVG_ALBEDO) as a state file variable ([GH#712](https://github.com/UW-Hydro/VIC/pull/712))
+
+    This is for use in the CESM driver for VIC to pass to WRF, but has been implemented in the core structure of VIC (in vic_run) for consistency with the classic and image drivers. Running VIC from a cold start now also includes calculation of gridcell-averaged albedo.
+
+6. Cleanup of the initialization sections of the ``image`` and ``cesm`` drivers ([GH#701](https://github.com/UW-Hydro/VIC/pull/701))
+
+	Codified behavior in the initialization of the ``image`` and `cesm` drivers that requires the parameter variables `AreaFract`, `Pfactor`, `zone_fract`, and `Cv` must sum exactly to 1.0. If using the `SNOW_BAND` option, the area weighted `elevation` must match the mean grid cell elevation (`elev`). VIC will print *warnings* if any of these criteria are violated.  
+
+7. Added thread parallelization using OPENMP ([GH#712](https://github.com/UW-Hydro/VIC/pull/712))
+
+	The VIC image and CESM drivers now may be optionally compiled with OPENMP to enable shared memory thread parallelization. This option should improve the parallel scaling of these drivers by reducing the number of MPI messages and increasing message size.
+
+#### Bug Fixes:
+
+1.	Renamed "fcov" to "fcan" in image driver to better match variable code name ([GH#673](https://github.com/UW-Hydro/VIC/pull/673))
 
 ------------------------------
 
@@ -50,10 +125,10 @@ To check which release of VIC you are running:
 
 2. Fixed forceskip rounding bug ([GH#639](https://github.com/UW-Hydro/VIC/pull/639))
 
-	After the fix, the `forceskip` variable in the global parameter structure (i.e., the number of timesteps to skip in the forcing data for the simulatin period) is rounded correctly (before the fix, rounding error might cause 1-timestep offset in the simulation results).
+	After the fix, the `forceskip` variable in the global parameter structure (i.e., the number of timesteps to skip in the forcing data for the simulation period) is rounded correctly (before the fix, rounding error might cause 1-timestep offset in the simulation results).
 
 3. Fixed a problem with image restarts when using multiple processors ([GH#638](https://github.com/UW-Hydro/VIC/pull/638))
-	
+
 	After the fix, only the master node is assigned the task of validating state file dimensions and coordinate variables. Multiprocessing was also added to the VIC testing framework.
 
 4. Ensured that the mask variable in the input domain file must be integer type; otherwise an error is raised. ([GH#645](https://github.com/UW-Hydro/VIC/pull/645))
@@ -64,8 +139,8 @@ To check which release of VIC you are running:
 
 6. Fixed a bug related to writing two-dimensional lat/lon variables to a state file ([GH#652](https://github.com/UW-Hydro/VIC/pull/652))
 
-	Before the bug fix, two-dimensional lat/lon variables were not populated correctly and were written as fill values to a state file. Now two-dimensional lat/lon variables are correctly populated and written. 
- 
+	Before the bug fix, two-dimensional lat/lon variables were not populated correctly and were written as fill values to a state file. Now two-dimensional lat/lon variables are correctly populated and written.
+
 7. Fixed a bug related to `dz_node` and `node_depth` variables in image driver output state file ([GH#657](https://github.com/UW-Hydro/VIC/pull/657))
 
 	Before the fix, `dz_node` and `node_depth` in image driver output state file were not spatially distributed, which was wrong. Now these two variables are spatially distributed in the output state file.
@@ -82,7 +157,6 @@ To check which release of VIC you are running:
 
 	Before the fix, there would be an error if the simulation start time is later than the forcing start time that year AND the simulation spans multiple years. Fixed this bug.
 
-
 ------------------------------
 
 ## VIC 5.0.0 [![DOI](https://zenodo.org/badge/doi/10.5281/zenodo.61422.svg)](http://dx.doi.org/10.5281/zenodo.61422)
@@ -260,7 +334,6 @@ This is a major update from VIC 4. The VIC 5.0.0 release aims to have nearly ide
 
 	Fixed a bug where volumetric heat capacity of water should be used in `func_canopy_energy_bal` (previously specific heat capacity was used).
 
-
 ------------------------------
 
 ## VIC 4.2.d [![DOI](https://zenodo.org/badge/doi/10.5281/zenodo.56058.svg)](http://dx.doi.org/10.5281/zenodo.56058)

docs/Documentation/Drivers/Image/Params.md

@@ -2,6 +2,9 @@
 
 The Image Driver uses the [NetCDF](http://www.unidata.ucar.edu/software/netcdf/) file format for its input model parameters. It is possible to convert the VIC ASCII style parameters to this format. We have put together an example ([Tutorial](Ascii_to_NetCDF_params.md) and [Ipython Notebook](https://github.com/UW-Hydro/VIC/blob/develop/samples/notebooks/example_reformat_vic4_parameters_to_vic5image.ipynb)) that provide examples of how to do this conversion. Our example uses the `tonic` [Python](https://www.python.org/) Package.
 
+!!! Note
+	It is the user's responsibility to ensure that parameter files are formatted appropriately. Notably, the variables `AreaFract`, `Pfactor`, `zone_fract`, and `Cv` must sum exactly to 1.0. If using the `SNOW_BAND` option, the area weighted `elevation` must match the mean grid cell elevation (`elev`). VIC will print *** warnings *** if any of these criteria are violated.
+
 # Soil Parameters
 
 The Soil Parameters serve three main purposes:

docs/Documentation/Drivers/Image/RunVIC.md

@@ -19,6 +19,9 @@ The Image Driver has three dependencies:
 
 3.  [netCDF4](http://www.unidata.ucar.edu/software/netcdf/)
 
+!!! Note
+    Compiling the Image Driver may also be done with [OpenMP](http://www.openmp.org/). Nearly all modern C compilers include the [OpenMP standard](http://www.openmp.org/resources/openmp-compilers/) and users will need to ensure that the makefile has the appropriate compiler flag (usually `-fopenmp`). See the discussion below for how to control OpenMP parallelization.
+
 ## Compiling
 In most cases, you will need to edit the `NETCDF_PATH` and `MPI_PATH` variables in the `Makefile`.
 
@@ -50,11 +53,23 @@ At the command prompt, type:
 
 where `global_parameter_filename` = name of the global parameter file corresponding to your project.
 
-To run VIC image driver using multiple processors, type the following instead:
+The VIC image driver can be run using parallel processing with MPI and/or OpenMP.
+
+!!! Note
+    Users are encouraged to consult their system administrator for assistance in configuring the VIC image driver for parallel processing applications.
+
+To run VIC image driver using multiple processors using MPI, type the following instead:
 
         mpiexec -np $n_proc ./vic_image.exe -g global_parameter_filename.txt
 
-where `n_proc` = number of processors to be used
+where `n_proc` = number of processors to be used. *Note that different MPI implementations may use different names for the MPI executable such as: `mpirun`, `mpiexec_mpt`, or `mpiexec.hydra`*.
+
+To run the VIC image driver using multiple processors with OpenMP (threads), set the environment variable `OMP_NUM_THREADS`:
+
+        export OMP_NUM_THREADS=8
+        ./vic_image.exe -g global_parameter_filename.txt
+
+These two parallelization methods may also be combined using a Hybrid OpenMP/MPI approach. However, that configuration is usually machine, compiler, or scheduler dependent.
 
 ## Other Command Line Options
 

tests/run_profiling.py

@@ -111,7 +111,96 @@ def log2_range(m):
 mpiexec_mpt -np ${BC_MPI_TASKS_ALLOC} $vic_exe -g $vic_global
 END=$(date +%s)
 DIFF=$(echo "$END - $START" | bc)
-printf "%5s | %f\n" ${BC_MPI_TASKS_ALLOC} $DIFF >> $timing_table_file''')}
+printf "%5s | %f\n" ${BC_MPI_TASKS_ALLOC} $DIFF >> $timing_table_file'''),
+    'thunder': host_config(profile=[dict(select=1, mpiprocs=36),
+                                    dict(select=2, mpiprocs=36),
+                                    dict(select=3, mpiprocs=36),
+                                    dict(select=4, mpiprocs=36),
+                                    dict(select=5, mpiprocs=36),
+                                    dict(select=6, mpiprocs=36),
+                                    dict(select=8, mpiprocs=36),
+                                    dict(select=10, mpiprocs=36),
+                                    dict(select=12, mpiprocs=36)],
+                           submit='qsub', mpiexec='mpiexec_mpt',
+                           template='''#!/bin/bash
+#!/bin/bash
+#PBS -N VIC$i
+#PBS -q standard
+#PBS -A NPSCA07935242
+#PBS -l application=VIC
+#PBS -l select=$select:ncpus=36:mpiprocs=$mpiprocs
+#PBS -l walltime=35:00:00
+#PBS -j oe
+
+# Qsub template for AFRL THUNDER
+# Scheduler: PBS
+
+module load netcdf-fortran/intel/4.4.2
+
+START=$(date +%s)
+mpiexec_mpt -np ${BC_MPI_TASKS_ALLOC} $vic_exe -g $vic_global
+END=$(date +%s)
+DIFF=$(echo "$END - $START" | bc)
+printf "%5s | %f\n" ${BC_MPI_TASKS_ALLOC} $DIFF >> $timing_table_file'''),
+    'gordon': host_config(profile=[dict(select=1, mpiprocs=32),
+                                   dict(select=2, mpiprocs=32),
+                                   dict(select=3, mpiprocs=32),
+                                   dict(select=4, mpiprocs=32),
+                                   dict(select=5, mpiprocs=32),
+                                   dict(select=6, mpiprocs=32),
+                                   dict(select=8, mpiprocs=32),
+                                   dict(select=10, mpiprocs=32),
+                                   dict(select=12, mpiprocs=32)],
+                          submit='qsub', mpiexec='aprun',
+                          template='''#!/bin/bash
+#PBS -N VIC$i
+#PBS -q frontier
+#PBS -A NPSCA07935YF5
+#PBS -l application=VIC
+#PBS -l select=$select:ncpus=32:mpiprocs=$mpiprocs
+#PBS -l walltime=35:00:00
+#PBS -j oe
+
+# Qsub template for DSRC GORDON
+# Scheduler: PBS
+
+module load cray-netcdf/4.3.2
+
+START=$(date +%s)
+aprun -n ${BC_MPI_TASKS_ALLOC} $vic_exe -g $vic_global
+END=$(date +%s)
+DIFF=$(echo "$END - $START" | bc)
+printf "%5s | %f\n" ${BC_MPI_TASKS_ALLOC} $DIFF >> $timing_table_file'''),
+    'cheyenne': host_config(profile=[dict(select=1, mpiprocs=36),
+                                     dict(select=2, mpiprocs=36),
+                                     dict(select=3, mpiprocs=36),
+                                     dict(select=4, mpiprocs=36),
+                                     dict(select=5, mpiprocs=36),
+                                     dict(select=6, mpiprocs=36),
+                                     dict(select=8, mpiprocs=36),
+                                     dict(select=10, mpiprocs=36),
+                                     dict(select=12, mpiprocs=36)],
+                            submit='qsub', mpiexec='mpiexec_mpt',
+                            template='''#!/bin/bash
+#!/bin/bash
+#PBS -N VIC$i
+#PBS -q regular
+#PBS -A P48500028
+#PBS -l select=$select:ncpus=36:mpiprocs=$mpiprocs
+#PBS -l walltime=12:00:00
+#PBS -j oe
+#PBS -m abe
+
+# Qsub template for UCAR CHEYENNE
+# Scheduler: PBS
+
+START=$(date +%s)
+
+$mpiexec $vic_exe -g $vic_global
+END=$(date +%s)
+DIFF=$(echo "$END - $START" | bc)
+printf "%5s | %f\n" ${BC_MPI_TASKS_ALLOC} $DIFF >> $timing_table_file''')
+}
 
 OUT_WIDTH = 100
 
@@ -217,7 +306,8 @@ def run_scaling(args):
 
             run_string = template.safe_substitute(
                 vic_exe=args.vic_exe, vic_global=args.global_param,
-                timing_table_file=args.timing, i=i, **kwargs)
+                timing_table_file=args.timing, i=i,
+                mpiexec=config.mpiexec, **kwargs)
             run_file = 'vic_{host}_{i}.sh'.format(host=args.host, i=i)
             with open(run_file, 'w') as f:
                 f.write(run_string)

tests/test_image_driver.py

@@ -291,4 +291,5 @@ def check_mpi_states(state_basedir, list_n_proc):
         for var in ds_first_run.data_vars:
             npt.assert_array_equal(ds_current_run[var].values,
                                    ds_first_run[var].values,
-                                   err_msg='States are not an exact match')
+                                   err_msg='States are not an exact match '
+                                   'for variable: {}'.format(var))

tests/test_restart.py

@@ -260,7 +260,7 @@ def check_exact_restart_fluxes(result_basedir, driver, run_periods):
             for var in ds_full_run.data_vars:
                 np.testing.assert_array_equal(
                     ds[var].values, ds_full_run_split_period[var].values,
-                    err_msg='Fluxes are not an exact match')
+                    err_msg='Fluxes are not an exact match for %s' % var)
 
 
 def check_exact_restart_states(state_basedir, driver, run_periods,
@@ -365,7 +365,7 @@ def check_exact_restart_states(state_basedir, driver, run_periods,
             np.testing.assert_array_equal(ds_states[var].values,
                                           ds_states_full_run[var].values,
                                           err_msg='states are not an '
-                                                  'exact match')
+                                                  'exact match for %s' % var)
 
 
 def read_ascii_state(state_fname):

vic/drivers/cesm/Makefile

@@ -95,6 +95,7 @@ LIBRARY = -lm -L${NETCDFPATH}/lib -lnetcdf
 
 # Set compiler flags
 CFLAGS  =  ${INCLUDES} -ggdb -O0 -Wall -Wextra -fPIC \
+					 -fopenmp \
 					 -DLOG_LVL=$(LOG_LVL) \
 					 -DGIT_VERSION=\"$(GIT_VERSION)\" \
 					 -DUSERNAME=\"$(USER)\" \

vic/drivers/cesm/cpl_mct/lnd_comp_mct.F90

@@ -415,7 +415,7 @@ SUBROUTINE lnd_final_mct(EClock, cdata, x2l, l2x, cdata_s, x2s, s2x)
     CHARACTER(len=*), PARAMETER     :: subname = '(lnd_final_mct)'
 
     !--- clean up
-    errno = vic_cesm_final()
+    errno = vic_cesm_final(vclock)
 
     IF (errno /= 0) THEN
        CALL shr_sys_abort(subname//' ERROR: vic_cesm_final returned a errno /= 0')