Merge pull request UW-Hydro#735 from jhamman/feature/openmp_threading

Feature/openmp threading

Author: bartnijssen

ci/image.travis

@@ -81,6 +81,10 @@ function vic_script {
     $DRIVER_EXE -v
     $DRIVER_EXE -o
 
+    # Set the number of OpenMP threads to use
+    # https://docs.travis-ci.com/user/languages/c/#OpenMP-projects 
+    export OMP_NUM_THREADS=4
+
     # Run test package
     ./tests/run_tests.py unit examples system \
         --image=${DRIVER_EXE} \

docs/Development/ReleaseNotes.md

@@ -95,6 +95,10 @@ To check which release of VIC you are running:
 
 	Codified behavior in the initialization of the ``image`` and `cesm` drivers that requires the parameter variables `AreaFract`, `Pfactor`, `zone_fract`, and `Cv` must sum exactly to 1.0. If using the `SNOW_BAND` option, the area weighted `elevation` must match the mean grid cell elevation (`elev`). VIC will print *warnings* if any of these criteria are violated.  
 
+7. Added thread parallelization using OPENMP ([GH#712](https://github.com/UW-Hydro/VIC/pull/712))
+
+	The VIC image and CESM drivers now may be optionally compiled with OPENMP to enable shared memory thread parallelization. This option should improve the parallel scaling of these drivers by reducing the number of MPI messages and increasing message size.
+
 ## VIC 5.0.1
 
 **Release date: (February 1, 2017)**

docs/Documentation/Drivers/Image/RunVIC.md

@@ -19,6 +19,9 @@ The Image Driver has three dependencies:
 
 3.  [netCDF4](http://www.unidata.ucar.edu/software/netcdf/)
 
+!!! Note
+    Compiling the Image Driver may also be done with [OpenMP](http://www.openmp.org/). Nearly all modern C compilers include the [OpenMP standard](http://www.openmp.org/resources/openmp-compilers/) and users will need to ensure that the makefile has the appropriate compiler flag (usually `-fopenmp`). See the discussion below for how to control OpenMP parallelization.
+
 ## Compiling
 In most cases, you will need to edit the `NETCDF_PATH` and `MPI_PATH` variables in the `Makefile`.
 
@@ -48,11 +51,23 @@ At the command prompt, type:
 
 where `global_parameter_filename` = name of the global parameter file corresponding to your project.
 
-To run VIC image driver using multiple processors, type the following instead:
+The VIC image driver can be run using parallel processing with MPI and/or OpenMP.
+
+!!! Note
+    Users are encouraged to consult their system administrator for assistance in configuring the VIC image driver for parallel processing applications.
+
+To run VIC image driver using multiple processors using MPI, type the following instead:
 
         mpiexec -np $n_proc ./vic_image.exe -g global_parameter_filename.txt
 
-where `n_proc` = number of processors to be used
+where `n_proc` = number of processors to be used. *Note that different MPI implementations may use different names for the MPI executable such as: `mpirun`, `mpiexec_mpt`, or `mpiexec.hydra`*.
+
+To run the VIC image driver using multiple processors with OpenMP (threads), set the environment variable `OMP_NUM_THREADS`:
+
+        export OMP_NUM_THREADS=8
+        ./vic_image.exe -g global_parameter_filename.txt
+
+These two parallelization methods may also be combined using a Hybrid OpenMP/MPI approach. However, that configuration is usually machine, compiler, or scheduler dependent.
 
 ## Other Command Line Options
 

tests/test_restart.py

@@ -260,7 +260,7 @@ def check_exact_restart_fluxes(result_basedir, driver, run_periods):
             for var in ds_full_run.data_vars:
                 np.testing.assert_array_equal(
                     ds[var].values, ds_full_run_split_period[var].values,
-                    err_msg='Fluxes are not an exact match')
+                    err_msg='Fluxes are not an exact match for %s' % var)
 
 
 def check_exact_restart_states(state_basedir, driver, run_periods,
@@ -365,7 +365,7 @@ def check_exact_restart_states(state_basedir, driver, run_periods,
             np.testing.assert_array_equal(ds_states[var].values,
                                           ds_states_full_run[var].values,
                                           err_msg='states are not an '
-                                                  'exact match')
+                                                  'exact match for %s' % var)
 
 
 def read_ascii_state(state_fname):

vic/drivers/cesm/Makefile

@@ -88,6 +88,7 @@ LIBRARY = -lm -L${NETCDFPATH}/lib -lnetcdf
 
 # Set compiler flags
 CFLAGS  =  ${INCLUDES} -ggdb -O0 -Wall -Wextra -fPIC \
+					 -fopenmp \
 					 -DLOG_LVL=$(LOG_LVL) \
 					 -DGIT_VERSION=\"$(GIT_VERSION)\" \
 					 -DUSERNAME=\"$(USER)\" \

vic/drivers/image/Makefile

@@ -74,7 +74,8 @@ INCLUDES = -I ${DRIVERPATH}/include \
 		   -I ${SHAREDIMAGEPATH}/include
 
 # Uncomment to include debugging information
-CFLAGS  =  ${INCLUDES} ${NC_CFLAGS}  -ggdb -O0 -Wall -Wextra -std=c99 \
+CFLAGS  =  ${INCLUDES} ${NC_CFLAGS} -ggdb -O0 -Wall -Wextra -std=c99 \
+					 -fopenmp \
 					 -DLOG_LVL=$(LOG_LVL) \
 					 -DGIT_VERSION=\"$(GIT_VERSION)\" \
 					 -DUSERNAME=\"$(USER)\" \

vic/drivers/shared_image/include/vic_mpi.h

@@ -29,6 +29,11 @@
 
 #include <vic_def.h>
 #include <mpi.h>
+#ifdef _OPENMP
+    #include <omp.h>
+#else
+    #define omp_get_max_threads() 1
+#endif
 
 #define VIC_MPI_ROOT 0
 

vic/drivers/shared_image/src/vic_image_run.c

@@ -55,6 +55,8 @@ vic_image_run(dmy_struct *dmy_current)
     sprint_dmy(dmy_str, dmy_current);
     debug("Running timestep %zu: %s", current, dmy_str);
 
+    // If running with OpenMP, run this for loop using multiple threads
+    #pragma omp parallel for default(shared) private(i, timer, vic_run_ref_str)
     for (i = 0; i < local_domain.ncells_active; i++) {
         // Set global reference string (for debugging inside vic_run)
         sprintf(vic_run_ref_str, "Gridcell io_idx: %zu, timestep info: %s",

vic/drivers/shared_image/src/vic_image_timing.c

@@ -46,6 +46,8 @@ write_vic_timing_table(timer_struct *timers,
     struct passwd             *pw;
     double                     ndays;
     double                     nyears;
+    int                        nprocs;
+    int                        nthreads;
 
     // datestr
     curr_date_time = time(NULL);
@@ -70,6 +72,10 @@ write_vic_timing_table(timer_struct *timers,
         strcpy(user, "unknown");
     }
 
+    // mpi/openmp
+    nthreads = omp_get_max_threads();
+    nprocs = mpi_size * nthreads;
+
     // calculate run length
     ndays = global_param.dt * global_param.nrecs / SEC_PER_DAY;
     nyears = ndays / DAYS_PER_YEAR;
@@ -107,7 +113,9 @@ write_vic_timing_table(timer_struct *timers,
             global_param.atmos_dt);
     fprintf(LOG_DEST, "\n");
 
-    fprintf(LOG_DEST, "  Total pes active          : %d\n", mpi_size);
+    fprintf(LOG_DEST, "  MPI Processes             : %d\n", mpi_size);
+    fprintf(LOG_DEST, "  OPENMP Threads            : %d\n", nthreads);
+    fprintf(LOG_DEST, "  Total pes active          : %d\n", nprocs);
     fprintf(LOG_DEST, "  pes per node              : %ld\n",
             sysconf(_SC_NPROCESSORS_ONLN));
 
@@ -116,7 +124,7 @@ write_vic_timing_table(timer_struct *timers,
     fprintf(LOG_DEST, "  Overall Metrics\n");
     fprintf(LOG_DEST, "  ---------------\n");
     fprintf(LOG_DEST, "    Model Cost       : %g pe-hrs/simulated_year\n",
-            mpi_size * timers[TIMER_VIC_ALL].delta_wall / SEC_PER_HOUR /
+            nprocs * timers[TIMER_VIC_ALL].delta_wall / SEC_PER_HOUR /
             nyears);
     fprintf(LOG_DEST, "    Model Throughput : %g simulated_years/day\n",
             nyears / (timers[TIMER_VIC_ALL].delta_wall / SEC_PER_DAY));

vic/vic_run/src/CalcBlowingSnow.c

@@ -271,18 +271,18 @@ CalcBlowingSnow(double   Dt,
  *****************************************************************************/
 double
 qromb(double (*funcd)(),
-      double es,
-      double Wind,
-      double AirDens,
-      double ZO,
-      double EactAir,
-      double F,
-      double hsalt,
-      double phi_r,
-      double ushear,
-      double Zrh,
-      double a,
-      double b)
+      double   es,
+      double   Wind,
+      double   AirDens,
+      double   ZO,
+      double   EactAir,
+      double   F,
+      double   hsalt,
+      double   phi_r,
+      double   ushear,
+      double   Zrh,
+      double   a,
+      double   b)
 {
     extern parameters_struct param;
 
@@ -363,23 +363,27 @@ polint(double  xa[],
  *****************************************************************************/
 double
 trapzd(double (*funcd)(),
-       double es,
-       double Wind,
-       double AirDens,
-       double ZO,
-       double EactAir,
-       double F,
-       double hsalt,
-       double phi_r,
-       double ushear,
-       double Zrh,
-       double a,
-       double b,
-       int    n)
+       double   es,
+       double   Wind,
+       double   AirDens,
+       double   ZO,
+       double   EactAir,
+       double   F,
+       double   hsalt,
+       double   phi_r,
+       double   ushear,
+       double   Zrh,
+       double   a,
+       double   b,
+       int      n)
 {
-    double        x, tnm, sum, del;
+    double x, tnm, sum, del;
+    int    it, j;
+
+    // TODO: remove use of static variables (see GH #735), for now:
+    // make static variables thread safe
     static double s;
-    int           it, j;
+    #pragma omp threadprivate(s)
 
     if (n == 1) {
         return (s = 0.5 *

vic/vic_run/src/frozen_soil.c

@@ -170,16 +170,18 @@ solve_T_profile(double   *T,
                 int       NOFLUX,
                 int       EXP_TRANS)
 {
+    double *aa, *bb, *cc, *dd, *ee, Bexp;
+    int     Error;
+    int     j;
+
+    // TODO: remove use of static variables (see GH #735), for now:
+    // make static variables thread safe
     static double A[MAX_NODES];
     static double B[MAX_NODES];
     static double C[MAX_NODES];
     static double D[MAX_NODES];
     static double E[MAX_NODES];
-
-    double       *aa, *bb, *cc, *dd, *ee, Bexp;
-
-    int           Error;
-    int           j;
+    #pragma omp threadprivate(A, B, C, D, E)
 
     if (FIRST_SOLN[0]) {
         if (EXP_TRANS) {
@@ -646,6 +648,18 @@ fda_heat_eqn(double T_2[],
              int    init,
              ...)
 {
+    char    PAST_BOTTOM;
+    double  storage_term, flux_term, phase_term, flux_term1, flux_term2;
+    double  Lsum;
+    int     i;
+    size_t  lidx;
+    int     focus, left, right;
+
+    // argument list handling
+    va_list arg_addr;
+
+    // TODO: remove use of static variables (see GH #735), for now:
+    // make static variables thread safe
     static double  deltat;
     static int     NOFLUX;
     static int     EXP_TRANS;
@@ -681,15 +695,12 @@ fda_heat_eqn(double T_2[],
     static double DT[MAX_NODES], DT_down[MAX_NODES], DT_up[MAX_NODES];
     static double Dkappa[MAX_NODES];
     static double Bexp;
-    char          PAST_BOTTOM;
-    double        storage_term, flux_term, phase_term, flux_term1, flux_term2;
-    double        Lsum;
-    int           i;
-    size_t        lidx;
-    int           focus, left, right;
 
-    // argument list handling
-    va_list       arg_addr;
+    #pragma omp threadprivate(deltat, NOFLUX, EXP_TRANS, T0, moist, ice, \
+    kappa, Cs, max_moist, bubble, expt, alpha, beta, gamma, Zsum, Dp, \
+    bulk_dens_min, soil_dens_min, quartz, bulk_density, soil_density, organic, \
+    depth, Nlayers, Ts, Tb, ice_new, Cs_new, kappa_new, DT, DT_down, DT_up, \
+    Dkappa, Bexp)
 
     // initialize variables if init==1
     if (init == 1) {