outsourced force monitoring into separate function

Marcogarofalo · Feb 20, 2013 · dcf31f5 · dcf31f5
1 parent 5905f46
commit dcf31f5
Show file tree

Hide file tree

Showing 4 changed files with 139 additions and 54 deletions.
diff --git a/monomial/Makefile.in b/monomial/Makefile.in
@@ -33,7 +33,7 @@ LIBRARIES = libmonomial
 libmonomial_TARGETS =  nddetratio_monomial monomial det_monomial detratio_monomial \
 	gauge_monomial ndpoly_monomial clover_trlog_monomial cloverdet_monomial cloverdetratio_monomial \
 	clovernd_trlog_monomial poly_monomial cloverndpoly_monomial moment_energy \
-	ndrat_monomial ndratcor_monomial rat_monomial ratcor_monomial 
+	ndrat_monomial ndratcor_monomial rat_monomial ratcor_monomial monitor_forces
 
 
 libmonomial_STARGETS = 

diff --git a/monomial/monitor_forces.c b/monomial/monitor_forces.c
@@ -0,0 +1,108 @@
+/***********************************************************************
+ *
+ * Copyright (C) 2001 Martin Hasebusch
+ *               2002,2003,2004,2005,2006,2007,2008,2012 Carsten Urbach
+ *
+ * This file is part of tmLQCD.
+ *
+ * tmLQCD is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ * 
+ * tmLQCD is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ * 
+ * You should have received a copy of the GNU General Public License
+ * along with tmLQCD.  If not, see <http://www.gnu.org/licenses/>.
+ ***********************************************************************/
+
+#ifdef HAVE_CONFIG_H
+# include<config.h>
+#endif
+#include <stdlib.h>
+#include <stdio.h>
+#include <math.h>
+#include <time.h>
+#ifdef MPI
+# include <mpi.h>
+#endif
+#ifdef OMP
+# include <omp.h>
+#endif
+#include "global.h"
+#include "su3.h"
+#include "su3adj.h"
+#include "su3spinor.h"
+#include "monomial/monomial.h"
+#include "xchange/xchange.h"
+#include "hamiltonian_field.h"
+#include "monitor_forces.h"
+#include "gettime.h"
+
+void monitor_forces(hamiltonian_field_t * const hf) {
+
+  for(int id = 0; id < no_monomials; id++) {
+    if(monomial_list[ id ].derivativefunction != NULL) {
+#ifdef OMP
+#pragma omp parallel for
+#endif
+      for(int i = 0; i < (VOLUMEPLUSRAND + g_dbw2rand);i++) { 
+	for(int mu=0;mu<4;mu++) { 
+	  _zero_su3adj(hf->derivative[i][mu]);
+	}
+      }
+
+      monomial_list[ id ].derivativefunction(id, hf);
+
+#ifdef MPI
+      xchange_deri(hf->derivative);
+#endif
+
+      double sum = 0., max = 0., sum2;
+#ifdef OMP
+#pragma omp parallel private(sum2)
+      {
+	int thread_num = omp_get_thread_num();
+	g_omp_acc_re[thread_num] = 0.;
+#pragma omp for reduction(+ : sum) nowait
+#endif
+	for(int i = 0; i < VOLUME; i++) {
+	  for(int mu = 0; mu < 4; mu++) {
+	    sum2 = _su3adj_square_norm(hf->derivative[i][mu]); 
+	    sum += sum2;
+#ifdef OMP
+	    if(sum2 > g_omp_acc_re[thread_num]) g_omp_acc_re[thread_num] = sum2;
+#else
+	    if(sum2 > max) max = sum2;
+#endif
+	  }
+	}
+#ifdef OMP
+      } /* OMP closing brace */
+      max = g_omp_acc_re[0];
+      for( int i = 1; i < omp_num_threads; i++) {
+	if(g_omp_acc_re[i] > max) max = g_omp_acc_re[i];
+      }
+#endif
+
+      // output for force monitoring
+#ifdef MPI
+      MPI_Reduce(&sum, &sum2, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
+      sum = sum2;
+      MPI_Reduce(&max, &sum2, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
+      max = sum2;
+#endif
+      if(g_proc_id == 0) {
+	printf("# squared force for monomial %s on timescale %d: aver: %1.2e max: %1.2e\n", 
+	       monomial_list[ id ].name,
+	       monomial_list[ id ].timescale,
+	       sum/((double)(VOLUME*g_nproc))/4., max);
+	fflush(stdout);
+      }
+    }
+  } 
+  return;
+}
diff --git a/monomial/monitor_forces.h b/monomial/monitor_forces.h
@@ -0,0 +1,26 @@
+/***********************************************************************
+ * Copyright (C) 2013 Carsten Urbach
+ *
+ * This file is part of tmLQCD.
+ *
+ * tmLQCD is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ * 
+ * tmLQCD is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ * 
+ * You should have received a copy of the GNU General Public License
+ * along with tmLQCD.  If not, see <http://www.gnu.org/licenses/>.
+ ***********************************************************************/
+#ifndef _MONITOR_FORCES_H
+#define _MONITOR_FORCES_H
+
+#include "hamiltonian_field.h"
+
+void monitor_forces(hamiltonian_field_t * const hf);
+
+#endif
diff --git a/update_momenta.c b/update_momenta.c
@@ -42,7 +42,6 @@
 void update_momenta(int * mnllist, double step, const int no, 
 		    hamiltonian_field_t * const hf) {
 
-  double sum = 0., max = 0., sum2 = 0., tmax = 0.;
 #ifdef OMP
 #pragma omp parallel for
 #endif
@@ -62,61 +61,13 @@ void update_momenta(int * mnllist, double step, const int no,
   xchange_deri(hf->derivative);
 #endif
 
-  // if we want to monitor the force magnitudes
-  // the logics is a bit more involved for openMP
-  // the debug level might need to be adjusted
-  // if this turns out to be performance critical
-  if(g_debug_level > 0) {
-#ifdef OMP
-#pragma omp parallel shared(max) private(sum2) firstprivate(tmax)
-    {
-      max = 0.;
-#pragma omp for reduction(+ : sum) nowait
-#endif
-      for(int i = 0; i < VOLUME; i++) {
-	for(int mu = 0; mu < 4; mu++) {
-	  /* the minus comes from an extra minus in trace_lambda */
-	  _su3adj_minus_const_times_su3adj(hf->momenta[i][mu], step, hf->derivative[i][mu]); 
-	  sum2 = _su3adj_square_norm(hf->derivative[i][mu]); 
-          sum += sum2;
-          if(fabs(sum2) > tmax) tmax = sum2;
-	}
-      }
-#ifdef OMP
-#pragma omp critical 
-      {
-	if(tmax > max) max = tmax;
-      }
-    }
-#else
-    max = tmax;
-#endif
-  }
-  // no monitoring of force magnitudes
-  else {
 #ifdef OMP
 #pragma omp parallel for
 #endif
-    for(int i = 0; i < VOLUME; i++) {
-      for(int mu = 0; mu < 4; mu++) {
-	/* the minus comes from an extra minus in trace_lambda */
-	_su3adj_minus_const_times_su3adj(hf->momenta[i][mu], step, hf->derivative[i][mu]); 
-      }
-    }
-  }
-  // output for force monitoring
-  if(g_debug_level > 0) {
-#ifdef MPI
-    MPI_Reduce(&sum, &sum2, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
-    sum = sum2;
-    MPI_Reduce(&max, &sum2, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
-    max = sum2;
-#endif
-    if(g_proc_id == 0) {
-      printf("# squared force on timescale %d: aver: %1.2e max: %1.2e dt: %1.4e\n", 
-	     monomial_list[ mnllist[0] ].timescale,
-	     sum/((double)(VOLUME*g_nproc))/4., max, step);
-      fflush(stdout);
+  for(int i = 0; i < VOLUME; i++) {
+    for(int mu = 0; mu < 4; mu++) {
+      /* the minus comes from an extra minus in trace_lambda */
+      _su3adj_minus_const_times_su3adj(hf->momenta[i][mu], step, hf->derivative[i][mu]); 
     }
   }