Cleanup/gf format fixes

include/enum_quda.h

@@ -549,6 +549,12 @@ typedef enum QudaWFlowType_s {
   QUDA_WFLOW_TYPE_INVALID = QUDA_INVALID_ENUM
 } QudaWFlowType;
 
+typedef enum QudaGaugeFixType_s {
+  QUDA_GAUGEFIX_TYPE_OVR = 0,
+  QUDA_GAUGEFIX_TYPE_FFT = 1,
+  QUDA_GAUGEFIX_TYPE_INVALID = QUDA_INVALID_ENUM
+} QudaGaugeFixType;
+
 // Allows to choose an appropriate external library
 typedef enum QudaExtLibType_s {
   QUDA_CUSOLVE_EXTLIB,

include/enum_quda_fortran.h

@@ -492,6 +492,16 @@
 #define QUDA_CONTRACT_GAMMA_S34 15
 #define QUDA_CONTRACT_GAMMA_INVALID QUDA_INVALID_ENUM
 
+#define QudaWFlowType integer(4)
+#define QUDA_WFLOW_TYPE_WILSON 0
+#define QUDA_WFLOW_TYPE_SYMANZIK 1
+#define QUDA_WFLOW_TYPE_INVALID QUDA_INVALID_ENUM
+
+#define QudaGaugeFixType integer(4)
+#define QUDA_GAUGEFIX_TYPE_OVR 0
+#define QUDA_GAUGEFIX_TYPE_FFT 1
+#define QUDA_GAUGEFIX_TYPE_INVALID QUDA_INVALID_ENUM
+
 #define QudaExtLibType integer(4)
 #define QUDA_CUSOLVE_EXTLIB 0
 #define QUDA_EIGEN_EXTLIB 1

include/gauge_tools.h

@@ -113,34 +113,16 @@ namespace quda
   /**
    * @brief Gauge fixing with overrelaxation with support for single and multi GPU.
    * @param[in,out] data, quda gauge field
-   * @param[in] gauge_dir, 3 for Coulomb gauge fixing, other for Landau gauge fixing
-   * @param[in] Nsteps, maximum number of steps to perform gauge fixing
-   * @param[in] verbose_interval, print gauge fixing info when iteration count is a multiple of this
-   * @param[in] relax_boost, gauge fixing parameter of the overrelaxation method, most common value is 1.5 or 1.7.
-   * @param[in] tolerance, torelance value to stop the method, if this
-   * value is zero then the method stops when iteration reachs the
-   * maximum number of steps defined by Nsteps
-   * @param[in] reunit_interval, reunitarize gauge field when iteration count is a multiple of this
-   * @param[in] stopWtheta, 0 for MILC criterion and 1 to use the theta value
+   * @param[in] fix_param Parameter struct that defines the gauge fixing
    */
-  void gaugeFixingOVR(GaugeField &data, const int gauge_dir, const int Nsteps, const int verbose_interval,
-                      const double relax_boost, const double tolerance, const int reunit_interval, const int stopWtheta);
+  void gaugeFixingOVR(GaugeField &data, QudaGaugeFixParam &fix_param);
 
   /**
    * @brief Gauge fixing with Steepest descent method with FFTs with support for single GPU only.
    * @param[in,out] data, quda gauge field
-   * @param[in] gauge_dir, 3 for Coulomb gauge fixing, other for Landau gauge fixing
-   * @param[in] Nsteps, maximum number of steps to perform gauge fixing
-   * @param[in] verbose_interval, print gauge fixing info when iteration count is a multiple of this
-   * @param[in] alpha, gauge fixing parameter of the method, most common value is 0.08
-   * @param[in] autotune, 1 to autotune the method, i.e., if the Fg inverts its tendency we decrease the alpha value
-   * @param[in] tolerance, torelance value to stop the method, if this
-   * value is zero then the method stops when iteration reachs the
-   * maximum number of steps defined by Nsteps
-   * @param[in] stopWtheta, 0 for MILC criterion and 1 to use the theta value
+   * @param[in] fix_param Parameter struct that defines the gauge fixing
    */
-  void gaugeFixingFFT(GaugeField &data, const int gauge_dir, const int Nsteps, const int verbose_interval,
-                      const double alpha, const int autotune, const double tolerance, const int stopWtheta);
+  void gaugeFixingFFT(GaugeField &data, QudaGaugeFixParam &fix_param);
 
   /**
      @brief Compute the Fmunu tensor

include/quda.h

@@ -778,6 +778,24 @@ extern "C" {
     QudaBLASDataOrder data_order; /**< Specifies if using Row or Column major */
   } QudaBLASParam;
 
+  typedef struct QudaGaugeFixParam_s {
+    size_t struct_size; /**< Size of this struct in bytes.  Used to ensure that the host application and QUDA see the same struct size */
+
+    QudaGaugeFixType fix_type;   /**< The aglorithm to use for gauge fixing */
+    int gauge_dir;               /**< The orthogonal direction of the gauge fixing, 3=Coulomb, 4=Landau. (default 4) */
+    int maxiter;                 /**< The maximum number of gauge fixing iterations to be applied (default 10000) */
+    int verbosity_interval;      /**< Print the gauge fixing progress every N steps (default 100) */
+    double ovr_relaxation_boost; /**< The overrelaxation boost parameter for the overrelaxation method (default 1.5) */
+    double fft_alpha;            /**< The Alpha parameter in the FFT method (default 0.8) */
+    QudaBoolean fft_autotune;    /**< Autotune the Alpha parameter in the FFT method (default true) */
+    int reunit_interval;         /**< Reunitarise the gauge field every N steps (default 10) */
+    double tolerance;            /**< The tolerance of the gauge fixing quality (default 1e-6) */
+    QudaBoolean theta_condition; /**< "Use the theta value to determine the gauge fixing if true. If false, use the
+                                    delta value (default false)" */
+    QudaPrecision precision;     /**< The precision used by the algorithm */
+
+  } QudaGaugeFixParam;
+
   /*
    * Interface functions, found in interface_quda.cpp
    */
@@ -954,6 +972,15 @@ extern "C" {
    */
   QudaBLASParam newQudaBLASParam(void);
 
+  /**
+   * A new QudaGaugeFixParam should always be initialized immediately
+   * after it's defined (and prior to explicitly setting its members)
+   * using this function.  Typical usage is as follows:
+   *
+   *   QudaGaugeFixParam fix_param = newQudaGaugeFixParam();
+   */
+  QudaGaugeFixParam newQudaGaugeFixParam(void);
+
   /**
    * Print the members of QudaGaugeParam.
    * @param param The QudaGaugeParam whose elements we are to print.
@@ -990,6 +1017,12 @@ extern "C" {
    */
   void printQudaBLASParam(QudaBLASParam *param);
 
+  /**
+   * Print the members of QudaGaugeFixParam.
+   * @param param The QudaGaugeFixParam whose elements we are to print.
+   */
+  void printQudaGaugeFixParam(QudaGaugeFixParam *param);
+
   /**
    * Load the gauge field from the host.
    * @param h_gauge Base pointer to host gauge field (regardless of dimensionality)
@@ -1519,41 +1552,14 @@ extern "C" {
                     const int *X);
 
   /**
-   * @brief Gauge fixing with overrelaxation with support for single and multi GPU.
-   * @param[in,out] gauge, gauge field to be fixed
-   * @param[in] gauge_dir, 3 for Coulomb gauge fixing, other for Landau gauge fixing
-   * @param[in] Nsteps, maximum number of steps to perform gauge fixing
-   * @param[in] verbose_interval, print gauge fixing info when iteration count is a multiple of this
-   * @param[in] relax_boost, gauge fixing parameter of the overrelaxation method, most common value is 1.5 or 1.7.
-   * @param[in] tolerance, torelance value to stop the method, if this value is zero then the method stops when
-   * iteration reachs the maximum number of steps defined by Nsteps
-   * @param[in] reunit_interval, reunitarize gauge field when iteration count is a multiple of this
-   * @param[in] stopWtheta, 0 for MILC criterion and 1 to use the theta value
-   * @param[in] param The parameters of the external fields and the computation settings
-   * @param[out] timeinfo
-   */
-  int computeGaugeFixingOVRQuda(void *gauge, const unsigned int gauge_dir, const unsigned int Nsteps,
-                                const unsigned int verbose_interval, const double relax_boost, const double tolerance,
-                                const unsigned int reunit_interval, const unsigned int stopWtheta,
-                                QudaGaugeParam *param, double *timeinfo);
-  /**
-   * @brief Gauge fixing with Steepest descent method with FFTs with support for single GPU only.
+   * @brief Gauge fixing with overrelaxation with support for single and multi GPU, and steepest descent FFT with
+   * support for single GPU only.
    * @param[in,out] gauge, gauge field to be fixed
-   * @param[in] gauge_dir, 3 for Coulomb gauge fixing, other for Landau gauge fixing
-   * @param[in] Nsteps, maximum number of steps to perform gauge fixing
-   * @param[in] verbose_interval, print gauge fixing info when iteration count is a multiple of this
-   * @param[in] alpha, gauge fixing parameter of the method, most common value is 0.08
-   * @param[in] autotune, 1 to autotune the method, i.e., if the Fg inverts its tendency we decrease the alpha value
-   * @param[in] tolerance, torelance value to stop the method, if this value is zero then the method stops when
-   * iteration reachs the maximum number of steps defined by Nsteps
-   * @param[in] stopWtheta, 0 for MILC criterion and 1 to use the theta value
-   * @param[in] param The parameters of the external fields and the computation settings
-   * @param[out] timeinfo
+   * @param[in] gauge_param The parameters of the external fields and the computation settings
+   * @param[in] fix_param Container for the gauge fixing algorithm and parameters to use.
+   * @param[out] timeinfo Array to track timings
    */
-  int computeGaugeFixingFFTQuda(void *gauge, const unsigned int gauge_dir, const unsigned int Nsteps,
-                                const unsigned int verbose_interval, const double alpha, const unsigned int autotune,
-                                const double tolerance, const unsigned int stopWtheta, QudaGaugeParam *param,
-                                double *timeinfo);
+  int computeGaugeFixingQuda(void *gauge, QudaGaugeParam *gauge_param, QudaGaugeFixParam *fix_param, double *timeinfo);
 
   /**
    * @brief Strided Batched GEMM

lib/check_params.h

@@ -268,7 +268,7 @@ void printQudaCloverParam(QudaInvertParam *param)
 
 #if defined CHECK_PARAM
   if (param->struct_size != (size_t)INVALID_INT && param->struct_size != sizeof(*param))
-    errorQuda("Unexpected QudaInvertParam struct size %lu, expected %lu", param->struct_size, sizeof(*param));
+    errorQuda("Unexpected QudaCloverParam struct size %lu, expected %lu", param->struct_size, sizeof(*param));
 #else
   P(struct_size, (size_t)INVALID_INT);
 #endif
@@ -1047,8 +1047,45 @@ void printQudaBLASParam(QudaBLASParam *param)
 #endif
 }
 
-// clean up
+#if defined INIT_PARAM
+QudaGaugeFixParam newQudaGaugeFixParam(void)
+{
+  QudaGaugeFixParam ret;
+#elif defined CHECK_PARAM
+static void checkGaugeFixParam(QudaGaugeFixParam *param)
+{
+#else
+void printQudaGaugeFixParam(QudaGaugeFixParam *param)
+{
+  printfQuda("QUDA gauge fix parameters:\n");
+#endif
+
+#if defined CHECK_PARAM
+  if (param->struct_size != (size_t)INVALID_INT && param->struct_size != sizeof(*param))
+    errorQuda("Unexpected QudaGaugeFixParam struct size %lu, expected %lu", param->struct_size, sizeof(*param));
+#else
+  P(struct_size, (size_t)INVALID_INT);
+#endif
+
+  P(gauge_dir, INVALID_INT);
+  P(maxiter, INVALID_INT);
+  P(verbosity_interval, INVALID_INT);
+  P(reunit_interval, INVALID_INT);
+  P(ovr_relaxation_boost, INVALID_DOUBLE);
+  P(fft_alpha, INVALID_DOUBLE);
+  P(tolerance, INVALID_DOUBLE);
+
+#ifndef CHECK_PARAM
+  P(fft_autotune, QUDA_BOOLEAN_FALSE);
+  P(theta_condition, QUDA_BOOLEAN_FALSE);
+#endif
 
+#ifdef INIT_PARAM
+  return ret;
+#endif
+}
+
+// clean up
 #undef INVALID_INT
 #undef INVALID_DOUBLE
 #undef P

lib/gauge_fix_fft.cu

@@ -182,19 +182,26 @@ namespace quda {
   };
 
   template <typename Float, QudaReconstructType recon, int gauge_dir>
-  void gaugeFixingFFT(GaugeField& data, int Nsteps, int verbose_interval,
-                      double alpha0, int autotune, double tolerance, int stopWtheta)
+  void gaugeFixingFFT(GaugeField& data, QudaGaugeFixParam &fix_param)
   {
     TimeProfile profileInternalGaugeFixFFT("InternalGaugeFixQudaFFT", false);
-
+
+    QudaBoolean autotune = fix_param.fft_autotune;
+    double alpha0 = fix_param.fft_alpha;
+    double tolerance = fix_param.tolerance;
+    QudaBoolean theta_condition = fix_param.theta_condition;
+    int steps = fix_param.maxiter;
+    int verbose_interval = fix_param.verbosity_interval;
+
     profileInternalGaugeFixFFT.TPSTART(QUDA_PROFILE_COMPUTE);
 
     if (getVerbosity() >= QUDA_SUMMARIZE) {
-      printfQuda("\tAuto tune active: %s\n", autotune ? "true" : "false");      
+      if(autotune == QUDA_BOOLEAN_TRUE) printfQuda("\tAuto tune active: alpha will be adjusted as the algorithm progresses\n");
+      else printfQuda("\tAuto tune not active: alpha will remain constant as the algorithm progresses\n");      
       printfQuda("\tAlpha parameter of the Steepest Descent Method: %e\n", alpha0);
-      printfQuda("\tTolerance: %lf\n", tolerance);
-      printfQuda("\tStop criterion method: %s\n", stopWtheta ? "Theta" : "Delta");
-      printfQuda("\tMaximum number of iterations: %d\n", Nsteps);
+      printfQuda("\tTolerance: %e\n", tolerance);
+      printfQuda("\tStop criterion method: %s\n", theta_condition == QUDA_BOOLEAN_TRUE ? "Theta" : "Delta");
+      printfQuda("\tMaximum number of iterations: %d\n", steps);
       printfQuda("\tPrint convergence results at every %d steps\n", verbose_interval);
     }
 
@@ -217,11 +224,11 @@ namespace quda {
     GaugeFixQuality<decltype(argQ)> gfixquality(argQ, data);
     gfixquality.apply(device::get_default_stream());
     double action0 = argQ.getAction();
-    if(getVerbosity() >= QUDA_SUMMARIZE) printf("Step: %d\tAction: %.16e\ttheta: %.16e\n", 0, argQ.getAction(), argQ.getTheta());
+    if(getVerbosity() >= QUDA_SUMMARIZE) printf("Step: %05d\tAction: %.16e\ttheta: %.16e\n", 0, argQ.getAction(), argQ.getTheta());
 
     double diff = 0.0;
     int iter = 0;
-    for (iter = 0; iter < Nsteps; iter++) {
+    for (iter = 0; iter < steps; iter++) {
       for (int k = 0; k < 6; k++) {
         //------------------------------------------------------------------------
         // Set a pointer do the element k in lattice volume
@@ -285,23 +292,23 @@ namespace quda {
       double action = argQ.getAction();
       diff = abs(action0 - action);
       if ((iter % verbose_interval) == (verbose_interval - 1) && getVerbosity() >= QUDA_SUMMARIZE)
-        printf("Step: %d\tAction: %.16e\ttheta: %.16e\tDelta: %.16e\n", iter + 1, argQ.getAction(), argQ.getTheta(), diff);
-      if ( autotune && ((action - action0) < -1e-14) ) {
+        printf("Step: %05d\tAction: %.16e\ttheta: %.16e\tDelta: %.16e\n", iter + 1, argQ.getAction(), argQ.getTheta(), diff);
+      if ( autotune == QUDA_BOOLEAN_TRUE && ((action - action0) < -1e-14) ) {
         if ( arg.alpha > 0.01 ) {
           arg.alpha = 0.95 * arg.alpha;
-          if(getVerbosity() >= QUDA_SUMMARIZE) printf(">>>>>>>>>>>>>> Warning: changing alpha down -> %.4e\n", arg.alpha);
+          if(getVerbosity() >= QUDA_SUMMARIZE) printf("Changing alpha down -> %.4e\n", arg.alpha);
         }
       }
       //------------------------------------------------------------------------
       // Check gauge fix quality criterion
       //------------------------------------------------------------------------
-      if ( stopWtheta ) {   if ( argQ.getTheta() < tolerance ) break; }
+      if ( theta_condition == QUDA_BOOLEAN_TRUE ) {   if ( argQ.getTheta() < tolerance ) break; }
       else { if ( diff < tolerance ) break; }
 
       action0 = action;
     }
     if ((iter % verbose_interval) != 0 && getVerbosity() >= QUDA_SUMMARIZE)
-      printf("Step: %d\tAction: %.16e\ttheta: %.16e\tDelta: %.16e\n", iter, argQ.getAction(), argQ.getTheta(), diff);
+      printf("Step: %05d\tAction: %.16e\ttheta: %.16e\tDelta: %.16e\n", iter, argQ.getAction(), argQ.getTheta(), diff);
 
     // Reunitarize at end
     const double unitarize_eps = 1e-14;
@@ -356,45 +363,40 @@ namespace quda {
 
     gflops = (gflops * 1e-9) / (secs);
     gbytes = gbytes / (secs * 1e9);
-    if (getVerbosity() > QUDA_SUMMARIZE) printfQuda("Time: %6.6f s, Gflop/s = %6.1f, GB/s = %6.1f\n", secs, gflops, gbytes);
-
+    if (getVerbosity() > QUDA_SUMMARIZE)
+      printfQuda("Time: %6.6f s, Gflop/s = %6.1f, GB/s = %6.1f\n", secs, gflops, gbytes);
+
     host_free(num_failures_h);
   }
 
   template<typename Float, int nColors, QudaReconstructType recon> struct GaugeFixingFFT {
-    GaugeFixingFFT(GaugeField& data, int gauge_dir, int Nsteps, int verbose_interval,
-                   double alpha, int autotune, double tolerance, int stopWtheta)
+    GaugeFixingFFT(GaugeField& data, QudaGaugeFixParam &fix_param)
     {
-      if (gauge_dir != 3) {
-	if (getVerbosity() > QUDA_SUMMARIZE) printfQuda("Starting Landau gauge fixing with FFTs...\n");
-        gaugeFixingFFT<Float, recon, 4>(data, Nsteps, verbose_interval, alpha, autotune, tolerance, stopWtheta);
+      if (fix_param.gauge_dir == 4) {
+	if (getVerbosity() > QUDA_SUMMARIZE) printfQuda("Starting Landau gauge fixing with FFTs\n");
+        gaugeFixingFFT<Float, recon, 4>(data, fix_param);
+      } else if (fix_param.gauge_dir == 3) {
+	if (getVerbosity() > QUDA_SUMMARIZE) printfQuda("Starting Coulomb gauge fixing with FFTs\n");
+        gaugeFixingFFT<Float, recon, 3>(data, fix_param);	
       } else {
-	if (getVerbosity() > QUDA_SUMMARIZE) printfQuda("Starting Coulomb gauge fixing with FFTs...\n");
-        gaugeFixingFFT<Float, recon, 3>(data, Nsteps, verbose_interval, alpha, autotune, tolerance, stopWtheta);
+	errorQuda("Unexpected gauge_dir = %d", fix_param.gauge_dir); 
       }
     }
   };
 
   /**
    * @brief Gauge fixing with Steepest descent method with FFTs with support for single GPU only.
    * @param[in,out] data, quda gauge field
-   * @param[in] gauge_dir, 3 for Coulomb gauge fixing, other for Landau gauge fixing
-   * @param[in] Nsteps, maximum number of steps to perform gauge fixing
-   * @param[in] verbose_interval, print gauge fixing info when iteration count is a multiple of this
-   * @param[in] alpha, gauge fixing parameter of the method, most common value is 0.08
-   * @param[in] autotune, 1 to autotune the method, i.e., if the Fg inverts its tendency we decrease the alpha value
-   * @param[in] tolerance, torelance value to stop the method, if this value is zero then the method stops when iteration reachs the maximum number of steps defined by Nsteps
-   * @param[in] stopWtheta, 0 for MILC criterion and 1 to use the theta value
+   * @param[in] fix_param Parameter struct defining the gauge fixing
    */
 #if defined(GPU_GAUGE_ALG)
-  void gaugeFixingFFT(GaugeField& data, const int gauge_dir, const int Nsteps, const int verbose_interval, const double alpha,
-                      const int autotune, const double tolerance, const int stopWtheta)
+  void gaugeFixingFFT(GaugeField& data, QudaGaugeFixParam &fix_param)
   {
     if (comm_partitioned()) errorQuda("Gauge Fixing with FFTs in multi-GPU support NOT implemented yet!");
-    instantiate<GaugeFixingFFT, ReconstructNo12>(data, gauge_dir, Nsteps, verbose_interval, alpha, autotune, tolerance, stopWtheta);
+    instantiate<GaugeFixingFFT, ReconstructNo12>(data, fix_param);
   }
 #else
-  void gaugeFixingFFT(GaugeField&, const int, const int, const int, const double, const int, const double, const int)
+  void gaugeFixingFFT(GaugeField&, QudaGaugeFixParam &)
   {
     errorQuda("Gauge fixing has bot been built");
   }