Hb solver

.github/workflows/basic-build.yaml

@@ -9,21 +9,28 @@ jobs:
     steps:
       # handling of environment variable: https://stackoverflow.com/a/57969570
       - name: lscpu
+        if: ${{ always() }}
         run: |
           lscpu; \
           echo "nb_cores=$(lscpu | grep \^CPU\(s\)\: | awk '{print $2}')" >> $GITHUB_ENV
 
       - name: gcc_version
+        if: ${{ always() }}
         run: gcc --version
 
       - name: echo_nb_cores
+        if: ${{ always() }}
         run: echo "Number of cores ${nb_cores}"
 
       - name: get_packages
-        run: sudo apt-get update && sudo apt install openmpi-bin openmpi-common numdiff
+        run: sudo apt-get update && sudo apt install openmpi-bin openmpi-common libopenmpi3 libopenmpi-dev numdiff
+
+      - name: mpicc_version
+        if: ${{ always() }}
+        run: mpicc --version
 
       - name: get_lime
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           repository: usqcd-software/c-lime
           path: lime
@@ -41,8 +48,15 @@ jobs:
           make -j
           make install
 
+      - name: Archive lime config.log
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: lime_config_output
+          path: ${{github.workspace}}/lime/build/config.log 
+
       - name: get_lemon
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           repository: etmc/lemon
           path: lemon
@@ -61,9 +75,16 @@ jobs:
             ../configure --prefix=$(pwd)/install_dir
           make -j
           make install
+
+      - name: Archive lemon config.log
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: lemon_config_output
+          path: ${{github.workspace}}/lemon/build/config.log 
 
       - name: get_tmlqcd
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           path: main
 
@@ -94,11 +115,25 @@ jobs:
             --with-lapack="-lblas -llapack" || cat config.log
           make -j
 
+      - name: Archive tmLQCD config.log
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: tmlqcd_config_output
+          path: ${{github.workspace}}/main/build/config.log 
+
       - name: nf2_rgmixedcg_hmc_tmcloverdetratio
         working-directory: ${{github.workspace}}/main/build
         run: |
           mpirun -np 2 ./hmc_tm \
             -f ../sample-input/sample-hmc-rgmixedcg-tmcloverdetratio.input
+
+      - name: Archive nf2_rgmixedcg_hmc_tmcloverdetratio output
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: tmlqcd_test_output
+          path: ${{github.workspace}}/main/build/output.data
 
       - name: compare_nf2_rgmixedcg_hmc_tmcloverdetratio
         working-directory: ${{github.workspace}}/main/build

.github/workflows/ddalphaamg-build.yaml

@@ -8,7 +8,7 @@ jobs:
 
     steps:
       - name: get_tmlqcd
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           path: main
 
@@ -20,16 +20,22 @@ jobs:
           echo "nb_cores=$(lscpu | grep \^CPU\(s\)\: | awk '{print $2}')" >> $GITHUB_ENV
 
       - name: gcc_version
+        if: ${{ always() }}
         run: gcc --version
 
       - name: echo_nb_cores
+        if: ${{ always() }}
         run: echo "Number of cores ${nb_cores}"
 
       - name: get_packages
-        run: sudo apt update && sudo apt install openmpi-bin openmpi-common numdiff
+        run: sudo apt update && sudo apt install openmpi-bin openmpi-common libopenmpi3 libopenmpi-dev numdiff
+
+      - name: mpicc_version
+        if: ${{ always() }}
+        run: mpicc --version
 
       - name: get_lime
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           repository: usqcd-software/c-lime
           path: lime
@@ -50,8 +56,15 @@ jobs:
           make -j
           make install
 
+      - name: Archive lime config.log
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: lime_config_output
+          path: ${{github.workspace}}/lime/build/config.log 
+
       - name: get_lemon
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           repository: etmc/lemon
           path: lemon
@@ -73,10 +86,18 @@ jobs:
           make -j
           make install
 
+      - name: Archive lemon config.log
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: lemon_config_output
+          path: ${{github.workspace}}/lemon/build/config.log 
+
       - name: get_ddalphaamg
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
-          repository: sbacchio/ddalphaamg
+          repository: kostrzewa/ddalphaamg #sbacchio/ddalphaamg
+          ref: fix_main_build
           path: ddalphaamg
 
       - name: build_ddalphaamg
@@ -120,6 +141,13 @@ jobs:
           mpirun -np 2 ./hmc_tm \
             -f ../sample-input/sample-hmc-ddalphaamg-tmcloverdetratio.input
 
+      - name: Archive nf2_ddalphaamg_hmc_tmcloverdetratio output
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: tmlqcd_test_output
+          path: ${{github.workspace}}/main/build/output.data
+
       - name: compare_nf2_ddalphaamg_hmc_tmcloverdetratio
         working-directory: ${{github.workspace}}/main/build
         run: |

.github/workflows/qphix-build.yaml

@@ -9,21 +9,28 @@ jobs:
     steps:
       # handling of environment variable: https://stackoverflow.com/a/57969570
       - name: lscpu
+        if: ${{ always() }}
         run: |
           lscpu; \
           echo "nb_cores=$(lscpu | grep \^CPU\(s\)\: | awk '{print $2}')" >> $GITHUB_ENV
 
       - name: gcc_version
+        if: ${{ always() }}
         run: gcc --version
 
       - name: echo_nb_cores
+        if: ${{ always() }}
         run: echo "Number of cores ${nb_cores}"
 
       - name: get_packages
-        run: sudo apt update && sudo apt install openmpi-bin openmpi-common numdiff
+        run: sudo apt update && sudo apt install openmpi-bin openmpi-common libopenmpi3 libopenmpi-dev numdiff
+
+      - name: mpicc_version
+        if: ${{ always() }}
+        run: mpicc --version
 
       - name: get_lime
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           repository: usqcd-software/c-lime
           path: lime
@@ -41,8 +48,15 @@ jobs:
           make -j
           make install
 
+      - name: Archive lime config.log
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: lime_config_output
+          path: ${{github.workspace}}/lime/build/config.log 
+
       - name: get_lemon
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           repository: etmc/lemon
           path: lemon
@@ -63,8 +77,16 @@ jobs:
           make -j
           make install
 
+      - name: Archive lemon config.log
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: lemon_config_output
+          path: ${{github.workspace}}/lemon/build/config.log 
+
+
       - name: get_qmp
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           repository: usqcd-software/qmp
           path: qmp
@@ -86,11 +108,18 @@ jobs:
           make -j
           make install
 
+      - name: Archive qmp config.log
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: qmp_config_output
+          path: ${{github.workspace}}/qmp/build/config.log 
+
       - name: get_qphix
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           repository: JeffersonLab/qphix
-          ref: fix_tmlqcd
+          ref: sysctl_deprecated
           path: qphix
 
       - name: create_qphix_builddir
@@ -99,7 +128,7 @@ jobs:
 
       - name: cache_qphix
         id: cache-qphix
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: qphix/build
           key: ${{ runner.os }}-haswell-qphix
@@ -116,6 +145,7 @@ jobs:
             -Dtm_clover=ON \
             -Dtwisted_mass=ON \
             -Dtesting=OFF \
+            -Drecursive_jN=$(( ${nb_cores} + 3 )) \
             -DCMAKE_CXX_COMPILER=mpicxx \
             -DCMAKE_CXX_FLAGS="-std=c++11 -O2 -mavx2 -mfma -mtune=haswell -march=haswell -fopenmp" \
             -DCMAKE_C_COMPILER=mpicc \
@@ -126,7 +156,7 @@ jobs:
           make install
 
       - name: get_tmlqcd
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           path: main
 
@@ -160,12 +190,27 @@ jobs:
             --enable-qphix-soalen=4 || cat config.log
           make -j
 
+      - name: Archive tmLQCD config.log
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: tmlqcd_config_output
+          path: ${{github.workspace}}/main/build/config.log 
+
       - name: nf2_qphix_hmc_tmcloverdetratio
         working-directory: ${{github.workspace}}/main/build
         run: |
           mpirun -np 2 ./hmc_tm \
             -f ../sample-input/sample-hmc-qphix-tmcloverdetratio.input
 
+      - name: Archive nf2_qphix_hmc_tmcloverdetratio output
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: tmlqcd_test_output
+          path: ${{github.workspace}}/main/build/output.data
+
+
       - name: compare_nf2_qphix_hmc_tmcloverdetratio
         working-directory: ${{github.workspace}}/main/build
         run: |

.gitignore

@@ -40,3 +40,4 @@ doc/*.txt
 doc/*.gpl
 doc/wrapper*
 history_hmc_tm
+.vscode/*

default_input_values.h

@@ -85,6 +85,7 @@
 #define _default_rlxd_level 1
 #define _default_solver_flag 1 // this is CG (see solver/solver_types.h)
 #define _default_nd_solver_flag 15 // this is CGMMSND (see solver/solver_types.h)
+#define _default_HB_solver_flag 26 // this is INVALID_SOLVER (see solver/solver_types.h)
 #define _default_startoption 0
 #define _default_Ntherm 0
 #define _default_Nmeas 1

doc/input.tex

@@ -422,7 +422,8 @@ \subsection{Input parameter for main program}
   \left[\det(P_{n}(Q^2(\kappa) + \mu^2)) det(Q^2(\kappa_2) + \mu^2_2)\right]^{-1}
   \]
 \end{itemize}
-Each of them has different options:
+Each of them has different options
+:
 \begin{itemize}
 \item {\ttfamily DET, CLOVERDET}:
   \begin{itemize}
@@ -450,19 +451,24 @@ \subsection{Input parameter for main program}
   \item {\ttfamily MaxSolverIterations}: default is $5000$
   \item {\ttfamily Solver}: the solver to be used, either CG or
     BiCGstab. Default is CG.
+  \item {\ttfamily HB\_Solver}: the solver to be used in the heatbath step, see section \ref{sec:hb.solver} for details.
   \item {\ttfamily Name}: a name to be assigned to the monomial. The
     default is {\ttfamily DET}
   \end{itemize}
+%
 \item {\ttfamily DETRATIO}: the same as for {\ttfamily DET}, but in
   addition:
   \begin{itemize}
   \item {\ttfamily 2KappaMu2}
   \item {\ttfamily Kappa2}
-  \item {\ttfamily Name}: a name to be assigned to the monomial. The
-    default is {\ttfamily DETRATIO}
+  \item {\ttfamily Name}: 
+    a name to be assigned to the monomial. 
+    The default is {\ttfamily DETRATIO}
   \end{itemize}
-
-
+%
+\item {\ttfamily CLOVERDETRATIO}: 
+  see {\ttfamily CLOVERDET} and {\ttfamily DETRATIO}.
+%
 \item {\ttfamily GAUGE}: 
   \begin{itemize}
   \item {\ttfamily Timescale}: the timescale on which to integrate
@@ -641,6 +647,24 @@ \subsection{Input parameter for main program}
 {\ttfamily <epsilon>} is to be replaced by the ratio  {\ttfamily Lmin/Lmax}.
 \end{itemize}
 
+\subsubsection{The heatbath solver}
+\label{sec:hb.solver}
+
+Inside each monomial the user specifies the solver for the inversion of the Dirac operator.
+The solver used in the HB (heatbath) can be different from the MD (Molecular Dynamics) and ACC step (acceptance step).
+%
+This is determined by the {\ttfamily HB\_solver} parameter inside the {\ttfamily BeginMonomial}-{\ttfamily EndMonomial}.
+Supported values: CG, MIXEDCG, RGMIXEDCG, BICGSTAB, MIXEDBICGSTAB, MG.
+If specified, the following (optional) parameters of the HB solver are parsed:
+\begin{itemize}
+    \item {\ttfamily HB\_MaxSolverIterations}: analogous to {\ttfamily MaxSolverIterations}
+    \item {\ttfamily HB\_UseExternalInverter}: analogous to {\ttfamily UseExternalInverter}
+    \item {\ttfamily HB\_UseSloppyPrecision}: analogous to {\ttfamily UseSloppyPrecision}
+    \item {\ttfamily HB\_UseCompression}: analogous to {\ttfamily UseCompression} 
+\end{itemize}
+%
+Their default values are taken from the corresponding MD solver parameters.
+
 \subsubsection{The Integrator}
 
 The Integrator can be specified similar to the monomials: