From 6d3343bd6a5c1866f1502caeada79605e77988ff Mon Sep 17 00:00:00 2001 From: Balint Joo Date: Wed, 16 Oct 2024 14:27:21 +0000 Subject: [PATCH 1/9] Mods to allow copy in/out of full QDP-JIT spinors --- include/color_spinor_field_order.h | 8 ++++---- include/kernels/copy_color_spinor.cuh | 19 +++++++++++++++++-- lib/copy_color_spinor.cuh | 5 ----- 3 files changed, 21 insertions(+), 11 deletions(-) diff --git a/include/color_spinor_field_order.h b/include/color_spinor_field_order.h index 25f5234390..b4be52fd8a 100644 --- a/include/color_spinor_field_order.h +++ b/include/color_spinor_field_order.h @@ -1799,8 +1799,8 @@ namespace quda { for (int s = 0; s < Ns; s++) { for (int c = 0; c < Nc; c++) { - v[s * Nc + c] = complex(field[(((0 * Nc + c) * Ns + s) * 2 + (1 - parity)) * volumeCB + x], - field[(((1 * Nc + c) * Ns + s) * 2 + (1 - parity)) * volumeCB + x]); + v[s * Nc + c] = complex(field[(((0 * Nc + c) * Ns + s) * 2 + parity) * volumeCB + x], + field[(((1 * Nc + c) * Ns + s) * 2 + parity) * volumeCB + x]); } } } @@ -1809,8 +1809,8 @@ namespace quda { for (int s = 0; s < Ns; s++) { for (int c = 0; c < Nc; c++) { - field[(((0 * Nc + c) * Ns + s) * 2 + (1 - parity)) * volumeCB + x] = v[s * Nc + c].real(); - field[(((1 * Nc + c) * Ns + s) * 2 + (1 - parity)) * volumeCB + x] = v[s * Nc + c].imag(); + field[(((0 * Nc + c) * Ns + s) * 2 + parity) * volumeCB + x] = v[s * Nc + c].real(); + field[(((1 * Nc + c) * Ns + s) * 2 + parity) * volumeCB + x] = v[s * Nc + c].imag(); } } } diff --git a/include/kernels/copy_color_spinor.cuh b/include/kernels/copy_color_spinor.cuh index 39bc226f3f..b57533d54a 100644 --- a/include/kernels/copy_color_spinor.cuh +++ b/include/kernels/copy_color_spinor.cuh @@ -16,6 +16,21 @@ namespace quda using namespace colorspinor; + /** Helper function for parity computation */ + inline int computeParity(const ColorSpinorField &f) + { + + // Account for odd-even vs. even-odd site orders + int ret_val = f.SiteOrder() == QUDA_ODD_EVEN_SITE_ORDER ? 1 : 0; + + // Account for potential parity flip to access single parity subset QDP-JIT fields + if (f.FieldOrder() == QUDA_QDPJIT_FIELD_ORDER && f.SiteSubset() == QUDA_PARITY_SITE_SUBSET) { + ret_val = 1 - ret_val; + } + + return ret_val; + } + template class Basis_> struct CopyColorSpinorArg : kernel_param<> { @@ -32,8 +47,8 @@ namespace quda kernel_param(dim3(in.VolumeCB(), in.SiteSubset(), 1)), out(out, 1, Out_), in(in, 1, const_cast(In_)), - outParity(out.SiteOrder() == QUDA_ODD_EVEN_SITE_ORDER ? 1 : 0), - inParity(in.SiteOrder() == QUDA_ODD_EVEN_SITE_ORDER ? 1 : 0) + outParity(computeParity(out)), + inParity(computeParity(in)) { } }; diff --git a/lib/copy_color_spinor.cuh b/lib/copy_color_spinor.cuh index f9c40ba9fc..02e6357a35 100644 --- a/lib/copy_color_spinor.cuh +++ b/lib/copy_color_spinor.cuh @@ -181,11 +181,6 @@ namespace quda errorQuda("Copying to full fields with lexicographical ordering is not currently supported"); } - if (dst.SiteSubset() == QUDA_FULL_SITE_SUBSET - && (src.FieldOrder() == QUDA_QDPJIT_FIELD_ORDER || dst.FieldOrder() == QUDA_QDPJIT_FIELD_ORDER)) { - errorQuda("QDPJIT field ordering not supported for full site fields"); - } - genericCopyColorSpinor(param); } From b7da89519f8fc127c1f2bd9c87c91f4a848fb0ec Mon Sep 17 00:00:00 2001 From: SaltyChiang <35213529+SaltyChiang@users.noreply.github.com> Date: Wed, 23 Oct 2024 21:27:16 +0800 Subject: [PATCH 2/9] Use references in the loop when performing solver normalization. --- lib/solve.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/solve.cpp b/lib/solve.cpp index d4eb2bed8d..c9ba9baf21 100644 --- a/lib/solve.cpp +++ b/lib/solve.cpp @@ -146,7 +146,7 @@ namespace quda // rescale the source and solution vectors to help prevent the onset of underflow if (param.solver_normalization == QUDA_SOURCE_NORMALIZATION) { auto nb_inv(nb); - for (auto bi : nb_inv) bi = 1 / sqrt(bi); + for (auto &bi : nb_inv) bi = 1 / sqrt(bi); blas::ax(nb_inv, b); blas::ax(nb_inv, x); } @@ -299,7 +299,7 @@ namespace quda if (param.solver_normalization == QUDA_SOURCE_NORMALIZATION) { // rescale the solution - for (auto bi : nb) bi = sqrt(bi); + for (auto &bi : nb) bi = sqrt(bi); blas::ax(nb, x); } From 4474826975df49651b2098e180c91b3411c80246 Mon Sep 17 00:00:00 2001 From: Balint Joo Date: Mon, 28 Oct 2024 16:15:15 +0000 Subject: [PATCH 3/9] QDPJIT parity only flopped on ODD parity --- include/kernels/copy_color_spinor.cuh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/include/kernels/copy_color_spinor.cuh b/include/kernels/copy_color_spinor.cuh index b57533d54a..5ed05c68ba 100644 --- a/include/kernels/copy_color_spinor.cuh +++ b/include/kernels/copy_color_spinor.cuh @@ -24,7 +24,9 @@ namespace quda int ret_val = f.SiteOrder() == QUDA_ODD_EVEN_SITE_ORDER ? 1 : 0; // Account for potential parity flip to access single parity subset QDP-JIT fields - if (f.FieldOrder() == QUDA_QDPJIT_FIELD_ORDER && f.SiteSubset() == QUDA_PARITY_SITE_SUBSET) { + // The Flip is only needed fir offsetting into Odd Parity Fields + if (f.FieldOrder() == QUDA_QDPJIT_FIELD_ORDER && f.SiteSubset() == QUDA_PARITY_SITE_SUBSET + && f.SuggestedParity() == QUDA_ODD_PARITY) { ret_val = 1 - ret_val; } From 25cfeeddf30be134d67e41fd2c87bd31dbbb9cfe Mon Sep 17 00:00:00 2001 From: Balint Joo Date: Mon, 28 Oct 2024 16:49:24 +0000 Subject: [PATCH 4/9] Prettied doxygen for compute parity --- include/kernels/copy_color_spinor.cuh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/include/kernels/copy_color_spinor.cuh b/include/kernels/copy_color_spinor.cuh index 5ed05c68ba..3751a42411 100644 --- a/include/kernels/copy_color_spinor.cuh +++ b/include/kernels/copy_color_spinor.cuh @@ -16,7 +16,14 @@ namespace quda using namespace colorspinor; - /** Helper function for parity computation */ + /** + * @brief A helper function to figure out what parity to use for input and output. + * @details Pick parity from input field site order. Addditionally QDPJIT fields + * may need a relative parity flip compared to what is expected when dealing with + * only the odd parity since the pointer is always to the top of the full spinort. + * @param f[in] Reference to the field for parity computation + * @return the computed parity + */ inline int computeParity(const ColorSpinorField &f) { From 76947a339c158f017334f71ad9ae381e6b59aae1 Mon Sep 17 00:00:00 2001 From: Balint Joo Date: Mon, 28 Oct 2024 17:37:49 +0000 Subject: [PATCH 5/9] Fixed another doxygen booboo --- include/kernels/copy_color_spinor.cuh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/kernels/copy_color_spinor.cuh b/include/kernels/copy_color_spinor.cuh index 3751a42411..505163cc93 100644 --- a/include/kernels/copy_color_spinor.cuh +++ b/include/kernels/copy_color_spinor.cuh @@ -21,7 +21,7 @@ namespace quda * @details Pick parity from input field site order. Addditionally QDPJIT fields * may need a relative parity flip compared to what is expected when dealing with * only the odd parity since the pointer is always to the top of the full spinort. - * @param f[in] Reference to the field for parity computation + * @param[in] f Reference to the field for parity computation * @return the computed parity */ inline int computeParity(const ColorSpinorField &f) From 219f38ff1f915df6124c421be8b5ef1cb4774e5e Mon Sep 17 00:00:00 2001 From: SaltyChiang Date: Fri, 1 Nov 2024 15:07:38 +0800 Subject: [PATCH 6/9] Use the first `r_coarse` to verify the MG setup. --- lib/multigrid.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/multigrid.cpp b/lib/multigrid.cpp index 5241fdeabc..f355203c6a 100644 --- a/lib/multigrid.cpp +++ b/lib/multigrid.cpp @@ -966,7 +966,7 @@ namespace quda } transfer->R(x_coarse[0], tmp2); - static_cast(diracCoarseResidual)->M(r_coarse, tmp_coarse); + static_cast(diracCoarseResidual)->M(r_coarse[0], tmp_coarse); #if 0 // enable to print out emulated and actual coarse-grid operator vectors for debugging setOutputPrefix(""); From e0f5143154e62fb14cb7024bfc05727d0a626fdf Mon Sep 17 00:00:00 2001 From: SaltyChiang Date: Fri, 1 Nov 2024 15:12:40 +0800 Subject: [PATCH 7/9] Enable `computeHISQForceQuda` for other gauge field orders than the MILC order. --- lib/interface_quda.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/interface_quda.cpp b/lib/interface_quda.cpp index e5b678acfd..7449a0e156 100644 --- a/lib/interface_quda.cpp +++ b/lib/interface_quda.cpp @@ -4256,7 +4256,6 @@ void computeHISQForceQuda(void* const milc_momentum, using namespace quda; using namespace quda::fermion_force; - if (gParam->gauge_order != QUDA_MILC_GAUGE_ORDER) errorQuda("Unsupported input field order %d", gParam->gauge_order); { // default settings for the unitarization @@ -4399,7 +4398,6 @@ void computeHISQForceQuda(void* const milc_momentum, GaugeFieldParam param(*gParam); param.location = QUDA_CPU_FIELD_LOCATION; param.create = QUDA_REFERENCE_FIELD_CREATE; - param.order = QUDA_MILC_GAUGE_ORDER; param.link_type = QUDA_ASQTAD_MOM_LINKS; param.reconstruct = QUDA_RECONSTRUCT_10; param.ghostExchange = QUDA_GHOST_EXCHANGE_NO; @@ -4421,7 +4419,6 @@ void computeHISQForceQuda(void* const milc_momentum, GaugeFieldParam wParam(gParam_field); wParam.location = QUDA_CPU_FIELD_LOCATION; wParam.create = QUDA_REFERENCE_FIELD_CREATE; - wParam.order = QUDA_MILC_GAUGE_ORDER; wParam.link_type = QUDA_GENERAL_LINKS; wParam.ghostExchange = QUDA_GHOST_EXCHANGE_NO; wParam.gauge = (void *)w_link; From 4385f9d89ab8fdc95acf670e3211a779c6fa964e Mon Sep 17 00:00:00 2001 From: SaltyChiang Date: Fri, 1 Nov 2024 15:14:02 +0800 Subject: [PATCH 8/9] Fix typo. --- lib/interface_quda.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/interface_quda.cpp b/lib/interface_quda.cpp index 7449a0e156..1d0f7d3688 100644 --- a/lib/interface_quda.cpp +++ b/lib/interface_quda.cpp @@ -5587,7 +5587,7 @@ void gaugeObservablesQuda(QudaGaugeObservableParam *param) auto profile = pushProfile(profileGaugeObs); checkGaugeObservableParam(param); - if (!gaugePrecise) errorQuda("Cannot compute Polyakov loop as there is no resident gauge field"); + if (!gaugePrecise) errorQuda("Cannot compute gauge observables as there is no resident gauge field"); GaugeField *gauge = nullptr; if (!gaugeSmeared) { From 086174bdd588e52e49f69d98f7ae6f5fd57834c1 Mon Sep 17 00:00:00 2001 From: SaltyChiang Date: Fri, 1 Nov 2024 15:21:46 +0800 Subject: [PATCH 9/9] Add hash for the Eigen tarball. --- CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 803f5dba41..88c83c5b8f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -421,6 +421,7 @@ if(QUDA_DOWNLOAD_EIGEN) NAME Eigen VERSION ${QUDA_EIGEN_VERSION} URL https://gitlab.com/libeigen/eigen/-/archive/${QUDA_EIGEN_VERSION}/eigen-${QUDA_EIGEN_VERSION}.tar.bz2 + URL_HASH SHA256=B4C198460EBA6F28D34894E3A5710998818515104D6E74E5CC331CE31E46E626 DOWNLOAD_ONLY YES SYSTEM YES) target_include_directories(Eigen SYSTEM INTERFACE ${Eigen_SOURCE_DIR})