From 8b2f9318688d73797b45e6563bb8e95d1d1774e0 Mon Sep 17 00:00:00 2001 From: Sigfried Haering Date: Mon, 3 Jun 2024 16:53:13 -0700 Subject: [PATCH 01/27] low-hanging fruit optimization using just solver/precon options --- src/calorically_perfect.cpp | 16 +++++++++++----- src/tomboulides.cpp | 26 ++++++++++++++++++++++---- src/tomboulides.hpp | 6 +++--- 3 files changed, 36 insertions(+), 12 deletions(-) diff --git a/src/calorically_perfect.cpp b/src/calorically_perfect.cpp index 91ba466b5..db01df506 100644 --- a/src/calorically_perfect.cpp +++ b/src/calorically_perfect.cpp @@ -366,7 +366,7 @@ void CaloricallyPerfectThermoChem::initializeSelf() { // Wall BCs { - std::cout << "There are " << pmesh_->bdr_attributes.Max() << " boundary attributes!" << std::endl; + if (rank0_) std::cout << "There are " << pmesh_->bdr_attributes.Max() << " boundary attributes" << std::endl; Array attr_wall(pmesh_->bdr_attributes.Max()); attr_wall = 0; @@ -379,7 +379,7 @@ void CaloricallyPerfectThermoChem::initializeSelf() { tpsP_->getRequiredInput((basepath + "/type").c_str(), type); if (type == "viscous_isothermal") { - std::cout << "Adding patch = " << patch << " to isothermal wall list!" << std::endl; + if (rank0_) std::cout << "Adding patch = " << patch << " to isothermal wall list" << std::endl; attr_wall = 0; attr_wall[patch - 1] = 1; @@ -504,7 +504,9 @@ void CaloricallyPerfectThermoChem::initializeOperators() { MsInvPC_ = new OperatorJacobiSmoother(diag_pa, empty); } else { MsInvPC_ = new HypreSmoother(*Ms_.As()); - dynamic_cast(MsInvPC_)->SetType(HypreSmoother::Jacobi, 1); + dynamic_cast(MsInvPC_)->SetType(HypreSmoother::Jacobi, 0); + dynamic_cast(MsInvPC_)->SetSOROptions(0.0, 1.0); + dynamic_cast(MsInvPC_)->SetPolyOptions(3, 0.01); } MsInv_ = new CGSolver(sfes_->GetComm()); MsInv_->iterative_mode = false; @@ -520,7 +522,9 @@ void CaloricallyPerfectThermoChem::initializeOperators() { HtInvPC_ = new OperatorJacobiSmoother(diag_pa, temp_ess_tdof_); } else { HtInvPC_ = new HypreSmoother(*Ht_.As()); - dynamic_cast(HtInvPC_)->SetType(HypreSmoother::Jacobi, 1); + dynamic_cast(HtInvPC_)->SetType(HypreSmoother::Jacobi, 0); + dynamic_cast(HtInvPC_)->SetSOROptions(0.0, 1.0); + dynamic_cast(HtInvPC_)->SetPolyOptions(3, 0.01); } HtInv_ = new CGSolver(sfes_->GetComm()); @@ -554,7 +558,9 @@ void CaloricallyPerfectThermoChem::initializeOperators() { MqInvPC_ = new OperatorJacobiSmoother(diag_pa, empty); } else { MqInvPC_ = new HypreSmoother(*Mq_.As()); - dynamic_cast(MqInvPC_)->SetType(HypreSmoother::Jacobi, 1); + dynamic_cast(MqInvPC_)->SetType(HypreSmoother::Jacobi, 0); + dynamic_cast(MqInvPC_)->SetSOROptions(0.0, 1.0); + dynamic_cast(MqInvPC_)->SetPolyOptions(3, 0.01); } MqInv_ = new CGSolver(sfes_->GetComm()); MqInv_->iterative_mode = false; diff --git a/src/tomboulides.cpp b/src/tomboulides.cpp index 9aeb78c69..0da57bc85 100644 --- a/src/tomboulides.cpp +++ b/src/tomboulides.cpp @@ -682,6 +682,11 @@ void Tomboulides::initializeOperators() { L_iorho_lor_ = new ParLORDiscretization(*L_iorho_form_, pres_ess_tdof_); L_iorho_inv_pc_ = new HypreBoomerAMG(L_iorho_lor_->GetAssembledMatrix()); L_iorho_inv_pc_->SetPrintLevel(0); + L_iorho_inv_pc_->SetCoarsening(0.4); + L_iorho_inv_pc_->SetAggressiveCoarsening(4); + L_iorho_inv_pc_->SetMaxLevels(200); + // L_iorho_inv_pc_->SetCycleType(1); + // L_iorho_inv_pc_->SetRelaxType(1); L_iorho_inv_ortho_pc_ = new OrthoSolver(pfes_->GetComm()); L_iorho_inv_ortho_pc_->SetSolver(*L_iorho_inv_pc_); @@ -809,7 +814,9 @@ void Tomboulides::initializeOperators() { Mv_inv_pc_ = new OperatorJacobiSmoother(diag_pa, empty); } else { Mv_inv_pc_ = new HypreSmoother(*Mv_op_.As()); - dynamic_cast(Mv_inv_pc_)->SetType(HypreSmoother::Jacobi, 1); + dynamic_cast(Mv_inv_pc_)->SetType(HypreSmoother::Jacobi, 0); + dynamic_cast(Mv_inv_pc_)->SetSOROptions(0.0, 1.0); + dynamic_cast(Mv_inv_pc_)->SetPolyOptions(3, 0.01); } Mv_inv_ = new CGSolver(vfes_->GetComm()); Mv_inv_->iterative_mode = false; @@ -825,7 +832,9 @@ void Tomboulides::initializeOperators() { Mv_rho_inv_pc_ = new OperatorJacobiSmoother(diag_pa, empty); } else { Mv_rho_inv_pc_ = new HypreSmoother(*Mv_rho_op_.As()); - dynamic_cast(Mv_rho_inv_pc_)->SetType(HypreSmoother::Jacobi, 1); + dynamic_cast(Mv_rho_inv_pc_)->SetType(HypreSmoother::Jacobi, 0); + dynamic_cast(Mv_rho_inv_pc_)->SetSOROptions(0.0, 1.0); + dynamic_cast(Mv_rho_inv_pc_)->SetPolyOptions(3, 0.01); } Mv_rho_inv_ = new CGSolver(vfes_->GetComm()); Mv_rho_inv_->iterative_mode = false; @@ -909,7 +918,9 @@ void Tomboulides::initializeOperators() { Hv_inv_pc_ = new OperatorJacobiSmoother(diag_pa, vel_ess_tdof_); } else { Hv_inv_pc_ = new HypreSmoother(*Hv_op_.As()); - dynamic_cast(Hv_inv_pc_)->SetType(HypreSmoother::Jacobi, 1); + dynamic_cast(Hv_inv_pc_)->SetType(HypreSmoother::Jacobi, 0); + dynamic_cast(Hv_inv_pc_)->SetSOROptions(0.0, 1.0); + dynamic_cast(Hv_inv_pc_)->SetPolyOptions(3, 0.01); } Hv_inv_ = new CGSolver(vfes_->GetComm()); Hv_inv_->iterative_mode = true; @@ -1001,7 +1012,9 @@ void Tomboulides::initializeOperators() { Hs_form_->FormSystemMatrix(swirl_ess_tdof_, Hs_op_); Hs_inv_pc_ = new HypreSmoother(*Hs_op_.As()); - dynamic_cast(Hs_inv_pc_)->SetType(HypreSmoother::Jacobi, 1); + dynamic_cast(Hs_inv_pc_)->SetType(HypreSmoother::Jacobi, 0); + dynamic_cast(Hs_inv_pc_)->SetSOROptions(0.0, 1.0); + dynamic_cast(Hs_inv_pc_)->SetPolyOptions(3, 0.01); Hs_inv_ = new CGSolver(pfes_->GetComm()); Hs_inv_->iterative_mode = true; @@ -1150,6 +1163,11 @@ void Tomboulides::step() { L_iorho_lor_ = new ParLORDiscretization(*L_iorho_form_, pres_ess_tdof_); L_iorho_inv_pc_ = new HypreBoomerAMG(L_iorho_lor_->GetAssembledMatrix()); L_iorho_inv_pc_->SetPrintLevel(0); + L_iorho_inv_pc_->SetCoarsening(0.4); + L_iorho_inv_pc_->SetAggressiveCoarsening(4); + L_iorho_inv_pc_->SetMaxLevels(200); + // L_iorho_inv_pc_->SetCycleType(1); + // L_iorho_inv_pc_->SetRelaxType(2); L_iorho_inv_ortho_pc_ = new OrthoSolver(pfes_->GetComm()); L_iorho_inv_ortho_pc_->SetSolver(*L_iorho_inv_pc_); diff --git a/src/tomboulides.hpp b/src/tomboulides.hpp index d6c0bf911..a5da0cd35 100644 --- a/src/tomboulides.hpp +++ b/src/tomboulides.hpp @@ -85,16 +85,16 @@ class Tomboulides final : public FlowBase { bool partial_assembly_ = false; int pressure_solve_pl_ = 0; - int pressure_solve_max_iter_ = 100; + int pressure_solve_max_iter_ = 1000; double pressure_solve_rtol_ = 1e-8; double pressure_solve_atol_ = 1e-18; int mass_inverse_pl_ = 0; - int mass_inverse_max_iter_ = 200; + int mass_inverse_max_iter_ = 1000; double mass_inverse_rtol_ = 1e-12; int hsolve_pl_ = 0; - int hsolve_max_iter_ = 200; + int hsolve_max_iter_ = 1000; double hsolve_rtol_ = 1e-12; // To use "numerical integration", quadrature rule must persist From 054a0755d4d3e82111c9b85ae1d9517cc47c47c0 Mon Sep 17 00:00:00 2001 From: Sigfried Haering Date: Mon, 3 Jun 2024 17:17:03 -0700 Subject: [PATCH 02/27] removed partial assembly if's where partial assembly is not supported. However, setting partial assembly to true is slightly slower --- src/calorically_perfect.cpp | 41 ++++------------------ src/tomboulides.cpp | 70 +++++++------------------------------ 2 files changed, 19 insertions(+), 92 deletions(-) diff --git a/src/calorically_perfect.cpp b/src/calorically_perfect.cpp index db01df506..39f7420cf 100644 --- a/src/calorically_perfect.cpp +++ b/src/calorically_perfect.cpp @@ -446,9 +446,6 @@ void CaloricallyPerfectThermoChem::initializeOperators() { at_blfi->SetIntRule(&ir_nli); } At_form_->AddDomainIntegrator(at_blfi); - if (partial_assembly_) { - At_form_->SetAssemblyLevel(AssemblyLevel::PARTIAL); - } At_form_->Assemble(); At_form_->FormSystemMatrix(empty, At_); if (rank0_) std::cout << "CaloricallyPerfectThermoChem At operator set" << endl; @@ -474,9 +471,6 @@ void CaloricallyPerfectThermoChem::initializeOperators() { // msrho_blfi->SetIntRule(&ir_di); } MsRho_form_->AddDomainIntegrator(msrho_blfi); - if (partial_assembly_) { - MsRho_form_->SetAssemblyLevel(AssemblyLevel::PARTIAL); - } MsRho_form_->Assemble(); MsRho_form_->FormSystemMatrix(empty, MsRho_); if (rank0_) std::cout << "CaloricallyPerfectThermoChem MsRho operator set" << endl; @@ -491,9 +485,6 @@ void CaloricallyPerfectThermoChem::initializeOperators() { } Ht_form_->AddDomainIntegrator(hmt_blfi); Ht_form_->AddDomainIntegrator(hdt_blfi); - if (partial_assembly_) { - Ht_form_->SetAssemblyLevel(AssemblyLevel::PARTIAL); - } Ht_form_->Assemble(); Ht_form_->FormSystemMatrix(temp_ess_tdof_, Ht_); if (rank0_) std::cout << "CaloricallyPerfectThermoChem Ht operator set" << endl; @@ -516,19 +507,12 @@ void CaloricallyPerfectThermoChem::initializeOperators() { MsInv_->SetRelTol(rtol_); MsInv_->SetMaxIter(max_iter_); - if (partial_assembly_) { - Vector diag_pa(sfes_->GetTrueVSize()); - Ht_form_->AssembleDiagonal(diag_pa); - HtInvPC_ = new OperatorJacobiSmoother(diag_pa, temp_ess_tdof_); - } else { - HtInvPC_ = new HypreSmoother(*Ht_.As()); - dynamic_cast(HtInvPC_)->SetType(HypreSmoother::Jacobi, 0); - dynamic_cast(HtInvPC_)->SetSOROptions(0.0, 1.0); - dynamic_cast(HtInvPC_)->SetPolyOptions(3, 0.01); - } - + HtInvPC_ = new HypreSmoother(*Ht_.As()); + dynamic_cast(HtInvPC_)->SetType(HypreSmoother::Jacobi, 0); + dynamic_cast(HtInvPC_)->SetSOROptions(0.0, 1.0); + dynamic_cast(HtInvPC_)->SetPolyOptions(3, 0.01); + HtInv_ = new CGSolver(sfes_->GetComm()); - HtInv_->iterative_mode = true; HtInv_->SetOperator(*Ht_); HtInv_->SetPreconditioner(*HtInvPC_); @@ -678,15 +662,7 @@ void CaloricallyPerfectThermoChem::step() { Ht_form_->Update(); Ht_form_->Assemble(); Ht_form_->FormSystemMatrix(temp_ess_tdof_, Ht_); - HtInv_->SetOperator(*Ht_); - if (partial_assembly_) { - delete HtInvPC_; - Vector diag_pa(sfes_->GetTrueVSize()); - Ht_form_->AssembleDiagonal(diag_pa); - HtInvPC_ = new OperatorJacobiSmoother(diag_pa, temp_ess_tdof_); - HtInv_->SetPreconditioner(*HtInvPC_); - } // Prepare for the solve for (auto &temp_dbc : temp_dbcs_) { @@ -695,12 +671,7 @@ void CaloricallyPerfectThermoChem::step() { sfes_->GetRestrictionMatrix()->MultTranspose(resT_, resT_gf_); Vector Xt2, Bt2; - if (partial_assembly_) { - auto *HC = Ht_.As(); - EliminateRHS(*Ht_form_, *HC, temp_ess_tdof_, Tn_next_gf_, resT_gf_, Xt2, Bt2, 1); - } else { - Ht_form_->FormLinearSystem(temp_ess_tdof_, Tn_next_gf_, resT_gf_, Ht_, Xt2, Bt2, 1); - } + Ht_form_->FormLinearSystem(temp_ess_tdof_, Tn_next_gf_, resT_gf_, Ht_, Xt2, Bt2, 1); // solve helmholtz eq for temp HtInv_->Mult(Bt2, Xt2); diff --git a/src/tomboulides.cpp b/src/tomboulides.cpp index 0da57bc85..525f333ff 100644 --- a/src/tomboulides.cpp +++ b/src/tomboulides.cpp @@ -672,9 +672,6 @@ void Tomboulides::initializeOperators() { L_iorho_blfi->SetIntRule(&ir_ni_p); } L_iorho_form_->AddDomainIntegrator(L_iorho_blfi); - if (partial_assembly_) { - L_iorho_form_->SetAssemblyLevel(AssemblyLevel::PARTIAL); - } L_iorho_form_->Assemble(); L_iorho_form_->FormSystemMatrix(pres_ess_tdof_, L_iorho_op_); @@ -765,9 +762,6 @@ void Tomboulides::initializeOperators() { msr_blfi->SetIntRule(&ir_ni_p); } Ms_rho_form_->AddDomainIntegrator(msr_blfi); - if (partial_assembly_) { - Ms_rho_form_->SetAssemblyLevel(AssemblyLevel::PARTIAL); - } Ms_rho_form_->Assemble(); Ms_rho_form_->FormSystemMatrix(empty, Ms_rho_op_); @@ -801,9 +795,6 @@ void Tomboulides::initializeOperators() { mvr_blfi->SetIntRule(&ir_ni_v); } Mv_rho_form_->AddDomainIntegrator(mvr_blfi); - if (partial_assembly_) { - Mv_rho_form_->SetAssemblyLevel(AssemblyLevel::PARTIAL); - } Mv_rho_form_->Assemble(); Mv_rho_form_->FormSystemMatrix(empty, Mv_rho_op_); @@ -826,16 +817,11 @@ void Tomboulides::initializeOperators() { Mv_inv_->SetRelTol(mass_inverse_rtol_); Mv_inv_->SetMaxIter(mass_inverse_max_iter_); - if (partial_assembly_) { - Vector diag_pa(vfes_->GetTrueVSize()); - Mv_rho_form_->AssembleDiagonal(diag_pa); - Mv_rho_inv_pc_ = new OperatorJacobiSmoother(diag_pa, empty); - } else { - Mv_rho_inv_pc_ = new HypreSmoother(*Mv_rho_op_.As()); - dynamic_cast(Mv_rho_inv_pc_)->SetType(HypreSmoother::Jacobi, 0); - dynamic_cast(Mv_rho_inv_pc_)->SetSOROptions(0.0, 1.0); - dynamic_cast(Mv_rho_inv_pc_)->SetPolyOptions(3, 0.01); - } + Mv_rho_inv_pc_ = new HypreSmoother(*Mv_rho_op_.As()); + dynamic_cast(Mv_rho_inv_pc_)->SetType(HypreSmoother::Jacobi, 0); + dynamic_cast(Mv_rho_inv_pc_)->SetSOROptions(0.0, 1.0); + dynamic_cast(Mv_rho_inv_pc_)->SetPolyOptions(3, 0.01); + Mv_rho_inv_ = new CGSolver(vfes_->GetComm()); Mv_rho_inv_->iterative_mode = false; Mv_rho_inv_->SetOperator(*Mv_rho_op_); @@ -902,26 +888,15 @@ void Tomboulides::initializeOperators() { auto *hfv_blfi = new VectorMassIntegrator(*visc_forcing_coeff_); Hv_form_->AddDomainIntegrator(hfv_blfi); } - if (partial_assembly_) { - // Partial assembly is not supported for variable coefficient - // VectorMassIntegrator (as of mfem 4.5.2 at least) - assert(false); - Hv_form_->SetAssemblyLevel(AssemblyLevel::PARTIAL); - } Hv_form_->Assemble(); Hv_form_->FormSystemMatrix(vel_ess_tdof_, Hv_op_); // Helmholtz solver - if (partial_assembly_) { - Vector diag_pa(vfes_->GetTrueVSize()); - Hv_form_->AssembleDiagonal(diag_pa); - Hv_inv_pc_ = new OperatorJacobiSmoother(diag_pa, vel_ess_tdof_); - } else { - Hv_inv_pc_ = new HypreSmoother(*Hv_op_.As()); - dynamic_cast(Hv_inv_pc_)->SetType(HypreSmoother::Jacobi, 0); - dynamic_cast(Hv_inv_pc_)->SetSOROptions(0.0, 1.0); - dynamic_cast(Hv_inv_pc_)->SetPolyOptions(3, 0.01); - } + Hv_inv_pc_ = new HypreSmoother(*Hv_op_.As()); + dynamic_cast(Hv_inv_pc_)->SetType(HypreSmoother::Jacobi, 0); + dynamic_cast(Hv_inv_pc_)->SetSOROptions(0.0, 1.0); + dynamic_cast(Hv_inv_pc_)->SetPolyOptions(3, 0.01); + Hv_inv_ = new CGSolver(vfes_->GetComm()); Hv_inv_->iterative_mode = true; Hv_inv_->SetOperator(*Hv_op_); @@ -1201,10 +1176,7 @@ void Tomboulides::step() { Hv_form_->FormSystemMatrix(vel_ess_tdof_, Hv_op_); Hv_inv_->SetOperator(*Hv_op_); - if (partial_assembly_) { - // TODO(trevilo): Support partial assembly - assert(false); - } + sw_helm_.Stop(); //------------------------------------------------------------------------ @@ -1408,15 +1380,7 @@ void Tomboulides::step() { pfes_->GetRestrictionMatrix()->MultTranspose(resp_vec_, *resp_gf_); Vector X1, B1; - if (partial_assembly_) { - // TODO(trevilo): Support partial assembly here - assert(false); - // auto *SpC = Sp.As(); - // EliminateRHS(*Sp_form, *SpC, pres_ess_tdof, pn_gf, resp_gf, X1, B1, 1); - } else { - L_iorho_form_->FormLinearSystem(pres_ess_tdof_, *p_gf_, *resp_gf_, L_iorho_op_, X1, B1, 1); - } - + L_iorho_form_->FormLinearSystem(pres_ess_tdof_, *p_gf_, *resp_gf_, L_iorho_op_, X1, B1, 1); L_iorho_inv_->Mult(B1, X1); if (!L_iorho_inv_->GetConverged()) { if (rank0_) std::cout << "ERROR: Poisson solve did not converge." << std::endl; @@ -1463,15 +1427,7 @@ void Tomboulides::step() { vfes_->GetRestrictionMatrix()->MultTranspose(resu_vec_, *resu_gf_); Vector X2, B2; - if (partial_assembly_) { - // TODO(trevilo): Add partial assembly support - assert(false); - // auto *HC = H.As(); - // EliminateRHS(*Hv_form_, *HC, vel_ess_tdof, un_next_gf, resu_gf, X2, B2, 1); - } else { - Hv_form_->FormLinearSystem(vel_ess_tdof_, *u_next_gf_, *resu_gf_, Hv_op_, X2, B2, 1); - } - + Hv_form_->FormLinearSystem(vel_ess_tdof_, *u_next_gf_, *resu_gf_, Hv_op_, X2, B2, 1); Hv_inv_->Mult(B2, X2); if (!Hv_inv_->GetConverged()) { if (rank0_) std::cout << "ERROR: Helmholtz solve did not converge." << std::endl; From 99debbad8939cb48c801a871c39b1bf875c20669 Mon Sep 17 00:00:00 2001 From: Sigfried Haering Date: Fri, 7 Jun 2024 03:17:24 -0700 Subject: [PATCH 03/27] lin-solve options pushed to hpp --- src/calorically_perfect.cpp | 21 ++++++++++-------- src/calorically_perfect.hpp | 8 ++++++- src/tomboulides.cpp | 43 ++++++++++++++++++++----------------- src/tomboulides.hpp | 11 ++++++++++ 4 files changed, 53 insertions(+), 30 deletions(-) diff --git a/src/calorically_perfect.cpp b/src/calorically_perfect.cpp index 39f7420cf..d1c1fb3e1 100644 --- a/src/calorically_perfect.cpp +++ b/src/calorically_perfect.cpp @@ -408,6 +408,9 @@ void CaloricallyPerfectThermoChem::initializeSelf() { void CaloricallyPerfectThermoChem::initializeOperators() { dt_ = time_coeff_.dt; + // polynomial order for smoother of precons + smoother_poly_order_ = order_ + 1; + Array empty; // GLL integration rule (Numerical Integration) @@ -495,9 +498,9 @@ void CaloricallyPerfectThermoChem::initializeOperators() { MsInvPC_ = new OperatorJacobiSmoother(diag_pa, empty); } else { MsInvPC_ = new HypreSmoother(*Ms_.As()); - dynamic_cast(MsInvPC_)->SetType(HypreSmoother::Jacobi, 0); - dynamic_cast(MsInvPC_)->SetSOROptions(0.0, 1.0); - dynamic_cast(MsInvPC_)->SetPolyOptions(3, 0.01); + dynamic_cast(MsInvPC_)->SetType(HypreSmoother::Jacobi, smoother_passes_); + dynamic_cast(MsInvPC_)->SetSOROptions(smoother_relax_weight_, smoother_relax_omega_); + dynamic_cast(MsInvPC_)->SetPolyOptions(smoother_poly_order_, smoother_poly_fraction_,smoother_eig_est_); } MsInv_ = new CGSolver(sfes_->GetComm()); MsInv_->iterative_mode = false; @@ -508,9 +511,9 @@ void CaloricallyPerfectThermoChem::initializeOperators() { MsInv_->SetMaxIter(max_iter_); HtInvPC_ = new HypreSmoother(*Ht_.As()); - dynamic_cast(HtInvPC_)->SetType(HypreSmoother::Jacobi, 0); - dynamic_cast(HtInvPC_)->SetSOROptions(0.0, 1.0); - dynamic_cast(HtInvPC_)->SetPolyOptions(3, 0.01); + dynamic_cast(HtInvPC_)->SetType(HypreSmoother::Jacobi, smoother_passes_); + dynamic_cast(HtInvPC_)->SetSOROptions(smoother_relax_weight_, smoother_relax_omega_); + dynamic_cast(HtInvPC_)->SetPolyOptions(smoother_poly_order_, smoother_poly_fraction_,smoother_eig_est_); HtInv_ = new CGSolver(sfes_->GetComm()); HtInv_->iterative_mode = true; @@ -542,9 +545,9 @@ void CaloricallyPerfectThermoChem::initializeOperators() { MqInvPC_ = new OperatorJacobiSmoother(diag_pa, empty); } else { MqInvPC_ = new HypreSmoother(*Mq_.As()); - dynamic_cast(MqInvPC_)->SetType(HypreSmoother::Jacobi, 0); - dynamic_cast(MqInvPC_)->SetSOROptions(0.0, 1.0); - dynamic_cast(MqInvPC_)->SetPolyOptions(3, 0.01); + dynamic_cast(MqInvPC_)->SetType(HypreSmoother::Jacobi, smoother_passes_); + dynamic_cast(MqInvPC_)->SetSOROptions(smoother_relax_weight_, smoother_relax_omega_); + dynamic_cast(MqInvPC_)->SetPolyOptions(smoother_poly_order_, smoother_poly_fraction_,smoother_eig_est_); } MqInv_ = new CGSolver(sfes_->GetComm()); MqInv_->iterative_mode = false; diff --git a/src/calorically_perfect.hpp b/src/calorically_perfect.hpp index 570124e02..d515b9364 100644 --- a/src/calorically_perfect.hpp +++ b/src/calorically_perfect.hpp @@ -85,7 +85,13 @@ class CaloricallyPerfectThermoChem : public ThermoChemModelBase { int pl_solve_ = 0; /**< Verbosity level passed to mfem solvers */ int max_iter_; /**< Maximum number of linear solver iterations */ double rtol_ = 1e-12; /**< Linear solver relative tolerance */ - + int smoother_poly_order_; + double smoother_poly_fraction_ = 0.1; + int smoother_eig_est_ = 10; + int smoother_passes_ = 1; + double smoother_relax_weight_ = 0.0; + double smoother_relax_omega_ = 0.001; + // Boundary condition info Array temp_ess_attr_; /**< List of patches with Dirichlet BC on temperature */ Array Qt_ess_attr_; /**< List of patches with Dirichlet BC on Q (thermal divergence) */ diff --git a/src/tomboulides.cpp b/src/tomboulides.cpp index 525f333ff..0b6475b71 100644 --- a/src/tomboulides.cpp +++ b/src/tomboulides.cpp @@ -577,6 +577,9 @@ void Tomboulides::initializeSelf() { void Tomboulides::initializeOperators() { assert(thermo_interface_ != NULL); + // polynomial order for smoother of precons + smoother_poly_order_ = vorder_ + 1; + // Create all the Coefficient objects we need rho_coeff_ = new GridFunctionCoefficient(thermo_interface_->density); @@ -679,9 +682,9 @@ void Tomboulides::initializeOperators() { L_iorho_lor_ = new ParLORDiscretization(*L_iorho_form_, pres_ess_tdof_); L_iorho_inv_pc_ = new HypreBoomerAMG(L_iorho_lor_->GetAssembledMatrix()); L_iorho_inv_pc_->SetPrintLevel(0); - L_iorho_inv_pc_->SetCoarsening(0.4); - L_iorho_inv_pc_->SetAggressiveCoarsening(4); - L_iorho_inv_pc_->SetMaxLevels(200); + L_iorho_inv_pc_->SetCoarsening(amg_coarsening_); + L_iorho_inv_pc_->SetAggressiveCoarsening(amg_aggresive_); + L_iorho_inv_pc_->SetMaxLevels(amg_max_levels_); // L_iorho_inv_pc_->SetCycleType(1); // L_iorho_inv_pc_->SetRelaxType(1); L_iorho_inv_ortho_pc_ = new OrthoSolver(pfes_->GetComm()); @@ -805,9 +808,9 @@ void Tomboulides::initializeOperators() { Mv_inv_pc_ = new OperatorJacobiSmoother(diag_pa, empty); } else { Mv_inv_pc_ = new HypreSmoother(*Mv_op_.As()); - dynamic_cast(Mv_inv_pc_)->SetType(HypreSmoother::Jacobi, 0); - dynamic_cast(Mv_inv_pc_)->SetSOROptions(0.0, 1.0); - dynamic_cast(Mv_inv_pc_)->SetPolyOptions(3, 0.01); + dynamic_cast(Mv_inv_pc_)->SetType(HypreSmoother::Jacobi, smoother_passes_); + dynamic_cast(Mv_inv_pc_)->SetSOROptions(smoother_relax_weight_, smoother_relax_omega_); + dynamic_cast(Mv_inv_pc_)->SetPolyOptions(smoother_poly_order_, smoother_poly_fraction_,smoother_eig_est_); } Mv_inv_ = new CGSolver(vfes_->GetComm()); Mv_inv_->iterative_mode = false; @@ -818,10 +821,10 @@ void Tomboulides::initializeOperators() { Mv_inv_->SetMaxIter(mass_inverse_max_iter_); Mv_rho_inv_pc_ = new HypreSmoother(*Mv_rho_op_.As()); - dynamic_cast(Mv_rho_inv_pc_)->SetType(HypreSmoother::Jacobi, 0); - dynamic_cast(Mv_rho_inv_pc_)->SetSOROptions(0.0, 1.0); - dynamic_cast(Mv_rho_inv_pc_)->SetPolyOptions(3, 0.01); - + dynamic_cast(Mv_rho_inv_pc_)->SetType(HypreSmoother::Jacobi, smoother_passes_); + dynamic_cast(Mv_rho_inv_pc_)->SetSOROptions(smoother_relax_weight_, smoother_relax_omega_); + dynamic_cast(Mv_rho_inv_pc_)->SetPolyOptions(smoother_poly_order_, smoother_poly_fraction_,smoother_eig_est_); + Mv_rho_inv_ = new CGSolver(vfes_->GetComm()); Mv_rho_inv_->iterative_mode = false; Mv_rho_inv_->SetOperator(*Mv_rho_op_); @@ -892,10 +895,10 @@ void Tomboulides::initializeOperators() { Hv_form_->FormSystemMatrix(vel_ess_tdof_, Hv_op_); // Helmholtz solver - Hv_inv_pc_ = new HypreSmoother(*Hv_op_.As()); - dynamic_cast(Hv_inv_pc_)->SetType(HypreSmoother::Jacobi, 0); - dynamic_cast(Hv_inv_pc_)->SetSOROptions(0.0, 1.0); - dynamic_cast(Hv_inv_pc_)->SetPolyOptions(3, 0.01); + Hv_inv_pc_ = new HypreSmoother(*Hv_op_.As()); + dynamic_cast(Hv_inv_pc_)->SetType(HypreSmoother::Jacobi, smoother_passes_); + dynamic_cast(Hv_inv_pc_)->SetSOROptions(smoother_relax_weight_, smoother_relax_omega_); + dynamic_cast(Hv_inv_pc_)->SetPolyOptions(smoother_poly_order_, smoother_poly_fraction_,smoother_eig_est_); Hv_inv_ = new CGSolver(vfes_->GetComm()); Hv_inv_->iterative_mode = true; @@ -987,9 +990,9 @@ void Tomboulides::initializeOperators() { Hs_form_->FormSystemMatrix(swirl_ess_tdof_, Hs_op_); Hs_inv_pc_ = new HypreSmoother(*Hs_op_.As()); - dynamic_cast(Hs_inv_pc_)->SetType(HypreSmoother::Jacobi, 0); - dynamic_cast(Hs_inv_pc_)->SetSOROptions(0.0, 1.0); - dynamic_cast(Hs_inv_pc_)->SetPolyOptions(3, 0.01); + dynamic_cast(Hs_inv_pc_)->SetType(HypreSmoother::Jacobi, smoother_passes_); + dynamic_cast(Hs_inv_pc_)->SetSOROptions(smoother_relax_weight_, smoother_relax_omega_); + dynamic_cast(Hs_inv_pc_)->SetPolyOptions(smoother_poly_order_, smoother_poly_fraction_,smoother_eig_est_); Hs_inv_ = new CGSolver(pfes_->GetComm()); Hs_inv_->iterative_mode = true; @@ -1138,9 +1141,9 @@ void Tomboulides::step() { L_iorho_lor_ = new ParLORDiscretization(*L_iorho_form_, pres_ess_tdof_); L_iorho_inv_pc_ = new HypreBoomerAMG(L_iorho_lor_->GetAssembledMatrix()); L_iorho_inv_pc_->SetPrintLevel(0); - L_iorho_inv_pc_->SetCoarsening(0.4); - L_iorho_inv_pc_->SetAggressiveCoarsening(4); - L_iorho_inv_pc_->SetMaxLevels(200); + L_iorho_inv_pc_->SetCoarsening(amg_coarsening_); + L_iorho_inv_pc_->SetAggressiveCoarsening(amg_aggresive_); + L_iorho_inv_pc_->SetMaxLevels(amg_max_levels_); // L_iorho_inv_pc_->SetCycleType(1); // L_iorho_inv_pc_->SetRelaxType(2); L_iorho_inv_ortho_pc_ = new OrthoSolver(pfes_->GetComm()); diff --git a/src/tomboulides.hpp b/src/tomboulides.hpp index a5da0cd35..545d01182 100644 --- a/src/tomboulides.hpp +++ b/src/tomboulides.hpp @@ -84,6 +84,17 @@ class Tomboulides final : public FlowBase { bool numerical_integ_ = false; bool partial_assembly_ = false; + // linear-solver options + int smoother_poly_order_; + double smoother_poly_fraction_ = 0.1; + int smoother_eig_est_ = 10; + int smoother_passes_ = 1; + double smoother_relax_weight_ = 0.0; + double smoother_relax_omega_ = 0.001; + double amg_coarsening_ = 0.4; + int amg_aggresive_ = 4; + int amg_max_levels_ = 100; + int pressure_solve_pl_ = 0; int pressure_solve_max_iter_ = 1000; double pressure_solve_rtol_ = 1e-8; From 735597943c07e1fa3df06a173c0ff2b18bb466aa Mon Sep 17 00:00:00 2001 From: Sigfried Haering Date: Sat, 8 Jun 2024 00:13:32 -0700 Subject: [PATCH 04/27] adding more AMG optimizations --- src/tomboulides.cpp | 51 ++++++++++++++++++++++++++++++++------------- src/tomboulides.hpp | 51 +++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 86 insertions(+), 16 deletions(-) diff --git a/src/tomboulides.cpp b/src/tomboulides.cpp index 0b6475b71..2305500e5 100644 --- a/src/tomboulides.cpp +++ b/src/tomboulides.cpp @@ -115,6 +115,19 @@ Tomboulides::Tomboulides(mfem::ParMesh *pmesh, int vorder, int porder, temporalS // locates quadrature points on the axis, which can lead to // singular mass matrices and evaluations of 1/r = 1/0. if (axisym_) assert(!numerical_integ_); + + // exposing solver tolerance options to user + tps->getInput("loMach/tomboulides/psolve_atol_", pressure_solve_atol_, 1.0e-18); + tps->getInput("loMach/tomboulides/hsolve_atol_", hsolve_atol_, 1.0e-12); + tps->getInput("loMach/tomboulides/msolve_atol_", mass_inverse_atol_, 1.0e-12); + + tps->getInput("loMach/tomboulides/psolve_rtol_", pressure_solve_rtol_, 1.0e-08); + tps->getInput("loMach/tomboulides/hsolve_rtol_", hsolve_rtol_, 1.0e-12); + tps->getInput("loMach/tomboulides/msolve_rtol_", mass_inverse_rtol_, 1.0e-12); + + tps->getInput("loMach/tomboulides/psolve_maxIters_", pressure_solve_max_iter_, 1000); + tps->getInput("loMach/tomboulides/hsolve_maxIters_", hsolve_max_iter_, 1000); + tps->getInput("loMach/tomboulides/msolve_maxIters_", mass_inverse_max_iter_, 1000); } } @@ -580,6 +593,13 @@ void Tomboulides::initializeOperators() { // polynomial order for smoother of precons smoother_poly_order_ = vorder_ + 1; + // AMG strength threshold + if (dim_ == 2) { + pressure_strength_thres_ = 0.25; + } else { + pressure_strength_thres_ = 0.6; + } + // Create all the Coefficient objects we need rho_coeff_ = new GridFunctionCoefficient(thermo_interface_->density); @@ -678,15 +698,10 @@ void Tomboulides::initializeOperators() { L_iorho_form_->Assemble(); L_iorho_form_->FormSystemMatrix(pres_ess_tdof_, L_iorho_op_); - // Variable coefficient Laplacian inverse + // Variable coefficient Laplacian inverse (gets destoryed and rebuilt every step) L_iorho_lor_ = new ParLORDiscretization(*L_iorho_form_, pres_ess_tdof_); L_iorho_inv_pc_ = new HypreBoomerAMG(L_iorho_lor_->GetAssembledMatrix()); - L_iorho_inv_pc_->SetPrintLevel(0); - L_iorho_inv_pc_->SetCoarsening(amg_coarsening_); - L_iorho_inv_pc_->SetAggressiveCoarsening(amg_aggresive_); - L_iorho_inv_pc_->SetMaxLevels(amg_max_levels_); - // L_iorho_inv_pc_->SetCycleType(1); - // L_iorho_inv_pc_->SetRelaxType(1); + L_iorho_inv_pc_->SetPrintLevel(pressure_solve_pl_); L_iorho_inv_ortho_pc_ = new OrthoSolver(pfes_->GetComm()); L_iorho_inv_ortho_pc_->SetSolver(*L_iorho_inv_pc_); @@ -808,7 +823,7 @@ void Tomboulides::initializeOperators() { Mv_inv_pc_ = new OperatorJacobiSmoother(diag_pa, empty); } else { Mv_inv_pc_ = new HypreSmoother(*Mv_op_.As()); - dynamic_cast(Mv_inv_pc_)->SetType(HypreSmoother::Jacobi, smoother_passes_); + dynamic_cast(Mv_inv_pc_)->SetType(smoother_type_, smoother_passes_); dynamic_cast(Mv_inv_pc_)->SetSOROptions(smoother_relax_weight_, smoother_relax_omega_); dynamic_cast(Mv_inv_pc_)->SetPolyOptions(smoother_poly_order_, smoother_poly_fraction_,smoother_eig_est_); } @@ -817,11 +832,12 @@ void Tomboulides::initializeOperators() { Mv_inv_->SetOperator(*Mv_op_); Mv_inv_->SetPreconditioner(*Mv_inv_pc_); Mv_inv_->SetPrintLevel(mass_inverse_pl_); + Mv_inv_->SetAbsTol(mass_inverse_atol_); Mv_inv_->SetRelTol(mass_inverse_rtol_); Mv_inv_->SetMaxIter(mass_inverse_max_iter_); Mv_rho_inv_pc_ = new HypreSmoother(*Mv_rho_op_.As()); - dynamic_cast(Mv_rho_inv_pc_)->SetType(HypreSmoother::Jacobi, smoother_passes_); + dynamic_cast(Mv_rho_inv_pc_)->SetType(smoother_type_, smoother_passes_); dynamic_cast(Mv_rho_inv_pc_)->SetSOROptions(smoother_relax_weight_, smoother_relax_omega_); dynamic_cast(Mv_rho_inv_pc_)->SetPolyOptions(smoother_poly_order_, smoother_poly_fraction_,smoother_eig_est_); @@ -830,6 +846,7 @@ void Tomboulides::initializeOperators() { Mv_rho_inv_->SetOperator(*Mv_rho_op_); Mv_rho_inv_->SetPreconditioner(*Mv_rho_inv_pc_); Mv_rho_inv_->SetPrintLevel(mass_inverse_pl_); + Mv_rho_inv_->SetAbsTol(mass_inverse_atol_); Mv_rho_inv_->SetRelTol(mass_inverse_rtol_); Mv_rho_inv_->SetMaxIter(mass_inverse_max_iter_); @@ -896,7 +913,7 @@ void Tomboulides::initializeOperators() { // Helmholtz solver Hv_inv_pc_ = new HypreSmoother(*Hv_op_.As()); - dynamic_cast(Hv_inv_pc_)->SetType(HypreSmoother::Jacobi, smoother_passes_); + dynamic_cast(Hv_inv_pc_)->SetType(smoother_type_, smoother_passes_); dynamic_cast(Hv_inv_pc_)->SetSOROptions(smoother_relax_weight_, smoother_relax_omega_); dynamic_cast(Hv_inv_pc_)->SetPolyOptions(smoother_poly_order_, smoother_poly_fraction_,smoother_eig_est_); @@ -905,6 +922,7 @@ void Tomboulides::initializeOperators() { Hv_inv_->SetOperator(*Hv_op_); Hv_inv_->SetPreconditioner(*Hv_inv_pc_); Hv_inv_->SetPrintLevel(hsolve_pl_); + Hv_inv_->SetAbsTol(hsolve_atol_); Hv_inv_->SetRelTol(hsolve_rtol_); Hv_inv_->SetMaxIter(hsolve_max_iter_); @@ -990,7 +1008,7 @@ void Tomboulides::initializeOperators() { Hs_form_->FormSystemMatrix(swirl_ess_tdof_, Hs_op_); Hs_inv_pc_ = new HypreSmoother(*Hs_op_.As()); - dynamic_cast(Hs_inv_pc_)->SetType(HypreSmoother::Jacobi, smoother_passes_); + dynamic_cast(Hs_inv_pc_)->SetType(smoother_type_, smoother_passes_); dynamic_cast(Hs_inv_pc_)->SetSOROptions(smoother_relax_weight_, smoother_relax_omega_); dynamic_cast(Hs_inv_pc_)->SetPolyOptions(smoother_poly_order_, smoother_poly_fraction_,smoother_eig_est_); @@ -999,6 +1017,7 @@ void Tomboulides::initializeOperators() { Hs_inv_->SetOperator(*Hs_op_); Hs_inv_->SetPreconditioner(*Hs_inv_pc_); Hs_inv_->SetPrintLevel(hsolve_pl_); + Hs_inv_->SetAbsTol(hsolve_atol_); Hs_inv_->SetRelTol(hsolve_rtol_); Hs_inv_->SetMaxIter(hsolve_max_iter_); @@ -1140,12 +1159,16 @@ void Tomboulides::step() { L_iorho_lor_ = new ParLORDiscretization(*L_iorho_form_, pres_ess_tdof_); L_iorho_inv_pc_ = new HypreBoomerAMG(L_iorho_lor_->GetAssembledMatrix()); - L_iorho_inv_pc_->SetPrintLevel(0); + L_iorho_inv_pc_->SetPrintLevel(pressure_solve_pl_); L_iorho_inv_pc_->SetCoarsening(amg_coarsening_); L_iorho_inv_pc_->SetAggressiveCoarsening(amg_aggresive_); + L_iorho_inv_pc_->SetInterpolation(amg_interpolation_); + L_iorho_inv_pc_->SetStrengthThresh(pressure_strength_thres_); + L_iorho_inv_pc_->SetRelaxType(amg_relax_); L_iorho_inv_pc_->SetMaxLevels(amg_max_levels_); - // L_iorho_inv_pc_->SetCycleType(1); - // L_iorho_inv_pc_->SetRelaxType(2); + L_iorho_inv_pc_->SetMaxIter(amg_max_iters_); + // NOTE: other cycle types (i.e. not V-cycle) do not scale well in parallel, so option removed for now + // L_iorho_inv_pc_->SetCycleType(1); L_iorho_inv_ortho_pc_ = new OrthoSolver(pfes_->GetComm()); L_iorho_inv_ortho_pc_->SetSolver(*L_iorho_inv_pc_); diff --git a/src/tomboulides.hpp b/src/tomboulides.hpp index 545d01182..2c6541dec 100644 --- a/src/tomboulides.hpp +++ b/src/tomboulides.hpp @@ -91,22 +91,69 @@ class Tomboulides final : public FlowBase { int smoother_passes_ = 1; double smoother_relax_weight_ = 0.0; double smoother_relax_omega_ = 0.001; - double amg_coarsening_ = 0.4; + double pressure_strength_thres_ = 0.6; int amg_aggresive_ = 4; int amg_max_levels_ = 100; + int amg_max_iters_ = 0; + int amg_relax_ = 0; // only 0 or 18 now + + // Options: Jacobi, l1Jacobi, l1GS, l1GStr, lumpedJacobi, + // GS, OPFS, Chebyshev, Taubin, FIR + mfem::HypreSmoother::Type smoother_type_ = HypreSmoother::Jacobi; + + /* + 0 : CLJP-coarsening (a parallel coarsening algorithm using independent sets. + 1 : classical Ruge-Stueben coarsening on each processor, no boundary treatment (not recommended!) + 3 : classical Ruge-Stueben coarsening on each processor, followed by a third pass, which adds coarse points on the boundaries + 6 : Falgout coarsening (uses 1 first, followed by CLJP using the interior coarse points generated by 1 as its first independent set) + 7 : CLJP-coarsening (using a fixed random vector, for debugging purposes only) + 8 : PMIS-coarsening (a parallel coarsening algorithm using independent sets, generating lower complexities than CLJP, might also lead to slower convergence) + 9 : PMIS-coarsening (using a fixed random vector, for debugging purposes only) + 10 : HMIS-coarsening (uses one pass Ruge-Stueben on each processor independently, followed by PMIS using the interior C-points generated as its first independent set) + 11 : one-pass Ruge-Stueben coarsening on each processor, no boundary treatment (not recommended!) + 21 : CGC coarsening by M. Griebel, B. Metsch and A. Schweitzer + 22 : CGC-E coarsening by M. Griebel, B. Metsch and A.Schweitzer + */ + int amg_coarsening_ = 8; + + /* + 0 : classical modified interpolation + 1 : LS interpolation (for use with GSMG) + 2 : classical modified interpolation for hyperbolic PDEs + 3 : direct interpolation (with separation of weights) (also for GPU use) + 4 : multipass interpolation + 5 : multipass interpolation (with separation of weights) + 6 : extended+i interpolation (also for GPU use) *** + 7 : extended+i (if no common C neighbor) interpolation + 8 : standard interpolation + 9 : standard interpolation (with separation of weights) + 10 : classical block interpolation (for use with nodal systems version only) + 11 : classical block interpolation (for use with nodal systems version only) with diagonalized diagonal blocks + 12 : FF interpolation + 13 : FF1 interpolation + 14 : extended interpolation (also for GPU use) + 15 : interpolation with adaptive weights (GPU use only) + 16 : extended interpolation in matrix-matrix form + 17 : extended+i interpolation in matrix-matrix form + 18 : extended+e interpolation in matrix-matrix form + */ + int amg_interpolation_ = 14; + // solver tolerance options int pressure_solve_pl_ = 0; int pressure_solve_max_iter_ = 1000; - double pressure_solve_rtol_ = 1e-8; + double pressure_solve_rtol_ = 1e-08; double pressure_solve_atol_ = 1e-18; int mass_inverse_pl_ = 0; int mass_inverse_max_iter_ = 1000; double mass_inverse_rtol_ = 1e-12; + double mass_inverse_atol_ = 1e-12; int hsolve_pl_ = 0; int hsolve_max_iter_ = 1000; double hsolve_rtol_ = 1e-12; + double hsolve_atol_ = 1e-12; // To use "numerical integration", quadrature rule must persist mfem::IntegrationRules gll_rules; From 00bebc201d055632d8be99458510de85bc0778fc Mon Sep 17 00:00:00 2001 From: Sigfried Haering Date: Sat, 8 Jun 2024 16:12:49 -0700 Subject: [PATCH 05/27] enforce style --- src/M2ulPhyS2Boltzmann.cpp | 16 +++++++------- src/calorically_perfect.cpp | 13 ++++++----- src/calorically_perfect.hpp | 2 +- src/chemistry.cpp | 2 +- src/chemistry.hpp | 2 +- src/gpu_constructor.cpp | 4 ++-- src/gpu_constructor.hpp | 4 ++-- src/reaction.cpp | 4 ++-- src/reaction.hpp | 4 ++-- src/tomboulides.cpp | 44 ++++++++++++++++++++----------------- src/tomboulides.hpp | 26 +++++++++++----------- src/tps.cpp | 3 +-- src/tps2Boltzmann.cpp | 2 -- 13 files changed, 65 insertions(+), 61 deletions(-) diff --git a/src/M2ulPhyS2Boltzmann.cpp b/src/M2ulPhyS2Boltzmann.cpp index a8aba27b6..8bdf6a5cc 100644 --- a/src/M2ulPhyS2Boltzmann.cpp +++ b/src/M2ulPhyS2Boltzmann.cpp @@ -90,12 +90,12 @@ void M2ulPhyS::fetch(TPS::Tps2Boltzmann &interface) { mfem::ParFiniteElementSpace *reaction_rates_fes(&(interface.NativeFes(TPS::Tps2Boltzmann::Index::ReactionRates))); externalReactionRates.reset(new mfem::ParGridFunction(reaction_rates_fes)); interface.interpolateToNativeFES(*externalReactionRates, TPS::Tps2Boltzmann::Index::ReactionRates); - #if defined(_CUDA_) || defined(_HIP_) - const double * data(externalReactionRates->Read() ); - int size(externalReactionRates->FESpace()->GetNDofs() ); - assert(externalReactionRates->FESpace()->GetOrdering() == mfem::Ordering::byNODES); - gpu::deviceSetChemistryReactionData<<<1, 1>>>(data, size, chemistry_); - #else - chemistry_->setGridFunctionRates(*externalReactionRates); - #endif +#if defined(_CUDA_) || defined(_HIP_) + const double *data(externalReactionRates->Read()); + int size(externalReactionRates->FESpace()->GetNDofs()); + assert(externalReactionRates->FESpace()->GetOrdering() == mfem::Ordering::byNODES); + gpu::deviceSetChemistryReactionData<<<1, 1>>>(data, size, chemistry_); +#else + chemistry_->setGridFunctionRates(*externalReactionRates); +#endif } diff --git a/src/calorically_perfect.cpp b/src/calorically_perfect.cpp index a13ea5f92..c053237da 100644 --- a/src/calorically_perfect.cpp +++ b/src/calorically_perfect.cpp @@ -402,7 +402,7 @@ void CaloricallyPerfectThermoChem::initializeOperators() { // polynomial order for smoother of precons smoother_poly_order_ = order_ + 1; - + Array empty; // GLL integration rule (Numerical Integration) @@ -492,7 +492,8 @@ void CaloricallyPerfectThermoChem::initializeOperators() { MsInvPC_ = new HypreSmoother(*Ms_.As()); dynamic_cast(MsInvPC_)->SetType(HypreSmoother::Jacobi, smoother_passes_); dynamic_cast(MsInvPC_)->SetSOROptions(smoother_relax_weight_, smoother_relax_omega_); - dynamic_cast(MsInvPC_)->SetPolyOptions(smoother_poly_order_, smoother_poly_fraction_,smoother_eig_est_); + dynamic_cast(MsInvPC_)->SetPolyOptions(smoother_poly_order_, smoother_poly_fraction_, + smoother_eig_est_); } MsInv_ = new CGSolver(sfes_->GetComm()); MsInv_->iterative_mode = false; @@ -505,8 +506,9 @@ void CaloricallyPerfectThermoChem::initializeOperators() { HtInvPC_ = new HypreSmoother(*Ht_.As()); dynamic_cast(HtInvPC_)->SetType(HypreSmoother::Jacobi, smoother_passes_); dynamic_cast(HtInvPC_)->SetSOROptions(smoother_relax_weight_, smoother_relax_omega_); - dynamic_cast(HtInvPC_)->SetPolyOptions(smoother_poly_order_, smoother_poly_fraction_,smoother_eig_est_); - + dynamic_cast(HtInvPC_)->SetPolyOptions(smoother_poly_order_, smoother_poly_fraction_, + smoother_eig_est_); + HtInv_ = new CGSolver(sfes_->GetComm()); HtInv_->iterative_mode = true; HtInv_->SetOperator(*Ht_); @@ -539,7 +541,8 @@ void CaloricallyPerfectThermoChem::initializeOperators() { MqInvPC_ = new HypreSmoother(*Mq_.As()); dynamic_cast(MqInvPC_)->SetType(HypreSmoother::Jacobi, smoother_passes_); dynamic_cast(MqInvPC_)->SetSOROptions(smoother_relax_weight_, smoother_relax_omega_); - dynamic_cast(MqInvPC_)->SetPolyOptions(smoother_poly_order_, smoother_poly_fraction_,smoother_eig_est_); + dynamic_cast(MqInvPC_)->SetPolyOptions(smoother_poly_order_, smoother_poly_fraction_, + smoother_eig_est_); } MqInv_ = new CGSolver(sfes_->GetComm()); MqInv_->iterative_mode = false; diff --git a/src/calorically_perfect.hpp b/src/calorically_perfect.hpp index d515b9364..8cc258afb 100644 --- a/src/calorically_perfect.hpp +++ b/src/calorically_perfect.hpp @@ -91,7 +91,7 @@ class CaloricallyPerfectThermoChem : public ThermoChemModelBase { int smoother_passes_ = 1; double smoother_relax_weight_ = 0.0; double smoother_relax_omega_ = 0.001; - + // Boundary condition info Array temp_ess_attr_; /**< List of patches with Dirichlet BC on temperature */ Array Qt_ess_attr_; /**< List of patches with Dirichlet BC on Q (thermal divergence) */ diff --git a/src/chemistry.cpp b/src/chemistry.cpp index fed7512d3..5960b463e 100644 --- a/src/chemistry.cpp +++ b/src/chemistry.cpp @@ -109,7 +109,7 @@ MFEM_HOST_DEVICE Chemistry::~Chemistry() { } } -MFEM_HOST_DEVICE void Chemistry::setRates(const double * data, int size) { +MFEM_HOST_DEVICE void Chemistry::setRates(const double *data, int size) { for (int r = 0; r < numReactions_; r++) { if (reactions_[r]->reactionModel == GRIDFUNCTION_RXN) { GridFunctionReaction *rx = (GridFunctionReaction *)(reactions_[r]); diff --git a/src/chemistry.hpp b/src/chemistry.hpp index 906d249af..79af5a885 100644 --- a/src/chemistry.hpp +++ b/src/chemistry.hpp @@ -97,7 +97,7 @@ class Chemistry { // Set the grid function rates for GRIDFUNCTION_RXN reaction types void setGridFunctionRates(mfem::GridFunction &f); - MFEM_HOST_DEVICE void setRates(const double * data, int size); + MFEM_HOST_DEVICE void setRates(const double *data, int size); // return Vector of reaction rate coefficients, with the size of numReaction_. // WARNING(marc) I have removed "virtual" qualifier here assuming these functions will not diff --git a/src/gpu_constructor.cpp b/src/gpu_constructor.cpp index bf58b88a4..62c5d6f1b 100644 --- a/src/gpu_constructor.cpp +++ b/src/gpu_constructor.cpp @@ -124,11 +124,11 @@ __global__ void freeDeviceRadiation(Radiation *radiation) { //--------------------------------------------------- // And finally device setters //--------------------------------------------------- -__global__ void deviceSetGridFunctionReactionData(const double * data, int size, GridFunctionReaction * reaction) { +__global__ void deviceSetGridFunctionReactionData(const double *data, int size, GridFunctionReaction *reaction) { reaction->setData(data, size); } -__global__ void deviceSetChemistryReactionData(const double * data, int size, Chemistry * chem) { +__global__ void deviceSetChemistryReactionData(const double *data, int size, Chemistry *chem) { chem->setRates(data, size); } diff --git a/src/gpu_constructor.hpp b/src/gpu_constructor.hpp index 64e98d34d..18d516dd2 100644 --- a/src/gpu_constructor.hpp +++ b/src/gpu_constructor.hpp @@ -161,8 +161,8 @@ __global__ void freeDeviceChemistry(Chemistry *chem); __global__ void freeDeviceRadiation(Radiation *radiation); //! Set the data to a GridFunctionReaction -__global__ void deviceSetGridFunctionReactionData(const double * data, int size, GridFunctionReaction * reaction); -__global__ void deviceSetChemistryReactionData(const double * data, int size, Chemistry * chem); +__global__ void deviceSetGridFunctionReactionData(const double *data, int size, GridFunctionReaction *reaction); +__global__ void deviceSetChemistryReactionData(const double *data, int size, Chemistry *chem); #endif // cuda or hip } // namespace gpu diff --git a/src/reaction.cpp b/src/reaction.cpp index 18efb6072..54a02e478 100644 --- a/src/reaction.cpp +++ b/src/reaction.cpp @@ -84,12 +84,12 @@ MFEM_HOST_DEVICE GridFunctionReaction::GridFunctionReaction(int comp) MFEM_HOST_DEVICE GridFunctionReaction::~GridFunctionReaction() {} -MFEM_HOST_DEVICE void GridFunctionReaction::setData(const double * data, int size) { +MFEM_HOST_DEVICE void GridFunctionReaction::setData(const double *data, int size) { data_ = data + comp_ * size_; size_ = size; } -void GridFunctionReaction::setGridFunction(const mfem::GridFunction & f) { +void GridFunctionReaction::setGridFunction(const mfem::GridFunction &f) { size_ = f.FESpace()->GetNDofs(); assert(comp_ < f.FESpace()->GetVDim()); assert(f.FESpace()->GetOrdering() == mfem::Ordering::byNODES); diff --git a/src/reaction.hpp b/src/reaction.hpp index c563e0dd8..078c26faf 100644 --- a/src/reaction.hpp +++ b/src/reaction.hpp @@ -129,9 +129,9 @@ class GridFunctionReaction : public Reaction { MFEM_HOST_DEVICE virtual ~GridFunctionReaction(); - void setGridFunction(const mfem::GridFunction & f); + void setGridFunction(const mfem::GridFunction &f); - MFEM_HOST_DEVICE void setData(const double * data, int size); + MFEM_HOST_DEVICE void setData(const double *data, int size); MFEM_HOST_DEVICE virtual double computeRateCoefficient([[maybe_unused]] const double &T_h, [[maybe_unused]] const double &T_e, const int &dofindex, diff --git a/src/tomboulides.cpp b/src/tomboulides.cpp index d46e6a3d9..2a55a8783 100644 --- a/src/tomboulides.cpp +++ b/src/tomboulides.cpp @@ -120,14 +120,14 @@ Tomboulides::Tomboulides(mfem::ParMesh *pmesh, int vorder, int porder, temporalS tps->getInput("loMach/tomboulides/psolve_atol_", pressure_solve_atol_, 1.0e-18); tps->getInput("loMach/tomboulides/hsolve_atol_", hsolve_atol_, 1.0e-12); tps->getInput("loMach/tomboulides/msolve_atol_", mass_inverse_atol_, 1.0e-12); - + tps->getInput("loMach/tomboulides/psolve_rtol_", pressure_solve_rtol_, 1.0e-08); tps->getInput("loMach/tomboulides/hsolve_rtol_", hsolve_rtol_, 1.0e-12); tps->getInput("loMach/tomboulides/msolve_rtol_", mass_inverse_rtol_, 1.0e-12); - + tps->getInput("loMach/tomboulides/psolve_maxIters_", pressure_solve_max_iter_, 1000); tps->getInput("loMach/tomboulides/hsolve_maxIters_", hsolve_max_iter_, 1000); - tps->getInput("loMach/tomboulides/msolve_maxIters_", mass_inverse_max_iter_, 1000); + tps->getInput("loMach/tomboulides/msolve_maxIters_", mass_inverse_max_iter_, 1000); } } @@ -578,7 +578,7 @@ void Tomboulides::initializeOperators() { // polynomial order for smoother of precons smoother_poly_order_ = vorder_ + 1; - // AMG strength threshold + // AMG strength threshold if (dim_ == 2) { pressure_strength_thres_ = 0.25; } else { @@ -810,28 +810,30 @@ void Tomboulides::initializeOperators() { Mv_inv_pc_ = new HypreSmoother(*Mv_op_.As()); dynamic_cast(Mv_inv_pc_)->SetType(smoother_type_, smoother_passes_); dynamic_cast(Mv_inv_pc_)->SetSOROptions(smoother_relax_weight_, smoother_relax_omega_); - dynamic_cast(Mv_inv_pc_)->SetPolyOptions(smoother_poly_order_, smoother_poly_fraction_,smoother_eig_est_); + dynamic_cast(Mv_inv_pc_) + ->SetPolyOptions(smoother_poly_order_, smoother_poly_fraction_, smoother_eig_est_); } Mv_inv_ = new CGSolver(vfes_->GetComm()); Mv_inv_->iterative_mode = false; Mv_inv_->SetOperator(*Mv_op_); Mv_inv_->SetPreconditioner(*Mv_inv_pc_); Mv_inv_->SetPrintLevel(mass_inverse_pl_); - Mv_inv_->SetAbsTol(mass_inverse_atol_); + Mv_inv_->SetAbsTol(mass_inverse_atol_); Mv_inv_->SetRelTol(mass_inverse_rtol_); Mv_inv_->SetMaxIter(mass_inverse_max_iter_); Mv_rho_inv_pc_ = new HypreSmoother(*Mv_rho_op_.As()); dynamic_cast(Mv_rho_inv_pc_)->SetType(smoother_type_, smoother_passes_); dynamic_cast(Mv_rho_inv_pc_)->SetSOROptions(smoother_relax_weight_, smoother_relax_omega_); - dynamic_cast(Mv_rho_inv_pc_)->SetPolyOptions(smoother_poly_order_, smoother_poly_fraction_,smoother_eig_est_); - + dynamic_cast(Mv_rho_inv_pc_) + ->SetPolyOptions(smoother_poly_order_, smoother_poly_fraction_, smoother_eig_est_); + Mv_rho_inv_ = new CGSolver(vfes_->GetComm()); Mv_rho_inv_->iterative_mode = false; Mv_rho_inv_->SetOperator(*Mv_rho_op_); Mv_rho_inv_->SetPreconditioner(*Mv_rho_inv_pc_); Mv_rho_inv_->SetPrintLevel(mass_inverse_pl_); - Mv_rho_inv_->SetAbsTol(mass_inverse_atol_); + Mv_rho_inv_->SetAbsTol(mass_inverse_atol_); Mv_rho_inv_->SetRelTol(mass_inverse_rtol_); Mv_rho_inv_->SetMaxIter(mass_inverse_max_iter_); @@ -897,17 +899,18 @@ void Tomboulides::initializeOperators() { Hv_form_->FormSystemMatrix(vel_ess_tdof_, Hv_op_); // Helmholtz solver - Hv_inv_pc_ = new HypreSmoother(*Hv_op_.As()); + Hv_inv_pc_ = new HypreSmoother(*Hv_op_.As()); dynamic_cast(Hv_inv_pc_)->SetType(smoother_type_, smoother_passes_); dynamic_cast(Hv_inv_pc_)->SetSOROptions(smoother_relax_weight_, smoother_relax_omega_); - dynamic_cast(Hv_inv_pc_)->SetPolyOptions(smoother_poly_order_, smoother_poly_fraction_,smoother_eig_est_); + dynamic_cast(Hv_inv_pc_) + ->SetPolyOptions(smoother_poly_order_, smoother_poly_fraction_, smoother_eig_est_); Hv_inv_ = new CGSolver(vfes_->GetComm()); Hv_inv_->iterative_mode = true; Hv_inv_->SetOperator(*Hv_op_); Hv_inv_->SetPreconditioner(*Hv_inv_pc_); Hv_inv_->SetPrintLevel(hsolve_pl_); - Hv_inv_->SetAbsTol(hsolve_atol_); + Hv_inv_->SetAbsTol(hsolve_atol_); Hv_inv_->SetRelTol(hsolve_rtol_); Hv_inv_->SetMaxIter(hsolve_max_iter_); @@ -993,16 +996,17 @@ void Tomboulides::initializeOperators() { Hs_form_->FormSystemMatrix(swirl_ess_tdof_, Hs_op_); Hs_inv_pc_ = new HypreSmoother(*Hs_op_.As()); - dynamic_cast(Hs_inv_pc_)->SetType(smoother_type_, smoother_passes_); - dynamic_cast(Hs_inv_pc_)->SetSOROptions(smoother_relax_weight_, smoother_relax_omega_); - dynamic_cast(Hs_inv_pc_)->SetPolyOptions(smoother_poly_order_, smoother_poly_fraction_,smoother_eig_est_); + dynamic_cast(Hs_inv_pc_)->SetType(smoother_type_, smoother_passes_); + dynamic_cast(Hs_inv_pc_)->SetSOROptions(smoother_relax_weight_, smoother_relax_omega_); + dynamic_cast(Hs_inv_pc_) + ->SetPolyOptions(smoother_poly_order_, smoother_poly_fraction_, smoother_eig_est_); Hs_inv_ = new CGSolver(pfes_->GetComm()); Hs_inv_->iterative_mode = true; Hs_inv_->SetOperator(*Hs_op_); Hs_inv_->SetPreconditioner(*Hs_inv_pc_); Hs_inv_->SetPrintLevel(hsolve_pl_); - Hs_inv_->SetAbsTol(hsolve_atol_); + Hs_inv_->SetAbsTol(hsolve_atol_); Hs_inv_->SetRelTol(hsolve_rtol_); Hs_inv_->SetMaxIter(hsolve_max_iter_); @@ -1150,10 +1154,10 @@ void Tomboulides::step() { L_iorho_inv_pc_->SetInterpolation(amg_interpolation_); L_iorho_inv_pc_->SetStrengthThresh(pressure_strength_thres_); L_iorho_inv_pc_->SetRelaxType(amg_relax_); - L_iorho_inv_pc_->SetMaxLevels(amg_max_levels_); - L_iorho_inv_pc_->SetMaxIter(amg_max_iters_); + L_iorho_inv_pc_->SetMaxLevels(amg_max_levels_); + L_iorho_inv_pc_->SetMaxIter(amg_max_iters_); // NOTE: other cycle types (i.e. not V-cycle) do not scale well in parallel, so option removed for now - // L_iorho_inv_pc_->SetCycleType(1); + // L_iorho_inv_pc_->SetCycleType(1); L_iorho_inv_ortho_pc_ = new OrthoSolver(pfes_->GetComm()); L_iorho_inv_ortho_pc_->SetSolver(*L_iorho_inv_pc_); @@ -1187,7 +1191,7 @@ void Tomboulides::step() { Hv_form_->FormSystemMatrix(vel_ess_tdof_, Hv_op_); Hv_inv_->SetOperator(*Hv_op_); - + sw_helm_.Stop(); //------------------------------------------------------------------------ diff --git a/src/tomboulides.hpp b/src/tomboulides.hpp index 3d3d33cc3..efcc19fec 100644 --- a/src/tomboulides.hpp +++ b/src/tomboulides.hpp @@ -95,7 +95,7 @@ class Tomboulides final : public FlowBase { int amg_aggresive_ = 4; int amg_max_levels_ = 100; int amg_max_iters_ = 0; - int amg_relax_ = 0; // only 0 or 18 now + int amg_relax_ = 0; // only 0 or 18 now // Options: Jacobi, l1Jacobi, l1GS, l1GStr, lumpedJacobi, // GS, OPFS, Chebyshev, Taubin, FIR @@ -104,15 +104,15 @@ class Tomboulides final : public FlowBase { /* 0 : CLJP-coarsening (a parallel coarsening algorithm using independent sets. 1 : classical Ruge-Stueben coarsening on each processor, no boundary treatment (not recommended!) - 3 : classical Ruge-Stueben coarsening on each processor, followed by a third pass, which adds coarse points on the boundaries - 6 : Falgout coarsening (uses 1 first, followed by CLJP using the interior coarse points generated by 1 as its first independent set) - 7 : CLJP-coarsening (using a fixed random vector, for debugging purposes only) - 8 : PMIS-coarsening (a parallel coarsening algorithm using independent sets, generating lower complexities than CLJP, might also lead to slower convergence) - 9 : PMIS-coarsening (using a fixed random vector, for debugging purposes only) - 10 : HMIS-coarsening (uses one pass Ruge-Stueben on each processor independently, followed by PMIS using the interior C-points generated as its first independent set) - 11 : one-pass Ruge-Stueben coarsening on each processor, no boundary treatment (not recommended!) - 21 : CGC coarsening by M. Griebel, B. Metsch and A. Schweitzer - 22 : CGC-E coarsening by M. Griebel, B. Metsch and A.Schweitzer + 3 : classical Ruge-Stueben coarsening on each processor, followed by a third pass, which adds coarse points on the + boundaries 6 : Falgout coarsening (uses 1 first, followed by CLJP using the interior coarse points generated by 1 as + its first independent set) 7 : CLJP-coarsening (using a fixed random vector, for debugging purposes only) 8 : + PMIS-coarsening (a parallel coarsening algorithm using independent sets, generating lower complexities than CLJP, + might also lead to slower convergence) 9 : PMIS-coarsening (using a fixed random vector, for debugging purposes only) + 10 : HMIS-coarsening (uses one pass Ruge-Stueben on each processor independently, followed by PMIS using the interior + C-points generated as its first independent set) 11 : one-pass Ruge-Stueben coarsening on each processor, no boundary + treatment (not recommended!) 21 : CGC coarsening by M. Griebel, B. Metsch and A. Schweitzer 22 : CGC-E coarsening by + M. Griebel, B. Metsch and A.Schweitzer */ int amg_coarsening_ = 8; @@ -137,8 +137,8 @@ class Tomboulides final : public FlowBase { 17 : extended+i interpolation in matrix-matrix form 18 : extended+e interpolation in matrix-matrix form */ - int amg_interpolation_ = 14; - + int amg_interpolation_ = 14; + // solver tolerance options int pressure_solve_pl_ = 0; int pressure_solve_max_iter_ = 1000; @@ -153,7 +153,7 @@ class Tomboulides final : public FlowBase { int hsolve_pl_ = 0; int hsolve_max_iter_ = 1000; double hsolve_rtol_ = 1e-12; - double hsolve_atol_ = 1e-12; + double hsolve_atol_ = 1e-12; // To use "numerical integration", quadrature rule must persist mfem::IntegrationRules gll_rules; diff --git a/src/tps.cpp b/src/tps.cpp index b82b7caf3..7e0cd6c39 100644 --- a/src/tps.cpp +++ b/src/tps.cpp @@ -140,8 +140,7 @@ void Tps::parseCommandLineArgs(int argc, char *argv[]) { args.AddOption(&visualMode, "-visual", "--visualization", "", "--no-visualization", "Launch post-process visualization."); gpu_aware_mpi_ = false; - args.AddOption(&gpu_aware_mpi_, "-ga", "--gpu-aware-mpi", "", "--no-gpu-aware-mpi", - "Set GPU-aware MPI."); + args.AddOption(&gpu_aware_mpi_, "-ga", "--gpu-aware-mpi", "", "--no-gpu-aware-mpi", "Set GPU-aware MPI."); args.Parse(); if (!args.Good()) { diff --git a/src/tps2Boltzmann.cpp b/src/tps2Boltzmann.cpp index 72f9b0678..689fded06 100644 --- a/src/tps2Boltzmann.cpp +++ b/src/tps2Boltzmann.cpp @@ -94,12 +94,10 @@ Tps2Boltzmann::Tps2Boltzmann(Tps *tps) save_to_paraview_dc = tps->getInput("boltzmannInterface/save_to_paraview", false); - offsets.SetSize(NIndexes + 1); ncomps.SetSize(NIndexes + 1); } - int Tps2Boltzmann::_countBTEReactions() { int total_reactions(0); int bte_reactions(0); From 2139a06766b8f6e3f7e9024462b72b90f4a8d969 Mon Sep 17 00:00:00 2001 From: Sigfried Haering Date: Sun, 9 Jun 2024 08:19:38 -0700 Subject: [PATCH 06/27] changing default tolerances back to previous values --- src/tomboulides.cpp | 6 +++--- src/tomboulides.hpp | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/tomboulides.cpp b/src/tomboulides.cpp index 2a55a8783..03cfcf62a 100644 --- a/src/tomboulides.cpp +++ b/src/tomboulides.cpp @@ -117,13 +117,13 @@ Tomboulides::Tomboulides(mfem::ParMesh *pmesh, int vorder, int porder, temporalS if (axisym_) assert(!numerical_integ_); // exposing solver tolerance options to user - tps->getInput("loMach/tomboulides/psolve_atol_", pressure_solve_atol_, 1.0e-18); + tps->getInput("loMach/tomboulides/psolve_atol_", pressure_solve_atol_, 1.0e-12); tps->getInput("loMach/tomboulides/hsolve_atol_", hsolve_atol_, 1.0e-12); tps->getInput("loMach/tomboulides/msolve_atol_", mass_inverse_atol_, 1.0e-12); tps->getInput("loMach/tomboulides/psolve_rtol_", pressure_solve_rtol_, 1.0e-08); - tps->getInput("loMach/tomboulides/hsolve_rtol_", hsolve_rtol_, 1.0e-12); - tps->getInput("loMach/tomboulides/msolve_rtol_", mass_inverse_rtol_, 1.0e-12); + tps->getInput("loMach/tomboulides/hsolve_rtol_", hsolve_rtol_, 1.0e-08); + tps->getInput("loMach/tomboulides/msolve_rtol_", mass_inverse_rtol_, 1.0e-08); tps->getInput("loMach/tomboulides/psolve_maxIters_", pressure_solve_max_iter_, 1000); tps->getInput("loMach/tomboulides/hsolve_maxIters_", hsolve_max_iter_, 1000); diff --git a/src/tomboulides.hpp b/src/tomboulides.hpp index efcc19fec..f00eacbba 100644 --- a/src/tomboulides.hpp +++ b/src/tomboulides.hpp @@ -147,12 +147,12 @@ class Tomboulides final : public FlowBase { int mass_inverse_pl_ = 0; int mass_inverse_max_iter_ = 1000; - double mass_inverse_rtol_ = 1e-12; + double mass_inverse_rtol_ = 1e-08; double mass_inverse_atol_ = 1e-12; int hsolve_pl_ = 0; int hsolve_max_iter_ = 1000; - double hsolve_rtol_ = 1e-12; + double hsolve_rtol_ = 1e-08; double hsolve_atol_ = 1e-12; // To use "numerical integration", quadrature rule must persist From b77e854afc00738e58e272b96f64ebc44e358dd2 Mon Sep 17 00:00:00 2001 From: Sigfried Haering Date: Mon, 10 Jun 2024 17:49:01 -0700 Subject: [PATCH 07/27] additional tolerance controls for cal-perf and updated ref soln to heatedBox --- src/calorically_perfect.cpp | 34 ++++++++++++++----- src/calorically_perfect.hpp | 18 ++++++++-- test/inputs/input.heatedBox.ini | 3 -- .../ref_solns/heatedBox/restart_output.sol.h5 | 4 +-- 4 files changed, 42 insertions(+), 17 deletions(-) diff --git a/src/calorically_perfect.cpp b/src/calorically_perfect.cpp index c053237da..e841c9549 100644 --- a/src/calorically_perfect.cpp +++ b/src/calorically_perfect.cpp @@ -118,7 +118,20 @@ CaloricallyPerfectThermoChem::CaloricallyPerfectThermoChem(mfem::ParMesh *pmesh, tpsP_->getInput("loMach/calperfect/linear-solver-rtol", rtol_, 1e-12); tpsP_->getInput("loMach/calperfect/linear-solver-max-iter", max_iter_, 1000); tpsP_->getInput("loMach/calperfect/linear-solver-verbosity", pl_solve_, 0); + + // NOTE: this should default to FALSE (but would break all related test cases...) tpsP_->getInput("loMach/calperfect/numerical-integ", numerical_integ_, true); + + // not deleting above block to maintain backwards-compatability + tpsP_->getInput("loMach/calperfect/hsolve-rtol", hsolve_rtol_, rtol_); + tpsP_->getInput("loMach/calperfect/hsolve-atol", hsolve_atol_, 1.0e-12); + tpsP_->getInput("loMach/calperfect/hsolve-max-iter", hsolve_max_iter_, max_iter_); + tpsP_->getInput("loMach/calperfect/hsolve-verbosity", hsolve_pl_, pl_solve_); + + tpsP_->getInput("loMach/calperfect/msolve-rtol", mass_inverse_rtol_, rtol_); + tpsP_->getInput("loMach/calperfect/msolve-atol", mass_inverse_atol_, 1.0e-12); + tpsP_->getInput("loMach/calperfect/msolve-max-iter", mass_inverse_max_iter_, max_iter_); + tpsP_->getInput("loMach/calperfect/msolve-verbosity", mass_inverse_pl_, pl_solve_); } CaloricallyPerfectThermoChem::~CaloricallyPerfectThermoChem() { @@ -499,9 +512,10 @@ void CaloricallyPerfectThermoChem::initializeOperators() { MsInv_->iterative_mode = false; MsInv_->SetOperator(*Ms_); MsInv_->SetPreconditioner(*MsInvPC_); - MsInv_->SetPrintLevel(pl_solve_); - MsInv_->SetRelTol(rtol_); - MsInv_->SetMaxIter(max_iter_); + MsInv_->SetPrintLevel(mass_inverse_pl_); + MsInv_->SetRelTol(mass_inverse_rtol_); + MsInv_->SetAbsTol(mass_inverse_atol_); + MsInv_->SetMaxIter(mass_inverse_max_iter_); HtInvPC_ = new HypreSmoother(*Ht_.As()); dynamic_cast(HtInvPC_)->SetType(HypreSmoother::Jacobi, smoother_passes_); @@ -513,9 +527,10 @@ void CaloricallyPerfectThermoChem::initializeOperators() { HtInv_->iterative_mode = true; HtInv_->SetOperator(*Ht_); HtInv_->SetPreconditioner(*HtInvPC_); - HtInv_->SetPrintLevel(pl_solve_); - HtInv_->SetRelTol(rtol_); - HtInv_->SetMaxIter(max_iter_); + HtInv_->SetPrintLevel(hsolve_pl_); + HtInv_->SetRelTol(hsolve_rtol_); + HtInv_->SetAbsTol(hsolve_atol_); + HtInv_->SetMaxIter(hsolve_max_iter_); if (rank0_) std::cout << "Temperature operators set" << endl; // Qt ..................................... @@ -548,9 +563,10 @@ void CaloricallyPerfectThermoChem::initializeOperators() { MqInv_->iterative_mode = false; MqInv_->SetOperator(*Mq_); MqInv_->SetPreconditioner(*MqInvPC_); - MqInv_->SetPrintLevel(pl_solve_); - MqInv_->SetRelTol(rtol_); - MqInv_->SetMaxIter(max_iter_); + MqInv_->SetPrintLevel(mass_inverse_pl_); + MqInv_->SetRelTol(mass_inverse_rtol_); + MqInv_->SetAbsTol(mass_inverse_atol_); + MqInv_->SetMaxIter(mass_inverse_max_iter_); LQ_form_ = new ParBilinearForm(sfes_); auto *lqd_blfi = new DiffusionIntegrator(*thermal_diff_total_coeff_); diff --git a/src/calorically_perfect.hpp b/src/calorically_perfect.hpp index 8cc258afb..c956796af 100644 --- a/src/calorically_perfect.hpp +++ b/src/calorically_perfect.hpp @@ -82,9 +82,6 @@ class CaloricallyPerfectThermoChem : public ThermoChemModelBase { bool domain_is_open_ = false; /**< true if domain is open */ // Linear-solver-related options - int pl_solve_ = 0; /**< Verbosity level passed to mfem solvers */ - int max_iter_; /**< Maximum number of linear solver iterations */ - double rtol_ = 1e-12; /**< Linear solver relative tolerance */ int smoother_poly_order_; double smoother_poly_fraction_ = 0.1; int smoother_eig_est_ = 10; @@ -92,6 +89,21 @@ class CaloricallyPerfectThermoChem : public ThermoChemModelBase { double smoother_relax_weight_ = 0.0; double smoother_relax_omega_ = 0.001; + // solver tolerance options + int pl_solve_; /**< Verbosity level passed to mfem solvers */ + int max_iter_; /**< Maximum number of linear solver iterations */ + double rtol_; /**< Linear solver relative tolerance */ + + int mass_inverse_pl_ = 0; + int mass_inverse_max_iter_ = 1000; + double mass_inverse_rtol_ = 1e-12; + double mass_inverse_atol_ = 1e-12; + + int hsolve_pl_ = 0; + int hsolve_max_iter_ = 1000; + double hsolve_rtol_ = 1e-12; + double hsolve_atol_ = 1e-12; + // Boundary condition info Array temp_ess_attr_; /**< List of patches with Dirichlet BC on temperature */ Array Qt_ess_attr_; /**< List of patches with Dirichlet BC on Q (thermal divergence) */ diff --git a/test/inputs/input.heatedBox.ini b/test/inputs/input.heatedBox.ini index c3b68817a..c603db677 100644 --- a/test/inputs/input.heatedBox.ini +++ b/test/inputs/input.heatedBox.ini @@ -4,9 +4,6 @@ type = loMach [loMach] mesh = meshes/lequereBoxCrs3wall.msh order = 1 -uOrder = 1 -pOrder = 1 -nOrder = 1 nFilter = 0 filterWeight = 1.0 maxIters = 5000 diff --git a/test/ref_solns/heatedBox/restart_output.sol.h5 b/test/ref_solns/heatedBox/restart_output.sol.h5 index dc428085f..27e02919a 100644 --- a/test/ref_solns/heatedBox/restart_output.sol.h5 +++ b/test/ref_solns/heatedBox/restart_output.sol.h5 @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:a7e7088904222b6e9c29b70a410f82885f389e2c9fb672686df5ff418ea92696 -size 3504 +oid sha256:c52baaa79d08408152e03a0da164102b72440afc811c7ac5d47fe5eb2c5cb663 +size 3568 From f202b72867c623d3b366ebc28b53e72bcb28e0dd Mon Sep 17 00:00:00 2001 From: Sigfried Haering Date: Mon, 10 Jun 2024 17:50:46 -0700 Subject: [PATCH 08/27] enforce style --- src/calorically_perfect.cpp | 14 +++++++------- src/calorically_perfect.hpp | 16 ++++++++-------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/calorically_perfect.cpp b/src/calorically_perfect.cpp index e841c9549..40e2a673d 100644 --- a/src/calorically_perfect.cpp +++ b/src/calorically_perfect.cpp @@ -124,14 +124,14 @@ CaloricallyPerfectThermoChem::CaloricallyPerfectThermoChem(mfem::ParMesh *pmesh, // not deleting above block to maintain backwards-compatability tpsP_->getInput("loMach/calperfect/hsolve-rtol", hsolve_rtol_, rtol_); - tpsP_->getInput("loMach/calperfect/hsolve-atol", hsolve_atol_, 1.0e-12); + tpsP_->getInput("loMach/calperfect/hsolve-atol", hsolve_atol_, 1.0e-12); tpsP_->getInput("loMach/calperfect/hsolve-max-iter", hsolve_max_iter_, max_iter_); tpsP_->getInput("loMach/calperfect/hsolve-verbosity", hsolve_pl_, pl_solve_); - + tpsP_->getInput("loMach/calperfect/msolve-rtol", mass_inverse_rtol_, rtol_); - tpsP_->getInput("loMach/calperfect/msolve-atol", mass_inverse_atol_, 1.0e-12); + tpsP_->getInput("loMach/calperfect/msolve-atol", mass_inverse_atol_, 1.0e-12); tpsP_->getInput("loMach/calperfect/msolve-max-iter", mass_inverse_max_iter_, max_iter_); - tpsP_->getInput("loMach/calperfect/msolve-verbosity", mass_inverse_pl_, pl_solve_); + tpsP_->getInput("loMach/calperfect/msolve-verbosity", mass_inverse_pl_, pl_solve_); } CaloricallyPerfectThermoChem::~CaloricallyPerfectThermoChem() { @@ -514,7 +514,7 @@ void CaloricallyPerfectThermoChem::initializeOperators() { MsInv_->SetPreconditioner(*MsInvPC_); MsInv_->SetPrintLevel(mass_inverse_pl_); MsInv_->SetRelTol(mass_inverse_rtol_); - MsInv_->SetAbsTol(mass_inverse_atol_); + MsInv_->SetAbsTol(mass_inverse_atol_); MsInv_->SetMaxIter(mass_inverse_max_iter_); HtInvPC_ = new HypreSmoother(*Ht_.As()); @@ -529,7 +529,7 @@ void CaloricallyPerfectThermoChem::initializeOperators() { HtInv_->SetPreconditioner(*HtInvPC_); HtInv_->SetPrintLevel(hsolve_pl_); HtInv_->SetRelTol(hsolve_rtol_); - HtInv_->SetAbsTol(hsolve_atol_); + HtInv_->SetAbsTol(hsolve_atol_); HtInv_->SetMaxIter(hsolve_max_iter_); if (rank0_) std::cout << "Temperature operators set" << endl; @@ -565,7 +565,7 @@ void CaloricallyPerfectThermoChem::initializeOperators() { MqInv_->SetPreconditioner(*MqInvPC_); MqInv_->SetPrintLevel(mass_inverse_pl_); MqInv_->SetRelTol(mass_inverse_rtol_); - MqInv_->SetAbsTol(mass_inverse_atol_); + MqInv_->SetAbsTol(mass_inverse_atol_); MqInv_->SetMaxIter(mass_inverse_max_iter_); LQ_form_ = new ParBilinearForm(sfes_); diff --git a/src/calorically_perfect.hpp b/src/calorically_perfect.hpp index c956796af..ec3e6081c 100644 --- a/src/calorically_perfect.hpp +++ b/src/calorically_perfect.hpp @@ -90,19 +90,19 @@ class CaloricallyPerfectThermoChem : public ThermoChemModelBase { double smoother_relax_omega_ = 0.001; // solver tolerance options - int pl_solve_; /**< Verbosity level passed to mfem solvers */ + int pl_solve_; /**< Verbosity level passed to mfem solvers */ int max_iter_; /**< Maximum number of linear solver iterations */ - double rtol_; /**< Linear solver relative tolerance */ - - int mass_inverse_pl_ = 0; - int mass_inverse_max_iter_ = 1000; - double mass_inverse_rtol_ = 1e-12; - double mass_inverse_atol_ = 1e-12; + double rtol_; /**< Linear solver relative tolerance */ + + int mass_inverse_pl_ = 0; + int mass_inverse_max_iter_ = 1000; + double mass_inverse_rtol_ = 1e-12; + double mass_inverse_atol_ = 1e-12; int hsolve_pl_ = 0; int hsolve_max_iter_ = 1000; double hsolve_rtol_ = 1e-12; - double hsolve_atol_ = 1e-12; + double hsolve_atol_ = 1e-12; // Boundary condition info Array temp_ess_attr_; /**< List of patches with Dirichlet BC on temperature */ From 31f8fcad2f2cfbb593dd3b26693d3660b2390de8 Mon Sep 17 00:00:00 2001 From: Sigfried Haering Date: Mon, 10 Jun 2024 18:33:30 -0700 Subject: [PATCH 09/27] fixed issure with SOR weight on helmholtz solve --- src/calorically_perfect.cpp | 6 +++--- src/calorically_perfect.hpp | 3 +++ src/tomboulides.cpp | 4 ++-- src/tomboulides.hpp | 2 ++ test/ref_solns/heatedBox/restart_output.sol.h5 | 2 +- 5 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/calorically_perfect.cpp b/src/calorically_perfect.cpp index 40e2a673d..ca2552b97 100644 --- a/src/calorically_perfect.cpp +++ b/src/calorically_perfect.cpp @@ -506,7 +506,7 @@ void CaloricallyPerfectThermoChem::initializeOperators() { dynamic_cast(MsInvPC_)->SetType(HypreSmoother::Jacobi, smoother_passes_); dynamic_cast(MsInvPC_)->SetSOROptions(smoother_relax_weight_, smoother_relax_omega_); dynamic_cast(MsInvPC_)->SetPolyOptions(smoother_poly_order_, smoother_poly_fraction_, - smoother_eig_est_); + smoother_eig_est_); } MsInv_ = new CGSolver(sfes_->GetComm()); MsInv_->iterative_mode = false; @@ -519,7 +519,7 @@ void CaloricallyPerfectThermoChem::initializeOperators() { HtInvPC_ = new HypreSmoother(*Ht_.As()); dynamic_cast(HtInvPC_)->SetType(HypreSmoother::Jacobi, smoother_passes_); - dynamic_cast(HtInvPC_)->SetSOROptions(smoother_relax_weight_, smoother_relax_omega_); + dynamic_cast(HtInvPC_)->SetSOROptions(hsmoother_relax_weight_, hsmoother_relax_omega_); dynamic_cast(HtInvPC_)->SetPolyOptions(smoother_poly_order_, smoother_poly_fraction_, smoother_eig_est_); @@ -557,7 +557,7 @@ void CaloricallyPerfectThermoChem::initializeOperators() { dynamic_cast(MqInvPC_)->SetType(HypreSmoother::Jacobi, smoother_passes_); dynamic_cast(MqInvPC_)->SetSOROptions(smoother_relax_weight_, smoother_relax_omega_); dynamic_cast(MqInvPC_)->SetPolyOptions(smoother_poly_order_, smoother_poly_fraction_, - smoother_eig_est_); + smoother_eig_est_); } MqInv_ = new CGSolver(sfes_->GetComm()); MqInv_->iterative_mode = false; diff --git a/src/calorically_perfect.hpp b/src/calorically_perfect.hpp index ec3e6081c..f8a8bd4de 100644 --- a/src/calorically_perfect.hpp +++ b/src/calorically_perfect.hpp @@ -88,7 +88,10 @@ class CaloricallyPerfectThermoChem : public ThermoChemModelBase { int smoother_passes_ = 1; double smoother_relax_weight_ = 0.0; double smoother_relax_omega_ = 0.001; + double hsmoother_relax_weight_ = 0.01; + double hsmoother_relax_omega_ = 0.001; + // solver tolerance options int pl_solve_; /**< Verbosity level passed to mfem solvers */ int max_iter_; /**< Maximum number of linear solver iterations */ diff --git a/src/tomboulides.cpp b/src/tomboulides.cpp index 03cfcf62a..6e3ca93cf 100644 --- a/src/tomboulides.cpp +++ b/src/tomboulides.cpp @@ -901,7 +901,7 @@ void Tomboulides::initializeOperators() { // Helmholtz solver Hv_inv_pc_ = new HypreSmoother(*Hv_op_.As()); dynamic_cast(Hv_inv_pc_)->SetType(smoother_type_, smoother_passes_); - dynamic_cast(Hv_inv_pc_)->SetSOROptions(smoother_relax_weight_, smoother_relax_omega_); + dynamic_cast(Hv_inv_pc_)->SetSOROptions(hsmoother_relax_weight_, hsmoother_relax_omega_); dynamic_cast(Hv_inv_pc_) ->SetPolyOptions(smoother_poly_order_, smoother_poly_fraction_, smoother_eig_est_); @@ -997,7 +997,7 @@ void Tomboulides::initializeOperators() { Hs_inv_pc_ = new HypreSmoother(*Hs_op_.As()); dynamic_cast(Hs_inv_pc_)->SetType(smoother_type_, smoother_passes_); - dynamic_cast(Hs_inv_pc_)->SetSOROptions(smoother_relax_weight_, smoother_relax_omega_); + dynamic_cast(Hs_inv_pc_)->SetSOROptions(hsmoother_relax_weight_, hsmoother_relax_omega_); dynamic_cast(Hs_inv_pc_) ->SetPolyOptions(smoother_poly_order_, smoother_poly_fraction_, smoother_eig_est_); diff --git a/src/tomboulides.hpp b/src/tomboulides.hpp index f00eacbba..d2c565723 100644 --- a/src/tomboulides.hpp +++ b/src/tomboulides.hpp @@ -91,6 +91,8 @@ class Tomboulides final : public FlowBase { int smoother_passes_ = 1; double smoother_relax_weight_ = 0.0; double smoother_relax_omega_ = 0.001; + double hsmoother_relax_weight_ = 0.01; + double hsmoother_relax_omega_ = 0.001; double pressure_strength_thres_ = 0.6; int amg_aggresive_ = 4; int amg_max_levels_ = 100; diff --git a/test/ref_solns/heatedBox/restart_output.sol.h5 b/test/ref_solns/heatedBox/restart_output.sol.h5 index 27e02919a..a70ced5fd 100644 --- a/test/ref_solns/heatedBox/restart_output.sol.h5 +++ b/test/ref_solns/heatedBox/restart_output.sol.h5 @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c52baaa79d08408152e03a0da164102b72440afc811c7ac5d47fe5eb2c5cb663 +oid sha256:d81816aaa43491b582aff0937a16c2bd1037340378c59bbecc3203c9ad26e78e size 3568 From eb376c381a443b1a3216fbf4988739f2e607b25d Mon Sep 17 00:00:00 2001 From: Sigfried Haering Date: Mon, 10 Jun 2024 18:34:34 -0700 Subject: [PATCH 10/27] enforce style --- src/calorically_perfect.cpp | 4 ++-- src/calorically_perfect.hpp | 1 - src/tomboulides.hpp | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/calorically_perfect.cpp b/src/calorically_perfect.cpp index ca2552b97..44be20c2d 100644 --- a/src/calorically_perfect.cpp +++ b/src/calorically_perfect.cpp @@ -506,7 +506,7 @@ void CaloricallyPerfectThermoChem::initializeOperators() { dynamic_cast(MsInvPC_)->SetType(HypreSmoother::Jacobi, smoother_passes_); dynamic_cast(MsInvPC_)->SetSOROptions(smoother_relax_weight_, smoother_relax_omega_); dynamic_cast(MsInvPC_)->SetPolyOptions(smoother_poly_order_, smoother_poly_fraction_, - smoother_eig_est_); + smoother_eig_est_); } MsInv_ = new CGSolver(sfes_->GetComm()); MsInv_->iterative_mode = false; @@ -557,7 +557,7 @@ void CaloricallyPerfectThermoChem::initializeOperators() { dynamic_cast(MqInvPC_)->SetType(HypreSmoother::Jacobi, smoother_passes_); dynamic_cast(MqInvPC_)->SetSOROptions(smoother_relax_weight_, smoother_relax_omega_); dynamic_cast(MqInvPC_)->SetPolyOptions(smoother_poly_order_, smoother_poly_fraction_, - smoother_eig_est_); + smoother_eig_est_); } MqInv_ = new CGSolver(sfes_->GetComm()); MqInv_->iterative_mode = false; diff --git a/src/calorically_perfect.hpp b/src/calorically_perfect.hpp index f8a8bd4de..ef5493aab 100644 --- a/src/calorically_perfect.hpp +++ b/src/calorically_perfect.hpp @@ -91,7 +91,6 @@ class CaloricallyPerfectThermoChem : public ThermoChemModelBase { double hsmoother_relax_weight_ = 0.01; double hsmoother_relax_omega_ = 0.001; - // solver tolerance options int pl_solve_; /**< Verbosity level passed to mfem solvers */ int max_iter_; /**< Maximum number of linear solver iterations */ diff --git a/src/tomboulides.hpp b/src/tomboulides.hpp index d2c565723..1a8fadcd0 100644 --- a/src/tomboulides.hpp +++ b/src/tomboulides.hpp @@ -92,7 +92,7 @@ class Tomboulides final : public FlowBase { double smoother_relax_weight_ = 0.0; double smoother_relax_omega_ = 0.001; double hsmoother_relax_weight_ = 0.01; - double hsmoother_relax_omega_ = 0.001; + double hsmoother_relax_omega_ = 0.001; double pressure_strength_thres_ = 0.6; int amg_aggresive_ = 4; int amg_max_levels_ = 100; From af6845fc398fc2a66e7eb43e36a95ff7efa59f21 Mon Sep 17 00:00:00 2001 From: Sigfried Haering Date: Wed, 12 Jun 2024 19:09:28 -0700 Subject: [PATCH 11/27] fix issue with smoother weight on mass inverse --- src/calorically_perfect.hpp | 2 +- src/tomboulides.hpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/calorically_perfect.hpp b/src/calorically_perfect.hpp index ef5493aab..95be425b4 100644 --- a/src/calorically_perfect.hpp +++ b/src/calorically_perfect.hpp @@ -86,7 +86,7 @@ class CaloricallyPerfectThermoChem : public ThermoChemModelBase { double smoother_poly_fraction_ = 0.1; int smoother_eig_est_ = 10; int smoother_passes_ = 1; - double smoother_relax_weight_ = 0.0; + double smoother_relax_weight_ = 0.01; double smoother_relax_omega_ = 0.001; double hsmoother_relax_weight_ = 0.01; double hsmoother_relax_omega_ = 0.001; diff --git a/src/tomboulides.hpp b/src/tomboulides.hpp index 1a8fadcd0..29f1bfe76 100644 --- a/src/tomboulides.hpp +++ b/src/tomboulides.hpp @@ -89,7 +89,7 @@ class Tomboulides final : public FlowBase { double smoother_poly_fraction_ = 0.1; int smoother_eig_est_ = 10; int smoother_passes_ = 1; - double smoother_relax_weight_ = 0.0; + double smoother_relax_weight_ = 0.01; double smoother_relax_omega_ = 0.001; double hsmoother_relax_weight_ = 0.01; double hsmoother_relax_omega_ = 0.001; From 2a03e5299df71e83d92ca03f9de537e83778c4e0 Mon Sep 17 00:00:00 2001 From: Sigfried Haering Date: Wed, 12 Jun 2024 21:24:51 -0700 Subject: [PATCH 12/27] fixes to options according to hypre guidance (HYPRE_BoomerAMGSetMaxIter as a precon) and apparent smoother relax weight not functioning as intended --- src/tomboulides.hpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/tomboulides.hpp b/src/tomboulides.hpp index 29f1bfe76..25f1eabdb 100644 --- a/src/tomboulides.hpp +++ b/src/tomboulides.hpp @@ -86,17 +86,17 @@ class Tomboulides final : public FlowBase { // linear-solver options int smoother_poly_order_; - double smoother_poly_fraction_ = 0.1; + double smoother_poly_fraction_ = 0.75; int smoother_eig_est_ = 10; int smoother_passes_ = 1; - double smoother_relax_weight_ = 0.01; - double smoother_relax_omega_ = 0.001; - double hsmoother_relax_weight_ = 0.01; - double hsmoother_relax_omega_ = 0.001; + double smoother_relax_weight_ = 0.4; + double smoother_relax_omega_ = 0.01; + double hsmoother_relax_weight_ = 0.4; + double hsmoother_relax_omega_ = 0.01; double pressure_strength_thres_ = 0.6; int amg_aggresive_ = 4; int amg_max_levels_ = 100; - int amg_max_iters_ = 0; + int amg_max_iters_ = 1; int amg_relax_ = 0; // only 0 or 18 now // Options: Jacobi, l1Jacobi, l1GS, l1GStr, lumpedJacobi, From 5f6d08854539fff16bce13bc692a9b5c3b2519a0 Mon Sep 17 00:00:00 2001 From: Sigfried Haering Date: Wed, 12 Jun 2024 22:33:24 -0700 Subject: [PATCH 13/27] max iters for amg precon set to 1, updates to input files for two tests and reduced rel tol --- src/calorically_perfect.hpp | 10 ++++---- src/tomboulides.cpp | 1 + src/tomboulides.hpp | 9 ++++--- test/autoPeriodic.test | 2 +- test/inputs/input.autoPeriodic.ini | 25 +++++++++---------- test/inputs/input.sgsLoMach.ini | 23 ++++++++--------- test/interpInlet.test | 2 +- test/lomach-flow.test | 14 +++++------ test/lomach-lequere.test | 2 +- .../ref_solns/sgsLoMach/restart_output.sol.h5 | 2 +- test/sgsLoMach.test | 2 +- 11 files changed, 46 insertions(+), 46 deletions(-) diff --git a/src/calorically_perfect.hpp b/src/calorically_perfect.hpp index 95be425b4..6a5ed581a 100644 --- a/src/calorically_perfect.hpp +++ b/src/calorically_perfect.hpp @@ -83,13 +83,13 @@ class CaloricallyPerfectThermoChem : public ThermoChemModelBase { // Linear-solver-related options int smoother_poly_order_; - double smoother_poly_fraction_ = 0.1; + double smoother_poly_fraction_ = 0.75; int smoother_eig_est_ = 10; int smoother_passes_ = 1; - double smoother_relax_weight_ = 0.01; - double smoother_relax_omega_ = 0.001; - double hsmoother_relax_weight_ = 0.01; - double hsmoother_relax_omega_ = 0.001; + double smoother_relax_weight_ = 0.4; + double smoother_relax_omega_ = 0.01; + double hsmoother_relax_weight_ = 0.4; + double hsmoother_relax_omega_ = 0.01; // solver tolerance options int pl_solve_; /**< Verbosity level passed to mfem solvers */ diff --git a/src/tomboulides.cpp b/src/tomboulides.cpp index 6e3ca93cf..52deac957 100644 --- a/src/tomboulides.cpp +++ b/src/tomboulides.cpp @@ -1156,6 +1156,7 @@ void Tomboulides::step() { L_iorho_inv_pc_->SetRelaxType(amg_relax_); L_iorho_inv_pc_->SetMaxLevels(amg_max_levels_); L_iorho_inv_pc_->SetMaxIter(amg_max_iters_); + L_iorho_inv_pc_->SetTol(0); // NOTE: other cycle types (i.e. not V-cycle) do not scale well in parallel, so option removed for now // L_iorho_inv_pc_->SetCycleType(1); L_iorho_inv_ortho_pc_ = new OrthoSolver(pfes_->GetComm()); diff --git a/src/tomboulides.hpp b/src/tomboulides.hpp index 25f1eabdb..431b28635 100644 --- a/src/tomboulides.hpp +++ b/src/tomboulides.hpp @@ -89,15 +89,16 @@ class Tomboulides final : public FlowBase { double smoother_poly_fraction_ = 0.75; int smoother_eig_est_ = 10; int smoother_passes_ = 1; - double smoother_relax_weight_ = 0.4; + double smoother_relax_weight_ = + 0.4; // NOTE: setting relax wt to 0 should determine wt on a per-level basis but this does not seem to work double smoother_relax_omega_ = 0.01; double hsmoother_relax_weight_ = 0.4; double hsmoother_relax_omega_ = 0.01; double pressure_strength_thres_ = 0.6; int amg_aggresive_ = 4; - int amg_max_levels_ = 100; - int amg_max_iters_ = 1; - int amg_relax_ = 0; // only 0 or 18 now + int amg_max_levels_ = 10; // 100; + int amg_max_iters_ = 1; // setting to zero is ~40% faster per step but effectively turns off precon? + int amg_relax_ = 0; // only 0 or 18 now // Options: Jacobi, l1Jacobi, l1GS, l1GStr, lumpedJacobi, // GS, OPFS, Chebyshev, Taubin, FIR diff --git a/test/autoPeriodic.test b/test/autoPeriodic.test index dc6a9d33e..1fb5cc41a 100755 --- a/test/autoPeriodic.test +++ b/test/autoPeriodic.test @@ -26,5 +26,5 @@ setup() { @test "[$TEST] verify tps output with input -> $RUNFILE" { test -s $SOLN_FILE test -s $REF_FILE - h5diff -r --delta=1e-12 $SOLN_FILE $REF_FILE /velocity + h5diff -r --delta=1e-11 $SOLN_FILE $REF_FILE /velocity } diff --git a/test/inputs/input.autoPeriodic.ini b/test/inputs/input.autoPeriodic.ini index 56fc43205..a462938b6 100644 --- a/test/inputs/input.autoPeriodic.ini +++ b/test/inputs/input.autoPeriodic.ini @@ -3,35 +3,34 @@ type = loMach [loMach] flow-solver = tomboulides +thermo-solver = constant-property mesh = meshes/flatBox.msh -uOrder = 1 -pOrder = 1 -nOrder = 1 +order = 1 maxIters = 10 outputFreq = 10 -fluid = dry_air -refLength = 0.05 -equation_system = navier-stokes enablePressureForcing = False -constantViscosity = 1.552e-5 -constantDensity = 1.1839 enableGravity = False +gravity = '0 0 0' openSystem = True ambientPressure = 101326. sgsModel = smagorinsky sgsModelConstant = 0.09 [loMach/tomboulides] -ic = tgv2d +numerical-integ = false +ic = tgv2d_uniform + +[loMach/constprop] +mu = 1.552e-5 +rho = 1.1839 [io] outdirBase = output [time] -cfl = 0.5 +integrator = curlcurl dt_fixed = 1.0e-3 -maxSolverIteration = 1000 -solverRelTolerance = 1.0e-12 +bdfOrder = 3 [initialConditions] rho = 1.1839 @@ -49,4 +48,4 @@ numOutlets = 0 enablePeriodic = True periodicX = True periodicY = True -periodicZ = True \ No newline at end of file +periodicZ = True diff --git a/test/inputs/input.sgsLoMach.ini b/test/inputs/input.sgsLoMach.ini index a884829c6..ad3e812db 100644 --- a/test/inputs/input.sgsLoMach.ini +++ b/test/inputs/input.sgsLoMach.ini @@ -3,35 +3,34 @@ type = loMach [loMach] flow-solver = tomboulides +thermo-solver = constant-property mesh = meshes/flatBox.msh -uOrder = 1 -pOrder = 1 -nOrder = 1 +order = 1 maxIters = 10 outputFreq = 10 -fluid = dry_air -refLength = 0.05 -equation_system = navier-stokes enablePressureForcing = False -constantViscosity = 1.552e-5 -constantDensity = 1.1839 enableGravity = False +gravity = '0 0 0' openSystem = True ambientPressure = 101326. sgsModel = smagorinsky sgsModelConstant = 0.09 [loMach/tomboulides] -ic = tgv2d +numerical-integ = false +ic = tgv2d_uniform + +[loMach/constprop] +mu = 1.552e-5 +rho = 1.1839 [io] outdirBase = output [time] -cfl = 0.5 +integrator = curlcurl dt_fixed = 1.0e-3 -maxSolverIteration = 1000 -solverRelTolerance = 1.0e-12 +bdfOrder = 3 [initialConditions] rho = 1.1839 diff --git a/test/interpInlet.test b/test/interpInlet.test index e9fc0e30f..92df8c21e 100755 --- a/test/interpInlet.test +++ b/test/interpInlet.test @@ -33,5 +33,5 @@ setup() { @test "[$TEST] verify tps output with input -> $RUNFILE" { test -s $SOLN_FILE test -s $REF_FILE - h5diff -r --relative=1e-12 $SOLN_FILE $REF_FILE /temperature/temperature + h5diff -r --relative=1e-11 $SOLN_FILE $REF_FILE /temperature/temperature } diff --git a/test/lomach-flow.test b/test/lomach-flow.test index a54cfaedd..4caec6700 100755 --- a/test/lomach-flow.test +++ b/test/lomach-flow.test @@ -138,7 +138,7 @@ setup() { $MPIRUN -n 2 $EXE --runFile $RUNFILE_LID_MOD # check solutions are the same! - h5diff -r --delta=1e-12 restart_output-lid.sol.h5 restart_output-lid.sol.2.h5 + h5diff -r --delta=1e-11 restart_output-lid.sol.h5 restart_output-lid.sol.2.h5 # delete intermediate files rm -f restart_output-lid*.h5 @@ -149,22 +149,22 @@ setup() { run $EXE --runFile $RUNFILE_PIPE [[ ${status} -eq 0 ]] test -s restart_output-pipe-lam.sol.h5 - h5diff -r --delta=1e-12 restart_output-pipe-lam.sol.h5 $REF_SOLN_PIPE /velocity - h5diff -r --delta=1e-12 restart_output-pipe-lam.sol.h5 $REF_SOLN_PIPE /swirl + h5diff -r --delta=1e-11 restart_output-pipe-lam.sol.h5 $REF_SOLN_PIPE /velocity + h5diff -r --delta=1e-11 restart_output-pipe-lam.sol.h5 $REF_SOLN_PIPE /swirl } @test "[$TEST] check that Taylor-Couette (axisymmetric, with swirl) runs and verify result" { run $EXE --runFile $RUNFILE_TC [[ ${status} -eq 0 ]] test -s restart_output-tc.sol.h5 - h5diff -r --delta=1e-12 restart_output-tc.sol.h5 $REF_SOLN_TC /velocity - h5diff -r --delta=1e-12 restart_output-tc.sol.h5 $REF_SOLN_TC /swirl + h5diff -r --delta=1e-11 restart_output-tc.sol.h5 $REF_SOLN_TC /velocity + h5diff -r --delta=1e-11 restart_output-tc.sol.h5 $REF_SOLN_TC /swirl } @test "[$TEST] check pipe (axisymmetric) flow with algebraic rans model" { run $EXE --runFile $RUNFILE_PIPE_ARANS [[ ${status} -eq 0 ]] test -s restart_output-pipe-arans.sol.h5 - h5diff -r --delta=1e-12 restart_output-pipe-arans.sol.h5 $REF_SOLN_PIPE_ARANS /velocity - h5diff -r --delta=1e-12 restart_output-pipe-arans.sol.h5 $REF_SOLN_PIPE_ARANS /swirl + h5diff -r --delta=1e-11 restart_output-pipe-arans.sol.h5 $REF_SOLN_PIPE_ARANS /velocity + h5diff -r --delta=1e-11 restart_output-pipe-arans.sol.h5 $REF_SOLN_PIPE_ARANS /swirl } diff --git a/test/lomach-lequere.test b/test/lomach-lequere.test index 4738f3ac9..b6d1e1132 100755 --- a/test/lomach-lequere.test +++ b/test/lomach-lequere.test @@ -24,7 +24,7 @@ setup() { $EXE --runFile $RUNFILE # check - h5diff -r --relative=1e-12 $SOLN_FILE $REF_SOLN /temperature/temperature + h5diff -r --relative=1e-10 $SOLN_FILE $REF_SOLN /temperature/temperature h5diff -r --relative=1e-10 $SOLN_FILE $REF_SOLN /velocity/x-comp h5diff -r --relative=1e-10 $SOLN_FILE $REF_SOLN /velocity/y-comp } diff --git a/test/ref_solns/sgsLoMach/restart_output.sol.h5 b/test/ref_solns/sgsLoMach/restart_output.sol.h5 index 333ab85e4..eb91f9a10 100644 --- a/test/ref_solns/sgsLoMach/restart_output.sol.h5 +++ b/test/ref_solns/sgsLoMach/restart_output.sol.h5 @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:a1d285d59acd77b7cc383d646e8b2ddf10b1da771dad96bf0a5f46afa08df0bc +oid sha256:4c6c9716c42370668ad883e258709b93083b2d7f25c6f8afdc547b680611fc73 size 78744 diff --git a/test/sgsLoMach.test b/test/sgsLoMach.test index d572d711d..0309c3755 100755 --- a/test/sgsLoMach.test +++ b/test/sgsLoMach.test @@ -26,5 +26,5 @@ setup() { @test "[$TEST] verify tps output with input -> $RUNFILE" { test -s $SOLN_FILE test -s $REF_FILE - h5diff -r --delta=1e-12 $SOLN_FILE $REF_FILE /velocity + h5diff -r --delta=1e-11 $SOLN_FILE $REF_FILE /velocity } From 4d8e970cf00c7c6062b1e0b0a675e1883cd82b9e Mon Sep 17 00:00:00 2001 From: Sigfried Haering Date: Wed, 12 Jun 2024 23:54:17 -0700 Subject: [PATCH 14/27] further reduing tolerance on lomach-flow tests --- test/lomach-flow.test | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/test/lomach-flow.test b/test/lomach-flow.test index 4caec6700..80e32b237 100755 --- a/test/lomach-flow.test +++ b/test/lomach-flow.test @@ -109,7 +109,7 @@ setup() { [[ ${status} -eq 0 ]] # check solutions are the same! - h5diff -r --delta=1e-12 restart_output-lid.sol.h5 restart_output-lid.sol.2.h5 + h5diff -r --delta=1e-11 restart_output-lid.sol.h5 restart_output-lid.sol.2.h5 # delete intermediate files rm -f restart_output-lid*.h5 @@ -138,7 +138,7 @@ setup() { $MPIRUN -n 2 $EXE --runFile $RUNFILE_LID_MOD # check solutions are the same! - h5diff -r --delta=1e-11 restart_output-lid.sol.h5 restart_output-lid.sol.2.h5 + h5diff -r --delta=1e-10 restart_output-lid.sol.h5 restart_output-lid.sol.2.h5 # delete intermediate files rm -f restart_output-lid*.h5 @@ -149,22 +149,22 @@ setup() { run $EXE --runFile $RUNFILE_PIPE [[ ${status} -eq 0 ]] test -s restart_output-pipe-lam.sol.h5 - h5diff -r --delta=1e-11 restart_output-pipe-lam.sol.h5 $REF_SOLN_PIPE /velocity - h5diff -r --delta=1e-11 restart_output-pipe-lam.sol.h5 $REF_SOLN_PIPE /swirl + h5diff -r --delta=1e-10 restart_output-pipe-lam.sol.h5 $REF_SOLN_PIPE /velocity + h5diff -r --delta=1e-10 restart_output-pipe-lam.sol.h5 $REF_SOLN_PIPE /swirl } @test "[$TEST] check that Taylor-Couette (axisymmetric, with swirl) runs and verify result" { run $EXE --runFile $RUNFILE_TC [[ ${status} -eq 0 ]] test -s restart_output-tc.sol.h5 - h5diff -r --delta=1e-11 restart_output-tc.sol.h5 $REF_SOLN_TC /velocity - h5diff -r --delta=1e-11 restart_output-tc.sol.h5 $REF_SOLN_TC /swirl + h5diff -r --delta=1e-10 restart_output-tc.sol.h5 $REF_SOLN_TC /velocity + h5diff -r --delta=1e-10 restart_output-tc.sol.h5 $REF_SOLN_TC /swirl } @test "[$TEST] check pipe (axisymmetric) flow with algebraic rans model" { run $EXE --runFile $RUNFILE_PIPE_ARANS [[ ${status} -eq 0 ]] test -s restart_output-pipe-arans.sol.h5 - h5diff -r --delta=1e-11 restart_output-pipe-arans.sol.h5 $REF_SOLN_PIPE_ARANS /velocity - h5diff -r --delta=1e-11 restart_output-pipe-arans.sol.h5 $REF_SOLN_PIPE_ARANS /swirl + h5diff -r --delta=1e-9 restart_output-pipe-arans.sol.h5 $REF_SOLN_PIPE_ARANS /velocity + h5diff -r --delta=1e-9 restart_output-pipe-arans.sol.h5 $REF_SOLN_PIPE_ARANS /swirl } From fb5ee8dcb0f64881b7e61fd561c2e518ac8e52ef Mon Sep 17 00:00:00 2001 From: Sigfried Haering Date: Thu, 13 Jun 2024 20:16:46 -0700 Subject: [PATCH 15/27] further reduction in test tols --- test/lomach-flow.test | 12 ++++++------ test/lomach-lequere.test | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/test/lomach-flow.test b/test/lomach-flow.test index 80e32b237..fb6b6f536 100755 --- a/test/lomach-flow.test +++ b/test/lomach-flow.test @@ -149,22 +149,22 @@ setup() { run $EXE --runFile $RUNFILE_PIPE [[ ${status} -eq 0 ]] test -s restart_output-pipe-lam.sol.h5 - h5diff -r --delta=1e-10 restart_output-pipe-lam.sol.h5 $REF_SOLN_PIPE /velocity - h5diff -r --delta=1e-10 restart_output-pipe-lam.sol.h5 $REF_SOLN_PIPE /swirl + h5diff -r --delta=1e-8 restart_output-pipe-lam.sol.h5 $REF_SOLN_PIPE /velocity + h5diff -r --delta=1e-8 restart_output-pipe-lam.sol.h5 $REF_SOLN_PIPE /swirl } @test "[$TEST] check that Taylor-Couette (axisymmetric, with swirl) runs and verify result" { run $EXE --runFile $RUNFILE_TC [[ ${status} -eq 0 ]] test -s restart_output-tc.sol.h5 - h5diff -r --delta=1e-10 restart_output-tc.sol.h5 $REF_SOLN_TC /velocity - h5diff -r --delta=1e-10 restart_output-tc.sol.h5 $REF_SOLN_TC /swirl + h5diff -r --delta=1e-8 restart_output-tc.sol.h5 $REF_SOLN_TC /velocity + h5diff -r --delta=1e-8 restart_output-tc.sol.h5 $REF_SOLN_TC /swirl } @test "[$TEST] check pipe (axisymmetric) flow with algebraic rans model" { run $EXE --runFile $RUNFILE_PIPE_ARANS [[ ${status} -eq 0 ]] test -s restart_output-pipe-arans.sol.h5 - h5diff -r --delta=1e-9 restart_output-pipe-arans.sol.h5 $REF_SOLN_PIPE_ARANS /velocity - h5diff -r --delta=1e-9 restart_output-pipe-arans.sol.h5 $REF_SOLN_PIPE_ARANS /swirl + h5diff -r --delta=1e-8 restart_output-pipe-arans.sol.h5 $REF_SOLN_PIPE_ARANS /velocity + h5diff -r --delta=1e-8 restart_output-pipe-arans.sol.h5 $REF_SOLN_PIPE_ARANS /swirl } diff --git a/test/lomach-lequere.test b/test/lomach-lequere.test index b6d1e1132..fe518c6b1 100755 --- a/test/lomach-lequere.test +++ b/test/lomach-lequere.test @@ -24,8 +24,8 @@ setup() { $EXE --runFile $RUNFILE # check - h5diff -r --relative=1e-10 $SOLN_FILE $REF_SOLN /temperature/temperature - h5diff -r --relative=1e-10 $SOLN_FILE $REF_SOLN /velocity/x-comp - h5diff -r --relative=1e-10 $SOLN_FILE $REF_SOLN /velocity/y-comp + h5diff -r --relative=1e-8 $SOLN_FILE $REF_SOLN /temperature/temperature + h5diff -r --relative=1e-8 $SOLN_FILE $REF_SOLN /velocity/x-comp + h5diff -r --relative=1e-8 $SOLN_FILE $REF_SOLN /velocity/y-comp } From e1c62261a591b559322a3dd13f1b1f15cb573330 Mon Sep 17 00:00:00 2001 From: Sigfried Haering Date: Fri, 14 Jun 2024 07:54:33 -0700 Subject: [PATCH 16/27] raised rel. tol to 1e-10 --- src/tomboulides.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tomboulides.hpp b/src/tomboulides.hpp index 431b28635..016b061db 100644 --- a/src/tomboulides.hpp +++ b/src/tomboulides.hpp @@ -150,12 +150,12 @@ class Tomboulides final : public FlowBase { int mass_inverse_pl_ = 0; int mass_inverse_max_iter_ = 1000; - double mass_inverse_rtol_ = 1e-08; + double mass_inverse_rtol_ = 1e-10; double mass_inverse_atol_ = 1e-12; int hsolve_pl_ = 0; int hsolve_max_iter_ = 1000; - double hsolve_rtol_ = 1e-08; + double hsolve_rtol_ = 1e-10; double hsolve_atol_ = 1e-12; // To use "numerical integration", quadrature rule must persist From 0486e55f3ad337dfad23af546635d36a0eb1a4ce Mon Sep 17 00:00:00 2001 From: Sigfried Haering Date: Fri, 14 Jun 2024 08:45:01 -0700 Subject: [PATCH 17/27] switch rel tol check to delta check in lequere test, all atol to 1e-12 --- src/calorically_perfect.hpp | 4 ++-- src/tomboulides.hpp | 8 ++++---- test/lomach-lequere.test | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/calorically_perfect.hpp b/src/calorically_perfect.hpp index 6a5ed581a..bfe2dc13c 100644 --- a/src/calorically_perfect.hpp +++ b/src/calorically_perfect.hpp @@ -98,12 +98,12 @@ class CaloricallyPerfectThermoChem : public ThermoChemModelBase { int mass_inverse_pl_ = 0; int mass_inverse_max_iter_ = 1000; - double mass_inverse_rtol_ = 1e-12; + double mass_inverse_rtol_ = 1e-8; double mass_inverse_atol_ = 1e-12; int hsolve_pl_ = 0; int hsolve_max_iter_ = 1000; - double hsolve_rtol_ = 1e-12; + double hsolve_rtol_ = 1e-8; double hsolve_atol_ = 1e-12; // Boundary condition info diff --git a/src/tomboulides.hpp b/src/tomboulides.hpp index 016b061db..1ce9b9e9b 100644 --- a/src/tomboulides.hpp +++ b/src/tomboulides.hpp @@ -145,17 +145,17 @@ class Tomboulides final : public FlowBase { // solver tolerance options int pressure_solve_pl_ = 0; int pressure_solve_max_iter_ = 1000; - double pressure_solve_rtol_ = 1e-08; - double pressure_solve_atol_ = 1e-18; + double pressure_solve_rtol_ = 1e-8; + double pressure_solve_atol_ = 1e-12; int mass_inverse_pl_ = 0; int mass_inverse_max_iter_ = 1000; - double mass_inverse_rtol_ = 1e-10; + double mass_inverse_rtol_ = 1e-8; double mass_inverse_atol_ = 1e-12; int hsolve_pl_ = 0; int hsolve_max_iter_ = 1000; - double hsolve_rtol_ = 1e-10; + double hsolve_rtol_ = 1e-8; double hsolve_atol_ = 1e-12; // To use "numerical integration", quadrature rule must persist diff --git a/test/lomach-lequere.test b/test/lomach-lequere.test index fe518c6b1..582aa3072 100755 --- a/test/lomach-lequere.test +++ b/test/lomach-lequere.test @@ -25,7 +25,7 @@ setup() { # check h5diff -r --relative=1e-8 $SOLN_FILE $REF_SOLN /temperature/temperature - h5diff -r --relative=1e-8 $SOLN_FILE $REF_SOLN /velocity/x-comp - h5diff -r --relative=1e-8 $SOLN_FILE $REF_SOLN /velocity/y-comp + h5diff -r --delta=1e-8 $SOLN_FILE $REF_SOLN /velocity/x-comp + h5diff -r --delta=1e-8 $SOLN_FILE $REF_SOLN /velocity/y-comp } From d3d1082a53e4e7328726fef7c6c5a0c87872f37e Mon Sep 17 00:00:00 2001 From: Sigfried Haering Date: Fri, 14 Jun 2024 10:33:35 -0700 Subject: [PATCH 18/27] collected default solver tols to single defaults --- src/calorically_perfect.cpp | 12 ++++++------ src/calorically_perfect.hpp | 16 ++++++++++------ src/tomboulides.cpp | 18 +++++++++--------- src/tomboulides.hpp | 22 +++++++++++++--------- 4 files changed, 38 insertions(+), 30 deletions(-) diff --git a/src/calorically_perfect.cpp b/src/calorically_perfect.cpp index 44be20c2d..c182c14e6 100644 --- a/src/calorically_perfect.cpp +++ b/src/calorically_perfect.cpp @@ -115,21 +115,21 @@ CaloricallyPerfectThermoChem::CaloricallyPerfectThermoChem(mfem::ParMesh *pmesh, tpsP_->getInput("loMach/openSystem", domain_is_open_, false); - tpsP_->getInput("loMach/calperfect/linear-solver-rtol", rtol_, 1e-12); - tpsP_->getInput("loMach/calperfect/linear-solver-max-iter", max_iter_, 1000); - tpsP_->getInput("loMach/calperfect/linear-solver-verbosity", pl_solve_, 0); - // NOTE: this should default to FALSE (but would break all related test cases...) tpsP_->getInput("loMach/calperfect/numerical-integ", numerical_integ_, true); + tpsP_->getInput("loMach/calperfect/linear-solver-rtol", rtol_, default_rtol_); + tpsP_->getInput("loMach/calperfect/linear-solver-max-iter", max_iter_, default_max_iter_); + tpsP_->getInput("loMach/calperfect/linear-solver-verbosity", pl_solve_, 0); + // not deleting above block to maintain backwards-compatability tpsP_->getInput("loMach/calperfect/hsolve-rtol", hsolve_rtol_, rtol_); - tpsP_->getInput("loMach/calperfect/hsolve-atol", hsolve_atol_, 1.0e-12); + tpsP_->getInput("loMach/calperfect/hsolve-atol", hsolve_atol_, default_atol_); tpsP_->getInput("loMach/calperfect/hsolve-max-iter", hsolve_max_iter_, max_iter_); tpsP_->getInput("loMach/calperfect/hsolve-verbosity", hsolve_pl_, pl_solve_); tpsP_->getInput("loMach/calperfect/msolve-rtol", mass_inverse_rtol_, rtol_); - tpsP_->getInput("loMach/calperfect/msolve-atol", mass_inverse_atol_, 1.0e-12); + tpsP_->getInput("loMach/calperfect/msolve-atol", mass_inverse_atol_, default_atol_); tpsP_->getInput("loMach/calperfect/msolve-max-iter", mass_inverse_max_iter_, max_iter_); tpsP_->getInput("loMach/calperfect/msolve-verbosity", mass_inverse_pl_, pl_solve_); } diff --git a/src/calorically_perfect.hpp b/src/calorically_perfect.hpp index bfe2dc13c..9829aeef0 100644 --- a/src/calorically_perfect.hpp +++ b/src/calorically_perfect.hpp @@ -96,15 +96,19 @@ class CaloricallyPerfectThermoChem : public ThermoChemModelBase { int max_iter_; /**< Maximum number of linear solver iterations */ double rtol_; /**< Linear solver relative tolerance */ + int default_max_iter_ = 1000; + double default_rtol_ = 1.0e-8; + double default_atol_ = 1.0e-10; + int mass_inverse_pl_ = 0; - int mass_inverse_max_iter_ = 1000; - double mass_inverse_rtol_ = 1e-8; - double mass_inverse_atol_ = 1e-12; + int mass_inverse_max_iter_; + double mass_inverse_rtol_; + double mass_inverse_atol_; int hsolve_pl_ = 0; - int hsolve_max_iter_ = 1000; - double hsolve_rtol_ = 1e-8; - double hsolve_atol_ = 1e-12; + int hsolve_max_iter_; + double hsolve_rtol_; + double hsolve_atol_; // Boundary condition info Array temp_ess_attr_; /**< List of patches with Dirichlet BC on temperature */ diff --git a/src/tomboulides.cpp b/src/tomboulides.cpp index 52deac957..21daf0f26 100644 --- a/src/tomboulides.cpp +++ b/src/tomboulides.cpp @@ -117,17 +117,17 @@ Tomboulides::Tomboulides(mfem::ParMesh *pmesh, int vorder, int porder, temporalS if (axisym_) assert(!numerical_integ_); // exposing solver tolerance options to user - tps->getInput("loMach/tomboulides/psolve_atol_", pressure_solve_atol_, 1.0e-12); - tps->getInput("loMach/tomboulides/hsolve_atol_", hsolve_atol_, 1.0e-12); - tps->getInput("loMach/tomboulides/msolve_atol_", mass_inverse_atol_, 1.0e-12); + tps->getInput("loMach/tomboulides/psolve_atol_", pressure_solve_atol_, default_atol_); + tps->getInput("loMach/tomboulides/hsolve_atol_", hsolve_atol_, default_atol_); + tps->getInput("loMach/tomboulides/msolve_atol_", mass_inverse_atol_, default_atol_); - tps->getInput("loMach/tomboulides/psolve_rtol_", pressure_solve_rtol_, 1.0e-08); - tps->getInput("loMach/tomboulides/hsolve_rtol_", hsolve_rtol_, 1.0e-08); - tps->getInput("loMach/tomboulides/msolve_rtol_", mass_inverse_rtol_, 1.0e-08); + tps->getInput("loMach/tomboulides/psolve_rtol_", pressure_solve_rtol_, default_rtol_); + tps->getInput("loMach/tomboulides/hsolve_rtol_", hsolve_rtol_, default_rtol_); + tps->getInput("loMach/tomboulides/msolve_rtol_", mass_inverse_rtol_, default_rtol_); - tps->getInput("loMach/tomboulides/psolve_maxIters_", pressure_solve_max_iter_, 1000); - tps->getInput("loMach/tomboulides/hsolve_maxIters_", hsolve_max_iter_, 1000); - tps->getInput("loMach/tomboulides/msolve_maxIters_", mass_inverse_max_iter_, 1000); + tps->getInput("loMach/tomboulides/psolve_maxIters_", pressure_solve_max_iter_, default_max_iter_); + tps->getInput("loMach/tomboulides/hsolve_maxIters_", hsolve_max_iter_, default_max_iter_); + tps->getInput("loMach/tomboulides/msolve_maxIters_", mass_inverse_max_iter_, default_max_iter_); } } diff --git a/src/tomboulides.hpp b/src/tomboulides.hpp index 1ce9b9e9b..515249cd3 100644 --- a/src/tomboulides.hpp +++ b/src/tomboulides.hpp @@ -143,20 +143,24 @@ class Tomboulides final : public FlowBase { int amg_interpolation_ = 14; // solver tolerance options + int default_max_iter_ = 1000; + double default_rtol_ = 1e-8; + double default_atol_ = 1e-10; + int pressure_solve_pl_ = 0; - int pressure_solve_max_iter_ = 1000; - double pressure_solve_rtol_ = 1e-8; - double pressure_solve_atol_ = 1e-12; + int pressure_solve_max_iter_; + double pressure_solve_rtol_; + double pressure_solve_atol_; int mass_inverse_pl_ = 0; - int mass_inverse_max_iter_ = 1000; - double mass_inverse_rtol_ = 1e-8; - double mass_inverse_atol_ = 1e-12; + int mass_inverse_max_iter_; + double mass_inverse_rtol_; + double mass_inverse_atol_; int hsolve_pl_ = 0; - int hsolve_max_iter_ = 1000; - double hsolve_rtol_ = 1e-8; - double hsolve_atol_ = 1e-12; + int hsolve_max_iter_; + double hsolve_rtol_; + double hsolve_atol_; // To use "numerical integration", quadrature rule must persist mfem::IntegrationRules gll_rules; From d2744cc4bd9a0d02b823ce232686915ce91319fa Mon Sep 17 00:00:00 2001 From: Sigfried Haering Date: Sat, 15 Jun 2024 11:45:21 -0700 Subject: [PATCH 19/27] further solver setting tweaks, reduced test tols --- src/calorically_perfect.hpp | 6 +++--- src/tomboulides.hpp | 20 +++++++++++--------- test/autoPeriodic.test | 2 +- test/aveLoMach.test | 2 +- test/interpInlet.test | 2 +- 5 files changed, 17 insertions(+), 15 deletions(-) diff --git a/src/calorically_perfect.hpp b/src/calorically_perfect.hpp index 9829aeef0..fd62692f4 100644 --- a/src/calorically_perfect.hpp +++ b/src/calorically_perfect.hpp @@ -87,9 +87,9 @@ class CaloricallyPerfectThermoChem : public ThermoChemModelBase { int smoother_eig_est_ = 10; int smoother_passes_ = 1; double smoother_relax_weight_ = 0.4; - double smoother_relax_omega_ = 0.01; - double hsmoother_relax_weight_ = 0.4; - double hsmoother_relax_omega_ = 0.01; + double smoother_relax_omega_ = 1.0; + double hsmoother_relax_weight_ = 0.8; + double hsmoother_relax_omega_ = 0.1; // solver tolerance options int pl_solve_; /**< Verbosity level passed to mfem solvers */ diff --git a/src/tomboulides.hpp b/src/tomboulides.hpp index 515249cd3..f6bf89fec 100644 --- a/src/tomboulides.hpp +++ b/src/tomboulides.hpp @@ -90,20 +90,22 @@ class Tomboulides final : public FlowBase { int smoother_eig_est_ = 10; int smoother_passes_ = 1; double smoother_relax_weight_ = - 0.4; // NOTE: setting relax wt to 0 should determine wt on a per-level basis but this does not seem to work - double smoother_relax_omega_ = 0.01; - double hsmoother_relax_weight_ = 0.4; - double hsmoother_relax_omega_ = 0.01; - double pressure_strength_thres_ = 0.6; - int amg_aggresive_ = 4; - int amg_max_levels_ = 10; // 100; - int amg_max_iters_ = 1; // setting to zero is ~40% faster per step but effectively turns off precon? - int amg_relax_ = 0; // only 0 or 18 now + 0.4; // NOTE: setting relax wt to 0 should determine wt dynamically but this does not seem to work + double smoother_relax_omega_ = 1.0; + double hsmoother_relax_weight_ = 0.8; + double hsmoother_relax_omega_ = 0.1; // Options: Jacobi, l1Jacobi, l1GS, l1GStr, lumpedJacobi, // GS, OPFS, Chebyshev, Taubin, FIR mfem::HypreSmoother::Type smoother_type_ = HypreSmoother::Jacobi; + double pressure_strength_thres_ = 0.6; + int amg_aggresive_ = 4; + int amg_max_levels_ = 10; + int amg_max_iters_ = + 1; // should be 1 for precon, setting to zero is ~30-40% faster per step but effectively turns off solve + int amg_relax_ = 18; // only 0 or 18 now + /* 0 : CLJP-coarsening (a parallel coarsening algorithm using independent sets. 1 : classical Ruge-Stueben coarsening on each processor, no boundary treatment (not recommended!) diff --git a/test/autoPeriodic.test b/test/autoPeriodic.test index 1fb5cc41a..3a46235e2 100755 --- a/test/autoPeriodic.test +++ b/test/autoPeriodic.test @@ -26,5 +26,5 @@ setup() { @test "[$TEST] verify tps output with input -> $RUNFILE" { test -s $SOLN_FILE test -s $REF_FILE - h5diff -r --delta=1e-11 $SOLN_FILE $REF_FILE /velocity + h5diff -r --delta=1e-10 $SOLN_FILE $REF_FILE /velocity } diff --git a/test/aveLoMach.test b/test/aveLoMach.test index 31af36add..283146d62 100755 --- a/test/aveLoMach.test +++ b/test/aveLoMach.test @@ -26,5 +26,5 @@ setup() { @test "[$TEST] verify tps output with input -> $RUNFILE" { test -s $SOLN_FILE test -s $REF_FILE - h5diff -r --delta=1e-12 $SOLN_FILE $REF_FILE meanVel + h5diff -r --delta=1e-11 $SOLN_FILE $REF_FILE meanVel } diff --git a/test/interpInlet.test b/test/interpInlet.test index 92df8c21e..3e1234ae6 100755 --- a/test/interpInlet.test +++ b/test/interpInlet.test @@ -33,5 +33,5 @@ setup() { @test "[$TEST] verify tps output with input -> $RUNFILE" { test -s $SOLN_FILE test -s $REF_FILE - h5diff -r --relative=1e-11 $SOLN_FILE $REF_FILE /temperature/temperature + h5diff -r --relative=1e-9 $SOLN_FILE $REF_FILE /temperature/temperature } From 128997778193425cb40f83fac31dd12c356b36e7 Mon Sep 17 00:00:00 2001 From: Sigfried Haering Date: Mon, 17 Jun 2024 08:11:57 -0700 Subject: [PATCH 20/27] removed unintentional underscore in tomboulides tol setting labels --- src/tomboulides.cpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/tomboulides.cpp b/src/tomboulides.cpp index 21daf0f26..331cf02fc 100644 --- a/src/tomboulides.cpp +++ b/src/tomboulides.cpp @@ -117,17 +117,17 @@ Tomboulides::Tomboulides(mfem::ParMesh *pmesh, int vorder, int porder, temporalS if (axisym_) assert(!numerical_integ_); // exposing solver tolerance options to user - tps->getInput("loMach/tomboulides/psolve_atol_", pressure_solve_atol_, default_atol_); - tps->getInput("loMach/tomboulides/hsolve_atol_", hsolve_atol_, default_atol_); - tps->getInput("loMach/tomboulides/msolve_atol_", mass_inverse_atol_, default_atol_); + tps->getInput("loMach/tomboulides/psolve_atol", pressure_solve_atol_, default_atol_); + tps->getInput("loMach/tomboulides/hsolve_atol", hsolve_atol_, default_atol_); + tps->getInput("loMach/tomboulides/msolve_atol", mass_inverse_atol_, default_atol_); - tps->getInput("loMach/tomboulides/psolve_rtol_", pressure_solve_rtol_, default_rtol_); - tps->getInput("loMach/tomboulides/hsolve_rtol_", hsolve_rtol_, default_rtol_); - tps->getInput("loMach/tomboulides/msolve_rtol_", mass_inverse_rtol_, default_rtol_); + tps->getInput("loMach/tomboulides/psolve_rtol", pressure_solve_rtol_, default_rtol_); + tps->getInput("loMach/tomboulides/hsolve_rtol", hsolve_rtol_, default_rtol_); + tps->getInput("loMach/tomboulides/msolve_rtol", mass_inverse_rtol_, default_rtol_); - tps->getInput("loMach/tomboulides/psolve_maxIters_", pressure_solve_max_iter_, default_max_iter_); - tps->getInput("loMach/tomboulides/hsolve_maxIters_", hsolve_max_iter_, default_max_iter_); - tps->getInput("loMach/tomboulides/msolve_maxIters_", mass_inverse_max_iter_, default_max_iter_); + tps->getInput("loMach/tomboulides/psolve_maxIters", pressure_solve_max_iter_, default_max_iter_); + tps->getInput("loMach/tomboulides/hsolve_maxIters", hsolve_max_iter_, default_max_iter_); + tps->getInput("loMach/tomboulides/msolve_maxIters", mass_inverse_max_iter_, default_max_iter_); } } From f721a33f32fbad82ae9af547837ad6cda3e2ad77 Mon Sep 17 00:00:00 2001 From: Sigfried Haering Date: Mon, 17 Jun 2024 08:16:40 -0700 Subject: [PATCH 21/27] switch tomboulides underscore tol labels to dashes for consitency --- src/tomboulides.cpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/tomboulides.cpp b/src/tomboulides.cpp index 331cf02fc..0932a1889 100644 --- a/src/tomboulides.cpp +++ b/src/tomboulides.cpp @@ -117,17 +117,17 @@ Tomboulides::Tomboulides(mfem::ParMesh *pmesh, int vorder, int porder, temporalS if (axisym_) assert(!numerical_integ_); // exposing solver tolerance options to user - tps->getInput("loMach/tomboulides/psolve_atol", pressure_solve_atol_, default_atol_); - tps->getInput("loMach/tomboulides/hsolve_atol", hsolve_atol_, default_atol_); - tps->getInput("loMach/tomboulides/msolve_atol", mass_inverse_atol_, default_atol_); + tps->getInput("loMach/tomboulides/psolve-atol", pressure_solve_atol_, default_atol_); + tps->getInput("loMach/tomboulides/hsolve-atol", hsolve_atol_, default_atol_); + tps->getInput("loMach/tomboulides/msolve-atol", mass_inverse_atol_, default_atol_); - tps->getInput("loMach/tomboulides/psolve_rtol", pressure_solve_rtol_, default_rtol_); - tps->getInput("loMach/tomboulides/hsolve_rtol", hsolve_rtol_, default_rtol_); - tps->getInput("loMach/tomboulides/msolve_rtol", mass_inverse_rtol_, default_rtol_); + tps->getInput("loMach/tomboulides/psolve-rtol", pressure_solve_rtol_, default_rtol_); + tps->getInput("loMach/tomboulides/hsolve-rtol", hsolve_rtol_, default_rtol_); + tps->getInput("loMach/tomboulides/msolve-rtol", mass_inverse_rtol_, default_rtol_); - tps->getInput("loMach/tomboulides/psolve_maxIters", pressure_solve_max_iter_, default_max_iter_); - tps->getInput("loMach/tomboulides/hsolve_maxIters", hsolve_max_iter_, default_max_iter_); - tps->getInput("loMach/tomboulides/msolve_maxIters", mass_inverse_max_iter_, default_max_iter_); + tps->getInput("loMach/tomboulides/psolve-maxIters", pressure_solve_max_iter_, default_max_iter_); + tps->getInput("loMach/tomboulides/hsolve-maxIters", hsolve_max_iter_, default_max_iter_); + tps->getInput("loMach/tomboulides/msolve-maxIters", mass_inverse_max_iter_, default_max_iter_); } } From 099782efc0ca23cfc7da0e5d433fcf15968e5427 Mon Sep 17 00:00:00 2001 From: Sigfried Haering Date: Thu, 20 Jun 2024 14:16:37 -0700 Subject: [PATCH 22/27] upped default tolerances and switched default NI to off --- src/calorically_perfect.hpp | 6 +++--- src/tomboulides.hpp | 7 +++---- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/calorically_perfect.hpp b/src/calorically_perfect.hpp index fd62692f4..054aee996 100644 --- a/src/calorically_perfect.hpp +++ b/src/calorically_perfect.hpp @@ -76,7 +76,7 @@ class CaloricallyPerfectThermoChem : public ThermoChemModelBase { // Flags bool rank0_; /**< true if this is rank 0 */ bool partial_assembly_ = false; /**< Enable/disable partial assembly of forms. */ - bool numerical_integ_ = true; /**< Enable/disable numerical integration rules of forms. */ + bool numerical_integ_ = false; /**< Enable/disable numerical integration rules of forms. */ bool constant_viscosity_ = false; /**< Enable/disable constant viscosity */ bool constant_density_ = false; /**< Enable/disable constant density */ bool domain_is_open_ = false; /**< true if domain is open */ @@ -97,8 +97,8 @@ class CaloricallyPerfectThermoChem : public ThermoChemModelBase { double rtol_; /**< Linear solver relative tolerance */ int default_max_iter_ = 1000; - double default_rtol_ = 1.0e-8; - double default_atol_ = 1.0e-10; + double default_rtol_ = 1.0e-10; + double default_atol_ = 1.0e-12; int mass_inverse_pl_ = 0; int mass_inverse_max_iter_; diff --git a/src/tomboulides.hpp b/src/tomboulides.hpp index f6bf89fec..8bc5d8584 100644 --- a/src/tomboulides.hpp +++ b/src/tomboulides.hpp @@ -80,8 +80,7 @@ class Tomboulides final : public FlowBase { bool axisym_; // Options - // TODO(trevilo): hardcoded for testing. Need to set based on input file. - bool numerical_integ_ = true; + bool numerical_integ_ = false; bool partial_assembly_ = false; // linear-solver options @@ -146,8 +145,8 @@ class Tomboulides final : public FlowBase { // solver tolerance options int default_max_iter_ = 1000; - double default_rtol_ = 1e-8; - double default_atol_ = 1e-10; + double default_rtol_ = 1e-10; + double default_atol_ = 1e-12; int pressure_solve_pl_ = 0; int pressure_solve_max_iter_; From 6d3c610041796cb1a6777ea6bb42b69d38363460 Mon Sep 17 00:00:00 2001 From: Sigfried Haering Date: Fri, 21 Jun 2024 12:24:47 -0700 Subject: [PATCH 23/27] switched tolerances of test checks to be consitent with default lin-sol tol and replacing ref soln. numerical-integ defaults to false now --- test/autoPeriodic.test | 2 ++ test/aveLoMach.test | 2 ++ test/heatEq.test | 4 +++- test/interpInlet.test | 4 +++- test/lomach-flow.test | 22 ++++++++++++------- test/lomach-lequere.test | 8 ++++--- .../ref_solns/aveLoMach/restart_output.sol.h5 | 2 +- .../ref_solns/heatedBox/restart_output.sol.h5 | 2 +- .../interpInlet/restart_output.sol.h5 | 4 ++-- .../reference-lequere-varmu.sol.h5 | 2 +- .../pipe/restart_output-pipe-lam.sol.h5 | 4 ++-- .../ref_solns/sgsLoMach/restart_output.sol.h5 | 2 +- test/sgsLoMach.test | 4 +++- 13 files changed, 40 insertions(+), 22 deletions(-) diff --git a/test/autoPeriodic.test b/test/autoPeriodic.test index 3a46235e2..9ae901de6 100755 --- a/test/autoPeriodic.test +++ b/test/autoPeriodic.test @@ -9,6 +9,7 @@ RESTART="ref_solns/sgsLoMach/restart_output.sol.h5" setup() { SOLN_FILE=restart_output.sol.h5 REF_FILE=ref_solns/sgsLoMach/restart_output.sol.h5 + OUT_FILE=output_solns/restart_output_autoPeriodic.sol.h5 } @test "[$TEST] check for input file $RUNFILE" { @@ -27,4 +28,5 @@ setup() { test -s $SOLN_FILE test -s $REF_FILE h5diff -r --delta=1e-10 $SOLN_FILE $REF_FILE /velocity + cp $SOLN_FILE $OUT_FILE } diff --git a/test/aveLoMach.test b/test/aveLoMach.test index 283146d62..8333dbb51 100755 --- a/test/aveLoMach.test +++ b/test/aveLoMach.test @@ -9,6 +9,7 @@ RESTART="ref_solns/aveLoMach/restart_output.sol.h5" setup() { SOLN_FILE=restart_output.sol.h5 REF_FILE=ref_solns/aveLoMach/restart_output.sol.h5 + OUT_FILE=output_solns/restart_output_aveLoMach.sol.h5 } @test "[$TEST] check for input file $RUNFILE" { @@ -27,4 +28,5 @@ setup() { test -s $SOLN_FILE test -s $REF_FILE h5diff -r --delta=1e-11 $SOLN_FILE $REF_FILE meanVel + cp $SOLN_FILE $OUT_FILE } diff --git a/test/heatEq.test b/test/heatEq.test index 771aa9296..4ddacbe54 100755 --- a/test/heatEq.test +++ b/test/heatEq.test @@ -9,6 +9,7 @@ RESTART="ref_solns/heatedBox/restart_output.sol.h5" setup() { SOLN_FILE=restart_output.sol.h5 REF_FILE=ref_solns/heatedBox/restart_output.sol.h5 + OUT_FILE=output_solns/restart_output_heatedBox.sol.h5 } @test "[$TEST] check for input file $RUNFILE" { @@ -25,5 +26,6 @@ setup() { @test "[$TEST] verify tps output with input -> $RUNFILE" { test -s $SOLN_FILE test -s $REF_FILE - h5diff -r --relative=1e-13 $SOLN_FILE $REF_FILE /temperature/temperature + h5diff -r --relative=1e-10 $SOLN_FILE $REF_FILE /temperature/temperature + cp $SOLN_FILE $OUT_FILE } diff --git a/test/interpInlet.test b/test/interpInlet.test index 3e1234ae6..ab48232b2 100755 --- a/test/interpInlet.test +++ b/test/interpInlet.test @@ -9,6 +9,7 @@ RESTART= setup() { SOLN_FILE=restart_output.sol.h5 REF_FILE=ref_solns/interpInlet/restart_output.sol.h5 + OUT_FILE=output_solns/restart_output_interpInlet.sol.h5 } @test "[$TEST] check for input file $RUNFILE" { @@ -33,5 +34,6 @@ setup() { @test "[$TEST] verify tps output with input -> $RUNFILE" { test -s $SOLN_FILE test -s $REF_FILE - h5diff -r --relative=1e-9 $SOLN_FILE $REF_FILE /temperature/temperature + h5diff -r --relative=1e-10 $SOLN_FILE $REF_FILE /temperature/temperature + cp $SOLN_FILE $OUT_FILE } diff --git a/test/lomach-flow.test b/test/lomach-flow.test index fb6b6f536..497f59c5e 100755 --- a/test/lomach-flow.test +++ b/test/lomach-flow.test @@ -16,12 +16,15 @@ setup() { RUNFILE_PIPE="inputs/lomach.pipe.ini" REF_SOLN_PIPE="ref_solns/pipe/restart_output-pipe-lam.sol.h5" + OUT_SOLN_PIPE="output_solns/restart_output-pipe-lam.sol.h5" RUNFILE_TC="inputs/lomach.taylor.couette.ini" REF_SOLN_TC="ref_solns/taylor-couette/restart_output-tc.Re100.h5" + OUT_SOLN_TC="output_solns/restart_output-tc.Re100.h5" RUNFILE_PIPE_ARANS="inputs/lomach.pipe.arans.ini" REF_SOLN_PIPE_ARANS="ref_solns/pipe/restart_output-pipe-arans.sol.h5" + OUT_SOLN_PIPE_ARANS="output_solns/restart_output-pipe-arans.sol.h5" MPIRUN=`./sniff_mpirun.sh` } @@ -109,8 +112,8 @@ setup() { [[ ${status} -eq 0 ]] # check solutions are the same! - h5diff -r --delta=1e-11 restart_output-lid.sol.h5 restart_output-lid.sol.2.h5 - + h5diff -r --delta=1e-11 restart_output-lid.sol.h5 restart_output-lid.sol.2.h5 + # delete intermediate files rm -f restart_output-lid*.h5 rm $RUNFILE_LID_MOD @@ -149,22 +152,25 @@ setup() { run $EXE --runFile $RUNFILE_PIPE [[ ${status} -eq 0 ]] test -s restart_output-pipe-lam.sol.h5 - h5diff -r --delta=1e-8 restart_output-pipe-lam.sol.h5 $REF_SOLN_PIPE /velocity - h5diff -r --delta=1e-8 restart_output-pipe-lam.sol.h5 $REF_SOLN_PIPE /swirl + h5diff -r --delta=1e-10 restart_output-pipe-lam.sol.h5 $REF_SOLN_PIPE /velocity + h5diff -r --delta=1e-10 restart_output-pipe-lam.sol.h5 $REF_SOLN_PIPE /swirl + cp restart_output-pipe-lam.sol.h5 $OUT_SOLN_PIPE } @test "[$TEST] check that Taylor-Couette (axisymmetric, with swirl) runs and verify result" { run $EXE --runFile $RUNFILE_TC [[ ${status} -eq 0 ]] test -s restart_output-tc.sol.h5 - h5diff -r --delta=1e-8 restart_output-tc.sol.h5 $REF_SOLN_TC /velocity - h5diff -r --delta=1e-8 restart_output-tc.sol.h5 $REF_SOLN_TC /swirl + h5diff -r --delta=1e-10 restart_output-tc.sol.h5 $REF_SOLN_TC /velocity + h5diff -r --delta=1e-10 restart_output-tc.sol.h5 $REF_SOLN_TC /swirl + cp restart_output-tc.sol.h5 $OUT_SOLN_TC } @test "[$TEST] check pipe (axisymmetric) flow with algebraic rans model" { run $EXE --runFile $RUNFILE_PIPE_ARANS [[ ${status} -eq 0 ]] test -s restart_output-pipe-arans.sol.h5 - h5diff -r --delta=1e-8 restart_output-pipe-arans.sol.h5 $REF_SOLN_PIPE_ARANS /velocity - h5diff -r --delta=1e-8 restart_output-pipe-arans.sol.h5 $REF_SOLN_PIPE_ARANS /swirl + h5diff -r --delta=1e-10 restart_output-pipe-arans.sol.h5 $REF_SOLN_PIPE_ARANS /velocity + h5diff -r --delta=1e-10 restart_output-pipe-arans.sol.h5 $REF_SOLN_PIPE_ARANS /swirl + cp restart_output-pipe-arans.sol.h5 $OUT_SOLN_PIPE_ARANS } diff --git a/test/lomach-lequere.test b/test/lomach-lequere.test index 582aa3072..70d9fba63 100755 --- a/test/lomach-lequere.test +++ b/test/lomach-lequere.test @@ -10,6 +10,7 @@ setup() { RUNFILE="inputs/lomach.lequere.varmu.ini" REF_SOLN="ref_solns/lequere-varmu/reference-lequere-varmu.sol.h5" SOLN_FILE="restart_output-lequere-varmu.sol.h5" + OUT_SOLN="output_solns/reference-lequere-varmu.sol.h5" } @test "[$TEST] Le Quere low Mach, thermally-driven cavity regression with variable viscosity" { @@ -24,8 +25,9 @@ setup() { $EXE --runFile $RUNFILE # check - h5diff -r --relative=1e-8 $SOLN_FILE $REF_SOLN /temperature/temperature - h5diff -r --delta=1e-8 $SOLN_FILE $REF_SOLN /velocity/x-comp - h5diff -r --delta=1e-8 $SOLN_FILE $REF_SOLN /velocity/y-comp + h5diff -r --relative=1e-10 $SOLN_FILE $REF_SOLN /temperature/temperature + h5diff -r --delta=1e-10 $SOLN_FILE $REF_SOLN /velocity/x-comp + h5diff -r --delta=1e-10 $SOLN_FILE $REF_SOLN /velocity/y-comp + cp $SOLN_FILE $OUT_SOLN } diff --git a/test/ref_solns/aveLoMach/restart_output.sol.h5 b/test/ref_solns/aveLoMach/restart_output.sol.h5 index 48d9b67a7..fc8a71bcd 100644 --- a/test/ref_solns/aveLoMach/restart_output.sol.h5 +++ b/test/ref_solns/aveLoMach/restart_output.sol.h5 @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c3d84bab6ef05f42cdad7b3e28a758b3a1f32b0eaaf55ec3cdf0a7b04a47fec0 +oid sha256:15bb2ec395f79dc7d28324a61467760e675f3501b5582b56ff7a294146cb337d size 382040 diff --git a/test/ref_solns/heatedBox/restart_output.sol.h5 b/test/ref_solns/heatedBox/restart_output.sol.h5 index a70ced5fd..cf108bb26 100644 --- a/test/ref_solns/heatedBox/restart_output.sol.h5 +++ b/test/ref_solns/heatedBox/restart_output.sol.h5 @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:d81816aaa43491b582aff0937a16c2bd1037340378c59bbecc3203c9ad26e78e +oid sha256:02d366574f35b75cc97fdfe330450832cdb98937bcbdf3fd39f4f79921a6bab9 size 3568 diff --git a/test/ref_solns/interpInlet/restart_output.sol.h5 b/test/ref_solns/interpInlet/restart_output.sol.h5 index 5fc3e60c0..78a7f4a38 100644 --- a/test/ref_solns/interpInlet/restart_output.sol.h5 +++ b/test/ref_solns/interpInlet/restart_output.sol.h5 @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:8c782a4cdbb7228c7ffd19f262ba1f2440ed132ca055e2e76b54c3a3c2630c5d -size 8664 +oid sha256:c0639671f8f1f9496eb313f87d4fa40bcada3713d9d6f4bbd67b4bbf81405a32 +size 8728 diff --git a/test/ref_solns/lequere-varmu/reference-lequere-varmu.sol.h5 b/test/ref_solns/lequere-varmu/reference-lequere-varmu.sol.h5 index 13e2013d8..b62cc8878 100644 --- a/test/ref_solns/lequere-varmu/reference-lequere-varmu.sol.h5 +++ b/test/ref_solns/lequere-varmu/reference-lequere-varmu.sol.h5 @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:992d58bb5010f3576f13fd076f2b2ab67b4ead1536b627d7e05649c6acfccc87 +oid sha256:222e94021b4b7ed239f24c8b486a45b89a7bf7d7a15d621273f2b504bf14900a size 106416 diff --git a/test/ref_solns/pipe/restart_output-pipe-lam.sol.h5 b/test/ref_solns/pipe/restart_output-pipe-lam.sol.h5 index 636445c61..953709889 100644 --- a/test/ref_solns/pipe/restart_output-pipe-lam.sol.h5 +++ b/test/ref_solns/pipe/restart_output-pipe-lam.sol.h5 @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:7fe39fdee999551ea2e82c2e492f0cafd8567e9c0bae0066a02063e05cac8bc4 -size 18416 +oid sha256:3af1089c90ac4118d913cf80d7e9e9ee260e2ad94e0a5c0310a2fde4ae63c574 +size 18480 diff --git a/test/ref_solns/sgsLoMach/restart_output.sol.h5 b/test/ref_solns/sgsLoMach/restart_output.sol.h5 index eb91f9a10..c56fe05a7 100644 --- a/test/ref_solns/sgsLoMach/restart_output.sol.h5 +++ b/test/ref_solns/sgsLoMach/restart_output.sol.h5 @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:4c6c9716c42370668ad883e258709b93083b2d7f25c6f8afdc547b680611fc73 +oid sha256:663299c7c61d64043b0e02a6da2e6c315bf0ff9d63736bbc7ba8ec27b64a4840 size 78744 diff --git a/test/sgsLoMach.test b/test/sgsLoMach.test index 0309c3755..e317f0e56 100755 --- a/test/sgsLoMach.test +++ b/test/sgsLoMach.test @@ -9,6 +9,7 @@ RESTART="ref_solns/sgsLoMach/restart_output.sol.h5" setup() { SOLN_FILE=restart_output.sol.h5 REF_FILE=ref_solns/sgsLoMach/restart_output.sol.h5 + OUT_FILE=output_solns/restart_output_sgsLoMach.sol.h5 } @test "[$TEST] check for input file $RUNFILE" { @@ -26,5 +27,6 @@ setup() { @test "[$TEST] verify tps output with input -> $RUNFILE" { test -s $SOLN_FILE test -s $REF_FILE - h5diff -r --delta=1e-11 $SOLN_FILE $REF_FILE /velocity + h5diff -r --delta=1e-10 $SOLN_FILE $REF_FILE /velocity + cp $SOLN_FILE $OUT_FILE } From b0b914e9ed6545ba626b208e2e3759489f34a10b Mon Sep 17 00:00:00 2001 From: Sigfried Haering Date: Fri, 21 Jun 2024 12:32:51 -0700 Subject: [PATCH 24/27] enforce style --- src/calorically_perfect.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calorically_perfect.hpp b/src/calorically_perfect.hpp index 054aee996..118cd3dd5 100644 --- a/src/calorically_perfect.hpp +++ b/src/calorically_perfect.hpp @@ -76,7 +76,7 @@ class CaloricallyPerfectThermoChem : public ThermoChemModelBase { // Flags bool rank0_; /**< true if this is rank 0 */ bool partial_assembly_ = false; /**< Enable/disable partial assembly of forms. */ - bool numerical_integ_ = false; /**< Enable/disable numerical integration rules of forms. */ + bool numerical_integ_ = false; /**< Enable/disable numerical integration rules of forms. */ bool constant_viscosity_ = false; /**< Enable/disable constant viscosity */ bool constant_density_ = false; /**< Enable/disable constant density */ bool domain_is_open_ = false; /**< true if domain is open */ From da1e5624c2bf06d60580e14c64c66ab88227e815 Mon Sep 17 00:00:00 2001 From: Sigfried Haering Date: Fri, 21 Jun 2024 12:54:46 -0700 Subject: [PATCH 25/27] commented copy lines for ref solns as this was causing issues on the gpu --- test/autoPeriodic.test | 2 +- test/aveLoMach.test | 2 +- test/heatEq.test | 2 +- test/lomach-flow.test | 6 +++--- test/lomach-lequere.test | 2 +- test/sgsLoMach.test | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/test/autoPeriodic.test b/test/autoPeriodic.test index 9ae901de6..de811ddf7 100755 --- a/test/autoPeriodic.test +++ b/test/autoPeriodic.test @@ -28,5 +28,5 @@ setup() { test -s $SOLN_FILE test -s $REF_FILE h5diff -r --delta=1e-10 $SOLN_FILE $REF_FILE /velocity - cp $SOLN_FILE $OUT_FILE + # cp $SOLN_FILE $OUT_FILE } diff --git a/test/aveLoMach.test b/test/aveLoMach.test index 8333dbb51..008c43efc 100755 --- a/test/aveLoMach.test +++ b/test/aveLoMach.test @@ -28,5 +28,5 @@ setup() { test -s $SOLN_FILE test -s $REF_FILE h5diff -r --delta=1e-11 $SOLN_FILE $REF_FILE meanVel - cp $SOLN_FILE $OUT_FILE + # cp $SOLN_FILE $OUT_FILE } diff --git a/test/heatEq.test b/test/heatEq.test index 4ddacbe54..ac6108dd5 100755 --- a/test/heatEq.test +++ b/test/heatEq.test @@ -27,5 +27,5 @@ setup() { test -s $SOLN_FILE test -s $REF_FILE h5diff -r --relative=1e-10 $SOLN_FILE $REF_FILE /temperature/temperature - cp $SOLN_FILE $OUT_FILE + # cp $SOLN_FILE $OUT_FILE } diff --git a/test/lomach-flow.test b/test/lomach-flow.test index 497f59c5e..fd02a6ca3 100755 --- a/test/lomach-flow.test +++ b/test/lomach-flow.test @@ -154,7 +154,7 @@ setup() { test -s restart_output-pipe-lam.sol.h5 h5diff -r --delta=1e-10 restart_output-pipe-lam.sol.h5 $REF_SOLN_PIPE /velocity h5diff -r --delta=1e-10 restart_output-pipe-lam.sol.h5 $REF_SOLN_PIPE /swirl - cp restart_output-pipe-lam.sol.h5 $OUT_SOLN_PIPE + # cp restart_output-pipe-lam.sol.h5 $OUT_SOLN_PIPE } @test "[$TEST] check that Taylor-Couette (axisymmetric, with swirl) runs and verify result" { @@ -163,7 +163,7 @@ setup() { test -s restart_output-tc.sol.h5 h5diff -r --delta=1e-10 restart_output-tc.sol.h5 $REF_SOLN_TC /velocity h5diff -r --delta=1e-10 restart_output-tc.sol.h5 $REF_SOLN_TC /swirl - cp restart_output-tc.sol.h5 $OUT_SOLN_TC + # cp restart_output-tc.sol.h5 $OUT_SOLN_TC } @test "[$TEST] check pipe (axisymmetric) flow with algebraic rans model" { @@ -172,5 +172,5 @@ setup() { test -s restart_output-pipe-arans.sol.h5 h5diff -r --delta=1e-10 restart_output-pipe-arans.sol.h5 $REF_SOLN_PIPE_ARANS /velocity h5diff -r --delta=1e-10 restart_output-pipe-arans.sol.h5 $REF_SOLN_PIPE_ARANS /swirl - cp restart_output-pipe-arans.sol.h5 $OUT_SOLN_PIPE_ARANS + # cp restart_output-pipe-arans.sol.h5 $OUT_SOLN_PIPE_ARANS } diff --git a/test/lomach-lequere.test b/test/lomach-lequere.test index 70d9fba63..546b91b13 100755 --- a/test/lomach-lequere.test +++ b/test/lomach-lequere.test @@ -28,6 +28,6 @@ setup() { h5diff -r --relative=1e-10 $SOLN_FILE $REF_SOLN /temperature/temperature h5diff -r --delta=1e-10 $SOLN_FILE $REF_SOLN /velocity/x-comp h5diff -r --delta=1e-10 $SOLN_FILE $REF_SOLN /velocity/y-comp - cp $SOLN_FILE $OUT_SOLN + # cp $SOLN_FILE $OUT_SOLN } diff --git a/test/sgsLoMach.test b/test/sgsLoMach.test index e317f0e56..8034a54b6 100755 --- a/test/sgsLoMach.test +++ b/test/sgsLoMach.test @@ -28,5 +28,5 @@ setup() { test -s $SOLN_FILE test -s $REF_FILE h5diff -r --delta=1e-10 $SOLN_FILE $REF_FILE /velocity - cp $SOLN_FILE $OUT_FILE + # cp $SOLN_FILE $OUT_FILE } From ac9fd5d92faec798ab34962b126726bbbb70040a Mon Sep 17 00:00:00 2001 From: Sigfried Haering Date: Fri, 21 Jun 2024 13:58:53 -0700 Subject: [PATCH 26/27] updated sponge ref soln, reduced tol of axi-symm arans test, and removed a leftover ref sol copy line --- test/interpInlet.test | 2 +- test/lomach-flow.test | 4 ++-- test/ref_solns/spongeBox/restart_output.sol.h5 | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/test/interpInlet.test b/test/interpInlet.test index ab48232b2..0af9a9283 100755 --- a/test/interpInlet.test +++ b/test/interpInlet.test @@ -35,5 +35,5 @@ setup() { test -s $SOLN_FILE test -s $REF_FILE h5diff -r --relative=1e-10 $SOLN_FILE $REF_FILE /temperature/temperature - cp $SOLN_FILE $OUT_FILE + # cp $SOLN_FILE $OUT_FILE } diff --git a/test/lomach-flow.test b/test/lomach-flow.test index fd02a6ca3..eba58c5b9 100755 --- a/test/lomach-flow.test +++ b/test/lomach-flow.test @@ -170,7 +170,7 @@ setup() { run $EXE --runFile $RUNFILE_PIPE_ARANS [[ ${status} -eq 0 ]] test -s restart_output-pipe-arans.sol.h5 - h5diff -r --delta=1e-10 restart_output-pipe-arans.sol.h5 $REF_SOLN_PIPE_ARANS /velocity - h5diff -r --delta=1e-10 restart_output-pipe-arans.sol.h5 $REF_SOLN_PIPE_ARANS /swirl + h5diff -r --delta=1e-9 restart_output-pipe-arans.sol.h5 $REF_SOLN_PIPE_ARANS /velocity + h5diff -r --delta=1e-9 restart_output-pipe-arans.sol.h5 $REF_SOLN_PIPE_ARANS /swirl # cp restart_output-pipe-arans.sol.h5 $OUT_SOLN_PIPE_ARANS } diff --git a/test/ref_solns/spongeBox/restart_output.sol.h5 b/test/ref_solns/spongeBox/restart_output.sol.h5 index 0251aed2e..75c4efff5 100644 --- a/test/ref_solns/spongeBox/restart_output.sol.h5 +++ b/test/ref_solns/spongeBox/restart_output.sol.h5 @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:6369b60adb2786bcd2c477743345cc303d33e73a105db3d39fc114990a6f96dd -size 8664 +oid sha256:197ed2f249869bc6f94a5649bb6d3f4aff73dc406fc94ca98681e48dbd01b6e3 +size 8728 From 4de93a50578c0d608b700093778807d7d3fb0e60 Mon Sep 17 00:00:00 2001 From: Sigfried Haering Date: Fri, 21 Jun 2024 14:48:46 -0700 Subject: [PATCH 27/27] update pipe arans soln and reduced sponge test tol --- test/ref_solns/pipe/restart_output-pipe-arans.sol.h5 | 2 +- test/sponge.test | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/ref_solns/pipe/restart_output-pipe-arans.sol.h5 b/test/ref_solns/pipe/restart_output-pipe-arans.sol.h5 index 5e71a3a80..dd9e8665d 100644 --- a/test/ref_solns/pipe/restart_output-pipe-arans.sol.h5 +++ b/test/ref_solns/pipe/restart_output-pipe-arans.sol.h5 @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:fc4128fc860c4f6f3a5419ed3117af719e8968cfee76b85bee28bb9019667e90 +oid sha256:39061c77f5cc54f664ce69f5f91ae3f5f5386a04616efa7223cce39793234b97 size 56496 diff --git a/test/sponge.test b/test/sponge.test index c06d1757a..d69ff6edc 100755 --- a/test/sponge.test +++ b/test/sponge.test @@ -33,5 +33,5 @@ setup() { @test "[$TEST] verify tps output with input -> $RUNFILE" { test -s $SOLN_FILE test -s $REF_FILE - h5diff -r --relative=1e-13 $SOLN_FILE $REF_FILE /temperature/temperature + h5diff -r --relative=1e-10 $SOLN_FILE $REF_FILE /temperature/temperature }