From 0e8f1541e1897880943a804fefe036e40182616d Mon Sep 17 00:00:00 2001 From: Chris Siefert Date: Tue, 8 Mar 2022 16:49:25 -0700 Subject: [PATCH 01/19] Tpetra: Fixing tests to work with SC != double --- packages/tpetra/core/test/BugTests/Bug7234.cpp | 3 ++- .../core/test/CrsMatrix/sumIntoStaticProfileExtraSpace.cpp | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/tpetra/core/test/BugTests/Bug7234.cpp b/packages/tpetra/core/test/BugTests/Bug7234.cpp index efcdbffa6141..30fb37c448d4 100644 --- a/packages/tpetra/core/test/BugTests/Bug7234.cpp +++ b/packages/tpetra/core/test/BugTests/Bug7234.cpp @@ -99,11 +99,12 @@ namespace { using GST = Tpetra::global_size_t; using LO = int; using GO = Tpetra::Map<>::global_ordinal_type; + using SC = Tpetra::Vector<>::scalar_type; using NodeType = Tpetra::Map<>::node_type; using TpetraMap = Tpetra::Map; using TpetraImport = Tpetra::Import; using TpetraExport = Tpetra::Export; - using TpetraVec = Tpetra::Vector; + using TpetraVec = Tpetra::Vector; Teuchos::RCP > comm = Tpetra::getDefaultComm(); const int rank = comm->getRank(); diff --git a/packages/tpetra/core/test/CrsMatrix/sumIntoStaticProfileExtraSpace.cpp b/packages/tpetra/core/test/CrsMatrix/sumIntoStaticProfileExtraSpace.cpp index a202399d2984..4bc458ae213b 100644 --- a/packages/tpetra/core/test/CrsMatrix/sumIntoStaticProfileExtraSpace.cpp +++ b/packages/tpetra/core/test/CrsMatrix/sumIntoStaticProfileExtraSpace.cpp @@ -44,6 +44,7 @@ #include "Tpetra_TestingUtilities.hpp" #include "Tpetra_CrsMatrix.hpp" #include "Tpetra_Core.hpp" +#include "Tpetra_Vector.hpp" #include "Tpetra_Map.hpp" #include "Tpetra_Details_Behavior.hpp" #include "Teuchos_CommHelpers.hpp" @@ -62,6 +63,7 @@ TEUCHOS_UNIT_TEST( CrsMatrix, sumIntoStaticProfileExtraSpace ) using crs_matrix_type = Tpetra::CrsMatrix<>; using LO = map_type::local_ordinal_type; using GO = map_type::global_ordinal_type; + using SC = Tpetra::Vector<>::scalar_type; int lclSuccess = 1; int gblSuccess = 0; @@ -102,7 +104,7 @@ TEUCHOS_UNIT_TEST( CrsMatrix, sumIntoStaticProfileExtraSpace ) out << "Insert local column indices 1,2 into each row" << endl; Teuchos::OSTab tab2 (out); - const double vals[2] = { 5.0, 6.0 }; + const SC vals[2] = { 5.0, 6.0 }; const LO inds[2] = { LO (1), LO (2) }; const LO numToInsert = 2; @@ -135,7 +137,7 @@ TEUCHOS_UNIT_TEST( CrsMatrix, sumIntoStaticProfileExtraSpace ) out << "Try to sumInto a local index that should not exist" << endl; Teuchos::OSTab tab2 (out); - const double vals[2] = { 20.0 }; + const SC vals[2] = { 20.0 }; const LO inds[2] = { LO (0) }; // not in graph/matrix yet const LO numToInsert = 1; From ee7d8125bb074807d829c4d90ba83599127ca9ed Mon Sep 17 00:00:00 2001 From: Chris Siefert Date: Tue, 8 Mar 2022 20:21:12 -0700 Subject: [PATCH 02/19] Amesos2: Fixing tests to work with SC != double --- packages/amesos2/example/SimpleSolve.cpp | 2 +- packages/amesos2/example/SimpleSolveNonContigMap.cpp | 2 +- packages/amesos2/example/SimpleSolveTranspose.cpp | 2 +- packages/amesos2/example/SimpleSolve_File.cpp | 2 +- packages/amesos2/example/quick_solve.cpp | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/amesos2/example/SimpleSolve.cpp b/packages/amesos2/example/SimpleSolve.cpp index cc4c712e28dc..7340fe8c5dae 100644 --- a/packages/amesos2/example/SimpleSolve.cpp +++ b/packages/amesos2/example/SimpleSolve.cpp @@ -70,7 +70,7 @@ int main(int argc, char *argv[]) { Tpetra::ScopeGuard tpetraScope(&argc,&argv); - typedef double Scalar; + typedef Tpetra::CrsMatrix<>::scalar_type Scalar; typedef Tpetra::Map<>::local_ordinal_type LO; typedef Tpetra::Map<>::global_ordinal_type GO; diff --git a/packages/amesos2/example/SimpleSolveNonContigMap.cpp b/packages/amesos2/example/SimpleSolveNonContigMap.cpp index e88bbdc71639..e045df12c2a3 100644 --- a/packages/amesos2/example/SimpleSolveNonContigMap.cpp +++ b/packages/amesos2/example/SimpleSolveNonContigMap.cpp @@ -94,7 +94,7 @@ int main(int argc, char *argv[]) { } - typedef double Scalar; + typedef Tpetra::CrsMatrix<>::scalar_type Scalar; typedef Tpetra::Map<>::local_ordinal_type LO; typedef Tpetra::Map<>::global_ordinal_type GO; diff --git a/packages/amesos2/example/SimpleSolveTranspose.cpp b/packages/amesos2/example/SimpleSolveTranspose.cpp index 0f80e0a7f2f5..162fbbf9c442 100644 --- a/packages/amesos2/example/SimpleSolveTranspose.cpp +++ b/packages/amesos2/example/SimpleSolveTranspose.cpp @@ -64,7 +64,7 @@ int main(int argc, char *argv[]) { Tpetra::ScopeGuard tpetraScope(&argc,&argv); - typedef double Scalar; + typedef Tpetra::CrsMatrix<>::scalar_type Scalar; typedef Tpetra::Map<>::local_ordinal_type LO; typedef Tpetra::Map<>::global_ordinal_type GO; diff --git a/packages/amesos2/example/SimpleSolve_File.cpp b/packages/amesos2/example/SimpleSolve_File.cpp index 4f47f536fa96..9b354d8d7444 100644 --- a/packages/amesos2/example/SimpleSolve_File.cpp +++ b/packages/amesos2/example/SimpleSolve_File.cpp @@ -63,7 +63,7 @@ int main(int argc, char *argv[]) { Tpetra::ScopeGuard tpetraScope(&argc,&argv); - typedef double Scalar; + typedef Tpetra::CrsMatrix<>::scalar_type Scalar; typedef Tpetra::Map<>::local_ordinal_type LO; typedef Tpetra::Map<>::global_ordinal_type GO; diff --git a/packages/amesos2/example/quick_solve.cpp b/packages/amesos2/example/quick_solve.cpp index ef4aed210465..27664a10a914 100644 --- a/packages/amesos2/example/quick_solve.cpp +++ b/packages/amesos2/example/quick_solve.cpp @@ -70,7 +70,7 @@ int main(int argc, char *argv[]) { Tpetra::ScopeGuard tpetraScope(&argc,&argv); - typedef double Scalar; + typedef Tpetra::CrsMatrix<>::scalar_type Scalar; typedef Teuchos::ScalarTraits::magnitudeType Magnitude; typedef Tpetra::Map<>::local_ordinal_type LO; typedef Tpetra::Map<>::global_ordinal_type GO; From 6304b4f8bc13eb5b7d36413b88f5a4fe3c5c6c16 Mon Sep 17 00:00:00 2001 From: Chris Siefert Date: Tue, 8 Mar 2022 20:41:14 -0700 Subject: [PATCH 03/19] Tpetra: Fixing precision issues in reader --- .../core/test/inout/MatrixMarket_Tpetra_CrsMatrix_Dist.cpp | 3 ++- .../test/inout/MatrixMarket_Tpetra_CrsMatrix_Dist_Binary.cpp | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/tpetra/core/test/inout/MatrixMarket_Tpetra_CrsMatrix_Dist.cpp b/packages/tpetra/core/test/inout/MatrixMarket_Tpetra_CrsMatrix_Dist.cpp index 46352e65511e..1ab1dddaeb8c 100644 --- a/packages/tpetra/core/test/inout/MatrixMarket_Tpetra_CrsMatrix_Dist.cpp +++ b/packages/tpetra/core/test/inout/MatrixMarket_Tpetra_CrsMatrix_Dist.cpp @@ -424,7 +424,8 @@ class TestReader { const Teuchos::RCP &y_test ) { - const scalar_t epsilon = 0.0000001; + const scalar_t epsilon = 10*Teuchos::ScalarTraits::squareroot(Teuchos::ScalarTraits::eps()); + int ierr = 0; // First compare the norms of the result vector to the baseline diff --git a/packages/tpetra/core/test/inout/MatrixMarket_Tpetra_CrsMatrix_Dist_Binary.cpp b/packages/tpetra/core/test/inout/MatrixMarket_Tpetra_CrsMatrix_Dist_Binary.cpp index 0e6342707908..c397923ffc1d 100644 --- a/packages/tpetra/core/test/inout/MatrixMarket_Tpetra_CrsMatrix_Dist_Binary.cpp +++ b/packages/tpetra/core/test/inout/MatrixMarket_Tpetra_CrsMatrix_Dist_Binary.cpp @@ -546,7 +546,7 @@ class TestReader { const Teuchos::RCP &y_test ) { - const scalar_t epsilon = 0.0000001; + const scalar_t epsilon = 10*Teuchos::ScalarTraits::squareroot(Teuchos::ScalarTraits::eps()); int ierr = 0; // First compare the norms of the result vector to the baseline From 8cb569952e174dcda0f335dbf2363261ac18d641 Mon Sep 17 00:00:00 2001 From: Chris Siefert Date: Tue, 8 Mar 2022 20:53:14 -0700 Subject: [PATCH 04/19] Ifpack2: Making the configure if SC=double is disabled --- packages/ifpack2/test/belos/CMakeLists.txt | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/packages/ifpack2/test/belos/CMakeLists.txt b/packages/ifpack2/test/belos/CMakeLists.txt index 7d7423829d69..0c0fb8f44ba0 100644 --- a/packages/ifpack2/test/belos/CMakeLists.txt +++ b/packages/ifpack2/test/belos/CMakeLists.txt @@ -112,9 +112,10 @@ TRIBITS_COPY_FILES_TO_BINARY_DIR(Ifpack2BelosCopyFiles SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR} DEST_DIR ${CMAKE_CURRENT_BINARY_DIR} - EXEDEPS tif_belos + #EXEDEPS tif_belos ) +IF (Tpetra_INST_DOUBLE) TRIBITS_ADD_TEST( tif_belos NAME small_gmres_belos @@ -123,6 +124,7 @@ TRIBITS_ADD_TEST( NUM_MPI_PROCS 1 STANDARD_PASS_OUTPUT ) +ENDIF() IF (Tpetra_INST_LONG_DOUBLE) TRIBITS_ADD_TEST( @@ -146,6 +148,8 @@ TRIBITS_ADD_TEST( ) ENDIF() + +IF (Tpetra_INST_DOUBLE) TRIBITS_ADD_TEST( tif_belos NAME small_pseudoblkcg_belos @@ -307,8 +311,10 @@ TRIBITS_ADD_TEST( NUM_MPI_PROCS 4 STANDARD_PASS_OUTPUT ) +ENDIF() -IF(Ifpack2_ENABLE_ShyLU_NodeHTS) + +IF(Tpetra_INST_DOUBLE AND Ifpack2_ENABLE_ShyLU_NodeHTS) TRIBITS_ADD_TEST( tif_belos NAME RILUK_HTS_hb_belos @@ -383,6 +389,7 @@ IF(Ifpack2_ENABLE_QD) ) ENDIF() +IF (Tpetra_INST_DOUBLE) TRIBITS_ADD_TEST( tif_belos NAME ILUT_5w_2 @@ -400,8 +407,9 @@ TRIBITS_ADD_TEST( NUM_MPI_PROCS 1 STANDARD_PASS_OUTPUT ) +ENDIF() -IF(Ifpack2_ENABLE_ShyLU_NodeFastILU) +IF(Tpetra_INST_DOUBLE AND Ifpack2_ENABLE_ShyLU_NodeFastILU) #calore1 TRIBITS_ADD_TEST( tif_belos Ifpack2BelosCopyFiles @@ -529,6 +537,7 @@ ENDIF() # STANDARD_PASS_OUTPUT # ) +IF (Tpetra_INST_DOUBLE) TRIBITS_ADD_TEST( tif_belos NAME Cheby_belos @@ -647,6 +656,7 @@ TRIBITS_ADD_TEST( NUM_MPI_PROCS 1 STANDARD_PASS_OUTPUT ) +ENDIF() TRIBITS_ADD_TEST( tif_tpetra_native From cd277d00955c1be099055cb78c580e297cbdadbf Mon Sep 17 00:00:00 2001 From: Chris Siefert Date: Tue, 8 Mar 2022 21:55:47 -0700 Subject: [PATCH 05/19] Ifpack2: Compiling w/ SC != double --- packages/ifpack2/example/CMakeLists.txt | 2 + .../src/Ifpack2_BlockRelaxation_def.hpp | 2 +- .../src/Ifpack2_LinePartitioner_decl.hpp | 6 +- .../src/Ifpack2_LinePartitioner_def.hpp | 58 ++++++++++--------- .../ifpack2/test/unit_tests/CMakeLists.txt | 2 +- 5 files changed, 38 insertions(+), 32 deletions(-) diff --git a/packages/ifpack2/example/CMakeLists.txt b/packages/ifpack2/example/CMakeLists.txt index 39bc6f16a22a..7cbe114a02c3 100644 --- a/packages/ifpack2/example/CMakeLists.txt +++ b/packages/ifpack2/example/CMakeLists.txt @@ -6,7 +6,9 @@ TRIBITS_ADD_EXECUTABLE( SOURCES ex1.cpp ) +IF (Tpetra_INST_DOUBLE) TRIBITS_ADD_EXECUTABLE( RelaxationWithEquilibration SOURCES RelaxationWithEquilibration.cpp ) +ENDIF() \ No newline at end of file diff --git a/packages/ifpack2/src/Ifpack2_BlockRelaxation_def.hpp b/packages/ifpack2/src/Ifpack2_BlockRelaxation_def.hpp index 4f0537ef0270..007758a9726d 100644 --- a/packages/ifpack2/src/Ifpack2_BlockRelaxation_def.hpp +++ b/packages/ifpack2/src/Ifpack2_BlockRelaxation_def.hpp @@ -158,7 +158,7 @@ getValidParameters () const validParams->set("partitioner: line detection threshold", 0.0); validParams->set("partitioner: PDE equations", 1); - Teuchos::RCP::magnitudeType, typename MatrixType::local_ordinal_type, typename MatrixType::global_ordinal_type, typename MatrixType::node_type> > dummy; diff --git a/packages/ifpack2/src/Ifpack2_LinePartitioner_decl.hpp b/packages/ifpack2/src/Ifpack2_LinePartitioner_decl.hpp index 1678526e5391..53a6b8c4546a 100644 --- a/packages/ifpack2/src/Ifpack2_LinePartitioner_decl.hpp +++ b/packages/ifpack2/src/Ifpack2_LinePartitioner_decl.hpp @@ -45,6 +45,7 @@ #include "Ifpack2_ConfigDefs.hpp" #include "Ifpack2_OverlappingPartitioner.hpp" +#include "Teuchos_ScalarTraits.hpp" #include "Tpetra_MultiVector.hpp" namespace Ifpack2 { @@ -80,7 +81,8 @@ class LinePartitioner : public OverlappingPartitioner { typedef typename GraphType::global_ordinal_type global_ordinal_type; typedef typename GraphType::node_type node_type; typedef Tpetra::RowGraph row_graph_type; - typedef Tpetra::MultiVector multivector_type; + typedef typename Teuchos::ScalarTraits::magnitudeType magnitude_type; + typedef Tpetra::MultiVector multivector_type; typedef typename row_graph_type::nonconst_global_inds_host_view_type nonconst_global_inds_host_view_type; typedef typename row_graph_type::nonconst_local_inds_host_view_type nonconst_local_inds_host_view_type; @@ -100,7 +102,7 @@ class LinePartitioner : public OverlappingPartitioner { private: // Useful functions int Compute_Blocks_AutoLine(Teuchos::ArrayView blockIndices) const; - void local_automatic_line_search(int NumEqns, Teuchos::ArrayView blockIndices, local_ordinal_type last, local_ordinal_type next, local_ordinal_type LineID, double tol, Teuchos::Array itemp, Teuchos::Array dtemp) const; + void local_automatic_line_search(int NumEqns, Teuchos::ArrayView blockIndices, local_ordinal_type last, local_ordinal_type next, local_ordinal_type LineID, double tol, Teuchos::Array itemp, Teuchos::Array dtemp) const; diff --git a/packages/ifpack2/src/Ifpack2_LinePartitioner_def.hpp b/packages/ifpack2/src/Ifpack2_LinePartitioner_def.hpp index 5d7ef779f58e..a908885046ce 100644 --- a/packages/ifpack2/src/Ifpack2_LinePartitioner_def.hpp +++ b/packages/ifpack2/src/Ifpack2_LinePartitioner_def.hpp @@ -109,11 +109,12 @@ void LinePartitioner::computePartitions() { template int LinePartitioner::Compute_Blocks_AutoLine(Teuchos::ArrayView blockIndices) const { typedef local_ordinal_type LO; + typedef magnitude_type MT; const LO invalid = Teuchos::OrdinalTraits::invalid(); - const double zero = Teuchos::ScalarTraits::zero(); + const MT zero = Teuchos::ScalarTraits::zero(); - Teuchos::ArrayRCP xvalsRCP, yvalsRCP, zvalsRCP; - Teuchos::ArrayView xvals, yvals, zvals; + Teuchos::ArrayRCP xvalsRCP, yvalsRCP, zvalsRCP; + Teuchos::ArrayView xvals, yvals, zvals; xvalsRCP = coord_->getData(0); xvals = xvalsRCP(); if(coord_->getNumVectors() > 1) { yvalsRCP = coord_->getData(1); yvals = yvalsRCP(); } if(coord_->getNumVectors() > 2) { zvalsRCP = coord_->getData(2); zvals = zvalsRCP(); } @@ -124,10 +125,10 @@ int LinePartitioner::Compute_Blocks_AutoLine(Teuchos::ArrayVie nonconst_local_inds_host_view_type cols("cols",allocated_space); Teuchos::Array indices(allocated_space); - Teuchos::Array dist(allocated_space); + Teuchos::Array dist(allocated_space); Teuchos::Array itemp(2*allocated_space); - Teuchos::Array dtemp(allocated_space); + Teuchos::Array dtemp(allocated_space); LO num_lines = 0; @@ -138,24 +139,24 @@ int LinePartitioner::Compute_Blocks_AutoLine(Teuchos::ArrayVie // Get neighbors and sort by distance this->Graph_->getLocalRowCopy(i,cols,nz); - double x0 = (!xvals.is_null()) ? xvals[i/NumEqns_] : zero; - double y0 = (!yvals.is_null()) ? yvals[i/NumEqns_] : zero; - double z0 = (!zvals.is_null()) ? zvals[i/NumEqns_] : zero; + MT x0 = (!xvals.is_null()) ? xvals[i/NumEqns_] : zero; + MT y0 = (!yvals.is_null()) ? yvals[i/NumEqns_] : zero; + MT z0 = (!zvals.is_null()) ? zvals[i/NumEqns_] : zero; LO neighbor_len=0; for(size_t j=0; j=(LO)N) continue; // Check for off-proc entries - if(!xvals.is_null()) mydist += square(x0 - xvals[nn]); - if(!yvals.is_null()) mydist += square(y0 - yvals[nn]); - if(!zvals.is_null()) mydist += square(z0 - zvals[nn]); - dist[neighbor_len] = Teuchos::ScalarTraits::squareroot(mydist); + if(!xvals.is_null()) mydist += square(x0 - xvals[nn]); + if(!yvals.is_null()) mydist += square(y0 - yvals[nn]); + if(!zvals.is_null()) mydist += square(z0 - zvals[nn]); + dist[neighbor_len] = Teuchos::ScalarTraits::squareroot(mydist); indices[neighbor_len]=cols[j]; neighbor_len++; } - Teuchos::ArrayView dist_view = dist(0,neighbor_len); + Teuchos::ArrayView dist_view = dist(0,neighbor_len); Tpetra::sort2(dist_view.begin(),dist_view.end(),indices.begin()); // Number myself @@ -177,13 +178,14 @@ int LinePartitioner::Compute_Blocks_AutoLine(Teuchos::ArrayVie } // ============================================================================ template -void LinePartitioner::local_automatic_line_search(int NumEqns, Teuchos::ArrayView blockIndices, local_ordinal_type last, local_ordinal_type next, local_ordinal_type LineID, double tol, Teuchos::Array itemp, Teuchos::Array dtemp) const { +void LinePartitioner::local_automatic_line_search(int NumEqns, Teuchos::ArrayView blockIndices, local_ordinal_type last, local_ordinal_type next, local_ordinal_type LineID, double tol, Teuchos::Array itemp, Teuchos::Array dtemp) const { typedef local_ordinal_type LO; + typedef magnitude_type MT; const LO invalid = Teuchos::OrdinalTraits::invalid(); - const double zero = Teuchos::ScalarTraits::zero(); + const MT zero = Teuchos::ScalarTraits::zero(); - Teuchos::ArrayRCP xvalsRCP, yvalsRCP, zvalsRCP; - Teuchos::ArrayView xvals, yvals, zvals; + Teuchos::ArrayRCP xvalsRCP, yvalsRCP, zvalsRCP; + Teuchos::ArrayView xvals, yvals, zvals; xvalsRCP = coord_->getData(0); xvals = xvalsRCP(); if(coord_->getNumVectors() > 1) { yvalsRCP = coord_->getData(1); yvals = yvalsRCP(); } if(coord_->getNumVectors() > 2) { zvalsRCP = coord_->getData(2); zvals = zvalsRCP(); } @@ -193,7 +195,7 @@ void LinePartitioner::local_automatic_line_search(int NumEqns, nonconst_local_inds_host_view_type cols(itemp.data(),allocated_space); Teuchos::ArrayView indices = itemp.view(allocated_space,allocated_space); - Teuchos::ArrayView dist= dtemp(); + Teuchos::ArrayView dist= dtemp(); while (blockIndices[next] == invalid) { // Get the next row @@ -201,21 +203,21 @@ void LinePartitioner::local_automatic_line_search(int NumEqns, LO neighbors_in_line=0; this->Graph_->getLocalRowCopy(next,cols,nz); - double x0 = (!xvals.is_null()) ? xvals[next/NumEqns_] : zero; - double y0 = (!yvals.is_null()) ? yvals[next/NumEqns_] : zero; - double z0 = (!zvals.is_null()) ? zvals[next/NumEqns_] : zero; + MT x0 = (!xvals.is_null()) ? xvals[next/NumEqns_] : zero; + MT y0 = (!yvals.is_null()) ? yvals[next/NumEqns_] : zero; + MT z0 = (!zvals.is_null()) ? zvals[next/NumEqns_] : zero; // Calculate neighbor distances & sort LO neighbor_len=0; for(size_t i=0; i=(LO)N) continue; // Check for off-proc entries LO nn = cols[i] / NumEqns; if(blockIndices[nn]==LineID) neighbors_in_line++; - if(!xvals.is_null()) mydist += square(x0 - xvals[nn]); - if(!yvals.is_null()) mydist += square(y0 - yvals[nn]); - if(!zvals.is_null()) mydist += square(z0 - zvals[nn]); - dist[neighbor_len] = Teuchos::ScalarTraits::squareroot(mydist); + if(!xvals.is_null()) mydist += square(x0 - xvals[nn]); + if(!yvals.is_null()) mydist += square(y0 - yvals[nn]); + if(!zvals.is_null()) mydist += square(z0 - zvals[nn]); + dist[neighbor_len] = Teuchos::ScalarTraits::squareroot(mydist); indices[neighbor_len]=cols[i]; neighbor_len++; } @@ -228,7 +230,7 @@ void LinePartitioner::local_automatic_line_search(int NumEqns, blockIndices[next + k] = LineID; // Try to find the next guy in the line (only check the closest two that aren't element 0 (diagonal)) - Teuchos::ArrayView dist_view = dist(0,neighbor_len); + Teuchos::ArrayView dist_view = dist(0,neighbor_len); Tpetra::sort2(dist_view.begin(),dist_view.end(),indices.begin()); if(neighbor_len > 2 && indices[1] != last && blockIndices[indices[1]] == -1 && dist[1]/dist[neighbor_len-1] < tol) { diff --git a/packages/ifpack2/test/unit_tests/CMakeLists.txt b/packages/ifpack2/test/unit_tests/CMakeLists.txt index 139136723deb..1d08c2b9e6af 100644 --- a/packages/ifpack2/test/unit_tests/CMakeLists.txt +++ b/packages/ifpack2/test/unit_tests/CMakeLists.txt @@ -133,7 +133,7 @@ TRIBITS_ADD_EXECUTABLE_AND_TEST( STANDARD_PASS_OUTPUT ) -IF(${PACKAGE_NAME}_ENABLE_Experimental_KokkosKernels_Features) +IF(Tpetra_INST_DOUBLE AND ${PACKAGE_NAME}_ENABLE_Experimental_KokkosKernels_Features) TRIBITS_ADD_EXECUTABLE_AND_TEST( BlockTriDiContainerUnitAndPerfTests SOURCES From 107e2b48aa53a6e0e12694d21ebdb7b5325a48f8 Mon Sep 17 00:00:00 2001 From: Chris Siefert Date: Wed, 9 Mar 2022 09:28:54 -0700 Subject: [PATCH 06/19] Belos: Making tests compile w/ SC!=double --- .../tpetra/src/BelosTpetraTestFramework.hpp | 8 +++++++ .../tpetra/test/BiCGStab/test_bicgstab_hb.cpp | 10 ++++----- .../test/BlockCG/test_bl_cg_complex_hb.cpp | 4 ++-- .../tpetra/test/BlockCG/test_bl_cg_hb.cpp | 3 +-- .../test/BlockCG/test_pseudo_bl_cg_hb.cpp | 3 +-- .../BlockCG/test_pseudo_stochastic_cg_hb.cpp | 3 +-- .../test/BlockGmres/test_bl_fgmres_hb.cpp | 5 ++--- .../test/BlockGmres/test_bl_gmres_hb.cpp | 3 +-- .../test/BlockGmres/test_bl_gmres_hb_df.cpp | 11 ++++++++-- .../test/BlockGmres/test_hybrid_gmres_hb.cpp | 3 +-- .../tpetra/test/FixedPoint/test_fp_hb.cpp | 3 +-- .../tpetra/test/GCRODR/test_block_gcrodr.cpp | 6 +++--- .../test/GCRODR/test_gcrodr_complex_hb.cpp | 4 ++-- .../tpetra/test/GCRODR/test_gcrodr_hb.cpp | 3 +-- packages/belos/tpetra/test/Issue_3235.cpp | 3 +-- .../tpetra/test/Native/nonsymm_matrix.cpp | 2 +- .../belos_orthomanager_tpetra.cpp | 7 ++++--- .../belos_orthomanager_tpetra_benchmark.cpp | 6 +++--- .../belos_orthomanager_tpetra_util.hpp | 21 ++++++++++++------- .../belos/tpetra/test/RCG/test_rcg_hb.cpp | 2 +- .../belos/tpetra/test/TFQMR/test_tfqmr_hb.cpp | 10 ++++----- 21 files changed, 66 insertions(+), 54 deletions(-) diff --git a/packages/belos/tpetra/src/BelosTpetraTestFramework.hpp b/packages/belos/tpetra/src/BelosTpetraTestFramework.hpp index f2e3769bcb68..e979908f5708 100644 --- a/packages/belos/tpetra/src/BelosTpetraTestFramework.hpp +++ b/packages/belos/tpetra/src/BelosTpetraTestFramework.hpp @@ -69,6 +69,14 @@ namespace Belos { retVals[i] = dPtr[i]; } + template<> + void convertData( const double* dPtr, const int nnz, float* retVals ) + { + for ( int i=0; i void convertData( const double* dPtr, const int nnz, std::complex* retVals ) { diff --git a/packages/belos/tpetra/test/BiCGStab/test_bicgstab_hb.cpp b/packages/belos/tpetra/test/BiCGStab/test_bicgstab_hb.cpp index 4479a36f327a..9bdfc38a6699 100644 --- a/packages/belos/tpetra/test/BiCGStab/test_bicgstab_hb.cpp +++ b/packages/belos/tpetra/test/BiCGStab/test_bicgstab_hb.cpp @@ -64,7 +64,7 @@ int main (int argc, char *argv[]) using std::cout; using std::endl; - typedef double ST; + typedef Tpetra::MultiVector<>::scalar_type ST; typedef Teuchos::ScalarTraits SCT; typedef SCT::magnitudeType MT; typedef Tpetra::Operator OP; @@ -151,8 +151,8 @@ int main (int argc, char *argv[]) // // Construct a preconditioned linear problem // - RCP > problem - = rcp (new Belos::LinearProblem (A, X, B)); + RCP > problem + = rcp (new Belos::LinearProblem (A, X, B)); bool set = problem->setProblem (); if (! set) { if (proc_verbose) { @@ -162,8 +162,8 @@ int main (int argc, char *argv[]) } // Create a Belos solver. - RCP > solver - = rcp (new Belos::BiCGStabSolMgr (problem, belosList)); + RCP > solver + = rcp (new Belos::BiCGStabSolMgr (problem, belosList)); if (proc_verbose) { cout << endl << endl; diff --git a/packages/belos/tpetra/test/BlockCG/test_bl_cg_complex_hb.cpp b/packages/belos/tpetra/test/BlockCG/test_bl_cg_complex_hb.cpp index d305df251ba5..f470943e96b9 100644 --- a/packages/belos/tpetra/test/BlockCG/test_bl_cg_complex_hb.cpp +++ b/packages/belos/tpetra/test/BlockCG/test_bl_cg_complex_hb.cpp @@ -70,8 +70,8 @@ using std::vector; using Teuchos::tuple; int main(int argc, char *argv[]) { - - typedef std::complex ST; + typedef Tpetra::MultiVector<>::scalar_type BSC; + typedef std::complex ST; typedef ScalarTraits SCT; typedef SCT::magnitudeType MT; typedef Tpetra::Operator OP; diff --git a/packages/belos/tpetra/test/BlockCG/test_bl_cg_hb.cpp b/packages/belos/tpetra/test/BlockCG/test_bl_cg_hb.cpp index 9eab2dab965f..6737d5536335 100644 --- a/packages/belos/tpetra/test/BlockCG/test_bl_cg_hb.cpp +++ b/packages/belos/tpetra/test/BlockCG/test_bl_cg_hb.cpp @@ -71,8 +71,7 @@ using std::vector; using Teuchos::tuple; int main(int argc, char *argv[]) { - - typedef double ST; + typedef Tpetra::MultiVector<>::scalar_type ST; typedef ScalarTraits SCT; typedef SCT::magnitudeType MT; typedef Tpetra::Operator OP; diff --git a/packages/belos/tpetra/test/BlockCG/test_pseudo_bl_cg_hb.cpp b/packages/belos/tpetra/test/BlockCG/test_pseudo_bl_cg_hb.cpp index 6eae93afa0da..f7f6f032b4a1 100644 --- a/packages/belos/tpetra/test/BlockCG/test_pseudo_bl_cg_hb.cpp +++ b/packages/belos/tpetra/test/BlockCG/test_pseudo_bl_cg_hb.cpp @@ -71,8 +71,7 @@ using std::vector; using Teuchos::tuple; int main(int argc, char *argv[]) { - - typedef double ST; + typedef Tpetra::MultiVector<>::scalar_type ST; typedef ScalarTraits SCT; typedef SCT::magnitudeType MT; typedef Tpetra::Operator OP; diff --git a/packages/belos/tpetra/test/BlockCG/test_pseudo_stochastic_cg_hb.cpp b/packages/belos/tpetra/test/BlockCG/test_pseudo_stochastic_cg_hb.cpp index 1b9740f67772..2969bdde91b3 100644 --- a/packages/belos/tpetra/test/BlockCG/test_pseudo_stochastic_cg_hb.cpp +++ b/packages/belos/tpetra/test/BlockCG/test_pseudo_stochastic_cg_hb.cpp @@ -70,8 +70,7 @@ main (int argc, char *argv[]) using Teuchos::rcpFromRef; using std::endl; using std::cout; - - typedef double ST; + typedef Tpetra::MultiVector<>::scalar_type ST; typedef Teuchos::ScalarTraits STS; typedef STS::magnitudeType MT; typedef Tpetra::Operator OP; diff --git a/packages/belos/tpetra/test/BlockGmres/test_bl_fgmres_hb.cpp b/packages/belos/tpetra/test/BlockGmres/test_bl_fgmres_hb.cpp index 4217c55ac16f..9c048327554e 100644 --- a/packages/belos/tpetra/test/BlockGmres/test_bl_fgmres_hb.cpp +++ b/packages/belos/tpetra/test/BlockGmres/test_bl_fgmres_hb.cpp @@ -72,8 +72,7 @@ using Teuchos::rcp; using Teuchos::ParameterList; int main(int argc, char *argv[]) { - - typedef double ST; + typedef Tpetra::MultiVector<>::scalar_type ST; typedef Teuchos::ScalarTraits SCT; typedef SCT::magnitudeType MT; typedef Tpetra::Operator OP; @@ -186,7 +185,7 @@ int main(int argc, char *argv[]) { innerBelosList.set( "Timer Label", "Belos Preconditioner Solve" );// Choose a different label for the inner solve // *****Construct linear problem for the inner iteration using A ***** - Belos::LinearProblem innerProblem; + Belos::LinearProblem innerProblem; innerProblem.setOperator( A ); innerProblem.setLabel( "Belos Preconditioner Solve" ); diff --git a/packages/belos/tpetra/test/BlockGmres/test_bl_gmres_hb.cpp b/packages/belos/tpetra/test/BlockGmres/test_bl_gmres_hb.cpp index a0e401b844b2..3bbf1a1c9f23 100644 --- a/packages/belos/tpetra/test/BlockGmres/test_bl_gmres_hb.cpp +++ b/packages/belos/tpetra/test/BlockGmres/test_bl_gmres_hb.cpp @@ -69,8 +69,7 @@ using std::vector; using Teuchos::tuple; int main(int argc, char *argv[]) { - - typedef double ST; + typedef Tpetra::MultiVector<>::scalar_type ST; typedef ScalarTraits SCT; typedef SCT::magnitudeType MT; typedef Tpetra::Operator OP; diff --git a/packages/belos/tpetra/test/BlockGmres/test_bl_gmres_hb_df.cpp b/packages/belos/tpetra/test/BlockGmres/test_bl_gmres_hb_df.cpp index 09a53f8b1828..2c36ad87a081 100644 --- a/packages/belos/tpetra/test/BlockGmres/test_bl_gmres_hb_df.cpp +++ b/packages/belos/tpetra/test/BlockGmres/test_bl_gmres_hb_df.cpp @@ -330,9 +330,16 @@ int main(int argc, char *argv[]) // run tests for different scalar types double ltol = tol; - double ftime[3]; int fiter; bool fpass = runTest(ltol,ftime,fiter,vmap); + double ftime[3] = {0,0,0}; int fiter=0; + bool fpass = true, dpass = true; +#ifdef HAVE_TPETRA_INST_FLOAT + fpass = runTest(ltol,ftime,fiter,vmap); +#endif ltol = (reduce_tol ? ltol*ltol : ltol); - double dtime[3]; int diter; bool dpass = runTest(ltol,dtime,diter,vmap); + double dtime[3] = {0,0,0}; int diter=0; +#ifdef HAVE_TPETRA_INST_DOUBLE + dpass = runTest(ltol,dtime,diter,vmap); +#endif // done with the matrix data now; delete it if (mptestmypid == 0) { diff --git a/packages/belos/tpetra/test/BlockGmres/test_hybrid_gmres_hb.cpp b/packages/belos/tpetra/test/BlockGmres/test_hybrid_gmres_hb.cpp index 28555e40a56b..b693dc0662ee 100644 --- a/packages/belos/tpetra/test/BlockGmres/test_hybrid_gmres_hb.cpp +++ b/packages/belos/tpetra/test/BlockGmres/test_hybrid_gmres_hb.cpp @@ -79,8 +79,7 @@ using std::vector; using Teuchos::tuple; int main(int argc, char *argv[]) { - - typedef double ST; + typedef Tpetra::MultiVector<>::scalar_type ST; typedef ScalarTraits SCT; typedef SCT::magnitudeType MT; typedef Tpetra::Operator OP; diff --git a/packages/belos/tpetra/test/FixedPoint/test_fp_hb.cpp b/packages/belos/tpetra/test/FixedPoint/test_fp_hb.cpp index d62886db239a..fdb921224032 100644 --- a/packages/belos/tpetra/test/FixedPoint/test_fp_hb.cpp +++ b/packages/belos/tpetra/test/FixedPoint/test_fp_hb.cpp @@ -71,8 +71,7 @@ using std::vector; using Teuchos::tuple; int main(int argc, char *argv[]) { - - typedef double ST; + typedef Tpetra::MultiVector<>::scalar_type ST; typedef ScalarTraits SCT; typedef SCT::magnitudeType MT; typedef Tpetra::Operator OP; diff --git a/packages/belos/tpetra/test/GCRODR/test_block_gcrodr.cpp b/packages/belos/tpetra/test/GCRODR/test_block_gcrodr.cpp index a209f8ef3df0..b6d1d9d53abe 100644 --- a/packages/belos/tpetra/test/GCRODR/test_block_gcrodr.cpp +++ b/packages/belos/tpetra/test/GCRODR/test_block_gcrodr.cpp @@ -73,9 +73,9 @@ main (int argc, char *argv[]) // // Typedefs for Tpetra template arguments. // - typedef double scalar_type; - typedef long int global_ordinal_type; - typedef int local_ordinal_type; + typedef Tpetra::MultiVector<>::scalar_type scalar_type; + typedef Tpetra::MultiVector<>::local_ordinal_type local_ordinal_type; + typedef Tpetra::MultiVector<>::global_ordinal_type global_ordinal_type; // // Tpetra objects which are the MV and OP template parameters of the // Belos specialization which we are testing. diff --git a/packages/belos/tpetra/test/GCRODR/test_gcrodr_complex_hb.cpp b/packages/belos/tpetra/test/GCRODR/test_gcrodr_complex_hb.cpp index 1c1999ab5e98..0e86cf30faa2 100644 --- a/packages/belos/tpetra/test/GCRODR/test_gcrodr_complex_hb.cpp +++ b/packages/belos/tpetra/test/GCRODR/test_gcrodr_complex_hb.cpp @@ -67,8 +67,8 @@ using std::vector; using Teuchos::tuple; int main(int argc, char *argv[]) { - - typedef std::complex ST; + typedef Tpetra::MultiVector<>::scalar_type BSC; + typedef std::complex ST; typedef ScalarTraits SCT; typedef SCT::magnitudeType MT; typedef Tpetra::Operator OP; diff --git a/packages/belos/tpetra/test/GCRODR/test_gcrodr_hb.cpp b/packages/belos/tpetra/test/GCRODR/test_gcrodr_hb.cpp index 7f58deca8f11..76b237b095a8 100644 --- a/packages/belos/tpetra/test/GCRODR/test_gcrodr_hb.cpp +++ b/packages/belos/tpetra/test/GCRODR/test_gcrodr_hb.cpp @@ -67,8 +67,7 @@ using std::vector; using Teuchos::tuple; int main(int argc, char *argv[]) { - - typedef double ST; + typedef Tpetra::MultiVector<>::scalar_type ST; typedef ScalarTraits SCT; typedef SCT::magnitudeType MT; typedef Tpetra::Operator OP; diff --git a/packages/belos/tpetra/test/Issue_3235.cpp b/packages/belos/tpetra/test/Issue_3235.cpp index 545402f0b081..6605ba570019 100644 --- a/packages/belos/tpetra/test/Issue_3235.cpp +++ b/packages/belos/tpetra/test/Issue_3235.cpp @@ -13,8 +13,7 @@ int main(int argc, char *argv[]) { using Teuchos::ParameterList; using Teuchos::RCP; using Teuchos::rcp; - - using SC = double; + using SC = Tpetra::MultiVector<>::scalar_type; using crs_matrix_type = Tpetra::CrsMatrix; using map_type = Tpetra::Map<>; using OP = Tpetra::Operator; diff --git a/packages/belos/tpetra/test/Native/nonsymm_matrix.cpp b/packages/belos/tpetra/test/Native/nonsymm_matrix.cpp index 27048c3dc4cb..cc2e54da0e2b 100644 --- a/packages/belos/tpetra/test/Native/nonsymm_matrix.cpp +++ b/packages/belos/tpetra/test/Native/nonsymm_matrix.cpp @@ -14,7 +14,7 @@ namespace { // (anonymous) struct CommandLineOptions { std::string solverName {"TPETRA GMRES"}; - double offDiagDiff = 1.0 / 8.0; + Tpetra::MultiVector<>::scalar_type offDiagDiff = 1.0 / 8.0; // mfh 14 Aug 2018: GMRES takes 20 iterations on this problem (with // offDiagDiff = 1/8). We add 10 iterations to allow for rounding // error and differences in the algorithm. diff --git a/packages/belos/tpetra/test/OrthoManager/belos_orthomanager_tpetra.cpp b/packages/belos/tpetra/test/OrthoManager/belos_orthomanager_tpetra.cpp index 7ec168450d20..92f729220d6e 100644 --- a/packages/belos/tpetra/test/OrthoManager/belos_orthomanager_tpetra.cpp +++ b/packages/belos/tpetra/test/OrthoManager/belos_orthomanager_tpetra.cpp @@ -91,13 +91,14 @@ static void getCmdLineArgs (const Teuchos::Comm& comm, int argc, char* argv[]) { using Teuchos::CommandLineProcessor; + typedef Tpetra::MultiVector<>::scalar_type ST; // Define a OrthoManagerFactory to use to get the names of valid // orthogonalization manager types. We won't use this factory to // create them, so we should be able to pick the Scalar, MV, and // OP template parameters as we wish. typedef Belos::OrthoManagerFactory, Tpetra::Operator > factory_type; + Tpetra::MultiVector, Tpetra::Operator > factory_type; factory_type factory; //////////////////////////////////////////////////////////////////// @@ -407,7 +408,7 @@ main (int argc, char *argv[]) typedef Tpetra::Map<>::node_type node_type; { - typedef double scalar_type; + typedef Tpetra::MultiVector<>::scalar_type scalar_type; success = runTest (comm); if (success) { @@ -425,7 +426,7 @@ main (int argc, char *argv[]) } #if defined(HAVE_TEUCHOS_COMPLEX) && defined(HAVE_TPETRA_COMPLEX) { - typedef std::complex scalar_type; + typedef std::complex::scalar_type> scalar_type; success = runTest (comm); if (success) { diff --git a/packages/belos/tpetra/test/OrthoManager/belos_orthomanager_tpetra_benchmark.cpp b/packages/belos/tpetra/test/OrthoManager/belos_orthomanager_tpetra_benchmark.cpp index 4d1e288a8203..ac729f0b481c 100644 --- a/packages/belos/tpetra/test/OrthoManager/belos_orthomanager_tpetra_benchmark.cpp +++ b/packages/belos/tpetra/test/OrthoManager/belos_orthomanager_tpetra_benchmark.cpp @@ -56,8 +56,8 @@ using std::endl; // // These typedefs make main() as generic as possible. -// -typedef double scalar_type; +// +typedef Tpetra::MultiVector<>::scalar_type scalar_type; typedef Tpetra::Map<>::local_ordinal_type local_ordinal_type; typedef Tpetra::Map<>::global_ordinal_type global_ordinal_type; typedef Tpetra::Map<>::node_type node_type; @@ -234,7 +234,7 @@ main (int argc, char *argv[]) // modify numRows to be the total number of rows in the sparse // matrix. Otherwise, it will leave numRows alone. std::pair, RCP > results = - loadSparseMatrix (pComm, filename, numRows, debugOut); + loadSparseMatrix (pComm, filename, numRows, debugOut); map = results.first; M = results.second; } diff --git a/packages/belos/tpetra/test/OrthoManager/belos_orthomanager_tpetra_util.hpp b/packages/belos/tpetra/test/OrthoManager/belos_orthomanager_tpetra_util.hpp index 3473fa68446f..ffb4655d3b40 100644 --- a/packages/belos/tpetra/test/OrthoManager/belos_orthomanager_tpetra_util.hpp +++ b/packages/belos/tpetra/test/OrthoManager/belos_orthomanager_tpetra_util.hpp @@ -44,6 +44,7 @@ #include // includes BelosConfigDefs.hpp #include #include +#include #include #include #include @@ -107,33 +108,36 @@ namespace Belos { /// distribution of the sparse matrix: we distribute in a way such /// that the domain, range, and row maps are the same) and a /// sparse_matrix_type (the sparse matrix itself). - template - std::pair >, Teuchos::RCP > > + template + std::pair >, Teuchos::RCP<::Tpetra::CrsMatrix > > loadSparseMatrix (const Teuchos::RCP< const Teuchos::Comm > pComm, const std::string& filename, int& numRows, std::ostream& debugOut) { - typedef double scalar_type; + typedef SC scalar_type; typedef LO local_ordinal_type; typedef GO global_ordinal_type; typedef NodeType node_type; - typedef Tpetra::Map map_type; - typedef Tpetra::CrsMatrix sparse_matrix_type; + typedef ::Tpetra::Map map_type; + typedef ::Tpetra::CrsMatrix sparse_matrix_type; using Teuchos::RCP; using Teuchos::rcp; using std::vector; - const int myRank = Teuchos::rank (*pComm); + // const int myRank = Teuchos::rank (*pComm); RCP pMap; RCP pMatrix; if (filename != "") { debugOut << "Loading sparse matrix file \"" << filename << "\"" << endl; - + Belos::Tpetra::HarwellBoeingReader reader(pComm); + pMatrix = reader.readFromFile(filename); + pMap = Teuchos::rcp_const_cast(pMatrix->getRowMap()); +#if 0 int loadedNumRows = 0; int numCols = 0; int nnz = -1; @@ -274,6 +278,7 @@ namespace Belos { // among the processes. The domain, range, and row maps are // the same (the matrix must be square). pMatrix->fillComplete(); +#endif debugOut << "Completed loading and distributing sparse matrix" << endl; } // else M == null else @@ -283,7 +288,7 @@ namespace Belos { // Let M remain null, and allocate map using the number of rows // (numRows) specified on the command line. pMap = rcp (new map_type (numRows, 0, pComm, - Tpetra::GloballyDistributed)); + ::Tpetra::GloballyDistributed)); } return std::make_pair (pMap, pMatrix); } diff --git a/packages/belos/tpetra/test/RCG/test_rcg_hb.cpp b/packages/belos/tpetra/test/RCG/test_rcg_hb.cpp index e66044df42df..0b98639de93b 100644 --- a/packages/belos/tpetra/test/RCG/test_rcg_hb.cpp +++ b/packages/belos/tpetra/test/RCG/test_rcg_hb.cpp @@ -71,7 +71,7 @@ using Teuchos::tuple; int main(int argc, char *argv[]) { - typedef double ST; + typedef Tpetra::MultiVector<>::scalar_type ST; typedef ScalarTraits SCT; typedef SCT::magnitudeType MT; typedef Tpetra::Operator OP; diff --git a/packages/belos/tpetra/test/TFQMR/test_tfqmr_hb.cpp b/packages/belos/tpetra/test/TFQMR/test_tfqmr_hb.cpp index bb4363d680ac..56b780c61f4d 100644 --- a/packages/belos/tpetra/test/TFQMR/test_tfqmr_hb.cpp +++ b/packages/belos/tpetra/test/TFQMR/test_tfqmr_hb.cpp @@ -60,7 +60,7 @@ int main(int argc, char *argv[]) { // - typedef double ST; + typedef Tpetra::MultiVector<>::scalar_type ST; typedef Teuchos::ScalarTraits SCT; typedef SCT::magnitudeType MT; typedef Tpetra::Operator OP; @@ -158,7 +158,7 @@ int main(int argc, char *argv[]) { // // Construct an unpreconditioned linear problem instance. // - Belos::LinearProblem problem( A, X, B ); + Belos::LinearProblem problem( A, X, B ); bool set = problem.setProblem(); if (set == false) { if (proc_verbose) @@ -168,12 +168,12 @@ int main(int argc, char *argv[]) { // // Create an iterative solver manager. // - RCP< Belos::SolverManager > solver; + RCP< Belos::SolverManager > solver; if (pseudo) - solver = rcp( new Belos::PseudoBlockTFQMRSolMgr(rcp(&problem,false), rcp(&belosList,false)) ); + solver = rcp( new Belos::PseudoBlockTFQMRSolMgr(rcp(&problem,false), rcp(&belosList,false)) ); else - solver = rcp( new Belos::TFQMRSolMgr(rcp(&problem,false), rcp(&belosList,false)) ); + solver = rcp( new Belos::TFQMRSolMgr(rcp(&problem,false), rcp(&belosList,false)) ); // // **********Print out information about problem******************* // From 283aa0ca1bbce9321a854ad23d0854caad0a06a0 Mon Sep 17 00:00:00 2001 From: Chris Siefert Date: Wed, 9 Mar 2022 10:39:41 -0700 Subject: [PATCH 07/19] Tpetra: Fixing examples --- .../advanced/Benchmarks/CrsMatrixDenseRowUnpack.cpp | 3 ++- .../advanced/Benchmarks/CrsMatrix_sumIntoLocalValues.cpp | 9 +++++---- packages/tpetra/core/example/inout/NewReaderExample.cpp | 3 ++- .../core/guide/src/Examples/SourceCode/data_redist_1.cpp | 2 +- .../core/guide/src/Examples/SourceCode/matrix_fill_1.cpp | 2 +- 5 files changed, 11 insertions(+), 8 deletions(-) diff --git a/packages/tpetra/core/example/advanced/Benchmarks/CrsMatrixDenseRowUnpack.cpp b/packages/tpetra/core/example/advanced/Benchmarks/CrsMatrixDenseRowUnpack.cpp index e406241c8325..e60b8d4f8929 100644 --- a/packages/tpetra/core/example/advanced/Benchmarks/CrsMatrixDenseRowUnpack.cpp +++ b/packages/tpetra/core/example/advanced/Benchmarks/CrsMatrixDenseRowUnpack.cpp @@ -60,12 +60,13 @@ using Teuchos::rcp; using Teuchos::Time; using Teuchos::TimeMonitor; +using SC = Tpetra::CrsMatrix<>::scalar_type; using GST = Tpetra::global_size_t; using map_type = Tpetra::Map<>; using LO = map_type::local_ordinal_type; using GO = map_type::global_ordinal_type; using crs_graph_type = Tpetra::CrsGraph<>; -using crs_matrix_type = Tpetra::CrsMatrix; +using crs_matrix_type = Tpetra::CrsMatrix; using export_type = Tpetra::Export<>; struct CmdLineArgs { diff --git a/packages/tpetra/core/example/advanced/Benchmarks/CrsMatrix_sumIntoLocalValues.cpp b/packages/tpetra/core/example/advanced/Benchmarks/CrsMatrix_sumIntoLocalValues.cpp index 9c26a26d9091..46687f41bf12 100644 --- a/packages/tpetra/core/example/advanced/Benchmarks/CrsMatrix_sumIntoLocalValues.cpp +++ b/packages/tpetra/core/example/advanced/Benchmarks/CrsMatrix_sumIntoLocalValues.cpp @@ -66,6 +66,7 @@ using crs_matrix_type = Tpetra::CrsMatrix<>; using map_type = Tpetra::Map<>; using LO = map_type::local_ordinal_type; using GO = map_type::global_ordinal_type; +using SC = crs_matrix_type::scalar_type; RCP createRowAndColMap (RCP > comm, @@ -105,7 +106,7 @@ void populateCrsMatrix (crs_matrix_type& A, const LO numToInsert, const LO lclColInds[], - const double vals[]) + const SC vals[]) { //TM mon (*TM::getNewCounter ("CrsMatrix::insertLocalValues loop")); @@ -145,7 +146,7 @@ doSumIntoLocalValues (const std::string& label, crs_matrix_type& A, const LO numToInsert, const LO lclColInds[], - const double vals[], + const SC vals[], const int numTrials) { TM mon (*TM::getNewCounter (label)); @@ -171,7 +172,7 @@ doKokkosSumIntoLocalValues (const std::string& label, crs_matrix_type& A, const LO numToInsert, const LO lclColInds[], - const double vals[], + const SC vals[], const int numTrials) { TM mon (*TM::getNewCounter (label)); @@ -213,7 +214,7 @@ benchmarkCrsMatrixSumIntoLocalValues (const CmdLineArgs args) std::vector lclColInds (size_t (args.numToInsert)); - std::vector vals (size_t (args.numToInsert), 0.0); + std::vector vals (size_t (args.numToInsert), 0.0); // Skip 0, so that search isn't completely trivial. std::iota (lclColInds.begin (), lclColInds.end (), LO (1)); diff --git a/packages/tpetra/core/example/inout/NewReaderExample.cpp b/packages/tpetra/core/example/inout/NewReaderExample.cpp index e546b6d5f83b..bfc7539e584c 100644 --- a/packages/tpetra/core/example/inout/NewReaderExample.cpp +++ b/packages/tpetra/core/example/inout/NewReaderExample.cpp @@ -102,7 +102,8 @@ int main(int narg, char *arg[]) params.set("chunkSize", (size_t)chunkSize); // Call readSparseFile to read the file - using matrix_t = Tpetra::CrsMatrix; + using scalar_t = Tpetra::CrsMatrix<>::scalar_type; + using matrix_t = Tpetra::CrsMatrix; Teuchos::RCP Amat; try { using reader_t = Tpetra::MatrixMarket::Reader; diff --git a/packages/tpetra/core/guide/src/Examples/SourceCode/data_redist_1.cpp b/packages/tpetra/core/guide/src/Examples/SourceCode/data_redist_1.cpp index 8b475facb6ef..5abf32446af0 100644 --- a/packages/tpetra/core/guide/src/Examples/SourceCode/data_redist_1.cpp +++ b/packages/tpetra/core/guide/src/Examples/SourceCode/data_redist_1.cpp @@ -123,7 +123,7 @@ example(const Teuchos::RCP >& comm, using GST = Tpetra::global_size_t; // Set up Tpetra typedefs. - using SC = double; + using SC = Tpetra::CrsMatrix<>::scalar_type; using crs_matrix_type = Tpetra::CrsMatrix; using map_type = Tpetra::Map<>; using GO = Tpetra::Map<>::global_ordinal_type; diff --git a/packages/tpetra/core/guide/src/Examples/SourceCode/matrix_fill_1.cpp b/packages/tpetra/core/guide/src/Examples/SourceCode/matrix_fill_1.cpp index b5876beaf4be..a933556aa7bb 100644 --- a/packages/tpetra/core/guide/src/Examples/SourceCode/matrix_fill_1.cpp +++ b/packages/tpetra/core/guide/src/Examples/SourceCode/matrix_fill_1.cpp @@ -100,7 +100,7 @@ main(int argc, char *argv[]) using Teuchos::rcp; using Teuchos::RCP; using GST = Tpetra::global_size_t; - using scalar_type = double; + using scalar_type = Tpetra::CrsMatrix<>::scalar_type; using crs_matrix_type = Tpetra::CrsMatrix; using map_type = Tpetra::Map<>; using global_ordinal_type = Tpetra::Map<>::global_ordinal_type; From fc799bdb6d31323a6bdcd3f79f539af6fc175287 Mon Sep 17 00:00:00 2001 From: Chris Siefert Date: Wed, 9 Mar 2022 10:58:21 -0700 Subject: [PATCH 08/19] Xpetra: Fixing code for SC!=double --- packages/xpetra/example/Simple/Laplace1D-Tpetra.cpp | 2 +- packages/xpetra/src/MultiVector/Xpetra_MultiVector_def.hpp | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/xpetra/example/Simple/Laplace1D-Tpetra.cpp b/packages/xpetra/example/Simple/Laplace1D-Tpetra.cpp index 365e54885db3..bbe8312c69cf 100644 --- a/packages/xpetra/example/Simple/Laplace1D-Tpetra.cpp +++ b/packages/xpetra/example/Simple/Laplace1D-Tpetra.cpp @@ -57,7 +57,7 @@ int main (int argc, char *argv[]) using Teuchos::RCP; using Teuchos::rcp; using Teuchos::tuple; - typedef double Scalar; + typedef Tpetra::CrsMatrix<>::scalar_type Scalar; typedef Tpetra::CrsMatrix crs_matrix_type; typedef Tpetra::Map<> map_type; typedef Tpetra::Map<>::global_ordinal_type GO; diff --git a/packages/xpetra/src/MultiVector/Xpetra_MultiVector_def.hpp b/packages/xpetra/src/MultiVector/Xpetra_MultiVector_def.hpp index dd2dcf40892a..d57d86484224 100644 --- a/packages/xpetra/src/MultiVector/Xpetra_MultiVector_def.hpp +++ b/packages/xpetra/src/MultiVector/Xpetra_MultiVector_def.hpp @@ -95,6 +95,7 @@ MultiVector:: Xpetra_randomize(const Scalar& minVal, const Scalar& maxVal) { typedef Teuchos::ScalarTraits SCT; + Scalar point5 = SCT::one()/ (SCT::one() + SCT::one()); const size_t numVectors = getNumVectors(); for(size_t i = 0; i < numVectors; i++) @@ -104,7 +105,7 @@ Xpetra_randomize(const Scalar& minVal, const Scalar& maxVal) const size_t myLength = getLocalLength(); for(size_t j = 0; j < myLength; j++) { - datai[ j ] = 0.5*(maxVal-minVal)*SCT::random()+0.5*(maxVal+minVal); + datai[ j ] = point5*(maxVal-minVal)*SCT::random()+point5*(maxVal+minVal); } } } From aff02400d4dd26632c85dd7f219429616de9b5f7 Mon Sep 17 00:00:00 2001 From: Chris Siefert Date: Wed, 9 Mar 2022 16:58:16 -0700 Subject: [PATCH 09/19] Galeri: Fixing SC!=double --- .../src-xpetra/Galeri_StencilProblems.hpp | 26 +++++++++++-------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/packages/galeri/src-xpetra/Galeri_StencilProblems.hpp b/packages/galeri/src-xpetra/Galeri_StencilProblems.hpp index 3f14faa0ac3a..c727df934cae 100644 --- a/packages/galeri/src-xpetra/Galeri_StencilProblems.hpp +++ b/packages/galeri/src-xpetra/Galeri_StencilProblems.hpp @@ -212,7 +212,7 @@ namespace Galeri { Teuchos::ParameterList list = this->list_; GlobalOrdinal nx = -1; GlobalOrdinal ny = -1; - + using MT = typename Teuchos::ScalarTraits::magnitudeType; if (list.isParameter("nx")) { if (list.isType("nx")) nx = Teuchos::as(list.get("nx")); @@ -226,10 +226,13 @@ namespace Galeri { ny = list.get("ny"); } - Scalar one = 1.0; - Scalar zero = 0.0; + Scalar zero = 0.0; + Scalar one = 1.0; Scalar two = one+one; + MT one_MT = 1.0; + MT two_MT = one_MT+one_MT; + if (nx == -1 || ny == -1) { GlobalOrdinal n = this->Map_->getGlobalNumElements(); nx = (GlobalOrdinal)sqrt((double)n); @@ -238,7 +241,7 @@ namespace Galeri { } bool keepBCs = this->list_.get("keepBCs", false); - Scalar dtInv = one / (Scalar) this->list_.get("dt", one); + MT dtInv = one_MT / this->list_.get("dt", one_MT); Scalar Kxx = (Scalar) this->list_.get("Kxx", one); Scalar Kxy = (Scalar) this->list_.get("Kxy", zero); Scalar Kyy = (Scalar) this->list_.get("Kyy", one); @@ -252,8 +255,9 @@ namespace Galeri { // z1 d z2 if (meshType == "tri") { - Scalar massDiag = one/two/((Scalar) ((nx+1)*(ny+1))); - Scalar massOffDiag = massDiag/6.0; + MT massDiag_MT = one_MT/two_MT/(Teuchos::as((nx+1)*(ny+1))); + Scalar massDiag = massDiag_MT; + Scalar massOffDiag = massDiag_MT/6.0; c = -Kxx + Kxy + dtInv * massOffDiag; b = -Kxx + Kxy + dtInv * massOffDiag; @@ -266,11 +270,11 @@ namespace Galeri { z4 = -Kxy + dtInv * massOffDiag; } else if (meshType == "quad") { Scalar mass = one/((Scalar) ((nx+1)*(ny+1))) / (Scalar) 36.0; - Scalar four_thirds = (Scalar) 4.0/3.0; - Scalar third = (Scalar) 1.0/3.0; - Scalar two_thirds = (Scalar) 2.0/3.0; - Scalar half = (Scalar) 1.0/2.0; - Scalar sixth = (Scalar) 1.0/6.0; + Scalar four_thirds = (Scalar) (4.0/3.0); + Scalar third = (Scalar) (1.0/3.0); + Scalar two_thirds = (Scalar) (2.0/3.0); + Scalar half = (Scalar) (1.0/2.0); + Scalar sixth = (Scalar) (1.0/6.0); a = four_thirds*Kxx + four_thirds*Kyy + dtInv * mass * (Scalar) 16.0; From 67590e6a5568d816568b8eb16ab64211213ff8fc Mon Sep 17 00:00:00 2001 From: Chris Siefert Date: Wed, 9 Mar 2022 16:58:26 -0700 Subject: [PATCH 10/19] MueLu: Fixing SC!=double --- packages/muelu/CMakeLists.txt | 29 +++++++++++++++-- .../advanced/memory/MueLu_MatrixDriver.cpp | 4 ++- .../advanced/memory/Tpetra1DLaplace.cpp | 2 +- ...ueLu_CoupledAggregationCommHelper_decl.hpp | 3 +- ...ueLu_LeftoverAggregationAlgorithm_decl.hpp | 6 ++-- ...MueLu_LeftoverAggregationAlgorithm_def.hpp | 31 +++++++++++-------- .../MueLu_CoalesceDropFactory_kokkos_def.hpp | 3 +- .../MueLu_SmooVecCoalesceDropFactory_def.hpp | 13 ++++---- .../MueLu_UncoupledAggregationFactory_def.hpp | 2 +- .../Headers/MueLu_Details_DefaultTypes.hpp | 14 ++++++++- .../MueLu_RebalanceMapFactory_decl.hpp | 2 +- .../MueLu_SaPFactory_def.hpp | 4 ++- .../MueLu_TentativePFactory_def.hpp | 4 +-- .../muelu/src/Utils/MueLu_PerfUtils_def.hpp | 5 ++- .../src/Utils/MueLu_UtilitiesBase_decl.hpp | 3 +- packages/muelu/test/scaling/MatrixLoad.hpp | 12 +++---- .../test/unit_tests/BlockedRepartition.cpp | 8 +++++ .../unit_tests_kokkos/SaPFactory_kokkos.cpp | 2 +- 18 files changed, 105 insertions(+), 42 deletions(-) diff --git a/packages/muelu/CMakeLists.txt b/packages/muelu/CMakeLists.txt index ac3ad637fbd7..702728563c6a 100644 --- a/packages/muelu/CMakeLists.txt +++ b/packages/muelu/CMakeLists.txt @@ -243,9 +243,14 @@ GLOBAL_SET(HAVE_${PACKAGE_NAME_UC}_DEFAULT_GO_LONG OFF) GLOBAL_SET(HAVE_${PACKAGE_NAME_UC}_DEFAULT_GO_LONGLONG OFF) IF(${PACKAGE_NAME}_ENABLE_EXPLICIT_INSTANTIATION) + GLOBAL_SET(${PACKAGE_NAME}_HAVE_GO_INT OFF) + GLOBAL_SET(${PACKAGE_NAME}_HAVE_GO_LONG OFF) + GLOBAL_SET(${PACKAGE_NAME}_HAVE_GO_LONG_LONG OFF) + # # Examples and tests need at least this one, also if Epetra is enabled IF(Tpetra_INST_DOUBLE AND Tpetra_INST_INT_INT) + GLOBAL_SET(${PACKAGE_NAME}_HAS_GO_INT ON) GLOBAL_SET(${PACKAGE_NAME}_INST_DOUBLE_INT_INT ON) GLOBAL_SET(HAVE_${PACKAGE_NAME_UC}_INST_DOUBLE_INT_INT ON) ELSE() @@ -267,6 +272,7 @@ IF(${PACKAGE_NAME}_ENABLE_EXPLICIT_INSTANTIATION) # IF(Tpetra_INST_DOUBLE AND Tpetra_INST_INT_LONG) + GLOBAL_SET(${PACKAGE_NAME}_HAVE_GO_LONG ON) GLOBAL_SET(${PACKAGE_NAME}_INST_DOUBLE_INT_LONGINT ON) GLOBAL_SET(HAVE_${PACKAGE_NAME_UC}_INST_DOUBLE_INT_LONGINT ON) ELSE() @@ -277,6 +283,7 @@ IF(${PACKAGE_NAME}_ENABLE_EXPLICIT_INSTANTIATION) # IF(Tpetra_INST_DOUBLE AND Tpetra_INST_INT_LONG_LONG) + GLOBAL_SET(${PACKAGE_NAME}_HAVE_GO_LONG_LONG ON) GLOBAL_SET(${PACKAGE_NAME}_INST_DOUBLE_INT_LONGLONGINT ON) GLOBAL_SET(HAVE_${PACKAGE_NAME_UC}_INST_DOUBLE_INT_LONGLONGINT ON) ELSE() @@ -286,6 +293,7 @@ IF(${PACKAGE_NAME}_ENABLE_EXPLICIT_INSTANTIATION) # IF(Tpetra_INST_COMPLEX_DOUBLE AND Tpetra_INST_INT_INT) + GLOBAL_SET(${PACKAGE_NAME}_HAVE_GO_INT ON) GLOBAL_SET(${PACKAGE_NAME}_INST_COMPLEX_INT_INT ON) GLOBAL_SET(HAVE_${PACKAGE_NAME_UC}_INST_COMPLEX_INT_INT ON) ELSE() @@ -295,6 +303,7 @@ IF(${PACKAGE_NAME}_ENABLE_EXPLICIT_INSTANTIATION) # IF(Tpetra_INST_COMPLEX_DOUBLE AND Tpetra_INST_INT_LONG_LONG) + GLOBAL_SET(${PACKAGE_NAME}_HAVE_GO_LONG_LONG ON) GLOBAL_SET(${PACKAGE_NAME}_INST_COMPLEX_INT_LONG_LONG ON) GLOBAL_SET(HAVE_${PACKAGE_NAME_UC}_INST_COMPLEX_INT_LONG_LONG ON) ELSE() @@ -304,6 +313,7 @@ IF(${PACKAGE_NAME}_ENABLE_EXPLICIT_INSTANTIATION) # IF(Tpetra_INST_FLOAT AND Tpetra_INST_INT_INT) + GLOBAL_SET(${PACKAGE_NAME}_HAVE_GO_LONG ON) GLOBAL_SET(${PACKAGE_NAME}_INST_FLOAT_INT_INT ON) GLOBAL_SET(HAVE_${PACKAGE_NAME_UC}_INST_FLOAT_INT_INT ON) ELSE() @@ -311,8 +321,19 @@ IF(${PACKAGE_NAME}_ENABLE_EXPLICIT_INSTANTIATION) GLOBAL_SET(HAVE_${PACKAGE_NAME_UC}_INST_FLOAT_INT_INT OFF) ENDIF() +# + IF(Tpetra_INST_FLOAT AND Tpetra_INST_INT_LONG_LONG) + GLOBAL_SET(${PACKAGE_NAME}_HAVE_GO_LONG_LONG ON) + GLOBAL_SET(${PACKAGE_NAME}_INST_FLOAT_INT_LONG_LONG ON) + GLOBAL_SET(HAVE_${PACKAGE_NAME_UC}_INST_FLOAT_INT_LONG_LONG ON) + ELSE() + GLOBAL_SET(${PACKAGE_NAME}_INST_FLOAT_INT_LONG_LONG OFF) + GLOBAL_SET(HAVE_${PACKAGE_NAME_UC}_INST_FLOAT_INT_LONG_LONG OFF) + ENDIF() + MESSAGE(STATUS "${PACKAGE_NAME}: Enabling ETI support") MESSAGE(STATUS " : ${${PACKAGE_NAME}_INST_FLOAT_INT_INT}") + MESSAGE(STATUS " : ${${PACKAGE_NAME}_INST_FLOAT_INT_LONG_LONG}") MESSAGE(STATUS " : ${${PACKAGE_NAME}_INST_DOUBLE_INT_INT}") MESSAGE(STATUS " : ${${PACKAGE_NAME}_INST_DOUBLE_INT_LONGINT}") MESSAGE(STATUS " : ${${PACKAGE_NAME}_INST_DOUBLE_INT_LONGLONGINT}") @@ -320,14 +341,18 @@ IF(${PACKAGE_NAME}_ENABLE_EXPLICIT_INSTANTIATION) MESSAGE(STATUS " : ${${PACKAGE_NAME}_INST_COMPLEX_INT_LONG_LONG}") # GO preference: int > long long > long - IF(NOT ${${PACKAGE_NAME}_INST_DOUBLE_INT_INT}) + IF(NOT ${${PACKAGE_NAME}_HAVE_GO_INT}) # No GO=int (first choice for default) - IF(NOT ${${PACKAGE_NAME}_INST_DOUBLE_INT_LONGLONGINT}) + IF(NOT ${${PACKAGE_NAME}_HAVE_GO_LONG_LONG}) #No GO=long long (second choice), must use long GLOBAL_SET(HAVE_${PACKAGE_NAME_UC}_DEFAULT_GO_LONG ON) + MESSAGE(STATUS "MueLu: Default GO: long") ELSE() GLOBAL_SET(HAVE_${PACKAGE_NAME_UC}_DEFAULT_GO_LONGLONG ON) + MESSAGE(STATUS "MueLu: Default GO: long long") ENDIF() + ELSE() + MESSAGE(STATUS "MueLu: Default GO: int") ENDIF() # If Tpetra is enabled, determine which Nodes are used diff --git a/packages/muelu/example/advanced/memory/MueLu_MatrixDriver.cpp b/packages/muelu/example/advanced/memory/MueLu_MatrixDriver.cpp index f2365f841445..d6b8fe929774 100644 --- a/packages/muelu/example/advanced/memory/MueLu_MatrixDriver.cpp +++ b/packages/muelu/example/advanced/memory/MueLu_MatrixDriver.cpp @@ -63,6 +63,8 @@ #include #include +#include + #ifdef _MSC_VER #define NOMINMAX #include // for Sleep @@ -83,7 +85,7 @@ int main(int argc, char** argv) typedef typename Tpetra::Map<>::local_ordinal_type LO; // LocalOrdinal typedef typename Tpetra::Map<>::global_ordinal_type GO; // GlobalOrdinal - typedef double SC; + typedef MueLu::DefaultScalar SC; Teuchos::oblackholestream blackhole; Teuchos::GlobalMPISession mpiSession(&argc,&argv,&blackhole); diff --git a/packages/muelu/example/advanced/memory/Tpetra1DLaplace.cpp b/packages/muelu/example/advanced/memory/Tpetra1DLaplace.cpp index 9c2fb47bc5ec..d356b0c3e991 100644 --- a/packages/muelu/example/advanced/memory/Tpetra1DLaplace.cpp +++ b/packages/muelu/example/advanced/memory/Tpetra1DLaplace.cpp @@ -62,7 +62,7 @@ int main(int argc, char *argv[]) { bool success = false; bool verbose = true; try { - typedef double Scalar; + typedef Tpetra::CrsMatrix<>::scalar_type Scalar; typedef Tpetra::Map<>::local_ordinal_type LO; #if defined(HAVE_TPETRA_INST_INT_INT) // mfh 07 Aug 2015: Prefer GO = int, for consistency with Epetra, diff --git a/packages/muelu/src/Graph/CoupledAggregation/MueLu_CoupledAggregationCommHelper_decl.hpp b/packages/muelu/src/Graph/CoupledAggregation/MueLu_CoupledAggregationCommHelper_decl.hpp index 50f2189a793a..ade76a880e53 100644 --- a/packages/muelu/src/Graph/CoupledAggregation/MueLu_CoupledAggregationCommHelper_decl.hpp +++ b/packages/muelu/src/Graph/CoupledAggregation/MueLu_CoupledAggregationCommHelper_decl.hpp @@ -57,6 +57,7 @@ #include "MueLu_Aggregates.hpp" + namespace MueLu { /*! @@ -70,8 +71,8 @@ namespace MueLu { class GlobalOrdinal = DefaultGlobalOrdinal, class Node = DefaultNode> class CoupledAggregationCommHelper : public BaseClass { + typedef DefaultScalar Scalar; - typedef double Scalar; // Scalar type only used for weight: always a double. #undef MUELU_COUPLEDAGGREGATIONCOMMHELPER_SHORT #include "MueLu_UseShortNames.hpp" diff --git a/packages/muelu/src/Graph/CoupledAggregation/MueLu_LeftoverAggregationAlgorithm_decl.hpp b/packages/muelu/src/Graph/CoupledAggregation/MueLu_LeftoverAggregationAlgorithm_decl.hpp index 008853f801d8..ca260d9f0aa2 100644 --- a/packages/muelu/src/Graph/CoupledAggregation/MueLu_LeftoverAggregationAlgorithm_decl.hpp +++ b/packages/muelu/src/Graph/CoupledAggregation/MueLu_LeftoverAggregationAlgorithm_decl.hpp @@ -58,6 +58,8 @@ #include "MueLu_CoupledAggregationCommHelper_fwd.hpp" + + namespace MueLu { template > & distWeights, const MueLu::CoupledAggregationCommHelper & myWidget) const; //RemoveSmallAggs + RCP > & distWeights, const MueLu::CoupledAggregationCommHelper & myWidget) const; //RemoveSmallAggs //@} diff --git a/packages/muelu/src/Graph/CoupledAggregation/MueLu_LeftoverAggregationAlgorithm_def.hpp b/packages/muelu/src/Graph/CoupledAggregation/MueLu_LeftoverAggregationAlgorithm_def.hpp index 088ee5e5a899..a1fc2e23ee5c 100644 --- a/packages/muelu/src/Graph/CoupledAggregation/MueLu_LeftoverAggregationAlgorithm_def.hpp +++ b/packages/muelu/src/Graph/CoupledAggregation/MueLu_LeftoverAggregationAlgorithm_def.hpp @@ -59,6 +59,11 @@ #include "MueLu_Exceptions.hpp" #include "MueLu_Monitor.hpp" +#ifdef HAVE_MUELU_TPETRA +#include +#endif + + namespace MueLu { template @@ -84,7 +89,7 @@ namespace MueLu { MueLu::CoupledAggregationCommHelper myWidget(uniqueMap, nonUniqueMap); //TODO JJH We want to skip this call - RCP > distWeights = Xpetra::VectorFactory::Build(nonUniqueMap); + RCP > distWeights = Xpetra::VectorFactory::Build(nonUniqueMap); // Aggregated vertices not "definitively" assigned to processors are // arbitrated by ArbitrateAndCommunicate(). There is some @@ -92,7 +97,7 @@ namespace MueLu { { ArrayRCP vertex2AggId = aggregates.GetVertex2AggId()->getData(0); ArrayRCP procWinner = aggregates.GetProcWinner()->getData(0); - ArrayRCP weights = distWeights->getDataNonConst(0); + ArrayRCP weights = distWeights->getDataNonConst(0); for (size_t i=0;igetLocalNumElements();i++) { if (procWinner[i] == MUELU_UNASSIGNED) { @@ -115,7 +120,7 @@ namespace MueLu { { ArrayRCP vertex2AggId = aggregates.GetVertex2AggId()->getDataNonConst(0); ArrayRCP procWinner = aggregates.GetProcWinner()->getData(0); - ArrayRCP weights = distWeights->getDataNonConst(0); + ArrayRCP weights = distWeights->getDataNonConst(0); for (my_size_t i = 0; i < nVertices; i++) { if ( aggregates.IsRoot(i) && (procWinner[i] == myPid) ) { @@ -162,7 +167,7 @@ namespace MueLu { /* base the number of new aggregates created on the percentage of */ /* unaggregated nodes. */ - ArrayRCP weights = distWeights->getDataNonConst(0); + ArrayRCP weights = distWeights->getDataNonConst(0); double factor = 1.; factor = ((double) total_phase_one_aggregated)/((double)(total_nVertices + 1)); @@ -228,16 +233,16 @@ namespace MueLu { // std::cout << "exp_nrows=" << exp_nRows << " (nVertices= " << nVertices << ", numGhost=" << graph.GetNodeNumGhost() << ")" << std::endl; // std::cout << "nonUniqueMap=" << nonUniqueMap->getLocalNumElements() << std::endl; - RCP > temp_ = Xpetra::VectorFactory ::Build(nonUniqueMap,false); //no need to zero out vector in ctor + RCP > temp_ = Xpetra::VectorFactory ::Build(nonUniqueMap,false); //no need to zero out vector in ctor temp_->putScalar(1.); - RCP > tempOutput_ = Xpetra::VectorFactory ::Build(nonUniqueMap); + RCP > tempOutput_ = Xpetra::VectorFactory ::Build(nonUniqueMap); myWidget.NonUnique2NonUnique(*temp_, *tempOutput_, Xpetra::ADD); std::vector gidNotShared(exp_nRows); { - ArrayRCP tempOutput = tempOutput_->getData(0); + ArrayRCP tempOutput = tempOutput_->getData(0); for (int i = 0; i < exp_nRows; i++) { if (tempOutput[i] > 1.) gidNotShared[i] = false; @@ -275,7 +280,7 @@ namespace MueLu { temp_->randomize(); - ArrayRCP temp = temp_->getDataNonConst(0); + ArrayRCP temp = temp_->getDataNonConst(0); // build a list of candidate root nodes (vertices not adjacent // to aggregated vertices) @@ -303,7 +308,7 @@ namespace MueLu { { ArrayRCP vertex2AggId = aggregates.GetVertex2AggId()->getDataNonConst(0); - ArrayRCP weights = distWeights->getDataNonConst(0); + ArrayRCP weights = distWeights->getDataNonConst(0); for (int k = 0; k < nCandidates; k++ ) { int i = candidates[k]; @@ -452,7 +457,7 @@ namespace MueLu { ArrayRCP vertex2AggId = aggregates.GetVertex2AggId()->getDataNonConst(0); ArrayRCP procWinner = aggregates.GetProcWinner()->getData(0); - ArrayRCP weights = distWeights->getDataNonConst(0); + ArrayRCP weights = distWeights->getDataNonConst(0); for (int i = 0; i < exp_nRows; i++) { @@ -580,7 +585,7 @@ namespace MueLu { { ArrayRCP vertex2AggId = aggregates.GetVertex2AggId()->getDataNonConst(0); - ArrayRCP weights = distWeights->getDataNonConst(0); + ArrayRCP weights = distWeights->getDataNonConst(0); ArrayRCP procWinner = aggregates.GetProcWinner()->getData(0); int count = 0; @@ -700,7 +705,7 @@ namespace MueLu { template int LeftoverAggregationAlgorithm::RemoveSmallAggs(Aggregates& aggregates, int min_size, - RCP > & distWeights, const MueLu::CoupledAggregationCommHelper & myWidget) const { + RCP > & distWeights, const MueLu::CoupledAggregationCommHelper & myWidget) const { int myPid = aggregates.GetMap()->getComm()->getRank(); LO nAggregates = aggregates.GetNumAggregates(); @@ -713,7 +718,7 @@ namespace MueLu { //aggregates.ComputeAggSizes(AggInfo); ArrayRCP AggInfo = aggregates.ComputeAggregateSizes(); - ArrayRCP weights = distWeights->getDataNonConst(0); + ArrayRCP weights = distWeights->getDataNonConst(0); // Make a list of all aggregates indicating New AggId // Use AggInfo array for this. diff --git a/packages/muelu/src/Graph/MatrixTransformation/MueLu_CoalesceDropFactory_kokkos_def.hpp b/packages/muelu/src/Graph/MatrixTransformation/MueLu_CoalesceDropFactory_kokkos_def.hpp index be3273db3f4e..adb1cbb8c9d4 100644 --- a/packages/muelu/src/Graph/MatrixTransformation/MueLu_CoalesceDropFactory_kokkos_def.hpp +++ b/packages/muelu/src/Graph/MatrixTransformation/MueLu_CoalesceDropFactory_kokkos_def.hpp @@ -495,7 +495,8 @@ namespace MueLu { FactoryMonitor m(*this, "Build", currentLevel); typedef Teuchos::ScalarTraits STS; - const SC zero = STS::zero(); + typedef typename STS::magnitudeType MT; + const MT zero = Teuchos::ScalarTraits::zero(); auto A = Get< RCP >(currentLevel, "A"); LO blkSize = A->GetFixedBlockSize(); diff --git a/packages/muelu/src/Graph/MatrixTransformation/MueLu_SmooVecCoalesceDropFactory_def.hpp b/packages/muelu/src/Graph/MatrixTransformation/MueLu_SmooVecCoalesceDropFactory_def.hpp index f5bb64e19a5c..997832ac98e5 100644 --- a/packages/muelu/src/Graph/MatrixTransformation/MueLu_SmooVecCoalesceDropFactory_def.hpp +++ b/packages/muelu/src/Graph/MatrixTransformation/MueLu_SmooVecCoalesceDropFactory_def.hpp @@ -421,10 +421,11 @@ namespace MueLu { template void SmooVecCoalesceDropFactory::badGuysDropfunc(LO row, const Teuchos::ArrayView& cols, const Teuchos::ArrayView& vals, const MultiVector& testVecs, LO nPDEs, Teuchos::ArrayRCP & penalties, const MultiVector& nearNull, Teuchos::ArrayRCP& Bcols, Teuchos::ArrayRCP& keepOrNot, LO &Nbcols, LO nLoc) const { + using TST=Teuchos::ScalarTraits; LO nLeng = cols.size(); - typename Teuchos::ScalarTraits::coordinateType temp; - temp = ((typename Teuchos::ScalarTraits::coordinateType) (row))/((typename Teuchos::ScalarTraits::coordinateType) (nPDEs)); + typename TST::coordinateType temp; + temp = ((typename TST::coordinateType) (row))/((typename TST::coordinateType) (nPDEs)); LO blkRow = as(floor(temp)); Teuchos::ArrayRCP badGuy( nLeng, 0.0); Teuchos::ArrayRCP subNull(nLeng, 0.0); /* subset of nearNull */ @@ -449,21 +450,21 @@ namespace MueLu { Teuchos::ArrayRCP< const Scalar > oneNull = nearNull.getData( as(rowDof)); for (LO i = 0; i < nLeng; i++) { - if ((cols[i] < nLoc ) && (vals[i] != 0.0)) { /* on processor */ - temp = ((typename Teuchos::ScalarTraits::coordinateType) (cols[i]))/((typename Teuchos::ScalarTraits::coordinateType) (nPDEs)); + if ((cols[i] < nLoc ) && (TST::magnitude(vals[i]) != 0.0)) { /* on processor */ + temp = ((typename TST::coordinateType) (cols[i]))/((typename TST::coordinateType) (nPDEs)); LO colDof = cols[i] - (as(floor( temp )))*nPDEs; if (colDof == rowDof) { /* same dof within node as row */ Bcols[ Nbcols] = (cols[i] - colDof)/nPDEs; subNull[Nbcols] = oneNull[cols[i]]; if (cols[i] != row) { /* not diagonal */ - Scalar worstRatio = -1.0; + Scalar worstRatio = -TST::one(); Scalar targetRatio = subNull[Nbcols]/oneNull[row]; Scalar actualRatio; for (size_t kk = 0; kk < testVecs.getNumVectors(); kk++ ) { Teuchos::ArrayRCP< const Scalar > curVec = testVecs.getData(kk); actualRatio = curVec[cols[i]]/curVec[row]; - if (Teuchos::ScalarTraits::magnitude(actualRatio - targetRatio) > Teuchos::ScalarTraits::magnitude(worstRatio)) { + if (TST::magnitude(actualRatio - targetRatio) > TST::magnitude(worstRatio)) { badGuy[Nbcols] = actualRatio; worstRatio = Teuchos::ScalarTraits::magnitude(actualRatio - targetRatio); } diff --git a/packages/muelu/src/Graph/UncoupledAggregation/MueLu_UncoupledAggregationFactory_def.hpp b/packages/muelu/src/Graph/UncoupledAggregation/MueLu_UncoupledAggregationFactory_def.hpp index 479bc5388b5f..18463940ba4d 100644 --- a/packages/muelu/src/Graph/UncoupledAggregation/MueLu_UncoupledAggregationFactory_def.hpp +++ b/packages/muelu/src/Graph/UncoupledAggregation/MueLu_UncoupledAggregationFactory_def.hpp @@ -299,7 +299,7 @@ namespace MueLu { Set(currentLevel, "Aggregates", aggregates); if (pL.get("aggregation: compute aggregate qualities")) { - RCP> aggQualities = Get>>(currentLevel, "AggregateQualities"); + RCP> aggQualities = Get>>(currentLevel, "AggregateQualities"); } if (IsPrint(Statistics1)) diff --git a/packages/muelu/src/Headers/MueLu_Details_DefaultTypes.hpp b/packages/muelu/src/Headers/MueLu_Details_DefaultTypes.hpp index bfda7733d9fb..23c66068585c 100644 --- a/packages/muelu/src/Headers/MueLu_Details_DefaultTypes.hpp +++ b/packages/muelu/src/Headers/MueLu_Details_DefaultTypes.hpp @@ -50,9 +50,21 @@ #include #include "MueLu_config.hpp" +#ifdef HAVE_MUELU_TPETRA +#include +#endif + namespace MueLu { - typedef double DefaultScalar; + +#ifdef HAVE_MUELU_TPETRA + typedef Tpetra::Details::DefaultTypes::scalar_type DefaultScalar; +#else + typedef double DefaultScalar; +#endif + + + typedef int DefaultLocalOrdinal; #if defined HAVE_MUELU_DEFAULT_GO_LONG diff --git a/packages/muelu/src/Rebalancing/MueLu_RebalanceMapFactory_decl.hpp b/packages/muelu/src/Rebalancing/MueLu_RebalanceMapFactory_decl.hpp index 9e0b57414c6e..cb223beee094 100644 --- a/packages/muelu/src/Rebalancing/MueLu_RebalanceMapFactory_decl.hpp +++ b/packages/muelu/src/Rebalancing/MueLu_RebalanceMapFactory_decl.hpp @@ -70,7 +70,7 @@ namespace MueLu { class GlobalOrdinal = DefaultGlobalOrdinal, class Node = DefaultNode> class RebalanceMapFactory : public SingleLevelFactoryBase { - typedef double Scalar; // FIXME this is just a dummy scalar for being able to access Xpetra::Matrix + typedef DefaultScalar Scalar; // FIXME this is just a dummy scalar for being able to access Xpetra::Matrix #undef MUELU_REBALANCEMAPFACTORY_SHORT #include "MueLu_UseShortNames.hpp" diff --git a/packages/muelu/src/Transfers/Smoothed-Aggregation/MueLu_SaPFactory_def.hpp b/packages/muelu/src/Transfers/Smoothed-Aggregation/MueLu_SaPFactory_def.hpp index 25594f147f3c..5ea16148eaad 100644 --- a/packages/muelu/src/Transfers/Smoothed-Aggregation/MueLu_SaPFactory_def.hpp +++ b/packages/muelu/src/Transfers/Smoothed-Aggregation/MueLu_SaPFactory_def.hpp @@ -119,6 +119,8 @@ namespace MueLu { const std::string prefix = "MueLu::SaPFactory(" + levelIDs + "): "; typedef typename Teuchos::ScalarTraits::coordinateType Coordinate; + typedef typename Teuchos::ScalarTraits::magnitudeType MT; + // Get default tentative prolongator factory // Getting it that way ensure that the same factory instance will be used for both SaPFactory and NullspaceFactory. @@ -162,7 +164,7 @@ namespace MueLu { const bool useAbsValueRowSum= pL.get ("sa: use rowsumabs diagonal scaling"); const bool doQRStep = pL.get("tentative: calculate qr"); const bool enforceConstraints= pL.get("sa: enforce constraints"); - const SC userDefinedMaxEigen = as(pL.get("sa: max eigenvalue")); + const MT userDefinedMaxEigen = as(pL.get("sa: max eigenvalue")); typedef typename Teuchos::ScalarTraits::magnitudeType Magnitude; double dTol = pL.get("sa: rowsumabs diagonal replacement tolerance"); const Magnitude diagonalReplacementTolerance = (dTol == as(-1) ? Teuchos::ScalarTraits::eps()*100 : as(pL.get("sa: rowsumabs diagonal replacement tolerance"))); diff --git a/packages/muelu/src/Transfers/Smoothed-Aggregation/MueLu_TentativePFactory_def.hpp b/packages/muelu/src/Transfers/Smoothed-Aggregation/MueLu_TentativePFactory_def.hpp index 96874b5f2b81..c5d76496c439 100644 --- a/packages/muelu/src/Transfers/Smoothed-Aggregation/MueLu_TentativePFactory_def.hpp +++ b/packages/muelu/src/Transfers/Smoothed-Aggregation/MueLu_TentativePFactory_def.hpp @@ -505,7 +505,7 @@ namespace MueLu { for (size_t k = 0, lnnz = 0; k < NSDim; k++) { // Skip zeros (there may be plenty of them, i.e., NSDim > 1 or boundary conditions) SC qr_jk = fineNS[k][aggToRowMapLO[aggStart[agg]+j]]; - if(constantColSums) qr_jk = qr_jk / (double)aggSizes[agg]; + if(constantColSums) qr_jk = qr_jk / (Magnitude)aggSizes[agg]; if (qr_jk != zero) { ja [rowStart+lnnz] = offset + k; val[rowStart+lnnz] = qr_jk; @@ -530,7 +530,7 @@ namespace MueLu { for (size_t k = 0, lnnz = 0; k < NSDim; ++k) { // Skip zeros (there may be plenty of them, i.e., NSDim > 1 or boundary conditions) SC qr_jk = fineNS[k][rowMap->getLocalElement(aggToRowMapGO[aggStart[agg]+j])]; - if(constantColSums) qr_jk = qr_jk / (double)aggSizes[agg]; + if(constantColSums) qr_jk = qr_jk / (Magnitude)aggSizes[agg]; if (qr_jk != zero) { ja [rowStart+lnnz] = offset + k; val[rowStart+lnnz] = qr_jk; diff --git a/packages/muelu/src/Utils/MueLu_PerfUtils_def.hpp b/packages/muelu/src/Utils/MueLu_PerfUtils_def.hpp index b9bd4fdf2c0a..3d715d88c2f3 100644 --- a/packages/muelu/src/Utils/MueLu_PerfUtils_def.hpp +++ b/packages/muelu/src/Utils/MueLu_PerfUtils_def.hpp @@ -68,8 +68,11 @@ namespace MueLu { void calculateStats(Type& minVal, Type& maxVal, double& avgVal, double& devVal, int& minProc, int& maxProc, const RCP >& comm, int numActiveProcs, const Type& v) { Type sumVal, sum2Val, v2 = v*v; + using MT = typename Teuchos::ScalarTraits::magnitudeType; double zero = Teuchos::ScalarTraits::zero(); + MT avgVal_MT = Teuchos::as(avgVal); + MueLu_sumAll(comm, v, sumVal); MueLu_sumAll(comm, v2, sum2Val); MueLu_minAll(comm, v, minVal); @@ -82,7 +85,7 @@ namespace MueLu { MueLu_maxAll(comm, w, minProc); avgVal = (numActiveProcs > 0 ? (as(Teuchos::ScalarTraits::real(sumVal)) / numActiveProcs) : zero); - devVal = (numActiveProcs > 1 ? sqrt((as(Teuchos::ScalarTraits::real(sum2Val - sumVal*avgVal)))/(numActiveProcs-1)) : zero); + devVal = (numActiveProcs > 1 ? sqrt((as(Teuchos::ScalarTraits::real(sum2Val - sumVal*avgVal_MT)))/(numActiveProcs-1)) : zero); } template diff --git a/packages/muelu/src/Utils/MueLu_UtilitiesBase_decl.hpp b/packages/muelu/src/Utils/MueLu_UtilitiesBase_decl.hpp index 1cfdc6b75323..dc115fe932ac 100644 --- a/packages/muelu/src/Utils/MueLu_UtilitiesBase_decl.hpp +++ b/packages/muelu/src/Utils/MueLu_UtilitiesBase_decl.hpp @@ -649,10 +649,11 @@ namespace MueLu { */ static typename Teuchos::ScalarTraits::magnitudeType Distance2(const Teuchos::ArrayView & weight,const Teuchos::Array> &v, LocalOrdinal i0, LocalOrdinal i1) { const size_t numVectors = v.size(); + using MT = typename Teuchos::ScalarTraits::magnitudeType; Scalar d = Teuchos::ScalarTraits::zero(); for (size_t j = 0; j < numVectors; j++) { - d += weight[j]*(v[j][i0] - v[j][i1])*(v[j][i0] - v[j][i1]); + d += Teuchos::as(weight[j])*(v[j][i0] - v[j][i1])*(v[j][i0] - v[j][i1]); } return Teuchos::ScalarTraits::magnitude(d); } diff --git a/packages/muelu/test/scaling/MatrixLoad.hpp b/packages/muelu/test/scaling/MatrixLoad.hpp index 91598f10cbc5..bb3e3eab28b3 100644 --- a/packages/muelu/test/scaling/MatrixLoad.hpp +++ b/packages/muelu/test/scaling/MatrixLoad.hpp @@ -120,16 +120,16 @@ void MatrixLoad(Teuchos::RCP > &comm, Xpetra::Underlyi // At the moment, however, things are fragile as we hope that the Problem uses same map and coordinates inside if (matrixType == "Laplace1D") { map = Galeri::Xpetra::CreateMap(xpetraParameters.GetLib(), "Cartesian1D", comm, galeriList); - coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates("1D", map, galeriList); + coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates("1D", map, galeriList); } else if (matrixType == "Laplace2D" || matrixType == "Star2D" || matrixType == "BigStar2D" || matrixType == "AnisotropicDiffusion" || matrixType == "Elasticity2D") { map = Galeri::Xpetra::CreateMap(xpetraParameters.GetLib(), "Cartesian2D", comm, galeriList); - coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates("2D", map, galeriList); + coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates("2D", map, galeriList); } else if (matrixType == "Laplace3D" || matrixType == "Brick3D" || matrixType == "Elasticity3D") { map = Galeri::Xpetra::CreateMap(xpetraParameters.GetLib(), "Cartesian3D", comm, galeriList); - coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates("3D", map, galeriList); + coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates("3D", map, galeriList); } // Expand map to do multiple DOF per node for block problems @@ -162,7 +162,7 @@ void MatrixLoad(Teuchos::RCP > &comm, Xpetra::Underlyi if(!coordinates.is_null() && (matrixType == "Elasticity2D" || matrixType == "Elasticity3D") ) { - Teuchos::RCP::magnitudeType,LocalOrdinal,GlobalOrdinal,Node> > newcoordinates; + Teuchos::RCP > newcoordinates; newcoordinates = Pr->BuildCoords(); // Galeri makes multiple copies of coordinates to deal with @@ -215,7 +215,7 @@ void MatrixLoad(Teuchos::RCP > &comm, Xpetra::Underlyi coordMap = Xpetra::IO::ReadMap(coordMapFile, lib, comm); else coordMap = map; - coordinates = Xpetra::IO::magnitudeType,LO,GO,Node>::ReadMultiVector(coordFile, coordMap); + coordinates = Xpetra::IO::ReadMultiVector(coordFile, coordMap); } if (!nullFile.empty()) @@ -234,7 +234,7 @@ void MatrixLoad(Teuchos::RCP > &comm, Xpetra::Underlyi X->randomize(); A->apply(*X, *B, Teuchos::NO_TRANS, one, zero); - Teuchos::Array norms(numVectors); + Teuchos::Array norms(numVectors); B->norm2(norms); B->scale(one/norms[0]); diff --git a/packages/muelu/test/unit_tests/BlockedRepartition.cpp b/packages/muelu/test/unit_tests/BlockedRepartition.cpp index 6c17423e892b..db0f853dde66 100644 --- a/packages/muelu/test/unit_tests/BlockedRepartition.cpp +++ b/packages/muelu/test/unit_tests/BlockedRepartition.cpp @@ -1188,25 +1188,33 @@ namespace MueLuTests { RepHeuFact->SetParameter("repartition: start level",Teuchos::ParameterEntry(0)); RepHeuFact->SetParameter("repartition: min rows per proc",Teuchos::ParameterEntry(40)); +#ifdef HAVE_MUELU_ZOLTAN2 RCP zoltan11 = rcp(new Zoltan2Interface()); zoltan11->SetFactory("A",AR11Fact); zoltan11->SetFactory("number of partitions",RepHeuFact); zoltan11->SetFactory("Coordinates",Coord11); +#endif RCP repart11 = rcp(new RepartitionFactory()); repart11->SetFactory("A",AR11Fact); repart11->SetFactory("number of partitions",RepHeuFact); +#ifdef HAVE_MUELU_ZOLTAN2 repart11->SetFactory("Partition", zoltan11); +#endif +#ifdef HAVE_MUELU_ZOLTAN2 RCP zoltan22 = rcp(new Zoltan2Interface()); zoltan22->SetFactory("A",AR22Fact); zoltan22->SetFactory("number of partitions",RepHeuFact); zoltan22->SetFactory("Coordinates",Coord22); +#endif RCP repart22 = rcp(new RepartitionFactory()); repart22->SetFactory("A",AR22Fact); repart22->SetFactory("number of partitions",RepHeuFact); +#ifdef HAVE_MUELU_ZOLTAN2 repart22->SetFactory("Partition", zoltan22); +#endif RCP MReb11 = rcp(new FactoryManager()); MReb11->SetFactory("A", AR11Fact); diff --git a/packages/muelu/test/unit_tests_kokkos/SaPFactory_kokkos.cpp b/packages/muelu/test/unit_tests_kokkos/SaPFactory_kokkos.cpp index dd50cf113330..193853575137 100644 --- a/packages/muelu/test/unit_tests_kokkos/SaPFactory_kokkos.cpp +++ b/packages/muelu/test/unit_tests_kokkos/SaPFactory_kokkos.cpp @@ -89,7 +89,7 @@ namespace MueLuTests { TestHelpers_kokkos::TestFactory::createTwoLevelHierarchy(fineLevel, coarseLevel); // construct matrices - const SC lambdaMax = 5; + const typename Teuchos::ScalarTraits::magnitudeType lambdaMax = 5; RCP A = TestHelpers_kokkos::TestFactory::Build2DPoisson(27*comm->getSize()); A->SetMaxEigenvalueEstimate(lambdaMax); RCP Ptent = TestHelpers_kokkos::TestFactory::Build2DPoisson(27*comm->getSize()); From 1830d07e47928bc83faa519e31efef2cb3b8ee90 Mon Sep 17 00:00:00 2001 From: Chris Siefert Date: Thu, 10 Mar 2022 10:27:40 -0700 Subject: [PATCH 11/19] Belos: Fixing bug --- .../tpetra/test/OrthoManager/belos_orthomanager_tpetra.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/belos/tpetra/test/OrthoManager/belos_orthomanager_tpetra.cpp b/packages/belos/tpetra/test/OrthoManager/belos_orthomanager_tpetra.cpp index 92f729220d6e..f08b2d9d868e 100644 --- a/packages/belos/tpetra/test/OrthoManager/belos_orthomanager_tpetra.cpp +++ b/packages/belos/tpetra/test/OrthoManager/belos_orthomanager_tpetra.cpp @@ -244,7 +244,7 @@ class SparseMatrixLoader // modify numRows to be the number of rows in the sparse matrix. // Otherwise, it will leave numRows alone. std::pair, Teuchos::RCP > results = - Belos::Test::loadSparseMatrix (comm, filename, numRows, debugOut); map = results.first; From b4e596b1a16365d30389b77a0b946f38a7988a30 Mon Sep 17 00:00:00 2001 From: Chris Siefert Date: Thu, 10 Mar 2022 12:50:04 -0700 Subject: [PATCH 12/19] MueLu: Fixing bug --- packages/muelu/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/muelu/CMakeLists.txt b/packages/muelu/CMakeLists.txt index 702728563c6a..9833457fd50f 100644 --- a/packages/muelu/CMakeLists.txt +++ b/packages/muelu/CMakeLists.txt @@ -250,7 +250,7 @@ IF(${PACKAGE_NAME}_ENABLE_EXPLICIT_INSTANTIATION) # # Examples and tests need at least this one, also if Epetra is enabled IF(Tpetra_INST_DOUBLE AND Tpetra_INST_INT_INT) - GLOBAL_SET(${PACKAGE_NAME}_HAS_GO_INT ON) + GLOBAL_SET(${PACKAGE_NAME}_HAVE_GO_INT ON) GLOBAL_SET(${PACKAGE_NAME}_INST_DOUBLE_INT_INT ON) GLOBAL_SET(HAVE_${PACKAGE_NAME_UC}_INST_DOUBLE_INT_INT ON) ELSE() @@ -272,7 +272,7 @@ IF(${PACKAGE_NAME}_ENABLE_EXPLICIT_INSTANTIATION) # IF(Tpetra_INST_DOUBLE AND Tpetra_INST_INT_LONG) - GLOBAL_SET(${PACKAGE_NAME}_HAVE_GO_LONG ON) + GLOBAL_SET(${PACKAGE_NAME}_HAVE_GO_INT ON) GLOBAL_SET(${PACKAGE_NAME}_INST_DOUBLE_INT_LONGINT ON) GLOBAL_SET(HAVE_${PACKAGE_NAME_UC}_INST_DOUBLE_INT_LONGINT ON) ELSE() From b0f62f93ca1af4ff6f8c520fe75bd8bd7cfcc4b6 Mon Sep 17 00:00:00 2001 From: Chris Siefert Date: Fri, 11 Mar 2022 15:05:13 -0700 Subject: [PATCH 13/19] MueLu: Fixing bad use of MT --- packages/muelu/src/Utils/MueLu_PerfUtils_def.hpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/muelu/src/Utils/MueLu_PerfUtils_def.hpp b/packages/muelu/src/Utils/MueLu_PerfUtils_def.hpp index 3d715d88c2f3..3f44a7e52d4a 100644 --- a/packages/muelu/src/Utils/MueLu_PerfUtils_def.hpp +++ b/packages/muelu/src/Utils/MueLu_PerfUtils_def.hpp @@ -70,8 +70,6 @@ namespace MueLu { Type sumVal, sum2Val, v2 = v*v; using MT = typename Teuchos::ScalarTraits::magnitudeType; double zero = Teuchos::ScalarTraits::zero(); - - MT avgVal_MT = Teuchos::as(avgVal); MueLu_sumAll(comm, v, sumVal); MueLu_sumAll(comm, v2, sum2Val); @@ -85,6 +83,7 @@ namespace MueLu { MueLu_maxAll(comm, w, minProc); avgVal = (numActiveProcs > 0 ? (as(Teuchos::ScalarTraits::real(sumVal)) / numActiveProcs) : zero); + MT avgVal_MT = Teuchos::as(avgVal); devVal = (numActiveProcs > 1 ? sqrt((as(Teuchos::ScalarTraits::real(sum2Val - sumVal*avgVal_MT)))/(numActiveProcs-1)) : zero); } From f91e0b7b130f16e616ceadee052c525e0e0e7422 Mon Sep 17 00:00:00 2001 From: Roger Pawlowski Date: Fri, 11 Mar 2022 14:46:52 -0700 Subject: [PATCH 14/19] PIRO: fixes for tempus forward only solver Thanks to @glhenni for providing the patch. --- packages/piro/src/Piro_TempusSolverForwardOnly_Def.hpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/piro/src/Piro_TempusSolverForwardOnly_Def.hpp b/packages/piro/src/Piro_TempusSolverForwardOnly_Def.hpp index 1fd69bc9e861..737377d82b86 100644 --- a/packages/piro/src/Piro_TempusSolverForwardOnly_Def.hpp +++ b/packages/piro/src/Piro_TempusSolverForwardOnly_Def.hpp @@ -125,12 +125,15 @@ void Piro::TempusSolverForwardOnly::initialize( *out << "\nA) Get the base parameter list ...\n"; // - TEUCHOS_TEST_FOR_EXCEPTION(appParams->isSublist("Tempus"), + TEUCHOS_TEST_FOR_EXCEPTION(!appParams->isSublist("Tempus"), Teuchos::Exceptions::InvalidParameter, std::endl << "Error! Piro::TempusSolverForwardOnly: must have Tempus sublist "); RCP tempusPL = sublist(appParams, "Tempus", true); - tempusPL->validateParameters(*getValidTempusParameters(),0); + + // this only works if the stepper is "Forward Euler".for now just + // disable validation + // tempusPL->validateParameters(*getValidTempusParameters(),0); // *out << "\nD) Create the stepper and integrator for the forward problem ...\n"; @@ -475,10 +478,8 @@ void Piro::TempusSolverForwardOnly::evalModelImpl( fwdStateIntegrator->initialize(); Scalar t_final = get_t_final(); - *out << "T final requested: " << t_final << " \n"; fwdStateIntegrator->advanceTime(t_final); double time = fwdStateIntegrator->getTime(); - *out << "T final actual: " << time << "\n"; Scalar diff = 0.0; if (abs(t_final) == 0) diff = abs(time-t_final); From 07e509f6fa832bcfcff1488f7ba901e2c689e9eb Mon Sep 17 00:00:00 2001 From: Roger Pawlowski Date: Fri, 11 Mar 2022 14:53:30 -0700 Subject: [PATCH 15/19] Panzer: support for PIRO Tempus forward only solver in ME factory Thanks to @glhenni for the patch! --- .../src/Panzer_STK_ModelEvaluatorFactory.hpp | 10 ++- .../Panzer_STK_ModelEvaluatorFactory_impl.hpp | 55 +++++++++++++- .../src/Panzer_STK_TempusObserverFactory.hpp | 76 +++++++++++++++++++ 3 files changed, 135 insertions(+), 6 deletions(-) create mode 100644 packages/panzer/adapters-stk/src/Panzer_STK_TempusObserverFactory.hpp diff --git a/packages/panzer/adapters-stk/src/Panzer_STK_ModelEvaluatorFactory.hpp b/packages/panzer/adapters-stk/src/Panzer_STK_ModelEvaluatorFactory.hpp index a37616e0b0e8..7010da6192d1 100644 --- a/packages/panzer/adapters-stk/src/Panzer_STK_ModelEvaluatorFactory.hpp +++ b/packages/panzer/adapters-stk/src/Panzer_STK_ModelEvaluatorFactory.hpp @@ -76,6 +76,7 @@ namespace Piro { template class RythmosSolver; + template class TempusSolverForwardOnly; } namespace Thyra { @@ -98,6 +99,7 @@ namespace panzer_stk { class STKConnManager; class NOXObserverFactory; class RythmosObserverFactory; + class TempusObserverFactory; class WorksetFactory; template @@ -134,6 +136,7 @@ namespace panzer_stk { void setNOXObserverFactory(const Teuchos::RCP& nox_observer_factory); void setRythmosObserverFactory(const Teuchos::RCP& rythmos_observer_factory); + void setTempusObserverFactory(const Teuchos::RCP& tempus_observer_factory); template int addResponse(const std::string & responseName,const std::vector & wkstDesc,const BuilderT & builder); @@ -148,8 +151,10 @@ namespace panzer_stk { buildResponseOnlyModelEvaluator(const Teuchos::RCP > & thyra_me, const Teuchos::RCP& global_data, const Teuchos::RCP > rythmosSolver = Teuchos::null, - const Teuchos::Ptr & in_nox_observer_factory=Teuchos::null, - const Teuchos::Ptr & in_rythmos_observer_factory=Teuchos::null); + const Teuchos::RCP > tempusSolver = Teuchos::null, + const Teuchos::Ptr & in_nox_observer_factory=Teuchos::null, + const Teuchos::Ptr & in_rythmos_observer_factory=Teuchos::null, + const Teuchos::Ptr & in_tempus_observer_factory=Teuchos::null); //@} @@ -327,6 +332,7 @@ namespace panzer_stk { Teuchos::RCP m_nox_observer_factory; Teuchos::RCP m_rythmos_observer_factory; + Teuchos::RCP m_tempus_observer_factory; Teuchos::RCP m_user_wkst_factory; Teuchos::RCP m_wkstContainer; diff --git a/packages/panzer/adapters-stk/src/Panzer_STK_ModelEvaluatorFactory_impl.hpp b/packages/panzer/adapters-stk/src/Panzer_STK_ModelEvaluatorFactory_impl.hpp index 65b82f6ec08a..219c8f1aea3b 100644 --- a/packages/panzer/adapters-stk/src/Panzer_STK_ModelEvaluatorFactory_impl.hpp +++ b/packages/panzer/adapters-stk/src/Panzer_STK_ModelEvaluatorFactory_impl.hpp @@ -90,6 +90,7 @@ #include "Panzer_STKConnManager.hpp" #include "Panzer_STK_NOXObserverFactory.hpp" #include "Panzer_STK_RythmosObserverFactory.hpp" +#include "Panzer_STK_TempusObserverFactory.hpp" #include "Panzer_STK_ParameterListCallback.hpp" #include "Panzer_STK_ParameterListCallbackBlocked.hpp" #include "Panzer_STK_IOClosureModel_Factory_TemplateBuilder.hpp" @@ -107,6 +108,7 @@ #include "Piro_NOXSolver.hpp" #include "Piro_LOCASolver.hpp" #include "Piro_RythmosSolver.hpp" +#include "Piro_TempusSolverForwardOnly.hpp" #include @@ -294,7 +296,8 @@ namespace panzer_stk { // this is weird...we are accessing the solution control to determine if things are transient // it is backwards! - bool is_transient = solncntl_params.get("Piro Solver") == "Rythmos" ? true : false; + bool is_transient = (solncntl_params.get("Piro Solver") == "Rythmos" || + solncntl_params.get("Piro Solver") == "Tempus") ? true : false; // for pseudo-transient, we need to enable transient solver support to get time derivatives into fill if (solncntl_params.get("Piro Solver") == "NOX") { if (solncntl_params.sublist("NOX").get("Nonlinear Solver") == "Pseudo-Transient") @@ -1159,6 +1162,12 @@ namespace panzer_stk { m_rythmos_observer_factory = rythmos_observer_factory; } + template + void ModelEvaluatorFactory::setTempusObserverFactory(const Teuchos::RCP& tempus_observer_factory) + { + m_tempus_observer_factory = tempus_observer_factory; + } + template void ModelEvaluatorFactory::setUserWorksetFactory(Teuchos::RCP& user_wkst_factory) { @@ -1177,10 +1186,12 @@ namespace panzer_stk { template Teuchos::RCP > ModelEvaluatorFactory:: buildResponseOnlyModelEvaluator(const Teuchos::RCP > & thyra_me, - const Teuchos::RCP& global_data, + const Teuchos::RCP& global_data, const Teuchos::RCP > rythmosSolver, + const Teuchos::RCP > tempusSolver, const Teuchos::Ptr & in_nox_observer_factory, - const Teuchos::Ptr & in_rythmos_observer_factory + const Teuchos::Ptr & in_rythmos_observer_factory, + const Teuchos::Ptr & in_tempus_observer_factory ) { using Teuchos::is_null; @@ -1196,6 +1207,8 @@ namespace panzer_stk { = is_null(in_nox_observer_factory) ? m_nox_observer_factory.ptr() : in_nox_observer_factory; Teuchos::Ptr rythmos_observer_factory = is_null(in_rythmos_observer_factory) ? m_rythmos_observer_factory.ptr() : in_rythmos_observer_factory; + Teuchos::Ptr tempus_observer_factory + = is_null(in_tempus_observer_factory) ? m_tempus_observer_factory.ptr() : in_tempus_observer_factory; Teuchos::ParameterList& p = *this->getNonconstParameterList(); Teuchos::ParameterList & solncntl_params = p.sublist("Solution Control"); @@ -1230,7 +1243,7 @@ namespace panzer_stk { else if (solver=="Rythmos") { TEUCHOS_TEST_FOR_EXCEPTION(Teuchos::is_null(rythmos_observer_factory), std::runtime_error, - "No NOX obersver built! Please call setrythmosObserverFactory() member function if you plan to use a Rythmos solver."); + "No Rythmos observer built! Please call setrythmosObserverFactory() member function if you plan to use a Rythmos solver."); // install the nox observer if(rythmos_observer_factory->useNOXObserver()) { @@ -1263,6 +1276,40 @@ namespace panzer_stk { piro = piro_rythmos; } + else if (solver=="Tempus") { + + TEUCHOS_TEST_FOR_EXCEPTION(Teuchos::is_null(tempus_observer_factory), std::runtime_error, + "No Tempus observer built! Please call setTempusObserverFactory() member function if you plan to use a Tempus solver."); + + // install the nox observer + if(tempus_observer_factory->useNOXObserver()) { + Teuchos::RCP ppo = nox_observer_factory->buildNOXObserver(m_mesh,m_global_indexer,m_lin_obj_factory); + piro_params->sublist("NOX").sublist("Solver Options").set("User Defined Pre/Post Operator", ppo); + } + + // override printing to use panzer ostream + piro_params->sublist("NOX").sublist("Printing").set >("Output Stream",global_data->os); + piro_params->sublist("NOX").sublist("Printing").set >("Error Stream",global_data->os); + piro_params->sublist("NOX").sublist("Printing").set("Output Processor",global_data->os->getOutputToRootOnly()); + + // use the user specfied tempus solver if they pass one in + Teuchos::RCP > piro_tempus; + + if(tempusSolver==Teuchos::null) + { + piro_tempus = + Teuchos::rcp(new Piro::TempusSolverForwardOnly(piro_params, thyra_me, + tempus_observer_factory->buildTempusObserver(m_mesh,m_global_indexer,m_lin_obj_factory))); + } + else + { + piro_tempus = tempusSolver; + piro_tempus->initialize(piro_params, thyra_me, + tempus_observer_factory->buildTempusObserver(m_mesh,m_global_indexer,m_lin_obj_factory)); + } + + piro = piro_tempus; + } else { TEUCHOS_TEST_FOR_EXCEPTION(true, std::logic_error, "Error: Unknown Piro Solver : " << solver); diff --git a/packages/panzer/adapters-stk/src/Panzer_STK_TempusObserverFactory.hpp b/packages/panzer/adapters-stk/src/Panzer_STK_TempusObserverFactory.hpp new file mode 100644 index 000000000000..7357386eddc1 --- /dev/null +++ b/packages/panzer/adapters-stk/src/Panzer_STK_TempusObserverFactory.hpp @@ -0,0 +1,76 @@ +// @HEADER +// *********************************************************************** +// +// Panzer: A partial differential equation assembly +// engine for strongly coupled complex multiphysics systems +// Copyright (2011) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact Roger P. Pawlowski (rppawlo@sandia.gov) and +// Eric C. Cyr (eccyr@sandia.gov) +// *********************************************************************** +// @HEADER + +#ifndef PANZER_STK_TEMPUS_OBSERVER_FACTORY_HPP +#define PANZER_STK_TEMPUS_OBSERVER_FACTORY_HPP + +#include "Tempus_IntegratorObserver.hpp" + +#include "Teuchos_RCP.hpp" +#include "Teuchos_Assert.hpp" + +#include "Panzer_STK_Interface.hpp" +#include "Panzer_GlobalIndexer.hpp" +#include "Panzer_LinearObjFactory.hpp" + +#include "Panzer_STK_Utilities.hpp" + +namespace panzer_stk { + + class TempusObserverFactory { + + public: + + virtual ~TempusObserverFactory() {} + + //! Use the NOX observer as well? + virtual bool useNOXObserver() const = 0; + + virtual Teuchos::RCP > + buildTempusObserver(const Teuchos::RCP& mesh, + const Teuchos::RCP & dof_manager, + const Teuchos::RCP >& lof) const = 0; + }; + +} + +#endif From e9f2aee22a49967639c7e05a2fe935946b6bcf47 Mon Sep 17 00:00:00 2001 From: Roger Pawlowski Date: Fri, 11 Mar 2022 15:11:06 -0700 Subject: [PATCH 16/19] Panzer: add tempus as a required package for adapters-stk --- packages/panzer/adapters-stk/cmake/Dependencies.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/panzer/adapters-stk/cmake/Dependencies.cmake b/packages/panzer/adapters-stk/cmake/Dependencies.cmake index 95fc7a2c9e6e..d793739f65e2 100644 --- a/packages/panzer/adapters-stk/cmake/Dependencies.cmake +++ b/packages/panzer/adapters-stk/cmake/Dependencies.cmake @@ -4,7 +4,7 @@ # Pamgen enabled to run, so Pamgen is only declared an optional test # dependency. -SET(LIB_REQUIRED_DEP_PACKAGES STKUtil STKTopology STKMesh STKIO Zoltan Stratimikos Piro NOX Rythmos PanzerCore PanzerDiscFE) +SET(LIB_REQUIRED_DEP_PACKAGES STKUtil STKTopology STKMesh STKIO Zoltan Stratimikos Piro NOX Rythmos Tempus PanzerCore PanzerDiscFE) SET(LIB_OPTIONAL_DEP_PACKAGES SEACASIoss SEACASExodus UMR Percept Teko MueLu Ifpack2) SET(TEST_REQUIRED_DEP_PACKAGES SEACASIoss SEACASExodus Teko MueLu Ifpack2) SET(TEST_OPTIONAL_DEP_PACKAGES Pamgen) From 0a6002d0b37c3d407a6e39d8ce081ba294c9f08a Mon Sep 17 00:00:00 2001 From: iyamazaki Date: Mon, 14 Mar 2022 03:20:42 -0600 Subject: [PATCH 17/19] Amesos2_Superludist : to be consistent with SuperLU_DIST --- packages/amesos2/src/Amesos2_Superludist_def.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/amesos2/src/Amesos2_Superludist_def.hpp b/packages/amesos2/src/Amesos2_Superludist_def.hpp index 8180ca558fa5..b19903080823 100644 --- a/packages/amesos2/src/Amesos2_Superludist_def.hpp +++ b/packages/amesos2/src/Amesos2_Superludist_def.hpp @@ -460,7 +460,8 @@ namespace Amesos2 { } // Retrieve the normI of A (required by gstrf). - double anorm = function_map::plangs((char *)"I", &(data_.A), &(data_.grid)); + bool notran = (data_.options.Trans == SLUD::NOTRANS); + double anorm = function_map::plangs((notran ? (char *)"1" : (char *)"I"), &(data_.A), &(data_.grid)); int info = 0; { From a301069c8d28b262c8c0dae1ed230a9de28a4d25 Mon Sep 17 00:00:00 2001 From: Roger Pawlowski Date: Mon, 14 Mar 2022 09:47:19 -0600 Subject: [PATCH 18/19] Panzer: make Tempus and optional package --- packages/panzer/adapters-stk/CMakeLists.txt | 8 ++++++++ .../adapters-stk/cmake/Dependencies.cmake | 4 ++-- .../cmake/PanzerAdaptersSTK_config.hpp.in | 1 + .../src/Panzer_STK_ModelEvaluatorFactory.hpp | 18 ++++++++++++++++-- .../Panzer_STK_ModelEvaluatorFactory_impl.hpp | 18 ++++++++++++++++-- 5 files changed, 43 insertions(+), 6 deletions(-) diff --git a/packages/panzer/adapters-stk/CMakeLists.txt b/packages/panzer/adapters-stk/CMakeLists.txt index ea24e332328e..0675ed17db00 100644 --- a/packages/panzer/adapters-stk/CMakeLists.txt +++ b/packages/panzer/adapters-stk/CMakeLists.txt @@ -48,6 +48,14 @@ ELSE() GLOBAL_SET(PANZER_HAVE_IFPACK2 OFF) ENDIF() +# Optional Tempus dependency +############################# +IF(${PACKAGE_NAME}_ENABLE_Tempus) + GLOBAL_SET(PANZER_HAVE_TEMPUS ON) +ELSE() + GLOBAL_SET(PANZER_HAVE_TEMPUS OFF) +ENDIF() + ADD_SUBDIRECTORY(src) TRIBITS_ADD_TEST_DIRECTORIES(test) TRIBITS_ADD_EXAMPLE_DIRECTORIES(example) diff --git a/packages/panzer/adapters-stk/cmake/Dependencies.cmake b/packages/panzer/adapters-stk/cmake/Dependencies.cmake index d793739f65e2..9f690b815313 100644 --- a/packages/panzer/adapters-stk/cmake/Dependencies.cmake +++ b/packages/panzer/adapters-stk/cmake/Dependencies.cmake @@ -4,8 +4,8 @@ # Pamgen enabled to run, so Pamgen is only declared an optional test # dependency. -SET(LIB_REQUIRED_DEP_PACKAGES STKUtil STKTopology STKMesh STKIO Zoltan Stratimikos Piro NOX Rythmos Tempus PanzerCore PanzerDiscFE) -SET(LIB_OPTIONAL_DEP_PACKAGES SEACASIoss SEACASExodus UMR Percept Teko MueLu Ifpack2) +SET(LIB_REQUIRED_DEP_PACKAGES STKUtil STKTopology STKMesh STKIO Zoltan Stratimikos Piro NOX Rythmos PanzerCore PanzerDiscFE) +SET(LIB_OPTIONAL_DEP_PACKAGES SEACASIoss SEACASExodus UMR Percept Teko MueLu Ifpack2 Tempus) SET(TEST_REQUIRED_DEP_PACKAGES SEACASIoss SEACASExodus Teko MueLu Ifpack2) SET(TEST_OPTIONAL_DEP_PACKAGES Pamgen) SET(LIB_REQUIRED_DEP_TPLS) diff --git a/packages/panzer/adapters-stk/cmake/PanzerAdaptersSTK_config.hpp.in b/packages/panzer/adapters-stk/cmake/PanzerAdaptersSTK_config.hpp.in index ee4c218bae79..ab9a2a62cef0 100644 --- a/packages/panzer/adapters-stk/cmake/PanzerAdaptersSTK_config.hpp.in +++ b/packages/panzer/adapters-stk/cmake/PanzerAdaptersSTK_config.hpp.in @@ -14,5 +14,6 @@ #cmakedefine PANZER_HAVE_TEKO #cmakedefine PANZER_HAVE_MUELU #cmakedefine PANZER_HAVE_IFPACK2 +#cmakedefine PANZER_HAVE_TEMPUS #endif diff --git a/packages/panzer/adapters-stk/src/Panzer_STK_ModelEvaluatorFactory.hpp b/packages/panzer/adapters-stk/src/Panzer_STK_ModelEvaluatorFactory.hpp index 7010da6192d1..17f1cbc01669 100644 --- a/packages/panzer/adapters-stk/src/Panzer_STK_ModelEvaluatorFactory.hpp +++ b/packages/panzer/adapters-stk/src/Panzer_STK_ModelEvaluatorFactory.hpp @@ -76,7 +76,9 @@ namespace Piro { template class RythmosSolver; +#ifdef PANZER_HAVE_TEMPUS template class TempusSolverForwardOnly; +#endif } namespace Thyra { @@ -99,7 +101,9 @@ namespace panzer_stk { class STKConnManager; class NOXObserverFactory; class RythmosObserverFactory; +#ifdef PANZER_HAVE_TEMPUS class TempusObserverFactory; +#endif class WorksetFactory; template @@ -136,7 +140,10 @@ namespace panzer_stk { void setNOXObserverFactory(const Teuchos::RCP& nox_observer_factory); void setRythmosObserverFactory(const Teuchos::RCP& rythmos_observer_factory); + +#ifdef PANZER_HAVE_TEMPUS void setTempusObserverFactory(const Teuchos::RCP& tempus_observer_factory); +#endif template int addResponse(const std::string & responseName,const std::vector & wkstDesc,const BuilderT & builder); @@ -151,10 +158,15 @@ namespace panzer_stk { buildResponseOnlyModelEvaluator(const Teuchos::RCP > & thyra_me, const Teuchos::RCP& global_data, const Teuchos::RCP > rythmosSolver = Teuchos::null, +#ifdef PANZER_HAVE_TEMPUS const Teuchos::RCP > tempusSolver = Teuchos::null, +#endif const Teuchos::Ptr & in_nox_observer_factory=Teuchos::null, - const Teuchos::Ptr & in_rythmos_observer_factory=Teuchos::null, - const Teuchos::Ptr & in_tempus_observer_factory=Teuchos::null); + const Teuchos::Ptr & in_rythmos_observer_factory=Teuchos::null +#ifdef PANZER_HAVE_TEMPUS + , const Teuchos::Ptr & in_tempus_observer_factory=Teuchos::null +#endif + ); //@} @@ -332,7 +344,9 @@ namespace panzer_stk { Teuchos::RCP m_nox_observer_factory; Teuchos::RCP m_rythmos_observer_factory; +#ifdef PANZER_HAVE_TEMPUS Teuchos::RCP m_tempus_observer_factory; +#endif Teuchos::RCP m_user_wkst_factory; Teuchos::RCP m_wkstContainer; diff --git a/packages/panzer/adapters-stk/src/Panzer_STK_ModelEvaluatorFactory_impl.hpp b/packages/panzer/adapters-stk/src/Panzer_STK_ModelEvaluatorFactory_impl.hpp index 219c8f1aea3b..8e7113a2de9e 100644 --- a/packages/panzer/adapters-stk/src/Panzer_STK_ModelEvaluatorFactory_impl.hpp +++ b/packages/panzer/adapters-stk/src/Panzer_STK_ModelEvaluatorFactory_impl.hpp @@ -90,7 +90,9 @@ #include "Panzer_STKConnManager.hpp" #include "Panzer_STK_NOXObserverFactory.hpp" #include "Panzer_STK_RythmosObserverFactory.hpp" +#ifdef PANZER_HAVE_TEMPUS #include "Panzer_STK_TempusObserverFactory.hpp" +#endif #include "Panzer_STK_ParameterListCallback.hpp" #include "Panzer_STK_ParameterListCallbackBlocked.hpp" #include "Panzer_STK_IOClosureModel_Factory_TemplateBuilder.hpp" @@ -108,7 +110,9 @@ #include "Piro_NOXSolver.hpp" #include "Piro_LOCASolver.hpp" #include "Piro_RythmosSolver.hpp" +#ifdef PANZER_HAVE_TEMPUS #include "Piro_TempusSolverForwardOnly.hpp" +#endif #include @@ -1162,11 +1166,13 @@ namespace panzer_stk { m_rythmos_observer_factory = rythmos_observer_factory; } +#ifdef PANZER_HAVE_TEMPUS template void ModelEvaluatorFactory::setTempusObserverFactory(const Teuchos::RCP& tempus_observer_factory) { m_tempus_observer_factory = tempus_observer_factory; } +#endif template void ModelEvaluatorFactory::setUserWorksetFactory(Teuchos::RCP& user_wkst_factory) @@ -1188,10 +1194,14 @@ namespace panzer_stk { buildResponseOnlyModelEvaluator(const Teuchos::RCP > & thyra_me, const Teuchos::RCP& global_data, const Teuchos::RCP > rythmosSolver, +#ifdef PANZER_HAVE_TEMPUS const Teuchos::RCP > tempusSolver, +#endif const Teuchos::Ptr & in_nox_observer_factory, - const Teuchos::Ptr & in_rythmos_observer_factory, - const Teuchos::Ptr & in_tempus_observer_factory + const Teuchos::Ptr & in_rythmos_observer_factory +#ifdef PANZER_HAVE_TEMPUS + , const Teuchos::Ptr & in_tempus_observer_factory +#endif ) { using Teuchos::is_null; @@ -1207,8 +1217,10 @@ namespace panzer_stk { = is_null(in_nox_observer_factory) ? m_nox_observer_factory.ptr() : in_nox_observer_factory; Teuchos::Ptr rythmos_observer_factory = is_null(in_rythmos_observer_factory) ? m_rythmos_observer_factory.ptr() : in_rythmos_observer_factory; +#ifdef PANZER_HAVE_TEMPUS Teuchos::Ptr tempus_observer_factory = is_null(in_tempus_observer_factory) ? m_tempus_observer_factory.ptr() : in_tempus_observer_factory; +#endif Teuchos::ParameterList& p = *this->getNonconstParameterList(); Teuchos::ParameterList & solncntl_params = p.sublist("Solution Control"); @@ -1276,6 +1288,7 @@ namespace panzer_stk { piro = piro_rythmos; } +#ifdef PANZER_HAVE_TEMPUS else if (solver=="Tempus") { TEUCHOS_TEST_FOR_EXCEPTION(Teuchos::is_null(tempus_observer_factory), std::runtime_error, @@ -1310,6 +1323,7 @@ namespace panzer_stk { piro = piro_tempus; } +#endif else { TEUCHOS_TEST_FOR_EXCEPTION(true, std::logic_error, "Error: Unknown Piro Solver : " << solver); From 196ced033b2d436941d4f11632aabb63da70bfb1 Mon Sep 17 00:00:00 2001 From: Roger Pawlowski Date: Mon, 14 Mar 2022 14:19:15 -0600 Subject: [PATCH 19/19] Panzer: add tempus example --- .../example/main_driver/CMakeLists.txt | 10 + .../energy-transient-tempus-blocked.xml | 453 ++++++++++++++++++ .../example/main_driver/main_driver.cpp | 12 + .../user_app_TempusObserverFactory.hpp | 99 ++++ .../user_app_TempusObserver_WriteToExodus.hpp | 127 +++++ 5 files changed, 701 insertions(+) create mode 100644 packages/panzer/adapters-stk/example/main_driver/energy-transient-tempus-blocked.xml create mode 100644 packages/panzer/adapters-stk/example/main_driver/user_app_TempusObserverFactory.hpp create mode 100644 packages/panzer/adapters-stk/example/main_driver/user_app_TempusObserver_WriteToExodus.hpp diff --git a/packages/panzer/adapters-stk/example/main_driver/CMakeLists.txt b/packages/panzer/adapters-stk/example/main_driver/CMakeLists.txt index 28314ce6e7f3..b9aba947ab6b 100644 --- a/packages/panzer/adapters-stk/example/main_driver/CMakeLists.txt +++ b/packages/panzer/adapters-stk/example/main_driver/CMakeLists.txt @@ -26,6 +26,7 @@ TRIBITS_COPY_FILES_TO_BINARY_DIR(main_driver_files energy-transient.xml energy-ss-blocked.xml energy-transient-blocked.xml + energy-transient-tempus-blocked.xml energy-neumann.xml energy-ss-blocked-tp.xml meshmotion.xml @@ -104,6 +105,15 @@ IF(${PACKAGE_NAME}_ENABLE_Teko) ARGS --i=energy-transient-blocked.xml PASS_REGULAR_EXPRESSION "panzer::MainDriver run completed." ) + + IF(${PACKAGE_NAME}_ENABLE_Tempus) + TRIBITS_ADD_ADVANCED_TEST( + main_driver_energy-transient-tempus-blocked + TEST_0 EXEC main_driver + ARGS --i=energy-transient-tempus-blocked.xml + PASS_REGULAR_EXPRESSION "panzer::MainDriver run completed." + ) + ENDIF() ENDIF() IF(${PACKAGE_NAME}_ENABLE_Pamgen) diff --git a/packages/panzer/adapters-stk/example/main_driver/energy-transient-tempus-blocked.xml b/packages/panzer/adapters-stk/example/main_driver/energy-transient-tempus-blocked.xml new file mode 100644 index 000000000000..922053b454ea --- /dev/null +++ b/packages/panzer/adapters-stk/example/main_driver/energy-transient-tempus-blocked.xml @@ -0,0 +1,453 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/packages/panzer/adapters-stk/example/main_driver/main_driver.cpp b/packages/panzer/adapters-stk/example/main_driver/main_driver.cpp index 4e1dd294c79c..54b1314a0311 100644 --- a/packages/panzer/adapters-stk/example/main_driver/main_driver.cpp +++ b/packages/panzer/adapters-stk/example/main_driver/main_driver.cpp @@ -72,6 +72,9 @@ #include "user_app_BCStrategy_Factory.hpp" #include "user_app_NOXObserverFactory.hpp" #include "user_app_RythmosObserverFactory.hpp" +#ifdef PANZER_HAVE_TEMPUS +#include "user_app_TempusObserverFactory.hpp" +#endif #include "user_app_ResponseEvaluatorFactory_HOFlux.hpp" #include @@ -243,7 +246,16 @@ int main(int argc, char *argv[]) } // solver = me_factory.getResponseOnlyModelEvaluator(); +#ifdef PANZER_HAVE_TEMPUS + Teuchos::RCP tof; + { + tof = Teuchos::rcp(new user_app::TempusObserverFactory(stkIOResponseLibrary,rLibrary->getWorksetContainer())); + } + + solver = me_factory.buildResponseOnlyModelEvaluator(physics,global_data,Teuchos::null,Teuchos::null,nof.ptr(),rof.ptr(),tof.ptr()); +#else solver = me_factory.buildResponseOnlyModelEvaluator(physics,global_data,Teuchos::null,nof.ptr(),rof.ptr()); +#endif } } diff --git a/packages/panzer/adapters-stk/example/main_driver/user_app_TempusObserverFactory.hpp b/packages/panzer/adapters-stk/example/main_driver/user_app_TempusObserverFactory.hpp new file mode 100644 index 000000000000..78fc76ae2c15 --- /dev/null +++ b/packages/panzer/adapters-stk/example/main_driver/user_app_TempusObserverFactory.hpp @@ -0,0 +1,99 @@ +// @HEADER +// *********************************************************************** +// +// Panzer: A partial differential equation assembly +// engine for strongly coupled complex multiphysics systems +// Copyright (2011) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact Roger P. Pawlowski (rppawlo@sandia.gov) and +// Eric C. Cyr (eccyr@sandia.gov) +// *********************************************************************** +// @HEADER + +#ifndef USER_APP_TEMPUS_OBSERVER_FACTORY_HPP +#define USER_APP_TEMPUS_OBSERVER_FACTORY_HPP + +#include "PanzerAdaptersSTK_config.hpp" + +#ifdef PANZER_HAVE_TEMPUS + +#include "Panzer_STK_TempusObserverFactory.hpp" +#include "Panzer_ResponseLibrary.hpp" +#include "Panzer_Traits.hpp" + +#include "Tempus_IntegratorObserverComposite.hpp" + +// Individual Observers +#include "user_app_TempusObserver_WriteToExodus.hpp" + +namespace user_app { + + class TempusObserverFactory : public panzer_stk::TempusObserverFactory { + + public: + TempusObserverFactory(const Teuchos::RCP > & stkIOResponseLibrary, + const Teuchos::RCP wkstContainer) + : stkIOResponseLibrary_(stkIOResponseLibrary) + , wkstContainer_(wkstContainer) + {} + + bool useNOXObserver() const { return false; } + + Teuchos::RCP > + buildTempusObserver(const Teuchos::RCP& mesh, + const Teuchos::RCP & dof_manager, + const Teuchos::RCP >& lof) const + { + Teuchos::RCP > composite_observer = + Teuchos::rcp(new Tempus::IntegratorObserverComposite); + + { + Teuchos::RCP observer + = Teuchos::rcp(new user_app::TempusObserver_WriteToExodus(mesh,dof_manager,lof,stkIOResponseLibrary_)); + composite_observer->addObserver(observer); + } + + return composite_observer; + } + + private: + //! Store STK IO response library...be careful, it will be modified externally + Teuchos::RCP > stkIOResponseLibrary_; + + Teuchos::RCP wkstContainer_; + }; + +} + +#endif // PANZER_HAVE_TEMPUS +#endif diff --git a/packages/panzer/adapters-stk/example/main_driver/user_app_TempusObserver_WriteToExodus.hpp b/packages/panzer/adapters-stk/example/main_driver/user_app_TempusObserver_WriteToExodus.hpp new file mode 100644 index 000000000000..bf2db3711cff --- /dev/null +++ b/packages/panzer/adapters-stk/example/main_driver/user_app_TempusObserver_WriteToExodus.hpp @@ -0,0 +1,127 @@ +// @HEADER +// *********************************************************************** +// +// Panzer: A partial differential equation assembly +// engine for strongly coupled complex multiphysics systems +// Copyright (2011) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact Roger P. Pawlowski (rppawlo@sandia.gov) and +// Eric C. Cyr (eccyr@sandia.gov) +// *********************************************************************** +// @HEADER + +#ifndef USER_APP_TEMPUS_OBSERVER_WRITE_TO_EXODUS_HPP +#define USER_APP_TEMPUS_OBSERVER_WRITE_TO_EXODUS_HPP + +#include "Tempus_Integrator.hpp" +#include "Tempus_IntegratorObserver.hpp" +#include "Teuchos_RCP.hpp" +#include "Teuchos_Assert.hpp" + +#include "Panzer_STK_Interface.hpp" +#include "Panzer_GlobalIndexer.hpp" +#include "Panzer_STK_ResponseEvaluatorFactory_SolutionWriter.hpp" +#include "Panzer_STK_Utilities.hpp" + +namespace user_app { + + class TempusObserver_WriteToExodus : public Tempus::IntegratorObserver { + + public: + + TempusObserver_WriteToExodus(const Teuchos::RCP& mesh, + const Teuchos::RCP& dof_manager, + const Teuchos::RCP >& lof, + const Teuchos::RCP > & response_library) : + m_mesh(mesh), + m_dof_manager(dof_manager), + m_lof(lof), + m_response_library(response_library) + { + // get all element blocks and add them to the list + std::vector eBlocks; + mesh->getElementBlockNames(eBlocks); + + panzer_stk::RespFactorySolnWriter_Builder builder; + builder.mesh = mesh; + m_response_library->addResponse("Main Field Output",eBlocks,builder); + } + + void observeStartIntegrator(const Tempus::Integrator& integrator) override {this->writeToExodus(integrator);}; + void observeStartTimeStep(const Tempus::Integrator& ) override {}; + void observeNextTimeStep(const Tempus::Integrator& ) override {}; + void observeBeforeTakeStep(const Tempus::Integrator& ) override {}; + void observeAfterTakeStep(const Tempus::Integrator& integrator) override {this->writeToExodus(integrator);} + void observeAfterCheckTimeStep(const Tempus::Integrator& ) override {}; + void observeEndTimeStep(const Tempus::Integrator& ) override {}; + void observeEndIntegrator(const Tempus::Integrator& ) override {}; + + void writeToExodus(const Tempus::Integrator& integrator) + { + + Teuchos::RCP > solution = integrator.getSolutionHistory()->getStateTimeIndexN()->getX(); + + // initialize the assembly container + panzer::AssemblyEngineInArgs ae_inargs; + ae_inargs.container_ = m_lof->buildLinearObjContainer(); + ae_inargs.ghostedContainer_ = m_lof->buildGhostedLinearObjContainer(); + ae_inargs.alpha = 0.0; + ae_inargs.beta = 1.0; + ae_inargs.evaluate_transient_terms = false; + + // initialize the ghosted container + m_lof->initializeGhostedContainer(panzer::LinearObjContainer::X,*ae_inargs.ghostedContainer_); + + { + // initialize the x vector + const Teuchos::RCP > thyraContainer + = Teuchos::rcp_dynamic_cast >(ae_inargs.container_,true); + thyraContainer->set_x_th(Teuchos::rcp_const_cast >(solution)); + } + + m_response_library->addResponsesToInArgs(ae_inargs); + m_response_library->evaluate(ae_inargs); + m_mesh->writeToExodus(integrator.getSolutionHistory()->getCurrentTime()); + } + + protected: + + Teuchos::RCP m_mesh; + Teuchos::RCP m_dof_manager; + Teuchos::RCP > m_lof; + Teuchos::RCP > m_response_library; + }; + +} + +#endif