From f78e1f5d1eada82b12e3affa1f2126476ff23c5f Mon Sep 17 00:00:00 2001 From: Damien L-G Date: Fri, 18 Oct 2024 09:04:32 -0400 Subject: [PATCH] Fixup atomic_{inc,dec}[rement] Signed-off-by: Damien L-G --- .../MueLu_AggregationPhase1Algorithm_def.hpp | 2 +- .../MueLu_AggregationPhase3Algorithm_def.hpp | 10 +++++----- packages/nox/test/tpetra/1DFEM_Functors.hpp | 10 +++++----- .../performance/fenl_assembly/fenl_functors.hpp | 16 ++++++++-------- .../fenl_assembly_view/fenl_functors.hpp | 4 ++-- .../shylu_node/basker/src/shylubasker_thread.hpp | 3 +-- .../stk_mesh/calculate_centroid.hpp | 2 +- .../src/Tpetra_Details_FixedHashTable_def.hpp | 4 ++-- .../examples/fenl/fenl_functors.hpp | 8 ++++---- 9 files changed, 29 insertions(+), 30 deletions(-) diff --git a/packages/muelu/src/Graph/UncoupledAggregation/MueLu_AggregationPhase1Algorithm_def.hpp b/packages/muelu/src/Graph/UncoupledAggregation/MueLu_AggregationPhase1Algorithm_def.hpp index 497595c81398..db1938ba0f0d 100644 --- a/packages/muelu/src/Graph/UncoupledAggregation/MueLu_AggregationPhase1Algorithm_def.hpp +++ b/packages/muelu/src/Graph/UncoupledAggregation/MueLu_AggregationPhase1Algorithm_def.hpp @@ -330,7 +330,7 @@ void AggregationPhase1Algorithm:: break; } else { // Decrement back the value of aggSizesView(agg) - Kokkos::atomic_decrement(&aggSizesView(agg)); + Kokkos::atomic_dec(&aggSizesView(agg)); } } } diff --git a/packages/muelu/src/Graph/UncoupledAggregation/MueLu_AggregationPhase3Algorithm_def.hpp b/packages/muelu/src/Graph/UncoupledAggregation/MueLu_AggregationPhase3Algorithm_def.hpp index f3cb94f1a18b..259fc734bf40 100644 --- a/packages/muelu/src/Graph/UncoupledAggregation/MueLu_AggregationPhase3Algorithm_def.hpp +++ b/packages/muelu/src/Graph/UncoupledAggregation/MueLu_AggregationPhase3Algorithm_def.hpp @@ -255,7 +255,7 @@ void AggregationPhase3Algorithm:: procWinner(nodeIdx, 0) = myRank; vertex2AggId(nodeIdx, 0) = aggId; // aggregates.SetIsRoot(nodeIdx); - Kokkos::atomic_decrement(&numNonAggregated()); + Kokkos::atomic_dec(&numNonAggregated()); for (int neigh = 0; neigh < neighbors.length; ++neigh) { neighIdx = neighbors(neigh); if ((neighIdx != nodeIdx) && @@ -264,7 +264,7 @@ void AggregationPhase3Algorithm:: aggStat(neighIdx) = AGGREGATED; procWinner(neighIdx, 0) = myRank; vertex2AggId(neighIdx, 0) = aggId; - Kokkos::atomic_decrement(&numNonAggregated()); + Kokkos::atomic_dec(&numNonAggregated()); } } return; @@ -279,7 +279,7 @@ void AggregationPhase3Algorithm:: aggStat(nodeIdx) = AGGREGATED; procWinner(nodeIdx, 0) = myRank; vertex2AggId(nodeIdx, 0) = vertex2AggId(neighIdx, 0); - Kokkos::atomic_decrement(&numNonAggregated()); + Kokkos::atomic_dec(&numNonAggregated()); return; } } @@ -293,7 +293,7 @@ void AggregationPhase3Algorithm:: aggStat(nodeIdx) = AGGREGATED; procWinner(nodeIdx, 0) = myRank; vertex2AggId(nodeIdx, 0) = vertex2AggId(otherNodeIdx, 0); - Kokkos::atomic_decrement(&numNonAggregated()); + Kokkos::atomic_dec(&numNonAggregated()); return; } } @@ -306,7 +306,7 @@ void AggregationPhase3Algorithm:: aggStat(nodeIdx) = AGGREGATED; procWinner(nodeIdx, 0) = myRank; vertex2AggId(nodeIdx, 0) = aggId; - Kokkos::atomic_decrement(&numNonAggregated()); + Kokkos::atomic_dec(&numNonAggregated()); } }); // LBV on 09/27/19: here we could copy numNonAggregated to host diff --git a/packages/nox/test/tpetra/1DFEM_Functors.hpp b/packages/nox/test/tpetra/1DFEM_Functors.hpp index 3cb9563ce396..892d5d9f2c31 100644 --- a/packages/nox/test/tpetra/1DFEM_Functors.hpp +++ b/packages/nox/test/tpetra/1DFEM_Functors.hpp @@ -67,26 +67,26 @@ struct RowCountsFunctor void operator() (const LO localRow, std::size_t& curNumLocalEntries) const { // Add a diagonal matrix entry - Kokkos::atomic_increment(&counts_(localRow)); + Kokkos::atomic_inc(&counts_(localRow)); ++curNumLocalEntries; // Contribute a matrix entry to the previous row if (localRow > 0) { - Kokkos::atomic_increment(&counts_(localRow-1)); + Kokkos::atomic_inc(&counts_(localRow-1)); ++curNumLocalEntries; } // Contribute a matrix entry to the next row if (localRow < numMyNodes_-1) { - Kokkos::atomic_increment(&counts_(localRow+1)); + Kokkos::atomic_inc(&counts_(localRow+1)); ++curNumLocalEntries; } // MPI process to the left sends us an entry if ((myRank_ > 0) && (localRow == 0)) { - Kokkos::atomic_increment(&counts_(localRow)); + Kokkos::atomic_inc(&counts_(localRow)); ++curNumLocalEntries; } // MPI process to the right sends us an entry if ((myRank_ < numProcs_-1) && (localRow == numMyNodes_-1)) { - Kokkos::atomic_increment(&counts_(localRow)); + Kokkos::atomic_inc(&counts_(localRow)); ++curNumLocalEntries; } } diff --git a/packages/sacado/test/performance/fenl_assembly/fenl_functors.hpp b/packages/sacado/test/performance/fenl_assembly/fenl_functors.hpp index 847e286a25c3..292757686cad 100644 --- a/packages/sacado/test/performance/fenl_assembly/fenl_functors.hpp +++ b/packages/sacado/test/performance/fenl_assembly/fenl_functors.hpp @@ -250,10 +250,10 @@ class NodeNodeGraph { if ( result.success() ) { // If row node is owned then increment count - if ( row_node < row_count.extent(0) ) { atomic_increment( & row_count( row_node ) ); } + if ( row_node < row_count.extent(0) ) { Kokkos::atomic_inc( & row_count( row_node ) ); } // If column node is owned and not equal to row node then increment count - if ( col_node < row_count.extent(0) && col_node != row_node ) { atomic_increment( & row_count( col_node ) ); } + if ( col_node < row_count.extent(0) && col_node != row_node ) { Kokkos::atomic_inc( & row_count( col_node ) ); } } else if ( result.failed() ) { ++count ; @@ -276,12 +276,12 @@ class NodeNodeGraph { const unsigned col_node = key.second ; if ( row_node < row_count.extent(0) ) { - const unsigned offset = graph.row_map( row_node ) + atomic_fetch_add( & row_count( row_node ) , atomic_incr_type(1) ); + const unsigned offset = graph.row_map( row_node ) + Kokkos::atomic_fetch_add( & row_count( row_node ) , atomic_incr_type(1) ); graph.entries( offset ) = col_node ; } if ( col_node < row_count.extent(0) && col_node != row_node ) { - const unsigned offset = graph.row_map( col_node ) + atomic_fetch_add( & row_count( col_node ) , atomic_incr_type(1) ); + const unsigned offset = graph.row_map( col_node ) + Kokkos::atomic_fetch_add( & row_count( col_node ) , atomic_incr_type(1) ); graph.entries( offset ) = row_node ; } } @@ -650,12 +650,12 @@ class ElementComputation for( unsigned i = 0 ; i < FunctionCount ; i++ ) { const unsigned row = node_index[i] ; if ( row < this->residual.extent(0) ) { - atomic_add( & this->residual( row ) , res[i] ); + Kokkos::atomic_add( & this->residual( row ) , res[i] ); for( unsigned j = 0 ; j < FunctionCount ; j++ ) { const unsigned entry = this->elem_graph( ielem , i , j ); if ( entry != ~0u ) { - atomic_add( & this->jacobian.coeff( entry ) , mat[i][j] ); + Kokkos::atomic_add( & this->jacobian.coeff( entry ) , mat[i][j] ); } } } @@ -835,12 +835,12 @@ class ElementComputation for( unsigned i = 0 ; i < FunctionCount ; i++ ) { const unsigned row = node_index[i] ; if ( row < this->residual.extent(0) ) { - atomic_add( & this->residual( row ) , res[i].val() ); + Kokkos::atomic_add( & this->residual( row ) , res[i].val() ); for( unsigned j = 0 ; j < FunctionCount ; j++ ) { const unsigned entry = this->elem_graph( ielem , i , j ); if ( entry != ~0u ) { - atomic_add( & this->jacobian.coeff( entry ) , + Kokkos::atomic_add( & this->jacobian.coeff( entry ) , res[i].fastAccessDx(j) ); } } diff --git a/packages/sacado/test/performance/fenl_assembly_view/fenl_functors.hpp b/packages/sacado/test/performance/fenl_assembly_view/fenl_functors.hpp index 85974a84f509..d92794953071 100644 --- a/packages/sacado/test/performance/fenl_assembly_view/fenl_functors.hpp +++ b/packages/sacado/test/performance/fenl_assembly_view/fenl_functors.hpp @@ -250,10 +250,10 @@ class NodeNodeGraph { if ( result.success() ) { // If row node is owned then increment count - if ( row_node < row_count.extent(0) ) { atomic_increment( & row_count( row_node ) ); } + if ( row_node < row_count.extent(0) ) { atomic_inc( & row_count( row_node ) ); } // If column node is owned and not equal to row node then increment count - if ( col_node < row_count.extent(0) && col_node != row_node ) { atomic_increment( & row_count( col_node ) ); } + if ( col_node < row_count.extent(0) && col_node != row_node ) { atomic_inc( & row_count( col_node ) ); } } else if ( result.failed() ) { ++count ; diff --git a/packages/shylu/shylu_node/basker/src/shylubasker_thread.hpp b/packages/shylu/shylu_node/basker/src/shylubasker_thread.hpp index ac078d10fe3e..0e43f7d5d96b 100644 --- a/packages/shylu/shylu_node/basker/src/shylubasker_thread.hpp +++ b/packages/shylu/shylu_node/basker/src/shylubasker_thread.hpp @@ -272,8 +272,7 @@ namespace BaskerNS BASKER_INLINE void atomic_barrier_fanout(volatile Int &value, const Int l_size) { - Kokkos::atomic_increment(&(value)); - while(value < l_size) + while(Kokkos::atomic_inc_fetch(&value) < l_size) { BASKER_NO_OP; } diff --git a/packages/stk/stk_performance_tests/stk_mesh/calculate_centroid.hpp b/packages/stk/stk_performance_tests/stk_mesh/calculate_centroid.hpp index 5f3dc5becf76..7688de12a1d3 100644 --- a/packages/stk/stk_performance_tests/stk_mesh/calculate_centroid.hpp +++ b/packages/stk/stk_performance_tests/stk_mesh/calculate_centroid.hpp @@ -237,7 +237,7 @@ std::vector get_centroid_average_from_device(stk::mesh::BulkData &bulk, for(size_t dim = 0; dim < 3; dim++) { Kokkos::atomic_add(&deviceAverageView(dim), ngpField(elem, dim)); } - Kokkos::atomic_increment(&deviceAverageView(3)); + Kokkos::atomic_inc(&deviceAverageView(3)); }); Kokkos::deep_copy(hostAverageView, deviceAverageView); diff --git a/packages/tpetra/core/src/Tpetra_Details_FixedHashTable_def.hpp b/packages/tpetra/core/src/Tpetra_Details_FixedHashTable_def.hpp index f616368048ad..5ea7148667a0 100644 --- a/packages/tpetra/core/src/Tpetra_Details_FixedHashTable_def.hpp +++ b/packages/tpetra/core/src/Tpetra_Details_FixedHashTable_def.hpp @@ -107,7 +107,7 @@ class CountBuckets { typedef typename hash_type::result_type hash_value_type; const hash_value_type hashVal = hash_type::hashFunc (keys_[i], size_); - Kokkos::atomic_increment (&counts_[hashVal]); + Kokkos::atomic_inc (&counts_[hashVal]); } using value_type = Kokkos::pair; @@ -128,7 +128,7 @@ class CountBuckets { dst.second = keyVal; } else { - Kokkos::atomic_increment (&counts_[hashVal]); + Kokkos::atomic_inc (&counts_[hashVal]); } } diff --git a/packages/trilinoscouplings/examples/fenl/fenl_functors.hpp b/packages/trilinoscouplings/examples/fenl/fenl_functors.hpp index 74abbd89d9be..1060467e03bd 100644 --- a/packages/trilinoscouplings/examples/fenl/fenl_functors.hpp +++ b/packages/trilinoscouplings/examples/fenl/fenl_functors.hpp @@ -221,8 +221,8 @@ class NodeNodeGraph { if ( result.success() ) { // First time this pair was inserted - if ( row_node < row_count.extent(0) ) { atomic_increment( & row_count( row_node ) ); } - if ( col_node < row_count.extent(0) && col_node != row_node ) { atomic_increment( & row_count( col_node ) ); } + if ( row_node < row_count.extent(0) ) { Kokkos::atomic_inc( & row_count( row_node ) ); } + if ( col_node < row_count.extent(0) && col_node != row_node ) { Kokkos::atomic_inc( & row_count( col_node ) ); } } else if ( result.failed() ) { // Ran out of memory for insertion. @@ -244,12 +244,12 @@ class NodeNodeGraph { using atomic_incr_type = typename std::remove_reference< decltype( row_count(0) ) >::type; if ( row_node < row_count.extent(0) ) { - const unsigned offset = graph.row_map( row_node ) + atomic_fetch_add( & row_count( row_node ) , atomic_incr_type(1) ); + const unsigned offset = graph.row_map( row_node ) + Kokkos::atomic_fetch_add( & row_count( row_node ) , atomic_incr_type(1) ); graph.entries( offset ) = col_node ; } if ( col_node < row_count.extent(0) && col_node != row_node ) { - const unsigned offset = graph.row_map( col_node ) + atomic_fetch_add( & row_count( col_node ) , atomic_incr_type(1) ); + const unsigned offset = graph.row_map( col_node ) + Kokkos::atomic_fetch_add( & row_count( col_node ) , atomic_incr_type(1) ); graph.entries( offset ) = row_node ; } }