From 4625664484fba62a9723a1221ee978c583d5e124 Mon Sep 17 00:00:00 2001
From: Rich Hornung <hornung1@llnl.gov>
Date: Wed, 30 Oct 2024 14:27:23 -0700
Subject: [PATCH] Add resource to omp target kernel variants

---
 src/polybench/POLYBENCH_2MM-OMPTarget.cpp      |  8 ++++++--
 src/polybench/POLYBENCH_3MM-OMPTarget.cpp      | 11 ++++++++---
 src/polybench/POLYBENCH_ADI-OMPTarget.cpp      |  8 ++++++--
 src/polybench/POLYBENCH_ATAX-OMPTarget.cpp     |  8 ++++++--
 src/polybench/POLYBENCH_FDTD_2D-OMPTarget.cpp  | 13 +++++++++----
 .../POLYBENCH_FLOYD_WARSHALL-OMPTarget.cpp     |  9 ++++++---
 src/polybench/POLYBENCH_GEMM-OMPTarget.cpp     |  5 ++++-
 src/polybench/POLYBENCH_GEMVER-OMPTarget.cpp   | 16 +++++++++++-----
 src/polybench/POLYBENCH_GESUMMV-OMPTarget.cpp  |  5 ++++-
 src/polybench/POLYBENCH_HEAT_3D-OMPTarget.cpp  | 18 ++++++++++++------
 .../POLYBENCH_JACOBI_1D-OMPTarget.cpp          |  6 ++++--
 .../POLYBENCH_JACOBI_2D-OMPTarget.cpp          |  8 ++++++--
 src/polybench/POLYBENCH_MVT-OMPTarget.cpp      |  8 ++++++--
 src/stream/ADD-OMPTarget.cpp                   |  4 +++-
 src/stream/COPY-OMPTarget.cpp                  |  4 +++-
 src/stream/DOT-OMPTarget.cpp                   |  4 +++-
 src/stream/MUL-OMPTarget.cpp                   |  4 +++-
 src/stream/TRIAD-OMPTarget.cpp                 |  4 +++-
 18 files changed, 103 insertions(+), 40 deletions(-)

diff --git a/src/polybench/POLYBENCH_2MM-OMPTarget.cpp b/src/polybench/POLYBENCH_2MM-OMPTarget.cpp
index 781139422..2a9115e89 100644
--- a/src/polybench/POLYBENCH_2MM-OMPTarget.cpp
+++ b/src/polybench/POLYBENCH_2MM-OMPTarget.cpp
@@ -61,6 +61,8 @@ void POLYBENCH_2MM::runOpenMPTargetVariant(VariantID vid, size_t RAJAPERF_UNUSED
 
   } else if ( vid == RAJA_OpenMPTarget ) {
 
+    auto res{getOmpTargetResource()};
+
     POLYBENCH_2MM_VIEWS_RAJA;
 
     using EXEC_POL =
@@ -78,11 +80,12 @@ void POLYBENCH_2MM::runOpenMPTargetVariant(VariantID vid, size_t RAJAPERF_UNUSED
     startTimer();
     for (RepIndex_type irep = 0; irep < run_reps; ++irep) {
 
-      RAJA::kernel_param<EXEC_POL>(
+      RAJA::kernel_param_resource<EXEC_POL>(
         RAJA::make_tuple(RAJA::RangeSegment{0, ni},
                          RAJA::RangeSegment{0, nj},
                          RAJA::RangeSegment{0, nk}),
         RAJA::tuple<Real_type>{0.0},
+        res,
 
         [=](Real_type &dot) {
           POLYBENCH_2MM_BODY1_RAJA;
@@ -95,11 +98,12 @@ void POLYBENCH_2MM::runOpenMPTargetVariant(VariantID vid, size_t RAJAPERF_UNUSED
         }
       );
 
-      RAJA::kernel_param<EXEC_POL>(
+      RAJA::kernel_param_resource<EXEC_POL>(
         RAJA::make_tuple(RAJA::RangeSegment{0, ni},
                          RAJA::RangeSegment{0, nl},
                          RAJA::RangeSegment{0, nj}),
         RAJA::tuple<Real_type>{0.0},
+        res,
 
         [=](Real_type &dot) {
           POLYBENCH_2MM_BODY4_RAJA;
diff --git a/src/polybench/POLYBENCH_3MM-OMPTarget.cpp b/src/polybench/POLYBENCH_3MM-OMPTarget.cpp
index d165426d1..70c19079f 100644
--- a/src/polybench/POLYBENCH_3MM-OMPTarget.cpp
+++ b/src/polybench/POLYBENCH_3MM-OMPTarget.cpp
@@ -73,6 +73,8 @@ void POLYBENCH_3MM::runOpenMPTargetVariant(VariantID vid, size_t RAJAPERF_UNUSED
 
   } else if ( vid == RAJA_OpenMPTarget ) {
 
+    auto res{getOmpTargetResource()};
+
     POLYBENCH_3MM_VIEWS_RAJA;
 
     using EXEC_POL =
@@ -90,11 +92,12 @@ void POLYBENCH_3MM::runOpenMPTargetVariant(VariantID vid, size_t RAJAPERF_UNUSED
     startTimer();
     for (RepIndex_type irep = 0; irep < run_reps; ++irep) {
 
-      RAJA::kernel_param<EXEC_POL>(
+      RAJA::kernel_param_resource<EXEC_POL>(
         RAJA::make_tuple(RAJA::RangeSegment{0, ni},
                          RAJA::RangeSegment{0, nj},
                          RAJA::RangeSegment{0, nk}),
         RAJA::tuple<Real_type>{0.0},
+        res,
 
         [=] (Real_type &dot) {
           POLYBENCH_3MM_BODY1_RAJA;
@@ -110,11 +113,12 @@ void POLYBENCH_3MM::runOpenMPTargetVariant(VariantID vid, size_t RAJAPERF_UNUSED
 
       );
 
-      RAJA::kernel_param<EXEC_POL>(
+      RAJA::kernel_param_resource<EXEC_POL>(
         RAJA::make_tuple(RAJA::RangeSegment{0, nj},
                          RAJA::RangeSegment{0, nl},
                          RAJA::RangeSegment{0, nm}),
         RAJA::tuple<Real_type>{0.0},
+        res,
 
         [=] (Real_type &dot) {
           POLYBENCH_3MM_BODY4_RAJA;
@@ -130,11 +134,12 @@ void POLYBENCH_3MM::runOpenMPTargetVariant(VariantID vid, size_t RAJAPERF_UNUSED
 
       );
 
-      RAJA::kernel_param<EXEC_POL>(
+      RAJA::kernel_param_resource<EXEC_POL>(
         RAJA::make_tuple(RAJA::RangeSegment{0, ni},
                          RAJA::RangeSegment{0, nl},
                          RAJA::RangeSegment{0, nj}),
         RAJA::tuple<Real_type>{0.0},
+        res,
 
         [=] (Real_type &dot) {
           POLYBENCH_3MM_BODY7_RAJA;
diff --git a/src/polybench/POLYBENCH_ADI-OMPTarget.cpp b/src/polybench/POLYBENCH_ADI-OMPTarget.cpp
index 8cee39e2c..3795f72ed 100644
--- a/src/polybench/POLYBENCH_ADI-OMPTarget.cpp
+++ b/src/polybench/POLYBENCH_ADI-OMPTarget.cpp
@@ -72,6 +72,8 @@ void POLYBENCH_ADI::runOpenMPTargetVariant(VariantID vid, size_t RAJAPERF_UNUSED
 
   } else if ( vid == RAJA_OpenMPTarget ) {
 
+    auto res{getOmpTargetResource()};
+
     POLYBENCH_ADI_VIEWS_RAJA;
 
     using EXEC_POL =
@@ -93,10 +95,11 @@ void POLYBENCH_ADI::runOpenMPTargetVariant(VariantID vid, size_t RAJAPERF_UNUSED
 
       for (Index_type t = 1; t <= tsteps; ++t) {
 
-        RAJA::kernel<EXEC_POL>(
+        RAJA::kernel_resource<EXEC_POL>(
           RAJA::make_tuple(RAJA::RangeSegment{1, n-1},
                            RAJA::RangeSegment{1, n-1},
                            RAJA::RangeStrideSegment{n-2, 0, -1}),
+          res,
 
           [=] (Index_type i) {
             POLYBENCH_ADI_BODY2_RAJA;
@@ -112,10 +115,11 @@ void POLYBENCH_ADI::runOpenMPTargetVariant(VariantID vid, size_t RAJAPERF_UNUSED
           }
         );
 
-        RAJA::kernel<EXEC_POL>(
+        RAJA::kernel_resource<EXEC_POL>(
           RAJA::make_tuple(RAJA::RangeSegment{1, n-1},
                            RAJA::RangeSegment{1, n-1},
                            RAJA::RangeStrideSegment{n-2, 0, -1}),
+          res,
 
           [=] (Index_type i) {
             POLYBENCH_ADI_BODY6_RAJA;
diff --git a/src/polybench/POLYBENCH_ATAX-OMPTarget.cpp b/src/polybench/POLYBENCH_ATAX-OMPTarget.cpp
index ce7ba4843..d377282be 100644
--- a/src/polybench/POLYBENCH_ATAX-OMPTarget.cpp
+++ b/src/polybench/POLYBENCH_ATAX-OMPTarget.cpp
@@ -62,6 +62,8 @@ void POLYBENCH_ATAX::runOpenMPTargetVariant(VariantID vid, size_t RAJAPERF_UNUSE
 
   } else if (vid == RAJA_OpenMPTarget) {
 
+    auto res{getOmpTargetResource()};
+
     POLYBENCH_ATAX_VIEWS_RAJA;
 
     using EXEC_POL1 =
@@ -89,10 +91,11 @@ void POLYBENCH_ATAX::runOpenMPTargetVariant(VariantID vid, size_t RAJAPERF_UNUSE
     startTimer();
     for (RepIndex_type irep = 0; irep < run_reps; ++irep) {
 
-      RAJA::kernel_param<EXEC_POL1>(
+      RAJA::kernel_param_resource<EXEC_POL1>(
         RAJA::make_tuple(RAJA::RangeSegment{0, N},
                          RAJA::RangeSegment{0, N}),
         RAJA::tuple<Real_type>{0.0},
+        res,
 
         [=] (Index_type i, Real_type &dot) {
           POLYBENCH_ATAX_BODY1_RAJA;
@@ -106,10 +109,11 @@ void POLYBENCH_ATAX::runOpenMPTargetVariant(VariantID vid, size_t RAJAPERF_UNUSE
 
       );
 
-      RAJA::kernel_param<EXEC_POL2>(
+      RAJA::kernel_param_resource<EXEC_POL2>(
         RAJA::make_tuple(RAJA::RangeSegment{0, N},
                          RAJA::RangeSegment{0, N}),
         RAJA::tuple<Real_type>{0.0},
+        res,
 
         [=] (Index_type j, Real_type &dot) {
           POLYBENCH_ATAX_BODY4_RAJA;
diff --git a/src/polybench/POLYBENCH_FDTD_2D-OMPTarget.cpp b/src/polybench/POLYBENCH_FDTD_2D-OMPTarget.cpp
index be8c0491e..008ea3188 100644
--- a/src/polybench/POLYBENCH_FDTD_2D-OMPTarget.cpp
+++ b/src/polybench/POLYBENCH_FDTD_2D-OMPTarget.cpp
@@ -76,6 +76,8 @@ void POLYBENCH_FDTD_2D::runOpenMPTargetVariant(VariantID vid, size_t RAJAPERF_UN
 
   } else if (vid == RAJA_OpenMPTarget) {
 
+    auto res{getOmpTargetResource()};
+
     POLYBENCH_FDTD_2D_VIEWS_RAJA;
 
     using EXEC_POL1 = RAJA::omp_target_parallel_for_exec<threads_per_team>;
@@ -93,30 +95,33 @@ void POLYBENCH_FDTD_2D::runOpenMPTargetVariant(VariantID vid, size_t RAJAPERF_UN
 
       for (t = 0; t < tsteps; ++t) {
 
-        RAJA::forall<EXEC_POL1>( RAJA::RangeSegment(0, ny),
+        RAJA::forall<EXEC_POL1>( res, RAJA::RangeSegment(0, ny),
          [=] (Index_type j) {
            POLYBENCH_FDTD_2D_BODY1_RAJA;
         });
 
-        RAJA::kernel<EXEC_POL234>(
+        RAJA::kernel_resource<EXEC_POL234>(
           RAJA::make_tuple(RAJA::RangeSegment{1, nx},
                            RAJA::RangeSegment{0, ny}),
+          res,
           [=] (Index_type i, Index_type j) {
             POLYBENCH_FDTD_2D_BODY2_RAJA;
           }
         );
 
-        RAJA::kernel<EXEC_POL234>(
+        RAJA::kernel_resource<EXEC_POL234>(
           RAJA::make_tuple(RAJA::RangeSegment{0, nx},
                            RAJA::RangeSegment{1, ny}),
+          res,
           [=] (Index_type i, Index_type j) {
             POLYBENCH_FDTD_2D_BODY3_RAJA;
           }
         );
 
-        RAJA::kernel<EXEC_POL234>(
+        RAJA::kernel_resource<EXEC_POL234>(
           RAJA::make_tuple(RAJA::RangeSegment{0, nx-1},
                            RAJA::RangeSegment{0, ny-1}),
+          res,
           [=] (Index_type i, Index_type j) {
             POLYBENCH_FDTD_2D_BODY4_RAJA;
           }
diff --git a/src/polybench/POLYBENCH_FLOYD_WARSHALL-OMPTarget.cpp b/src/polybench/POLYBENCH_FLOYD_WARSHALL-OMPTarget.cpp
index f3af4a088..78e9dd0dd 100644
--- a/src/polybench/POLYBENCH_FLOYD_WARSHALL-OMPTarget.cpp
+++ b/src/polybench/POLYBENCH_FLOYD_WARSHALL-OMPTarget.cpp
@@ -49,6 +49,8 @@ void POLYBENCH_FLOYD_WARSHALL::runOpenMPTargetVariant(VariantID vid, size_t RAJA
 
   } else if ( vid == RAJA_OpenMPTarget ) {
 
+    auto res{getOmpTargetResource()};
+
     POLYBENCH_FLOYD_WARSHALL_VIEWS_RAJA;
 
     using EXEC_POL =
@@ -64,9 +66,10 @@ void POLYBENCH_FLOYD_WARSHALL::runOpenMPTargetVariant(VariantID vid, size_t RAJA
     startTimer();
     for (RepIndex_type irep = 0; irep < run_reps; ++irep) {
 
-      RAJA::kernel<EXEC_POL>( RAJA::make_tuple(RAJA::RangeSegment{0, N},
-                                               RAJA::RangeSegment{0, N},
-                                               RAJA::RangeSegment{0, N}),
+      RAJA::kernel_resource<EXEC_POL>( RAJA::make_tuple(RAJA::RangeSegment{0, N},
+                                                        RAJA::RangeSegment{0, N},
+                                                        RAJA::RangeSegment{0, N}),
+        res,
         [=] (Index_type k, Index_type i, Index_type j) {
           POLYBENCH_FLOYD_WARSHALL_BODY_RAJA;
         }
diff --git a/src/polybench/POLYBENCH_GEMM-OMPTarget.cpp b/src/polybench/POLYBENCH_GEMM-OMPTarget.cpp
index a660ec35e..057938dc3 100644
--- a/src/polybench/POLYBENCH_GEMM-OMPTarget.cpp
+++ b/src/polybench/POLYBENCH_GEMM-OMPTarget.cpp
@@ -50,6 +50,8 @@ void POLYBENCH_GEMM::runOpenMPTargetVariant(VariantID vid, size_t RAJAPERF_UNUSE
 
   } else if (vid == RAJA_OpenMPTarget) {
 
+    auto res{getOmpTargetResource()};
+
     POLYBENCH_GEMM_VIEWS_RAJA;
 
     using EXEC_POL =
@@ -68,12 +70,13 @@ void POLYBENCH_GEMM::runOpenMPTargetVariant(VariantID vid, size_t RAJAPERF_UNUSE
       startTimer();
       for (RepIndex_type irep = 0; irep < run_reps; ++irep) {
 
-        RAJA::kernel_param<EXEC_POL>(
+        RAJA::kernel_param_resource<EXEC_POL>(
 
           RAJA::make_tuple( RAJA::RangeSegment{0, ni},
                             RAJA::RangeSegment{0, nj},
                             RAJA::RangeSegment{0, nk} ),
           RAJA::tuple<Real_type>{0.0},  // variable for dot
+          res,
 
           [=] (Real_type& dot) {
             POLYBENCH_GEMM_BODY1_RAJA;
diff --git a/src/polybench/POLYBENCH_GEMVER-OMPTarget.cpp b/src/polybench/POLYBENCH_GEMVER-OMPTarget.cpp
index 29f487d73..776c2683a 100644
--- a/src/polybench/POLYBENCH_GEMVER-OMPTarget.cpp
+++ b/src/polybench/POLYBENCH_GEMVER-OMPTarget.cpp
@@ -76,6 +76,8 @@ void POLYBENCH_GEMVER::runOpenMPTargetVariant(VariantID vid, size_t RAJAPERF_UNU
 
   } else if ( vid == RAJA_OpenMPTarget ) {
 
+    auto res{getOmpTargetResource()}; 
+
     POLYBENCH_GEMVER_VIEWS_RAJA;
 
     using EXEC_POL1 =
@@ -102,17 +104,20 @@ void POLYBENCH_GEMVER::runOpenMPTargetVariant(VariantID vid, size_t RAJAPERF_UNU
     startTimer();
     for (RepIndex_type irep = 0; irep < run_reps; ++irep) {
 
-      RAJA::kernel<EXEC_POL1>( RAJA::make_tuple(RAJA::RangeSegment{0, n},
-                                                RAJA::RangeSegment{0, n}),
+      RAJA::kernel_resource<EXEC_POL1>(
+        RAJA::make_tuple(RAJA::RangeSegment{0, n},
+                         RAJA::RangeSegment{0, n}),
+        res,
         [=] (Index_type i, Index_type j) {
           POLYBENCH_GEMVER_BODY1_RAJA;
         }
       );
 
-      RAJA::kernel_param<EXEC_POL24>(
+      RAJA::kernel_param_resource<EXEC_POL24>(
         RAJA::make_tuple(RAJA::RangeSegment{0, n},
                          RAJA::RangeSegment{0, n}),
         RAJA::tuple<Real_type>{0.0},
+        res,
 
         [=] (Index_type /* i */, Real_type &dot) {
           POLYBENCH_GEMVER_BODY2_RAJA;
@@ -125,16 +130,17 @@ void POLYBENCH_GEMVER::runOpenMPTargetVariant(VariantID vid, size_t RAJAPERF_UNU
         }
       );
 
-      RAJA::forall<EXEC_POL3> (RAJA::RangeSegment{0, n},
+      RAJA::forall<EXEC_POL3> (res, RAJA::RangeSegment{0, n},
         [=] (Index_type i) {
           POLYBENCH_GEMVER_BODY5_RAJA;
         }
       );
 
-      RAJA::kernel_param<EXEC_POL24>(
+      RAJA::kernel_param_resource<EXEC_POL24>(
         RAJA::make_tuple(RAJA::RangeSegment{0, n},
                          RAJA::RangeSegment{0, n}),
         RAJA::tuple<Real_type>{0.0},
+        res,
 
         [=] (Index_type i, Real_type &dot) {
           POLYBENCH_GEMVER_BODY6_RAJA;
diff --git a/src/polybench/POLYBENCH_GESUMMV-OMPTarget.cpp b/src/polybench/POLYBENCH_GESUMMV-OMPTarget.cpp
index 8f572c16a..3e7dd33ad 100644
--- a/src/polybench/POLYBENCH_GESUMMV-OMPTarget.cpp
+++ b/src/polybench/POLYBENCH_GESUMMV-OMPTarget.cpp
@@ -52,6 +52,8 @@ void POLYBENCH_GESUMMV::runOpenMPTargetVariant(VariantID vid, size_t RAJAPERF_UN
 
   } else if (vid == RAJA_OpenMPTarget) {
 
+    auto res{getOmpTargetResource()};
+
     POLYBENCH_GESUMMV_VIEWS_RAJA;
 
     using EXEC_POL =
@@ -68,11 +70,12 @@ void POLYBENCH_GESUMMV::runOpenMPTargetVariant(VariantID vid, size_t RAJAPERF_UN
       startTimer();
       for (RepIndex_type irep = 0; irep < run_reps; ++irep) {
 
-        RAJA::kernel_param<EXEC_POL>(
+        RAJA::kernel_param_resource<EXEC_POL>(
           RAJA::make_tuple( RAJA::RangeSegment{0, N},
                             RAJA::RangeSegment{0, N} ),
           RAJA::make_tuple(static_cast<Real_type>(0.0),
                            static_cast<Real_type>(0.0)),
+          res,
 
           [=] (Real_type& tmpdot,
                Real_type& ydot) {
diff --git a/src/polybench/POLYBENCH_HEAT_3D-OMPTarget.cpp b/src/polybench/POLYBENCH_HEAT_3D-OMPTarget.cpp
index 1c3999279..63a103baf 100644
--- a/src/polybench/POLYBENCH_HEAT_3D-OMPTarget.cpp
+++ b/src/polybench/POLYBENCH_HEAT_3D-OMPTarget.cpp
@@ -61,6 +61,8 @@ void POLYBENCH_HEAT_3D::runOpenMPTargetVariant(VariantID vid, size_t RAJAPERF_UN
 
   } else if (vid == RAJA_OpenMPTarget) {
 
+    auto res{getOmpTargetResource()};
+
     POLYBENCH_HEAT_3D_VIEWS_RAJA;
 
     using EXEC_POL =
@@ -76,17 +78,21 @@ void POLYBENCH_HEAT_3D::runOpenMPTargetVariant(VariantID vid, size_t RAJAPERF_UN
 
       for (Index_type t = 0; t < tsteps; ++t) {
 
-        RAJA::kernel<EXEC_POL>( RAJA::make_tuple(RAJA::RangeSegment{1, N-1},
-                                                 RAJA::RangeSegment{1, N-1},
-                                                 RAJA::RangeSegment{1, N-1}),
+        RAJA::kernel_resource<EXEC_POL>(
+          RAJA::make_tuple(RAJA::RangeSegment{1, N-1},
+                           RAJA::RangeSegment{1, N-1},
+                           RAJA::RangeSegment{1, N-1}),
+          res,
           [=] (Index_type i, Index_type j, Index_type k) {
             POLYBENCH_HEAT_3D_BODY1_RAJA;
           }
         );
 
-        RAJA::kernel<EXEC_POL>( RAJA::make_tuple(RAJA::RangeSegment{1, N-1},
-                                                 RAJA::RangeSegment{1, N-1},
-                                                 RAJA::RangeSegment{1, N-1}),
+        RAJA::kernel_resource<EXEC_POL>(
+          RAJA::make_tuple(RAJA::RangeSegment{1, N-1},
+                           RAJA::RangeSegment{1, N-1},
+                           RAJA::RangeSegment{1, N-1}),
+          res,
           [=] (Index_type i, Index_type j, Index_type k) {
             POLYBENCH_HEAT_3D_BODY2_RAJA;
           }
diff --git a/src/polybench/POLYBENCH_JACOBI_1D-OMPTarget.cpp b/src/polybench/POLYBENCH_JACOBI_1D-OMPTarget.cpp
index 35089be71..1b026dfd0 100644
--- a/src/polybench/POLYBENCH_JACOBI_1D-OMPTarget.cpp
+++ b/src/polybench/POLYBENCH_JACOBI_1D-OMPTarget.cpp
@@ -57,17 +57,19 @@ void POLYBENCH_JACOBI_1D::runOpenMPTargetVariant(VariantID vid, size_t RAJAPERF_
 
   } else if (vid == RAJA_OpenMPTarget ) {
 
+    auto res{getOmpTargetResource()};
+
     startTimer();
     for (RepIndex_type irep = 0; irep < run_reps; ++irep) {
 
       for (Index_type t = 0; t < tsteps; ++t) {
 
-        RAJA::forall<RAJA::omp_target_parallel_for_exec<threads_per_team>> (
+        RAJA::forall<RAJA::omp_target_parallel_for_exec<threads_per_team>> ( res,
           RAJA::RangeSegment{1, N-1}, [=] (Index_type i) {
             POLYBENCH_JACOBI_1D_BODY1;
         });
 
-        RAJA::forall<RAJA::omp_target_parallel_for_exec<threads_per_team>> (
+        RAJA::forall<RAJA::omp_target_parallel_for_exec<threads_per_team>> ( res,
           RAJA::RangeSegment{1, N-1}, [=] (Index_type i) {
             POLYBENCH_JACOBI_1D_BODY2;
         });
diff --git a/src/polybench/POLYBENCH_JACOBI_2D-OMPTarget.cpp b/src/polybench/POLYBENCH_JACOBI_2D-OMPTarget.cpp
index e711660cc..fe2a87119 100644
--- a/src/polybench/POLYBENCH_JACOBI_2D-OMPTarget.cpp
+++ b/src/polybench/POLYBENCH_JACOBI_2D-OMPTarget.cpp
@@ -57,6 +57,8 @@ void POLYBENCH_JACOBI_2D::runOpenMPTargetVariant(VariantID vid, size_t RAJAPERF_
 
   } else if (vid == RAJA_OpenMPTarget) {
 
+    auto res{getOmpTargetResource()};
+
     POLYBENCH_JACOBI_2D_VIEWS_RAJA;
 
     using EXEC_POL =
@@ -76,8 +78,10 @@ void POLYBENCH_JACOBI_2D::runOpenMPTargetVariant(VariantID vid, size_t RAJAPERF_
 
       for (Index_type t = 0; t < tsteps; ++t) {
 
-        RAJA::kernel<EXEC_POL>( RAJA::make_tuple(RAJA::RangeSegment{1, N-1},
-                                                 RAJA::RangeSegment{1, N-1}),
+        RAJA::kernel_resource<EXEC_POL>(
+          RAJA::make_tuple(RAJA::RangeSegment{1, N-1},
+                           RAJA::RangeSegment{1, N-1}),
+          res, 
           [=] (Index_type i, Index_type j) {
             POLYBENCH_JACOBI_2D_BODY1_RAJA;
           },
diff --git a/src/polybench/POLYBENCH_MVT-OMPTarget.cpp b/src/polybench/POLYBENCH_MVT-OMPTarget.cpp
index 5b278628d..ab06ef4c7 100644
--- a/src/polybench/POLYBENCH_MVT-OMPTarget.cpp
+++ b/src/polybench/POLYBENCH_MVT-OMPTarget.cpp
@@ -62,6 +62,8 @@ void POLYBENCH_MVT::runOpenMPTargetVariant(VariantID vid, size_t RAJAPERF_UNUSED
 
   } else if (vid == RAJA_OpenMPTarget) {
 
+    auto res{getOmpTargetResource()};
+
     POLYBENCH_MVT_VIEWS_RAJA;
 
     using EXEC_POL =
@@ -80,10 +82,11 @@ void POLYBENCH_MVT::runOpenMPTargetVariant(VariantID vid, size_t RAJAPERF_UNUSED
 
       RAJA::region<RAJA::seq_region>( [=]() {
 
-        RAJA::kernel_param<EXEC_POL>(
+        RAJA::kernel_param_resource<EXEC_POL>(
           RAJA::make_tuple(RAJA::RangeSegment{0, N},
                            RAJA::RangeSegment{0, N}),
           RAJA::tuple<Real_type>{0.0},
+          res,
 
           [=] (Real_type &dot) {
             POLYBENCH_MVT_BODY1_RAJA;
@@ -97,10 +100,11 @@ void POLYBENCH_MVT::runOpenMPTargetVariant(VariantID vid, size_t RAJAPERF_UNUSED
 
         );
 
-        RAJA::kernel_param<EXEC_POL>(
+        RAJA::kernel_param_resource<EXEC_POL>(
           RAJA::make_tuple(RAJA::RangeSegment{0, N},
                            RAJA::RangeSegment{0, N}),
           RAJA::tuple<Real_type>{0.0},
+          res,
 
           [=] (Real_type &dot) {
             POLYBENCH_MVT_BODY4_RAJA;
diff --git a/src/stream/ADD-OMPTarget.cpp b/src/stream/ADD-OMPTarget.cpp
index c1a1480cf..5f6cd8d0a 100644
--- a/src/stream/ADD-OMPTarget.cpp
+++ b/src/stream/ADD-OMPTarget.cpp
@@ -50,10 +50,12 @@ void ADD::runOpenMPTargetVariant(VariantID vid, size_t RAJAPERF_UNUSED_ARG(tune_
 
   } else if ( vid == RAJA_OpenMPTarget ) {
 
+    auto res{getOmpTargetResource()};
+
     startTimer();
     for (RepIndex_type irep = 0; irep < run_reps; ++irep) {
 
-      RAJA::forall<RAJA::omp_target_parallel_for_exec<threads_per_team>>(
+      RAJA::forall<RAJA::omp_target_parallel_for_exec<threads_per_team>>( res,
         RAJA::RangeSegment(ibegin, iend), [=](Index_type i) {
         ADD_BODY;
       });
diff --git a/src/stream/COPY-OMPTarget.cpp b/src/stream/COPY-OMPTarget.cpp
index f1dd5017d..334b72972 100644
--- a/src/stream/COPY-OMPTarget.cpp
+++ b/src/stream/COPY-OMPTarget.cpp
@@ -50,10 +50,12 @@ void COPY::runOpenMPTargetVariant(VariantID vid, size_t RAJAPERF_UNUSED_ARG(tune
 
   } else if ( vid == RAJA_OpenMPTarget ) {
 
+    auto res{getOmpTargetResource()};
+
     startTimer();
     for (RepIndex_type irep = 0; irep < run_reps; ++irep) {
 
-      RAJA::forall<RAJA::omp_target_parallel_for_exec<threads_per_team>>(
+      RAJA::forall<RAJA::omp_target_parallel_for_exec<threads_per_team>>( res,
         RAJA::RangeSegment(ibegin, iend), [=](Index_type i) {
         COPY_BODY;
       });
diff --git a/src/stream/DOT-OMPTarget.cpp b/src/stream/DOT-OMPTarget.cpp
index fd7d02a70..b16b77ccd 100644
--- a/src/stream/DOT-OMPTarget.cpp
+++ b/src/stream/DOT-OMPTarget.cpp
@@ -60,12 +60,14 @@ void DOT::runOpenMPTargetVariant(VariantID vid, size_t RAJAPERF_UNUSED_ARG(tune_
 
     case RAJA_OpenMPTarget : {
 
+      auto res{getOmpTargetResource()};
+
       startTimer();
       for (RepIndex_type irep = 0; irep < run_reps; ++irep) {
 
         Real_type tdot = m_dot_init;
 
-        RAJA::forall<RAJA::omp_target_parallel_for_exec<threads_per_team>>(
+        RAJA::forall<RAJA::omp_target_parallel_for_exec<threads_per_team>>( res,
           RAJA::RangeSegment(ibegin, iend),
           RAJA::expt::Reduce<RAJA::operators::plus>(&tdot),
           [=] (Index_type i, Real_type& dot) {
diff --git a/src/stream/MUL-OMPTarget.cpp b/src/stream/MUL-OMPTarget.cpp
index 07edb732f..8c7ce7140 100644
--- a/src/stream/MUL-OMPTarget.cpp
+++ b/src/stream/MUL-OMPTarget.cpp
@@ -50,10 +50,12 @@ void MUL::runOpenMPTargetVariant(VariantID vid, size_t RAJAPERF_UNUSED_ARG(tune_
 
   } else if ( vid == RAJA_OpenMPTarget ) {
 
+    auto res{getOmpTargetResource()};
+
     startTimer();
     for (RepIndex_type irep = 0; irep < run_reps; ++irep) {
 
-      RAJA::forall<RAJA::omp_target_parallel_for_exec<threads_per_team>>(
+      RAJA::forall<RAJA::omp_target_parallel_for_exec<threads_per_team>>( res,
         RAJA::RangeSegment(ibegin, iend), [=](Index_type i) {
         MUL_BODY;
       });
diff --git a/src/stream/TRIAD-OMPTarget.cpp b/src/stream/TRIAD-OMPTarget.cpp
index 5ec18d155..ba1b6e0f7 100644
--- a/src/stream/TRIAD-OMPTarget.cpp
+++ b/src/stream/TRIAD-OMPTarget.cpp
@@ -50,10 +50,12 @@ void TRIAD::runOpenMPTargetVariant(VariantID vid, size_t RAJAPERF_UNUSED_ARG(tun
 
   } else if ( vid == RAJA_OpenMPTarget ) {
 
+    auto res{getOmpTargetResource()};
+
     startTimer();
     for (RepIndex_type irep = 0; irep < run_reps; ++irep) {
 
-      RAJA::forall<RAJA::omp_target_parallel_for_exec<threads_per_team>>(
+      RAJA::forall<RAJA::omp_target_parallel_for_exec<threads_per_team>>( res,
           RAJA::RangeSegment(ibegin, iend), [=](Index_type i) {
         TRIAD_BODY;
       });