Mpi

src/parcsr_block_mv/par_csr_block_matvec.c

@@ -208,7 +208,7 @@ hypre_ParCSRBlockMatrixMatvecT( HYPRE_Complex    alpha,
    hypre_CSRBlockMatrixMatvecT(alpha, diag, x_local, beta, y_local);
 
 
-   hypre_ParCSRCommHandleDestroy(comm_handle);
+   hypre_ParCSRBlockCommHandleDestroy(comm_handle);
    comm_handle = NULL;
 
    index = 0;

src/parcsr_ls/par_rap.c

@@ -48,7 +48,6 @@ hypre_BoomerAMGBuildCoarseOperatorKT( hypre_ParCSRMatrix  *RT,
    HYPRE_Int              num_cols_offd_RT = hypre_CSRMatrixNumCols(RT_offd);
    HYPRE_Int              num_rows_offd_RT = hypre_CSRMatrixNumRows(RT_offd);
    hypre_ParCSRCommPkg   *comm_pkg_RT = hypre_ParCSRMatrixCommPkg(RT);
-   HYPRE_Int              num_recvs_RT = 0;
    HYPRE_Int              num_sends_RT = 0;
    HYPRE_Int             *send_map_starts_RT = NULL;
    HYPRE_Int             *send_map_elmts_RT = NULL;
@@ -192,7 +191,6 @@ hypre_BoomerAMGBuildCoarseOperatorKT( hypre_ParCSRMatrix  *RT,
 
    if (comm_pkg_RT)
    {
-      num_recvs_RT = hypre_ParCSRCommPkgNumRecvs(comm_pkg_RT);
       num_sends_RT = hypre_ParCSRCommPkgNumSends(comm_pkg_RT);
       send_map_starts_RT = hypre_ParCSRCommPkgSendMapStarts(comm_pkg_RT);
       send_map_elmts_RT = hypre_ParCSRCommPkgSendMapElmts(comm_pkg_RT);
@@ -201,7 +199,6 @@ hypre_BoomerAMGBuildCoarseOperatorKT( hypre_ParCSRMatrix  *RT,
    {
       hypre_MatvecCommPkgCreate(RT);
       comm_pkg_RT = hypre_ParCSRMatrixCommPkg(RT);
-      num_recvs_RT = hypre_ParCSRCommPkgNumRecvs(comm_pkg_RT);
       num_sends_RT = hypre_ParCSRCommPkgNumSends(comm_pkg_RT);
       send_map_starts_RT = hypre_ParCSRCommPkgSendMapStarts(comm_pkg_RT);
       send_map_elmts_RT = hypre_ParCSRCommPkgSendMapElmts(comm_pkg_RT);
@@ -1040,16 +1037,15 @@ hypre_BoomerAMGBuildCoarseOperatorKT( hypre_ParCSRMatrix  *RT,
 #endif
 
    RAP_ext_size = 0;
-   if (num_sends_RT || num_recvs_RT)
-   {
-      void *request;
-      hypre_ExchangeExternalRowsInit(RAP_int, comm_pkg_RT, &request);
-      RAP_ext = hypre_ExchangeExternalRowsWait(request);
-      RAP_ext_i = hypre_CSRMatrixI(RAP_ext);
-      RAP_ext_j = hypre_CSRMatrixBigJ(RAP_ext);
-      RAP_ext_data = hypre_CSRMatrixData(RAP_ext);
-      RAP_ext_size = RAP_ext_i[hypre_CSRMatrixNumRows(RAP_ext)];
-   }
+
+   void *request;
+   hypre_ExchangeExternalRowsInit(RAP_int, comm_pkg_RT, &request);
+   RAP_ext = hypre_ExchangeExternalRowsWait(request);
+   RAP_ext_i = hypre_CSRMatrixI(RAP_ext);
+   RAP_ext_j = hypre_CSRMatrixBigJ(RAP_ext);
+   RAP_ext_data = hypre_CSRMatrixData(RAP_ext);
+   RAP_ext_size = RAP_ext_i[hypre_CSRMatrixNumRows(RAP_ext)];
+
    if (num_cols_offd_RT)
    {
       hypre_CSRMatrixDestroy(RAP_int);
@@ -2039,11 +2035,8 @@ hypre_BoomerAMGBuildCoarseOperatorKT( hypre_ParCSRMatrix  *RT,
       R_offd = NULL;
    }
 
-   if (num_sends_RT || num_recvs_RT)
-   {
-      hypre_CSRMatrixDestroy(RAP_ext);
-      RAP_ext = NULL;
-   }
+   hypre_CSRMatrixDestroy(RAP_ext);
+   RAP_ext = NULL;
    hypre_TFree(P_mark_array, HYPRE_MEMORY_HOST);
    hypre_TFree(A_mark_array, HYPRE_MEMORY_HOST);
    hypre_TFree(P_ext_diag_i, HYPRE_MEMORY_HOST);

src/parcsr_ls/par_relax.c

@@ -787,9 +787,9 @@ hypre_BoomerAMGRelaxHybridGaussSeidel_core( hypre_ParCSRMatrix *A,
       num_sends = hypre_ParCSRCommPkgNumSends(comm_pkg);
 
 #if defined(HYPRE_USING_PERSISTENT_COMM)
-      persistent_comm_handle = hypre_ParCSRCommPkgGetPersistentCommHandle(1, comm_pkg);
-      v_buf_data = (HYPRE_Real *) hypre_ParCSRCommHandleSendDataBuffer(persistent_comm_handle);
-      v_ext_data = (HYPRE_Real *) hypre_ParCSRCommHandleRecvDataBuffer(persistent_comm_handle);
+      persistent_comm_handle = hypre_ParCSRCommPkgGetPersistentCommHandle(1, comm_pkg, HYPRE_MEMORY_HOST, HYPRE_MEMORY_HOST);
+      v_buf_data = (HYPRE_Real *) hypre_ParCSRCommHandleSendData(persistent_comm_handle);
+      v_ext_data = (HYPRE_Real *) hypre_ParCSRCommHandleRecvData(persistent_comm_handle);
 #else
       v_buf_data = hypre_CTAlloc(HYPRE_Real,
                                  hypre_ParCSRCommPkgSendMapStart(comm_pkg, num_sends),
@@ -813,13 +813,13 @@ hypre_BoomerAMGRelaxHybridGaussSeidel_core( hypre_ParCSRMatrix *A,
 #endif
 
 #if defined(HYPRE_USING_PERSISTENT_COMM)
-      hypre_ParCSRPersistentCommHandleStart(persistent_comm_handle, HYPRE_MEMORY_HOST, v_buf_data);
+      hypre_ParCSRPersistentCommHandleStart(persistent_comm_handle);
 #else
       comm_handle = hypre_ParCSRCommHandleCreate(1, comm_pkg, v_buf_data, v_ext_data);
 #endif
 
 #if defined(HYPRE_USING_PERSISTENT_COMM)
-      hypre_ParCSRPersistentCommHandleWait(persistent_comm_handle, HYPRE_MEMORY_HOST, v_ext_data);
+      hypre_ParCSRCommHandleWait(persistent_comm_handle);
 #else
       hypre_ParCSRCommHandleDestroy(comm_handle);
 #endif

src/parcsr_mv/_hypre_parcsr_mv.h

@@ -29,7 +29,6 @@ extern "C" {
  *   Structure containing information for doing communications
  *--------------------------------------------------------------------------*/
 
-#ifdef HYPRE_USING_PERSISTENT_COMM
 typedef enum CommPkgJobType
 {
    HYPRE_COMM_PKG_JOB_COMPLEX = 0,
@@ -38,9 +37,90 @@ typedef enum CommPkgJobType
    HYPRE_COMM_PKG_JOB_INT_TRANSPOSE,
    HYPRE_COMM_PKG_JOB_BIGINT,
    HYPRE_COMM_PKG_JOB_BIGINT_TRANSPOSE,
-   NUM_OF_COMM_PKG_JOB_TYPE,
+   NUM_OF_COMM_PKG_JOB_TYPE
 } CommPkgJobType;
-#endif
+
+static inline CommPkgJobType
+hypre_ParCSRCommHandleGetJobType(HYPRE_Int job)
+{
+   CommPkgJobType job_type = HYPRE_COMM_PKG_JOB_COMPLEX;
+   switch (job)
+   {
+      case  1:
+         job_type = HYPRE_COMM_PKG_JOB_COMPLEX;
+         break;
+      case  2:
+         job_type = HYPRE_COMM_PKG_JOB_COMPLEX_TRANSPOSE;
+         break;
+      case 11:
+         job_type = HYPRE_COMM_PKG_JOB_INT;
+         break;
+      case 12:
+         job_type = HYPRE_COMM_PKG_JOB_INT_TRANSPOSE;
+         break;
+      case 21:
+         job_type = HYPRE_COMM_PKG_JOB_BIGINT;
+         break;
+      case 22:
+         job_type = HYPRE_COMM_PKG_JOB_BIGINT_TRANSPOSE;
+         break;
+   }
+
+   return job_type;
+}
+
+static inline HYPRE_Int
+hypre_ParCSRCommHandleIsTransposeJob(HYPRE_Int job)
+{
+   HYPRE_Int trans = 0;
+
+   switch (hypre_ParCSRCommHandleGetJobType(job))
+   {
+      case HYPRE_COMM_PKG_JOB_COMPLEX:
+      case HYPRE_COMM_PKG_JOB_INT:
+      case HYPRE_COMM_PKG_JOB_BIGINT:
+      {
+         trans = 0;
+         break;
+      }
+      case HYPRE_COMM_PKG_JOB_COMPLEX_TRANSPOSE:
+      case HYPRE_COMM_PKG_JOB_INT_TRANSPOSE:
+      case HYPRE_COMM_PKG_JOB_BIGINT_TRANSPOSE:
+      {
+         trans = 1;
+         break;
+      }
+      default:
+         break;
+   }
+   return trans;
+}
+
+static inline hypre_MPI_Datatype
+hypre_ParCSRCommHandleGetMPIDataType(HYPRE_Int job)
+{
+   hypre_MPI_Datatype dtype = HYPRE_MPI_COMPLEX;
+
+   switch (hypre_ParCSRCommHandleGetJobType(job))
+   {
+      case HYPRE_COMM_PKG_JOB_COMPLEX:
+      case HYPRE_COMM_PKG_JOB_COMPLEX_TRANSPOSE:
+         dtype = HYPRE_MPI_COMPLEX;
+         break;
+      case HYPRE_COMM_PKG_JOB_INT:
+      case HYPRE_COMM_PKG_JOB_INT_TRANSPOSE:
+         dtype = HYPRE_MPI_INT;
+         break;
+      case HYPRE_COMM_PKG_JOB_BIGINT:
+      case HYPRE_COMM_PKG_JOB_BIGINT_TRANSPOSE:
+         dtype = HYPRE_MPI_BIG_INT;
+         break;
+      default:
+         break;
+   }
+
+   return dtype;
+}
 
 /*--------------------------------------------------------------------------
  * hypre_ParCSRCommHandle, hypre_ParCSRPersistentCommHandle
@@ -50,18 +130,31 @@ struct _hypre_ParCSRCommPkg;
 typedef struct
 {
    struct _hypre_ParCSRCommPkg *comm_pkg;
-   HYPRE_MemoryLocation  send_memory_location;
-   HYPRE_MemoryLocation  recv_memory_location;
-   HYPRE_Int             num_send_bytes;
-   HYPRE_Int             num_recv_bytes;
-   void                 *send_data;
-   void                 *recv_data;
-   void                 *send_data_buffer;
-   void                 *recv_data_buffer;
-   HYPRE_Int             num_requests;
-   hypre_MPI_Request    *requests;
+   HYPRE_Int                    persistent;
+   void                        *send_data;
+   void                        *recv_data;
+   HYPRE_MemoryLocation         send_location;
+   HYPRE_MemoryLocation         recv_location;
+   HYPRE_Int                    num_requests;
+   hypre_MPI_Request           *requests;
+   hypre_MPICommWrapper        *comm;
 } hypre_ParCSRCommHandle;
 
+/*--------------------------------------------------------------------------
+ * Accessor macros: hypre_ParCSRCommHandle
+ *--------------------------------------------------------------------------*/
+
+#define hypre_ParCSRCommHandleCommPkg(comm_handle)                (comm_handle -> comm_pkg)
+#define hypre_ParCSRCommHandlePersistent(comm_handle)             (comm_handle -> persistent)
+#define hypre_ParCSRCommHandleSendData(comm_handle)               (comm_handle -> send_data)
+#define hypre_ParCSRCommHandleRecvData(comm_handle)               (comm_handle -> recv_data)
+#define hypre_ParCSRCommHandleSendLocation(comm_handle)           (comm_handle -> send_location)
+#define hypre_ParCSRCommHandleRecvLocation(comm_handle)           (comm_handle -> recv_location)
+#define hypre_ParCSRCommHandleNumRequests(comm_handle)            (comm_handle -> num_requests)
+#define hypre_ParCSRCommHandleRequests(comm_handle)               (comm_handle -> requests)
+#define hypre_ParCSRCommHandleRequest(comm_handle, i)             (comm_handle -> requests[i])
+#define hypre_ParCSRCommHandleComm(comm_handle)                   (comm_handle -> comm)
+
 typedef hypre_ParCSRCommHandle hypre_ParCSRPersistentCommHandle;
 
 typedef struct _hypre_ParCSRCommPkg
@@ -79,7 +172,7 @@ typedef struct _hypre_ParCSRCommPkg
    /* remote communication information */
    hypre_MPI_Datatype               *send_mpi_types;
    hypre_MPI_Datatype               *recv_mpi_types;
-#ifdef HYPRE_USING_PERSISTENT_COMM
+#if defined(HYPRE_USING_PERSISTENT_COMM)
    hypre_ParCSRPersistentCommHandle *persistent_comm_handles[NUM_OF_COMM_PKG_JOB_TYPE];
 #endif
 #if defined(HYPRE_USING_GPU) || defined(HYPRE_USING_DEVICE_OPENMP)
@@ -94,31 +187,33 @@ typedef struct _hypre_ParCSRCommPkg
  * Accessor macros: hypre_ParCSRCommPkg
  *--------------------------------------------------------------------------*/
 
-#define hypre_ParCSRCommPkgComm(comm_pkg)                (comm_pkg -> comm)
-#define hypre_ParCSRCommPkgNumComponents(comm_pkg)       (comm_pkg -> num_components)
-#define hypre_ParCSRCommPkgNumSends(comm_pkg)            (comm_pkg -> num_sends)
-#define hypre_ParCSRCommPkgSendProcs(comm_pkg)           (comm_pkg -> send_procs)
-#define hypre_ParCSRCommPkgSendProc(comm_pkg, i)         (comm_pkg -> send_procs[i])
-#define hypre_ParCSRCommPkgSendMapStarts(comm_pkg)       (comm_pkg -> send_map_starts)
-#define hypre_ParCSRCommPkgSendMapStart(comm_pkg,i)      (comm_pkg -> send_map_starts[i])
-#define hypre_ParCSRCommPkgSendMapElmts(comm_pkg)        (comm_pkg -> send_map_elmts)
-#define hypre_ParCSRCommPkgDeviceSendMapElmts(comm_pkg)  (comm_pkg -> device_send_map_elmts)
-#define hypre_ParCSRCommPkgSendMapElmt(comm_pkg,i)       (comm_pkg -> send_map_elmts[i])
-#define hypre_ParCSRCommPkgDeviceSendMapElmt(comm_pkg,i) (comm_pkg -> device_send_map_elmts[i])
-#define hypre_ParCSRCommPkgNumRecvs(comm_pkg)            (comm_pkg -> num_recvs)
-#define hypre_ParCSRCommPkgRecvProcs(comm_pkg)           (comm_pkg -> recv_procs)
-#define hypre_ParCSRCommPkgRecvProc(comm_pkg, i)         (comm_pkg -> recv_procs[i])
-#define hypre_ParCSRCommPkgRecvVecStarts(comm_pkg)       (comm_pkg -> recv_vec_starts)
-#define hypre_ParCSRCommPkgRecvVecStart(comm_pkg,i)      (comm_pkg -> recv_vec_starts[i])
-#define hypre_ParCSRCommPkgSendMPITypes(comm_pkg)        (comm_pkg -> send_mpi_types)
-#define hypre_ParCSRCommPkgSendMPIType(comm_pkg,i)       (comm_pkg -> send_mpi_types[i])
-#define hypre_ParCSRCommPkgRecvMPITypes(comm_pkg)        (comm_pkg -> recv_mpi_types)
-#define hypre_ParCSRCommPkgRecvMPIType(comm_pkg,i)       (comm_pkg -> recv_mpi_types[i])
+#define hypre_ParCSRCommPkgComm(comm_pkg)                        (comm_pkg -> comm)
+#define hypre_ParCSRCommPkgNumComponents(comm_pkg)               (comm_pkg -> num_components)
+#define hypre_ParCSRCommPkgNumSends(comm_pkg)                    (comm_pkg -> num_sends)
+#define hypre_ParCSRCommPkgSendProcs(comm_pkg)                   (comm_pkg -> send_procs)
+#define hypre_ParCSRCommPkgSendProc(comm_pkg, i)                 (comm_pkg -> send_procs[i])
+#define hypre_ParCSRCommPkgSendMapStarts(comm_pkg)               (comm_pkg -> send_map_starts)
+#define hypre_ParCSRCommPkgSendMapStart(comm_pkg,i)              (comm_pkg -> send_map_starts[i])
+#define hypre_ParCSRCommPkgSendMapElmts(comm_pkg)                (comm_pkg -> send_map_elmts)
+#define hypre_ParCSRCommPkgDeviceSendMapElmts(comm_pkg)          (comm_pkg -> device_send_map_elmts)
+#define hypre_ParCSRCommPkgSendMapElmt(comm_pkg,i)               (comm_pkg -> send_map_elmts[i])
+#define hypre_ParCSRCommPkgDeviceSendMapElmt(comm_pkg,i)         (comm_pkg -> device_send_map_elmts[i])
+#define hypre_ParCSRCommPkgNumRecvs(comm_pkg)                    (comm_pkg -> num_recvs)
+#define hypre_ParCSRCommPkgRecvProcs(comm_pkg)                   (comm_pkg -> recv_procs)
+#define hypre_ParCSRCommPkgRecvProc(comm_pkg, i)                 (comm_pkg -> recv_procs[i])
+#define hypre_ParCSRCommPkgRecvVecStarts(comm_pkg)               (comm_pkg -> recv_vec_starts)
+#define hypre_ParCSRCommPkgRecvVecStart(comm_pkg,i)              (comm_pkg -> recv_vec_starts[i])
+#define hypre_ParCSRCommPkgSendMPITypes(comm_pkg)                (comm_pkg -> send_mpi_types)
+#define hypre_ParCSRCommPkgSendMPIType(comm_pkg,i)               (comm_pkg -> send_mpi_types[i])
+#define hypre_ParCSRCommPkgRecvMPITypes(comm_pkg)                (comm_pkg -> recv_mpi_types)
+#define hypre_ParCSRCommPkgRecvMPIType(comm_pkg,i)               (comm_pkg -> recv_mpi_types[i])
+#define hypre_ParCSRCommPkgPersistentCommHandles(comm_pkg)       (comm_pkg -> persistent_comm_handles)
+#define hypre_ParCSRCommPkgPersistentCommHandle(comm_pkg,i)      (comm_pkg -> persistent_comm_handles[i])
 
 #if defined(HYPRE_USING_GPU) || defined(HYPRE_USING_DEVICE_OPENMP)
-#define hypre_ParCSRCommPkgTmpData(comm_pkg)             ((comm_pkg) -> tmp_data)
-#define hypre_ParCSRCommPkgBufData(comm_pkg)             ((comm_pkg) -> buf_data)
-#define hypre_ParCSRCommPkgMatrixE(comm_pkg)             ((comm_pkg) -> matrix_E)
+#define hypre_ParCSRCommPkgTmpData(comm_pkg)                     ((comm_pkg) -> tmp_data)
+#define hypre_ParCSRCommPkgBufData(comm_pkg)                     ((comm_pkg) -> buf_data)
+#define hypre_ParCSRCommPkgMatrixE(comm_pkg)                     ((comm_pkg) -> matrix_E)
 #endif
 
 static inline HYPRE_MAYBE_UNUSED_FUNC void
@@ -146,23 +241,6 @@ hypre_ParCSRCommPkgCopySendMapElmtsToDevice(hypre_ParCSRCommPkg *comm_pkg)
 #endif
 }
 
-/*--------------------------------------------------------------------------
- * Accessor macros: hypre_ParCSRCommHandle
- *--------------------------------------------------------------------------*/
-
-#define hypre_ParCSRCommHandleCommPkg(comm_handle)                (comm_handle -> comm_pkg)
-#define hypre_ParCSRCommHandleSendMemoryLocation(comm_handle)     (comm_handle -> send_memory_location)
-#define hypre_ParCSRCommHandleRecvMemoryLocation(comm_handle)     (comm_handle -> recv_memory_location)
-#define hypre_ParCSRCommHandleNumSendBytes(comm_handle)           (comm_handle -> num_send_bytes)
-#define hypre_ParCSRCommHandleNumRecvBytes(comm_handle)           (comm_handle -> num_recv_bytes)
-#define hypre_ParCSRCommHandleSendData(comm_handle)               (comm_handle -> send_data)
-#define hypre_ParCSRCommHandleRecvData(comm_handle)               (comm_handle -> recv_data)
-#define hypre_ParCSRCommHandleSendDataBuffer(comm_handle)         (comm_handle -> send_data_buffer)
-#define hypre_ParCSRCommHandleRecvDataBuffer(comm_handle)         (comm_handle -> recv_data_buffer)
-#define hypre_ParCSRCommHandleNumRequests(comm_handle)            (comm_handle -> num_requests)
-#define hypre_ParCSRCommHandleRequests(comm_handle)               (comm_handle -> requests)
-#define hypre_ParCSRCommHandleRequest(comm_handle, i)             (comm_handle -> requests[i])
-
 #endif /* HYPRE_PAR_CSR_COMMUNICATION_HEADER */
 /******************************************************************************
  * Copyright (c) 1998 Lawrence Livermore National Security, LLC and other
@@ -898,12 +976,20 @@ HYPRE_Int hypre_BooleanGenerateDiagAndOffd ( hypre_CSRBooleanMatrix *A,
 /* par_csr_communication.c */
 hypre_ParCSRCommHandle *hypre_ParCSRCommHandleCreate ( HYPRE_Int job, hypre_ParCSRCommPkg *comm_pkg,
                                                        void *send_data, void *recv_data );
+hypre_ParCSRCommHandle *hypre_ParCSRCommHandleCreate_core ( HYPRE_Int persistent,
+                                                            HYPRE_Int job,
+                                                            hypre_ParCSRCommPkg *comm_pkg,
+                                                            HYPRE_MemoryLocation send_memory_location,
+                                                            void *send_data_in,
+                                                            HYPRE_MemoryLocation recv_memory_location,
+                                                            void *recv_data_in );
 hypre_ParCSRCommHandle *hypre_ParCSRCommHandleCreate_v2 ( HYPRE_Int job,
                                                           hypre_ParCSRCommPkg *comm_pkg,
                                                           HYPRE_MemoryLocation send_memory_location,
                                                           void *send_data_in,
                                                           HYPRE_MemoryLocation recv_memory_location,
                                                           void *recv_data_in );
+HYPRE_Int hypre_ParCSRCommHandleWait(hypre_ParCSRCommHandle *comm_handle);
 HYPRE_Int hypre_ParCSRCommHandleDestroy ( hypre_ParCSRCommHandle *comm_handle );
 void hypre_ParCSRCommPkgCreate_core ( MPI_Comm comm, HYPRE_BigInt *col_map_offd,
                                       HYPRE_BigInt first_col_diag, HYPRE_BigInt *col_starts, HYPRE_Int num_cols_diag,
@@ -1034,15 +1120,18 @@ HYPRE_Int hypre_ParCSRMatrixDropSmallEntriesDevice( hypre_ParCSRMatrix *A, HYPRE
 HYPRE_Int hypre_ParCSRCommPkgCreateMatrixE( hypre_ParCSRCommPkg *comm_pkg, HYPRE_Int local_ncols );
 
 #ifdef HYPRE_USING_PERSISTENT_COMM
-hypre_ParCSRPersistentCommHandle* hypre_ParCSRPersistentCommHandleCreate(HYPRE_Int job,
-                                                                         hypre_ParCSRCommPkg *comm_pkg);
-hypre_ParCSRPersistentCommHandle* hypre_ParCSRCommPkgGetPersistentCommHandle(HYPRE_Int job,
-                                                                             hypre_ParCSRCommPkg *comm_pkg);
-void hypre_ParCSRPersistentCommHandleDestroy(hypre_ParCSRPersistentCommHandle *comm_handle);
-void hypre_ParCSRPersistentCommHandleStart(hypre_ParCSRPersistentCommHandle *comm_handle,
-                                           HYPRE_MemoryLocation send_memory_location, void *send_data);
-void hypre_ParCSRPersistentCommHandleWait(hypre_ParCSRPersistentCommHandle *comm_handle,
-                                          HYPRE_MemoryLocation recv_memory_location, void *recv_data);
+hypre_ParCSRCommHandle* hypre_ParCSRPersistentCommHandleCreate(HYPRE_Int job,
+                                                               hypre_ParCSRCommPkg *comm_pkg,
+                                                               HYPRE_MemoryLocation send_memory_location,
+                                                               HYPRE_MemoryLocation recv_memory_location);
+hypre_ParCSRCommHandle* hypre_ParCSRCommPkgGetPersistentCommHandle(HYPRE_Int job,
+                                                                   hypre_ParCSRCommPkg *comm_pkg,
+                                                                   HYPRE_MemoryLocation send_memory_location,
+                                                                   HYPRE_MemoryLocation recv_memory_location);
+#define hypre_ParCSRPersistentCommHandleWait hypre_ParCSRCommHandleWait
+#define hypre_ParCSRPersistentCommHandleDestroy hypre_ParCSRCommHandleDestroy
+
+HYPRE_Int hypre_ParCSRPersistentCommHandleStart(hypre_ParCSRCommHandle *comm_handle);
 #endif
 
 HYPRE_Int hypre_ParcsrGetExternalRowsInit( hypre_ParCSRMatrix *A, HYPRE_Int indices_len,