Correct size of message in mpi send/recv functions. Now unit tests pa…

…ss repeatedly without seg faults. Implement an array_recv method, although I don't foresee using it.

unit/mctest_mpi_comm.c

@@ -422,6 +422,8 @@ test_n2_array_send_irecv_1d()
     TEST_CHECK( f[0] == recvval );
   }
 
+  gkyl_comm_barrier(comm);
+
   gkyl_comm_state_release(comm, cstate);
   gkyl_array_release(arrA);
   gkyl_array_release(arrB);

zero/gkyl_comm.h

@@ -30,6 +30,10 @@ typedef int (*gkyl_array_send_t)(struct gkyl_array *array, int dest, int tag,
 typedef int (*gkyl_array_isend_t)(struct gkyl_array *array, int dest, int tag,
   struct gkyl_comm *comm, struct gkyl_comm_state *state);
 
+// Blocking receive @a array from @a src process using @a tag.
+typedef int (*gkyl_array_recv_t)(struct gkyl_array *array, int src, int tag,
+  struct gkyl_comm *comm);
+
 // Nonblocking receive @a array from @a src process using @a tag, and
 // store the status of this comm in @a state.
 typedef int (*gkyl_array_irecv_t)(struct gkyl_array *array, int src, int tag,
@@ -85,6 +89,7 @@ struct gkyl_comm {
   get_size_t get_size; // get number of ranks
   gkyl_array_send_t gkyl_array_send; // blocking send array.
   gkyl_array_isend_t gkyl_array_isend; // nonblocking send array.
+  gkyl_array_recv_t gkyl_array_recv; // blocking recv array.
   gkyl_array_irecv_t gkyl_array_irecv; // nonblocking recv array.
   all_reduce_t all_reduce; // all reduce function
   gkyl_array_sync_t gkyl_array_sync; // sync array
@@ -159,6 +164,21 @@ gkyl_comm_array_isend(struct gkyl_comm *comm, struct gkyl_array *array,
   return comm->gkyl_array_isend(array, dest, tag, comm, state);
 }
 
+/**
+ * Blocking recv a gkyl array from another process.
+ * @param comm Communicator.
+ * @param array Array to receive into.
+ * @param src MPI rank we are receiving from. 
+ * @param tag MPI tag.
+ * @return error code: 0 for success
+ */
+static int
+gkyl_comm_array_recv(struct gkyl_comm *comm, struct gkyl_array *array,
+  int src, int tag)
+{
+  return comm->gkyl_array_recv(array, src, tag, comm);
+}
+
 /**
  * Nonblocking recv a gkyl array from another process.
  * @param comm Communicator.

zero/mpi_comm.c

@@ -108,7 +108,7 @@ get_size(struct gkyl_comm *comm, int *sz)
 static int
 array_send(struct gkyl_array *array, int dest, int tag, struct gkyl_comm *comm)
 {
-  size_t vol = array->esznc*array->size;
+  size_t vol = array->ncomp*array->size;
   struct mpi_comm *mpi = container_of(comm, struct mpi_comm, base);  
   int ret = MPI_Send(array->data, vol, g2_mpi_datatype[array->type], dest, tag, mpi->mcomm); 
   return ret == MPI_SUCCESS ? 0 : 1;
@@ -117,16 +117,26 @@ array_send(struct gkyl_array *array, int dest, int tag, struct gkyl_comm *comm)
 static int
 array_isend(struct gkyl_array *array, int dest, int tag, struct gkyl_comm *comm, struct gkyl_comm_state *state)
 {
-  size_t vol = array->esznc*array->size;
+  size_t vol = array->ncomp*array->size;
   struct mpi_comm *mpi = container_of(comm, struct mpi_comm, base);  
   int ret = MPI_Isend(array->data, vol, g2_mpi_datatype[array->type], dest, tag, mpi->mcomm, &state->req); 
   return ret == MPI_SUCCESS ? 0 : 1;
 }
 
+static int
+array_recv(struct gkyl_array *array, int src, int tag, struct gkyl_comm *comm)
+{
+  size_t vol = array->ncomp*array->size;
+  struct mpi_comm *mpi = container_of(comm, struct mpi_comm, base);  
+  MPI_Status stat;
+  int ret = MPI_Recv(array->data, vol, g2_mpi_datatype[array->type], src, tag, mpi->mcomm, &stat); 
+  return ret == MPI_SUCCESS ? 0 : 1;
+}
+
 static int
 array_irecv(struct gkyl_array *array, int src, int tag, struct gkyl_comm *comm, struct gkyl_comm_state *state)
 {
-  size_t vol = array->esznc*array->size;
+  size_t vol = array->ncomp*array->size;
   struct mpi_comm *mpi = container_of(comm, struct mpi_comm, base);  
   int ret = MPI_Irecv(array->data, vol, g2_mpi_datatype[array->type], src, tag, mpi->mcomm, &state->req); 
   return ret == MPI_SUCCESS ? 0 : 1;
@@ -579,6 +589,7 @@ gkyl_mpi_comm_new(const struct gkyl_mpi_comm_inp *inp)
   mpi->base.barrier = barrier;
   mpi->base.gkyl_array_send = array_send;
   mpi->base.gkyl_array_isend = array_isend;
+  mpi->base.gkyl_array_recv = array_recv;
   mpi->base.gkyl_array_irecv = array_irecv;
   mpi->base.all_reduce = all_reduce;
   mpi->base.gkyl_array_sync = array_sync;