prov/efa: add device to host copy for inject rdma write

fi_writemsg with FI_INJECT needs to copy from outbound buffer to internal bounce buffer. The outbound buffer can be non-system. Extract and call a helper function for device to host copy. Signed-off-by: Jessie Yang <[email protected]>
ofiwg · Aug 31, 2024 · 1b3fb77 · 1b3fb77
1 parent 22be7e0
commit 1b3fb77
Show file tree

Hide file tree

Showing 3 changed files with 42 additions and 16 deletions.
diff --git a/prov/efa/src/rdm/efa_rdm_ope.c b/prov/efa/src/rdm/efa_rdm_ope.c
@@ -1507,8 +1507,10 @@ int efa_rdm_ope_post_remote_write(struct efa_rdm_ope *ope)
 		if (ope->fi_flags & FI_INJECT) {
 			assert(ope->iov_count == 1);
 			assert(ope->total_len <= ep->inject_size);
-			copied = ofi_copy_from_hmem_iov(pkt_entry->wiredata + sizeof(struct efa_rdm_rma_context_pkt),
-				ope->total_len, FI_HMEM_SYSTEM, 0, ope->iov, ope->iov_count, 0);
+			copied = efa_rdm_pke_copy_from_hmem_iov(
+				ope->desc[iov_idx], pkt_entry, ope,
+				sizeof(struct efa_rdm_rma_context_pkt), 0,
+				ope->total_len);
 			assert(copied == ope->total_len);
 			(void) copied; /* suppress compiler warning for non-debug build */
 			ope->desc[0] = fi_mr_desc(pkt_entry->mr);

diff --git a/prov/efa/src/rdm/efa_rdm_pke_utils.c b/prov/efa/src/rdm/efa_rdm_pke_utils.c
@@ -93,20 +93,8 @@ ssize_t efa_rdm_pke_init_payload_from_ope(struct efa_rdm_pke *pke,
 		return 0;
 	}
 
-	if (iov_mr && (iov_mr->peer.flags & OFI_HMEM_DATA_DEV_REG_HANDLE)) {
-		assert(iov_mr->peer.hmem_data);
-		copied = ofi_dev_reg_copy_from_hmem_iov(pke->wiredata + payload_offset,
-							data_size, iov_mr->peer.iface,
-							(uint64_t)iov_mr->peer.hmem_data,
-							ope->iov, ope->iov_count,
-							segment_offset);
-	} else {
-		copied = ofi_copy_from_hmem_iov(pke->wiredata + payload_offset,
-						data_size,
-		                                iov_mr ? iov_mr->peer.iface : FI_HMEM_SYSTEM,
-		                                iov_mr ? iov_mr->peer.device.reserved : 0,
-		                                ope->iov, ope->iov_count, segment_offset);
-	}
+	copied = efa_rdm_pke_copy_from_hmem_iov(
+		iov_mr, pke, ope, payload_offset, segment_offset, data_size);
 
 	assert(copied == data_size);
 	pke->payload = pke->wiredata + payload_offset;

diff --git a/prov/efa/src/rdm/efa_rdm_pke_utils.h b/prov/efa/src/rdm/efa_rdm_pke_utils.h
@@ -58,6 +58,42 @@ size_t efa_rdm_pke_get_segment_offset(struct efa_rdm_pke *pke)
 	return 0;
 }
 
+/**
+ * @brief copy data from the user's buffer to the packet's wiredata.
+ *
+ * @param[in]		iov_mr  memory descriptors of the user's data buffer.
+ * @param[in,out]	pke	    packet entry. Header must have been set when the function is called.
+ * @param[in]		ope		operation entry that has user buffer information.
+ * @param[in]		payload_offset	the data offset in reference to pkt_entry->wiredata.
+ * @param[in]		segment_offset	the data offset in reference to user's buffer
+ * @param[in]		data_size	length of the data to be set up.
+ * @return   		length of data copied.
+ */
+static inline size_t
+efa_rdm_pke_copy_from_hmem_iov(struct efa_mr *iov_mr, struct efa_rdm_pke *pke,
+			       struct efa_rdm_ope *ope, size_t payload_offset,
+			       size_t segment_offset, size_t data_size)
+{
+	size_t copied;
+
+	if (iov_mr && (iov_mr->peer.flags & OFI_HMEM_DATA_DEV_REG_HANDLE)) {
+		assert(iov_mr->peer.hmem_data);
+		copied = ofi_dev_reg_copy_from_hmem_iov(pke->wiredata + payload_offset,
+							data_size, iov_mr->peer.iface,
+							(uint64_t)iov_mr->peer.hmem_data,
+							ope->iov, ope->iov_count,
+							segment_offset);
+	} else {
+		copied = ofi_copy_from_hmem_iov(pke->wiredata + payload_offset,
+		                                data_size,
+		                                iov_mr ? iov_mr->peer.iface : FI_HMEM_SYSTEM,
+		                                iov_mr ? iov_mr->peer.device.reserved : 0,
+		                                ope->iov, ope->iov_count, segment_offset);
+	}
+
+	return copied;
+}
+
 size_t efa_rdm_pke_get_payload_offset(struct efa_rdm_pke *pkt_entry);
 
 ssize_t efa_rdm_pke_init_payload_from_ope(struct efa_rdm_pke *pke,