prov/sm2: Add self_region and mmap to struct sm2_ep

Clean up code, and save instructions by adding self_region and mmap to struct sm2_ep Signed-off-by: Seth Zegelstein <[email protected]>
ofiwg · Jun 30, 2023 · a097685 · a097685
1 parent 45e59fd
commit a097685
Show file tree

Hide file tree

Showing 4 changed files with 23 additions and 50 deletions.
diff --git a/prov/sm2/src/sm2.h b/prov/sm2/src/sm2.h
@@ -215,6 +215,8 @@ struct sm2_ep {
 	size_t rx_size;
 	size_t tx_size;
 	const char *name;
+	struct sm2_mmap *mmap;
+	struct sm2_region *self_region;
 	sm2_gid_t gid;
 	struct fid_ep *srx;
 	struct ofi_bufpool *xfer_ctx_pool;
@@ -273,26 +275,17 @@ int sm2_unexp_start(struct fi_peer_rx_entry *rx_entry);
 
 static inline struct sm2_region *sm2_peer_region(struct sm2_ep *ep, int id)
 {
-	struct sm2_av *av;
-
 	assert(id < SM2_MAX_UNIVERSE_SIZE);
-	av = container_of(ep->util_ep.av, struct sm2_av, util_av);
-
-	return sm2_mmap_ep_region(&av->mmap, id);
+	return sm2_mmap_ep_region(ep->mmap, id);
 }
 
 static inline size_t sm2_pop_xfer_entry(struct sm2_ep *ep,
 					struct sm2_xfer_entry **xfer_entry)
 {
-	struct sm2_av *av =
-		container_of(ep->util_ep.av, struct sm2_av, util_av);
-	struct sm2_mmap *map = &av->mmap;
-	struct sm2_region *self_region = sm2_mmap_ep_region(map, ep->gid);
-
-	if (smr_freestack_isempty(sm2_freestack(self_region)))
+	if (smr_freestack_isempty(sm2_freestack(ep->self_region)))
 		return -FI_EAGAIN;
 
-	*xfer_entry = smr_freestack_pop(sm2_freestack(self_region));
+	*xfer_entry = smr_freestack_pop(sm2_freestack(ep->self_region));
 	return FI_SUCCESS;
 }
 

diff --git a/prov/sm2/src/sm2_ep.c b/prov/sm2/src/sm2_ep.c
@@ -173,7 +173,7 @@ ssize_t sm2_verify_peer(struct sm2_ep *ep, fi_addr_t fi_addr, sm2_gid_t *gid)
 	if (sm2_av->reverse_lookup[*gid] == FI_ADDR_NOTAVAIL)
 		return -FI_EINVAL;
 
-	entries = sm2_mmap_entries(&sm2_av->mmap);
+	entries = sm2_mmap_entries(ep->mmap);
 	/* TODO... should this be atomic? */
 	if (entries[*gid].startup_ready == false)
 		return -FI_EAGAIN;
@@ -217,27 +217,22 @@ static ssize_t sm2_do_inject(struct sm2_ep *ep, struct sm2_region *peer_smr,
 static void cleanup_shm_resources(struct sm2_ep *ep)
 {
 	struct sm2_xfer_entry *xfer_entry;
-	struct sm2_av *av =
-		container_of(ep->util_ep.av, struct sm2_av, util_av);
-	struct sm2_mmap *map = &av->mmap;
-	struct sm2_region *self_region = sm2_mmap_ep_region(map, ep->gid);
 	bool retry = true;
 
 	/* Return all free queue entries in queue without processing them */
 return_incoming:
 	while (NULL != (xfer_entry = sm2_fifo_read(ep))) {
 		if (xfer_entry->hdr.proto == sm2_proto_return) {
-			smr_freestack_push(
-				sm2_freestack(sm2_mmap_ep_region(map, ep->gid)),
-				xfer_entry);
+			smr_freestack_push(sm2_freestack(ep->self_region),
+					   xfer_entry);
 		} else {
 			/* TODO Tell other side that we haven't processed their
 			 * message, just returned xfer_entry */
 			sm2_fifo_write_back(ep, xfer_entry);
 		}
 	}
 
-	if (smr_freestack_isfull(sm2_freestack(self_region))) {
+	if (smr_freestack_isfull(sm2_freestack(ep->self_region))) {
 		/* TODO Set head/tail of FIFO queue to show peers we aren't
 		   accepting new entires */
 		FI_INFO(&sm2_prov, FI_LOG_EP_CTRL,
@@ -263,12 +258,6 @@ static int sm2_ep_close(struct fid *fid)
 {
 	struct sm2_ep *ep =
 		container_of(fid, struct sm2_ep, util_ep.ep_fid.fid);
-	struct sm2_av *av =
-		container_of(ep->util_ep.av, struct sm2_av, util_av);
-	struct sm2_mmap *map = &av->mmap;
-	struct sm2_region *self_region;
-
-	self_region = sm2_mmap_ep_region(map, ep->gid);
 
 	cleanup_shm_resources(ep);
 
@@ -282,10 +271,10 @@ static int sm2_ep_close(struct fid *fid)
 	 */
 	/* TODO Do we want to mark our entry as zombie now if we don't have all
 	   our xfer_entry? */
-	if (smr_freestack_isfull(sm2_freestack(self_region))) {
-		sm2_file_lock(map);
-		sm2_entry_free(map, ep->gid);
-		sm2_file_unlock(map);
+	if (smr_freestack_isfull(sm2_freestack(ep->self_region))) {
+		sm2_file_lock(ep->mmap);
+		sm2_entry_free(ep->mmap, ep->gid);
+		sm2_file_unlock(ep->mmap);
 	}
 
 	if (ep->xfer_ctx_pool)
@@ -456,6 +445,8 @@ static int sm2_ep_ctrl(struct fid *fid, int command, void *arg)
 
 		ret = sm2_create(&sm2_prov, &attr, &av->mmap, &self_gid);
 		ep->gid = self_gid;
+		ep->mmap = &av->mmap;
+		ep->self_region = sm2_mmap_ep_region(ep->mmap, ep->gid);
 
 		if (ret)
 			return ret;

diff --git a/prov/sm2/src/sm2_fifo.h b/prov/sm2/src/sm2_fifo.h
@@ -165,13 +165,10 @@ static inline void sm2_fifo_init(struct sm2_fifo *fifo)
 static inline void sm2_fifo_write(struct sm2_ep *ep, sm2_gid_t peer_gid,
 				  struct sm2_xfer_entry *xfer_entry)
 {
-	struct sm2_av *av =
-		container_of(ep->util_ep.av, struct sm2_av, util_av);
-	struct sm2_mmap *map = &av->mmap;
-	struct sm2_region *peer_region = sm2_mmap_ep_region(map, peer_gid);
+	struct sm2_region *peer_region = sm2_mmap_ep_region(ep->mmap, peer_gid);
 	struct sm2_fifo *peer_fifo = sm2_recv_queue(peer_region);
+	long int offset = sm2_absptr_to_relptr(xfer_entry, ep->mmap);
 	struct sm2_xfer_entry *prev_xfer_entry;
-	long int offset = sm2_absptr_to_relptr(xfer_entry, map);
 	long int prev;
 
 	assert(peer_fifo->head != 0);
@@ -187,7 +184,7 @@ static inline void sm2_fifo_write(struct sm2_ep *ep, sm2_gid_t peer_gid,
 	assert(prev != offset);
 
 	if (SM2_FIFO_FREE != prev) {
-		prev_xfer_entry = sm2_relptr_to_absptr(prev, map);
+		prev_xfer_entry = sm2_relptr_to_absptr(prev, ep->mmap);
 		prev_xfer_entry->hdr.next = offset;
 	} else {
 		peer_fifo->head = offset;
@@ -199,11 +196,7 @@ static inline void sm2_fifo_write(struct sm2_ep *ep, sm2_gid_t peer_gid,
 /* Read, Dequeue */
 static inline struct sm2_xfer_entry *sm2_fifo_read(struct sm2_ep *ep)
 {
-	struct sm2_av *av =
-		container_of(ep->util_ep.av, struct sm2_av, util_av);
-	struct sm2_mmap *map = &av->mmap;
-	struct sm2_region *self_region = sm2_mmap_ep_region(map, ep->gid);
-	struct sm2_fifo *self_fifo = sm2_recv_queue(self_region);
+	struct sm2_fifo *self_fifo = sm2_recv_queue(ep->self_region);
 	struct sm2_xfer_entry *xfer_entry;
 	long int prev_head;
 
@@ -216,8 +209,8 @@ static inline struct sm2_xfer_entry *sm2_fifo_read(struct sm2_ep *ep)
 	atomic_rmb();
 
 	prev_head = self_fifo->head;
-	xfer_entry =
-		(struct sm2_xfer_entry *) sm2_relptr_to_absptr(prev_head, map);
+	xfer_entry = (struct sm2_xfer_entry *) sm2_relptr_to_absptr(prev_head,
+								    ep->mmap);
 	self_fifo->head = SM2_FIFO_FREE;
 
 	assert(xfer_entry->hdr.next != prev_head);

diff --git a/prov/sm2/src/sm2_progress.c b/prov/sm2/src/sm2_progress.c
@@ -331,9 +331,6 @@ static int sm2_progress_atomic(struct sm2_ep *ep,
 
 void sm2_progress_recv(struct sm2_ep *ep)
 {
-	struct sm2_av *av =
-		container_of(ep->util_ep.av, struct sm2_av, util_av);
-	struct sm2_mmap *map = &av->mmap;
 	struct sm2_atomic_entry *atomic_entry;
 	struct sm2_xfer_entry *xfer_entry;
 	int ret = 0, i;
@@ -366,9 +363,8 @@ void sm2_progress_recv(struct sm2_ep *ep)
 						"completion\n");
 			}
 
-			smr_freestack_push(
-				sm2_freestack(sm2_mmap_ep_region(map, ep->gid)),
-				xfer_entry);
+			smr_freestack_push(sm2_freestack(ep->self_region),
+					   xfer_entry);
 			continue;
 		}