From 11dce894bc5da89bb83f4230b9723644224dd70c Mon Sep 17 00:00:00 2001 From: Shi Jin Date: Wed, 24 Jul 2024 20:18:31 +0000 Subject: [PATCH] prov/efa: Allow disabling unsolicited write recv via env Introduce FI_EFA_USE_UNSOLICITED_WRITE_RECV which means whether to use the unsolicited write recv feature from device when it's available. Setting this variable to 0 will disable this feature. Signed-off-by: Shi Jin --- man/fi_efa.7.md | 4 ++++ prov/efa/src/efa_base_ep.c | 2 +- prov/efa/src/efa_cq.h | 2 +- prov/efa/src/efa_env.c | 4 ++++ prov/efa/src/efa_env.h | 1 + prov/efa/src/rdm/efa_rdm_cq.c | 2 +- prov/efa/src/rdm/efa_rdm_util.h | 6 ++++++ 7 files changed, 18 insertions(+), 3 deletions(-) diff --git a/man/fi_efa.7.md b/man/fi_efa.7.md index f2bb7a3e9ce..686c5e74a67 100644 --- a/man/fi_efa.7.md +++ b/man/fi_efa.7.md @@ -317,6 +317,10 @@ due to other requirements. See https://github.com/ofiwg/libfabric/blob/main/prov/efa/docs/efa_rdm_protocol_v4.md for details. +*FI_EFA_USE_UNSOLICITED_WRITE_RECV* +: Use device's unsolicited write recv functionality when it's available. (Default: 1). +Setting this environment variable to 0 can disable this feature. + # SEE ALSO [`fabric`(7)](fabric.7.html), diff --git a/prov/efa/src/efa_base_ep.c b/prov/efa/src/efa_base_ep.c index 5eb69ad5892..8c55fee2387 100644 --- a/prov/efa/src/efa_base_ep.c +++ b/prov/efa/src/efa_base_ep.c @@ -181,7 +181,7 @@ int efa_qp_create(struct efa_qp **qp, struct ibv_qp_init_attr_ex *init_attr_ex) } else { assert(init_attr_ex->qp_type == IBV_QPT_DRIVER); #if HAVE_CAPS_UNSOLICITED_WRITE_RECV - if (efa_device_support_unsolicited_write_recv()) + if (efa_rdm_use_unsolicited_write_recv()) efa_attr.flags |= EFADV_QP_FLAGS_UNSOLICITED_WRITE_RECV; #endif efa_attr.driver_qp_type = EFADV_QP_DRIVER_TYPE_SRD; diff --git a/prov/efa/src/efa_cq.h b/prov/efa/src/efa_cq.h index e3486f43f46..18cf435023b 100644 --- a/prov/efa/src/efa_cq.h +++ b/prov/efa/src/efa_cq.h @@ -131,7 +131,7 @@ static inline int efa_cq_ibv_cq_ex_open(struct fi_cq_attr *attr, }; #if HAVE_CAPS_UNSOLICITED_WRITE_RECV - if (efa_device_support_unsolicited_write_recv()) + if (efa_rdm_use_unsolicited_write_recv()) efadv_cq_init_attr.wc_flags |= EFADV_WC_EX_WITH_IS_UNSOLICITED; #endif diff --git a/prov/efa/src/efa_env.c b/prov/efa/src/efa_env.c index 1c54851acf5..84614a06426 100644 --- a/prov/efa/src/efa_env.c +++ b/prov/efa/src/efa_env.c @@ -37,6 +37,7 @@ struct efa_env efa_env = { .host_id_file = "/sys/devices/virtual/dmi/id/board_asset_tag", /* Available on EC2 instances and containers */ .use_sm2 = false, .huge_page_setting = EFA_ENV_HUGE_PAGE_UNSPEC, + .use_unsolicited_write_recv = 1, }; /** @@ -141,6 +142,7 @@ void efa_env_param_get(void) fi_param_get_size_t(&efa_prov, "inter_max_gdrcopy_message_size", &efa_env.efa_max_gdrcopy_msg_size); fi_param_get_bool(&efa_prov, "use_sm2", &efa_env.use_sm2); + fi_param_get_bool(&efa_prov, "use_unsolicited_write_recv", &efa_env.use_unsolicited_write_recv); int use_huge_page; if (fi_param_get_bool(&efa_prov, "use_huge_page", &use_huge_page) ==0) { @@ -216,6 +218,8 @@ void efa_env_define() "Using huge page memory has a small performance advantage, but can " "cause system to run out of huge page memory. By default, EFA provider " "will use huge page unless FI_EFA_FORK_SAFE is set to 1/on/true."); + fi_param_define(&efa_prov, "use_unsolicited_write_recv", FI_PARAM_BOOL, + "Use device's unsolicited write recv functionality when it's available. (Default: true)"); } diff --git a/prov/efa/src/efa_env.h b/prov/efa/src/efa_env.h index 2d315d7bedb..48742fe4097 100644 --- a/prov/efa/src/efa_env.h +++ b/prov/efa/src/efa_env.h @@ -77,6 +77,7 @@ struct efa_env { char *host_id_file; int use_sm2; enum efa_env_huge_page_setting huge_page_setting; + int use_unsolicited_write_recv; }; /** diff --git a/prov/efa/src/rdm/efa_rdm_cq.c b/prov/efa/src/rdm/efa_rdm_cq.c index ba1b218a31a..b4085aa96bc 100644 --- a/prov/efa/src/rdm/efa_rdm_cq.c +++ b/prov/efa/src/rdm/efa_rdm_cq.c @@ -83,7 +83,7 @@ static struct fi_ops efa_rdm_cq_fi_ops = { static inline bool efa_rdm_cq_wc_is_unsolicited(struct ibv_cq_ex *ibv_cq_ex) { - return efa_device_support_unsolicited_write_recv() ? efadv_wc_is_unsolicited(efadv_cq_from_ibv_cq_ex(ibv_cq_ex)) : false; + return efa_rdm_use_unsolicited_write_recv() && efadv_wc_is_unsolicited(efadv_cq_from_ibv_cq_ex(ibv_cq_ex)); } #else diff --git a/prov/efa/src/rdm/efa_rdm_util.h b/prov/efa/src/rdm/efa_rdm_util.h index 02c91772c33..1b2fc1da0a2 100644 --- a/prov/efa/src/rdm/efa_rdm_util.h +++ b/prov/efa/src/rdm/efa_rdm_util.h @@ -30,4 +30,10 @@ static inline void efa_rdm_poison_mem_region(void *ptr, size_t size) } #endif +static inline +bool efa_rdm_use_unsolicited_write_recv() +{ + return efa_env.use_unsolicited_write_recv && efa_device_support_unsolicited_write_recv(); +} + #endif /* _EFA_RDM_UTIL_H */