Merge remote-tracking branch 'comm/release/2.6' into jeffolivier/goog…

…le/2.6

Change-Id: I7e6c15c07ad7fcb94622ec8d6081624641c44441
Signed-off-by: Jeff Olivier <[email protected]>

TAG

@@ -1 +1 @@
-2.6.3-rc1
+2.6.3-rc3

debian/changelog

@@ -1,3 +1,15 @@
+daos (2.6.3-3) unstable; urgency=medium
+  [ Phillip Henderson ]
+  * Third release candidate for 2.6.3
+
+ -- Phillip Henderson <[email protected]>  Tue, 11 Feb 2025 09:16:00 -0500
+
+daos (2.6.3-2) unstable; urgency=medium
+  [ Phillip Henderson ]
+  * Second release candidate for 2.6.3
+
+ -- Phillip Henderson <[email protected]>  Tue, 04 Feb 2025 17:42:00 -0500
+
 daos (2.6.3-1) unstable; urgency=medium
   [ Dalton Bohning ]
   * First release candidate for 2.6.3

src/cart/README.env

@@ -207,3 +207,5 @@ This file lists the environment variables used in CaRT.
    traffic congestion. Available options are: "unspec" (default), "best_effort",
    "low_latency", "bulk_data".
 
+ . CRT_CXI_INIT_RETRY
+   Retry count for HG_Init_opt2() when initializing the CXI provider (default = 3).

src/cart/crt_hg.c

@@ -1,6 +1,7 @@
 /*
  * (C) Copyright 2016-2024 Intel Corporation.
  * (C) Copyright 2025 Google LLC
+ * (C) Copyright 2025 Hewlett Packard Enterprise Development LP
  *
  * SPDX-License-Identifier: BSD-2-Clause-Patent
  */
@@ -834,6 +835,7 @@ crt_hg_class_init(crt_provider_t provider, int ctx_idx, bool primary, int iface_
 	char			addr_str[CRT_ADDR_STR_MAX_LEN] = {'\0'};
 	size_t			str_size = CRT_ADDR_STR_MAX_LEN;
 	struct crt_prov_gdata	*prov_data;
+	uint32_t                 retry_count = 0;
 	int			rc = DER_SUCCESS;
 
 	prov_data = crt_get_prov_gdata(primary, provider);
@@ -867,9 +869,17 @@ crt_hg_class_init(crt_provider_t provider, int ctx_idx, bool primary, int iface_
 	/* Separate SWIM traffic in an effort to prevent potential congestion. */
 	if (crt_is_service() && ctx_idx == crt_gdata.cg_swim_crt_idx)
 		init_info.traffic_class = (enum na_traffic_class)crt_gdata.cg_swim_tc;
-
+retry:
 	hg_class = HG_Init_opt2(info_string, crt_is_service(), HG_VERSION(2, 4), &init_info);
 	if (hg_class == NULL) {
+		/** workaround for DAOS-16990, DAOS-17011 - retry a few times on init */
+		if (provider == CRT_PROV_OFI_CXI && !crt_is_service() &&
+		    retry_count < crt_gdata.cg_hg_init_retry_cnt) {
+			retry_count++;
+			D_WARN("Could not initialize HG class; retrying (%d)\n", retry_count);
+			sleep(retry_count * 5);
+			goto retry;
+		}
 		D_ERROR("Could not initialize HG class.\n");
 		D_GOTO(out, rc = -DER_HG);
 	}

src/cart/crt_init.c

@@ -1,5 +1,6 @@
 /*
  * (C) Copyright 2016-2024 Intel Corporation.
+ * (C) Copyright 2025 Hewlett Packard Enterprise Development LP
  *
  * SPDX-License-Identifier: BSD-2-Clause-Patent
  */
@@ -287,11 +288,17 @@ static int data_init(int server, crt_init_options_t *opt)
 		if (mem_pin_enable == 1)
 			mem_pin_workaround();
 	} else {
+		int retry_count = 3;
+
 		/*
 		 * Client-side envariable to indicate that the cluster
 		 * is running using a secondary provider
 		 */
 		crt_env_get(CRT_SECONDARY_PROVIDER, &is_secondary);
+
+		/** Client side env for hg_init() retries */
+		crt_env_get(CRT_CXI_INIT_RETRY, &retry_count);
+		crt_gdata.cg_hg_init_retry_cnt = retry_count;
 	}
 	crt_gdata.cg_provider_is_primary = (is_secondary) ? 0 : 1;
 

src/cart/crt_internal_types.h

@@ -1,5 +1,6 @@
 /*
  * (C) Copyright 2016-2024 Intel Corporation.
+ * (C) Copyright 2025 Hewlett Packard Enterprise Development LP
  *
  * SPDX-License-Identifier: BSD-2-Clause-Patent
  */
@@ -167,6 +168,8 @@ struct crt_gdata {
 	long			 cg_num_cores;
 	/** Inflight rpc quota limit */
 	uint32_t		cg_rpc_quota;
+	/** Retry count of HG_Init_opt2() on failure when using CXI provider */
+	uint32_t                 cg_hg_init_retry_cnt;
 };
 
 extern struct crt_gdata		crt_gdata;
@@ -194,6 +197,7 @@ struct crt_event_cb_priv {
 	ENV_STR(CRT_ATTACH_INFO_PATH)                                                              \
 	ENV(CRT_CREDIT_EP_CTX)                                                                     \
 	ENV(CRT_CTX_NUM)                                                                           \
+	ENV(CRT_CXI_INIT_RETRY)                                                                    \
 	ENV(CRT_ENABLE_MEM_PIN)                                                                    \
 	ENV_STR(CRT_L_GRP_CFG)                                                                     \
 	ENV(CRT_L_RANK)                                                                            \

src/client/api/init.c

@@ -1,5 +1,6 @@
 /**
  * (C) Copyright 2016-2024 Intel Corporation.
+ * (C) Copyright 2025 Hewlett Packard Enterprise Development LP
  *
  * SPDX-License-Identifier: BSD-2-Clause-Patent
  */
@@ -270,6 +271,7 @@ daos_init(void)
 	if (rc != 0)
 		D_GOTO(out_obj, rc);
 #endif
+	daos_array_env_init();
 	module_initialized++;
 	D_GOTO(unlock, rc = 0);
 

src/client/array/dc_array.c

@@ -1,5 +1,6 @@
 /**
  * (C) Copyright 2016-2024 Intel Corporation.
+ * (C) Copyright 2025 Hewlett Packard Enterprise Development LP
  *
  * SPDX-License-Identifier: BSD-2-Clause-Patent
  */
@@ -69,6 +70,24 @@ struct io_params {
 	char			akey_val;
 };
 
+unsigned int array_list_io_limit;
+
+void
+daos_array_env_init()
+{
+	array_list_io_limit = DAOS_ARRAY_LIST_IO_LIMIT;
+	d_getenv_uint("DAOS_ARRAY_LIST_IO_LIMIT", &array_list_io_limit);
+	if (array_list_io_limit == 0) {
+		array_list_io_limit = UINT_MAX;
+	}
+	if (array_list_io_limit > DAOS_ARRAY_LIST_IO_LIMIT) {
+		D_WARN("Setting a high limit for list io descriptors (%u) is not recommended\n",
+		       array_list_io_limit);
+	} else {
+		D_DEBUG(DB_TRACE, "ARRAY List IO limit = %u\n", array_list_io_limit);
+	}
+}
+
 static void
 array_free(struct d_hlink *hlink)
 {
@@ -1436,6 +1455,38 @@ dc_array_io(daos_handle_t array_oh, daos_handle_t th,
 		D_GOTO(err_task, rc = -DER_INVAL);
 	}
 
+	/*
+	 * If we are above the limit, check for small recx size. Just a best effort check for
+	 * extreme cases to reject.
+	 */
+	if (rg_iod->arr_nr > array_list_io_limit) {
+		daos_size_t i;
+		daos_size_t tiny_count = 0;
+
+		/* quick shortcut check */
+		for (i = 0; i < rg_iod->arr_nr; i = i * 2) {
+			if (rg_iod->arr_rgs[i].rg_len > DAOS_ARRAY_RG_LEN_THD)
+				break;
+			if (i == 0)
+				i++;
+		}
+
+		/** Full check if quick check fails */
+		if (i >= rg_iod->arr_nr) {
+			for (i = 0; i < rg_iod->arr_nr; i++) {
+				if (rg_iod->arr_rgs[i].rg_len <= DAOS_ARRAY_RG_LEN_THD)
+					tiny_count++;
+				if (tiny_count > array_list_io_limit)
+					break;
+			}
+			if (tiny_count > array_list_io_limit) {
+				D_ERROR("List io supports a max of %u offsets (using %zu)",
+					array_list_io_limit, rg_iod->arr_nr);
+				D_GOTO(err_task, rc = -DER_NOTSUPPORTED);
+			}
+		}
+	}
+
 	array = array_hdl2ptr(array_oh);
 	if (array == NULL) {
 		D_ERROR("Invalid array handle: "DF_RC"\n", DP_RC(-DER_NO_HDL));

src/client/dfuse/pil4dfs/aio.c

@@ -61,11 +61,30 @@ static int (*next_io_cancel)(io_context_t ctx, struct iocb *iocb, struct io_even
 static int (*next_io_getevents)(io_context_t ctx, long min_nr, long nr, struct io_event *events,
 				struct timespec *timeout);
 
+static int (*next_io_queue_init)(int maxevents, io_context_t *ctxp);
 /* aio functions return negative errno in case of failure */
 
 static int
 create_ev_eq_for_aio(d_aio_ctx_t *aio_ctx);
 
+int
+io_setup(int maxevents, io_context_t *ctxp);
+
+int
+io_queue_init(int maxevents, io_context_t *ctxp)
+{
+	if (next_io_queue_init == NULL) {
+		next_io_queue_init = dlsym(RTLD_NEXT, "io_queue_init");
+		D_ASSERT(next_io_queue_init != NULL);
+	}
+
+	if (maxevents > 0) {
+		return io_setup(maxevents, ctxp);
+	}
+
+	return -EINVAL;
+}
+
 int
 io_setup(int maxevents, io_context_t *ctxp)
 {

src/client/dfuse/pil4dfs/int_dfs.c

@@ -1,5 +1,6 @@
 /**
  * (C) Copyright 2022-2024 Intel Corporation.
+ * (C) Copyright 2025 Hewlett Packard Enterprise Development LP
  *
  * SPDX-License-Identifier: BSD-2-Clause-Patent
  */
@@ -6070,14 +6071,18 @@ futimens(int fd, const struct timespec times[2])
 static int
 new_fcntl(int fd, int cmd, ...)
 {
-	int     fd_directed, param, OrgFunc = 1;
+	int     fd_directed, OrgFunc = 1;
 	int     next_dirfd, next_fd, rc;
+	void   *param;
 	va_list arg;
 
 	va_start(arg, cmd);
-	param = va_arg(arg, int);
+	param = va_arg(arg, void *);
 	va_end(arg);
 
+	if (!d_hook_enabled)
+		return libc_fcntl(fd, cmd, param);
+
 	if (fd < FD_FILE_BASE && d_compatible_mode)
 		return libc_fcntl(fd, cmd, param);
 
@@ -6096,9 +6101,6 @@ new_fcntl(int fd, int cmd, ...)
 	case F_ADD_SEALS:
 		fd_directed = d_get_fd_redirected(fd);
 
-		if (!d_hook_enabled)
-			return libc_fcntl(fd, cmd, param);
-
 		if (cmd == F_GETFL) {
 			if (fd_directed >= FD_DIR_BASE)
 				return dir_list[fd_directed - FD_DIR_BASE]->open_flag;
@@ -6147,12 +6149,15 @@ new_fcntl(int fd, int cmd, ...)
 	case F_OFD_GETLK:
 	case F_GETOWN_EX:
 	case F_SETOWN_EX:
-		if (!d_hook_enabled)
+		fd_directed = d_get_fd_redirected(fd);
+		if (fd_directed >= FD_FILE_BASE) {
+			errno = ENOTSUP;
+			return (-1);
+		} else {
 			return libc_fcntl(fd, cmd, param);
-
-		return libc_fcntl(fd, cmd, param);
+		}
 	default:
-		return libc_fcntl(fd, cmd);
+		return libc_fcntl(fd, cmd, param);
 	}
 }
 

src/container/srv_target.c

@@ -1,6 +1,7 @@
 /**
  * (C) Copyright 2016-2024 Intel Corporation.
  * (C) Copyright 2025 Google LLC
+ * (C) Copyright 2025 Hewlett Packard Enterprise Development LP
  *
  * SPDX-License-Identifier: BSD-2-Clause-Patent
  */
@@ -1759,8 +1760,9 @@ ds_cont_tgt_open(uuid_t pool_uuid, uuid_t cont_hdl_uuid,
 	D_DEBUG(DB_TRACE, "open pool/cont/hdl "DF_UUID"/"DF_UUID"/"DF_UUID"\n",
 		DP_UUID(pool_uuid), DP_UUID(cont_uuid), DP_UUID(cont_hdl_uuid));
 
-	rc = ds_pool_thread_collective(pool_uuid, PO_COMP_ST_NEW | PO_COMP_ST_DOWN |
-				       PO_COMP_ST_DOWNOUT, cont_open_one, &arg, 0);
+	rc = ds_pool_thread_collective(pool_uuid,
+				       PO_COMP_ST_NEW | PO_COMP_ST_DOWN | PO_COMP_ST_DOWNOUT,
+				       cont_open_one, &arg, DSS_ULT_DEEP_STACK);
 	if (rc != 0)
 		/* Once it exclude the target from the pool, since the target
 		 * might still in the cart group, so IV cont open might still
@@ -2065,9 +2067,9 @@ ds_cont_tgt_snapshots_update(uuid_t pool_uuid, uuid_t cont_uuid,
 	 * the up targets in this scenario. The target property will be updated
 	 * upon initiating container aggregation.
 	 */
-	return ds_pool_thread_collective(pool_uuid, PO_COMP_ST_NEW | PO_COMP_ST_DOWN |
-					 PO_COMP_ST_DOWNOUT | PO_COMP_ST_UP,
-					 cont_snap_update_one, &args, 0);
+	return ds_pool_thread_collective(
+	    pool_uuid, PO_COMP_ST_NEW | PO_COMP_ST_DOWN | PO_COMP_ST_DOWNOUT | PO_COMP_ST_UP,
+	    cont_snap_update_one, &args, DSS_ULT_DEEP_STACK);
 }
 
 void

src/control/cmd/dmg/system.go

@@ -172,6 +172,7 @@ type systemStopCmd struct {
 	cmdutil.JSONOutputCmd
 	rankListCmd
 	Force bool `long:"force" description:"Force stop DAOS system members"`
+	Full  bool `long:"full" description:"Attempt a graceful shutdown of DAOS system. Experimental and not for use in production environments"`
 }
 
 // Execute is run when systemStopCmd activates.
@@ -181,11 +182,20 @@ func (cmd *systemStopCmd) Execute(_ []string) (errOut error) {
 	defer func() {
 		errOut = errors.Wrap(errOut, "system stop failed")
 	}()
+	if cmd.Force && cmd.Full {
+		return errIncompatFlags("force", "full")
+	}
+	if cmd.Full && !cmd.Hosts.Empty() {
+		return errIncompatFlags("full", "rank-hosts")
+	}
+	if cmd.Full && !cmd.Ranks.Empty() {
+		return errIncompatFlags("full", "ranks")
+	}
 
 	if err := cmd.validateHostsRanks(); err != nil {
 		return err
 	}
-	req := &control.SystemStopReq{Force: cmd.Force}
+	req := &control.SystemStopReq{Force: cmd.Force, Full: cmd.Full}
 	req.Hosts.Replace(&cmd.Hosts.HostSet)
 	req.Ranks.Replace(&cmd.Ranks.RankSet)
 

src/control/cmd/dmg/system_test.go

@@ -192,6 +192,32 @@ func TestDmg_SystemCommands(t *testing.T) {
 			"",
 			errors.New("--ranks and --rank-hosts options cannot be set together"),
 		},
+		{
+			"system stop with full option",
+			"system stop --full",
+			strings.Join([]string{
+				printRequest(t, &control.SystemStopReq{Full: true}),
+			}, " "),
+			nil,
+		},
+		{
+			"system stop with full and force options",
+			"system stop --full --force",
+			"",
+			errors.New(`may not be mixed`),
+		},
+		{
+			"system stop with full and rank-hosts options",
+			"system stop --full --rank-hosts foo-[0-2]",
+			"",
+			errors.New(`may not be mixed`),
+		},
+		{
+			"system stop with full and ranks options",
+			"system stop --full --ranks 0-2",
+			"",
+			errors.New(`may not be mixed`),
+		},
 		{
 			"system start with no arguments",
 			"system start",