Merge pull request #729 from shefty/v1.5.x

cherry-picked commits for 1.5.x branch

benchmarks/rma_bw.c

@@ -97,13 +97,18 @@ int main(int argc, char **argv)
 	if (!hints)
 		return EXIT_FAILURE;
 
+	hints->caps = FI_MSG | FI_RMA;
+	hints->domain_attr->resource_mgmt = FI_RM_ENABLED;
+	hints->mode = FI_CONTEXT;
+	hints->domain_attr->mr_mode = FI_MR_LOCAL | OFI_MR_BASIC_MAP;
+
 	while ((op = getopt(argc, argv, "ho:" CS_OPTS INFO_OPTS BENCHMARK_OPTS)) != -1) {
 		switch (op) {
 		default:
 			ft_parse_benchmark_opts(op, optarg);
 			ft_parseinfo(op, optarg, hints);
 			ft_parsecsopts(op, optarg, &opts);
-			ret = ft_parse_rma_opts(op, optarg, &opts);
+			ret = ft_parse_rma_opts(op, optarg, hints, &opts);
 			if (ret)
 				return ret;
 			break;
@@ -123,11 +128,6 @@ int main(int argc, char **argv)
 	if (optind < argc)
 		opts.dst_addr = argv[optind];
 
-	hints->caps = FI_MSG | FI_RMA;
-	hints->domain_attr->resource_mgmt = FI_RM_ENABLED;
-	hints->mode = FI_CONTEXT | FI_RX_CQ_DATA;
-	hints->domain_attr->mr_mode = FI_MR_LOCAL | OFI_MR_BASIC_MAP;
-
 	ret = run();
 
 	ft_free_res();

common/shared.c

@@ -435,11 +435,6 @@ int ft_open_fabric_res(void)
 int ft_alloc_ep_res(struct fi_info *fi)
 {
 	int ret;
-	if (hints->caps & FI_RMA) {
-		ret = ft_set_rma_caps(fi, opts.rma_op);
-		if (ret)
-			return ret;
-	}
 
 	ret = ft_alloc_msgs();
 	if (ret)
@@ -452,14 +447,12 @@ int ft_alloc_ep_res(struct fi_info *fi)
 			cq_attr.format = FI_CQ_FORMAT_CONTEXT;
 	}
 
-	if (opts.options & FT_OPT_TX_CQ) {
-		ft_cq_set_wait_attr();
-		cq_attr.size = fi->tx_attr->size;
-		ret = fi_cq_open(domain, &cq_attr, &txcq, &txcq);
-		if (ret) {
-			FT_PRINTERR("fi_cq_open", ret);
-			return ret;
-		}
+	ft_cq_set_wait_attr();
+	cq_attr.size = fi->tx_attr->size;
+	ret = fi_cq_open(domain, &cq_attr, &txcq, &txcq);
+	if (ret) {
+		FT_PRINTERR("fi_cq_open", ret);
+		return ret;
 	}
 
 	if (opts.options & FT_OPT_TX_CNTR) {
@@ -471,14 +464,12 @@ int ft_alloc_ep_res(struct fi_info *fi)
 		}
 	}
 
-	if (opts.options & FT_OPT_RX_CQ) {
-		ft_cq_set_wait_attr();
-		cq_attr.size = fi->rx_attr->size;
-		ret = fi_cq_open(domain, &cq_attr, &rxcq, &rxcq);
-		if (ret) {
-			FT_PRINTERR("fi_cq_open", ret);
-			return ret;
-		}
+	ft_cq_set_wait_attr();
+	cq_attr.size = fi->rx_attr->size;
+	ret = fi_cq_open(domain, &cq_attr, &rxcq, &rxcq);
+	if (ret) {
+		FT_PRINTERR("fi_cq_open", ret);
+		return ret;
 	}
 
 	if (opts.options & FT_OPT_RX_CNTR) {
@@ -523,27 +514,6 @@ int ft_alloc_active_res(struct fi_info *fi)
 	return 0;
 }
 
-int ft_set_rma_caps(struct fi_info *fi, enum ft_rma_opcodes rma_op)
-{
-	switch (rma_op) {
-	case FT_RMA_READ:
-		fi->caps |= FI_REMOTE_READ;
-		if (fi->mode & FI_LOCAL_MR)
-			fi->caps |= FI_READ;
-		break;
-	case FT_RMA_WRITE:
-	case FT_RMA_WRITEDATA:
-		fi->caps |= FI_REMOTE_WRITE;
-		if (fi->mode & FI_LOCAL_MR)
-			fi->caps |= FI_WRITE;
-		break;
-	default:
-		FT_ERR("Invalid rma op type\n");
-		return -FI_EINVAL;
-	}
-	return 0;
-}
-
 int ft_getinfo(struct fi_info *hints, struct fi_info **info)
 {
 	char *node, *service;
@@ -780,12 +750,6 @@ int ft_init_fabric(void)
 	if (ret)
 		return ret;
 
-	if (hints->caps & FI_RMA) {
-		ret = ft_set_rma_caps(fi, opts.rma_op);
-		if (ret)
-			return ret;
-	}
-
 	ret = ft_alloc_active_res(fi);
 	if (ret)
 		return ret;
@@ -852,13 +816,20 @@ int ft_setup_ep(struct fid_ep *ep, struct fid_eq *eq,
 		return ret;
 
 	/* TODO: use control structure to select counter bindings explicitly */
-	flags = !txcq ? FI_SEND : 0;
+	if (opts.options & FT_OPT_TX_CQ)
+		flags = 0;
+	else
+		flags = FI_SEND;
 	if (hints->caps & (FI_WRITE | FI_READ))
 		flags |= hints->caps & (FI_WRITE | FI_READ);
 	else if (hints->caps & FI_RMA)
 		flags |= FI_WRITE | FI_READ;
 	FT_EP_BIND(ep, txcntr, flags);
-	flags = !rxcq ? FI_RECV : 0;
+
+	if (opts.options & FT_OPT_RX_CQ)
+		flags = 0;
+	else
+		flags = FI_RECV;
 	if (hints->caps & (FI_REMOTE_WRITE | FI_REMOTE_READ))
 		flags |= hints->caps & (FI_REMOTE_WRITE | FI_REMOTE_READ);
 	else if (hints->caps & FI_RMA)
@@ -1320,7 +1291,7 @@ static int ft_inject_progress(uint64_t total)
 	struct fi_cq_err_entry comp;
 	int ret;
 
-	if (txcq) {
+	if (opts.options & FT_OPT_TX_CQ) {
 		if (opts.comp_method == FT_COMP_SREAD)
 			ret = fi_cq_sread(txcq, &comp, 1, NULL, 0);
 		else
@@ -1743,7 +1714,7 @@ int ft_get_rx_comp(uint64_t total)
 {
 	int ret = FI_SUCCESS;
 
-	if (rxcq) {
+	if (opts.options & FT_OPT_RX_CQ) {
 		ret = ft_get_cq_comp(rxcq, &rx_cq_cntr, total, timeout);
 	} else if (rxcntr) {
 		while (fi_cntr_read(rxcntr) < total) {
@@ -1764,7 +1735,7 @@ int ft_get_tx_comp(uint64_t total)
 {
 	int ret;
 
-	if (txcq) {
+	if (opts.options & FT_OPT_TX_CQ) {
 		ret = ft_get_cq_comp(txcq, &tx_cq_cntr, total, -1);
 	} else if (txcntr) {
 		ret = fi_cntr_wait(txcntr, total, -1);
@@ -2230,16 +2201,22 @@ void ft_parsecsopts(int op, char *optarg, struct ft_opts *opts)
 	}
 }
 
-int ft_parse_rma_opts(int op, char *optarg, struct ft_opts *opts)
+int ft_parse_rma_opts(int op, char *optarg, struct fi_info *hints,
+		      struct ft_opts *opts)
 {
 	switch (op) {
 	case 'o':
 		if (!strcmp(optarg, "read")) {
+			hints->caps |= FI_READ | FI_REMOTE_READ;
 			opts->rma_op = FT_RMA_READ;
 		} else if (!strcmp(optarg, "writedata")) {
+			hints->caps |= FI_WRITE | FI_REMOTE_WRITE;
+			hints->mode |= FI_RX_CQ_DATA;
+			hints->domain_attr->cq_data_size = 4;
 			opts->rma_op = FT_RMA_WRITEDATA;
 			cq_attr.format = FI_CQ_FORMAT_DATA;
 		} else if (!strcmp(optarg, "write")) {
+			hints->caps |= FI_WRITE | FI_REMOTE_WRITE;
 			opts->rma_op = FT_RMA_WRITE;
 		} else {
 			fprintf(stderr, "Invalid operation type: \"%s\". Usage:\n"

complex/ft_comp.c

@@ -114,15 +114,18 @@ int ft_open_comp(void)
 {
 	int ret;
 
+	ret = ft_open_cqs();
+	if (ret)
+		return ret;
+
 	switch (test_info.comp_type) {
 	case FT_COMP_QUEUE:
-		ret = ft_open_cqs();
 		break;
 	case FT_COMP_CNTR:
 		ret = ft_open_cntrs();
 		break;
 	default:
-		ret = -FI_ENOSYS; 
+		ret = -FI_ENOSYS;
 	}
 
 	return ret;
@@ -132,25 +135,38 @@ int ft_bind_comp(struct fid_ep *ep)
 {
 	int ret;
 	uint64_t flags;
-	struct fid *txfid = (test_info.comp_type == FT_COMP_CNTR) ? &txcntr->fid : &txcq->fid;
-	struct fid *rxfid = (test_info.comp_type == FT_COMP_CNTR) ? &rxcntr->fid : &rxcq->fid;
 
 	flags = FI_TRANSMIT;
-	if (test_info.comp_type == FT_COMP_CNTR)
-		flags |= FI_READ | FI_WRITE;
-	ret = fi_ep_bind(ep, txfid, flags);
+	ret = fi_ep_bind(ep, &txcq->fid, flags);
 	if (ret) {
 		FT_PRINTERR("fi_ep_bind", ret);
 		return ret;
 	}
 
+	if (test_info.comp_type == FT_COMP_CNTR) {
+		flags |= FI_READ | FI_WRITE;
+		ret = fi_ep_bind(ep, &txcntr->fid, flags);
+		if (ret) {
+			FT_PRINTERR("fi_ep_bind", ret);
+			return ret;
+		}
+	}
+
 	flags = FI_RECV;
-	ret = fi_ep_bind(ep, rxfid, flags);
+	ret = fi_ep_bind(ep, &rxcq->fid, flags);
 	if (ret) {
 		FT_PRINTERR("fi_ep_bind", ret);
 		return ret;
 	}
 
+	if (test_info.comp_type == FT_COMP_CNTR) {
+		ret = fi_ep_bind(ep, &rxcntr->fid, flags);
+		if (ret) {
+			FT_PRINTERR("fi_ep_bind", ret);
+			return ret;
+		}
+	}
+
 	return 0;
 }
 

complex/ft_config.c

@@ -776,6 +776,9 @@ void fts_cur_info(struct ft_series *series, struct ft_info *info)
 	info->ep_type = set->ep_type[series->cur_ep];
 	info->av_type = set->av_type[series->cur_av];
 	info->comp_type = set->comp_type[series->cur_comp];
+	if (info->caps & (FT_CAP_RMA | FT_CAP_ATOMIC) &&
+		(info->comp_type == FT_COMP_CNTR))
+		info->caps |= FI_RMA_EVENT;
 	info->eq_wait_obj = set->eq_wait_obj[series->cur_eq_wait_obj];
 	info->cntr_wait_obj = set->cntr_wait_obj[series->cur_cntr_wait_obj];
 

complex/ft_main.c

@@ -221,6 +221,13 @@ static int ft_check_info(struct fi_info *hints, struct fi_info *info)
 static void ft_fw_convert_info(struct fi_info *info, struct ft_info *test_info)
 {
 	info->caps = test_info->caps;
+
+	if ((test_info->class_function == FT_FUNC_WRITEDATA) ||
+	    (test_info->class_function == FT_FUNC_INJECT_WRITEDATA) ||
+	    (test_info->class_function == FT_FUNC_INJECTDATA) ||
+	    (test_info->class_function == FT_FUNC_SENDDATA))
+		info->domain_attr->cq_data_size = 4;
+
 	info->mode = test_info->mode;
 
 	info->domain_attr->av_type = test_info->av_type;

include/shared.h

@@ -184,7 +184,8 @@ extern struct ft_opts opts;
 void ft_parseinfo(int op, char *optarg, struct fi_info *hints);
 void ft_parse_addr_opts(int op, char *optarg, struct ft_opts *opts);
 void ft_parsecsopts(int op, char *optarg, struct ft_opts *opts);
-int ft_parse_rma_opts(int op, char *optarg, struct ft_opts *opts);
+int ft_parse_rma_opts(int op, char *optarg, struct fi_info *hints,
+		      struct ft_opts *opts);
 void ft_addr_usage();
 void ft_usage(char *name, char *desc);
 void ft_mcusage(char *name, char *desc);
@@ -305,7 +306,6 @@ size_t datatype_to_size(enum fi_datatype datatype);
 
 int ft_alloc_bufs();
 int ft_open_fabric_res();
-int ft_set_rma_caps(struct fi_info *fi, enum ft_rma_opcodes rma_op);
 int ft_getinfo(struct fi_info *hints, struct fi_info **info);
 int ft_init_fabric();
 int ft_start_server();

scripts/runfabtests.cmd

@@ -11,8 +11,9 @@ set TEST_FAIL_TIMEOUT=90
 
 set unit_tests=^
 	"av_test -g 192.168.10.1 -n 1 -s 127.0.0.1"^
-	"dom_test -n 2"^
 	"eq_test"
+:: Disabling this test since it fails on windows (appveyor). Re-enable after root cause is identified and fixed
+:: "dom_test -n 2"^
 
 set simple_tests=^
 	"cq_data"^

scripts/runfabtests.sh

@@ -103,9 +103,6 @@ simple_tests=(
 	"rdm_shared_av"
 	"multi_mr -e msg -V"
 	"multi_mr -e rdm -V"
-	"rdm_multi_domain -V"
-	"multi_ep -e msg"
-	"multi_ep -e rdm"
 	"recv_cancel -e rdm -V"
 	"unexpected_msg -e msg -i 10"
 	"unexpected_msg -e rdm -i 10"

simple/rdm_deferred_wq.c

@@ -165,7 +165,7 @@ static int check_data()
 	case FI_OP_ATOMIC:
 		for (i = 0; i < strlen(welcome_text); i++) {
 			if (rx_buf[i] != c) {
-				printf("Data mismatch found at byte %d", i);
+				printf("Data mismatch found at byte %d...", i);
 				return 1;
 			}
 		}
@@ -332,13 +332,13 @@ static int compare_atomic_trigger()
 	struct fi_ioc compare_iov;
 	struct fi_rma_ioc rma_iov;
 
-	ret = check_compare_atomic_op(ep, FI_CSWAP, FI_UINT8, &count);
+	ret = check_compare_atomic_op(ep, FI_CSWAP_LE, FI_UINT8, &count);
 	if (ret)
 		return ret;
 
 	format_simple_msg_atomic(&msg, &iov, &rma_iov,
 				 tx_buf + strlen(welcome_text),
-				 strlen(welcome_text), &work.context, FI_UINT8, FI_CSWAP_LT);
+				 strlen(welcome_text), &work.context, FI_UINT8, FI_CSWAP_LE);
 	format_simple_msg_fetch(&fetch, &fetch_iov, result_buf, strlen(welcome_text));
 	format_simple_msg_compare(&compare, &compare_iov, compare_buf, strlen(welcome_text));
 
@@ -483,7 +483,7 @@ static void init_buf_vals()
 	case FI_OP_ATOMIC:
 	case FI_OP_FETCH_ATOMIC:
 	case FI_OP_COMPARE_ATOMIC:
-		memset(compare_buf, ~0, strlen(compare_buf));
+		memset(compare_buf, ~0, strlen(welcome_text) * 2);
 		sprintf(tx_buf, "%s", welcome_text);
 		memset(&tx_buf[strlen(welcome_text)], ~0, strlen(welcome_text));
 		if (opts.dst_addr)