Merge pull request ofiwg#1254 from jithinjosepkl/pr/devel

sockets: scalability fixes/enhancements
hoopoepg · Sep 1, 2015 · 09d9c81 · 09d9c81
2 parents b4a2155 + 9caf308
commit 09d9c81
Show file tree

Hide file tree

Showing 6 changed files with 45 additions and 14 deletions.
diff --git a/prov/sockets/src/sock.h b/prov/sockets/src/sock.h
@@ -87,6 +87,7 @@
 #define SOCK_EQ_DEF_SZ (1<<8)
 #define SOCK_CQ_DEF_SZ (1<<8)
 #define SOCK_AV_DEF_SZ (1<<8)
+#define SOCK_CMAP_DEF_SZ (1<<10)
 
 #define SOCK_CQ_DATA_SIZE (sizeof(uint64_t))
 #define SOCK_TAG_SIZE (sizeof(uint64_t))
@@ -96,6 +97,7 @@
 #define SOCK_EP_MAX_RETRY (5)
 #define SOCK_EP_MAX_CM_DATA_SZ (256)
 #define SOCK_CM_DEF_BACKLOG (128)
+#define SOCK_CM_DEF_RETRY (5)
 
 #define SOCK_EP_RDM_PRI_CAP (FI_MSG | FI_RMA | FI_TAGGED | FI_ATOMICS |	\
 			 FI_NAMED_RX_CTX | \

diff --git a/prov/sockets/src/sock_av.c b/prov/sockets/src/sock_av.c
@@ -551,7 +551,7 @@ int sock_av_open(struct fid_domain *domain, struct fi_av_attr *attr,
 		return -FI_ENOMEM;
 
 	_av->attr = *attr;
-	_av->attr.count = (attr->count) ? attr->count : SOCK_AV_DEF_SZ;
+	_av->attr.count = (attr->count) ? attr->count : sock_av_def_sz;
 
 	_av->key = calloc(_av->attr.count, sizeof(uint16_t));
 	if (!_av->key) {

diff --git a/prov/sockets/src/sock_conn.c b/prov/sockets/src/sock_conn.c
@@ -214,27 +214,20 @@ int sock_conn_map_connect(struct sock_ep *ep,
 	socklen_t optlen;
 	fd_set fds;
 	struct sockaddr_in src_addr;
-	int do_retry = 5;
+	int do_retry = sock_conn_retry;
 
 	*index = 0;
+	memcpy(&src_addr, ep->src_addr, sizeof src_addr);
+
+bind_retry:
 	conn_fd = socket(AF_INET, SOCK_STREAM, 0);
 	if (conn_fd < 0) {
 		SOCK_LOG_ERROR("failed to create conn_fd, errno: %d\n", errno);
 		errno = FI_EOTHER;
 		return -errno;
 	}
 
-	memcpy(&src_addr, ep->src_addr, sizeof src_addr);
-	src_addr.sin_port = 0;
-
-	sock_set_sockopt_reuseaddr(conn_fd);
-	if (bind(conn_fd, (struct sockaddr*) &src_addr, sizeof(src_addr))) {
-		SOCK_LOG_ERROR("bind failed : %s\n", strerror(errno));
-		close(conn_fd);
-		errno = FI_EOTHER;
-		return -errno;
-	}
-
+	sock_set_sockopt_reuseaddr(conn_fd);	
 	SOCK_LOG_DBG("Connecting to: %s:%d\n", inet_ntoa(addr->sin_addr),
 		      ntohs(addr->sin_port));
 	SOCK_LOG_DBG("Connecting using address:%s\n",
@@ -267,6 +260,12 @@ int sock_conn_map_connect(struct sock_ep *ep,
 			SOCK_LOG_ERROR("Connect timed out, retrying - %s\n",
 				       strerror(errno));
 			goto retry;
+		} else if (errno == EADDRNOTAVAIL && do_retry) {
+			do_retry--;
+			SOCK_LOG_ERROR("Connect failed with address not available, retrying - %s\n",
+				       strerror(errno));
+			close(conn_fd);
+			goto bind_retry;
 		} else {
 			SOCK_LOG_ERROR("Error connecting %d - %s\n", errno,
 				       strerror(errno));

diff --git a/prov/sockets/src/sock_dom.c b/prov/sockets/src/sock_dom.c
@@ -496,7 +496,7 @@ int sock_domain(struct fid_fabric *fabric, struct fi_info *info,
 		goto err;
 	}
 
-	if (sock_conn_map_init(&sock_domain->r_cmap, 128))
+	if (sock_conn_map_init(&sock_domain->r_cmap, sock_cm_def_map_sz))
 		goto err;
 
 	sock_domain->r_cmap.domain = sock_domain;

diff --git a/prov/sockets/src/sock_fabric.c b/prov/sockets/src/sock_fabric.c
@@ -54,6 +54,11 @@ int sock_pe_waittime = SOCK_PE_WAITTIME;
 const char sock_fab_name[] = "IP";
 const char sock_dom_name[] = "sockets";
 const char sock_prov_name[] = "sockets";
+int sock_conn_retry = SOCK_CM_DEF_RETRY;
+int sock_cm_def_map_sz = SOCK_CMAP_DEF_SZ;
+int sock_av_def_sz = SOCK_AV_DEF_SZ;
+int sock_cq_def_sz = SOCK_CQ_DEF_SZ;
+int sock_eq_def_sz = SOCK_EQ_DEF_SZ;
 #if ENABLE_DEBUG
 int sock_dgram_drop_rate = 0;
 #endif
@@ -598,6 +603,26 @@ SOCKETS_INI
                         "How many milliseconds to spin while waiting for progress");
 	fi_param_get_int(&sock_prov, "pe_waittime", &sock_pe_waittime);
 
+	fi_param_define(&sock_prov, "max_conn_retry", FI_PARAM_INT,
+			"Number of connection retries before reporting as failure");
+	fi_param_get_int(&sock_prov, "max_conn_retry", &sock_conn_retry);
+
+	fi_param_define(&sock_prov, "def_conn_map_sz", FI_PARAM_INT,
+			"Default connection map size");
+	fi_param_get_int(&sock_prov, "def_conn_map_sz", &sock_cm_def_map_sz);
+
+	fi_param_define(&sock_prov, "def_av_sz", FI_PARAM_INT,
+			"Default address vector size");
+	fi_param_get_int(&sock_prov, "def_av_sz", &sock_av_def_sz);
+
+	fi_param_define(&sock_prov, "def_cq_sz", FI_PARAM_INT,
+			"Default completion queue size");
+	fi_param_get_int(&sock_prov, "def_cq_sz", &sock_cq_def_sz);
+
+	fi_param_define(&sock_prov, "def_eq_sz", FI_PARAM_INT,
+			"Default event queue size");
+	fi_param_get_int(&sock_prov, "def_eq_sz", &sock_eq_def_sz);
+
 	fastlock_init(&sock_list_lock);
 	dlist_init(&sock_fab_list);
 	dlist_init(&sock_dom_list);

diff --git a/prov/sockets/src/sock_util.h b/prov/sockets/src/sock_util.h
@@ -42,6 +42,11 @@ extern const char sock_dom_name[];
 extern const char sock_prov_name[];
 extern struct fi_provider sock_prov;
 extern int sock_pe_waittime;
+extern int sock_conn_retry;
+extern int sock_cm_def_map_sz;
+extern int sock_av_def_sz;
+extern int sock_cq_def_sz;
+extern int sock_eq_def_sz;
 #if ENABLE_DEBUG
 extern int sock_dgram_drop_rate;
 #endif