From 7884e4879d75f59c1299b5d8fbaebbb9f482d1e2 Mon Sep 17 00:00:00 2001 From: Julian Benda Date: Wed, 4 Aug 2021 10:05:49 +0200 Subject: [PATCH 01/60] Add network port to jconfiguration --- include/core/jconfiguration.h | 3 +++ lib/core/jconfiguration.c | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/include/core/jconfiguration.h b/include/core/jconfiguration.h index e20dc4e70..b7cb2ade3 100644 --- a/include/core/jconfiguration.h +++ b/include/core/jconfiguration.h @@ -113,6 +113,9 @@ guint16 j_configuration_get_port(JConfiguration*); guint32 j_configuration_get_max_connections(JConfiguration*); guint64 j_configuration_get_stripe_size(JConfiguration*); +/// network port to communicate +guint16 j_configuration_get_port(JConfiguration*); + G_END_DECLS /** diff --git a/lib/core/jconfiguration.c b/lib/core/jconfiguration.c index 9d22c7ffd..094501d99 100644 --- a/lib/core/jconfiguration.c +++ b/lib/core/jconfiguration.c @@ -319,7 +319,8 @@ j_configuration_new_for_data(GKeyFile* key_file) || kv_path == NULL || db_backend == NULL || db_component == NULL - || db_path == NULL) + || db_path == NULL + || port >= 0xFFFF) { g_free(db_backend); g_free(db_component); From faced313e791d37ae78c6a183289bd5339e6a62e Mon Sep 17 00:00:00 2001 From: Julian Benda Date: Wed, 4 Aug 2021 10:19:58 +0200 Subject: [PATCH 02/60] Replaced magic port numbers with configuration --- server/server.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/server/server.c b/server/server.c index a2627ec77..fea94cc8e 100644 --- a/server/server.c +++ b/server/server.c @@ -279,6 +279,16 @@ main(int argc, char** argv) socket_service = g_threaded_socket_service_new(-1); g_socket_listener_set_backlog(G_SOCKET_LISTENER(socket_service), 128); + jd_configuration = j_configuration_new(); + if(!jd_configuration) { + g_warning("Failed to load configuration!"); + return 1; + } + + if(opt_port == 0) { + opt_port = j_configuration_get_port(jd_configuration); + } + while (TRUE) { if (!g_socket_listener_add_inet_port(G_SOCKET_LISTENER(socket_service), opt_port, NULL, &error)) From 0d120754465ddbdb771cf4bb0eb033ed86f30112 Mon Sep 17 00:00:00 2001 From: Julian Benda Date: Wed, 4 Aug 2021 11:02:03 +0200 Subject: [PATCH 03/60] Add port configuration to config-tool also adapt tests --- test/core/configuration.c | 2 ++ test/core/distribution.c | 1 + tools/config.c | 9 +++++++++ 3 files changed, 12 insertions(+) diff --git a/test/core/configuration.c b/test/core/configuration.c index 20799e2a2..6be934fd1 100644 --- a/test/core/configuration.c +++ b/test/core/configuration.c @@ -49,6 +49,7 @@ test_configuration_new_for_data(void) J_TEST_TRAP_START; key_file = g_key_file_new(); + g_key_file_set_uint64(key_file, "core", "port", 4700); g_key_file_set_string_list(key_file, "servers", "object", servers, 1); g_key_file_set_string_list(key_file, "servers", "kv", servers, 1); g_key_file_set_string_list(key_file, "servers", "db", servers, 1); @@ -81,6 +82,7 @@ test_configuration_get(void) J_TEST_TRAP_START; key_file = g_key_file_new(); + g_key_file_set_uint64(key_file, "core", "port", 4700); g_key_file_set_string_list(key_file, "servers", "object", object_servers, 2); g_key_file_set_string_list(key_file, "servers", "kv", kv_servers, 1); g_key_file_set_string_list(key_file, "servers", "db", db_servers, 2); diff --git a/test/core/distribution.c b/test/core/distribution.c index 79a1ebd0c..6b990e6f3 100644 --- a/test/core/distribution.c +++ b/test/core/distribution.c @@ -33,6 +33,7 @@ test_distribution_fixture_setup(JConfiguration** configuration, gconstpointer da (void)data; key_file = g_key_file_new(); + g_key_file_set_uint64(key_file, "core", "port", 4700); g_key_file_set_string_list(key_file, "servers", "object", servers, 2); g_key_file_set_string_list(key_file, "servers", "kv", servers, 2); g_key_file_set_string_list(key_file, "servers", "db", servers, 2); diff --git a/tools/config.c b/tools/config.c index 02909a2fb..e4b30589c 100644 --- a/tools/config.c +++ b/tools/config.c @@ -45,6 +45,8 @@ static gint64 opt_max_operation_size = 0; static gint opt_port = 0; static gint opt_max_connections = 0; static gint64 opt_stripe_size = 0; +static gint64 opt_network_port = 0; +static gint64 opt_user_id = 0; static gchar** string_split(gchar const* string) @@ -154,6 +156,8 @@ main(gint argc, gchar** argv) { "system", 0, 0, G_OPTION_ARG_NONE, &opt_system, "Write system configuration", NULL }, { "read", 0, 0, G_OPTION_ARG_NONE, &opt_read, "Read configuration", NULL }, { "name", 0, 0, G_OPTION_ARG_STRING, &opt_name, "Configuration name", "julea" }, + { "port", 0, 0, G_OPTION_ARG_INT64, &opt_network_port, "Network communication port", "4000 + user_id%100"}, + { "user-id", 0, 0, G_OPTION_ARG_INT64, &opt_user_id, "User Id used to determined communication port", NULL}, { "object-servers", 0, 0, G_OPTION_ARG_STRING, &opt_servers_object, "Object servers to use", "host1,host2:port" }, { "kv-servers", 0, 0, G_OPTION_ARG_STRING, &opt_servers_kv, "Key-value servers to use", "host1,host2:port" }, { "db-servers", 0, 0, G_OPTION_ARG_STRING, &opt_servers_db, "Database servers to use", "host1,host2:port" }, @@ -208,6 +212,11 @@ main(gint argc, gchar** argv) return 1; } + if(opt_network_port == 0) + { + opt_network_port = 4000 + (opt_user_id % 1000); + } + if (opt_user) { path = g_build_filename(g_get_user_config_dir(), "julea", opt_name, NULL); From 0527b66c6755764c7ac2264a3ab1895b411e8182 Mon Sep 17 00:00:00 2001 From: Julian Benda Date: Wed, 4 Aug 2021 11:13:25 +0200 Subject: [PATCH 04/60] Adapt documentation for new feature --- README.md | 1 + tools/config.c | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 24a5574a1..cf895f6c5 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,7 @@ $ julea-config --user \ --object-backend=posix --object-component=server --object-path="/tmp/julea-$(id -u)/posix" \ --kv-backend=lmdb --kv-component=server --kv-path="/tmp/julea-$(id -u)/lmdb" \ --db-backend=sqlite --db-component=server --db-path="/tmp/julea-$(id -u)/sqlite" + --user-id="$(id --user)" ``` You can check whether JULEA works by executing the integrated test suite. diff --git a/tools/config.c b/tools/config.c index e4b30589c..77a08b47c 100644 --- a/tools/config.c +++ b/tools/config.c @@ -156,7 +156,7 @@ main(gint argc, gchar** argv) { "system", 0, 0, G_OPTION_ARG_NONE, &opt_system, "Write system configuration", NULL }, { "read", 0, 0, G_OPTION_ARG_NONE, &opt_read, "Read configuration", NULL }, { "name", 0, 0, G_OPTION_ARG_STRING, &opt_name, "Configuration name", "julea" }, - { "port", 0, 0, G_OPTION_ARG_INT64, &opt_network_port, "Network communication port", "4000 + user_id%100"}, + { "port", 0, 0, G_OPTION_ARG_INT64, &opt_network_port, "Network communication port", "4000 + user-id%100"}, { "user-id", 0, 0, G_OPTION_ARG_INT64, &opt_user_id, "User Id used to determined communication port", NULL}, { "object-servers", 0, 0, G_OPTION_ARG_STRING, &opt_servers_object, "Object servers to use", "host1,host2:port" }, { "kv-servers", 0, 0, G_OPTION_ARG_STRING, &opt_servers_kv, "Key-value servers to use", "host1,host2:port" }, From 2516f77160213c69e5f19cbce05ba05e8c9fd7f3 Mon Sep 17 00:00:00 2001 From: Julian Benda Date: Tue, 29 Jun 2021 14:56:41 +0200 Subject: [PATCH 05/60] Add libfabirc example --- example/libfabric_basic.c | 989 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 989 insertions(+) create mode 100644 example/libfabric_basic.c diff --git a/example/libfabric_basic.c b/example/libfabric_basic.c new file mode 100644 index 000000000..f4a3b66ba --- /dev/null +++ b/example/libfabric_basic.c @@ -0,0 +1,989 @@ +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include + +// TODO: only no wait calls for cq? + +#define EXE(a) do{if(!a) { goto end; }} while(FALSE) +#define CHECK(msg) do{if(res < 0) { g_error("%s: "msg"\nDetails:\t%s", location, fi_strerror(-res)); goto end; }} while(FALSE) +// TODO add to config! +#define PORT 47592 +#define MSG_PARTS 2 +#define MAX_SEGMENTS 5 +#define ACK 42 + +const char* const usage = "call with: (server|client )"; + +enum ConnectionType {CLIENT, SERVER}; +enum ConnectionDirection {TX, RX}; +enum Event {ERROR = 0, CONNECTION_REQUEST, CONNECTED, SHUTDOWN}; + +struct addrinfo; + +struct message { size_t len; struct { uint64_t buff_addr; size_t buff_size; uint64_t key; } *data; }; + +struct jfabric { + struct fi_info* info; + struct fid_fabric* fabric; + struct fid_eq* pep_eq; + struct fid_pep* pep; + const char* location; + enum ConnectionType con_type; + struct Config* config; +}; +struct EndpointMemory { + struct fid_mr* mr; + void* buffer; + size_t buffer_size; + size_t rx_prefix_size; + size_t tx_prefix_size; +}; + +struct jendpoint { + struct jfabric* fabric; + struct fi_info* info; + struct fid_domain* domain; + struct fid_ep* ep; + struct fid_eq* eq; + struct EndpointMemory memory; + struct fid_cq* txcq; + struct fid_cq* rxcq; + int inject_size; +}; + +struct MyThreadData { + struct jfabric* fabric; + struct fi_eq_cm_entry* con_req; +}; + +struct Config { struct fi_info* hints; int version; size_t max_op_size; }; + +struct Config* config_init(void); +void config_fini(struct Config*); +int config_get_version(struct Config*); +struct fi_info* config_get_hints(struct Config* conf); +void config_set_max_op_size(struct Config* conf, size_t size); +size_t config_get_max_op_size(struct Config* conf); + +int config_get_version(struct Config* conf) { return conf->version; } +struct fi_info* config_get_hints(struct Config* conf) { return conf->hints; } +void config_set_max_op_size(struct Config* conf, size_t size) { conf->max_op_size = size; } +size_t config_get_max_op_size(struct Config* conf) { return conf->max_op_size; } +struct Config* config_init(void) +{ + struct Config* res = malloc(sizeof(struct Config)); + memset(res, 0, sizeof(struct Config)); + res->version = FI_VERSION(1, 11); + res->hints = fi_allocinfo(); + res->hints->caps = FI_MSG | FI_SEND | FI_RECV | FI_READ | FI_RMA | FI_REMOTE_READ; + res->hints->mode = FI_MSG_PREFIX; + res->hints->domain_attr->mr_mode = FI_MR_LOCAL | FI_MR_ALLOCATED | FI_MR_PROV_KEY | FI_MR_VIRT_ADDR; + res->hints->ep_attr->type = FI_EP_MSG; + res->hints->fabric_attr->prov_name = g_strdup("verbs"); + res->max_op_size = 64; + return res; +} +void config_fini(struct Config* config) +{ + fi_freeinfo(config->hints); + free(config); +} + + +gboolean fabric_init_server(struct Config* config, struct jfabric** fabric); +gboolean fabric_init_client(struct Config* config, struct fi_info* hints, struct jfabric** fabirc); +gboolean fabric_fini(struct jfabric* fabric); +gboolean read_pep_eq(struct jfabric* fabric, enum Event* evt, struct fi_eq_cm_entry* entry); + +gboolean fabric_init_client(struct Config* config, struct fi_info* hints, struct jfabric** fabric_ptr) { + int res; + struct jfabric* fabric; + const char* location; + + *fabric_ptr = malloc(sizeof(struct jfabric)); + fabric = *fabric_ptr; + memset(fabric, 0, sizeof(struct jfabric)); + + fabric->config = config; + fabric->location = "Client"; + fabric->con_type = CLIENT; + + location = fabric->location; + + res = fi_getinfo( + config_get_version(config), + NULL, NULL, 0, + hints, + &fabric->info); + CHECK("failed to find fabric!"); + + fi_freeinfo(fabric->info->next); + fabric->info->next = NULL; + + res = fi_fabric(fabric->info->fabric_attr, &fabric->fabric, NULL); + return TRUE; +end: + *fabric_ptr = NULL; + return FALSE; +} +gboolean fabric_init_server(struct Config* config, struct jfabric** fabric_ptr) +{ + int res; + struct jfabric* fabric; + const char* location; + + *fabric_ptr = malloc(sizeof(struct jfabric)); // TODO: alloc macro? + fabric = *fabric_ptr; + memset(fabric, 0, sizeof(struct jfabric)); + + fabric->config = config; + fabric->location = "Server"; + fabric->con_type = SERVER; + + location = fabric->location; + + res = fi_getinfo( + config_get_version(config), + NULL, NULL, FI_SOURCE, // node and service specification (we don't care?); TODO: maybe can replace socket? + config_get_hints(config), + &fabric->info + ); + CHECK("Failed to get fabric info!"); + + // throw other matches away, we only need one + fi_freeinfo(fabric->info->next); + fabric->info->next = NULL; + + res = fi_fabric(fabric->info->fabric_attr, &fabric->fabric, NULL); // no context needed + CHECK("failed to open fabric!"); + + res = fi_eq_open( + fabric->fabric, + &(struct fi_eq_attr){.wait_obj = FI_WAIT_UNSPEC}, + &fabric->pep_eq, NULL); + CHECK("failed to create eq for fabric!"); + res = fi_passive_ep(fabric->fabric, fabric->info, &fabric->pep, NULL); // no context needed + CHECK("failed to create pep"); + res = fi_pep_bind(fabric->pep, &fabric->pep_eq->fid, 0); // no special flags needed + CHECK("failed to bind eq to pep"); + res = fi_listen(fabric->pep); + CHECK("failed to set pep to listening!"); + + return TRUE; +end: + fabric_fini(fabric); + *fabric_ptr = NULL; + return FALSE; +} +gboolean fabric_fini(struct jfabric* fabric) +{ + int res; + const char* location = fabric->location; + + g_message("close: %s", location); + + fi_freeinfo(fabric->info); + fabric->info = NULL; + if(fabric->con_type == SERVER) { + res = fi_close(&fabric->pep->fid); + fabric->pep = NULL; + CHECK("failed to close pep!"); + res = fi_close(&fabric->pep_eq->fid); + CHECK("failed to close pep_eq!"); + fabric->pep_eq = NULL; + } + res = fi_close(&fabric->fabric->fid); + CHECK("failed to close fabric!"); + fabric->fabric = NULL; + free(fabric); + return TRUE; +end: + return FALSE; +} +gboolean read_pep_eq(struct jfabric* fabric, enum Event* event, struct fi_eq_cm_entry* entry) +{ + int res; + const char* location; + uint32_t fi_event; + + location = fabric->location; + + do{ + res = fi_eq_sread(fabric->pep_eq, &fi_event, entry, sizeof(struct fi_eq_cm_entry), -1, 0); // no timeout, no special flags + } while(res == -FI_EAGAIN); + CHECK("failed to read pep event queue!"); + + switch(fi_event) { + case FI_CONNREQ: *event = CONNECTION_REQUEST; break; + case FI_CONNECTED: g_error("should not connect!"); goto end; break; + case FI_SHUTDOWN: *event = SHUTDOWN; break; + default: g_assert_not_reached(); goto end; + } + + return TRUE; +end: + *event = ERROR; + memset(entry, 0, sizeof(struct fi_eq_cm_entry)); + return FALSE; +} + +gboolean open_socket(int* fd, uint32_t port); +gboolean socket_wait_for_connection(int fd, int* active_socket); +gboolean socket_write_addr(int fd, struct jfabric* fabric); +gboolean socket_request_addr(char* addr, uint32_t port, struct fi_info* hints); +gboolean parse_address(char* addr, uint32_t port, struct addrinfo** infos); + +gboolean open_socket(int* res, uint32_t port) { + int fd; + int error; + struct sockaddr_in ctrl_addr = {0}; + int one = 1; + + fd = socket(AF_INET, SOCK_STREAM, 0); // orderd ipv4 socket + if(fd == -1) { g_error("failed to open listening socket! error: %s", strerror(errno)); goto end; } + error = setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, (const char*)&one, sizeof(one)); + if(error == -1) { g_error("failed to set socket option! error: %s", strerror(errno)); goto end; } + + ctrl_addr.sin_family = AF_INET; + ctrl_addr.sin_port = htons(port); + ctrl_addr.sin_addr.s_addr = htonl(INADDR_ANY); + error = bind(fd, (struct sockaddr*)&ctrl_addr, sizeof(ctrl_addr)); + if(error == -1) { g_error("filed to set address for socket (binding)! error: %s", strerror(errno)); goto end; } + + *res = fd; + return TRUE; +end: + *res = -1; + return FALSE; +} + +gboolean socket_wait_for_connection(int fd, int* active_socket) +{ + int error; + int a_fd; + + g_message("wait for connection!"); + + error = listen(fd, 0); // lagecy argument: length is now defined in system settings, this argument is ignored + if(error == -1) { g_error("failed to listen for connections! error: %s", strerror(errno)); goto end; } + g_message("listen finished"); + a_fd = accept(fd, NULL, 0); // we accept every thing what is comming! + if(a_fd == -1) { g_error("failed to accept connection request! error: %s", strerror(errno)); goto end; } + + *active_socket = a_fd; + return TRUE; +end: + *active_socket = 0; + return FALSE; +} +gboolean socket_write_addr(int fd, struct jfabric* fabric) +{ + int res; + size_t addrlen = 0; + const char* location; + char* addr; + uint32_t len; + + location = fabric->location; + + res = fi_getname(&fabric->pep->fid, NULL, &addrlen); + if((res != -FI_ETOOSMALL) || addrlen <= 0) { CHECK("failed to fetch address size!"); } + addr = malloc(addrlen); + res = fi_getname(&fabric->pep->fid, addr, &addrlen); + CHECK("failed to get addres!"); + + // DEBUG: + { + char str[16*3 + 1]; + int i; + for(i = 0; i < 16; ++i) { snprintf(str+i*3, 4, "%02x ", ((uint8_t*)addr)[i]); } + g_message("SendAddr: len: %lu, data: %s", addrlen, str); + } + + len = htonl(addrlen); + res = send(fd, (char*)&len, sizeof(len), 0); + if(res == -1 || (size_t)res != sizeof(len)) { g_error("failed to send addrlen!(%i)", res); goto end; } + res = send(fd, &fabric->info->addr_format, sizeof(fabric->info->addr_format), 0); + if(res == -1 || (size_t)res != sizeof(fabric->info->addr_format)) { g_error("failed to send addr format!(%i)", res); goto end; } + res = send(fd, addr, addrlen, 0); + if(res == -1 || (size_t)res != addrlen) { g_error("failed to send addr!(%i)", res); goto end; } + + return TRUE; +end: + return FALSE; +} +gboolean parse_address(char* addr, uint32_t port, struct addrinfo** infos) +{ + const char* err_msg; + char port_s[6]; + int res; + + struct addrinfo hints = { + .ai_family = AF_INET, + .ai_socktype = SOCK_STREAM, + .ai_protocol = IPPROTO_TCP, + .ai_flags = AI_NUMERICSERV + }; + snprintf(port_s, 6, "%"PRIu16, port); + res = getaddrinfo(addr, port_s, &hints, infos); + if(res != 0) { + err_msg = gai_strerror(res); + g_error("failed to get socket addr: error: %s", err_msg); + goto end; + } else if (*infos == NULL) { + g_error("unable to find matching socket!"); + goto end; + } + return TRUE; +end: + return FALSE; +} +gboolean socket_request_addr(char* s_addr, uint32_t s_port, struct fi_info* hints) +{ + int res; + struct addrinfo* itr; + int fd; + uint32_t addr_len; + uint32_t addr_format; + void* addr = NULL; + struct addrinfo* addresses; + + EXE(parse_address(s_addr, s_port, &addresses)); + + for(itr = addresses; itr; itr = itr->ai_next) { + fd = socket(itr->ai_family, itr->ai_socktype, itr->ai_protocol); + if(fd == -1) { g_error("failed to open connection: error: %s", strerror(errno)); continue; } + res = connect(fd, itr->ai_addr, itr->ai_addrlen); + if(res == -1) { + g_error("filed to connect to host: error: %s", strerror(errno)); + // close(fd); + continue; + } + break; + } + if(!itr) { g_error("Unable to connect to any host!"); goto end; } + freeaddrinfo(addresses); + + res = recv(fd, (void*)&addr_len, sizeof(addr_len), 0); + if(res == -1 || res != sizeof(addr_len)) { g_error("failed to recive address len!"); goto end; } + addr_len = ntohl(addr_len); + res = recv(fd, (void*)&addr_format, sizeof(addr_format), 0); + if(res == -1 || res != sizeof(addr_format)) { g_error("failed to recive address format!"); goto end; } + addr = malloc(addr_len); + res = recv(fd, addr, addr_len, 0); + if(res == -1 || (uint32_t)res != addr_len) { g_error("failed to recive address!"); goto end; } + + hints->dest_addr = addr; + hints->dest_addrlen = addr_len; + hints->addr_format = addr_format; + // DEBUG: + { + char str[16*3 + 1]; + int i; + for(i = 0; i < 16; ++i) { snprintf(str+i*3, 4, "%02x ", ((uint8_t*)addr)[i]); } + g_message("RecvAddr: len: %i, data: %s", addr_len, str); + } + return TRUE; +end: + return FALSE; +} + + +gboolean endpoint_init_client(struct jendpoint** endpoint, struct Config* config, char* addr, uint32_t port); +gboolean endpoint_init_server(struct jendpoint** endpoint, struct jfabric*, struct fi_eq_cm_entry* con_req); +gboolean endpoint_fini(struct jendpoint* ednpoint); +// alloc memory, bind and create fi_ep +gboolean endpoint_create(struct jendpoint* endpoint); +gboolean _endpoint_create_mr(struct jendpoint* endpoint); +gboolean endpoint_check_connection(struct jendpoint* endpoint); +gboolean endpoint_read_eq(struct jendpoint* endpoint, enum Event* evt, struct fi_eq_cm_entry* entry); +gboolean endpoint_wait_for_completion(struct jendpoint* endpoint, enum ConnectionDirection con_dir, int len, gboolean* check, void** contexts); + +gboolean endpoint_fini(struct jendpoint* endpoint) +{ + int res; + const char* location = endpoint->fabric->location; + + res = fi_close(&endpoint->ep->fid); + CHECK("failed to close ep!"); + res = fi_close(&endpoint->txcq->fid); + CHECK("failed to close txcq!"); + res = fi_close(&endpoint->rxcq->fid); + CHECK("failed to close rxqc!"); + res = fi_close(&endpoint->eq->fid); + CHECK("failed to close eq!"); + if(endpoint->memory.mr) { + res = fi_close(&endpoint->memory.mr->fid); + CHECK("failed to close memory of endpoint!"); + free(endpoint->memory.buffer); + } + res = fi_close(&endpoint->domain->fid); + CHECK("failed to close domain!"); + if(endpoint->fabric->con_type == CLIENT) { + EXE(fabric_fini(endpoint->fabric)); + } else { + fi_freeinfo(endpoint->info); + } + free(endpoint); + return TRUE; +end: + return FALSE; +} +gboolean _endpoint_create_mr(struct jendpoint* endpoint) +{ + + int res; + gboolean tx_prefix; + gboolean rx_prefix; + const char* location; + size_t prefix_size; + size_t size; + + location = endpoint->fabric->location; + size = config_get_max_op_size(endpoint->fabric->config); + tx_prefix = (endpoint->fabric->info->tx_attr->mode & FI_MSG_PREFIX) != 0; + rx_prefix = (endpoint->fabric->info->rx_attr->mode & FI_MSG_PREFIX) != 0; + prefix_size = endpoint->fabric->info->ep_attr->msg_prefix_size; + g_message("%s: prefix_size: %lu", location, prefix_size); + + if(size + (tx_prefix | rx_prefix) * prefix_size > endpoint->fabric->info->ep_attr->max_msg_size) { + // TODO: message size != operation size! + size = endpoint->fabric->info->ep_attr->max_msg_size - (tx_prefix | rx_prefix) * prefix_size; + config_set_max_op_size(endpoint->fabric->config, size); + } + + if(endpoint->fabric->info->domain_attr->mr_mode & FI_MR_LOCAL) { + g_message("%s: local memory!", location); + size += (rx_prefix | tx_prefix) * prefix_size * MSG_PARTS; + endpoint->memory.buffer_size = size; + endpoint->memory.buffer = malloc(size); + endpoint->memory.tx_prefix_size = tx_prefix * prefix_size; + endpoint->memory.rx_prefix_size = rx_prefix * prefix_size; + res = fi_mr_reg(endpoint->domain, endpoint->memory.buffer, endpoint->memory.buffer_size, FI_SEND | FI_RECV, 0, 0, 0, &endpoint->memory.mr, NULL); + CHECK("failed to register memory for msg communication"); + } + return TRUE; +end: + if(endpoint->memory.buffer) { free(endpoint->memory.buffer); } + return FALSE; +} + +gboolean endpoint_create(struct jendpoint* endpoint) +{ + int res; + const char* location; + + location = endpoint->fabric->location; + + res = fi_eq_open( + endpoint->fabric->fabric, + &(struct fi_eq_attr){.wait_obj = FI_WAIT_UNSPEC}, + &endpoint->eq, NULL); + CHECK("failed to create eq!"); + res = fi_domain(endpoint->fabric->fabric, endpoint->info, &endpoint->domain, NULL); + CHECK("failed to create domain!"); + + endpoint->inject_size = endpoint->fabric->info->tx_attr->inject_size; + + EXE(_endpoint_create_mr(endpoint)); + + res = fi_cq_open(endpoint->domain, &(struct fi_cq_attr){ + .wait_obj = FI_WAIT_UNSPEC, + .format = FI_CQ_FORMAT_CONTEXT, + .size = endpoint->info->tx_attr->size + }, + &endpoint->txcq, &endpoint->rxcq); + CHECK("failed to create txcq!"); + res = fi_cq_open(endpoint->domain, &(struct fi_cq_attr){ + .wait_obj = FI_WAIT_UNSPEC, + .format = FI_CQ_FORMAT_CONTEXT, + .size = endpoint->info->rx_attr->size + }, + &endpoint->rxcq, &endpoint->rxcq); + CHECK("failed to create rxcq!"); + res = fi_endpoint(endpoint->domain, endpoint->info, &endpoint->ep, NULL); + CHECK("failed to create endpoint!"); + + res = fi_ep_bind(endpoint->ep, &endpoint->eq->fid, 0); + CHECK("failed to bind eq to ep"); + res = fi_ep_bind(endpoint->ep, &endpoint->txcq->fid, FI_TRANSMIT); + CHECK("failed to bind txcq to ep"); + res = fi_ep_bind(endpoint->ep, &endpoint->rxcq->fid, FI_RECV); + CHECK("failed to bind rxcq to ep"); + res = fi_enable(endpoint->ep); + CHECK("failed to enable ep"); + + return TRUE; +end: + return FALSE; +} +gboolean endpoint_init_client(struct jendpoint** endpoint_ptr, struct Config* config, char* addr, uint32_t port) +{ + struct fi_info* hints; + struct jendpoint* endpoint = NULL; + int res; + const char* location; + enum Event event; + struct fi_eq_cm_entry cm_entry; + + hints = fi_dupinfo(config_get_hints(config)); + EXE(socket_request_addr(addr, port, hints)); + + *endpoint_ptr = malloc(sizeof(struct jendpoint)); + endpoint = *endpoint_ptr; + memset(endpoint, 0, sizeof(struct jendpoint)); + EXE(fabric_init_client(config, hints, &endpoint->fabric)); + + location = endpoint->fabric->location; + endpoint->info = endpoint->fabric->info; + + EXE(endpoint_create(endpoint)); + + res = fi_connect(endpoint->ep, hints->dest_addr, NULL, 0); // TODO: con data id exchange? + CHECK("failed to send connection request!"); + + EXE(endpoint_read_eq(endpoint, &event, &cm_entry)); + if(event != CONNECTED || cm_entry.fid != &endpoint->ep->fid) { + g_error("failed to connect!"); goto end; + } + + fi_freeinfo(hints); + return TRUE; +end: + fi_freeinfo(hints); + if(endpoint) { + free(endpoint); + *endpoint_ptr = NULL; + } + return FALSE; +} +gboolean endpoint_init_server(struct jendpoint** endpoint_ptr, struct jfabric* fabric, struct fi_eq_cm_entry* con_req) +{ + int res; + const char* location; + struct jendpoint* endpoint; + enum Event event; + struct fi_eq_cm_entry con_ack; + + location = fabric->location; + *endpoint_ptr = malloc(sizeof(struct jendpoint)); + endpoint = *endpoint_ptr; + memset(endpoint, 0, sizeof(struct jendpoint)); + + endpoint->fabric = fabric; + endpoint->info = con_req->info; + + EXE(endpoint_create(endpoint)); + + res = fi_accept(endpoint->ep, NULL, 0); // TODO: con data exchange + CHECK("failed to accept connection"); + EXE(endpoint_read_eq(endpoint, &event, &con_ack)); + if(event != CONNECTED || con_ack.fid != &endpoint->ep->fid) { + g_error("failed to ack connection!"); goto end; + } + + return TRUE; +end: + free(endpoint); + *endpoint_ptr = NULL; + res = fi_reject(fabric->pep, con_req->fid, NULL, 0); + if(res < 0) { g_error("%s: failed to reject connection:\n\tDetails: %s", location, fi_strerror(-res)); } + return FALSE; +} +gboolean endpoint_read_eq(struct jendpoint* endpoint, enum Event* evt, struct fi_eq_cm_entry* entry) +{ + int res; + const char* location; + uint32_t fi_event; + + location = endpoint->fabric->location; + + do{ + res = fi_eq_sread(endpoint->eq, &fi_event, entry, sizeof(struct fi_eq_cm_entry), -1, 0); + }while(res == -FI_EAGAIN); + CHECK("failed to read endpoint eq!"); + + switch(fi_event) { + case FI_CONNREQ: g_error("endpoint can't get connection request!"); goto end; + case FI_CONNECTED: *evt = CONNECTED; break; + case FI_SHUTDOWN: *evt = SHUTDOWN; break; + default: g_assert_not_reached(); goto end; + } + + return TRUE; +end: + *evt = ERROR; + memset(entry, 0, sizeof(struct fi_eq_cm_entry)); + return FALSE; +} + +gboolean endpoint_check_connection(struct jendpoint* endpoint) +{ + return endpoint->ep != NULL; +} + +gboolean endpoint_wait_for_completion(struct jendpoint* endpoint, enum ConnectionDirection con_dir, int len, gboolean* check, void** contexts) +{ + int i, count = 0; + struct fid_cq* cq; + struct fi_cq_err_entry err_entry; + struct fi_cq_entry entry; + int res; + const char* location = endpoint->fabric->location; + + switch(con_dir) { + case TX: cq = endpoint->txcq; break; + case RX: cq = endpoint->rxcq; break; + default: g_assert_not_reached(); goto end; + } + + for(i = 0; i < len; ++i) { + if(contexts[i] == NULL) { check[i] = TRUE; } + if(check[i] == TRUE) {++count; } + } + while(count != len) { + res = fi_cq_sread(cq, &entry, 1, NULL, -1); + if(res == -FI_EAGAIN) { continue; } + if(res < 0) { + int err = res; + res = fi_cq_readerr(cq, &err_entry, 0); + CHECK("failed to fetch cq error!"); + g_error("%s: completion error!\n\tDetails:%s\n\tProvider:%s", + location, + fi_strerror(-err), + fi_cq_strerror(cq, err_entry.prov_errno, err_entry.err_data, NULL, 0)); + goto end; + } + for(i = 0; i < len; ++i) { + if(!check[i]) { + if(contexts[i] == entry.op_context) { + ++count; + check[i] = TRUE; + } + } + } + } + return TRUE; +end: + return FALSE; +} + +gboolean ssend_message(struct jendpoint* endpoint, struct message* msg); +gboolean srecv_message(struct jendpoint* endpoint, struct message** msg); + +gboolean ssend_message(struct jendpoint* endpoint, struct message* msg) +{ + int res; + const char* location = endpoint->fabric->location; + gboolean checks[2] = {FALSE, FALSE}; + void* contexts[2] = {NULL, NULL}; + int size = 0; + int offset = 0; + int i; + struct fid_mr* mrs[MAX_SEGMENTS]; + int ack = ~ACK; + + for(i = 0; (size_t)i < msg->len; ++i) { + res = fi_mr_reg( + endpoint->domain, + (void*)msg->data[i].buff_addr, + msg->data[i].buff_size, FI_REMOTE_READ, + 0, 0, 0, &mrs[i], NULL); + CHECK("failed to register memory for rma send!"); + msg->data[i].key = fi_mr_key(mrs[i]); + } + + // don't need dst_addr because we are connected! + size = sizeof(msg->len); + if(size < endpoint->inject_size) { + res = fi_inject(endpoint->ep, &msg->len, size, 0); + CHECK("failed to inject msg header!"); + } else { + memcpy( + (char*)endpoint->memory.buffer + offset + endpoint->memory.tx_prefix_size, + &msg->len, size); + res = fi_send(endpoint->ep, + (char*)endpoint->memory.buffer + offset, + size + endpoint->memory.tx_prefix_size, + fi_mr_desc(endpoint->memory.mr), 0, &msg->len); + CHECK("failed to initeiate header send!"); + contexts[0] = &msg->len; + offset += size + endpoint->memory.tx_prefix_size; + } + + size = msg->len * sizeof(*msg->data); + if(size < endpoint->inject_size) { + res = fi_inject(endpoint->ep, msg->data, sizeof(*msg->data) * msg->len, 0); + CHECK("failed to inject msg body!"); + } else { + memcpy( + (char*)endpoint->memory.buffer + offset + endpoint->memory.tx_prefix_size, + msg->data, size); + res = fi_send(endpoint->ep, + (char*)endpoint->memory.buffer + offset, + size + endpoint->memory.tx_prefix_size, + fi_mr_desc(endpoint->memory.mr), 0, msg->data); + CHECK("failed to initeiate body send!"); + contexts[1] = msg->data; + offset += size + endpoint->memory.tx_prefix_size; + } + + + res = fi_recv(endpoint->ep, endpoint->memory.buffer, sizeof(int) + endpoint->memory.rx_prefix_size, + fi_mr_desc(endpoint->memory.mr), 0, &ack); + CHECK("failed to read ack flag!"); + checks[0] = FALSE; contexts[0] = &ack; + EXE(endpoint_wait_for_completion(endpoint, RX, 1, checks, contexts)); + ack = *(int*)((char*)endpoint->memory.buffer + endpoint->memory.rx_prefix_size); + if(ack != ACK) { g_error("recived wrong ack!: %i, expected: %i", ack, ACK); goto end; } + + for(i = 0; (size_t)i< msg->len; ++i) { + res = fi_close(&mrs[i]->fid); + CHECK("failed to free memory after rma send!"); + } + + EXE(endpoint_wait_for_completion(endpoint, TX, 2, checks, contexts)); + + return TRUE; +end: + return FALSE; +} + +gboolean srecv_message(struct jendpoint* endpoint, struct message** msg_ptr) +{ + int res; + const char* location = endpoint->fabric->location; + gboolean checks[MAX_SEGMENTS] = {FALSE}; + void* contexts[MAX_SEGMENTS]; + int i; + int ack = ACK; + struct fid_mr* mrs[MAX_SEGMENTS]; + struct message* msg = malloc(sizeof(struct message)); + + res = fi_recv(endpoint->ep, endpoint->memory.buffer, sizeof(msg->len) + endpoint->memory.rx_prefix_size, fi_mr_desc(endpoint->memory.mr), 0, &msg->len); + CHECK("failed to recive msg len!"); + checks[0] = FALSE; contexts[0] = &msg->len; + EXE(endpoint_wait_for_completion(endpoint, RX, 1, checks, contexts)); + memcpy(&msg->len, (char*)endpoint->memory.buffer + endpoint->memory.rx_prefix_size, sizeof(msg->len)); + + res = fi_recv(endpoint->ep, endpoint->memory.buffer, sizeof(*msg->data)*msg->len + endpoint->memory.rx_prefix_size, fi_mr_desc(endpoint->memory.mr), 0, msg->data); + CHECK("failed to recive body!"); + checks[0] = FALSE; contexts[0] = msg->data; + msg->data = malloc(msg->len * sizeof(*msg->data)); + EXE(endpoint_wait_for_completion(endpoint, RX, 1, checks, contexts)); + memcpy(msg->data, (char*)endpoint->memory.buffer + endpoint->memory.rx_prefix_size, msg->len * sizeof(*msg->data)); + + for( i = 0; (size_t)i < msg->len; ++i) { + uint64_t addr = msg->data[i].buff_addr; + contexts[i] = (void*) addr; + checks[i] = FALSE; + + msg->data[i].buff_addr = (uint64_t)malloc(msg->data[i].buff_size); + memset((void*)msg->data[i].buff_addr, 0, msg->data[i].buff_size); + res = fi_mr_reg(endpoint->domain, + (void*)msg->data[i].buff_addr, + msg->data[i].buff_size, FI_READ, + 0, 0, 0, &mrs[i], 0); + CHECK("failed to register recive memory for rma!"); + + res = fi_read(endpoint->ep, + (void*)msg->data[i].buff_addr, + msg->data[i].buff_size, + fi_mr_desc(mrs[i]), 0, addr, msg->data[i].key, contexts[i]); + CHECK("failed to initeate rma read!"); + } + + EXE(endpoint_wait_for_completion(endpoint, TX, msg->len, checks, contexts)); + fi_inject(endpoint->ep, &ack, sizeof(ack), 0); + + for(i = 0; (size_t)i < msg->len; ++i) { + res = fi_close(&mrs[i]->fid); + CHECK("failed to unregister memory!"); + } + + *msg_ptr = msg; + return TRUE; +end: + *msg_ptr = NULL; + if(msg->data) { free(msg->data); } + free(msg); + return FALSE; +} + +void server(void); +void client(char* addr, uint32_t port); +struct message* construct_message(void); +void free_message(struct message* msg, gboolean sender); +void print_message(struct message* msg); +gboolean start_new_thread(struct jfabric* fabric, struct fi_eq_cm_entry* con_req); +void* thread(void* thread_data); + +const char* const buffers[] = {"Hello", "World!"}; +struct message* construct_message(void) { + struct message* msg = malloc(sizeof(struct message)); + + msg->len = 2; + msg->data = malloc(sizeof(*msg->data) * msg->len); + msg->data[0].buff_size = 6; msg->data[0].buff_addr = (uint64_t)g_strdup(buffers[0]); + msg->data[1].buff_size = 7; msg->data[1].buff_addr = (uint64_t)g_strdup(buffers[1]); + return msg; +} +void free_message(struct message* msg, gboolean sender) { + size_t i; + if(msg == NULL) { return; } + if(msg->data != NULL) { + for(i=0; i < msg->len; ++i) { free((void*)msg->data[i].buff_addr); } + free(msg->data); + } + free(msg); +} +void print_message(struct message* msg) { + int i; + + g_print("message:\n\tlen: %lu\n\tdata:\n", msg->len); + for(i = 0; (size_t)i < msg->len; ++i) { + g_print("\t\tseg %i[l=%lu]: >%s<\n", i, msg->data[i].buff_size, (char*)msg->data[i].buff_addr); + // g_print("\t\tseg %i[l=%lu]: addr=%lu\n", i, msg->data[i].buff_size, msg->data[i].buff_addr); + } +} + +void* thread(void* thread_data) { + struct MyThreadData* data; + struct jendpoint* endpoint; + struct message* msg; + // enum Event event; + + data = thread_data; + + EXE(endpoint_init_server(&endpoint, data->fabric, data->con_req)); + while(endpoint_check_connection(endpoint)) + { + EXE(srecv_message(endpoint, &msg)); + print_message(msg); + free_message(msg, FALSE); + msg = construct_message(); + g_print("SEND:"); + print_message(msg); + EXE(ssend_message(endpoint, msg)); + free_message(msg, TRUE); + /*EXE(endpoint_read_eq(endpoint, &event)); // TODO: check for shutdown + switch(event) { + case ERROR: goto end; + case CONNECTED: run = TRUE; break; + case SHUTDOWN: run = FALSE; break; + case CONNECTION_REQUEST: g_error("ep can't recive an connection request?!"); goto end; + }*/ + sleep(10); + } +end: + g_thread_exit(NULL); + return NULL; +} + +gboolean start_new_thread(struct jfabric* fabric, struct fi_eq_cm_entry* con_req) { + struct MyThreadData* thread_data = malloc(sizeof(struct MyThreadData)); + thread_data->fabric = fabric; + thread_data->con_req = con_req; + g_thread_new(NULL, thread, (gpointer)thread_data); + return TRUE; +} + + +void server(void) { + struct jfabric* fabric; + gboolean run; + enum Event event; + struct fi_eq_cm_entry con_req; + int p_socket; + int socket; + struct Config* config; + + config = config_init(); + + EXE(fabric_init_server(config, &fabric)); + EXE(open_socket(&p_socket, PORT)); + run = TRUE; + do{ + EXE(socket_wait_for_connection(p_socket, &socket)); + g_message("write_addr"); + EXE(socket_write_addr(socket, fabric)); + close(socket); + EXE(read_pep_eq(fabric, &event, &con_req)); + switch(event) { + case ERROR: goto end; + case CONNECTED: g_error("pep can't connect!?"); goto end; + case CONNECTION_REQUEST: + EXE(start_new_thread(fabric, &con_req)); + break; + case SHUTDOWN: run = FALSE; break; + default: g_assert_not_reached(); goto end; + } + } while(run); +end: + fabric_fini(fabric); + config_fini(config); +} + +void client(char* addr, uint32_t port) { + struct jendpoint* endpoint; + struct message* msg = NULL; + struct Config* config; + + config = config_init(); + + EXE(endpoint_init_client(&endpoint, config, addr, port)); + + msg = construct_message(); + ((char*)msg->data[0].buff_addr)[3] = 'w'; + ((char*)msg->data[1].buff_addr)[3] = 'w'; + g_print("SEND"); + print_message(msg); + EXE(ssend_message(endpoint, msg)); + free_message(msg, TRUE); + EXE(srecv_message(endpoint, &msg)); + print_message(msg); +end: + free_message(msg, FALSE); + config_fini(config); + endpoint_fini(endpoint); + return; +} + +int main(int argc, char** argv) { + + if(argc < 2) { + fprintf(stderr, usage); + return EXIT_FAILURE; + } + if(strcmp(argv[1], "server") == 0) { + server(); + } else if (strcmp(argv[1], "client") == 0) { + if(argc < 3) { + fprintf(stderr, "client can only connect when ip address is given!"); + return EXIT_FAILURE; + } + client(argv[2], PORT); + } else { + fprintf(stderr, usage); + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} From 340cce733ea2d46450432c5ddecd7dff6f388492 Mon Sep 17 00:00:00 2001 From: Julian Benda Date: Tue, 29 Jun 2021 16:12:39 +0200 Subject: [PATCH 06/60] Fix memory leaks and add closing server --- example/libfabric_basic.c | 67 +++++++++++++++++++++++++++------------ meson.build | 3 ++ 2 files changed, 50 insertions(+), 20 deletions(-) diff --git a/example/libfabric_basic.c b/example/libfabric_basic.c index f4a3b66ba..c87838bd0 100644 --- a/example/libfabric_basic.c +++ b/example/libfabric_basic.c @@ -29,6 +29,7 @@ #define MSG_PARTS 2 #define MAX_SEGMENTS 5 #define ACK 42 +#define TIMEOUT 5 const char* const usage = "call with: (server|client )"; @@ -67,6 +68,7 @@ struct jendpoint { struct fid_cq* txcq; struct fid_cq* rxcq; int inject_size; + gboolean shutdown; }; struct MyThreadData { @@ -299,7 +301,7 @@ gboolean socket_write_addr(int fd, struct jfabric* fabric) int res; size_t addrlen = 0; const char* location; - char* addr; + char* addr = NULL; uint32_t len; location = fabric->location; @@ -326,8 +328,10 @@ gboolean socket_write_addr(int fd, struct jfabric* fabric) res = send(fd, addr, addrlen, 0); if(res == -1 || (size_t)res != addrlen) { g_error("failed to send addr!(%i)", res); goto end; } + free(addr); return TRUE; end: + if(addr) { free(addr); } return FALSE; } gboolean parse_address(char* addr, uint32_t port, struct addrinfo** infos) @@ -443,6 +447,7 @@ gboolean endpoint_fini(struct jendpoint* endpoint) fi_freeinfo(endpoint->info); } free(endpoint); + g_message("finisihed endpoint!"); return TRUE; end: return FALSE; @@ -637,7 +642,7 @@ gboolean endpoint_read_eq(struct jendpoint* endpoint, enum Event* evt, struct fi gboolean endpoint_check_connection(struct jendpoint* endpoint) { - return endpoint->ep != NULL; + return endpoint->ep != NULL && !endpoint->shutdown; } gboolean endpoint_wait_for_completion(struct jendpoint* endpoint, enum ConnectionDirection con_dir, int len, gboolean* check, void** contexts) @@ -660,8 +665,23 @@ gboolean endpoint_wait_for_completion(struct jendpoint* endpoint, enum Connectio if(check[i] == TRUE) {++count; } } while(count != len) { - res = fi_cq_sread(cq, &entry, 1, NULL, -1); - if(res == -FI_EAGAIN) { continue; } + res = fi_cq_sread(cq, &entry, 1, NULL, TIMEOUT); + if (res == -FI_EAGAIN) { + uint32_t event; + struct fi_eq_cm_entry eq_entry; + res = fi_eq_read(endpoint->eq, &event, &eq_entry, sizeof(eq_entry), 0); + if(res != -FI_EAGAIN) { + CHECK("failed to read eq in cq test!"); + if(event == FI_SHUTDOWN) { + g_message("shutdown"); + endpoint->shutdown = TRUE; + return FALSE; + } + g_error("invalid event recived!"); + goto end; + } + continue; + } if(res < 0) { int err = res; res = fi_cq_readerr(cq, &err_entry, 0); @@ -777,6 +797,8 @@ gboolean srecv_message(struct jendpoint* endpoint, struct message** msg_ptr) int ack = ACK; struct fid_mr* mrs[MAX_SEGMENTS]; struct message* msg = malloc(sizeof(struct message)); + + msg->data = NULL; res = fi_recv(endpoint->ep, endpoint->memory.buffer, sizeof(msg->len) + endpoint->memory.rx_prefix_size, fi_mr_desc(endpoint->memory.mr), 0, &msg->len); CHECK("failed to recive msg len!"); @@ -784,10 +806,10 @@ gboolean srecv_message(struct jendpoint* endpoint, struct message** msg_ptr) EXE(endpoint_wait_for_completion(endpoint, RX, 1, checks, contexts)); memcpy(&msg->len, (char*)endpoint->memory.buffer + endpoint->memory.rx_prefix_size, sizeof(msg->len)); + msg->data = malloc(msg->len * sizeof(*msg->data)); res = fi_recv(endpoint->ep, endpoint->memory.buffer, sizeof(*msg->data)*msg->len + endpoint->memory.rx_prefix_size, fi_mr_desc(endpoint->memory.mr), 0, msg->data); CHECK("failed to recive body!"); checks[0] = FALSE; contexts[0] = msg->data; - msg->data = malloc(msg->len * sizeof(*msg->data)); EXE(endpoint_wait_for_completion(endpoint, RX, 1, checks, contexts)); memcpy(msg->data, (char*)endpoint->memory.buffer + endpoint->memory.rx_prefix_size, msg->len * sizeof(*msg->data)); @@ -825,6 +847,7 @@ gboolean srecv_message(struct jendpoint* endpoint, struct message** msg_ptr) *msg_ptr = NULL; if(msg->data) { free(msg->data); } free(msg); + g_message("cancle read!"); return FALSE; } @@ -833,7 +856,7 @@ void client(char* addr, uint32_t port); struct message* construct_message(void); void free_message(struct message* msg, gboolean sender); void print_message(struct message* msg); -gboolean start_new_thread(struct jfabric* fabric, struct fi_eq_cm_entry* con_req); +gboolean start_new_thread(struct jfabric* fabric, struct fi_eq_cm_entry* con_req, GArray* threads); void* thread(void* thread_data); const char* const buffers[] = {"Hello", "World!"}; @@ -884,43 +907,42 @@ void* thread(void* thread_data) { print_message(msg); EXE(ssend_message(endpoint, msg)); free_message(msg, TRUE); - /*EXE(endpoint_read_eq(endpoint, &event)); // TODO: check for shutdown - switch(event) { - case ERROR: goto end; - case CONNECTED: run = TRUE; break; - case SHUTDOWN: run = FALSE; break; - case CONNECTION_REQUEST: g_error("ep can't recive an connection request?!"); goto end; - }*/ - sleep(10); } end: - g_thread_exit(NULL); + g_message("close thread!"); + endpoint_fini(endpoint); + free(thread_data); return NULL; } -gboolean start_new_thread(struct jfabric* fabric, struct fi_eq_cm_entry* con_req) { +gboolean start_new_thread(struct jfabric* fabric, struct fi_eq_cm_entry* con_req, GArray* threads) { struct MyThreadData* thread_data = malloc(sizeof(struct MyThreadData)); + GThread* new_thread; thread_data->fabric = fabric; thread_data->con_req = con_req; - g_thread_new(NULL, thread, (gpointer)thread_data); + new_thread = g_thread_new(NULL, thread, (gpointer)thread_data); + g_array_append_val(threads, new_thread); return TRUE; } void server(void) { struct jfabric* fabric; - gboolean run; + int run = 3; + guint i; enum Event event; struct fi_eq_cm_entry con_req; int p_socket; int socket; struct Config* config; + GArray* threads; config = config_init(); + threads = g_array_new(FALSE, FALSE, sizeof(GThread*)); EXE(fabric_init_server(config, &fabric)); EXE(open_socket(&p_socket, PORT)); - run = TRUE; + // run = TRUE; do{ EXE(socket_wait_for_connection(p_socket, &socket)); g_message("write_addr"); @@ -931,13 +953,18 @@ void server(void) { case ERROR: goto end; case CONNECTED: g_error("pep can't connect!?"); goto end; case CONNECTION_REQUEST: - EXE(start_new_thread(fabric, &con_req)); + EXE(start_new_thread(fabric, &con_req, threads)); + --run; break; case SHUTDOWN: run = FALSE; break; default: g_assert_not_reached(); goto end; } } while(run); end: + for(i = 0; i < threads->len; ++i) { + g_thread_join(g_array_index(threads, GThread*, i)); + } + g_array_free(threads, TRUE); fabric_fini(fabric); config_fini(config); } diff --git a/meson.build b/meson.build index 2f49ebbbc..ddf68311e 100644 --- a/meson.build +++ b/meson.build @@ -732,6 +732,9 @@ endif # ) #endforeach +executable('libfabric_basic', files('example/libfabric_basic.c'), + dependencies: [glib_dep, libfabric_dep]) + # Install julea_client_hdrs = { From a619897f20306be39fd4be6aee9aacd5926d4a6a Mon Sep 17 00:00:00 2001 From: Julian Benda Date: Thu, 1 Jul 2021 10:51:21 +0200 Subject: [PATCH 07/60] First docu of network --- lib/jnetwork.c | 26 ++++ lib/jnetwork.h | 396 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 422 insertions(+) create mode 100644 lib/jnetwork.c create mode 100644 lib/jnetwork.h diff --git a/lib/jnetwork.c b/lib/jnetwork.c new file mode 100644 index 000000000..2b70a71be --- /dev/null +++ b/lib/jnetwork.c @@ -0,0 +1,26 @@ +#include "./jnetwork.h" + +#include + +/// Used to initialize common parts between different connection. +/** This will create the following libfabric resources: + * * the domain + * * a event queue + * * a rx and tx completion queue + * * the endpoint + * + * Also it will bind them accordingly and enable the endpoint. + * \protected \memberof JConnection + * \retval FALSE if initialisation failed + */ +gboolean +j_connection_init(struct JConnection* instance); + +/// Allocated and bind memory needed for message transfer. +/** + * \protected \memberof JConnection + * \retval FALSE if allocation or binding failed + */ +gboolean +j_connection_create_memory_resources(struct JConnection* instance); + diff --git a/lib/jnetwork.h b/lib/jnetwork.h new file mode 100644 index 000000000..d04a4a41b --- /dev/null +++ b/lib/jnetwork.h @@ -0,0 +1,396 @@ +/** \file jnetwork.h + * \ingroup network + * + * \copyright + * JULEA - Flexible storage framework + * Copyright (C) 2010-2021 Michael Kuhn + * \n + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * \n + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * \n + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + */ + +#include + +#include + +struct JConfiguration; +struct GSocketConnection; + +/** \defgroup network Network + * Interface for communication over network + */ + +/** \struct JFabricConnectionRequest lib/jnetwork.h + * \brief A connection request read from an event queue. + * + * Connection request are mendetory to establish a connection from server side. + * \sa j_connection_init_server, j_fabric_sread_event + * \ingroup network + */ +struct JFabricConnectionRequest; + +/** \struct JFabricAddr jnetwork.h + * \brief Fabrics address data needed to send an connection request. + * \ingroup network + * \sa j_fabric_init_client + */ +struct JFabricAddr; + +/// Possible events for paired connections +/** \ingroup network + * \sa j_connection_sread_event, j_connection_read_event + */ +enum JConnectionEvents { + J_CONNECTION_EVENT_ERROR = 0, ///< An error was reported, the connection is probably in an invalid state! + J_CONNECTION_EVENT_TIMEOUT, ///< there was no event to read in the given time frame + J_CONNECTION_EVENT_CONNECTED, ///< First event after successful established connection. + J_CONNECTION_EVENT_SHUTDOWN ///< connection was closed +}; + +/// Possible events for listening fabircs +/** \ingroup network + * \sa j_fabric_sread_event + */ +enum JFabricEvents { + J_FABRIC_EVENT_ERROR = 0, ///< An error was reported, fabric is probably in a invalid state! + J_FABRIC_EVENT_TIMEOUT, ///< No event received in the given time frame. + J_FABRIC_EVENT_CONNECTION_REQUEST, ///< A connection request was received. + J_FABRIC_EVENT_SHUTDOWN ///< fabric socket was closed +}; + +/** \class JFabric lib/jnetwork.h + * \brief Manage access to network. + * + * Mandetory to communicate via a network and to initelize connections. + * Also handles unpaired communication (connection requests!) + * \ingroup network + */ +struct JFabric; + +/// Initelize a fabric for the server side. +/** \public \memberof JFabric + * + * In differents to j_fabric_init_client, the resulting fabric is capable of reciving connection requests. + * There fore events can be readed via j_fabric_sread_event + * \retval FALSE on failure + */ +gboolean +j_fabric_init_server( + struct JConfiguration* configuration, ///< [in] of JULEA, to fetch details + struct JFabric** instance ///< [out] pointer to resulting fabric +); + +/// Initelize a fabric for the client side. +/** \pulbic \memberof JFabric + * + * Contains data to building a paired connection. + * Addess of JULEA server is needed, to enforce that both communicate via the same network. + * \warning will be called from j_connection_init_client(), so no need for explicit a call + * \retval FALSE on failure + */ +gboolean +j_fabric_init_client( + struct JConfiguration* configuration, ///< [in] of JULEA to fetch details + struct JFabricAddr* addr, ///< [in] of JULEA server fabric + struct JFabric** instance ///< [out] pointer to resulting fabric +); + +/// Closes a fabirc and frees used memory. +/** \public \memberof JFabric + * \retval FALSE on failure + * \pre Finish all connections created from this fabric! + */ +gboolean +j_fabric_fini(struct JFabric* instance); + +/// Read a event of a listening fabric. +/** \public \memberof JFabric + * \warning only usable for fabrics initelized with j_fabric_init_server() + * \retval TRUE if fetching succeed or the request timeouted + * \retval FALSE if fetching an event fails + */ +gboolean +j_fabric_sread_event(struct JFabric* instance, + int timeout, ///< [in] set to -1 for no timeout. + enum JFabricEvents* event, ///< [out] reeded from event queue + struct JFabricConnectionRequest* con_req ///< [out] contains connection request, + /// if event == J_FABRIC_EVENT_CONNECTION_REQUEST +); + +/** \class JConnection jnetwork.h + * \brief A paired connection over a network. + * + * Connections are used to send data from server to client or vis a vi. + * They support usage for: + * * messaging: easy to serialize and verify + * * direct memory read: sender must "protect memory" until receiver verifies completion. + * \ingroup network + * \par Example for client side connection usage. + * You must check the success of each function individual, we spare this to reduce the code length! + * \code + * struct JConnection* connection; + * int ack; + * int data_size = 12, rdata_size = 16; + * gboolean ret; + * void* data_a = malloc(data_size), data_b = malloc(data_size), rdata = malloc(rdata_size); + * // fill data ! + * + * j_connection_init_client(config, J_BACKEND_TYPE_OBJECT, &connection); + * + * // message exchange + * j_connection_srecv(connection, rdata, rdata_size); // sync receive operation + * // handle message and write result in data + * j_connection_send(connection, data_a, data_size); + * j_connection_send(connection, data_b, data_size); + * j_connection_wait_for_completion(connection, ret, data_a, data_b); + * + * // provide memory for rma read action of other party. + * JConnectionMemory rma_mem; + * j_connection_rma_register(connection, data, data_size, &rma_mem); + * // wait for other party to signal that they finished reading + * j_connection_srecv(connection, &ack, sizeof(ack)); + * j_connection_rma_unregister(connection, &rma_mem); + * + * // rma read + * j_connection_rma_read(connection, data, data_size); + * j_connection_wait_for_completion(connection, ret, data); + * + * j_connection_fini(connection); + * \endcode + * + * \par Establish a connection from server side + * You must check the success of each function individual, we spare this to reduce the code length! + * This code should be placed in the TCP server connection request handler. + * \code + * JConfiguration* config; JFabric* fabric; // are probably initialized + * GSocketConnection* gconnection = [>newly established g_socket connection<]; + * JConnection* jconnection; + * j_connection_init_server(config, fabric, gconnection, &jconnection); + * + * // same communication code then client side + * + * j_connection_fini(jconnection); + * \endcode + * \todo expose more connection posibilities (eg. connection with an JFabricAddr) + */ +struct JConnection; + +/// Handle to recognize async operations. +/** \public \memberof JConnection + * Used for async operations to check later if the completed or wait for completion.\n + * The value is equal NULL means no waiting is necessary. + * \sa j_connection_recv, j_connection_send, j_connection_rma_read, j_connection_wait_for_completion + */ +typedef void* JConnectionOperationHandle; + +/// Handle for memory regions available via rma. +/** \public \memberof JConnection + * \sa j_connection_rma_read, j_connection_rma_register, j_connection_rma_unregister + */ +typedef void* JConnectionMemory; + +/// Builds a connection to an active fabric (direct). +/** \public \memberof JConnection + * + * This constructor will fetch the fabric address via TCP and then building an paired + * connection to this, which later allows for messaging and RMA data transfer. \n + * This will also construct a fabric for that connection, which is freed when the connection is finished. + * + * \attention this function may reduces j_configuration_max_operation_size() + * accordingly to network capabilities. + * + * \msc "Connection process" + * Client, Server; + * Client => Server [label="g_socket_client_connect_to_host(...)"]; + * Server => Client [label="JFabricAddr"]; + * Client => Server [label="g_socket_close(...)"]; + * Client => Server [label="J_FABRIC_EVENT_CONNECTION_REQUEST"]; + * Server => Client [label="J_CONNECTION_EVENT_CONNECTED"]; + * Server => Server [label="J_CONNECTION_EVENT_CONNECTED"]; + * \endmsc + * \retval FALSE if building a connection failed + * \sa j_connection_init_server + */ +gboolean +j_connection_init_client ( + struct JConfiguration* configuration, ///< [in] for server address, and network configuration + enum JBackendType backend, ///< [in] backend server to connect to + struct JConnection** instance ///< [out] constructed instance +); + + +/// Establish connection to client based on established GSocketConnection. +/** \public \memberof JConnection + * + * The GSocketConnection will be used to send the server fabric data. + * For the connection process see j_connection_init_client(). + * + * \attention this function may reduces j_configuration_max_operation_size() + * accordingly to network capabilities. + * + * \retval FALSE if establishing the connection failed + * \sa j_connection_init_client + */ +gboolean +j_connection_init_server ( + struct JConfiguration* configuration, ///< [in] for the network configuration + struct JFabric* fabric, ///< [in] via which the connection should be established + struct GSocketConnection* gconnection, ///< [in] valid GSocketConnection for address exchange + struct JConnection** instance ///< [out] constructed instance +); +/// Closes a connection and free all related resources. +/** \public \memberof JConnection + * + * If this an client side connection with private fabric, it will also clear the fabric. + * \retval FALSE if closing the connection failed. The connection will still be unusable! + */ +gboolean +j_connection_fini ( struct JConnection* instance); + +/// Check if the connection is still valid. +/** \public \memberof JConnection + * + * Check if the connection was established and if was no J_CONNECTION_EVENT_SHUTDOWN recognized. + * \attention events will only be checked if j_connection_read_event() is called!. + * \retval FALSE if connection is not longer valid + * \retval TRUE if connection was created and no shutdown event was reported. + * \todo advance connection checking! + */ +gboolean +j_connection_check_connection(struct JConnection* instance); + +/// blocking read one event from connection +/** \public \memberof JConnection + * Used to check wait for connected state, and recognized errors and shutdown signals. + * For a version which returns immediate when no event exists see j_connection_read_event() + * \retval TRUE if fetching succeed or the request timeouted + * \retval FALSE if fetching event fails + */ +gboolean +j_connection_sread_event(struct JConnection* instance, + int timeout, ///< [in] set to -1 for no timeout + enum JConnectionEvents* event ///< [out] reeded from queue +); + +/// check for event on connection +/** \public \memberof JConnection + * Returns a J_CONNECTION_EVENT_TIMEOUT event if no event is ready to report. + * \retval TRUE if fetching succeed or no events to record + * \retval FALSE if fetching failed + */ +gboolean +j_connection_read_event(struct JConnection* instance, + enum JConnectionEvents* event ///< [out] reeded from queue +); + +/// Async send data via MSG connection +/** \public \memberof JConnection + * + * Asynchronous sends a message, to recognize for completion use j_connection_wait_for_completion().\n + * If the message is small enough it can "injected" to the network, in that case the actions finished + * immediate and the \p handle is NULL + * \attention it is only allowed to have J_CONNECTION_MAX_SEND + * \retval FALSE if an error occurred. + * \sa j_connection_srecv, j_connection_wait_for_completion + */ +gboolean +j_connection_send(struct JConnection* instance, + void* data, ///< [in] to send + size_t data_len, ///< [in] in bytes + JConnectionOperationHandle* handle ///< [out] to recognize completion +); + +/// Blocking receive data via MSG connection. +/** \public \memberof JConnection + * + * Blocks until an error occures or the data were received. + * If you want read multiple data via the same pair connection at the same time you will probably use + * j_connection_rma_read().\n + * To wait for completion use j_connection_wait_for_completion(). + * \retval FALSE if an error occurred + * \sa j_connection_send, j_connection_wait_for_completion + */ +gboolean +j_connection_srecv(struct JConnection* instance, + size_t data_len, ///< [in] in bytes to receive + void* data ///< [out] received +); + + +/// Async direct memory read. +/** \public \memberof JConnection + * + * Initiate an direct memory read, the completion can be recognized with \p handle. + * \sa j_connection_wait_for_completion + * \retval FALSE if an error occurred -> handle will then also invalid + */ +gboolean +j_connection_rma_read(struct JConnection* instance, + size_t data_len, ///< [in] in bytes to read + void* data, ///< [out] received + JConnectionOperationHandle* handle ///< [out] to recognize completion +); + + +/// Wait until operation associated with handles are completed. +/** \public \memberof JConnection + * + * \p handle which equals NULL will not be waited for. Also \p check is even on error valid. + * Use \ref j_connection_wait_for_completion for a simpler interface. + * \attention memory for check must be provided by caller! + * \sa j_connection_rma_read, j_connection_send, j_connection_wait_for_completion + */ +gboolean +j_connection_wait_for_completion_detail(struct JConnection* instance, + JConnectionOperationHandle handles[], ///< [in] array of handle to wait for. + int count, ///< [in] of handle + gboolean check[] ///< [out] logs which handle are completed now +); + +#define NUMARGS(...) (sizeof((int[]){__VA_ARGS__})/sizeof(int)) + +/// Wait until operations associated with handles are completed +/** \relates JConnection + * \param[in] instance of JConnection to work on + * \param[in] ... JConnectionOperationHandle to wait for completion + * \param[out] result of operation, is FALSE on failure + * \sa j_connection_rma_read, j_connection_send, j_connection_wait_for_completion_detail + */ +#define j_connection_wait_for_completion(instance, result, ...) \ + do { \ + gboolean check[NUMARGS(__VA_ARGS__)] = {0}; \ + JConnectionOperationHandle handles = {__VA_ARGS__}; \ + result = j_connection_wait_for_completion_detail(instance, handles, NUMARGS(__VA_ARGS__), check); \ + } while(FALSE) + +/// Register memory to make it rma readable. +/** \public \memberof JConnection + * Memory access rights must changed to allow for an rma read of other party. + * There for before an j_connection_rma_read() can succeed the provider of the data must + * register the memory first! + */ +gboolean +j_connection_rma_register(struct JConnection* intsance, + void* data, ///< [in] begin of memory region to share + size_t data_len, ///< [in] size of memory region in bytes + JConnectionMemory* handle ///< [out] for memory region to unregister with j_connection_rma_unregister +); + +/// Unregister memory from rma availablity. +/** \public \memberof JConnection + * Counterpart to j_connection_rma_register(). + */ +gboolean +j_connection_rma_unregister(struct JConnection* instance, + JConnectionMemory* handle ///< [in] for memory region to unregister +); From 479335572b561f4f84d9bfc0ec1702e83c500946 Mon Sep 17 00:00:00 2001 From: Julian Benda Date: Thu, 1 Jul 2021 11:30:44 +0200 Subject: [PATCH 08/60] Changed srecv to recv --- lib/core/jhelper.c | 35 --------------- lib/{ => core}/jnetwork.c | 0 lib/{ => core}/jnetwork.h | 95 ++++++++++++++++----------------------- 3 files changed, 38 insertions(+), 92 deletions(-) rename lib/{ => core}/jnetwork.c (100%) rename lib/{ => core}/jnetwork.h (80%) diff --git a/lib/core/jhelper.c b/lib/core/jhelper.c index cf6282c44..c562ebe6d 100644 --- a/lib/core/jhelper.c +++ b/lib/core/jhelper.c @@ -49,41 +49,6 @@ * @{ **/ -void -j_helper_set_nodelay(GSocketConnection* connection, gboolean enable) -{ - J_TRACE_FUNCTION(NULL); - - gint const flag = (enable) ? 1 : 0; - - GSocket* socket_; - gint fd; - - g_return_if_fail(connection != NULL); - - socket_ = g_socket_connection_get_socket(connection); - fd = g_socket_get_fd(socket_); - - setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &flag, sizeof(gint)); -} - -void -j_helper_set_cork(GSocketConnection* connection, gboolean enable) -{ - J_TRACE_FUNCTION(NULL); - - gint const flag = (enable) ? 1 : 0; - - GSocket* socket_; - gint fd; - - g_return_if_fail(connection != NULL); - - socket_ = g_socket_connection_get_socket(connection); - fd = g_socket_get_fd(socket_); - - setsockopt(fd, IPPROTO_TCP, TCP_CORK, &flag, sizeof(gint)); -} void j_helper_get_number_string(gchar* string, guint32 length, guint32 number) diff --git a/lib/jnetwork.c b/lib/core/jnetwork.c similarity index 100% rename from lib/jnetwork.c rename to lib/core/jnetwork.c diff --git a/lib/jnetwork.h b/lib/core/jnetwork.h similarity index 80% rename from lib/jnetwork.h rename to lib/core/jnetwork.h index d04a4a41b..6801afa72 100644 --- a/lib/jnetwork.h +++ b/lib/core/jnetwork.h @@ -141,29 +141,30 @@ j_fabric_sread_event(struct JFabric* instance, * struct JConnection* connection; * int ack; * int data_size = 12, rdata_size = 16; - * gboolean ret; * void* data_a = malloc(data_size), data_b = malloc(data_size), rdata = malloc(rdata_size); * // fill data ! * * j_connection_init_client(config, J_BACKEND_TYPE_OBJECT, &connection); * * // message exchange - * j_connection_srecv(connection, rdata, rdata_size); // sync receive operation + * j_connection_recv(connection, rdata, rdata_size); + * j_connection_wait_for_completion(connection); * // handle message and write result in data * j_connection_send(connection, data_a, data_size); * j_connection_send(connection, data_b, data_size); - * j_connection_wait_for_completion(connection, ret, data_a, data_b); + * j_connection_wait_for_completion(connection); * * // provide memory for rma read action of other party. * JConnectionMemory rma_mem; * j_connection_rma_register(connection, data, data_size, &rma_mem); * // wait for other party to signal that they finished reading - * j_connection_srecv(connection, &ack, sizeof(ack)); + * j_connection_recv(connection, &ack, sizeof(ack)); + * j_connection_wait_for_completion(connection); * j_connection_rma_unregister(connection, &rma_mem); * * // rma read * j_connection_rma_read(connection, data, data_size); - * j_connection_wait_for_completion(connection, ret, data); + * j_connection_wait_for_completion(connection); * * j_connection_fini(connection); * \endcode @@ -185,13 +186,13 @@ j_fabric_sread_event(struct JFabric* instance, */ struct JConnection; -/// Handle to recognize async operations. -/** \public \memberof JConnection - * Used for async operations to check later if the completed or wait for completion.\n - * The value is equal NULL means no waiting is necessary. - * \sa j_connection_recv, j_connection_send, j_connection_rma_read, j_connection_wait_for_completion - */ -typedef void* JConnectionOperationHandle; +/// Highest number of j_connection_send() calls before a j_connection_wait_for_completion(). +/** \public \memberof JConnection */ +#define J_CONNECTION_MAX_SEND 2 + +/// Highest number of j_connection_recv() calls before a j_connection_wait_for_completion(). +/** \public \memberof JConnection */ +#define J_CONNECTION_MAX_RECV 1 /// Handle for memory regions available via rma. /** \public \memberof JConnection @@ -298,30 +299,37 @@ j_connection_read_event(struct JConnection* instance, * * Asynchronous sends a message, to recognize for completion use j_connection_wait_for_completion().\n * If the message is small enough it can "injected" to the network, in that case the actions finished - * immediate and the \p handle is NULL - * \attention it is only allowed to have J_CONNECTION_MAX_SEND + * immediate (j_connection_wait_for_completion() still works). + * + * \todo feedback if message was injected! + * + * \attention it is only allowed to have J_CONNECTION_MAX_SEND send + * operation pending at the same time. Each has a max size + * of j_configuration_max_operation_size() (the connection initialization may changes this value!). + * * \retval FALSE if an error occurred. - * \sa j_connection_srecv, j_connection_wait_for_completion + * \sa j_connection_recv, j_connection_wait_for_completion */ gboolean j_connection_send(struct JConnection* instance, - void* data, ///< [in] to send - size_t data_len, ///< [in] in bytes - JConnectionOperationHandle* handle ///< [out] to recognize completion + void* data, ///< [in] to send + size_t data_len ///< [in] in bytes ); -/// Blocking receive data via MSG connection. +/// Asynchronous receive data via MSG connection. /** \public \memberof JConnection * - * Blocks until an error occures or the data were received. - * If you want read multiple data via the same pair connection at the same time you will probably use - * j_connection_rma_read().\n - * To wait for completion use j_connection_wait_for_completion(). + * Asynchronous receive a message, to wait for completion use j_connection_wait_for_completion(). + * + * \attention it is only allowed to have J_CONNECTION_MAX_RECV recv + * operation pending at the same time. Each has a max size + * of j_configuration_max_operation_size() (the connection initialization may has changed this value!). + * * \retval FALSE if an error occurred * \sa j_connection_send, j_connection_wait_for_completion */ gboolean -j_connection_srecv(struct JConnection* instance, +j_connection_recv(struct JConnection* instance, size_t data_len, ///< [in] in bytes to receive void* data ///< [out] received ); @@ -330,48 +338,21 @@ j_connection_srecv(struct JConnection* instance, /// Async direct memory read. /** \public \memberof JConnection * - * Initiate an direct memory read, the completion can be recognized with \p handle. - * \sa j_connection_wait_for_completion + * Initiate an direct memory read, to wait for completion use j_connection_wait_for_completion(). * \retval FALSE if an error occurred -> handle will then also invalid */ gboolean j_connection_rma_read(struct JConnection* instance, size_t data_len, ///< [in] in bytes to read - void* data, ///< [out] received - JConnectionOperationHandle* handle ///< [out] to recognize completion + void* data ///< [out] received ); - -/// Wait until operation associated with handles are completed. -/** \public \memberof JConnection - * - * \p handle which equals NULL will not be waited for. Also \p check is even on error valid. - * Use \ref j_connection_wait_for_completion for a simpler interface. - * \attention memory for check must be provided by caller! - * \sa j_connection_rma_read, j_connection_send, j_connection_wait_for_completion +/// Wait until operations initiated at his connection finished. +/** \public \memberof JConnection + * \sa j_connection_rma_read, j_connection_send, j_connection_recv */ gboolean -j_connection_wait_for_completion_detail(struct JConnection* instance, - JConnectionOperationHandle handles[], ///< [in] array of handle to wait for. - int count, ///< [in] of handle - gboolean check[] ///< [out] logs which handle are completed now -); - -#define NUMARGS(...) (sizeof((int[]){__VA_ARGS__})/sizeof(int)) - -/// Wait until operations associated with handles are completed -/** \relates JConnection - * \param[in] instance of JConnection to work on - * \param[in] ... JConnectionOperationHandle to wait for completion - * \param[out] result of operation, is FALSE on failure - * \sa j_connection_rma_read, j_connection_send, j_connection_wait_for_completion_detail - */ -#define j_connection_wait_for_completion(instance, result, ...) \ - do { \ - gboolean check[NUMARGS(__VA_ARGS__)] = {0}; \ - JConnectionOperationHandle handles = {__VA_ARGS__}; \ - result = j_connection_wait_for_completion_detail(instance, handles, NUMARGS(__VA_ARGS__), check); \ - } while(FALSE) +j_connection_wait_for_completion(struct JConnection* instance); /// Register memory to make it rma readable. /** \public \memberof JConnection From 5d504652ee3fb16f5dc1be02aa8c9a3e16277fac Mon Sep 17 00:00:00 2001 From: Julian Benda Date: Thu, 1 Jul 2021 11:44:52 +0200 Subject: [PATCH 09/60] Place jnetwork in correct location & edit build file --- {lib => include}/core/jnetwork.h | 9 +++++++++ include/julea.h | 1 + lib/core/jnetwork.c | 2 +- meson.build | 2 ++ 4 files changed, 13 insertions(+), 1 deletion(-) rename {lib => include}/core/jnetwork.h (98%) diff --git a/lib/core/jnetwork.h b/include/core/jnetwork.h similarity index 98% rename from lib/core/jnetwork.h rename to include/core/jnetwork.h index 6801afa72..0767420f4 100644 --- a/lib/core/jnetwork.h +++ b/include/core/jnetwork.h @@ -19,6 +19,13 @@ * along with this program. If not, see . */ +#ifndef JULEA_NETWORK_H +#define JULEA_NETWORK_H + +#if !defined(JULEA_H) && !defined(JULEA_COMPILATION) +#error "Only can be included directly." +#endif + #include #include @@ -375,3 +382,5 @@ gboolean j_connection_rma_unregister(struct JConnection* instance, JConnectionMemory* handle ///< [in] for memory region to unregister ); + +#endif diff --git a/include/julea.h b/include/julea.h index fd9aa7bdc..994f54023 100644 --- a/include/julea.h +++ b/include/julea.h @@ -38,6 +38,7 @@ #include #include #include +#include #include #include #include diff --git a/lib/core/jnetwork.c b/lib/core/jnetwork.c index 2b70a71be..15b0655a1 100644 --- a/lib/core/jnetwork.c +++ b/lib/core/jnetwork.c @@ -1,4 +1,4 @@ -#include "./jnetwork.h" +#include #include diff --git a/meson.build b/meson.build index ddf68311e..d9fb017e0 100644 --- a/meson.build +++ b/meson.build @@ -373,6 +373,7 @@ julea_srcs = files([ 'lib/core/jcache.c', 'lib/core/jcommon.c', 'lib/core/jconfiguration.c', + 'lib/core/jnetwork.c', 'lib/core/jconnection-pool.c', 'lib/core/jcredentials.c', 'lib/core/jdir-iterator.c', @@ -745,6 +746,7 @@ julea_client_hdrs = { 'include/core/jbatch.h', 'include/core/jcache.h', 'include/core/jconfiguration.h', + 'include/core/jnetwork.h', 'include/core/jconnection-pool.h', 'include/core/jcredentials.h', 'include/core/jdir-iterator.h', From ed03ab4e843b89db0a2826299fb39079926ff9e8 Mon Sep 17 00:00:00 2001 From: Julian Benda Date: Fri, 2 Jul 2021 10:59:42 +0200 Subject: [PATCH 10/60] introduce rma_mem_id --- include/core/jnetwork.h | 36 +++++++++++++++++++++++++++++++----- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/include/core/jnetwork.h b/include/core/jnetwork.h index 0767420f4..6baa7992a 100644 --- a/include/core/jnetwork.h +++ b/include/core/jnetwork.h @@ -29,9 +29,9 @@ #include #include +#include struct JConfiguration; -struct GSocketConnection; /** \defgroup network Network * Interface for communication over network @@ -163,14 +163,19 @@ j_fabric_sread_event(struct JFabric* instance, * * // provide memory for rma read action of other party. * JConnectionMemory rma_mem; + * struct JConnectionMemoryID rma_mem_id; * j_connection_rma_register(connection, data, data_size, &rma_mem); + * j_connection_memory_get_id(rma_mem, &rma_mem_id); + * j_connection_send(connection, &rma_mem_id, sizeof(rma_mem_id)); * // wait for other party to signal that they finished reading * j_connection_recv(connection, &ack, sizeof(ack)); * j_connection_wait_for_completion(connection); * j_connection_rma_unregister(connection, &rma_mem); * * // rma read - * j_connection_rma_read(connection, data, data_size); + * j_connection_recv(connection, &rma_mem_id, sizeof(rma_mem_id)); + * j_connection_wait_for_completion(connection); + * j_connection_rma_read(connection, &rma_mem_id, data); * j_connection_wait_for_completion(connection); * * j_connection_fini(connection); @@ -207,6 +212,27 @@ struct JConnection; */ typedef void* JConnectionMemory; +/// Identifier to read memory. +/** \public \memberof JConnectionMemory + * \sa j_connection_rma_read, j_connection_rma_register, j_connection_memory_get_id + */ +struct JConnectionMemoryID { + uint64_t key; ///< access key for authentication + uint64_t offset; ///< used to identifie memory region + uint64_t size; ///< size in bytes of memery region +}; + +/// Get identifier of memory region +/** \public \memberof JConnectionMemory + * \retval FALSE on failure + * \sa j_connection_rma_read, j_connection_rma_register + */ +gboolean +j_connection_memory_get_id(JConnectionMemory instance, + struct JConnectionMemoryID* id ///< [out] of registerd memory +); + + /// Builds a connection to an active fabric (direct). /** \public \memberof JConnection * @@ -253,7 +279,7 @@ gboolean j_connection_init_server ( struct JConfiguration* configuration, ///< [in] for the network configuration struct JFabric* fabric, ///< [in] via which the connection should be established - struct GSocketConnection* gconnection, ///< [in] valid GSocketConnection for address exchange + GSocketConnection* gconnection, ///< [in] valid GSocketConnection for address exchange struct JConnection** instance ///< [out] constructed instance ); /// Closes a connection and free all related resources. @@ -350,8 +376,8 @@ j_connection_recv(struct JConnection* instance, */ gboolean j_connection_rma_read(struct JConnection* instance, - size_t data_len, ///< [in] in bytes to read - void* data ///< [out] received + struct JConnectionMemoryID* memoryID, ///< [in] for segment which should be copied + void* data ///< [out] received ); /// Wait until operations initiated at his connection finished. From d4f067344e08addc8263c602dcd12452a3dca044 Mon Sep 17 00:00:00 2001 From: Julian Benda Date: Fri, 2 Jul 2021 11:44:27 +0200 Subject: [PATCH 11/60] Adapt jmessage to jnetwork --- include/core/jmessage.h | 43 +++++++++--------- include/core/jnetwork.h | 15 ++++++- lib/core/jmessage.c | 96 +++++++++++++++++++++++------------------ 3 files changed, 91 insertions(+), 63 deletions(-) diff --git a/include/core/jmessage.h b/include/core/jmessage.h index eb6774819..ebd453bea 100644 --- a/include/core/jmessage.h +++ b/include/core/jmessage.h @@ -68,6 +68,7 @@ enum JMessageType typedef enum JMessageType JMessageType; struct JMessage; +struct JConnection; typedef struct JMessage JMessage; @@ -292,33 +293,30 @@ gpointer j_message_get_n(JMessage* message, gsize length); gchar const* j_message_get_string(JMessage* message); /** - * Writes a message to the network. + * Adds new data to send to a message. * * \code * \endcode * * \param message A message. - * \param stream A network stream. - * - * \return TRUE on success, FALSE if an error occurred. + * \param data Data. + * \param length A length. **/ -gboolean j_message_send(JMessage* message, gpointer stream); +void j_message_add_send(JMessage* message, gconstpointer data, guint64 length); /** - * Reads a message from the network. + * Adds a new operation to a message. * * \code * \endcode * * \param message A message. - * \param stream A network stream. - * - * \return TRUE on success, FALSE if an error occurred. + * \param length A length. **/ -gboolean j_message_receive(JMessage* message, gpointer stream); +void j_message_add_operation(JMessage* message, gsize length); /** - * Reads a message from the network. + * Writes a message to the network. * * \code * \endcode @@ -328,10 +326,10 @@ gboolean j_message_receive(JMessage* message, gpointer stream); * * \return TRUE on success, FALSE if an error occurred. **/ -gboolean j_message_read(JMessage* message, GInputStream* stream); +gboolean j_message_send(JMessage*, struct JConnection*); /** - * Writes a message to the network. + * Reads a message from the network. * * \code * \endcode @@ -341,30 +339,33 @@ gboolean j_message_read(JMessage* message, GInputStream* stream); * * \return TRUE on success, FALSE if an error occurred. **/ -gboolean j_message_write(JMessage* message, GOutputStream* stream); +gboolean j_message_receive(JMessage*, struct JConnection*); /** - * Adds new data to send to a message. + * Reads a message from the network. * * \code * \endcode * * \param message A message. - * \param data Data. - * \param length A length. + * \param stream A network stream. + * + * \return TRUE on success, FALSE if an error occurred. **/ -void j_message_add_send(JMessage* message, gconstpointer data, guint64 length); +gboolean j_message_read(JMessage*, struct JConnection*); /** - * Adds a new operation to a message. + * Writes a message to the network. * * \code * \endcode * * \param message A message. - * \param length A length. + * \param stream A network stream. + * + * \return TRUE on success, FALSE if an error occurred. **/ -void j_message_add_operation(JMessage* message, gsize length); +gboolean j_message_write(JMessage*, struct JConnection*); /** * Set the semantics of a message. diff --git a/include/core/jnetwork.h b/include/core/jnetwork.h index 6baa7992a..d0f1494de 100644 --- a/include/core/jnetwork.h +++ b/include/core/jnetwork.h @@ -206,6 +206,19 @@ struct JConnection; /** \public \memberof JConnection */ #define J_CONNECTION_MAX_RECV 1 +/// Acknowladgement value. +/** \public \memberof JConnection + * Value used to send an ack for any reason. Maybe to verify that the remote memory was successful readed. + * \sa j_connection_rma_unregister, j_connection_rma_read, j_connection_rma_register, JConnectionAck + */ +#define J_CONNECTION_ACK 42 + +/// type of ack value. +/** \public \memberof JConnection + * \sa J_CONNECTION_ACK + */ +typedef int JConnectionAck; + /// Handle for memory regions available via rma. /** \public \memberof JConnection * \sa j_connection_rma_read, j_connection_rma_register, j_connection_rma_unregister @@ -395,7 +408,7 @@ j_connection_wait_for_completion(struct JConnection* instance); */ gboolean j_connection_rma_register(struct JConnection* intsance, - void* data, ///< [in] begin of memory region to share + const void* data, ///< [in] begin of memory region to share size_t data_len, ///< [in] size of memory region in bytes JConnectionMemory* handle ///< [out] for memory region to unregister with j_connection_rma_unregister ); diff --git a/lib/core/jmessage.c b/lib/core/jmessage.c index c37a9aade..e9e5c2659 100644 --- a/lib/core/jmessage.c +++ b/lib/core/jmessage.c @@ -30,12 +30,15 @@ #include +#include #include #include #include #include #include +#define EXE(cmd, ...) do { if(cmd == FALSE) { g_warning(__VA_ARGS__); goto end; } } while(FALSE) + /** * \addtogroup JMessage Message * @@ -595,65 +598,54 @@ j_message_get_string(JMessage* message) } gboolean -j_message_receive(JMessage* message, gpointer connection) +j_message_receive(JMessage* message, struct JConnection* connection) { J_TRACE_FUNCTION(NULL); - GInputStream* stream; - g_return_val_if_fail(message != NULL, FALSE); g_return_val_if_fail(connection != NULL, FALSE); - stream = g_io_stream_get_input_stream(G_IO_STREAM(connection)); - return j_message_read(message, stream); + return j_message_read(message, connection); } gboolean -j_message_send(JMessage* message, gpointer connection) +j_message_send(JMessage* message, struct JConnection* connection) { J_TRACE_FUNCTION(NULL); gboolean ret; - GOutputStream* stream; - g_return_val_if_fail(message != NULL, FALSE); g_return_val_if_fail(connection != NULL, FALSE); - j_helper_set_cork(connection, TRUE); - - stream = g_io_stream_get_output_stream(G_IO_STREAM(connection)); - ret = j_message_write(message, stream); - - j_helper_set_cork(connection, FALSE); + ret = j_message_write(message, connection); return ret; } gboolean -j_message_read(JMessage* message, GInputStream* stream) +j_message_read(JMessage* message, struct JConnection* connection) { J_TRACE_FUNCTION(NULL); gboolean ret = FALSE; GError* error = NULL; - gsize bytes_read; g_return_val_if_fail(message != NULL, FALSE); - g_return_val_if_fail(stream != NULL, FALSE); + g_return_val_if_fail(connection != NULL, FALSE); - if (!g_input_stream_read_all(stream, &(message->header), sizeof(JMessageHeader), &bytes_read, NULL, &error) || bytes_read != sizeof(JMessageHeader)) - { - goto end; - } + EXE(j_connection_recv(connection, sizeof(message->header), &(message->header)), + "Failed to initiated header receive!"); + EXE(j_connection_wait_for_completion(connection), + "Failed to wait for header receive!"); j_message_ensure_size(message, j_message_length(message)); - if (!g_input_stream_read_all(stream, message->data, j_message_length(message), &bytes_read, NULL, &error) || bytes_read != j_message_length(message)) - { - goto end; - } + EXE(j_connection_recv(connection, j_message_length(message), message->data), + "Failed to initiated message body receive!"); + EXE(j_connection_wait_for_completion(connection), + "Failed to wait for message body receive!"); message->current = message->data; @@ -675,7 +667,7 @@ j_message_read(JMessage* message, GInputStream* stream) } gboolean -j_message_write(JMessage* message, GOutputStream* stream) +j_message_write(JMessage* message, struct JConnection* connection) { J_TRACE_FUNCTION(NULL); @@ -683,20 +675,22 @@ j_message_write(JMessage* message, GOutputStream* stream) g_autoptr(JListIterator) iterator = NULL; GError* error = NULL; - gsize bytes_written; + JConnectionMemory* memory_regions = NULL; + JConnectionMemory* memory_itr; + int n_memory_regions = 0; + JConnectionAck ack; g_return_val_if_fail(message != NULL, FALSE); - g_return_val_if_fail(stream != NULL, FALSE); + g_return_val_if_fail(connection != NULL, FALSE); - if (!g_output_stream_write_all(stream, &(message->header), sizeof(JMessageHeader), &bytes_written, NULL, &error) || bytes_written != sizeof(JMessageHeader)) - { - goto end; - } + EXE(j_connection_send(connection, &(message->header), sizeof(message->header)), + "Failed to initiated sending message header."); + EXE(j_connection_send(connection, message->data, j_message_length(message)), + "Failed to initiated sending message body."); - if (!g_output_stream_write_all(stream, message->data, j_message_length(message), &bytes_written, NULL, &error) || bytes_written != j_message_length(message)) - { - goto end; - } + n_memory_regions = j_list_length(message->send_list); + memory_regions = calloc(sizeof(JConnectionMemory), n_memory_regions); + memory_itr = memory_regions; if (message->send_list != NULL) { @@ -706,14 +700,26 @@ j_message_write(JMessage* message, GOutputStream* stream) { JMessageData* message_data = j_list_iterator_get(iterator); - if (!g_output_stream_write_all(stream, message_data->data, message_data->length, &bytes_written, NULL, &error)) - { - goto end; - } + EXE(j_connection_rma_register(connection, message_data->data, message_data->length, memory_itr), + "Failed to register message data memory!"); + ++memory_itr; } } + EXE(j_connection_recv(connection, sizeof(ack), &ack), + "Failed to initiated ACK receive."); + EXE(j_connection_wait_for_completion(connection), + "Failed to wait for completion of message send!"); + if(ack != J_CONNECTION_ACK) { + g_warning("Wrong ACK flag received! got: %i, instead of: %i", ack, J_CONNECTION_ACK); + goto end; + } - g_output_stream_flush(stream, NULL, NULL); + for(memory_itr = memory_regions; memory_itr - memory_regions < n_memory_regions; ++memory_itr) + { + EXE(j_connection_rma_unregister(connection, memory_itr), + "Failed to unregister message data memory!"); + } + free(memory_regions); ret = TRUE; @@ -724,6 +730,14 @@ j_message_write(JMessage* message, GOutputStream* stream) g_error_free(error); } + if(memory_regions != NULL) { + JConnectionMemory* mrs = memory_regions; + while(memory_regions != memory_itr) { + j_connection_rma_unregister(connection, memory_regions); + } + free(mrs); + } + return ret; } From d04b2b03ae447879e9ebbab274a14f602c74b26e Mon Sep 17 00:00:00 2001 From: Julian Benda Date: Wed, 7 Jul 2021 15:21:09 +0200 Subject: [PATCH 12/60] Introduce new interface in loop and jobject * start implementing jfabric functions --- include/core/jconfiguration.h | 4 + include/core/jmessage.h | 27 +++-- include/core/jnetwork.h | 28 ++--- lib/core/jconfiguration.c | 33 ++++++ lib/core/jmessage.c | 115 +++++++++++------- lib/core/jnetwork.c | 193 +++++++++++++++++++++++++++++++ lib/object/jdistributed-object.c | 5 +- lib/object/jobject.c | 22 ++-- server/loop.c | 23 ++-- server/server.c | 19 ++- server/server.h | 3 +- 11 files changed, 373 insertions(+), 99 deletions(-) diff --git a/include/core/jconfiguration.h b/include/core/jconfiguration.h index b7cb2ade3..bf6aaad00 100644 --- a/include/core/jconfiguration.h +++ b/include/core/jconfiguration.h @@ -40,6 +40,7 @@ G_BEGIN_DECLS **/ struct JConfiguration; +struct fi_info; typedef struct JConfiguration JConfiguration; @@ -116,6 +117,9 @@ guint64 j_configuration_get_stripe_size(JConfiguration*); /// network port to communicate guint16 j_configuration_get_port(JConfiguration*); +gint64 j_configuration_get_libfabric_version(JConfiguration*); +struct fi_info* j_configuration_get_libfabric_hints(JConfiguration*); + G_END_DECLS /** diff --git a/include/core/jmessage.h b/include/core/jmessage.h index ebd453bea..f1aded100 100644 --- a/include/core/jmessage.h +++ b/include/core/jmessage.h @@ -68,13 +68,13 @@ enum JMessageType typedef enum JMessageType JMessageType; struct JMessage; -struct JConnection; typedef struct JMessage JMessage; G_END_DECLS #include +#include G_BEGIN_DECLS @@ -229,6 +229,9 @@ gboolean j_message_append_n(JMessage* message, gconstpointer data, gsize length) **/ gboolean j_message_append_string(JMessage* message, gchar const* str); +/// Appends a memory identifier to a message +gboolean j_message_append_memory_id(JMessage*, const struct JConnectionMemoryID*); + /** * Gets 1 byte from a message. * @@ -292,17 +295,19 @@ gpointer j_message_get_n(JMessage* message, gsize length); **/ gchar const* j_message_get_string(JMessage* message); +/// Gets an memory identifier from a message. +const struct JConnectionMemoryID* j_message_get_memory_id(JMessage*); + /** * Adds new data to send to a message. * * \code * \endcode * - * \param message A message. - * \param data Data. - * \param length A length. + * \param data,length Data segment to send. + * \param header,h_size header data (included in message) **/ -void j_message_add_send(JMessage* message, gconstpointer data, guint64 length); +void j_message_add_send(JMessage* this, gconstpointer data, guint64 length, void* header, guint64 h_size); /** * Adds a new operation to a message. @@ -326,6 +331,8 @@ void j_message_add_operation(JMessage* message, gsize length); * * \return TRUE on success, FALSE if an error occurred. **/ + +/** Append sendend data memory ids to end of message! */ gboolean j_message_send(JMessage*, struct JConnection*); /** @@ -340,6 +347,9 @@ gboolean j_message_send(JMessage*, struct JConnection*); * \return TRUE on success, FALSE if an error occurred. **/ gboolean j_message_receive(JMessage*, struct JConnection*); +/// signal when rma read actions are finished. +/** Sends ACK flag, to signal that host can free data memory. Also wait for all network actions to coplete!*/ +gboolean j_message_send_ack(JMessage*, struct JConnection*); /** * Reads a message from the network. @@ -354,9 +364,9 @@ gboolean j_message_receive(JMessage*, struct JConnection*); **/ gboolean j_message_read(JMessage*, struct JConnection*); -/** - * Writes a message to the network. +/** Writes a message to the network. * + * \attention send without data segment! * \code * \endcode * @@ -367,8 +377,7 @@ gboolean j_message_read(JMessage*, struct JConnection*); **/ gboolean j_message_write(JMessage*, struct JConnection*); -/** - * Set the semantics of a message. +/** Set the semantics of a message. * * \code * \endcode diff --git a/include/core/jnetwork.h b/include/core/jnetwork.h index d0f1494de..5467a81d1 100644 --- a/include/core/jnetwork.h +++ b/include/core/jnetwork.h @@ -87,8 +87,8 @@ struct JFabric; /// Initelize a fabric for the server side. /** \public \memberof JFabric * - * In differents to j_fabric_init_client, the resulting fabric is capable of reciving connection requests. - * There fore events can be readed via j_fabric_sread_event + * In differents to j_fabric_init_client(), the resulting fabric is capable of reciving connection requests. + * There fore events can be readed via j_fabric_sread_event() * \retval FALSE on failure */ gboolean @@ -98,7 +98,7 @@ j_fabric_init_server( ); /// Initelize a fabric for the client side. -/** \pulbic \memberof JFabric +/** \public \memberof JFabric * * Contains data to building a paired connection. * Addess of JULEA server is needed, to enforce that both communicate via the same network. @@ -223,7 +223,7 @@ typedef int JConnectionAck; /** \public \memberof JConnection * \sa j_connection_rma_read, j_connection_rma_register, j_connection_rma_unregister */ -typedef void* JConnectionMemory; +struct JConnectionMemory; /// Identifier to read memory. /** \public \memberof JConnectionMemory @@ -238,10 +238,10 @@ struct JConnectionMemoryID { /// Get identifier of memory region /** \public \memberof JConnectionMemory * \retval FALSE on failure - * \sa j_connection_rma_read, j_connection_rma_register + * \sa j_connection_rma_read, j_connection_rma_register, j_connection_rma_unregister */ gboolean -j_connection_memory_get_id(JConnectionMemory instance, +j_connection_memory_get_id(struct JConnectionMemory* instance, struct JConnectionMemoryID* id ///< [out] of registerd memory ); @@ -358,8 +358,8 @@ j_connection_read_event(struct JConnection* instance, */ gboolean j_connection_send(struct JConnection* instance, - void* data, ///< [in] to send - size_t data_len ///< [in] in bytes + const void* data, ///< [in] to send + size_t data_len ///< [in] in bytes ); /// Asynchronous receive data via MSG connection. @@ -389,8 +389,8 @@ j_connection_recv(struct JConnection* instance, */ gboolean j_connection_rma_read(struct JConnection* instance, - struct JConnectionMemoryID* memoryID, ///< [in] for segment which should be copied - void* data ///< [out] received + const struct JConnectionMemoryID* memoryID, ///< [in] for segment which should be copied + void* data ///< [out] received ); /// Wait until operations initiated at his connection finished. @@ -408,9 +408,9 @@ j_connection_wait_for_completion(struct JConnection* instance); */ gboolean j_connection_rma_register(struct JConnection* intsance, - const void* data, ///< [in] begin of memory region to share - size_t data_len, ///< [in] size of memory region in bytes - JConnectionMemory* handle ///< [out] for memory region to unregister with j_connection_rma_unregister + const void* data, ///< [in] begin of memory region to share + size_t data_len, ///< [in] size of memory region in bytes + struct JConnectionMemory* handle ///< [out] for memory region to unregister with j_connection_rma_unregister ); /// Unregister memory from rma availablity. @@ -419,7 +419,7 @@ j_connection_rma_register(struct JConnection* intsance, */ gboolean j_connection_rma_unregister(struct JConnection* instance, - JConnectionMemory* handle ///< [in] for memory region to unregister + struct JConnectionMemory* handle ///< [in] for memory region to unregister ); #endif diff --git a/lib/core/jconfiguration.c b/lib/core/jconfiguration.c index 094501d99..a7d80661c 100644 --- a/lib/core/jconfiguration.c +++ b/lib/core/jconfiguration.c @@ -23,6 +23,7 @@ #include #include +#include #include @@ -140,6 +141,12 @@ struct JConfiguration gchar* path; } db; + /// libfabric configuration + struct { + int32_t version; + struct fi_info* hints; + } libfabric; + guint64 max_operation_size; guint16 port; @@ -281,6 +288,7 @@ j_configuration_new_for_data(GKeyFile* key_file) gchar* db_backend; gchar* db_component; gchar* db_path; + gchar* libfabric_provider; guint64 max_operation_size; guint32 port; guint32 max_connections; @@ -304,6 +312,7 @@ j_configuration_new_for_data(GKeyFile* key_file) db_backend = g_key_file_get_string(key_file, "db", "backend", NULL); db_component = g_key_file_get_string(key_file, "db", "component", NULL); db_path = g_key_file_get_string(key_file, "db", "path", NULL); + libfabric_provider = g_key_file_get_string(key_file, "libfabric", "provider", NULL); /// \todo check value ranges (max_operation_size, port, max_connections, stripe_size) // configuration->port < 0 || configuration->port > 65535 @@ -331,6 +340,7 @@ j_configuration_new_for_data(GKeyFile* key_file) g_free(object_backend); g_free(object_component); g_free(object_path); + g_free(libfabric_provider); g_strfreev(servers_object); g_strfreev(servers_kv); g_strfreev(servers_db); @@ -359,6 +369,15 @@ j_configuration_new_for_data(GKeyFile* key_file) configuration->max_connections = max_connections; configuration->stripe_size = stripe_size; configuration->ref_count = 1; + configuration->libfabric.version = FI_VERSION(1, 11); + configuration->libfabric.hints = fi_allocinfo(); + configuration->libfabric.hints->caps = + FI_MSG | FI_SEND | FI_RECV | FI_READ | FI_RMA | FI_REMOTE_READ; + configuration->libfabric.hints->mode = FI_MSG_PREFIX; + configuration->libfabric.hints->domain_attr->mr_mode = + FI_MR_LOCAL | FI_MR_ALLOCATED | FI_MR_PROV_KEY | FI_MR_VIRT_ADDR; + configuration->libfabric.hints->ep_attr->type = FI_EP_MSG; + configuration->libfabric.hints->fabric_attr->prov_name = libfabric_provider; if (configuration->max_operation_size == 0) { @@ -422,6 +441,9 @@ j_configuration_unref(JConfiguration* configuration) g_strfreev(configuration->servers.kv); g_strfreev(configuration->servers.db); + g_free(configuration->libfabric.hints->fabric_attr->prov_name); + fi_freeinfo(configuration->libfabric.hints); + g_slice_free(JConfiguration, configuration); } } @@ -579,6 +601,17 @@ j_configuration_get_port(JConfiguration* configuration) return configuration->port; } +struct fi_info* +j_configuration_get_libfabric_hints(JConfiguration* configuration) +{ +} + +guint64 +j_configuration_get_libfabric_version(JConfiguration* configuration) +{ + +} + /** * @} **/ diff --git a/lib/core/jmessage.c b/lib/core/jmessage.c index e9e5c2659..f4efce8f3 100644 --- a/lib/core/jmessage.c +++ b/lib/core/jmessage.c @@ -85,6 +85,9 @@ struct JMessageData * The data length. **/ guint64 length; + + void* header; + guint64 header_size; }; typedef struct JMessageData JMessageData; @@ -608,18 +611,81 @@ j_message_receive(JMessage* message, struct JConnection* connection) return j_message_read(message, connection); } +gboolean +j_message_send_ack(JMessage* message, struct JConnection* connection) +{ + const JConnectionAck ack = J_CONNECTION_ACK; + + // No data where send -> no need to acknowledge + if(j_message_get_count(message) == 0) { return TRUE; } + + EXE(j_connection_wait_for_completion(connection), + "Failed to wait to finishe all operations before sending ack!"); + EXE(j_connection_send(connection, &ack, sizeof(ack)), + "Failed to initiated sending ACK!"); + EXE(j_connection_wait_for_completion(connection), + "Failed to verify completion of sending ACK!"); + return TRUE; +end: + return FALSE; +} + gboolean j_message_send(JMessage* message, struct JConnection* connection) { J_TRACE_FUNCTION(NULL); gboolean ret; + int n_memory_regions = 0; + JConnectionMemory* memory_itr = NULL; + JConnectionMemory* memory_regions = NULL; + struct JConnectionMemoryID mem_id; + g_autoptr(JListIterator) iterator = NULL; g_return_val_if_fail(message != NULL, FALSE); g_return_val_if_fail(connection != NULL, FALSE); + ret = FALSE; + + n_memory_regions = j_list_length(message->send_list); + memory_regions = calloc(sizeof(JConnectionMemory), n_memory_regions); + memset(memory_regions, 0, sizeof(JConnectionMemory) * n_memory_regions); + memory_itr = memory_regions; + + if (message->send_list != NULL) + { + iterator = j_list_iterator_new(message->send_list); + + while (j_list_iterator_next(iterator)) + { + JMessageData* message_data = j_list_iterator_get(iterator); + + EXE(j_connection_rma_register(connection, message_data->data, message_data->length, memory_itr), + "Failed to register message data memory!"); + + j_message_add_operation(message, sizeof(struct JConnectionMemoryID) + message_data->header_size); + + EXE(j_connection_memory_get_id(connection, &mem_id), + "Failed to get memory it!"); + EXE(j_message_append_n(message, message_data->header, message_data->header_size), + "Failed to append header"); + EXE(j_message_append_memory_id(message, &mem_id), + "Failed to append memory id to message!"); + ++memory_itr; + } + } + ret = j_message_write(message, connection); +end: + if(memory_regions != NULL) { + JConnectionMemory* mrs = memory_regions; + while(memory_regions != memory_itr) { + j_connection_rma_unregister(connection, memory_regions); + } + if(*memory_itr) { j_connection_rma_unregister(connection, memory_itr); } + free(mrs); + } return ret; } @@ -673,11 +739,7 @@ j_message_write(JMessage* message, struct JConnection* connection) gboolean ret = FALSE; - g_autoptr(JListIterator) iterator = NULL; GError* error = NULL; - JConnectionMemory* memory_regions = NULL; - JConnectionMemory* memory_itr; - int n_memory_regions = 0; JConnectionAck ack; g_return_val_if_fail(message != NULL, FALSE); @@ -688,39 +750,19 @@ j_message_write(JMessage* message, struct JConnection* connection) EXE(j_connection_send(connection, message->data, j_message_length(message)), "Failed to initiated sending message body."); - n_memory_regions = j_list_length(message->send_list); - memory_regions = calloc(sizeof(JConnectionMemory), n_memory_regions); - memory_itr = memory_regions; - - if (message->send_list != NULL) - { - iterator = j_list_iterator_new(message->send_list); - - while (j_list_iterator_next(iterator)) - { - JMessageData* message_data = j_list_iterator_get(iterator); - - EXE(j_connection_rma_register(connection, message_data->data, message_data->length, memory_itr), - "Failed to register message data memory!"); - ++memory_itr; - } + if(message->send_list != NULL) { + EXE(j_connection_recv(connection, sizeof(ack), &ack), + "Failed to initiated ACK receive."); } - EXE(j_connection_recv(connection, sizeof(ack), &ack), - "Failed to initiated ACK receive."); + EXE(j_connection_wait_for_completion(connection), "Failed to wait for completion of message send!"); - if(ack != J_CONNECTION_ACK) { + + if(message->send_list != NULL && ack != J_CONNECTION_ACK) { g_warning("Wrong ACK flag received! got: %i, instead of: %i", ack, J_CONNECTION_ACK); goto end; } - for(memory_itr = memory_regions; memory_itr - memory_regions < n_memory_regions; ++memory_itr) - { - EXE(j_connection_rma_unregister(connection, memory_itr), - "Failed to unregister message data memory!"); - } - free(memory_regions); - ret = TRUE; end: @@ -729,20 +771,11 @@ j_message_write(JMessage* message, struct JConnection* connection) g_critical("%s", error->message); g_error_free(error); } - - if(memory_regions != NULL) { - JConnectionMemory* mrs = memory_regions; - while(memory_regions != memory_itr) { - j_connection_rma_unregister(connection, memory_regions); - } - free(mrs); - } - return ret; } void -j_message_add_send(JMessage* message, gconstpointer data, guint64 length) +j_message_add_send(JMessage* message, gconstpointer data, guint64 length, void* header, guint64 h_size) { J_TRACE_FUNCTION(NULL); @@ -755,6 +788,8 @@ j_message_add_send(JMessage* message, gconstpointer data, guint64 length) message_data = g_slice_new(JMessageData); message_data->data = data; message_data->length = length; + message_data->header = header; + message_data->header_size = h_size; j_list_append(message->send_list, message_data); } diff --git a/lib/core/jnetwork.c b/lib/core/jnetwork.c index 15b0655a1..dab1da44a 100644 --- a/lib/core/jnetwork.c +++ b/lib/core/jnetwork.c @@ -2,6 +2,11 @@ #include +#include +#include +#include +#include + /// Used to initialize common parts between different connection. /** This will create the following libfabric resources: * * the domain @@ -24,3 +29,191 @@ j_connection_init(struct JConnection* instance); gboolean j_connection_create_memory_resources(struct JConnection* instance); +/// flag used to different between client and server fabric +enum JFabricSide { + JF_SERVER, + JF_CLIENT +}; + +struct JFabric { + struct fi_info* info; + struct fid_fabric* fabric; + struct fid_eq* pep_eq; + struct fid_pep* pep; + struct JConfiguration* config; + enum JFabricSide con_side; +}; + +struct JConnection { + struct jfabric* fabric; + struct fi_info* info; + struct fid_domain* domain; + struct fid_ep* ep; + struct fid_eq* eq; + struct fid_cq* cq; + int inject_size; + struct { + struct fid_mr* mr; + void* buffer; + size_t buffer_size; + size_t rx_prefix_size; + size_t tx_prefix_size; + } rx_mem, tx_mem; +}; + +struct JConnectionMemory { + struct fid_mr* memory_region; + void* addr; + guint64 size; +}; + +struct JFabricAddr { + void* addr; + uint64_t addr_len; + uint32_t addr_format; +}; + +struct JFabricConnectionRequest { + struct fi_eq_cm_entry request_entry; +}; + +#define EXE(cmd, ...) do { if (cmd == FALSE) { g_warning(__VA_ARGS__); goto end; } } while(FALSE) +#define CHECK(msg) do { if (res < 0) { g_warning("%s: "msg"\nDetails:\t%s", "??TODO??", fi_strerror(-res)); goto end; } } while(FALSE) +void free_dangeling_infos(struct fi_info* info) { + fi_freeinfo(info->next); + info->next = NULL; +} + +gboolean +j_fabric_init_server(struct JConfiguration* configuration, struct JFabric** instnace_ptr) +{ + struct JFabric* this; + int res; + + *instnace_ptr = malloc(sizeof(*this)); + this = *instnace_ptr; + memset(this, 0, sizeof(*this)); + + this->config = configuration; + this->con_side = JF_SERVER; + + res = fi_getinfo( + j_configuration_get_libfabric_version(this->config), + NULL, NULL, FI_SOURCE, + j_configuration_get_libfabric_hints(this->config), + &this->info); + CHECK("Failed to find fabric for server!"); + free_dangeling_infos(this->info); + + // no context needed, because we are in sync + res = fi_fabric(this->info->fabric_attr, &this->fabric, NULL); + CHECK("Failed to open server fabric!"); + + // wait obj defined to use synced actions + res = fi_eq_open(this->fabric, + &(struct fi_eq_attr){.wait_obj = FI_WAIT_UNSPEC}, + &this->pep_eq, NULL); + CHECK("failed to create eq for fabric!"); + res= fi_passive_ep(this->fabric, this->info, &this->pep, NULL); + CHECK("failed to create pep for fabric!"); + res = fi_pep_bind(this->pep, &this->pep_eq->fid, 0); + CHECK("failed to bind event queue to pep!"); + res = fi_listen(this->pep); + CHECK("failed to start listening on pep!"); + + return TRUE; +end: + return FALSE; +} + +gboolean +j_fabric_init_client(struct JConfiguration* configuration, struct JFabricAddr* addr, struct JFabric** instance_ptr) +{ + struct JFabric* this; + struct fi_info* hints; + int res; + gboolean ret = FALSE; + + *instance_ptr = malloc(sizeof(*this)); + this = *instance_ptr; + memset(this, 0, sizeof(*this)); + + this->config = configuration; + this->con_side = JF_CLIENT; + + hints = fi_dupinfo(j_configuration_get_libfabric_hints(configuration)); + hints->addr_format = addr->addr_format; + hints->dest_addr = addr->addr; + hints->dest_addrlen = addr->addr_len; + + res = fi_getinfo( + j_configuration_get_libfabric_version(configuration), + NULL, NULL, 0, + hints, &this->info); + CHECK("Failed to find fabric!"); + free_dangeling_infos(this->info); + + res = fi_fabric(this->info->fabric_attr, &this->fabric, NULL); + CHECK("failed to initelize client fabric!"); + + ret = TRUE; +end: + fi_freeinfo(hints); + return ret; +} + +gboolean +j_fabric_fini(struct JFabric* this) +{ + int res; + + fi_freeinfo(this->info); + this->info = NULL; + if(this->con_side == JF_SERVER) { + res = fi_close(&this->pep->fid); + CHECK("Failed to close PEP!"); + this->pep = NULL; + + res = fi_close(&this->pep_eq->fid); + CHECK("Failed to close EQ for PEP!"); + this->pep_eq = NULL; + } + + res = fi_close(&this->fabric->fid); + CHECK("failed to close fabric!"); + this->fabric = NULL; + free(this); + return TRUE; +end: + return FALSE; +} + +///! \todo read error! +gboolean +j_fabric_sread_event(struct JFabric* this, int timeout, enum JFabricEvents* event, struct JFabricConnectionRequest* con_req) +{ + int res; + gboolean ret = FALSE; + uint32_t fi_event; + + res = fi_eq_sread(this->pep_eq, &fi_event, + &con_req->request_entry, sizeof(con_req->request_entry), + timeout, 0); + if (res == -FI_EAGAIN) { + *event = J_FABRIC_EVENT_TIMEOUT; + ret = TRUE; goto end; + } else if (res == -FI_EAVAIL) { + *event = J_FABRIC_EVENT_ERROR; + // TODO: fetch error! + ret = TRUE; goto end; + } + CHECK("failed to read pep event queue!"); + switch(fi_event) { + case FI_CONNREQ: *event = J_FABRIC_EVENT_CONNECTION_REQUEST; break; + case FI_SHUTDOWN: *event = J_FABRIC_EVENT_SHUTDOWN; break; + default: g_assert_not_reached(); goto end; + } + ret = TRUE; +end: + return ret; +} diff --git a/lib/object/jdistributed-object.c b/lib/object/jdistributed-object.c index d2f8d2fb8..c6ce68ce0 100644 --- a/lib/object/jdistributed-object.c +++ b/lib/object/jdistributed-object.c @@ -1056,10 +1056,7 @@ j_distributed_object_write_exec(JList* operations, JSemantics* semantics) bw_lists[index] = j_list_new(NULL); } - j_message_add_operation(messages[index], sizeof(guint64) + sizeof(guint64)); - j_message_append_8(messages[index], &new_length); - j_message_append_8(messages[index], &new_offset); - j_message_add_send(messages[index], new_data, new_length); + j_message_add_send(messages[index], new_data, new_length, &new_offset, sizeof(new_offset)); j_list_append(bw_lists[index], bytes_written); diff --git a/lib/object/jobject.c b/lib/object/jobject.c index 52c57819c..1d28fb2b0 100644 --- a/lib/object/jobject.c +++ b/lib/object/jobject.c @@ -513,7 +513,7 @@ j_object_read_exec(JList* operations, JSemantics* semantics) if (object_backend == NULL) { g_autoptr(JMessage) reply = NULL; - gpointer object_connection; + struct JConnection* object_connection; guint32 operations_done; guint32 operation_count; @@ -549,23 +549,20 @@ j_object_read_exec(JList* operations, JSemantics* semantics) for (guint i = 0; i < reply_operation_count && j_list_iterator_next(it); i++) { JObjectOperation* operation = j_list_iterator_get(it); + const struct JConnectionMemoryID* mem_id; gpointer data = operation->read.data; guint64* bytes_read = operation->read.bytes_read; - guint64 nbytes; - - nbytes = j_message_get_8(reply); - j_helper_atomic_add(bytes_read, nbytes); + mem_id = j_message_get_memory_id(reply); + j_helper_atomic_add(bytes_read, mem_id->size); - if (nbytes > 0) + if (mem_id->size > 0) { - GInputStream* input; - - input = g_io_stream_get_input_stream(G_IO_STREAM(object_connection)); - g_input_stream_read_all(input, data, nbytes, NULL, NULL, NULL); + j_connection_rma_read(object_connection, mem_id, data); } } + j_message_send_ack(reply, object_connection); operations_done += reply_operation_count; } @@ -667,10 +664,7 @@ j_object_write_exec(JList* operations, JSemantics* semantics) if (object_backend == NULL) { - j_message_add_operation(message, sizeof(guint64) + sizeof(guint64)); - j_message_append_8(message, &length); - j_message_append_8(message, &offset); - j_message_add_send(message, data, length); + j_message_add_send(message, data, length, &operation->write.offset, sizeof(operation->write.offset)); // Fake bytes_written here instead of doing another loop further down if (j_semantics_get(semantics, J_SEMANTICS_SAFETY) == J_SEMANTICS_SAFETY_NONE) diff --git a/server/loop.c b/server/loop.c index 33adbc1c1..a5ed41f84 100644 --- a/server/loop.c +++ b/server/loop.c @@ -29,7 +29,7 @@ static guint jd_thread_num = 0; gboolean -jd_handle_message(JMessage* message, GSocketConnection* connection, JMemoryChunk* memory_chunk, guint64 memory_chunk_size, JStatistics* statistics) +jd_handle_message(JMessage* message, struct JConnection* connection, JMemoryChunk* memory_chunk, guint64 memory_chunk_size, JStatistics* statistics) { J_TRACE_FUNCTION(NULL); @@ -219,6 +219,7 @@ jd_handle_message(JMessage* message, GSocketConnection* connection, JMemoryChunk if (length > memory_chunk_size) { /// \todo return proper error + g_warning("memory object don't fit in memory chunk!"); j_message_add_operation(reply, sizeof(guint64)); j_message_append_8(reply, &bytes_read); continue; @@ -241,12 +242,9 @@ jd_handle_message(JMessage* message, GSocketConnection* connection, JMemoryChunk j_backend_object_read(jd_object_backend, object, buf, length, offset, &bytes_read); j_statistics_add(statistics, J_STATISTICS_BYTES_READ, bytes_read); - j_message_add_operation(reply, sizeof(guint64)); - j_message_append_8(reply, &bytes_read); - if (bytes_read > 0) { - j_message_add_send(reply, buf, bytes_read); + j_message_add_send(reply, buf, bytes_read, NULL, 0); } j_statistics_add(statistics, J_STATISTICS_BYTES_SENT, bytes_read); @@ -281,16 +279,15 @@ jd_handle_message(JMessage* message, GSocketConnection* connection, JMemoryChunk for (i = 0; i < operation_count; i++) { - GInputStream* input; gchar* buf; - guint64 length; + const struct JConnectionMemoryID* memoryID; guint64 offset; guint64 bytes_written = 0; - length = j_message_get_8(message); + memoryID = j_message_get_memory_id(message); offset = j_message_get_8(message); - if (length > memory_chunk_size && reply != NULL && G_LIKELY(ret)) + if (memoryID->size > memory_chunk_size && reply != NULL && G_LIKELY(ret)) { /// \todo return proper error j_message_add_operation(reply, sizeof(guint64)); @@ -299,12 +296,12 @@ jd_handle_message(JMessage* message, GSocketConnection* connection, JMemoryChunk } // Guaranteed to work because memory_chunk is reset below - buf = j_memory_chunk_get(memory_chunk, length); + buf = j_memory_chunk_get(memory_chunk, memoryID->size); g_assert(buf != NULL); - input = g_io_stream_get_input_stream(G_IO_STREAM(connection)); - g_input_stream_read_all(input, buf, length, NULL, NULL, NULL); - j_statistics_add(statistics, J_STATISTICS_BYTES_RECEIVED, length); + j_connection_rma_read(connection, memoryID, buf); + j_statistics_add(statistics, J_STATISTICS_BYTES_RECEIVED, memoryID->size); + // CONTINUE if (G_LIKELY(ret)) { diff --git a/server/server.c b/server/server.c index fea94cc8e..ba9e1f9aa 100644 --- a/server/server.c +++ b/server/server.c @@ -42,6 +42,8 @@ JBackend* jd_db_backend = NULL; static JConfiguration* jd_configuration = NULL; +static struct JFabric* jd_fabric = NULL; + static gboolean jd_signal(gpointer data) { @@ -58,7 +60,7 @@ jd_signal(gpointer data) } static gboolean -jd_on_run(GThreadedSocketService* service, GSocketConnection* connection, GObject* source_object, gpointer user_data) +jd_on_run(GThreadedSocketService* service, GSocketConnection* gconnection, GObject* source_object, gpointer user_data) { J_TRACE_FUNCTION(NULL); @@ -66,12 +68,13 @@ jd_on_run(GThreadedSocketService* service, GSocketConnection* connection, GObjec g_autoptr(JMessage) message = NULL; JStatistics* statistics; guint64 memory_chunk_size; + struct JConnection* jconnection; (void)service; (void)source_object; (void)user_data; - j_helper_set_nodelay(connection, TRUE); + j_helper_set_nodelay(gconnection, TRUE); statistics = j_statistics_new(TRUE); memory_chunk_size = j_configuration_get_max_operation_size(jd_configuration); @@ -79,9 +82,11 @@ jd_on_run(GThreadedSocketService* service, GSocketConnection* connection, GObjec message = j_message_new(J_MESSAGE_NONE, 0); - while (j_message_receive(message, connection)) + j_connection_init_server(jd_configuration, jd_fabric, gconnection, &jconnection); + + while (j_message_receive(message, jconnection)) { - jd_handle_message(message, connection, memory_chunk, memory_chunk_size, statistics); + jd_handle_message(message, jconnection, memory_chunk, memory_chunk_size, statistics); } { @@ -374,6 +379,12 @@ main(int argc, char** argv) g_debug("Initialized db backend %s.", db_backend); } + if(!j_fabric_init_server(jd_configuration, &jd_fabric)) + { + g_warning("Failed to initialize server fabric!"); + return 1; + } + jd_statistics = j_statistics_new(FALSE); g_mutex_init(jd_statistics_mutex); diff --git a/server/server.h b/server/server.h index 522c75f62..a128fc14b 100644 --- a/server/server.h +++ b/server/server.h @@ -25,6 +25,7 @@ #include #include #include +#include #include G_GNUC_INTERNAL extern JStatistics* jd_statistics; @@ -34,6 +35,6 @@ G_GNUC_INTERNAL extern JBackend* jd_object_backend; G_GNUC_INTERNAL extern JBackend* jd_kv_backend; G_GNUC_INTERNAL extern JBackend* jd_db_backend; -G_GNUC_INTERNAL gboolean jd_handle_message(JMessage*, GSocketConnection*, JMemoryChunk*, guint64, JStatistics*); +G_GNUC_INTERNAL gboolean jd_handle_message(JMessage*, struct JConnection*, JMemoryChunk*, guint64, JStatistics*); #endif From 1968022200eb0b4a8eb78f1aa2d980d906ca4797 Mon Sep 17 00:00:00 2001 From: Julian Benda Date: Wed, 7 Jul 2021 17:43:18 +0200 Subject: [PATCH 13/60] It's compiled (what is already coded) --- include/core/jnetwork.h | 9 +- lib/core/jconfiguration.c | 5 +- lib/core/jconnection-pool.c | 46 +++---- lib/core/jmessage.c | 20 ++- lib/core/jnetwork.c | 254 ++++++++++++++++++++++++++++++++++-- lib/db/jdb-internal.c | 2 +- meson.build | 2 +- server/server.c | 2 +- 8 files changed, 281 insertions(+), 59 deletions(-) diff --git a/include/core/jnetwork.h b/include/core/jnetwork.h index 5467a81d1..f616eecd7 100644 --- a/include/core/jnetwork.h +++ b/include/core/jnetwork.h @@ -32,6 +32,7 @@ #include struct JConfiguration; +struct fi_eq_cm_entry; /** \defgroup network Network * Interface for communication over network @@ -44,9 +45,10 @@ struct JConfiguration; * \sa j_connection_init_server, j_fabric_sread_event * \ingroup network */ -struct JFabricConnectionRequest; +typedef struct fi_eq_cm_entry JFabricConnectionRequest; /** \struct JFabricAddr jnetwork.h + * \public \memberof * \brief Fabrics address data needed to send an connection request. * \ingroup network * \sa j_fabric_init_client @@ -130,7 +132,7 @@ gboolean j_fabric_sread_event(struct JFabric* instance, int timeout, ///< [in] set to -1 for no timeout. enum JFabricEvents* event, ///< [out] reeded from event queue - struct JFabricConnectionRequest* con_req ///< [out] contains connection request, + JFabricConnectionRequest* con_req ///< [out] contains connection request, /// if event == J_FABRIC_EVENT_CONNECTION_REQUEST ); @@ -267,11 +269,13 @@ j_connection_memory_get_id(struct JConnectionMemory* instance, * \endmsc * \retval FALSE if building a connection failed * \sa j_connection_init_server + * \todo document index parameter! */ gboolean j_connection_init_client ( struct JConfiguration* configuration, ///< [in] for server address, and network configuration enum JBackendType backend, ///< [in] backend server to connect to + guint index, ///< [in] ?? struct JConnection** instance ///< [out] constructed instance ); @@ -290,7 +294,6 @@ j_connection_init_client ( */ gboolean j_connection_init_server ( - struct JConfiguration* configuration, ///< [in] for the network configuration struct JFabric* fabric, ///< [in] via which the connection should be established GSocketConnection* gconnection, ///< [in] valid GSocketConnection for address exchange struct JConnection** instance ///< [out] constructed instance diff --git a/lib/core/jconfiguration.c b/lib/core/jconfiguration.c index a7d80661c..1b6c3576b 100644 --- a/lib/core/jconfiguration.c +++ b/lib/core/jconfiguration.c @@ -604,12 +604,13 @@ j_configuration_get_port(JConfiguration* configuration) struct fi_info* j_configuration_get_libfabric_hints(JConfiguration* configuration) { + return configuration->libfabric.hints; } -guint64 +gint64 j_configuration_get_libfabric_version(JConfiguration* configuration) { - + return configuration->libfabric.version; } /** diff --git a/lib/core/jconnection-pool.c b/lib/core/jconnection-pool.c index 7998237a0..46f3385c9 100644 --- a/lib/core/jconnection-pool.c +++ b/lib/core/jconnection-pool.c @@ -167,12 +167,12 @@ j_connection_pool_fini(void) g_slice_free(JConnectionPool, pool); } -static GSocketConnection* -j_connection_pool_pop_internal(GAsyncQueue* queue, guint* count, gchar const* server) +static struct JConnection* +j_connection_pool_pop_internal(GAsyncQueue* queue, guint* count, enum JBackendType backend, guint64 index) { J_TRACE_FUNCTION(NULL); - GSocketConnection* connection; + struct JConnection* connection; g_return_val_if_fail(queue != NULL, NULL); g_return_val_if_fail(count != NULL, NULL); @@ -188,30 +188,16 @@ j_connection_pool_pop_internal(GAsyncQueue* queue, guint* count, gchar const* se { if ((guint)g_atomic_int_add(count, 1) < j_connection_pool->max_count) { - GError* error = NULL; - g_autoptr(GSocketClient) client = NULL; - g_autoptr(JMessage) message = NULL; g_autoptr(JMessage) reply = NULL; guint op_count; - client = g_socket_client_new(); - connection = g_socket_client_connect_to_host(client, server, j_configuration_get_port(j_configuration()), NULL, &error); - - if (error != NULL) - { - g_critical("%s", error->message); - g_error_free(error); - } - - if (connection == NULL) + if(!j_connection_init_client(j_connection_pool->configuration, backend, index, &connection)) { - g_critical("Can not connect to %s [%d].", server, g_atomic_int_get(count)); + g_critical("Can not connect to %s [%d].", "TODO(SERVER ADDR)", g_atomic_int_get(count)); } - j_helper_set_nodelay(connection, TRUE); - message = j_message_new(J_MESSAGE_PING, 0); j_message_send(message, connection); @@ -222,21 +208,21 @@ j_connection_pool_pop_internal(GAsyncQueue* queue, guint* count, gchar const* se for (guint i = 0; i < op_count; i++) { - gchar const* backend; + gchar const* backend_str; - backend = j_message_get_string(reply); + backend_str = j_message_get_string(reply); - if (g_strcmp0(backend, "object") == 0) + if (g_strcmp0(backend_str, "object") == 0) { - //g_print("Server has object backend.\n"); + //g_print("Server has object backend_str.\n"); } - else if (g_strcmp0(backend, "kv") == 0) + else if (g_strcmp0(backend_str, "kv") == 0) { - //g_print("Server has kv backend.\n"); + //g_print("Server has kv backend_str.\n"); } - else if (g_strcmp0(backend, "db") == 0) + else if (g_strcmp0(backend_str, "db") == 0) { - //g_print("Server has db backend.\n"); + //g_print("Server has db backend_str.\n"); } } } @@ -278,13 +264,13 @@ j_connection_pool_pop(JBackendType backend, guint index) { case J_BACKEND_TYPE_OBJECT: g_return_val_if_fail(index < j_connection_pool->object_len, NULL); - return j_connection_pool_pop_internal(j_connection_pool->object_queues[index].queue, &(j_connection_pool->object_queues[index].count), j_configuration_get_server(j_connection_pool->configuration, J_BACKEND_TYPE_OBJECT, index)); + return j_connection_pool_pop_internal(j_connection_pool->object_queues[index].queue, &(j_connection_pool->object_queues[index].count), backend, index); case J_BACKEND_TYPE_KV: g_return_val_if_fail(index < j_connection_pool->kv_len, NULL); - return j_connection_pool_pop_internal(j_connection_pool->kv_queues[index].queue, &(j_connection_pool->kv_queues[index].count), j_configuration_get_server(j_connection_pool->configuration, J_BACKEND_TYPE_KV, index)); + return j_connection_pool_pop_internal(j_connection_pool->kv_queues[index].queue, &(j_connection_pool->kv_queues[index].count), backend, index); case J_BACKEND_TYPE_DB: g_return_val_if_fail(index < j_connection_pool->db_len, NULL); - return j_connection_pool_pop_internal(j_connection_pool->db_queues[index].queue, &(j_connection_pool->db_queues[index].count), j_configuration_get_server(j_connection_pool->configuration, J_BACKEND_TYPE_DB, index)); + return j_connection_pool_pop_internal(j_connection_pool->db_queues[index].queue, &(j_connection_pool->db_queues[index].count), backend, index); default: g_assert_not_reached(); } diff --git a/lib/core/jmessage.c b/lib/core/jmessage.c index f4efce8f3..f937dc1cf 100644 --- a/lib/core/jmessage.c +++ b/lib/core/jmessage.c @@ -630,6 +630,12 @@ j_message_send_ack(JMessage* message, struct JConnection* connection) return FALSE; } +struct JConnectionMemory { + struct fid_mr* memory_region; + void* addr; + guint64 size; +}; + gboolean j_message_send(JMessage* message, struct JConnection* connection) { @@ -637,8 +643,8 @@ j_message_send(JMessage* message, struct JConnection* connection) gboolean ret; int n_memory_regions = 0; - JConnectionMemory* memory_itr = NULL; - JConnectionMemory* memory_regions = NULL; + struct JConnectionMemory* memory_itr = NULL; + struct JConnectionMemory* memory_regions = NULL; struct JConnectionMemoryID mem_id; g_autoptr(JListIterator) iterator = NULL; @@ -648,8 +654,8 @@ j_message_send(JMessage* message, struct JConnection* connection) ret = FALSE; n_memory_regions = j_list_length(message->send_list); - memory_regions = calloc(sizeof(JConnectionMemory), n_memory_regions); - memset(memory_regions, 0, sizeof(JConnectionMemory) * n_memory_regions); + memory_regions = calloc(sizeof(struct JConnectionMemory), n_memory_regions); + memset(memory_regions, 0, sizeof(struct JConnectionMemory) * n_memory_regions); memory_itr = memory_regions; if (message->send_list != NULL) @@ -665,7 +671,7 @@ j_message_send(JMessage* message, struct JConnection* connection) j_message_add_operation(message, sizeof(struct JConnectionMemoryID) + message_data->header_size); - EXE(j_connection_memory_get_id(connection, &mem_id), + EXE(j_connection_memory_get_id(memory_itr, &mem_id), "Failed to get memory it!"); EXE(j_message_append_n(message, message_data->header, message_data->header_size), "Failed to append header"); @@ -679,11 +685,11 @@ j_message_send(JMessage* message, struct JConnection* connection) end: if(memory_regions != NULL) { - JConnectionMemory* mrs = memory_regions; + struct JConnectionMemory* mrs = memory_regions; while(memory_regions != memory_itr) { j_connection_rma_unregister(connection, memory_regions); } - if(*memory_itr) { j_connection_rma_unregister(connection, memory_itr); } + if(memory_itr->memory_region) { j_connection_rma_unregister(connection, memory_itr); } free(mrs); } return ret; diff --git a/lib/core/jnetwork.c b/lib/core/jnetwork.c index dab1da44a..73d9b725c 100644 --- a/lib/core/jnetwork.c +++ b/lib/core/jnetwork.c @@ -2,11 +2,15 @@ #include +#include + #include #include #include #include +#include + /// Used to initialize common parts between different connection. /** This will create the following libfabric resources: * * the domain @@ -35,17 +39,24 @@ enum JFabricSide { JF_CLIENT }; +struct JFabricAddr { + void* addr; + uint64_t addr_len; + uint32_t addr_format; +}; + struct JFabric { struct fi_info* info; struct fid_fabric* fabric; struct fid_eq* pep_eq; struct fid_pep* pep; + struct JFabricAddr fabric_addr_network; struct JConfiguration* config; enum JFabricSide con_side; }; struct JConnection { - struct jfabric* fabric; + struct JFabric* fabric; struct fi_info* info; struct fid_domain* domain; struct fid_ep* ep; @@ -55,10 +66,11 @@ struct JConnection { struct { struct fid_mr* mr; void* buffer; + size_t used; size_t buffer_size; size_t rx_prefix_size; size_t tx_prefix_size; - } rx_mem, tx_mem; + } memory; }; struct JConnectionMemory { @@ -67,18 +79,16 @@ struct JConnectionMemory { guint64 size; }; -struct JFabricAddr { - void* addr; - uint64_t addr_len; - uint32_t addr_format; -}; - -struct JFabricConnectionRequest { - struct fi_eq_cm_entry request_entry; -}; - #define EXE(cmd, ...) do { if (cmd == FALSE) { g_warning(__VA_ARGS__); goto end; } } while(FALSE) #define CHECK(msg) do { if (res < 0) { g_warning("%s: "msg"\nDetails:\t%s", "??TODO??", fi_strerror(-res)); goto end; } } while(FALSE) +#define G_CHECK(msg) do { \ + if(error != NULL) {\ + g_warning(msg"\n\tWith:%s", error->message);\ + g_error_free(error);\ + goto end; } } while(FALSE) + +void free_dangeling_infos(struct fi_info*); + void free_dangeling_infos(struct fi_info* info) { fi_freeinfo(info->next); info->next = NULL; @@ -121,6 +131,17 @@ j_fabric_init_server(struct JConfiguration* configuration, struct JFabric** inst res = fi_listen(this->pep); CHECK("failed to start listening on pep!"); + // initelize addr field! + res = fi_getname(&this->pep->fid, NULL, &this->fabric_addr_network.addr_len); + CHECK("failed to fetch address len from libfabirc!"); + this->fabric_addr_network.addr = malloc(this->fabric_addr_network.addr_len); + res = fi_getname(&this->pep->fid, this->fabric_addr_network.addr, + &this->fabric_addr_network.addr_len); + CHECK("failed to fetch address from libfabric!"); + + this->fabric_addr_network.addr_len = htonl(this->fabric_addr_network.addr_len); + this->fabric_addr_network.addr_format = htonl(this->info->addr_format); + return TRUE; end: return FALSE; @@ -190,14 +211,14 @@ j_fabric_fini(struct JFabric* this) ///! \todo read error! gboolean -j_fabric_sread_event(struct JFabric* this, int timeout, enum JFabricEvents* event, struct JFabricConnectionRequest* con_req) +j_fabric_sread_event(struct JFabric* this, int timeout, enum JFabricEvents* event, JFabricConnectionRequest* con_req) { int res; gboolean ret = FALSE; uint32_t fi_event; res = fi_eq_sread(this->pep_eq, &fi_event, - &con_req->request_entry, sizeof(con_req->request_entry), + con_req, sizeof(*con_req), timeout, 0); if (res == -FI_EAGAIN) { *event = J_FABRIC_EVENT_TIMEOUT; @@ -217,3 +238,208 @@ j_fabric_sread_event(struct JFabric* this, int timeout, enum JFabricEvents* even end: return ret; } + +/** \todo Port is a magic number ! 4711 */ +gboolean +j_connection_init_client(struct JConfiguration* configuration, enum JBackendType backend, guint index, struct JConnection** instance_ptr) +{ + struct JConnection* this; + gboolean ret = FALSE; + int res; + g_autoptr(GSocketClient) g_client = NULL; + GSocketConnection* g_connection; + GInputStream* g_input; + GError* error; + const gchar* server; + struct JFabricAddr jf_addr; + enum JConnectionEvents event; + + *instance_ptr = malloc(sizeof(this)); + this = *instance_ptr; + memset(this, 0, sizeof(*this)); + + g_client = g_socket_client_new(); + server = j_configuration_get_server(configuration, backend, index); + g_connection = g_socket_client_connect_to_host(g_client, + server, + 4711, NULL, &error); + G_CHECK("Failed to build gsocket connection to host"); + if(g_connection == NULL) { + g_warning("Can not connect to %s.", server); + goto end; + } + + g_input = g_io_stream_get_input_stream(G_IO_STREAM(g_connection)); + + g_input_stream_read(g_input, &jf_addr.addr_format, sizeof(jf_addr.addr_format), NULL, &error); + G_CHECK("Failed to read addr format from g_connection!"); + jf_addr.addr_format = ntohl(jf_addr.addr_format); + + g_input_stream_read(g_input, &jf_addr.addr_len, sizeof(jf_addr.addr_len), NULL, &error); + G_CHECK("Failed to read addr len from g_connection!"); + jf_addr.addr_len = ntohl(jf_addr.addr_len); + + jf_addr.addr = malloc(jf_addr.addr_len); + g_input_stream_read(g_input, jf_addr.addr, jf_addr.addr_len, NULL, &error); + G_CHECK("Failed to read addr from g_connection!"); + + g_input_stream_close(g_input, NULL, &error); + G_CHECK("Failed to close input stream!"); + g_io_stream_close(G_IO_STREAM(g_connection), NULL, &error); + G_CHECK("Failed to close gsocket!"); + + EXE(j_fabric_init_client(configuration, &jf_addr, &this->fabric), + "Failed to initelize fabric for client!"); + this->info = this->fabric->info; + + EXE(j_connection_init(this), + "Failed to initelze connection!"); + + res = fi_connect(this->ep, jf_addr.addr, NULL, 0); + CHECK("Failed to fire connection request!"); + + EXE(j_connection_sread_event(this, -1, &event), + "Failed to read event queue, waiting for CONNECTED signal!"); + + if(event != J_CONNECTION_EVENT_CONNECTED) { + g_warning("Failed to connect to host!"); + goto end; + } + + ret = TRUE; +end: + /// FIXME memory leak on error in addr.addr + return ret; +} + +gboolean +j_connection_init_server(struct JFabric* fabric, GSocketConnection* gconnection, struct JConnection** instance_ptr) +{ + struct JConnection* this; + gboolean ret = FALSE; + int res; + GOutputStream* g_out; + GError* error; + struct JFabricAddr* addr = &fabric->fabric_addr_network; + enum JFabricEvents event; + enum JConnectionEvents con_event; + JFabricConnectionRequest request; + + instance_ptr = malloc(sizeof(*this)); + this = *instance_ptr; + memset(this, 0, sizeof(*this)); + + // send addr + g_out = g_io_stream_get_output_stream(G_IO_STREAM(gconnection)); + g_output_stream_write(g_out, &addr->addr_format, sizeof(addr->addr_format), NULL, &error); + G_CHECK("Failed to write addr_format to stream!"); + g_output_stream_write(g_out, &addr->addr_len, sizeof(addr->addr_len), NULL, &error); + G_CHECK("Failed to write addr_len to stream!"); + g_output_stream_write(g_out, addr->addr, addr->addr_len, NULL, &error); + G_CHECK("Failed to write addr to stream!"); + g_output_stream_close(g_out, NULL, &error); + G_CHECK("Failed to close output stream!"); + + EXE(j_fabric_sread_event(fabric, -1, &event, &request), + "Failed to wait for connection request"); + if(event != J_FABRIC_EVENT_CONNECTION_REQUEST) { + g_warning("expected an connection request and nothing else!"); + goto end; + } + this->fabric = fabric; + this->info = request.info; + + EXE(j_connection_init(this), "Failed to initelize connection server side!"); + + res = fi_accept(this->ep, NULL, 0); + CHECK("Failed to accept connection!"); + EXE(j_connection_sread_event(this, -1, &con_event), "Failed to verify connection!"); + if(con_event != J_CONNECTION_EVENT_CONNECTED) { + g_warning("expected and connection ack and nothing else!"); + goto end; + } + + ret = TRUE; +end: + return ret; +} + +gboolean +j_connection_init(struct JConnection* this) +{ + gboolean ret = FALSE; + int res; + + res = fi_eq_open(this->fabric->fabric, &(struct fi_eq_attr){.wait_obj = FI_WAIT_UNSPEC}, + &this->eq, NULL); + CHECK("Failed to open event queue for connection!"); + res = fi_domain(this->fabric->fabric, this->info, + &this->domain, NULL); + CHECK("Failed to open connection domain!"); + + this->inject_size = this->fabric->info->tx_attr->inject_size; + EXE(j_connection_create_memory_resources(this), + "Failed to create memory resources for connection!"); + + res = fi_cq_open(this->domain, &(struct fi_cq_attr){ + .wait_obj = FI_WAIT_UNSPEC, + .format = FI_CQ_FORMAT_CONTEXT, + .size = this->info->tx_attr->size + }, + &this->cq, &this->cq); + // tx completion size differs from rx completion size! + g_assert_true(this->info->tx_attr->size == this->info->rx_attr->size); + CHECK("Failed to create completion queue!"); + res = fi_endpoint(this->domain, this->info, &this->ep, NULL); + CHECK("Failed to open endpoint for connection!"); + res = fi_ep_bind(this->ep, &this->eq->fid, 0); + CHECK("Failed to bind event queue to endpoint!"); + res = fi_ep_bind(this->ep, &this->cq->fid, FI_TRANSMIT | FI_RECV); + CHECK("Failed to bind completion queue to endpoint!"); + res = fi_enable(this->ep); + CHECK("Failed to enable connection!"); + + ret = TRUE; +end: + return ret; +} + +gboolean +j_connection_create_memory_resources(struct JConnection* this) +{ + gboolean ret = FALSE; + int res; + guint64 op_size, size; + size_t prefix_size; + gboolean tx_prefix, rx_prefix; + + op_size = j_configuration_get_max_operation_size(this->fabric->config); + tx_prefix = (this->info->tx_attr->mode & FI_MSG_PREFIX) != 0; + rx_prefix = (this->info->rx_attr->mode & FI_MSG_PREFIX) != 0; + prefix_size = this->info->ep_attr->msg_prefix_size; + + if(op_size + (tx_prefix | rx_prefix) * prefix_size > this->info->ep_attr->msg_prefix_size) { + op_size = this->info->ep_attr->max_msg_size - (tx_prefix | rx_prefix) * prefix_size; + g_critical("Fabric supported memory size is too smal! please configure a max operation size less equal to %lu!", op_size); + goto end; + } + size = 0; + + if(this->info->domain_attr->mr_mode & FI_MR_LOCAL) { + size += + (rx_prefix * prefix_size) * J_CONNECTION_MAX_RECV * op_size + + (tx_prefix * prefix_size) * J_CONNECTION_MAX_SEND * op_size; + this->memory.buffer_size = size; + this->memory.buffer = malloc(size); + this->memory.rx_prefix_size = rx_prefix * prefix_size; + this->memory.tx_prefix_size = tx_prefix * prefix_size; + this->memory.used = 0; + res = fi_mr_reg(this->domain, this->memory.buffer, this->memory.buffer_size, + FI_SEND | FI_RECV, 0, 0, 0, &this->memory.mr, NULL); + CHECK("Failed to register memory for msg communication!"); + } + + ret = TRUE; +end: + return ret; +} diff --git a/lib/db/jdb-internal.c b/lib/db/jdb-internal.c index c54d00a99..8178db913 100644 --- a/lib/db/jdb-internal.c +++ b/lib/db/jdb-internal.c @@ -58,7 +58,7 @@ j_backend_db_func_exec(JList* operations, JSemantics* semantics, JMessageType ty JBackendOperation* data = NULL; gboolean ret = TRUE; - GSocketConnection* db_connection; + struct JConnection* db_connection; g_autoptr(JListIterator) iter_send = NULL; g_autoptr(JListIterator) iter_recieve = NULL; g_autoptr(JMessage) message = NULL; diff --git a/meson.build b/meson.build index d9fb017e0..e1f84c485 100644 --- a/meson.build +++ b/meson.build @@ -354,7 +354,7 @@ configure_file( # Build -common_deps = [m_dep, glib_dep, gio_dep, gmodule_dep, gthread_dep, gobject_dep, libbson_dep, otf_dep] +common_deps = [m_dep, glib_dep, gio_dep, gmodule_dep, gthread_dep, gobject_dep, libbson_dep, otf_dep, libfabric_dep] # FIXME Remove core directory julea_incs = include_directories([ diff --git a/server/server.c b/server/server.c index ba9e1f9aa..063858869 100644 --- a/server/server.c +++ b/server/server.c @@ -82,7 +82,7 @@ jd_on_run(GThreadedSocketService* service, GSocketConnection* gconnection, GObje message = j_message_new(J_MESSAGE_NONE, 0); - j_connection_init_server(jd_configuration, jd_fabric, gconnection, &jconnection); + j_connection_init_server(jd_fabric, gconnection, &jconnection); while (j_message_receive(message, jconnection)) { From 8666f0fbf44e649435bbe72bc54f8c41f56694f6 Mon Sep 17 00:00:00 2001 From: Julian Benda Date: Wed, 4 Aug 2021 09:28:22 +0200 Subject: [PATCH 14/60] WIP --- lib/core/jconnection-pool.c | 1 + lib/core/jmessage.c | 37 +++- lib/core/jnetwork.c | 341 ++++++++++++++++++++++++++++++++++-- server/server.c | 4 +- 4 files changed, 362 insertions(+), 21 deletions(-) diff --git a/lib/core/jconnection-pool.c b/lib/core/jconnection-pool.c index 46f3385c9..bebdff651 100644 --- a/lib/core/jconnection-pool.c +++ b/lib/core/jconnection-pool.c @@ -196,6 +196,7 @@ j_connection_pool_pop_internal(GAsyncQueue* queue, guint* count, enum JBackendTy if(!j_connection_init_client(j_connection_pool->configuration, backend, index, &connection)) { g_critical("Can not connect to %s [%d].", "TODO(SERVER ADDR)", g_atomic_int_get(count)); + exit(1); } message = j_message_new(J_MESSAGE_PING, 0); diff --git a/lib/core/jmessage.c b/lib/core/jmessage.c index f937dc1cf..bc9659b28 100644 --- a/lib/core/jmessage.c +++ b/lib/core/jmessage.c @@ -508,6 +508,26 @@ j_message_append_n(JMessage* message, gconstpointer data, gsize length) return TRUE; } +gboolean +j_message_append_memory_id(JMessage* message, const struct JConnectionMemoryID* id) +{ + J_TRACE_FUNCTION(NULL); + + guint32 new_length; + + g_return_val_if_fail(message != NULL, FALSE); + g_return_val_if_fail(id != NULL, FALSE); + g_return_val_if_fail(j_message_can_append(message, sizeof(*id)), FALSE); + + memcpy(message->current, id, sizeof(*id)); + message->current += sizeof(*id); + + new_length = j_message_length(message) + sizeof(*id); + message->header.length = GUINT32_TO_LE(new_length); + + return TRUE; +} + gboolean j_message_append_string(JMessage* message, gchar const* str) { @@ -600,6 +620,21 @@ j_message_get_string(JMessage* message) return ret; } +const struct JConnectionMemoryID* +j_message_get_memory_id(JMessage* message) +{ + J_TRACE_FUNCTION(NULL); + + const struct JConnectionMemoryID* ret; + + g_return_val_if_fail(message != NULL, NULL); + + ret = (const void*)message->current; + message->current += sizeof(*ret); + + return ret; +} + gboolean j_message_receive(JMessage* message, struct JConnection* connection) { @@ -632,7 +667,7 @@ j_message_send_ack(JMessage* message, struct JConnection* connection) struct JConnectionMemory { struct fid_mr* memory_region; - void* addr; + guint64 addr; guint64 size; }; diff --git a/lib/core/jnetwork.c b/lib/core/jnetwork.c index 73d9b725c..771b98d70 100644 --- a/lib/core/jnetwork.c +++ b/lib/core/jnetwork.c @@ -41,7 +41,7 @@ enum JFabricSide { struct JFabricAddr { void* addr; - uint64_t addr_len; + uint32_t addr_len; uint32_t addr_format; }; @@ -62,7 +62,7 @@ struct JConnection { struct fid_ep* ep; struct fid_eq* eq; struct fid_cq* cq; - int inject_size; + size_t inject_size; struct { struct fid_mr* mr; void* buffer; @@ -71,11 +71,20 @@ struct JConnection { size_t rx_prefix_size; size_t tx_prefix_size; } memory; + struct { + struct { + void* context; + void* dest; + size_t len; + struct fid_mr* mr; + } entry[J_CONNECTION_MAX_RECV + J_CONNECTION_MAX_SEND]; + int len; + } running_actions; }; struct JConnectionMemory { struct fid_mr* memory_region; - void* addr; + guint64 addr; guint64 size; }; @@ -99,6 +108,7 @@ j_fabric_init_server(struct JConfiguration* configuration, struct JFabric** inst { struct JFabric* this; int res; + size_t addrlen; *instnace_ptr = malloc(sizeof(*this)); this = *instnace_ptr; @@ -128,17 +138,22 @@ j_fabric_init_server(struct JConfiguration* configuration, struct JFabric** inst CHECK("failed to create pep for fabric!"); res = fi_pep_bind(this->pep, &this->pep_eq->fid, 0); CHECK("failed to bind event queue to pep!"); - res = fi_listen(this->pep); - CHECK("failed to start listening on pep!"); // initelize addr field! - res = fi_getname(&this->pep->fid, NULL, &this->fabric_addr_network.addr_len); - CHECK("failed to fetch address len from libfabirc!"); + res = fi_getname(&this->pep->fid, NULL, &addrlen); + if(res != -FI_ETOOSMALL) { CHECK("failed to fetch address len from libfabirc!"); } + this->fabric_addr_network.addr_len = addrlen; + g_message("test: %u", this->fabric_addr_network.addr_len); this->fabric_addr_network.addr = malloc(this->fabric_addr_network.addr_len); res = fi_getname(&this->pep->fid, this->fabric_addr_network.addr, - &this->fabric_addr_network.addr_len); + &addrlen); CHECK("failed to fetch address from libfabric!"); + g_message("wait for pep request!"); + res = fi_listen(this->pep); + CHECK("failed to start listening on pep!"); + + this->fabric_addr_network.addr_len = htonl(this->fabric_addr_network.addr_len); this->fabric_addr_network.addr_format = htonl(this->info->addr_format); @@ -166,6 +181,13 @@ j_fabric_init_client(struct JConfiguration* configuration, struct JFabricAddr* a hints->addr_format = addr->addr_format; hints->dest_addr = addr->addr; hints->dest_addrlen = addr->addr_len; + { // DEBUG TODO + char* str = malloc(hints->dest_addrlen * 3 + 1); + size_t i; + for(i = 0; i < hints->dest_addrlen; ++i) { snprintf(str+i*3, 4, "%02x ", ((uint8_t*)hints->dest_addr)[i]); } + g_message("Addr: format: %u, len: %lu\n\t%s", hints->addr_format, hints->dest_addrlen, str); + free(str); + } res = fi_getinfo( j_configuration_get_libfabric_version(configuration), @@ -178,8 +200,8 @@ j_fabric_init_client(struct JConfiguration* configuration, struct JFabricAddr* a CHECK("failed to initelize client fabric!"); ret = TRUE; + g_message("initelize client!"); end: - fi_freeinfo(hints); return ret; } @@ -254,7 +276,7 @@ j_connection_init_client(struct JConfiguration* configuration, enum JBackendType struct JFabricAddr jf_addr; enum JConnectionEvents event; - *instance_ptr = malloc(sizeof(this)); + *instance_ptr = malloc(sizeof(*this)); this = *instance_ptr; memset(this, 0, sizeof(*this)); @@ -298,8 +320,10 @@ j_connection_init_client(struct JConfiguration* configuration, enum JBackendType res = fi_connect(this->ep, jf_addr.addr, NULL, 0); CHECK("Failed to fire connection request!"); - EXE(j_connection_sread_event(this, -1, &event), + do {EXE(j_connection_sread_event(this, -1, &event), "Failed to read event queue, waiting for CONNECTED signal!"); + } while(event == J_CONNECTION_EVENT_TIMEOUT); + g_message("Event FI: %u", event); if(event != J_CONNECTION_EVENT_CONNECTED) { g_warning("Failed to connect to host!"); @@ -325,34 +349,48 @@ j_connection_init_server(struct JFabric* fabric, GSocketConnection* gconnection, enum JConnectionEvents con_event; JFabricConnectionRequest request; - instance_ptr = malloc(sizeof(*this)); + *instance_ptr = malloc(sizeof(*this)); this = *instance_ptr; + g_message("debug: size(*this) = %lu, this = %p", sizeof(*this), (void*)this); memset(this, 0, sizeof(*this)); // send addr + { // DEBUG TODO + char* str = malloc(ntohl(addr->addr_len) * 3 + 1); + size_t i; + g_message("tada: addr_len: %u", ntohl(addr->addr_len)); + for(i = 0; i < ntohl(addr->addr_len); ++i) { snprintf(str+i*3, 4, "%02x ", ((uint8_t*)addr->addr)[i]); } + g_message("Addr: format: %u, len: %u\n\t%s", ntohl(addr->addr_format), ntohl(addr->addr_len), str); + free(str); + } g_out = g_io_stream_get_output_stream(G_IO_STREAM(gconnection)); g_output_stream_write(g_out, &addr->addr_format, sizeof(addr->addr_format), NULL, &error); G_CHECK("Failed to write addr_format to stream!"); g_output_stream_write(g_out, &addr->addr_len, sizeof(addr->addr_len), NULL, &error); G_CHECK("Failed to write addr_len to stream!"); - g_output_stream_write(g_out, addr->addr, addr->addr_len, NULL, &error); + g_output_stream_write(g_out, addr->addr, ntohl(addr->addr_len), NULL, &error); G_CHECK("Failed to write addr to stream!"); + g_message("close g_stream"); g_output_stream_close(g_out, NULL, &error); G_CHECK("Failed to close output stream!"); + g_message("wait for conrequest!"); EXE(j_fabric_sread_event(fabric, -1, &event, &request), "Failed to wait for connection request"); if(event != J_FABRIC_EVENT_CONNECTION_REQUEST) { g_warning("expected an connection request and nothing else!"); goto end; } + g_message("inet connection!"); this->fabric = fabric; this->info = request.info; EXE(j_connection_init(this), "Failed to initelize connection server side!"); + g_message("Except!"); res = fi_accept(this->ep, NULL, 0); CHECK("Failed to accept connection!"); + g_message("wait for connection !"); EXE(j_connection_sread_event(this, -1, &con_event), "Failed to verify connection!"); if(con_event != J_CONNECTION_EVENT_CONNECTED) { g_warning("expected and connection ack and nothing else!"); @@ -370,6 +408,8 @@ j_connection_init(struct JConnection* this) gboolean ret = FALSE; int res; + this->running_actions.len = 0; + res = fi_eq_open(this->fabric->fabric, &(struct fi_eq_attr){.wait_obj = FI_WAIT_UNSPEC}, &this->eq, NULL); CHECK("Failed to open event queue for connection!"); @@ -400,6 +440,7 @@ j_connection_init(struct JConnection* this) CHECK("Failed to enable connection!"); ret = TRUE; + g_message("enabled endpoint!"); end: return ret; } @@ -418,17 +459,19 @@ j_connection_create_memory_resources(struct JConnection* this) rx_prefix = (this->info->rx_attr->mode & FI_MSG_PREFIX) != 0; prefix_size = this->info->ep_attr->msg_prefix_size; - if(op_size + (tx_prefix | rx_prefix) * prefix_size > this->info->ep_attr->msg_prefix_size) { - op_size = this->info->ep_attr->max_msg_size - (tx_prefix | rx_prefix) * prefix_size; - g_critical("Fabric supported memory size is too smal! please configure a max operation size less equal to %lu!", op_size); + if(op_size + (tx_prefix | rx_prefix) * prefix_size > this->info->ep_attr->max_msg_size) { + guint64 max_size = this->info->ep_attr->max_msg_size - (tx_prefix | rx_prefix) * prefix_size; + g_critical("Fabric supported memory size is too smal! please configure a max operation size less equal to %lu! instead of %lu", + max_size, op_size + (tx_prefix | rx_prefix) * prefix_size); goto end; } size = 0; if(this->info->domain_attr->mr_mode & FI_MR_LOCAL) { + g_message("Localmemory!"); size += - (rx_prefix * prefix_size) * J_CONNECTION_MAX_RECV * op_size - + (tx_prefix * prefix_size) * J_CONNECTION_MAX_SEND * op_size; + (rx_prefix * prefix_size) + J_CONNECTION_MAX_RECV * op_size + + (tx_prefix * prefix_size) + J_CONNECTION_MAX_SEND * op_size; this->memory.buffer_size = size; this->memory.buffer = malloc(size); this->memory.rx_prefix_size = rx_prefix * prefix_size; @@ -437,7 +480,269 @@ j_connection_create_memory_resources(struct JConnection* this) res = fi_mr_reg(this->domain, this->memory.buffer, this->memory.buffer_size, FI_SEND | FI_RECV, 0, 0, 0, &this->memory.mr, NULL); CHECK("Failed to register memory for msg communication!"); + } else { g_message("NO MEM!"); } + + ret = TRUE; +end: + return ret; +} + +gboolean +j_connection_sread_event(struct JConnection* this, int timeout, enum JConnectionEvents* event) +{ + int res; + uint32_t fi_event; + gboolean ret = FALSE; + struct fi_eq_cm_entry entry; + + g_message("Call sread event! %i", timeout); + + { + struct fi_eq_err_entry err; + do { + res = fi_eq_readerr(this->eq, &err, 0); + g_message("err: %s", fi_eq_strerror(this->eq, + err.prov_errno, err.err_data, NULL, 0)); + } while(res > 0); + + } + g_message("wait for read!"); + res = fi_eq_sread(this->eq, &fi_event, &entry, sizeof(entry), timeout, 5); + g_message("evaluate read!"); + { + struct fi_eq_err_entry err; + do { + res = fi_eq_readerr(this->eq, &err, 0); + g_message("err: %s", fi_eq_strerror(this->eq, + err.prov_errno, err.err_data, NULL, 0)); + } while(res > 0); + + } + g_message("res = %i", res); + if(res == -FI_EAGAIN) { + *event = J_CONNECTION_EVENT_TIMEOUT; + ret = TRUE; goto end; + } else if (res == -FI_EAVAIL) { + struct fi_eq_err_entry error = {0}; + *event = J_CONNECTION_EVENT_ERROR; + do { + res = fi_eq_readerr(this->eq, &error, 0); + CHECK("Failed to read error!"); + g_warning("event queue contains following error (%s|c:%p):\n\t%s", + fi_strerror(FI_EAVAIL), error.context, + fi_eq_strerror(this->eq, error.prov_errno, error.err_data, NULL, 0)); + } while(res > 0); + goto end; /// \todo consider return TRUE on event queue error + } + CHECK("Failed to read event of connection!"); + g_message("Event: %u", fi_event); + + switch(fi_event) { + case FI_CONNECTED: *event = J_CONNECTION_EVENT_CONNECTED; break; + case FI_SHUTDOWN: *event = J_CONNECTION_EVENT_SHUTDOWN; break; + default: g_assert_not_reached(); goto end; } + ret = TRUE; +end: + return ret; +} + +gboolean +j_connection_read_event(struct JConnection* this, enum JConnectionEvents* event) +{ + int res; + uint32_t fi_event; + gboolean ret = FALSE; + + res = fi_eq_read(this->eq, &fi_event, NULL, 0, 0); + if(res == -FI_EAGAIN) { + *event = J_CONNECTION_EVENT_TIMEOUT; + ret = TRUE; goto end; + } else if (res == -FI_EAVAIL) { + *event = J_CONNECTION_EVENT_ERROR; + // TODO: fetch error! + ret = TRUE; goto end; + } + CHECK("Failed to read event of connection!"); + switch(fi_event) { + case FI_CONNECTED: *event = J_CONNECTION_EVENT_CONNECTED; break; + case FI_SHUTDOWN: *event = J_CONNECTION_EVENT_SHUTDOWN; break; + default: g_assert_not_reached(); goto end; + } + ret = TRUE; +end: + return ret; +} + +gboolean +j_connection_send(struct JConnection* this, const void* data, size_t data_len) +{ + int res; + gboolean ret = FALSE; + void* segment; + size_t size; + + // we used paired endponits -> inject and send don't need destination addr (last parameter) + + if(data_len < this->inject_size) { + do { res = fi_inject(this->ep, data, data_len, 0); } while(res == -FI_EAGAIN); + CHECK("Failed to inject data!"); + ret = TRUE; goto end; + } + + // normal send + memcpy((char*)this->memory.buffer + this->memory.used + this->memory.tx_prefix_size, + data, data_len); + segment = (char*)this->memory.buffer + this->memory.used; + size = data_len + this->memory.tx_prefix_size; + res = fi_send(this->ep, segment, size, fi_mr_desc(this->memory.mr), 0, segment); + CHECK("Failed to initelize sending!"); + this->memory.used += size; + g_assert_true(this->memory.used <= this->memory.buffer_size); + g_assert_true(this->running_actions.len + 1< J_CONNECTION_MAX_SEND + J_CONNECTION_MAX_RECV); + this->running_actions.entry[this->running_actions.len].context = segment; + this->running_actions.entry[this->running_actions.len].dest = NULL; + this->running_actions.entry[this->running_actions.len].len = 0; + this->running_actions.entry[this->running_actions.len].mr = NULL; + ++this->running_actions.len; + + ret = TRUE; +end: + return ret; +} + +gboolean +j_connection_recv(struct JConnection* this, size_t data_len, void* data) +{ + gboolean ret = FALSE; + int res; + void* segment; + size_t size; + + segment = (char*)this->memory.buffer + this->memory.used; + size = data_len + this->memory.rx_prefix_size; + res = fi_recv(this->ep, segment, size, fi_mr_desc(this->memory.mr), 0, segment); + CHECK("Failed to initelized receiving!"); + this->memory.used += size; + g_message("memory useag: %lu/%lu", this->memory.used, this->memory.buffer_size); + g_assert_true(this->memory.used <= this->memory.buffer_size); + g_assert_true(this->running_actions.len + 1< J_CONNECTION_MAX_SEND + J_CONNECTION_MAX_RECV); + this->running_actions.entry[this->running_actions.len].context = segment; + this->running_actions.entry[this->running_actions.len].dest = data; + this->running_actions.entry[this->running_actions.len].len = data_len; + this->running_actions.entry[this->running_actions.len].mr = NULL; + ++this->running_actions.len; + + ret = TRUE; +end: + return ret; +} + +gboolean +j_connection_wait_for_completion(struct JConnection* this) +{ + gboolean ret = FALSE; + int res; + struct fi_cq_entry entry; + int i; + + while(this->running_actions.len) { + res = fi_cq_sread(this->cq, &entry, 1, NULL, -1); /// \todo handle shutdown msgs! + /// \todo error message fetch! + CHECK("Failed to read completion queue!"); + for(i = 0; i <= this->running_actions.len; ++i) { + if(i == this->running_actions.len) { + g_warning("unable to find completet context!"); + goto end; + } + if(this->running_actions.entry[i].context == entry.op_context) { + --this->running_actions.len; + if(this->running_actions.entry[i].dest) { + memcpy(this->running_actions.entry[i].dest, + this->running_actions.entry[i].context, + this->running_actions.entry[i].len); + } + if (this->running_actions.entry[i].mr) { + res = fi_close( + &this->running_actions.entry[i].mr->fid); + CHECK("Failed to free receiving memory!"); + } + this->running_actions.entry[i] = this->running_actions.entry[this->running_actions.len]; + break; + } + } + } + + this->running_actions.len = 0; + ret = TRUE; +end: + return ret; +} + +gboolean +j_connection_rma_register(struct JConnection* this, const void* data, size_t data_len, struct JConnectionMemory* handle) +{ + int res; + gboolean ret = FALSE; + + res = fi_mr_reg(this->domain, + data, + data_len, + FI_REMOTE_READ, + 0, 0, 0, &handle->memory_region, NULL); + CHECK("Failed to register memory region!"); + handle->addr = (guint64)data; + handle->size = data_len; + + ret = TRUE; +end: + return ret; +} + +gboolean +j_connection_rma_unregister(struct JConnection* this, struct JConnectionMemory* handle) +{ + int res; + res = fi_close(&handle->memory_region->fid); + CHECK("Failed to unregistrer rma memory!"); + return TRUE; +end: + return FALSE; +} + +gboolean +j_connection_memory_get_id(struct JConnectionMemory* this, struct JConnectionMemoryID* id) +{ + id->size = this->size; + id->key = fi_mr_key(this->memory_region); + id->offset = this->addr; + return TRUE; +} + +gboolean +j_connection_rma_read(struct JConnection* this, const struct JConnectionMemoryID* memoryID, void* data) +{ + int res; + gboolean ret = FALSE; + struct fid_mr* mr; + + res = fi_mr_reg(this->domain, data, memoryID->size, + FI_READ, 0, 0, 0, &mr, 0); + CHECK("Failed to register receiving memory!"); + res = fi_read(this->ep, + data, + memoryID->size, + fi_mr_desc(mr), + 0, + memoryID->offset, + memoryID->key, + data); + CHECK("Failed to initate reading"); + this->running_actions.entry[this->running_actions.len].context = data; + this->running_actions.entry[this->running_actions.len].dest = NULL; + this->running_actions.entry[this->running_actions.len].len = 0; + this->running_actions.entry[this->running_actions.len].mr = mr; + ++this->running_actions.len; ret = TRUE; end: diff --git a/server/server.c b/server/server.c index 063858869..9bb2064cd 100644 --- a/server/server.c +++ b/server/server.c @@ -74,14 +74,13 @@ jd_on_run(GThreadedSocketService* service, GSocketConnection* gconnection, GObje (void)source_object; (void)user_data; - j_helper_set_nodelay(gconnection, TRUE); - statistics = j_statistics_new(TRUE); memory_chunk_size = j_configuration_get_max_operation_size(jd_configuration); memory_chunk = j_memory_chunk_new(memory_chunk_size); message = j_message_new(J_MESSAGE_NONE, 0); + g_message("fabric: %p", (void*)jd_fabric); j_connection_init_server(jd_fabric, gconnection, &jconnection); while (j_message_receive(message, jconnection)) @@ -384,6 +383,7 @@ main(int argc, char** argv) g_warning("Failed to initialize server fabric!"); return 1; } + g_message("fabric: %p", (void*)jd_fabric); jd_statistics = j_statistics_new(FALSE); g_mutex_init(jd_statistics_mutex); From 859f91e45982ee7a651798616149290b5763b85d Mon Sep 17 00:00:00 2001 From: Julian Benda Date: Wed, 4 Aug 2021 14:45:39 +0200 Subject: [PATCH 15/60] Working hand shake and no connection stall --- lib/core/jnetwork.c | 83 +++++++++++++++++++++++---------------------- 1 file changed, 43 insertions(+), 40 deletions(-) diff --git a/lib/core/jnetwork.c b/lib/core/jnetwork.c index 771b98d70..dad7413e5 100644 --- a/lib/core/jnetwork.c +++ b/lib/core/jnetwork.c @@ -61,7 +61,10 @@ struct JConnection { struct fid_domain* domain; struct fid_ep* ep; struct fid_eq* eq; - struct fid_cq* cq; + struct { + struct fid_cq* tx; + struct fid_cq* rx; + } cq; size_t inject_size; struct { struct fid_mr* mr; @@ -283,7 +286,7 @@ j_connection_init_client(struct JConfiguration* configuration, enum JBackendType g_client = g_socket_client_new(); server = j_configuration_get_server(configuration, backend, index); g_connection = g_socket_client_connect_to_host(g_client, - server, + "10.0.10.1", 4711, NULL, &error); G_CHECK("Failed to build gsocket connection to host"); if(g_connection == NULL) { @@ -320,7 +323,7 @@ j_connection_init_client(struct JConfiguration* configuration, enum JBackendType res = fi_connect(this->ep, jf_addr.addr, NULL, 0); CHECK("Failed to fire connection request!"); - do {EXE(j_connection_sread_event(this, -1, &event), + do {EXE(j_connection_sread_event(this, 1, &event), "Failed to read event queue, waiting for CONNECTED signal!"); } while(event == J_CONNECTION_EVENT_TIMEOUT); g_message("Event FI: %u", event); @@ -330,6 +333,10 @@ j_connection_init_client(struct JConfiguration* configuration, enum JBackendType goto end; } + *(gint32*)(this->memory.buffer) = 12; + do { res = fi_inject(this->ep, this->memory.buffer, 4, 0); } while(res == -FI_EAGAIN); + CHECK("failed to inject test message!"); + ret = TRUE; end: /// FIXME memory leak on error in addr.addr @@ -375,10 +382,11 @@ j_connection_init_server(struct JFabric* fabric, GSocketConnection* gconnection, G_CHECK("Failed to close output stream!"); g_message("wait for conrequest!"); - EXE(j_fabric_sread_event(fabric, -1, &event, &request), + do {EXE(j_fabric_sread_event(fabric, 2, &event, &request), "Failed to wait for connection request"); + } while(event == J_FABRIC_EVENT_TIMEOUT); if(event != J_FABRIC_EVENT_CONNECTION_REQUEST) { - g_warning("expected an connection request and nothing else!"); + g_warning("expected an connection request and nothing else! (%i)", event); goto end; } g_message("inet connection!"); @@ -391,7 +399,7 @@ j_connection_init_server(struct JFabric* fabric, GSocketConnection* gconnection, res = fi_accept(this->ep, NULL, 0); CHECK("Failed to accept connection!"); g_message("wait for connection !"); - EXE(j_connection_sread_event(this, -1, &con_event), "Failed to verify connection!"); + EXE(j_connection_sread_event(this, 2, &con_event), "Failed to verify connection!"); if(con_event != J_CONNECTION_EVENT_CONNECTED) { g_warning("expected and connection ack and nothing else!"); goto end; @@ -426,16 +434,22 @@ j_connection_init(struct JConnection* this) .format = FI_CQ_FORMAT_CONTEXT, .size = this->info->tx_attr->size }, - &this->cq, &this->cq); - // tx completion size differs from rx completion size! - g_assert_true(this->info->tx_attr->size == this->info->rx_attr->size); + &this->cq.tx, &this->cq.tx); + res = fi_cq_open(this->domain, &(struct fi_cq_attr){ + .wait_obj = FI_WAIT_UNSPEC, + .format = FI_CQ_FORMAT_CONTEXT, + .size = this->info->rx_attr->size + }, + &this->cq.rx, &this->cq.rx); CHECK("Failed to create completion queue!"); res = fi_endpoint(this->domain, this->info, &this->ep, NULL); CHECK("Failed to open endpoint for connection!"); res = fi_ep_bind(this->ep, &this->eq->fid, 0); CHECK("Failed to bind event queue to endpoint!"); - res = fi_ep_bind(this->ep, &this->cq->fid, FI_TRANSMIT | FI_RECV); - CHECK("Failed to bind completion queue to endpoint!"); + res = fi_ep_bind(this->ep, &this->cq.tx->fid, FI_TRANSMIT); + CHECK("Failed to bind tx completion queue to endpoint!"); + res = fi_ep_bind(this->ep, &this->cq.rx->fid, FI_RECV); + CHECK("Failed to bind rx completion queue to endpoint!"); res = fi_enable(this->ep); CHECK("Failed to enable connection!"); @@ -495,30 +509,8 @@ j_connection_sread_event(struct JConnection* this, int timeout, enum JConnection gboolean ret = FALSE; struct fi_eq_cm_entry entry; - g_message("Call sread event! %i", timeout); - { - struct fi_eq_err_entry err; - do { - res = fi_eq_readerr(this->eq, &err, 0); - g_message("err: %s", fi_eq_strerror(this->eq, - err.prov_errno, err.err_data, NULL, 0)); - } while(res > 0); - - } - g_message("wait for read!"); res = fi_eq_sread(this->eq, &fi_event, &entry, sizeof(entry), timeout, 5); - g_message("evaluate read!"); - { - struct fi_eq_err_entry err; - do { - res = fi_eq_readerr(this->eq, &err, 0); - g_message("err: %s", fi_eq_strerror(this->eq, - err.prov_errno, err.err_data, NULL, 0)); - } while(res > 0); - - } - g_message("res = %i", res); if(res == -FI_EAGAIN) { *event = J_CONNECTION_EVENT_TIMEOUT; ret = TRUE; goto end; @@ -584,18 +576,21 @@ j_connection_send(struct JConnection* this, const void* data, size_t data_len) // we used paired endponits -> inject and send don't need destination addr (last parameter) - if(data_len < this->inject_size) { + g_message("send"); + /*if(data_len < this->inject_size) { do { res = fi_inject(this->ep, data, data_len, 0); } while(res == -FI_EAGAIN); CHECK("Failed to inject data!"); ret = TRUE; goto end; - } + }*/ // normal send memcpy((char*)this->memory.buffer + this->memory.used + this->memory.tx_prefix_size, data, data_len); segment = (char*)this->memory.buffer + this->memory.used; size = data_len + this->memory.tx_prefix_size; - res = fi_send(this->ep, segment, size, fi_mr_desc(this->memory.mr), 0, segment); + do{ + res = fi_send(this->ep, segment, size, fi_mr_desc(this->memory.mr), 0, segment); + } while(res == -FI_EAGAIN); CHECK("Failed to initelize sending!"); this->memory.used += size; g_assert_true(this->memory.used <= this->memory.buffer_size); @@ -618,21 +613,22 @@ j_connection_recv(struct JConnection* this, size_t data_len, void* data) int res; void* segment; size_t size; - + g_message("receive"); segment = (char*)this->memory.buffer + this->memory.used; size = data_len + this->memory.rx_prefix_size; res = fi_recv(this->ep, segment, size, fi_mr_desc(this->memory.mr), 0, segment); CHECK("Failed to initelized receiving!"); this->memory.used += size; g_message("memory useag: %lu/%lu", this->memory.used, this->memory.buffer_size); + g_message("actions: %i", this->running_actions.len); g_assert_true(this->memory.used <= this->memory.buffer_size); - g_assert_true(this->running_actions.len + 1< J_CONNECTION_MAX_SEND + J_CONNECTION_MAX_RECV); + g_assert_true(this->running_actions.len < J_CONNECTION_MAX_SEND + J_CONNECTION_MAX_RECV); this->running_actions.entry[this->running_actions.len].context = segment; this->running_actions.entry[this->running_actions.len].dest = data; this->running_actions.entry[this->running_actions.len].len = data_len; this->running_actions.entry[this->running_actions.len].mr = NULL; ++this->running_actions.len; - + g_message("fin recive"); ret = TRUE; end: return ret; @@ -646,8 +642,14 @@ j_connection_wait_for_completion(struct JConnection* this) struct fi_cq_entry entry; int i; + g_message("start waiting"); while(this->running_actions.len) { - res = fi_cq_sread(this->cq, &entry, 1, NULL, -1); /// \todo handle shutdown msgs! + do { + res = fi_cq_sread(this->cq.rx, &entry, 1, NULL, 2); /// \todo handle shutdown msgs! + if(res == -FI_EAGAIN) { + res = fi_cq_sread(this->cq.tx, &entry, 1, NULL, 2); /// \todo handle shutdown msgs! + } + } while (res == -FI_EAGAIN); /// \todo error message fetch! CHECK("Failed to read completion queue!"); for(i = 0; i <= this->running_actions.len; ++i) { @@ -672,6 +674,7 @@ j_connection_wait_for_completion(struct JConnection* this) } } } + g_message("fin waiting"); this->running_actions.len = 0; ret = TRUE; From 578fc2bd1012fe84ca6400581c66d2760e3985f6 Mon Sep 17 00:00:00 2001 From: Julian Benda Date: Wed, 4 Aug 2021 18:17:02 +0200 Subject: [PATCH 16/60] Transfare first header correct --- lib/core/jmessage.c | 12 +++++++++++- lib/core/jnetwork.c | 24 ++++++++++++++++++------ 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/lib/core/jmessage.c b/lib/core/jmessage.c index bc9659b28..9045192f0 100644 --- a/lib/core/jmessage.c +++ b/lib/core/jmessage.c @@ -739,13 +739,19 @@ j_message_read(JMessage* message, struct JConnection* connection) GError* error = NULL; + gint32 buffer; + g_return_val_if_fail(message != NULL, FALSE); g_return_val_if_fail(connection != NULL, FALSE); - + g_message("read: msg hedaer: %lu", sizeof(message->header)); + EXE(j_connection_recv(connection, 4, &buffer), "Failed to read stuff"); EXE(j_connection_recv(connection, sizeof(message->header), &(message->header)), "Failed to initiated header receive!"); EXE(j_connection_wait_for_completion(connection), "Failed to wait for header receive!"); + g_print("Header:\n\t"); + for(unsigned i = 0; i < sizeof(message->header); ++i) { g_print("%02x ", ((uint8_t*)&message->header)[i]); } + g_print("\n"); j_message_ensure_size(message, j_message_length(message)); @@ -786,6 +792,10 @@ j_message_write(JMessage* message, struct JConnection* connection) g_return_val_if_fail(message != NULL, FALSE); g_return_val_if_fail(connection != NULL, FALSE); + g_message("send: msg hedaer: %lu", sizeof(message->header)); + g_print("Header:\n\t"); + for(unsigned i = 0; i < sizeof(message->header); ++i) { g_print("%02x ", ((uint8_t*)&message->header)[i]); } + g_print("\n"); EXE(j_connection_send(connection, &(message->header), sizeof(message->header)), "Failed to initiated sending message header."); EXE(j_connection_send(connection, message->data, j_message_length(message)), diff --git a/lib/core/jnetwork.c b/lib/core/jnetwork.c index dad7413e5..8679669e3 100644 --- a/lib/core/jnetwork.c +++ b/lib/core/jnetwork.c @@ -571,7 +571,7 @@ j_connection_send(struct JConnection* this, const void* data, size_t data_len) { int res; gboolean ret = FALSE; - void* segment; + uint8_t* segment; size_t size; // we used paired endponits -> inject and send don't need destination addr (last parameter) @@ -584,11 +584,12 @@ j_connection_send(struct JConnection* this, const void* data, size_t data_len) }*/ // normal send - memcpy((char*)this->memory.buffer + this->memory.used + this->memory.tx_prefix_size, + segment = (uint8_t*)this->memory.buffer + this->memory.used; + memcpy(segment + this->memory.tx_prefix_size, data, data_len); - segment = (char*)this->memory.buffer + this->memory.used; size = data_len + this->memory.tx_prefix_size; do{ + g_message("send: prefix: %lu, size: %lu", this->memory.tx_prefix_size, size); res = fi_send(this->ep, segment, size, fi_mr_desc(this->memory.mr), 0, segment); } while(res == -FI_EAGAIN); CHECK("Failed to initelize sending!"); @@ -613,10 +614,10 @@ j_connection_recv(struct JConnection* this, size_t data_len, void* data) int res; void* segment; size_t size; - g_message("receive"); segment = (char*)this->memory.buffer + this->memory.used; size = data_len + this->memory.rx_prefix_size; res = fi_recv(this->ep, segment, size, fi_mr_desc(this->memory.mr), 0, segment); + g_message("receive: prefix: %lu, size: %lu", this->memory.rx_prefix_size, size); CHECK("Failed to initelized receiving!"); this->memory.used += size; g_message("memory useag: %lu/%lu", this->memory.used, this->memory.buffer_size); @@ -628,7 +629,6 @@ j_connection_recv(struct JConnection* this, size_t data_len, void* data) this->running_actions.entry[this->running_actions.len].len = data_len; this->running_actions.entry[this->running_actions.len].mr = NULL; ++this->running_actions.len; - g_message("fin recive"); ret = TRUE; end: return ret; @@ -644,14 +644,26 @@ j_connection_wait_for_completion(struct JConnection* this) g_message("start waiting"); while(this->running_actions.len) { + bool rx; do { + rx = true; res = fi_cq_sread(this->cq.rx, &entry, 1, NULL, 2); /// \todo handle shutdown msgs! if(res == -FI_EAGAIN) { + rx = false; res = fi_cq_sread(this->cq.tx, &entry, 1, NULL, 2); /// \todo handle shutdown msgs! } } while (res == -FI_EAGAIN); /// \todo error message fetch! - CHECK("Failed to read completion queue!"); + if(res == -FI_EAVAIL) { + struct fi_cq_err_entry err_entry; + res = fi_cq_readerr(rx ? this->cq.rx : this->cq.tx, + &err_entry, 0); + CHECK("Failed to read error of cq!"); + g_warning("Failed to read completion queue\nWidth:\t%s", + fi_cq_strerror(rx ? this->cq.rx : this->cq.tx, + err_entry.prov_errno, err_entry.err_data, NULL, 0)); + goto end; + } else { CHECK("Failed to read completion queue!"); } for(i = 0; i <= this->running_actions.len; ++i) { if(i == this->running_actions.len) { g_warning("unable to find completet context!"); From 20a7e00e1f2847ff9f1dc9bba9faf9e9093fb7b1 Mon Sep 17 00:00:00 2001 From: Julian Benda Date: Wed, 4 Aug 2021 18:18:03 +0200 Subject: [PATCH 17/60] Remove debug code --- lib/core/jmessage.c | 3 --- lib/core/jnetwork.c | 4 ---- 2 files changed, 7 deletions(-) diff --git a/lib/core/jmessage.c b/lib/core/jmessage.c index 9045192f0..05e69949c 100644 --- a/lib/core/jmessage.c +++ b/lib/core/jmessage.c @@ -739,12 +739,9 @@ j_message_read(JMessage* message, struct JConnection* connection) GError* error = NULL; - gint32 buffer; - g_return_val_if_fail(message != NULL, FALSE); g_return_val_if_fail(connection != NULL, FALSE); g_message("read: msg hedaer: %lu", sizeof(message->header)); - EXE(j_connection_recv(connection, 4, &buffer), "Failed to read stuff"); EXE(j_connection_recv(connection, sizeof(message->header), &(message->header)), "Failed to initiated header receive!"); EXE(j_connection_wait_for_completion(connection), diff --git a/lib/core/jnetwork.c b/lib/core/jnetwork.c index 8679669e3..c0b161335 100644 --- a/lib/core/jnetwork.c +++ b/lib/core/jnetwork.c @@ -333,10 +333,6 @@ j_connection_init_client(struct JConfiguration* configuration, enum JBackendType goto end; } - *(gint32*)(this->memory.buffer) = 12; - do { res = fi_inject(this->ep, this->memory.buffer, 4, 0); } while(res == -FI_EAGAIN); - CHECK("failed to inject test message!"); - ret = TRUE; end: /// FIXME memory leak on error in addr.addr From 16b693bbf6c6ee52d4b45d390f059dbafc71ee7e Mon Sep 17 00:00:00 2001 From: Julian Benda Date: Thu, 5 Aug 2021 13:28:03 +0200 Subject: [PATCH 18/60] Hello World works --- example/hello-world.c | 15 +++++--- include/core/jmessage.h | 6 ++-- lib/core/jconnection-pool.c | 21 ++++------- lib/core/jmessage.c | 69 ++++++++++++++++++++++--------------- lib/core/jnetwork.c | 45 +++++++++++++++--------- lib/object/jobject.c | 1 + server/loop.c | 9 ++++- 7 files changed, 98 insertions(+), 68 deletions(-) diff --git a/example/hello-world.c b/example/hello-world.c index 94071853c..fcbebdcd5 100644 --- a/example/hello-world.c +++ b/example/hello-world.c @@ -33,7 +33,7 @@ main(int argc, char** argv) g_autoptr(JKV) kv = NULL; g_autoptr(JObject) object = NULL; - gchar const* hello_world = "Hello World!"; + gchar const* hello_world = g_strdup("Hello World!"); guint64 nbytes; (void)argc; @@ -58,9 +58,10 @@ main(int argc, char** argv) j_kv_put(kv, g_strdup(hello_world), strlen(hello_world) + 1, g_free, batch); j_db_schema_create(schema, batch, NULL); j_db_entry_insert(entry, batch, NULL); - + g_message("start execution!"); if (j_batch_execute(batch)) { + g_message("Execution finished"); g_autoptr(JDBIterator) iterator = NULL; g_autoptr(JDBSelector) selector = NULL; @@ -70,19 +71,23 @@ main(int argc, char** argv) g_autofree gchar* db_field = NULL; guint64 db_length = 0; + g_message("obj read start"); j_object_read(object, buffer, 128, 0, &nbytes, batch); + g_message("obj read fin"); if (j_batch_execute(batch)) { - printf("Object contains: %s (%" G_GUINT64_FORMAT " bytes)\n", buffer, nbytes); + g_message("Object contains: %s (%" G_GUINT64_FORMAT " bytes)\n", buffer, nbytes); } + g_message("batch fin"); j_kv_get(kv, &value, &length, batch); if (j_batch_execute(batch)) { - printf("KV contains: %s (%" G_GUINT32_FORMAT " bytes)\n", (gchar*)value, length); + g_message("KV contains: %s (%" G_GUINT32_FORMAT " bytes)\n", (gchar*)value, length); } + g_message("kv fin"); selector = j_db_selector_new(schema, J_DB_SELECTOR_MODE_AND, NULL); j_db_selector_add_field(selector, "hello", J_DB_SELECTOR_OPERATOR_EQ, hello_world, strlen(hello_world) + 1, NULL); @@ -93,7 +98,7 @@ main(int argc, char** argv) JDBType type; j_db_iterator_get_field(iterator, "hello", &type, (gpointer*)&db_field, &db_length, NULL); - printf("DB contains: %s (%" G_GUINT64_FORMAT " bytes)\n", db_field, db_length); + g_message("DB contains: %s (%" G_GUINT64_FORMAT " bytes)\n", db_field, db_length); } } diff --git a/include/core/jmessage.h b/include/core/jmessage.h index f1aded100..22dc55e37 100644 --- a/include/core/jmessage.h +++ b/include/core/jmessage.h @@ -348,8 +348,10 @@ gboolean j_message_send(JMessage*, struct JConnection*); **/ gboolean j_message_receive(JMessage*, struct JConnection*); /// signal when rma read actions are finished. -/** Sends ACK flag, to signal that host can free data memory. Also wait for all network actions to coplete!*/ -gboolean j_message_send_ack(JMessage*, struct JConnection*); +/** Sends ACK flag, to signal that host can free data memory. Also wait for all network actions to coplete! + * @param message checks if message has memory regions, only then sends ack. Can be NULL to force ack sending + */ +gboolean j_message_send_ack(JMessage* message, struct JConnection*); /** * Reads a message from the network. diff --git a/lib/core/jconnection-pool.c b/lib/core/jconnection-pool.c index bebdff651..2a7c772a7 100644 --- a/lib/core/jconnection-pool.c +++ b/lib/core/jconnection-pool.c @@ -121,39 +121,30 @@ j_connection_pool_fini(void) for (guint i = 0; i < pool->object_len; i++) { - GSocketConnection* connection; + struct JConnection* connection; while ((connection = g_async_queue_try_pop(pool->object_queues[i].queue)) != NULL) - { - g_io_stream_close(G_IO_STREAM(connection), NULL, NULL); - g_object_unref(connection); - } + { j_connection_fini(connection); } g_async_queue_unref(pool->object_queues[i].queue); } for (guint i = 0; i < pool->kv_len; i++) { - GSocketConnection* connection; + struct JConnection* connection; while ((connection = g_async_queue_try_pop(pool->kv_queues[i].queue)) != NULL) - { - g_io_stream_close(G_IO_STREAM(connection), NULL, NULL); - g_object_unref(connection); - } + { j_connection_fini(connection); } g_async_queue_unref(pool->kv_queues[i].queue); } for (guint i = 0; i < pool->db_len; i++) { - GSocketConnection* connection; + struct JConnection* connection; while ((connection = g_async_queue_try_pop(pool->db_queues[i].queue)) != NULL) - { - g_io_stream_close(G_IO_STREAM(connection), NULL, NULL); - g_object_unref(connection); - } + { j_connection_fini(connection); } g_async_queue_unref(pool->db_queues[i].queue); } diff --git a/lib/core/jmessage.c b/lib/core/jmessage.c index 05e69949c..010c5ac41 100644 --- a/lib/core/jmessage.c +++ b/lib/core/jmessage.c @@ -511,18 +511,20 @@ j_message_append_n(JMessage* message, gconstpointer data, gsize length) gboolean j_message_append_memory_id(JMessage* message, const struct JConnectionMemoryID* id) { + g_message("ID size: %lu", id->size); J_TRACE_FUNCTION(NULL); guint32 new_length; + guint64 padding = (guint64)message->current % 8; g_return_val_if_fail(message != NULL, FALSE); g_return_val_if_fail(id != NULL, FALSE); - g_return_val_if_fail(j_message_can_append(message, sizeof(*id)), FALSE); + g_return_val_if_fail(j_message_can_append(message, sizeof(*id)+padding), FALSE); - memcpy(message->current, id, sizeof(*id)); - message->current += sizeof(*id); + memcpy(message->current+padding, id, sizeof(*id)); + message->current += sizeof(*id) + padding; - new_length = j_message_length(message) + sizeof(*id); + new_length = j_message_length(message) + sizeof(*id) + padding; message->header.length = GUINT32_TO_LE(new_length); return TRUE; @@ -629,8 +631,11 @@ j_message_get_memory_id(JMessage* message) g_return_val_if_fail(message != NULL, NULL); - ret = (const void*)message->current; - message->current += sizeof(*ret); + guint64 padding = (guint64)message->current % 8; + + g_message("read mem id: %lu", (message->current + padding) - message->data); + ret = (const void*)(message->current + padding); + message->current += sizeof(*ret) + padding; return ret; } @@ -649,11 +654,12 @@ j_message_receive(JMessage* message, struct JConnection* connection) gboolean j_message_send_ack(JMessage* message, struct JConnection* connection) { + g_message("ACK call"); const JConnectionAck ack = J_CONNECTION_ACK; // No data where send -> no need to acknowledge - if(j_message_get_count(message) == 0) { return TRUE; } - + if(message != NULL && j_message_get_count(message) == 0) { return TRUE; } + g_message("send ack"); EXE(j_connection_wait_for_completion(connection), "Failed to wait to finishe all operations before sending ack!"); EXE(j_connection_send(connection, &ack, sizeof(ack)), @@ -680,6 +686,7 @@ j_message_send(JMessage* message, struct JConnection* connection) int n_memory_regions = 0; struct JConnectionMemory* memory_itr = NULL; struct JConnectionMemory* memory_regions = NULL; + struct JConnectionMemory* memory_regions_end = NULL; struct JConnectionMemoryID mem_id; g_autoptr(JListIterator) iterator = NULL; @@ -689,8 +696,12 @@ j_message_send(JMessage* message, struct JConnection* connection) ret = FALSE; n_memory_regions = j_list_length(message->send_list); - memory_regions = calloc(sizeof(struct JConnectionMemory), n_memory_regions); - memset(memory_regions, 0, sizeof(struct JConnectionMemory) * n_memory_regions); + if(n_memory_regions) { + size_t size = sizeof(struct JConnectionMemory) * n_memory_regions; + memory_regions = malloc(size); + memset(memory_regions, 0, size); + memory_regions_end = memory_regions + n_memory_regions; + } memory_itr = memory_regions; if (message->send_list != NULL) @@ -708,8 +719,10 @@ j_message_send(JMessage* message, struct JConnection* connection) EXE(j_connection_memory_get_id(memory_itr, &mem_id), "Failed to get memory it!"); - EXE(j_message_append_n(message, message_data->header, message_data->header_size), - "Failed to append header"); + if(message_data->header != NULL) { + EXE(j_message_append_n(message, message_data->header, message_data->header_size), + "Failed to append header"); + } EXE(j_message_append_memory_id(message, &mem_id), "Failed to append memory id to message!"); ++memory_itr; @@ -722,9 +735,11 @@ j_message_send(JMessage* message, struct JConnection* connection) if(memory_regions != NULL) { struct JConnectionMemory* mrs = memory_regions; while(memory_regions != memory_itr) { - j_connection_rma_unregister(connection, memory_regions); + j_connection_rma_unregister(connection, memory_regions++); + } + if(memory_itr != memory_regions_end && memory_itr->memory_region) { + j_connection_rma_unregister(connection, memory_itr); } - if(memory_itr->memory_region) { j_connection_rma_unregister(connection, memory_itr); } free(mrs); } return ret; @@ -746,16 +761,15 @@ j_message_read(JMessage* message, struct JConnection* connection) "Failed to initiated header receive!"); EXE(j_connection_wait_for_completion(connection), "Failed to wait for header receive!"); - g_print("Header:\n\t"); - for(unsigned i = 0; i < sizeof(message->header); ++i) { g_print("%02x ", ((uint8_t*)&message->header)[i]); } - g_print("\n"); j_message_ensure_size(message, j_message_length(message)); - EXE(j_connection_recv(connection, j_message_length(message), message->data), - "Failed to initiated message body receive!"); - EXE(j_connection_wait_for_completion(connection), - "Failed to wait for message body receive!"); + if(j_message_length(message) > 0) { + EXE(j_connection_recv(connection, j_message_length(message), message->data), + "Failed to initiated message body receive!"); + EXE(j_connection_wait_for_completion(connection), + "Failed to wait for message body receive!"); + } message->current = message->data; @@ -790,15 +804,14 @@ j_message_write(JMessage* message, struct JConnection* connection) g_return_val_if_fail(connection != NULL, FALSE); g_message("send: msg hedaer: %lu", sizeof(message->header)); - g_print("Header:\n\t"); - for(unsigned i = 0; i < sizeof(message->header); ++i) { g_print("%02x ", ((uint8_t*)&message->header)[i]); } - g_print("\n"); EXE(j_connection_send(connection, &(message->header), sizeof(message->header)), "Failed to initiated sending message header."); - EXE(j_connection_send(connection, message->data, j_message_length(message)), - "Failed to initiated sending message body."); + if(j_message_length(message) > 0) { + EXE(j_connection_send(connection, message->data, j_message_length(message)), + "Failed to initiated sending message body."); + } - if(message->send_list != NULL) { + if(j_list_length(message->send_list)) { EXE(j_connection_recv(connection, sizeof(ack), &ack), "Failed to initiated ACK receive."); } @@ -806,7 +819,7 @@ j_message_write(JMessage* message, struct JConnection* connection) EXE(j_connection_wait_for_completion(connection), "Failed to wait for completion of message send!"); - if(message->send_list != NULL && ack != J_CONNECTION_ACK) { + if(j_list_length(message->send_list) && ack != J_CONNECTION_ACK) { g_warning("Wrong ACK flag received! got: %i, instead of: %i", ack, J_CONNECTION_ACK); goto end; } diff --git a/lib/core/jnetwork.c b/lib/core/jnetwork.c index c0b161335..f38d430cd 100644 --- a/lib/core/jnetwork.c +++ b/lib/core/jnetwork.c @@ -92,29 +92,29 @@ struct JConnectionMemory { }; #define EXE(cmd, ...) do { if (cmd == FALSE) { g_warning(__VA_ARGS__); goto end; } } while(FALSE) -#define CHECK(msg) do { if (res < 0) { g_warning("%s: "msg"\nDetails:\t%s", "??TODO??", fi_strerror(-res)); goto end; } } while(FALSE) +#define CHECK(msg) do { if (res < 0) { g_warning("%s: "msg"\t(%s:%d)\nDetails:\t%s", "??TODO??", __FILE__, __LINE__,fi_strerror(-res)); goto end; } } while(FALSE) #define G_CHECK(msg) do { \ if(error != NULL) {\ g_warning(msg"\n\tWith:%s", error->message);\ g_error_free(error);\ goto end; } } while(FALSE) -void free_dangeling_infos(struct fi_info*); +void free_dangling_infos(struct fi_info*); -void free_dangeling_infos(struct fi_info* info) { +void free_dangling_infos(struct fi_info* info) { fi_freeinfo(info->next); info->next = NULL; } gboolean -j_fabric_init_server(struct JConfiguration* configuration, struct JFabric** instnace_ptr) +j_fabric_init_server(struct JConfiguration* configuration, struct JFabric** instance_ptr) { struct JFabric* this; int res; size_t addrlen; - *instnace_ptr = malloc(sizeof(*this)); - this = *instnace_ptr; + *instance_ptr = malloc(sizeof(*this)); + this = *instance_ptr; memset(this, 0, sizeof(*this)); this->config = configuration; @@ -126,7 +126,7 @@ j_fabric_init_server(struct JConfiguration* configuration, struct JFabric** inst j_configuration_get_libfabric_hints(this->config), &this->info); CHECK("Failed to find fabric for server!"); - free_dangeling_infos(this->info); + free_dangling_infos(this->info); // no context needed, because we are in sync res = fi_fabric(this->info->fabric_attr, &this->fabric, NULL); @@ -197,7 +197,7 @@ j_fabric_init_client(struct JConfiguration* configuration, struct JFabricAddr* a NULL, NULL, 0, hints, &this->info); CHECK("Failed to find fabric!"); - free_dangeling_infos(this->info); + free_dangling_infos(this->info); res = fi_fabric(this->info->fabric_attr, &this->fabric, NULL); CHECK("failed to initelize client fabric!"); @@ -274,7 +274,7 @@ j_connection_init_client(struct JConfiguration* configuration, enum JBackendType g_autoptr(GSocketClient) g_client = NULL; GSocketConnection* g_connection; GInputStream* g_input; - GError* error; + GError* error = NULL; const gchar* server; struct JFabricAddr jf_addr; enum JConnectionEvents event; @@ -346,7 +346,7 @@ j_connection_init_server(struct JFabric* fabric, GSocketConnection* gconnection, gboolean ret = FALSE; int res; GOutputStream* g_out; - GError* error; + GError* error = NULL; struct JFabricAddr* addr = &fabric->fabric_addr_network; enum JFabricEvents event; enum JConnectionEvents con_event; @@ -486,7 +486,6 @@ j_connection_create_memory_resources(struct JConnection* this) this->memory.buffer = malloc(size); this->memory.rx_prefix_size = rx_prefix * prefix_size; this->memory.tx_prefix_size = tx_prefix * prefix_size; - this->memory.used = 0; res = fi_mr_reg(this->domain, this->memory.buffer, this->memory.buffer_size, FI_SEND | FI_RECV, 0, 0, 0, &this->memory.mr, NULL); CHECK("Failed to register memory for msg communication!"); @@ -572,7 +571,6 @@ j_connection_send(struct JConnection* this, const void* data, size_t data_len) // we used paired endponits -> inject and send don't need destination addr (last parameter) - g_message("send"); /*if(data_len < this->inject_size) { do { res = fi_inject(this->ep, data, data_len, 0); } while(res == -FI_EAGAIN); CHECK("Failed to inject data!"); @@ -617,7 +615,6 @@ j_connection_recv(struct JConnection* this, size_t data_len, void* data) CHECK("Failed to initelized receiving!"); this->memory.used += size; g_message("memory useag: %lu/%lu", this->memory.used, this->memory.buffer_size); - g_message("actions: %i", this->running_actions.len); g_assert_true(this->memory.used <= this->memory.buffer_size); g_assert_true(this->running_actions.len < J_CONNECTION_MAX_SEND + J_CONNECTION_MAX_RECV); this->running_actions.entry[this->running_actions.len].context = segment; @@ -673,8 +670,7 @@ j_connection_wait_for_completion(struct JConnection* this) this->running_actions.entry[i].len); } if (this->running_actions.entry[i].mr) { - res = fi_close( - &this->running_actions.entry[i].mr->fid); + res = fi_close(&this->running_actions.entry[i].mr->fid); CHECK("Failed to free receiving memory!"); } this->running_actions.entry[i] = this->running_actions.entry[this->running_actions.len]; @@ -696,13 +692,20 @@ j_connection_rma_register(struct JConnection* this, const void* data, size_t dat int res; gboolean ret = FALSE; + { // TODO: DEBUG + for(unsigned i = 0; i < data_len; ++i) { + g_print("%c ", ((const char*)data)[i]); + } + g_print("\n"); + } + void* ptr = g_strdup(data); // TODO: change! res = fi_mr_reg(this->domain, - data, + ptr, data_len, FI_REMOTE_READ, 0, 0, 0, &handle->memory_region, NULL); CHECK("Failed to register memory region!"); - handle->addr = (guint64)data; + handle->addr = (guint64)ptr; handle->size = data_len; ret = TRUE; @@ -714,7 +717,9 @@ gboolean j_connection_rma_unregister(struct JConnection* this, struct JConnectionMemory* handle) { int res; + g_message("close memory start"); res = fi_close(&handle->memory_region->fid); + g_message("close memory fin"); CHECK("Failed to unregistrer rma memory!"); return TRUE; end: @@ -740,6 +745,7 @@ j_connection_rma_read(struct JConnection* this, const struct JConnectionMemoryID res = fi_mr_reg(this->domain, data, memoryID->size, FI_READ, 0, 0, 0, &mr, 0); CHECK("Failed to register receiving memory!"); + g_message("try to read: %lu, %lu", memoryID->offset, memoryID->key); res = fi_read(this->ep, data, memoryID->size, @@ -759,3 +765,8 @@ j_connection_rma_read(struct JConnection* this, const struct JConnectionMemoryID end: return ret; } + +gboolean +j_connection_fini ( struct JConnection* instance) { + // TODO: implement! +} diff --git a/lib/object/jobject.c b/lib/object/jobject.c index 1d28fb2b0..2cd697e93 100644 --- a/lib/object/jobject.c +++ b/lib/object/jobject.c @@ -558,6 +558,7 @@ j_object_read_exec(JList* operations, JSemantics* semantics) if (mem_id->size > 0) { + g_message("RMA read"); j_connection_rma_read(object_connection, mem_id, data); } } diff --git a/server/loop.c b/server/loop.c index a5ed41f84..9a1437c6c 100644 --- a/server/loop.c +++ b/server/loop.c @@ -284,8 +284,9 @@ jd_handle_message(JMessage* message, struct JConnection* connection, JMemoryChun guint64 offset; guint64 bytes_written = 0; - memoryID = j_message_get_memory_id(message); offset = j_message_get_8(message); + memoryID = j_message_get_memory_id(message); + g_message("memory size: %lu", memoryID->size); if (memoryID->size > memory_chunk_size && reply != NULL && G_LIKELY(ret)) { @@ -300,6 +301,9 @@ jd_handle_message(JMessage* message, struct JConnection* connection, JMemoryChun g_assert(buf != NULL); j_connection_rma_read(connection, memoryID, buf); + // TODO: pararlleize + j_connection_wait_for_completion(connection); + g_message("rma read message: %s", buf); j_statistics_add(statistics, J_STATISTICS_BYTES_RECEIVED, memoryID->size); // CONTINUE @@ -329,6 +333,9 @@ jd_handle_message(JMessage* message, struct JConnection* connection, JMemoryChun j_backend_object_close(jd_object_backend, object); } + if(operation_count) { + j_message_send_ack(NULL, connection); + } if (reply != NULL) { j_message_send(reply, connection); From 279418ead2e49330833da50ac57a17a55db634df Mon Sep 17 00:00:00 2001 From: Julian Benda Date: Thu, 5 Aug 2021 13:35:41 +0200 Subject: [PATCH 19/60] Remove prints and dummy vars --- example/hello-world.c | 7 +---- lib/core/jmessage.c | 12 ++++----- lib/core/jnetwork.c | 59 +++++++++++++++++++------------------------ lib/object/jobject.c | 1 - server/loop.c | 4 +-- server/server.c | 2 -- 6 files changed, 35 insertions(+), 50 deletions(-) diff --git a/example/hello-world.c b/example/hello-world.c index fcbebdcd5..76e1348b5 100644 --- a/example/hello-world.c +++ b/example/hello-world.c @@ -58,10 +58,9 @@ main(int argc, char** argv) j_kv_put(kv, g_strdup(hello_world), strlen(hello_world) + 1, g_free, batch); j_db_schema_create(schema, batch, NULL); j_db_entry_insert(entry, batch, NULL); - g_message("start execution!"); + if (j_batch_execute(batch)) { - g_message("Execution finished"); g_autoptr(JDBIterator) iterator = NULL; g_autoptr(JDBSelector) selector = NULL; @@ -71,15 +70,12 @@ main(int argc, char** argv) g_autofree gchar* db_field = NULL; guint64 db_length = 0; - g_message("obj read start"); j_object_read(object, buffer, 128, 0, &nbytes, batch); - g_message("obj read fin"); if (j_batch_execute(batch)) { g_message("Object contains: %s (%" G_GUINT64_FORMAT " bytes)\n", buffer, nbytes); } - g_message("batch fin"); j_kv_get(kv, &value, &length, batch); @@ -87,7 +83,6 @@ main(int argc, char** argv) { g_message("KV contains: %s (%" G_GUINT32_FORMAT " bytes)\n", (gchar*)value, length); } - g_message("kv fin"); selector = j_db_selector_new(schema, J_DB_SELECTOR_MODE_AND, NULL); j_db_selector_add_field(selector, "hello", J_DB_SELECTOR_OPERATOR_EQ, hello_world, strlen(hello_world) + 1, NULL); diff --git a/lib/core/jmessage.c b/lib/core/jmessage.c index 010c5ac41..ba85f164f 100644 --- a/lib/core/jmessage.c +++ b/lib/core/jmessage.c @@ -511,7 +511,7 @@ j_message_append_n(JMessage* message, gconstpointer data, gsize length) gboolean j_message_append_memory_id(JMessage* message, const struct JConnectionMemoryID* id) { - g_message("ID size: %lu", id->size); + // g_message("ID size: %lu", id->size); J_TRACE_FUNCTION(NULL); guint32 new_length; @@ -633,7 +633,7 @@ j_message_get_memory_id(JMessage* message) guint64 padding = (guint64)message->current % 8; - g_message("read mem id: %lu", (message->current + padding) - message->data); + // g_message("read mem id: %lu", (message->current + padding) - message->data); ret = (const void*)(message->current + padding); message->current += sizeof(*ret) + padding; @@ -654,12 +654,12 @@ j_message_receive(JMessage* message, struct JConnection* connection) gboolean j_message_send_ack(JMessage* message, struct JConnection* connection) { - g_message("ACK call"); + // g_message("ACK call"); const JConnectionAck ack = J_CONNECTION_ACK; // No data where send -> no need to acknowledge if(message != NULL && j_message_get_count(message) == 0) { return TRUE; } - g_message("send ack"); + // g_message("send ack"); EXE(j_connection_wait_for_completion(connection), "Failed to wait to finishe all operations before sending ack!"); EXE(j_connection_send(connection, &ack, sizeof(ack)), @@ -756,7 +756,7 @@ j_message_read(JMessage* message, struct JConnection* connection) g_return_val_if_fail(message != NULL, FALSE); g_return_val_if_fail(connection != NULL, FALSE); - g_message("read: msg hedaer: %lu", sizeof(message->header)); + // g_message("read: msg hedaer: %lu", sizeof(message->header)); EXE(j_connection_recv(connection, sizeof(message->header), &(message->header)), "Failed to initiated header receive!"); EXE(j_connection_wait_for_completion(connection), @@ -803,7 +803,7 @@ j_message_write(JMessage* message, struct JConnection* connection) g_return_val_if_fail(message != NULL, FALSE); g_return_val_if_fail(connection != NULL, FALSE); - g_message("send: msg hedaer: %lu", sizeof(message->header)); + // g_message("send: msg hedaer: %lu", sizeof(message->header)); EXE(j_connection_send(connection, &(message->header), sizeof(message->header)), "Failed to initiated sending message header."); if(j_message_length(message) > 0) { diff --git a/lib/core/jnetwork.c b/lib/core/jnetwork.c index f38d430cd..558a5d0bf 100644 --- a/lib/core/jnetwork.c +++ b/lib/core/jnetwork.c @@ -146,13 +146,13 @@ j_fabric_init_server(struct JConfiguration* configuration, struct JFabric** inst res = fi_getname(&this->pep->fid, NULL, &addrlen); if(res != -FI_ETOOSMALL) { CHECK("failed to fetch address len from libfabirc!"); } this->fabric_addr_network.addr_len = addrlen; - g_message("test: %u", this->fabric_addr_network.addr_len); + // g_message("test: %u", this->fabric_addr_network.addr_len); this->fabric_addr_network.addr = malloc(this->fabric_addr_network.addr_len); res = fi_getname(&this->pep->fid, this->fabric_addr_network.addr, &addrlen); CHECK("failed to fetch address from libfabric!"); - g_message("wait for pep request!"); + // g_message("wait for pep request!"); res = fi_listen(this->pep); CHECK("failed to start listening on pep!"); @@ -188,7 +188,7 @@ j_fabric_init_client(struct JConfiguration* configuration, struct JFabricAddr* a char* str = malloc(hints->dest_addrlen * 3 + 1); size_t i; for(i = 0; i < hints->dest_addrlen; ++i) { snprintf(str+i*3, 4, "%02x ", ((uint8_t*)hints->dest_addr)[i]); } - g_message("Addr: format: %u, len: %lu\n\t%s", hints->addr_format, hints->dest_addrlen, str); + // g_message("Addr: format: %u, len: %lu\n\t%s", hints->addr_format, hints->dest_addrlen, str); free(str); } @@ -203,7 +203,7 @@ j_fabric_init_client(struct JConfiguration* configuration, struct JFabricAddr* a CHECK("failed to initelize client fabric!"); ret = TRUE; - g_message("initelize client!"); + // g_message("initelize client!"); end: return ret; } @@ -326,7 +326,7 @@ j_connection_init_client(struct JConfiguration* configuration, enum JBackendType do {EXE(j_connection_sread_event(this, 1, &event), "Failed to read event queue, waiting for CONNECTED signal!"); } while(event == J_CONNECTION_EVENT_TIMEOUT); - g_message("Event FI: %u", event); + // g_message("Event FI: %u", event); if(event != J_CONNECTION_EVENT_CONNECTED) { g_warning("Failed to connect to host!"); @@ -354,16 +354,16 @@ j_connection_init_server(struct JFabric* fabric, GSocketConnection* gconnection, *instance_ptr = malloc(sizeof(*this)); this = *instance_ptr; - g_message("debug: size(*this) = %lu, this = %p", sizeof(*this), (void*)this); + // g_message("debug: size(*this) = %lu, this = %p", sizeof(*this), (void*)this); memset(this, 0, sizeof(*this)); // send addr { // DEBUG TODO char* str = malloc(ntohl(addr->addr_len) * 3 + 1); size_t i; - g_message("tada: addr_len: %u", ntohl(addr->addr_len)); + // g_message("tada: addr_len: %u", ntohl(addr->addr_len)); for(i = 0; i < ntohl(addr->addr_len); ++i) { snprintf(str+i*3, 4, "%02x ", ((uint8_t*)addr->addr)[i]); } - g_message("Addr: format: %u, len: %u\n\t%s", ntohl(addr->addr_format), ntohl(addr->addr_len), str); + // g_message("Addr: format: %u, len: %u\n\t%s", ntohl(addr->addr_format), ntohl(addr->addr_len), str); free(str); } g_out = g_io_stream_get_output_stream(G_IO_STREAM(gconnection)); @@ -373,11 +373,11 @@ j_connection_init_server(struct JFabric* fabric, GSocketConnection* gconnection, G_CHECK("Failed to write addr_len to stream!"); g_output_stream_write(g_out, addr->addr, ntohl(addr->addr_len), NULL, &error); G_CHECK("Failed to write addr to stream!"); - g_message("close g_stream"); + // g_message("close g_stream"); g_output_stream_close(g_out, NULL, &error); G_CHECK("Failed to close output stream!"); - g_message("wait for conrequest!"); + // g_message("wait for conrequest!"); do {EXE(j_fabric_sread_event(fabric, 2, &event, &request), "Failed to wait for connection request"); } while(event == J_FABRIC_EVENT_TIMEOUT); @@ -385,16 +385,16 @@ j_connection_init_server(struct JFabric* fabric, GSocketConnection* gconnection, g_warning("expected an connection request and nothing else! (%i)", event); goto end; } - g_message("inet connection!"); + // g_message("inet connection!"); this->fabric = fabric; this->info = request.info; EXE(j_connection_init(this), "Failed to initelize connection server side!"); - g_message("Except!"); + // g_message("Except!"); res = fi_accept(this->ep, NULL, 0); CHECK("Failed to accept connection!"); - g_message("wait for connection !"); + // g_message("wait for connection !"); EXE(j_connection_sread_event(this, 2, &con_event), "Failed to verify connection!"); if(con_event != J_CONNECTION_EVENT_CONNECTED) { g_warning("expected and connection ack and nothing else!"); @@ -450,7 +450,7 @@ j_connection_init(struct JConnection* this) CHECK("Failed to enable connection!"); ret = TRUE; - g_message("enabled endpoint!"); + // g_message("enabled endpoint!"); end: return ret; } @@ -478,7 +478,7 @@ j_connection_create_memory_resources(struct JConnection* this) size = 0; if(this->info->domain_attr->mr_mode & FI_MR_LOCAL) { - g_message("Localmemory!"); + // g_message("Localmemory!"); size += (rx_prefix * prefix_size) + J_CONNECTION_MAX_RECV * op_size + (tx_prefix * prefix_size) + J_CONNECTION_MAX_SEND * op_size; @@ -522,7 +522,7 @@ j_connection_sread_event(struct JConnection* this, int timeout, enum JConnection goto end; /// \todo consider return TRUE on event queue error } CHECK("Failed to read event of connection!"); - g_message("Event: %u", fi_event); + // g_message("Event: %u", fi_event); switch(fi_event) { case FI_CONNECTED: *event = J_CONNECTION_EVENT_CONNECTED; break; @@ -583,7 +583,7 @@ j_connection_send(struct JConnection* this, const void* data, size_t data_len) data, data_len); size = data_len + this->memory.tx_prefix_size; do{ - g_message("send: prefix: %lu, size: %lu", this->memory.tx_prefix_size, size); + // g_message("send: prefix: %lu, size: %lu", this->memory.tx_prefix_size, size); res = fi_send(this->ep, segment, size, fi_mr_desc(this->memory.mr), 0, segment); } while(res == -FI_EAGAIN); CHECK("Failed to initelize sending!"); @@ -611,10 +611,10 @@ j_connection_recv(struct JConnection* this, size_t data_len, void* data) segment = (char*)this->memory.buffer + this->memory.used; size = data_len + this->memory.rx_prefix_size; res = fi_recv(this->ep, segment, size, fi_mr_desc(this->memory.mr), 0, segment); - g_message("receive: prefix: %lu, size: %lu", this->memory.rx_prefix_size, size); + // g_message("receive: prefix: %lu, size: %lu", this->memory.rx_prefix_size, size); CHECK("Failed to initelized receiving!"); this->memory.used += size; - g_message("memory useag: %lu/%lu", this->memory.used, this->memory.buffer_size); + // g_message("memory useag: %lu/%lu", this->memory.used, this->memory.buffer_size); g_assert_true(this->memory.used <= this->memory.buffer_size); g_assert_true(this->running_actions.len < J_CONNECTION_MAX_SEND + J_CONNECTION_MAX_RECV); this->running_actions.entry[this->running_actions.len].context = segment; @@ -635,7 +635,7 @@ j_connection_wait_for_completion(struct JConnection* this) struct fi_cq_entry entry; int i; - g_message("start waiting"); + // g_message("start waiting"); while(this->running_actions.len) { bool rx; do { @@ -678,7 +678,7 @@ j_connection_wait_for_completion(struct JConnection* this) } } } - g_message("fin waiting"); + // g_message("fin waiting"); this->running_actions.len = 0; ret = TRUE; @@ -692,20 +692,13 @@ j_connection_rma_register(struct JConnection* this, const void* data, size_t dat int res; gboolean ret = FALSE; - { // TODO: DEBUG - for(unsigned i = 0; i < data_len; ++i) { - g_print("%c ", ((const char*)data)[i]); - } - g_print("\n"); - } - void* ptr = g_strdup(data); // TODO: change! res = fi_mr_reg(this->domain, - ptr, + data, data_len, FI_REMOTE_READ, 0, 0, 0, &handle->memory_region, NULL); CHECK("Failed to register memory region!"); - handle->addr = (guint64)ptr; + handle->addr = (guint64)data; handle->size = data_len; ret = TRUE; @@ -717,9 +710,9 @@ gboolean j_connection_rma_unregister(struct JConnection* this, struct JConnectionMemory* handle) { int res; - g_message("close memory start"); + // g_message("close memory start"); res = fi_close(&handle->memory_region->fid); - g_message("close memory fin"); + // g_message("close memory fin"); CHECK("Failed to unregistrer rma memory!"); return TRUE; end: @@ -745,7 +738,7 @@ j_connection_rma_read(struct JConnection* this, const struct JConnectionMemoryID res = fi_mr_reg(this->domain, data, memoryID->size, FI_READ, 0, 0, 0, &mr, 0); CHECK("Failed to register receiving memory!"); - g_message("try to read: %lu, %lu", memoryID->offset, memoryID->key); + // g_message("try to read: %lu, %lu", memoryID->offset, memoryID->key); res = fi_read(this->ep, data, memoryID->size, diff --git a/lib/object/jobject.c b/lib/object/jobject.c index 2cd697e93..1d28fb2b0 100644 --- a/lib/object/jobject.c +++ b/lib/object/jobject.c @@ -558,7 +558,6 @@ j_object_read_exec(JList* operations, JSemantics* semantics) if (mem_id->size > 0) { - g_message("RMA read"); j_connection_rma_read(object_connection, mem_id, data); } } diff --git a/server/loop.c b/server/loop.c index 9a1437c6c..410b1d499 100644 --- a/server/loop.c +++ b/server/loop.c @@ -286,7 +286,7 @@ jd_handle_message(JMessage* message, struct JConnection* connection, JMemoryChun offset = j_message_get_8(message); memoryID = j_message_get_memory_id(message); - g_message("memory size: %lu", memoryID->size); + // g_message("memory size: %lu", memoryID->size); if (memoryID->size > memory_chunk_size && reply != NULL && G_LIKELY(ret)) { @@ -303,7 +303,7 @@ jd_handle_message(JMessage* message, struct JConnection* connection, JMemoryChun j_connection_rma_read(connection, memoryID, buf); // TODO: pararlleize j_connection_wait_for_completion(connection); - g_message("rma read message: %s", buf); + // g_message("rma read message: %s", buf); j_statistics_add(statistics, J_STATISTICS_BYTES_RECEIVED, memoryID->size); // CONTINUE diff --git a/server/server.c b/server/server.c index 9bb2064cd..258ee6d8c 100644 --- a/server/server.c +++ b/server/server.c @@ -80,7 +80,6 @@ jd_on_run(GThreadedSocketService* service, GSocketConnection* gconnection, GObje message = j_message_new(J_MESSAGE_NONE, 0); - g_message("fabric: %p", (void*)jd_fabric); j_connection_init_server(jd_fabric, gconnection, &jconnection); while (j_message_receive(message, jconnection)) @@ -383,7 +382,6 @@ main(int argc, char** argv) g_warning("Failed to initialize server fabric!"); return 1; } - g_message("fabric: %p", (void*)jd_fabric); jd_statistics = j_statistics_new(FALSE); g_mutex_init(jd_statistics_mutex); From f31b35e72b8319bdb3fc5b69a18aa8cddcca7200 Mon Sep 17 00:00:00 2001 From: Julian Benda Date: Thu, 5 Aug 2021 14:32:45 +0200 Subject: [PATCH 20/60] Remove hints memory leak --- lib/core/jnetwork.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/core/jnetwork.c b/lib/core/jnetwork.c index 558a5d0bf..96323386e 100644 --- a/lib/core/jnetwork.c +++ b/lib/core/jnetwork.c @@ -205,6 +205,7 @@ j_fabric_init_client(struct JConfiguration* configuration, struct JFabricAddr* a ret = TRUE; // g_message("initelize client!"); end: + fi_freeinfo(hints); return ret; } @@ -762,4 +763,5 @@ j_connection_rma_read(struct JConnection* this, const struct JConnectionMemoryID gboolean j_connection_fini ( struct JConnection* instance) { // TODO: implement! + return TRUE; } From 9cd6dd049b88e8e36f3faaca966a065509a0a1ca Mon Sep 17 00:00:00 2001 From: Julian Benda Date: Fri, 13 Aug 2021 13:26:38 +0200 Subject: [PATCH 21/60] Free message buffer memory after finish tranmition --- lib/core/jnetwork.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/core/jnetwork.c b/lib/core/jnetwork.c index 96323386e..ea9b78d17 100644 --- a/lib/core/jnetwork.c +++ b/lib/core/jnetwork.c @@ -483,6 +483,7 @@ j_connection_create_memory_resources(struct JConnection* this) size += (rx_prefix * prefix_size) + J_CONNECTION_MAX_RECV * op_size + (tx_prefix * prefix_size) + J_CONNECTION_MAX_SEND * op_size; + this->memory.used = 0; this->memory.buffer_size = size; this->memory.buffer = malloc(size); this->memory.rx_prefix_size = rx_prefix * prefix_size; @@ -682,6 +683,7 @@ j_connection_wait_for_completion(struct JConnection* this) // g_message("fin waiting"); this->running_actions.len = 0; + this->memory.used = 0; ret = TRUE; end: return ret; From 71d6438205af4ec5bc2c0a2eb30eb97e96aae68a Mon Sep 17 00:00:00 2001 From: Julian Benda Date: Sun, 15 Aug 2021 11:44:07 +0200 Subject: [PATCH 22/60] Enable injecting of small messages --- lib/core/jnetwork.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/core/jnetwork.c b/lib/core/jnetwork.c index ea9b78d17..ed7906252 100644 --- a/lib/core/jnetwork.c +++ b/lib/core/jnetwork.c @@ -573,11 +573,11 @@ j_connection_send(struct JConnection* this, const void* data, size_t data_len) // we used paired endponits -> inject and send don't need destination addr (last parameter) - /*if(data_len < this->inject_size) { + if(data_len < this->inject_size) { do { res = fi_inject(this->ep, data, data_len, 0); } while(res == -FI_EAGAIN); CHECK("Failed to inject data!"); ret = TRUE; goto end; - }*/ + } // normal send segment = (uint8_t*)this->memory.buffer + this->memory.used; From 304b596022db6d3b1109945e2338ca1d15fded61 Mon Sep 17 00:00:00 2001 From: Julian Benda Date: Sun, 15 Aug 2021 15:24:20 +0200 Subject: [PATCH 23/60] Fix new header ack communication for distrubted-objectk --- include/core/jnetwork.h | 1 + lib/core/jmessage.c | 33 +++++++++++++++++++------------- lib/core/jnetwork.c | 5 +++-- lib/object/jdistributed-object.c | 14 ++++++-------- server/loop.c | 3 +-- 5 files changed, 31 insertions(+), 25 deletions(-) diff --git a/include/core/jnetwork.h b/include/core/jnetwork.h index f616eecd7..63d1bc23d 100644 --- a/include/core/jnetwork.h +++ b/include/core/jnetwork.h @@ -389,6 +389,7 @@ j_connection_recv(struct JConnection* instance, * * Initiate an direct memory read, to wait for completion use j_connection_wait_for_completion(). * \retval FALSE if an error occurred -> handle will then also invalid + * \todo evaluate if paralisation possible */ gboolean j_connection_rma_read(struct JConnection* instance, diff --git a/lib/core/jmessage.c b/lib/core/jmessage.c index ba85f164f..773dbcd55 100644 --- a/lib/core/jmessage.c +++ b/lib/core/jmessage.c @@ -628,10 +628,11 @@ j_message_get_memory_id(JMessage* message) J_TRACE_FUNCTION(NULL); const struct JConnectionMemoryID* ret; + guint64 padding; g_return_val_if_fail(message != NULL, NULL); - guint64 padding = (guint64)message->current % 8; + padding = (guint64)message->current % 8; // g_message("read mem id: %lu", (message->current + padding) - message->data); ret = (const void*)(message->current + padding); @@ -654,7 +655,6 @@ j_message_receive(JMessage* message, struct JConnection* connection) gboolean j_message_send_ack(JMessage* message, struct JConnection* connection) { - // g_message("ACK call"); const JConnectionAck ack = J_CONNECTION_ACK; // No data where send -> no need to acknowledge @@ -719,9 +719,14 @@ j_message_send(JMessage* message, struct JConnection* connection) EXE(j_connection_memory_get_id(memory_itr, &mem_id), "Failed to get memory it!"); - if(message_data->header != NULL) { - EXE(j_message_append_n(message, message_data->header, message_data->header_size), - "Failed to append header"); + if(message_data->header_size > 0) { + if(message_data->header_size <= sizeof(message_data->header)) { + EXE(j_message_append_n(message, &message_data->header, message_data->header_size), + "Failed to append header"); + } else { + EXE(j_message_append_n(message, message_data->header, message_data->header_size), + "Failed to append header"); + } } EXE(j_message_append_memory_id(message, &mem_id), "Failed to append memory id to message!"); @@ -797,7 +802,6 @@ j_message_write(JMessage* message, struct JConnection* connection) gboolean ret = FALSE; - GError* error = NULL; JConnectionAck ack; g_return_val_if_fail(message != NULL, FALSE); @@ -827,11 +831,6 @@ j_message_write(JMessage* message, struct JConnection* connection) ret = TRUE; end: - if (error != NULL) - { - g_critical("%s", error->message); - g_error_free(error); - } return ret; } @@ -849,8 +848,16 @@ j_message_add_send(JMessage* message, gconstpointer data, guint64 length, void* message_data = g_slice_new(JMessageData); message_data->data = data; message_data->length = length; - message_data->header = header; - message_data->header_size = h_size; + if(h_size == 0) { + message_data->header = NULL; + message_data->header_size = 0; + } else if(h_size <= sizeof(message_data->header)) { + memcpy(&message_data->header, header, h_size); + message_data->header_size = h_size; + } else { + message_data->header = header; + message_data->header_size = h_size; + } j_list_append(message->send_list, message_data); } diff --git a/lib/core/jnetwork.c b/lib/core/jnetwork.c index ed7906252..3b779bf95 100644 --- a/lib/core/jnetwork.c +++ b/lib/core/jnetwork.c @@ -288,7 +288,7 @@ j_connection_init_client(struct JConfiguration* configuration, enum JBackendType server = j_configuration_get_server(configuration, backend, index); g_connection = g_socket_client_connect_to_host(g_client, "10.0.10.1", - 4711, NULL, &error); + 4711, NULL, &error); ///< \todo: !!!! read from config G_CHECK("Failed to build gsocket connection to host"); if(g_connection == NULL) { g_warning("Can not connect to %s.", server); @@ -610,6 +610,7 @@ j_connection_recv(struct JConnection* this, size_t data_len, void* data) int res; void* segment; size_t size; + segment = (char*)this->memory.buffer + this->memory.used; size = data_len + this->memory.rx_prefix_size; res = fi_recv(this->ep, segment, size, fi_mr_desc(this->memory.mr), 0, segment); @@ -741,7 +742,7 @@ j_connection_rma_read(struct JConnection* this, const struct JConnectionMemoryID res = fi_mr_reg(this->domain, data, memoryID->size, FI_READ, 0, 0, 0, &mr, 0); CHECK("Failed to register receiving memory!"); - // g_message("try to read: %lu, %lu", memoryID->offset, memoryID->key); + res = fi_read(this->ep, data, memoryID->size, diff --git a/lib/object/jdistributed-object.c b/lib/object/jdistributed-object.c index c6ce68ce0..3e25fa740 100644 --- a/lib/object/jdistributed-object.c +++ b/lib/object/jdistributed-object.c @@ -365,21 +365,19 @@ j_distributed_object_read_background_operation(gpointer data) gchar* read_data = buffer->data; guint64* bytes_read = buffer->bytes_read; - guint64 nbytes; + const struct JConnectionMemoryID* memoryID; - nbytes = j_message_get_8(reply); - j_helper_atomic_add(bytes_read, nbytes); + memoryID = j_message_get_memory_id(reply); + j_helper_atomic_add(bytes_read, memoryID->size); - if (nbytes > 0) + if (memoryID->size > 0) { - GInputStream* input; - - input = g_io_stream_get_input_stream(G_IO_STREAM(object_connection)); - g_input_stream_read_all(input, read_data, nbytes, NULL, NULL, NULL); + j_connection_rma_read(object_connection, memoryID, read_data); } g_slice_free(JDistributedObjectReadBuffer, buffer); } + j_message_send_ack(reply, object_connection); operations_done += reply_operation_count; } diff --git a/server/loop.c b/server/loop.c index 410b1d499..f39736ecd 100644 --- a/server/loop.c +++ b/server/loop.c @@ -301,8 +301,7 @@ jd_handle_message(JMessage* message, struct JConnection* connection, JMemoryChun g_assert(buf != NULL); j_connection_rma_read(connection, memoryID, buf); - // TODO: pararlleize - j_connection_wait_for_completion(connection); + j_connection_wait_for_completion(connection); ///< \todo pararlleize // g_message("rma read message: %s", buf); j_statistics_add(statistics, J_STATISTICS_BYTES_RECEIVED, memoryID->size); // CONTINUE From 48f590d7a28f1f92bb2d34df0453b6b8a445057e Mon Sep 17 00:00:00 2001 From: Julian Benda Date: Sun, 15 Aug 2021 15:47:13 +0200 Subject: [PATCH 24/60] Disable test test case assumed communication via GSocket, we need to redesign this test to work with the new api --- test/core/message.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/core/message.c b/test/core/message.c index 90cd81b84..cce21a808 100644 --- a/test/core/message.c +++ b/test/core/message.c @@ -118,10 +118,11 @@ test_message_append(void) J_TEST_TRAP_END; } +/// @todo adapt to new network static void test_message_write_read(void) { - g_autoptr(JMessage) message_recv = NULL; + /*g_autoptr(JMessage) message_recv = NULL; g_autoptr(JMessage) message_send = NULL; g_autoptr(GOutputStream) output = NULL; g_autoptr(GInputStream) input = NULL; @@ -173,7 +174,7 @@ test_message_write_read(void) g_assert_cmpstr(dummy_str, ==, "2"); dummy_str = j_message_get_string(message_recv); g_assert_cmpstr(dummy_str, ==, "42"); - J_TEST_TRAP_END; + J_TEST_TRAP_END; */ } static void From dff90ef51b52a483cd36cdb3c8d66dda944fc01a Mon Sep 17 00:00:00 2001 From: Julian Benda Date: Fri, 6 Aug 2021 18:58:54 +0200 Subject: [PATCH 25/60] Use JCredentials to fetch user id for port --- meson.build | 8 +++++++- tools/config.c | 9 +++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/meson.build b/meson.build index e1f84c485..eb0bcafbc 100644 --- a/meson.build +++ b/meson.build @@ -685,9 +685,15 @@ executable('julea-cli', julea_cli_srcs, install: true, ) -executable('julea-config', 'tools/config.c', +julea_config_srcs = files([ + 'lib/core/jcredentials.c', + 'lib/core/jtrace.c', + 'tools/config.c', +]) +executable('julea-config', julea_config_srcs, dependencies: common_deps, include_directories: julea_incs, + c_args: ['-DJULEA_COMPILATION'], install: true, ) diff --git a/tools/config.c b/tools/config.c index 77a08b47c..172972e08 100644 --- a/tools/config.c +++ b/tools/config.c @@ -17,6 +17,7 @@ */ #include +#include #include #include @@ -46,7 +47,6 @@ static gint opt_port = 0; static gint opt_max_connections = 0; static gint64 opt_stripe_size = 0; static gint64 opt_network_port = 0; -static gint64 opt_user_id = 0; static gchar** string_split(gchar const* string) @@ -156,8 +156,7 @@ main(gint argc, gchar** argv) { "system", 0, 0, G_OPTION_ARG_NONE, &opt_system, "Write system configuration", NULL }, { "read", 0, 0, G_OPTION_ARG_NONE, &opt_read, "Read configuration", NULL }, { "name", 0, 0, G_OPTION_ARG_STRING, &opt_name, "Configuration name", "julea" }, - { "port", 0, 0, G_OPTION_ARG_INT64, &opt_network_port, "Network communication port", "4000 + user-id%100"}, - { "user-id", 0, 0, G_OPTION_ARG_INT64, &opt_user_id, "User Id used to determined communication port", NULL}, + { "port", 0, 0, G_OPTION_ARG_INT64, &opt_network_port, "Network communication port", "4000 + user-id%1000"}, { "object-servers", 0, 0, G_OPTION_ARG_STRING, &opt_servers_object, "Object servers to use", "host1,host2:port" }, { "kv-servers", 0, 0, G_OPTION_ARG_STRING, &opt_servers_kv, "Key-value servers to use", "host1,host2:port" }, { "db-servers", 0, 0, G_OPTION_ARG_STRING, &opt_servers_db, "Database servers to use", "host1,host2:port" }, @@ -214,7 +213,9 @@ main(gint argc, gchar** argv) if(opt_network_port == 0) { - opt_network_port = 4000 + (opt_user_id % 1000); + JCredentials* credentials = j_credentials_new(); + opt_network_port = 4000 + ( j_credentials_get_user(credentials) % 1000 ); + j_credentials_unref(credentials); } if (opt_user) From dfb3ca26b978729f863b2ae2f969a5e53c4c0613 Mon Sep 17 00:00:00 2001 From: Julian Benda Date: Sun, 15 Aug 2021 17:40:28 +0200 Subject: [PATCH 26/60] Read server data from config --- lib/core/jnetwork.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/core/jnetwork.c b/lib/core/jnetwork.c index 3b779bf95..862d0367d 100644 --- a/lib/core/jnetwork.c +++ b/lib/core/jnetwork.c @@ -265,7 +265,6 @@ j_fabric_sread_event(struct JFabric* this, int timeout, enum JFabricEvents* even return ret; } -/** \todo Port is a magic number ! 4711 */ gboolean j_connection_init_client(struct JConfiguration* configuration, enum JBackendType backend, guint index, struct JConnection** instance_ptr) { @@ -287,8 +286,8 @@ j_connection_init_client(struct JConfiguration* configuration, enum JBackendType g_client = g_socket_client_new(); server = j_configuration_get_server(configuration, backend, index); g_connection = g_socket_client_connect_to_host(g_client, - "10.0.10.1", - 4711, NULL, &error); ///< \todo: !!!! read from config + server, + j_configuration_get_port(configuration), NULL, &error); G_CHECK("Failed to build gsocket connection to host"); if(g_connection == NULL) { g_warning("Can not connect to %s.", server); From f0db28d293b65f5b20f455cb7b32ab9bcd66612c Mon Sep 17 00:00:00 2001 From: Julian Benda Date: Tue, 24 Aug 2021 10:57:27 +0200 Subject: [PATCH 27/60] Remove Debug message --- lib/core/jnetwork.c | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/lib/core/jnetwork.c b/lib/core/jnetwork.c index 862d0367d..a01085924 100644 --- a/lib/core/jnetwork.c +++ b/lib/core/jnetwork.c @@ -184,13 +184,6 @@ j_fabric_init_client(struct JConfiguration* configuration, struct JFabricAddr* a hints->addr_format = addr->addr_format; hints->dest_addr = addr->addr; hints->dest_addrlen = addr->addr_len; - { // DEBUG TODO - char* str = malloc(hints->dest_addrlen * 3 + 1); - size_t i; - for(i = 0; i < hints->dest_addrlen; ++i) { snprintf(str+i*3, 4, "%02x ", ((uint8_t*)hints->dest_addr)[i]); } - // g_message("Addr: format: %u, len: %lu\n\t%s", hints->addr_format, hints->dest_addrlen, str); - free(str); - } res = fi_getinfo( j_configuration_get_libfabric_version(configuration), @@ -251,6 +244,7 @@ j_fabric_sread_event(struct JFabric* this, int timeout, enum JFabricEvents* even ret = TRUE; goto end; } else if (res == -FI_EAVAIL) { *event = J_FABRIC_EVENT_ERROR; + g_warning("error"); // TODO: fetch error! ret = TRUE; goto end; } From 9eb831f920f5810feeb032be410f5cc08530f8dc Mon Sep 17 00:00:00 2001 From: Julian Benda Date: Tue, 24 Aug 2021 12:59:56 +0200 Subject: [PATCH 28/60] Editing to work via tcp --- lib/core/jmessage.c | 1 + lib/core/jnetwork.c | 83 +++++++++++++++++++++++++++++---------------- server/loop.c | 8 +++++ 3 files changed, 62 insertions(+), 30 deletions(-) diff --git a/lib/core/jmessage.c b/lib/core/jmessage.c index 773dbcd55..fefa38f57 100644 --- a/lib/core/jmessage.c +++ b/lib/core/jmessage.c @@ -656,6 +656,7 @@ gboolean j_message_send_ack(JMessage* message, struct JConnection* connection) { const JConnectionAck ack = J_CONNECTION_ACK; + g_message("send ACK!"); // No data where send -> no need to acknowledge if(message != NULL && j_message_get_count(message) == 0) { return TRUE; } diff --git a/lib/core/jnetwork.c b/lib/core/jnetwork.c index a01085924..5e1fee382 100644 --- a/lib/core/jnetwork.c +++ b/lib/core/jnetwork.c @@ -67,6 +67,7 @@ struct JConnection { } cq; size_t inject_size; struct { + gboolean active; struct fid_mr* mr; void* buffer; size_t used; @@ -122,7 +123,7 @@ j_fabric_init_server(struct JConfiguration* configuration, struct JFabric** inst res = fi_getinfo( j_configuration_get_libfabric_version(this->config), - NULL, NULL, FI_SOURCE, + NULL, NULL, 0, j_configuration_get_libfabric_hints(this->config), &this->info); CHECK("Failed to find fabric for server!"); @@ -181,9 +182,11 @@ j_fabric_init_client(struct JConfiguration* configuration, struct JFabricAddr* a this->con_side = JF_CLIENT; hints = fi_dupinfo(j_configuration_get_libfabric_hints(configuration)); - hints->addr_format = addr->addr_format; - hints->dest_addr = addr->addr; - hints->dest_addrlen = addr->addr_len; + /// \todo check if verbs works without this! + // Causes invalid state for tcp connection + // hints->addr_format = addr->addr_format; + // hints->dest_addr = addr->addr; + // hints->dest_addrlen = addr->addr_len; res = fi_getinfo( j_configuration_get_libfabric_version(configuration), @@ -476,6 +479,7 @@ j_connection_create_memory_resources(struct JConnection* this) size += (rx_prefix * prefix_size) + J_CONNECTION_MAX_RECV * op_size + (tx_prefix * prefix_size) + J_CONNECTION_MAX_SEND * op_size; + this->memory.active = true; this->memory.used = 0; this->memory.buffer_size = size; this->memory.buffer = malloc(size); @@ -484,7 +488,7 @@ j_connection_create_memory_resources(struct JConnection* this) res = fi_mr_reg(this->domain, this->memory.buffer, this->memory.buffer_size, FI_SEND | FI_RECV, 0, 0, 0, &this->memory.mr, NULL); CHECK("Failed to register memory for msg communication!"); - } else { g_message("NO MEM!"); } + } else { this->memory.active = false; } ret = TRUE; end: @@ -529,6 +533,7 @@ j_connection_sread_event(struct JConnection* this, int timeout, enum JConnection return ret; } +/// \todo check usage and maybe merge with j_connection_read_event() gboolean j_connection_read_event(struct JConnection* this, enum JConnectionEvents* event) { @@ -573,18 +578,26 @@ j_connection_send(struct JConnection* this, const void* data, size_t data_len) } // normal send - segment = (uint8_t*)this->memory.buffer + this->memory.used; - memcpy(segment + this->memory.tx_prefix_size, - data, data_len); - size = data_len + this->memory.tx_prefix_size; - do{ - // g_message("send: prefix: %lu, size: %lu", this->memory.tx_prefix_size, size); - res = fi_send(this->ep, segment, size, fi_mr_desc(this->memory.mr), 0, segment); - } while(res == -FI_EAGAIN); - CHECK("Failed to initelize sending!"); - this->memory.used += size; - g_assert_true(this->memory.used <= this->memory.buffer_size); - g_assert_true(this->running_actions.len + 1< J_CONNECTION_MAX_SEND + J_CONNECTION_MAX_RECV); + if (this->memory.active) { + segment = (uint8_t*)this->memory.buffer + this->memory.used; + memcpy(segment + this->memory.tx_prefix_size, + data, data_len); + size = data_len + this->memory.tx_prefix_size; + do{ + // g_message("send: prefix: %lu, size: %lu", this->memory.tx_prefix_size, size); + res = fi_send(this->ep, segment, size, fi_mr_desc(this->memory.mr), 0, segment); + } while(res == -FI_EAGAIN); + CHECK("Failed to initelize sending!"); + this->memory.used += size; + g_assert_true(this->memory.used <= this->memory.buffer_size); + g_assert_true(this->running_actions.len + 1< J_CONNECTION_MAX_SEND + J_CONNECTION_MAX_RECV); + } else { + segment = (uint8_t*)data; + size = data_len; + do{ + res = fi_send(this->ep, segment, size, NULL, 0, segment); + } while(res == -FI_EAGAIN); + } this->running_actions.entry[this->running_actions.len].context = segment; this->running_actions.entry[this->running_actions.len].dest = NULL; this->running_actions.entry[this->running_actions.len].len = 0; @@ -604,19 +617,27 @@ j_connection_recv(struct JConnection* this, size_t data_len, void* data) void* segment; size_t size; - segment = (char*)this->memory.buffer + this->memory.used; + segment = this->memory.active ? (char*)this->memory.buffer + this->memory.used : data; size = data_len + this->memory.rx_prefix_size; - res = fi_recv(this->ep, segment, size, fi_mr_desc(this->memory.mr), 0, segment); - // g_message("receive: prefix: %lu, size: %lu", this->memory.rx_prefix_size, size); + res = fi_recv(this->ep, segment, size, + this->memory.active ? fi_mr_desc(this->memory.mr) : NULL, + 0, segment); CHECK("Failed to initelized receiving!"); - this->memory.used += size; - // g_message("memory useag: %lu/%lu", this->memory.used, this->memory.buffer_size); - g_assert_true(this->memory.used <= this->memory.buffer_size); - g_assert_true(this->running_actions.len < J_CONNECTION_MAX_SEND + J_CONNECTION_MAX_RECV); - this->running_actions.entry[this->running_actions.len].context = segment; - this->running_actions.entry[this->running_actions.len].dest = data; - this->running_actions.entry[this->running_actions.len].len = data_len; - this->running_actions.entry[this->running_actions.len].mr = NULL; + + if (this->memory.active) { + this->memory.used += size; + g_assert_true(this->memory.used <= this->memory.buffer_size); + g_assert_true(this->running_actions.len < J_CONNECTION_MAX_SEND + J_CONNECTION_MAX_RECV); + this->running_actions.entry[this->running_actions.len].context = segment; + this->running_actions.entry[this->running_actions.len].dest = data; + this->running_actions.entry[this->running_actions.len].len = data_len; + this->running_actions.entry[this->running_actions.len].mr = NULL; + } else { + this->running_actions.entry[this->running_actions.len].context = segment; + this->running_actions.entry[this->running_actions.len].dest = NULL; + this->running_actions.entry[this->running_actions.len].len = 0; + this->running_actions.entry[this->running_actions.len].mr = NULL; + } ++this->running_actions.len; ret = TRUE; end: @@ -682,7 +703,7 @@ j_connection_wait_for_completion(struct JConnection* this) end: return ret; } - +int offset = 0; gboolean j_connection_rma_register(struct JConnection* this, const void* data, size_t data_len, struct JConnectionMemory* handle) { @@ -695,8 +716,10 @@ j_connection_rma_register(struct JConnection* this, const void* data, size_t dat FI_REMOTE_READ, 0, 0, 0, &handle->memory_region, NULL); CHECK("Failed to register memory region!"); - handle->addr = (guint64)data; + /// \todo tcp/verbs why? + handle->addr = offset; // (guint64)data; handle->size = data_len; + offset += handle->size; ret = TRUE; end: diff --git a/server/loop.c b/server/loop.c index f39736ecd..d28591437 100644 --- a/server/loop.c +++ b/server/loop.c @@ -300,8 +300,16 @@ jd_handle_message(JMessage* message, struct JConnection* connection, JMemoryChun buf = j_memory_chunk_get(memory_chunk, memoryID->size); g_assert(buf != NULL); + g_message("start rma read!"); j_connection_rma_read(connection, memoryID, buf); + g_message("start waiting!"); j_connection_wait_for_completion(connection); ///< \todo pararlleize + g_message("fin rma read!"); + g_print("data: "); + for(guint64 j = 0; j < memoryID->size; ++j) { + g_print("%x ", buf[j]); + } + g_print("\n"); // g_message("rma read message: %s", buf); j_statistics_add(statistics, J_STATISTICS_BYTES_RECEIVED, memoryID->size); // CONTINUE From 039d313d6607c4cea60f2d94ee8c0e0324e16a77 Mon Sep 17 00:00:00 2001 From: Julian Benda Date: Tue, 24 Aug 2021 14:46:37 +0200 Subject: [PATCH 29/60] Working tcp `hello_world` * fixes bufnr usage * proper shutdown behavior * offset hack for 0 based address --- include/core/jnetwork.h | 6 ++++ lib/core/jmessage.c | 7 ++-- lib/core/jnetwork.c | 72 ++++++++++++++++++++++++++++++++--------- server/server.c | 2 ++ 4 files changed, 69 insertions(+), 18 deletions(-) diff --git a/include/core/jnetwork.h b/include/core/jnetwork.h index 63d1bc23d..f4d7dc0ad 100644 --- a/include/core/jnetwork.h +++ b/include/core/jnetwork.h @@ -399,11 +399,17 @@ j_connection_rma_read(struct JConnection* instance, /// Wait until operations initiated at his connection finished. /** \public \memberof JConnection + * \retval FALSE if waiting finished. This may occures because the connection was closed: check this with: j_connection_active() * \sa j_connection_rma_read, j_connection_send, j_connection_recv */ gboolean j_connection_wait_for_completion(struct JConnection* instance); +/// Check if the connection was closed from the other party. +/** \sa j_connection_wait_for_completion */ +gboolean +j_connection_closed(struct JConnection* this); + /// Register memory to make it rma readable. /** \public \memberof JConnection * Memory access rights must changed to allow for an rma read of other party. diff --git a/lib/core/jmessage.c b/lib/core/jmessage.c index fefa38f57..3e3d4bed4 100644 --- a/lib/core/jmessage.c +++ b/lib/core/jmessage.c @@ -762,11 +762,14 @@ j_message_read(JMessage* message, struct JConnection* connection) g_return_val_if_fail(message != NULL, FALSE); g_return_val_if_fail(connection != NULL, FALSE); - // g_message("read: msg hedaer: %lu", sizeof(message->header)); + EXE(j_connection_recv(connection, sizeof(message->header), &(message->header)), "Failed to initiated header receive!"); - EXE(j_connection_wait_for_completion(connection), + if(!j_connection_wait_for_completion(connection)) { + EXE(j_connection_closed(connection), "Failed to wait for header receive!"); + goto end; + } j_message_ensure_size(message, j_message_length(message)); diff --git a/lib/core/jnetwork.c b/lib/core/jnetwork.c index 5e1fee382..977c6fd6d 100644 --- a/lib/core/jnetwork.c +++ b/lib/core/jnetwork.c @@ -84,6 +84,8 @@ struct JConnection { } entry[J_CONNECTION_MAX_RECV + J_CONNECTION_MAX_SEND]; int len; } running_actions; + guint addr_offset; + gboolean closed; }; struct JConnectionMemory { @@ -181,7 +183,7 @@ j_fabric_init_client(struct JConfiguration* configuration, struct JFabricAddr* a this->config = configuration; this->con_side = JF_CLIENT; - hints = fi_dupinfo(j_configuration_get_libfabric_hints(configuration)); + hints = j_configuration_get_libfabric_hints(configuration); /// \todo check if verbs works without this! // Causes invalid state for tcp connection // hints->addr_format = addr->addr_format; @@ -199,9 +201,7 @@ j_fabric_init_client(struct JConfiguration* configuration, struct JFabricAddr* a CHECK("failed to initelize client fabric!"); ret = TRUE; - // g_message("initelize client!"); end: - fi_freeinfo(hints); return ret; } @@ -332,7 +332,8 @@ j_connection_init_client(struct JConfiguration* configuration, enum JBackendType ret = TRUE; end: - /// FIXME memory leak on error in addr.addr + /// \todo memory leak on error in addr.addr + free(jf_addr.addr); return ret; } @@ -446,6 +447,8 @@ j_connection_init(struct JConnection* this) res = fi_enable(this->ep); CHECK("Failed to enable connection!"); + this->closed = FALSE; + ret = TRUE; // g_message("enabled endpoint!"); end: @@ -479,7 +482,7 @@ j_connection_create_memory_resources(struct JConnection* this) size += (rx_prefix * prefix_size) + J_CONNECTION_MAX_RECV * op_size + (tx_prefix * prefix_size) + J_CONNECTION_MAX_SEND * op_size; - this->memory.active = true; + this->memory.active = TRUE; this->memory.used = 0; this->memory.buffer_size = size; this->memory.buffer = malloc(size); @@ -488,7 +491,7 @@ j_connection_create_memory_resources(struct JConnection* this) res = fi_mr_reg(this->domain, this->memory.buffer, this->memory.buffer_size, FI_SEND | FI_RECV, 0, 0, 0, &this->memory.mr, NULL); CHECK("Failed to register memory for msg communication!"); - } else { this->memory.active = false; } + } else { this->memory.active = FALSE; } ret = TRUE; end: @@ -644,6 +647,12 @@ j_connection_recv(struct JConnection* this, size_t data_len, void* data) return ret; } +gboolean +j_connection_closed(struct JConnection* this) +{ + return this->closed; +} + gboolean j_connection_wait_for_completion(struct JConnection* this) { @@ -656,15 +665,18 @@ j_connection_wait_for_completion(struct JConnection* this) while(this->running_actions.len) { bool rx; do { - rx = true; + rx = TRUE; res = fi_cq_sread(this->cq.rx, &entry, 1, NULL, 2); /// \todo handle shutdown msgs! if(res == -FI_EAGAIN) { - rx = false; + rx = FALSE; res = fi_cq_sread(this->cq.tx, &entry, 1, NULL, 2); /// \todo handle shutdown msgs! } } while (res == -FI_EAGAIN); /// \todo error message fetch! if(res == -FI_EAVAIL) { + enum JConnectionEvents event; + j_connection_sread_event(this, 0, &event); + if(event == J_CONNECTION_EVENT_SHUTDOWN) { this->closed = TRUE; goto end; } struct fi_cq_err_entry err_entry; res = fi_cq_readerr(rx ? this->cq.rx : this->cq.tx, &err_entry, 0); @@ -703,7 +715,7 @@ j_connection_wait_for_completion(struct JConnection* this) end: return ret; } -int offset = 0; + gboolean j_connection_rma_register(struct JConnection* this, const void* data, size_t data_len, struct JConnectionMemory* handle) { @@ -717,9 +729,9 @@ j_connection_rma_register(struct JConnection* this, const void* data, size_t dat 0, 0, 0, &handle->memory_region, NULL); CHECK("Failed to register memory region!"); /// \todo tcp/verbs why? - handle->addr = offset; // (guint64)data; + handle->addr = this->addr_offset; // (guint64)data; handle->size = data_len; - offset += handle->size; + this->addr_offset += handle->size; ret = TRUE; end: @@ -730,9 +742,8 @@ gboolean j_connection_rma_unregister(struct JConnection* this, struct JConnectionMemory* handle) { int res; - // g_message("close memory start"); + this->addr_offset = 0; /// \todo ensure that this works! (breaks one incremental freeing and realloc) res = fi_close(&handle->memory_region->fid); - // g_message("close memory fin"); CHECK("Failed to unregistrer rma memory!"); return TRUE; end: @@ -780,7 +791,36 @@ j_connection_rma_read(struct JConnection* this, const struct JConnectionMemoryID } gboolean -j_connection_fini ( struct JConnection* instance) { - // TODO: implement! - return TRUE; +j_connection_fini ( struct JConnection* this) { + int res; + gboolean ret = FALSE; + + res = fi_shutdown(this->ep, 0); + CHECK("failed to send shutdown signal"); + + if(this->memory.active) { + res = fi_close(&this->memory.mr->fid); + CHECK("failed to free memory region!"); + free(this->memory.buffer); + } + + res = fi_close(&this->ep->fid); + CHECK("failed to close endpoint!"); + res = fi_close(&this->cq.tx->fid); + CHECK("failed to close tx cq!"); + res = fi_close(&this->cq.rx->fid); + CHECK("failed to close rx cq!"); + res = fi_close(&this->eq->fid); + CHECK("failed to close event queue!"); + res = fi_close(&this->domain->fid); + CHECK("failed to close domain!"); + + if(this->fabric->con_side == JF_CLIENT) { + j_fabric_fini(this->fabric); + } + + free(this); + ret = TRUE; +end: + return ret; } diff --git a/server/server.c b/server/server.c index 258ee6d8c..41fd65937 100644 --- a/server/server.c +++ b/server/server.c @@ -87,6 +87,8 @@ jd_on_run(GThreadedSocketService* service, GSocketConnection* gconnection, GObje jd_handle_message(message, jconnection, memory_chunk, memory_chunk_size, statistics); } + j_connection_fini(jconnection); + { guint64 value; From 13c61174113fe08eb7c0906756574c454e8cb439 Mon Sep 17 00:00:00 2001 From: Julian Benda Date: Tue, 24 Aug 2021 23:00:44 +0200 Subject: [PATCH 30/60] More tcp fixes * tcp connections have default request key * tcp memory reads are identified via key and not address! --- lib/core/jnetwork.c | 4 ++-- server/loop.c | 9 --------- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/lib/core/jnetwork.c b/lib/core/jnetwork.c index 977c6fd6d..2d536e0c1 100644 --- a/lib/core/jnetwork.c +++ b/lib/core/jnetwork.c @@ -726,7 +726,7 @@ j_connection_rma_register(struct JConnection* this, const void* data, size_t dat data, data_len, FI_REMOTE_READ, - 0, 0, 0, &handle->memory_region, NULL); + 0, this->addr_offset, 0, &handle->memory_region, NULL); CHECK("Failed to register memory region!"); /// \todo tcp/verbs why? handle->addr = this->addr_offset; // (guint64)data; @@ -775,7 +775,7 @@ j_connection_rma_read(struct JConnection* this, const struct JConnectionMemoryID memoryID->size, fi_mr_desc(mr), 0, - memoryID->offset, + 0, // memoryID->offset, memoryID->key, data); CHECK("Failed to initate reading"); diff --git a/server/loop.c b/server/loop.c index d28591437..f13b7af1d 100644 --- a/server/loop.c +++ b/server/loop.c @@ -300,17 +300,8 @@ jd_handle_message(JMessage* message, struct JConnection* connection, JMemoryChun buf = j_memory_chunk_get(memory_chunk, memoryID->size); g_assert(buf != NULL); - g_message("start rma read!"); j_connection_rma_read(connection, memoryID, buf); - g_message("start waiting!"); j_connection_wait_for_completion(connection); ///< \todo pararlleize - g_message("fin rma read!"); - g_print("data: "); - for(guint64 j = 0; j < memoryID->size; ++j) { - g_print("%x ", buf[j]); - } - g_print("\n"); - // g_message("rma read message: %s", buf); j_statistics_add(statistics, J_STATISTICS_BYTES_RECEIVED, memoryID->size); // CONTINUE From f63e567773cdb71a80c735477b2041f3d4cd2863 Mon Sep 17 00:00:00 2001 From: Julian Benda Date: Tue, 24 Aug 2021 23:02:03 +0200 Subject: [PATCH 31/60] Remove debug messages --- lib/core/jmessage.c | 5 ----- lib/core/jnetwork.c | 17 ----------------- 2 files changed, 22 deletions(-) diff --git a/lib/core/jmessage.c b/lib/core/jmessage.c index 3e3d4bed4..3df607ec9 100644 --- a/lib/core/jmessage.c +++ b/lib/core/jmessage.c @@ -511,7 +511,6 @@ j_message_append_n(JMessage* message, gconstpointer data, gsize length) gboolean j_message_append_memory_id(JMessage* message, const struct JConnectionMemoryID* id) { - // g_message("ID size: %lu", id->size); J_TRACE_FUNCTION(NULL); guint32 new_length; @@ -634,7 +633,6 @@ j_message_get_memory_id(JMessage* message) padding = (guint64)message->current % 8; - // g_message("read mem id: %lu", (message->current + padding) - message->data); ret = (const void*)(message->current + padding); message->current += sizeof(*ret) + padding; @@ -656,11 +654,9 @@ gboolean j_message_send_ack(JMessage* message, struct JConnection* connection) { const JConnectionAck ack = J_CONNECTION_ACK; - g_message("send ACK!"); // No data where send -> no need to acknowledge if(message != NULL && j_message_get_count(message) == 0) { return TRUE; } - // g_message("send ack"); EXE(j_connection_wait_for_completion(connection), "Failed to wait to finishe all operations before sending ack!"); EXE(j_connection_send(connection, &ack, sizeof(ack)), @@ -811,7 +807,6 @@ j_message_write(JMessage* message, struct JConnection* connection) g_return_val_if_fail(message != NULL, FALSE); g_return_val_if_fail(connection != NULL, FALSE); - // g_message("send: msg hedaer: %lu", sizeof(message->header)); EXE(j_connection_send(connection, &(message->header), sizeof(message->header)), "Failed to initiated sending message header."); if(j_message_length(message) > 0) { diff --git a/lib/core/jnetwork.c b/lib/core/jnetwork.c index 2d536e0c1..b5a440525 100644 --- a/lib/core/jnetwork.c +++ b/lib/core/jnetwork.c @@ -149,13 +149,11 @@ j_fabric_init_server(struct JConfiguration* configuration, struct JFabric** inst res = fi_getname(&this->pep->fid, NULL, &addrlen); if(res != -FI_ETOOSMALL) { CHECK("failed to fetch address len from libfabirc!"); } this->fabric_addr_network.addr_len = addrlen; - // g_message("test: %u", this->fabric_addr_network.addr_len); this->fabric_addr_network.addr = malloc(this->fabric_addr_network.addr_len); res = fi_getname(&this->pep->fid, this->fabric_addr_network.addr, &addrlen); CHECK("failed to fetch address from libfabric!"); - // g_message("wait for pep request!"); res = fi_listen(this->pep); CHECK("failed to start listening on pep!"); @@ -323,7 +321,6 @@ j_connection_init_client(struct JConfiguration* configuration, enum JBackendType do {EXE(j_connection_sread_event(this, 1, &event), "Failed to read event queue, waiting for CONNECTED signal!"); } while(event == J_CONNECTION_EVENT_TIMEOUT); - // g_message("Event FI: %u", event); if(event != J_CONNECTION_EVENT_CONNECTED) { g_warning("Failed to connect to host!"); @@ -352,16 +349,13 @@ j_connection_init_server(struct JFabric* fabric, GSocketConnection* gconnection, *instance_ptr = malloc(sizeof(*this)); this = *instance_ptr; - // g_message("debug: size(*this) = %lu, this = %p", sizeof(*this), (void*)this); memset(this, 0, sizeof(*this)); // send addr { // DEBUG TODO char* str = malloc(ntohl(addr->addr_len) * 3 + 1); size_t i; - // g_message("tada: addr_len: %u", ntohl(addr->addr_len)); for(i = 0; i < ntohl(addr->addr_len); ++i) { snprintf(str+i*3, 4, "%02x ", ((uint8_t*)addr->addr)[i]); } - // g_message("Addr: format: %u, len: %u\n\t%s", ntohl(addr->addr_format), ntohl(addr->addr_len), str); free(str); } g_out = g_io_stream_get_output_stream(G_IO_STREAM(gconnection)); @@ -371,11 +365,9 @@ j_connection_init_server(struct JFabric* fabric, GSocketConnection* gconnection, G_CHECK("Failed to write addr_len to stream!"); g_output_stream_write(g_out, addr->addr, ntohl(addr->addr_len), NULL, &error); G_CHECK("Failed to write addr to stream!"); - // g_message("close g_stream"); g_output_stream_close(g_out, NULL, &error); G_CHECK("Failed to close output stream!"); - // g_message("wait for conrequest!"); do {EXE(j_fabric_sread_event(fabric, 2, &event, &request), "Failed to wait for connection request"); } while(event == J_FABRIC_EVENT_TIMEOUT); @@ -383,16 +375,13 @@ j_connection_init_server(struct JFabric* fabric, GSocketConnection* gconnection, g_warning("expected an connection request and nothing else! (%i)", event); goto end; } - // g_message("inet connection!"); this->fabric = fabric; this->info = request.info; EXE(j_connection_init(this), "Failed to initelize connection server side!"); - // g_message("Except!"); res = fi_accept(this->ep, NULL, 0); CHECK("Failed to accept connection!"); - // g_message("wait for connection !"); EXE(j_connection_sread_event(this, 2, &con_event), "Failed to verify connection!"); if(con_event != J_CONNECTION_EVENT_CONNECTED) { g_warning("expected and connection ack and nothing else!"); @@ -450,7 +439,6 @@ j_connection_init(struct JConnection* this) this->closed = FALSE; ret = TRUE; - // g_message("enabled endpoint!"); end: return ret; } @@ -478,7 +466,6 @@ j_connection_create_memory_resources(struct JConnection* this) size = 0; if(this->info->domain_attr->mr_mode & FI_MR_LOCAL) { - // g_message("Localmemory!"); size += (rx_prefix * prefix_size) + J_CONNECTION_MAX_RECV * op_size + (tx_prefix * prefix_size) + J_CONNECTION_MAX_SEND * op_size; @@ -524,7 +511,6 @@ j_connection_sread_event(struct JConnection* this, int timeout, enum JConnection goto end; /// \todo consider return TRUE on event queue error } CHECK("Failed to read event of connection!"); - // g_message("Event: %u", fi_event); switch(fi_event) { case FI_CONNECTED: *event = J_CONNECTION_EVENT_CONNECTED; break; @@ -587,7 +573,6 @@ j_connection_send(struct JConnection* this, const void* data, size_t data_len) data, data_len); size = data_len + this->memory.tx_prefix_size; do{ - // g_message("send: prefix: %lu, size: %lu", this->memory.tx_prefix_size, size); res = fi_send(this->ep, segment, size, fi_mr_desc(this->memory.mr), 0, segment); } while(res == -FI_EAGAIN); CHECK("Failed to initelize sending!"); @@ -661,7 +646,6 @@ j_connection_wait_for_completion(struct JConnection* this) struct fi_cq_entry entry; int i; - // g_message("start waiting"); while(this->running_actions.len) { bool rx; do { @@ -707,7 +691,6 @@ j_connection_wait_for_completion(struct JConnection* this) } } } - // g_message("fin waiting"); this->running_actions.len = 0; this->memory.used = 0; From de255cad562cbc2f107a4afd922dacafccbe9b07 Mon Sep 17 00:00:00 2001 From: Julian Benda Date: Tue, 24 Aug 2021 23:05:40 +0200 Subject: [PATCH 32/60] Clean up * a counter key is sufficent for tcp --- lib/core/jnetwork.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/core/jnetwork.c b/lib/core/jnetwork.c index b5a440525..c7bf376b4 100644 --- a/lib/core/jnetwork.c +++ b/lib/core/jnetwork.c @@ -84,7 +84,7 @@ struct JConnection { } entry[J_CONNECTION_MAX_RECV + J_CONNECTION_MAX_SEND]; int len; } running_actions; - guint addr_offset; + guint next_key; gboolean closed; }; @@ -709,12 +709,12 @@ j_connection_rma_register(struct JConnection* this, const void* data, size_t dat data, data_len, FI_REMOTE_READ, - 0, this->addr_offset, 0, &handle->memory_region, NULL); + 0, this->next_key, 0, &handle->memory_region, NULL); CHECK("Failed to register memory region!"); /// \todo tcp/verbs why? - handle->addr = this->addr_offset; // (guint64)data; + handle->addr = 0; // (guint64)data; handle->size = data_len; - this->addr_offset += handle->size; + this->next_key += 1; ret = TRUE; end: @@ -725,7 +725,7 @@ gboolean j_connection_rma_unregister(struct JConnection* this, struct JConnectionMemory* handle) { int res; - this->addr_offset = 0; /// \todo ensure that this works! (breaks one incremental freeing and realloc) + this->next_key = 0; /// \todo ensure that this works! (breaks one incremental freeing and realloc) res = fi_close(&handle->memory_region->fid); CHECK("Failed to unregistrer rma memory!"); return TRUE; @@ -758,7 +758,7 @@ j_connection_rma_read(struct JConnection* this, const struct JConnectionMemoryID memoryID->size, fi_mr_desc(mr), 0, - 0, // memoryID->offset, + memoryID->offset, memoryID->key, data); CHECK("Failed to initate reading"); From 0003fd8b0808b53b0f9561314e01d104d64b5c01 Mon Sep 17 00:00:00 2001 From: Julian Benda Date: Tue, 24 Aug 2021 23:59:20 +0200 Subject: [PATCH 33/60] Fix rma addresses for non virt address support? --- lib/core/jnetwork.c | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/lib/core/jnetwork.c b/lib/core/jnetwork.c index c7bf376b4..49ece083c 100644 --- a/lib/core/jnetwork.c +++ b/lib/core/jnetwork.c @@ -181,12 +181,10 @@ j_fabric_init_client(struct JConfiguration* configuration, struct JFabricAddr* a this->config = configuration; this->con_side = JF_CLIENT; - hints = j_configuration_get_libfabric_hints(configuration); - /// \todo check if verbs works without this! - // Causes invalid state for tcp connection - // hints->addr_format = addr->addr_format; - // hints->dest_addr = addr->addr; - // hints->dest_addrlen = addr->addr_len; + hints = fi_dupinfo(j_configuration_get_libfabric_hints(configuration)); + hints->addr_format = addr->addr_format; + hints->dest_addr = addr->addr; + hints->dest_addrlen = addr->addr_len; res = fi_getinfo( j_configuration_get_libfabric_version(configuration), @@ -200,6 +198,7 @@ j_fabric_init_client(struct JConfiguration* configuration, struct JFabricAddr* a ret = TRUE; end: + // fi_freeinfo(hints); return ret; } @@ -711,8 +710,7 @@ j_connection_rma_register(struct JConnection* this, const void* data, size_t dat FI_REMOTE_READ, 0, this->next_key, 0, &handle->memory_region, NULL); CHECK("Failed to register memory region!"); - /// \todo tcp/verbs why? - handle->addr = 0; // (guint64)data; + handle->addr = (this->info->domain_attr->mr_mode & FI_MR_VIRT_ADDR ) ? (guint64)data : 0; handle->size = data_len; this->next_key += 1; @@ -725,7 +723,7 @@ gboolean j_connection_rma_unregister(struct JConnection* this, struct JConnectionMemory* handle) { int res; - this->next_key = 0; /// \todo ensure that this works! (breaks one incremental freeing and realloc) + this->next_key = 0; /// \todo may just count key to overflow? (max value is stored in domain_attr) res = fi_close(&handle->memory_region->fid); CHECK("Failed to unregistrer rma memory!"); return TRUE; From 90c4f43e95398cfcace7bddf42280c674f1575ca Mon Sep 17 00:00:00 2001 From: Julian Benda Date: Wed, 15 Sep 2021 21:12:38 +0200 Subject: [PATCH 34/60] Remove rma read limit --- lib/core/jnetwork.c | 100 +++++++++++++++++++++---------------------- lib/object/jobject.c | 1 + 2 files changed, 51 insertions(+), 50 deletions(-) diff --git a/lib/core/jnetwork.c b/lib/core/jnetwork.c index 49ece083c..f2c1992e7 100644 --- a/lib/core/jnetwork.c +++ b/lib/core/jnetwork.c @@ -80,9 +80,9 @@ struct JConnection { void* context; void* dest; size_t len; - struct fid_mr* mr; - } entry[J_CONNECTION_MAX_RECV + J_CONNECTION_MAX_SEND]; - int len; + } msg_entry[J_CONNECTION_MAX_RECV + J_CONNECTION_MAX_SEND]; + int msg_len; + int rma_len; } running_actions; guint next_key; gboolean closed; @@ -398,7 +398,8 @@ j_connection_init(struct JConnection* this) gboolean ret = FALSE; int res; - this->running_actions.len = 0; + this->running_actions.msg_len = 0; + this->running_actions.rma_len = 0; res = fi_eq_open(this->fabric->fabric, &(struct fi_eq_attr){.wait_obj = FI_WAIT_UNSPEC}, &this->eq, NULL); @@ -577,7 +578,7 @@ j_connection_send(struct JConnection* this, const void* data, size_t data_len) CHECK("Failed to initelize sending!"); this->memory.used += size; g_assert_true(this->memory.used <= this->memory.buffer_size); - g_assert_true(this->running_actions.len + 1< J_CONNECTION_MAX_SEND + J_CONNECTION_MAX_RECV); + g_assert_true(this->running_actions.msg_len + 1< J_CONNECTION_MAX_SEND + J_CONNECTION_MAX_RECV); } else { segment = (uint8_t*)data; size = data_len; @@ -585,11 +586,10 @@ j_connection_send(struct JConnection* this, const void* data, size_t data_len) res = fi_send(this->ep, segment, size, NULL, 0, segment); } while(res == -FI_EAGAIN); } - this->running_actions.entry[this->running_actions.len].context = segment; - this->running_actions.entry[this->running_actions.len].dest = NULL; - this->running_actions.entry[this->running_actions.len].len = 0; - this->running_actions.entry[this->running_actions.len].mr = NULL; - ++this->running_actions.len; + this->running_actions.msg_entry[this->running_actions.msg_len].context = segment; + this->running_actions.msg_entry[this->running_actions.msg_len].dest = NULL; + this->running_actions.msg_entry[this->running_actions.msg_len].len = 0; + ++this->running_actions.msg_len; ret = TRUE; end: @@ -614,18 +614,16 @@ j_connection_recv(struct JConnection* this, size_t data_len, void* data) if (this->memory.active) { this->memory.used += size; g_assert_true(this->memory.used <= this->memory.buffer_size); - g_assert_true(this->running_actions.len < J_CONNECTION_MAX_SEND + J_CONNECTION_MAX_RECV); - this->running_actions.entry[this->running_actions.len].context = segment; - this->running_actions.entry[this->running_actions.len].dest = data; - this->running_actions.entry[this->running_actions.len].len = data_len; - this->running_actions.entry[this->running_actions.len].mr = NULL; + g_assert_true(this->running_actions.msg_len < J_CONNECTION_MAX_SEND + J_CONNECTION_MAX_RECV); + this->running_actions.msg_entry[this->running_actions.msg_len].context = segment; + this->running_actions.msg_entry[this->running_actions.msg_len].dest = data; + this->running_actions.msg_entry[this->running_actions.msg_len].len = data_len; } else { - this->running_actions.entry[this->running_actions.len].context = segment; - this->running_actions.entry[this->running_actions.len].dest = NULL; - this->running_actions.entry[this->running_actions.len].len = 0; - this->running_actions.entry[this->running_actions.len].mr = NULL; + this->running_actions.msg_entry[this->running_actions.msg_len].context = segment; + this->running_actions.msg_entry[this->running_actions.msg_len].dest = NULL; + this->running_actions.msg_entry[this->running_actions.msg_len].len = 0; } - ++this->running_actions.len; + ++this->running_actions.msg_len; ret = TRUE; end: return ret; @@ -645,14 +643,14 @@ j_connection_wait_for_completion(struct JConnection* this) struct fi_cq_entry entry; int i; - while(this->running_actions.len) { + while(this->running_actions.rma_len + this->running_actions.msg_len) { bool rx; do { rx = TRUE; - res = fi_cq_sread(this->cq.rx, &entry, 1, NULL, 2); /// \todo handle shutdown msgs! + res = fi_cq_read(this->cq.rx, &entry, 1); /// \todo handle shutdown msgs! if(res == -FI_EAGAIN) { rx = FALSE; - res = fi_cq_sread(this->cq.tx, &entry, 1, NULL, 2); /// \todo handle shutdown msgs! + res = fi_cq_read(this->cq.tx, &entry, 1); /// \todo handle shutdown msgs! } } while (res == -FI_EAGAIN); /// \todo error message fetch! @@ -669,29 +667,25 @@ j_connection_wait_for_completion(struct JConnection* this) err_entry.prov_errno, err_entry.err_data, NULL, 0)); goto end; } else { CHECK("Failed to read completion queue!"); } - for(i = 0; i <= this->running_actions.len; ++i) { - if(i == this->running_actions.len) { - g_warning("unable to find completet context!"); - goto end; + for(i = 0; i <= this->running_actions.msg_len; ++i) { + if(i == this->running_actions.msg_len) { + // If there is no match -> it's an rma transafre -> context = memory region + res = fi_close(&((struct fid_mr*)entry.op_context)->fid); + CHECK("Failed to free receiving memory!"); + --this->running_actions.rma_len; } - if(this->running_actions.entry[i].context == entry.op_context) { - --this->running_actions.len; - if(this->running_actions.entry[i].dest) { - memcpy(this->running_actions.entry[i].dest, - this->running_actions.entry[i].context, - this->running_actions.entry[i].len); + if(this->running_actions.msg_entry[i].context == entry.op_context) { + --this->running_actions.msg_len; + if(this->running_actions.msg_entry[i].dest) { + memcpy(this->running_actions.msg_entry[i].dest, + this->running_actions.msg_entry[i].context, + this->running_actions.msg_entry[i].len); } - if (this->running_actions.entry[i].mr) { - res = fi_close(&this->running_actions.entry[i].mr->fid); - CHECK("Failed to free receiving memory!"); - } - this->running_actions.entry[i] = this->running_actions.entry[this->running_actions.len]; + this->running_actions.msg_entry[i] = this->running_actions.msg_entry[this->running_actions.msg_len]; break; } } } - - this->running_actions.len = 0; this->memory.used = 0; ret = TRUE; end: @@ -746,28 +740,34 @@ j_connection_rma_read(struct JConnection* this, const struct JConnectionMemoryID int res; gboolean ret = FALSE; struct fid_mr* mr; + static unsigned key = 0; res = fi_mr_reg(this->domain, data, memoryID->size, - FI_READ, 0, 0, 0, &mr, 0); + FI_READ, 0, ++key, 0, &mr, 0); CHECK("Failed to register receiving memory!"); - - res = fi_read(this->ep, + do { res = fi_read(this->ep, data, memoryID->size, fi_mr_desc(mr), 0, memoryID->offset, memoryID->key, - data); + mr); + /// \todo fix the timing for minimal timeout. This is needed when device is too bussy + if(res == -FI_EAGAIN) { + struct timespec ts; + int tres; + ts.tv_nsec = 1000000; + do { + tres = nanosleep(&ts, &ts); + } while(tres && errno == EINTR); + } + } while ( res == -FI_EAGAIN); CHECK("Failed to initate reading"); - this->running_actions.entry[this->running_actions.len].context = data; - this->running_actions.entry[this->running_actions.len].dest = NULL; - this->running_actions.entry[this->running_actions.len].len = 0; - this->running_actions.entry[this->running_actions.len].mr = mr; - ++this->running_actions.len; - + ++this->running_actions.rma_len; ret = TRUE; end: + if(!ret) { fi_close(&mr->fid); } return ret; } diff --git a/lib/object/jobject.c b/lib/object/jobject.c index 1d28fb2b0..500ec40fb 100644 --- a/lib/object/jobject.c +++ b/lib/object/jobject.c @@ -559,6 +559,7 @@ j_object_read_exec(JList* operations, JSemantics* semantics) if (mem_id->size > 0) { j_connection_rma_read(object_connection, mem_id, data); + j_connection_wait_for_completion(object_connection); ///\todo deserelize } } From e99a8fdc4a2cdffbda044c84bd829f8d12172d5b Mon Sep 17 00:00:00 2001 From: Julian Benda Date: Mon, 11 Oct 2021 17:42:38 +0200 Subject: [PATCH 35/60] Add trace functions for network code --- lib/core/jnetwork.c | 47 +++++++++++++++++++++++++++++++++++++++------ 1 file changed, 41 insertions(+), 6 deletions(-) diff --git a/lib/core/jnetwork.c b/lib/core/jnetwork.c index f2c1992e7..09403a7ee 100644 --- a/lib/core/jnetwork.c +++ b/lib/core/jnetwork.c @@ -10,6 +10,7 @@ #include #include +#include /// Used to initialize common parts between different connection. /** This will create the following libfabric resources: @@ -105,6 +106,8 @@ struct JConnectionMemory { void free_dangling_infos(struct fi_info*); void free_dangling_infos(struct fi_info* info) { + J_TRACE_FUNCTION(NULL); + fi_freeinfo(info->next); info->next = NULL; } @@ -112,6 +115,8 @@ void free_dangling_infos(struct fi_info* info) { gboolean j_fabric_init_server(struct JConfiguration* configuration, struct JFabric** instance_ptr) { + J_TRACE_FUNCTION(NULL); + struct JFabric* this; int res; size_t addrlen; @@ -169,6 +174,8 @@ j_fabric_init_server(struct JConfiguration* configuration, struct JFabric** inst gboolean j_fabric_init_client(struct JConfiguration* configuration, struct JFabricAddr* addr, struct JFabric** instance_ptr) { + J_TRACE_FUNCTION(NULL); + struct JFabric* this; struct fi_info* hints; int res; @@ -205,6 +212,8 @@ j_fabric_init_client(struct JConfiguration* configuration, struct JFabricAddr* a gboolean j_fabric_fini(struct JFabric* this) { + J_TRACE_FUNCTION(NULL); + int res; fi_freeinfo(this->info); @@ -232,6 +241,8 @@ j_fabric_fini(struct JFabric* this) gboolean j_fabric_sread_event(struct JFabric* this, int timeout, enum JFabricEvents* event, JFabricConnectionRequest* con_req) { + J_TRACE_FUNCTION(NULL); + int res; gboolean ret = FALSE; uint32_t fi_event; @@ -262,6 +273,8 @@ j_fabric_sread_event(struct JFabric* this, int timeout, enum JFabricEvents* even gboolean j_connection_init_client(struct JConfiguration* configuration, enum JBackendType backend, guint index, struct JConnection** instance_ptr) { + J_TRACE_FUNCTION(NULL); + struct JConnection* this; gboolean ret = FALSE; int res; @@ -336,6 +349,8 @@ j_connection_init_client(struct JConfiguration* configuration, enum JBackendType gboolean j_connection_init_server(struct JFabric* fabric, GSocketConnection* gconnection, struct JConnection** instance_ptr) { + J_TRACE_FUNCTION(NULL); + struct JConnection* this; gboolean ret = FALSE; int res; @@ -351,12 +366,6 @@ j_connection_init_server(struct JFabric* fabric, GSocketConnection* gconnection, memset(this, 0, sizeof(*this)); // send addr - { // DEBUG TODO - char* str = malloc(ntohl(addr->addr_len) * 3 + 1); - size_t i; - for(i = 0; i < ntohl(addr->addr_len); ++i) { snprintf(str+i*3, 4, "%02x ", ((uint8_t*)addr->addr)[i]); } - free(str); - } g_out = g_io_stream_get_output_stream(G_IO_STREAM(gconnection)); g_output_stream_write(g_out, &addr->addr_format, sizeof(addr->addr_format), NULL, &error); G_CHECK("Failed to write addr_format to stream!"); @@ -395,6 +404,8 @@ j_connection_init_server(struct JFabric* fabric, GSocketConnection* gconnection, gboolean j_connection_init(struct JConnection* this) { + J_TRACE_FUNCTION(NULL); + gboolean ret = FALSE; int res; @@ -446,6 +457,8 @@ j_connection_init(struct JConnection* this) gboolean j_connection_create_memory_resources(struct JConnection* this) { + J_TRACE_FUNCTION(NULL); + gboolean ret = FALSE; int res; guint64 op_size, size; @@ -488,6 +501,8 @@ j_connection_create_memory_resources(struct JConnection* this) gboolean j_connection_sread_event(struct JConnection* this, int timeout, enum JConnectionEvents* event) { + J_TRACE_FUNCTION(NULL); + int res; uint32_t fi_event; gboolean ret = FALSE; @@ -526,6 +541,8 @@ j_connection_sread_event(struct JConnection* this, int timeout, enum JConnection gboolean j_connection_read_event(struct JConnection* this, enum JConnectionEvents* event) { + J_TRACE_FUNCTION(NULL); + int res; uint32_t fi_event; gboolean ret = FALSE; @@ -553,6 +570,8 @@ j_connection_read_event(struct JConnection* this, enum JConnectionEvents* event) gboolean j_connection_send(struct JConnection* this, const void* data, size_t data_len) { + J_TRACE_FUNCTION(NULL); + int res; gboolean ret = FALSE; uint8_t* segment; @@ -599,6 +618,8 @@ j_connection_send(struct JConnection* this, const void* data, size_t data_len) gboolean j_connection_recv(struct JConnection* this, size_t data_len, void* data) { + J_TRACE_FUNCTION(NULL); + gboolean ret = FALSE; int res; void* segment; @@ -632,12 +653,16 @@ j_connection_recv(struct JConnection* this, size_t data_len, void* data) gboolean j_connection_closed(struct JConnection* this) { + J_TRACE_FUNCTION(NULL); + return this->closed; } gboolean j_connection_wait_for_completion(struct JConnection* this) { + J_TRACE_FUNCTION(NULL); + gboolean ret = FALSE; int res; struct fi_cq_entry entry; @@ -695,6 +720,8 @@ j_connection_wait_for_completion(struct JConnection* this) gboolean j_connection_rma_register(struct JConnection* this, const void* data, size_t data_len, struct JConnectionMemory* handle) { + J_TRACE_FUNCTION(NULL); + int res; gboolean ret = FALSE; @@ -716,6 +743,8 @@ j_connection_rma_register(struct JConnection* this, const void* data, size_t dat gboolean j_connection_rma_unregister(struct JConnection* this, struct JConnectionMemory* handle) { + J_TRACE_FUNCTION(NULL); + int res; this->next_key = 0; /// \todo may just count key to overflow? (max value is stored in domain_attr) res = fi_close(&handle->memory_region->fid); @@ -728,6 +757,8 @@ j_connection_rma_unregister(struct JConnection* this, struct JConnectionMemory* gboolean j_connection_memory_get_id(struct JConnectionMemory* this, struct JConnectionMemoryID* id) { + J_TRACE_FUNCTION(NULL); + id->size = this->size; id->key = fi_mr_key(this->memory_region); id->offset = this->addr; @@ -737,6 +768,8 @@ j_connection_memory_get_id(struct JConnectionMemory* this, struct JConnectionMem gboolean j_connection_rma_read(struct JConnection* this, const struct JConnectionMemoryID* memoryID, void* data) { + J_TRACE_FUNCTION(NULL); + int res; gboolean ret = FALSE; struct fid_mr* mr; @@ -773,6 +806,8 @@ j_connection_rma_read(struct JConnection* this, const struct JConnectionMemoryID gboolean j_connection_fini ( struct JConnection* this) { + J_TRACE_FUNCTION(NULL); + int res; gboolean ret = FALSE; From 65a31eb9d6d6c109a01625e3749fe512a68d11f9 Mon Sep 17 00:00:00 2001 From: Julian Benda Date: Wed, 13 Oct 2021 11:35:15 +0200 Subject: [PATCH 36/60] Enchant OBJECT_WRITE don't wait after each part from message -> better performance --- lib/core/jmessage.c | 4 +++ lib/core/jnetwork.c | 8 ++--- lib/object/jobject.c | 2 +- server/loop.c | 71 +++++++++++++++++++++++++++++++++----------- 4 files changed, 60 insertions(+), 25 deletions(-) diff --git a/lib/core/jmessage.c b/lib/core/jmessage.c index 3df607ec9..58f182398 100644 --- a/lib/core/jmessage.c +++ b/lib/core/jmessage.c @@ -686,6 +686,7 @@ j_message_send(JMessage* message, struct JConnection* connection) struct JConnectionMemory* memory_regions_end = NULL; struct JConnectionMemoryID mem_id; g_autoptr(JListIterator) iterator = NULL; + clock_t stop, start = clock(); g_return_val_if_fail(message != NULL, FALSE); g_return_val_if_fail(connection != NULL, FALSE); @@ -730,6 +731,8 @@ j_message_send(JMessage* message, struct JConnection* connection) ++memory_itr; } } + stop = clock(); + start = stop; ret = j_message_write(message, connection); @@ -744,6 +747,7 @@ j_message_send(JMessage* message, struct JConnection* connection) } free(mrs); } + stop = clock(); return ret; } diff --git a/lib/core/jnetwork.c b/lib/core/jnetwork.c index 09403a7ee..94791ecd5 100644 --- a/lib/core/jnetwork.c +++ b/lib/core/jnetwork.c @@ -714,6 +714,7 @@ j_connection_wait_for_completion(struct JConnection* this) this->memory.used = 0; ret = TRUE; end: + // g_message("\t\toverhead: %f", (double)(sum)/CLOCKS_PER_SEC); return ret; } @@ -788,12 +789,7 @@ j_connection_rma_read(struct JConnection* this, const struct JConnectionMemoryID mr); /// \todo fix the timing for minimal timeout. This is needed when device is too bussy if(res == -FI_EAGAIN) { - struct timespec ts; - int tres; - ts.tv_nsec = 1000000; - do { - tres = nanosleep(&ts, &ts); - } while(tres && errno == EINTR); + j_connection_wait_for_completion(this); } } while ( res == -FI_EAGAIN); CHECK("Failed to initate reading"); diff --git a/lib/object/jobject.c b/lib/object/jobject.c index 500ec40fb..8827310a8 100644 --- a/lib/object/jobject.c +++ b/lib/object/jobject.c @@ -695,7 +695,6 @@ j_object_write_exec(JList* operations, JSemantics* semantics) safety = j_semantics_get(semantics, J_SEMANTICS_SAFETY); object_connection = j_connection_pool_pop(J_BACKEND_TYPE_OBJECT, object->index); j_message_send(message, object_connection); - if (safety == J_SEMANTICS_SAFETY_NETWORK || safety == J_SEMANTICS_SAFETY_STORAGE) { g_autoptr(JMessage) reply = NULL; @@ -732,6 +731,7 @@ j_object_write_exec(JList* operations, JSemantics* semantics) ret = j_backend_object_close(object_backend, object_handle) && ret; } + /* if (lock != NULL) { diff --git a/server/loop.c b/server/loop.c index f13b7af1d..42c753c8e 100644 --- a/server/loop.c +++ b/server/loop.c @@ -28,6 +28,12 @@ static guint jd_thread_num = 0; +struct ObjectWriteEntry { + uint64_t size; + uint64_t offset; + void* data; +}; + gboolean jd_handle_message(JMessage* message, struct JConnection* connection, JMemoryChunk* memory_chunk, guint64 memory_chunk_size, JStatistics* statistics) { @@ -266,6 +272,8 @@ jd_handle_message(JMessage* message, struct JConnection* connection, JMemoryChun g_autoptr(JMessage) reply = NULL; gpointer object; gboolean ret; + struct ObjectWriteEntry* writeEntries; + guint64 entryItr; if (safety == J_SEMANTICS_SAFETY_NETWORK || safety == J_SEMANTICS_SAFETY_STORAGE) { @@ -277,6 +285,8 @@ jd_handle_message(JMessage* message, struct JConnection* connection, JMemoryChun ret = j_backend_object_open(jd_object_backend, namespace, path, &object); + writeEntries = malloc(sizeof(struct ObjectWriteEntry) * operation_count); + entryItr = 0; for (i = 0; i < operation_count; i++) { gchar* buf; @@ -286,28 +296,53 @@ jd_handle_message(JMessage* message, struct JConnection* connection, JMemoryChun offset = j_message_get_8(message); memoryID = j_message_get_memory_id(message); - // g_message("memory size: %lu", memoryID->size); - if (memoryID->size > memory_chunk_size && reply != NULL && G_LIKELY(ret)) + if (memoryID->size <= memory_chunk_size && G_LIKELY(ret)) { - /// \todo return proper error + buf = j_memory_chunk_get(memory_chunk, memoryID->size); + if (buf == NULL) + { + j_connection_wait_for_completion(connection); ///< \todo parallelize more + for(; entryItr < i; ++entryItr) { + struct ObjectWriteEntry* entry = writeEntries + entryItr; + j_statistics_add(statistics, J_STATISTICS_BYTES_RECEIVED, entry->size); + j_backend_object_write(jd_object_backend, object, entry->data, entry->size, entry->offset, &bytes_written); + j_statistics_add(statistics, J_STATISTICS_BYTES_WRITTEN, bytes_written); + if(reply != NULL) + { + j_message_add_operation(reply, sizeof(guint64)); + j_message_append_8(reply, &bytes_written); + } + } + j_memory_chunk_reset(memory_chunk); + buf = j_memory_chunk_get(memory_chunk, memoryID->size); + /// \todo check for oversized memory areas! + } + + // Guaranteed to work because is reseted on demand + g_assert(buf != NULL); + + j_connection_rma_read(connection, memoryID, buf); + writeEntries[i] = (struct ObjectWriteEntry){ + .size = memoryID->size, + .offset = offset, + .data = buf}; + } else if (reply != NULL) { + /// @todo return write error + bytes_written = 0; j_message_add_operation(reply, sizeof(guint64)); j_message_append_8(reply, &bytes_written); - continue; } + } - // Guaranteed to work because memory_chunk is reset below - buf = j_memory_chunk_get(memory_chunk, memoryID->size); - g_assert(buf != NULL); - - j_connection_rma_read(connection, memoryID, buf); - j_connection_wait_for_completion(connection); ///< \todo pararlleize - j_statistics_add(statistics, J_STATISTICS_BYTES_RECEIVED, memoryID->size); - // CONTINUE - - if (G_LIKELY(ret)) - { - j_backend_object_write(jd_object_backend, object, buf, length, offset, &bytes_written); + j_connection_wait_for_completion(connection); + if(G_LIKELY(ret)) + { + for(; entryItr < operation_count; ++entryItr) { + guint64 bytes_written = 0; + struct ObjectWriteEntry* entry = writeEntries + entryItr; + j_statistics_add(statistics, J_STATISTICS_BYTES_RECEIVED, entry->size); + j_backend_object_write(jd_object_backend, object, entry->data, entry->size, entry->offset, &bytes_written); j_statistics_add(statistics, J_STATISTICS_BYTES_WRITTEN, bytes_written); if (reply != NULL) @@ -316,9 +351,9 @@ jd_handle_message(JMessage* message, struct JConnection* connection, JMemoryChun j_message_append_8(reply, &bytes_written); } } - - j_memory_chunk_reset(memory_chunk); } + j_memory_chunk_reset(memory_chunk); + free(writeEntries); if (safety == J_SEMANTICS_SAFETY_STORAGE) { From 108f2a26c050ae8e45b02f569926d802719cf8fc Mon Sep 17 00:00:00 2001 From: Julian Benda Date: Wed, 13 Oct 2021 11:52:31 +0200 Subject: [PATCH 37/60] Reduce serialisation for READ --- lib/core/jmessage.c | 1 - lib/object/jobject.c | 1 - 2 files changed, 2 deletions(-) diff --git a/lib/core/jmessage.c b/lib/core/jmessage.c index 58f182398..395e878ed 100644 --- a/lib/core/jmessage.c +++ b/lib/core/jmessage.c @@ -686,7 +686,6 @@ j_message_send(JMessage* message, struct JConnection* connection) struct JConnectionMemory* memory_regions_end = NULL; struct JConnectionMemoryID mem_id; g_autoptr(JListIterator) iterator = NULL; - clock_t stop, start = clock(); g_return_val_if_fail(message != NULL, FALSE); g_return_val_if_fail(connection != NULL, FALSE); diff --git a/lib/object/jobject.c b/lib/object/jobject.c index 8827310a8..acb8fbfd2 100644 --- a/lib/object/jobject.c +++ b/lib/object/jobject.c @@ -559,7 +559,6 @@ j_object_read_exec(JList* operations, JSemantics* semantics) if (mem_id->size > 0) { j_connection_rma_read(object_connection, mem_id, data); - j_connection_wait_for_completion(object_connection); ///\todo deserelize } } From 918b94b9990b3a2b6c543d67671ec50e6c33716b Mon Sep 17 00:00:00 2001 From: Julian Benda Date: Mon, 18 Oct 2021 22:24:59 +0200 Subject: [PATCH 38/60] Enchantet writing/reading of small objects not batched reads/writes now part of the message -> less overhead --- include/core/jnetwork.h | 3 +++ lib/core/jmessage.c | 57 +++++++++++++++++++++++++++++------------ lib/core/jnetwork.c | 13 +++++++++- lib/object/jobject.c | 6 ++++- server/loop.c | 56 ++++++++++++++++++++++------------------ 5 files changed, 92 insertions(+), 43 deletions(-) diff --git a/include/core/jnetwork.h b/include/core/jnetwork.h index f4d7dc0ad..636265fe5 100644 --- a/include/core/jnetwork.h +++ b/include/core/jnetwork.h @@ -432,4 +432,7 @@ j_connection_rma_unregister(struct JConnection* instance, struct JConnectionMemory* handle ///< [in] for memory region to unregister ); +struct JConfiguration* +j_connection_get_configuration(struct JConnection* this); + #endif diff --git a/lib/core/jmessage.c b/lib/core/jmessage.c index 395e878ed..28bb42f27 100644 --- a/lib/core/jmessage.c +++ b/lib/core/jmessage.c @@ -36,6 +36,7 @@ #include #include #include +#include #define EXE(cmd, ...) do { if(cmd == FALSE) { g_warning(__VA_ARGS__); goto end; } } while(FALSE) @@ -673,7 +674,6 @@ struct JConnectionMemory { guint64 addr; guint64 size; }; - gboolean j_message_send(JMessage* message, struct JConnection* connection) { @@ -686,6 +686,7 @@ j_message_send(JMessage* message, struct JConnection* connection) struct JConnectionMemory* memory_regions_end = NULL; struct JConnectionMemoryID mem_id; g_autoptr(JListIterator) iterator = NULL; + gboolean fits = false; g_return_val_if_fail(message != NULL, FALSE); g_return_val_if_fail(connection != NULL, FALSE); @@ -703,19 +704,30 @@ j_message_send(JMessage* message, struct JConnection* connection) if (message->send_list != NULL) { + { + guint64 total_data_length = + sizeof(message->header) + + j_message_length(message); + iterator = j_list_iterator_new(message->send_list); + while(j_list_iterator_next(iterator)) { + JMessageData* message_data = j_list_iterator_get(iterator); + total_data_length += message_data->header_size + message_data->length; + } + j_list_iterator_free(iterator); + fits = total_data_length < 10 * 1024; ///\todo fix magic number + } iterator = j_list_iterator_new(message->send_list); while (j_list_iterator_next(iterator)) { JMessageData* message_data = j_list_iterator_get(iterator); - EXE(j_connection_rma_register(connection, message_data->data, message_data->length, memory_itr), - "Failed to register message data memory!"); - j_message_add_operation(message, sizeof(struct JConnectionMemoryID) + message_data->header_size); + j_message_add_operation(message, + sizeof(struct JConnectionMemoryID) + + message_data->header_size + + fits ? message_data->length : 0); - EXE(j_connection_memory_get_id(memory_itr, &mem_id), - "Failed to get memory it!"); if(message_data->header_size > 0) { if(message_data->header_size <= sizeof(message_data->header)) { EXE(j_message_append_n(message, &message_data->header, message_data->header_size), @@ -725,28 +737,41 @@ j_message_send(JMessage* message, struct JConnection* connection) "Failed to append header"); } } - EXE(j_message_append_memory_id(message, &mem_id), - "Failed to append memory id to message!"); + if(!fits) { + EXE(j_connection_rma_register(connection, message_data->data, message_data->length, memory_itr), + "Failed to register message data memory!"); + EXE(j_connection_memory_get_id(memory_itr, &mem_id), + "Failed to get memory it!"); + EXE(j_message_append_memory_id(message, &mem_id), + "Failed to append memory id to message!"); + } else { + mem_id.size = message_data->length; + mem_id.key = 0; + mem_id.offset = 0; + EXE(j_message_append_memory_id(message, &mem_id), + "Failed to append memory information!"); + EXE(j_message_append_n(message, message_data->data, message_data->length), + "Failed to append message data!"); + } ++memory_itr; } } - stop = clock(); - start = stop; ret = j_message_write(message, connection); end: if(memory_regions != NULL) { struct JConnectionMemory* mrs = memory_regions; - while(memory_regions != memory_itr) { - j_connection_rma_unregister(connection, memory_regions++); - } - if(memory_itr != memory_regions_end && memory_itr->memory_region) { - j_connection_rma_unregister(connection, memory_itr); + if(!fits) { + while(memory_regions != memory_itr) { + j_connection_rma_unregister(connection, memory_regions++); + } + if(memory_itr != memory_regions_end && memory_itr->memory_region) { + j_connection_rma_unregister(connection, memory_itr); + } } free(mrs); } - stop = clock(); return ret; } diff --git a/lib/core/jnetwork.c b/lib/core/jnetwork.c index 94791ecd5..938bf295b 100644 --- a/lib/core/jnetwork.c +++ b/lib/core/jnetwork.c @@ -12,6 +12,8 @@ #include #include +#define KEY_MIN 1 + /// Used to initialize common parts between different connection. /** This will create the following libfabric resources: * * the domain @@ -411,6 +413,7 @@ j_connection_init(struct JConnection* this) this->running_actions.msg_len = 0; this->running_actions.rma_len = 0; + this->next_key = KEY_MIN; res = fi_eq_open(this->fabric->fabric, &(struct fi_eq_attr){.wait_obj = FI_WAIT_UNSPEC}, &this->eq, NULL); @@ -747,7 +750,7 @@ j_connection_rma_unregister(struct JConnection* this, struct JConnectionMemory* J_TRACE_FUNCTION(NULL); int res; - this->next_key = 0; /// \todo may just count key to overflow? (max value is stored in domain_attr) + this->next_key = KEY_MIN; /// \todo may just count key to overflow? (max value is stored in domain_attr) res = fi_close(&handle->memory_region->fid); CHECK("Failed to unregistrer rma memory!"); return TRUE; @@ -836,3 +839,11 @@ j_connection_fini ( struct JConnection* this) { end: return ret; } + +struct JConfiguration* +j_connection_get_configuration(struct JConnection* this) +{ + J_TRACE_FUNCTION(NULL); + + return this->fabric->config; +} diff --git a/lib/object/jobject.c b/lib/object/jobject.c index acb8fbfd2..a0bcb032a 100644 --- a/lib/object/jobject.c +++ b/lib/object/jobject.c @@ -558,7 +558,11 @@ j_object_read_exec(JList* operations, JSemantics* semantics) if (mem_id->size > 0) { - j_connection_rma_read(object_connection, mem_id, data); + if(mem_id->key == 0 && mem_id->offset == 0) { + memcpy(data, j_message_get_n(reply, mem_id->size), mem_id->size); + } else { + j_connection_rma_read(object_connection, mem_id, data); + } } } diff --git a/server/loop.c b/server/loop.c index 42c753c8e..246693b73 100644 --- a/server/loop.c +++ b/server/loop.c @@ -297,36 +297,42 @@ jd_handle_message(JMessage* message, struct JConnection* connection, JMemoryChun offset = j_message_get_8(message); memoryID = j_message_get_memory_id(message); - if (memoryID->size <= memory_chunk_size && G_LIKELY(ret)) + if (G_LIKELY(memoryID->size <= memory_chunk_size) && G_LIKELY(ret)) { - buf = j_memory_chunk_get(memory_chunk, memoryID->size); - if (buf == NULL) - { - j_connection_wait_for_completion(connection); ///< \todo parallelize more - for(; entryItr < i; ++entryItr) { - struct ObjectWriteEntry* entry = writeEntries + entryItr; - j_statistics_add(statistics, J_STATISTICS_BYTES_RECEIVED, entry->size); - j_backend_object_write(jd_object_backend, object, entry->data, entry->size, entry->offset, &bytes_written); - j_statistics_add(statistics, J_STATISTICS_BYTES_WRITTEN, bytes_written); - if(reply != NULL) - { - j_message_add_operation(reply, sizeof(guint64)); - j_message_append_8(reply, &bytes_written); + if (memoryID->key == 0 && memoryID->offset == 0) { + writeEntries[i] = (struct ObjectWriteEntry){ + .size = memoryID->size, + .offset = offset, + .data = j_message_get_n(message, memoryID->size)}; + } else { + buf = j_memory_chunk_get(memory_chunk, memoryID->size); + if (buf == NULL) + { + j_connection_wait_for_completion(connection); ///< \todo paralleilize more + for(; entryItr < i; ++entryItr) { + struct ObjectWriteEntry* entry = writeEntries + entryItr; + j_statistics_add(statistics, J_STATISTICS_BYTES_RECEIVED, entry->size); + j_backend_object_write(jd_object_backend, object, entry->data, entry->size, entry->offset, &bytes_written); + j_statistics_add(statistics, J_STATISTICS_BYTES_WRITTEN, bytes_written); + if(reply != NULL) + { + j_message_add_operation(reply, sizeof(guint64)); + j_message_append_8(reply, &bytes_written); + } } + j_memory_chunk_reset(memory_chunk); + buf = j_memory_chunk_get(memory_chunk, memoryID->size); } - j_memory_chunk_reset(memory_chunk); - buf = j_memory_chunk_get(memory_chunk, memoryID->size); - /// \todo check for oversized memory areas! - } - // Guaranteed to work because is reseted on demand - g_assert(buf != NULL); + // Guaranteed to work, because chunk get reseted if to full + g_assert(buf != NULL); - j_connection_rma_read(connection, memoryID, buf); - writeEntries[i] = (struct ObjectWriteEntry){ - .size = memoryID->size, - .offset = offset, - .data = buf}; + j_connection_rma_read(connection, memoryID, buf); + writeEntries[i] = (struct ObjectWriteEntry){ + .size = memoryID->size, + .offset = offset, + .data = buf}; + } } else if (reply != NULL) { /// @todo return write error bytes_written = 0; From 73f0c8547581d50c68d0eb57e31c6cc082cdaeb2 Mon Sep 17 00:00:00 2001 From: Julian Benda Date: Wed, 20 Oct 2021 11:13:43 +0200 Subject: [PATCH 39/60] Implement new read logic in distributed object --- lib/object/jdistributed-object.c | 6 +++++- lib/object/jobject.c | 3 +-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/object/jdistributed-object.c b/lib/object/jdistributed-object.c index 3e25fa740..bab4124ec 100644 --- a/lib/object/jdistributed-object.c +++ b/lib/object/jdistributed-object.c @@ -372,7 +372,11 @@ j_distributed_object_read_background_operation(gpointer data) if (memoryID->size > 0) { - j_connection_rma_read(object_connection, memoryID, read_data); + if(memoryID->key == 0 && memoryID->offset == 0) { + memcpy(read_data, j_message_get_n(reply, memoryID->size), memoryID->size); + } else { + j_connection_rma_read(object_connection, memoryID, read_data); + } } g_slice_free(JDistributedObjectReadBuffer, buffer); diff --git a/lib/object/jobject.c b/lib/object/jobject.c index a0bcb032a..f3d864cd6 100644 --- a/lib/object/jobject.c +++ b/lib/object/jobject.c @@ -446,10 +446,9 @@ j_object_read_exec(JList* operations, JSemantics* semantics) { JObjectOperation* operation = j_list_get_first(operations); + g_assert(operation != NULL); object = operation->read.object; - - g_assert(operation != NULL); g_assert(object != NULL); } From 561e313a37d356c1466e28a7f320389248134138 Mon Sep 17 00:00:00 2001 From: Julian Benda Date: Wed, 20 Oct 2021 13:44:52 +0200 Subject: [PATCH 40/60] Check if list exist before query size --- lib/core/jmessage.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/core/jmessage.c b/lib/core/jmessage.c index 28bb42f27..adf876028 100644 --- a/lib/core/jmessage.c +++ b/lib/core/jmessage.c @@ -842,7 +842,7 @@ j_message_write(JMessage* message, struct JConnection* connection) "Failed to initiated sending message body."); } - if(j_list_length(message->send_list)) { + if(message->send_list && j_list_length(message->send_list)) { EXE(j_connection_recv(connection, sizeof(ack), &ack), "Failed to initiated ACK receive."); } From c3a608c0656ec8d2ac7f36327c4f64f14a9837a4 Mon Sep 17 00:00:00 2001 From: Julian Benda Date: Wed, 20 Oct 2021 13:45:09 +0200 Subject: [PATCH 41/60] Add large object benchmarks large objects = 256KB small objects = 4KB --- benchmark/object/distributed-object.c | 39 ++++++++++++++++++++++++--- benchmark/object/object.c | 38 +++++++++++++++++++++++--- 2 files changed, 69 insertions(+), 8 deletions(-) diff --git a/benchmark/object/distributed-object.c b/benchmark/object/distributed-object.c index fc2e970c5..af459d4d6 100644 --- a/benchmark/object/distributed-object.c +++ b/benchmark/object/distributed-object.c @@ -27,6 +27,9 @@ #include "benchmark.h" +#define SMALL_OBJECT_SIZE 4 * 1024 +#define LARGE_OBJECT_SIZE 256 * 1024 + static void _benchmark_distributed_object_create(BenchmarkRun* run, gboolean use_batch) { @@ -302,13 +305,25 @@ _benchmark_distributed_object_read(BenchmarkRun* run, gboolean use_batch, guint static void benchmark_distributed_object_read(BenchmarkRun* run) { - _benchmark_distributed_object_read(run, FALSE, 4 * 1024); + _benchmark_distributed_object_read(run, FALSE, SMALL_OBJECT_SIZE); +} + +static void +benchmark_distributed_object_read_large(BenchmarkRun* run) +{ + _benchmark_distributed_object_read(run, FALSE, LARGE_OBJECT_SIZE); } static void benchmark_distributed_object_read_batch(BenchmarkRun* run) { - _benchmark_distributed_object_read(run, TRUE, 4 * 1024); + _benchmark_distributed_object_read(run, TRUE, SMALL_OBJECT_SIZE); +} + +static void +benchmark_distributed_object_read_large_batch(BenchmarkRun* run) +{ + _benchmark_distributed_object_read(run, TRUE, LARGE_OBJECT_SIZE); } static void @@ -372,13 +387,25 @@ _benchmark_distributed_object_write(BenchmarkRun* run, gboolean use_batch, guint static void benchmark_distributed_object_write(BenchmarkRun* run) { - _benchmark_distributed_object_write(run, FALSE, 4 * 1024); + _benchmark_distributed_object_write(run, FALSE, SMALL_OBJECT_SIZE); +} + +static void +benchmark_distributed_object_write_large(BenchmarkRun* run) +{ + _benchmark_distributed_object_write(run, FALSE, LARGE_OBJECT_SIZE); } static void benchmark_distributed_object_write_batch(BenchmarkRun* run) { - _benchmark_distributed_object_write(run, TRUE, 4 * 1024); + _benchmark_distributed_object_write(run, TRUE, SMALL_OBJECT_SIZE); +} + +static void +benchmark_distributed_object_write_large_batch(BenchmarkRun* run) +{ + _benchmark_distributed_object_write(run, TRUE, LARGE_OBJECT_SIZE); } static void @@ -451,9 +478,13 @@ benchmark_distributed_object(void) j_benchmark_add("/object/distributed-object/status-batch", benchmark_distributed_object_status_batch); /// \todo get j_benchmark_add("/object/distributed-object/read", benchmark_distributed_object_read); + j_benchmark_add("/object/distributed-object/read-large", benchmark_distributed_object_read_large); j_benchmark_add("/object/distributed-object/read-batch", benchmark_distributed_object_read_batch); + j_benchmark_add("/object/distributed-object/read-large-batch", benchmark_distributed_object_read_large_batch); j_benchmark_add("/object/distributed-object/write", benchmark_distributed_object_write); + j_benchmark_add("/object/distributed-object/write-large", benchmark_distributed_object_write_large); j_benchmark_add("/object/distributed-object/write-batch", benchmark_distributed_object_write_batch); + j_benchmark_add("/object/distributed-object/write-large-batch", benchmark_distributed_object_write_large_batch); j_benchmark_add("/object/distributed-object/unordered-create-delete", benchmark_distributed_object_unordered_create_delete); j_benchmark_add("/object/distributed-object/unordered-create-delete-batch", benchmark_distributed_object_unordered_create_delete_batch); } diff --git a/benchmark/object/object.c b/benchmark/object/object.c index b2cc48648..ef7edf0a3 100644 --- a/benchmark/object/object.c +++ b/benchmark/object/object.c @@ -27,6 +27,9 @@ #include "benchmark.h" +#define SMALL_OBJECT_SIZE 4 * 1024 +#define LARGE_OBJECT_SIZE 256 * 1024 + static void _benchmark_object_create(BenchmarkRun* run, gboolean use_batch) { @@ -294,13 +297,25 @@ _benchmark_object_read(BenchmarkRun* run, gboolean use_batch, guint block_size) static void benchmark_object_read(BenchmarkRun* run) { - _benchmark_object_read(run, FALSE, 4 * 1024); + _benchmark_object_read(run, FALSE, SMALL_OBJECT_SIZE); +} + +static void +benchmark_object_read_large(BenchmarkRun* run) +{ + _benchmark_object_read(run, FALSE, LARGE_OBJECT_SIZE); } static void benchmark_object_read_batch(BenchmarkRun* run) { - _benchmark_object_read(run, TRUE, 4 * 1024); + _benchmark_object_read(run, TRUE, SMALL_OBJECT_SIZE); +} + +static void +benchmark_object_read_large_batch(BenchmarkRun* run) +{ + _benchmark_object_read(run, TRUE, LARGE_OBJECT_SIZE); } static void @@ -362,13 +377,24 @@ _benchmark_object_write(BenchmarkRun* run, gboolean use_batch, guint block_size) static void benchmark_object_write(BenchmarkRun* run) { - _benchmark_object_write(run, FALSE, 4 * 1024); + _benchmark_object_write(run, FALSE, SMALL_OBJECT_SIZE); +} + +static void +benchmark_object_write_large(BenchmarkRun* run) +{ + _benchmark_object_write(run, FALSE, LARGE_OBJECT_SIZE); } static void benchmark_object_write_batch(BenchmarkRun* run) { - _benchmark_object_write(run, TRUE, 4 * 1024); + _benchmark_object_write(run, TRUE, SMALL_OBJECT_SIZE); +} +static void +benchmark_object_write_large_batch(BenchmarkRun* run) +{ + _benchmark_object_write(run, TRUE, LARGE_OBJECT_SIZE); } static void @@ -439,9 +465,13 @@ benchmark_object(void) j_benchmark_add("/object/object/status-batch", benchmark_object_status_batch); /// \todo get j_benchmark_add("/object/object/read", benchmark_object_read); + j_benchmark_add("/object/object/read-large", benchmark_object_read_large); j_benchmark_add("/object/object/read-batch", benchmark_object_read_batch); + j_benchmark_add("/object/object/read-large-batch", benchmark_object_read_large_batch); j_benchmark_add("/object/object/write", benchmark_object_write); + j_benchmark_add("/object/object/write-large", benchmark_object_write_large); j_benchmark_add("/object/object/write-batch", benchmark_object_write_batch); + j_benchmark_add("/object/object/write-large-batch", benchmark_object_write_large_batch); j_benchmark_add("/object/object/unordered-create-delete", benchmark_object_unordered_create_delete); j_benchmark_add("/object/object/unordered-create-delete-batch", benchmark_object_unordered_create_delete_batch); } From e4e38578dc39c3c453b386f3f745a4d1bd74c692 Mon Sep 17 00:00:00 2001 From: Julian Benda Date: Fri, 22 Oct 2021 17:46:25 +0200 Subject: [PATCH 42/60] Update Docu for network code --- include/core/jmessage.h | 398 +++++++++++++--------------------------- include/core/jnetwork.h | 47 ++--- lib/core/jmessage.c | 9 - 3 files changed, 156 insertions(+), 298 deletions(-) diff --git a/include/core/jmessage.h b/include/core/jmessage.h index 22dc55e37..3b5373e5f 100644 --- a/include/core/jmessage.h +++ b/include/core/jmessage.h @@ -16,10 +16,6 @@ * along with this program. If not, see . */ -/** - * \file - **/ - #ifndef JULEA_MESSAGE_H #define JULEA_MESSAGE_H @@ -32,12 +28,12 @@ G_BEGIN_DECLS -/** - * \defgroup JMessage Message - * - * @{ - **/ - +/** \defgroup JMessage Message + * Sends message between server and client + */ +/** \public \memberof JMessage + * \sa j_message_new, j_message_get_type + */ enum JMessageType { J_MESSAGE_NONE, @@ -64,11 +60,12 @@ enum JMessageType J_MESSAGE_DB_DELETE, J_MESSAGE_DB_QUERY }; - typedef enum JMessageType JMessageType; +/** \class JMessage lib/jmessage.h + * \ingroup network JMessage + */ struct JMessage; - typedef struct JMessage JMessage; G_END_DECLS @@ -78,332 +75,201 @@ G_END_DECLS G_BEGIN_DECLS -/** - * Creates a new message. - * - * \code - * \endcode - * - * \param op_type An operation type. - * \param length A length. - * - * \return A new message. Should be freed with j_message_unref(). +/// Creates a new Message +/** \public \memberof JMessage + * \param op_type[in] message type + * \param length[in] message body size. + * \return A new message. Should be freed with j_message_unref(). **/ JMessage* j_message_new(JMessageType op_type, gsize length); -/** - * Creates a new reply message. - * - * \code - * \endcode - * - * \param message A message. - * - * \return A new reply message. Should be freed with j_message_unref(). +/// Creates a new reply message +/** \public \memberof JMessage + * \param message[in] message to reply to + * \return A new reply message. Should be freed with j_message_unref(). **/ JMessage* j_message_new_reply(JMessage* message); -/** - * Increases a message's reference count. - * - * \code - * JMessage* m; - * - * j_message_ref(m); - * \endcode - * - * \param message A message. - * - * \return \p message. +/// Increases a message's reference count. +/** \public \memberof JMessage + * \return this + * \sa j_message_unref **/ -JMessage* j_message_ref(JMessage* message); +JMessage* j_message_ref(JMessage* this); -/** - * Decreases a message's reference count. - * When the reference count reaches zero, frees the memory allocated for the message. - * - * \code - * \endcode - * - * \param message A message. - **/ -void j_message_unref(JMessage* message); +/// Decreases a message's reference count. +/** When the reference count reaches zero, frees the memory allocated for the message. + * \public \memberof JMessage + * \sa j_message_ref + */ +void j_message_unref(JMessage* this); G_DEFINE_AUTOPTR_CLEANUP_FUNC(JMessage, j_message_unref) -/** - * Returns a message's type. - * - * \code - * \endcode - * - * \param message A message. - * - * \return The message's operation type. - **/ -JMessageType j_message_get_type(JMessage const* message); +/// Returns a message's type. +/** \public \memberof JMessage + * \return The message's operation type. + */ +JMessageType j_message_get_type(JMessage const* this); -/** - * Returns a message's count. - * - * \code - * \endcode - * - * \param message A message. - * - * \return The message's operation count. - **/ -guint32 j_message_get_count(JMessage const* message); +/// Returns a message's operation count. +/** \public \memberof JMessage + */ +guint32 j_message_get_count(JMessage const* this); -/** - * Appends 1 byte to a message. - * - * \code - * \endcode - * - * \param message A message. - * \param data Data to append. - * - * \return TRUE on success, FALSE if an error occurred. +/// Appends 1 byte to a message. +/** \public \memberof JMessage + * \param data Data to append. + * \retval TRUE on success. **/ -gboolean j_message_append_1(JMessage* message, gconstpointer data); +gboolean j_message_append_1(JMessage* this, gconstpointer data); -/** - * Appends 4 bytes to a message. - * The bytes are converted to little endian automatically. - * - * \code - * \endcode - * - * \param message A message. - * \param data Data to append. - * - * \return TRUE on success, FALSE if an error occurred. +/// Appends 4 byte to a message. +/** \public \memberof JMessage + * \param data Data to append. + * \retval TRUE on success. **/ -gboolean j_message_append_4(JMessage* message, gconstpointer data); +gboolean j_message_append_4(JMessage* this, gconstpointer data); -/** - * Appends 8 bytes to a message. - * The bytes are converted to little endian automatically. - * - * \code - * \endcode - * - * \param message A message. - * \param data Data to append. - * - * \return TRUE on success, FALSE if an error occurred. +/// Appends 8 byte to a message. +/** \public \memberof JMessage + * \param data Data to append. + * \retval TRUE on success. **/ -gboolean j_message_append_8(JMessage* message, gconstpointer data); +gboolean j_message_append_8(JMessage* this, gconstpointer data); -/** - * Appends a number of bytes to a message. - * +///Appends a number of bytes to a message. +/** \public \memberof JMessage * \code - * gchar* str = "Hello world!"; + * int data[32]; * ... - * j_message_append_n(message, str, strlen(str) + 1); + * j_message_append_n(message, data, 32); * \endcode * - * \param message A message. * \param data Data to append. * \param length Length of data. - * - * \return TRUE on success, FALSE if an error occurred. + * \retval TRUE on success. + * \sa j_message_append_string, j_message_append_memory_id **/ gboolean j_message_append_n(JMessage* message, gconstpointer data, gsize length); -/** - * Appends a string to a message. - * +/// Appends a string to a message. +/** \public \memberof JMessage * \code * gchar* str = "Hello world!"; * ... * j_message_append_string(message, str); * \endcode * - * \param message A message. * \param str String to append. - * - * \return TRUE on success, FALSE if an error occurred. + * \retval TRUE on success. **/ -gboolean j_message_append_string(JMessage* message, gchar const* str); +gboolean j_message_append_string(JMessage* this, gchar const* str); /// Appends a memory identifier to a message -gboolean j_message_append_memory_id(JMessage*, const struct JConnectionMemoryID*); +/** \public \memberof JMessage + * \retval TRUE on success. + */ +gboolean j_message_append_memory_id(JMessage* this, const struct JConnectionMemoryID* memory_id); -/** - * Gets 1 byte from a message. - * - * \code - * \endcode - * - * \param message A message. - * - * \return A character. - **/ -gchar j_message_get_1(JMessage* message); +/// Gets 1 byte from a message. +/** \public \memberof JMessage */ +gchar j_message_get_1(JMessage* this); -/** - * Gets 4 bytes from a message. - * The bytes are converted from little endian automatically. - * - * \code - * \endcode - * - * \param message A message. - * - * \return A 4-bytes integer. +/// Gets 4 bytes from a message. +/** The bytes are converted from little endian automatically. + * \public \memberof JMessage + * \attention byte order depends on endian on system. + * \return 4 byte integer containing data **/ -gint32 j_message_get_4(JMessage* message); +gint32 j_message_get_4(JMessage* this); -/** - * Gets 8 bytes from a message. - * The bytes are converted from little endian automatically. - * - * \code - * \endcode - * - * \param message A message. - * - * \return An 8-bytes integer. +/// Gets 8 bytes from a message. +/** The bytes are converted from little endian automatically. + * \public \memberof JMessage + * \attention byte order depends on endian + * \return An 8-bytes integer containing data. **/ -gint64 j_message_get_8(JMessage* message); +gint64 j_message_get_8(JMessage* this); -/** - * Gets n bytes from a message. - * - * \code - * \endcode - * - * \param message A message. +/// Gets n bytes from a message. +/** \public \memberof JMessage * \param length Number of bytes to get. - * + * \attention the data are still owned and managed by the message! * \return A pointer to the data. **/ -gpointer j_message_get_n(JMessage* message, gsize length); +gpointer j_message_get_n(JMessage* this, gsize length); -/** - * Gets a string from a message. - * - * \code - * \endcode - * - * \param message A message. - * +/// Gets a string from a message. +/** \public \memberof JMessage + * \attention the string is still owned and managed by the message! * \return A string. **/ -gchar const* j_message_get_string(JMessage* message); +gchar const* j_message_get_string(JMessage* this); /// Gets an memory identifier from a message. +/** \public \memberof JMessage + * \attention The memory is still owned and managed by the message! + */ const struct JConnectionMemoryID* j_message_get_memory_id(JMessage*); -/** - * Adds new data to send to a message. - * - * \code - * \endcode - * +/// Add new data block to send with message. +/** \public \memberof JMessage * \param data,length Data segment to send. * \param header,h_size header data (included in message) **/ void j_message_add_send(JMessage* this, gconstpointer data, guint64 length, void* header, guint64 h_size); -/** - * Adds a new operation to a message. - * - * \code - * \endcode - * - * \param message A message. - * \param length A length. - **/ -void j_message_add_operation(JMessage* message, gsize length); - -/** - * Writes a message to the network. - * - * \code - * \endcode - * - * \param message A message. - * \param stream A network stream. - * - * \return TRUE on success, FALSE if an error occurred. +/// Adds a new operation to a message. +/** \public \memberof JMessage + * \remark Operation data must appended with the j_message_append functions. + * \param length length of operation. + * \sa j_message_append_1, j_message_append_4, j_message_append_8, j_message_append_n, j_message_append_string, j_message_append_memory_id. **/ +void j_message_add_operation(JMessage* this, gsize length); -/** Append sendend data memory ids to end of message! */ -gboolean j_message_send(JMessage*, struct JConnection*); +/// Sends message via connection +/** \public \memberof JMessage + * \remark append one MemoryID for each data segment send with the message + * \retval TRUE on success. + */ +gboolean j_message_send(JMessage* this, struct JConnection* connection); -/** - * Reads a message from the network. - * - * \code - * \endcode - * - * \param message A message. - * \param stream A network stream. - * - * \return TRUE on success, FALSE if an error occurred. +/// Reads a message from connection. +/* \private \memberof JMessage + * \retval TRUE on success. + * \public \memberof JMessage **/ -gboolean j_message_receive(JMessage*, struct JConnection*); +gboolean j_message_receive(JMessage* this, struct JConnection* connection); + /// signal when rma read actions are finished. -/** Sends ACK flag, to signal that host can free data memory. Also wait for all network actions to coplete! - * @param message checks if message has memory regions, only then sends ack. Can be NULL to force ack sending +/** Sends ACK flag, to signal that host can free data memory. Also wait for all network actions to complete! + * \param message checks if message has memory regions, only then sends ack. Can be NULL to force ack sending + * \retval TRUE on success */ -gboolean j_message_send_ack(JMessage* message, struct JConnection*); +gboolean j_message_send_ack(JMessage* this, struct JConnection* connection); -/** - * Reads a message from the network. - * - * \code - * \endcode - * - * \param message A message. - * \param stream A network stream. - * - * \return TRUE on success, FALSE if an error occurred. +///Reads a message from the network. +/** \public \memberof + * \retval TRUE on success. + * \attention for internal usage only, please use: j_message_receive() **/ -gboolean j_message_read(JMessage*, struct JConnection*); +gboolean j_message_read(JMessage* this, struct JConnection* connection); -/** Writes a message to the network. - * - * \attention send without data segment! - * \code - * \endcode - * - * \param message A message. - * \param stream A network stream. - * +/// Writes a message to the network. +/**\private \memberof JMessage * \return TRUE on success, FALSE if an error occurred. + * \attention for internal usage only, please use: j_message_send() **/ -gboolean j_message_write(JMessage*, struct JConnection*); - -/** Set the semantics of a message. - * - * \code - * \endcode - * - * \param message A message. - * \param semantics A semantics object. - **/ -void j_message_set_semantics(JMessage* message, JSemantics* semantics); +gboolean j_message_write(JMessage* this, struct JConnection* connection); -/** - * get the semantics of a message. - * - * \code - * \endcode - * - * \param message A message. - * - * \return A semantics object. - **/ -JSemantics* j_message_get_semantics(JMessage* message); +/// Set the semantics of a message. +/** \public \memberof JMessage */ +void j_message_set_semantics(JMessage* this, JSemantics* semantics); -/** - * @} - **/ +/// get the semantics of a message. +/** \public \memberof JMessage */ +JSemantics* j_message_get_semantics(JMessage* this); G_END_DECLS diff --git a/include/core/jnetwork.h b/include/core/jnetwork.h index 636265fe5..33dc1039c 100644 --- a/include/core/jnetwork.h +++ b/include/core/jnetwork.h @@ -43,39 +43,39 @@ struct fi_eq_cm_entry; * * Connection request are mendetory to establish a connection from server side. * \sa j_connection_init_server, j_fabric_sread_event - * \ingroup network */ typedef struct fi_eq_cm_entry JFabricConnectionRequest; -/** \struct JFabricAddr jnetwork.h - * \public \memberof +/** \struct JFabricAddr lib/jnetwork.h + * \public \memberof JFabric * \brief Fabrics address data needed to send an connection request. - * \ingroup network * \sa j_fabric_init_client */ struct JFabricAddr; +typedef struct JFabricAddr JFabricAddr; /// Possible events for paired connections /** \ingroup network * \sa j_connection_sread_event, j_connection_read_event */ -enum JConnectionEvents { +typedef enum { J_CONNECTION_EVENT_ERROR = 0, ///< An error was reported, the connection is probably in an invalid state! J_CONNECTION_EVENT_TIMEOUT, ///< there was no event to read in the given time frame J_CONNECTION_EVENT_CONNECTED, ///< First event after successful established connection. J_CONNECTION_EVENT_SHUTDOWN ///< connection was closed -}; +} JConnectionEvents; -/// Possible events for listening fabircs -/** \ingroup network + +/// Possible events for listening fabrics +/** \public \memberof JFabric * \sa j_fabric_sread_event */ -enum JFabricEvents { +typedef enum { J_FABRIC_EVENT_ERROR = 0, ///< An error was reported, fabric is probably in a invalid state! J_FABRIC_EVENT_TIMEOUT, ///< No event received in the given time frame. J_FABRIC_EVENT_CONNECTION_REQUEST, ///< A connection request was received. J_FABRIC_EVENT_SHUTDOWN ///< fabric socket was closed -}; +} JFabricEvents; /** \class JFabric lib/jnetwork.h * \brief Manage access to network. @@ -85,6 +85,7 @@ enum JFabricEvents { * \ingroup network */ struct JFabric; +typedef struct JFabric JFabric; /// Initelize a fabric for the server side. /** \public \memberof JFabric @@ -120,7 +121,7 @@ j_fabric_init_client( * \pre Finish all connections created from this fabric! */ gboolean -j_fabric_fini(struct JFabric* instance); +j_fabric_fini(struct JFabric* this); /// Read a event of a listening fabric. /** \public \memberof JFabric @@ -129,7 +130,7 @@ j_fabric_fini(struct JFabric* instance); * \retval FALSE if fetching an event fails */ gboolean -j_fabric_sread_event(struct JFabric* instance, +j_fabric_sread_event(struct JFabric* this, int timeout, ///< [in] set to -1 for no timeout. enum JFabricEvents* event, ///< [out] reeded from event queue JFabricConnectionRequest* con_req ///< [out] contains connection request, @@ -243,7 +244,7 @@ struct JConnectionMemoryID { * \sa j_connection_rma_read, j_connection_rma_register, j_connection_rma_unregister */ gboolean -j_connection_memory_get_id(struct JConnectionMemory* instance, +j_connection_memory_get_id(struct JConnectionMemory* this, struct JConnectionMemoryID* id ///< [out] of registerd memory ); @@ -305,7 +306,7 @@ j_connection_init_server ( * \retval FALSE if closing the connection failed. The connection will still be unusable! */ gboolean -j_connection_fini ( struct JConnection* instance); +j_connection_fini ( struct JConnection* this); /// Check if the connection is still valid. /** \public \memberof JConnection @@ -317,7 +318,7 @@ j_connection_fini ( struct JConnection* instance); * \todo advance connection checking! */ gboolean -j_connection_check_connection(struct JConnection* instance); +j_connection_check_connection(struct JConnection* this); /// blocking read one event from connection /** \public \memberof JConnection @@ -327,7 +328,7 @@ j_connection_check_connection(struct JConnection* instance); * \retval FALSE if fetching event fails */ gboolean -j_connection_sread_event(struct JConnection* instance, +j_connection_sread_event(struct JConnection* this, int timeout, ///< [in] set to -1 for no timeout enum JConnectionEvents* event ///< [out] reeded from queue ); @@ -339,7 +340,7 @@ j_connection_sread_event(struct JConnection* instance, * \retval FALSE if fetching failed */ gboolean -j_connection_read_event(struct JConnection* instance, +j_connection_read_event(struct JConnection* this, enum JConnectionEvents* event ///< [out] reeded from queue ); @@ -360,7 +361,7 @@ j_connection_read_event(struct JConnection* instance, * \sa j_connection_recv, j_connection_wait_for_completion */ gboolean -j_connection_send(struct JConnection* instance, +j_connection_send(struct JConnection* this, const void* data, ///< [in] to send size_t data_len ///< [in] in bytes ); @@ -378,7 +379,7 @@ j_connection_send(struct JConnection* instance, * \sa j_connection_send, j_connection_wait_for_completion */ gboolean -j_connection_recv(struct JConnection* instance, +j_connection_recv(struct JConnection* this, size_t data_len, ///< [in] in bytes to receive void* data ///< [out] received ); @@ -392,7 +393,7 @@ j_connection_recv(struct JConnection* instance, * \todo evaluate if paralisation possible */ gboolean -j_connection_rma_read(struct JConnection* instance, +j_connection_rma_read(struct JConnection* this, const struct JConnectionMemoryID* memoryID, ///< [in] for segment which should be copied void* data ///< [out] received ); @@ -403,7 +404,7 @@ j_connection_rma_read(struct JConnection* instance, * \sa j_connection_rma_read, j_connection_send, j_connection_recv */ gboolean -j_connection_wait_for_completion(struct JConnection* instance); +j_connection_wait_for_completion(struct JConnection* this); /// Check if the connection was closed from the other party. /** \sa j_connection_wait_for_completion */ @@ -417,7 +418,7 @@ j_connection_closed(struct JConnection* this); * register the memory first! */ gboolean -j_connection_rma_register(struct JConnection* intsance, +j_connection_rma_register(struct JConnection* this, const void* data, ///< [in] begin of memory region to share size_t data_len, ///< [in] size of memory region in bytes struct JConnectionMemory* handle ///< [out] for memory region to unregister with j_connection_rma_unregister @@ -428,7 +429,7 @@ j_connection_rma_register(struct JConnection* intsance, * Counterpart to j_connection_rma_register(). */ gboolean -j_connection_rma_unregister(struct JConnection* instance, +j_connection_rma_unregister(struct JConnection* this, struct JConnectionMemory* handle ///< [in] for memory region to unregister ); diff --git a/lib/core/jmessage.c b/lib/core/jmessage.c index adf876028..a296478e6 100644 --- a/lib/core/jmessage.c +++ b/lib/core/jmessage.c @@ -40,11 +40,6 @@ #define EXE(cmd, ...) do { if(cmd == FALSE) { g_warning(__VA_ARGS__); goto end; } } while(FALSE) -/** - * \addtogroup JMessage Message - * - * @{ - **/ enum JMessageSemantics { @@ -998,7 +993,3 @@ j_message_get_semantics(JMessage* message) return semantics; } - -/** - * @} - **/ From 639b53cd0b6ac33d75c44eae83c9995cef94b3fe Mon Sep 17 00:00:00 2001 From: Julian Benda Date: Fri, 22 Oct 2021 18:19:02 +0200 Subject: [PATCH 43/60] Rework todos in network code --- include/core/jnetwork.h | 6 ++--- lib/core/jnetwork.c | 56 +++++++++++++++++++++++------------------ 2 files changed, 34 insertions(+), 28 deletions(-) diff --git a/include/core/jnetwork.h b/include/core/jnetwork.h index 33dc1039c..a997ab8ee 100644 --- a/include/core/jnetwork.h +++ b/include/core/jnetwork.h @@ -132,7 +132,7 @@ j_fabric_fini(struct JFabric* this); gboolean j_fabric_sread_event(struct JFabric* this, int timeout, ///< [in] set to -1 for no timeout. - enum JFabricEvents* event, ///< [out] reeded from event queue + JFabricEvents* event, ///< [out] reeded from event queue JFabricConnectionRequest* con_req ///< [out] contains connection request, /// if event == J_FABRIC_EVENT_CONNECTION_REQUEST ); @@ -330,7 +330,7 @@ j_connection_check_connection(struct JConnection* this); gboolean j_connection_sread_event(struct JConnection* this, int timeout, ///< [in] set to -1 for no timeout - enum JConnectionEvents* event ///< [out] reeded from queue + JConnectionEvents* event ///< [out] reeded from queue ); /// check for event on connection @@ -341,7 +341,7 @@ j_connection_sread_event(struct JConnection* this, */ gboolean j_connection_read_event(struct JConnection* this, - enum JConnectionEvents* event ///< [out] reeded from queue + JConnectionEvents* event ///< [out] reeded from queue ); /// Async send data via MSG connection diff --git a/lib/core/jnetwork.c b/lib/core/jnetwork.c index 938bf295b..d1c0f8fce 100644 --- a/lib/core/jnetwork.c +++ b/lib/core/jnetwork.c @@ -239,9 +239,8 @@ j_fabric_fini(struct JFabric* this) return FALSE; } -///! \todo read error! gboolean -j_fabric_sread_event(struct JFabric* this, int timeout, enum JFabricEvents* event, JFabricConnectionRequest* con_req) +j_fabric_sread_event(struct JFabric* this, int timeout, JFabricEvents* event, JFabricConnectionRequest* con_req) { J_TRACE_FUNCTION(NULL); @@ -256,9 +255,13 @@ j_fabric_sread_event(struct JFabric* this, int timeout, enum JFabricEvents* even *event = J_FABRIC_EVENT_TIMEOUT; ret = TRUE; goto end; } else if (res == -FI_EAVAIL) { + struct fi_eq_err_entry error = {0}; *event = J_FABRIC_EVENT_ERROR; - g_warning("error"); - // TODO: fetch error! + res = fi_eq_readerr(this->pep_eq, &error, 0); + CHECK("Failed to read error!"); + g_warning("event queue contains following error (%s|c:%p):\n\t%s", + fi_strerror(FI_EAVAIL), error.context, + fi_eq_strerror(this->pep_eq, error.prov_errno, error.err_data, NULL, 0)); ret = TRUE; goto end; } CHECK("failed to read pep event queue!"); @@ -286,7 +289,7 @@ j_connection_init_client(struct JConfiguration* configuration, enum JBackendType GError* error = NULL; const gchar* server; struct JFabricAddr jf_addr; - enum JConnectionEvents event; + JConnectionEvents event; *instance_ptr = malloc(sizeof(*this)); this = *instance_ptr; @@ -343,7 +346,6 @@ j_connection_init_client(struct JConfiguration* configuration, enum JBackendType ret = TRUE; end: - /// \todo memory leak on error in addr.addr free(jf_addr.addr); return ret; } @@ -359,8 +361,8 @@ j_connection_init_server(struct JFabric* fabric, GSocketConnection* gconnection, GOutputStream* g_out; GError* error = NULL; struct JFabricAddr* addr = &fabric->fabric_addr_network; - enum JFabricEvents event; - enum JConnectionEvents con_event; + JFabricEvents event; + JConnectionEvents con_event; JFabricConnectionRequest request; *instance_ptr = malloc(sizeof(*this)); @@ -502,7 +504,7 @@ j_connection_create_memory_resources(struct JConnection* this) } gboolean -j_connection_sread_event(struct JConnection* this, int timeout, enum JConnectionEvents* event) +j_connection_sread_event(struct JConnection* this, int timeout, JConnectionEvents* event) { J_TRACE_FUNCTION(NULL); @@ -526,7 +528,7 @@ j_connection_sread_event(struct JConnection* this, int timeout, enum JConnection fi_strerror(FI_EAVAIL), error.context, fi_eq_strerror(this->eq, error.prov_errno, error.err_data, NULL, 0)); } while(res > 0); - goto end; /// \todo consider return TRUE on event queue error + goto end; } CHECK("Failed to read event of connection!"); @@ -540,9 +542,8 @@ j_connection_sread_event(struct JConnection* this, int timeout, enum JConnection return ret; } -/// \todo check usage and maybe merge with j_connection_read_event() gboolean -j_connection_read_event(struct JConnection* this, enum JConnectionEvents* event) +j_connection_read_event(struct JConnection* this, JConnectionEvents* event) { J_TRACE_FUNCTION(NULL); @@ -555,8 +556,13 @@ j_connection_read_event(struct JConnection* this, enum JConnectionEvents* event) *event = J_CONNECTION_EVENT_TIMEOUT; ret = TRUE; goto end; } else if (res == -FI_EAVAIL) { + struct fi_eq_err_entry error = {0}; *event = J_CONNECTION_EVENT_ERROR; - // TODO: fetch error! + res = fi_eq_readerr(this->eq, &error, 0); + CHECK("Failed to read error!"); + g_warning("event queue contains following error (%s|c:%p):\n\t%s", + fi_strerror(FI_EAVAIL), error.context, + fi_eq_strerror(this->eq, error.prov_errno, error.err_data, NULL, 0)); ret = TRUE; goto end; } CHECK("Failed to read event of connection!"); @@ -577,7 +583,7 @@ j_connection_send(struct JConnection* this, const void* data, size_t data_len) int res; gboolean ret = FALSE; - uint8_t* segment; + void* context; size_t size; // we used paired endponits -> inject and send don't need destination addr (last parameter) @@ -590,25 +596,26 @@ j_connection_send(struct JConnection* this, const void* data, size_t data_len) // normal send if (this->memory.active) { - segment = (uint8_t*)this->memory.buffer + this->memory.used; + uint8_t* segment = (uint8_t*)this->memory.buffer + this->memory.used; + context = segment; memcpy(segment + this->memory.tx_prefix_size, data, data_len); size = data_len + this->memory.tx_prefix_size; do{ - res = fi_send(this->ep, segment, size, fi_mr_desc(this->memory.mr), 0, segment); + res = fi_send(this->ep, segment, size, fi_mr_desc(this->memory.mr), 0, context); } while(res == -FI_EAGAIN); CHECK("Failed to initelize sending!"); this->memory.used += size; g_assert_true(this->memory.used <= this->memory.buffer_size); g_assert_true(this->running_actions.msg_len + 1< J_CONNECTION_MAX_SEND + J_CONNECTION_MAX_RECV); } else { - segment = (uint8_t*)data; + context = (void*)((const char*)data - (const char*)NULL); size = data_len; do{ - res = fi_send(this->ep, segment, size, NULL, 0, segment); + res = fi_send(this->ep, data, size, NULL, 0, context); } while(res == -FI_EAGAIN); } - this->running_actions.msg_entry[this->running_actions.msg_len].context = segment; + this->running_actions.msg_entry[this->running_actions.msg_len].context = context; this->running_actions.msg_entry[this->running_actions.msg_len].dest = NULL; this->running_actions.msg_entry[this->running_actions.msg_len].len = 0; ++this->running_actions.msg_len; @@ -675,15 +682,14 @@ j_connection_wait_for_completion(struct JConnection* this) bool rx; do { rx = TRUE; - res = fi_cq_read(this->cq.rx, &entry, 1); /// \todo handle shutdown msgs! + res = fi_cq_read(this->cq.rx, &entry, 1); if(res == -FI_EAGAIN) { rx = FALSE; - res = fi_cq_read(this->cq.tx, &entry, 1); /// \todo handle shutdown msgs! + res = fi_cq_read(this->cq.tx, &entry, 1); } } while (res == -FI_EAGAIN); - /// \todo error message fetch! if(res == -FI_EAVAIL) { - enum JConnectionEvents event; + JConnectionEvents event; j_connection_sread_event(this, 0, &event); if(event == J_CONNECTION_EVENT_SHUTDOWN) { this->closed = TRUE; goto end; } struct fi_cq_err_entry err_entry; @@ -750,7 +756,7 @@ j_connection_rma_unregister(struct JConnection* this, struct JConnectionMemory* J_TRACE_FUNCTION(NULL); int res; - this->next_key = KEY_MIN; /// \todo may just count key to overflow? (max value is stored in domain_attr) + this->next_key = KEY_MIN; res = fi_close(&handle->memory_region->fid); CHECK("Failed to unregistrer rma memory!"); return TRUE; @@ -790,8 +796,8 @@ j_connection_rma_read(struct JConnection* this, const struct JConnectionMemoryID memoryID->offset, memoryID->key, mr); - /// \todo fix the timing for minimal timeout. This is needed when device is too bussy if(res == -FI_EAGAIN) { + /// \todo evaluate: only wait for partcial finished jobs j_connection_wait_for_completion(this); } } while ( res == -FI_EAGAIN); From 9e570fb436a3bf6bb80b5a4cb09a4c6ce3896976 Mon Sep 17 00:00:00 2001 From: Julian Benda Date: Fri, 22 Oct 2021 18:36:54 +0200 Subject: [PATCH 44/60] Run autoformat --- example/libfabric_basic.c | 720 +++++++++++++++++++++---------- include/core/jmessage.h | 4 +- include/core/jnetwork.h | 135 +++--- lib/core/jconfiguration.c | 7 +- lib/core/jconnection-pool.c | 14 +- lib/core/jhelper.c | 1 - lib/core/jmessage.c | 137 +++--- lib/core/jnetwork.c | 442 ++++++++++++------- lib/object/jdistributed-object.c | 7 +- lib/object/jobject.c | 8 +- server/loop.c | 6 +- server/server.c | 8 +- tools/config.c | 6 +- 13 files changed, 961 insertions(+), 534 deletions(-) diff --git a/example/libfabric_basic.c b/example/libfabric_basic.c index c87838bd0..2b3596c51 100644 --- a/example/libfabric_basic.c +++ b/example/libfabric_basic.c @@ -22,8 +22,23 @@ // TODO: only no wait calls for cq? -#define EXE(a) do{if(!a) { goto end; }} while(FALSE) -#define CHECK(msg) do{if(res < 0) { g_error("%s: "msg"\nDetails:\t%s", location, fi_strerror(-res)); goto end; }} while(FALSE) +#define EXE(a) \ + do \ + { \ + if (!a) \ + { \ + goto end; \ + } \ + } while (FALSE) +#define CHECK(msg) \ + do \ + { \ + if (res < 0) \ + { \ + g_error("%s: " msg "\nDetails:\t%s", location, fi_strerror(-res)); \ + goto end; \ + } \ + } while (FALSE) // TODO add to config! #define PORT 47592 #define MSG_PARTS 2 @@ -33,15 +48,39 @@ const char* const usage = "call with: (server|client )"; -enum ConnectionType {CLIENT, SERVER}; -enum ConnectionDirection {TX, RX}; -enum Event {ERROR = 0, CONNECTION_REQUEST, CONNECTED, SHUTDOWN}; +enum ConnectionType +{ + CLIENT, + SERVER +}; +enum ConnectionDirection +{ + TX, + RX +}; +enum Event +{ + ERROR = 0, + CONNECTION_REQUEST, + CONNECTED, + SHUTDOWN +}; struct addrinfo; -struct message { size_t len; struct { uint64_t buff_addr; size_t buff_size; uint64_t key; } *data; }; +struct message +{ + size_t len; + struct + { + uint64_t buff_addr; + size_t buff_size; + uint64_t key; + } * data; +}; -struct jfabric { +struct jfabric +{ struct fi_info* info; struct fid_fabric* fabric; struct fid_eq* pep_eq; @@ -50,7 +89,8 @@ struct jfabric { enum ConnectionType con_type; struct Config* config; }; -struct EndpointMemory { +struct EndpointMemory +{ struct fid_mr* mr; void* buffer; size_t buffer_size; @@ -58,7 +98,8 @@ struct EndpointMemory { size_t tx_prefix_size; }; -struct jendpoint { +struct jendpoint +{ struct jfabric* fabric; struct fi_info* info; struct fid_domain* domain; @@ -71,12 +112,18 @@ struct jendpoint { gboolean shutdown; }; -struct MyThreadData { +struct MyThreadData +{ struct jfabric* fabric; struct fi_eq_cm_entry* con_req; }; -struct Config { struct fi_info* hints; int version; size_t max_op_size; }; +struct Config +{ + struct fi_info* hints; + int version; + size_t max_op_size; +}; struct Config* config_init(void); void config_fini(struct Config*); @@ -85,11 +132,28 @@ struct fi_info* config_get_hints(struct Config* conf); void config_set_max_op_size(struct Config* conf, size_t size); size_t config_get_max_op_size(struct Config* conf); -int config_get_version(struct Config* conf) { return conf->version; } -struct fi_info* config_get_hints(struct Config* conf) { return conf->hints; } -void config_set_max_op_size(struct Config* conf, size_t size) { conf->max_op_size = size; } -size_t config_get_max_op_size(struct Config* conf) { return conf->max_op_size; } -struct Config* config_init(void) +int +config_get_version(struct Config* conf) +{ + return conf->version; +} +struct fi_info* +config_get_hints(struct Config* conf) +{ + return conf->hints; +} +void +config_set_max_op_size(struct Config* conf, size_t size) +{ + conf->max_op_size = size; +} +size_t +config_get_max_op_size(struct Config* conf) +{ + return conf->max_op_size; +} +struct Config* +config_init(void) { struct Config* res = malloc(sizeof(struct Config)); memset(res, 0, sizeof(struct Config)); @@ -98,24 +162,26 @@ struct Config* config_init(void) res->hints->caps = FI_MSG | FI_SEND | FI_RECV | FI_READ | FI_RMA | FI_REMOTE_READ; res->hints->mode = FI_MSG_PREFIX; res->hints->domain_attr->mr_mode = FI_MR_LOCAL | FI_MR_ALLOCATED | FI_MR_PROV_KEY | FI_MR_VIRT_ADDR; - res->hints->ep_attr->type = FI_EP_MSG; + res->hints->ep_attr->type = FI_EP_MSG; res->hints->fabric_attr->prov_name = g_strdup("verbs"); res->max_op_size = 64; return res; } -void config_fini(struct Config* config) +void +config_fini(struct Config* config) { fi_freeinfo(config->hints); free(config); } - gboolean fabric_init_server(struct Config* config, struct jfabric** fabric); gboolean fabric_init_client(struct Config* config, struct fi_info* hints, struct jfabric** fabirc); gboolean fabric_fini(struct jfabric* fabric); gboolean read_pep_eq(struct jfabric* fabric, enum Event* evt, struct fi_eq_cm_entry* entry); -gboolean fabric_init_client(struct Config* config, struct fi_info* hints, struct jfabric** fabric_ptr) { +gboolean +fabric_init_client(struct Config* config, struct fi_info* hints, struct jfabric** fabric_ptr) +{ int res; struct jfabric* fabric; const char* location; @@ -131,10 +197,10 @@ gboolean fabric_init_client(struct Config* config, struct fi_info* hints, struct location = fabric->location; res = fi_getinfo( - config_get_version(config), - NULL, NULL, 0, - hints, - &fabric->info); + config_get_version(config), + NULL, NULL, 0, + hints, + &fabric->info); CHECK("failed to find fabric!"); fi_freeinfo(fabric->info->next); @@ -146,7 +212,8 @@ gboolean fabric_init_client(struct Config* config, struct fi_info* hints, struct *fabric_ptr = NULL; return FALSE; } -gboolean fabric_init_server(struct Config* config, struct jfabric** fabric_ptr) +gboolean +fabric_init_server(struct Config* config, struct jfabric** fabric_ptr) { int res; struct jfabric* fabric; @@ -163,13 +230,12 @@ gboolean fabric_init_server(struct Config* config, struct jfabric** fabric_ptr) location = fabric->location; res = fi_getinfo( - config_get_version(config), - NULL, NULL, FI_SOURCE, // node and service specification (we don't care?); TODO: maybe can replace socket? - config_get_hints(config), - &fabric->info - ); + config_get_version(config), + NULL, NULL, FI_SOURCE, // node and service specification (we don't care?); TODO: maybe can replace socket? + config_get_hints(config), + &fabric->info); CHECK("Failed to get fabric info!"); - + // throw other matches away, we only need one fi_freeinfo(fabric->info->next); fabric->info->next = NULL; @@ -178,9 +244,9 @@ gboolean fabric_init_server(struct Config* config, struct jfabric** fabric_ptr) CHECK("failed to open fabric!"); res = fi_eq_open( - fabric->fabric, - &(struct fi_eq_attr){.wait_obj = FI_WAIT_UNSPEC}, - &fabric->pep_eq, NULL); + fabric->fabric, + &(struct fi_eq_attr){ .wait_obj = FI_WAIT_UNSPEC }, + &fabric->pep_eq, NULL); CHECK("failed to create eq for fabric!"); res = fi_passive_ep(fabric->fabric, fabric->info, &fabric->pep, NULL); // no context needed CHECK("failed to create pep"); @@ -195,16 +261,18 @@ gboolean fabric_init_server(struct Config* config, struct jfabric** fabric_ptr) *fabric_ptr = NULL; return FALSE; } -gboolean fabric_fini(struct jfabric* fabric) +gboolean +fabric_fini(struct jfabric* fabric) { int res; const char* location = fabric->location; g_message("close: %s", location); - + fi_freeinfo(fabric->info); fabric->info = NULL; - if(fabric->con_type == SERVER) { + if (fabric->con_type == SERVER) + { res = fi_close(&fabric->pep->fid); fabric->pep = NULL; CHECK("failed to close pep!"); @@ -217,27 +285,39 @@ gboolean fabric_fini(struct jfabric* fabric) fabric->fabric = NULL; free(fabric); return TRUE; -end: +end: return FALSE; } -gboolean read_pep_eq(struct jfabric* fabric, enum Event* event, struct fi_eq_cm_entry* entry) +gboolean +read_pep_eq(struct jfabric* fabric, enum Event* event, struct fi_eq_cm_entry* entry) { int res; const char* location; uint32_t fi_event; location = fabric->location; - - do{ + + do + { res = fi_eq_sread(fabric->pep_eq, &fi_event, entry, sizeof(struct fi_eq_cm_entry), -1, 0); // no timeout, no special flags - } while(res == -FI_EAGAIN); + } while (res == -FI_EAGAIN); CHECK("failed to read pep event queue!"); - - switch(fi_event) { - case FI_CONNREQ: *event = CONNECTION_REQUEST; break; - case FI_CONNECTED: g_error("should not connect!"); goto end; break; - case FI_SHUTDOWN: *event = SHUTDOWN; break; - default: g_assert_not_reached(); goto end; + + switch (fi_event) + { + case FI_CONNREQ: + *event = CONNECTION_REQUEST; + break; + case FI_CONNECTED: + g_error("should not connect!"); + goto end; + break; + case FI_SHUTDOWN: + *event = SHUTDOWN; + break; + default: + g_assert_not_reached(); + goto end; } return TRUE; @@ -253,22 +333,36 @@ gboolean socket_write_addr(int fd, struct jfabric* fabric); gboolean socket_request_addr(char* addr, uint32_t port, struct fi_info* hints); gboolean parse_address(char* addr, uint32_t port, struct addrinfo** infos); -gboolean open_socket(int* res, uint32_t port) { +gboolean +open_socket(int* res, uint32_t port) +{ int fd; int error; - struct sockaddr_in ctrl_addr = {0}; + struct sockaddr_in ctrl_addr = { 0 }; int one = 1; fd = socket(AF_INET, SOCK_STREAM, 0); // orderd ipv4 socket - if(fd == -1) { g_error("failed to open listening socket! error: %s", strerror(errno)); goto end; } + if (fd == -1) + { + g_error("failed to open listening socket! error: %s", strerror(errno)); + goto end; + } error = setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, (const char*)&one, sizeof(one)); - if(error == -1) { g_error("failed to set socket option! error: %s", strerror(errno)); goto end; } + if (error == -1) + { + g_error("failed to set socket option! error: %s", strerror(errno)); + goto end; + } ctrl_addr.sin_family = AF_INET; ctrl_addr.sin_port = htons(port); ctrl_addr.sin_addr.s_addr = htonl(INADDR_ANY); error = bind(fd, (struct sockaddr*)&ctrl_addr, sizeof(ctrl_addr)); - if(error == -1) { g_error("filed to set address for socket (binding)! error: %s", strerror(errno)); goto end; } + if (error == -1) + { + g_error("filed to set address for socket (binding)! error: %s", strerror(errno)); + goto end; + } *res = fd; return TRUE; @@ -277,7 +371,8 @@ gboolean open_socket(int* res, uint32_t port) { return FALSE; } -gboolean socket_wait_for_connection(int fd, int* active_socket) +gboolean +socket_wait_for_connection(int fd, int* active_socket) { int error; int a_fd; @@ -285,10 +380,18 @@ gboolean socket_wait_for_connection(int fd, int* active_socket) g_message("wait for connection!"); error = listen(fd, 0); // lagecy argument: length is now defined in system settings, this argument is ignored - if(error == -1) { g_error("failed to listen for connections! error: %s", strerror(errno)); goto end; } + if (error == -1) + { + g_error("failed to listen for connections! error: %s", strerror(errno)); + goto end; + } g_message("listen finished"); a_fd = accept(fd, NULL, 0); // we accept every thing what is comming! - if(a_fd == -1) { g_error("failed to accept connection request! error: %s", strerror(errno)); goto end; } + if (a_fd == -1) + { + g_error("failed to accept connection request! error: %s", strerror(errno)); + goto end; + } *active_socket = a_fd; return TRUE; @@ -296,7 +399,8 @@ gboolean socket_wait_for_connection(int fd, int* active_socket) *active_socket = 0; return FALSE; } -gboolean socket_write_addr(int fd, struct jfabric* fabric) +gboolean +socket_write_addr(int fd, struct jfabric* fabric) { int res; size_t addrlen = 0; @@ -307,34 +411,56 @@ gboolean socket_write_addr(int fd, struct jfabric* fabric) location = fabric->location; res = fi_getname(&fabric->pep->fid, NULL, &addrlen); - if((res != -FI_ETOOSMALL) || addrlen <= 0) { CHECK("failed to fetch address size!"); } + if ((res != -FI_ETOOSMALL) || addrlen <= 0) + { + CHECK("failed to fetch address size!"); + } addr = malloc(addrlen); res = fi_getname(&fabric->pep->fid, addr, &addrlen); CHECK("failed to get addres!"); // DEBUG: - { - char str[16*3 + 1]; + { + char str[16 * 3 + 1]; int i; - for(i = 0; i < 16; ++i) { snprintf(str+i*3, 4, "%02x ", ((uint8_t*)addr)[i]); } + for (i = 0; i < 16; ++i) + { + snprintf(str + i * 3, 4, "%02x ", ((uint8_t*)addr)[i]); + } g_message("SendAddr: len: %lu, data: %s", addrlen, str); } len = htonl(addrlen); res = send(fd, (char*)&len, sizeof(len), 0); - if(res == -1 || (size_t)res != sizeof(len)) { g_error("failed to send addrlen!(%i)", res); goto end; } + if (res == -1 || (size_t)res != sizeof(len)) + { + g_error("failed to send addrlen!(%i)", res); + goto end; + } res = send(fd, &fabric->info->addr_format, sizeof(fabric->info->addr_format), 0); - if(res == -1 || (size_t)res != sizeof(fabric->info->addr_format)) { g_error("failed to send addr format!(%i)", res); goto end; } + if (res == -1 || (size_t)res != sizeof(fabric->info->addr_format)) + { + g_error("failed to send addr format!(%i)", res); + goto end; + } res = send(fd, addr, addrlen, 0); - if(res == -1 || (size_t)res != addrlen) { g_error("failed to send addr!(%i)", res); goto end; } + if (res == -1 || (size_t)res != addrlen) + { + g_error("failed to send addr!(%i)", res); + goto end; + } free(addr); return TRUE; end: - if(addr) { free(addr); } + if (addr) + { + free(addr); + } return FALSE; } -gboolean parse_address(char* addr, uint32_t port, struct addrinfo** infos) +gboolean +parse_address(char* addr, uint32_t port, struct addrinfo** infos) { const char* err_msg; char port_s[6]; @@ -346,13 +472,16 @@ gboolean parse_address(char* addr, uint32_t port, struct addrinfo** infos) .ai_protocol = IPPROTO_TCP, .ai_flags = AI_NUMERICSERV }; - snprintf(port_s, 6, "%"PRIu16, port); + snprintf(port_s, 6, "%" PRIu16, port); res = getaddrinfo(addr, port_s, &hints, infos); - if(res != 0) { + if (res != 0) + { err_msg = gai_strerror(res); g_error("failed to get socket addr: error: %s", err_msg); - goto end; - } else if (*infos == NULL) { + goto end; + } + else if (*infos == NULL) + { g_error("unable to find matching socket!"); goto end; } @@ -360,7 +489,8 @@ gboolean parse_address(char* addr, uint32_t port, struct addrinfo** infos) end: return FALSE; } -gboolean socket_request_addr(char* s_addr, uint32_t s_port, struct fi_info* hints) +gboolean +socket_request_addr(char* s_addr, uint32_t s_port, struct fi_info* hints) { int res; struct addrinfo* itr; @@ -372,37 +502,62 @@ gboolean socket_request_addr(char* s_addr, uint32_t s_port, struct fi_info* hint EXE(parse_address(s_addr, s_port, &addresses)); - for(itr = addresses; itr; itr = itr->ai_next) { + for (itr = addresses; itr; itr = itr->ai_next) + { fd = socket(itr->ai_family, itr->ai_socktype, itr->ai_protocol); - if(fd == -1) { g_error("failed to open connection: error: %s", strerror(errno)); continue; } + if (fd == -1) + { + g_error("failed to open connection: error: %s", strerror(errno)); + continue; + } res = connect(fd, itr->ai_addr, itr->ai_addrlen); - if(res == -1) { - g_error("filed to connect to host: error: %s", strerror(errno)); + if (res == -1) + { + g_error("filed to connect to host: error: %s", strerror(errno)); // close(fd); continue; } break; } - if(!itr) { g_error("Unable to connect to any host!"); goto end; } + if (!itr) + { + g_error("Unable to connect to any host!"); + goto end; + } freeaddrinfo(addresses); res = recv(fd, (void*)&addr_len, sizeof(addr_len), 0); - if(res == -1 || res != sizeof(addr_len)) { g_error("failed to recive address len!"); goto end; } + if (res == -1 || res != sizeof(addr_len)) + { + g_error("failed to recive address len!"); + goto end; + } addr_len = ntohl(addr_len); res = recv(fd, (void*)&addr_format, sizeof(addr_format), 0); - if(res == -1 || res != sizeof(addr_format)) { g_error("failed to recive address format!"); goto end; } + if (res == -1 || res != sizeof(addr_format)) + { + g_error("failed to recive address format!"); + goto end; + } addr = malloc(addr_len); res = recv(fd, addr, addr_len, 0); - if(res == -1 || (uint32_t)res != addr_len) { g_error("failed to recive address!"); goto end; } + if (res == -1 || (uint32_t)res != addr_len) + { + g_error("failed to recive address!"); + goto end; + } hints->dest_addr = addr; hints->dest_addrlen = addr_len; hints->addr_format = addr_format; // DEBUG: - { - char str[16*3 + 1]; + { + char str[16 * 3 + 1]; int i; - for(i = 0; i < 16; ++i) { snprintf(str+i*3, 4, "%02x ", ((uint8_t*)addr)[i]); } + for (i = 0; i < 16; ++i) + { + snprintf(str + i * 3, 4, "%02x ", ((uint8_t*)addr)[i]); + } g_message("RecvAddr: len: %i, data: %s", addr_len, str); } return TRUE; @@ -410,7 +565,6 @@ gboolean socket_request_addr(char* s_addr, uint32_t s_port, struct fi_info* hint return FALSE; } - gboolean endpoint_init_client(struct jendpoint** endpoint, struct Config* config, char* addr, uint32_t port); gboolean endpoint_init_server(struct jendpoint** endpoint, struct jfabric*, struct fi_eq_cm_entry* con_req); gboolean endpoint_fini(struct jendpoint* ednpoint); @@ -421,7 +575,8 @@ gboolean endpoint_check_connection(struct jendpoint* endpoint); gboolean endpoint_read_eq(struct jendpoint* endpoint, enum Event* evt, struct fi_eq_cm_entry* entry); gboolean endpoint_wait_for_completion(struct jendpoint* endpoint, enum ConnectionDirection con_dir, int len, gboolean* check, void** contexts); -gboolean endpoint_fini(struct jendpoint* endpoint) +gboolean +endpoint_fini(struct jendpoint* endpoint) { int res; const char* location = endpoint->fabric->location; @@ -434,16 +589,20 @@ gboolean endpoint_fini(struct jendpoint* endpoint) CHECK("failed to close rxqc!"); res = fi_close(&endpoint->eq->fid); CHECK("failed to close eq!"); - if(endpoint->memory.mr) { + if (endpoint->memory.mr) + { res = fi_close(&endpoint->memory.mr->fid); CHECK("failed to close memory of endpoint!"); free(endpoint->memory.buffer); } res = fi_close(&endpoint->domain->fid); CHECK("failed to close domain!"); - if(endpoint->fabric->con_type == CLIENT) { + if (endpoint->fabric->con_type == CLIENT) + { EXE(fabric_fini(endpoint->fabric)); - } else { + } + else + { fi_freeinfo(endpoint->info); } free(endpoint); @@ -452,9 +611,9 @@ gboolean endpoint_fini(struct jendpoint* endpoint) end: return FALSE; } -gboolean _endpoint_create_mr(struct jendpoint* endpoint) +gboolean +_endpoint_create_mr(struct jendpoint* endpoint) { - int res; gboolean tx_prefix; gboolean rx_prefix; @@ -469,13 +628,15 @@ gboolean _endpoint_create_mr(struct jendpoint* endpoint) prefix_size = endpoint->fabric->info->ep_attr->msg_prefix_size; g_message("%s: prefix_size: %lu", location, prefix_size); - if(size + (tx_prefix | rx_prefix) * prefix_size > endpoint->fabric->info->ep_attr->max_msg_size) { + if (size + (tx_prefix | rx_prefix) * prefix_size > endpoint->fabric->info->ep_attr->max_msg_size) + { // TODO: message size != operation size! size = endpoint->fabric->info->ep_attr->max_msg_size - (tx_prefix | rx_prefix) * prefix_size; config_set_max_op_size(endpoint->fabric->config, size); } - if(endpoint->fabric->info->domain_attr->mr_mode & FI_MR_LOCAL) { + if (endpoint->fabric->info->domain_attr->mr_mode & FI_MR_LOCAL) + { g_message("%s: local memory!", location); size += (rx_prefix | tx_prefix) * prefix_size * MSG_PARTS; endpoint->memory.buffer_size = size; @@ -487,11 +648,15 @@ gboolean _endpoint_create_mr(struct jendpoint* endpoint) } return TRUE; end: - if(endpoint->memory.buffer) { free(endpoint->memory.buffer); } + if (endpoint->memory.buffer) + { + free(endpoint->memory.buffer); + } return FALSE; } -gboolean endpoint_create(struct jendpoint* endpoint) +gboolean +endpoint_create(struct jendpoint* endpoint) { int res; const char* location; @@ -499,9 +664,9 @@ gboolean endpoint_create(struct jendpoint* endpoint) location = endpoint->fabric->location; res = fi_eq_open( - endpoint->fabric->fabric, - &(struct fi_eq_attr){.wait_obj = FI_WAIT_UNSPEC}, - &endpoint->eq, NULL); + endpoint->fabric->fabric, + &(struct fi_eq_attr){ .wait_obj = FI_WAIT_UNSPEC }, + &endpoint->eq, NULL); CHECK("failed to create eq!"); res = fi_domain(endpoint->fabric->fabric, endpoint->info, &endpoint->domain, NULL); CHECK("failed to create domain!"); @@ -510,19 +675,11 @@ gboolean endpoint_create(struct jendpoint* endpoint) EXE(_endpoint_create_mr(endpoint)); - res = fi_cq_open(endpoint->domain, &(struct fi_cq_attr){ - .wait_obj = FI_WAIT_UNSPEC, - .format = FI_CQ_FORMAT_CONTEXT, - .size = endpoint->info->tx_attr->size - }, - &endpoint->txcq, &endpoint->rxcq); + res = fi_cq_open(endpoint->domain, &(struct fi_cq_attr){ .wait_obj = FI_WAIT_UNSPEC, .format = FI_CQ_FORMAT_CONTEXT, .size = endpoint->info->tx_attr->size }, + &endpoint->txcq, &endpoint->rxcq); CHECK("failed to create txcq!"); - res = fi_cq_open(endpoint->domain, &(struct fi_cq_attr){ - .wait_obj = FI_WAIT_UNSPEC, - .format = FI_CQ_FORMAT_CONTEXT, - .size = endpoint->info->rx_attr->size - }, - &endpoint->rxcq, &endpoint->rxcq); + res = fi_cq_open(endpoint->domain, &(struct fi_cq_attr){ .wait_obj = FI_WAIT_UNSPEC, .format = FI_CQ_FORMAT_CONTEXT, .size = endpoint->info->rx_attr->size }, + &endpoint->rxcq, &endpoint->rxcq); CHECK("failed to create rxcq!"); res = fi_endpoint(endpoint->domain, endpoint->info, &endpoint->ep, NULL); CHECK("failed to create endpoint!"); @@ -540,7 +697,8 @@ gboolean endpoint_create(struct jendpoint* endpoint) end: return FALSE; } -gboolean endpoint_init_client(struct jendpoint** endpoint_ptr, struct Config* config, char* addr, uint32_t port) +gboolean +endpoint_init_client(struct jendpoint** endpoint_ptr, struct Config* config, char* addr, uint32_t port) { struct fi_info* hints; struct jendpoint* endpoint = NULL; @@ -550,7 +708,7 @@ gboolean endpoint_init_client(struct jendpoint** endpoint_ptr, struct Config* co struct fi_eq_cm_entry cm_entry; hints = fi_dupinfo(config_get_hints(config)); - EXE(socket_request_addr(addr, port, hints)); + EXE(socket_request_addr(addr, port, hints)); *endpoint_ptr = malloc(sizeof(struct jendpoint)); endpoint = *endpoint_ptr; @@ -566,21 +724,25 @@ gboolean endpoint_init_client(struct jendpoint** endpoint_ptr, struct Config* co CHECK("failed to send connection request!"); EXE(endpoint_read_eq(endpoint, &event, &cm_entry)); - if(event != CONNECTED || cm_entry.fid != &endpoint->ep->fid) { - g_error("failed to connect!"); goto end; + if (event != CONNECTED || cm_entry.fid != &endpoint->ep->fid) + { + g_error("failed to connect!"); + goto end; } - + fi_freeinfo(hints); return TRUE; end: fi_freeinfo(hints); - if(endpoint) { + if (endpoint) + { free(endpoint); *endpoint_ptr = NULL; } return FALSE; } -gboolean endpoint_init_server(struct jendpoint** endpoint_ptr, struct jfabric* fabric, struct fi_eq_cm_entry* con_req) +gboolean +endpoint_init_server(struct jendpoint** endpoint_ptr, struct jfabric* fabric, struct fi_eq_cm_entry* con_req) { int res; const char* location; @@ -592,17 +754,19 @@ gboolean endpoint_init_server(struct jendpoint** endpoint_ptr, struct jfabric* f *endpoint_ptr = malloc(sizeof(struct jendpoint)); endpoint = *endpoint_ptr; memset(endpoint, 0, sizeof(struct jendpoint)); - + endpoint->fabric = fabric; endpoint->info = con_req->info; EXE(endpoint_create(endpoint)); - + res = fi_accept(endpoint->ep, NULL, 0); // TODO: con data exchange CHECK("failed to accept connection"); EXE(endpoint_read_eq(endpoint, &event, &con_ack)); - if(event != CONNECTED || con_ack.fid != &endpoint->ep->fid) { - g_error("failed to ack connection!"); goto end; + if (event != CONNECTED || con_ack.fid != &endpoint->ep->fid) + { + g_error("failed to ack connection!"); + goto end; } return TRUE; @@ -610,10 +774,14 @@ gboolean endpoint_init_server(struct jendpoint** endpoint_ptr, struct jfabric* f free(endpoint); *endpoint_ptr = NULL; res = fi_reject(fabric->pep, con_req->fid, NULL, 0); - if(res < 0) { g_error("%s: failed to reject connection:\n\tDetails: %s", location, fi_strerror(-res)); } + if (res < 0) + { + g_error("%s: failed to reject connection:\n\tDetails: %s", location, fi_strerror(-res)); + } return FALSE; } -gboolean endpoint_read_eq(struct jendpoint* endpoint, enum Event* evt, struct fi_eq_cm_entry* entry) +gboolean +endpoint_read_eq(struct jendpoint* endpoint, enum Event* evt, struct fi_eq_cm_entry* entry) { int res; const char* location; @@ -621,16 +789,26 @@ gboolean endpoint_read_eq(struct jendpoint* endpoint, enum Event* evt, struct fi location = endpoint->fabric->location; - do{ + do + { res = fi_eq_sread(endpoint->eq, &fi_event, entry, sizeof(struct fi_eq_cm_entry), -1, 0); - }while(res == -FI_EAGAIN); + } while (res == -FI_EAGAIN); CHECK("failed to read endpoint eq!"); - switch(fi_event) { - case FI_CONNREQ: g_error("endpoint can't get connection request!"); goto end; - case FI_CONNECTED: *evt = CONNECTED; break; - case FI_SHUTDOWN: *evt = SHUTDOWN; break; - default: g_assert_not_reached(); goto end; + switch (fi_event) + { + case FI_CONNREQ: + g_error("endpoint can't get connection request!"); + goto end; + case FI_CONNECTED: + *evt = CONNECTED; + break; + case FI_SHUTDOWN: + *evt = SHUTDOWN; + break; + default: + g_assert_not_reached(); + goto end; } return TRUE; @@ -640,12 +818,14 @@ gboolean endpoint_read_eq(struct jendpoint* endpoint, enum Event* evt, struct fi return FALSE; } -gboolean endpoint_check_connection(struct jendpoint* endpoint) +gboolean +endpoint_check_connection(struct jendpoint* endpoint) { return endpoint->ep != NULL && !endpoint->shutdown; } -gboolean endpoint_wait_for_completion(struct jendpoint* endpoint, enum ConnectionDirection con_dir, int len, gboolean* check, void** contexts) +gboolean +endpoint_wait_for_completion(struct jendpoint* endpoint, enum ConnectionDirection con_dir, int len, gboolean* check, void** contexts) { int i, count = 0; struct fid_cq* cq; @@ -654,25 +834,43 @@ gboolean endpoint_wait_for_completion(struct jendpoint* endpoint, enum Connectio int res; const char* location = endpoint->fabric->location; - switch(con_dir) { - case TX: cq = endpoint->txcq; break; - case RX: cq = endpoint->rxcq; break; - default: g_assert_not_reached(); goto end; + switch (con_dir) + { + case TX: + cq = endpoint->txcq; + break; + case RX: + cq = endpoint->rxcq; + break; + default: + g_assert_not_reached(); + goto end; } - for(i = 0; i < len; ++i) { - if(contexts[i] == NULL) { check[i] = TRUE; } - if(check[i] == TRUE) {++count; } + for (i = 0; i < len; ++i) + { + if (contexts[i] == NULL) + { + check[i] = TRUE; + } + if (check[i] == TRUE) + { + ++count; + } } - while(count != len) { + while (count != len) + { res = fi_cq_sread(cq, &entry, 1, NULL, TIMEOUT); - if (res == -FI_EAGAIN) { + if (res == -FI_EAGAIN) + { uint32_t event; struct fi_eq_cm_entry eq_entry; res = fi_eq_read(endpoint->eq, &event, &eq_entry, sizeof(eq_entry), 0); - if(res != -FI_EAGAIN) { + if (res != -FI_EAGAIN) + { CHECK("failed to read eq in cq test!"); - if(event == FI_SHUTDOWN) { + if (event == FI_SHUTDOWN) + { g_message("shutdown"); endpoint->shutdown = TRUE; return FALSE; @@ -682,19 +880,23 @@ gboolean endpoint_wait_for_completion(struct jendpoint* endpoint, enum Connectio } continue; } - if(res < 0) { + if (res < 0) + { int err = res; res = fi_cq_readerr(cq, &err_entry, 0); CHECK("failed to fetch cq error!"); g_error("%s: completion error!\n\tDetails:%s\n\tProvider:%s", - location, - fi_strerror(-err), - fi_cq_strerror(cq, err_entry.prov_errno, err_entry.err_data, NULL, 0)); + location, + fi_strerror(-err), + fi_cq_strerror(cq, err_entry.prov_errno, err_entry.err_data, NULL, 0)); goto end; } - for(i = 0; i < len; ++i) { - if(!check[i]) { - if(contexts[i] == entry.op_context) { + for (i = 0; i < len; ++i) + { + if (!check[i]) + { + if (contexts[i] == entry.op_context) + { ++count; check[i] = TRUE; } @@ -709,89 +911,103 @@ gboolean endpoint_wait_for_completion(struct jendpoint* endpoint, enum Connectio gboolean ssend_message(struct jendpoint* endpoint, struct message* msg); gboolean srecv_message(struct jendpoint* endpoint, struct message** msg); -gboolean ssend_message(struct jendpoint* endpoint, struct message* msg) +gboolean +ssend_message(struct jendpoint* endpoint, struct message* msg) { int res; const char* location = endpoint->fabric->location; - gboolean checks[2] = {FALSE, FALSE}; - void* contexts[2] = {NULL, NULL}; + gboolean checks[2] = { FALSE, FALSE }; + void* contexts[2] = { NULL, NULL }; int size = 0; int offset = 0; int i; struct fid_mr* mrs[MAX_SEGMENTS]; int ack = ~ACK; - for(i = 0; (size_t)i < msg->len; ++i) { + for (i = 0; (size_t)i < msg->len; ++i) + { res = fi_mr_reg( - endpoint->domain, - (void*)msg->data[i].buff_addr, - msg->data[i].buff_size, FI_REMOTE_READ, - 0, 0, 0, &mrs[i], NULL); + endpoint->domain, + (void*)msg->data[i].buff_addr, + msg->data[i].buff_size, FI_REMOTE_READ, + 0, 0, 0, &mrs[i], NULL); CHECK("failed to register memory for rma send!"); msg->data[i].key = fi_mr_key(mrs[i]); } // don't need dst_addr because we are connected! size = sizeof(msg->len); - if(size < endpoint->inject_size) { + if (size < endpoint->inject_size) + { res = fi_inject(endpoint->ep, &msg->len, size, 0); CHECK("failed to inject msg header!"); - } else { + } + else + { memcpy( - (char*)endpoint->memory.buffer + offset + endpoint->memory.tx_prefix_size, - &msg->len, size); + (char*)endpoint->memory.buffer + offset + endpoint->memory.tx_prefix_size, + &msg->len, size); res = fi_send(endpoint->ep, - (char*)endpoint->memory.buffer + offset, - size + endpoint->memory.tx_prefix_size, - fi_mr_desc(endpoint->memory.mr), 0, &msg->len); + (char*)endpoint->memory.buffer + offset, + size + endpoint->memory.tx_prefix_size, + fi_mr_desc(endpoint->memory.mr), 0, &msg->len); CHECK("failed to initeiate header send!"); contexts[0] = &msg->len; offset += size + endpoint->memory.tx_prefix_size; } size = msg->len * sizeof(*msg->data); - if(size < endpoint->inject_size) { + if (size < endpoint->inject_size) + { res = fi_inject(endpoint->ep, msg->data, sizeof(*msg->data) * msg->len, 0); CHECK("failed to inject msg body!"); - } else { + } + else + { memcpy( - (char*)endpoint->memory.buffer + offset + endpoint->memory.tx_prefix_size, - msg->data, size); + (char*)endpoint->memory.buffer + offset + endpoint->memory.tx_prefix_size, + msg->data, size); res = fi_send(endpoint->ep, - (char*)endpoint->memory.buffer + offset, - size + endpoint->memory.tx_prefix_size, - fi_mr_desc(endpoint->memory.mr), 0, msg->data); + (char*)endpoint->memory.buffer + offset, + size + endpoint->memory.tx_prefix_size, + fi_mr_desc(endpoint->memory.mr), 0, msg->data); CHECK("failed to initeiate body send!"); contexts[1] = msg->data; offset += size + endpoint->memory.tx_prefix_size; } - res = fi_recv(endpoint->ep, endpoint->memory.buffer, sizeof(int) + endpoint->memory.rx_prefix_size, - fi_mr_desc(endpoint->memory.mr), 0, &ack); + fi_mr_desc(endpoint->memory.mr), 0, &ack); CHECK("failed to read ack flag!"); - checks[0] = FALSE; contexts[0] = &ack; + checks[0] = FALSE; + contexts[0] = &ack; EXE(endpoint_wait_for_completion(endpoint, RX, 1, checks, contexts)); ack = *(int*)((char*)endpoint->memory.buffer + endpoint->memory.rx_prefix_size); - if(ack != ACK) { g_error("recived wrong ack!: %i, expected: %i", ack, ACK); goto end; } + if (ack != ACK) + { + g_error("recived wrong ack!: %i, expected: %i", ack, ACK); + goto end; + } - for(i = 0; (size_t)i< msg->len; ++i) { + for (i = 0; (size_t)i < msg->len; ++i) + { res = fi_close(&mrs[i]->fid); CHECK("failed to free memory after rma send!"); } EXE(endpoint_wait_for_completion(endpoint, TX, 2, checks, contexts)); - + return TRUE; end: return FALSE; } -gboolean srecv_message(struct jendpoint* endpoint, struct message** msg_ptr) +gboolean +srecv_message(struct jendpoint* endpoint, struct message** msg_ptr) { int res; const char* location = endpoint->fabric->location; - gboolean checks[MAX_SEGMENTS] = {FALSE}; + gboolean checks[MAX_SEGMENTS] = { FALSE }; void* contexts[MAX_SEGMENTS]; int i; int ack = ACK; @@ -799,23 +1015,26 @@ gboolean srecv_message(struct jendpoint* endpoint, struct message** msg_ptr) struct message* msg = malloc(sizeof(struct message)); msg->data = NULL; - + res = fi_recv(endpoint->ep, endpoint->memory.buffer, sizeof(msg->len) + endpoint->memory.rx_prefix_size, fi_mr_desc(endpoint->memory.mr), 0, &msg->len); CHECK("failed to recive msg len!"); - checks[0] = FALSE; contexts[0] = &msg->len; + checks[0] = FALSE; + contexts[0] = &msg->len; EXE(endpoint_wait_for_completion(endpoint, RX, 1, checks, contexts)); memcpy(&msg->len, (char*)endpoint->memory.buffer + endpoint->memory.rx_prefix_size, sizeof(msg->len)); - + msg->data = malloc(msg->len * sizeof(*msg->data)); - res = fi_recv(endpoint->ep, endpoint->memory.buffer, sizeof(*msg->data)*msg->len + endpoint->memory.rx_prefix_size, fi_mr_desc(endpoint->memory.mr), 0, msg->data); + res = fi_recv(endpoint->ep, endpoint->memory.buffer, sizeof(*msg->data) * msg->len + endpoint->memory.rx_prefix_size, fi_mr_desc(endpoint->memory.mr), 0, msg->data); CHECK("failed to recive body!"); - checks[0] = FALSE; contexts[0] = msg->data; + checks[0] = FALSE; + contexts[0] = msg->data; EXE(endpoint_wait_for_completion(endpoint, RX, 1, checks, contexts)); memcpy(msg->data, (char*)endpoint->memory.buffer + endpoint->memory.rx_prefix_size, msg->len * sizeof(*msg->data)); - - for( i = 0; (size_t)i < msg->len; ++i) { + + for (i = 0; (size_t)i < msg->len; ++i) + { uint64_t addr = msg->data[i].buff_addr; - contexts[i] = (void*) addr; + contexts[i] = (void*)addr; checks[i] = FALSE; msg->data[i].buff_addr = (uint64_t)malloc(msg->data[i].buff_size); @@ -827,25 +1046,29 @@ gboolean srecv_message(struct jendpoint* endpoint, struct message** msg_ptr) CHECK("failed to register recive memory for rma!"); res = fi_read(endpoint->ep, - (void*)msg->data[i].buff_addr, - msg->data[i].buff_size, - fi_mr_desc(mrs[i]), 0, addr, msg->data[i].key, contexts[i]); + (void*)msg->data[i].buff_addr, + msg->data[i].buff_size, + fi_mr_desc(mrs[i]), 0, addr, msg->data[i].key, contexts[i]); CHECK("failed to initeate rma read!"); } EXE(endpoint_wait_for_completion(endpoint, TX, msg->len, checks, contexts)); fi_inject(endpoint->ep, &ack, sizeof(ack), 0); - for(i = 0; (size_t)i < msg->len; ++i) { + for (i = 0; (size_t)i < msg->len; ++i) + { res = fi_close(&mrs[i]->fid); CHECK("failed to unregister memory!"); } - + *msg_ptr = msg; return TRUE; end: *msg_ptr = NULL; - if(msg->data) { free(msg->data); } + if (msg->data) + { + free(msg->data); + } free(msg); g_message("cancle read!"); return FALSE; @@ -859,45 +1082,63 @@ void print_message(struct message* msg); gboolean start_new_thread(struct jfabric* fabric, struct fi_eq_cm_entry* con_req, GArray* threads); void* thread(void* thread_data); -const char* const buffers[] = {"Hello", "World!"}; -struct message* construct_message(void) { +const char* const buffers[] = { "Hello", "World!" }; +struct message* +construct_message(void) +{ struct message* msg = malloc(sizeof(struct message)); msg->len = 2; msg->data = malloc(sizeof(*msg->data) * msg->len); - msg->data[0].buff_size = 6; msg->data[0].buff_addr = (uint64_t)g_strdup(buffers[0]); - msg->data[1].buff_size = 7; msg->data[1].buff_addr = (uint64_t)g_strdup(buffers[1]); + msg->data[0].buff_size = 6; + msg->data[0].buff_addr = (uint64_t)g_strdup(buffers[0]); + msg->data[1].buff_size = 7; + msg->data[1].buff_addr = (uint64_t)g_strdup(buffers[1]); return msg; } -void free_message(struct message* msg, gboolean sender) { +void +free_message(struct message* msg, gboolean sender) +{ size_t i; - if(msg == NULL) { return; } - if(msg->data != NULL) { - for(i=0; i < msg->len; ++i) { free((void*)msg->data[i].buff_addr); } + if (msg == NULL) + { + return; + } + if (msg->data != NULL) + { + for (i = 0; i < msg->len; ++i) + { + free((void*)msg->data[i].buff_addr); + } free(msg->data); } free(msg); } -void print_message(struct message* msg) { +void +print_message(struct message* msg) +{ int i; g_print("message:\n\tlen: %lu\n\tdata:\n", msg->len); - for(i = 0; (size_t)i < msg->len; ++i) { + for (i = 0; (size_t)i < msg->len; ++i) + { g_print("\t\tseg %i[l=%lu]: >%s<\n", i, msg->data[i].buff_size, (char*)msg->data[i].buff_addr); // g_print("\t\tseg %i[l=%lu]: addr=%lu\n", i, msg->data[i].buff_size, msg->data[i].buff_addr); } } -void* thread(void* thread_data) { +void* +thread(void* thread_data) +{ struct MyThreadData* data; - struct jendpoint* endpoint; + struct jendpoint* endpoint; struct message* msg; // enum Event event; data = thread_data; EXE(endpoint_init_server(&endpoint, data->fabric, data->con_req)); - while(endpoint_check_connection(endpoint)) + while (endpoint_check_connection(endpoint)) { EXE(srecv_message(endpoint, &msg)); print_message(msg); @@ -915,7 +1156,9 @@ void* thread(void* thread_data) { return NULL; } -gboolean start_new_thread(struct jfabric* fabric, struct fi_eq_cm_entry* con_req, GArray* threads) { +gboolean +start_new_thread(struct jfabric* fabric, struct fi_eq_cm_entry* con_req, GArray* threads) +{ struct MyThreadData* thread_data = malloc(sizeof(struct MyThreadData)); GThread* new_thread; thread_data->fabric = fabric; @@ -925,8 +1168,9 @@ gboolean start_new_thread(struct jfabric* fabric, struct fi_eq_cm_entry* con_req return TRUE; } - -void server(void) { +void +server(void) +{ struct jfabric* fabric; int run = 3; guint i; @@ -943,25 +1187,35 @@ void server(void) { EXE(fabric_init_server(config, &fabric)); EXE(open_socket(&p_socket, PORT)); // run = TRUE; - do{ + do + { EXE(socket_wait_for_connection(p_socket, &socket)); g_message("write_addr"); EXE(socket_write_addr(socket, fabric)); close(socket); EXE(read_pep_eq(fabric, &event, &con_req)); - switch(event) { - case ERROR: goto end; - case CONNECTED: g_error("pep can't connect!?"); goto end; - case CONNECTION_REQUEST: - EXE(start_new_thread(fabric, &con_req, threads)); - --run; - break; - case SHUTDOWN: run = FALSE; break; - default: g_assert_not_reached(); goto end; + switch (event) + { + case ERROR: + goto end; + case CONNECTED: + g_error("pep can't connect!?"); + goto end; + case CONNECTION_REQUEST: + EXE(start_new_thread(fabric, &con_req, threads)); + --run; + break; + case SHUTDOWN: + run = FALSE; + break; + default: + g_assert_not_reached(); + goto end; } - } while(run); + } while (run); end: - for(i = 0; i < threads->len; ++i) { + for (i = 0; i < threads->len; ++i) + { g_thread_join(g_array_index(threads, GThread*, i)); } g_array_free(threads, TRUE); @@ -969,7 +1223,9 @@ void server(void) { config_fini(config); } -void client(char* addr, uint32_t port) { +void +client(char* addr, uint32_t port) +{ struct jendpoint* endpoint; struct message* msg = NULL; struct Config* config; @@ -994,21 +1250,29 @@ void client(char* addr, uint32_t port) { return; } -int main(int argc, char** argv) { - - if(argc < 2) { +int +main(int argc, char** argv) +{ + if (argc < 2) + { fprintf(stderr, usage); return EXIT_FAILURE; } - if(strcmp(argv[1], "server") == 0) { + if (strcmp(argv[1], "server") == 0) + { server(); - } else if (strcmp(argv[1], "client") == 0) { - if(argc < 3) { + } + else if (strcmp(argv[1], "client") == 0) + { + if (argc < 3) + { fprintf(stderr, "client can only connect when ip address is given!"); return EXIT_FAILURE; } client(argv[2], PORT); - } else { + } + else + { fprintf(stderr, usage); return EXIT_FAILURE; } diff --git a/include/core/jmessage.h b/include/core/jmessage.h index 3b5373e5f..2027dfd30 100644 --- a/include/core/jmessage.h +++ b/include/core/jmessage.h @@ -77,7 +77,7 @@ G_BEGIN_DECLS /// Creates a new Message /** \public \memberof JMessage - * \param op_type[in] message type + * \param op_type[in] message type * \param length[in] message body size. * \return A new message. Should be freed with j_message_unref(). **/ @@ -85,7 +85,7 @@ JMessage* j_message_new(JMessageType op_type, gsize length); /// Creates a new reply message /** \public \memberof JMessage - * \param message[in] message to reply to + * \param message[in] message to reply to * \return A new reply message. Should be freed with j_message_unref(). **/ JMessage* j_message_new_reply(JMessage* message); diff --git a/include/core/jnetwork.h b/include/core/jnetwork.h index a997ab8ee..104fde00a 100644 --- a/include/core/jnetwork.h +++ b/include/core/jnetwork.h @@ -1,4 +1,4 @@ -/** \file jnetwork.h +/** \file jnetwork.h * \ingroup network * * \copyright @@ -40,7 +40,7 @@ struct fi_eq_cm_entry; /** \struct JFabricConnectionRequest lib/jnetwork.h * \brief A connection request read from an event queue. - * + * * Connection request are mendetory to establish a connection from server side. * \sa j_connection_init_server, j_fabric_sread_event */ @@ -58,23 +58,24 @@ typedef struct JFabricAddr JFabricAddr; /** \ingroup network * \sa j_connection_sread_event, j_connection_read_event */ -typedef enum { +typedef enum +{ J_CONNECTION_EVENT_ERROR = 0, ///< An error was reported, the connection is probably in an invalid state! - J_CONNECTION_EVENT_TIMEOUT, ///< there was no event to read in the given time frame + J_CONNECTION_EVENT_TIMEOUT, ///< there was no event to read in the given time frame J_CONNECTION_EVENT_CONNECTED, ///< First event after successful established connection. - J_CONNECTION_EVENT_SHUTDOWN ///< connection was closed + J_CONNECTION_EVENT_SHUTDOWN ///< connection was closed } JConnectionEvents; - /// Possible events for listening fabrics /** \public \memberof JFabric * \sa j_fabric_sread_event */ -typedef enum { - J_FABRIC_EVENT_ERROR = 0, ///< An error was reported, fabric is probably in a invalid state! - J_FABRIC_EVENT_TIMEOUT, ///< No event received in the given time frame. - J_FABRIC_EVENT_CONNECTION_REQUEST, ///< A connection request was received. - J_FABRIC_EVENT_SHUTDOWN ///< fabric socket was closed +typedef enum +{ + J_FABRIC_EVENT_ERROR = 0, ///< An error was reported, fabric is probably in a invalid state! + J_FABRIC_EVENT_TIMEOUT, ///< No event received in the given time frame. + J_FABRIC_EVENT_CONNECTION_REQUEST, ///< A connection request was received. + J_FABRIC_EVENT_SHUTDOWN ///< fabric socket was closed } JFabricEvents; /** \class JFabric lib/jnetwork.h @@ -96,8 +97,8 @@ typedef struct JFabric JFabric; */ gboolean j_fabric_init_server( - struct JConfiguration* configuration, ///< [in] of JULEA, to fetch details - struct JFabric** instance ///< [out] pointer to resulting fabric + struct JConfiguration* configuration, ///< [in] of JULEA, to fetch details + struct JFabric** instance ///< [out] pointer to resulting fabric ); /// Initelize a fabric for the client side. @@ -105,36 +106,36 @@ j_fabric_init_server( * * Contains data to building a paired connection. * Addess of JULEA server is needed, to enforce that both communicate via the same network. - * \warning will be called from j_connection_init_client(), so no need for explicit a call + * \warning will be called from j_connection_init_client(), so no need for explicit a call * \retval FALSE on failure */ gboolean j_fabric_init_client( - struct JConfiguration* configuration, ///< [in] of JULEA to fetch details - struct JFabricAddr* addr, ///< [in] of JULEA server fabric - struct JFabric** instance ///< [out] pointer to resulting fabric + struct JConfiguration* configuration, ///< [in] of JULEA to fetch details + struct JFabricAddr* addr, ///< [in] of JULEA server fabric + struct JFabric** instance ///< [out] pointer to resulting fabric ); /// Closes a fabirc and frees used memory. /** \public \memberof JFabric * \retval FALSE on failure - * \pre Finish all connections created from this fabric! + * \pre Finish all connections created from this fabric! */ gboolean j_fabric_fini(struct JFabric* this); -/// Read a event of a listening fabric. +/// Read a event of a listening fabric. /** \public \memberof JFabric * \warning only usable for fabrics initelized with j_fabric_init_server() - * \retval TRUE if fetching succeed or the request timeouted + * \retval TRUE if fetching succeed or the request timeouted * \retval FALSE if fetching an event fails */ gboolean j_fabric_sread_event(struct JFabric* this, - int timeout, ///< [in] set to -1 for no timeout. - JFabricEvents* event, ///< [out] reeded from event queue - JFabricConnectionRequest* con_req ///< [out] contains connection request, - /// if event == J_FABRIC_EVENT_CONNECTION_REQUEST + int timeout, ///< [in] set to -1 for no timeout. + JFabricEvents* event, ///< [out] reeded from event queue + JFabricConnectionRequest* con_req ///< [out] contains connection request, + /// if event == J_FABRIC_EVENT_CONNECTION_REQUEST ); /** \class JConnection jnetwork.h @@ -175,7 +176,7 @@ j_fabric_sread_event(struct JFabric* this, * j_connection_wait_for_completion(connection); * j_connection_rma_unregister(connection, &rma_mem); * - * // rma read + * // rma read * j_connection_recv(connection, &rma_mem_id, sizeof(rma_mem_id)); * j_connection_wait_for_completion(connection); * j_connection_rma_read(connection, &rma_mem_id, data); @@ -197,7 +198,7 @@ j_fabric_sread_event(struct JFabric* this, * * j_connection_fini(jconnection); * \endcode - * \todo expose more connection posibilities (eg. connection with an JFabricAddr) + * \todo expose more connection posibilities (eg. connection with an JFabricAddr) */ struct JConnection; @@ -210,7 +211,7 @@ struct JConnection; #define J_CONNECTION_MAX_RECV 1 /// Acknowladgement value. -/** \public \memberof JConnection +/** \public \memberof JConnection * Value used to send an ack for any reason. Maybe to verify that the remote memory was successful readed. * \sa j_connection_rma_unregister, j_connection_rma_read, j_connection_rma_register, JConnectionAck */ @@ -232,10 +233,11 @@ struct JConnectionMemory; /** \public \memberof JConnectionMemory * \sa j_connection_rma_read, j_connection_rma_register, j_connection_memory_get_id */ -struct JConnectionMemoryID { - uint64_t key; ///< access key for authentication - uint64_t offset; ///< used to identifie memory region - uint64_t size; ///< size in bytes of memery region +struct JConnectionMemoryID +{ + uint64_t key; ///< access key for authentication + uint64_t offset; ///< used to identifie memory region + uint64_t size; ///< size in bytes of memery region }; /// Get identifier of memory region @@ -245,16 +247,15 @@ struct JConnectionMemoryID { */ gboolean j_connection_memory_get_id(struct JConnectionMemory* this, - struct JConnectionMemoryID* id ///< [out] of registerd memory + struct JConnectionMemoryID* id ///< [out] of registerd memory ); - /// Builds a connection to an active fabric (direct). /** \public \memberof JConnection * - * This constructor will fetch the fabric address via TCP and then building an paired + * This constructor will fetch the fabric address via TCP and then building an paired * connection to this, which later allows for messaging and RMA data transfer. \n - * This will also construct a fabric for that connection, which is freed when the connection is finished. + * This will also construct a fabric for that connection, which is freed when the connection is finished. * * \attention this function may reduces j_configuration_max_operation_size() * accordingly to network capabilities. @@ -273,20 +274,19 @@ j_connection_memory_get_id(struct JConnectionMemory* this, * \todo document index parameter! */ gboolean -j_connection_init_client ( - struct JConfiguration* configuration, ///< [in] for server address, and network configuration - enum JBackendType backend, ///< [in] backend server to connect to - guint index, ///< [in] ?? - struct JConnection** instance ///< [out] constructed instance +j_connection_init_client( + struct JConfiguration* configuration, ///< [in] for server address, and network configuration + enum JBackendType backend, ///< [in] backend server to connect to + guint index, ///< [in] ?? + struct JConnection** instance ///< [out] constructed instance ); - /// Establish connection to client based on established GSocketConnection. /** \public \memberof JConnection * * The GSocketConnection will be used to send the server fabric data. * For the connection process see j_connection_init_client(). - * + * * \attention this function may reduces j_configuration_max_operation_size() * accordingly to network capabilities. * @@ -294,10 +294,10 @@ j_connection_init_client ( * \sa j_connection_init_client */ gboolean -j_connection_init_server ( - struct JFabric* fabric, ///< [in] via which the connection should be established - GSocketConnection* gconnection, ///< [in] valid GSocketConnection for address exchange - struct JConnection** instance ///< [out] constructed instance +j_connection_init_server( + struct JFabric* fabric, ///< [in] via which the connection should be established + GSocketConnection* gconnection, ///< [in] valid GSocketConnection for address exchange + struct JConnection** instance ///< [out] constructed instance ); /// Closes a connection and free all related resources. /** \public \memberof JConnection @@ -306,11 +306,11 @@ j_connection_init_server ( * \retval FALSE if closing the connection failed. The connection will still be unusable! */ gboolean -j_connection_fini ( struct JConnection* this); +j_connection_fini(struct JConnection* this); /// Check if the connection is still valid. /** \public \memberof JConnection - * + * * Check if the connection was established and if was no J_CONNECTION_EVENT_SHUTDOWN recognized. * \attention events will only be checked if j_connection_read_event() is called!. * \retval FALSE if connection is not longer valid @@ -324,13 +324,13 @@ j_connection_check_connection(struct JConnection* this); /** \public \memberof JConnection * Used to check wait for connected state, and recognized errors and shutdown signals. * For a version which returns immediate when no event exists see j_connection_read_event() - * \retval TRUE if fetching succeed or the request timeouted + * \retval TRUE if fetching succeed or the request timeouted * \retval FALSE if fetching event fails */ gboolean j_connection_sread_event(struct JConnection* this, - int timeout, ///< [in] set to -1 for no timeout - JConnectionEvents* event ///< [out] reeded from queue + int timeout, ///< [in] set to -1 for no timeout + JConnectionEvents* event ///< [out] reeded from queue ); /// check for event on connection @@ -341,14 +341,14 @@ j_connection_sread_event(struct JConnection* this, */ gboolean j_connection_read_event(struct JConnection* this, - JConnectionEvents* event ///< [out] reeded from queue + JConnectionEvents* event ///< [out] reeded from queue ); /// Async send data via MSG connection /** \public \memberof JConnection - * + * * Asynchronous sends a message, to recognize for completion use j_connection_wait_for_completion().\n - * If the message is small enough it can "injected" to the network, in that case the actions finished + * If the message is small enough it can "injected" to the network, in that case the actions finished * immediate (j_connection_wait_for_completion() still works). * * \todo feedback if message was injected! @@ -362,29 +362,28 @@ j_connection_read_event(struct JConnection* this, */ gboolean j_connection_send(struct JConnection* this, - const void* data, ///< [in] to send - size_t data_len ///< [in] in bytes + const void* data, ///< [in] to send + size_t data_len ///< [in] in bytes ); /// Asynchronous receive data via MSG connection. /** \public \memberof JConnection - * + * * Asynchronous receive a message, to wait for completion use j_connection_wait_for_completion(). * * \attention it is only allowed to have J_CONNECTION_MAX_RECV recv * operation pending at the same time. Each has a max size * of j_configuration_max_operation_size() (the connection initialization may has changed this value!). - * + * * \retval FALSE if an error occurred * \sa j_connection_send, j_connection_wait_for_completion */ gboolean j_connection_recv(struct JConnection* this, - size_t data_len, ///< [in] in bytes to receive - void* data ///< [out] received + size_t data_len, ///< [in] in bytes to receive + void* data ///< [out] received ); - /// Async direct memory read. /** \public \memberof JConnection * @@ -394,12 +393,12 @@ j_connection_recv(struct JConnection* this, */ gboolean j_connection_rma_read(struct JConnection* this, - const struct JConnectionMemoryID* memoryID, ///< [in] for segment which should be copied - void* data ///< [out] received + const struct JConnectionMemoryID* memoryID, ///< [in] for segment which should be copied + void* data ///< [out] received ); /// Wait until operations initiated at his connection finished. -/** \public \memberof JConnection +/** \public \memberof JConnection * \retval FALSE if waiting finished. This may occures because the connection was closed: check this with: j_connection_active() * \sa j_connection_rma_read, j_connection_send, j_connection_recv */ @@ -419,9 +418,9 @@ j_connection_closed(struct JConnection* this); */ gboolean j_connection_rma_register(struct JConnection* this, - const void* data, ///< [in] begin of memory region to share - size_t data_len, ///< [in] size of memory region in bytes - struct JConnectionMemory* handle ///< [out] for memory region to unregister with j_connection_rma_unregister + const void* data, ///< [in] begin of memory region to share + size_t data_len, ///< [in] size of memory region in bytes + struct JConnectionMemory* handle ///< [out] for memory region to unregister with j_connection_rma_unregister ); /// Unregister memory from rma availablity. @@ -430,7 +429,7 @@ j_connection_rma_register(struct JConnection* this, */ gboolean j_connection_rma_unregister(struct JConnection* this, - struct JConnectionMemory* handle ///< [in] for memory region to unregister + struct JConnectionMemory* handle ///< [in] for memory region to unregister ); struct JConfiguration* diff --git a/lib/core/jconfiguration.c b/lib/core/jconfiguration.c index 1b6c3576b..ea881bed0 100644 --- a/lib/core/jconfiguration.c +++ b/lib/core/jconfiguration.c @@ -142,7 +142,8 @@ struct JConfiguration } db; /// libfabric configuration - struct { + struct + { int32_t version; struct fi_info* hints; } libfabric; @@ -371,10 +372,10 @@ j_configuration_new_for_data(GKeyFile* key_file) configuration->ref_count = 1; configuration->libfabric.version = FI_VERSION(1, 11); configuration->libfabric.hints = fi_allocinfo(); - configuration->libfabric.hints->caps = + configuration->libfabric.hints->caps = FI_MSG | FI_SEND | FI_RECV | FI_READ | FI_RMA | FI_REMOTE_READ; configuration->libfabric.hints->mode = FI_MSG_PREFIX; - configuration->libfabric.hints->domain_attr->mr_mode = + configuration->libfabric.hints->domain_attr->mr_mode = FI_MR_LOCAL | FI_MR_ALLOCATED | FI_MR_PROV_KEY | FI_MR_VIRT_ADDR; configuration->libfabric.hints->ep_attr->type = FI_EP_MSG; configuration->libfabric.hints->fabric_attr->prov_name = libfabric_provider; diff --git a/lib/core/jconnection-pool.c b/lib/core/jconnection-pool.c index 2a7c772a7..b2d6e8b39 100644 --- a/lib/core/jconnection-pool.c +++ b/lib/core/jconnection-pool.c @@ -124,7 +124,9 @@ j_connection_pool_fini(void) struct JConnection* connection; while ((connection = g_async_queue_try_pop(pool->object_queues[i].queue)) != NULL) - { j_connection_fini(connection); } + { + j_connection_fini(connection); + } g_async_queue_unref(pool->object_queues[i].queue); } @@ -134,7 +136,9 @@ j_connection_pool_fini(void) struct JConnection* connection; while ((connection = g_async_queue_try_pop(pool->kv_queues[i].queue)) != NULL) - { j_connection_fini(connection); } + { + j_connection_fini(connection); + } g_async_queue_unref(pool->kv_queues[i].queue); } @@ -144,7 +148,9 @@ j_connection_pool_fini(void) struct JConnection* connection; while ((connection = g_async_queue_try_pop(pool->db_queues[i].queue)) != NULL) - { j_connection_fini(connection); } + { + j_connection_fini(connection); + } g_async_queue_unref(pool->db_queues[i].queue); } @@ -184,7 +190,7 @@ j_connection_pool_pop_internal(GAsyncQueue* queue, guint* count, enum JBackendTy guint op_count; - if(!j_connection_init_client(j_connection_pool->configuration, backend, index, &connection)) + if (!j_connection_init_client(j_connection_pool->configuration, backend, index, &connection)) { g_critical("Can not connect to %s [%d].", "TODO(SERVER ADDR)", g_atomic_int_get(count)); exit(1); diff --git a/lib/core/jhelper.c b/lib/core/jhelper.c index c562ebe6d..964120584 100644 --- a/lib/core/jhelper.c +++ b/lib/core/jhelper.c @@ -49,7 +49,6 @@ * @{ **/ - void j_helper_get_number_string(gchar* string, guint32 length, guint32 number) { diff --git a/lib/core/jmessage.c b/lib/core/jmessage.c index a296478e6..f9aa36a8f 100644 --- a/lib/core/jmessage.c +++ b/lib/core/jmessage.c @@ -38,8 +38,15 @@ #include #include -#define EXE(cmd, ...) do { if(cmd == FALSE) { g_warning(__VA_ARGS__); goto end; } } while(FALSE) - +#define EXE(cmd, ...) \ + do \ + { \ + if (cmd == FALSE) \ + { \ + g_warning(__VA_ARGS__); \ + goto end; \ + } \ + } while (FALSE) enum JMessageSemantics { @@ -514,9 +521,9 @@ j_message_append_memory_id(JMessage* message, const struct JConnectionMemoryID* g_return_val_if_fail(message != NULL, FALSE); g_return_val_if_fail(id != NULL, FALSE); - g_return_val_if_fail(j_message_can_append(message, sizeof(*id)+padding), FALSE); + g_return_val_if_fail(j_message_can_append(message, sizeof(*id) + padding), FALSE); - memcpy(message->current+padding, id, sizeof(*id)); + memcpy(message->current + padding, id, sizeof(*id)); message->current += sizeof(*id) + padding; new_length = j_message_length(message) + sizeof(*id) + padding; @@ -650,21 +657,25 @@ gboolean j_message_send_ack(JMessage* message, struct JConnection* connection) { const JConnectionAck ack = J_CONNECTION_ACK; - + // No data where send -> no need to acknowledge - if(message != NULL && j_message_get_count(message) == 0) { return TRUE; } + if (message != NULL && j_message_get_count(message) == 0) + { + return TRUE; + } EXE(j_connection_wait_for_completion(connection), - "Failed to wait to finishe all operations before sending ack!"); + "Failed to wait to finishe all operations before sending ack!"); EXE(j_connection_send(connection, &ack, sizeof(ack)), - "Failed to initiated sending ACK!"); + "Failed to initiated sending ACK!"); EXE(j_connection_wait_for_completion(connection), - "Failed to verify completion of sending ACK!"); + "Failed to verify completion of sending ACK!"); return TRUE; end: return FALSE; } -struct JConnectionMemory { +struct JConnectionMemory +{ struct fid_mr* memory_region; guint64 addr; guint64 size; @@ -689,7 +700,8 @@ j_message_send(JMessage* message, struct JConnection* connection) ret = FALSE; n_memory_regions = j_list_length(message->send_list); - if(n_memory_regions) { + if (n_memory_regions) + { size_t size = sizeof(struct JConnectionMemory) * n_memory_regions; memory_regions = malloc(size); memset(memory_regions, 0, size); @@ -700,11 +712,12 @@ j_message_send(JMessage* message, struct JConnection* connection) if (message->send_list != NULL) { { - guint64 total_data_length = - sizeof(message->header) + guint64 total_data_length = + sizeof(message->header) + j_message_length(message); iterator = j_list_iterator_new(message->send_list); - while(j_list_iterator_next(iterator)) { + while (j_list_iterator_next(iterator)) + { JMessageData* message_data = j_list_iterator_get(iterator); total_data_length += message_data->header_size + message_data->length; } @@ -717,36 +730,44 @@ j_message_send(JMessage* message, struct JConnection* connection) { JMessageData* message_data = j_list_iterator_get(iterator); - j_message_add_operation(message, - sizeof(struct JConnectionMemoryID) - + message_data->header_size - + fits ? message_data->length : 0); - - if(message_data->header_size > 0) { - if(message_data->header_size <= sizeof(message_data->header)) { + sizeof(struct JConnectionMemoryID) + + message_data->header_size + + fits + ? message_data->length + : 0); + + if (message_data->header_size > 0) + { + if (message_data->header_size <= sizeof(message_data->header)) + { EXE(j_message_append_n(message, &message_data->header, message_data->header_size), - "Failed to append header"); - } else { + "Failed to append header"); + } + else + { EXE(j_message_append_n(message, message_data->header, message_data->header_size), - "Failed to append header"); + "Failed to append header"); } } - if(!fits) { + if (!fits) + { EXE(j_connection_rma_register(connection, message_data->data, message_data->length, memory_itr), - "Failed to register message data memory!"); + "Failed to register message data memory!"); EXE(j_connection_memory_get_id(memory_itr, &mem_id), - "Failed to get memory it!"); + "Failed to get memory it!"); EXE(j_message_append_memory_id(message, &mem_id), - "Failed to append memory id to message!"); - } else { + "Failed to append memory id to message!"); + } + else + { mem_id.size = message_data->length; mem_id.key = 0; mem_id.offset = 0; EXE(j_message_append_memory_id(message, &mem_id), - "Failed to append memory information!"); + "Failed to append memory information!"); EXE(j_message_append_n(message, message_data->data, message_data->length), - "Failed to append message data!"); + "Failed to append message data!"); } ++memory_itr; } @@ -755,13 +776,17 @@ j_message_send(JMessage* message, struct JConnection* connection) ret = j_message_write(message, connection); end: - if(memory_regions != NULL) { + if (memory_regions != NULL) + { struct JConnectionMemory* mrs = memory_regions; - if(!fits) { - while(memory_regions != memory_itr) { + if (!fits) + { + while (memory_regions != memory_itr) + { j_connection_rma_unregister(connection, memory_regions++); } - if(memory_itr != memory_regions_end && memory_itr->memory_region) { + if (memory_itr != memory_regions_end && memory_itr->memory_region) + { j_connection_rma_unregister(connection, memory_itr); } } @@ -783,20 +808,22 @@ j_message_read(JMessage* message, struct JConnection* connection) g_return_val_if_fail(connection != NULL, FALSE); EXE(j_connection_recv(connection, sizeof(message->header), &(message->header)), - "Failed to initiated header receive!"); - if(!j_connection_wait_for_completion(connection)) { + "Failed to initiated header receive!"); + if (!j_connection_wait_for_completion(connection)) + { EXE(j_connection_closed(connection), - "Failed to wait for header receive!"); + "Failed to wait for header receive!"); goto end; } j_message_ensure_size(message, j_message_length(message)); - if(j_message_length(message) > 0) { + if (j_message_length(message) > 0) + { EXE(j_connection_recv(connection, j_message_length(message), message->data), - "Failed to initiated message body receive!"); + "Failed to initiated message body receive!"); EXE(j_connection_wait_for_completion(connection), - "Failed to wait for message body receive!"); + "Failed to wait for message body receive!"); } message->current = message->data; @@ -831,21 +858,24 @@ j_message_write(JMessage* message, struct JConnection* connection) g_return_val_if_fail(connection != NULL, FALSE); EXE(j_connection_send(connection, &(message->header), sizeof(message->header)), - "Failed to initiated sending message header."); - if(j_message_length(message) > 0) { + "Failed to initiated sending message header."); + if (j_message_length(message) > 0) + { EXE(j_connection_send(connection, message->data, j_message_length(message)), - "Failed to initiated sending message body."); + "Failed to initiated sending message body."); } - if(message->send_list && j_list_length(message->send_list)) { + if (message->send_list && j_list_length(message->send_list)) + { EXE(j_connection_recv(connection, sizeof(ack), &ack), - "Failed to initiated ACK receive."); + "Failed to initiated ACK receive."); } EXE(j_connection_wait_for_completion(connection), - "Failed to wait for completion of message send!"); + "Failed to wait for completion of message send!"); - if(j_list_length(message->send_list) && ack != J_CONNECTION_ACK) { + if (j_list_length(message->send_list) && ack != J_CONNECTION_ACK) + { g_warning("Wrong ACK flag received! got: %i, instead of: %i", ack, J_CONNECTION_ACK); goto end; } @@ -870,13 +900,18 @@ j_message_add_send(JMessage* message, gconstpointer data, guint64 length, void* message_data = g_slice_new(JMessageData); message_data->data = data; message_data->length = length; - if(h_size == 0) { + if (h_size == 0) + { message_data->header = NULL; message_data->header_size = 0; - } else if(h_size <= sizeof(message_data->header)) { + } + else if (h_size <= sizeof(message_data->header)) + { memcpy(&message_data->header, header, h_size); message_data->header_size = h_size; - } else { + } + else + { message_data->header = header; message_data->header_size = h_size; } diff --git a/lib/core/jnetwork.c b/lib/core/jnetwork.c index d1c0f8fce..285d6c122 100644 --- a/lib/core/jnetwork.c +++ b/lib/core/jnetwork.c @@ -29,7 +29,7 @@ gboolean j_connection_init(struct JConnection* instance); /// Allocated and bind memory needed for message transfer. -/** +/** * \protected \memberof JConnection * \retval FALSE if allocation or binding failed */ @@ -37,18 +37,21 @@ gboolean j_connection_create_memory_resources(struct JConnection* instance); /// flag used to different between client and server fabric -enum JFabricSide { +enum JFabricSide +{ JF_SERVER, JF_CLIENT }; -struct JFabricAddr { +struct JFabricAddr +{ void* addr; uint32_t addr_len; uint32_t addr_format; }; -struct JFabric { +struct JFabric +{ struct fi_info* info; struct fid_fabric* fabric; struct fid_eq* pep_eq; @@ -58,18 +61,21 @@ struct JFabric { enum JFabricSide con_side; }; -struct JConnection { +struct JConnection +{ struct JFabric* fabric; struct fi_info* info; struct fid_domain* domain; struct fid_ep* ep; struct fid_eq* eq; - struct { + struct + { struct fid_cq* tx; struct fid_cq* rx; } cq; size_t inject_size; - struct { + struct + { gboolean active; struct fid_mr* mr; void* buffer; @@ -78,11 +84,13 @@ struct JConnection { size_t rx_prefix_size; size_t tx_prefix_size; } memory; - struct { - struct { - void* context; - void* dest; - size_t len; + struct + { + struct + { + void* context; + void* dest; + size_t len; } msg_entry[J_CONNECTION_MAX_RECV + J_CONNECTION_MAX_SEND]; int msg_len; int rma_len; @@ -91,23 +99,47 @@ struct JConnection { gboolean closed; }; -struct JConnectionMemory { +struct JConnectionMemory +{ struct fid_mr* memory_region; guint64 addr; guint64 size; }; -#define EXE(cmd, ...) do { if (cmd == FALSE) { g_warning(__VA_ARGS__); goto end; } } while(FALSE) -#define CHECK(msg) do { if (res < 0) { g_warning("%s: "msg"\t(%s:%d)\nDetails:\t%s", "??TODO??", __FILE__, __LINE__,fi_strerror(-res)); goto end; } } while(FALSE) -#define G_CHECK(msg) do { \ - if(error != NULL) {\ - g_warning(msg"\n\tWith:%s", error->message);\ - g_error_free(error);\ - goto end; } } while(FALSE) +#define EXE(cmd, ...) \ + do \ + { \ + if (cmd == FALSE) \ + { \ + g_warning(__VA_ARGS__); \ + goto end; \ + } \ + } while (FALSE) +#define CHECK(msg) \ + do \ + { \ + if (res < 0) \ + { \ + g_warning("%s: " msg "\t(%s:%d)\nDetails:\t%s", "??TODO??", __FILE__, __LINE__, fi_strerror(-res)); \ + goto end; \ + } \ + } while (FALSE) +#define G_CHECK(msg) \ + do \ + { \ + if (error != NULL) \ + { \ + g_warning(msg "\n\tWith:%s", error->message); \ + g_error_free(error); \ + goto end; \ + } \ + } while (FALSE) void free_dangling_infos(struct fi_info*); -void free_dangling_infos(struct fi_info* info) { +void +free_dangling_infos(struct fi_info* info) +{ J_TRACE_FUNCTION(NULL); fi_freeinfo(info->next); @@ -131,10 +163,10 @@ j_fabric_init_server(struct JConfiguration* configuration, struct JFabric** inst this->con_side = JF_SERVER; res = fi_getinfo( - j_configuration_get_libfabric_version(this->config), - NULL, NULL, 0, - j_configuration_get_libfabric_hints(this->config), - &this->info); + j_configuration_get_libfabric_version(this->config), + NULL, NULL, 0, + j_configuration_get_libfabric_hints(this->config), + &this->info); CHECK("Failed to find fabric for server!"); free_dangling_infos(this->info); @@ -144,27 +176,29 @@ j_fabric_init_server(struct JConfiguration* configuration, struct JFabric** inst // wait obj defined to use synced actions res = fi_eq_open(this->fabric, - &(struct fi_eq_attr){.wait_obj = FI_WAIT_UNSPEC}, - &this->pep_eq, NULL); + &(struct fi_eq_attr){ .wait_obj = FI_WAIT_UNSPEC }, + &this->pep_eq, NULL); CHECK("failed to create eq for fabric!"); - res= fi_passive_ep(this->fabric, this->info, &this->pep, NULL); + res = fi_passive_ep(this->fabric, this->info, &this->pep, NULL); CHECK("failed to create pep for fabric!"); res = fi_pep_bind(this->pep, &this->pep_eq->fid, 0); CHECK("failed to bind event queue to pep!"); // initelize addr field! res = fi_getname(&this->pep->fid, NULL, &addrlen); - if(res != -FI_ETOOSMALL) { CHECK("failed to fetch address len from libfabirc!"); } + if (res != -FI_ETOOSMALL) + { + CHECK("failed to fetch address len from libfabirc!"); + } this->fabric_addr_network.addr_len = addrlen; this->fabric_addr_network.addr = malloc(this->fabric_addr_network.addr_len); res = fi_getname(&this->pep->fid, this->fabric_addr_network.addr, - &addrlen); + &addrlen); CHECK("failed to fetch address from libfabric!"); res = fi_listen(this->pep); CHECK("failed to start listening on pep!"); - this->fabric_addr_network.addr_len = htonl(this->fabric_addr_network.addr_len); this->fabric_addr_network.addr_format = htonl(this->info->addr_format); @@ -196,9 +230,9 @@ j_fabric_init_client(struct JConfiguration* configuration, struct JFabricAddr* a hints->dest_addrlen = addr->addr_len; res = fi_getinfo( - j_configuration_get_libfabric_version(configuration), - NULL, NULL, 0, - hints, &this->info); + j_configuration_get_libfabric_version(configuration), + NULL, NULL, 0, + hints, &this->info); CHECK("Failed to find fabric!"); free_dangling_infos(this->info); @@ -220,12 +254,13 @@ j_fabric_fini(struct JFabric* this) fi_freeinfo(this->info); this->info = NULL; - if(this->con_side == JF_SERVER) { + if (this->con_side == JF_SERVER) + { res = fi_close(&this->pep->fid); CHECK("Failed to close PEP!"); this->pep = NULL; - - res = fi_close(&this->pep_eq->fid); + + res = fi_close(&this->pep_eq->fid); CHECK("Failed to close EQ for PEP!"); this->pep_eq = NULL; } @@ -249,26 +284,38 @@ j_fabric_sread_event(struct JFabric* this, int timeout, JFabricEvents* event, JF uint32_t fi_event; res = fi_eq_sread(this->pep_eq, &fi_event, - con_req, sizeof(*con_req), - timeout, 0); - if (res == -FI_EAGAIN) { + con_req, sizeof(*con_req), + timeout, 0); + if (res == -FI_EAGAIN) + { *event = J_FABRIC_EVENT_TIMEOUT; - ret = TRUE; goto end; - } else if (res == -FI_EAVAIL) { - struct fi_eq_err_entry error = {0}; + ret = TRUE; + goto end; + } + else if (res == -FI_EAVAIL) + { + struct fi_eq_err_entry error = { 0 }; *event = J_FABRIC_EVENT_ERROR; res = fi_eq_readerr(this->pep_eq, &error, 0); CHECK("Failed to read error!"); g_warning("event queue contains following error (%s|c:%p):\n\t%s", - fi_strerror(FI_EAVAIL), error.context, - fi_eq_strerror(this->pep_eq, error.prov_errno, error.err_data, NULL, 0)); - ret = TRUE; goto end; + fi_strerror(FI_EAVAIL), error.context, + fi_eq_strerror(this->pep_eq, error.prov_errno, error.err_data, NULL, 0)); + ret = TRUE; + goto end; } CHECK("failed to read pep event queue!"); - switch(fi_event) { - case FI_CONNREQ: *event = J_FABRIC_EVENT_CONNECTION_REQUEST; break; - case FI_SHUTDOWN: *event = J_FABRIC_EVENT_SHUTDOWN; break; - default: g_assert_not_reached(); goto end; + switch (fi_event) + { + case FI_CONNREQ: + *event = J_FABRIC_EVENT_CONNECTION_REQUEST; + break; + case FI_SHUTDOWN: + *event = J_FABRIC_EVENT_SHUTDOWN; + break; + default: + g_assert_not_reached(); + goto end; } ret = TRUE; end: @@ -298,10 +345,11 @@ j_connection_init_client(struct JConfiguration* configuration, enum JBackendType g_client = g_socket_client_new(); server = j_configuration_get_server(configuration, backend, index); g_connection = g_socket_client_connect_to_host(g_client, - server, - j_configuration_get_port(configuration), NULL, &error); + server, + j_configuration_get_port(configuration), NULL, &error); G_CHECK("Failed to build gsocket connection to host"); - if(g_connection == NULL) { + if (g_connection == NULL) + { g_warning("Can not connect to %s.", server); goto end; } @@ -326,20 +374,23 @@ j_connection_init_client(struct JConfiguration* configuration, enum JBackendType G_CHECK("Failed to close gsocket!"); EXE(j_fabric_init_client(configuration, &jf_addr, &this->fabric), - "Failed to initelize fabric for client!"); + "Failed to initelize fabric for client!"); this->info = this->fabric->info; EXE(j_connection_init(this), - "Failed to initelze connection!"); + "Failed to initelze connection!"); res = fi_connect(this->ep, jf_addr.addr, NULL, 0); CHECK("Failed to fire connection request!"); - do {EXE(j_connection_sread_event(this, 1, &event), - "Failed to read event queue, waiting for CONNECTED signal!"); - } while(event == J_CONNECTION_EVENT_TIMEOUT); - - if(event != J_CONNECTION_EVENT_CONNECTED) { + do + { + EXE(j_connection_sread_event(this, 1, &event), + "Failed to read event queue, waiting for CONNECTED signal!"); + } while (event == J_CONNECTION_EVENT_TIMEOUT); + + if (event != J_CONNECTION_EVENT_CONNECTED) + { g_warning("Failed to connect to host!"); goto end; } @@ -368,7 +419,7 @@ j_connection_init_server(struct JFabric* fabric, GSocketConnection* gconnection, *instance_ptr = malloc(sizeof(*this)); this = *instance_ptr; memset(this, 0, sizeof(*this)); - + // send addr g_out = g_io_stream_get_output_stream(G_IO_STREAM(gconnection)); g_output_stream_write(g_out, &addr->addr_format, sizeof(addr->addr_format), NULL, &error); @@ -380,10 +431,13 @@ j_connection_init_server(struct JFabric* fabric, GSocketConnection* gconnection, g_output_stream_close(g_out, NULL, &error); G_CHECK("Failed to close output stream!"); - do {EXE(j_fabric_sread_event(fabric, 2, &event, &request), - "Failed to wait for connection request"); - } while(event == J_FABRIC_EVENT_TIMEOUT); - if(event != J_FABRIC_EVENT_CONNECTION_REQUEST) { + do + { + EXE(j_fabric_sread_event(fabric, 2, &event, &request), + "Failed to wait for connection request"); + } while (event == J_FABRIC_EVENT_TIMEOUT); + if (event != J_FABRIC_EVENT_CONNECTION_REQUEST) + { g_warning("expected an connection request and nothing else! (%i)", event); goto end; } @@ -395,13 +449,14 @@ j_connection_init_server(struct JFabric* fabric, GSocketConnection* gconnection, res = fi_accept(this->ep, NULL, 0); CHECK("Failed to accept connection!"); EXE(j_connection_sread_event(this, 2, &con_event), "Failed to verify connection!"); - if(con_event != J_CONNECTION_EVENT_CONNECTED) { + if (con_event != J_CONNECTION_EVENT_CONNECTED) + { g_warning("expected and connection ack and nothing else!"); goto end; } ret = TRUE; -end: +end: return ret; } @@ -410,36 +465,28 @@ j_connection_init(struct JConnection* this) { J_TRACE_FUNCTION(NULL); - gboolean ret = FALSE; + gboolean ret = FALSE; int res; this->running_actions.msg_len = 0; this->running_actions.rma_len = 0; this->next_key = KEY_MIN; - res = fi_eq_open(this->fabric->fabric, &(struct fi_eq_attr){.wait_obj = FI_WAIT_UNSPEC}, - &this->eq, NULL); + res = fi_eq_open(this->fabric->fabric, &(struct fi_eq_attr){ .wait_obj = FI_WAIT_UNSPEC }, + &this->eq, NULL); CHECK("Failed to open event queue for connection!"); res = fi_domain(this->fabric->fabric, this->info, &this->domain, NULL); CHECK("Failed to open connection domain!"); - + this->inject_size = this->fabric->info->tx_attr->inject_size; EXE(j_connection_create_memory_resources(this), - "Failed to create memory resources for connection!"); - - res = fi_cq_open(this->domain, &(struct fi_cq_attr){ - .wait_obj = FI_WAIT_UNSPEC, - .format = FI_CQ_FORMAT_CONTEXT, - .size = this->info->tx_attr->size - }, - &this->cq.tx, &this->cq.tx); - res = fi_cq_open(this->domain, &(struct fi_cq_attr){ - .wait_obj = FI_WAIT_UNSPEC, - .format = FI_CQ_FORMAT_CONTEXT, - .size = this->info->rx_attr->size - }, - &this->cq.rx, &this->cq.rx); + "Failed to create memory resources for connection!"); + + res = fi_cq_open(this->domain, &(struct fi_cq_attr){ .wait_obj = FI_WAIT_UNSPEC, .format = FI_CQ_FORMAT_CONTEXT, .size = this->info->tx_attr->size }, + &this->cq.tx, &this->cq.tx); + res = fi_cq_open(this->domain, &(struct fi_cq_attr){ .wait_obj = FI_WAIT_UNSPEC, .format = FI_CQ_FORMAT_CONTEXT, .size = this->info->rx_attr->size }, + &this->cq.rx, &this->cq.rx); CHECK("Failed to create completion queue!"); res = fi_endpoint(this->domain, this->info, &this->ep, NULL); CHECK("Failed to open endpoint for connection!"); @@ -475,17 +522,19 @@ j_connection_create_memory_resources(struct JConnection* this) rx_prefix = (this->info->rx_attr->mode & FI_MSG_PREFIX) != 0; prefix_size = this->info->ep_attr->msg_prefix_size; - if(op_size + (tx_prefix | rx_prefix) * prefix_size > this->info->ep_attr->max_msg_size) { + if (op_size + (tx_prefix | rx_prefix) * prefix_size > this->info->ep_attr->max_msg_size) + { guint64 max_size = this->info->ep_attr->max_msg_size - (tx_prefix | rx_prefix) * prefix_size; g_critical("Fabric supported memory size is too smal! please configure a max operation size less equal to %lu! instead of %lu", - max_size, op_size + (tx_prefix | rx_prefix) * prefix_size); + max_size, op_size + (tx_prefix | rx_prefix) * prefix_size); goto end; } size = 0; - if(this->info->domain_attr->mr_mode & FI_MR_LOCAL) { + if (this->info->domain_attr->mr_mode & FI_MR_LOCAL) + { size += - (rx_prefix * prefix_size) + J_CONNECTION_MAX_RECV * op_size + (rx_prefix * prefix_size) + J_CONNECTION_MAX_RECV * op_size + (tx_prefix * prefix_size) + J_CONNECTION_MAX_SEND * op_size; this->memory.active = TRUE; this->memory.used = 0; @@ -496,7 +545,11 @@ j_connection_create_memory_resources(struct JConnection* this) res = fi_mr_reg(this->domain, this->memory.buffer, this->memory.buffer_size, FI_SEND | FI_RECV, 0, 0, 0, &this->memory.mr, NULL); CHECK("Failed to register memory for msg communication!"); - } else { this->memory.active = FALSE; } + } + else + { + this->memory.active = FALSE; + } ret = TRUE; end: @@ -513,29 +566,40 @@ j_connection_sread_event(struct JConnection* this, int timeout, JConnectionEvent gboolean ret = FALSE; struct fi_eq_cm_entry entry; - res = fi_eq_sread(this->eq, &fi_event, &entry, sizeof(entry), timeout, 5); - if(res == -FI_EAGAIN) { + if (res == -FI_EAGAIN) + { *event = J_CONNECTION_EVENT_TIMEOUT; - ret = TRUE; goto end; - } else if (res == -FI_EAVAIL) { - struct fi_eq_err_entry error = {0}; + ret = TRUE; + goto end; + } + else if (res == -FI_EAVAIL) + { + struct fi_eq_err_entry error = { 0 }; *event = J_CONNECTION_EVENT_ERROR; - do { - res = fi_eq_readerr(this->eq, &error, 0); - CHECK("Failed to read error!"); - g_warning("event queue contains following error (%s|c:%p):\n\t%s", - fi_strerror(FI_EAVAIL), error.context, - fi_eq_strerror(this->eq, error.prov_errno, error.err_data, NULL, 0)); - } while(res > 0); + do + { + res = fi_eq_readerr(this->eq, &error, 0); + CHECK("Failed to read error!"); + g_warning("event queue contains following error (%s|c:%p):\n\t%s", + fi_strerror(FI_EAVAIL), error.context, + fi_eq_strerror(this->eq, error.prov_errno, error.err_data, NULL, 0)); + } while (res > 0); goto end; } CHECK("Failed to read event of connection!"); - switch(fi_event) { - case FI_CONNECTED: *event = J_CONNECTION_EVENT_CONNECTED; break; - case FI_SHUTDOWN: *event = J_CONNECTION_EVENT_SHUTDOWN; break; - default: g_assert_not_reached(); goto end; + switch (fi_event) + { + case FI_CONNECTED: + *event = J_CONNECTION_EVENT_CONNECTED; + break; + case FI_SHUTDOWN: + *event = J_CONNECTION_EVENT_SHUTDOWN; + break; + default: + g_assert_not_reached(); + goto end; } ret = TRUE; end: @@ -552,24 +616,36 @@ j_connection_read_event(struct JConnection* this, JConnectionEvents* event) gboolean ret = FALSE; res = fi_eq_read(this->eq, &fi_event, NULL, 0, 0); - if(res == -FI_EAGAIN) { + if (res == -FI_EAGAIN) + { *event = J_CONNECTION_EVENT_TIMEOUT; - ret = TRUE; goto end; - } else if (res == -FI_EAVAIL) { - struct fi_eq_err_entry error = {0}; + ret = TRUE; + goto end; + } + else if (res == -FI_EAVAIL) + { + struct fi_eq_err_entry error = { 0 }; *event = J_CONNECTION_EVENT_ERROR; res = fi_eq_readerr(this->eq, &error, 0); CHECK("Failed to read error!"); g_warning("event queue contains following error (%s|c:%p):\n\t%s", - fi_strerror(FI_EAVAIL), error.context, - fi_eq_strerror(this->eq, error.prov_errno, error.err_data, NULL, 0)); - ret = TRUE; goto end; + fi_strerror(FI_EAVAIL), error.context, + fi_eq_strerror(this->eq, error.prov_errno, error.err_data, NULL, 0)); + ret = TRUE; + goto end; } CHECK("Failed to read event of connection!"); - switch(fi_event) { - case FI_CONNECTED: *event = J_CONNECTION_EVENT_CONNECTED; break; - case FI_SHUTDOWN: *event = J_CONNECTION_EVENT_SHUTDOWN; break; - default: g_assert_not_reached(); goto end; + switch (fi_event) + { + case FI_CONNECTED: + *event = J_CONNECTION_EVENT_CONNECTED; + break; + case FI_SHUTDOWN: + *event = J_CONNECTION_EVENT_SHUTDOWN; + break; + default: + g_assert_not_reached(); + goto end; } ret = TRUE; end: @@ -588,32 +664,42 @@ j_connection_send(struct JConnection* this, const void* data, size_t data_len) // we used paired endponits -> inject and send don't need destination addr (last parameter) - if(data_len < this->inject_size) { - do { res = fi_inject(this->ep, data, data_len, 0); } while(res == -FI_EAGAIN); + if (data_len < this->inject_size) + { + do + { + res = fi_inject(this->ep, data, data_len, 0); + } while (res == -FI_EAGAIN); CHECK("Failed to inject data!"); - ret = TRUE; goto end; + ret = TRUE; + goto end; } // normal send - if (this->memory.active) { + if (this->memory.active) + { uint8_t* segment = (uint8_t*)this->memory.buffer + this->memory.used; context = segment; memcpy(segment + this->memory.tx_prefix_size, - data, data_len); + data, data_len); size = data_len + this->memory.tx_prefix_size; - do{ + do + { res = fi_send(this->ep, segment, size, fi_mr_desc(this->memory.mr), 0, context); - } while(res == -FI_EAGAIN); + } while (res == -FI_EAGAIN); CHECK("Failed to initelize sending!"); this->memory.used += size; g_assert_true(this->memory.used <= this->memory.buffer_size); - g_assert_true(this->running_actions.msg_len + 1< J_CONNECTION_MAX_SEND + J_CONNECTION_MAX_RECV); - } else { + g_assert_true(this->running_actions.msg_len + 1 < J_CONNECTION_MAX_SEND + J_CONNECTION_MAX_RECV); + } + else + { context = (void*)((const char*)data - (const char*)NULL); size = data_len; - do{ + do + { res = fi_send(this->ep, data, size, NULL, 0, context); - } while(res == -FI_EAGAIN); + } while (res == -FI_EAGAIN); } this->running_actions.msg_entry[this->running_actions.msg_len].context = context; this->running_actions.msg_entry[this->running_actions.msg_len].dest = NULL; @@ -638,18 +724,21 @@ j_connection_recv(struct JConnection* this, size_t data_len, void* data) segment = this->memory.active ? (char*)this->memory.buffer + this->memory.used : data; size = data_len + this->memory.rx_prefix_size; res = fi_recv(this->ep, segment, size, - this->memory.active ? fi_mr_desc(this->memory.mr) : NULL, - 0, segment); + this->memory.active ? fi_mr_desc(this->memory.mr) : NULL, + 0, segment); CHECK("Failed to initelized receiving!"); - if (this->memory.active) { + if (this->memory.active) + { this->memory.used += size; g_assert_true(this->memory.used <= this->memory.buffer_size); g_assert_true(this->running_actions.msg_len < J_CONNECTION_MAX_SEND + J_CONNECTION_MAX_RECV); this->running_actions.msg_entry[this->running_actions.msg_len].context = segment; this->running_actions.msg_entry[this->running_actions.msg_len].dest = data; this->running_actions.msg_entry[this->running_actions.msg_len].len = data_len; - } else { + } + else + { this->running_actions.msg_entry[this->running_actions.msg_len].context = segment; this->running_actions.msg_entry[this->running_actions.msg_len].dest = NULL; this->running_actions.msg_entry[this->running_actions.msg_len].len = 0; @@ -677,43 +766,59 @@ j_connection_wait_for_completion(struct JConnection* this) int res; struct fi_cq_entry entry; int i; - - while(this->running_actions.rma_len + this->running_actions.msg_len) { + + while (this->running_actions.rma_len + this->running_actions.msg_len) + { bool rx; - do { + do + { rx = TRUE; res = fi_cq_read(this->cq.rx, &entry, 1); - if(res == -FI_EAGAIN) { + if (res == -FI_EAGAIN) + { rx = FALSE; res = fi_cq_read(this->cq.tx, &entry, 1); } } while (res == -FI_EAGAIN); - if(res == -FI_EAVAIL) { + if (res == -FI_EAVAIL) + { JConnectionEvents event; j_connection_sread_event(this, 0, &event); - if(event == J_CONNECTION_EVENT_SHUTDOWN) { this->closed = TRUE; goto end; } + if (event == J_CONNECTION_EVENT_SHUTDOWN) + { + this->closed = TRUE; + goto end; + } struct fi_cq_err_entry err_entry; res = fi_cq_readerr(rx ? this->cq.rx : this->cq.tx, - &err_entry, 0); + &err_entry, 0); CHECK("Failed to read error of cq!"); g_warning("Failed to read completion queue\nWidth:\t%s", - fi_cq_strerror(rx ? this->cq.rx : this->cq.tx, - err_entry.prov_errno, err_entry.err_data, NULL, 0)); + fi_cq_strerror(rx ? this->cq.rx : this->cq.tx, + err_entry.prov_errno, err_entry.err_data, NULL, 0)); goto end; - } else { CHECK("Failed to read completion queue!"); } - for(i = 0; i <= this->running_actions.msg_len; ++i) { - if(i == this->running_actions.msg_len) { + } + else + { + CHECK("Failed to read completion queue!"); + } + for (i = 0; i <= this->running_actions.msg_len; ++i) + { + if (i == this->running_actions.msg_len) + { // If there is no match -> it's an rma transafre -> context = memory region res = fi_close(&((struct fid_mr*)entry.op_context)->fid); CHECK("Failed to free receiving memory!"); --this->running_actions.rma_len; } - if(this->running_actions.msg_entry[i].context == entry.op_context) { + if (this->running_actions.msg_entry[i].context == entry.op_context) + { --this->running_actions.msg_len; - if(this->running_actions.msg_entry[i].dest) { + if (this->running_actions.msg_entry[i].dest) + { memcpy(this->running_actions.msg_entry[i].dest, - this->running_actions.msg_entry[i].context, - this->running_actions.msg_entry[i].len); + this->running_actions.msg_entry[i].context, + this->running_actions.msg_entry[i].len); } this->running_actions.msg_entry[i] = this->running_actions.msg_entry[this->running_actions.msg_len]; break; @@ -741,7 +846,7 @@ j_connection_rma_register(struct JConnection* this, const void* data, size_t dat FI_REMOTE_READ, 0, this->next_key, 0, &handle->memory_region, NULL); CHECK("Failed to register memory region!"); - handle->addr = (this->info->domain_attr->mr_mode & FI_MR_VIRT_ADDR ) ? (guint64)data : 0; + handle->addr = (this->info->domain_attr->mr_mode & FI_MR_VIRT_ADDR) ? (guint64)data : 0; handle->size = data_len; this->next_key += 1; @@ -788,29 +893,36 @@ j_connection_rma_read(struct JConnection* this, const struct JConnectionMemoryID res = fi_mr_reg(this->domain, data, memoryID->size, FI_READ, 0, ++key, 0, &mr, 0); CHECK("Failed to register receiving memory!"); - do { res = fi_read(this->ep, - data, - memoryID->size, - fi_mr_desc(mr), - 0, - memoryID->offset, - memoryID->key, - mr); - if(res == -FI_EAGAIN) { + do + { + res = fi_read(this->ep, + data, + memoryID->size, + fi_mr_desc(mr), + 0, + memoryID->offset, + memoryID->key, + mr); + if (res == -FI_EAGAIN) + { /// \todo evaluate: only wait for partcial finished jobs j_connection_wait_for_completion(this); } - } while ( res == -FI_EAGAIN); + } while (res == -FI_EAGAIN); CHECK("Failed to initate reading"); ++this->running_actions.rma_len; ret = TRUE; end: - if(!ret) { fi_close(&mr->fid); } + if (!ret) + { + fi_close(&mr->fid); + } return ret; } gboolean -j_connection_fini ( struct JConnection* this) { +j_connection_fini(struct JConnection* this) +{ J_TRACE_FUNCTION(NULL); int res; @@ -819,7 +931,8 @@ j_connection_fini ( struct JConnection* this) { res = fi_shutdown(this->ep, 0); CHECK("failed to send shutdown signal"); - if(this->memory.active) { + if (this->memory.active) + { res = fi_close(&this->memory.mr->fid); CHECK("failed to free memory region!"); free(this->memory.buffer); @@ -836,14 +949,15 @@ j_connection_fini ( struct JConnection* this) { res = fi_close(&this->domain->fid); CHECK("failed to close domain!"); - if(this->fabric->con_side == JF_CLIENT) { + if (this->fabric->con_side == JF_CLIENT) + { j_fabric_fini(this->fabric); } free(this); ret = TRUE; end: - return ret; + return ret; } struct JConfiguration* diff --git a/lib/object/jdistributed-object.c b/lib/object/jdistributed-object.c index bab4124ec..688252d11 100644 --- a/lib/object/jdistributed-object.c +++ b/lib/object/jdistributed-object.c @@ -372,9 +372,12 @@ j_distributed_object_read_background_operation(gpointer data) if (memoryID->size > 0) { - if(memoryID->key == 0 && memoryID->offset == 0) { + if (memoryID->key == 0 && memoryID->offset == 0) + { memcpy(read_data, j_message_get_n(reply, memoryID->size), memoryID->size); - } else { + } + else + { j_connection_rma_read(object_connection, memoryID, read_data); } } diff --git a/lib/object/jobject.c b/lib/object/jobject.c index f3d864cd6..b747ee900 100644 --- a/lib/object/jobject.c +++ b/lib/object/jobject.c @@ -557,9 +557,12 @@ j_object_read_exec(JList* operations, JSemantics* semantics) if (mem_id->size > 0) { - if(mem_id->key == 0 && mem_id->offset == 0) { + if (mem_id->key == 0 && mem_id->offset == 0) + { memcpy(data, j_message_get_n(reply, mem_id->size), mem_id->size); - } else { + } + else + { j_connection_rma_read(object_connection, mem_id, data); } } @@ -733,7 +736,6 @@ j_object_write_exec(JList* operations, JSemantics* semantics) ret = j_backend_object_close(object_backend, object_handle) && ret; } - /* if (lock != NULL) { diff --git a/server/loop.c b/server/loop.c index 246693b73..6d54374f8 100644 --- a/server/loop.c +++ b/server/loop.c @@ -28,7 +28,8 @@ static guint jd_thread_num = 0; -struct ObjectWriteEntry { +struct ObjectWriteEntry +{ uint64_t size; uint64_t offset; void* data; @@ -372,7 +373,8 @@ jd_handle_message(JMessage* message, struct JConnection* connection, JMemoryChun j_backend_object_close(jd_object_backend, object); } - if(operation_count) { + if (operation_count) + { j_message_send_ack(NULL, connection); } if (reply != NULL) diff --git a/server/server.c b/server/server.c index 41fd65937..8e7053fec 100644 --- a/server/server.c +++ b/server/server.c @@ -285,12 +285,14 @@ main(int argc, char** argv) g_socket_listener_set_backlog(G_SOCKET_LISTENER(socket_service), 128); jd_configuration = j_configuration_new(); - if(!jd_configuration) { + if (!jd_configuration) + { g_warning("Failed to load configuration!"); return 1; } - if(opt_port == 0) { + if (opt_port == 0) + { opt_port = j_configuration_get_port(jd_configuration); } @@ -379,7 +381,7 @@ main(int argc, char** argv) g_debug("Initialized db backend %s.", db_backend); } - if(!j_fabric_init_server(jd_configuration, &jd_fabric)) + if (!j_fabric_init_server(jd_configuration, &jd_fabric)) { g_warning("Failed to initialize server fabric!"); return 1; diff --git a/tools/config.c b/tools/config.c index 172972e08..dfa9fea11 100644 --- a/tools/config.c +++ b/tools/config.c @@ -156,7 +156,7 @@ main(gint argc, gchar** argv) { "system", 0, 0, G_OPTION_ARG_NONE, &opt_system, "Write system configuration", NULL }, { "read", 0, 0, G_OPTION_ARG_NONE, &opt_read, "Read configuration", NULL }, { "name", 0, 0, G_OPTION_ARG_STRING, &opt_name, "Configuration name", "julea" }, - { "port", 0, 0, G_OPTION_ARG_INT64, &opt_network_port, "Network communication port", "4000 + user-id%1000"}, + { "port", 0, 0, G_OPTION_ARG_INT64, &opt_network_port, "Network communication port", "4000 + user-id%1000" }, { "object-servers", 0, 0, G_OPTION_ARG_STRING, &opt_servers_object, "Object servers to use", "host1,host2:port" }, { "kv-servers", 0, 0, G_OPTION_ARG_STRING, &opt_servers_kv, "Key-value servers to use", "host1,host2:port" }, { "db-servers", 0, 0, G_OPTION_ARG_STRING, &opt_servers_db, "Database servers to use", "host1,host2:port" }, @@ -211,10 +211,10 @@ main(gint argc, gchar** argv) return 1; } - if(opt_network_port == 0) + if (opt_network_port == 0) { JCredentials* credentials = j_credentials_new(); - opt_network_port = 4000 + ( j_credentials_get_user(credentials) % 1000 ); + opt_network_port = 4000 + (j_credentials_get_user(credentials) % 1000); j_credentials_unref(credentials); } From e0d69271e4f31fdeb795ae4b2c60c3c8fb0552a5 Mon Sep 17 00:00:00 2001 From: Julian Benda Date: Sat, 23 Oct 2021 14:37:27 +0200 Subject: [PATCH 45/60] Add RDMA dependency * removed libfabric_example from builds --- meson.build | 4 ++-- scripts/spack | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/meson.build b/meson.build index eb0bcafbc..795e44038 100644 --- a/meson.build +++ b/meson.build @@ -739,8 +739,8 @@ endif # ) #endforeach -executable('libfabric_basic', files('example/libfabric_basic.c'), - dependencies: [glib_dep, libfabric_dep]) +# executable('libfabric_basic', files('example/libfabric_basic.c'), +# dependencies: [glib_dep, libfabric_dep]) # Install diff --git a/scripts/spack b/scripts/spack index a1d963541..9356e352b 100644 --- a/scripts/spack +++ b/scripts/spack @@ -97,7 +97,7 @@ spack_get_dependencies () # Mandatory dependencies dependencies="${dependencies} glib" dependencies="${dependencies} libbson" - #dependencies="${dependencies} libfabric#fabrics=sockets,tcp,udp,verbs,rxd,rxm" + dependencies="${dependencies} libfabric#fabrics=sockets,tcp,udp,verbs,rxd,rxm" # Recommended dependencies dependencies="${dependencies} hdf5~mpi@1.12:" From 925207f5f7f75da3d49f1899d7764b2a48a6a6c6 Mon Sep 17 00:00:00 2001 From: Julian Benda Date: Sat, 23 Oct 2021 15:00:27 +0200 Subject: [PATCH 46/60] clean up compiler warnings --- backend/db/sql-generic.c | 2 +- lib/core/jnetwork.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/db/sql-generic.c b/backend/db/sql-generic.c index 4a70bbf3a..9e914c7e0 100644 --- a/backend/db/sql-generic.c +++ b/backend/db/sql-generic.c @@ -1109,7 +1109,7 @@ _backend_schema_get(gpointer backend_data, gpointer _batch, gchar const* name, b return TRUE; _error: - if (G_UNLIKELY(!j_sql_reset(thread_variables->sql_backend, prepared->stmt, NULL))) + if (G_UNLIKELY(!thread_variables || !prepared || !j_sql_reset(thread_variables->sql_backend, prepared->stmt, NULL))) { goto _error2; } diff --git a/lib/core/jnetwork.c b/lib/core/jnetwork.c index 285d6c122..8d8e14035 100644 --- a/lib/core/jnetwork.c +++ b/lib/core/jnetwork.c @@ -4,7 +4,6 @@ #include -#include #include #include #include @@ -783,13 +782,14 @@ j_connection_wait_for_completion(struct JConnection* this) if (res == -FI_EAVAIL) { JConnectionEvents event; + struct fi_cq_err_entry err_entry; + j_connection_sread_event(this, 0, &event); if (event == J_CONNECTION_EVENT_SHUTDOWN) { this->closed = TRUE; goto end; } - struct fi_cq_err_entry err_entry; res = fi_cq_readerr(rx ? this->cq.rx : this->cq.tx, &err_entry, 0); CHECK("Failed to read error of cq!"); From 0515521b729ed2988d8452f21ca241fc32806070 Mon Sep 17 00:00:00 2001 From: Julian Benda Date: Sat, 23 Oct 2021 17:19:17 +0200 Subject: [PATCH 47/60] Add libfabric to system dependencies --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d94ffc459..4ed26b36d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -109,9 +109,9 @@ jobs: sudo apt --yes --purge autoremove sudo aa-remove-unknown sudo apt update || true - sudo apt --yes --no-install-recommends install ninja-build pkgconf libglib2.0-dev libbson-dev liblmdb-dev libsqlite3-dev libleveldb-dev libmongoc-dev libmariadb-dev librocksdb-dev libfuse-dev libopen-trace-format-dev librados-dev - sudo apt --yes --no-install-recommends install python3 python3-pip python3-setuptools python3-wheel - sudo pip3 install meson + sudo apt --yes --no-install-recommends install ninja-build pkgconf libglib2.0-dev libbson-dev liblmdb-dev libsqlite3-dev libleveldb-dev libmongoc-dev libmariadb-dev librocksdb-dev libfuse-dev libopen-trace-format-dev librados-dev libfabric-dev + sudo apt --yes --no-install-recommends install python3 python3-pip python3-setuptools python3-wheel + sudo pip3 install meson - name: Cache dependencies id: cache if: matrix.dependencies == 'spack' From 833fa3d511cc93804665dbbd6d3579507fdc574b Mon Sep 17 00:00:00 2001 From: Julian Benda Date: Sun, 24 Oct 2021 11:09:15 +0200 Subject: [PATCH 48/60] Add libfabric-dev also for test build --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4ed26b36d..756699e32 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -208,9 +208,9 @@ jobs: sudo apt --yes --purge autoremove sudo aa-remove-unknown sudo apt update || true - sudo apt --yes --no-install-recommends install ninja-build pkgconf libglib2.0-dev libbson-dev liblmdb-dev libsqlite3-dev libleveldb-dev libmongoc-dev libmariadb-dev librocksdb-dev libfuse-dev libopen-trace-format-dev librados-dev - sudo apt --yes --no-install-recommends install python3 python3-pip python3-setuptools python3-wheel - sudo pip3 install meson + sudo apt --yes --no-install-recommends install ninja-build pkgconf libglib2.0-dev libbson-dev liblmdb-dev libsqlite3-dev libleveldb-dev libmongoc-dev libmariadb-dev librocksdb-dev libfuse-dev libopen-trace-format-dev librados-dev libfabric-dev + sudo apt --yes --no-install-recommends install python3 python3-pip python3-setuptools python3-wheel + sudo pip3 install meson - name: Cache dependencies id: cache if: matrix.dependencies == 'spack' From 2ae6e6113e35ccdf991830790b310441fedee8f1 Mon Sep 17 00:00:00 2001 From: Julian Benda Date: Wed, 27 Oct 2021 12:21:31 +0200 Subject: [PATCH 49/60] Fix missing initelization --- lib/core/jnetwork.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/core/jnetwork.c b/lib/core/jnetwork.c index 8d8e14035..4e0c8e3f6 100644 --- a/lib/core/jnetwork.c +++ b/lib/core/jnetwork.c @@ -150,9 +150,9 @@ j_fabric_init_server(struct JConfiguration* configuration, struct JFabric** inst { J_TRACE_FUNCTION(NULL); - struct JFabric* this; - int res; - size_t addrlen; + struct JFabric* this = NULL; + int res = 0; + size_t addrlen = 0; *instance_ptr = malloc(sizeof(*this)); this = *instance_ptr; From f6ea57333d85096f24fc140d7471ef6e1c6254c2 Mon Sep 17 00:00:00 2001 From: Julian Benda Date: Wed, 27 Oct 2021 12:21:47 +0200 Subject: [PATCH 50/60] Moved message injection size to config --- include/core/jconfiguration.h | 2 ++ lib/core/jconfiguration.c | 18 ++++++++++++++++++ lib/core/jmessage.c | 6 ++++-- 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/include/core/jconfiguration.h b/include/core/jconfiguration.h index bf6aaad00..dbe9ce1bb 100644 --- a/include/core/jconfiguration.h +++ b/include/core/jconfiguration.h @@ -111,6 +111,8 @@ gchar const* j_configuration_get_backend_path(JConfiguration*, JBackendType); guint64 j_configuration_get_max_operation_size(JConfiguration*); guint16 j_configuration_get_port(JConfiguration*); +/// size of data segment which should be injected in message +guint64 j_configuration_get_message_inject_size(JConfiguration*); guint32 j_configuration_get_max_connections(JConfiguration*); guint64 j_configuration_get_stripe_size(JConfiguration*); diff --git a/lib/core/jconfiguration.c b/lib/core/jconfiguration.c index ea881bed0..31c48666d 100644 --- a/lib/core/jconfiguration.c +++ b/lib/core/jconfiguration.c @@ -151,6 +151,7 @@ struct JConfiguration guint64 max_operation_size; guint16 port; + guint64 max_message_injection_size; guint32 max_connections; guint64 stripe_size; @@ -292,6 +293,7 @@ j_configuration_new_for_data(GKeyFile* key_file) gchar* libfabric_provider; guint64 max_operation_size; guint32 port; + guint64 max_message_injection_size; guint32 max_connections; guint64 stripe_size; @@ -299,6 +301,7 @@ j_configuration_new_for_data(GKeyFile* key_file) max_operation_size = g_key_file_get_uint64(key_file, "core", "max-operation-size", NULL); port = g_key_file_get_integer(key_file, "core", "port", NULL); + max_message_injection_size = g_key_file_get_uint64(key_file, "core", "max-message-injection-size", NULL); max_connections = g_key_file_get_integer(key_file, "clients", "max-connections", NULL); stripe_size = g_key_file_get_uint64(key_file, "clients", "stripe-size", NULL); servers_object = g_key_file_get_string_list(key_file, "servers", "object", NULL, NULL); @@ -367,6 +370,7 @@ j_configuration_new_for_data(GKeyFile* key_file) configuration->db.path = db_path; configuration->max_operation_size = max_operation_size; configuration->port = port; + configuration->max_message_injection_size = max_message_injection_size; configuration->max_connections = max_connections; configuration->stripe_size = stripe_size; configuration->ref_count = 1; @@ -391,6 +395,10 @@ j_configuration_new_for_data(GKeyFile* key_file) configuration->port = 4711 + (j_credentials_get_user(credentials) % 1000); } + if (configuration->max_message_injection_size == 0) + { + configuration->max_message_injection_size = configuration->max_operation_size / 1024; + } if (configuration->max_connections == 0) { @@ -572,6 +580,16 @@ j_configuration_get_max_operation_size(JConfiguration* configuration) return configuration->max_operation_size; } +guint64 +j_configuration_get_message_inject_size(JConfiguration* configuration) +{ + J_TRACE_FUNCTION(NULL); + + g_return_val_if_fail(configuration != NULL, 0); + + return configuration->max_message_injection_size; +} + guint32 j_configuration_get_max_connections(JConfiguration* configuration) { diff --git a/lib/core/jmessage.c b/lib/core/jmessage.c index f9aa36a8f..e9a32a1c0 100644 --- a/lib/core/jmessage.c +++ b/lib/core/jmessage.c @@ -719,10 +719,12 @@ j_message_send(JMessage* message, struct JConnection* connection) while (j_list_iterator_next(iterator)) { JMessageData* message_data = j_list_iterator_get(iterator); - total_data_length += message_data->header_size + message_data->length; + total_data_length += sizeof(struct JConnectionMemoryID) + message_data->header_size + message_data->length; } j_list_iterator_free(iterator); - fits = total_data_length < 10 * 1024; ///\todo fix magic number + fits = + total_data_length + < j_configuration_get_message_inject_size(j_connection_get_configuration(connection)); } iterator = j_list_iterator_new(message->send_list); From d51d50e653ca67c57391467e54c4a25eede6c278 Mon Sep 17 00:00:00 2001 From: Julian Benda Date: Wed, 27 Oct 2021 13:05:28 +0200 Subject: [PATCH 51/60] Print julea server log on test fail --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 756699e32..8b6865c8a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -280,9 +280,9 @@ jobs: run: | . scripts/environment.sh ./scripts/setup.sh start - ./scripts/test.sh + ./scripts/test.sh || journalctl GLIB_DOMAIN=JULEA sleep 10 - ./scripts/test.sh + ./scripts/test.sh || journalctl GLIB_DOMAIN=JULEA ./scripts/setup.sh stop - name: HDF5 Tests if: matrix.julea.db != 'memory' From 93c0a95db03132f6f63d21dd335f2db3767d678b Mon Sep 17 00:00:00 2001 From: Julian Benda Date: Wed, 27 Oct 2021 23:01:23 +0200 Subject: [PATCH 52/60] Keep fabric hints around until finalisation some parts are still referenced in the fi_info, and to early freed if hints get freed --- lib/core/jnetwork.c | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/lib/core/jnetwork.c b/lib/core/jnetwork.c index 4e0c8e3f6..145201916 100644 --- a/lib/core/jnetwork.c +++ b/lib/core/jnetwork.c @@ -52,6 +52,7 @@ struct JFabricAddr struct JFabric { struct fi_info* info; + struct fi_info* hints; struct fid_fabric* fabric; struct fid_eq* pep_eq; struct fid_pep* pep; @@ -212,7 +213,6 @@ j_fabric_init_client(struct JConfiguration* configuration, struct JFabricAddr* a J_TRACE_FUNCTION(NULL); struct JFabric* this; - struct fi_info* hints; int res; gboolean ret = FALSE; @@ -223,15 +223,15 @@ j_fabric_init_client(struct JConfiguration* configuration, struct JFabricAddr* a this->config = configuration; this->con_side = JF_CLIENT; - hints = fi_dupinfo(j_configuration_get_libfabric_hints(configuration)); - hints->addr_format = addr->addr_format; - hints->dest_addr = addr->addr; - hints->dest_addrlen = addr->addr_len; + this->hints = fi_dupinfo(j_configuration_get_libfabric_hints(configuration)); + this->hints->addr_format = addr->addr_format; + this->hints->dest_addr = addr->addr; + this->hints->dest_addrlen = addr->addr_len; res = fi_getinfo( j_configuration_get_libfabric_version(configuration), NULL, NULL, 0, - hints, &this->info); + this->hints, &this->info); CHECK("Failed to find fabric!"); free_dangling_infos(this->info); @@ -240,7 +240,6 @@ j_fabric_init_client(struct JConfiguration* configuration, struct JFabricAddr* a ret = TRUE; end: - // fi_freeinfo(hints); return ret; } @@ -267,6 +266,11 @@ j_fabric_fini(struct JFabric* this) res = fi_close(&this->fabric->fid); CHECK("failed to close fabric!"); this->fabric = NULL; + if(this->hints) + { + fi_freeinfo(this->hints); + this->hints = NULL; + } free(this); return TRUE; end: @@ -396,7 +400,6 @@ j_connection_init_client(struct JConfiguration* configuration, enum JBackendType ret = TRUE; end: - free(jf_addr.addr); return ret; } From 99d136cffdc78517aef4e94d7b8e3261a635dc42 Mon Sep 17 00:00:00 2001 From: Julian Benda Date: Thu, 28 Oct 2021 11:08:11 +0200 Subject: [PATCH 53/60] Fix test workflow results --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8b6865c8a..030d73881 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -280,9 +280,9 @@ jobs: run: | . scripts/environment.sh ./scripts/setup.sh start - ./scripts/test.sh || journalctl GLIB_DOMAIN=JULEA + ./scripts/test.sh || (journalctl GLIB_DOMAIN=JULEA && $(exit 1)) sleep 10 - ./scripts/test.sh || journalctl GLIB_DOMAIN=JULEA + ./scripts/test.sh || (journalctl GLIB_DOMAIN=JULEA && $(exit 1)) ./scripts/setup.sh stop - name: HDF5 Tests if: matrix.julea.db != 'memory' From a15b56b0690171e8abf3b6ac56a2b79c7c86bfda Mon Sep 17 00:00:00 2001 From: Julian Benda Date: Thu, 28 Oct 2021 12:49:03 +0200 Subject: [PATCH 54/60] Run autoformatter --- lib/core/jmessage.c | 6 +++--- lib/core/jnetwork.c | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/core/jmessage.c b/lib/core/jmessage.c index e9a32a1c0..b0906ea08 100644 --- a/lib/core/jmessage.c +++ b/lib/core/jmessage.c @@ -722,9 +722,9 @@ j_message_send(JMessage* message, struct JConnection* connection) total_data_length += sizeof(struct JConnectionMemoryID) + message_data->header_size + message_data->length; } j_list_iterator_free(iterator); - fits = - total_data_length - < j_configuration_get_message_inject_size(j_connection_get_configuration(connection)); + fits = + total_data_length + < j_configuration_get_message_inject_size(j_connection_get_configuration(connection)); } iterator = j_list_iterator_new(message->send_list); diff --git a/lib/core/jnetwork.c b/lib/core/jnetwork.c index 145201916..18a6eb4fc 100644 --- a/lib/core/jnetwork.c +++ b/lib/core/jnetwork.c @@ -266,7 +266,7 @@ j_fabric_fini(struct JFabric* this) res = fi_close(&this->fabric->fid); CHECK("failed to close fabric!"); this->fabric = NULL; - if(this->hints) + if (this->hints) { fi_freeinfo(this->hints); this->hints = NULL; From 919ff5c7f21c62f4de0fa74b35367fca521b9b15 Mon Sep 17 00:00:00 2001 From: Julian Benda Date: Sun, 31 Oct 2021 13:12:32 +0100 Subject: [PATCH 55/60] Rebase to master, set libfabric version to 1.6 --- .github/workflows/ci.yml | 6 +++++- include/core/jconfiguration.h | 3 --- lib/core/jconfiguration.c | 2 +- lib/object/jdistributed-object.c | 22 +++++++++++++++++++++- lib/object/jobject.c | 11 +++++++++++ 5 files changed, 38 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 030d73881..8d2e18e75 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -193,8 +193,12 @@ jobs: exclude: # GLib is too old for TAP on Ubuntu 18.04 - os: - dist: ubuntu-18.04 + dist: ubuntu-18.04 dependencies: system + # libfaibrc 1.11 is not available on Ubuntu 20.04 + # - os: + # dist: ubuntu-20.04 + # depednencies: system steps: - name: Checkout uses: actions/checkout@v2 diff --git a/include/core/jconfiguration.h b/include/core/jconfiguration.h index dbe9ce1bb..703b04cfb 100644 --- a/include/core/jconfiguration.h +++ b/include/core/jconfiguration.h @@ -116,9 +116,6 @@ guint64 j_configuration_get_message_inject_size(JConfiguration*); guint32 j_configuration_get_max_connections(JConfiguration*); guint64 j_configuration_get_stripe_size(JConfiguration*); -/// network port to communicate -guint16 j_configuration_get_port(JConfiguration*); - gint64 j_configuration_get_libfabric_version(JConfiguration*); struct fi_info* j_configuration_get_libfabric_hints(JConfiguration*); diff --git a/lib/core/jconfiguration.c b/lib/core/jconfiguration.c index 31c48666d..580431135 100644 --- a/lib/core/jconfiguration.c +++ b/lib/core/jconfiguration.c @@ -374,7 +374,7 @@ j_configuration_new_for_data(GKeyFile* key_file) configuration->max_connections = max_connections; configuration->stripe_size = stripe_size; configuration->ref_count = 1; - configuration->libfabric.version = FI_VERSION(1, 11); + configuration->libfabric.version = FI_VERSION(1, 6); configuration->libfabric.hints = fi_allocinfo(); configuration->libfabric.hints->caps = FI_MSG | FI_SEND | FI_RECV | FI_READ | FI_RMA | FI_REMOTE_READ; diff --git a/lib/object/jdistributed-object.c b/lib/object/jdistributed-object.c index 688252d11..30ea3e471 100644 --- a/lib/object/jdistributed-object.c +++ b/lib/object/jdistributed-object.c @@ -72,6 +72,7 @@ struct JDistributedObjectBackgroundData struct { JList* bytes_written; + guint64 total_bytes_written; } write; }; }; @@ -440,6 +441,7 @@ j_distributed_object_write_background_operation(gpointer data) guint64* bytes_written = j_list_iterator_get(it); nbytes = j_message_get_8(reply); + background_data->write.total_bytes_written += nbytes; j_helper_atomic_add(bytes_written, nbytes); } } @@ -983,6 +985,8 @@ j_distributed_object_write_exec(JList* operations, JSemantics* semantics) gsize name_len = 0; gsize namespace_len = 0; guint32 server_count = 0; + guint64 total_data_length = 0; + JSemanticsSafety safety; /// \todo //JLock* lock = NULL; @@ -990,6 +994,8 @@ j_distributed_object_write_exec(JList* operations, JSemantics* semantics) g_return_val_if_fail(operations != NULL, FALSE); g_return_val_if_fail(semantics != NULL, FALSE); + safety = j_semantics_get(semantics, J_SEMANTICS_SAFETY); + { JDistributedObjectOperation* operation = j_list_get_first(operations); g_assert(operation != NULL); @@ -1075,10 +1081,14 @@ j_distributed_object_write_exec(JList* operations, JSemantics* semantics) new_data += new_length; // Fake bytes_written here instead of doing another loop further down - if (j_semantics_get(semantics, J_SEMANTICS_SAFETY) == J_SEMANTICS_SAFETY_NONE) + if (safety == J_SEMANTICS_SAFETY_NONE) { j_helper_atomic_add(bytes_written, new_length); } + else + { + total_data_length += new_length; + } } } else @@ -1114,6 +1124,7 @@ j_distributed_object_write_exec(JList* operations, JSemantics* semantics) data->operations = NULL; data->semantics = semantics; data->write.bytes_written = bw_lists[i]; + data->write.total_bytes_written = 0; data->ret = TRUE; background_data[i] = data; @@ -1121,6 +1132,7 @@ j_distributed_object_write_exec(JList* operations, JSemantics* semantics) j_helper_execute_parallel(j_distributed_object_write_background_operation, background_data, server_count); + guint64 total_written= 0; for (guint i = 0; i < server_count; i++) { JDistributedObjectBackgroundData* data; @@ -1132,9 +1144,17 @@ j_distributed_object_write_exec(JList* operations, JSemantics* semantics) data = background_data[i]; ret = data->ret && ret; + total_written += data->write.total_bytes_written; g_slice_free(JDistributedObjectBackgroundData, data); } + if(safety == J_SEMANTICS_SAFETY_STORAGE || safety == J_SEMANTICS_SAFETY_NETWORK) + { + if(total_written != total_data_length) + { + ret = FALSE; + } + } } else { diff --git a/lib/object/jobject.c b/lib/object/jobject.c index b747ee900..e626090f2 100644 --- a/lib/object/jobject.c +++ b/lib/object/jobject.c @@ -607,6 +607,7 @@ j_object_write_exec(JList* operations, JSemantics* semantics) g_autoptr(JMessage) message = NULL; JObject* object; gpointer object_handle; + guint64 total_data_length = 0; /// \todo //JLock* lock = NULL; @@ -677,6 +678,10 @@ j_object_write_exec(JList* operations, JSemantics* semantics) { j_helper_atomic_add(bytes_written, length); } + else + { + total_data_length += length; + } } else { @@ -710,6 +715,7 @@ j_object_write_exec(JList* operations, JSemantics* semantics) if (j_message_get_count(reply) > 0) { + guint64 total_received_length = 0; it = j_list_iterator_new(operations); while (j_list_iterator_next(it)) @@ -718,10 +724,15 @@ j_object_write_exec(JList* operations, JSemantics* semantics) guint64* bytes_written = operation->write.bytes_written; nbytes = j_message_get_8(reply); + total_received_length += nbytes; j_helper_atomic_add(bytes_written, nbytes); } j_list_iterator_free(it); + + if(total_data_length != total_received_length) { + ret = FALSE; + } } else { From 770927e784718d681c7ec7036002f5999135bf92 Mon Sep 17 00:00:00 2001 From: Julian Benda Date: Sun, 31 Oct 2021 13:17:18 +0100 Subject: [PATCH 56/60] Fix ci --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8d2e18e75..fb25752e7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -193,8 +193,8 @@ jobs: exclude: # GLib is too old for TAP on Ubuntu 18.04 - os: - dist: ubuntu-18.04 - dependencies: system + dist: ubuntu-18.04 + dependencies: system # libfaibrc 1.11 is not available on Ubuntu 20.04 # - os: # dist: ubuntu-20.04 From cb7d9ad576a58dd3113385ba13c8358e66fc4e37 Mon Sep 17 00:00:00 2001 From: Julian Benda Date: Sun, 31 Oct 2021 17:33:34 +0100 Subject: [PATCH 57/60] Autoformat + disable system checks for Ubuntu20.04 --- .github/workflows/ci.yml | 10 ++++----- lib/core/jconfiguration.c | 2 +- lib/object/jdistributed-object.c | 6 +++--- lib/object/jobject.c | 3 ++- server/loop.c | 35 ++++++++++++++++++++------------ 5 files changed, 33 insertions(+), 23 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fb25752e7..57a75135e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -194,11 +194,11 @@ jobs: # GLib is too old for TAP on Ubuntu 18.04 - os: dist: ubuntu-18.04 - dependencies: system - # libfaibrc 1.11 is not available on Ubuntu 20.04 - # - os: - # dist: ubuntu-20.04 - # depednencies: system + dependencies: system + # libfaibrc don't work on Ubuntu 20.04 + - os: + dist: ubuntu-20.04 + depednencies: system steps: - name: Checkout uses: actions/checkout@v2 diff --git a/lib/core/jconfiguration.c b/lib/core/jconfiguration.c index 580431135..df6efff43 100644 --- a/lib/core/jconfiguration.c +++ b/lib/core/jconfiguration.c @@ -333,7 +333,7 @@ j_configuration_new_for_data(GKeyFile* key_file) || db_backend == NULL || db_component == NULL || db_path == NULL - || port >= 0xFFFF) + || port >= 0xFFFF) { g_free(db_backend); g_free(db_component); diff --git a/lib/object/jdistributed-object.c b/lib/object/jdistributed-object.c index 30ea3e471..5ce27fdc4 100644 --- a/lib/object/jdistributed-object.c +++ b/lib/object/jdistributed-object.c @@ -1132,7 +1132,7 @@ j_distributed_object_write_exec(JList* operations, JSemantics* semantics) j_helper_execute_parallel(j_distributed_object_write_background_operation, background_data, server_count); - guint64 total_written= 0; + guint64 total_written = 0; for (guint i = 0; i < server_count; i++) { JDistributedObjectBackgroundData* data; @@ -1148,9 +1148,9 @@ j_distributed_object_write_exec(JList* operations, JSemantics* semantics) g_slice_free(JDistributedObjectBackgroundData, data); } - if(safety == J_SEMANTICS_SAFETY_STORAGE || safety == J_SEMANTICS_SAFETY_NETWORK) + if (safety == J_SEMANTICS_SAFETY_STORAGE || safety == J_SEMANTICS_SAFETY_NETWORK) { - if(total_written != total_data_length) + if (total_written != total_data_length) { ret = FALSE; } diff --git a/lib/object/jobject.c b/lib/object/jobject.c index e626090f2..95fe15d23 100644 --- a/lib/object/jobject.c +++ b/lib/object/jobject.c @@ -730,7 +730,8 @@ j_object_write_exec(JList* operations, JSemantics* semantics) j_list_iterator_free(it); - if(total_data_length != total_received_length) { + if (total_data_length != total_received_length) + { ret = FALSE; } } diff --git a/server/loop.c b/server/loop.c index 6d54374f8..0a2556d01 100644 --- a/server/loop.c +++ b/server/loop.c @@ -300,22 +300,27 @@ jd_handle_message(JMessage* message, struct JConnection* connection, JMemoryChun if (G_LIKELY(memoryID->size <= memory_chunk_size) && G_LIKELY(ret)) { - if (memoryID->key == 0 && memoryID->offset == 0) { + if (memoryID->key == 0 && memoryID->offset == 0) + { writeEntries[i] = (struct ObjectWriteEntry){ - .size = memoryID->size, - .offset = offset, - .data = j_message_get_n(message, memoryID->size)}; - } else { + .size = memoryID->size, + .offset = offset, + .data = j_message_get_n(message, memoryID->size) + }; + } + else + { buf = j_memory_chunk_get(memory_chunk, memoryID->size); if (buf == NULL) { j_connection_wait_for_completion(connection); ///< \todo paralleilize more - for(; entryItr < i; ++entryItr) { + for (; entryItr < i; ++entryItr) + { struct ObjectWriteEntry* entry = writeEntries + entryItr; j_statistics_add(statistics, J_STATISTICS_BYTES_RECEIVED, entry->size); j_backend_object_write(jd_object_backend, object, entry->data, entry->size, entry->offset, &bytes_written); j_statistics_add(statistics, J_STATISTICS_BYTES_WRITTEN, bytes_written); - if(reply != NULL) + if (reply != NULL) { j_message_add_operation(reply, sizeof(guint64)); j_message_append_8(reply, &bytes_written); @@ -330,11 +335,14 @@ jd_handle_message(JMessage* message, struct JConnection* connection, JMemoryChun j_connection_rma_read(connection, memoryID, buf); writeEntries[i] = (struct ObjectWriteEntry){ - .size = memoryID->size, - .offset = offset, - .data = buf}; + .size = memoryID->size, + .offset = offset, + .data = buf + }; } - } else if (reply != NULL) { + } + else if (reply != NULL) + { /// @todo return write error bytes_written = 0; j_message_add_operation(reply, sizeof(guint64)); @@ -343,9 +351,10 @@ jd_handle_message(JMessage* message, struct JConnection* connection, JMemoryChun } j_connection_wait_for_completion(connection); - if(G_LIKELY(ret)) + if (G_LIKELY(ret)) { - for(; entryItr < operation_count; ++entryItr) { + for (; entryItr < operation_count; ++entryItr) + { guint64 bytes_written = 0; struct ObjectWriteEntry* entry = writeEntries + entryItr; j_statistics_add(statistics, J_STATISTICS_BYTES_RECEIVED, entry->size); From 830a888e70c5e756b322b2c9ec95123180e53458 Mon Sep 17 00:00:00 2001 From: Julian Benda Date: Sun, 31 Oct 2021 17:35:07 +0100 Subject: [PATCH 58/60] Fix spelling mistake --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 57a75135e..bef836529 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -198,7 +198,7 @@ jobs: # libfaibrc don't work on Ubuntu 20.04 - os: dist: ubuntu-20.04 - depednencies: system + dependencies: system steps: - name: Checkout uses: actions/checkout@v2 From 82ea7f3673a4b3466ca4d9325b5f91337221ea84 Mon Sep 17 00:00:00 2001 From: Julian Benda Date: Tue, 2 Nov 2021 09:02:52 +0100 Subject: [PATCH 59/60] Set Libfabric version back to 11 --- lib/core/jconfiguration.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/core/jconfiguration.c b/lib/core/jconfiguration.c index df6efff43..38fa38695 100644 --- a/lib/core/jconfiguration.c +++ b/lib/core/jconfiguration.c @@ -374,7 +374,7 @@ j_configuration_new_for_data(GKeyFile* key_file) configuration->max_connections = max_connections; configuration->stripe_size = stripe_size; configuration->ref_count = 1; - configuration->libfabric.version = FI_VERSION(1, 6); + configuration->libfabric.version = FI_VERSION(1, 11); configuration->libfabric.hints = fi_allocinfo(); configuration->libfabric.hints->caps = FI_MSG | FI_SEND | FI_RECV | FI_READ | FI_RMA | FI_REMOTE_READ; From 9c036dd07a15401ba8c8b6dad61b2ca8173d48ac Mon Sep 17 00:00:00 2001 From: JBenda Date: Tue, 2 Nov 2021 09:06:24 +0100 Subject: [PATCH 60/60] Removed wrong flag from julea-config example --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index cf895f6c5..24a5574a1 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,6 @@ $ julea-config --user \ --object-backend=posix --object-component=server --object-path="/tmp/julea-$(id -u)/posix" \ --kv-backend=lmdb --kv-component=server --kv-path="/tmp/julea-$(id -u)/lmdb" \ --db-backend=sqlite --db-component=server --db-path="/tmp/julea-$(id -u)/sqlite" - --user-id="$(id --user)" ``` You can check whether JULEA works by executing the integrated test suite.