From 94e08da89962c86e3665604c78c87e9831af4e88 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Wed, 26 Jun 2024 08:49:48 +0000 Subject: [PATCH 1/8] delete --- src_py/apiServer/statsTiles.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 src_py/apiServer/statsTiles.py diff --git a/src_py/apiServer/statsTiles.py b/src_py/apiServer/statsTiles.py new file mode 100644 index 00000000..6067d3e1 --- /dev/null +++ b/src_py/apiServer/statsTiles.py @@ -0,0 +1,29 @@ +from IPython.display import display +import matplotlib.pyplot as plt +from datetime import datetime +from pathlib import Path +from experiment_phase import * +import globalVars as globe +from definitions import * +import pandas as pd +import numpy as np +import seaborn as sns +sns.set_theme() + +class StatsTiles(Stats): + + def __init__(self): + pass + + def get_confusion_matrices(self , normalize : bool = False ,plot : bool = False , saveToFile : bool = False): + assert self.experiment_flow_type == "classification", "This function is only available for classification experiments" + assert self.phase == PHASE_PREDICTION_STR, "This function is only available for predict phase" + sources_pieces_list = self.experiment_phase.get_sources_pieces() + workers_model_db_list = self.nerl_model_db.get_workers_model_db_list() + confusion_matrix_source_dict = {} + confusion_matrix_worker_dict = {} + + + + return confusion_matrix_source_dict, confusion_matrix_worker_dict + \ No newline at end of file From 388b4dee3b83a26b10ecc4480355ee5c0addafb8 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Tue, 2 Jul 2024 21:16:08 +0000 Subject: [PATCH 2/8] check stats tiles --- .../dc_test_synt_1d_2c_1s_4r_4w.json | 2 +- src_py/apiServer/experiment_flow.py | 7 ++--- src_py/apiServer/statsTiles.py | 26 ++++++++++++++++++- 3 files changed, 30 insertions(+), 5 deletions(-) diff --git a/inputJsonsFiles/DistributedConfig/dc_test_synt_1d_2c_1s_4r_4w.json b/inputJsonsFiles/DistributedConfig/dc_test_synt_1d_2c_1s_4r_4w.json index 01e46ffc..3fdb5c98 100644 --- a/inputJsonsFiles/DistributedConfig/dc_test_synt_1d_2c_1s_4r_4w.json +++ b/inputJsonsFiles/DistributedConfig/dc_test_synt_1d_2c_1s_4r_4w.json @@ -14,7 +14,7 @@ "devices": [ { "name": "pc1", - "ipv4": "10.211.55.3", + "ipv4": "10.0.0.16", "entities": "c1,c2,r2,r1,r3,r4,s1,apiServer,mainServer" } ], diff --git a/src_py/apiServer/experiment_flow.py b/src_py/apiServer/experiment_flow.py index fd54890f..579c854b 100644 --- a/src_py/apiServer/experiment_flow.py +++ b/src_py/apiServer/experiment_flow.py @@ -9,6 +9,7 @@ from events_sync import * from networkComponents import * from stats import * +from statsTiles import * from experiment_phase import * # Todo check imports and remove unused ones @@ -57,12 +58,12 @@ def get_events_sync(self): def get_csv_dataset(self): return self.csv_dataset - def generate_stats(self, experiment_phase = None) -> Stats: + def generate_stats(self, experiment_phase = None) -> StatsTiles: if experiment_phase is None: experiment_phase = self.get_current_experiment_phase() - return Stats(experiment_phase) + return StatsTiles(experiment_phase) - def merge_stats(self, stats_list: list) -> Stats: + def merge_stats(self, stats_list: list) -> StatsTiles: pass def parse_experiment_flow_json(self, json_path : str, override_csv_path = ""): diff --git a/src_py/apiServer/statsTiles.py b/src_py/apiServer/statsTiles.py index 6067d3e1..eb25966d 100644 --- a/src_py/apiServer/statsTiles.py +++ b/src_py/apiServer/statsTiles.py @@ -8,6 +8,8 @@ import pandas as pd import numpy as np import seaborn as sns +from stats import * + sns.set_theme() class StatsTiles(Stats): @@ -22,7 +24,29 @@ def get_confusion_matrices(self , normalize : bool = False ,plot : bool = False workers_model_db_list = self.nerl_model_db.get_workers_model_db_list() confusion_matrix_source_dict = {} confusion_matrix_worker_dict = {} - + for source_piece_inst in sources_pieces_list: + sourcePiece_csv_labels_path = source_piece_inst.get_pointer_to_sourcePiece_CsvDataSet_labels() + df_actual_labels = pd.read_csv(sourcePiece_csv_labels_path) + print(df_actual_labels,"df_actual_labels read_csv") + num_of_labels = df_actual_labels.shape[1] + print(num_of_labels,"num_of_labels") + header_list = range(num_of_labels) + df_actual_labels.columns = header_list + df_actual_labels = self.expend_labels_df(df_actual_labels) + print(df_actual_labels,"df_actual_labels") + #print(df_actual_labels) + source_name = source_piece_inst.get_source_name() + print(source_name,"source_name") + # build confusion matrix for each worker + target_workers = source_piece_inst.get_target_workers() + print(target_workers,"target_workers") + batch_size = source_piece_inst.get_batch_size() + print(batch_size,"batch_size") + for worker_db in workers_model_db_list: + worker_name = worker_db.get_worker_name() + print(worker_name,"worker_name") + total_batches_per_source = worker_db.get_total_batches_per_source(source_name) + print(total_batches_per_source,"total_batches_per_source") return confusion_matrix_source_dict, confusion_matrix_worker_dict From c108521f7ff106098eef28f6cd956ddb49f83a64 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Sun, 7 Jul 2024 12:41:16 +0000 Subject: [PATCH 3/8] change ip --- .../DistributedConfig/dc_test_synt_1d_2c_1s_4r_4w.json | 2 +- src_py/apiServer/statsTiles.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/inputJsonsFiles/DistributedConfig/dc_test_synt_1d_2c_1s_4r_4w.json b/inputJsonsFiles/DistributedConfig/dc_test_synt_1d_2c_1s_4r_4w.json index 3fdb5c98..0d2c3168 100644 --- a/inputJsonsFiles/DistributedConfig/dc_test_synt_1d_2c_1s_4r_4w.json +++ b/inputJsonsFiles/DistributedConfig/dc_test_synt_1d_2c_1s_4r_4w.json @@ -14,7 +14,7 @@ "devices": [ { "name": "pc1", - "ipv4": "10.0.0.16", + "ipv4": "10.0.0.6", "entities": "c1,c2,r2,r1,r3,r4,s1,apiServer,mainServer" } ], diff --git a/src_py/apiServer/statsTiles.py b/src_py/apiServer/statsTiles.py index eb25966d..324559c3 100644 --- a/src_py/apiServer/statsTiles.py +++ b/src_py/apiServer/statsTiles.py @@ -14,7 +14,7 @@ class StatsTiles(Stats): - def __init__(self): + def __init__(self, experiment_phase: ExperimentPhase): pass def get_confusion_matrices(self , normalize : bool = False ,plot : bool = False , saveToFile : bool = False): From f7aaa809799cede14a77a1fcc947b46d3e2b2f4f Mon Sep 17 00:00:00 2001 From: Orisadek Date: Fri, 12 Jul 2024 13:03:03 +0000 Subject: [PATCH 4/8] add new dc --- .../DistributedConfig/dc_5d_4c_4s_4r_12w.json | 383 ++++++++++++++++++ src_py/apiServer/statsTiles.py | 126 +++++- 2 files changed, 497 insertions(+), 12 deletions(-) create mode 100644 inputJsonsFiles/DistributedConfig/dc_5d_4c_4s_4r_12w.json diff --git a/inputJsonsFiles/DistributedConfig/dc_5d_4c_4s_4r_12w.json b/inputJsonsFiles/DistributedConfig/dc_5d_4c_4s_4r_12w.json new file mode 100644 index 00000000..22f6e91b --- /dev/null +++ b/inputJsonsFiles/DistributedConfig/dc_5d_4c_4s_4r_12w.json @@ -0,0 +1,383 @@ +{ + + "nerlnetSettings": { + + "frequency": "50", + + "batchSize": "30" + + }, + + "mainServer": { + + "port": "8080", + + "args": "" + + }, + + "apiServer": { + "port": "8081", + "args": "" + }, + "devices": [ + { + "name": "vm1", + "ipv4": "10.0.0.1", + "entities": "mainServer,apiServer" + }, + { + "name": "vm2", + "ipv4": "10.0.0.2", + "entities": "c1,r1,s1" + + }, + + { + "name": "vm4", + "ipv4": "10.0.0.4", + "entities": "c3,r3,s3" + + }, + + { + "name": "vm5", + "ipv4": "10.0.0.5", + "entities": "c4,r4,s4" + + }, + + { + + "name": "vm3", + "ipv4": "10.0.0.3", + "entities": "c2,r2,s2" + + } + + ], + + "routers": [ + + { + "name": "r1", + "port": "8090", + "policy": "0" + + }, + + { + + "name": "r2", + + "port": "8091", + + "policy": "0" + + }, + + { + + "name": "r3", + + "port": "8092", + + "policy": "0" + + }, + + { + + "name": "r4", + + "port": "8093", + + "policy": "0" + + } + + ], + + "sources": [ + + { + + "name": "s1", + + "port": "8086", + + "frequency": "50", + + "policy": "0", + + "epochs": "1", + + "type": "0" + + }, + + { + + "name": "s2", + + "port": "8087", + + "frequency": "50", + + "policy": "0", + + "epochs": "1", + + "type": "0" + + }, + + { + + "name": "s3", + + "port": "8088", + + "frequency": "50", + + "policy": "0", + + "epochs": "1", + + "type": "0" + + }, + + { + + "name": "s4", + + "port": "8089", + + "frequency": "50", + + "policy": "0", + + "epochs": "1", + + "type": "0" + + } + + ], + + "clients": [ + + { + + "name": "c2", + + "port": "8083", + + "workers": "w2_2,w3_2,w1_2" + + }, + + { + + "name": "c3", + + "port": "8084", + + "workers": "w1_3,w2_3,w3_3" + + }, + + { + + "name": "c4", + + "port": "8085", + + "workers": "w1_4,w2_4,w3_4" + + }, + + { + + "name": "c1", + + "port": "8082", + + "workers": "w1_1,w2_1,w3_1" + + } + + ], + + "workers": [ + + { + + "name": "w2_2", + + "model_sha": "42c55325abe802a45af6cc46b9f1a26d3693bef8efcebde63395dbc3a2c7d2ae" + + }, + + { + + "name": "w3_2", + + "model_sha": "42c55325abe802a45af6cc46b9f1a26d3693bef8efcebde63395dbc3a2c7d2ae" + + }, + + { + + "name": "w1_2", + + "model_sha": "42c55325abe802a45af6cc46b9f1a26d3693bef8efcebde63395dbc3a2c7d2ae" + + }, + + { + + "name": "w1_3", + + "model_sha": "42c55325abe802a45af6cc46b9f1a26d3693bef8efcebde63395dbc3a2c7d2ae" + + }, + + { + + "name": "w2_3", + + "model_sha": "42c55325abe802a45af6cc46b9f1a26d3693bef8efcebde63395dbc3a2c7d2ae" + + }, + + { + + "name": "w3_3", + + "model_sha": "42c55325abe802a45af6cc46b9f1a26d3693bef8efcebde63395dbc3a2c7d2ae" + + }, + + { + + "name": "w1_4", + + "model_sha": "42c55325abe802a45af6cc46b9f1a26d3693bef8efcebde63395dbc3a2c7d2ae" + + }, + + { + + "name": "w2_4", + + "model_sha": "42c55325abe802a45af6cc46b9f1a26d3693bef8efcebde63395dbc3a2c7d2ae" + + }, + + { + + "name": "w3_4", + + "model_sha": "42c55325abe802a45af6cc46b9f1a26d3693bef8efcebde63395dbc3a2c7d2ae" + + }, + + { + + "name": "w1_1", + + "model_sha": "42c55325abe802a45af6cc46b9f1a26d3693bef8efcebde63395dbc3a2c7d2ae" + + }, + + { + + "name": "w2_1", + + "model_sha": "42c55325abe802a45af6cc46b9f1a26d3693bef8efcebde63395dbc3a2c7d2ae" + + }, + + { + + "name": "w3_1", + + "model_sha": "42c55325abe802a45af6cc46b9f1a26d3693bef8efcebde63395dbc3a2c7d2ae" + + } + + ], + + "model_sha": { + + "42c55325abe802a45af6cc46b9f1a26d3693bef8efcebde63395dbc3a2c7d2ae": { + + "modelType": "0", + + "_doc_modelType": " nn:0 | approximation:1 | classification:2 | forecasting:3 | image_classification:4 | text_classification:5 | text_generation:6 | auto_association:7 | autoencoder:8 | ae_classifier:9 |", + + "modelArgs": "", + + "_doc_modelArgs": "Extra arguments to model", + + "layersSizes": "9x28x1k5x5x1x6p0s1t1,9x28x6k2x2p0s2,5x14x6k4x4x6x12p0s1t0,1,32,10", + + "_doc_layersSizes": "List of postive integers [L0, L1, ..., LN]", + + "layerTypesList": "2,4,2,10,3,5", + + "_doc_LayerTypes": " Default:0 | Scaling:1 | Conv:2 | Perceptron:3 | Pooling:4 | Probabilistic:5 | LSTM:6 | Reccurrent:7 | Unscaling:8 | Flatten:9 | Bounding:10 |", + + "layers_functions": "6,2,6,1,6,4", + + "_doc_layers_functions_activation": " Threshold:1 | Sign:2 | Logistic:3 | Tanh:4 | Linear:5 | ReLU:6 | eLU:7 | SeLU:8 | Soft-plus:9 | Soft-sign:10 | Hard-sigmoid:11 |", + + "_doc_layer_functions_pooling": " none:1 | Max:2 | Avg:3 |", + + "_doc_layer_functions_probabilistic": " Binary:1 | Logistic:2 | Competitive:3 | Softmax:4 |", + + "_doc_layer_functions_scaler": " none:1 | MinMax:2 | MeanStd:3 | STD:4 | Log:5 |", + + "lossMethod": "6", + + "_doc_lossMethod": " SSE:1 | MSE:2 | NSE:3 | MinkowskiE:4 | WSE:5 | CEE:6 |", + + "lr": "0.01", + + "_doc_lr": "Positve float", + + "epochs": "1", + + "_doc_epochs": "Positve Integer", + + "optimizer": "5", + + "_doc_optimizer": " GD:0 | CGD:1 | SGD:2 | QuasiNeuton:3 | LVM:4 | ADAM:5 |", + + "optimizerArgs": "none", + + "_doc_optimizerArgs": "String", + + "infraType": "0", + + "_doc_infraType": " opennn:0 | wolfengine:1 |", + + "distributedSystemType": "0", + + "_doc_distributedSystemType": " none:0 | FedClientAvg:1 | FedServerAvg:2 | FedClientWeightedAvgClassification:3 | FedServerWeightedAvgClassification:4 | FedClientAE:5 | FedServerAE:6 |", + + "distributedSystemArgs": "none", + + "_doc_distributedSystemArgs": "String", + + "distributedSystemToken": "none", + + "_doc_distributedSystemToken": "Token that associates distributed group of workers and parameter-server" + + } + + } + +} \ No newline at end of file diff --git a/src_py/apiServer/statsTiles.py b/src_py/apiServer/statsTiles.py index 324559c3..7132bdb1 100644 --- a/src_py/apiServer/statsTiles.py +++ b/src_py/apiServer/statsTiles.py @@ -15,7 +15,8 @@ class StatsTiles(Stats): def __init__(self, experiment_phase: ExperimentPhase): - pass + super().__init__(experiment_phase) + def get_confusion_matrices(self , normalize : bool = False ,plot : bool = False , saveToFile : bool = False): assert self.experiment_flow_type == "classification", "This function is only available for classification experiments" @@ -24,30 +25,131 @@ def get_confusion_matrices(self , normalize : bool = False ,plot : bool = False workers_model_db_list = self.nerl_model_db.get_workers_model_db_list() confusion_matrix_source_dict = {} confusion_matrix_worker_dict = {} + distributed_tokens_dict = {} + worker_name_labels_dict={} for source_piece_inst in sources_pieces_list: sourcePiece_csv_labels_path = source_piece_inst.get_pointer_to_sourcePiece_CsvDataSet_labels() df_actual_labels = pd.read_csv(sourcePiece_csv_labels_path) - print(df_actual_labels,"df_actual_labels read_csv") num_of_labels = df_actual_labels.shape[1] - print(num_of_labels,"num_of_labels") header_list = range(num_of_labels) df_actual_labels.columns = header_list df_actual_labels = self.expend_labels_df(df_actual_labels) - print(df_actual_labels,"df_actual_labels") - #print(df_actual_labels) source_name = source_piece_inst.get_source_name() - print(source_name,"source_name") - # build confusion matrix for each worker target_workers = source_piece_inst.get_target_workers() - print(target_workers,"target_workers") + worker_missed_batches = {} batch_size = source_piece_inst.get_batch_size() - print(batch_size,"batch_size") for worker_db in workers_model_db_list: worker_name = worker_db.get_worker_name() - print(worker_name,"worker_name") + if worker_name not in target_workers: + continue + df_worker_labels = df_actual_labels.copy() total_batches_per_source = worker_db.get_total_batches_per_source(source_name) - print(total_batches_per_source,"total_batches_per_source") + print("total_batches_per_source: ", total_batches_per_source) + for batch_id in range(total_batches_per_source): + batch_db = worker_db.get_batch(source_name, str(batch_id)) + if not batch_db: # if batch is missing + if not self.missed_batches_warning_msg: + LOG_WARNING(f"missed batches") + self.missed_batches_warning_msg = True + starting_offset = source_piece_inst.get_starting_offset() + df_worker_labels.iloc[batch_id * batch_size: (batch_id + 1) * batch_size, num_of_labels:] = None # set the actual label to None for the predict labels in the df + worker_missed_batches[(worker_name, source_name, str(batch_id))] = (starting_offset + batch_id * batch_size, batch_size) # save the missing batch + + #df_worker_labels = df_worker_labels.dropna() + for batch_id in range(total_batches_per_source): + batch_db = worker_db.get_batch(source_name, str(batch_id)) + if batch_db: + distributed_token_db = batch_db.get_distributed_token() + cycle = int(batch_db.get_batch_id()) + tensor_data = batch_db.get_tensor_data() + tensor_data = tensor_data.reshape(batch_size, num_of_labels) + start_index = cycle * batch_size + end_index = (cycle + 1) * batch_size + df_worker_labels.iloc[start_index:end_index, num_of_labels:] = None # Fix an issue of pandas of incompatible dtype + df_worker_labels.iloc[start_index:end_index, num_of_labels:] = tensor_data + try: + distributed_tokens_dict[distributed_token_db][worker_name].append(batch_id) + except: + distributed_tokens_dict[distributed_token_db] = {worker_name : [batch_id]} + print("df_worker_labels: ", df_worker_labels) + else: + cycle = int(batch_id) + start_index = cycle * batch_size + end_index = (cycle + 1) * batch_size + df_worker_labels.iloc[start_index:end_index, num_of_labels:] = 0 + # Take 2 list from the df, one for the actual labels and one for the predict labels to build the confusion matrix + max_column_predict_index = df_worker_labels.iloc[:, num_of_labels:].idxmax(axis=1) + max_column_predict_index = max_column_predict_index.tolist() + max_column_predict_index = [int(predict_index) - num_of_labels for predict_index in max_column_predict_index] # fix the index to original labels index + max_column_labels_index = df_worker_labels.iloc[:, :num_of_labels].idxmax(axis=1) + max_column_labels_index = max_column_labels_index.tolist() + worker_name_labels_dict[worker_name] = {"labels": [max_column_labels_index,max_column_predict_index]} + + + for distributed_token_key in distributed_tokens_dict.keys(): + workers = distributed_tokens_dict[distributed_token_key].keys() + distributed_token_arr = [] + batch_id_arr = [] + for worker in workers: + labels = worker_name_labels_dict[worker]["labels"] + distributed_token_arr.append(labels) + batch_id_arr.append(distributed_tokens_dict[distributed_token_key][worker].sort(key=lambda x: x[0])) + + batch_id_dict = {} + for i in range(len(batch_id_arr)): + for batch_id in batch_id_arr[i]: + try: + batch_id_dict[batch_id].append(i) + except: + batch_id_dict[batch_id] = [i] + + for class_index, class_name in enumerate(self.headers_list): + class_actual_list_full = [] + class_actual_predict_full = [] + for batch_id_val,worker_num_list in enumerate(batch_id_dict): + class_actual_list = [] + class_predict_list = [] + workers_list = [workers[i] for i in worker_num_list] + labels_list = [distributed_token_arr[i] for i in worker_num_list] + for worker_id,worker in enumerate(workers_list): + cycle = int(batch_id_val) + start_index = cycle * batch_size + end_index = (cycle + 1) * batch_size + class_actual_list = [1 if label_num == class_index else 0 for label_num in labels_list[worker_id][0][start_index:end_index]] + try: + class_predict_list += [1 if label_num == class_index else 0 for label_num in labels_list[worker_id][1][start_index:end_index]] + except: + class_predict_list = [1 if label_num == class_index else 0 for label_num in labels_list[worker_id][1][start_index:end_index]] + + class_predict_list = [1 if (i>=len(workers_list)//2 or i == len(workers_list)) else 0 for i in class_predict_list] + class_actual_list_full.extend(class_actual_list) + class_actual_predict_full.extend(class_predict_list) + confusion_matrix = metrics.confusion_matrix(class_actual_list_full, class_actual_predict_full) + try: + confusion_matrix_worker_dict[(distributed_token_key, class_name)] += confusion_matrix + except: + confusion_matrix_worker_dict[(distributed_token_key, class_name)] = confusion_matrix + + + if plot: + workers = sorted(list({tup[0] for tup in confusion_matrix_worker_dict.keys()})) + classes = sorted(list({tup[1] for tup in confusion_matrix_worker_dict.keys()})) + fig, ax = plt.subplots(nrows=len(workers), ncols=len(classes),figsize=(4*len(classes),4*len(workers)),dpi=140) + for i , worker in enumerate(workers): + for j , pred_class in enumerate(classes): + conf_mat = confusion_matrix_worker_dict[(worker , pred_class)] + heatmap = sns.heatmap(data=conf_mat ,ax=ax[i,j], annot=True , fmt="d", cmap='Blues',annot_kws={"size": 8}, cbar_kws={'pad': 0.1}) + cbar = heatmap.collections[0].colorbar + cbar.ax.tick_params(labelsize = 8) + ax[i, j].set_title(f"{worker} , Class '{pred_class}'" , fontsize=12) + ax[i, j].tick_params(axis='both', which='major', labelsize=8) + ax[i, j].set_xlabel("Predicted Label" , fontsize=8) + ax[i, j].set_ylabel("True Label" , fontsize=8) + ax[i, j].set_aspect('equal') + fig.subplots_adjust(wspace=0.4 , hspace=0.4) + plt.show() + return confusion_matrix_source_dict, confusion_matrix_worker_dict - \ No newline at end of file + \ No newline at end of file From 5527a8d4b611972cb8209206cabd9a102af905a8 Mon Sep 17 00:00:00 2001 From: Orisadek Date: Thu, 18 Jul 2024 09:18:11 +0000 Subject: [PATCH 5/8] add new dc and exp files --- .../ConnectionMap/conn_4Router12Client4S.json | 9 + .../ConnectionMap/conn_4Router4Client4S.json | 9 + .../dc_12w_12c_13d_4s_4r_tiles_rr.json | 383 ++++++++++++++++++ .../DistributedConfig/dc_5d_4c_4s_4r_12w.json | 341 ++++++---------- .../dc_mnist_4w_5d_4r_4s_rr.json | 172 ++++++++ .../exp_mnist_5d_4c_4s_4w_4r.json | 84 ++++ .../exp_tiles_13d_12c_4s_4r_12w_rr.json | 70 ++++ .../exp_tiles_4d_4c_4s_4r_12w.json | 70 ++++ src_py/apiServer/experiment_flow.py | 9 +- src_py/apiServer/hf_repo_ids.json | 16 +- src_py/apiServer/stats.py | 87 +++- src_py/apiServer/statsTiles.py | 178 +++++--- 12 files changed, 1155 insertions(+), 273 deletions(-) create mode 100644 inputJsonsFiles/ConnectionMap/conn_4Router12Client4S.json create mode 100644 inputJsonsFiles/ConnectionMap/conn_4Router4Client4S.json create mode 100644 inputJsonsFiles/DistributedConfig/dc_12w_12c_13d_4s_4r_tiles_rr.json create mode 100644 inputJsonsFiles/DistributedConfig/dc_mnist_4w_5d_4r_4s_rr.json create mode 100644 inputJsonsFiles/experimentsFlow/exp_mnist_5d_4c_4s_4w_4r.json create mode 100644 inputJsonsFiles/experimentsFlow/exp_tiles_13d_12c_4s_4r_12w_rr.json create mode 100644 inputJsonsFiles/experimentsFlow/exp_tiles_4d_4c_4s_4r_12w.json diff --git a/inputJsonsFiles/ConnectionMap/conn_4Router12Client4S.json b/inputJsonsFiles/ConnectionMap/conn_4Router12Client4S.json new file mode 100644 index 00000000..33d444aa --- /dev/null +++ b/inputJsonsFiles/ConnectionMap/conn_4Router12Client4S.json @@ -0,0 +1,9 @@ +{ + "connectionsMap": + { + "r1":["apiServer","mainServer", "c1", "s1", "r2","c2","c3"], + "r2":["c4", "s2","r3","c7","c12"], + "r3":["c5","s3" ,"r4","c8","c11"], + "r4":["c6","s4", "r1","c9","c10"] + } +} diff --git a/inputJsonsFiles/ConnectionMap/conn_4Router4Client4S.json b/inputJsonsFiles/ConnectionMap/conn_4Router4Client4S.json new file mode 100644 index 00000000..bb82a9da --- /dev/null +++ b/inputJsonsFiles/ConnectionMap/conn_4Router4Client4S.json @@ -0,0 +1,9 @@ +{ + "connectionsMap": + { + "r1":["apiServer","mainServer", "c1", "s1", "r2"], + "r2":["c2", "s2","r3"], + "r3":["c3","s3" ,"r4"], + "r4":["c4","s4", "r1"] + } +} diff --git a/inputJsonsFiles/DistributedConfig/dc_12w_12c_13d_4s_4r_tiles_rr.json b/inputJsonsFiles/DistributedConfig/dc_12w_12c_13d_4s_4r_tiles_rr.json new file mode 100644 index 00000000..50d5e282 --- /dev/null +++ b/inputJsonsFiles/DistributedConfig/dc_12w_12c_13d_4s_4r_tiles_rr.json @@ -0,0 +1,383 @@ +{ + "nerlnetSettings": { + "frequency": "100", + "batchSize": "20" + }, + "mainServer": { + "port": "8080", + "args": "" + }, + "apiServer": { + "port": "8081", + "args": "" + }, + "devices": [ + { + "name": "vm1", + "ipv4": "10.0.0.6", + "entities": "mainServer,apiServer" + }, + { + "name": "vm2", + "ipv4": "10.0.0.44", + "entities": "c1,r1" + }, + { + "name": "vm4", + "ipv4": "10.0.0.43", + "entities": "c3,s1" + }, + { + "name": "vm5", + "ipv4": "10.0.0.41", + "entities": "c4,s2" + }, + { + "name": "vm3", + "ipv4": "10.0.0.42", + "entities": "c2,r4" + }, + { + "name": "vm6", + "ipv4": "10.0.0.40", + "entities": "c5,s3" + }, + { + "name": "vm7", + "ipv4": "10.0.0.45", + "entities": "c6,r2" + }, + { + "name": "vm8", + "ipv4": "10.0.0.37", + "entities": "c7" + }, + { + "name": "vm9", + "ipv4": "10.0.0.46", + "entities": "c8,r3" + }, + { + "name": "vm10", + "ipv4": "10.0.0.39", + "entities": "c9" + }, + { + "name": "vm11", + "ipv4": "10.0.0.38", + "entities": "c10" + }, + { + "name": "vm12", + "ipv4": "10.0.0.46", + "entities": "c11,s4" + }, + { + "name": "vm13", + "ipv4": "10.0.0.16", + "entities": "c12" + } + ], + "routers": [ + { + "name": "r1", + "port": "8098", + "policy": "0" + }, + { + "name": "r2", + "port": "8099", + "policy": "0" + }, + { + "name": "r3", + "port": "8100", + "policy": "0" + }, + { + "name": "r4", + "port": "8101", + "policy": "0" + } + ], + "sources": [ + { + "name": "s1", + "port": "8094", + "frequency": "50", + "policy": "1", + "epochs": "1", + "type": "0" + }, + { + "name": "s2", + "port": "8095", + "frequency": "50", + "policy": "1", + "epochs": "1", + "type": "0" + }, + { + "name": "s3", + "port": "8096", + "frequency": "50", + "policy": "1", + "epochs": "1", + "type": "0" + }, + { + "name": "s4", + "port": "8097", + "frequency": "50", + "policy": "1", + "epochs": "1", + "type": "0" + } + ], + "clients": [ + { + "name": "c2", + "port": "8083", + "workers": "w2_0" + }, + { + "name": "c3", + "port": "8084", + "workers": "w3_0" + }, + { + "name": "c4", + "port": "8085", + "workers": "w1_1" + }, + { + "name": "c1", + "port": "8082", + "workers": "w1_0" + }, + { + "name": "c5", + "port": "8086", + "workers": "w2_1" + }, + { + "name": "c6", + "port": "8087", + "workers": "w3_1" + }, + { + "name": "c7", + "port": "8088", + "workers": "w1_2" + }, + { + "name": "c8", + "port": "8089", + "workers": "w2_2" + }, + { + "name": "c9", + "port": "8090", + "workers": "w3_2" + }, + { + "name": "c10", + "port": "8091", + "workers": "w1_3" + }, + { + "name": "c11", + "port": "8092", + "workers": "w2_3" + }, + { + "name": "c12", + "port": "8093", + "workers": "w3_3" + } + ], + "workers": [ + { + "name": "w2_0", + "model_sha": "4bf6d1d5bc458db326e5cd17ccfeb4db3550373b8f5b2d260240ef4683184e38" + }, + { + "name": "w3_0", + "model_sha": "4bf6d1d5bc458db326e5cd17ccfeb4db3550373b8f5b2d260240ef4683184e38" + }, + { + "name": "w1_1", + "model_sha": "27623ebcd31c718209e874b6869ca62ec9842a5b05ee469cb12cc14a878d0c58" + }, + { + "name": "w1_0", + "model_sha": "4bf6d1d5bc458db326e5cd17ccfeb4db3550373b8f5b2d260240ef4683184e38" + }, + { + "name": "w2_1", + "model_sha": "27623ebcd31c718209e874b6869ca62ec9842a5b05ee469cb12cc14a878d0c58" + }, + { + "name": "w3_1", + "model_sha": "27623ebcd31c718209e874b6869ca62ec9842a5b05ee469cb12cc14a878d0c58" + }, + { + "name": "w1_2", + "model_sha": "8527b95bd75ec8d0eb95d06d5d1cdd1194d5706c5eada665c50240efac424ca4" + }, + { + "name": "w2_2", + "model_sha": "8527b95bd75ec8d0eb95d06d5d1cdd1194d5706c5eada665c50240efac424ca4" + }, + { + "name": "w3_2", + "model_sha": "8527b95bd75ec8d0eb95d06d5d1cdd1194d5706c5eada665c50240efac424ca4" + }, + { + "name": "w1_3", + "model_sha": "e09032ba977d35e5c39fb9c96dc4cfb3d73a8238b4e94635c031c00ec019a268" + }, + { + "name": "w2_3", + "model_sha": "e09032ba977d35e5c39fb9c96dc4cfb3d73a8238b4e94635c031c00ec019a268" + }, + { + "name": "w3_3", + "model_sha": "e09032ba977d35e5c39fb9c96dc4cfb3d73a8238b4e94635c031c00ec019a268" + } + ], + "model_sha": { + "4bf6d1d5bc458db326e5cd17ccfeb4db3550373b8f5b2d260240ef4683184e38": { + "modelType": "0", + "_doc_modelType": " nn:0 | approximation:1 | classification:2 | forecasting:3 | image_classification:4 | text_classification:5 | text_generation:6 | auto_association:7 | autoencoder:8 | ae_classifier:9 |", + "modelArgs": "", + "_doc_modelArgs": "Extra arguments to model", + "layersSizes": "9x28x1k3x3x1x6p0s1t1,9x28x6k2x2p0s2,4x14x6k3x3x6x16p0s1t0,1,120,84,10", + "_doc_layersSizes": "List of postive integers [L0, L1, ..., LN]", + "layerTypesList": "2,4,2,9,3,3,5", + "_doc_LayerTypes": " Default:0 | Scaling:1 | Conv:2 | Perceptron:3 | Pooling:4 | Probabilistic:5 | LSTM:6 | Reccurrent:7 | Unscaling:8 | Flatten:9 | Bounding:10 |", + "layers_functions": "6,2,6,1,6,6,4", + "_doc_layers_functions_activation": " Threshold:1 | Sign:2 | Logistic:3 | Tanh:4 | Linear:5 | ReLU:6 | eLU:7 | SeLU:8 | Soft-plus:9 | Soft-sign:10 | Hard-sigmoid:11 |", + "_doc_layer_functions_pooling": " none:1 | Max:2 | Avg:3 |", + "_doc_layer_functions_probabilistic": " Binary:1 | Logistic:2 | Competitive:3 | Softmax:4 |", + "_doc_layer_functions_scaler": " none:1 | MinMax:2 | MeanStd:3 | STD:4 | Log:5 |", + "lossMethod": "6", + "_doc_lossMethod": " SSE:1 | MSE:2 | NSE:3 | MinkowskiE:4 | WSE:5 | CEE:6 |", + "lr": "0.01", + "_doc_lr": "Positve float", + "epochs": "1", + "_doc_epochs": "Positve Integer", + "optimizer": "5", + "_doc_optimizer": " GD:0 | CGD:1 | SGD:2 | QuasiNeuton:3 | LVM:4 | ADAM:5 |", + "optimizerArgs": "none", + "_doc_optimizerArgs": "String", + "infraType": "0", + "_doc_infraType": " opennn:0 | wolfengine:1 |", + "distributedSystemType": "7", + "_doc_distributedSystemType": " none:0 | FedClientAvg:1 | FedServerAvg:2 | FedClientWeightedAvgClassification:3 | FedServerWeightedAvgClassification:4 | FedClientAE:5 | FedServerAE:6 | tiles:7 |", + "distributedSystemArgs": "none", + "_doc_distributedSystemArgs": "String", + "distributedSystemToken": "cl0", + "_doc_distributedSystemToken": "Token that associates distributed group of workers and parameter-server" + }, + "27623ebcd31c718209e874b6869ca62ec9842a5b05ee469cb12cc14a878d0c58": { + "modelType": "0", + "_doc_modelType": " nn:0 | approximation:1 | classification:2 | forecasting:3 | image_classification:4 | text_classification:5 | text_generation:6 | auto_association:7 | autoencoder:8 | ae_classifier:9 |", + "modelArgs": "", + "_doc_modelArgs": "Extra arguments to model", + "layersSizes": "9x28x1k3x3x1x6p0s1t1,9x28x6k2x2p0s2,4x14x6k3x3x6x16p0s1t0,1,120,84,10", + "_doc_layersSizes": "List of postive integers [L0, L1, ..., LN]", + "layerTypesList": "2,4,2,9,3,3,5", + "_doc_LayerTypes": " Default:0 | Scaling:1 | Conv:2 | Perceptron:3 | Pooling:4 | Probabilistic:5 | LSTM:6 | Reccurrent:7 | Unscaling:8 | Flatten:9 | Bounding:10 |", + "layers_functions": "6,2,6,1,6,6,4", + "_doc_layers_functions_activation": " Threshold:1 | Sign:2 | Logistic:3 | Tanh:4 | Linear:5 | ReLU:6 | eLU:7 | SeLU:8 | Soft-plus:9 | Soft-sign:10 | Hard-sigmoid:11 |", + "_doc_layer_functions_pooling": " none:1 | Max:2 | Avg:3 |", + "_doc_layer_functions_probabilistic": " Binary:1 | Logistic:2 | Competitive:3 | Softmax:4 |", + "_doc_layer_functions_scaler": " none:1 | MinMax:2 | MeanStd:3 | STD:4 | Log:5 |", + "lossMethod": "6", + "_doc_lossMethod": " SSE:1 | MSE:2 | NSE:3 | MinkowskiE:4 | WSE:5 | CEE:6 |", + "lr": "0.01", + "_doc_lr": "Positve float", + "epochs": "1", + "_doc_epochs": "Positve Integer", + "optimizer": "5", + "_doc_optimizer": " GD:0 | CGD:1 | SGD:2 | QuasiNeuton:3 | LVM:4 | ADAM:5 |", + "optimizerArgs": "none", + "_doc_optimizerArgs": "String", + "infraType": "0", + "_doc_infraType": " opennn:0 | wolfengine:1 |", + "distributedSystemType": "7", + "_doc_distributedSystemType": " none:0 | FedClientAvg:1 | FedServerAvg:2 | FedClientWeightedAvgClassification:3 | FedServerWeightedAvgClassification:4 | FedClientAE:5 | FedServerAE:6 | tiles:7 |", + "distributedSystemArgs": "none", + "_doc_distributedSystemArgs": "String", + "distributedSystemToken": "cl1", + "_doc_distributedSystemToken": "Token that associates distributed group of workers and parameter-server" + }, + "8527b95bd75ec8d0eb95d06d5d1cdd1194d5706c5eada665c50240efac424ca4": { + "modelType": "0", + "_doc_modelType": " nn:0 | approximation:1 | classification:2 | forecasting:3 | image_classification:4 | text_classification:5 | text_generation:6 | auto_association:7 | autoencoder:8 | ae_classifier:9 |", + "modelArgs": "", + "_doc_modelArgs": "Extra arguments to model", + "layersSizes": "9x28x1k3x3x1x6p0s1t1,9x28x6k2x2p0s2,4x14x6k3x3x6x16p0s1t0,1,120,84,10", + "_doc_layersSizes": "List of postive integers [L0, L1, ..., LN]", + "layerTypesList": "2,4,2,9,3,3,5", + "_doc_LayerTypes": " Default:0 | Scaling:1 | Conv:2 | Perceptron:3 | Pooling:4 | Probabilistic:5 | LSTM:6 | Reccurrent:7 | Unscaling:8 | Flatten:9 | Bounding:10 |", + "layers_functions": "6,2,6,1,6,6,4", + "_doc_layers_functions_activation": " Threshold:1 | Sign:2 | Logistic:3 | Tanh:4 | Linear:5 | ReLU:6 | eLU:7 | SeLU:8 | Soft-plus:9 | Soft-sign:10 | Hard-sigmoid:11 |", + "_doc_layer_functions_pooling": " none:1 | Max:2 | Avg:3 |", + "_doc_layer_functions_probabilistic": " Binary:1 | Logistic:2 | Competitive:3 | Softmax:4 |", + "_doc_layer_functions_scaler": " none:1 | MinMax:2 | MeanStd:3 | STD:4 | Log:5 |", + "lossMethod": "6", + "_doc_lossMethod": " SSE:1 | MSE:2 | NSE:3 | MinkowskiE:4 | WSE:5 | CEE:6 |", + "lr": "0.01", + "_doc_lr": "Positve float", + "epochs": "1", + "_doc_epochs": "Positve Integer", + "optimizer": "5", + "_doc_optimizer": " GD:0 | CGD:1 | SGD:2 | QuasiNeuton:3 | LVM:4 | ADAM:5 |", + "optimizerArgs": "none", + "_doc_optimizerArgs": "String", + "infraType": "0", + "_doc_infraType": " opennn:0 | wolfengine:1 |", + "distributedSystemType": "7", + "_doc_distributedSystemType": " none:0 | FedClientAvg:1 | FedServerAvg:2 | FedClientWeightedAvgClassification:3 | FedServerWeightedAvgClassification:4 | FedClientAE:5 | FedServerAE:6 | tiles:7 |", + "distributedSystemArgs": "none", + "_doc_distributedSystemArgs": "String", + "distributedSystemToken": "cl2", + "_doc_distributedSystemToken": "Token that associates distributed group of workers and parameter-server" + }, + "e09032ba977d35e5c39fb9c96dc4cfb3d73a8238b4e94635c031c00ec019a268": { + "modelType": "0", + "_doc_modelType": " nn:0 | approximation:1 | classification:2 | forecasting:3 | image_classification:4 | text_classification:5 | text_generation:6 | auto_association:7 | autoencoder:8 | ae_classifier:9 |", + "modelArgs": "", + "_doc_modelArgs": "Extra arguments to model", + "layersSizes": "9x28x1k3x3x1x6p0s1t1,9x28x6k2x2p0s2,4x14x6k3x3x6x16p0s1t0,1,120,84,10", + "_doc_layersSizes": "List of postive integers [L0, L1, ..., LN]", + "layerTypesList": "2,4,2,9,3,3,5", + "_doc_LayerTypes": " Default:0 | Scaling:1 | Conv:2 | Perceptron:3 | Pooling:4 | Probabilistic:5 | LSTM:6 | Reccurrent:7 | Unscaling:8 | Flatten:9 | Bounding:10 |", + "layers_functions": "6,2,6,1,6,6,4", + "_doc_layers_functions_activation": " Threshold:1 | Sign:2 | Logistic:3 | Tanh:4 | Linear:5 | ReLU:6 | eLU:7 | SeLU:8 | Soft-plus:9 | Soft-sign:10 | Hard-sigmoid:11 |", + "_doc_layer_functions_pooling": " none:1 | Max:2 | Avg:3 |", + "_doc_layer_functions_probabilistic": " Binary:1 | Logistic:2 | Competitive:3 | Softmax:4 |", + "_doc_layer_functions_scaler": " none:1 | MinMax:2 | MeanStd:3 | STD:4 | Log:5 |", + "lossMethod": "6", + "_doc_lossMethod": " SSE:1 | MSE:2 | NSE:3 | MinkowskiE:4 | WSE:5 | CEE:6 |", + "lr": "0.01", + "_doc_lr": "Positve float", + "epochs": "1", + "_doc_epochs": "Positve Integer", + "optimizer": "5", + "_doc_optimizer": " GD:0 | CGD:1 | SGD:2 | QuasiNeuton:3 | LVM:4 | ADAM:5 |", + "optimizerArgs": "none", + "_doc_optimizerArgs": "String", + "infraType": "0", + "_doc_infraType": " opennn:0 | wolfengine:1 |", + "distributedSystemType": "7", + "_doc_distributedSystemType": " none:0 | FedClientAvg:1 | FedServerAvg:2 | FedClientWeightedAvgClassification:3 | FedServerWeightedAvgClassification:4 | FedClientAE:5 | FedServerAE:6 | tiles:7 |", + "distributedSystemArgs": "none", + "_doc_distributedSystemArgs": "String", + "distributedSystemToken": "cl3", + "_doc_distributedSystemToken": "Token that associates distributed group of workers and parameter-server" + } + } +} \ No newline at end of file diff --git a/inputJsonsFiles/DistributedConfig/dc_5d_4c_4s_4r_12w.json b/inputJsonsFiles/DistributedConfig/dc_5d_4c_4s_4r_12w.json index 22f6e91b..03713309 100644 --- a/inputJsonsFiles/DistributedConfig/dc_5d_4c_4s_4r_12w.json +++ b/inputJsonsFiles/DistributedConfig/dc_5d_4c_4s_4r_12w.json @@ -1,21 +1,12 @@ { - "nerlnetSettings": { - - "frequency": "50", - - "batchSize": "30" - + "frequency": "100", + "batchSize": "20" }, - "mainServer": { - "port": "8080", - "args": "" - }, - "apiServer": { "port": "8081", "args": "" @@ -23,361 +14,291 @@ "devices": [ { "name": "vm1", - "ipv4": "10.0.0.1", + "ipv4": "10.0.0.6", "entities": "mainServer,apiServer" }, { "name": "vm2", - "ipv4": "10.0.0.2", + "ipv4": "10.0.0.16", "entities": "c1,r1,s1" - }, - { "name": "vm4", - "ipv4": "10.0.0.4", + "ipv4": "10.0.0.15", "entities": "c3,r3,s3" - }, - { "name": "vm5", - "ipv4": "10.0.0.5", + "ipv4": "10.0.0.13", "entities": "c4,r4,s4" - }, - { - "name": "vm3", - "ipv4": "10.0.0.3", + "ipv4": "10.0.0.7", "entities": "c2,r2,s2" - } - ], - "routers": [ - { "name": "r1", "port": "8090", "policy": "0" - }, - { - "name": "r2", - "port": "8091", - "policy": "0" - }, - { - "name": "r3", - "port": "8092", - "policy": "0" - }, - { - "name": "r4", - "port": "8093", - "policy": "0" - } - ], - "sources": [ - { - "name": "s1", - "port": "8086", - "frequency": "50", - "policy": "0", - "epochs": "1", - "type": "0" - }, - { - "name": "s2", - "port": "8087", - "frequency": "50", - "policy": "0", - "epochs": "1", - "type": "0" - }, - { - "name": "s3", - "port": "8088", - "frequency": "50", - "policy": "0", - "epochs": "1", - "type": "0" - }, - { - "name": "s4", - "port": "8089", - "frequency": "50", - "policy": "0", - "epochs": "1", - "type": "0" - } - ], - "clients": [ - { - "name": "c2", - "port": "8083", - - "workers": "w2_2,w3_2,w1_2" - + "workers": "w1_2,w2_2,w3_2" }, - { - "name": "c3", - "port": "8084", - "workers": "w1_3,w2_3,w3_3" - }, - { - "name": "c4", - "port": "8085", - - "workers": "w1_4,w2_4,w3_4" - + "workers": "w2_4,w1_4,w3_4" }, - { - "name": "c1", - "port": "8082", - "workers": "w1_1,w2_1,w3_1" - } - ], - "workers": [ - { - - "name": "w2_2", - - "model_sha": "42c55325abe802a45af6cc46b9f1a26d3693bef8efcebde63395dbc3a2c7d2ae" - + "name": "w1_2", + "model_sha": "905b6a65ff8b1a71f45ace0490bfd655f540fa298555faf83718311718eb69be" }, - { - - "name": "w3_2", - - "model_sha": "42c55325abe802a45af6cc46b9f1a26d3693bef8efcebde63395dbc3a2c7d2ae" - + "name": "w2_2", + "model_sha": "905b6a65ff8b1a71f45ace0490bfd655f540fa298555faf83718311718eb69be" }, - { - - "name": "w1_2", - - "model_sha": "42c55325abe802a45af6cc46b9f1a26d3693bef8efcebde63395dbc3a2c7d2ae" - + "name": "w3_2", + "model_sha": "905b6a65ff8b1a71f45ace0490bfd655f540fa298555faf83718311718eb69be" }, - { - "name": "w1_3", - - "model_sha": "42c55325abe802a45af6cc46b9f1a26d3693bef8efcebde63395dbc3a2c7d2ae" - + "model_sha": "5c6540e1e312b9a360193b4c0e80c561ae80f9249fdef48c28484ee7f4c9a3de" }, - { - "name": "w2_3", - - "model_sha": "42c55325abe802a45af6cc46b9f1a26d3693bef8efcebde63395dbc3a2c7d2ae" - + "model_sha": "5c6540e1e312b9a360193b4c0e80c561ae80f9249fdef48c28484ee7f4c9a3de" }, - { - "name": "w3_3", - - "model_sha": "42c55325abe802a45af6cc46b9f1a26d3693bef8efcebde63395dbc3a2c7d2ae" - + "model_sha": "5c6540e1e312b9a360193b4c0e80c561ae80f9249fdef48c28484ee7f4c9a3de" }, - { - - "name": "w1_4", - - "model_sha": "42c55325abe802a45af6cc46b9f1a26d3693bef8efcebde63395dbc3a2c7d2ae" - + "name": "w2_4", + "model_sha": "15606b956b4d1b4a240162a5cb34e61aa8a2658ee851f904ca698c3556d33779" }, - { - - "name": "w2_4", - - "model_sha": "42c55325abe802a45af6cc46b9f1a26d3693bef8efcebde63395dbc3a2c7d2ae" - + "name": "w1_4", + "model_sha": "15606b956b4d1b4a240162a5cb34e61aa8a2658ee851f904ca698c3556d33779" }, - { - "name": "w3_4", - - "model_sha": "42c55325abe802a45af6cc46b9f1a26d3693bef8efcebde63395dbc3a2c7d2ae" - + "model_sha": "15606b956b4d1b4a240162a5cb34e61aa8a2658ee851f904ca698c3556d33779" }, - { - "name": "w1_1", - - "model_sha": "42c55325abe802a45af6cc46b9f1a26d3693bef8efcebde63395dbc3a2c7d2ae" - + "model_sha": "4a86e14fa4c1dcb346d4c175a3c3bc359ef590beb5b4aad08605ddd0d4ed9f22" }, - { - "name": "w2_1", - - "model_sha": "42c55325abe802a45af6cc46b9f1a26d3693bef8efcebde63395dbc3a2c7d2ae" - + "model_sha": "4a86e14fa4c1dcb346d4c175a3c3bc359ef590beb5b4aad08605ddd0d4ed9f22" }, - { - "name": "w3_1", - - "model_sha": "42c55325abe802a45af6cc46b9f1a26d3693bef8efcebde63395dbc3a2c7d2ae" - + "model_sha": "4a86e14fa4c1dcb346d4c175a3c3bc359ef590beb5b4aad08605ddd0d4ed9f22" } - ], - + "model_sha": { - - "42c55325abe802a45af6cc46b9f1a26d3693bef8efcebde63395dbc3a2c7d2ae": { - + "905b6a65ff8b1a71f45ace0490bfd655f540fa298555faf83718311718eb69be": { "modelType": "0", - "_doc_modelType": " nn:0 | approximation:1 | classification:2 | forecasting:3 | image_classification:4 | text_classification:5 | text_generation:6 | auto_association:7 | autoencoder:8 | ae_classifier:9 |", - "modelArgs": "", - "_doc_modelArgs": "Extra arguments to model", - - "layersSizes": "9x28x1k5x5x1x6p0s1t1,9x28x6k2x2p0s2,5x14x6k4x4x6x12p0s1t0,1,32,10", - + "layersSizes": "9x28x1k3x3x1x6p0s1t1,9x28x6k2x2p0s2,4x14x6k3x3x6x16p0s1t0,1,120,84,10", "_doc_layersSizes": "List of postive integers [L0, L1, ..., LN]", - - "layerTypesList": "2,4,2,10,3,5", - + "layerTypesList": "2,4,2,9,3,3,5", "_doc_LayerTypes": " Default:0 | Scaling:1 | Conv:2 | Perceptron:3 | Pooling:4 | Probabilistic:5 | LSTM:6 | Reccurrent:7 | Unscaling:8 | Flatten:9 | Bounding:10 |", - - "layers_functions": "6,2,6,1,6,4", - + "layers_functions": "6,2,6,1,6,6,4", "_doc_layers_functions_activation": " Threshold:1 | Sign:2 | Logistic:3 | Tanh:4 | Linear:5 | ReLU:6 | eLU:7 | SeLU:8 | Soft-plus:9 | Soft-sign:10 | Hard-sigmoid:11 |", - "_doc_layer_functions_pooling": " none:1 | Max:2 | Avg:3 |", - "_doc_layer_functions_probabilistic": " Binary:1 | Logistic:2 | Competitive:3 | Softmax:4 |", - "_doc_layer_functions_scaler": " none:1 | MinMax:2 | MeanStd:3 | STD:4 | Log:5 |", - "lossMethod": "6", - "_doc_lossMethod": " SSE:1 | MSE:2 | NSE:3 | MinkowskiE:4 | WSE:5 | CEE:6 |", - "lr": "0.01", - "_doc_lr": "Positve float", - "epochs": "1", - "_doc_epochs": "Positve Integer", - "optimizer": "5", - "_doc_optimizer": " GD:0 | CGD:1 | SGD:2 | QuasiNeuton:3 | LVM:4 | ADAM:5 |", - "optimizerArgs": "none", - "_doc_optimizerArgs": "String", - "infraType": "0", - "_doc_infraType": " opennn:0 | wolfengine:1 |", - "distributedSystemType": "0", - - "_doc_distributedSystemType": " none:0 | FedClientAvg:1 | FedServerAvg:2 | FedClientWeightedAvgClassification:3 | FedServerWeightedAvgClassification:4 | FedClientAE:5 | FedServerAE:6 |", - + "_doc_distributedSystemType": " none:0 | FedClientAvg:1 | FedServerAvg:2 | FedClientWeightedAvgClassification:3 | FedServerWeightedAvgClassification:4 | FedClientAE:5 | FedServerAE:6 | tiles:7 |", "distributedSystemArgs": "none", - "_doc_distributedSystemArgs": "String", - "distributedSystemToken": "none", - "_doc_distributedSystemToken": "Token that associates distributed group of workers and parameter-server" - + }, + "5c6540e1e312b9a360193b4c0e80c561ae80f9249fdef48c28484ee7f4c9a3de": { + "modelType": "0", + "_doc_modelType": " nn:0 | approximation:1 | classification:2 | forecasting:3 | image_classification:4 | text_classification:5 | text_generation:6 | auto_association:7 | autoencoder:8 | ae_classifier:9 |", + "modelArgs": "", + "_doc_modelArgs": "Extra arguments to model", + "layersSizes": "9x28x1k3x3x1x6p0s1t1,9x28x6k2x2p0s2,4x14x6k3x3x6x16p0s1t0,1,120,84,10", + "_doc_layersSizes": "List of postive integers [L0, L1, ..., LN]", + "layerTypesList": "2,4,2,9,3,3,5", + "_doc_LayerTypes": " Default:0 | Scaling:1 | Conv:2 | Perceptron:3 | Pooling:4 | Probabilistic:5 | LSTM:6 | Reccurrent:7 | Unscaling:8 | Flatten:9 | Bounding:10 |", + "layers_functions":"6,2,6,1,6,6,4", + "_doc_layers_functions_activation": " Threshold:1 | Sign:2 | Logistic:3 | Tanh:4 | Linear:5 | ReLU:6 | eLU:7 | SeLU:8 | Soft-plus:9 | Soft-sign:10 | Hard-sigmoid:11 |", + "_doc_layer_functions_pooling": " none:1 | Max:2 | Avg:3 |", + "_doc_layer_functions_probabilistic": " Binary:1 | Logistic:2 | Competitive:3 | Softmax:4 |", + "_doc_layer_functions_scaler": " none:1 | MinMax:2 | MeanStd:3 | STD:4 | Log:5 |", + "lossMethod": "6", + "_doc_lossMethod": " SSE:1 | MSE:2 | NSE:3 | MinkowskiE:4 | WSE:5 | CEE:6 |", + "lr": "0.01", + "_doc_lr": "Positve float", + "epochs": "1", + "_doc_epochs": "Positve Integer", + "optimizer": "5", + "_doc_optimizer": " GD:0 | CGD:1 | SGD:2 | QuasiNeuton:3 | LVM:4 | ADAM:5 |", + "optimizerArgs": "none", + "_doc_optimizerArgs": "String", + "infraType": "0", + "_doc_infraType": " opennn:0 | wolfengine:1 |", + "distributedSystemType": "7", + "_doc_distributedSystemType": " none:0 | FedClientAvg:1 | FedServerAvg:2 | FedClientWeightedAvgClassification:3 | FedServerWeightedAvgClassification:4 | FedClientAE:5 | FedServerAE:6 | tiles:7 |", + "distributedSystemArgs": "none", + "_doc_distributedSystemArgs": "String", + "distributedSystemToken": "5299T", + "_doc_distributedSystemToken": "Token that associates distributed group of workers and parameter-server" + }, + "15606b956b4d1b4a240162a5cb34e61aa8a2658ee851f904ca698c3556d33779": { + "modelType": "0", + "_doc_modelType": " nn:0 | approximation:1 | classification:2 | forecasting:3 | image_classification:4 | text_classification:5 | text_generation:6 | auto_association:7 | autoencoder:8 | ae_classifier:9 |", + "modelArgs": "", + "_doc_modelArgs": "Extra arguments to model", + "layersSizes": "9x28x1k3x3x1x6p0s1t1,9x28x6k2x2p0s2,4x14x6k3x3x6x16p0s1t0,1,120,84,10", + "_doc_layersSizes": "List of postive integers [L0, L1, ..., LN]", + "layerTypesList":"2,4,2,9,3,3,5", + "_doc_LayerTypes": " Default:0 | Scaling:1 | Conv:2 | Perceptron:3 | Pooling:4 | Probabilistic:5 | LSTM:6 | Reccurrent:7 | Unscaling:8 | Flatten:9 | Bounding:10 |", + "layers_functions": "6,2,6,1,6,6,4", + "_doc_layers_functions_activation": " Threshold:1 | Sign:2 | Logistic:3 | Tanh:4 | Linear:5 | ReLU:6 | eLU:7 | SeLU:8 | Soft-plus:9 | Soft-sign:10 | Hard-sigmoid:11 |", + "_doc_layer_functions_pooling": " none:1 | Max:2 | Avg:3 |", + "_doc_layer_functions_probabilistic": " Binary:1 | Logistic:2 | Competitive:3 | Softmax:4 |", + "_doc_layer_functions_scaler": " none:1 | MinMax:2 | MeanStd:3 | STD:4 | Log:5 |", + "lossMethod": "6", + "_doc_lossMethod": " SSE:1 | MSE:2 | NSE:3 | MinkowskiE:4 | WSE:5 | CEE:6 |", + "lr": "0.01", + "_doc_lr": "Positve float", + "epochs": "1", + "_doc_epochs": "Positve Integer", + "optimizer": "5", + "_doc_optimizer": " GD:0 | CGD:1 | SGD:2 | QuasiNeuton:3 | LVM:4 | ADAM:5 |", + "optimizerArgs": "none", + "_doc_optimizerArgs": "String", + "infraType": "0", + "_doc_infraType": " opennn:0 | wolfengine:1 |", + "distributedSystemType": "7", + "_doc_distributedSystemType": " none:0 | FedClientAvg:1 | FedServerAvg:2 | FedClientWeightedAvgClassification:3 | FedServerWeightedAvgClassification:4 | FedClientAE:5 | FedServerAE:6 | tiles:7 |", + "distributedSystemArgs": "none", + "_doc_distributedSystemArgs": "String", + "distributedSystemToken": "0122j", + "_doc_distributedSystemToken": "Token that associates distributed group of workers and parameter-server" + }, + "4a86e14fa4c1dcb346d4c175a3c3bc359ef590beb5b4aad08605ddd0d4ed9f22": { + "modelType": "0", + "_doc_modelType": " nn:0 | approximation:1 | classification:2 | forecasting:3 | image_classification:4 | text_classification:5 | text_generation:6 | auto_association:7 | autoencoder:8 | ae_classifier:9 |", + "modelArgs": "", + "_doc_modelArgs": "Extra arguments to model", + "layersSizes": "9x28x1k3x3x1x6p0s1t1,9x28x6k2x2p0s2,4x14x6k3x3x6x16p0s1t0,1,120,84,10", + "_doc_layersSizes": "List of postive integers [L0, L1, ..., LN]", + "layerTypesList": "2,4,2,9,3,3,5", + "_doc_LayerTypes": " Default:0 | Scaling:1 | Conv:2 | Perceptron:3 | Pooling:4 | Probabilistic:5 | LSTM:6 | Reccurrent:7 | Unscaling:8 | Flatten:9 | Bounding:10 |", + "layers_functions": "6,2,6,1,6,6,4", + "_doc_layers_functions_activation": " Threshold:1 | Sign:2 | Logistic:3 | Tanh:4 | Linear:5 | ReLU:6 | eLU:7 | SeLU:8 | Soft-plus:9 | Soft-sign:10 | Hard-sigmoid:11 |", + "_doc_layer_functions_pooling": " none:1 | Max:2 | Avg:3 |", + "_doc_layer_functions_probabilistic": " Binary:1 | Logistic:2 | Competitive:3 | Softmax:4 |", + "_doc_layer_functions_scaler": " none:1 | MinMax:2 | MeanStd:3 | STD:4 | Log:5 |", + "lossMethod": "6", + "_doc_lossMethod": " SSE:1 | MSE:2 | NSE:3 | MinkowskiE:4 | WSE:5 | CEE:6 |", + "lr": "0.01", + "_doc_lr": "Positve float", + "epochs": "1", + "_doc_epochs": "Positve Integer", + "optimizer": "5", + "_doc_optimizer": " GD:0 | CGD:1 | SGD:2 | QuasiNeuton:3 | LVM:4 | ADAM:5 |", + "optimizerArgs": "none", + "_doc_optimizerArgs": "String", + "infraType": "0", + "_doc_infraType": " opennn:0 | wolfengine:1 |", + "distributedSystemType": "7", + "_doc_distributedSystemType": " none:0 | FedClientAvg:1 | FedServerAvg:2 | FedClientWeightedAvgClassification:3 | FedServerWeightedAvgClassification:4 | FedClientAE:5 | FedServerAE:6 | tiles:7 |", + "distributedSystemArgs": "none", + "_doc_distributedSystemArgs": "String", + "distributedSystemToken": "6355R", + "_doc_distributedSystemToken": "Token that associates distributed group of workers and parameter-server" } - } - } \ No newline at end of file diff --git a/inputJsonsFiles/DistributedConfig/dc_mnist_4w_5d_4r_4s_rr.json b/inputJsonsFiles/DistributedConfig/dc_mnist_4w_5d_4r_4s_rr.json new file mode 100644 index 00000000..ac75da83 --- /dev/null +++ b/inputJsonsFiles/DistributedConfig/dc_mnist_4w_5d_4r_4s_rr.json @@ -0,0 +1,172 @@ +{ + "nerlnetSettings": { + "frequency": "100", + "batchSize": "20" + }, + "mainServer": { + "port": "8080", + "args": "" + }, + "apiServer": { + "port": "8081", + "args": "" + }, + "devices": [ + { + "name": "vm1", + "ipv4": "10.0.0.6", + "entities": "mainServer,apiServer" + }, + { + "name": "vm2", + "ipv4": "10.0.0.16", + "entities": "c1,r1,s1" + }, + { + "name": "vm3", + "ipv4": "10.0.0.15", + "entities": "c2,s2,r2" + }, + { + "name": "vm4", + "ipv4": "10.0.0.13", + "entities": "c3,r3,s3" + }, + { + "name": "vm5", + "ipv4": "10.0.0.7", + "entities": "c4,r4,s4" + } + ], + "routers": [ + { + "name": "r1", + "port": "8090", + "policy": "0" + }, + { + "name": "r2", + "port": "8091", + "policy": "0" + }, + { + "name": "r3", + "port": "8092", + "policy": "0" + }, + { + "name": "r4", + "port": "8093", + "policy": "0" + } + ], + "sources": [ + { + "name": "s1", + "port": "8086", + "frequency": "50", + "policy": "1", + "epochs": "1", + "type": "0" + }, + { + "name": "s2", + "port": "8087", + "frequency": "50", + "policy": "1", + "epochs": "1", + "type": "0" + }, + { + "name": "s3", + "port": "8088", + "frequency": "50", + "policy": "1", + "epochs": "1", + "type": "0" + }, + { + "name": "s4", + "port": "8089", + "frequency": "50", + "policy": "1", + "epochs": "1", + "type": "0" + } + ], + "clients": [ + { + "name": "c1", + "port": "8082", + "workers": "w1" + }, + { + "name": "c2", + "port": "8083", + "workers": "w2" + }, + { + "name": "c3", + "port": "8084", + "workers": "w3" + }, + { + "name": "c4", + "port": "8085", + "workers": "w4" + } + ], + "workers": [ + { + "name": "w1", + "model_sha": "9c5f1261068be7be96487a2cae282aa22e8c1cb482a5bf8d557bc8e1e2b6fef0" + }, + { + "name": "w2", + "model_sha": "9c5f1261068be7be96487a2cae282aa22e8c1cb482a5bf8d557bc8e1e2b6fef0" + }, + { + "name": "w3", + "model_sha": "9c5f1261068be7be96487a2cae282aa22e8c1cb482a5bf8d557bc8e1e2b6fef0" + }, + { + "name": "w4", + "model_sha": "9c5f1261068be7be96487a2cae282aa22e8c1cb482a5bf8d557bc8e1e2b6fef0" + } + ], + "model_sha": { + "9c5f1261068be7be96487a2cae282aa22e8c1cb482a5bf8d557bc8e1e2b6fef0": { + "modelType": "0", + "_doc_modelType": " nn:0 | approximation:1 | classification:2 | forecasting:3 | image_classification:4 | text_classification:5 | text_generation:6 | auto_association:7 | autoencoder:8 | ae_classifier:9 |", + "modelArgs": "", + "_doc_modelArgs": "Extra arguments to model", + "layersSizes": "28x28x1k5x5x1x6p0s1t1,28x28x6k2x2p0s2,14x14x6k4x4x6x12p0s1t0,1,32,10", + "_doc_layersSizes": "List of postive integers [L0, L1, ..., LN]", + "layerTypesList": "2,4,2,9,3,5", + "_doc_LayerTypes": " Default:0 | Scaling:1 | Conv:2 | Perceptron:3 | Pooling:4 | Probabilistic:5 | LSTM:6 | Reccurrent:7 | Unscaling:8 | Flatten:9 | Bounding:10 |", + "layers_functions": "6,2,6,1,6,4", + "_doc_layers_functions_activation": " Threshold:1 | Sign:2 | Logistic:3 | Tanh:4 | Linear:5 | ReLU:6 | eLU:7 | SeLU:8 | Soft-plus:9 | Soft-sign:10 | Hard-sigmoid:11 |", + "_doc_layer_functions_pooling": " none:1 | Max:2 | Avg:3 |", + "_doc_layer_functions_probabilistic": " Binary:1 | Logistic:2 | Competitive:3 | Softmax:4 |", + "_doc_layer_functions_scaler": " none:1 | MinMax:2 | MeanStd:3 | STD:4 | Log:5 |", + "lossMethod": "6", + "_doc_lossMethod": " SSE:1 | MSE:2 | NSE:3 | MinkowskiE:4 | WSE:5 | CEE:6 |", + "lr": "0.01", + "_doc_lr": "Positve float", + "epochs": "1", + "_doc_epochs": "Positve Integer", + "optimizer": "5", + "_doc_optimizer": " GD:0 | CGD:1 | SGD:2 | QuasiNeuton:3 | LVM:4 | ADAM:5 |", + "optimizerArgs": "none", + "_doc_optimizerArgs": "String", + "infraType": "0", + "_doc_infraType": " opennn:0 | wolfengine:1 |", + "distributedSystemType": "0", + "_doc_distributedSystemType": " none:0 | FedClientAvg:1 | FedServerAvg:2 | FedClientWeightedAvgClassification:3 | FedServerWeightedAvgClassification:4 | FedClientAE:5 | FedServerAE:6 | tiles:7 |", + "distributedSystemArgs": "none", + "_doc_distributedSystemArgs": "String", + "distributedSystemToken": "none", + "_doc_distributedSystemToken": "Token that associates distributed group of workers and parameter-server" + } + } +} \ No newline at end of file diff --git a/inputJsonsFiles/experimentsFlow/exp_mnist_5d_4c_4s_4w_4r.json b/inputJsonsFiles/experimentsFlow/exp_mnist_5d_4c_4s_4w_4r.json new file mode 100644 index 00000000..a7008856 --- /dev/null +++ b/inputJsonsFiles/experimentsFlow/exp_mnist_5d_4c_4s_4w_4r.json @@ -0,0 +1,84 @@ +{ + "experimentName": "mnist_rr", + "experimentType": "classification", + "batchSize": 20, + "csvFilePath": "/tmp/nerlnet/data/NerlnetData-master/nerlnet/Nerlnet/Mnist_normalized/mnist_train_255_norm.csv", + "numOfFeatures": "784", + "numOfLabels": "10", + "headersNames": "0,1,2,3,4,5,6,7,8,9", + "Phases": + [ + { + "phaseName": "training_phase", + "phaseType": "training", + "sourcePieces": + [ + { + "sourceName": "s1", + "startingSample": "", + "numOfBatches": "500", + "workers": "w1", + "nerltensorType": "float" + }, + { + "sourceName": "s2", + "startingSample": "", + "numOfBatches": "500", + "workers": "w2", + "nerltensorType": "float" + }, + { + "sourceName": "s3", + "startingSample": "", + "numOfBatches": "500", + "workers": "w3", + "nerltensorType": "float" + }, + { + "sourceName": "s4", + "startingSample": "", + "numOfBatches": "500", + "workers": "w4", + "nerltensorType": "float" + } + ] + }, + { + "phaseName": "prediction_phase", + "phaseType": "prediction", + "sourcePieces": + [ + { + "sourceName": "s1", + "startingSample": "", + "numOfBatches": "50", + "workers": "w1", + "nerltensorType": "float" + }, + { + "sourceName": "s2", + "startingSample": "", + "numOfBatches": "50", + "workers": "w2", + "nerltensorType": "float" + }, + { + "sourceName": "s3", + "startingSample": "", + "numOfBatches": "50", + "workers": "w3", + "nerltensorType": "float" + }, + { + "sourceName": "s4", + "startingSample": "", + "numOfBatches": "50", + "workers": "w4", + "nerltensorType": "float" + } + ] + } + ] + } + + \ No newline at end of file diff --git a/inputJsonsFiles/experimentsFlow/exp_tiles_13d_12c_4s_4r_12w_rr.json b/inputJsonsFiles/experimentsFlow/exp_tiles_13d_12c_4s_4r_12w_rr.json new file mode 100644 index 00000000..621e5ffc --- /dev/null +++ b/inputJsonsFiles/experimentsFlow/exp_tiles_13d_12c_4s_4r_12w_rr.json @@ -0,0 +1,70 @@ +{ + "experimentName": "tiles_rr", + "experimentType": "classification", + "batchSize": 20, + "csvFilePath": "/tmp/nerlnet/data/NerlnetData-master/nerlnet/Tiles_Mnist_sorted/mnist_tiles_sorted.csv", + "numOfFeatures": "252", + "numOfLabels": "10", + "headersNames": "0,1,2,3,4,5,6,7,8,9", + "Phases": + [ + { + "phaseName": "training_phase", + "phaseType": "training", + "sourcePieces": + [ + { + "sourceName": "s1", + "startingSample": "0", + "numOfBatches": "500", + "workers": "w1_0,w1_1,w1_2,w1_3", + "nerltensorType": "float" + }, + { + "sourceName": "s2", + "startingSample": "59999", + "numOfBatches": "500", + "workers": "w2_0,w2_1,w2_2,w2_3", + "nerltensorType": "float" + }, + { + "sourceName": "s3", + "startingSample": "119998", + "numOfBatches": "500", + "workers": "w3_1,w3_2,w3_3,w3_0", + "nerltensorType": "float" + } + ] + }, + { + "phaseName": "prediction_phase", + "phaseType": "prediction", + "sourcePieces": + [ + { + "sourceName": "s1", + "startingSample": "15000", + "numOfBatches": "50", + "workers": "w1_1,w1_2,w1_3,w1_0", + "nerltensorType": "float" + }, + { + "sourceName": "s2", + "startingSample": "74999", + "numOfBatches": "50", + "workers": "w2_1,w2_2,w2_3,w2_0", + "nerltensorType": "float" + }, + { + "sourceName": "s3", + "startingSample": "134998", + "numOfBatches": "50", + "workers": "w3_1,w3_2,w3_3,w3_0", + "nerltensorType": "float" + } + ] + } + ] + } + + \ No newline at end of file diff --git a/inputJsonsFiles/experimentsFlow/exp_tiles_4d_4c_4s_4r_12w.json b/inputJsonsFiles/experimentsFlow/exp_tiles_4d_4c_4s_4r_12w.json new file mode 100644 index 00000000..3a145f8d --- /dev/null +++ b/inputJsonsFiles/experimentsFlow/exp_tiles_4d_4c_4s_4r_12w.json @@ -0,0 +1,70 @@ +{ + "experimentName": "tiles_1", + "experimentType": "classification", + "batchSize": 20, + "csvFilePath": "/tmp/nerlnet/data/NerlnetData-master/nerlnet/Tiles_Mnist_sorted/mnist_tiles_sorted.csv", + "numOfFeatures": "252", + "numOfLabels": "10", + "headersNames": "0,1,2,3,4,5,6,7,8,9", + "Phases": + [ + { + "phaseName": "training_phase", + "phaseType": "training", + "sourcePieces": + [ + { + "sourceName": "s1", + "startingSample": "0", + "numOfBatches": "500", + "workers": "w1_1,w1_2,w1_3,w1_4", + "nerltensorType": "float" + }, + { + "sourceName": "s2", + "startingSample": "59999", + "numOfBatches": "500", + "workers": "w2_4,w2_1,w2_2,w2_3", + "nerltensorType": "float" + }, + { + "sourceName": "s3", + "startingSample": "119998", + "numOfBatches": "500", + "workers": "w3_1,w3_2,w3_3,w3_4", + "nerltensorType": "float" + } + ] + }, + { + "phaseName": "prediction_phase", + "phaseType": "prediction", + "sourcePieces": + [ + { + "sourceName": "s1", + "startingSample": "15000", + "numOfBatches": "50", + "workers": "w1_1,w1_2,w1_3,w1_4", + "nerltensorType": "float" + }, + { + "sourceName": "s2", + "startingSample": "74999", + "numOfBatches": "50", + "workers": "w2_1,w2_2,w2_3,w2_4", + "nerltensorType": "float" + }, + { + "sourceName": "s3", + "startingSample": "134998", + "numOfBatches": "50", + "workers": "w3_1,w3_2,w3_3,w3_4", + "nerltensorType": "float" + } + ] + } + ] + } + + \ No newline at end of file diff --git a/src_py/apiServer/experiment_flow.py b/src_py/apiServer/experiment_flow.py index 579c854b..b2f20eac 100644 --- a/src_py/apiServer/experiment_flow.py +++ b/src_py/apiServer/experiment_flow.py @@ -58,12 +58,17 @@ def get_events_sync(self): def get_csv_dataset(self): return self.csv_dataset - def generate_stats(self, experiment_phase = None) -> StatsTiles: + def generate_stats(self, experiment_phase = None) -> Stats: + if experiment_phase is None: + experiment_phase = self.get_current_experiment_phase() + return Stats(experiment_phase) + + def generate_stats_tiles(self, experiment_phase = None) -> StatsTiles: if experiment_phase is None: experiment_phase = self.get_current_experiment_phase() return StatsTiles(experiment_phase) - def merge_stats(self, stats_list: list) -> StatsTiles: + def merge_stats(self, stats_list: list) -> Stats: pass def parse_experiment_flow_json(self, json_path : str, override_csv_path = ""): diff --git a/src_py/apiServer/hf_repo_ids.json b/src_py/apiServer/hf_repo_ids.json index 04e31482..2c22c48c 100644 --- a/src_py/apiServer/hf_repo_ids.json +++ b/src_py/apiServer/hf_repo_ids.json @@ -7,9 +7,9 @@ "description": "SKAB Dataset for AEC anomaly detection" }, { - "id": "Nerlnet/MNist", + "id": "Nerlnet/MNist_test", "idx": 1, - "name": "mnist", + "name": "mnist_test", "description": "MNist Dataset for CNN experiments" }, { @@ -17,6 +17,18 @@ "idx": 2, "name": "synthetic_norm", "description": "Gaussian Distributions Classification Syntetic Dataset for baseline experiments" + }, + { + "id": "Nerlnet/tiles_sorted", + "idx": 3, + "name": "Tiles_Mnist_sorted", + "description": "Mnist tiled,sorted by parts - tiled to 3 parts" + }, + { + "id": "Nerlnet/Mnist_normalized", + "idx": 4, + "name": "mnist_normalized", + "description": "MNist Dataset for CNN experiments - 255 normalized" } ] } \ No newline at end of file diff --git a/src_py/apiServer/stats.py b/src_py/apiServer/stats.py index e7811b6c..275d4991 100644 --- a/src_py/apiServer/stats.py +++ b/src_py/apiServer/stats.py @@ -144,6 +144,90 @@ def expend_labels_df(self, df): df = df.reindex(columns = [*df.columns.tolist(), *temp_list], fill_value = 0) assert df.shape[1] == 2 * num_of_labels, "Error in expend_labels_df function" return df + + + def survey(results, category_names): + """ + Parameters + ---------- + results : dict + A mapping from question labels to a list of answers per category. + It is assumed all lists contain the same number of entries and that + it matches the length of *category_names*. + category_names : list of str + The category labels. + """ + labels = list(results.keys()) + data = np.array(list(results.values())) + data_cum = data.cumsum(axis=1) + category_colors = plt.colormaps['RdYlGn'](np.linspace(0.15, 0.85, data.shape[1])) + fig, ax = plt.subplots(figsize=(9.2, 5)) + ax.invert_yaxis() + ax.xaxis.set_visible(False) + ax.set_xlim(0, np.sum(data, axis=1).max()) + + for i, (colname, color) in enumerate(zip(category_names, category_colors)): + widths = data[:, i] + starts = data_cum[:, i] - widths + rects = ax.barh(labels, widths, left=starts, height=0.5,label=colname, color=color) + + r, g, b, _ = color + text_color = 'white' if r * g * b < 0.5 else 'darkgrey' + ax.bar_label(rects, label_type='center', color=text_color) + ax.legend(ncols=len(category_names), bbox_to_anchor=(0, 1),loc='lower left', fontsize='small') + + return fig, ax + + def get_distributed_train_labels(self , normalize : bool = False ,plot : bool = False , saveToFile : bool = False): + assert self.experiment_flow_type == "training", "This function is only available for training experiments" + assert self.phase == PHASE_TRAINING_STR, "This function is only available for train phase" + sources_pieces_list = self.experiment_phase.get_sources_pieces() + workers_model_db_list = self.nerl_model_db.get_workers_model_db_list() + dict_worker = {} + for worker in workers_model_db_list: + dict_worker[worker.get_worker_name()] = [] + labels_d = [] + category_names = ['Strongly disagree', 'Disagree', + 'Neither agree nor disagree', 'Agree', 'Strongly agree'] + results = { + 'Question 1': [10, 15, 17, 32, 26], + 'Question 2': [26, 22, 29, 10, 13], + 'Question 3': [35, 37, 7, 2, 19], + 'Question 4': [32, 11, 9, 15, 33], + 'Question 5': [21, 29, 5, 5, 40], + 'Question 6': [8, 19, 5, 30, 38] + } + + for source_piece_inst in sources_pieces_list: + sourcePiece_csv_labels_path = source_piece_inst.get_pointer_to_sourcePiece_CsvDataSet_labels() + df_actual_labels = pd.read_csv(sourcePiece_csv_labels_path) + num_of_labels = df_actual_labels.shape[1] + header_list = range(num_of_labels) + df_actual_labels.columns = header_list + labels_d = header_list + source_name = source_piece_inst.get_source_name() + # build confusion matrix for each worker + target_workers = source_piece_inst.get_target_workers() + batch_size = source_piece_inst.get_batch_size() + for worker_db in workers_model_db_list: + worker_name = worker_db.get_worker_name() + if worker_name not in target_workers: + continue + if(dict_worker[worker_name] == []): + dict_worker[worker_name] = [0] * num_of_labels + df_worker_labels = df_actual_labels.copy() + total_batches_per_source = worker_db.get_total_batches_per_source(source_name) + for batch_id in range(total_batches_per_source): + batch_db = worker_db.get_batch(source_name, str(batch_id)) + if not batch_db: # if batch is missing + if not self.missed_batches_warning_msg: + LOG_WARNING(f"missed batches") + self.missed_batches_warning_msg = True + dict_worker[worker_name][sum(df_worker_labels.iloc[batch_id * batch_size: (batch_id + 1) * batch_size])] +=1 # set the actual label to None for the predict labels in the df + + self.survey(dict_worker, labels_d) + plt.show() + def get_confusion_matrices(self , normalize : bool = False ,plot : bool = False , saveToFile : bool = False): assert self.experiment_flow_type == "classification", "This function is only available for classification experiments" @@ -192,13 +276,10 @@ def get_confusion_matrices(self , normalize : bool = False ,plot : bool = False cycle = int(batch_db.get_batch_id()) tensor_data = batch_db.get_tensor_data() tensor_data = tensor_data.reshape(batch_size, num_of_labels) - #print(df_worker_labels) - #print(tensor_data) start_index = cycle * batch_size end_index = (cycle + 1) * batch_size df_worker_labels.iloc[start_index:end_index, num_of_labels:] = None # Fix an issue of pandas of incompatible dtype df_worker_labels.iloc[start_index:end_index, num_of_labels:] = tensor_data - #print(df_worker_labels) # Take 2 list from the df, one for the actual labels and one for the predict labels to build the confusion matrix diff --git a/src_py/apiServer/statsTiles.py b/src_py/apiServer/statsTiles.py index 7132bdb1..d74d762d 100644 --- a/src_py/apiServer/statsTiles.py +++ b/src_py/apiServer/statsTiles.py @@ -17,14 +17,17 @@ class StatsTiles(Stats): def __init__(self, experiment_phase: ExperimentPhase): super().__init__(experiment_phase) - def get_confusion_matrices(self , normalize : bool = False ,plot : bool = False , saveToFile : bool = False): + confusion_matrix_source_dict, confusion_matrix_distributed_dict = self.get_confusion_matrices_tiles(normalize ,plot ,saveToFile ) + return confusion_matrix_source_dict, confusion_matrix_distributed_dict + + def get_confusion_matrices_tiles(self , normalize : bool = False ,plot : bool = False , saveToFile : bool = False): assert self.experiment_flow_type == "classification", "This function is only available for classification experiments" assert self.phase == PHASE_PREDICTION_STR, "This function is only available for predict phase" sources_pieces_list = self.experiment_phase.get_sources_pieces() workers_model_db_list = self.nerl_model_db.get_workers_model_db_list() confusion_matrix_source_dict = {} - confusion_matrix_worker_dict = {} + confusion_matrix_distributed_dict = {} distributed_tokens_dict = {} worker_name_labels_dict={} for source_piece_inst in sources_pieces_list: @@ -44,7 +47,6 @@ def get_confusion_matrices(self , normalize : bool = False ,plot : bool = False continue df_worker_labels = df_actual_labels.copy() total_batches_per_source = worker_db.get_total_batches_per_source(source_name) - print("total_batches_per_source: ", total_batches_per_source) for batch_id in range(total_batches_per_source): batch_db = worker_db.get_batch(source_name, str(batch_id)) if not batch_db: # if batch is missing @@ -68,10 +70,15 @@ def get_confusion_matrices(self , normalize : bool = False ,plot : bool = False df_worker_labels.iloc[start_index:end_index, num_of_labels:] = None # Fix an issue of pandas of incompatible dtype df_worker_labels.iloc[start_index:end_index, num_of_labels:] = tensor_data try: - distributed_tokens_dict[distributed_token_db][worker_name].append(batch_id) - except: + distributed_dict = distributed_tokens_dict[distributed_token_db] + try: + batch_id_arr_worker = distributed_dict[worker_name] + batch_id_arr_worker.append(batch_id) + distributed_tokens_dict[distributed_token_db][worker_name] = batch_id_arr_worker + except: + distributed_tokens_dict[distributed_token_db].update({worker_name : [batch_id]}) + except: distributed_tokens_dict[distributed_token_db] = {worker_name : [batch_id]} - print("df_worker_labels: ", df_worker_labels) else: cycle = int(batch_id) start_index = cycle * batch_size @@ -86,58 +93,62 @@ def get_confusion_matrices(self , normalize : bool = False ,plot : bool = False worker_name_labels_dict[worker_name] = {"labels": [max_column_labels_index,max_column_predict_index]} - for distributed_token_key in distributed_tokens_dict.keys(): - workers = distributed_tokens_dict[distributed_token_key].keys() - distributed_token_arr = [] - batch_id_arr = [] - for worker in workers: - labels = worker_name_labels_dict[worker]["labels"] - distributed_token_arr.append(labels) - batch_id_arr.append(distributed_tokens_dict[distributed_token_key][worker].sort(key=lambda x: x[0])) - - batch_id_dict = {} - for i in range(len(batch_id_arr)): - for batch_id in batch_id_arr[i]: - try: - batch_id_dict[batch_id].append(i) - except: - batch_id_dict[batch_id] = [i] - - for class_index, class_name in enumerate(self.headers_list): - class_actual_list_full = [] - class_actual_predict_full = [] - for batch_id_val,worker_num_list in enumerate(batch_id_dict): - class_actual_list = [] - class_predict_list = [] - workers_list = [workers[i] for i in worker_num_list] - labels_list = [distributed_token_arr[i] for i in worker_num_list] - for worker_id,worker in enumerate(workers_list): - cycle = int(batch_id_val) - start_index = cycle * batch_size - end_index = (cycle + 1) * batch_size - class_actual_list = [1 if label_num == class_index else 0 for label_num in labels_list[worker_id][0][start_index:end_index]] - try: - class_predict_list += [1 if label_num == class_index else 0 for label_num in labels_list[worker_id][1][start_index:end_index]] - except: - class_predict_list = [1 if label_num == class_index else 0 for label_num in labels_list[worker_id][1][start_index:end_index]] - - class_predict_list = [1 if (i>=len(workers_list)//2 or i == len(workers_list)) else 0 for i in class_predict_list] - class_actual_list_full.extend(class_actual_list) - class_actual_predict_full.extend(class_predict_list) - confusion_matrix = metrics.confusion_matrix(class_actual_list_full, class_actual_predict_full) + for distributed_token_key in list(distributed_tokens_dict.keys()): + workers = list(distributed_tokens_dict[distributed_token_key].keys()) + distributed_token_arr = [] + batch_id_arr = [] + for worker in list(workers): + labels = worker_name_labels_dict[worker]["labels"] + distributed_token_arr.append(labels) + distributed_token_key_worker = distributed_tokens_dict[distributed_token_key][worker] + batch_id_arr.append([distributed_token_key_worker,worker]) + batch_id_dict = {} + for i in range(len(batch_id_arr)): + if(batch_id_arr[i][0] is None): + continue + for batch_id in batch_id_arr[i][0]: try: - confusion_matrix_worker_dict[(distributed_token_key, class_name)] += confusion_matrix + batch_id_dict[batch_id].append(batch_id_arr[i][1]) except: - confusion_matrix_worker_dict[(distributed_token_key, class_name)] = confusion_matrix - - + batch_id_dict[batch_id] = [batch_id_arr[i][1]] + for class_index, class_name in enumerate(self.headers_list): + class_actual_list_full = [] + class_actual_predict_full = [] + for batch_id_val in list(batch_id_dict.keys()): + class_actual_list = [] + class_predict_list = [] + workers_list = batch_id_dict[batch_id_val] + print("workers_list: ",workers_list) + labels_list = distributed_token_arr + cycle = int(batch_id_val) + start_index = cycle * batch_size + end_index = (cycle + 1) * batch_size + for worker_id,worker in enumerate(workers_list): + class_actual_list = [1 if label_num == class_index else 0 for label_num in labels_list[worker_id][0][start_index:end_index]] + label_list_worker = labels_list[worker_id][1][start_index:end_index] + class_predict_temp = [1 if label_num == class_index else 0 for label_num in label_list_worker] + if(class_predict_list == []): + class_predict_list = [0]*len(class_predict_temp) + try: + temp_list = [class_predict_list,class_predict_temp] + class_predict_list = [sum(x) for x in zip(*temp_list)] + except: + class_predict_list = [1 if label_num == class_index else 0 for label_num in label_list_worker] + class_predict_list_new = [1 if (i>len(workers_list)/2 or i == len(workers_list)) else 0 for i in class_predict_list] + class_actual_list_full.extend(class_actual_list) + class_actual_predict_full.extend(class_predict_list_new) + confusion_matrix = metrics.confusion_matrix(class_actual_list_full, class_actual_predict_full) + try: + confusion_matrix_distributed_dict[(distributed_token_key, class_name)] += confusion_matrix + except: + confusion_matrix_distributed_dict[(distributed_token_key, class_name)] = confusion_matrix if plot: - workers = sorted(list({tup[0] for tup in confusion_matrix_worker_dict.keys()})) - classes = sorted(list({tup[1] for tup in confusion_matrix_worker_dict.keys()})) - fig, ax = plt.subplots(nrows=len(workers), ncols=len(classes),figsize=(4*len(classes),4*len(workers)),dpi=140) - for i , worker in enumerate(workers): + distributed_token_worker = sorted(list({tup[0] for tup in confusion_matrix_distributed_dict.keys()})) + classes = sorted(list({tup[1] for tup in confusion_matrix_distributed_dict.keys()})) + fig, ax = plt.subplots(nrows=len(distributed_token_worker), ncols=len(classes),figsize=(4*len(classes),4*len(distributed_token_worker)),dpi=140) + for i , worker in enumerate(distributed_token_worker): for j , pred_class in enumerate(classes): - conf_mat = confusion_matrix_worker_dict[(worker , pred_class)] + conf_mat = confusion_matrix_distributed_dict[(worker , pred_class)] heatmap = sns.heatmap(data=conf_mat ,ax=ax[i,j], annot=True , fmt="d", cmap='Blues',annot_kws={"size": 8}, cbar_kws={'pad': 0.1}) cbar = heatmap.collections[0].colorbar cbar.ax.tick_params(labelsize = 8) @@ -149,7 +160,62 @@ def get_confusion_matrices(self , normalize : bool = False ,plot : bool = False fig.subplots_adjust(wspace=0.4 , hspace=0.4) plt.show() + return confusion_matrix_source_dict, confusion_matrix_distributed_dict + + + def get_model_performence_stats_tiles(self , confusion_matrix_worker_dict , show : bool = False , saveToFile : bool = False, printStats = False) -> dict: + """ + Returns a dictionary of {(worker, class): {Performence_Stat : VALUE}}} for each worker and class in the experiment. + Performence Statistics Available are: TN, FP, FN, TP, Accuracy, Balanced Accuracy, Precision, Recall, True Negative Rate, Informedness, F1 + """ + assert self.experiment_flow_type == "classification", "This function is only available for classification experiments" + distributed_token_performence = OrderedDict() + for (token_name, class_name) in confusion_matrix_worker_dict.keys(): + distributed_token_performence[(token_name, class_name)] = OrderedDict() + tn, fp, fn, tp = confusion_matrix_worker_dict[(token_name, class_name)].ravel() + if printStats: + LOG_INFO(f"distributed token {token_name} class: {class_name} tn: {tn}, fp: {fp}, fn: {fn}, tp: {tp}") + tn = int(tn) + fp = int(fp) + fn = int(fn) + tp = int(tp) + acc = (tp + tn) / (tp + tn + fp + fn) + ppv = tp / (tp + fp) if tp > 0 else 0 # Precision + tpr = tp / (tp + fn) if tp > 0 else 0 # Recall + tnr = tn / (tn + fp) if tn > 0 else 0 + bacc = (tpr + tnr) / 2 + inf = tpr + tnr - 1 + f1 = 2 * (ppv * tpr) / (ppv + tpr) if (ppv + tpr) > 0 else 0 # F1-Score + + distributed_token_performence[(token_name, class_name)]['TN'] = tn + distributed_token_performence[(token_name, class_name)]['FP'] = fp + distributed_token_performence[(token_name, class_name)]['FN'] = fn + distributed_token_performence[(token_name, class_name)]['TP'] = tp + distributed_token_performence[(token_name, class_name)]['Accuracy'] = acc + distributed_token_performence[(token_name, class_name)]['Balanced Accuracy'] = bacc + distributed_token_performence[(token_name, class_name)]['Precision'] = ppv + distributed_token_performence[(token_name, class_name)]['Recall'] = tpr + distributed_token_performence[(token_name, class_name)]['True Negative Rate'] = tnr + distributed_token_performence[(token_name, class_name)]['Informedness'] = inf + distributed_token_performence[(token_name, class_name)]['F1'] = f1 + + df = pd.DataFrame.from_dict(distributed_token_performence, orient='index') + stats = list(df.columns) + df.reset_index(inplace=True) + df.columns = ['Worker', 'Class'] + stats + if show: + centered_df = df.style.set_properties(**{'text-align': 'center'}).set_table_styles([ # Center all + {'selector': 'th', 'props': [('text-align', 'center')]}, + {'selector': 'th.col_heading', 'props': [('text-align', 'center')]}, + {'selector': 'th.row_heading', 'props': [('text-align', 'center')]} + ]) + display(centered_df) + if saveToFile: + LOG_INFO(f"Saving model performence stats to csv file: {EXPERIMENT_RESULTS_PATH}/{self.exp_path}/{MODEL_PERFORMANCE_FILENAME}") + export_df_csv(f'{EXPERIMENT_RESULTS_PATH}/{self.exp_path}/{MODEL_PERFORMANCE_FILENAME}', df) + + return df + - return confusion_matrix_source_dict, confusion_matrix_worker_dict - \ No newline at end of file + \ No newline at end of file From b990079dc7bbf8790f1a0e623cdec3d0b053c1f9 Mon Sep 17 00:00:00 2001 From: Orisadek Date: Sun, 28 Jul 2024 11:10:58 +0000 Subject: [PATCH 6/8] add jsons and stats tiles works --- .../conn_4Router4Clients4S_mnist.json | 9 +++ .../dc_12w_12c_13d_4s_4r_tiles_rr.json | 2 +- .../dc_mnist_4w_5d_4r_4s_rr.json | 18 ++--- .../exp_mnist_5d_4c_4s_4w_4r.json | 62 +++++++---------- .../exp_tiles_13d_12c_4s_4r_12w_rr.json | 68 +++++++++++++++++-- src_py/apiServer/stats.py | 12 ---- src_py/apiServer/statsTiles.py | 1 - 7 files changed, 105 insertions(+), 67 deletions(-) create mode 100644 inputJsonsFiles/ConnectionMap/conn_4Router4Clients4S_mnist.json diff --git a/inputJsonsFiles/ConnectionMap/conn_4Router4Clients4S_mnist.json b/inputJsonsFiles/ConnectionMap/conn_4Router4Clients4S_mnist.json new file mode 100644 index 00000000..eccd4304 --- /dev/null +++ b/inputJsonsFiles/ConnectionMap/conn_4Router4Clients4S_mnist.json @@ -0,0 +1,9 @@ +{ + "connectionsMap": + { + "r1":["apiServer","mainServer", "c1", "s1", "r2"], + "r2":["c2", "s2","r3"], + "r3":["c3","s3" ,"r4","c1"], + "r4":["c4","c2","s4", "r1"] + } +} diff --git a/inputJsonsFiles/DistributedConfig/dc_12w_12c_13d_4s_4r_tiles_rr.json b/inputJsonsFiles/DistributedConfig/dc_12w_12c_13d_4s_4r_tiles_rr.json index 50d5e282..9853d7f1 100644 --- a/inputJsonsFiles/DistributedConfig/dc_12w_12c_13d_4s_4r_tiles_rr.json +++ b/inputJsonsFiles/DistributedConfig/dc_12w_12c_13d_4s_4r_tiles_rr.json @@ -54,7 +54,7 @@ }, { "name": "vm9", - "ipv4": "10.0.0.46", + "ipv4": "10.0.0.36", "entities": "c8,r3" }, { diff --git a/inputJsonsFiles/DistributedConfig/dc_mnist_4w_5d_4r_4s_rr.json b/inputJsonsFiles/DistributedConfig/dc_mnist_4w_5d_4r_4s_rr.json index ac75da83..073dd3e8 100644 --- a/inputJsonsFiles/DistributedConfig/dc_mnist_4w_5d_4r_4s_rr.json +++ b/inputJsonsFiles/DistributedConfig/dc_mnist_4w_5d_4r_4s_rr.json @@ -1,6 +1,6 @@ { "nerlnetSettings": { - "frequency": "100", + "frequency": "50", "batchSize": "20" }, "mainServer": { @@ -64,32 +64,32 @@ { "name": "s1", "port": "8086", - "frequency": "50", - "policy": "1", + "frequency": "30", + "policy": "0", "epochs": "1", "type": "0" }, { "name": "s2", "port": "8087", - "frequency": "50", - "policy": "1", + "frequency": "30", + "policy": "0", "epochs": "1", "type": "0" }, { "name": "s3", "port": "8088", - "frequency": "50", - "policy": "1", + "frequency": "30", + "policy": "0", "epochs": "1", "type": "0" }, { "name": "s4", "port": "8089", - "frequency": "50", - "policy": "1", + "frequency": "30", + "policy": "0", "epochs": "1", "type": "0" } diff --git a/inputJsonsFiles/experimentsFlow/exp_mnist_5d_4c_4s_4w_4r.json b/inputJsonsFiles/experimentsFlow/exp_mnist_5d_4c_4s_4w_4r.json index a7008856..0e2fbaa2 100644 --- a/inputJsonsFiles/experimentsFlow/exp_mnist_5d_4c_4s_4w_4r.json +++ b/inputJsonsFiles/experimentsFlow/exp_mnist_5d_4c_4s_4w_4r.json @@ -2,7 +2,7 @@ "experimentName": "mnist_rr", "experimentType": "classification", "batchSize": 20, - "csvFilePath": "/tmp/nerlnet/data/NerlnetData-master/nerlnet/Nerlnet/Mnist_normalized/mnist_train_255_norm.csv", + "csvFilePath": "/tmp/nerlnet/data/NerlnetData-master/nerlnet/mnist_normalized/mnist_train_255_norm.csv", "numOfFeatures": "784", "numOfLabels": "10", "headersNames": "0,1,2,3,4,5,6,7,8,9", @@ -15,32 +15,32 @@ [ { "sourceName": "s1", - "startingSample": "", - "numOfBatches": "500", + "startingSample": "0", + "numOfBatches": "300", "workers": "w1", "nerltensorType": "float" }, { "sourceName": "s2", - "startingSample": "", - "numOfBatches": "500", + "startingSample": "200", + "numOfBatches": "300", "workers": "w2", "nerltensorType": "float" }, { - "sourceName": "s3", - "startingSample": "", - "numOfBatches": "500", - "workers": "w3", - "nerltensorType": "float" - }, - { - "sourceName": "s4", - "startingSample": "", - "numOfBatches": "500", - "workers": "w4", - "nerltensorType": "float" - } + "sourceName": "s3", + "startingSample": "600", + "numOfBatches": "300", + "workers": "w3", + "nerltensorType": "float" + }, + { + "sourceName": "s4", + "startingSample": "1000", + "numOfBatches": "300", + "workers": "w4", + "nerltensorType": "float" + } ] }, { @@ -48,32 +48,18 @@ "phaseType": "prediction", "sourcePieces": [ - { - "sourceName": "s1", - "startingSample": "", - "numOfBatches": "50", - "workers": "w1", - "nerltensorType": "float" - }, - { - "sourceName": "s2", - "startingSample": "", - "numOfBatches": "50", - "workers": "w2", - "nerltensorType": "float" - }, { "sourceName": "s3", - "startingSample": "", - "numOfBatches": "50", - "workers": "w3", + "startingSample": "3000", + "numOfBatches": "30", + "workers": "w3,w1", "nerltensorType": "float" }, { "sourceName": "s4", - "startingSample": "", - "numOfBatches": "50", - "workers": "w4", + "startingSample": "1000", + "numOfBatches": "30", + "workers": "w4,w2", "nerltensorType": "float" } ] diff --git a/inputJsonsFiles/experimentsFlow/exp_tiles_13d_12c_4s_4r_12w_rr.json b/inputJsonsFiles/experimentsFlow/exp_tiles_13d_12c_4s_4r_12w_rr.json index 621e5ffc..26bd007c 100644 --- a/inputJsonsFiles/experimentsFlow/exp_tiles_13d_12c_4s_4r_12w_rr.json +++ b/inputJsonsFiles/experimentsFlow/exp_tiles_13d_12c_4s_4r_12w_rr.json @@ -36,6 +36,62 @@ } ] }, + { + "phaseName": "training1_phase", + "phaseType": "training", + "sourcePieces": + [ + { + "sourceName": "s1", + "startingSample": "0", + "numOfBatches": "500", + "workers": "w1_0,w1_1,w1_2,w1_3", + "nerltensorType": "float" + }, + { + "sourceName": "s2", + "startingSample": "59999", + "numOfBatches": "500", + "workers": "w2_0,w2_1,w2_2,w2_3", + "nerltensorType": "float" + }, + { + "sourceName": "s3", + "startingSample": "119998", + "numOfBatches": "500", + "workers": "w3_1,w3_2,w3_3,w3_0", + "nerltensorType": "float" + } + ] + }, + { + "phaseName": "training2_phase", + "phaseType": "training", + "sourcePieces": + [ + { + "sourceName": "s1", + "startingSample": "0", + "numOfBatches": "500", + "workers": "w1_0,w1_1,w1_2,w1_3", + "nerltensorType": "float" + }, + { + "sourceName": "s2", + "startingSample": "59999", + "numOfBatches": "500", + "workers": "w2_0,w2_1,w2_2,w2_3", + "nerltensorType": "float" + }, + { + "sourceName": "s3", + "startingSample": "119998", + "numOfBatches": "500", + "workers": "w3_1,w3_2,w3_3,w3_0", + "nerltensorType": "float" + } + ] + }, { "phaseName": "prediction_phase", "phaseType": "prediction", @@ -43,22 +99,22 @@ [ { "sourceName": "s1", - "startingSample": "15000", - "numOfBatches": "50", + "startingSample": "32000", + "numOfBatches": "60", "workers": "w1_1,w1_2,w1_3,w1_0", "nerltensorType": "float" }, { "sourceName": "s2", - "startingSample": "74999", - "numOfBatches": "50", + "startingSample": "91999", + "numOfBatches": "60", "workers": "w2_1,w2_2,w2_3,w2_0", "nerltensorType": "float" }, { "sourceName": "s3", - "startingSample": "134998", - "numOfBatches": "50", + "startingSample": "151998", + "numOfBatches": "60", "workers": "w3_1,w3_2,w3_3,w3_0", "nerltensorType": "float" } diff --git a/src_py/apiServer/stats.py b/src_py/apiServer/stats.py index 5420b37a..0a6085c0 100644 --- a/src_py/apiServer/stats.py +++ b/src_py/apiServer/stats.py @@ -179,7 +179,6 @@ def survey(results, category_names): return fig, ax def get_distributed_train_labels(self , normalize : bool = False ,plot : bool = False , saveToFile : bool = False): - assert self.experiment_flow_type == "training", "This function is only available for training experiments" assert self.phase == PHASE_TRAINING_STR, "This function is only available for train phase" sources_pieces_list = self.experiment_phase.get_sources_pieces() workers_model_db_list = self.nerl_model_db.get_workers_model_db_list() @@ -187,17 +186,6 @@ def get_distributed_train_labels(self , normalize : bool = False ,plot : bool = for worker in workers_model_db_list: dict_worker[worker.get_worker_name()] = [] labels_d = [] - category_names = ['Strongly disagree', 'Disagree', - 'Neither agree nor disagree', 'Agree', 'Strongly agree'] - results = { - 'Question 1': [10, 15, 17, 32, 26], - 'Question 2': [26, 22, 29, 10, 13], - 'Question 3': [35, 37, 7, 2, 19], - 'Question 4': [32, 11, 9, 15, 33], - 'Question 5': [21, 29, 5, 5, 40], - 'Question 6': [8, 19, 5, 30, 38] - } - for source_piece_inst in sources_pieces_list: sourcePiece_csv_labels_path = source_piece_inst.get_pointer_to_sourcePiece_CsvDataSet_labels() df_actual_labels = pd.read_csv(sourcePiece_csv_labels_path) diff --git a/src_py/apiServer/statsTiles.py b/src_py/apiServer/statsTiles.py index d74d762d..81b7c549 100644 --- a/src_py/apiServer/statsTiles.py +++ b/src_py/apiServer/statsTiles.py @@ -118,7 +118,6 @@ def get_confusion_matrices_tiles(self , normalize : bool = False ,plot : bool = class_actual_list = [] class_predict_list = [] workers_list = batch_id_dict[batch_id_val] - print("workers_list: ",workers_list) labels_list = distributed_token_arr cycle = int(batch_id_val) start_index = cycle * batch_size From 9e83dd632a3e832c3cd0525b27b5a52d286e8542 Mon Sep 17 00:00:00 2001 From: Orisadek Date: Tue, 30 Jul 2024 21:01:15 +0000 Subject: [PATCH 7/8] add new confusion matrix --- .../dc_mnist_13d_12w_4r_3s_3tokens.json | 342 ++++++++++++++++++ .../exp_tiles_13d_12c_4s_4r_12w_rr.json | 30 +- src_py/apiServer/stats.py | 3 +- src_py/apiServer/statsTiles.py | 209 +++++++++-- 4 files changed, 530 insertions(+), 54 deletions(-) create mode 100644 inputJsonsFiles/DistributedConfig/dc_mnist_13d_12w_4r_3s_3tokens.json diff --git a/inputJsonsFiles/DistributedConfig/dc_mnist_13d_12w_4r_3s_3tokens.json b/inputJsonsFiles/DistributedConfig/dc_mnist_13d_12w_4r_3s_3tokens.json new file mode 100644 index 00000000..deb91695 --- /dev/null +++ b/inputJsonsFiles/DistributedConfig/dc_mnist_13d_12w_4r_3s_3tokens.json @@ -0,0 +1,342 @@ +{ + "nerlnetSettings": { + "frequency": "100", + "batchSize": "20" + }, + "mainServer": { + "port": "8080", + "args": "" + }, + "apiServer": { + "port": "8081", + "args": "" + }, + "devices": [ + { + "name": "vm1", + "ipv4": "10.0.0.0", + "entities": "apiServer,mainServer" + }, + { + "name": "vm2", + "ipv4": "10.0.0.1", + "entities": "c1,r1" + }, + { + "name": "vm3", + "ipv4": "10.0.0.2", + "entities": "c2" + }, + { + "name": "vm4", + "ipv4": "10.0.0.3", + "entities": "c3,s1" + }, + { + "name": "vm5", + "ipv4": "10.0.0.4", + "entities": "c4,r3" + }, + { + "name": "vm6", + "ipv4": "10.0.0.5", + "entities": "c5" + }, + { + "name": "vm7", + "ipv4": "10.0.0.6", + "entities": "c6" + }, + { + "name": "vm8", + "ipv4": "10.0.0.7", + "entities": "c7" + }, + { + "name": "vm9", + "ipv4": "10.0.0.8", + "entities": "c8" + }, + { + "name": "vm10", + "ipv4": "10.0.0.9", + "entities": "c9,s2" + }, + { + "name": "vm11", + "ipv4": "10.0.0.10", + "entities": "c10,r4" + }, + { + "name": "vm12", + "ipv4": "10.0.0.11", + "entities": "c11,r2" + }, + { + "name": "vm13", + "ipv4": "10.0.0.12", + "entities": "c12,s3" + } + ], + "routers": [ + { + "name": "r1", + "port": "8085", + "policy": "0" + }, + { + "name": "r2", + "port": "8086", + "policy": "0" + }, + { + "name": "r3", + "port": "8087", + "policy": "0" + }, + { + "name": "r4", + "port": "8088", + "policy": "0" + } + ], + "sources": [ + { + "name": "s1", + "port": "8082", + "frequency": "100", + "policy": "1", + "epochs": "1", + "type": "0" + }, + { + "name": "s2", + "port": "8083", + "frequency": "100", + "policy": "1", + "epochs": "1", + "type": "0" + }, + { + "name": "s3", + "port": "8084", + "frequency": "100", + "policy": "1", + "epochs": "1", + "type": "0" + } + ], + "clients": [ + { + "name": "c1", + "port": "8089", + "workers": "w1_1" + }, + { + "name": "c2", + "port": "8090", + "workers": "w2_1" + }, + { + "name": "c3", + "port": "8091", + "workers": "w3_1" + }, + { + "name": "c4", + "port": "8092", + "workers": "w4_1" + }, + { + "name": "c5", + "port": "8093", + "workers": "w1_2" + }, + { + "name": "c6", + "port": "8094", + "workers": "w2_2" + }, + { + "name": "c7", + "port": "8095", + "workers": "w3_2" + }, + { + "name": "c8", + "port": "8096", + "workers": "w4_2" + }, + { + "name": "c9", + "port": "8097", + "workers": "w1_3" + }, + { + "name": "c10", + "port": "8098", + "workers": "w2_3" + }, + { + "name": "c11", + "port": "8099", + "workers": "w3_3" + }, + { + "name": "c12", + "port": "8100", + "workers": "w4_3" + } + ], + "workers": [ + { + "name": "w1_1", + "model_sha": "ffef532224331dde3a92aa0c30c51efea41eac213c308e137445ae4248c35529" + }, + { + "name": "w2_1", + "model_sha": "ffef532224331dde3a92aa0c30c51efea41eac213c308e137445ae4248c35529" + }, + { + "name": "w3_1", + "model_sha": "ffef532224331dde3a92aa0c30c51efea41eac213c308e137445ae4248c35529" + }, + { + "name": "w4_1", + "model_sha": "ffef532224331dde3a92aa0c30c51efea41eac213c308e137445ae4248c35529" + }, + { + "name": "w1_2", + "model_sha": "8bfb23bac4c5c8b82cde1901d5410f1268f2794ad295ab4947755aa13db43541" + }, + { + "name": "w2_2", + "model_sha": "8bfb23bac4c5c8b82cde1901d5410f1268f2794ad295ab4947755aa13db43541" + }, + { + "name": "w3_2", + "model_sha": "8bfb23bac4c5c8b82cde1901d5410f1268f2794ad295ab4947755aa13db43541" + }, + { + "name": "w4_2", + "model_sha": "8bfb23bac4c5c8b82cde1901d5410f1268f2794ad295ab4947755aa13db43541" + }, + { + "name": "w1_3", + "model_sha": "b7c0c6e0cce7968d2864c4701764fb90a3473b75ba6c173b3a92d8df6d8e5f35" + }, + { + "name": "w2_3", + "model_sha": "b7c0c6e0cce7968d2864c4701764fb90a3473b75ba6c173b3a92d8df6d8e5f35" + }, + { + "name": "w3_3", + "model_sha": "b7c0c6e0cce7968d2864c4701764fb90a3473b75ba6c173b3a92d8df6d8e5f35" + }, + { + "name": "w4_3", + "model_sha": "b7c0c6e0cce7968d2864c4701764fb90a3473b75ba6c173b3a92d8df6d8e5f35" + } + ], + "model_sha": { + "ffef532224331dde3a92aa0c30c51efea41eac213c308e137445ae4248c35529": { + "modelType": "0", + "_doc_modelType": " nn:0 | approximation:1 | classification:2 | forecasting:3 | image_classification:4 | text_classification:5 | text_generation:6 | auto_association:7 | autoencoder:8 | ae_classifier:9 |", + "modelArgs": "", + "_doc_modelArgs": "Extra arguments to model", + "layersSizes": "14x14x1k3x3x1x6p0s1t1,14x14x6k2x2p0s2,7x7x6k3x3x6x16p0s1t0,1,120,84,10", + "_doc_layersSizes": "List of postive integers [L0, L1, ..., LN]", + "layerTypesList": "2,4,2,9,3,3,5", + "_doc_LayerTypes": " Default:0 | Scaling:1 | Conv:2 | Perceptron:3 | Pooling:4 | Probabilistic:5 | LSTM:6 | Reccurrent:7 | Unscaling:8 | Flatten:9 | Bounding:10 |", + "layers_functions": "6,2,6,1,6,6,4", + "_doc_layers_functions_activation": " Threshold:1 | Sign:2 | Logistic:3 | Tanh:4 | Linear:5 | ReLU:6 | eLU:7 | SeLU:8 | Soft-plus:9 | Soft-sign:10 | Hard-sigmoid:11 |", + "_doc_layer_functions_pooling": " none:1 | Max:2 | Avg:3 |", + "_doc_layer_functions_probabilistic": " Binary:1 | Logistic:2 | Competitive:3 | Softmax:4 |", + "_doc_layer_functions_scaler": " none:1 | MinMax:2 | MeanStd:3 | STD:4 | Log:5 |", + "lossMethod": "6", + "_doc_lossMethod": " SSE:1 | MSE:2 | NSE:3 | MinkowskiE:4 | WSE:5 | CEE:6 |", + "lr": "0.01", + "_doc_lr": "Positve float", + "epochs": "1", + "_doc_epochs": "Positve Integer", + "optimizer": "5", + "_doc_optimizer": " GD:0 | CGD:1 | SGD:2 | QuasiNeuton:3 | LVM:4 | ADAM:5 |", + "optimizerArgs": "none", + "_doc_optimizerArgs": "String", + "infraType": "0", + "_doc_infraType": " opennn:0 | wolfengine:1 |", + "distributedSystemType": "7", + "_doc_distributedSystemType": " none:0 | FedClientAvg:1 | FedServerAvg:2 | FedClientWeightedAvgClassification:3 | FedServerWeightedAvgClassification:4 | FedClientAE:5 | FedServerAE:6 | tiles:7 |", + "distributedSystemArgs": "none", + "_doc_distributedSystemArgs": "String", + "distributedSystemToken": "cl_token0", + "_doc_distributedSystemToken": "Token that associates distributed group of workers and parameter-server" + }, + "8bfb23bac4c5c8b82cde1901d5410f1268f2794ad295ab4947755aa13db43541": { + "modelType": "0", + "_doc_modelType": " nn:0 | approximation:1 | classification:2 | forecasting:3 | image_classification:4 | text_classification:5 | text_generation:6 | auto_association:7 | autoencoder:8 | ae_classifier:9 |", + "modelArgs": "", + "_doc_modelArgs": "Extra arguments to model", + "layersSizes": "14x14x1k3x3x1x6p0s1t1,14x14x6k2x2p0s2,7x7x6k3x3x6x16p0s1t0,1,120,84,10", + "_doc_layersSizes": "List of postive integers [L0, L1, ..., LN]", + "layerTypesList": "2,4,2,9,3,3,5", + "_doc_LayerTypes": " Default:0 | Scaling:1 | Conv:2 | Perceptron:3 | Pooling:4 | Probabilistic:5 | LSTM:6 | Reccurrent:7 | Unscaling:8 | Flatten:9 | Bounding:10 |", + "layers_functions": "6,2,6,1,6,6,4", + "_doc_layers_functions_activation": " Threshold:1 | Sign:2 | Logistic:3 | Tanh:4 | Linear:5 | ReLU:6 | eLU:7 | SeLU:8 | Soft-plus:9 | Soft-sign:10 | Hard-sigmoid:11 |", + "_doc_layer_functions_pooling": " none:1 | Max:2 | Avg:3 |", + "_doc_layer_functions_probabilistic": " Binary:1 | Logistic:2 | Competitive:3 | Softmax:4 |", + "_doc_layer_functions_scaler": " none:1 | MinMax:2 | MeanStd:3 | STD:4 | Log:5 |", + "lossMethod": "6", + "_doc_lossMethod": " SSE:1 | MSE:2 | NSE:3 | MinkowskiE:4 | WSE:5 | CEE:6 |", + "lr": "0.01", + "_doc_lr": "Positve float", + "epochs": "1", + "_doc_epochs": "Positve Integer", + "optimizer": "5", + "_doc_optimizer": " GD:0 | CGD:1 | SGD:2 | QuasiNeuton:3 | LVM:4 | ADAM:5 |", + "optimizerArgs": "none", + "_doc_optimizerArgs": "String", + "infraType": "0", + "_doc_infraType": " opennn:0 | wolfengine:1 |", + "distributedSystemType": "7", + "_doc_distributedSystemType": " none:0 | FedClientAvg:1 | FedServerAvg:2 | FedClientWeightedAvgClassification:3 | FedServerWeightedAvgClassification:4 | FedClientAE:5 | FedServerAE:6 | tiles:7 |", + "distributedSystemArgs": "none", + "_doc_distributedSystemArgs": "String", + "distributedSystemToken": "cl_token1", + "_doc_distributedSystemToken": "Token that associates distributed group of workers and parameter-server" + }, + "b7c0c6e0cce7968d2864c4701764fb90a3473b75ba6c173b3a92d8df6d8e5f35": { + "modelType": "0", + "_doc_modelType": " nn:0 | approximation:1 | classification:2 | forecasting:3 | image_classification:4 | text_classification:5 | text_generation:6 | auto_association:7 | autoencoder:8 | ae_classifier:9 |", + "modelArgs": "", + "_doc_modelArgs": "Extra arguments to model", + "layersSizes": "14x14x1k3x3x1x6p0s1t1,14x14x6k2x2p0s2,7x7x6k3x3x6x16p0s1t0,1,120,84,10", + "_doc_layersSizes": "List of postive integers [L0, L1, ..., LN]", + "layerTypesList": "2,4,2,9,3,3,5", + "_doc_LayerTypes": " Default:0 | Scaling:1 | Conv:2 | Perceptron:3 | Pooling:4 | Probabilistic:5 | LSTM:6 | Reccurrent:7 | Unscaling:8 | Flatten:9 | Bounding:10 |", + "layers_functions": "6,2,6,1,6,6,4", + "_doc_layers_functions_activation": " Threshold:1 | Sign:2 | Logistic:3 | Tanh:4 | Linear:5 | ReLU:6 | eLU:7 | SeLU:8 | Soft-plus:9 | Soft-sign:10 | Hard-sigmoid:11 |", + "_doc_layer_functions_pooling": " none:1 | Max:2 | Avg:3 |", + "_doc_layer_functions_probabilistic": " Binary:1 | Logistic:2 | Competitive:3 | Softmax:4 |", + "_doc_layer_functions_scaler": " none:1 | MinMax:2 | MeanStd:3 | STD:4 | Log:5 |", + "lossMethod": "6", + "_doc_lossMethod": " SSE:1 | MSE:2 | NSE:3 | MinkowskiE:4 | WSE:5 | CEE:6 |", + "lr": "0.01", + "_doc_lr": "Positve float", + "epochs": "1", + "_doc_epochs": "Positve Integer", + "optimizer": "5", + "_doc_optimizer": " GD:0 | CGD:1 | SGD:2 | QuasiNeuton:3 | LVM:4 | ADAM:5 |", + "optimizerArgs": "none", + "_doc_optimizerArgs": "String", + "infraType": "0", + "_doc_infraType": " opennn:0 | wolfengine:1 |", + "distributedSystemType": "7", + "_doc_distributedSystemType": " none:0 | FedClientAvg:1 | FedServerAvg:2 | FedClientWeightedAvgClassification:3 | FedServerWeightedAvgClassification:4 | FedClientAE:5 | FedServerAE:6 | tiles:7 |", + "distributedSystemArgs": "none", + "_doc_distributedSystemArgs": "String", + "distributedSystemToken": "cl_token2", + "_doc_distributedSystemToken": "Token that associates distributed group of workers and parameter-server" + } + } +} \ No newline at end of file diff --git a/inputJsonsFiles/experimentsFlow/exp_tiles_13d_12c_4s_4r_12w_rr.json b/inputJsonsFiles/experimentsFlow/exp_tiles_13d_12c_4s_4r_12w_rr.json index 26bd007c..86a7ca3d 100644 --- a/inputJsonsFiles/experimentsFlow/exp_tiles_13d_12c_4s_4r_12w_rr.json +++ b/inputJsonsFiles/experimentsFlow/exp_tiles_13d_12c_4s_4r_12w_rr.json @@ -31,7 +31,7 @@ "sourceName": "s3", "startingSample": "119998", "numOfBatches": "500", - "workers": "w3_1,w3_2,w3_3,w3_0", + "workers": "w3_0,w3_1,w3_2,w3_3", "nerltensorType": "float" } ] @@ -43,23 +43,23 @@ [ { "sourceName": "s1", - "startingSample": "0", + "startingSample": "10000", "numOfBatches": "500", "workers": "w1_0,w1_1,w1_2,w1_3", "nerltensorType": "float" }, { "sourceName": "s2", - "startingSample": "59999", + "startingSample": "69999", "numOfBatches": "500", "workers": "w2_0,w2_1,w2_2,w2_3", "nerltensorType": "float" }, { "sourceName": "s3", - "startingSample": "119998", + "startingSample": "129998", "numOfBatches": "500", - "workers": "w3_1,w3_2,w3_3,w3_0", + "workers": "w3_0,w3_1,w3_2,w3_3", "nerltensorType": "float" } ] @@ -71,23 +71,23 @@ [ { "sourceName": "s1", - "startingSample": "0", + "startingSample": "20000", "numOfBatches": "500", "workers": "w1_0,w1_1,w1_2,w1_3", "nerltensorType": "float" }, { "sourceName": "s2", - "startingSample": "59999", + "startingSample": "79999", "numOfBatches": "500", "workers": "w2_0,w2_1,w2_2,w2_3", "nerltensorType": "float" }, { "sourceName": "s3", - "startingSample": "119998", + "startingSample": "139998", "numOfBatches": "500", - "workers": "w3_1,w3_2,w3_3,w3_0", + "workers": "w3_0,w3_1,w3_2,w3_3", "nerltensorType": "float" } ] @@ -100,22 +100,22 @@ { "sourceName": "s1", "startingSample": "32000", - "numOfBatches": "60", - "workers": "w1_1,w1_2,w1_3,w1_0", + "numOfBatches": "500", + "workers": "w1_0,w1_1,w1_2,w1_3", "nerltensorType": "float" }, { "sourceName": "s2", "startingSample": "91999", - "numOfBatches": "60", - "workers": "w2_1,w2_2,w2_3,w2_0", + "numOfBatches": "500", + "workers": "w2_0,w2_1,w2_2,w2_3", "nerltensorType": "float" }, { "sourceName": "s3", "startingSample": "151998", - "numOfBatches": "60", - "workers": "w3_1,w3_2,w3_3,w3_0", + "numOfBatches": "500", + "workers": "w3_0,w3_1,w3_2,w3_3", "nerltensorType": "float" } ] diff --git a/src_py/apiServer/stats.py b/src_py/apiServer/stats.py index 81daca8b..d185ffa0 100644 --- a/src_py/apiServer/stats.py +++ b/src_py/apiServer/stats.py @@ -187,8 +187,9 @@ def get_distributed_train_labels(self , normalize : bool = False ,plot : bool = dict_worker[worker.get_worker_name()] = [] labels_d = [] for source_piece_inst in sources_pieces_list: - sourcePiece_csv_labels_path = source_piece_inst.get_pointer_to_sourcePiece_CsvDataSet_labels() + sourcePiece_csv_labels_path = source_piece_inst.get_pointer_to_sourcePiece_CsvDataSet() df_actual_labels = pd.read_csv(sourcePiece_csv_labels_path) + print(df_actual_labels,"df_actual_labels") num_of_labels = df_actual_labels.shape[1] header_list = range(num_of_labels) df_actual_labels.columns = header_list diff --git a/src_py/apiServer/statsTiles.py b/src_py/apiServer/statsTiles.py index 81b7c549..cfb26610 100644 --- a/src_py/apiServer/statsTiles.py +++ b/src_py/apiServer/statsTiles.py @@ -50,14 +50,10 @@ def get_confusion_matrices_tiles(self , normalize : bool = False ,plot : bool = for batch_id in range(total_batches_per_source): batch_db = worker_db.get_batch(source_name, str(batch_id)) if not batch_db: # if batch is missing - if not self.missed_batches_warning_msg: - LOG_WARNING(f"missed batches") - self.missed_batches_warning_msg = True starting_offset = source_piece_inst.get_starting_offset() df_worker_labels.iloc[batch_id * batch_size: (batch_id + 1) * batch_size, num_of_labels:] = None # set the actual label to None for the predict labels in the df worker_missed_batches[(worker_name, source_name, str(batch_id))] = (starting_offset + batch_id * batch_size, batch_size) # save the missing batch - #df_worker_labels = df_worker_labels.dropna() for batch_id in range(total_batches_per_source): batch_db = worker_db.get_batch(source_name, str(batch_id)) if batch_db: @@ -95,22 +91,17 @@ def get_confusion_matrices_tiles(self , normalize : bool = False ,plot : bool = for distributed_token_key in list(distributed_tokens_dict.keys()): workers = list(distributed_tokens_dict[distributed_token_key].keys()) - distributed_token_arr = [] - batch_id_arr = [] + distributed_token_arr = {} + batch_id_dict = {} for worker in list(workers): labels = worker_name_labels_dict[worker]["labels"] - distributed_token_arr.append(labels) - distributed_token_key_worker = distributed_tokens_dict[distributed_token_key][worker] - batch_id_arr.append([distributed_token_key_worker,worker]) - batch_id_dict = {} - for i in range(len(batch_id_arr)): - if(batch_id_arr[i][0] is None): - continue - for batch_id in batch_id_arr[i][0]: + distributed_token_arr[worker] = labels + batch_list_for_worker = distributed_tokens_dict[distributed_token_key][worker] + for batch_id in batch_list_for_worker: try: - batch_id_dict[batch_id].append(batch_id_arr[i][1]) + batch_id_dict[batch_id].append(worker) except: - batch_id_dict[batch_id] = [batch_id_arr[i][1]] + batch_id_dict[batch_id] = [worker] for class_index, class_name in enumerate(self.headers_list): class_actual_list_full = [] class_actual_predict_full = [] @@ -118,13 +109,13 @@ def get_confusion_matrices_tiles(self , normalize : bool = False ,plot : bool = class_actual_list = [] class_predict_list = [] workers_list = batch_id_dict[batch_id_val] - labels_list = distributed_token_arr + labels_dict = distributed_token_arr cycle = int(batch_id_val) start_index = cycle * batch_size end_index = (cycle + 1) * batch_size - for worker_id,worker in enumerate(workers_list): - class_actual_list = [1 if label_num == class_index else 0 for label_num in labels_list[worker_id][0][start_index:end_index]] - label_list_worker = labels_list[worker_id][1][start_index:end_index] + for worker in workers_list: + class_actual_list = [1 if label_num == class_index else 0 for label_num in labels_dict[worker][0][start_index:end_index]] + label_list_worker = labels_dict[worker][1][start_index:end_index] class_predict_temp = [1 if label_num == class_index else 0 for label_num in label_list_worker] if(class_predict_list == []): class_predict_list = [0]*len(class_predict_temp) @@ -142,25 +133,167 @@ def get_confusion_matrices_tiles(self , normalize : bool = False ,plot : bool = except: confusion_matrix_distributed_dict[(distributed_token_key, class_name)] = confusion_matrix if plot: - distributed_token_worker = sorted(list({tup[0] for tup in confusion_matrix_distributed_dict.keys()})) - classes = sorted(list({tup[1] for tup in confusion_matrix_distributed_dict.keys()})) - fig, ax = plt.subplots(nrows=len(distributed_token_worker), ncols=len(classes),figsize=(4*len(classes),4*len(distributed_token_worker)),dpi=140) - for i , worker in enumerate(distributed_token_worker): - for j , pred_class in enumerate(classes): - conf_mat = confusion_matrix_distributed_dict[(worker , pred_class)] - heatmap = sns.heatmap(data=conf_mat ,ax=ax[i,j], annot=True , fmt="d", cmap='Blues',annot_kws={"size": 8}, cbar_kws={'pad': 0.1}) - cbar = heatmap.collections[0].colorbar - cbar.ax.tick_params(labelsize = 8) - ax[i, j].set_title(f"{worker} , Class '{pred_class}'" , fontsize=12) - ax[i, j].tick_params(axis='both', which='major', labelsize=8) - ax[i, j].set_xlabel("Predicted Label" , fontsize=8) - ax[i, j].set_ylabel("True Label" , fontsize=8) - ax[i, j].set_aspect('equal') - fig.subplots_adjust(wspace=0.4 , hspace=0.4) - plt.show() - + self.print_plot(confusion_matrix_distributed_dict) return confusion_matrix_source_dict, confusion_matrix_distributed_dict + def print_plot(self, confusion_matrix): + distributed_token_worker = sorted(list({tup[0] for tup in confusion_matrix.keys()})) + classes = sorted(list({tup[1] for tup in confusion_matrix.keys()})) + fig, ax = plt.subplots(nrows=len(distributed_token_worker), ncols=len(classes),figsize=(4*len(classes),4*len(distributed_token_worker)),dpi=140) + for i , token in enumerate(distributed_token_worker): + for j , pred_class in enumerate(classes): + conf_mat = confusion_matrix[(token , pred_class)] + heatmap = sns.heatmap(data=conf_mat ,ax=ax[i,j], annot=True , fmt="d", cmap='Blues',annot_kws={"size": 8}, cbar_kws={'pad': 0.1}) + cbar = heatmap.collections[0].colorbar + cbar.ax.tick_params(labelsize = 8) + ax[i, j].set_title(f"{token} , Class '{pred_class}'" , fontsize=12) + ax[i, j].tick_params(axis='both', which='major', labelsize=8) + ax[i, j].set_xlabel("Predicted Label" , fontsize=8) + ax[i, j].set_ylabel("True Label" , fontsize=8) + ax[i, j].set_aspect('equal') + fig.subplots_adjust(wspace=0.4 , hspace=0.4) + plt.show() + + def add_lists_of_lists(self,list1, list2): + if len(list1) != len(list2): + raise ValueError("Both lists of lists must have the same number of sublists") + result = [ + [item1 + item2 for item1, item2 in zip(sublist1, sublist2)] + for sublist1, sublist2 in zip(list1, list2) + ] + return result + + def argmax_axis_1(self,list_of_lists): + # Get the index of the maximum value in each sublist + result = [sublist.index(max(sublist)) for sublist in list_of_lists] + return result + + + def get_confusion_matrices_tiles_new(self , normalize : bool = False ,plot : bool = False , saveToFile : bool = False): + assert self.experiment_flow_type == "classification", "This function is only available for classification experiments" + assert self.phase == PHASE_PREDICTION_STR, "This function is only available for predict phase" + sources_pieces_list = self.experiment_phase.get_sources_pieces() + workers_model_db_list = self.nerl_model_db.get_workers_model_db_list() + confusion_matrix_source_dict = {} + confusion_matrix_distributed_dict = {} + distributed_tokens_dict = {} + worker_name_labels_dict={} + for source_piece_inst in sources_pieces_list: + sourcePiece_csv_labels_path = source_piece_inst.get_pointer_to_sourcePiece_CsvDataSet_labels() + df_actual_labels = pd.read_csv(sourcePiece_csv_labels_path) + num_of_labels = df_actual_labels.shape[1] + header_list = range(num_of_labels) + df_actual_labels.columns = header_list + df_actual_labels = self.expend_labels_df(df_actual_labels) + source_name = source_piece_inst.get_source_name() + target_workers = source_piece_inst.get_target_workers() + worker_missed_batches = {} + batch_size = source_piece_inst.get_batch_size() + for worker_db in workers_model_db_list: + worker_name = worker_db.get_worker_name() + if worker_name not in target_workers: + continue + df_worker_labels = df_actual_labels.copy() + total_batches_per_source = worker_db.get_total_batches_per_source(source_name) + for batch_id in range(total_batches_per_source): + batch_db = worker_db.get_batch(source_name, str(batch_id)) + if not batch_db: # if batch is missing + starting_offset = source_piece_inst.get_starting_offset() + df_worker_labels.iloc[batch_id * batch_size: (batch_id + 1) * batch_size, num_of_labels:] = None # set the actual label to None for the predict labels in the df + worker_missed_batches[(worker_name, source_name, str(batch_id))] = (starting_offset + batch_id * batch_size, batch_size) # save the missing batch + + for batch_id in range(total_batches_per_source): + batch_db = worker_db.get_batch(source_name, str(batch_id)) + if batch_db: + distributed_token_db = batch_db.get_distributed_token() + cycle = int(batch_db.get_batch_id()) + tensor_data = batch_db.get_tensor_data() + tensor_data = tensor_data.reshape(batch_size, num_of_labels) + start_index = cycle * batch_size + end_index = (cycle + 1) * batch_size + df_worker_labels.iloc[start_index:end_index, num_of_labels:] = None # Fix an issue of pandas of incompatible dtype + df_worker_labels.iloc[start_index:end_index, num_of_labels:] = tensor_data + try: + distributed_dict = distributed_tokens_dict[distributed_token_db] + try: + batch_id_arr_worker = distributed_dict[worker_name] + batch_id_arr_worker.append(batch_id) + distributed_tokens_dict[distributed_token_db][worker_name] = batch_id_arr_worker + except: + distributed_tokens_dict[distributed_token_db].update({worker_name : [batch_id]}) + except: + distributed_tokens_dict[distributed_token_db] = {worker_name : [batch_id]} + else: + cycle = int(batch_id) + start_index = cycle * batch_size + end_index = (cycle + 1) * batch_size + df_worker_labels.iloc[start_index:end_index, num_of_labels:] = 0 + # Take 2 list from the df, one for the actual labels and one for the predict labels to build the confusion matrix + max_column_predict_index = df_worker_labels.iloc[:, num_of_labels:] + max_column_predict_index = max_column_predict_index + max_column_labels_index = df_worker_labels.iloc[:, :num_of_labels].idxmax(axis=1) + max_column_labels_index = max_column_labels_index.tolist() + worker_name_labels_dict[worker_name] = {"labels": [max_column_labels_index,max_column_predict_index]} + + + for distributed_token_key in list(distributed_tokens_dict.keys()): + workers = list(distributed_tokens_dict[distributed_token_key].keys()) + distributed_token_arr = [] + batch_dict_worker_labels = {} + batch_id_dict = {} + for worker in list(workers): + labels = worker_name_labels_dict[worker]["labels"] + distributed_token_arr.append(labels) + distributed_token_key_worker = distributed_tokens_dict[distributed_token_key][worker] + for batch_id in distributed_token_key_worker: + batch_dict_worker_labels[batch_id] = labels[0] + try: + batch_id_dict[batch_id].append(worker) + except: + batch_id_dict[batch_id] = [worker] + class_actual_predict_dict = {} + for batch_id_val in list(batch_id_dict.keys()): + class_predict_list = [] + workers_list = batch_id_dict[batch_id_val] + labels_list = distributed_token_arr + cycle = int(batch_id_val) + start_index = cycle * batch_size + end_index = (cycle + 1) * batch_size + for worker_id,worker in enumerate(workers_list): + label_list_worker = labels_list[worker_id][1][start_index:end_index] + class_predict_temp = label_list_worker.values.tolist() + if(class_predict_list == []): + class_predict_list = class_predict_temp + else: + class_predict_list = self.add_lists_of_lists(class_predict_list, class_predict_temp) + class_actual_predict_dict[batch_id_val] = class_predict_list + + + for class_index, class_name in enumerate(self.headers_list): + class_actual_predict_full = [] + class_actual_list_full = [] + for batch_id_val in list(batch_id_dict.keys()): + cycle = int(batch_id_val) + start_index = cycle * batch_size + end_index = (cycle + 1) * batch_size + class_predict_list_prev = class_actual_predict_dict[batch_id_val] + class_predict_list = [] + class_actual_list = [1 if label_num == class_index else 0 for label_num in batch_dict_worker_labels[batch_id_val][start_index:end_index]] + max_column_predict_index = self.argmax_axis_1(class_predict_list_prev) + max_column_predict_index = [int(predict_index) for predict_index in max_column_predict_index] + class_predict_list = [1 if label_num == class_index else 0 for label_num in max_column_predict_index] + class_actual_predict_full.extend(class_predict_list) + class_actual_list_full.extend(class_actual_list) + + confusion_matrix = metrics.confusion_matrix(class_actual_list_full, class_actual_predict_full) + try: + confusion_matrix_distributed_dict[(distributed_token_key, class_name)] += confusion_matrix + except: + confusion_matrix_distributed_dict[(distributed_token_key, class_name)] = confusion_matrix + + if plot: + self.print_plot(confusion_matrix_distributed_dict) + return confusion_matrix_source_dict, confusion_matrix_distributed_dict def get_model_performence_stats_tiles(self , confusion_matrix_worker_dict , show : bool = False , saveToFile : bool = False, printStats = False) -> dict: """ @@ -201,7 +334,7 @@ def get_model_performence_stats_tiles(self , confusion_matrix_worker_dict , show df = pd.DataFrame.from_dict(distributed_token_performence, orient='index') stats = list(df.columns) df.reset_index(inplace=True) - df.columns = ['Worker', 'Class'] + stats + df.columns = ['Cluster', 'Class'] + stats if show: centered_df = df.style.set_properties(**{'text-align': 'center'}).set_table_styles([ # Center all {'selector': 'th', 'props': [('text-align', 'center')]}, From f9569da88703f04f20798a86af5ab0992b25f457 Mon Sep 17 00:00:00 2001 From: Orisadek Date: Sat, 3 Aug 2024 21:10:01 +0000 Subject: [PATCH 8/8] change for PR --- src_py/apiServer/stats.py | 72 ---------------------------------- src_py/apiServer/statsTiles.py | 11 ++++++ 2 files changed, 11 insertions(+), 72 deletions(-) diff --git a/src_py/apiServer/stats.py b/src_py/apiServer/stats.py index d185ffa0..738ee1f1 100644 --- a/src_py/apiServer/stats.py +++ b/src_py/apiServer/stats.py @@ -146,78 +146,6 @@ def expend_labels_df(self, df): return df - def survey(results, category_names): - """ - Parameters - ---------- - results : dict - A mapping from question labels to a list of answers per category. - It is assumed all lists contain the same number of entries and that - it matches the length of *category_names*. - category_names : list of str - The category labels. - """ - labels = list(results.keys()) - data = np.array(list(results.values())) - data_cum = data.cumsum(axis=1) - category_colors = plt.colormaps['RdYlGn'](np.linspace(0.15, 0.85, data.shape[1])) - fig, ax = plt.subplots(figsize=(9.2, 5)) - ax.invert_yaxis() - ax.xaxis.set_visible(False) - ax.set_xlim(0, np.sum(data, axis=1).max()) - - for i, (colname, color) in enumerate(zip(category_names, category_colors)): - widths = data[:, i] - starts = data_cum[:, i] - widths - rects = ax.barh(labels, widths, left=starts, height=0.5,label=colname, color=color) - - r, g, b, _ = color - text_color = 'white' if r * g * b < 0.5 else 'darkgrey' - ax.bar_label(rects, label_type='center', color=text_color) - ax.legend(ncols=len(category_names), bbox_to_anchor=(0, 1),loc='lower left', fontsize='small') - - return fig, ax - - def get_distributed_train_labels(self , normalize : bool = False ,plot : bool = False , saveToFile : bool = False): - assert self.phase == PHASE_TRAINING_STR, "This function is only available for train phase" - sources_pieces_list = self.experiment_phase.get_sources_pieces() - workers_model_db_list = self.nerl_model_db.get_workers_model_db_list() - dict_worker = {} - for worker in workers_model_db_list: - dict_worker[worker.get_worker_name()] = [] - labels_d = [] - for source_piece_inst in sources_pieces_list: - sourcePiece_csv_labels_path = source_piece_inst.get_pointer_to_sourcePiece_CsvDataSet() - df_actual_labels = pd.read_csv(sourcePiece_csv_labels_path) - print(df_actual_labels,"df_actual_labels") - num_of_labels = df_actual_labels.shape[1] - header_list = range(num_of_labels) - df_actual_labels.columns = header_list - labels_d = header_list - source_name = source_piece_inst.get_source_name() - # build confusion matrix for each worker - target_workers = source_piece_inst.get_target_workers() - batch_size = source_piece_inst.get_batch_size() - for worker_db in workers_model_db_list: - worker_name = worker_db.get_worker_name() - if worker_name not in target_workers: - continue - if(dict_worker[worker_name] == []): - dict_worker[worker_name] = [0] * num_of_labels - df_worker_labels = df_actual_labels.copy() - total_batches_per_source = worker_db.get_total_batches_per_source(source_name) - for batch_id in range(total_batches_per_source): - batch_db = worker_db.get_batch(source_name, str(batch_id)) - if not batch_db: # if batch is missing - if not self.missed_batches_warning_msg: - LOG_WARNING(f"missed batches") - self.missed_batches_warning_msg = True - dict_worker[worker_name][sum(df_worker_labels.iloc[batch_id * batch_size: (batch_id + 1) * batch_size])] +=1 # set the actual label to None for the predict labels in the df - - self.survey(dict_worker, labels_d) - plt.show() - - def get_confusion_matrices(self , normalize : bool = False ,plot : bool = False , saveToFile : bool = False): assert self.experiment_flow_type == "classification", "This function is only available for classification experiments" assert self.phase == PHASE_PREDICTION_STR, "This function is only available for predict phase" diff --git a/src_py/apiServer/statsTiles.py b/src_py/apiServer/statsTiles.py index cfb26610..7df0ccc7 100644 --- a/src_py/apiServer/statsTiles.py +++ b/src_py/apiServer/statsTiles.py @@ -21,6 +21,12 @@ def get_confusion_matrices(self , normalize : bool = False ,plot : bool = False confusion_matrix_source_dict, confusion_matrix_distributed_dict = self.get_confusion_matrices_tiles(normalize ,plot ,saveToFile ) return confusion_matrix_source_dict, confusion_matrix_distributed_dict + + """ + This function is used to calculate the confusion matrix for each cluster and each class in the distributed tokens + attention! get_confusion_matrices_tiles take the sum of the labels and from that do majority vote and + then decide 0 or 1 for each class + """ def get_confusion_matrices_tiles(self , normalize : bool = False ,plot : bool = False , saveToFile : bool = False): assert self.experiment_flow_type == "classification", "This function is only available for classification experiments" assert self.phase == PHASE_PREDICTION_STR, "This function is only available for predict phase" @@ -168,7 +174,12 @@ def argmax_axis_1(self,list_of_lists): result = [sublist.index(max(sublist)) for sublist in list_of_lists] return result + """ + This function is used to calculate the confusion matrix for each cluster and each class in the distributed tokens + attention! get_confusion_matrices_tiles_new take the sum of the probabilities and from that do majority vote and + then decide 0 or 1 for each class + """ def get_confusion_matrices_tiles_new(self , normalize : bool = False ,plot : bool = False , saveToFile : bool = False): assert self.experiment_flow_type == "classification", "This function is only available for classification experiments" assert self.phase == PHASE_PREDICTION_STR, "This function is only available for predict phase"