diff --git a/src_py/apiServer/NerlCsvDataSetDB.py b/src_py/apiServer/NerlCsvDataSetDB.py new file mode 100644 index 00000000..34b2684f --- /dev/null +++ b/src_py/apiServer/NerlCsvDataSetDB.py @@ -0,0 +1,52 @@ + + +class CsvDataSet(): + def __init__(self, csv_path, csv_temp_path, batch_size, num_of_features, num_of_labels): + self.csv_path = csv_path + self.csv_temp_path = csv_temp_path + self.batch_size = batch_size + self.num_of_features = num_of_features + self.num_of_labels = num_of_labels + + def get_csv_path(self): + return self.csv_path + + def get_csv_temp_path(self): + return self.csv_temp_path + + def get_batch_size(self): + return self.batch_size + + def get_num_of_features(self): + return self.num_of_features + + def get_num_of_labels(self): + return self.num_of_labels + + def generate_source_pieceDS(self, source_name : str, batch_size, phase : str, starting_offset = 0, num_of_batches = 0): + return SourcePieceDS(source_name, batch_size, phase, starting_offset, num_of_batches) + + + +class SourcePieceDS(): + def __init__(self, source_name : str, batch_size, phase : str, starting_offset = 0, num_of_batches = 0): + self.source_name = source_name + self.batch_size = batch_size + self.phase = phase + self.starting_offset = starting_offset + self.num_of_batches = num_of_batches + #self.workers + self.pointer_to_CsvDataSet = None # which csvDataSet + + def get_source_name(self): + return self.source_name + + def get_batch_size(self): + return self.batch_size + + def get_phase(self): + return self.phase + + def get_starting_offset(self): + return self.starting_offset + diff --git a/src_py/apiServer/experiment_flow.py b/src_py/apiServer/experiment_flow.py new file mode 100644 index 00000000..6ba8f0b8 --- /dev/null +++ b/src_py/apiServer/experiment_flow.py @@ -0,0 +1,64 @@ + +import pandas as pd +import globalVars as globe +from definitions import * +from workerResult import * +from collections import OrderedDict +from NerlComDB import * +from NerlDB import * +from NerlDatasetDB import * +# Todo check imports and remove unused ones + +PARAM_CSV_DB_PATH = "csv_db_path" +PARAM_BATCH_SIZE = "batch_size" + + +class Experiment(): + + DATA_SOURCE_TYPE_CSV = 0 + DATA_SOURCE_TYPE_CAMERA = 1 + def __init__(self ,experiment_name = "Untitled", temp_data_path = NERLNET_TEMP_DATA_DIR, data_source_type = DATA_SOURCE_TYPE_CSV): + self.exp_name = experiment_name + #self.gentrate_stats = + #self.csv_dataset = CsvDataset() + self.exp_phase_list = [] + self.exp_flow_json = None + + def get_exp_name(self): + return self.exp_name + + def get_exp_phase_list(self): + return self.exp_phase_list + + def new_exp_phase(self, phase : str): + exp_phase = ExperimentPhase(phase) + self.exp_phase_list.append(exp_phase) + + # def parse_experiment_flow_json(self, json_path : str): + # # read json file from nerlPlanner output + # with open(json_path) as json_file: + # self.exp_flow_json = json.load(json_file) + # # parse experiment flow json to dataset_db + # csv_dataset_path = self.exp_flow_json[PARAM_CSV_DB_PATH] + # csv_temp_path = self.temp_data_path + # batch_size = self.exp_flow_json[PARAM_BATCH_SIZE] + # self.dataset_db = DataSetCsvDB(csv_dataset_path, csv_temp_path, batch_size) + # # add training and prediction phases source_dbs to dataset_db + # phases_list = [TRAINING_DBS, PREDICTION_DBS] + # for phase in phases_list: + # for source_db_dict in self.exp_flow_json[phase]: + # source_db_name = source_db_dict[PARAM_SOURCE_DB_NAME] + # target_workers_names_list = source_db_dict[PARAM_TARGET_WORKERS_NAMES_LIST] + # starting_offset = source_db_dict[PARAM_STARTING_OFFSET] + # numof_batches = source_db_dict[PARAM_NUMOF_BATCHES] + # self.dataset_db.add_source_db(source_db_name, target_workers_names_list, starting_offset, starting_offset + numof_batches * batch_size, DataSetCsvDB.PHASE_TRAINING) + +class ExperimentPhase(): + def __init__(self, phase : str): + self.phase = phase # training/prediction + self.nerl_comm_db = NerlComDB(globe.components) # Todo check if this is the right place to put it + self.nerl_model_db = NerlDB() # Todo will change to NerlModelDB + self.source_pieces_list = [] # list of SourcePieceDS + + def get_phase(self): + return self.phase diff --git a/src_py/apiServer/receiver.py b/src_py/apiServer/receiver.py index a7b3732c..36df4c21 100644 --- a/src_py/apiServer/receiver.py +++ b/src_py/apiServer/receiver.py @@ -94,7 +94,7 @@ def post(self): # receiver.logger.info("Training result received") # Result preprocessing: # Receiving from Erlang: "worker#loss" - # TODO example "s1$w1#source_name|batch_id|loss_value|duration" + # TODO example "w1#source_name|batch_id|loss_value|duration" # TODO GUY - Add all attributes of nerl_db (batch_id etc.) source_csv_dict = {} resData = request.get_data().decode('utf-8')