From 99b39d03dccd21594a5ec004fa8866e8cceb443e Mon Sep 17 00:00:00 2001 From: leondavi Date: Sat, 24 Jun 2023 16:10:15 +0300 Subject: [PATCH 01/40] init --- src_erl/NerlMonitor/Instructions.txt | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 src_erl/NerlMonitor/Instructions.txt diff --git a/src_erl/NerlMonitor/Instructions.txt b/src_erl/NerlMonitor/Instructions.txt new file mode 100644 index 00000000..1ef658fc --- /dev/null +++ b/src_erl/NerlMonitor/Instructions.txt @@ -0,0 +1,2 @@ + +Create a new cowboy server of NerlMonitor \ No newline at end of file From b660a347f502938427ba0767f5e26262b7f539a4 Mon Sep 17 00:00:00 2001 From: GuyPerets106 Date: Sun, 16 Jul 2023 07:45:59 +0000 Subject: [PATCH 02/40] Update READme.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 970f588e..9401e187 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# NErlNet +#NErlNet ![version](https://img.shields.io/github/v/release/leondavi/NErlNet) From afe543b7e69da9c392aaafbc4fb6f5410ae5ff3d Mon Sep 17 00:00:00 2001 From: vboxuser Date: Sun, 16 Jul 2023 11:18:59 +0300 Subject: [PATCH 03/40] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9401e187..970f588e 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -#NErlNet +# NErlNet ![version](https://img.shields.io/github/v/release/leondavi/NErlNet) From 054e573d897a4613cf331f57f863947547c913d5 Mon Sep 17 00:00:00 2001 From: leondavi Date: Mon, 17 Jul 2023 14:06:47 +0300 Subject: [PATCH 04/40] [NERLMONITOR] add attacher files --- .../http_Nerlserver/src/utilities_attacher.erl | 6 ++++++ .../http_Nerlserver/src/utilities_handler.erl | 0 2 files changed, 6 insertions(+) create mode 100644 src_erl/Communication_Layer/http_Nerlserver/src/utilities_attacher.erl create mode 100644 src_erl/Communication_Layer/http_Nerlserver/src/utilities_handler.erl diff --git a/src_erl/Communication_Layer/http_Nerlserver/src/utilities_attacher.erl b/src_erl/Communication_Layer/http_Nerlserver/src/utilities_attacher.erl new file mode 100644 index 00000000..c248f141 --- /dev/null +++ b/src_erl/Communication_Layer/http_Nerlserver/src/utilities_attacher.erl @@ -0,0 +1,6 @@ +-module(utilities_attacher). + + +client_utilities_attach(ListOfFunctions, Data) -> + {ClientPid, ClientEtsRef} = Data, + lists:foreach(fun(X, Data) -> X() end,ListOfFunctions). diff --git a/src_erl/Communication_Layer/http_Nerlserver/src/utilities_handler.erl b/src_erl/Communication_Layer/http_Nerlserver/src/utilities_handler.erl new file mode 100644 index 00000000..e69de29b From ae130177c4a7b593bd038c25d6fa17aed58169dc Mon Sep 17 00:00:00 2001 From: GuyPerets106 Date: Tue, 18 Jul 2023 07:10:04 +0000 Subject: [PATCH 05/40] [NerlMonitor] Added files --- src_erl/NerlMonitor/MonitorGUI.py | 16 ++++++ .../handlers/nerlMonitor_handler.erl | 50 +++++++++++++++++++ src_erl/NerlMonitor/nerlMonitor_app.erl | 33 ++++++++++++ src_erl/NerlMonitor/nerlMonitor_sup.erl | 35 +++++++++++++ 4 files changed, 134 insertions(+) create mode 100644 src_erl/NerlMonitor/MonitorGUI.py create mode 100644 src_erl/NerlMonitor/handlers/nerlMonitor_handler.erl create mode 100644 src_erl/NerlMonitor/nerlMonitor_app.erl create mode 100644 src_erl/NerlMonitor/nerlMonitor_sup.erl diff --git a/src_erl/NerlMonitor/MonitorGUI.py b/src_erl/NerlMonitor/MonitorGUI.py new file mode 100644 index 00000000..257087a4 --- /dev/null +++ b/src_erl/NerlMonitor/MonitorGUI.py @@ -0,0 +1,16 @@ +from pyrlang.node import Node +import PySimpleGUI as sg + +layout = [ + [sg.Text("NerlNet Monitor" , size=(30,1) ,text_color='Black' , font=('SFPro' , 20) , background_color='#A9D8E6' , justification='center' , pad=((0,0) , (10,0)))] , + [sg.Button("Close" , button_color=('#94D5F2' , '#000000') , font=('SFPro' , 12))] + ] + +MainWindow = sg.Window("NErlNet" , layout , margins=(10,10) , size=(640,400) , background_color='#A9D8E6' , finalize=True , resizable=True) + +while True: + event , values = MainWindow.read() + if event == "Close" or event == sg.WIN_CLOSED: + break + +MainWindow.close() \ No newline at end of file diff --git a/src_erl/NerlMonitor/handlers/nerlMonitor_handler.erl b/src_erl/NerlMonitor/handlers/nerlMonitor_handler.erl new file mode 100644 index 00000000..bdb62325 --- /dev/null +++ b/src_erl/NerlMonitor/handlers/nerlMonitor_handler.erl @@ -0,0 +1,50 @@ +-module(hello_handler). + +-export([init/2, http_request/4]). + +init(Req0, State = [MainScreen]) -> + {_,Body,_} = cowboy_req:read_body(Req0), + Data = binary_to_list(Body), + io:format("hello_handler got: ~p~n", [Data]), + case Data of + "nerlGUI" -> ignore; + Any -> + [ScreenName, Info] = string:split(Any, "@"), + %body = "toScreen@info" + wx_object:cast(MainScreen, {passInfo, list_to_atom(ScreenName), Info}) + end, + + Req = cowboy_req:reply(200, + #{<<"content-type">> => <<"text/plain">>}, + <<"Got that">>, + Req0), + {ok, Req, State}. + +%%sending Body as an http request to {Host, Port} to path Path (=String) +%%Example: http_request(RouterHost,RouterPort,"start_training", <<"client1,client2">>), +http_request(Host, Port, Path, Body)-> + URL = "http://" ++ Host ++ ":"++Port ++ "/" ++ Path, + httpc:set_options([{proxy, {{Host, list_to_integer(Port)},[Host]}}]), +%% io:format("sending: ~p~nto HostPo: ~p~n",[Body,{Host, Port}]), + {ok,Res} = httpc:request(post,{URL, [],"application/x-www-form-urlencoded",Body}, [], []), + %io:format("Got response ~p~n", [Res]), + Approve = element(2,element(1,Res)), + case Approve of + 404 -> + % io:format("sending: ~p~nto HostPo: ~p~n Res: ~p",[Body,{Host, Port},R]), + % io:format("Trying again in 0.01 second~n"), + timer:sleep(10), + spawn(fun() ->http_request(Host, Port,Path, Body) end); + _ -> ok + end, + % if py + Ans = lists:sublist(element(3,Res),8), + io:format("returning from req: ~p~n", [element(3,Res)]), + case Ans of + "Somthing" -> + % io:format("sending: ~p~nto HostPo: ~p~nRes: ~p",[Body,{Host, Port},R]), + % io:format("Trying again in 0.01 second~n"), + timer:sleep(10), + spawn(fun() ->http_request(Host, Port,Path, Body) end); + _ -> element(3,Res) + end. \ No newline at end of file diff --git a/src_erl/NerlMonitor/nerlMonitor_app.erl b/src_erl/NerlMonitor/nerlMonitor_app.erl new file mode 100644 index 00000000..728e31ce --- /dev/null +++ b/src_erl/NerlMonitor/nerlMonitor_app.erl @@ -0,0 +1,33 @@ +%%%------------------------------------------------------------------- +%% @doc nerlGUI public API +%% @end +%%%------------------------------------------------------------------- + +-module(nerlGUI_app). + +-behaviour(application). + +-export([start/2, stop/1]). + +start(_StartType, _StartArgs) -> + application:start(sasl), + application:start(ranch), + application:start(inets), + + Dispatch = cowboy_router:compile([ + {'_', [ + {"/[...]", nerlMonitor_handler, [**GUI_PID**]} + + ]} + ]), + {ok, _} = cowboy:start_clear(gui_listener, + [{port, 8096}], + #{env => #{dispatch => Dispatch}} + ), + + nerlGUI_sup:start_link(). + +stop(_State) -> + ok. + +%% internal functions diff --git a/src_erl/NerlMonitor/nerlMonitor_sup.erl b/src_erl/NerlMonitor/nerlMonitor_sup.erl new file mode 100644 index 00000000..089a33f8 --- /dev/null +++ b/src_erl/NerlMonitor/nerlMonitor_sup.erl @@ -0,0 +1,35 @@ +%%%------------------------------------------------------------------- +%% @doc nerlGUI top level supervisor. +%% @end +%%%------------------------------------------------------------------- + +-module(nerlMonitor_sup). + +-behaviour(supervisor). + +-export([start_link/0]). + +-export([init/1]). + +-define(SERVER, ?MODULE). + +start_link() -> + supervisor:start_link({local, ?SERVER}, ?MODULE, []). + +%% sup_flags() = #{strategy => strategy(), % optional +%% intensity => non_neg_integer(), % optional +%% period => pos_integer()} % optional +%% child_spec() = #{id => child_id(), % mandatory +%% start => mfargs(), % mandatory +%% restart => restart(), % optional +%% shutdown => shutdown(), % optional +%% type => worker(), % optional +%% modules => modules()} % optional +init([]) -> + SupFlags = #{strategy => one_for_all, + intensity => 0, + period => 1}, + ChildSpecs = [], + {ok, {SupFlags, ChildSpecs}}. + +%% internal functions From 253c187de44de34c127d5fa1e45d7e5021e23040 Mon Sep 17 00:00:00 2001 From: vboxuser Date: Thu, 20 Jul 2023 15:24:37 +0300 Subject: [PATCH 06/40] [NerlMonitor] Handler updates --- .../src/Client/clientStatem.erl | 5 ++ .../handlers/nerlMonitor_handler.erl | 39 ++------------ src_erl/NerlMonitor/nerlMonitor_app.erl | 54 ++++++++++++++++--- 3 files changed, 55 insertions(+), 43 deletions(-) diff --git a/src_erl/Communication_Layer/http_Nerlserver/src/Client/clientStatem.erl b/src_erl/Communication_Layer/http_Nerlserver/src/Client/clientStatem.erl index 52459b6b..1511ead6 100644 --- a/src_erl/Communication_Layer/http_Nerlserver/src/Client/clientStatem.erl +++ b/src_erl/Communication_Layer/http_Nerlserver/src/Client/clientStatem.erl @@ -124,6 +124,11 @@ waitforWorkers(cast, EventContent, State = #client_statem_state{etsRef = EtsRef} ets:update_counter(EtsRef, infoIn, nerl_tools:calculate_size(EventContent)), ?LOG_WARNING("client waitforWorkers ignored!!!: ~p ~n",[EventContent]), {next_state, waitforWorkers, State}. + +waitforWorkers(info, EventContent, State) -> + case EventContent of + {'DOWN',Ref,process,Pid,Reason}->%tell main server + %% initiating workers when they include federated workers. init stage == handshake between federated worker client and server diff --git a/src_erl/NerlMonitor/handlers/nerlMonitor_handler.erl b/src_erl/NerlMonitor/handlers/nerlMonitor_handler.erl index bdb62325..7884b8cb 100644 --- a/src_erl/NerlMonitor/handlers/nerlMonitor_handler.erl +++ b/src_erl/NerlMonitor/handlers/nerlMonitor_handler.erl @@ -1,17 +1,14 @@ -module(hello_handler). --export([init/2, http_request/4]). +-export([init/2]). init(Req0, State = [MainScreen]) -> {_,Body,_} = cowboy_req:read_body(Req0), Data = binary_to_list(Body), - io:format("hello_handler got: ~p~n", [Data]), case Data of - "nerlGUI" -> ignore; - Any -> - [ScreenName, Info] = string:split(Any, "@"), - %body = "toScreen@info" - wx_object:cast(MainScreen, {passInfo, list_to_atom(ScreenName), Info}) + worker_death_caught -> ok;%tell gui to change graph display and write in event log + _ -> + ok % got unknown messge, ignore. end, Req = cowboy_req:reply(200, @@ -20,31 +17,3 @@ init(Req0, State = [MainScreen]) -> Req0), {ok, Req, State}. -%%sending Body as an http request to {Host, Port} to path Path (=String) -%%Example: http_request(RouterHost,RouterPort,"start_training", <<"client1,client2">>), -http_request(Host, Port, Path, Body)-> - URL = "http://" ++ Host ++ ":"++Port ++ "/" ++ Path, - httpc:set_options([{proxy, {{Host, list_to_integer(Port)},[Host]}}]), -%% io:format("sending: ~p~nto HostPo: ~p~n",[Body,{Host, Port}]), - {ok,Res} = httpc:request(post,{URL, [],"application/x-www-form-urlencoded",Body}, [], []), - %io:format("Got response ~p~n", [Res]), - Approve = element(2,element(1,Res)), - case Approve of - 404 -> - % io:format("sending: ~p~nto HostPo: ~p~n Res: ~p",[Body,{Host, Port},R]), - % io:format("Trying again in 0.01 second~n"), - timer:sleep(10), - spawn(fun() ->http_request(Host, Port,Path, Body) end); - _ -> ok - end, - % if py - Ans = lists:sublist(element(3,Res),8), - io:format("returning from req: ~p~n", [element(3,Res)]), - case Ans of - "Somthing" -> - % io:format("sending: ~p~nto HostPo: ~p~nRes: ~p",[Body,{Host, Port},R]), - % io:format("Trying again in 0.01 second~n"), - timer:sleep(10), - spawn(fun() ->http_request(Host, Port,Path, Body) end); - _ -> element(3,Res) - end. \ No newline at end of file diff --git a/src_erl/NerlMonitor/nerlMonitor_app.erl b/src_erl/NerlMonitor/nerlMonitor_app.erl index 728e31ce..e16b5beb 100644 --- a/src_erl/NerlMonitor/nerlMonitor_app.erl +++ b/src_erl/NerlMonitor/nerlMonitor_app.erl @@ -3,12 +3,16 @@ %% @end %%%------------------------------------------------------------------- --module(nerlGUI_app). +-module(nerlMonitor_app). -behaviour(application). -export([start/2, stop/1]). +-define(UTILLNAME,nerlMonitor ). +-define(PORT,8096 ). %port place holder +-define(MSADDRES,"ip:port" ). %place holder + start(_StartType, _StartArgs) -> application:start(sasl), application:start(ranch), @@ -16,18 +20,52 @@ start(_StartType, _StartArgs) -> Dispatch = cowboy_router:compile([ {'_', [ - {"/[...]", nerlMonitor_handler, [**GUI_PID**]} + {"/utillInfo",nerlUtill_info_handler, []} ]} ]), - {ok, _} = cowboy:start_clear(gui_listener, - [{port, 8096}], - #{env => #{dispatch => Dispatch}} - ), + {ok, _} = cowboy:start_clear(?UTILLNAME,[{port, ?PORT}],#{env => #{dispatch => Dispatch}}), + %GUI start + URL = "http://" ++ ?MSADDRES ++ "/toolConnectionReq", + mainServerPing(URL,{?UTILLNAME,functions()}). + + +%ping main server in 0.5 sec intervals with connection request. will stop when got valid response. +mainServerPing(URL,Body)-> + Response=httpc:request(post,{URL, [],"application/x-www-form-urlencoded",Body}, [], []), + case Response of + {error,_}-> + timer:sleep(500), + mainServerPing(URL,Body); + {ok,{_ResCode, _Headers, Data}}-> + initInfoProc(Data) + end. + - nerlGUI_sup:start_link(). +initInfoProc(Body)-> + DevicesInfo = string:split(Body, "#", all), + Devices = [string:split(DeviceInfo, ",", all) || DeviceInfo <- DevicesInfo, DevicesInfo /=[[]]], + Edges = lists:droplast(lists:last(Devices)), + DeviceList = lists:droplast(Devices), + {DeviceList,Edges}. %need to send to gui for display + +functions()-> + F1 = fun + F(EtsRef) -> + WorkersNames = ets:lookup(EtsRef,workersNames), + WorkerPids = [element(2,ets:lookup(EtsRef,Name))|| Name<-WorkersNames], + + lists:foreach(fun(Pid)->monitor(process , Pid)end , WorkerPids), + + F2 = fun + FF(EtsRef) -> + receive + {'DOWN',_,process,DownPid,Reason}-> + + end. + stop(_State) -> ok. -%% internal functions + From 79a0e067d5357fecf0ab6470eb66034ee4d3efc4 Mon Sep 17 00:00:00 2001 From: GuyPerets106 Date: Tue, 18 Jul 2023 12:52:23 +0000 Subject: [PATCH 07/40] [NERLMONITOR] Added MonitorGUI.py --- src_erl/NerlGUI/src/graphScreen.erl | 4 ++-- src_erl/NerlMonitor/MonitorGUI.py | 19 ++++++++++++++++--- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/src_erl/NerlGUI/src/graphScreen.erl b/src_erl/NerlGUI/src/graphScreen.erl index 6254dd8f..9711a61d 100644 --- a/src_erl/NerlGUI/src/graphScreen.erl +++ b/src_erl/NerlGUI/src/graphScreen.erl @@ -67,13 +67,13 @@ init([Parent, Gen])-> io:format("got body: ~p~n", [Body]), DevicesInfo = string:split(Body, "#", all), - Devices = [string:split(DeviceInfo, ",", all) || DeviceInfo <- DevicesInfo, DevicesInfo /=[[]]], + Devices = [string:split(DeviceInfo, ",", all) || DeviceInfo <- DevicesInfo, DevicesInfo /= [[]]], Edges = lists:droplast(lists:last(Devices)), io:format("got graph: ~p~n", [Devices]), DeviceList = lists:droplast(Devices), - {FileName, G} = gui_tools:makeGraphIMG(DeviceList, Edges), + {FileName, G} = gui_tools:makeGraphIMG(22), mainScreen:updateGraph(Gen, gui_tools:serialize(G)), mainScreen:addInfo(Gen, "updated graph"), diff --git a/src_erl/NerlMonitor/MonitorGUI.py b/src_erl/NerlMonitor/MonitorGUI.py index 257087a4..5a6cf3df 100644 --- a/src_erl/NerlMonitor/MonitorGUI.py +++ b/src_erl/NerlMonitor/MonitorGUI.py @@ -2,11 +2,24 @@ import PySimpleGUI as sg layout = [ - [sg.Text("NerlNet Monitor" , size=(30,1) ,text_color='Black' , font=('SFPro' , 20) , background_color='#A9D8E6' , justification='center' , pad=((0,0) , (10,0)))] , - [sg.Button("Close" , button_color=('#94D5F2' , '#000000') , font=('SFPro' , 12))] + [ + sg.Text("NerlNet Monitor" , key='-TEXT-' , size=(30,1) ,text_color='Black' , font=('SFPro' , 20) , background_color='#078FCA' , justification='center' , pad=(0,0)) , + sg.Image('NerlnetLogo.png' , size=(50,50)) + ] , + [ sg.Button("NerlNet Graph" , button_color=('#94D5F2' , '#000000') , font=('SFPro' , 12) , size=(10,5)) , + sg.Frame('Event Log:' , + layout=[[sg.Column( + [[sg.Multiline('', size=(100, 20), key='-LOG-', autoscroll=True)]]) + ]] , + background_color=('#078FCA') , font=('SFPro' , 20) , size=(400,200) , title_color='Black' , vertical_alignment='bottom') + ] , + [ + sg.Button("Close" , button_color=('#94D5F2' , '#000000') , font=('SFPro' , 12) , size=(5,2)) + ] + ] -MainWindow = sg.Window("NErlNet" , layout , margins=(10,10) , size=(640,400) , background_color='#A9D8E6' , finalize=True , resizable=True) +MainWindow = sg.Window("NErlNet" , layout , margins=(5,5) , size=(640,400) , background_color='#078FCA' , finalize=True , resizable=True , element_justification='c') while True: event , values = MainWindow.read() From b303eec9be60e8232d883ed219b49c277d0128d6 Mon Sep 17 00:00:00 2001 From: vboxuser Date: Fri, 21 Jul 2023 11:32:39 +0300 Subject: [PATCH 08/40] [nerlMonitor] updated files --- .../http_Nerlserver/src/Client/clientStatem.erl | 16 +++++++++++++--- src_erl/NerlMonitor/nerlMonitor_app.erl | 15 ++------------- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/src_erl/Communication_Layer/http_Nerlserver/src/Client/clientStatem.erl b/src_erl/Communication_Layer/http_Nerlserver/src/Client/clientStatem.erl index 1511ead6..d5d10c43 100644 --- a/src_erl/Communication_Layer/http_Nerlserver/src/Client/clientStatem.erl +++ b/src_erl/Communication_Layer/http_Nerlserver/src/Client/clientStatem.erl @@ -123,12 +123,22 @@ waitforWorkers(cast, EventContent, State = #client_statem_state{etsRef = EtsRef} ets:update_counter(EtsRef, msgCounter, 1), ets:update_counter(EtsRef, infoIn, nerl_tools:calculate_size(EventContent)), ?LOG_WARNING("client waitforWorkers ignored!!!: ~p ~n",[EventContent]), - {next_state, waitforWorkers, State}. + {next_state, waitforWorkers, State}; -waitforWorkers(info, EventContent, State) -> +%---------------------------------------------------------------- +waitforWorkers(info, EventContent, State = #client_statem_state{etsRef = EtsRef}) -> case EventContent of - {'DOWN',Ref,process,Pid,Reason}->%tell main server + {'DOWN',_Ref,process,Pid,_Reason}-> %worker down + [[Workername]] = ets:match(EtsRef,{'$1',Pid,'_','_','_'}), + %report worker down to main server + ets:delete(EtsRef,Workername), %delete worker from ets so client will not wait for it in the future + ok; + _Any->ok %recevied unexpected messege, print to log/tell main server for debuging + end, + {next_state, waitforWorkers, State#client_statem_state{etsRef = EtsRef}}. + +%----------------------------------------------------------------- %% initiating workers when they include federated workers. init stage == handshake between federated worker client and server diff --git a/src_erl/NerlMonitor/nerlMonitor_app.erl b/src_erl/NerlMonitor/nerlMonitor_app.erl index e16b5beb..1278a15e 100644 --- a/src_erl/NerlMonitor/nerlMonitor_app.erl +++ b/src_erl/NerlMonitor/nerlMonitor_app.erl @@ -49,20 +49,9 @@ initInfoProc(Body)-> DeviceList = lists:droplast(Devices), {DeviceList,Edges}. %need to send to gui for display -functions()-> - F1 = fun - F(EtsRef) -> - WorkersNames = ets:lookup(EtsRef,workersNames), - WorkerPids = [element(2,ets:lookup(EtsRef,Name))|| Name<-WorkersNames], - lists:foreach(fun(Pid)->monitor(process , Pid)end , WorkerPids), - - F2 = fun - FF(EtsRef) -> - receive - {'DOWN',_,process,DownPid,Reason}-> - - end. +%functionalty of the tool, will create a list of tuples when each tuple is {entity that will run the func (atom),function itself {func)} +functions()->ok. stop(_State) -> From 991b3429d577804c0123eabb40940bab88cfc6a2 Mon Sep 17 00:00:00 2001 From: GuyPerets106 Date: Fri, 21 Jul 2023 08:36:13 +0000 Subject: [PATCH 09/40] [NerlMonitor] Updated MonitorGUI --- src_erl/NerlMonitor/MonitorGUI.py | 129 +++++++++++++++++++++++++----- 1 file changed, 108 insertions(+), 21 deletions(-) diff --git a/src_erl/NerlMonitor/MonitorGUI.py b/src_erl/NerlMonitor/MonitorGUI.py index 5a6cf3df..4d6d27fa 100644 --- a/src_erl/NerlMonitor/MonitorGUI.py +++ b/src_erl/NerlMonitor/MonitorGUI.py @@ -1,29 +1,116 @@ +from term import Atom from pyrlang.node import Node +from pyrlang.process import Process import PySimpleGUI as sg +import multiprocessing +from time import sleep +import networkx as nx +import matplotlib.pyplot as plt +from datetime import datetime + +def draw_gradient(canvas, start_color, end_color): + for y in range(0, 200): # Adjust the range to your desired height + r = start_color[0] + (end_color[0] - start_color[0]) * y / 200 + g = start_color[1] + (end_color[1] - start_color[1]) * y / 200 + b = start_color[2] + (end_color[2] - start_color[2]) * y / 200 + color = f'#{int(r):02x}{int(g):02x}{int(b):02x}' + canvas.TKCanvas.create_line(0, y, 200, y, fill=color) + + +Msg_log = [] layout = [ - [ - sg.Text("NerlNet Monitor" , key='-TEXT-' , size=(30,1) ,text_color='Black' , font=('SFPro' , 20) , background_color='#078FCA' , justification='center' , pad=(0,0)) , - sg.Image('NerlnetLogo.png' , size=(50,50)) - ] , - [ sg.Button("NerlNet Graph" , button_color=('#94D5F2' , '#000000') , font=('SFPro' , 12) , size=(10,5)) , - sg.Frame('Event Log:' , - layout=[[sg.Column( - [[sg.Multiline('', size=(100, 20), key='-LOG-', autoscroll=True)]]) - ]] , - background_color=('#078FCA') , font=('SFPro' , 20) , size=(400,200) , title_color='Black' , vertical_alignment='bottom') - ] , - [ - sg.Button("Close" , button_color=('#94D5F2' , '#000000') , font=('SFPro' , 12) , size=(5,2)) + [ + sg.Text("NerlNet Monitor" , key='-TEXT-' , size=(30,1) ,text_color='Black' , font=('SFPro' , 20) , background_color='#930707' , justification='center' , pad=(0,0)) + ] , + [ sg.Text("Waiting For\n NerlNet Graph..." , text_color='Black' , font=('SFPro' , 12) , size=(50,5) , background_color='#930707' , justification='center' , pad=(0,0)) , + sg.Image(key='-IMAGE-' , visible=False , size=(500,400)) , + sg.Frame(title="Event Log:" , + layout=[[sg.Multiline('', size=(100, 20), key='-LOG-', autoscroll=True , font=('SFPro' , 12))]], + background_color=('#930707') , font=('SFPro' , 20) , size=(300,400) , title_color='Black' , element_justification='right') + ] , + [ + sg.Button(button_text="Close" , button_color=('#C30404' , '#000000') , font=('SFPro' , 12) , size=(5,2)), + sg.Button(button_text="Clear Log" , button_color=('#C30404' , '#000000') , font=('SFPro' , 12) , size=(5,2)) + ] + ] - - ] -MainWindow = sg.Window("NErlNet" , layout , margins=(5,5) , size=(640,400) , background_color='#078FCA' , finalize=True , resizable=True , element_justification='c') +MainWindow = sg.Window("NErlNet" , layout , margins=(5,5) , size=(800,500) , background_color='#930707' , finalize=True , resizable=False , element_justification='c') + + + +def formatted_time(): + return f'[{datetime.now().day}/{datetime.now().month}/{datetime.now().year}|{datetime.now().hour}:{datetime.now().minute}:{datetime.now().second}]' + +def GUI(): + while True: + event , values = MainWindow.read(timeout=100) + if event == "Close" or event == sg.WIN_CLOSED: + break + if event == "Clear Log": + MainWindow['-LOG-'].update('') + if event == "NerlNet Graph": + Show_Nerlnet_Graph() + if not msg_queue.empty(): + msg = msg_queue.get_nowait() + if "Graph" in msg: + Show_Nerlnet_Graph(msg) + elif values['-LOG-'] != '': + existing_text = values['-LOG-'] + updated_text = f'{existing_text}\n{formatted_time()}: {msg}' + else: + updated_text = f'{formatted_time()}: {msg}' + MainWindow['-LOG-'].update(updated_text) + + + + MainWindow.close() + +def Show_Nerlnet_Graph(NerlGraph): + # Graph got in string format: "Entity1Name,Entity1IP,Entity1Port#Entity2Name,Entity2IP,Entity2Port#Entity1-Entity2,Entity2-Entity1" etc. + # Node is defined by a triplet 'Name,IP,Port' seperated by '#' + # Edge is defined by a string 'Entity1-Entity2' seperated by ',' + + Nodes = NerlGraph.split('#')[0:-1] + Edges = NerlGraph.split('#')[-1].split(',') + EdgesSeperated = [(Edge.split('-')[0],Edge.split('-')[1]) for Edge in Edges if len(Edges) > 1] + NodesNames = [NodeTriplet.split(',')[0] for NodeTriplet in Nodes] + + graph = nx.Graph() + graph.add_nodes_from(NodesNames) + graph.add_edges_from(EdgesSeperated) + + spring_pos = nx.spring_layout(graph) + nx.draw_networkx(graph, spring_pos, with_labels=True, node_color='lightblue', node_size=500, font_size=12, font_color='black') + plt.axis('off') + plt.show() + + +class MyProcess(Process): + def __init__(self , msg_queue) -> None: + Process.__init__(self) + self.get_node().register_name(self, Atom('PyrlangProcess')) + self.msg_queue = msg_queue + + + def handle_one_inbox_message(self, msg): + print(f'From Pyrlang: {msg}') + self.msg_queue.put(msg) + if not self.msg_queue.empty(): + print(f'Queue is not Empty: {msg} added.') + +if __name__ == "__main__": + msg_queue = multiprocessing.Queue() + + GUI_Process = multiprocessing.Process(target=GUI) + GUI_Process.start() + + print("Starting a Pyrlang node...") + PyNode = Node(node_name="py@127.0.0.1" , cookie="COOKIE") + MyProcess(msg_queue) + PyNode.run() -while True: - event , values = MainWindow.read() - if event == "Close" or event == sg.WIN_CLOSED: - break + -MainWindow.close() \ No newline at end of file + From ac2fc31b7c2faa21d11a2167a1d7fdb92ccc1068 Mon Sep 17 00:00:00 2001 From: GuyPerets106 Date: Fri, 21 Jul 2023 10:35:17 +0000 Subject: [PATCH 10/40] [NerlMonitor] Updated files for messages passing --- .../src/MainServer/mainGenserver.erl | 6 +++--- .../http_Nerlserver/src/nerlNetServer_app.erl | 2 +- .../http_Nerlserver/src/utilities_handler.erl | 18 ++++++++++++++++++ src_erl/NerlMonitor/MonitorGUI.py | 6 +++--- src_erl/NerlMonitor/nerlMonitor_app.erl | 17 +++++++---------- 5 files changed, 32 insertions(+), 17 deletions(-) diff --git a/src_erl/Communication_Layer/http_Nerlserver/src/MainServer/mainGenserver.erl b/src_erl/Communication_Layer/http_Nerlserver/src/MainServer/mainGenserver.erl index 09e84c24..210177d2 100644 --- a/src_erl/Communication_Layer/http_Nerlserver/src/MainServer/mainGenserver.erl +++ b/src_erl/Communication_Layer/http_Nerlserver/src/MainServer/mainGenserver.erl @@ -22,7 +22,7 @@ -define(SERVER, ?MODULE). --record(main_genserver_state, {statisticsCounter = 0, myName, state, workersMap, clients, nerlnetGraph, sourcesCastingList = [], sourcesWaitingList = [], clientsWaitingList = [], statisticsMap, msgCounter = 0, batchSize}). +-record(main_genserver_state, {statisticsCounter = 0, myName, state, workersMap, clients, nerlnetGraph, sourcesCastingList = [], sourcesWaitingList = [], clientsWaitingList = [], statisticsMap, msgCounter = 0, batchSize , etsRef}). %%%=============================================================== %%% API @@ -56,10 +56,10 @@ init({MyName,Clients,BatchSize,WorkersMap,NerlnetGraph}) -> ?LOG_NOTICE("Main Server is connected to: ~p~n",[ConnectedEntities]), put(nerlnetGraph, NerlnetGraph), % nerl_tools:start_connection([digraph:vertex(NerlnetGraph,Vertex) || Vertex <- digraph:out_neighbours(NerlnetGraph,MyName)]), - + EtsRef = ets:new() , %% ! In some point the whole record will replaced by ets NewStatisticsMap = getNewStatisticsMap([digraph:vertex(NerlnetGraph,Vertex) || Vertex <- digraph:vertices(NerlnetGraph)--?LIST_OF_SPECIAL_SERVERS]), % io:format("New StatisticsMap = ~p~n",[NewStatisticsMap]), - {ok, #main_genserver_state{myName = MyNameStr, workersMap = WorkersMap, batchSize = BatchSize, state=idle, clients = Clients, nerlnetGraph = NerlnetGraph, msgCounter = 1,statisticsMap = NewStatisticsMap}}. + {ok, #main_genserver_state{myName = MyNameStr, workersMap = WorkersMap, batchSize = BatchSize, state=idle, clients = Clients, nerlnetGraph = NerlnetGraph, msgCounter = 1,statisticsMap = NewStatisticsMap , etsRef = EtsRef}}. %% @private %% @doc Handling call messages diff --git a/src_erl/Communication_Layer/http_Nerlserver/src/nerlNetServer_app.erl b/src_erl/Communication_Layer/http_Nerlserver/src/nerlNetServer_app.erl index 70e9454f..fcad087c 100644 --- a/src_erl/Communication_Layer/http_Nerlserver/src/nerlNetServer_app.erl +++ b/src_erl/Communication_Layer/http_Nerlserver/src/nerlNetServer_app.erl @@ -304,7 +304,7 @@ createMainServer(true,BatchSize,HostName) -> %GUI actions {"/getGraph",[],guiHandler, [getGraph, MainGenServerPid]}, {"/getStats",[],guiHandler, [getStats, MainGenServerPid]}, - + {"/toolConnectionReq" , [] , utilities_handler , [MainGenServerPid]} , %% Added with NerlMonitor Project {"/[...]", [],noMatchingRouteHandler, [MainGenServerPid]} ]} ]), diff --git a/src_erl/Communication_Layer/http_Nerlserver/src/utilities_handler.erl b/src_erl/Communication_Layer/http_Nerlserver/src/utilities_handler.erl index e69de29b..c8d87922 100644 --- a/src_erl/Communication_Layer/http_Nerlserver/src/utilities_handler.erl +++ b/src_erl/Communication_Layer/http_Nerlserver/src/utilities_handler.erl @@ -0,0 +1,18 @@ +-module(utilities_handler). + +-export([init/2]). + +init(Req0 , [Pid]) -> + {_,Body,_} = cowboy_req:read_body(Req0), + [UtilityName , _Funcs , IP , Port] = binary_to_list(Body), + case UtilityName of + nerlMonitor -> + {reply , Reply , _ } = gen_server:call(Pid , getGraph) + + end, + gen_server:cast(), %% TODO Add IP , Port to ets in main server record + Req = cowboy_req:reply(200, + #{<<"content-type">> => <<"text/plain">>}, + Reply, + Req0), + {ok, Req, Pid}. diff --git a/src_erl/NerlMonitor/MonitorGUI.py b/src_erl/NerlMonitor/MonitorGUI.py index 4d6d27fa..9b542614 100644 --- a/src_erl/NerlMonitor/MonitorGUI.py +++ b/src_erl/NerlMonitor/MonitorGUI.py @@ -54,8 +54,8 @@ def GUI(): Show_Nerlnet_Graph() if not msg_queue.empty(): msg = msg_queue.get_nowait() - if "Graph" in msg: - Show_Nerlnet_Graph(msg) + if msg[0] == 'graph': + Show_Nerlnet_Graph(msg[1]) elif values['-LOG-'] != '': existing_text = values['-LOG-'] updated_text = f'{existing_text}\n{formatted_time()}: {msg}' @@ -74,7 +74,7 @@ def Show_Nerlnet_Graph(NerlGraph): Nodes = NerlGraph.split('#')[0:-1] Edges = NerlGraph.split('#')[-1].split(',') - EdgesSeperated = [(Edge.split('-')[0],Edge.split('-')[1]) for Edge in Edges if len(Edges) > 1] + EdgesSeperated = [(Edge.split('-')[0],Edge.split('-')[1]) for Edge in Edges if len(Edges) > 1] # ? What if no edges? NodesNames = [NodeTriplet.split(',')[0] for NodeTriplet in Nodes] graph = nx.Graph() diff --git a/src_erl/NerlMonitor/nerlMonitor_app.erl b/src_erl/NerlMonitor/nerlMonitor_app.erl index 1278a15e..2de7a786 100644 --- a/src_erl/NerlMonitor/nerlMonitor_app.erl +++ b/src_erl/NerlMonitor/nerlMonitor_app.erl @@ -12,6 +12,7 @@ -define(UTILLNAME,nerlMonitor ). -define(PORT,8096 ). %port place holder -define(MSADDRES,"ip:port" ). %place holder +-define(GUI , {'PyrlangProcess' , 'py@127.0.0.1'}). % Erlang node should be long name to communicate with pyrlang node start(_StartType, _StartArgs) -> application:start(sasl), @@ -25,29 +26,25 @@ start(_StartType, _StartArgs) -> ]} ]), {ok, _} = cowboy:start_clear(?UTILLNAME,[{port, ?PORT}],#{env => #{dispatch => Dispatch}}), - %GUI start + os:cmd('python3 MonitorGUI.py'), %% PyrlangNode: ('PyralngProcess' , 'py@127.0.0.1' , 'COOKIE') , sending message by: 'GUI ! HELLO.' URL = "http://" ++ ?MSADDRES ++ "/toolConnectionReq", - mainServerPing(URL,{?UTILLNAME,functions()}). + mainServerPing(URL,[?UTILLNAME,functions(),nerl_tools:getdeviceIP() , ?PORT]). %% TODO How to "import" nerl_tools %ping main server in 0.5 sec intervals with connection request. will stop when got valid response. mainServerPing(URL,Body)-> - Response=httpc:request(post,{URL, [],"application/x-www-form-urlencoded",Body}, [], []), + Response = httpc:request(post,{URL, [],"application/x-www-form-urlencoded",Body}, [], []), case Response of {error,_}-> timer:sleep(500), mainServerPing(URL,Body); - {ok,{_ResCode, _Headers, Data}}-> + {ok,{_ResCode, _Headers, Data}}-> initInfoProc(Data) end. -initInfoProc(Body)-> - DevicesInfo = string:split(Body, "#", all), - Devices = [string:split(DeviceInfo, ",", all) || DeviceInfo <- DevicesInfo, DevicesInfo /=[[]]], - Edges = lists:droplast(lists:last(Devices)), - DeviceList = lists:droplast(Devices), - {DeviceList,Edges}. %need to send to gui for display +initInfoProc(Body)-> %% MainServer replies with Nerlnet-Graph when nerlMonitor tool is used + ?GUI ! {graph , Body}. %functionalty of the tool, will create a list of tuples when each tuple is {entity that will run the func (atom),function itself {func)} From 4021fcf9b4ab615498c5351e49ab652be46502c2 Mon Sep 17 00:00:00 2001 From: GuyPerets106 Date: Tue, 1 Aug 2023 11:19:15 +0000 Subject: [PATCH 11/40] Updated NerlMonitorGUI and First Communication with MainServer --- .../src/MainServer/mainGenserver.erl | 4 ++ .../http_Nerlserver/src/utilities_handler.erl | 9 ++-- src_erl/NerlMonitor/MonitorGUI.py | 41 ++++++++++++------- src_erl/NerlMonitor/nerlMonitor_app.erl | 7 ++-- 4 files changed, 38 insertions(+), 23 deletions(-) diff --git a/src_erl/Communication_Layer/http_Nerlserver/src/MainServer/mainGenserver.erl b/src_erl/Communication_Layer/http_Nerlserver/src/MainServer/mainGenserver.erl index 210177d2..6f43cbcf 100644 --- a/src_erl/Communication_Layer/http_Nerlserver/src/MainServer/mainGenserver.erl +++ b/src_erl/Communication_Layer/http_Nerlserver/src/MainServer/mainGenserver.erl @@ -113,6 +113,10 @@ edgeString([Edge |EdgesList], Str)-> {noreply, NewState :: #main_genserver_state{}, timeout() | hibernate} | {stop, Reason :: term(), NewState :: #main_genserver_state{}}). +handle_cast({saveUtility , Body} , State = #main_genserver_state{etsRef = EtsRef , msgCounter = MsgCounter}) -> + ets:insert(EtsRef , Body), + {noreply, State#main_genserver_state{msgCounter = MsgCounter+1}}; + handle_cast({initCSV, Source,SourceData}, State = #main_genserver_state{state = idle, myName = MyName, sourcesWaitingList = SourcesWaitingList,nerlnetGraph = NerlnetGraph,msgCounter = MsgCounter}) -> %% send router http request, to rout this message to all sensors diff --git a/src_erl/Communication_Layer/http_Nerlserver/src/utilities_handler.erl b/src_erl/Communication_Layer/http_Nerlserver/src/utilities_handler.erl index c8d87922..5ea193ad 100644 --- a/src_erl/Communication_Layer/http_Nerlserver/src/utilities_handler.erl +++ b/src_erl/Communication_Layer/http_Nerlserver/src/utilities_handler.erl @@ -2,17 +2,16 @@ -export([init/2]). -init(Req0 , [Pid]) -> +init(Req0 , [MainServerPid]) -> {_,Body,_} = cowboy_req:read_body(Req0), [UtilityName , _Funcs , IP , Port] = binary_to_list(Body), case UtilityName of nerlMonitor -> - {reply , Reply , _ } = gen_server:call(Pid , getGraph) - + {reply , Reply , _ } = gen_server:call(MainServerPid , getGraph) end, - gen_server:cast(), %% TODO Add IP , Port to ets in main server record + gen_server:cast(MainServerPid , {saveUtility , {UtilityName , IP , Port}}), %% TODO Add IP , Port to ets in main server record Req = cowboy_req:reply(200, #{<<"content-type">> => <<"text/plain">>}, Reply, Req0), - {ok, Req, Pid}. + {ok, Req, MainServerPid}. diff --git a/src_erl/NerlMonitor/MonitorGUI.py b/src_erl/NerlMonitor/MonitorGUI.py index 9b542614..10588072 100644 --- a/src_erl/NerlMonitor/MonitorGUI.py +++ b/src_erl/NerlMonitor/MonitorGUI.py @@ -7,6 +7,7 @@ import networkx as nx import matplotlib.pyplot as plt from datetime import datetime +import os def draw_gradient(canvas, start_color, end_color): for y in range(0, 200): # Adjust the range to your desired height @@ -23,11 +24,12 @@ def draw_gradient(canvas, start_color, end_color): [ sg.Text("NerlNet Monitor" , key='-TEXT-' , size=(30,1) ,text_color='Black' , font=('SFPro' , 20) , background_color='#930707' , justification='center' , pad=(0,0)) ] , - [ sg.Text("Waiting For\n NerlNet Graph..." , text_color='Black' , font=('SFPro' , 12) , size=(50,5) , background_color='#930707' , justification='center' , pad=(0,0)) , - sg.Image(key='-IMAGE-' , visible=False , size=(500,400)) , - sg.Frame(title="Event Log:" , - layout=[[sg.Multiline('', size=(100, 20), key='-LOG-', autoscroll=True , font=('SFPro' , 12))]], - background_color=('#930707') , font=('SFPro' , 20) , size=(300,400) , title_color='Black' , element_justification='right') + [ sg.Frame(title="Event Log:" , + layout=[[sg.Multiline('', size=(100, 20), key='-LOG-', autoscroll=True , font=('SFPro' , 12) , no_scrollbar=True)]], + background_color=('#930707') , font=('SFPro' , 20) , size=(300,400) , title_color='Black' , element_justification='right') , + sg.Text("Waiting For\n NerlNet Graph..." , key='-PHOLD-', text_color='Black' , font=('SFPro' , 12) , size=(50,5) , background_color='#930707' , justification='center' , pad=(0,0)) , + sg.Image(key='-IMAGE-' , visible=False) + ] , [ sg.Button(button_text="Close" , button_color=('#C30404' , '#000000') , font=('SFPro' , 12) , size=(5,2)), @@ -36,17 +38,20 @@ def draw_gradient(canvas, start_color, end_color): ] -MainWindow = sg.Window("NErlNet" , layout , margins=(5,5) , size=(800,500) , background_color='#930707' , finalize=True , resizable=False , element_justification='c') +MainWindow = sg.Window("NErlNet" , layout , margins=(5,5) , size=(800,500) , background_color='#930707' , finalize=True , resizable=True , element_justification='c') def formatted_time(): return f'[{datetime.now().day}/{datetime.now().month}/{datetime.now().year}|{datetime.now().hour}:{datetime.now().minute}:{datetime.now().second}]' -def GUI(): +def GUI(MainPid): while True: event , values = MainWindow.read(timeout=100) + updated_text = '' if event == "Close" or event == sg.WIN_CLOSED: + os.kill(MainPid , 9) + print("GUI Closed.") break if event == "Clear Log": MainWindow['-LOG-'].update('') @@ -56,19 +61,23 @@ def GUI(): msg = msg_queue.get_nowait() if msg[0] == 'graph': Show_Nerlnet_Graph(msg[1]) + MainWindow['-PHOLD-'].update(visible=False) + MainWindow['-IMAGE-'].update(filename='NerlNetGraph.png' , visible=True , size=(410,310)) elif values['-LOG-'] != '': existing_text = values['-LOG-'] updated_text = f'{existing_text}\n{formatted_time()}: {msg}' else: updated_text = f'{formatted_time()}: {msg}' - MainWindow['-LOG-'].update(updated_text) + if updated_text != '': + MainWindow['-LOG-'].update(updated_text) MainWindow.close() + def Show_Nerlnet_Graph(NerlGraph): - # Graph got in string format: "Entity1Name,Entity1IP,Entity1Port#Entity2Name,Entity2IP,Entity2Port#Entity1-Entity2,Entity2-Entity1" etc. + # Graph got in string format: "Entity1Name,Entity1IP,Entity1Port#Entity2Name,Entity2IP,Entity2Port#Entity1Name-Entity2Name,Entity2Name-Entity1Name" etc. # Node is defined by a triplet 'Name,IP,Port' seperated by '#' # Edge is defined by a string 'Entity1-Entity2' seperated by ',' @@ -80,11 +89,11 @@ def Show_Nerlnet_Graph(NerlGraph): graph = nx.Graph() graph.add_nodes_from(NodesNames) graph.add_edges_from(EdgesSeperated) - + #print(graph.nodes , graph.edges) spring_pos = nx.spring_layout(graph) nx.draw_networkx(graph, spring_pos, with_labels=True, node_color='lightblue', node_size=500, font_size=12, font_color='black') - plt.axis('off') - plt.show() + plt.savefig('NerlNetGraph.png' ,bbox_inches='tight' , dpi=80) + plt.close() class MyProcess(Process): @@ -99,11 +108,13 @@ def handle_one_inbox_message(self, msg): self.msg_queue.put(msg) if not self.msg_queue.empty(): print(f'Queue is not Empty: {msg} added.') + if __name__ == "__main__": msg_queue = multiprocessing.Queue() - - GUI_Process = multiprocessing.Process(target=GUI) + + PyrlangPid = os.getpid() + GUI_Process = multiprocessing.Process(target=GUI , args=(PyrlangPid,)) GUI_Process.start() print("Starting a Pyrlang node...") @@ -113,4 +124,4 @@ def handle_one_inbox_message(self, msg): - + diff --git a/src_erl/NerlMonitor/nerlMonitor_app.erl b/src_erl/NerlMonitor/nerlMonitor_app.erl index 2de7a786..f50dd16e 100644 --- a/src_erl/NerlMonitor/nerlMonitor_app.erl +++ b/src_erl/NerlMonitor/nerlMonitor_app.erl @@ -9,7 +9,8 @@ -export([start/2, stop/1]). --define(UTILLNAME,nerlMonitor ). +-define(UTILLNAME,nerlMonitor). +-define(IP , nerl_tools:getdeviceIP()). -define(PORT,8096 ). %port place holder -define(MSADDRES,"ip:port" ). %place holder -define(GUI , {'PyrlangProcess' , 'py@127.0.0.1'}). % Erlang node should be long name to communicate with pyrlang node @@ -28,7 +29,7 @@ start(_StartType, _StartArgs) -> {ok, _} = cowboy:start_clear(?UTILLNAME,[{port, ?PORT}],#{env => #{dispatch => Dispatch}}), os:cmd('python3 MonitorGUI.py'), %% PyrlangNode: ('PyralngProcess' , 'py@127.0.0.1' , 'COOKIE') , sending message by: 'GUI ! HELLO.' URL = "http://" ++ ?MSADDRES ++ "/toolConnectionReq", - mainServerPing(URL,[?UTILLNAME,functions(),nerl_tools:getdeviceIP() , ?PORT]). %% TODO How to "import" nerl_tools + mainServerPing(URL,[?UTILLNAME,functions(), ?IP , ?PORT]). %% TODO How to "import" nerl_tools %ping main server in 0.5 sec intervals with connection request. will stop when got valid response. @@ -48,7 +49,7 @@ initInfoProc(Body)-> %% MainServer replies with Nerlnet-Graph when nerlMonitor t %functionalty of the tool, will create a list of tuples when each tuple is {entity that will run the func (atom),function itself {func)} -functions()->ok. +functions()-> ok. stop(_State) -> From 9ad8cde504d0ade3959c2bfc86a96666cc7b0f23 Mon Sep 17 00:00:00 2001 From: GuyPerets106 Date: Tue, 1 Aug 2023 13:19:33 +0000 Subject: [PATCH 12/40] Added WorkersMap to NerlGraph --- .../http_Nerlserver/src/utilities_handler.erl | 6 +++++- src_erl/NerlMonitor/MonitorGUI.py | 13 ++++++++----- src_erl/NerlMonitor/nerlMonitor_app.erl | 2 +- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/src_erl/Communication_Layer/http_Nerlserver/src/utilities_handler.erl b/src_erl/Communication_Layer/http_Nerlserver/src/utilities_handler.erl index 5ea193ad..fce98a99 100644 --- a/src_erl/Communication_Layer/http_Nerlserver/src/utilities_handler.erl +++ b/src_erl/Communication_Layer/http_Nerlserver/src/utilities_handler.erl @@ -1,5 +1,7 @@ -module(utilities_handler). +-define(DATA_IDX , 2). + -export([init/2]). init(Req0 , [MainServerPid]) -> @@ -7,7 +9,9 @@ init(Req0 , [MainServerPid]) -> [UtilityName , _Funcs , IP , Port] = binary_to_list(Body), case UtilityName of nerlMonitor -> - {reply , Reply , _ } = gen_server:call(MainServerPid , getGraph) + {reply , Graph , _ } = gen_server:call(MainServerPid , getGraph), + WorkersMap = ets:lookup_element(nerlnet_data , workers , ?DATA_IDX), + Reply = {Graph , WorkersMap} end, gen_server:cast(MainServerPid , {saveUtility , {UtilityName , IP , Port}}), %% TODO Add IP , Port to ets in main server record Req = cowboy_req:reply(200, diff --git a/src_erl/NerlMonitor/MonitorGUI.py b/src_erl/NerlMonitor/MonitorGUI.py index 10588072..153a2aa3 100644 --- a/src_erl/NerlMonitor/MonitorGUI.py +++ b/src_erl/NerlMonitor/MonitorGUI.py @@ -60,7 +60,7 @@ def GUI(MainPid): if not msg_queue.empty(): msg = msg_queue.get_nowait() if msg[0] == 'graph': - Show_Nerlnet_Graph(msg[1]) + Show_Nerlnet_Graph(msg[1] , msg[2]) MainWindow['-PHOLD-'].update(visible=False) MainWindow['-IMAGE-'].update(filename='NerlNetGraph.png' , visible=True , size=(410,310)) elif values['-LOG-'] != '': @@ -76,15 +76,18 @@ def GUI(MainPid): MainWindow.close() -def Show_Nerlnet_Graph(NerlGraph): - # Graph got in string format: "Entity1Name,Entity1IP,Entity1Port#Entity2Name,Entity2IP,Entity2Port#Entity1Name-Entity2Name,Entity2Name-Entity1Name" etc. +def Show_Nerlnet_Graph(NerlGraph , Workers): + # Graph in string format: "Entity1Name,Entity1IP,Entity1Port#Entity2Name,Entity2IP,Entity2Port#Entity1Name-Entity2Name,Entity2Name-Entity1Name" etc. + # Workers in list format: [WorkerName , ClientName] # Node is defined by a triplet 'Name,IP,Port' seperated by '#' # Edge is defined by a string 'Entity1-Entity2' seperated by ',' - - Nodes = NerlGraph.split('#')[0:-1] + WorkersNames = [Worker[0] for Worker in Workers] + Nodes = NerlGraph.split('#')[0:-1] Edges = NerlGraph.split('#')[-1].split(',') EdgesSeperated = [(Edge.split('-')[0],Edge.split('-')[1]) for Edge in Edges if len(Edges) > 1] # ? What if no edges? + EdgesSeperated.append(Workers) NodesNames = [NodeTriplet.split(',')[0] for NodeTriplet in Nodes] + NodesNames.append(WorkersNames) graph = nx.Graph() graph.add_nodes_from(NodesNames) diff --git a/src_erl/NerlMonitor/nerlMonitor_app.erl b/src_erl/NerlMonitor/nerlMonitor_app.erl index f50dd16e..96ea9d5d 100644 --- a/src_erl/NerlMonitor/nerlMonitor_app.erl +++ b/src_erl/NerlMonitor/nerlMonitor_app.erl @@ -29,7 +29,7 @@ start(_StartType, _StartArgs) -> {ok, _} = cowboy:start_clear(?UTILLNAME,[{port, ?PORT}],#{env => #{dispatch => Dispatch}}), os:cmd('python3 MonitorGUI.py'), %% PyrlangNode: ('PyralngProcess' , 'py@127.0.0.1' , 'COOKIE') , sending message by: 'GUI ! HELLO.' URL = "http://" ++ ?MSADDRES ++ "/toolConnectionReq", - mainServerPing(URL,[?UTILLNAME,functions(), ?IP , ?PORT]). %% TODO How to "import" nerl_tools + mainServerPing(URL,[?UTILLNAME, functions() , ?IP , ?PORT]). %% TODO How to "import" nerl_tools %ping main server in 0.5 sec intervals with connection request. will stop when got valid response. From 9e76992974fda661bbed122143cc9dc54a52f329 Mon Sep 17 00:00:00 2001 From: vboxuser Date: Tue, 1 Aug 2023 17:50:54 +0300 Subject: [PATCH 13/40] add monitor functionality to entities --- .../src/Client/clientStatem.erl | 21 +++++++++++++------ .../src/MainServer/actionHandler.erl | 4 +++- .../src/MainServer/mainGenserver.erl | 3 +++ .../src/Router/routerGenserver.erl | 6 ++++++ .../src/Router/routingHandler.erl | 4 +++- .../http_Nerlserver/src/nerlNetServer_app.erl | 8 +++++-- src_erl/erlBridge/workers/workerGeneric.erl | 7 ++++++- 7 files changed, 42 insertions(+), 11 deletions(-) diff --git a/src_erl/Communication_Layer/http_Nerlserver/src/Client/clientStatem.erl b/src_erl/Communication_Layer/http_Nerlserver/src/Client/clientStatem.erl index d5d10c43..76187464 100644 --- a/src_erl/Communication_Layer/http_Nerlserver/src/Client/clientStatem.erl +++ b/src_erl/Communication_Layer/http_Nerlserver/src/Client/clientStatem.erl @@ -126,13 +126,13 @@ waitforWorkers(cast, EventContent, State = #client_statem_state{etsRef = EtsRef} {next_state, waitforWorkers, State}; %---------------------------------------------------------------- -waitforWorkers(info, EventContent, State = #client_statem_state{etsRef = EtsRef}) -> +waitforWorkers(info, EventContent, State = #client_statem_state{myName = MyName,etsRef = EtsRef}) -> case EventContent of {'DOWN',_Ref,process,Pid,_Reason}-> %worker down - [[Workername]] = ets:match(EtsRef,{'$1',Pid,'_','_','_'}), + [[WorkerName]] = ets:match(EtsRef,{'$1',Pid,'_','_','_'}), %report worker down to main server - ets:delete(EtsRef,Workername), %delete worker from ets so client will not wait for it in the future - ok; + delete_worker(EtsRef,MyName,WorkerName); %delete worker from ets so client will not wait for it in the future + _Any->ok %recevied unexpected messege, print to log/tell main server for debuging end, {next_state, waitforWorkers, State#client_statem_state{etsRef = EtsRef}}. @@ -438,7 +438,7 @@ createWorkers(ClientName, EtsRef) -> WorkerArgs = {WorkerName,ModelId,ModelType,ScalingMethod, LayerTypesList, LayersSizes, LayersActivationFunctions, Optimizer, LossMethod, LearningRate, self(), CustomFunc, WorkerData}, - WorkerPid = workerGeneric:start_link(WorkerArgs), + WorkerPid = workerGeneric:start_monitor(WorkerArgs), ets:insert(EtsRef, {WorkerName, WorkerPid, WorkerArgs, {0,0,0.0}, 0}), WorkerName @@ -479,4 +479,13 @@ cast_message_to_workers(EtsRef, Msg) -> WorkerPid = ets:lookup_element(EtsRef, WorkerKey, ?WORKER_PID_IDX), gen_statem:cast(WorkerPid, Msg) end, - lists:foreach(Func, Workers). \ No newline at end of file + lists:foreach(Func, Workers). + +%activated if client detected that a worker stoped, will delete it from ets so as not to wait for responsed from it in the future. +delete_worker(EtsRef,MyName,WorkerName)-> + ets:delete(EtsRef,WorkerName), + NameList= ets:lookup_element(EtsRef, workersNames, ?ETS_KV_VAL_IDX), + ets:delete(EtsRef,workersNames), + ets:insert(EtsRef, {workersNames, NameList--[WorkerName]}), + nerl_tools:sendHTTP(MyName, ?MAIN_SERVER_ATOM, "worker_down", term_to_binary({MyName,WorkerName})), + EtsRef. \ No newline at end of file diff --git a/src_erl/Communication_Layer/http_Nerlserver/src/MainServer/actionHandler.erl b/src_erl/Communication_Layer/http_Nerlserver/src/MainServer/actionHandler.erl index adf2b06f..9adb9bb9 100644 --- a/src_erl/Communication_Layer/http_Nerlserver/src/MainServer/actionHandler.erl +++ b/src_erl/Communication_Layer/http_Nerlserver/src/MainServer/actionHandler.erl @@ -25,7 +25,9 @@ init(Req0, [Action, Main_genserver_Pid]) -> predictRes -> gen_statem:cast(Main_genserver_Pid, {predictRes,Body}); statistics -> gen_statem:cast(Main_genserver_Pid, {statistics,Body}); startCasting -> gen_statem:cast(Main_genserver_Pid, {startCasting,Body}); - stopCasting -> gen_statem:cast(Main_genserver_Pid, {stopCasting,Body}) + stopCasting -> gen_statem:cast(Main_genserver_Pid, {stopCasting,Body}); + %monitor + worker_down -> gen_statem:cast(Main_genserver_Pid, {worker_down,Body}) end, Reply = io_lib:format("Body Received: ~p ~n ", [Body]), Req = cowboy_req:reply(200, diff --git a/src_erl/Communication_Layer/http_Nerlserver/src/MainServer/mainGenserver.erl b/src_erl/Communication_Layer/http_Nerlserver/src/MainServer/mainGenserver.erl index 6f43cbcf..82ca3144 100644 --- a/src_erl/Communication_Layer/http_Nerlserver/src/MainServer/mainGenserver.erl +++ b/src_erl/Communication_Layer/http_Nerlserver/src/MainServer/mainGenserver.erl @@ -314,6 +314,9 @@ handle_cast({predictRes,Body}, State = #main_genserver_state{batchSize = BatchSi end, {noreply, State#main_genserver_state{msgCounter = MsgCounter+1}}; +handle_cast({worker_down,Body}, State = #main_genserver_state{}) -> + + {noreply, State}; handle_cast(Request, State = #main_genserver_state{}) -> io:format("main server cast ignored: ~p~n",[Request]), diff --git a/src_erl/Communication_Layer/http_Nerlserver/src/Router/routerGenserver.erl b/src_erl/Communication_Layer/http_Nerlserver/src/Router/routerGenserver.erl index e1da9edf..a3540dbc 100644 --- a/src_erl/Communication_Layer/http_Nerlserver/src/Router/routerGenserver.erl +++ b/src_erl/Communication_Layer/http_Nerlserver/src/Router/routerGenserver.erl @@ -177,6 +177,12 @@ handle_cast({getStats,_Body}, State = #router_genserver_state{myName = MyName, {noreply, State#router_genserver_state{msgCounter = MsgCounter+1}}; +%monitor +handle_cast({worker_down,Body}, State = #router_genserver_state{myName = MyName, msgCounter = MsgCounter, nerlnetGraph = NerlnetGraph}) -> + nerl_tools:sendHTTP(MyName, ?MAIN_SERVER_ATOM, "worker_down", Body), + {noreply, State#router_genserver_state{msgCounter = MsgCounter+1}}; + + handle_cast(_Request, State = #router_genserver_state{msgCounter = MsgCounter }) -> {noreply, State#router_genserver_state{msgCounter = MsgCounter+1}}. diff --git a/src_erl/Communication_Layer/http_Nerlserver/src/Router/routingHandler.erl b/src_erl/Communication_Layer/http_Nerlserver/src/Router/routingHandler.erl index 4dd51b29..235a50d2 100644 --- a/src_erl/Communication_Layer/http_Nerlserver/src/Router/routingHandler.erl +++ b/src_erl/Communication_Layer/http_Nerlserver/src/Router/routingHandler.erl @@ -74,7 +74,9 @@ init(Req0, State) -> % gen_server:cast(Router_genserver_Pid, {federatedWeights,Body}); %%%%%%%%%%%%%%GUI actions - getStats -> gen_server:cast(Router_genserver_Pid, {getStats,Body}) + getStats -> gen_server:cast(Router_genserver_Pid, {getStats,Body}); + %monitor + worker_down-> gen_server:cast(Router_genserver_Pid, {worker_down,Body}) end, Reply = io_lib:format(" ", []), diff --git a/src_erl/Communication_Layer/http_Nerlserver/src/nerlNetServer_app.erl b/src_erl/Communication_Layer/http_Nerlserver/src/nerlNetServer_app.erl index fcad087c..91eec1f2 100644 --- a/src_erl/Communication_Layer/http_Nerlserver/src/nerlNetServer_app.erl +++ b/src_erl/Communication_Layer/http_Nerlserver/src/nerlNetServer_app.erl @@ -265,7 +265,9 @@ createRouters(MapOfRouters, HostName) -> {"/federatedWeights",routingHandler, [federatedWeights,RouterGenServerPid]}, %%GUI actions - {"/getStats",routingHandler, [getStats,RouterGenServerPid]} + {"/getStats",routingHandler, [getStats,RouterGenServerPid]}, + %monitor actions + {"/worker_down",routingHandler, [worker_down,RouterGenServerPid]} ]} ]), %% cowboy:start_clear(Name, TransOpts, ProtoOpts) - an http_listener @@ -305,7 +307,9 @@ createMainServer(true,BatchSize,HostName) -> {"/getGraph",[],guiHandler, [getGraph, MainGenServerPid]}, {"/getStats",[],guiHandler, [getStats, MainGenServerPid]}, {"/toolConnectionReq" , [] , utilities_handler , [MainGenServerPid]} , %% Added with NerlMonitor Project - {"/[...]", [],noMatchingRouteHandler, [MainGenServerPid]} + {"/[...]", [],noMatchingRouteHandler, [MainGenServerPid]}, + %monitor actions + {"/worker_down",actionHandler, [worker_down,MainGenServerPid]} ]} ]), %% cowboy:start_clear(Name, TransOpts, ProtoOpts) - an http_listener diff --git a/src_erl/erlBridge/workers/workerGeneric.erl b/src_erl/erlBridge/workers/workerGeneric.erl index fb859377..40fbb35d 100644 --- a/src_erl/erlBridge/workers/workerGeneric.erl +++ b/src_erl/erlBridge/workers/workerGeneric.erl @@ -16,7 +16,7 @@ -behaviour(gen_statem). %% API --export([start_link/1]). +-export([start_link/1,start_monitor/1]). %% gen_statem callbacks -export([init/1, format_status/2, state_name/3, handle_event/4, terminate/3, code_change/4, callback_mode/0]). @@ -37,6 +37,11 @@ start_link(ARGS) -> {ok,Pid} = gen_statem:start_link(?MODULE, ARGS, []), Pid. +start_monitor(ARGS) -> + %{ok,Pid} = gen_statem:start_link({local, element(1, ARGS)}, ?MODULE, ARGS, []), %% name this machine by unique name + {ok,{Pid,_Ref}} = gen_statem:start_monitor(?MODULE, ARGS, []), + Pid. + %%%=================================================================== %%% gen_statem callbacks %%%=================================================================== From 86f32e4b76c79f6270ca581f54b4cd50e9a1f828 Mon Sep 17 00:00:00 2001 From: GuyPerets106 Date: Tue, 1 Aug 2023 14:56:50 +0000 Subject: [PATCH 14/40] Changed WorkersMap Format --- src_erl/NerlMonitor/MonitorGUI.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src_erl/NerlMonitor/MonitorGUI.py b/src_erl/NerlMonitor/MonitorGUI.py index 153a2aa3..1637c176 100644 --- a/src_erl/NerlMonitor/MonitorGUI.py +++ b/src_erl/NerlMonitor/MonitorGUI.py @@ -60,7 +60,7 @@ def GUI(MainPid): if not msg_queue.empty(): msg = msg_queue.get_nowait() if msg[0] == 'graph': - Show_Nerlnet_Graph(msg[1] , msg[2]) + Show_Nerlnet_Graph(msg[1][0] , msg[1][1]) MainWindow['-PHOLD-'].update(visible=False) MainWindow['-IMAGE-'].update(filename='NerlNetGraph.png' , visible=True , size=(410,310)) elif values['-LOG-'] != '': @@ -78,16 +78,24 @@ def GUI(MainPid): def Show_Nerlnet_Graph(NerlGraph , Workers): # Graph in string format: "Entity1Name,Entity1IP,Entity1Port#Entity2Name,Entity2IP,Entity2Port#Entity1Name-Entity2Name,Entity2Name-Entity1Name" etc. - # Workers in list format: [WorkerName , ClientName] + # Workers in string format: "Worker1-Client1,Worker2-Client1,Worker3-Client2" etc. # Node is defined by a triplet 'Name,IP,Port' seperated by '#' # Edge is defined by a string 'Entity1-Entity2' seperated by ',' + #print(f'Graph: {NerlGraph} , Workers: {Workers}') WorkersNames = [Worker[0] for Worker in Workers] + #print(f'WorkersNames: {WorkersNames}') Nodes = NerlGraph.split('#')[0:-1] + #print(f'Nodes: {Nodes}') Edges = NerlGraph.split('#')[-1].split(',') + #print(f'Edges: {Edges}') EdgesSeperated = [(Edge.split('-')[0],Edge.split('-')[1]) for Edge in Edges if len(Edges) > 1] # ? What if no edges? + #print(f'EdgesSeperated: {EdgesSeperated}') EdgesSeperated.append(Workers) + #print(f'EdgesSeperated+Workers: {EdgesSeperated}') NodesNames = [NodeTriplet.split(',')[0] for NodeTriplet in Nodes] + #print(f'NodesNames: {NodesNames}') NodesNames.append(WorkersNames) + #print(f'NodesNames+WorkersNames: {NodesNames}') graph = nx.Graph() graph.add_nodes_from(NodesNames) From 6ef37b8bc18a1533241d35a6468dfb3fc359b437 Mon Sep 17 00:00:00 2001 From: galhilu Date: Wed, 2 Aug 2023 16:17:21 +0300 Subject: [PATCH 15/40] added worker down cast handel for main server --- .../http_Nerlserver/src/MainServer/mainGenserver.erl | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src_erl/Communication_Layer/http_Nerlserver/src/MainServer/mainGenserver.erl b/src_erl/Communication_Layer/http_Nerlserver/src/MainServer/mainGenserver.erl index 82ca3144..7bebadd4 100644 --- a/src_erl/Communication_Layer/http_Nerlserver/src/MainServer/mainGenserver.erl +++ b/src_erl/Communication_Layer/http_Nerlserver/src/MainServer/mainGenserver.erl @@ -314,9 +314,15 @@ handle_cast({predictRes,Body}, State = #main_genserver_state{batchSize = BatchSi end, {noreply, State#main_genserver_state{msgCounter = MsgCounter+1}}; -handle_cast({worker_down,Body}, State = #main_genserver_state{}) -> - - {noreply, State}; +handle_cast({worker_down,Body}, State = #main_genserver_state{msgCounter = MsgCounter,etsRef=EtsRef}) -> + case ets:member(EtsRef,nerlMonitor) of + true-> + [{nerlMonitor,Ip,Port}]=ets:lookup(EtsRef,nerlMonitor), + URL = "http://" ++ Ip ++ ":"++integer_to_list(Port) ++ "/utillInfo", + httpc:request(post,{URL, [],"application/x-www-form-urlencoded",Body}, [], []); + false->ok + end, + {noreply, State#main_genserver_state{msgCounter = MsgCounter+1,etsRef=EtsRef}}; handle_cast(Request, State = #main_genserver_state{}) -> io:format("main server cast ignored: ~p~n",[Request]), From 3aa8babc71e862b61061f12c7c72e8e6f9fb8ff0 Mon Sep 17 00:00:00 2001 From: GuyPerets106 Date: Wed, 2 Aug 2023 13:42:22 +0000 Subject: [PATCH 16/40] Updated Json and Port for exp --- inputJsonFiles/Architecture/arch_1PCSIM6WorkerSynth.json | 6 +++--- src_erl/NerlMonitor/nerlMonitor_app.erl | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/inputJsonFiles/Architecture/arch_1PCSIM6WorkerSynth.json b/inputJsonFiles/Architecture/arch_1PCSIM6WorkerSynth.json index 8d1bb185..58821811 100755 --- a/inputJsonFiles/Architecture/arch_1PCSIM6WorkerSynth.json +++ b/inputJsonFiles/Architecture/arch_1PCSIM6WorkerSynth.json @@ -6,13 +6,13 @@ }, "devices": [ { - "host": "192.168.0.108", + "host": "192.168.64.7", "entities": "mainServer,c1,c2,c3,c4,c5,c6,s1,r1,r2,r3,r4,r5,r6,apiServer" } ], "apiServer": { - "host": "192.168.0.108", + "host": "192.168.64.7", "port": "8095", "args": "" } @@ -26,7 +26,7 @@ , "mainServer": { - "host": "192.168.0.108", + "host": "192.168.64.7", "port": "8080", "args": "" } diff --git a/src_erl/NerlMonitor/nerlMonitor_app.erl b/src_erl/NerlMonitor/nerlMonitor_app.erl index 96ea9d5d..dd8f46e4 100644 --- a/src_erl/NerlMonitor/nerlMonitor_app.erl +++ b/src_erl/NerlMonitor/nerlMonitor_app.erl @@ -12,7 +12,7 @@ -define(UTILLNAME,nerlMonitor). -define(IP , nerl_tools:getdeviceIP()). -define(PORT,8096 ). %port place holder --define(MSADDRES,"ip:port" ). %place holder +-define(MSADDRES,"192.168.64.7:8080" ). %place holder -define(GUI , {'PyrlangProcess' , 'py@127.0.0.1'}). % Erlang node should be long name to communicate with pyrlang node start(_StartType, _StartArgs) -> From 3dcf86314c506598e46d1cabf9885e852972b2c5 Mon Sep 17 00:00:00 2001 From: GuyPerets106 Date: Wed, 2 Aug 2023 15:12:47 +0000 Subject: [PATCH 17/40] Fixed ets:new declaration in main server --- .../http_Nerlserver/src/MainServer/mainGenserver.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src_erl/Communication_Layer/http_Nerlserver/src/MainServer/mainGenserver.erl b/src_erl/Communication_Layer/http_Nerlserver/src/MainServer/mainGenserver.erl index 7bebadd4..6adfac36 100644 --- a/src_erl/Communication_Layer/http_Nerlserver/src/MainServer/mainGenserver.erl +++ b/src_erl/Communication_Layer/http_Nerlserver/src/MainServer/mainGenserver.erl @@ -56,7 +56,7 @@ init({MyName,Clients,BatchSize,WorkersMap,NerlnetGraph}) -> ?LOG_NOTICE("Main Server is connected to: ~p~n",[ConnectedEntities]), put(nerlnetGraph, NerlnetGraph), % nerl_tools:start_connection([digraph:vertex(NerlnetGraph,Vertex) || Vertex <- digraph:out_neighbours(NerlnetGraph,MyName)]), - EtsRef = ets:new() , %% ! In some point the whole record will replaced by ets + EtsRef = ets:new(main_server_data , [set]) , %% ! In some point the whole record will replaced by ets NewStatisticsMap = getNewStatisticsMap([digraph:vertex(NerlnetGraph,Vertex) || Vertex <- digraph:vertices(NerlnetGraph)--?LIST_OF_SPECIAL_SERVERS]), % io:format("New StatisticsMap = ~p~n",[NewStatisticsMap]), {ok, #main_genserver_state{myName = MyNameStr, workersMap = WorkersMap, batchSize = BatchSize, state=idle, clients = Clients, nerlnetGraph = NerlnetGraph, msgCounter = 1,statisticsMap = NewStatisticsMap , etsRef = EtsRef}}. From 5e45ba894c272e333e40657ec244b118346336fb Mon Sep 17 00:00:00 2001 From: David Date: Tue, 1 Aug 2023 21:13:47 +0300 Subject: [PATCH 18/40] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 970f588e..17ec3b35 100644 --- a/README.md +++ b/README.md @@ -76,4 +76,4 @@ Minimum Python version: 3.8 4. Run Jupyter notebook with ```jupyter-notebook``` and create a new notebook in the created dir from step 3. 5. Follow the example: https://github.com/leondavi/NErlNet/blob/master/examples/example_run.ipynb -Contact Email: nerlnet@outlook.com +Contact Email: leondavi@post.bgu.ac.il From afb9e94e57a36f5ac1d13cae91f889fbb58b0b78 Mon Sep 17 00:00:00 2001 From: GuyPerets106 Date: Thu, 3 Aug 2023 07:22:02 +0000 Subject: [PATCH 19/40] Updated NerlMonitor directory as rebar3 app --- src_erl/NerlMonitor/.gitignore | 20 ++ src_erl/NerlMonitor/Instructions.txt | 2 - src_erl/NerlMonitor/LICENSE.md | 186 ++++++++++++++++++ src_erl/NerlMonitor/README.md | 9 + src_erl/NerlMonitor/rebar.config | 9 + src_erl/NerlMonitor/{ => src}/MonitorGUI.py | 0 .../handlers/nerlMonitor_handler.erl | 2 +- src_erl/NerlMonitor/src/nerlMonitor.app.src | 15 ++ .../NerlMonitor/{ => src}/nerlMonitor_app.erl | 7 +- .../NerlMonitor/{ => src}/nerlMonitor_sup.erl | 2 +- 10 files changed, 246 insertions(+), 6 deletions(-) create mode 100644 src_erl/NerlMonitor/.gitignore delete mode 100644 src_erl/NerlMonitor/Instructions.txt create mode 100644 src_erl/NerlMonitor/LICENSE.md create mode 100644 src_erl/NerlMonitor/README.md create mode 100644 src_erl/NerlMonitor/rebar.config rename src_erl/NerlMonitor/{ => src}/MonitorGUI.py (100%) rename src_erl/NerlMonitor/{ => src}/handlers/nerlMonitor_handler.erl (93%) create mode 100644 src_erl/NerlMonitor/src/nerlMonitor.app.src rename src_erl/NerlMonitor/{ => src}/nerlMonitor_app.erl (91%) rename src_erl/NerlMonitor/{ => src}/nerlMonitor_sup.erl (96%) diff --git a/src_erl/NerlMonitor/.gitignore b/src_erl/NerlMonitor/.gitignore new file mode 100644 index 00000000..df53f7d9 --- /dev/null +++ b/src_erl/NerlMonitor/.gitignore @@ -0,0 +1,20 @@ +.rebar3 +_build +_checkouts +_vendor +.eunit +*.o +*.beam +*.plt +*.swp +*.swo +.erlang.cookie +ebin +log +erl_crash.dump +.rebar +logs +.idea +*.iml +rebar3.crashdump +*~ diff --git a/src_erl/NerlMonitor/Instructions.txt b/src_erl/NerlMonitor/Instructions.txt deleted file mode 100644 index 1ef658fc..00000000 --- a/src_erl/NerlMonitor/Instructions.txt +++ /dev/null @@ -1,2 +0,0 @@ - -Create a new cowboy server of NerlMonitor \ No newline at end of file diff --git a/src_erl/NerlMonitor/LICENSE.md b/src_erl/NerlMonitor/LICENSE.md new file mode 100644 index 00000000..8cdc1e6f --- /dev/null +++ b/src_erl/NerlMonitor/LICENSE.md @@ -0,0 +1,186 @@ +# Apache License +Version 2.0, January 2004 + +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +## 1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, and +distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by the copyright +owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all other entities +that control, are controlled by, or are under common control with that entity. +For the purposes of this definition, "control" means (i) the power, direct or +indirect, to cause the direction or management of such entity, whether by +contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the +outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity exercising +permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, including +but not limited to software source code, documentation source, and configuration +files. + +"Object" form shall mean any form resulting from mechanical transformation or +translation of a Source form, including but not limited to compiled object code, +generated documentation, and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or Object form, made +available under the License, as indicated by a copyright notice that is included +in or attached to the work (an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object form, that +is based on (or derived from) the Work and for which the editorial revisions, +annotations, elaborations, or other modifications represent, as a whole, an +original work of authorship. For the purposes of this License, Derivative Works +shall not include works that remain separable from, or merely link (or bind by +name) to the interfaces of, the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including the original version +of the Work and any modifications or additions to that Work or Derivative Works +thereof, that is intentionally submitted to Licensor for inclusion in the Work +by the copyright owner or by an individual or Legal Entity authorized to submit +on behalf of the copyright owner. For the purposes of this definition, +"submitted" means any form of electronic, verbal, or written communication sent +to the Licensor or its representatives, including but not limited to +communication on electronic mailing lists, source code control systems, and +issue tracking systems that are managed by, or on behalf of, the Licensor for +the purpose of discussing and improving the Work, but excluding communication +that is conspicuously marked or otherwise designated in writing by the copyright +owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity on behalf +of whom a Contribution has been received by Licensor and subsequently +incorporated within the Work. + +## 2. Grant of Copyright License. + +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable copyright license to reproduce, prepare Derivative Works of, +publicly display, publicly perform, sublicense, and distribute the Work and such +Derivative Works in Source or Object form. + +## 3. Grant of Patent License. + +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable (except as stated in this section) patent license to make, have +made, use, offer to sell, sell, import, and otherwise transfer the Work, where +such license applies only to those patent claims licensable by such Contributor +that are necessarily infringed by their Contribution(s) alone or by combination +of their Contribution(s) with the Work to which such Contribution(s) was +submitted. If You institute patent litigation against any entity (including a +cross-claim or counterclaim in a lawsuit) alleging that the Work or a +Contribution incorporated within the Work constitutes direct or contributory +patent infringement, then any patent licenses granted to You under this License +for that Work shall terminate as of the date such litigation is filed. + +## 4. Redistribution. + +You may reproduce and distribute copies of the Work or Derivative Works thereof +in any medium, with or without modifications, and in Source or Object form, +provided that You meet the following conditions: + +1. You must give any other recipients of the Work or Derivative Works a copy of + this License; and + +2. You must cause any modified files to carry prominent notices stating that + You changed the files; and + +3. You must retain, in the Source form of any Derivative Works that You + distribute, all copyright, patent, trademark, and attribution notices from + the Source form of the Work, excluding those notices that do not pertain to + any part of the Derivative Works; and + +4. If the Work includes a "NOTICE" text file as part of its distribution, then + any Derivative Works that You distribute must include a readable copy of the + attribution notices contained within such NOTICE file, excluding those + notices that do not pertain to any part of the Derivative Works, in at least + one of the following places: within a NOTICE text file distributed as part + of the Derivative Works; within the Source form or documentation, if + provided along with the Derivative Works; or, within a display generated by + the Derivative Works, if and wherever such third-party notices normally + appear. The contents of the NOTICE file are for informational purposes only + and do not modify the License. You may add Your own attribution notices + within Derivative Works that You distribute, alongside or as an addendum to + the NOTICE text from the Work, provided that such additional attribution + notices cannot be construed as modifying the License. + +You may add Your own copyright statement to Your modifications and may provide +additional or different license terms and conditions for use, reproduction, or +distribution of Your modifications, or for any such Derivative Works as a whole, +provided Your use, reproduction, and distribution of the Work otherwise complies +with the conditions stated in this License. + +## 5. Submission of Contributions. + +Unless You explicitly state otherwise, any Contribution intentionally submitted +for inclusion in the Work by You to the Licensor shall be under the terms and +conditions of this License, without any additional terms or conditions. +Notwithstanding the above, nothing herein shall supersede or modify the terms of +any separate license agreement you may have executed with Licensor regarding +such Contributions. + +## 6. Trademarks. + +This License does not grant permission to use the trade names, trademarks, +service marks, or product names of the Licensor, except as required for +reasonable and customary use in describing the origin of the Work and +reproducing the content of the NOTICE file. + +## 7. Disclaimer of Warranty. + +Unless required by applicable law or agreed to in writing, Licensor provides the +Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, +including, without limitation, any warranties or conditions of TITLE, NON- +INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are +solely responsible for determining the appropriateness of using or +redistributing the Work and assume any risks associated with Your exercise of +permissions under this License. + +## 8. Limitation of Liability. + +In no event and under no legal theory, whether in tort (including negligence), +contract, or otherwise, unless required by applicable law (such as deliberate +and grossly negligent acts) or agreed to in writing, shall any Contributor be +liable to You for damages, including any direct, indirect, special, incidental, +or consequential damages of any character arising as a result of this License or +out of the use or inability to use the Work (including but not limited to +damages for loss of goodwill, work stoppage, computer failure or malfunction, or +any and all other commercial damages or losses), even if such Contributor has +been advised of the possibility of such damages. + +## 9. Accepting Warranty or Additional Liability. + +While redistributing the Work or Derivative Works thereof, You may choose to +offer, and charge a fee for, acceptance of support, warranty, indemnity, or +other liability obligations and/or rights consistent with this License. However, +in accepting such obligations, You may act only on Your own behalf and on Your +sole responsibility, not on behalf of any other Contributor, and only if You +agree to indemnify, defend, and hold each Contributor harmless for any liability +incurred by, or claims asserted against, such Contributor by reason of your +accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +Copyright 2023, GuyPerets106 . + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. \ No newline at end of file diff --git a/src_erl/NerlMonitor/README.md b/src_erl/NerlMonitor/README.md new file mode 100644 index 00000000..a690a438 --- /dev/null +++ b/src_erl/NerlMonitor/README.md @@ -0,0 +1,9 @@ +NerlMonitor +===== + +An OTP application + +Build +----- + + $ rebar3 compile diff --git a/src_erl/NerlMonitor/rebar.config b/src_erl/NerlMonitor/rebar.config new file mode 100644 index 00000000..cc4df178 --- /dev/null +++ b/src_erl/NerlMonitor/rebar.config @@ -0,0 +1,9 @@ +{erl_opts, [debug_info]}. +{deps, [ + {cowboy, {git, "https://github.com/ninenines/cowboy.git" , {tag,"2.9.0"}}} +]}. + +{shell, [ + % {config, "config/sys.config"}, + {apps, [nerlMonitor]} +]}. diff --git a/src_erl/NerlMonitor/MonitorGUI.py b/src_erl/NerlMonitor/src/MonitorGUI.py similarity index 100% rename from src_erl/NerlMonitor/MonitorGUI.py rename to src_erl/NerlMonitor/src/MonitorGUI.py diff --git a/src_erl/NerlMonitor/handlers/nerlMonitor_handler.erl b/src_erl/NerlMonitor/src/handlers/nerlMonitor_handler.erl similarity index 93% rename from src_erl/NerlMonitor/handlers/nerlMonitor_handler.erl rename to src_erl/NerlMonitor/src/handlers/nerlMonitor_handler.erl index 7884b8cb..cc773ee6 100644 --- a/src_erl/NerlMonitor/handlers/nerlMonitor_handler.erl +++ b/src_erl/NerlMonitor/src/handlers/nerlMonitor_handler.erl @@ -1,4 +1,4 @@ --module(hello_handler). +-module(nerlMonitor_handler). -export([init/2]). diff --git a/src_erl/NerlMonitor/src/nerlMonitor.app.src b/src_erl/NerlMonitor/src/nerlMonitor.app.src new file mode 100644 index 00000000..3625d754 --- /dev/null +++ b/src_erl/NerlMonitor/src/nerlMonitor.app.src @@ -0,0 +1,15 @@ +{application, nerlMonitor, + [{description, "An OTP application"}, + {vsn, "0.1.0"}, + {registered, []}, + {mod, {nerlMonitor_app, []}}, + {applications, + [kernel, + stdlib + ]}, + {env,[]}, + {modules, []}, + + {licenses, ["Apache-2.0"]}, + {links, []} + ]}. diff --git a/src_erl/NerlMonitor/nerlMonitor_app.erl b/src_erl/NerlMonitor/src/nerlMonitor_app.erl similarity index 91% rename from src_erl/NerlMonitor/nerlMonitor_app.erl rename to src_erl/NerlMonitor/src/nerlMonitor_app.erl index dd8f46e4..c997b024 100644 --- a/src_erl/NerlMonitor/nerlMonitor_app.erl +++ b/src_erl/NerlMonitor/src/nerlMonitor_app.erl @@ -7,10 +7,12 @@ -behaviour(application). +-include("../../Communication_Layer/http_Nerlserver/src/nerl_tools.hrl"). + -export([start/2, stop/1]). -define(UTILLNAME,nerlMonitor). --define(IP , nerl_tools:getdeviceIP()). +-define(IP , "192.168.64.7"). -define(PORT,8096 ). %port place holder -define(MSADDRES,"192.168.64.7:8080" ). %place holder -define(GUI , {'PyrlangProcess' , 'py@127.0.0.1'}). % Erlang node should be long name to communicate with pyrlang node @@ -33,7 +35,8 @@ start(_StartType, _StartArgs) -> %ping main server in 0.5 sec intervals with connection request. will stop when got valid response. -mainServerPing(URL,Body)-> +mainServerPing(URL,Body)-> + io:format("pinging main server~n"), Response = httpc:request(post,{URL, [],"application/x-www-form-urlencoded",Body}, [], []), case Response of {error,_}-> diff --git a/src_erl/NerlMonitor/nerlMonitor_sup.erl b/src_erl/NerlMonitor/src/nerlMonitor_sup.erl similarity index 96% rename from src_erl/NerlMonitor/nerlMonitor_sup.erl rename to src_erl/NerlMonitor/src/nerlMonitor_sup.erl index 089a33f8..02225cc6 100644 --- a/src_erl/NerlMonitor/nerlMonitor_sup.erl +++ b/src_erl/NerlMonitor/src/nerlMonitor_sup.erl @@ -1,5 +1,5 @@ %%%------------------------------------------------------------------- -%% @doc nerlGUI top level supervisor. +%% @doc NerlMonitor top level supervisor. %% @end %%%------------------------------------------------------------------- From c79f1d6fc929226806f775304d19dd3e3b79f1e7 Mon Sep 17 00:00:00 2001 From: galhilu Date: Thu, 3 Aug 2023 10:27:02 +0300 Subject: [PATCH 20/40] added hadle info for client states for worker fall detection --- .../src/Client/clientStatem.erl | 56 ++++++++++++++++--- 1 file changed, 48 insertions(+), 8 deletions(-) diff --git a/src_erl/Communication_Layer/http_Nerlserver/src/Client/clientStatem.erl b/src_erl/Communication_Layer/http_Nerlserver/src/Client/clientStatem.erl index 76187464..080d713c 100644 --- a/src_erl/Communication_Layer/http_Nerlserver/src/Client/clientStatem.erl +++ b/src_erl/Communication_Layer/http_Nerlserver/src/Client/clientStatem.erl @@ -126,16 +126,19 @@ waitforWorkers(cast, EventContent, State = #client_statem_state{etsRef = EtsRef} {next_state, waitforWorkers, State}; %---------------------------------------------------------------- -waitforWorkers(info, EventContent, State = #client_statem_state{myName = MyName,etsRef = EtsRef}) -> +waitforWorkers(info, EventContent, State = #client_statem_state{myName = MyName,waitforWorkers = WaitforWorkers ,etsRef = EtsRef}) -> case EventContent of {'DOWN',_Ref,process,Pid,_Reason}-> %worker down [[WorkerName]] = ets:match(EtsRef,{'$1',Pid,'_','_','_'}), + NewWaitforWorkers = WaitforWorkers--[WorkerName], %report worker down to main server - delete_worker(EtsRef,MyName,WorkerName); %delete worker from ets so client will not wait for it in the future + delete_worker(EtsRef,MyName,WorkerName), + {next_state, waitforWorkers, State#client_statem_state{waitforWorkers = NewWaitforWorkers,etsRef = EtsRef}}; %delete worker from ets so client will not wait for it in the future - _Any->ok %recevied unexpected messege, print to log/tell main server for debuging - end, - {next_state, waitforWorkers, State#client_statem_state{etsRef = EtsRef}}. + _Any-> + %recevied unexpected messege, print to log/tell main server for debuging + {next_state, waitforWorkers, State#client_statem_state{etsRef = EtsRef}} + end. %----------------------------------------------------------------- @@ -188,7 +191,20 @@ idle(cast, EventContent, State = #client_statem_state{etsRef = EtsRef}) -> ets:update_counter(EtsRef, msgCounter, 1), ets:update_counter(EtsRef, infoIn, nerl_tools:calculate_size(EventContent)), io:format("client idle ignored!!!: ~p ~n",[EventContent]), - {next_state, training, State#client_statem_state{etsRef = EtsRef}}. + {next_state, training, State#client_statem_state{etsRef = EtsRef}}; + + +idle(info, EventContent, State = #client_statem_state{myName = MyName,etsRef = EtsRef}) -> + case EventContent of + {'DOWN',_Ref,process,Pid,_Reason}-> %worker down + [[WorkerName]] = ets:match(EtsRef,{'$1',Pid,'_','_','_'}), + %report worker down to main server + delete_worker(EtsRef,MyName,WorkerName); %delete worker from ets so client will not wait for it in the future + + _Any->ok + %recevied unexpected messege, print to log/tell main server for debuging + end, + {next_state, idle, State#client_statem_state{etsRef = EtsRef}}. %% passing vector from FedClient to FedServer training(cast, In = {update, {From, To, Data}}, State = #client_statem_state{etsRef = EtsRef, myName = MyName}) -> @@ -296,7 +312,19 @@ training(cast, In = {loss,WorkerName,LossFunction,_Time_NIF}, State = #client_st training(cast, EventContent, State = #client_statem_state{etsRef = EtsRef, myName = MyName}) -> ?LOG_WARNING("client ~p training ignored!!!: ~p ~n!!!",[MyName, EventContent]), ets:update_counter(EtsRef, infoIn, nerl_tools:calculate_size(EventContent)), - {next_state, training, State#client_statem_state{etsRef = EtsRef}}. + {next_state, training, State#client_statem_state{etsRef = EtsRef}}; + +training(info, EventContent, State = #client_statem_state{myName = MyName,etsRef = EtsRef}) -> + case EventContent of + {'DOWN',_Ref,process,Pid,_Reason}-> %worker down + [[WorkerName]] = ets:match(EtsRef,{'$1',Pid,'_','_','_'}), + %report worker down to main server + delete_worker(EtsRef,MyName,WorkerName); %delete worker from ets so client will not wait for it in the future + + _Any->ok + %recevied unexpected messege, print to log/tell main server for debuging + end, + {next_state, training, State#client_statem_state{etsRef = EtsRef}}. predict(cast, In = {sample,Body}, State = #client_statem_state{etsRef = EtsRef}) -> %% Body: ClientName#WorkerName#CSVName#BatchNumber#BatchOfSamples @@ -354,7 +382,19 @@ predict(cast, EventContent, State = #client_statem_state{etsRef = EtsRef}) -> ets:update_counter(EtsRef, msgCounter, 1), ets:update_counter(EtsRef, infoIn, erts_debug:flat_size(EventContent)), ?LOG_WARNING("client predict ignored: ~p ~n",[EventContent]), - {next_state, predict, State#client_statem_state{etsRef = EtsRef}}. + {next_state, predict, State#client_statem_state{etsRef = EtsRef}}; + +predict(info, EventContent, State = #client_statem_state{myName = MyName,etsRef = EtsRef}) -> + case EventContent of + {'DOWN',_Ref,process,Pid,_Reason}-> %worker down + [[WorkerName]] = ets:match(EtsRef,{'$1',Pid,'_','_','_'}), + %report worker down to main server + delete_worker(EtsRef,MyName,WorkerName); %delete worker from ets so client will not wait for it in the future + + _Any->ok + %recevied unexpected messege, print to log/tell main server for debuging + end, + {next_state, predict, State#client_statem_state{etsRef = EtsRef}}. %% @private From 3943ccbf0a08b1552e92a4bde5f6c24ec3b8c856 Mon Sep 17 00:00:00 2001 From: GuyPerets106 Date: Thu, 3 Aug 2023 11:58:48 +0000 Subject: [PATCH 21/40] NerlGraph sent from MainServer in right format --- .../src/MainServer/mainGenserver.erl | 11 ++--- .../http_Nerlserver/src/utilities_handler.erl | 15 +++++-- src_erl/NerlMonitor/src/MonitorGUI.py | 44 +++++++++---------- src_erl/NerlMonitor/src/nerlMonitor_app.erl | 17 ++++--- 4 files changed, 49 insertions(+), 38 deletions(-) diff --git a/src_erl/Communication_Layer/http_Nerlserver/src/MainServer/mainGenserver.erl b/src_erl/Communication_Layer/http_Nerlserver/src/MainServer/mainGenserver.erl index 6adfac36..97b3096e 100644 --- a/src_erl/Communication_Layer/http_Nerlserver/src/MainServer/mainGenserver.erl +++ b/src_erl/Communication_Layer/http_Nerlserver/src/MainServer/mainGenserver.erl @@ -76,10 +76,11 @@ init({MyName,Clients,BatchSize,WorkersMap,NerlnetGraph}) -> handle_call(getGraph, _From, State) -> NerlGraph = State#main_genserver_state.nerlnetGraph, FullNodes = [digraph:vertex(NerlGraph,Vertex) || Vertex <- digraph:vertices(NerlGraph)], - %io:format("Full graph is: ~p~n", [FullNodes]), - NodesList = [Entity++","++IP++","++integer_to_list(Port)++"#"||{Entity, {IP, Port}} <- FullNodes], + %%io:format("Full graph is: ~p~n", [FullNodes]), + NodesList = [atom_to_list(Entity)++","++binary_to_list(IP)++","++integer_to_list(Port)++"#"||{Entity, {IP, Port}} <- FullNodes], + %%io:format("graph nodes are: ~p~n", [NodesList]), EdgesList = [digraph:edge(NerlGraph,Edge) || Edge <- digraph:edges(NerlGraph)], - %io:format("graph edges are: ~p~n", [EdgesList]), + %%io:format("graph edges are: ~p~n", [EdgesList]), Nodes = nodeString(NodesList), Edges = edgeString(EdgesList), @@ -100,11 +101,11 @@ nodeString([Node |NodeList]) -> nodeString(NodeList, Node). nodeString([], Str) -> Str; nodeString([Node |NodeList], Str)-> nodeString(NodeList, Node++Str). -edgeString([Edge |EdgesList])-> {_ID, V1, V2, _Label} = Edge, edgeString(EdgesList, V1++"-"++V2). +edgeString([Edge |EdgesList])-> {_ID, V1, V2, _Label} = Edge, edgeString(EdgesList, atom_to_list(V1)++"-"++atom_to_list(V2)). edgeString([], Str)-> Str; edgeString([Edge |EdgesList], Str)-> {_ID, V1, V2, _Label} = Edge, - edgeString(EdgesList, V1++"-"++V2++","++Str). + edgeString(EdgesList, atom_to_list(V1)++"-"++atom_to_list(V2)++","++Str). %% @private %% @doc Handling cast messages diff --git a/src_erl/Communication_Layer/http_Nerlserver/src/utilities_handler.erl b/src_erl/Communication_Layer/http_Nerlserver/src/utilities_handler.erl index fce98a99..c79ffe9e 100644 --- a/src_erl/Communication_Layer/http_Nerlserver/src/utilities_handler.erl +++ b/src_erl/Communication_Layer/http_Nerlserver/src/utilities_handler.erl @@ -6,12 +6,19 @@ init(Req0 , [MainServerPid]) -> {_,Body,_} = cowboy_req:read_body(Req0), - [UtilityName , _Funcs , IP , Port] = binary_to_list(Body), - case UtilityName of + io:format("Body: ~p~n" , [Body]) , + Formatted = binary_to_list(Body), + [UtilityName , IP , Port] = [X || X <- re:split(Formatted , "#" , [{return , list}])], + io:format("UtilityName: ~p , IP: ~p , Port: ~p~n" , [UtilityName , IP , Port]), + case list_to_atom(UtilityName) of nerlMonitor -> - {reply , Graph , _ } = gen_server:call(MainServerPid , getGraph), + io:format("I'm HERE , args are: ~p , ~p , ~p~n" , [UtilityName , IP , Port]), + Graph = gen_server:call(MainServerPid , getGraph), + io:format("Graph: ~p~n" , [Graph]), WorkersMap = ets:lookup_element(nerlnet_data , workers , ?DATA_IDX), - Reply = {Graph , WorkersMap} + WorkersClients = maps:to_list(WorkersMap), + Workers = lists:flatten([atom_to_list(X)++"-"++atom_to_list(Y)++"!" || {X , Y} <- WorkersClients]), + Reply = Graph++","++Workers end, gen_server:cast(MainServerPid , {saveUtility , {UtilityName , IP , Port}}), %% TODO Add IP , Port to ets in main server record Req = cowboy_req:reply(200, diff --git a/src_erl/NerlMonitor/src/MonitorGUI.py b/src_erl/NerlMonitor/src/MonitorGUI.py index 1637c176..f948084e 100644 --- a/src_erl/NerlMonitor/src/MonitorGUI.py +++ b/src_erl/NerlMonitor/src/MonitorGUI.py @@ -8,6 +8,7 @@ import matplotlib.pyplot as plt from datetime import datetime import os +import math def draw_gradient(canvas, start_color, end_color): for y in range(0, 200): # Adjust the range to your desired height @@ -25,9 +26,9 @@ def draw_gradient(canvas, start_color, end_color): sg.Text("NerlNet Monitor" , key='-TEXT-' , size=(30,1) ,text_color='Black' , font=('SFPro' , 20) , background_color='#930707' , justification='center' , pad=(0,0)) ] , [ sg.Frame(title="Event Log:" , - layout=[[sg.Multiline('', size=(100, 20), key='-LOG-', autoscroll=True , font=('SFPro' , 12) , no_scrollbar=True)]], - background_color=('#930707') , font=('SFPro' , 20) , size=(300,400) , title_color='Black' , element_justification='right') , - sg.Text("Waiting For\n NerlNet Graph..." , key='-PHOLD-', text_color='Black' , font=('SFPro' , 12) , size=(50,5) , background_color='#930707' , justification='center' , pad=(0,0)) , + layout=[[sg.Multiline('', size=(140, 60), key='-LOG-', autoscroll=True , font=('SFPro' , 12) , no_scrollbar=True)]], + background_color=('#930707') , font=('SFPro' , 20) , size=(500,650) , title_color='Black' , element_justification='right') , + sg.Text("Waiting For\n NerlNet Graph..." , key='-PHOLD-', text_color='Black' , font=('SFPro' , 12) , size=(70,5) , background_color='#930707' , justification='center' , pad=(0,0)) , sg.Image(key='-IMAGE-' , visible=False) ] , @@ -38,7 +39,7 @@ def draw_gradient(canvas, start_color, end_color): ] -MainWindow = sg.Window("NErlNet" , layout , margins=(5,5) , size=(800,500) , background_color='#930707' , finalize=True , resizable=True , element_justification='c') +MainWindow = sg.Window("NErlNet" , layout , margins=(5,5) , size=(1400,800) , background_color='#930707' , finalize=True , resizable=True , element_justification='c') @@ -60,9 +61,9 @@ def GUI(MainPid): if not msg_queue.empty(): msg = msg_queue.get_nowait() if msg[0] == 'graph': - Show_Nerlnet_Graph(msg[1][0] , msg[1][1]) + Show_Nerlnet_Graph(msg[1]) MainWindow['-PHOLD-'].update(visible=False) - MainWindow['-IMAGE-'].update(filename='NerlNetGraph.png' , visible=True , size=(410,310)) + MainWindow['-IMAGE-'].update(filename='NerlNetGraph.png' , visible=True , size=(800,600)) elif values['-LOG-'] != '': existing_text = values['-LOG-'] updated_text = f'{existing_text}\n{formatted_time()}: {msg}' @@ -76,34 +77,29 @@ def GUI(MainPid): MainWindow.close() -def Show_Nerlnet_Graph(NerlGraph , Workers): - # Graph in string format: "Entity1Name,Entity1IP,Entity1Port#Entity2Name,Entity2IP,Entity2Port#Entity1Name-Entity2Name,Entity2Name-Entity1Name" etc. +def Show_Nerlnet_Graph(NerlGraph): + # Graph in string format: "Entity1Name,Entity1IP,Entity1Port#Entity2Name,Entity2IP,Entity2Port#Entity1Name-Entity2Name,Entity2Name-Entity1Name#Worker1-Client1#Worker2-Client2" etc. # Workers in string format: "Worker1-Client1,Worker2-Client1,Worker3-Client2" etc. # Node is defined by a triplet 'Name,IP,Port' seperated by '#' # Edge is defined by a string 'Entity1-Entity2' seperated by ',' - #print(f'Graph: {NerlGraph} , Workers: {Workers}') - WorkersNames = [Worker[0] for Worker in Workers] - #print(f'WorkersNames: {WorkersNames}') Nodes = NerlGraph.split('#')[0:-1] - #print(f'Nodes: {Nodes}') - Edges = NerlGraph.split('#')[-1].split(',') - #print(f'Edges: {Edges}') + Edges = NerlGraph.split('#')[-1].split(',')[0:-1] + Workers = NerlGraph.split('#')[-1].split(',')[-1].split('!')[0:-1] + WorkersNames = [Worker.split('-')[0] for Worker in Workers] + Edges += Workers EdgesSeperated = [(Edge.split('-')[0],Edge.split('-')[1]) for Edge in Edges if len(Edges) > 1] # ? What if no edges? - #print(f'EdgesSeperated: {EdgesSeperated}') - EdgesSeperated.append(Workers) - #print(f'EdgesSeperated+Workers: {EdgesSeperated}') NodesNames = [NodeTriplet.split(',')[0] for NodeTriplet in Nodes] - #print(f'NodesNames: {NodesNames}') - NodesNames.append(WorkersNames) - #print(f'NodesNames+WorkersNames: {NodesNames}') + NodesNames += WorkersNames graph = nx.Graph() graph.add_nodes_from(NodesNames) graph.add_edges_from(EdgesSeperated) - #print(graph.nodes , graph.edges) - spring_pos = nx.spring_layout(graph) - nx.draw_networkx(graph, spring_pos, with_labels=True, node_color='lightblue', node_size=500, font_size=12, font_color='black') - plt.savefig('NerlNetGraph.png' ,bbox_inches='tight' , dpi=80) + pos = nx.spectral_layout(graph) + angle = 40 + rotated_pos = {node: (x*math.cos(angle) -y*math.sin(angle), x*math.sin(angle) + y*math.cos(angle)) for node, (x, y) in pos.items()} + plt.figure(figsize=(8,6)) + nx.draw_networkx(graph, rotated_pos, with_labels=True, node_color='skyblue', node_size=200, font_size=8, font_color='black' , edge_color='black' , width=1.5) + plt.savefig('NerlNetGraph.png' ,bbox_inches='tight' , dpi=125) plt.close() diff --git a/src_erl/NerlMonitor/src/nerlMonitor_app.erl b/src_erl/NerlMonitor/src/nerlMonitor_app.erl index c997b024..000a31f8 100644 --- a/src_erl/NerlMonitor/src/nerlMonitor_app.erl +++ b/src_erl/NerlMonitor/src/nerlMonitor_app.erl @@ -13,7 +13,7 @@ -define(UTILLNAME,nerlMonitor). -define(IP , "192.168.64.7"). --define(PORT,8096 ). %port place holder +-define(PORT, 8096). %port place holder -define(MSADDRES,"192.168.64.7:8080" ). %place holder -define(GUI , {'PyrlangProcess' , 'py@127.0.0.1'}). % Erlang node should be long name to communicate with pyrlang node @@ -24,14 +24,17 @@ start(_StartType, _StartArgs) -> Dispatch = cowboy_router:compile([ {'_', [ - {"/utillInfo",nerlUtill_info_handler, []} + {"/utillInfo",nerlMonitor_handler, []} ]} ]), {ok, _} = cowboy:start_clear(?UTILLNAME,[{port, ?PORT}],#{env => #{dispatch => Dispatch}}), - os:cmd('python3 MonitorGUI.py'), %% PyrlangNode: ('PyralngProcess' , 'py@127.0.0.1' , 'COOKIE') , sending message by: 'GUI ! HELLO.' + io:format("nerlMonitor started , opening GUI...~n"), + %%os:cmd('python3 src/MonitorGUI.py'), %% PyrlangNode: ('PyralngProcess' , 'py@127.0.0.1' , 'COOKIE') , sending message by: 'GUI ! HELLO.' URL = "http://" ++ ?MSADDRES ++ "/toolConnectionReq", - mainServerPing(URL,[?UTILLNAME, functions() , ?IP , ?PORT]). %% TODO How to "import" nerl_tools + mainServerPing(URL,[list_to_binary(atom_to_list(?UTILLNAME) ++ "#") , list_to_binary(?IP ++ "#") , list_to_binary(integer_to_list(?PORT))]), %% TODO How to "import" nerl_tools + nerlMonitor_sup:start_link(). + %ping main server in 0.5 sec intervals with connection request. will stop when got valid response. @@ -43,11 +46,15 @@ mainServerPing(URL,Body)-> timer:sleep(500), mainServerPing(URL,Body); {ok,{_ResCode, _Headers, Data}}-> - initInfoProc(Data) + io:format("got1 response from main server~n"), + initInfoProc(Data); + {ok , _} -> + io:format("got2 response from main server~n") end. initInfoProc(Body)-> %% MainServer replies with Nerlnet-Graph when nerlMonitor tool is used + io:format("Sending graph to GUI , graph is ~p~n" , [Body]), ?GUI ! {graph , Body}. From a53dd3194f84752980b6977e317f0112a9797749 Mon Sep 17 00:00:00 2001 From: GuyPerets106 Date: Sat, 5 Aug 2023 12:30:27 +0000 Subject: [PATCH 22/40] [NerlMonitor] Added worker failure tolerance --- .../src/Client/clientStatem.erl | 40 +++++++++++++------ .../src/MainServer/mainGenserver.erl | 11 ++--- .../src/Router/routingHandler.erl | 2 +- .../http_Nerlserver/src/nerlNetServer_app.erl | 5 +-- .../http_Nerlserver/src/utilities_handler.erl | 6 +-- src_erl/NerlMonitor/src/MonitorGUI.py | 34 +++++++++++++--- .../src/handlers/nerlMonitor_handler.erl | 12 ++++-- src_erl/NerlMonitor/src/nerlMonitor_app.erl | 8 ++-- 8 files changed, 80 insertions(+), 38 deletions(-) diff --git a/src_erl/Communication_Layer/http_Nerlserver/src/Client/clientStatem.erl b/src_erl/Communication_Layer/http_Nerlserver/src/Client/clientStatem.erl index 080d713c..7779fc88 100644 --- a/src_erl/Communication_Layer/http_Nerlserver/src/Client/clientStatem.erl +++ b/src_erl/Communication_Layer/http_Nerlserver/src/Client/clientStatem.erl @@ -99,7 +99,14 @@ format_status(_Opt, [_PDict, _StateName, _State]) -> Status = some_term, Status. %% ==============STATES================= waitforWorkers(cast, In = {stateChange,WorkerName,MissedBatchesCount}, State = #client_statem_state{myName = MyName,waitforWorkers = WaitforWorkers,nextState = NextState, etsRef = EtsRef}) -> - NewWaitforWorkers = WaitforWorkers--[WorkerName], + NewWaitforWorkers = WaitforWorkers -- [WorkerName], + case WorkerName of + w5 -> + Pid = ets:lookup_element(EtsRef, w5, ?WORKER_PID_IDX), + io:format("---------------------Killin w5-----------------------~n"), + exit(Pid,kill); + _ -> ok + end, % io:format("remaining workers = ~p~n",[NewWaitforWorkers]), ets:update_counter(EtsRef, msgCounter, 1), % last is increment value ets:update_counter(EtsRef, infoIn, nerl_tools:calculate_size(In)), @@ -114,7 +121,7 @@ waitforWorkers(cast, In = {stateChange,WorkerName,MissedBatchesCount}, State = # waitforWorkers(cast, In = {NewState}, State = #client_statem_state{myName = MyName, etsRef = EtsRef}) -> ets:update_counter(EtsRef, msgCounter, 1), ets:update_counter(EtsRef, infoIn, nerl_tools:calculate_size(In)), - % ?LOG_INFO("~p in waiting going to state ~p~n",[MyName, State]), + %% ?LOG_INFO("~p in waiting going to state ~p~n",[MyName, State]), Workers = ets:lookup_element(EtsRef, workersNames, ?ETS_KV_VAL_IDX), cast_message_to_workers(EtsRef, {NewState}), {next_state, waitforWorkers, State#client_statem_state{nextState = NewState, waitforWorkers = Workers}}; @@ -268,7 +275,8 @@ training(cast, In = {sample,Body}, State = #client_statem_state{etsRef = EtsRef} NewTimingTuple = {Start,TotalBatches+1,TotalTime}, ets:update_element(EtsRef, WorkerName,[{?WORKER_PID_IDX, WorkerPid},{?WORKER_TIMING_IDX,NewTimingTuple}]), gen_statem:cast(WorkerPid, {sample, BatchOfSamples}); - true -> ?LOG_ERROR("Given worker ~p isn't found in client ~p",[WorkerName, ClientName]) end, + true -> ok %%'?'LOG_ERROR("Given worker ~p isn't found in client ~p",[WorkerName, ClientName]) + end, {next_state, training, State#client_statem_state{etsRef = EtsRef}}; training(cast, In = {idle}, State = #client_statem_state{myName = MyName, etsRef = EtsRef}) -> @@ -316,7 +324,7 @@ training(cast, EventContent, State = #client_statem_state{etsRef = EtsRef, myNam training(info, EventContent, State = #client_statem_state{myName = MyName,etsRef = EtsRef}) -> case EventContent of - {'DOWN',_Ref,process,Pid,_Reason}-> %worker down + {'DOWN' , _Ref , process , Pid , _Reason}-> %worker down [[WorkerName]] = ets:match(EtsRef,{'$1',Pid,'_','_','_'}), %report worker down to main server delete_worker(EtsRef,MyName,WorkerName); %delete worker from ets so client will not wait for it in the future @@ -340,12 +348,11 @@ predict(cast, In = {sample,Body}, State = #client_statem_state{etsRef = EtsRef}) TimingTuple = ets:lookup_element(EtsRef, WorkerName, ?WORKER_TIMING_IDX), %todo refactor timing map {_LastBatchReceivedTime,TotalBatches,TotalTime} = TimingTuple, NewTimingTuple = {Start,TotalBatches+1,TotalTime}, - ets:update_element(EtsRef, WorkerName,[{?WORKER_TIMING_IDX,NewTimingTuple}]); - true -> ?LOG_ERROR("Given worker ~p isn't found in client ~p",[WorkerName, ClientName]) + ets:update_element(EtsRef, WorkerName,[{?WORKER_TIMING_IDX,NewTimingTuple}]) , + WorkerPid = ets:lookup_element(EtsRef, WorkerName, ?WORKER_PID_IDX), + gen_statem:cast(WorkerPid, {sample, CSVName, BatchNumber, BatchOfSamples}); + true -> ok %%'?'LOG_ERROR("Given worker ~p isn't found in client ~p",[WorkerName, ClientName]) end, - - WorkerPid = ets:lookup_element(EtsRef, WorkerName, ?WORKER_PID_IDX), - gen_statem:cast(WorkerPid, {sample, CSVName, BatchNumber, BatchOfSamples}), {next_state, predict, State#client_statem_state{etsRef = EtsRef}}; %% TODO: add nif timing statistics @@ -515,6 +522,11 @@ sendStatistics(EtsRef)-> cast_message_to_workers(EtsRef, Msg) -> Workers = ets:lookup_element(EtsRef, workersNames, ?ETS_KV_VAL_IDX), + case Workers of + [] -> io:format("No workers to send message to~n"), + gen_statem:cast(self(), {stateChange , [] , 0}); + _ -> ok + end, Func = fun(WorkerKey) -> WorkerPid = ets:lookup_element(EtsRef, WorkerKey, ?WORKER_PID_IDX), gen_statem:cast(WorkerPid, Msg) @@ -523,9 +535,11 @@ cast_message_to_workers(EtsRef, Msg) -> %activated if client detected that a worker stoped, will delete it from ets so as not to wait for responsed from it in the future. delete_worker(EtsRef,MyName,WorkerName)-> - ets:delete(EtsRef,WorkerName), - NameList= ets:lookup_element(EtsRef, workersNames, ?ETS_KV_VAL_IDX), + ets:delete(EtsRef , WorkerName), + NameList = ets:lookup_element(EtsRef, workersNames, ?ETS_KV_VAL_IDX), ets:delete(EtsRef,workersNames), - ets:insert(EtsRef, {workersNames, NameList--[WorkerName]}), - nerl_tools:sendHTTP(MyName, ?MAIN_SERVER_ATOM, "worker_down", term_to_binary({MyName,WorkerName})), + ets:insert(EtsRef, {workersNames, NameList -- [WorkerName]}), + NerlGraph = ets:lookup_element(EtsRef, nerlnetGraph, ?ETS_KV_VAL_IDX), + {Host , Port} = nerl_tools:getShortPath(MyName, ?MAIN_SERVER_ATOM, NerlGraph), + nerl_tools:http_request(Host,Port,"worker_down",list_to_binary(atom_to_list(MyName) ++ "-" ++ atom_to_list(WorkerName))), EtsRef. \ No newline at end of file diff --git a/src_erl/Communication_Layer/http_Nerlserver/src/MainServer/mainGenserver.erl b/src_erl/Communication_Layer/http_Nerlserver/src/MainServer/mainGenserver.erl index 97b3096e..a7307efc 100644 --- a/src_erl/Communication_Layer/http_Nerlserver/src/MainServer/mainGenserver.erl +++ b/src_erl/Communication_Layer/http_Nerlserver/src/MainServer/mainGenserver.erl @@ -315,13 +315,14 @@ handle_cast({predictRes,Body}, State = #main_genserver_state{batchSize = BatchSi end, {noreply, State#main_genserver_state{msgCounter = MsgCounter+1}}; -handle_cast({worker_down,Body}, State = #main_genserver_state{msgCounter = MsgCounter,etsRef=EtsRef}) -> +handle_cast({worker_down,Body}, State = #main_genserver_state{msgCounter = MsgCounter,etsRef = EtsRef}) -> case ets:member(EtsRef,nerlMonitor) of true-> - [{nerlMonitor,Ip,Port}]=ets:lookup(EtsRef,nerlMonitor), - URL = "http://" ++ Ip ++ ":"++integer_to_list(Port) ++ "/utillInfo", + [{nerlMonitor , IP , Port}] = ets:lookup(EtsRef , nerlMonitor), + + URL = "http://" ++ IP ++ ":" ++ Port ++ "/utilInfo", httpc:request(post,{URL, [],"application/x-www-form-urlencoded",Body}, [], []); - false->ok + false -> ok end, {noreply, State#main_genserver_state{msgCounter = MsgCounter+1,etsRef=EtsRef}}; @@ -391,7 +392,7 @@ getNewStatisticsMap([{Name,{_Host, _Port}}|Tail],StatisticsMap) -> getNewStatisticsMap(Tail,maps:put(atom_to_list(Name), 0, StatisticsMap)). -startCasting([],_NumOfSampleToSend,_MyName, _NerlnetGraph)->done; +startCasting([],_NumOfSampleToSend,_MyName, _NerlnetGraph) -> done; startCasting([SourceName|SourceNames],NumOfSampleToSend, MyName, NerlnetGraph)-> ?LOG_NOTICE("~p sending start casting command to: ~p",[MyName, SourceName]), {RouterHost,RouterPort} = nerl_tools:getShortPath(MyName,SourceName,NerlnetGraph), diff --git a/src_erl/Communication_Layer/http_Nerlserver/src/Router/routingHandler.erl b/src_erl/Communication_Layer/http_Nerlserver/src/Router/routingHandler.erl index 235a50d2..22ec64cf 100644 --- a/src_erl/Communication_Layer/http_Nerlserver/src/Router/routingHandler.erl +++ b/src_erl/Communication_Layer/http_Nerlserver/src/Router/routingHandler.erl @@ -76,7 +76,7 @@ init(Req0, State) -> %%%%%%%%%%%%%%GUI actions getStats -> gen_server:cast(Router_genserver_Pid, {getStats,Body}); %monitor - worker_down-> gen_server:cast(Router_genserver_Pid, {worker_down,Body}) + worker_down -> gen_server:cast(Router_genserver_Pid, {worker_down,Body}) end, Reply = io_lib:format(" ", []), diff --git a/src_erl/Communication_Layer/http_Nerlserver/src/nerlNetServer_app.erl b/src_erl/Communication_Layer/http_Nerlserver/src/nerlNetServer_app.erl index 91eec1f2..13b5e174 100644 --- a/src_erl/Communication_Layer/http_Nerlserver/src/nerlNetServer_app.erl +++ b/src_erl/Communication_Layer/http_Nerlserver/src/nerlNetServer_app.erl @@ -307,9 +307,8 @@ createMainServer(true,BatchSize,HostName) -> {"/getGraph",[],guiHandler, [getGraph, MainGenServerPid]}, {"/getStats",[],guiHandler, [getStats, MainGenServerPid]}, {"/toolConnectionReq" , [] , utilities_handler , [MainGenServerPid]} , %% Added with NerlMonitor Project - {"/[...]", [],noMatchingRouteHandler, [MainGenServerPid]}, - %monitor actions - {"/worker_down",actionHandler, [worker_down,MainGenServerPid]} + {"/worker_down",actionHandler, [worker_down,MainGenServerPid]}, + {"/[...]", [],noMatchingRouteHandler, [MainGenServerPid]} ]} ]), %% cowboy:start_clear(Name, TransOpts, ProtoOpts) - an http_listener diff --git a/src_erl/Communication_Layer/http_Nerlserver/src/utilities_handler.erl b/src_erl/Communication_Layer/http_Nerlserver/src/utilities_handler.erl index c79ffe9e..563361e1 100644 --- a/src_erl/Communication_Layer/http_Nerlserver/src/utilities_handler.erl +++ b/src_erl/Communication_Layer/http_Nerlserver/src/utilities_handler.erl @@ -11,16 +11,16 @@ init(Req0 , [MainServerPid]) -> [UtilityName , IP , Port] = [X || X <- re:split(Formatted , "#" , [{return , list}])], io:format("UtilityName: ~p , IP: ~p , Port: ~p~n" , [UtilityName , IP , Port]), case list_to_atom(UtilityName) of - nerlMonitor -> + nerlMonitor -> io:format("I'm HERE , args are: ~p , ~p , ~p~n" , [UtilityName , IP , Port]), Graph = gen_server:call(MainServerPid , getGraph), io:format("Graph: ~p~n" , [Graph]), WorkersMap = ets:lookup_element(nerlnet_data , workers , ?DATA_IDX), WorkersClients = maps:to_list(WorkersMap), Workers = lists:flatten([atom_to_list(X)++"-"++atom_to_list(Y)++"!" || {X , Y} <- WorkersClients]), - Reply = Graph++","++Workers + Reply = Graph ++ "," ++ Workers end, - gen_server:cast(MainServerPid , {saveUtility , {UtilityName , IP , Port}}), %% TODO Add IP , Port to ets in main server record + gen_server:cast(MainServerPid , {saveUtility , {list_to_atom(UtilityName) , IP , Port}}), %% TODO Add IP , Port to ets in main server record Req = cowboy_req:reply(200, #{<<"content-type">> => <<"text/plain">>}, Reply, diff --git a/src_erl/NerlMonitor/src/MonitorGUI.py b/src_erl/NerlMonitor/src/MonitorGUI.py index f948084e..cdd29251 100644 --- a/src_erl/NerlMonitor/src/MonitorGUI.py +++ b/src_erl/NerlMonitor/src/MonitorGUI.py @@ -56,14 +56,36 @@ def GUI(MainPid): break if event == "Clear Log": MainWindow['-LOG-'].update('') - if event == "NerlNet Graph": - Show_Nerlnet_Graph() if not msg_queue.empty(): msg = msg_queue.get_nowait() if msg[0] == 'graph': - Show_Nerlnet_Graph(msg[1]) + Graph = Show_Nerlnet_Graph(msg[1]) MainWindow['-PHOLD-'].update(visible=False) MainWindow['-IMAGE-'].update(filename='NerlNetGraph.png' , visible=True , size=(800,600)) + elif msg[0] == 'update': + ClientName , WorkerName = msg[1].split('-') + default_color = 'skyblue' + node_colors = {node:default_color for node in Graph.nodes()} + node_colors[WorkerName] = 'red' + node_colors[ClientName] = 'yellow' + nx.set_node_attributes(Graph, node_colors, 'color') + colors = nx.get_node_attributes(Graph, 'color').values() + pos = nx.spectral_layout(Graph) + angle = 100 + rotated_pos = {node: (x*math.cos(angle) -y*math.sin(angle), x*math.sin(angle) + y*math.cos(angle)) for node, (x, y) in pos.items()} + + plt.figure(figsize=(8,6)) + nx.draw_networkx(Graph, rotated_pos, with_labels=True, node_color=colors , node_size=200, font_size=8, font_color='black' , edge_color='black' , width=1.5) + plt.savefig('NerlNetGraph.png' ,bbox_inches='tight' , dpi=125) + plt.close() + MainWindow['-IMAGE-'].update(filename='NerlNetGraph.png' , visible=True , size=(800,600)) + existing_text = values['-LOG-'] + if existing_text == '': + updated_text = f'{formatted_time()}: Worker {WorkerName} of Client {ClientName} is down.' + else: + updated_text = f'{existing_text}\n{formatted_time()}: Worker {WorkerName} of Client {ClientName} is down.' + MainWindow['-LOG-'].update(updated_text) + elif values['-LOG-'] != '': existing_text = values['-LOG-'] updated_text = f'{existing_text}\n{formatted_time()}: {msg}' @@ -91,16 +113,18 @@ def Show_Nerlnet_Graph(NerlGraph): NodesNames = [NodeTriplet.split(',')[0] for NodeTriplet in Nodes] NodesNames += WorkersNames + graph = nx.Graph() graph.add_nodes_from(NodesNames) graph.add_edges_from(EdgesSeperated) pos = nx.spectral_layout(graph) - angle = 40 + angle = 100 rotated_pos = {node: (x*math.cos(angle) -y*math.sin(angle), x*math.sin(angle) + y*math.cos(angle)) for node, (x, y) in pos.items()} plt.figure(figsize=(8,6)) - nx.draw_networkx(graph, rotated_pos, with_labels=True, node_color='skyblue', node_size=200, font_size=8, font_color='black' , edge_color='black' , width=1.5) + nx.draw_networkx(graph, rotated_pos, with_labels=True, node_color='skyblue' , node_size=200, font_size=8, font_color='black' , edge_color='black' , width=1.5) plt.savefig('NerlNetGraph.png' ,bbox_inches='tight' , dpi=125) plt.close() + return graph class MyProcess(Process): diff --git a/src_erl/NerlMonitor/src/handlers/nerlMonitor_handler.erl b/src_erl/NerlMonitor/src/handlers/nerlMonitor_handler.erl index cc773ee6..f68dfc04 100644 --- a/src_erl/NerlMonitor/src/handlers/nerlMonitor_handler.erl +++ b/src_erl/NerlMonitor/src/handlers/nerlMonitor_handler.erl @@ -2,11 +2,15 @@ -export([init/2]). -init(Req0, State = [MainScreen]) -> +-define(GUI , {'PyrlangProcess' , 'py@127.0.0.1'}). + +init(Req0, [Msg]) -> {_,Body,_} = cowboy_req:read_body(Req0), Data = binary_to_list(Body), - case Data of - worker_death_caught -> ok;%tell gui to change graph display and write in event log + io:format("-------------------Data ~p--------------~n" , [Data]), + case Msg of + utilInfo -> io:format("Update graph with: worker_down ~p~n" , [Data]) , + ?GUI ! {update ,Data}; _ -> ok % got unknown messge, ignore. end, @@ -15,5 +19,5 @@ init(Req0, State = [MainScreen]) -> #{<<"content-type">> => <<"text/plain">>}, <<"Got that">>, Req0), - {ok, Req, State}. + {ok, Req, Msg}. diff --git a/src_erl/NerlMonitor/src/nerlMonitor_app.erl b/src_erl/NerlMonitor/src/nerlMonitor_app.erl index 000a31f8..d08b5150 100644 --- a/src_erl/NerlMonitor/src/nerlMonitor_app.erl +++ b/src_erl/NerlMonitor/src/nerlMonitor_app.erl @@ -11,7 +11,7 @@ -export([start/2, stop/1]). --define(UTILLNAME,nerlMonitor). +-define(UTILNAME,nerlMonitor). -define(IP , "192.168.64.7"). -define(PORT, 8096). %port place holder -define(MSADDRES,"192.168.64.7:8080" ). %place holder @@ -24,15 +24,15 @@ start(_StartType, _StartArgs) -> Dispatch = cowboy_router:compile([ {'_', [ - {"/utillInfo",nerlMonitor_handler, []} + {"/utilInfo",nerlMonitor_handler, [utilInfo]} ]} ]), - {ok, _} = cowboy:start_clear(?UTILLNAME,[{port, ?PORT}],#{env => #{dispatch => Dispatch}}), + {ok, _} = cowboy:start_clear(?UTILNAME,[{port, ?PORT}],#{env => #{dispatch => Dispatch}}), io:format("nerlMonitor started , opening GUI...~n"), %%os:cmd('python3 src/MonitorGUI.py'), %% PyrlangNode: ('PyralngProcess' , 'py@127.0.0.1' , 'COOKIE') , sending message by: 'GUI ! HELLO.' URL = "http://" ++ ?MSADDRES ++ "/toolConnectionReq", - mainServerPing(URL,[list_to_binary(atom_to_list(?UTILLNAME) ++ "#") , list_to_binary(?IP ++ "#") , list_to_binary(integer_to_list(?PORT))]), %% TODO How to "import" nerl_tools + mainServerPing(URL,[list_to_binary(atom_to_list(?UTILNAME) ++ "#") , list_to_binary(?IP ++ "#") , list_to_binary(integer_to_list(?PORT))]), %% TODO How to "import" nerl_tools nerlMonitor_sup:start_link(). From 6024cc4bfbf44627138e78baa6cb75b881234a39 Mon Sep 17 00:00:00 2001 From: GuyPerets106 Date: Sat, 5 Aug 2023 15:22:33 +0000 Subject: [PATCH 23/40] [NerlMonitor] Fix aServer behaviour, more LOG msgs --- .../Architecture/arch_1PCSIM6WorkerMNist.json | 6 +++--- .../http_Nerlserver/src/Client/clientStatem.erl | 11 ++++++----- .../http_Nerlserver/src/MainServer/mainGenserver.erl | 2 +- src_erl/NerlMonitor/src/MonitorGUI.py | 6 ++++-- src_erl/erlBridge/workers/workerGeneric.erl | 2 +- src_py/apiServer/apiServer.py | 7 ++++++- 6 files changed, 21 insertions(+), 13 deletions(-) diff --git a/inputJsonFiles/Architecture/arch_1PCSIM6WorkerMNist.json b/inputJsonFiles/Architecture/arch_1PCSIM6WorkerMNist.json index 72b3c2c0..80b5df8b 100755 --- a/inputJsonFiles/Architecture/arch_1PCSIM6WorkerMNist.json +++ b/inputJsonFiles/Architecture/arch_1PCSIM6WorkerMNist.json @@ -6,13 +6,13 @@ }, "devices": [ { - "host": "192.168.0.108", + "host": "192.168.64.7", "entities": "mainServer,c1,c2,c3,c4,c5,c6,s1,r1,r2,r3,r4,r5,r6,apiServer" } ], "apiServer": { - "host": "192.168.0.108", + "host": "192.168.64.7", "port": "8095", "args": "" } @@ -26,7 +26,7 @@ , "mainServer": { - "host": "192.168.0.108", + "host": "192.168.64.7", "port": "8080", "args": "" } diff --git a/src_erl/Communication_Layer/http_Nerlserver/src/Client/clientStatem.erl b/src_erl/Communication_Layer/http_Nerlserver/src/Client/clientStatem.erl index 7779fc88..e43adc51 100644 --- a/src_erl/Communication_Layer/http_Nerlserver/src/Client/clientStatem.erl +++ b/src_erl/Communication_Layer/http_Nerlserver/src/Client/clientStatem.erl @@ -104,7 +104,7 @@ waitforWorkers(cast, In = {stateChange,WorkerName,MissedBatchesCount}, State = # w5 -> Pid = ets:lookup_element(EtsRef, w5, ?WORKER_PID_IDX), io:format("---------------------Killin w5-----------------------~n"), - exit(Pid,kill); + gen_statem:stop(Pid , shutdown , infinity); _ -> ok end, % io:format("remaining workers = ~p~n",[NewWaitforWorkers]), @@ -142,7 +142,7 @@ waitforWorkers(info, EventContent, State = #client_statem_state{myName = MyName, delete_worker(EtsRef,MyName,WorkerName), {next_state, waitforWorkers, State#client_statem_state{waitforWorkers = NewWaitforWorkers,etsRef = EtsRef}}; %delete worker from ets so client will not wait for it in the future - _Any-> + _Any-> ?LOG_INFO("client ~p got unexpected info: ~p~n",[MyName, EventContent]), %recevied unexpected messege, print to log/tell main server for debuging {next_state, waitforWorkers, State#client_statem_state{etsRef = EtsRef}} end. @@ -208,7 +208,7 @@ idle(info, EventContent, State = #client_statem_state{myName = MyName,etsRef = E %report worker down to main server delete_worker(EtsRef,MyName,WorkerName); %delete worker from ets so client will not wait for it in the future - _Any->ok + _Any-> ?LOG_INFO("client ~p got unexpected info: ~p~n",[MyName, EventContent]) %recevied unexpected messege, print to log/tell main server for debuging end, {next_state, idle, State#client_statem_state{etsRef = EtsRef}}. @@ -329,7 +329,7 @@ training(info, EventContent, State = #client_statem_state{myName = MyName,etsRef %report worker down to main server delete_worker(EtsRef,MyName,WorkerName); %delete worker from ets so client will not wait for it in the future - _Any->ok + _Any-> ?LOG_INFO("client ~p got unexpected info: ~p~n",[MyName, EventContent]) %recevied unexpected messege, print to log/tell main server for debuging end, {next_state, training, State#client_statem_state{etsRef = EtsRef}}. @@ -398,7 +398,7 @@ predict(info, EventContent, State = #client_statem_state{myName = MyName,etsRef %report worker down to main server delete_worker(EtsRef,MyName,WorkerName); %delete worker from ets so client will not wait for it in the future - _Any->ok + _Any-> ?LOG_INFO("client ~p got unexpected info: ~p~n",[MyName, EventContent]) %recevied unexpected messege, print to log/tell main server for debuging end, {next_state, predict, State#client_statem_state{etsRef = EtsRef}}. @@ -542,4 +542,5 @@ delete_worker(EtsRef,MyName,WorkerName)-> NerlGraph = ets:lookup_element(EtsRef, nerlnetGraph, ?ETS_KV_VAL_IDX), {Host , Port} = nerl_tools:getShortPath(MyName, ?MAIN_SERVER_ATOM, NerlGraph), nerl_tools:http_request(Host,Port,"worker_down",list_to_binary(atom_to_list(MyName) ++ "-" ++ atom_to_list(WorkerName))), + ?LOG_WARNING("Worker ~p is down, deleting it from client ~p~n",[WorkerName,MyName]), EtsRef. \ No newline at end of file diff --git a/src_erl/Communication_Layer/http_Nerlserver/src/MainServer/mainGenserver.erl b/src_erl/Communication_Layer/http_Nerlserver/src/MainServer/mainGenserver.erl index a7307efc..c4826240 100644 --- a/src_erl/Communication_Layer/http_Nerlserver/src/MainServer/mainGenserver.erl +++ b/src_erl/Communication_Layer/http_Nerlserver/src/MainServer/mainGenserver.erl @@ -319,11 +319,11 @@ handle_cast({worker_down,Body}, State = #main_genserver_state{msgCounter = MsgCo case ets:member(EtsRef,nerlMonitor) of true-> [{nerlMonitor , IP , Port}] = ets:lookup(EtsRef , nerlMonitor), - URL = "http://" ++ IP ++ ":" ++ Port ++ "/utilInfo", httpc:request(post,{URL, [],"application/x-www-form-urlencoded",Body}, [], []); false -> ok end, + ?LOG_WARNING(?LOG_HEADER++"Worker down , ~p disconneted~n",[binary_to_list(Body)]), {noreply, State#main_genserver_state{msgCounter = MsgCounter+1,etsRef=EtsRef}}; handle_cast(Request, State = #main_genserver_state{}) -> diff --git a/src_erl/NerlMonitor/src/MonitorGUI.py b/src_erl/NerlMonitor/src/MonitorGUI.py index cdd29251..a2d1e72c 100644 --- a/src_erl/NerlMonitor/src/MonitorGUI.py +++ b/src_erl/NerlMonitor/src/MonitorGUI.py @@ -107,7 +107,7 @@ def Show_Nerlnet_Graph(NerlGraph): Nodes = NerlGraph.split('#')[0:-1] Edges = NerlGraph.split('#')[-1].split(',')[0:-1] Workers = NerlGraph.split('#')[-1].split(',')[-1].split('!')[0:-1] - WorkersNames = [Worker.split('-')[0] for Worker in Workers] + WorkersNames = [Worker.split('-')[0] for Worker in Workers ] Edges += Workers EdgesSeperated = [(Edge.split('-')[0],Edge.split('-')[1]) for Edge in Edges if len(Edges) > 1] # ? What if no edges? NodesNames = [NodeTriplet.split(',')[0] for NodeTriplet in Nodes] @@ -117,7 +117,9 @@ def Show_Nerlnet_Graph(NerlGraph): graph = nx.Graph() graph.add_nodes_from(NodesNames) graph.add_edges_from(EdgesSeperated) - pos = nx.spectral_layout(graph) + my_labels = {'mainServer': 'mS' , 'apiServer': 'aS'} + nx.relabel_nodes(graph, my_labels , copy=False) + pos = nx.nx_agraph.graphviz_layout(graph, prog='dot') angle = 100 rotated_pos = {node: (x*math.cos(angle) -y*math.sin(angle), x*math.sin(angle) + y*math.cos(angle)) for node, (x, y) in pos.items()} plt.figure(figsize=(8,6)) diff --git a/src_erl/erlBridge/workers/workerGeneric.erl b/src_erl/erlBridge/workers/workerGeneric.erl index 40fbb35d..dfb7cbaa 100644 --- a/src_erl/erlBridge/workers/workerGeneric.erl +++ b/src_erl/erlBridge/workers/workerGeneric.erl @@ -109,7 +109,7 @@ handle_event(_EventType, _EventContent, _StateName, State = #workerGeneric_state %% necessary cleaning up. When it returns, the gen_statem terminates with %% Reason. The return value is ignored. terminate(_Reason, _StateName, _State) -> - ok. + nerlNIF:destroy_nif(ets:lookup_element(get(generic_worker_ets) , model_id , ?DATA_IDX)). %% @private %% @doc Convert process state when code is changed diff --git a/src_py/apiServer/apiServer.py b/src_py/apiServer/apiServer.py index ac6a4cad..b9699eed 100644 --- a/src_py/apiServer/apiServer.py +++ b/src_py/apiServer/apiServer.py @@ -413,8 +413,11 @@ def accuracy_matrix(self, expNum): # print(f"worker {worker}, has {len(workerNeuronRes[worker][TRUE_LABLE_IND])} labels, with {len(workerNeuronRes[worker][TRUE_LABLE_IND][j])} samples") # print(f"confusion {worker}:{j}, has is of {workerNeuronRes[worker][TRUE_LABLE_IND][j]}, {workerNeuronRes[worker][PRED_LABLE_IND][j]}") confMatList[worker][j] = confusion_matrix(workerNeuronRes[worker][globe.TRUE_LABLE_IND][j], workerNeuronRes[worker][globe.PRED_LABLE_IND][j]) - # print(confMatList[worker][j]) disp = ConfusionMatrixDisplay(confMatList[worker][j], display_labels=["X", labelNames[j]]) + if confMatList[worker][j].shape == (0,0): # ! Worker is down + disp = ConfusionMatrixDisplay(np.array([[0,0],[0,0]]), display_labels=["X", labelNames[j]]) + workerNeuronRes[worker][globe.TRUE_LABLE_IND][j] = [0,0] + workerNeuronRes[worker][globe.PRED_LABLE_IND][j] = [1,1] disp.plot(ax=axes[i, j], colorbar=False) disp.ax_.set_title(f'{worker}, class #{j}\nAccuracy={round(accuracy_score(workerNeuronRes[worker][globe.TRUE_LABLE_IND][j], workerNeuronRes[worker][globe.PRED_LABLE_IND][j]), 3)}') if i < len(workersList) - 1: @@ -437,6 +440,8 @@ def accuracy_matrix(self, expNum): statFile = open(statFileName, "a") for worker in confMatList: + if confMatList[worker][0].shape == (0,0): # ! Worker is down + continue for j, label in enumerate(confMatList[worker]): # Calculate the accuracy and other stats: tn, fp, fn, tp = label.ravel() From e2ee54b5d2b004bad69880cb67ce9a738552d287 Mon Sep 17 00:00:00 2001 From: GuyPerets106 Date: Mon, 7 Aug 2023 15:22:26 +0000 Subject: [PATCH 24/40] [NerlMonitor] Added worker termination for tests --- NerlnetMonitor.sh | 12 ++++++ .../src/Client/clientStatem.erl | 40 +++++++++++++++---- src_erl/NerlMonitor/src/MonitorGUI.py | 18 ++++++--- src_erl/NerlMonitor/src/nerlMonitor_app.erl | 10 +++-- src_py/apiServer/apiServer.py | 2 +- 5 files changed, 65 insertions(+), 17 deletions(-) create mode 100755 NerlnetMonitor.sh diff --git a/NerlnetMonitor.sh b/NerlnetMonitor.sh new file mode 100755 index 00000000..a95ee9d2 --- /dev/null +++ b/NerlnetMonitor.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +MONITOR_PATH="src_erl/NerlMonitor" +GUI_PATH="src_erl/NerlMonitor/src" + +echo "NerlnetMonitor Activated" + + +cd $MONITOR_PATH +rebar3 shell --name erl@127.0.0.1 --setcookie COOKIE + +cd ../../ diff --git a/src_erl/Communication_Layer/http_Nerlserver/src/Client/clientStatem.erl b/src_erl/Communication_Layer/http_Nerlserver/src/Client/clientStatem.erl index e43adc51..cfc5679f 100644 --- a/src_erl/Communication_Layer/http_Nerlserver/src/Client/clientStatem.erl +++ b/src_erl/Communication_Layer/http_Nerlserver/src/Client/clientStatem.erl @@ -100,14 +100,41 @@ format_status(_Opt, [_PDict, _StateName, _State]) -> Status = some_term, Status. %% ==============STATES================= waitforWorkers(cast, In = {stateChange,WorkerName,MissedBatchesCount}, State = #client_statem_state{myName = MyName,waitforWorkers = WaitforWorkers,nextState = NextState, etsRef = EtsRef}) -> NewWaitforWorkers = WaitforWorkers -- [WorkerName], - case WorkerName of - w5 -> - Pid = ets:lookup_element(EtsRef, w5, ?WORKER_PID_IDX), - io:format("---------------------Killin w5-----------------------~n"), - gen_statem:stop(Pid , shutdown , infinity); + % case WorkerName of + % w5 -> + % Pid = ets:lookup_element(EtsRef, w5, ?WORKER_PID_IDX), + % io:format("---------------------Killin w5-----------------------~n"), + % gen_statem:stop(Pid , shutdown , infinity); + % _ -> ok + % end, + % io:format("remaining workers = ~p~n",[NewWaitforWorkers]), + case NextState of + training -> + case WorkerName of + w5 -> + case ets:member(EtsRef , w5) of + true -> + Pid = ets:lookup_element(EtsRef, w5, ?WORKER_PID_IDX), + io:format("---------------------Killin w5-----------------------~n"), + gen_statem:stop(Pid , shutdown , infinity); + _ -> ok + end; + _ -> ok + end; + predict -> + case WorkerName of + w6 -> + case ets:member(EtsRef , w6) of + true -> + Pid = ets:lookup_element(EtsRef, w6, ?WORKER_PID_IDX), + io:format("---------------------Killin w6-----------------------~n"), + gen_statem:stop(Pid , shutdown , infinity); + _ -> ok + end; + _ -> ok + end; _ -> ok end, - % io:format("remaining workers = ~p~n",[NewWaitforWorkers]), ets:update_counter(EtsRef, msgCounter, 1), % last is increment value ets:update_counter(EtsRef, infoIn, nerl_tools:calculate_size(In)), ets:update_element(EtsRef, WorkerName,[{?WORKER_TRAIN_MISSED_IDX,MissedBatchesCount}]), %% update missed batches count @@ -340,7 +367,6 @@ predict(cast, In = {sample,Body}, State = #client_statem_state{etsRef = EtsRef}) ets:update_counter(EtsRef, infoIn, nerl_tools:calculate_size(In)), {ClientName, WorkerNameStr, CSVName, BatchNumber, BatchOfSamples} = binary_to_term(Body), WorkerName = list_to_atom(WorkerNameStr), - Start = os:timestamp(), WorkerOfThisClient = ets:member(EtsRef, WorkerName), if diff --git a/src_erl/NerlMonitor/src/MonitorGUI.py b/src_erl/NerlMonitor/src/MonitorGUI.py index a2d1e72c..28a14d15 100644 --- a/src_erl/NerlMonitor/src/MonitorGUI.py +++ b/src_erl/NerlMonitor/src/MonitorGUI.py @@ -59,18 +59,18 @@ def GUI(MainPid): if not msg_queue.empty(): msg = msg_queue.get_nowait() if msg[0] == 'graph': - Graph = Show_Nerlnet_Graph(msg[1]) + Graph , node_colors = Show_Nerlnet_Graph(msg[1]) MainWindow['-PHOLD-'].update(visible=False) MainWindow['-IMAGE-'].update(filename='NerlNetGraph.png' , visible=True , size=(800,600)) elif msg[0] == 'update': ClientName , WorkerName = msg[1].split('-') - default_color = 'skyblue' - node_colors = {node:default_color for node in Graph.nodes()} + node_colors[WorkerName] = 'red' node_colors[ClientName] = 'yellow' nx.set_node_attributes(Graph, node_colors, 'color') colors = nx.get_node_attributes(Graph, 'color').values() - pos = nx.spectral_layout(Graph) + + pos = nx.nx_agraph.graphviz_layout(Graph, prog='dot') angle = 100 rotated_pos = {node: (x*math.cos(angle) -y*math.sin(angle), x*math.sin(angle) + y*math.cos(angle)) for node, (x, y) in pos.items()} @@ -117,16 +117,22 @@ def Show_Nerlnet_Graph(NerlGraph): graph = nx.Graph() graph.add_nodes_from(NodesNames) graph.add_edges_from(EdgesSeperated) + my_labels = {'mainServer': 'mS' , 'apiServer': 'aS'} nx.relabel_nodes(graph, my_labels , copy=False) + default_color = 'skyblue' + node_colors = {node:default_color for node in graph.nodes()} + nx.set_node_attributes(graph, node_colors, 'color') + colors = nx.get_node_attributes(graph, 'color').values() + pos = nx.nx_agraph.graphviz_layout(graph, prog='dot') angle = 100 rotated_pos = {node: (x*math.cos(angle) -y*math.sin(angle), x*math.sin(angle) + y*math.cos(angle)) for node, (x, y) in pos.items()} plt.figure(figsize=(8,6)) - nx.draw_networkx(graph, rotated_pos, with_labels=True, node_color='skyblue' , node_size=200, font_size=8, font_color='black' , edge_color='black' , width=1.5) + nx.draw_networkx(graph, rotated_pos, with_labels=True, node_color=colors , node_size=200, font_size=8, font_color='black' , edge_color='black' , width=1.5) plt.savefig('NerlNetGraph.png' ,bbox_inches='tight' , dpi=125) plt.close() - return graph + return graph , node_colors class MyProcess(Process): diff --git a/src_erl/NerlMonitor/src/nerlMonitor_app.erl b/src_erl/NerlMonitor/src/nerlMonitor_app.erl index d08b5150..dd80a85c 100644 --- a/src_erl/NerlMonitor/src/nerlMonitor_app.erl +++ b/src_erl/NerlMonitor/src/nerlMonitor_app.erl @@ -9,7 +9,7 @@ -include("../../Communication_Layer/http_Nerlserver/src/nerl_tools.hrl"). --export([start/2, stop/1]). +-export([start/2, stop/1 , link_GUI/1]). -define(UTILNAME,nerlMonitor). -define(IP , "192.168.64.7"). @@ -30,7 +30,7 @@ start(_StartType, _StartArgs) -> ]), {ok, _} = cowboy:start_clear(?UTILNAME,[{port, ?PORT}],#{env => #{dispatch => Dispatch}}), io:format("nerlMonitor started , opening GUI...~n"), - %%os:cmd('python3 src/MonitorGUI.py'), %% PyrlangNode: ('PyralngProcess' , 'py@127.0.0.1' , 'COOKIE') , sending message by: 'GUI ! HELLO.' + GUI_PID = spawn_link(?MODULE , link_GUI , [self()]) , %% PyrlangNode: ('PyralngProcess' , 'py@127.0.0.1' , 'COOKIE') , sending message by: 'GUI ! HELLO.' URL = "http://" ++ ?MSADDRES ++ "/toolConnectionReq", mainServerPing(URL,[list_to_binary(atom_to_list(?UTILNAME) ++ "#") , list_to_binary(?IP ++ "#") , list_to_binary(integer_to_list(?PORT))]), %% TODO How to "import" nerl_tools nerlMonitor_sup:start_link(). @@ -43,7 +43,7 @@ mainServerPing(URL,Body)-> Response = httpc:request(post,{URL, [],"application/x-www-form-urlencoded",Body}, [], []), case Response of {error,_}-> - timer:sleep(500), + timer:sleep(1000), mainServerPing(URL,Body); {ok,{_ResCode, _Headers, Data}}-> io:format("got1 response from main server~n"), @@ -65,4 +65,8 @@ functions()-> ok. stop(_State) -> ok. +link_GUI(Pid) -> + os:cmd('python3 src/MonitorGUI.py'), + io:format("GUI Closed~n"). + diff --git a/src_py/apiServer/apiServer.py b/src_py/apiServer/apiServer.py index b9699eed..2f7bb31f 100644 --- a/src_py/apiServer/apiServer.py +++ b/src_py/apiServer/apiServer.py @@ -420,7 +420,7 @@ def accuracy_matrix(self, expNum): workerNeuronRes[worker][globe.PRED_LABLE_IND][j] = [1,1] disp.plot(ax=axes[i, j], colorbar=False) disp.ax_.set_title(f'{worker}, class #{j}\nAccuracy={round(accuracy_score(workerNeuronRes[worker][globe.TRUE_LABLE_IND][j], workerNeuronRes[worker][globe.PRED_LABLE_IND][j]), 3)}') - if i < len(workersList) - 1: + if i < len(workersList) - 1: # ? Ask David why this is needed disp.ax_.set_xlabel('') #remove "predicted label" if j != 0: disp.ax_.set_ylabel('') #remove "true label" From 2173654be2f22618439839be3a76c09f9efad0fa Mon Sep 17 00:00:00 2001 From: GuyPerets106 Date: Tue, 8 Aug 2023 06:50:18 +0000 Subject: [PATCH 25/40] [NerlMonitor] Changed NerlGraph colors added log-msg --- .../src/Client/clientStatem.erl | 57 ++++++++++--------- src_erl/NerlMonitor/src/MonitorGUI.py | 11 ++-- 2 files changed, 35 insertions(+), 33 deletions(-) diff --git a/src_erl/Communication_Layer/http_Nerlserver/src/Client/clientStatem.erl b/src_erl/Communication_Layer/http_Nerlserver/src/Client/clientStatem.erl index cfc5679f..ac899b17 100644 --- a/src_erl/Communication_Layer/http_Nerlserver/src/Client/clientStatem.erl +++ b/src_erl/Communication_Layer/http_Nerlserver/src/Client/clientStatem.erl @@ -108,33 +108,33 @@ waitforWorkers(cast, In = {stateChange,WorkerName,MissedBatchesCount}, State = # % _ -> ok % end, % io:format("remaining workers = ~p~n",[NewWaitforWorkers]), - case NextState of - training -> - case WorkerName of - w5 -> - case ets:member(EtsRef , w5) of - true -> - Pid = ets:lookup_element(EtsRef, w5, ?WORKER_PID_IDX), - io:format("---------------------Killin w5-----------------------~n"), - gen_statem:stop(Pid , shutdown , infinity); - _ -> ok - end; - _ -> ok - end; - predict -> - case WorkerName of - w6 -> - case ets:member(EtsRef , w6) of - true -> - Pid = ets:lookup_element(EtsRef, w6, ?WORKER_PID_IDX), - io:format("---------------------Killin w6-----------------------~n"), - gen_statem:stop(Pid , shutdown , infinity); - _ -> ok - end; - _ -> ok - end; - _ -> ok - end, + % case NextState of + % training -> + % case WorkerName of + % w5 -> + % case ets:member(EtsRef , w5) of + % true -> + % Pid = ets:lookup_element(EtsRef, w5, ?WORKER_PID_IDX), + % io:format("---------------------Killin w5-----------------------~n"), + % gen_statem:stop(Pid , shutdown , infinity); + % _ -> ok + % end; + % _ -> ok + % end; + % predict -> + % case WorkerName of + % w6 -> + % case ets:member(EtsRef , w6) of + % true -> + % Pid = ets:lookup_element(EtsRef, w6, ?WORKER_PID_IDX), + % io:format("---------------------Killin w6-----------------------~n"), + % gen_statem:stop(Pid , shutdown , infinity); + % _ -> ok + % end; + % _ -> ok + % end; + % _ -> ok + % end, ets:update_counter(EtsRef, msgCounter, 1), % last is increment value ets:update_counter(EtsRef, infoIn, nerl_tools:calculate_size(In)), ets:update_element(EtsRef, WorkerName,[{?WORKER_TRAIN_MISSED_IDX,MissedBatchesCount}]), %% update missed batches count @@ -559,7 +559,7 @@ cast_message_to_workers(EtsRef, Msg) -> end, lists:foreach(Func, Workers). -%activated if client detected that a worker stoped, will delete it from ets so as not to wait for responsed from it in the future. +%activated if client detected that a worker stopped, will delete it from ets so as not to wait for responsed from it in the future. delete_worker(EtsRef,MyName,WorkerName)-> ets:delete(EtsRef , WorkerName), NameList = ets:lookup_element(EtsRef, workersNames, ?ETS_KV_VAL_IDX), @@ -569,4 +569,5 @@ delete_worker(EtsRef,MyName,WorkerName)-> {Host , Port} = nerl_tools:getShortPath(MyName, ?MAIN_SERVER_ATOM, NerlGraph), nerl_tools:http_request(Host,Port,"worker_down",list_to_binary(atom_to_list(MyName) ++ "-" ++ atom_to_list(WorkerName))), ?LOG_WARNING("Worker ~p is down, deleting it from client ~p~n",[WorkerName,MyName]), + %% TODO Verify NIF active_model_id list EtsRef. \ No newline at end of file diff --git a/src_erl/NerlMonitor/src/MonitorGUI.py b/src_erl/NerlMonitor/src/MonitorGUI.py index 28a14d15..3cee1ce5 100644 --- a/src_erl/NerlMonitor/src/MonitorGUI.py +++ b/src_erl/NerlMonitor/src/MonitorGUI.py @@ -62,11 +62,12 @@ def GUI(MainPid): Graph , node_colors = Show_Nerlnet_Graph(msg[1]) MainWindow['-PHOLD-'].update(visible=False) MainWindow['-IMAGE-'].update(filename='NerlNetGraph.png' , visible=True , size=(800,600)) + MainWindow['-LOG-'].update(f'{formatted_time()}: NerlNet Graph Received.') elif msg[0] == 'update': ClientName , WorkerName = msg[1].split('-') - node_colors[WorkerName] = 'red' - node_colors[ClientName] = 'yellow' + node_colors[WorkerName] = 'black' + node_colors[ClientName] = 'gray' nx.set_node_attributes(Graph, node_colors, 'color') colors = nx.get_node_attributes(Graph, 'color').values() @@ -75,7 +76,7 @@ def GUI(MainPid): rotated_pos = {node: (x*math.cos(angle) -y*math.sin(angle), x*math.sin(angle) + y*math.cos(angle)) for node, (x, y) in pos.items()} plt.figure(figsize=(8,6)) - nx.draw_networkx(Graph, rotated_pos, with_labels=True, node_color=colors , node_size=200, font_size=8, font_color='black' , edge_color='black' , width=1.5) + nx.draw_networkx(Graph, rotated_pos, with_labels=True, node_color=colors , node_size=200, font_size=8, font_color='white' , edge_color='black' , width=1.5) plt.savefig('NerlNetGraph.png' ,bbox_inches='tight' , dpi=125) plt.close() MainWindow['-IMAGE-'].update(filename='NerlNetGraph.png' , visible=True , size=(800,600)) @@ -120,7 +121,7 @@ def Show_Nerlnet_Graph(NerlGraph): my_labels = {'mainServer': 'mS' , 'apiServer': 'aS'} nx.relabel_nodes(graph, my_labels , copy=False) - default_color = 'skyblue' + default_color = 'darkred' node_colors = {node:default_color for node in graph.nodes()} nx.set_node_attributes(graph, node_colors, 'color') colors = nx.get_node_attributes(graph, 'color').values() @@ -129,7 +130,7 @@ def Show_Nerlnet_Graph(NerlGraph): angle = 100 rotated_pos = {node: (x*math.cos(angle) -y*math.sin(angle), x*math.sin(angle) + y*math.cos(angle)) for node, (x, y) in pos.items()} plt.figure(figsize=(8,6)) - nx.draw_networkx(graph, rotated_pos, with_labels=True, node_color=colors , node_size=200, font_size=8, font_color='black' , edge_color='black' , width=1.5) + nx.draw_networkx(graph, rotated_pos, with_labels=True, node_color=colors , node_size=200, font_size=8, font_color='white' , edge_color='black' , width=1.5) plt.savefig('NerlNetGraph.png' ,bbox_inches='tight' , dpi=125) plt.close() return graph , node_colors From 3aa34ab4d99eddd8e413bb273ccc38cfc17decad Mon Sep 17 00:00:00 2001 From: GuyPerets106 Date: Tue, 8 Aug 2023 14:29:39 +0000 Subject: [PATCH 26/40] [NerlMonitor] Added worker_kill option --- .../src/Client/clientStateHandler.erl | 3 +- .../src/Client/clientStatem.erl | 34 ++++++++++++++++++- .../src/MainServer/mainGenserver.erl | 7 ++++ .../src/Router/routerGenserver.erl | 5 +++ .../src/Router/routingHandler.erl | 3 +- .../http_Nerlserver/src/nerlNetServer_app.erl | 7 ++-- .../http_Nerlserver/src/utilities_handler.erl | 15 ++++++++ src_erl/NerlMonitor/src/nerlMonitor_app.erl | 3 +- 8 files changed, 71 insertions(+), 6 deletions(-) diff --git a/src_erl/Communication_Layer/http_Nerlserver/src/Client/clientStateHandler.erl b/src_erl/Communication_Layer/http_Nerlserver/src/Client/clientStateHandler.erl index e1945527..a9601a68 100644 --- a/src_erl/Communication_Layer/http_Nerlserver/src/Client/clientStateHandler.erl +++ b/src_erl/Communication_Layer/http_Nerlserver/src/Client/clientStateHandler.erl @@ -30,7 +30,8 @@ init(Req0, [Action,Client_StateM_Pid]) -> idle -> gen_statem:cast(Client_StateM_Pid,{idle}); training -> gen_statem:cast(Client_StateM_Pid,{training}); predict -> gen_statem:cast(Client_StateM_Pid,{predict}); - statistics -> gen_statem:cast(Client_StateM_Pid,{statistics}) + statistics -> gen_statem:cast(Client_StateM_Pid,{statistics}); + worker_kill -> gen_statem:cast(Client_StateM_Pid,{worker_kill , Body}) end, %% reply ACKnowledge to main server for initiating, later send finished initiating http_request from client_stateM diff --git a/src_erl/Communication_Layer/http_Nerlserver/src/Client/clientStatem.erl b/src_erl/Communication_Layer/http_Nerlserver/src/Client/clientStatem.erl index ac899b17..f1c43c9b 100644 --- a/src_erl/Communication_Layer/http_Nerlserver/src/Client/clientStatem.erl +++ b/src_erl/Communication_Layer/http_Nerlserver/src/Client/clientStatem.erl @@ -20,7 +20,7 @@ code_change/4, callback_mode/0, idle/3, training/3,waitforWorkers/3]). --import(nerlNIF,[validate_nerltensor_erl/1]). +-import(nerlNIF,[validate_nerltensor_erl/1 , get_active_models_ids_list/0]). -define(ETS_KV_VAL_IDX, 2). % key value pairs --> value index is 2 -define(WORKER_PID_IDX, 2). @@ -153,6 +153,14 @@ waitforWorkers(cast, In = {NewState}, State = #client_statem_state{myName = MyNa cast_message_to_workers(EtsRef, {NewState}), {next_state, waitforWorkers, State#client_statem_state{nextState = NewState, waitforWorkers = Workers}}; +waitforWorkers(cast, {worker_kill , Body}, State = #client_statem_state{etsRef = EtsRef}) -> + ets:update_counter(EtsRef, msgCounter, 1), + [_ , Target] = string:split(binary_to_list(Body), "-"), + WorkerName = list_to_atom(Target), + Pid = ets:lookup_element(EtsRef, WorkerName, ?WORKER_PID_IDX), + gen_statem:stop(Pid, shutdown , infinity), + {next_state, waitforWorkers, State#client_statem_state{etsRef = EtsRef}}; + waitforWorkers(cast, EventContent, State = #client_statem_state{etsRef = EtsRef}) -> ets:update_counter(EtsRef, msgCounter, 1), ets:update_counter(EtsRef, infoIn, nerl_tools:calculate_size(EventContent)), @@ -213,6 +221,14 @@ idle(cast, In = {training}, State = #client_statem_state{etsRef = EtsRef}) -> cast_message_to_workers(EtsRef, MessageToCast), {next_state, waitforWorkers, State#client_statem_state{waitforWorkers= ets:lookup_element(EtsRef, workersNames, ?ETS_KV_VAL_IDX), nextState = training}}; +idle(cast, {worker_kill , Body}, State = #client_statem_state{etsRef = EtsRef}) -> + ets:update_counter(EtsRef, msgCounter, 1), + [_ , Target] = string:split(binary_to_list(Body), "-"), + WorkerName = list_to_atom(Target), + Pid = ets:lookup_element(EtsRef, WorkerName, ?WORKER_PID_IDX), + gen_statem:stop(Pid, shutdown , infinity), + {next_state, idle, State#client_statem_state{etsRef = EtsRef}}; + idle(cast, In = {predict}, State = #client_statem_state{etsRef = EtsRef}) -> io:format("client going to state predict~n",[]), ets:update_counter(EtsRef, msgCounter, 1), @@ -277,6 +293,14 @@ training(cast, In = {custom_worker_message, WorkersList, WeightsTensor}, State = end, lists:foreach(Func, WorkersList), {keep_state, State}; + +training(cast, {worker_kill , Body}, State = #client_statem_state{etsRef = EtsRef}) -> + ets:update_counter(EtsRef, msgCounter, 1), + [_ , Target] = string:split(binary_to_list(Body), "-"), + WorkerName = list_to_atom(Target), + Pid = ets:lookup_element(EtsRef, WorkerName, ?WORKER_PID_IDX), + gen_statem:stop(Pid, shutdown , infinity), + {next_state, training, State#client_statem_state{etsRef = EtsRef}}; % TODO Validate this state - sample and empty list @@ -392,6 +416,14 @@ predict(cast, In = {predictRes,WorkerName,InputName,ResultID,PredictNerlTensor, nerl_tools:http_request(RouterHost,RouterPort,"predictRes", term_to_binary({atom_to_list(WorkerName), InputName, ResultID, {PredictNerlTensor, Type}})), {next_state, predict, State#client_statem_state{etsRef = EtsRef}}; +predict(cast, {worker_kill , Body}, State = #client_statem_state{etsRef = EtsRef}) -> + ets:update_counter(EtsRef, msgCounter, 1), + [_ , Target] = string:split(binary_to_list(Body), "-"), + WorkerName = list_to_atom(Target), + Pid = ets:lookup_element(EtsRef, WorkerName, ?WORKER_PID_IDX), + gen_statem:stop(Pid, shutdown , infinity), + {next_state, predict, State#client_statem_state{etsRef = EtsRef}}; + % TODO from predict directly to training?!?!? predict(cast, In = {training}, State = #client_statem_state{etsRef = EtsRef}) -> ets:update_counter(EtsRef, msgCounter, 1), diff --git a/src_erl/Communication_Layer/http_Nerlserver/src/MainServer/mainGenserver.erl b/src_erl/Communication_Layer/http_Nerlserver/src/MainServer/mainGenserver.erl index c4826240..de6798a7 100644 --- a/src_erl/Communication_Layer/http_Nerlserver/src/MainServer/mainGenserver.erl +++ b/src_erl/Communication_Layer/http_Nerlserver/src/MainServer/mainGenserver.erl @@ -326,6 +326,13 @@ handle_cast({worker_down,Body}, State = #main_genserver_state{msgCounter = MsgCo ?LOG_WARNING(?LOG_HEADER++"Worker down , ~p disconneted~n",[binary_to_list(Body)]), {noreply, State#main_genserver_state{msgCounter = MsgCounter+1,etsRef=EtsRef}}; +handle_cast({worker_kill , WorkerName} , State = #main_genserver_state{workersMap = WorkersMap , msgCounter = MsgCounter}) -> + ?LOG_WARNING(?LOG_HEADER++"Worker ~p killed~n",[WorkerName]), + ClientName = maps:get(WorkerName,WorkersMap), % ! Worker Name is binary? + Body = list_to_binary(atom_to_list(ClientName)) ++ "-" ++ WorkerName, + nerl_tools:sendHTTP(?MAIN_SERVER_ATOM , ClientName , atom_to_list(worker_kill) , Body), + {noreply, State#main_genserver_state{workersMap = WorkersMap , msgCounter = MsgCounter+1}}; + handle_cast(Request, State = #main_genserver_state{}) -> io:format("main server cast ignored: ~p~n",[Request]), {noreply, State}. diff --git a/src_erl/Communication_Layer/http_Nerlserver/src/Router/routerGenserver.erl b/src_erl/Communication_Layer/http_Nerlserver/src/Router/routerGenserver.erl index a3540dbc..9f4f0ac3 100644 --- a/src_erl/Communication_Layer/http_Nerlserver/src/Router/routerGenserver.erl +++ b/src_erl/Communication_Layer/http_Nerlserver/src/Router/routerGenserver.erl @@ -176,6 +176,11 @@ handle_cast({getStats,_Body}, State = #router_genserver_state{myName = MyName, nerl_tools:http_request(Host,Port,"routerStats",Mes), {noreply, State#router_genserver_state{msgCounter = MsgCounter+1}}; +handle_cast({worker_kill , Body} , State = #router_genserver_state{msgCounter = MsgCounter, myName = MyName}) -> + [Target , _] = string:split(binary_to_list(Body) , "-"), + nerl_tools:sendHTTP(MyName, list_to_atom(Target), "worker_kill", Body), + {noreply, State#router_genserver_state{msgCounter = MsgCounter+1}}; + %monitor handle_cast({worker_down,Body}, State = #router_genserver_state{myName = MyName, msgCounter = MsgCounter, nerlnetGraph = NerlnetGraph}) -> diff --git a/src_erl/Communication_Layer/http_Nerlserver/src/Router/routingHandler.erl b/src_erl/Communication_Layer/http_Nerlserver/src/Router/routingHandler.erl index 22ec64cf..c391e481 100644 --- a/src_erl/Communication_Layer/http_Nerlserver/src/Router/routingHandler.erl +++ b/src_erl/Communication_Layer/http_Nerlserver/src/Router/routingHandler.erl @@ -76,7 +76,8 @@ init(Req0, State) -> %%%%%%%%%%%%%%GUI actions getStats -> gen_server:cast(Router_genserver_Pid, {getStats,Body}); %monitor - worker_down -> gen_server:cast(Router_genserver_Pid, {worker_down,Body}) + worker_down -> gen_server:cast(Router_genserver_Pid, {worker_down,Body}); + worker_kill -> gen_server:cast(Router_genserver_Pid, {worker_kill,Body}) end, Reply = io_lib:format(" ", []), diff --git a/src_erl/Communication_Layer/http_Nerlserver/src/nerlNetServer_app.erl b/src_erl/Communication_Layer/http_Nerlserver/src/nerlNetServer_app.erl index 13b5e174..4d20239a 100644 --- a/src_erl/Communication_Layer/http_Nerlserver/src/nerlNetServer_app.erl +++ b/src_erl/Communication_Layer/http_Nerlserver/src/nerlNetServer_app.erl @@ -187,7 +187,8 @@ createClientsAndWorkers() -> {"/clientTraining",clientStateHandler, [training,ClientStatemPid]}, {"/clientIdle",clientStateHandler, [idle,ClientStatemPid]}, {"/clientPredict",clientStateHandler, [predict,ClientStatemPid]}, - {"/weightsVector",clientStateHandler, [vector,ClientStatemPid]} + {"/weightsVector",clientStateHandler, [vector,ClientStatemPid]}, + {"/worker_kill" , clientStateHandler , [worker_kill , ClientStatemPid]} ]} ]), init_cowboy_start_clear(Client, {HostName, Port},NerlClientDispatch) @@ -267,7 +268,8 @@ createRouters(MapOfRouters, HostName) -> %%GUI actions {"/getStats",routingHandler, [getStats,RouterGenServerPid]}, %monitor actions - {"/worker_down",routingHandler, [worker_down,RouterGenServerPid]} + {"/worker_down",routingHandler, [worker_down,RouterGenServerPid]}, + {"/worker_kill" , routingHandler , [worker_kill , RouterGenServerPid]} ]} ]), %% cowboy:start_clear(Name, TransOpts, ProtoOpts) - an http_listener @@ -307,6 +309,7 @@ createMainServer(true,BatchSize,HostName) -> {"/getGraph",[],guiHandler, [getGraph, MainGenServerPid]}, {"/getStats",[],guiHandler, [getStats, MainGenServerPid]}, {"/toolConnectionReq" , [] , utilities_handler , [MainGenServerPid]} , %% Added with NerlMonitor Project + {"/worker_kill" , [] , utilities_handler , [worker_kill , MainGenServerPid]}, {"/worker_down",actionHandler, [worker_down,MainGenServerPid]}, {"/[...]", [],noMatchingRouteHandler, [MainGenServerPid]} ]} diff --git a/src_erl/Communication_Layer/http_Nerlserver/src/utilities_handler.erl b/src_erl/Communication_Layer/http_Nerlserver/src/utilities_handler.erl index 563361e1..b1bc8b67 100644 --- a/src_erl/Communication_Layer/http_Nerlserver/src/utilities_handler.erl +++ b/src_erl/Communication_Layer/http_Nerlserver/src/utilities_handler.erl @@ -5,6 +5,7 @@ -export([init/2]). init(Req0 , [MainServerPid]) -> + io:format("Req is ~p~n" , [Req0]), {_,Body,_} = cowboy_req:read_body(Req0), io:format("Body: ~p~n" , [Body]) , Formatted = binary_to_list(Body), @@ -21,6 +22,20 @@ init(Req0 , [MainServerPid]) -> Reply = Graph ++ "," ++ Workers end, gen_server:cast(MainServerPid , {saveUtility , {list_to_atom(UtilityName) , IP , Port}}), %% TODO Add IP , Port to ets in main server record + Req = cowboy_req:reply(200, + #{<<"content-type">> => <<"text/plain">>}, + Reply, + Req0), + {ok, Req, MainServerPid}; + +init(Req0 , [Action , MainServerPid]) -> + {_,Body,_} = cowboy_req:read_body(Req0), + case Action of + worker_kill -> + gen_statem:cast(MainServerPid , {worker_kill , Body}); + _ -> ok + end, + Reply = io_lib:format("ACK", []), Req = cowboy_req:reply(200, #{<<"content-type">> => <<"text/plain">>}, Reply, diff --git a/src_erl/NerlMonitor/src/nerlMonitor_app.erl b/src_erl/NerlMonitor/src/nerlMonitor_app.erl index dd80a85c..dd8c70e0 100644 --- a/src_erl/NerlMonitor/src/nerlMonitor_app.erl +++ b/src_erl/NerlMonitor/src/nerlMonitor_app.erl @@ -39,7 +39,8 @@ start(_StartType, _StartArgs) -> %ping main server in 0.5 sec intervals with connection request. will stop when got valid response. mainServerPing(URL,Body)-> - io:format("pinging main server~n"), + io:format("pinging main server~n"), + io:format("Body: ~p~n" , [Body]), Response = httpc:request(post,{URL, [],"application/x-www-form-urlencoded",Body}, [], []), case Response of {error,_}-> From d86129d774b4829a274e5cae4b92e0d2a1a1fc53 Mon Sep 17 00:00:00 2001 From: GuyPerets106 Date: Thu, 10 Aug 2023 11:51:04 +0000 Subject: [PATCH 27/40] [NerlMonitor] Added statistics --- .../src/Client/clientStatem.erl | 15 +++++- .../src/MainServer/mainGenserver.erl | 29 ++++++++--- .../http_Nerlserver/src/utilities_handler.erl | 2 - src_erl/NerlMonitor/src/MonitorGUI.py | 49 ++++++++++++++++--- .../src/handlers/nerlMonitor_handler.erl | 3 +- src_erl/NerlMonitor/src/nerlMonitor_app.erl | 3 +- 6 files changed, 81 insertions(+), 20 deletions(-) diff --git a/src_erl/Communication_Layer/http_Nerlserver/src/Client/clientStatem.erl b/src_erl/Communication_Layer/http_Nerlserver/src/Client/clientStatem.erl index f1c43c9b..d470a864 100644 --- a/src_erl/Communication_Layer/http_Nerlserver/src/Client/clientStatem.erl +++ b/src_erl/Communication_Layer/http_Nerlserver/src/Client/clientStatem.erl @@ -209,9 +209,9 @@ idle(cast, In = {custom_worker_message, {From, To}}, State = #client_statem_stat {keep_state, State}; idle(cast, In = {statistics}, State = #client_statem_state{ myName = MyName, etsRef = EtsRef}) -> - sendStatistics(EtsRef), ets:update_counter(EtsRef, msgCounter, 1), % last param is increment value ets:update_counter(EtsRef, infoIn, nerl_tools:calculate_size(In)), + sendStatistics(EtsRef), {next_state, idle, State}; idle(cast, In = {training}, State = #client_statem_state{etsRef = EtsRef}) -> @@ -368,6 +368,13 @@ training(cast, In = {loss,WorkerName,LossFunction,_Time_NIF}, State = #client_st nerl_tools:http_request(RouterHost,RouterPort,"lossFunction", term_to_binary({WorkerName,LossFunction})), {next_state, training, State#client_statem_state{myName = MyName,etsRef = EtsRef}}; +training(cast, In = {statistics}, State = #client_statem_state{ myName = MyName, etsRef = EtsRef}) -> + io:format("client ~p got statistics ~n",[MyName]), + ets:update_counter(EtsRef, msgCounter, 1), % last param is increment value + ets:update_counter(EtsRef, infoIn, nerl_tools:calculate_size(In)), + sendStatistics(EtsRef), + {next_state, training, State}; + training(cast, EventContent, State = #client_statem_state{etsRef = EtsRef, myName = MyName}) -> ?LOG_WARNING("client ~p training ignored!!!: ~p ~n!!!",[MyName, EventContent]), ets:update_counter(EtsRef, infoIn, nerl_tools:calculate_size(EventContent)), @@ -443,6 +450,12 @@ predict(cast, In = {idle}, State = #client_statem_state{etsRef = EtsRef}) -> Workers = ets:lookup_element(EtsRef, workersNames, ?ETS_KV_VAL_IDX), {next_state, waitforWorkers, State#client_statem_state{nextState = idle, waitforWorkers = Workers, etsRef = EtsRef}}; +predict(cast, In = {statistics}, State = #client_statem_state{ myName = MyName, etsRef = EtsRef}) -> + ets:update_counter(EtsRef, msgCounter, 1), % last param is increment value + ets:update_counter(EtsRef, infoIn, nerl_tools:calculate_size(In)), + sendStatistics(EtsRef), + {next_state, predict, State}; + predict(cast, EventContent, State = #client_statem_state{etsRef = EtsRef}) -> ets:update_counter(EtsRef, msgCounter, 1), ets:update_counter(EtsRef, infoIn, erts_debug:flat_size(EventContent)), diff --git a/src_erl/Communication_Layer/http_Nerlserver/src/MainServer/mainGenserver.erl b/src_erl/Communication_Layer/http_Nerlserver/src/MainServer/mainGenserver.erl index de6798a7..c288ad82 100644 --- a/src_erl/Communication_Layer/http_Nerlserver/src/MainServer/mainGenserver.erl +++ b/src_erl/Communication_Layer/http_Nerlserver/src/MainServer/mainGenserver.erl @@ -163,14 +163,13 @@ handle_cast({clientsIdle}, State = #main_genserver_state{state = idle, myName = {noreply, State#main_genserver_state{clientsWaitingList = ListOfClients,msgCounter = MsgCounter+1}}; %%%get Statistics from all Entities in the network -handle_cast({statistics,Body}, State = #main_genserver_state{myName = MyName, statisticsCounter = StatisticsCounter, nerlnetGraph = NerlnetGraph,statisticsMap = StatisticsMap,msgCounter = MsgCounter}) -> +handle_cast({statistics,Body}, State = #main_genserver_state{myName = MyName, statisticsCounter = StatisticsCounter, nerlnetGraph = NerlnetGraph,statisticsMap = StatisticsMap,msgCounter = MsgCounter , etsRef = EtsRef}) -> if Body == <<"getStatistics">> -> %% initial message from APIServer, get stats from entities NewStatisticsMap = getNewStatisticsMap([digraph:vertex(NerlnetGraph,Vertex) || Vertex <- digraph:vertices(NerlnetGraph)--?LIST_OF_SPECIAL_SERVERS]), [findroutAndsendStatistics(MyName, Name)||{Name,_Counter}<-maps:to_list(StatisticsMap)], NewState = State#main_genserver_state{msgCounter = MsgCounter+1,statisticsMap = NewStatisticsMap, statisticsCounter = length(maps:to_list(StatisticsMap))}; - Body == <<>> -> io:format("in Statistcs, State has: StatsCount=~p, MsgCount=~p~n", [StatisticsCounter, MsgCounter]), NewState = State; true -> @@ -185,10 +184,18 @@ handle_cast({statistics,Body}, State = #main_genserver_state{myName = MyName, st S = mapToString(Statistics,[]) , ?LOG_NOTICE("Sending stats: ~p~n",[S]), {RouterHost,RouterPort} = nerl_tools:getShortPath(MyName,?API_SERVER_ATOM,NerlnetGraph), - nerl_tools:http_request(RouterHost,RouterPort,"statistics", S ++ "|mainServer:" ++integer_to_list(MsgCounter)); + nerl_tools:http_request(RouterHost,RouterPort,"statistics", S ++ "|mainServer:" ++integer_to_list(MsgCounter)), + + case ets:member(EtsRef , nerlMonitor) of + true -> + [{_ , MonitorIP , MonitorPort}] = ets:lookup(EtsRef, nerlMonitor), + nerl_tools:http_request(MonitorIP,list_to_integer(MonitorPort),"stats", S ++ "|mainServer:" ++integer_to_list(MsgCounter)); + _ -> ok + end; %% wait for more stats - true -> pass end + true -> pass + end end, {noreply, NewState}; @@ -228,10 +235,16 @@ handle_cast({sourceAck,Body}, State = #main_genserver_state{sourcesWaitingList = {noreply, State#main_genserver_state{sourcesWaitingList = NewWaitingList,msgCounter = MsgCounter+1}}; -handle_cast({clientAck,Body}, State = #main_genserver_state{clientsWaitingList = WaitingList,msgCounter = MsgCounter}) -> - NewWaitingList = WaitingList--[list_to_atom(binary_to_list(Body))], - % io:format("new Waiting List: ~p ~n",[NewWaitingList]), - if length(NewWaitingList) == 0 -> ack(); +handle_cast({clientAck,Body}, State = #main_genserver_state{clientsWaitingList = WaitingList,msgCounter = MsgCounter , etsRef = EtsRef}) -> + ClientName = list_to_atom(binary_to_list(Body)), + NewWaitingList = WaitingList--[ClientName], + if length(NewWaitingList) == 0 -> + case ets:member(EtsRef , nerlMonitor) of + true -> + gen_server:cast(self(),{statistics , list_to_binary("getStatistics")}); + _ -> ok + end, + ack(); true-> ok end, {noreply, State#main_genserver_state{clientsWaitingList = NewWaitingList, msgCounter = MsgCounter+1}}; diff --git a/src_erl/Communication_Layer/http_Nerlserver/src/utilities_handler.erl b/src_erl/Communication_Layer/http_Nerlserver/src/utilities_handler.erl index b1bc8b67..61d969d3 100644 --- a/src_erl/Communication_Layer/http_Nerlserver/src/utilities_handler.erl +++ b/src_erl/Communication_Layer/http_Nerlserver/src/utilities_handler.erl @@ -5,9 +5,7 @@ -export([init/2]). init(Req0 , [MainServerPid]) -> - io:format("Req is ~p~n" , [Req0]), {_,Body,_} = cowboy_req:read_body(Req0), - io:format("Body: ~p~n" , [Body]) , Formatted = binary_to_list(Body), [UtilityName , IP , Port] = [X || X <- re:split(Formatted , "#" , [{return , list}])], io:format("UtilityName: ~p , IP: ~p , Port: ~p~n" , [UtilityName , IP , Port]), diff --git a/src_erl/NerlMonitor/src/MonitorGUI.py b/src_erl/NerlMonitor/src/MonitorGUI.py index 3cee1ce5..35a65f12 100644 --- a/src_erl/NerlMonitor/src/MonitorGUI.py +++ b/src_erl/NerlMonitor/src/MonitorGUI.py @@ -21,23 +21,37 @@ def draw_gradient(canvas, start_color, end_color): Msg_log = [] +DataColumn = [ + [sg.Frame(title="Event Log:" , + layout=[[sg.Multiline('', size=(140, 60), key='-LOG-', autoscroll=True , font=('SFPro' , 12) , no_scrollbar=True)]], + background_color=('#930707') , font=('SFPro' , 20) , size=(500,325) , title_color='Black' , element_justification='right') + ] , + [sg.Frame(title="Statistics:" , + layout=[[sg.Multiline('', size=(140, 60), key='-STATS-', autoscroll=True , font=('SFPro' , 12) , no_scrollbar=True)]], + background_color=('#930707') , font=('SFPro' , 20) , size=(500,325) , title_color='Black' , element_justification='right') + ] + ] + +GraphColumn = [ + [ sg.Text("Waiting For\n NerlNet Graph..." , key='-PHOLD-', text_color='Black' , font=('SFPro' , 12) , size=(70,5) , background_color='#930707' , justification='center' , pad=(0,0)) , + sg.Image(key='-IMAGE-' , visible=False) + ] + ] + layout = [ [ sg.Text("NerlNet Monitor" , key='-TEXT-' , size=(30,1) ,text_color='Black' , font=('SFPro' , 20) , background_color='#930707' , justification='center' , pad=(0,0)) ] , - [ sg.Frame(title="Event Log:" , - layout=[[sg.Multiline('', size=(140, 60), key='-LOG-', autoscroll=True , font=('SFPro' , 12) , no_scrollbar=True)]], - background_color=('#930707') , font=('SFPro' , 20) , size=(500,650) , title_color='Black' , element_justification='right') , - sg.Text("Waiting For\n NerlNet Graph..." , key='-PHOLD-', text_color='Black' , font=('SFPro' , 12) , size=(70,5) , background_color='#930707' , justification='center' , pad=(0,0)) , - sg.Image(key='-IMAGE-' , visible=False) - + [ sg.Column(DataColumn , background_color='#930707') , + sg.VSeperator() , + sg.Column(GraphColumn , background_color='#930707') ] , [ sg.Button(button_text="Close" , button_color=('#C30404' , '#000000') , font=('SFPro' , 12) , size=(5,2)), sg.Button(button_text="Clear Log" , button_color=('#C30404' , '#000000') , font=('SFPro' , 12) , size=(5,2)) ] - ] + ] MainWindow = sg.Window("NErlNet" , layout , margins=(5,5) , size=(1400,800) , background_color='#930707' , finalize=True , resizable=True , element_justification='c') @@ -86,6 +100,27 @@ def GUI(MainPid): else: updated_text = f'{existing_text}\n{formatted_time()}: Worker {WorkerName} of Client {ClientName} is down.' MainWindow['-LOG-'].update(updated_text) + + elif msg[0] == 'stats': + Data = msg[1] + statDict = {"workers": {}} + for items in str(Data).split('|'): + key, val = items.split(':') + if '=' in val: + for worker in val.split(','): + workerName, time = worker.split('=') + statDict["workers"][workerName] = time + else: + statDict[key] = val + # print(statDict) + for key, val in statDict.items(): + if isinstance(val, dict): + print(key, '--') + for key2, val2 in val.items(): + print("\t", key2, ' : ', val2) + else: + print(key, ' : ', val) + elif values['-LOG-'] != '': existing_text = values['-LOG-'] diff --git a/src_erl/NerlMonitor/src/handlers/nerlMonitor_handler.erl b/src_erl/NerlMonitor/src/handlers/nerlMonitor_handler.erl index f68dfc04..7cdc3302 100644 --- a/src_erl/NerlMonitor/src/handlers/nerlMonitor_handler.erl +++ b/src_erl/NerlMonitor/src/handlers/nerlMonitor_handler.erl @@ -7,10 +7,11 @@ init(Req0, [Msg]) -> {_,Body,_} = cowboy_req:read_body(Req0), Data = binary_to_list(Body), - io:format("-------------------Data ~p--------------~n" , [Data]), case Msg of utilInfo -> io:format("Update graph with: worker_down ~p~n" , [Data]) , ?GUI ! {update ,Data}; + stats -> io:format("Update graph with: stats ~p~n" , [Data]) , + ?GUI ! {stats ,Data}; _ -> ok % got unknown messge, ignore. end, diff --git a/src_erl/NerlMonitor/src/nerlMonitor_app.erl b/src_erl/NerlMonitor/src/nerlMonitor_app.erl index dd8c70e0..72563f35 100644 --- a/src_erl/NerlMonitor/src/nerlMonitor_app.erl +++ b/src_erl/NerlMonitor/src/nerlMonitor_app.erl @@ -24,7 +24,8 @@ start(_StartType, _StartArgs) -> Dispatch = cowboy_router:compile([ {'_', [ - {"/utilInfo",nerlMonitor_handler, [utilInfo]} + {"/utilInfo",nerlMonitor_handler, [utilInfo]}, + {"/stats" , nerlMonitor_handler , [stats]} ]} ]), From 748d94be69834b10de5697c1984ace8bbc10f0c2 Mon Sep 17 00:00:00 2001 From: GuyPerets106 Date: Thu, 10 Aug 2023 15:04:55 +0000 Subject: [PATCH 28/40] [NerlMonitor] Added worker termination from GUI --- .../src/Client/clientStatem.erl | 22 ++++++------ src_erl/NerlMonitor/src/MonitorGUI.py | 35 ++++++++++++++----- 2 files changed, 37 insertions(+), 20 deletions(-) diff --git a/src_erl/Communication_Layer/http_Nerlserver/src/Client/clientStatem.erl b/src_erl/Communication_Layer/http_Nerlserver/src/Client/clientStatem.erl index d470a864..794efb63 100644 --- a/src_erl/Communication_Layer/http_Nerlserver/src/Client/clientStatem.erl +++ b/src_erl/Communication_Layer/http_Nerlserver/src/Client/clientStatem.erl @@ -368,12 +368,12 @@ training(cast, In = {loss,WorkerName,LossFunction,_Time_NIF}, State = #client_st nerl_tools:http_request(RouterHost,RouterPort,"lossFunction", term_to_binary({WorkerName,LossFunction})), {next_state, training, State#client_statem_state{myName = MyName,etsRef = EtsRef}}; -training(cast, In = {statistics}, State = #client_statem_state{ myName = MyName, etsRef = EtsRef}) -> - io:format("client ~p got statistics ~n",[MyName]), - ets:update_counter(EtsRef, msgCounter, 1), % last param is increment value - ets:update_counter(EtsRef, infoIn, nerl_tools:calculate_size(In)), - sendStatistics(EtsRef), - {next_state, training, State}; +% training(cast, In = {statistics}, State = #client_statem_state{ myName = MyName, etsRef = EtsRef}) -> +% io:format("client ~p got statistics ~n",[MyName]), +% ets:update_counter(EtsRef, msgCounter, 1), % last param is increment value +% ets:update_counter(EtsRef, infoIn, nerl_tools:calculate_size(In)), +% sendStatistics(EtsRef), +% {next_state, training, State}; training(cast, EventContent, State = #client_statem_state{etsRef = EtsRef, myName = MyName}) -> ?LOG_WARNING("client ~p training ignored!!!: ~p ~n!!!",[MyName, EventContent]), @@ -450,11 +450,11 @@ predict(cast, In = {idle}, State = #client_statem_state{etsRef = EtsRef}) -> Workers = ets:lookup_element(EtsRef, workersNames, ?ETS_KV_VAL_IDX), {next_state, waitforWorkers, State#client_statem_state{nextState = idle, waitforWorkers = Workers, etsRef = EtsRef}}; -predict(cast, In = {statistics}, State = #client_statem_state{ myName = MyName, etsRef = EtsRef}) -> - ets:update_counter(EtsRef, msgCounter, 1), % last param is increment value - ets:update_counter(EtsRef, infoIn, nerl_tools:calculate_size(In)), - sendStatistics(EtsRef), - {next_state, predict, State}; +% predict(cast, In = {statistics}, State = #client_statem_state{ myName = MyName, etsRef = EtsRef}) -> +% ets:update_counter(EtsRef, msgCounter, 1), % last param is increment value +% ets:update_counter(EtsRef, infoIn, nerl_tools:calculate_size(In)), +% sendStatistics(EtsRef), +% {next_state, predict, State}; predict(cast, EventContent, State = #client_statem_state{etsRef = EtsRef}) -> ets:update_counter(EtsRef, msgCounter, 1), diff --git a/src_erl/NerlMonitor/src/MonitorGUI.py b/src_erl/NerlMonitor/src/MonitorGUI.py index 35a65f12..940e55a8 100644 --- a/src_erl/NerlMonitor/src/MonitorGUI.py +++ b/src_erl/NerlMonitor/src/MonitorGUI.py @@ -35,6 +35,11 @@ def draw_gradient(canvas, start_color, end_color): GraphColumn = [ [ sg.Text("Waiting For\n NerlNet Graph..." , key='-PHOLD-', text_color='Black' , font=('SFPro' , 12) , size=(70,5) , background_color='#930707' , justification='center' , pad=(0,0)) , sg.Image(key='-IMAGE-' , visible=False) + ], + [ + sg.Text("Enter the name of the worker you wish to terminate:" ,key='-INTEXT-', size=(42,1) ,text_color='white' , font=('SFPro' , 12) , background_color='#930707' , justification='left' , pad=(0,0) , visible=False) , + sg.Input('',key='-INPUT-' , visible=False , justification='left' , size=(20,1) , font=('SFPro' , 12) , background_color='white' , text_color='black' , pad=(0,0) , enable_events=True), + sg.Button(button_text="Terminate" , button_color=('#C30404' , '#000000') , font=('SFPro' , 12) , size=(10,1) , pad=(0,0) , visible=False, key='-TERM-', enable_events=True) ] ] @@ -63,13 +68,23 @@ def formatted_time(): def GUI(MainPid): while True: event , values = MainWindow.read(timeout=100) + existing_text = values['-LOG-'] updated_text = '' if event == "Close" or event == sg.WIN_CLOSED: os.kill(MainPid , 9) print("GUI Closed.") break - if event == "Clear Log": - MainWindow['-LOG-'].update('') + elif event == "Clear Log": + MainWindow['-LOG-'].update('') + elif event == "-TERM-": + Workers = [Graph.nodes[node]['label'] for node in Graph.nodes() if Graph.nodes[node]['label'][0] == 'w' or node_colors[node] == 'black'] + if values['-INPUT-'] not in Workers: + updated_text = f'{existing_text}\n{formatted_time()}: Invalid Worker Name {values["-INPUT-"]}.' + MainWindow['-LOG-'].update(updated_text) + else: + updated_text = f'{existing_text}\n{formatted_time()}: Worker {values["-INPUT-"]} terminated.' + MainWindow['-LOG-'].update(updated_text) + MainWindow['-INPUT-'].update('') if not msg_queue.empty(): msg = msg_queue.get_nowait() if msg[0] == 'graph': @@ -77,6 +92,9 @@ def GUI(MainPid): MainWindow['-PHOLD-'].update(visible=False) MainWindow['-IMAGE-'].update(filename='NerlNetGraph.png' , visible=True , size=(800,600)) MainWindow['-LOG-'].update(f'{formatted_time()}: NerlNet Graph Received.') + MainWindow['-INTEXT-'].update(visible=True) + MainWindow['-INPUT-'].update(visible=True) + MainWindow['-TERM-'].update(visible=True) elif msg[0] == 'update': ClientName , WorkerName = msg[1].split('-') @@ -94,7 +112,6 @@ def GUI(MainPid): plt.savefig('NerlNetGraph.png' ,bbox_inches='tight' , dpi=125) plt.close() MainWindow['-IMAGE-'].update(filename='NerlNetGraph.png' , visible=True , size=(800,600)) - existing_text = values['-LOG-'] if existing_text == '': updated_text = f'{formatted_time()}: Worker {WorkerName} of Client {ClientName} is down.' else: @@ -112,14 +129,13 @@ def GUI(MainPid): statDict["workers"][workerName] = time else: statDict[key] = val - # print(statDict) for key, val in statDict.items(): if isinstance(val, dict): - print(key, '--') + MainWindow['-STATS-'].update(f'{formatted_time()}: {key}') for key2, val2 in val.items(): - print("\t", key2, ' : ', val2) + MainWindow['-STATS-'].update(f'{formatted_time()}: {key2} : {val2}') else: - print(key, ' : ', val) + MainWindow['-STATS-'].update(f'{formatted_time()}: {key} : {val}') elif values['-LOG-'] != '': @@ -149,12 +165,13 @@ def Show_Nerlnet_Graph(NerlGraph): NodesNames = [NodeTriplet.split(',')[0] for NodeTriplet in Nodes] NodesNames += WorkersNames - + NodeWithLabels = [(NodeName , {'label' : NodeName}) for NodeName in NodesNames] graph = nx.Graph() - graph.add_nodes_from(NodesNames) + graph.add_nodes_from(NodeWithLabels) graph.add_edges_from(EdgesSeperated) my_labels = {'mainServer': 'mS' , 'apiServer': 'aS'} + nx.relabel_nodes(graph, my_labels , copy=False) default_color = 'darkred' node_colors = {node:default_color for node in graph.nodes()} From 298dea912a78f74ee6e774bad8b7320402a3d021 Mon Sep 17 00:00:00 2001 From: GuyPerets106 Date: Sun, 13 Aug 2023 07:53:18 +0000 Subject: [PATCH 29/40] [NerlMonitor] Updated Worker Termination NerlGraph --- src_cpp/opennn | 2 +- src_erl/NerlMonitor/src/MonitorGUI.py | 76 ++++++++++++++++----------- 2 files changed, 46 insertions(+), 32 deletions(-) diff --git a/src_cpp/opennn b/src_cpp/opennn index 4d6b9484..1de26c61 160000 --- a/src_cpp/opennn +++ b/src_cpp/opennn @@ -1 +1 @@ -Subproject commit 4d6b94848d5dd5356084a54f1b1be58b7cdc1a21 +Subproject commit 1de26c611fda2fe6cf026a0e4b329c7509a68042 diff --git a/src_erl/NerlMonitor/src/MonitorGUI.py b/src_erl/NerlMonitor/src/MonitorGUI.py index 940e55a8..3dace634 100644 --- a/src_erl/NerlMonitor/src/MonitorGUI.py +++ b/src_erl/NerlMonitor/src/MonitorGUI.py @@ -10,6 +10,20 @@ import os import math +class MyProcess(Process): + def __init__(self , msg_queue) -> None: + Process.__init__(self) + self.get_node().register_name(self, Atom('PyrlangProcess')) + self.msg_queue = msg_queue + + + def handle_one_inbox_message(self, msg): + print(f'From Pyrlang: {msg}') + self.msg_queue.put(msg) + if not self.msg_queue.empty(): + print(f'Queue is not Empty: {msg} added.') + + def draw_gradient(canvas, start_color, end_color): for y in range(0, 200): # Adjust the range to your desired height r = start_color[0] + (end_color[0] - start_color[0]) * y / 200 @@ -60,12 +74,13 @@ def draw_gradient(canvas, start_color, end_color): MainWindow = sg.Window("NErlNet" , layout , margins=(5,5) , size=(1400,800) , background_color='#930707' , finalize=True , resizable=True , element_justification='c') - +def RemoteRecv(): + return Atom('erl@127.0.0.1') , Atom("recvPyrlang") def formatted_time(): return f'[{datetime.now().day}/{datetime.now().month}/{datetime.now().year}|{datetime.now().hour}:{datetime.now().minute}:{datetime.now().second}]' -def GUI(MainPid): +def GUI(MainPid , CommProc : MyProcess , PyNode : Node): while True: event , values = MainWindow.read(timeout=100) existing_text = values['-LOG-'] @@ -77,12 +92,24 @@ def GUI(MainPid): elif event == "Clear Log": MainWindow['-LOG-'].update('') elif event == "-TERM-": - Workers = [Graph.nodes[node]['label'] for node in Graph.nodes() if Graph.nodes[node]['label'][0] == 'w' or node_colors[node] == 'black'] + Workers = [Graph.nodes[node]['label'] for node in Graph.nodes() if Graph.nodes[node]['label'][0] == 'w' and node_colors[node] != 'gray'] if values['-INPUT-'] not in Workers: - updated_text = f'{existing_text}\n{formatted_time()}: Invalid Worker Name {values["-INPUT-"]}.' + updated_text = f'{existing_text}\n{formatted_time()}: Invalid Worker Name {values["-INPUT-"]} , Available Workers: {Workers}.' MainWindow['-LOG-'].update(updated_text) else: - updated_text = f'{existing_text}\n{formatted_time()}: Worker {values["-INPUT-"]} terminated.' + Workers.remove(values['-INPUT-']) + node_colors[values['-INPUT-']] = 'gray' + nx.set_node_attributes(Graph, node_colors, 'color') + colors = nx.get_node_attributes(Graph, 'color').values() + pos = nx.nx_agraph.graphviz_layout(Graph, prog='dot') + plt.figure(figsize=(8,6)) + nx.draw_networkx(Graph, pos, with_labels=True, node_color=colors , node_size=200, font_size=8, font_color='white' , edge_color='black' , width=1.5) + plt.savefig('NerlNetGraph.png' ,bbox_inches='tight' , dpi=125) + plt.close() + MainWindow['-IMAGE-'].update(filename='NerlNetGraph.png' , visible=True , size=(800,600)) + + updated_text = f'{existing_text}\n{formatted_time()}: Worker {values["-INPUT-"]} terminated , Available Workers: {Workers}.' + PyNode.send_nowait(sender = CommProc.pid_ , receiver = ('erl@127.0.0.1' , 'recvPyrlang') , message = (Atom('terminate'),Atom(f'{values["-INPUT-"]}'))) MainWindow['-LOG-'].update(updated_text) MainWindow['-INPUT-'].update('') if not msg_queue.empty(): @@ -98,17 +125,17 @@ def GUI(MainPid): elif msg[0] == 'update': ClientName , WorkerName = msg[1].split('-') - node_colors[WorkerName] = 'black' - node_colors[ClientName] = 'gray' + node_colors[WorkerName] = 'gray' + #node_colors[ClientName] = 'gray' nx.set_node_attributes(Graph, node_colors, 'color') colors = nx.get_node_attributes(Graph, 'color').values() pos = nx.nx_agraph.graphviz_layout(Graph, prog='dot') angle = 100 - rotated_pos = {node: (x*math.cos(angle) -y*math.sin(angle), x*math.sin(angle) + y*math.cos(angle)) for node, (x, y) in pos.items()} + #rotated_pos = {node: (x*math.cos(angle) -y*math.sin(angle), x*math.sin(angle) + y*math.cos(angle)) for node, (x, y) in pos.items()} plt.figure(figsize=(8,6)) - nx.draw_networkx(Graph, rotated_pos, with_labels=True, node_color=colors , node_size=200, font_size=8, font_color='white' , edge_color='black' , width=1.5) + nx.draw_networkx(Graph, pos, with_labels=True, node_color=colors , node_size=200, font_size=8, font_color='white' , edge_color='black' , width=1.5) plt.savefig('NerlNetGraph.png' ,bbox_inches='tight' , dpi=125) plt.close() MainWindow['-IMAGE-'].update(filename='NerlNetGraph.png' , visible=True , size=(800,600)) @@ -173,45 +200,32 @@ def Show_Nerlnet_Graph(NerlGraph): my_labels = {'mainServer': 'mS' , 'apiServer': 'aS'} nx.relabel_nodes(graph, my_labels , copy=False) - default_color = 'darkred' - node_colors = {node:default_color for node in graph.nodes()} + default_colors = {node:'darkred' for node in graph.nodes()} + node_colors = {node:default_colors[node] for node in graph.nodes()} nx.set_node_attributes(graph, node_colors, 'color') colors = nx.get_node_attributes(graph, 'color').values() pos = nx.nx_agraph.graphviz_layout(graph, prog='dot') angle = 100 - rotated_pos = {node: (x*math.cos(angle) -y*math.sin(angle), x*math.sin(angle) + y*math.cos(angle)) for node, (x, y) in pos.items()} + #rotated_pos = {node: (x*math.cos(angle) -y*math.sin(angle), x*math.sin(angle) + y*math.cos(angle)) for node, (x, y) in pos.items()} plt.figure(figsize=(8,6)) - nx.draw_networkx(graph, rotated_pos, with_labels=True, node_color=colors , node_size=200, font_size=8, font_color='white' , edge_color='black' , width=1.5) + nx.draw_networkx(graph, pos, with_labels=True, node_color=colors , node_size=200, font_size=8, font_color='white' , edge_color='black' , width=1.5) plt.savefig('NerlNetGraph.png' ,bbox_inches='tight' , dpi=125) plt.close() return graph , node_colors - -class MyProcess(Process): - def __init__(self , msg_queue) -> None: - Process.__init__(self) - self.get_node().register_name(self, Atom('PyrlangProcess')) - self.msg_queue = msg_queue - - - def handle_one_inbox_message(self, msg): - print(f'From Pyrlang: {msg}') - self.msg_queue.put(msg) - if not self.msg_queue.empty(): - print(f'Queue is not Empty: {msg} added.') - + if __name__ == "__main__": msg_queue = multiprocessing.Queue() - + PyNode = Node(node_name="py@127.0.0.1" , cookie="COOKIE") + #event_loop = PyNode.get_loop() PyrlangPid = os.getpid() - GUI_Process = multiprocessing.Process(target=GUI , args=(PyrlangPid,)) + CommProc = MyProcess(msg_queue) + GUI_Process = multiprocessing.Process(target=GUI , args=(PyrlangPid,CommProc,PyNode)) GUI_Process.start() print("Starting a Pyrlang node...") - PyNode = Node(node_name="py@127.0.0.1" , cookie="COOKIE") - MyProcess(msg_queue) PyNode.run() From df3c0171c7cae56f99b890b64f6126e1a6eb2094 Mon Sep 17 00:00:00 2001 From: galhilu Date: Sun, 13 Aug 2023 10:59:28 +0300 Subject: [PATCH 30/40] [NerlMonitor] debug of worker kill --- .../http_Nerlserver/src/Client/clientStatem.erl | 12 ++++-------- .../http_Nerlserver/src/MainServer/mainGenserver.erl | 6 ++++-- .../http_Nerlserver/src/Router/routerGenserver.erl | 4 ++-- .../http_Nerlserver/src/utilities_handler.erl | 9 +++------ 4 files changed, 13 insertions(+), 18 deletions(-) diff --git a/src_erl/Communication_Layer/http_Nerlserver/src/Client/clientStatem.erl b/src_erl/Communication_Layer/http_Nerlserver/src/Client/clientStatem.erl index f1c43c9b..34bfed6f 100644 --- a/src_erl/Communication_Layer/http_Nerlserver/src/Client/clientStatem.erl +++ b/src_erl/Communication_Layer/http_Nerlserver/src/Client/clientStatem.erl @@ -155,8 +155,7 @@ waitforWorkers(cast, In = {NewState}, State = #client_statem_state{myName = MyNa waitforWorkers(cast, {worker_kill , Body}, State = #client_statem_state{etsRef = EtsRef}) -> ets:update_counter(EtsRef, msgCounter, 1), - [_ , Target] = string:split(binary_to_list(Body), "-"), - WorkerName = list_to_atom(Target), + {_, WorkerName} = binary_to_term(Body), Pid = ets:lookup_element(EtsRef, WorkerName, ?WORKER_PID_IDX), gen_statem:stop(Pid, shutdown , infinity), {next_state, waitforWorkers, State#client_statem_state{etsRef = EtsRef}}; @@ -223,8 +222,7 @@ idle(cast, In = {training}, State = #client_statem_state{etsRef = EtsRef}) -> idle(cast, {worker_kill , Body}, State = #client_statem_state{etsRef = EtsRef}) -> ets:update_counter(EtsRef, msgCounter, 1), - [_ , Target] = string:split(binary_to_list(Body), "-"), - WorkerName = list_to_atom(Target), + {_, WorkerName} = binary_to_term(Body), Pid = ets:lookup_element(EtsRef, WorkerName, ?WORKER_PID_IDX), gen_statem:stop(Pid, shutdown , infinity), {next_state, idle, State#client_statem_state{etsRef = EtsRef}}; @@ -296,8 +294,7 @@ training(cast, In = {custom_worker_message, WorkersList, WeightsTensor}, State = training(cast, {worker_kill , Body}, State = #client_statem_state{etsRef = EtsRef}) -> ets:update_counter(EtsRef, msgCounter, 1), - [_ , Target] = string:split(binary_to_list(Body), "-"), - WorkerName = list_to_atom(Target), + {_, WorkerName} = binary_to_term(Body), Pid = ets:lookup_element(EtsRef, WorkerName, ?WORKER_PID_IDX), gen_statem:stop(Pid, shutdown , infinity), {next_state, training, State#client_statem_state{etsRef = EtsRef}}; @@ -418,8 +415,7 @@ predict(cast, In = {predictRes,WorkerName,InputName,ResultID,PredictNerlTensor, predict(cast, {worker_kill , Body}, State = #client_statem_state{etsRef = EtsRef}) -> ets:update_counter(EtsRef, msgCounter, 1), - [_ , Target] = string:split(binary_to_list(Body), "-"), - WorkerName = list_to_atom(Target), + {_, WorkerName} = binary_to_term(Body), Pid = ets:lookup_element(EtsRef, WorkerName, ?WORKER_PID_IDX), gen_statem:stop(Pid, shutdown , infinity), {next_state, predict, State#client_statem_state{etsRef = EtsRef}}; diff --git a/src_erl/Communication_Layer/http_Nerlserver/src/MainServer/mainGenserver.erl b/src_erl/Communication_Layer/http_Nerlserver/src/MainServer/mainGenserver.erl index de6798a7..8532c40c 100644 --- a/src_erl/Communication_Layer/http_Nerlserver/src/MainServer/mainGenserver.erl +++ b/src_erl/Communication_Layer/http_Nerlserver/src/MainServer/mainGenserver.erl @@ -328,8 +328,10 @@ handle_cast({worker_down,Body}, State = #main_genserver_state{msgCounter = MsgCo handle_cast({worker_kill , WorkerName} , State = #main_genserver_state{workersMap = WorkersMap , msgCounter = MsgCounter}) -> ?LOG_WARNING(?LOG_HEADER++"Worker ~p killed~n",[WorkerName]), - ClientName = maps:get(WorkerName,WorkersMap), % ! Worker Name is binary? - Body = list_to_binary(atom_to_list(ClientName)) ++ "-" ++ WorkerName, + WorkerNameAtom=binary_to_term(WorkerName), + ClientName = maps:get(WorkerNameAtom,WorkersMap), % ! Worker Name is binary? + Body = term_to_binary({ClientName, WorkerNameAtom}), + io:format("-------------worker kill got is: ~p sending:~p~n",[binary_to_term(WorkerName),binary_to_term(Body)]), nerl_tools:sendHTTP(?MAIN_SERVER_ATOM , ClientName , atom_to_list(worker_kill) , Body), {noreply, State#main_genserver_state{workersMap = WorkersMap , msgCounter = MsgCounter+1}}; diff --git a/src_erl/Communication_Layer/http_Nerlserver/src/Router/routerGenserver.erl b/src_erl/Communication_Layer/http_Nerlserver/src/Router/routerGenserver.erl index 9f4f0ac3..28333980 100644 --- a/src_erl/Communication_Layer/http_Nerlserver/src/Router/routerGenserver.erl +++ b/src_erl/Communication_Layer/http_Nerlserver/src/Router/routerGenserver.erl @@ -177,8 +177,8 @@ handle_cast({getStats,_Body}, State = #router_genserver_state{myName = MyName, {noreply, State#router_genserver_state{msgCounter = MsgCounter+1}}; handle_cast({worker_kill , Body} , State = #router_genserver_state{msgCounter = MsgCounter, myName = MyName}) -> - [Target , _] = string:split(binary_to_list(Body) , "-"), - nerl_tools:sendHTTP(MyName, list_to_atom(Target), "worker_kill", Body), + {ClientName , _} = binary_to_term(Body), + nerl_tools:sendHTTP(MyName, ClientName, "worker_kill", Body), {noreply, State#router_genserver_state{msgCounter = MsgCounter+1}}; diff --git a/src_erl/Communication_Layer/http_Nerlserver/src/utilities_handler.erl b/src_erl/Communication_Layer/http_Nerlserver/src/utilities_handler.erl index b1bc8b67..a0540460 100644 --- a/src_erl/Communication_Layer/http_Nerlserver/src/utilities_handler.erl +++ b/src_erl/Communication_Layer/http_Nerlserver/src/utilities_handler.erl @@ -5,17 +5,13 @@ -export([init/2]). init(Req0 , [MainServerPid]) -> - io:format("Req is ~p~n" , [Req0]), {_,Body,_} = cowboy_req:read_body(Req0), - io:format("Body: ~p~n" , [Body]) , + io:format("got kill-------------------messege: ~p~n",[Body]), Formatted = binary_to_list(Body), [UtilityName , IP , Port] = [X || X <- re:split(Formatted , "#" , [{return , list}])], - io:format("UtilityName: ~p , IP: ~p , Port: ~p~n" , [UtilityName , IP , Port]), case list_to_atom(UtilityName) of - nerlMonitor -> - io:format("I'm HERE , args are: ~p , ~p , ~p~n" , [UtilityName , IP , Port]), + nerlMonitor -> Graph = gen_server:call(MainServerPid , getGraph), - io:format("Graph: ~p~n" , [Graph]), WorkersMap = ets:lookup_element(nerlnet_data , workers , ?DATA_IDX), WorkersClients = maps:to_list(WorkersMap), Workers = lists:flatten([atom_to_list(X)++"-"++atom_to_list(Y)++"!" || {X , Y} <- WorkersClients]), @@ -30,6 +26,7 @@ init(Req0 , [MainServerPid]) -> init(Req0 , [Action , MainServerPid]) -> {_,Body,_} = cowboy_req:read_body(Req0), + io:format("got kill-------------------messege: ~p~n",[Body]), case Action of worker_kill -> gen_statem:cast(MainServerPid , {worker_kill , Body}); From 874136eaeca65266c7536b6b5a127342b0a4526c Mon Sep 17 00:00:00 2001 From: GuyPerets106 Date: Sun, 13 Aug 2023 13:06:05 +0000 Subject: [PATCH 31/40] [NerlMonitor] Tests --- src_erl/NerlMonitor/src/MonitorGUI.py | 59 ++++++++++++++++++--- src_erl/NerlMonitor/src/nerlMonitor_app.erl | 11 +++- 2 files changed, 60 insertions(+), 10 deletions(-) diff --git a/src_erl/NerlMonitor/src/MonitorGUI.py b/src_erl/NerlMonitor/src/MonitorGUI.py index 3dace634..ad058804 100644 --- a/src_erl/NerlMonitor/src/MonitorGUI.py +++ b/src_erl/NerlMonitor/src/MonitorGUI.py @@ -18,12 +18,19 @@ def __init__(self , msg_queue) -> None: def handle_one_inbox_message(self, msg): - print(f'From Pyrlang: {msg}') - self.msg_queue.put(msg) + print(f'From ErlProcess: {msg}') + if msg[0] == Atom('send'): + print(f'From ErlProcess: {msg[1]}') + else: + self.msg_queue.put(msg) if not self.msg_queue.empty(): print(f'Queue is not Empty: {msg} added.') +msg_queue = multiprocessing.Queue() +PyNode = Node(node_name="py@127.0.0.1" , cookie="COOKIE") +CommProc = MyProcess(msg_queue=msg_queue) + def draw_gradient(canvas, start_color, end_color): for y in range(0, 200): # Adjust the range to your desired height r = start_color[0] + (end_color[0] - start_color[0]) * y / 200 @@ -80,7 +87,26 @@ def RemoteRecv(): def formatted_time(): return f'[{datetime.now().day}/{datetime.now().month}/{datetime.now().year}|{datetime.now().hour}:{datetime.now().minute}:{datetime.now().second}]' -def GUI(MainPid , CommProc : MyProcess , PyNode : Node): +# def SendMsg(Msg): +# SendNode = Node(node_name='pysend@127.0.0.1' , cookie='COOKIE') +# SendProc = MyProcess() +# event_loop = SendNode.get_loop() + +# print(SendNode.where_is(Atom('recvPyrlang'))) + +# def task(): +# SendNode.send_nowait(sender = SendProc.pid_ , receiver = RemoteRecv() , message = (Atom('send'),Atom(Msg))) +# SendNode.destroy() + +# event_loop.call_soon(task) + +# SendNode.run() + +def GUI(MainPid , CommProc : MyProcess , CommProcInbox): + async def task2(): + print("Sending Hello World!") + PyNode.send_nowait(sender = CommProc.pid_ , receiver = RemoteRecv() , message = (Atom('send'),Atom('Hello World!'))) + print("Sent Hello World!") while True: event , values = MainWindow.read(timeout=100) existing_text = values['-LOG-'] @@ -90,6 +116,9 @@ def GUI(MainPid , CommProc : MyProcess , PyNode : Node): print("GUI Closed.") break elif event == "Clear Log": + #CommProcInbox.put_nowait((Atom('send'),Atom('Hello World!'))) + PyNode.send_nowait(sender = CommProc.pid_ , receiver = RemoteRecv() , message = (Atom('send'),Atom('Hello World!'))) + print("Trying to send hello world") MainWindow['-LOG-'].update('') elif event == "-TERM-": Workers = [Graph.nodes[node]['label'] for node in Graph.nodes() if Graph.nodes[node]['label'][0] == 'w' and node_colors[node] != 'gray'] @@ -109,7 +138,11 @@ def GUI(MainPid , CommProc : MyProcess , PyNode : Node): MainWindow['-IMAGE-'].update(filename='NerlNetGraph.png' , visible=True , size=(800,600)) updated_text = f'{existing_text}\n{formatted_time()}: Worker {values["-INPUT-"]} terminated , Available Workers: {Workers}.' - PyNode.send_nowait(sender = CommProc.pid_ , receiver = ('erl@127.0.0.1' , 'recvPyrlang') , message = (Atom('terminate'),Atom(f'{values["-INPUT-"]}'))) + #PyNode.send_nowait(sender = CommProc.pid_ , receiver = RemoteRecv() , message = (Atom('terminate'),Atom(f'{values["-INPUT-"]}'))) + + event_loop = PyNode.get_loop() + event_loop.call_soon(task2) + MainWindow['-LOG-'].update(updated_text) MainWindow['-INPUT-'].update('') if not msg_queue.empty(): @@ -217,12 +250,22 @@ def Show_Nerlnet_Graph(NerlGraph): if __name__ == "__main__": - msg_queue = multiprocessing.Queue() - PyNode = Node(node_name="py@127.0.0.1" , cookie="COOKIE") + #msg_queue = multiprocessing.Queue() + #PyNode = Node(node_name="py@127.0.0.1" , cookie="COOKIE") + #event_loop = PyNode.get_loop() + PyrlangPid = os.getpid() - CommProc = MyProcess(msg_queue) - GUI_Process = multiprocessing.Process(target=GUI , args=(PyrlangPid,CommProc,PyNode)) + #CommProc = MyProcess(msg_queue=msg_queue) + + # def task(): + # Res = PyNode.send_nowait(sender = CommProc.pid_ , receiver = RemoteRecv() , message = (Atom('send'),Atom('Hello World!'))) + # print("Sent Hello World!") + # print(Res) + + # event_loop.call_soon(task) + + GUI_Process = multiprocessing.Process(target=GUI , args=(PyrlangPid,CommProc,CommProc.inbox_)) GUI_Process.start() print("Starting a Pyrlang node...") diff --git a/src_erl/NerlMonitor/src/nerlMonitor_app.erl b/src_erl/NerlMonitor/src/nerlMonitor_app.erl index 72563f35..dddfc7e4 100644 --- a/src_erl/NerlMonitor/src/nerlMonitor_app.erl +++ b/src_erl/NerlMonitor/src/nerlMonitor_app.erl @@ -31,7 +31,8 @@ start(_StartType, _StartArgs) -> ]), {ok, _} = cowboy:start_clear(?UTILNAME,[{port, ?PORT}],#{env => #{dispatch => Dispatch}}), io:format("nerlMonitor started , opening GUI...~n"), - GUI_PID = spawn_link(?MODULE , link_GUI , [self()]) , %% PyrlangNode: ('PyralngProcess' , 'py@127.0.0.1' , 'COOKIE') , sending message by: 'GUI ! HELLO.' + erlang:register(recvPyrlang , self()), + _GUI_PID = spawn_link(?MODULE , link_GUI , [self()]) , %% PyrlangNode: ('PyralngProcess' , 'py@127.0.0.1' , 'COOKIE') , sending message by: 'GUI ! HELLO.' URL = "http://" ++ ?MSADDRES ++ "/toolConnectionReq", mainServerPing(URL,[list_to_binary(atom_to_list(?UTILNAME) ++ "#") , list_to_binary(?IP ++ "#") , list_to_binary(integer_to_list(?PORT))]), %% TODO How to "import" nerl_tools nerlMonitor_sup:start_link(). @@ -57,7 +58,13 @@ mainServerPing(URL,Body)-> initInfoProc(Body)-> %% MainServer replies with Nerlnet-Graph when nerlMonitor tool is used io:format("Sending graph to GUI , graph is ~p~n" , [Body]), - ?GUI ! {graph , Body}. + ?GUI ! {graph , Body}, + receive + {send , Msg} -> + io:format("got message from GUI: ~p~n" , [Msg]); + _ -> io:format("got unknown message from GUI~n") + end, + io:format("initInfoProc finished~n"). %functionalty of the tool, will create a list of tuples when each tuple is {entity that will run the func (atom),function itself {func)} From 431ff909e1b7e5ef154edaf238dd6b05b8bec1d0 Mon Sep 17 00:00:00 2001 From: GuyPerets106 Date: Sun, 13 Aug 2023 14:28:37 +0000 Subject: [PATCH 32/40] [NerlMonitor] Added worker termination --- src_erl/NerlMonitor/src/MonitorGUI.py | 64 +++++++++------------ src_erl/NerlMonitor/src/nerlMonitor_app.erl | 22 ++++--- 2 files changed, 38 insertions(+), 48 deletions(-) diff --git a/src_erl/NerlMonitor/src/MonitorGUI.py b/src_erl/NerlMonitor/src/MonitorGUI.py index ad058804..e801474c 100644 --- a/src_erl/NerlMonitor/src/MonitorGUI.py +++ b/src_erl/NerlMonitor/src/MonitorGUI.py @@ -9,6 +9,10 @@ from datetime import datetime import os import math +import asyncio +import nest_asyncio +nest_asyncio.apply() + class MyProcess(Process): def __init__(self , msg_queue) -> None: @@ -22,14 +26,12 @@ def handle_one_inbox_message(self, msg): if msg[0] == Atom('send'): print(f'From ErlProcess: {msg[1]}') else: - self.msg_queue.put(msg) + self.msg_queue.put_nowait(msg) if not self.msg_queue.empty(): print(f'Queue is not Empty: {msg} added.') -msg_queue = multiprocessing.Queue() -PyNode = Node(node_name="py@127.0.0.1" , cookie="COOKIE") -CommProc = MyProcess(msg_queue=msg_queue) + def draw_gradient(canvas, start_color, end_color): for y in range(0, 200): # Adjust the range to your desired height @@ -102,23 +104,22 @@ def formatted_time(): # SendNode.run() -def GUI(MainPid , CommProc : MyProcess , CommProcInbox): - async def task2(): - print("Sending Hello World!") - PyNode.send_nowait(sender = CommProc.pid_ , receiver = RemoteRecv() , message = (Atom('send'),Atom('Hello World!'))) - print("Sent Hello World!") +async def GUI(msg_queue): + print("GUI task started...") + PyNode , CommProc = await msg_queue.get() + print("Got Message from queue") + print(msg_queue.empty()) + print(f"Got PyNode and CommProc from Queue.") while True: + await asyncio.sleep(.1) event , values = MainWindow.read(timeout=100) existing_text = values['-LOG-'] updated_text = '' if event == "Close" or event == sg.WIN_CLOSED: - os.kill(MainPid , 9) + os.kill(os.getpid() , 9) print("GUI Closed.") break elif event == "Clear Log": - #CommProcInbox.put_nowait((Atom('send'),Atom('Hello World!'))) - PyNode.send_nowait(sender = CommProc.pid_ , receiver = RemoteRecv() , message = (Atom('send'),Atom('Hello World!'))) - print("Trying to send hello world") MainWindow['-LOG-'].update('') elif event == "-TERM-": Workers = [Graph.nodes[node]['label'] for node in Graph.nodes() if Graph.nodes[node]['label'][0] == 'w' and node_colors[node] != 'gray'] @@ -138,10 +139,8 @@ async def task2(): MainWindow['-IMAGE-'].update(filename='NerlNetGraph.png' , visible=True , size=(800,600)) updated_text = f'{existing_text}\n{formatted_time()}: Worker {values["-INPUT-"]} terminated , Available Workers: {Workers}.' - #PyNode.send_nowait(sender = CommProc.pid_ , receiver = RemoteRecv() , message = (Atom('terminate'),Atom(f'{values["-INPUT-"]}'))) + PyNode.send_nowait(sender = CommProc.pid_ , receiver = RemoteRecv() , message = (Atom('terminate'),Atom(f'{values["-INPUT-"]}'))) - event_loop = PyNode.get_loop() - event_loop.call_soon(task2) MainWindow['-LOG-'].update(updated_text) MainWindow['-INPUT-'].update('') @@ -247,29 +246,22 @@ def Show_Nerlnet_Graph(NerlGraph): plt.close() return graph , node_colors - - -if __name__ == "__main__": - #msg_queue = multiprocessing.Queue() - #PyNode = Node(node_name="py@127.0.0.1" , cookie="COOKIE") - - #event_loop = PyNode.get_loop() - - PyrlangPid = os.getpid() - #CommProc = MyProcess(msg_queue=msg_queue) - - # def task(): - # Res = PyNode.send_nowait(sender = CommProc.pid_ , receiver = RemoteRecv() , message = (Atom('send'),Atom('Hello World!'))) - # print("Sent Hello World!") - # print(Res) - - # event_loop.call_soon(task) - GUI_Process = multiprocessing.Process(target=GUI , args=(PyrlangPid,CommProc,CommProc.inbox_)) - GUI_Process.start() - print("Starting a Pyrlang node...") +async def Pyrlang(msg_queue): + print("Pyrlang task started...") + PyNode = Node(node_name="py@127.0.0.1" , cookie="COOKIE") + CommProc = MyProcess(msg_queue=msg_queue) + msg_queue.put_nowait((PyNode , CommProc)) + print("Pyrlang task finished.") PyNode.run() + +async def main_func(): + msg_queue = asyncio.Queue() + await asyncio.gather(GUI(msg_queue) , Pyrlang(msg_queue)) + +if __name__ == "__main__": + asyncio.run(main_func()) diff --git a/src_erl/NerlMonitor/src/nerlMonitor_app.erl b/src_erl/NerlMonitor/src/nerlMonitor_app.erl index dddfc7e4..5b1a6b00 100644 --- a/src_erl/NerlMonitor/src/nerlMonitor_app.erl +++ b/src_erl/NerlMonitor/src/nerlMonitor_app.erl @@ -49,26 +49,24 @@ mainServerPing(URL,Body)-> timer:sleep(1000), mainServerPing(URL,Body); {ok,{_ResCode, _Headers, Data}}-> - io:format("got1 response from main server~n"), - initInfoProc(Data); + io:format("Sending graph to GUI , graph is ~p~n" , [Data]), + ?GUI ! {graph , Data} , + recvLoop(); {ok , _} -> io:format("got2 response from main server~n") end. -initInfoProc(Body)-> %% MainServer replies with Nerlnet-Graph when nerlMonitor tool is used - io:format("Sending graph to GUI , graph is ~p~n" , [Body]), - ?GUI ! {graph , Body}, +recvLoop()-> %% MainServer replies with Nerlnet-Graph when nerlMonitor tool is used receive - {send , Msg} -> - io:format("got message from GUI: ~p~n" , [Msg]); + {terminate , WorkerName} -> + io:format("got termination for Worker ~p from GUI~n" , [WorkerName]), + URL = "http://" ++ ?MSADDRES ++ "/worker_kill", + Body = term_to_binary(WorkerName), + httpc:request(post,{URL, [],"application/x-www-form-urlencoded",Body}, [], []); _ -> io:format("got unknown message from GUI~n") end, - io:format("initInfoProc finished~n"). - - -%functionalty of the tool, will create a list of tuples when each tuple is {entity that will run the func (atom),function itself {func)} -functions()-> ok. + recvLoop(). stop(_State) -> From aa48eae160e6115fca1381bfd0030b910ab388d4 Mon Sep 17 00:00:00 2001 From: GuyPerets106 Date: Sun, 13 Aug 2023 16:11:57 +0000 Subject: [PATCH 33/40] [NerlMonitor] Statistics receive --- .../src/Client/clientStatem.erl | 14 ++++++---- .../http_Nerlserver/src/utilities_handler.erl | 7 +++-- src_erl/NerlMonitor/src/MonitorGUI.py | 26 +++++++++++-------- src_erl/NerlMonitor/src/nerlMonitor_app.erl | 2 +- 4 files changed, 28 insertions(+), 21 deletions(-) diff --git a/src_erl/Communication_Layer/http_Nerlserver/src/Client/clientStatem.erl b/src_erl/Communication_Layer/http_Nerlserver/src/Client/clientStatem.erl index 960d983b..f5dd2b2e 100644 --- a/src_erl/Communication_Layer/http_Nerlserver/src/Client/clientStatem.erl +++ b/src_erl/Communication_Layer/http_Nerlserver/src/Client/clientStatem.erl @@ -78,7 +78,7 @@ init({MyName,NerlnetGraph, WorkersToClientsMap}) -> ets:insert(EtsRef, {msgCounter, 1}), ets:insert(EtsRef, {infoIn, 0}), ets:insert(EtsRef, {myName, MyName}), - + ets:insert(EtsRef , {deadWorkers , []}), createWorkers(MyName,EtsRef), %% send pre_idle signal to workers @@ -573,7 +573,8 @@ updateTimingMap(EtsRef, WorkerName) when is_atom(WorkerName) -> %% adding client c1=MsgNum,w1=... sendStatistics(EtsRef)-> NerlnetGraph = ets:lookup_element(EtsRef, nerlnetGraph, ?ETS_KV_VAL_IDX), - Workers = ets:lookup_element(EtsRef, workersNames, ?ETS_KV_VAL_IDX), + [{deadWorkers , DeadWorkers}] = ets:lookup(EtsRef, deadWorkers), + Workers = ets:lookup_element(EtsRef, workersNames, ?ETS_KV_VAL_IDX) ++ DeadWorkers, TimingMap = [{WorkerKey,ets:lookup_element(EtsRef, WorkerKey, ?WORKER_TIMING_IDX)} || WorkerKey <- Workers], MissedCounts = [{WorkerKey,ets:lookup_element(EtsRef, WorkerKey, ?WORKER_TRAIN_MISSED_IDX)} || WorkerKey <- Workers], Counter = ets:lookup_element(EtsRef, msgCounter, ?ETS_KV_VAL_IDX), @@ -602,10 +603,13 @@ cast_message_to_workers(EtsRef, Msg) -> %activated if client detected that a worker stopped, will delete it from ets so as not to wait for responsed from it in the future. delete_worker(EtsRef,MyName,WorkerName)-> - ets:delete(EtsRef , WorkerName), - NameList = ets:lookup_element(EtsRef, workersNames, ?ETS_KV_VAL_IDX), + %ets:delete(EtsRef , WorkerName), + AliveList = ets:lookup_element(EtsRef, workersNames, ?ETS_KV_VAL_IDX), + DeadList = ets:lookup_element(EtsRef, deadWorkers, ?ETS_KV_VAL_IDX), ets:delete(EtsRef,workersNames), - ets:insert(EtsRef, {workersNames, NameList -- [WorkerName]}), + ets:delete(EtsRef,deadWorkers), + ets:insert(EtsRef, {workersNames, AliveList -- [WorkerName]}), + ets:insert(EtsRef, {deadWorkers, DeadList ++ [WorkerName]}), NerlGraph = ets:lookup_element(EtsRef, nerlnetGraph, ?ETS_KV_VAL_IDX), {Host , Port} = nerl_tools:getShortPath(MyName, ?MAIN_SERVER_ATOM, NerlGraph), nerl_tools:http_request(Host,Port,"worker_down",list_to_binary(atom_to_list(MyName) ++ "-" ++ atom_to_list(WorkerName))), diff --git a/src_erl/Communication_Layer/http_Nerlserver/src/utilities_handler.erl b/src_erl/Communication_Layer/http_Nerlserver/src/utilities_handler.erl index 11476f19..4ac0d7b4 100644 --- a/src_erl/Communication_Layer/http_Nerlserver/src/utilities_handler.erl +++ b/src_erl/Communication_Layer/http_Nerlserver/src/utilities_handler.erl @@ -7,9 +7,8 @@ init(Req0 , [MainServerPid]) -> {_,Body,_} = cowboy_req:read_body(Req0), io:format("Body: ~p~n" , [Body]) , - Formatted = binary_to_list(Body), - [UtilityName , IP , Port] = [X || X <- re:split(Formatted , "#" , [{return , list}])], - case list_to_atom(UtilityName) of + [UtilityName , IP , Port] = binary_to_term(Body), + case UtilityName of nerlMonitor -> Graph = gen_server:call(MainServerPid , getGraph), WorkersMap = ets:lookup_element(nerlnet_data , workers , ?DATA_IDX), @@ -17,7 +16,7 @@ init(Req0 , [MainServerPid]) -> Workers = lists:flatten([atom_to_list(X)++"-"++atom_to_list(Y)++"!" || {X , Y} <- WorkersClients]), Reply = Graph ++ "," ++ Workers end, - gen_server:cast(MainServerPid , {saveUtility , {list_to_atom(UtilityName) , IP , Port}}), %% TODO Add IP , Port to ets in main server record + gen_server:cast(MainServerPid , {saveUtility , {UtilityName , IP , Port}}), %% TODO Add IP , Port to ets in main server record Req = cowboy_req:reply(200, #{<<"content-type">> => <<"text/plain">>}, Reply, diff --git a/src_erl/NerlMonitor/src/MonitorGUI.py b/src_erl/NerlMonitor/src/MonitorGUI.py index e801474c..049a2f8e 100644 --- a/src_erl/NerlMonitor/src/MonitorGUI.py +++ b/src_erl/NerlMonitor/src/MonitorGUI.py @@ -178,23 +178,27 @@ async def GUI(msg_queue): MainWindow['-LOG-'].update(updated_text) elif msg[0] == 'stats': + existing_text = values['-STATS-'] Data = msg[1] - statDict = {"workers": {}} + statDict = {"entities": {}} for items in str(Data).split('|'): - key, val = items.split(':') + Entity, val = items.split(':') if '=' in val: - for worker in val.split(','): - workerName, time = worker.split('=') - statDict["workers"][workerName] = time + for EntityStat in val.split(','): + Stat, Result = EntityStat.split('=') + statDict["entities"][Stat] = Result else: - statDict[key] = val - for key, val in statDict.items(): + statDict[Entity] = val + for Entity, val in statDict.items(): + existing_text = values['-STATS-'] if isinstance(val, dict): - MainWindow['-STATS-'].update(f'{formatted_time()}: {key}') - for key2, val2 in val.items(): - MainWindow['-STATS-'].update(f'{formatted_time()}: {key2} : {val2}') + MainWindow['-STATS-'].update(f'{existing_text}\n{formatted_time()}: {Entity} Stats:\n') + for Stat, Res in val.items(): + existing_text = values['-STATS-'] + updated_text = f'{existing_text}\n{formatted_time()}: {Stat}: {Res}' + MainWindow['-STATS-'].update(f'{existing_text}\n{formatted_time()}: {updated_text}') else: - MainWindow['-STATS-'].update(f'{formatted_time()}: {key} : {val}') + MainWindow['-STATS-'].update(f'{existing_text}\n{formatted_time()}: {Entity} Received {val} Messages') elif values['-LOG-'] != '': diff --git a/src_erl/NerlMonitor/src/nerlMonitor_app.erl b/src_erl/NerlMonitor/src/nerlMonitor_app.erl index 5b1a6b00..c178876b 100644 --- a/src_erl/NerlMonitor/src/nerlMonitor_app.erl +++ b/src_erl/NerlMonitor/src/nerlMonitor_app.erl @@ -34,7 +34,7 @@ start(_StartType, _StartArgs) -> erlang:register(recvPyrlang , self()), _GUI_PID = spawn_link(?MODULE , link_GUI , [self()]) , %% PyrlangNode: ('PyralngProcess' , 'py@127.0.0.1' , 'COOKIE') , sending message by: 'GUI ! HELLO.' URL = "http://" ++ ?MSADDRES ++ "/toolConnectionReq", - mainServerPing(URL,[list_to_binary(atom_to_list(?UTILNAME) ++ "#") , list_to_binary(?IP ++ "#") , list_to_binary(integer_to_list(?PORT))]), %% TODO How to "import" nerl_tools + mainServerPing(URL,term_to_binary([?UTILNAME , ?IP , integer_to_list(?PORT)])), %% TODO How to "import" nerl_tools nerlMonitor_sup:start_link(). From 594d77b56fd42a6b04a38253f3a9fafde56f41d3 Mon Sep 17 00:00:00 2001 From: GuyPerets106 Date: Mon, 14 Aug 2023 13:13:05 +0000 Subject: [PATCH 34/40] [NerlMonitor] Final commit before testing --- .gitignore | 3 +- .../exp_6Workers1SourceMNist.json | 2 +- .../src/Client/clientStatem.erl | 30 ++++-- .../src/MainServer/mainGenserver.erl | 36 ++++--- .../http_Nerlserver/src/utilities_handler.erl | 2 - src_erl/NerlMonitor/src/MonitorGUI.py | 101 ++++++++++++------ .../src/handlers/nerlMonitor_handler.erl | 6 +- src_erl/NerlMonitor/src/nerlMonitor_app.erl | 40 ++++--- 8 files changed, 136 insertions(+), 84 deletions(-) diff --git a/.gitignore b/.gitignore index 64abdb48..393cae9b 100644 --- a/.gitignore +++ b/.gitignore @@ -37,4 +37,5 @@ examples/.ipynb_checkpoints/simple_run-checkpoint.ipynb /Results arch.json conn.json -/inputDataDir \ No newline at end of file +/inputDataDir +/NerlNetGraph.png diff --git a/inputJsonFiles/experimentsFlow/exp_6Workers1SourceMNist.json b/inputJsonFiles/experimentsFlow/exp_6Workers1SourceMNist.json index 75b46199..3c15b57e 100644 --- a/inputJsonFiles/experimentsFlow/exp_6Workers1SourceMNist.json +++ b/inputJsonFiles/experimentsFlow/exp_6Workers1SourceMNist.json @@ -1,7 +1,7 @@ { "Features": 784, "Labels": ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"], - "CSV path": "mnist", + "CSV path": "mnist-o", "Batches per source": { "Training": 10000, diff --git a/src_erl/Communication_Layer/http_Nerlserver/src/Client/clientStatem.erl b/src_erl/Communication_Layer/http_Nerlserver/src/Client/clientStatem.erl index f5dd2b2e..9a52605a 100644 --- a/src_erl/Communication_Layer/http_Nerlserver/src/Client/clientStatem.erl +++ b/src_erl/Communication_Layer/http_Nerlserver/src/Client/clientStatem.erl @@ -163,7 +163,7 @@ waitforWorkers(cast, {worker_kill , Body}, State = #client_statem_state{etsRef = waitforWorkers(cast, EventContent, State = #client_statem_state{etsRef = EtsRef}) -> ets:update_counter(EtsRef, msgCounter, 1), ets:update_counter(EtsRef, infoIn, nerl_tools:calculate_size(EventContent)), - ?LOG_WARNING("client waitforWorkers ignored!!!: ~p ~n",[EventContent]), + ?LOG_WARNING("client waitforWorkers ignored!!!: ~p~n",[EventContent]), {next_state, waitforWorkers, State}; %---------------------------------------------------------------- @@ -214,6 +214,7 @@ idle(cast, In = {statistics}, State = #client_statem_state{ myName = MyName, ets {next_state, idle, State}; idle(cast, In = {training}, State = #client_statem_state{etsRef = EtsRef}) -> + io:format("client going to state training~n",[]), ets:update_counter(EtsRef, msgCounter, 1), ets:update_counter(EtsRef, infoIn, nerl_tools:calculate_size(In)), MessageToCast = {training}, @@ -370,12 +371,19 @@ training(cast, In = {loss,WorkerName,LossFunction,_Time_NIF}, State = #client_st % ets:update_counter(EtsRef, msgCounter, 1), % last param is increment value % ets:update_counter(EtsRef, infoIn, nerl_tools:calculate_size(In)), % sendStatistics(EtsRef), -% {next_state, training, State}; +% {next_state, training , State}; training(cast, EventContent, State = #client_statem_state{etsRef = EtsRef, myName = MyName}) -> - ?LOG_WARNING("client ~p training ignored!!!: ~p ~n!!!",[MyName, EventContent]), - ets:update_counter(EtsRef, infoIn, nerl_tools:calculate_size(EventContent)), - {next_state, training, State#client_statem_state{etsRef = EtsRef}}; + case EventContent of % ! Not the best way to handle this + {statistics} -> + ets:update_counter(EtsRef, infoIn, nerl_tools:calculate_size(EventContent)), + {next_state, training, State#client_statem_state{etsRef = EtsRef}}; + _ -> + ?LOG_WARNING("client ~p training ignored!!!: ~p ~n!!!",[MyName, EventContent]), + ets:update_counter(EtsRef, infoIn, nerl_tools:calculate_size(EventContent)), + {next_state, training, State#client_statem_state{etsRef = EtsRef}} + end; + training(info, EventContent, State = #client_statem_state{myName = MyName,etsRef = EtsRef}) -> case EventContent of @@ -446,11 +454,11 @@ predict(cast, In = {idle}, State = #client_statem_state{etsRef = EtsRef}) -> Workers = ets:lookup_element(EtsRef, workersNames, ?ETS_KV_VAL_IDX), {next_state, waitforWorkers, State#client_statem_state{nextState = idle, waitforWorkers = Workers, etsRef = EtsRef}}; -% predict(cast, In = {statistics}, State = #client_statem_state{ myName = MyName, etsRef = EtsRef}) -> -% ets:update_counter(EtsRef, msgCounter, 1), % last param is increment value -% ets:update_counter(EtsRef, infoIn, nerl_tools:calculate_size(In)), -% sendStatistics(EtsRef), -% {next_state, predict, State}; +predict(cast, In = {statistics}, State = #client_statem_state{ myName = MyName, etsRef = EtsRef}) -> + ets:update_counter(EtsRef, msgCounter, 1), % last param is increment value + ets:update_counter(EtsRef, infoIn, nerl_tools:calculate_size(In)), + sendStatistics(EtsRef), + {next_state, predict, State}; predict(cast, EventContent, State = #client_statem_state{etsRef = EtsRef}) -> ets:update_counter(EtsRef, msgCounter, 1), @@ -580,7 +588,7 @@ sendStatistics(EtsRef)-> Counter = ets:lookup_element(EtsRef, msgCounter, ?ETS_KV_VAL_IDX), InfoSize = ets:lookup_element(EtsRef, infoIn, ?ETS_KV_VAL_IDX), MyName = ets:lookup_element(EtsRef, myName, ?ETS_KV_VAL_IDX), - + % io:format("Timing Map is ~p~n",[TimingMap]), TimingStats = lists:flatten([atom_to_list(WorkerName)++"_Train_Avg_Time="++float_to_list(TotalTime/TotalBatches,[{decimals, 3}])++","||{WorkerName,{_LastTime,TotalBatches,TotalTime}}<-TimingMap]), MissingStats = lists:flatten([atom_to_list(WorkerName)++"_Train_Miss="++integer_to_list(MissCount)++","||{WorkerName,MissCount}<-MissedCounts]), MyStats = atom_to_list(MyName)++"_Msg_Count="++integer_to_list(Counter)++","++atom_to_list(MyName)++"_info_Size="++integer_to_list(InfoSize)++",", diff --git a/src_erl/Communication_Layer/http_Nerlserver/src/MainServer/mainGenserver.erl b/src_erl/Communication_Layer/http_Nerlserver/src/MainServer/mainGenserver.erl index 2b3ad1cb..77f423c0 100644 --- a/src_erl/Communication_Layer/http_Nerlserver/src/MainServer/mainGenserver.erl +++ b/src_erl/Communication_Layer/http_Nerlserver/src/MainServer/mainGenserver.erl @@ -163,7 +163,7 @@ handle_cast({clientsIdle}, State = #main_genserver_state{state = idle, myName = {noreply, State#main_genserver_state{clientsWaitingList = ListOfClients,msgCounter = MsgCounter+1}}; %%%get Statistics from all Entities in the network -handle_cast({statistics,Body}, State = #main_genserver_state{myName = MyName, statisticsCounter = StatisticsCounter, nerlnetGraph = NerlnetGraph,statisticsMap = StatisticsMap,msgCounter = MsgCounter , etsRef = EtsRef}) -> +handle_cast({statistics,Body}, State = #main_genserver_state{myName = MyName, statisticsCounter = StatisticsCounter, nerlnetGraph = NerlnetGraph,statisticsMap = StatisticsMap,msgCounter = MsgCounter , etsRef = EtsRef , state = CurrState}) -> if Body == <<"getStatistics">> -> %% initial message from APIServer, get stats from entities @@ -182,21 +182,26 @@ handle_cast({statistics,Body}, State = #main_genserver_state{myName = MyName, st if StatisticsCounter == 1 -> %% got stats from all entities Statistics = maps:to_list(NewStatisticsMap), S = mapToString(Statistics,[]) , - ?LOG_NOTICE("Sending stats: ~p~n",[S]), - {RouterHost,RouterPort} = nerl_tools:getShortPath(MyName,?API_SERVER_ATOM,NerlnetGraph), - nerl_tools:http_request(RouterHost,RouterPort,"statistics", S ++ "|mainServer:" ++integer_to_list(MsgCounter)), - - case ets:member(EtsRef , nerlMonitor) of - true -> - [{_ , MonitorIP , MonitorPort}] = ets:lookup(EtsRef, nerlMonitor), - nerl_tools:http_request(MonitorIP,list_to_integer(MonitorPort),"stats", S ++ "|mainServer:" ++integer_to_list(MsgCounter)); - _ -> ok + %?LOG_NOTICE("Sending stats: ~p~n",[S]), + case CurrState of + idle -> + case ets:member(EtsRef,nerlMonitor) of + true-> + [{nerlMonitor , IP , Port}] = ets:lookup(EtsRef , nerlMonitor), + nerl_tools:http_request(IP,list_to_integer(Port),"stats", S ++ "|mainServer:" ++integer_to_list(MsgCounter)); + false -> ok + end, + {RouterHost,RouterPort} = nerl_tools:getShortPath(MyName,?API_SERVER_ATOM,NerlnetGraph), + nerl_tools:http_request(RouterHost,RouterPort,"statistics", S ++ "|mainServer:" ++integer_to_list(MsgCounter)); + casting -> + [{_ , MonitorIP , MonitorPort}] = ets:lookup(EtsRef, nerlMonitor), + nerl_tools:http_request(MonitorIP,list_to_integer(MonitorPort),"stats", S ++ "|mainServer:" ++integer_to_list(MsgCounter)) end; - %% wait for more stats - true -> pass - end - end, + %% wait for more stats + true -> pass + end + end, {noreply, NewState}; %%handle_cast({startPredicting}, State = #main_genserver_state{clients = ListOfClients, nerlnetGraph = NerlnetGraph}) -> @@ -344,7 +349,6 @@ handle_cast({worker_kill , WorkerName} , State = #main_genserver_state{workersMa WorkerNameAtom=binary_to_term(WorkerName), ClientName = maps:get(WorkerNameAtom,WorkersMap), % ! Worker Name is binary? Body = term_to_binary({ClientName, WorkerNameAtom}), - io:format("-------------worker kill got is: ~p sending:~p~n",[binary_to_term(WorkerName),binary_to_term(Body)]), nerl_tools:sendHTTP(?MAIN_SERVER_ATOM , ClientName , atom_to_list(worker_kill) , Body), {noreply, State#main_genserver_state{workersMap = WorkersMap , msgCounter = MsgCounter+1}}; @@ -416,7 +420,7 @@ getNewStatisticsMap([{Name,{_Host, _Port}}|Tail],StatisticsMap) -> startCasting([],_NumOfSampleToSend,_MyName, _NerlnetGraph) -> done; startCasting([SourceName|SourceNames],NumOfSampleToSend, MyName, NerlnetGraph)-> - ?LOG_NOTICE("~p sending start casting command to: ~p",[MyName, SourceName]), + ?LOG_NOTICE("~p sending start casting command to: ~p~n",[MyName, SourceName]), {RouterHost,RouterPort} = nerl_tools:getShortPath(MyName,SourceName,NerlnetGraph), nerl_tools:http_request(RouterHost,RouterPort,"startCasting", SourceName++[","]++NumOfSampleToSend), startCasting(SourceNames,NumOfSampleToSend, MyName, NerlnetGraph). diff --git a/src_erl/Communication_Layer/http_Nerlserver/src/utilities_handler.erl b/src_erl/Communication_Layer/http_Nerlserver/src/utilities_handler.erl index 4ac0d7b4..f1c01224 100644 --- a/src_erl/Communication_Layer/http_Nerlserver/src/utilities_handler.erl +++ b/src_erl/Communication_Layer/http_Nerlserver/src/utilities_handler.erl @@ -6,7 +6,6 @@ init(Req0 , [MainServerPid]) -> {_,Body,_} = cowboy_req:read_body(Req0), - io:format("Body: ~p~n" , [Body]) , [UtilityName , IP , Port] = binary_to_term(Body), case UtilityName of nerlMonitor -> @@ -25,7 +24,6 @@ init(Req0 , [MainServerPid]) -> init(Req0 , [Action , MainServerPid]) -> {_,Body,_} = cowboy_req:read_body(Req0), - io:format("got kill-------------------messege: ~p~n",[Body]), case Action of worker_kill -> gen_statem:cast(MainServerPid , {worker_kill , Body}); diff --git a/src_erl/NerlMonitor/src/MonitorGUI.py b/src_erl/NerlMonitor/src/MonitorGUI.py index 049a2f8e..c9032767 100644 --- a/src_erl/NerlMonitor/src/MonitorGUI.py +++ b/src_erl/NerlMonitor/src/MonitorGUI.py @@ -47,41 +47,41 @@ def draw_gradient(canvas, start_color, end_color): DataColumn = [ [sg.Frame(title="Event Log:" , layout=[[sg.Multiline('', size=(140, 60), key='-LOG-', autoscroll=True , font=('SFPro' , 12) , no_scrollbar=True)]], - background_color=('#930707') , font=('SFPro' , 20) , size=(500,325) , title_color='Black' , element_justification='right') + background_color=('#A90433') , font=('SFPro' , 20) , size=(500,325) , title_color='White' , element_justification='right') ] , [sg.Frame(title="Statistics:" , layout=[[sg.Multiline('', size=(140, 60), key='-STATS-', autoscroll=True , font=('SFPro' , 12) , no_scrollbar=True)]], - background_color=('#930707') , font=('SFPro' , 20) , size=(500,325) , title_color='Black' , element_justification='right') + background_color=('#A90433') , font=('SFPro' , 20) , size=(500,325) , title_color='White' , element_justification='right') ] ] GraphColumn = [ - [ sg.Text("Waiting For\n NerlNet Graph..." , key='-PHOLD-', text_color='Black' , font=('SFPro' , 12) , size=(70,5) , background_color='#930707' , justification='center' , pad=(0,0)) , + [ sg.Text("Waiting For\n NerlNet Graph..." , key='-PHOLD-', text_color='White' , font=('SFPro' , 12) , size=(70,5) , background_color='#A90433' , justification='center' , pad=(0,0)) , sg.Image(key='-IMAGE-' , visible=False) ], [ - sg.Text("Enter the name of the worker you wish to terminate:" ,key='-INTEXT-', size=(42,1) ,text_color='white' , font=('SFPro' , 12) , background_color='#930707' , justification='left' , pad=(0,0) , visible=False) , + sg.Text("Enter the name of the worker you wish to terminate:" ,key='-INTEXT-', size=(42,1) ,text_color='white' , font=('SFPro' , 12) , background_color='#A90433' , justification='left' , pad=(0,0) , visible=False) , sg.Input('',key='-INPUT-' , visible=False , justification='left' , size=(20,1) , font=('SFPro' , 12) , background_color='white' , text_color='black' , pad=(0,0) , enable_events=True), - sg.Button(button_text="Terminate" , button_color=('#C30404' , '#000000') , font=('SFPro' , 12) , size=(10,1) , pad=(0,0) , visible=False, key='-TERM-', enable_events=True) + sg.Button(button_text="Terminate" , button_color=('#A90433' , '#FFFFFF') , font=('SFPro' , 12) , size=(10,1) , pad=(0,0) , visible=False, key='-TERM-', enable_events=True) ] ] layout = [ [ - sg.Text("NerlNet Monitor" , key='-TEXT-' , size=(30,1) ,text_color='Black' , font=('SFPro' , 20) , background_color='#930707' , justification='center' , pad=(0,0)) + sg.Text("NerlNet Monitor" , key='-TEXT-' , size=(30,1) ,text_color='White' , font=('SFPro' , 20) , background_color='#A90433' , justification='center' , pad=(0,0)) ] , - [ sg.Column(DataColumn , background_color='#930707') , + [ sg.Column(DataColumn , background_color='#A90433') , sg.VSeperator() , - sg.Column(GraphColumn , background_color='#930707') + sg.Column(GraphColumn , background_color='#A90433') ] , [ - sg.Button(button_text="Close" , button_color=('#C30404' , '#000000') , font=('SFPro' , 12) , size=(5,2)), - sg.Button(button_text="Clear Log" , button_color=('#C30404' , '#000000') , font=('SFPro' , 12) , size=(5,2)) + sg.Button(button_text="Close" , button_color=('#A90433' , '#FFFFFF') , font=('SFPro' , 12) , size=(5,2)), + sg.Button(button_text="Clear Log" , button_color=('#A90433' , '#FFFFFF') , font=('SFPro' , 12) , size=(5,2)) ] ] -MainWindow = sg.Window("NErlNet" , layout , margins=(5,5) , size=(1400,800) , background_color='#930707' , finalize=True , resizable=True , element_justification='c') +MainWindow = sg.Window("NErlNet" , layout , margins=(5,5) , size=(1400,800) , background_color='#A90433' , finalize=True , resizable=True , element_justification='c' , icon='../../../NerlnetLogo.ico') def RemoteRecv(): return Atom('erl@127.0.0.1') , Atom("recvPyrlang") @@ -110,16 +110,20 @@ async def GUI(msg_queue): print("Got Message from queue") print(msg_queue.empty()) print(f"Got PyNode and CommProc from Queue.") + StatsInfo = {"workers": {} , "clients": {}} while True: - await asyncio.sleep(.1) + await asyncio.sleep(.01) event , values = MainWindow.read(timeout=100) existing_text = values['-LOG-'] updated_text = '' if event == "Close" or event == sg.WIN_CLOSED: + PyNode.send_nowait(sender = CommProc.pid_ , receiver = RemoteRecv() , message = (Atom('close'))) + await asyncio.sleep(.2) os.kill(os.getpid() , 9) print("GUI Closed.") break elif event == "Clear Log": + ShowStats(StatsInfo) MainWindow['-LOG-'].update('') elif event == "-TERM-": Workers = [Graph.nodes[node]['label'] for node in Graph.nodes() if Graph.nodes[node]['label'][0] == 'w' and node_colors[node] != 'gray'] @@ -138,7 +142,7 @@ async def GUI(msg_queue): plt.close() MainWindow['-IMAGE-'].update(filename='NerlNetGraph.png' , visible=True , size=(800,600)) - updated_text = f'{existing_text}\n{formatted_time()}: Worker {values["-INPUT-"]} terminated , Available Workers: {Workers}.' + updated_text = f'{existing_text}\n{formatted_time()}: Sending termination message for {values["-INPUT-"]} to Main Server.' PyNode.send_nowait(sender = CommProc.pid_ , receiver = RemoteRecv() , message = (Atom('terminate'),Atom(f'{values["-INPUT-"]}'))) @@ -174,31 +178,35 @@ async def GUI(msg_queue): if existing_text == '': updated_text = f'{formatted_time()}: Worker {WorkerName} of Client {ClientName} is down.' else: - updated_text = f'{existing_text}\n{formatted_time()}: Worker {WorkerName} of Client {ClientName} is down.' + Workers = [Graph.nodes[node]['label'] for node in Graph.nodes() if Graph.nodes[node]['label'][0] == 'w' and node_colors[node] != 'gray'] + updated_text = f'{existing_text}\n{formatted_time()}: Worker {WorkerName} of Client {ClientName} is down , Available workers: {Workers}' MainWindow['-LOG-'].update(updated_text) elif msg[0] == 'stats': - existing_text = values['-STATS-'] - Data = msg[1] - statDict = {"entities": {}} - for items in str(Data).split('|'): - Entity, val = items.split(':') - if '=' in val: - for EntityStat in val.split(','): - Stat, Result = EntityStat.split('=') - statDict["entities"][Stat] = Result - else: - statDict[Entity] = val - for Entity, val in statDict.items(): - existing_text = values['-STATS-'] - if isinstance(val, dict): - MainWindow['-STATS-'].update(f'{existing_text}\n{formatted_time()}: {Entity} Stats:\n') - for Stat, Res in val.items(): - existing_text = values['-STATS-'] - updated_text = f'{existing_text}\n{formatted_time()}: {Stat}: {Res}' - MainWindow['-STATS-'].update(f'{existing_text}\n{formatted_time()}: {updated_text}') + try: + Data = msg[1] + for items in str(Data).split('|'): + Entity, val = items.split(':') + if '=' in val: + for EntityStat in val.split(','): + Stat, Result = EntityStat.split('=') + if "Train" in Stat: + StatsInfo["workers"][Stat] = Result + else: + StatsInfo["clients"][Stat] = Result + else: + StatsInfo[Entity] = val # Messages for entities other than clients/workers + CurrentStats = StatsInfo.copy() # copy the stats to a new variable + StatsText = ShowStats(CurrentStats) + existing_stats = values['-STATS-'] + if existing_stats != '': + MainWindow['-STATS-'].update(f'{existing_stats}\n{StatsText}') else: - MainWindow['-STATS-'].update(f'{existing_text}\n{formatted_time()}: {Entity} Received {val} Messages') + MainWindow['-STATS-'].update(StatsText) + existing_text = values['-LOG-'] + MainWindow['-LOG-'].update(f'{existing_text}\n{formatted_time()}: Statistics Received.') + except Exception as err: + MainWindow['-LOG-'].update(f"Error in Stats {err} , Got {StatsInfo}") elif values['-LOG-'] != '': @@ -214,6 +222,29 @@ async def GUI(msg_queue): MainWindow.close() +def ShowStats(CurrentStats): + MainWindow['-LOG-'].update(f'{formatted_time()}: Printing Statistics...') + StatsText = '' + for key in CurrentStats: + if key == 'workers': + StatsText += f'Workers:\n' + for stat in CurrentStats[key]: + if "Time" in stat: + StatsText += f'\t{stat.replace("_Train_" , " Working ")}: {CurrentStats[key][stat]} seconds\n' + else: + StatsText += f'\t{stat.replace("_" , " ")}: {CurrentStats[key][stat]}\n' + elif key == 'clients': + StatsText += f'Clients:\n' + for stat in CurrentStats[key]: + if "info" in stat: + StatsText += f'\t{stat.replace("_info_" , " Info ")}: {CurrentStats[key][stat]} bytes\n' + else: + StatsText += f'\t{stat.replace("_Msg_" , " Message ")}: {CurrentStats[key][stat]}\n' + else: + StatsText += f'{key} Message Count: {CurrentStats[key]}\n' + return StatsText + + def Show_Nerlnet_Graph(NerlGraph): # Graph in string format: "Entity1Name,Entity1IP,Entity1Port#Entity2Name,Entity2IP,Entity2Port#Entity1Name-Entity2Name,Entity2Name-Entity1Name#Worker1-Client1#Worker2-Client2" etc. # Workers in string format: "Worker1-Client1,Worker2-Client1,Worker3-Client2" etc. @@ -236,7 +267,7 @@ def Show_Nerlnet_Graph(NerlGraph): my_labels = {'mainServer': 'mS' , 'apiServer': 'aS'} nx.relabel_nodes(graph, my_labels , copy=False) - default_colors = {node:'darkred' for node in graph.nodes()} + default_colors = {node:'#A90433' for node in graph.nodes()} node_colors = {node:default_colors[node] for node in graph.nodes()} nx.set_node_attributes(graph, node_colors, 'color') colors = nx.get_node_attributes(graph, 'color').values() diff --git a/src_erl/NerlMonitor/src/handlers/nerlMonitor_handler.erl b/src_erl/NerlMonitor/src/handlers/nerlMonitor_handler.erl index 7cdc3302..59f19934 100644 --- a/src_erl/NerlMonitor/src/handlers/nerlMonitor_handler.erl +++ b/src_erl/NerlMonitor/src/handlers/nerlMonitor_handler.erl @@ -8,10 +8,8 @@ init(Req0, [Msg]) -> {_,Body,_} = cowboy_req:read_body(Req0), Data = binary_to_list(Body), case Msg of - utilInfo -> io:format("Update graph with: worker_down ~p~n" , [Data]) , - ?GUI ! {update ,Data}; - stats -> io:format("Update graph with: stats ~p~n" , [Data]) , - ?GUI ! {stats ,Data}; + utilInfo -> ?GUI ! {update ,Data}; + stats -> ?GUI ! {stats ,Data}; _ -> ok % got unknown messge, ignore. end, diff --git a/src_erl/NerlMonitor/src/nerlMonitor_app.erl b/src_erl/NerlMonitor/src/nerlMonitor_app.erl index c178876b..62d6fe7b 100644 --- a/src_erl/NerlMonitor/src/nerlMonitor_app.erl +++ b/src_erl/NerlMonitor/src/nerlMonitor_app.erl @@ -9,7 +9,7 @@ -include("../../Communication_Layer/http_Nerlserver/src/nerl_tools.hrl"). --export([start/2, stop/1 , link_GUI/1]). +-export([start/2, stop/1 , link_GUI/0]). -define(UTILNAME,nerlMonitor). -define(IP , "192.168.64.7"). @@ -32,7 +32,7 @@ start(_StartType, _StartArgs) -> {ok, _} = cowboy:start_clear(?UTILNAME,[{port, ?PORT}],#{env => #{dispatch => Dispatch}}), io:format("nerlMonitor started , opening GUI...~n"), erlang:register(recvPyrlang , self()), - _GUI_PID = spawn_link(?MODULE , link_GUI , [self()]) , %% PyrlangNode: ('PyralngProcess' , 'py@127.0.0.1' , 'COOKIE') , sending message by: 'GUI ! HELLO.' + _GUI_PID = spawn_link(?MODULE , link_GUI , []) , %% PyrlangNode: ('PyralngProcess' , 'py@127.0.0.1' , 'COOKIE') , sending message by: 'GUI ! HELLO.' URL = "http://" ++ ?MSADDRES ++ "/toolConnectionReq", mainServerPing(URL,term_to_binary([?UTILNAME , ?IP , integer_to_list(?PORT)])), %% TODO How to "import" nerl_tools nerlMonitor_sup:start_link(). @@ -41,38 +41,50 @@ start(_StartType, _StartArgs) -> %ping main server in 0.5 sec intervals with connection request. will stop when got valid response. mainServerPing(URL,Body)-> - io:format("pinging main server~n"), - io:format("Body: ~p~n" , [Body]), + io:format("pinging main server...~n"), Response = httpc:request(post,{URL, [],"application/x-www-form-urlencoded",Body}, [], []), case Response of {error,_}-> timer:sleep(1000), - mainServerPing(URL,Body); + receive + close -> + io:format("Quitting NerlMonitor...~n"), + {ok , AppName} = application:get_application(), + stop(AppName) + after 0 -> + mainServerPing(URL,Body) + end; {ok,{_ResCode, _Headers, Data}}-> - io:format("Sending graph to GUI , graph is ~p~n" , [Data]), - ?GUI ! {graph , Data} , + io:format("Got NerlGraph , Sending to GUI...~n" , []), + ?GUI ! {graph , Data}, recvLoop(); {ok , _} -> - io:format("got2 response from main server~n") + io:format("Got unknown response from main server~n") end. recvLoop()-> %% MainServer replies with Nerlnet-Graph when nerlMonitor tool is used receive {terminate , WorkerName} -> - io:format("got termination for Worker ~p from GUI~n" , [WorkerName]), + io:format("Got termination message for Worker ~p from GUI~n" , [WorkerName]), URL = "http://" ++ ?MSADDRES ++ "/worker_kill", Body = term_to_binary(WorkerName), - httpc:request(post,{URL, [],"application/x-www-form-urlencoded",Body}, [], []); - _ -> io:format("got unknown message from GUI~n") - end, - recvLoop(). + httpc:request(post,{URL, [],"application/x-www-form-urlencoded",Body}, [], []), + recvLoop(); + close -> + io:format("Quitting NerlMonitor...~n"), + {ok , AppName} = application:get_application(), + stop(AppName); + Msg -> + io:format("Got unknown message from GUI: ~p~n", [Msg]), + recvLoop() + end. stop(_State) -> ok. -link_GUI(Pid) -> +link_GUI() -> os:cmd('python3 src/MonitorGUI.py'), io:format("GUI Closed~n"). From 61c9a9a59ece9c640e377f1305a3664e9e753b8f Mon Sep 17 00:00:00 2001 From: galhilu Date: Tue, 15 Aug 2023 18:32:13 +0300 Subject: [PATCH 35/40] [NerlMonitor] comment editing --- .../src/Client/clientStatem.erl | 83 +++++++------------ .../src/MainServer/mainGenserver.erl | 35 ++++++-- .../src/utilities_attacher.erl | 6 -- .../http_Nerlserver/src/utilities_handler.erl | 6 +- src_erl/NerlMonitor/src/nerlMonitor_app.erl | 6 +- 5 files changed, 64 insertions(+), 72 deletions(-) delete mode 100644 src_erl/Communication_Layer/http_Nerlserver/src/utilities_attacher.erl diff --git a/src_erl/Communication_Layer/http_Nerlserver/src/Client/clientStatem.erl b/src_erl/Communication_Layer/http_Nerlserver/src/Client/clientStatem.erl index 9a52605a..21ae597c 100644 --- a/src_erl/Communication_Layer/http_Nerlserver/src/Client/clientStatem.erl +++ b/src_erl/Communication_Layer/http_Nerlserver/src/Client/clientStatem.erl @@ -100,41 +100,6 @@ format_status(_Opt, [_PDict, _StateName, _State]) -> Status = some_term, Status. %% ==============STATES================= waitforWorkers(cast, In = {stateChange,WorkerName,MissedBatchesCount}, State = #client_statem_state{myName = MyName,waitforWorkers = WaitforWorkers,nextState = NextState, etsRef = EtsRef}) -> NewWaitforWorkers = WaitforWorkers -- [WorkerName], - % case WorkerName of - % w5 -> - % Pid = ets:lookup_element(EtsRef, w5, ?WORKER_PID_IDX), - % io:format("---------------------Killin w5-----------------------~n"), - % gen_statem:stop(Pid , shutdown , infinity); - % _ -> ok - % end, - % io:format("remaining workers = ~p~n",[NewWaitforWorkers]), - % case NextState of - % training -> - % case WorkerName of - % w5 -> - % case ets:member(EtsRef , w5) of - % true -> - % Pid = ets:lookup_element(EtsRef, w5, ?WORKER_PID_IDX), - % io:format("---------------------Killin w5-----------------------~n"), - % gen_statem:stop(Pid , shutdown , infinity); - % _ -> ok - % end; - % _ -> ok - % end; - % predict -> - % case WorkerName of - % w6 -> - % case ets:member(EtsRef , w6) of - % true -> - % Pid = ets:lookup_element(EtsRef, w6, ?WORKER_PID_IDX), - % io:format("---------------------Killin w6-----------------------~n"), - % gen_statem:stop(Pid , shutdown , infinity); - % _ -> ok - % end; - % _ -> ok - % end; - % _ -> ok - % end, ets:update_counter(EtsRef, msgCounter, 1), % last is increment value ets:update_counter(EtsRef, infoIn, nerl_tools:calculate_size(In)), ets:update_element(EtsRef, WorkerName,[{?WORKER_TRAIN_MISSED_IDX,MissedBatchesCount}]), %% update missed batches count @@ -153,12 +118,15 @@ waitforWorkers(cast, In = {NewState}, State = #client_statem_state{myName = MyNa cast_message_to_workers(EtsRef, {NewState}), {next_state, waitforWorkers, State#client_statem_state{nextState = NewState, waitforWorkers = Workers}}; -waitforWorkers(cast, {worker_kill , Body}, State = #client_statem_state{etsRef = EtsRef}) -> +waitforWorkers(cast, In ={worker_kill , Body}, State = #client_statem_state{waitforWorkers = WaitforWorkers,etsRef = EtsRef}) -> + %got kill command for a worker, kiil and take off waitforWorkers list ets:update_counter(EtsRef, msgCounter, 1), + ets:update_counter(EtsRef, infoIn, nerl_tools:calculate_size(In)), {_, WorkerName} = binary_to_term(Body), Pid = ets:lookup_element(EtsRef, WorkerName, ?WORKER_PID_IDX), gen_statem:stop(Pid, shutdown , infinity), - {next_state, waitforWorkers, State#client_statem_state{etsRef = EtsRef}}; + NewWaitforWorkers = WaitforWorkers -- [WorkerName], + {next_state, waitforWorkers, State#client_statem_state{waitforWorkers = NewWaitforWorkers,etsRef = EtsRef}}; waitforWorkers(cast, EventContent, State = #client_statem_state{etsRef = EtsRef}) -> ets:update_counter(EtsRef, msgCounter, 1), @@ -166,23 +134,22 @@ waitforWorkers(cast, EventContent, State = #client_statem_state{etsRef = EtsRef} ?LOG_WARNING("client waitforWorkers ignored!!!: ~p~n",[EventContent]), {next_state, waitforWorkers, State}; -%---------------------------------------------------------------- waitforWorkers(info, EventContent, State = #client_statem_state{myName = MyName,waitforWorkers = WaitforWorkers ,etsRef = EtsRef}) -> + ets:update_counter(EtsRef, msgCounter, 1), + ets:update_counter(EtsRef, infoIn, nerl_tools:calculate_size(EventContent)), case EventContent of {'DOWN',_Ref,process,Pid,_Reason}-> %worker down [[WorkerName]] = ets:match(EtsRef,{'$1',Pid,'_','_','_'}), NewWaitforWorkers = WaitforWorkers--[WorkerName], %report worker down to main server delete_worker(EtsRef,MyName,WorkerName), - {next_state, waitforWorkers, State#client_statem_state{waitforWorkers = NewWaitforWorkers,etsRef = EtsRef}}; %delete worker from ets so client will not wait for it in the future + {next_state, waitforWorkers, State#client_statem_state{waitforWorkers = NewWaitforWorkers,etsRef = EtsRef}}; %delete worker from ets so client will not wait for it in the future _Any-> ?LOG_INFO("client ~p got unexpected info: ~p~n",[MyName, EventContent]), %recevied unexpected messege, print to log/tell main server for debuging {next_state, waitforWorkers, State#client_statem_state{etsRef = EtsRef}} end. - -%----------------------------------------------------------------- %% initiating workers when they include federated workers. init stage == handshake between federated worker client and server @@ -221,8 +188,9 @@ idle(cast, In = {training}, State = #client_statem_state{etsRef = EtsRef}) -> cast_message_to_workers(EtsRef, MessageToCast), {next_state, waitforWorkers, State#client_statem_state{waitforWorkers= ets:lookup_element(EtsRef, workersNames, ?ETS_KV_VAL_IDX), nextState = training}}; -idle(cast, {worker_kill , Body}, State = #client_statem_state{etsRef = EtsRef}) -> +idle(cast, In={worker_kill , Body}, State = #client_statem_state{etsRef = EtsRef}) -> ets:update_counter(EtsRef, msgCounter, 1), + ets:update_counter(EtsRef, infoIn, nerl_tools:calculate_size(In)), {_, WorkerName} = binary_to_term(Body), Pid = ets:lookup_element(EtsRef, WorkerName, ?WORKER_PID_IDX), gen_statem:stop(Pid, shutdown , infinity), @@ -244,6 +212,8 @@ idle(cast, EventContent, State = #client_statem_state{etsRef = EtsRef}) -> idle(info, EventContent, State = #client_statem_state{myName = MyName,etsRef = EtsRef}) -> + ets:update_counter(EtsRef, msgCounter, 1), + ets:update_counter(EtsRef, infoIn, nerl_tools:calculate_size(EventContent)), case EventContent of {'DOWN',_Ref,process,Pid,_Reason}-> %worker down [[WorkerName]] = ets:match(EtsRef,{'$1',Pid,'_','_','_'}), @@ -292,13 +262,6 @@ training(cast, In = {custom_worker_message, WorkersList, WeightsTensor}, State = end, lists:foreach(Func, WorkersList), {keep_state, State}; - -training(cast, {worker_kill , Body}, State = #client_statem_state{etsRef = EtsRef}) -> - ets:update_counter(EtsRef, msgCounter, 1), - {_, WorkerName} = binary_to_term(Body), - Pid = ets:lookup_element(EtsRef, WorkerName, ?WORKER_PID_IDX), - gen_statem:stop(Pid, shutdown , infinity), - {next_state, training, State#client_statem_state{etsRef = EtsRef}}; % TODO Validate this state - sample and empty list @@ -366,6 +329,15 @@ training(cast, In = {loss,WorkerName,LossFunction,_Time_NIF}, State = #client_st nerl_tools:http_request(RouterHost,RouterPort,"lossFunction", term_to_binary({WorkerName,LossFunction})), {next_state, training, State#client_statem_state{myName = MyName,etsRef = EtsRef}}; + +training(cast, In={worker_kill , Body}, State = #client_statem_state{etsRef = EtsRef}) -> + ets:update_counter(EtsRef, msgCounter, 1), + ets:update_counter(EtsRef, infoIn, nerl_tools:calculate_size(In)), + {_, WorkerName} = binary_to_term(Body), + Pid = ets:lookup_element(EtsRef, WorkerName, ?WORKER_PID_IDX), + gen_statem:stop(Pid, shutdown , infinity), + {next_state, training, State#client_statem_state{etsRef = EtsRef}}; + % training(cast, In = {statistics}, State = #client_statem_state{ myName = MyName, etsRef = EtsRef}) -> % io:format("client ~p got statistics ~n",[MyName]), % ets:update_counter(EtsRef, msgCounter, 1), % last param is increment value @@ -386,6 +358,8 @@ training(cast, EventContent, State = #client_statem_state{etsRef = EtsRef, myNam training(info, EventContent, State = #client_statem_state{myName = MyName,etsRef = EtsRef}) -> + ets:update_counter(EtsRef, msgCounter, 1), + ets:update_counter(EtsRef, infoIn, nerl_tools:calculate_size(EventContent)), case EventContent of {'DOWN' , _Ref , process , Pid , _Reason}-> %worker down [[WorkerName]] = ets:match(EtsRef,{'$1',Pid,'_','_','_'}), @@ -428,8 +402,9 @@ predict(cast, In = {predictRes,WorkerName,InputName,ResultID,PredictNerlTensor, nerl_tools:http_request(RouterHost,RouterPort,"predictRes", term_to_binary({atom_to_list(WorkerName), InputName, ResultID, {PredictNerlTensor, Type}})), {next_state, predict, State#client_statem_state{etsRef = EtsRef}}; -predict(cast, {worker_kill , Body}, State = #client_statem_state{etsRef = EtsRef}) -> +predict(cast, In={worker_kill , Body}, State = #client_statem_state{etsRef = EtsRef}) -> ets:update_counter(EtsRef, msgCounter, 1), + ets:update_counter(EtsRef, infoIn, nerl_tools:calculate_size(In)), {_, WorkerName} = binary_to_term(Body), Pid = ets:lookup_element(EtsRef, WorkerName, ?WORKER_PID_IDX), gen_statem:stop(Pid, shutdown , infinity), @@ -467,6 +442,8 @@ predict(cast, EventContent, State = #client_statem_state{etsRef = EtsRef}) -> {next_state, predict, State#client_statem_state{etsRef = EtsRef}}; predict(info, EventContent, State = #client_statem_state{myName = MyName,etsRef = EtsRef}) -> + ets:update_counter(EtsRef, msgCounter, 1), + ets:update_counter(EtsRef, infoIn, nerl_tools:calculate_size(EventContent)), case EventContent of {'DOWN',_Ref,process,Pid,_Reason}-> %worker down [[WorkerName]] = ets:match(EtsRef,{'$1',Pid,'_','_','_'}), @@ -594,7 +571,7 @@ sendStatistics(EtsRef)-> MyStats = atom_to_list(MyName)++"_Msg_Count="++integer_to_list(Counter)++","++atom_to_list(MyName)++"_info_Size="++integer_to_list(InfoSize)++",", {RouterHost,RouterPort} = nerl_tools:getShortPath(MyName, ?MAIN_SERVER_ATOM, NerlnetGraph), - nerl_tools:http_request(RouterHost,RouterPort,"statistics", list_to_binary(atom_to_list(MyName)++":"++MyStats++MissingStats++lists:droplast(TimingStats))). + nerl_tools:http_request(RouterHost,RouterPort,"statistics", list_to_binary(atom_to_list(MyName)++":"++MyStats++MissingStats++lists:droplast(TimingStats)++"#"++DeadWorkers)). cast_message_to_workers(EtsRef, Msg) -> Workers = ets:lookup_element(EtsRef, workersNames, ?ETS_KV_VAL_IDX), @@ -609,9 +586,9 @@ cast_message_to_workers(EtsRef, Msg) -> end, lists:foreach(Func, Workers). -%activated if client detected that a worker stopped, will delete it from ets so as not to wait for responsed from it in the future. + delete_worker(EtsRef,MyName,WorkerName)-> - %ets:delete(EtsRef , WorkerName), + %activated if client detected that a worker stopped, will delete it from workersNames and move to deadWorkers so as not to wait for responsed from it in the future. AliveList = ets:lookup_element(EtsRef, workersNames, ?ETS_KV_VAL_IDX), DeadList = ets:lookup_element(EtsRef, deadWorkers, ?ETS_KV_VAL_IDX), ets:delete(EtsRef,workersNames), diff --git a/src_erl/Communication_Layer/http_Nerlserver/src/MainServer/mainGenserver.erl b/src_erl/Communication_Layer/http_Nerlserver/src/MainServer/mainGenserver.erl index 77f423c0..021a1c6b 100644 --- a/src_erl/Communication_Layer/http_Nerlserver/src/MainServer/mainGenserver.erl +++ b/src_erl/Communication_Layer/http_Nerlserver/src/MainServer/mainGenserver.erl @@ -114,7 +114,8 @@ edgeString([Edge |EdgesList], Str)-> {noreply, NewState :: #main_genserver_state{}, timeout() | hibernate} | {stop, Reason :: term(), NewState :: #main_genserver_state{}}). -handle_cast({saveUtility , Body} , State = #main_genserver_state{etsRef = EtsRef , msgCounter = MsgCounter}) -> +handle_cast({saveUtility , Body} , State = #main_genserver_state{etsRef = EtsRef , msgCounter = MsgCounter}) -> + %gets called after a tool tried to connect, saves its addres in the ets. ets:insert(EtsRef , Body), {noreply, State#main_genserver_state{msgCounter = MsgCounter+1}}; @@ -174,16 +175,32 @@ handle_cast({statistics,Body}, State = #main_genserver_state{myName = MyName, st true -> %% statistics arrived from Entity - [From|[NewCounter]] = re:split(binary_to_list(Body), ":", [{return, list}]), + CheckForDead=re:split(binary_to_list(Body), "#", [{return, list}]), + case CheckForDead of + [_]-> + Data=CheckForDead, + UpdateStatMap=StatisticsMap; %entity is not a client + [Data,[]]-> + UpdateStatMap=StatisticsMap;%entity is client without dead workers + [Data,DeadWorkers]-> %client with dead workers + case maps:is_key("Dead workers") of + true-> + MapWithKey=StatisticsMap; %key was already put in + false-> + MapWithKey=maps:put("Dead workers","",StatisticsMap) + end, + UpdateStatMap=maps:put("Dead workers",maps:get("Dead workers",MapWithKey)++DeadWorkers,MapWithKey) + end, + [From|[NewCounter]] = re:split(Data, ":", [{return, list}]), - NewStatisticsMap = maps:put(From,NewCounter,StatisticsMap), + NewStatisticsMap = maps:put(From,NewCounter,UpdateStatMap), NewState = State#main_genserver_state{msgCounter = MsgCounter+1,statisticsMap = NewStatisticsMap,statisticsCounter = StatisticsCounter-1}, if StatisticsCounter == 1 -> %% got stats from all entities Statistics = maps:to_list(NewStatisticsMap), S = mapToString(Statistics,[]) , %?LOG_NOTICE("Sending stats: ~p~n",[S]), - case CurrState of + case CurrState of %statistics gets called in idle state by api at the end of the experiment or nerlMonitorin casting at the start of predict idle -> case ets:member(EtsRef,nerlMonitor) of true-> @@ -193,7 +210,7 @@ handle_cast({statistics,Body}, State = #main_genserver_state{myName = MyName, st end, {RouterHost,RouterPort} = nerl_tools:getShortPath(MyName,?API_SERVER_ATOM,NerlnetGraph), nerl_tools:http_request(RouterHost,RouterPort,"statistics", S ++ "|mainServer:" ++integer_to_list(MsgCounter)); - casting -> + casting -> [{_ , MonitorIP , MonitorPort}] = ets:lookup(EtsRef, nerlMonitor), nerl_tools:http_request(MonitorIP,list_to_integer(MonitorPort),"stats", S ++ "|mainServer:" ++integer_to_list(MsgCounter)) end; @@ -244,7 +261,7 @@ handle_cast({clientAck,Body}, State = #main_genserver_state{clientsWaitingList = ClientName = list_to_atom(binary_to_list(Body)), NewWaitingList = WaitingList--[ClientName], if length(NewWaitingList) == 0 -> - case ets:member(EtsRef , nerlMonitor) of + case ets:member(EtsRef , nerlMonitor) of %if NerlMonitor is up, initiate mid experiment statistics true -> gen_server:cast(self(),{statistics , list_to_binary("getStatistics")}); _ -> ok @@ -334,6 +351,7 @@ handle_cast({predictRes,Body}, State = #main_genserver_state{batchSize = BatchSi {noreply, State#main_genserver_state{msgCounter = MsgCounter+1}}; handle_cast({worker_down,Body}, State = #main_genserver_state{msgCounter = MsgCounter,etsRef = EtsRef}) -> + % some worker terminated, print to log and notify NerlMonitor if it is up case ets:member(EtsRef,nerlMonitor) of true-> [{nerlMonitor , IP , Port}] = ets:lookup(EtsRef , nerlMonitor), @@ -345,9 +363,10 @@ handle_cast({worker_down,Body}, State = #main_genserver_state{msgCounter = MsgCo {noreply, State#main_genserver_state{msgCounter = MsgCounter+1,etsRef=EtsRef}}; handle_cast({worker_kill , WorkerName} , State = #main_genserver_state{workersMap = WorkersMap , msgCounter = MsgCounter}) -> - ?LOG_WARNING(?LOG_HEADER++"Worker ~p killed~n",[WorkerName]), + %got a user kill command for a worker from NerlMonitor + ?LOG_WARNING(?LOG_HEADER++"Killing worker ~p ~n",[WorkerName]), WorkerNameAtom=binary_to_term(WorkerName), - ClientName = maps:get(WorkerNameAtom,WorkersMap), % ! Worker Name is binary? + ClientName = maps:get(WorkerNameAtom,WorkersMap), Body = term_to_binary({ClientName, WorkerNameAtom}), nerl_tools:sendHTTP(?MAIN_SERVER_ATOM , ClientName , atom_to_list(worker_kill) , Body), {noreply, State#main_genserver_state{workersMap = WorkersMap , msgCounter = MsgCounter+1}}; diff --git a/src_erl/Communication_Layer/http_Nerlserver/src/utilities_attacher.erl b/src_erl/Communication_Layer/http_Nerlserver/src/utilities_attacher.erl deleted file mode 100644 index c248f141..00000000 --- a/src_erl/Communication_Layer/http_Nerlserver/src/utilities_attacher.erl +++ /dev/null @@ -1,6 +0,0 @@ --module(utilities_attacher). - - -client_utilities_attach(ListOfFunctions, Data) -> - {ClientPid, ClientEtsRef} = Data, - lists:foreach(fun(X, Data) -> X() end,ListOfFunctions). diff --git a/src_erl/Communication_Layer/http_Nerlserver/src/utilities_handler.erl b/src_erl/Communication_Layer/http_Nerlserver/src/utilities_handler.erl index f1c01224..fb1bf309 100644 --- a/src_erl/Communication_Layer/http_Nerlserver/src/utilities_handler.erl +++ b/src_erl/Communication_Layer/http_Nerlserver/src/utilities_handler.erl @@ -5,6 +5,7 @@ -export([init/2]). init(Req0 , [MainServerPid]) -> + %handler for a tool connection req {_,Body,_} = cowboy_req:read_body(Req0), [UtilityName , IP , Port] = binary_to_term(Body), case UtilityName of @@ -15,14 +16,15 @@ init(Req0 , [MainServerPid]) -> Workers = lists:flatten([atom_to_list(X)++"-"++atom_to_list(Y)++"!" || {X , Y} <- WorkersClients]), Reply = Graph ++ "," ++ Workers end, - gen_server:cast(MainServerPid , {saveUtility , {UtilityName , IP , Port}}), %% TODO Add IP , Port to ets in main server record + gen_server:cast(MainServerPid , {saveUtility , {UtilityName , IP , Port}}), Req = cowboy_req:reply(200, #{<<"content-type">> => <<"text/plain">>}, Reply, Req0), {ok, Req, MainServerPid}; -init(Req0 , [Action , MainServerPid]) -> +init(Req0 , [Action , MainServerPid]) -> + %handler for a tool's requested action (not connection) {_,Body,_} = cowboy_req:read_body(Req0), case Action of worker_kill -> diff --git a/src_erl/NerlMonitor/src/nerlMonitor_app.erl b/src_erl/NerlMonitor/src/nerlMonitor_app.erl index 62d6fe7b..6c2f54e3 100644 --- a/src_erl/NerlMonitor/src/nerlMonitor_app.erl +++ b/src_erl/NerlMonitor/src/nerlMonitor_app.erl @@ -13,8 +13,8 @@ -define(UTILNAME,nerlMonitor). -define(IP , "192.168.64.7"). --define(PORT, 8096). %port place holder --define(MSADDRES,"192.168.64.7:8080" ). %place holder +-define(PORT, 8096). +-define(MSADDRES,"192.168.64.7:8080" ). -define(GUI , {'PyrlangProcess' , 'py@127.0.0.1'}). % Erlang node should be long name to communicate with pyrlang node start(_StartType, _StartArgs) -> @@ -63,7 +63,7 @@ mainServerPing(URL,Body)-> end. -recvLoop()-> %% MainServer replies with Nerlnet-Graph when nerlMonitor tool is used +recvLoop()-> %% MainServer replies with Nerlnet-Graph receive {terminate , WorkerName} -> io:format("Got termination message for Worker ~p from GUI~n" , [WorkerName]), From eda9e96ecd52d21d9b614cd3235e9c0795d18c2b Mon Sep 17 00:00:00 2001 From: GuyPerets106 Date: Wed, 16 Aug 2023 11:02:02 +0000 Subject: [PATCH 36/40] [NerlMonitor] Final commit --- .../http_Nerlserver/src/Client/clientStatem.erl | 4 ++-- .../http_Nerlserver/src/MainServer/mainGenserver.erl | 10 +++++----- src_erl/NerlMonitor/src/MonitorGUI.py | 2 ++ 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src_erl/Communication_Layer/http_Nerlserver/src/Client/clientStatem.erl b/src_erl/Communication_Layer/http_Nerlserver/src/Client/clientStatem.erl index 21ae597c..a1cb1674 100644 --- a/src_erl/Communication_Layer/http_Nerlserver/src/Client/clientStatem.erl +++ b/src_erl/Communication_Layer/http_Nerlserver/src/Client/clientStatem.erl @@ -569,9 +569,9 @@ sendStatistics(EtsRef)-> TimingStats = lists:flatten([atom_to_list(WorkerName)++"_Train_Avg_Time="++float_to_list(TotalTime/TotalBatches,[{decimals, 3}])++","||{WorkerName,{_LastTime,TotalBatches,TotalTime}}<-TimingMap]), MissingStats = lists:flatten([atom_to_list(WorkerName)++"_Train_Miss="++integer_to_list(MissCount)++","||{WorkerName,MissCount}<-MissedCounts]), MyStats = atom_to_list(MyName)++"_Msg_Count="++integer_to_list(Counter)++","++atom_to_list(MyName)++"_info_Size="++integer_to_list(InfoSize)++",", - + MyDeadWorkers=lists:flatten([" "++atom_to_list(WorkerName)||WorkerName<-DeadWorkers]), {RouterHost,RouterPort} = nerl_tools:getShortPath(MyName, ?MAIN_SERVER_ATOM, NerlnetGraph), - nerl_tools:http_request(RouterHost,RouterPort,"statistics", list_to_binary(atom_to_list(MyName)++":"++MyStats++MissingStats++lists:droplast(TimingStats)++"#"++DeadWorkers)). + nerl_tools:http_request(RouterHost,RouterPort,"statistics", list_to_binary(atom_to_list(MyName)++":"++MyStats++MissingStats++lists:droplast(TimingStats)++"#"++MyDeadWorkers)). cast_message_to_workers(EtsRef, Msg) -> Workers = ets:lookup_element(EtsRef, workersNames, ?ETS_KV_VAL_IDX), diff --git a/src_erl/Communication_Layer/http_Nerlserver/src/MainServer/mainGenserver.erl b/src_erl/Communication_Layer/http_Nerlserver/src/MainServer/mainGenserver.erl index 021a1c6b..7802900f 100644 --- a/src_erl/Communication_Layer/http_Nerlserver/src/MainServer/mainGenserver.erl +++ b/src_erl/Communication_Layer/http_Nerlserver/src/MainServer/mainGenserver.erl @@ -167,10 +167,9 @@ handle_cast({clientsIdle}, State = #main_genserver_state{state = idle, myName = handle_cast({statistics,Body}, State = #main_genserver_state{myName = MyName, statisticsCounter = StatisticsCounter, nerlnetGraph = NerlnetGraph,statisticsMap = StatisticsMap,msgCounter = MsgCounter , etsRef = EtsRef , state = CurrState}) -> if Body == <<"getStatistics">> -> %% initial message from APIServer, get stats from entities - NewStatisticsMap = getNewStatisticsMap([digraph:vertex(NerlnetGraph,Vertex) || Vertex <- digraph:vertices(NerlnetGraph)--?LIST_OF_SPECIAL_SERVERS]), - [findroutAndsendStatistics(MyName, Name)||{Name,_Counter}<-maps:to_list(StatisticsMap)], - NewState = State#main_genserver_state{msgCounter = MsgCounter+1,statisticsMap = NewStatisticsMap, statisticsCounter = length(maps:to_list(StatisticsMap))}; + [findroutAndsendStatistics(MyName, Name)||{Name,_Counter}<-maps:to_list(NewStatisticsMap)], + NewState = State#main_genserver_state{msgCounter = MsgCounter+1,statisticsMap = NewStatisticsMap, statisticsCounter = length(maps:to_list(NewStatisticsMap))}; Body == <<>> -> io:format("in Statistcs, State has: StatsCount=~p, MsgCount=~p~n", [StatisticsCounter, MsgCounter]), NewState = State; true -> @@ -183,13 +182,13 @@ handle_cast({statistics,Body}, State = #main_genserver_state{myName = MyName, st [Data,[]]-> UpdateStatMap=StatisticsMap;%entity is client without dead workers [Data,DeadWorkers]-> %client with dead workers - case maps:is_key("Dead workers") of + case maps:is_key("Dead workers",StatisticsMap) of true-> MapWithKey=StatisticsMap; %key was already put in false-> MapWithKey=maps:put("Dead workers","",StatisticsMap) end, - UpdateStatMap=maps:put("Dead workers",maps:get("Dead workers",MapWithKey)++DeadWorkers,MapWithKey) + UpdateStatMap=maps:put("Dead workers",maps:get("Dead workers",MapWithKey)++DeadWorkers++" ",MapWithKey) end, [From|[NewCounter]] = re:split(Data, ":", [{return, list}]), @@ -263,6 +262,7 @@ handle_cast({clientAck,Body}, State = #main_genserver_state{clientsWaitingList = if length(NewWaitingList) == 0 -> case ets:member(EtsRef , nerlMonitor) of %if NerlMonitor is up, initiate mid experiment statistics true -> + ?LOG_INFO(?LOG_HEADER ++ "Sending statistics request"), gen_server:cast(self(),{statistics , list_to_binary("getStatistics")}); _ -> ok end, diff --git a/src_erl/NerlMonitor/src/MonitorGUI.py b/src_erl/NerlMonitor/src/MonitorGUI.py index c9032767..27cde3ab 100644 --- a/src_erl/NerlMonitor/src/MonitorGUI.py +++ b/src_erl/NerlMonitor/src/MonitorGUI.py @@ -240,6 +240,8 @@ def ShowStats(CurrentStats): StatsText += f'\t{stat.replace("_info_" , " Info ")}: {CurrentStats[key][stat]} bytes\n' else: StatsText += f'\t{stat.replace("_Msg_" , " Message ")}: {CurrentStats[key][stat]}\n' + elif key == 'Dead workers': + StatsText += f'Dead Workers are:{CurrentStats[key]}\n' else: StatsText += f'{key} Message Count: {CurrentStats[key]}\n' return StatsText From 102a6f2485bdf34f25209388a1dafa1ee70a5d69 Mon Sep 17 00:00:00 2001 From: GuyPerets106 Date: Thu, 17 Aug 2023 12:33:37 +0000 Subject: [PATCH 37/40] [NerlMonitor] Router bug --- .../http_Nerlserver/src/Router/routerGenserver.erl | 1 + 1 file changed, 1 insertion(+) diff --git a/src_erl/Communication_Layer/http_Nerlserver/src/Router/routerGenserver.erl b/src_erl/Communication_Layer/http_Nerlserver/src/Router/routerGenserver.erl index 28333980..43ae44ba 100644 --- a/src_erl/Communication_Layer/http_Nerlserver/src/Router/routerGenserver.erl +++ b/src_erl/Communication_Layer/http_Nerlserver/src/Router/routerGenserver.erl @@ -177,6 +177,7 @@ handle_cast({getStats,_Body}, State = #router_genserver_state{myName = MyName, {noreply, State#router_genserver_state{msgCounter = MsgCounter+1}}; handle_cast({worker_kill , Body} , State = #router_genserver_state{msgCounter = MsgCounter, myName = MyName}) -> + io:format("Body is ~p~n",[binary_to_term(Body)]), {ClientName , _} = binary_to_term(Body), nerl_tools:sendHTTP(MyName, ClientName, "worker_kill", Body), {noreply, State#router_genserver_state{msgCounter = MsgCounter+1}}; From aea179620dff20cda19376455d27095a28f981c3 Mon Sep 17 00:00:00 2001 From: Guy Perets <79912473+GuyPerets106@users.noreply.github.com> Date: Sun, 20 Aug 2023 20:40:21 +0300 Subject: [PATCH 38/40] Update README.md --- src_erl/NerlMonitor/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src_erl/NerlMonitor/README.md b/src_erl/NerlMonitor/README.md index a690a438..f3c185ca 100644 --- a/src_erl/NerlMonitor/README.md +++ b/src_erl/NerlMonitor/README.md @@ -1,7 +1,7 @@ NerlMonitor ===== -An OTP application +An OTP application , Youtube Video Demo: https://youtu.be/X5RHLUTqBWk Build ----- From cb4196e20e4d6ca4838fd944d2a27697ed3f63b9 Mon Sep 17 00:00:00 2001 From: Guy Perets <79912473+GuyPerets106@users.noreply.github.com> Date: Mon, 21 Aug 2023 11:09:56 +0300 Subject: [PATCH 39/40] Update README.md --- src_erl/NerlMonitor/README.md | 40 +++++++++++++++++++++++++++++++---- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/src_erl/NerlMonitor/README.md b/src_erl/NerlMonitor/README.md index f3c185ca..bb7f9928 100644 --- a/src_erl/NerlMonitor/README.md +++ b/src_erl/NerlMonitor/README.md @@ -1,9 +1,41 @@ NerlMonitor ===== +NerlMonitor is an external tool that helps NErlNet users get more knowledge on the experiment flow and also gain the ability to terminate workers mid-experiment. This can help in monitoring your model behavior for different kinds of failures. The app is also used to gain various statistics. + + +# Dependencies +`pip install` these libraries (In addition to the src_py/requirements.txt file): +- NetowrkX +- PySimpleGUI +- PyGraphviz +- Nest-Asyncio + +Also, install **Pyrlang** and **Term** libraries for Python-Erlang communication (follow their instructions **carefully**): + +Pyrlang - https://github.com/Pyrlang/Pyrlang + +Term - https://github.com/Pyrlang/Term + +# Run The App +Run `./NerlnetMonitor.sh` script from a different shell (make sure you're using the same Python virtual environment where you installed all dependencies) + +# Demo +Youtube Video Demo: https://youtu.be/X5RHLUTqBWk + +https://github.com/leondavi/NErlNet/assets/79912473/4e69ad09-3a07-436e-9741-84a64baa4e47 + +When running the app: +1. Start up screen: + +SCR-20230815-lews + +2. Main Server is up: + +SCR-20230815-lghc + +3. Worker termination: + +SCR-20230815-lghc -An OTP application , Youtube Video Demo: https://youtu.be/X5RHLUTqBWk -Build ------ - $ rebar3 compile From 06507ed665799291167df47ce2e2990f4867d515 Mon Sep 17 00:00:00 2001 From: GuyPerets106 Date: Thu, 31 Aug 2023 10:43:27 +0000 Subject: [PATCH 40/40] Small fixes before merge --- .../Architecture/arch_1PCSIM6WorkerMNist.json | 6 +- src_erl/NerlGUI/src/graphScreen.erl | 2 +- src_erl/NerlMonitor/LICENSE.md | 186 ------------------ src_erl/NerlMonitor/src/requirements.txt | 4 + 4 files changed, 8 insertions(+), 190 deletions(-) delete mode 100644 src_erl/NerlMonitor/LICENSE.md create mode 100644 src_erl/NerlMonitor/src/requirements.txt diff --git a/inputJsonFiles/Architecture/arch_1PCSIM6WorkerMNist.json b/inputJsonFiles/Architecture/arch_1PCSIM6WorkerMNist.json index 80b5df8b..72b3c2c0 100755 --- a/inputJsonFiles/Architecture/arch_1PCSIM6WorkerMNist.json +++ b/inputJsonFiles/Architecture/arch_1PCSIM6WorkerMNist.json @@ -6,13 +6,13 @@ }, "devices": [ { - "host": "192.168.64.7", + "host": "192.168.0.108", "entities": "mainServer,c1,c2,c3,c4,c5,c6,s1,r1,r2,r3,r4,r5,r6,apiServer" } ], "apiServer": { - "host": "192.168.64.7", + "host": "192.168.0.108", "port": "8095", "args": "" } @@ -26,7 +26,7 @@ , "mainServer": { - "host": "192.168.64.7", + "host": "192.168.0.108", "port": "8080", "args": "" } diff --git a/src_erl/NerlGUI/src/graphScreen.erl b/src_erl/NerlGUI/src/graphScreen.erl index 9711a61d..97ef9cb2 100644 --- a/src_erl/NerlGUI/src/graphScreen.erl +++ b/src_erl/NerlGUI/src/graphScreen.erl @@ -73,7 +73,7 @@ init([Parent, Gen])-> io:format("got graph: ~p~n", [Devices]), DeviceList = lists:droplast(Devices), - {FileName, G} = gui_tools:makeGraphIMG(22), + {FileName, G} = gui_tools:makeGraphIMG(DeviceList , Edges), mainScreen:updateGraph(Gen, gui_tools:serialize(G)), mainScreen:addInfo(Gen, "updated graph"), diff --git a/src_erl/NerlMonitor/LICENSE.md b/src_erl/NerlMonitor/LICENSE.md deleted file mode 100644 index 8cdc1e6f..00000000 --- a/src_erl/NerlMonitor/LICENSE.md +++ /dev/null @@ -1,186 +0,0 @@ -# Apache License -Version 2.0, January 2004 - -http://www.apache.org/licenses/ - -TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - -## 1. Definitions. - -"License" shall mean the terms and conditions for use, reproduction, and -distribution as defined by Sections 1 through 9 of this document. - -"Licensor" shall mean the copyright owner or entity authorized by the copyright -owner that is granting the License. - -"Legal Entity" shall mean the union of the acting entity and all other entities -that control, are controlled by, or are under common control with that entity. -For the purposes of this definition, "control" means (i) the power, direct or -indirect, to cause the direction or management of such entity, whether by -contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the -outstanding shares, or (iii) beneficial ownership of such entity. - -"You" (or "Your") shall mean an individual or Legal Entity exercising -permissions granted by this License. - -"Source" form shall mean the preferred form for making modifications, including -but not limited to software source code, documentation source, and configuration -files. - -"Object" form shall mean any form resulting from mechanical transformation or -translation of a Source form, including but not limited to compiled object code, -generated documentation, and conversions to other media types. - -"Work" shall mean the work of authorship, whether in Source or Object form, made -available under the License, as indicated by a copyright notice that is included -in or attached to the work (an example is provided in the Appendix below). - -"Derivative Works" shall mean any work, whether in Source or Object form, that -is based on (or derived from) the Work and for which the editorial revisions, -annotations, elaborations, or other modifications represent, as a whole, an -original work of authorship. For the purposes of this License, Derivative Works -shall not include works that remain separable from, or merely link (or bind by -name) to the interfaces of, the Work and Derivative Works thereof. - -"Contribution" shall mean any work of authorship, including the original version -of the Work and any modifications or additions to that Work or Derivative Works -thereof, that is intentionally submitted to Licensor for inclusion in the Work -by the copyright owner or by an individual or Legal Entity authorized to submit -on behalf of the copyright owner. For the purposes of this definition, -"submitted" means any form of electronic, verbal, or written communication sent -to the Licensor or its representatives, including but not limited to -communication on electronic mailing lists, source code control systems, and -issue tracking systems that are managed by, or on behalf of, the Licensor for -the purpose of discussing and improving the Work, but excluding communication -that is conspicuously marked or otherwise designated in writing by the copyright -owner as "Not a Contribution." - -"Contributor" shall mean Licensor and any individual or Legal Entity on behalf -of whom a Contribution has been received by Licensor and subsequently -incorporated within the Work. - -## 2. Grant of Copyright License. - -Subject to the terms and conditions of this License, each Contributor hereby -grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, -irrevocable copyright license to reproduce, prepare Derivative Works of, -publicly display, publicly perform, sublicense, and distribute the Work and such -Derivative Works in Source or Object form. - -## 3. Grant of Patent License. - -Subject to the terms and conditions of this License, each Contributor hereby -grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, -irrevocable (except as stated in this section) patent license to make, have -made, use, offer to sell, sell, import, and otherwise transfer the Work, where -such license applies only to those patent claims licensable by such Contributor -that are necessarily infringed by their Contribution(s) alone or by combination -of their Contribution(s) with the Work to which such Contribution(s) was -submitted. If You institute patent litigation against any entity (including a -cross-claim or counterclaim in a lawsuit) alleging that the Work or a -Contribution incorporated within the Work constitutes direct or contributory -patent infringement, then any patent licenses granted to You under this License -for that Work shall terminate as of the date such litigation is filed. - -## 4. Redistribution. - -You may reproduce and distribute copies of the Work or Derivative Works thereof -in any medium, with or without modifications, and in Source or Object form, -provided that You meet the following conditions: - -1. You must give any other recipients of the Work or Derivative Works a copy of - this License; and - -2. You must cause any modified files to carry prominent notices stating that - You changed the files; and - -3. You must retain, in the Source form of any Derivative Works that You - distribute, all copyright, patent, trademark, and attribution notices from - the Source form of the Work, excluding those notices that do not pertain to - any part of the Derivative Works; and - -4. If the Work includes a "NOTICE" text file as part of its distribution, then - any Derivative Works that You distribute must include a readable copy of the - attribution notices contained within such NOTICE file, excluding those - notices that do not pertain to any part of the Derivative Works, in at least - one of the following places: within a NOTICE text file distributed as part - of the Derivative Works; within the Source form or documentation, if - provided along with the Derivative Works; or, within a display generated by - the Derivative Works, if and wherever such third-party notices normally - appear. The contents of the NOTICE file are for informational purposes only - and do not modify the License. You may add Your own attribution notices - within Derivative Works that You distribute, alongside or as an addendum to - the NOTICE text from the Work, provided that such additional attribution - notices cannot be construed as modifying the License. - -You may add Your own copyright statement to Your modifications and may provide -additional or different license terms and conditions for use, reproduction, or -distribution of Your modifications, or for any such Derivative Works as a whole, -provided Your use, reproduction, and distribution of the Work otherwise complies -with the conditions stated in this License. - -## 5. Submission of Contributions. - -Unless You explicitly state otherwise, any Contribution intentionally submitted -for inclusion in the Work by You to the Licensor shall be under the terms and -conditions of this License, without any additional terms or conditions. -Notwithstanding the above, nothing herein shall supersede or modify the terms of -any separate license agreement you may have executed with Licensor regarding -such Contributions. - -## 6. Trademarks. - -This License does not grant permission to use the trade names, trademarks, -service marks, or product names of the Licensor, except as required for -reasonable and customary use in describing the origin of the Work and -reproducing the content of the NOTICE file. - -## 7. Disclaimer of Warranty. - -Unless required by applicable law or agreed to in writing, Licensor provides the -Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, -including, without limitation, any warranties or conditions of TITLE, NON- -INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are -solely responsible for determining the appropriateness of using or -redistributing the Work and assume any risks associated with Your exercise of -permissions under this License. - -## 8. Limitation of Liability. - -In no event and under no legal theory, whether in tort (including negligence), -contract, or otherwise, unless required by applicable law (such as deliberate -and grossly negligent acts) or agreed to in writing, shall any Contributor be -liable to You for damages, including any direct, indirect, special, incidental, -or consequential damages of any character arising as a result of this License or -out of the use or inability to use the Work (including but not limited to -damages for loss of goodwill, work stoppage, computer failure or malfunction, or -any and all other commercial damages or losses), even if such Contributor has -been advised of the possibility of such damages. - -## 9. Accepting Warranty or Additional Liability. - -While redistributing the Work or Derivative Works thereof, You may choose to -offer, and charge a fee for, acceptance of support, warranty, indemnity, or -other liability obligations and/or rights consistent with this License. However, -in accepting such obligations, You may act only on Your own behalf and on Your -sole responsibility, not on behalf of any other Contributor, and only if You -agree to indemnify, defend, and hold each Contributor harmless for any liability -incurred by, or claims asserted against, such Contributor by reason of your -accepting any such warranty or additional liability. - -END OF TERMS AND CONDITIONS - -Copyright 2023, GuyPerets106 . - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. \ No newline at end of file diff --git a/src_erl/NerlMonitor/src/requirements.txt b/src_erl/NerlMonitor/src/requirements.txt new file mode 100644 index 00000000..b4bc4464 --- /dev/null +++ b/src_erl/NerlMonitor/src/requirements.txt @@ -0,0 +1,4 @@ +matplotlib==3.7.1 +nest-asyncio==1.5.7 +networkx==3.1 +PySimpleGUI==4.60.5