diff --git a/zmeventnotification.ini b/zmeventnotification.ini index 04e2741e..791f3ac8 100644 --- a/zmeventnotification.ini +++ b/zmeventnotification.ini @@ -229,6 +229,23 @@ use_hooks = yes # NOTE: This entire section is only valid if use_hooks is yes above +# When a hook is invoked, the ES forks a child. If you are in a situation +# where your motion sensititivy in ZM is not set properly, you may land up +# triggering hundreds of child processes of zm_detect that may potentially +# crash your system. Note that there are global locks around the ML code which +# are controlled by xxx_max_processes in the objectconfig/mlapiconfig.files +# which will avoid parallel running of models. But this is if you are facing issues +# by the simple fact that too many zm_detect processes are forked (which will apply +# whether you use mlapi or not). While I do feel the core issue is that you need +# to fix your ZM sensitivity, this parameter helps control. + +# NOTE: When you put in value for this, any hooks that attempt to kick off +# beyond this limit will simply be ignored. There is no queueing. + +# A value of 0 (default) means there are no limits +max_parallel_hooks=0 + + # Shell script name here to be called every time an alarm is detected # the script will get passed $1=alarmEventID, $2=alarmMonitorId # $3 monitor Name, $4 alarm cause diff --git a/zmeventnotification.pl b/zmeventnotification.pl index 0a8fd6ec..687c3851 100755 --- a/zmeventnotification.pl +++ b/zmeventnotification.pl @@ -138,7 +138,9 @@ '/var/lib/zmeventnotification/misc/escontrol_interface.dat', DEFAULT_FCM_DATE_FORMAT => '%I:%M %p, %d-%b', DEFAULT_FCM_ANDROID_PRIORITY=>'high', - DEFAULT_MAX_FCM_PER_MONTH_PER_TOKEN => 8000 + DEFAULT_MAX_FCM_PER_MONTH_PER_TOKEN => 8000, + + DEFAULT_MAX_PARALLEL_HOOKS => 0, }; # connection state @@ -173,6 +175,7 @@ my $es_terminate = 0; my $child_forks = 0; # Global tracker of active children +my $parallel_hooks = 0; # Global tracker for active hooks my $total_forks = 0; # Global tracker of all forks since start # Declare options. @@ -279,6 +282,8 @@ my %fcm_tokens_map; +my $max_parallel_hooks= 0; + my %monitors = (); my %active_events = (); my $monitor_reload_time = 0; @@ -696,6 +701,12 @@ sub loadEsConfigSettings { DEFAULT_EVENT_END_NOTIFY_ON_HOOK_SUCCESS ); + $max_parallel_hooks = config_get_val( + $config, 'hook', + 'max_parallel_hooks', + DEFAULT_MAX_PARALLEL_HOOKS + ); + # get channels and convert to hash %event_start_notify_on_hook_fail = map { $_ => 1 } @@ -807,10 +818,11 @@ sub print_config { Monitor rules JSON file................${\(value_or_undefined($es_rules_file))} Use Hooks............................. ${\(yes_or_no($use_hooks))} +Max Parallel Hooks.................... ${\(value_or_undefined($max_parallel_hooks))} Hook Script on Event Start ........... ${\(value_or_undefined($event_start_hook))} User Script on Event Start.............${\(value_or_undefined($event_start_hook_notify_userscript))} Hook Script on Event End.............. ${\(value_or_undefined($event_end_hook))} -User Script on Event End.............${\(value_or_undefined($event_end_hook_notify_userscript))} +User Script on Event End...............${\(value_or_undefined($event_end_hook_notify_userscript))} Hook Skipped monitors................. ${\(value_or_undefined($hook_skip_monitors))} Notify on Event Start (hook success).. ${\(value_or_undefined($event_start_notify_on_hook_success))} @@ -2374,6 +2386,17 @@ sub processJobs { delete( $active_events{$mid}->{$eid} ); $child_forks--; } + elsif ( $job eq 'update_parallel_hooks' ) { + if ($msg eq "add") { + $parallel_hooks++; + } + elsif ($msg eq "del") { + $parallel_hooks--; + } + else { + printError ("Parallel hooks update: command not understood: $msg"); + } + } elsif ( $job eq 'mqtt_publish' ) { my ( $id, $topic, $payload ) = split( '--SPLIT--', $msg ); printDebug( "Job: MQTT Publish on topic: $topic", 2 ); @@ -3861,7 +3884,10 @@ sub processNewAlarmsInFork { if ( $cmd =~ /^(.*)$/ ) { $cmd = $1; } + print WRITER "update_parallel_hooks--TYPE--add\n"; my $res = `$cmd`; + print WRITER "update_parallel_hooks--TYPE--del\n"; + chomp($res); my ( $resTxt, $resJsonString ) = parseDetectResults($res); $hookResult = $? >> 8; @@ -4093,7 +4119,11 @@ sub processNewAlarmsInFork { if ( $cmd =~ /^(.*)$/ ) { $cmd = $1; } + + print WRITER "update_parallel_hooks--TYPE--add\n"; my $res = `$cmd`; + print WRITER "update_parallel_hooks--TYPE--del\n"; + chomp($res); my ( $resTxt, $resJsonString ) = parseDetectResults($res); $hookResult = $? >> 8; @@ -4420,7 +4450,7 @@ sub initSocketServer { exit(0); } my $elapsed_time_min = ceil((time() - $es_start_time)/60); - printDebug( "----------> Tick START (active forks:$child_forks, total forks:$total_forks, running for:$elapsed_time_min min)<--------------", 2 ); + printDebug( "----------> Tick START (active forks:$child_forks, total forks:$total_forks, active hooks: $parallel_hooks running for:$elapsed_time_min min)<--------------", 2 ); if ( $restart_interval && ( ( time() - $es_start_time ) > $restart_interval ) ) { @@ -4446,7 +4476,7 @@ sub initSocketServer { checkConnection(); processJobs(); - printDebug( "There are $child_forks active child forks...", 2 ); + printDebug( "There are $child_forks active child forks & $parallel_hooks zm_detect processes running...", 2 ); my (@newEvents) = checkNewEvents(); #print Dumper(\@newEvents); @@ -4468,6 +4498,10 @@ sub initSocketServer { } foreach (@newEvents) { + if (($parallel_hooks >= $max_parallel_hooks) && ($max_parallel_hooks != 0)) { + printError("There are $parallel_hooks hooks running as of now. This exceeds your set limit of max_parallel_hooks=$max_parallel_hooks. Ignoring this event. Either increase your max_parallel_hooks value, or, adjust your ZM motion sensitivity "); + return; + } $child_forks++; $total_forks++; if ($cpid = fork() ) { @@ -4523,7 +4557,7 @@ sub initSocketServer { } check_for_duplicate_token(); - printDebug( "---------->Tick END (active forks:$child_forks, total forks:$total_forks)<--------------", 2 ); + printDebug( "---------->Tick END (active forks:$child_forks, total forks:$total_forks, active hooks: $parallel_hooks)<--------------", 2 ); }, # called when a new connection comes in