Allow programmatically starting additional TorQ processes from within…

… q (#394)

* Added launchprocess.sh and relevant q wrapper function for launching processes in the cloud

* Fixed typos, removed unnecessary env var code, etc. according to feedback

* Added defaults in launchprocess.q using .Q.def and functionality to handle exit codes from launchprocess.sh to log messages/errors

* Fixed behaviour to accept any number of custom arguments to the command line

* Simplified wrapper functions according to feedback and added killprocess.sh and its corresponding wrapper

* Removed files whose functionality has now been moved into code/common/cloudutils.q

* Made possible change of if-else, for review

* Simplied previous commit

* Added unit testing

* Improved method of finding process port in unit tests

* Implemented more TorQ functionality in unit tests

* Tweaked comments and some final checks in unit tests

* Added documentation for cloudutils.q

* Corrected formatting of docs

* Modify name of functions for broader uses

* Updated formatting in utilities.md

bin/killprocess.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+#script to terminate a specified TorQ process (standard or custom)
+#For now the exit codes are: 
+#1:failed to terminate process
+#2:process wasnot running
+#3:process not found
+
+#only argument is the process name
+procname=$1
+
+#fn to check if a process is running, returns a non-zero search result for success, returns null for failure
+findproc()  {
+    pgrep -lf "$1" -u "$USER" | grep -ow "q"
+}
+
+#make sure process is running before attempting to kill it
+if [[ -z $(findproc "$procname") ]]; then
+    exit 2
+else
+    #search for PID based on process name
+    pr_id=$( ps -ef -u $USER | grep "$procname" | awk -v procname="$procname" '{if ($15 == procname)  print $2 }' )
+    echo $procname
+    echo $pr_id
+    #only if this search returns a result do we kill the process
+    if [[ $pr_id ]]; then
+        kill "$pr_id"
+
+        #check that process has been killed
+        temp_exit=$?
+        if [ $temp_exit -eq 0 ]; then
+            exit 0
+        else
+            exit 1
+        fi
+    else
+        exit 3
+    fi
+fi
+

bin/launchprocess.sh

@@ -0,0 +1,79 @@
+#!/bin/bash
+
+#script to launch a specified TorQ process (standard or custom)
+#For now the exit codes are: 
+#1:process failed to start
+#2:process had already been running
+
+#all defaults have been assigned in the q wrspper script: cloudutils.q
+#for each cmd line argument, extract the argument and assign to relevant variable
+cnt=0
+extra_flag_cnt=0
+IFS='-'
+read -ra arg_arr <<< "$@"
+for arg in "${arg_arr[@]}"
+do
+    if [ "$cnt" -gt 0 ]; then
+    IFS=' '
+    read -ra sub_arg_arr <<< "$arg"
+    case ${sub_arg_arr[0]} in 
+        procname )
+	    procname=${sub_arg_arr[1]}
+	    ;;
+	proctype )
+            proctype=${sub_arg_arr[1]}
+	    ;;
+	U )
+	    U_arg=${sub_arg_arr[1]}
+	    ;;
+	localtime )
+	    localtime_arg=${sub_arg_arr[1]}
+	    ;;
+	qcmd )
+	    qcmd_arg=${sub_arg_arr[1]}
+	    ;;
+	p )
+	    p_arg=${sub_arg_arr[1]}
+	    ;;
+	* )
+	    extra_flag_cnt=$(($extra_flag_cnt + 1))
+      	    ;;
+    esac
+    fi
+    cnt=$((cnt+1))	
+done
+
+#to deal with any number of extra flags and args, take the last (extra_flag_cnt) arguments from the arg_arr 
+extra_arg_arr=" "
+i=0
+a=$((extra_flag_cnt * -1))
+while [ $i -lt "$extra_flag_cnt" ]
+do 
+    extra_arg_arr+="-"${arg_arr[$(( $a+i ))]}
+    i=$(($i + 1))
+done
+
+#define the startline
+sline="$qcmd_arg ${TORQHOME}/torq.q -stackid ${KDBBASEPORT} -proctype $proctype -procname $procname -U $U_arg -localtime $localtime_arg -p $p_arg $extra_arg_arr"
+
+#fn to check if a process is running, returns a non-zero search result for success, returns null for failure
+findproc()  {
+    pgrep -lf "$1" -u "$USER" | grep -ow "$qcmd_arg"
+}
+
+#check process isn't already running before attempting to start it
+if [[ -z $(findproc "$procname") ]]; then
+    #run process in background and redirect input & output 
+    eval "nohup $sline </dev/null >${KDBLOG}/torq${procname}.txt 2>&1 &"
+
+    #check exit code of nohup $sline to make sure process is starting successfully
+    temp_exit=$?
+    if [ $temp_exit -eq 0 ]; then
+        exit 0
+    else
+        exit 1
+    fi
+else
+    exit 2
+fi
+

code/common/bglaunchutils.q

@@ -0,0 +1,46 @@
+\d .sys
+
+/a function to execute system commands and return a log message depending on the resulting exit code:
+/used for both launchprocess.sh and killprocess.sh
+syscomm:{[params;cmd]
+    /params is (i) a dictionary if syscomm has been called by launch, and (ii) a string if it has been called by killproc
+    /cmd is the command to be executed, as a string
+
+    prevexitcode:first "I"$system cmd,"; echo $?"; 
+    id:$[lok:99h=type params;`launchprocess;`killprocess];
+    pname:$[lok;params[`procname];params];
+    $[0=prevexitcode;
+        .lg.o[id;"Successful execution: ",$[lok;"Starting ";"Terminating "],pname];
+      1=prevexitcode;
+        .lg.e[id;"Failed to ",$[lok;"start ";"terminate "],pname];
+      2=prevexitcode;
+        .lg.e[id;pname,$[lok;" already ";" not "],"running"];
+      3=prevexitcode;
+        .lg.e[id;pname," not found"];
+        .lg.e[id;"Unknown error encountered"]
+     ] 
+ }
+
+/function which lets us call launchprocess.sh from inside a TorQ process
+/it takes a dictionary of parameters which will be passed to launchprocess.sh, i.e "-procname rdb1 -proctype rdb" etc.
+bglaunch:{[params]
+    /exit immediately if process name and type aren't provided
+    if[not all `procname`proctype in key params;
+        .lg.e[`launchprocess;"Process name and type must be provided"];
+        :()];
+
+    /set default values with .Q.def and string the result:
+    pass_arg:first `$.Q.opt[.z.X]`U;
+    if[2>count pass_arg;pass_arg:`$getenv[`KDBAPPCONFIG],"/passwords/accesslist.txt"];
+    deflt:`procname`proctype`U`localtime`p`qcmd!(`;`;pass_arg;first string .proc.localtime;`$"0W";"q");
+    params:string each .Q.def[deflt] params;
+
+    /format the params dictionary as a series of command line args
+    f_args:{"-",string[x]," ",y}'[key params;value params];
+    sline:"bash ",getenv[`TORQHOME],"/bin/launchprocess.sh "," " sv f_args;
+    syscomm[params;] sline}
+
+/this function calls killprocess.sh from within a TorQ process, 
+/takes a single parameter, a string procname
+bgkill:{[procname] syscomm[procname;] "bash ",getenv[`TORQHOME],"/bin/killprocess.sh ",procname}
+

docs/utilities.md

@@ -1316,6 +1316,46 @@ this, we’ve modified u.q. This can be turned off by setting .u.broadcast
 to false. It is enabled by default, but will only override default
 publishing if the kdb+ version being used is 3.4 or after.
 
+## bglaunchutils.q 
+
+The background launch utilities allow other processes to be programmatically launched, or
+terminated, from inside a TorQ process. The two main functions here are
+.sys.bglaunch and .sys.bgkill.
+
+.sys.bglaunch is the q function which takes a dictionary of input parameters which
+are then passed to a bash script. This bash script functions like torq.sh in how
+it starts processes. It is important to note that the background launch utilities
+are only supported on Linux as a result.
+
+```q
+
+q)input:`procname`proctype`localtime`p!("hdb1";"hdb";"0";"1234");
+
+```
+
+The input parameter dictionary, as shown above, should contain symbols as keys
+and strings as values. Any standard or custom TorQ process can be launched
+using .sys.bglaunch, and as such the function can accept any desired command line
+parameters in the input dictionary. The minimum required are `` `procname`` and
+`` `proctype``. In the case that only these two are used the other arguments will be
+given default values.
+
+|    Parameter   |                                                    Default Value                                                          |
+| :------------: | :-----------------------------------------------------------------------------------------------------------------------: |
+|      U         | The password file used for the parent launching process, if none exists ${KDBAPPCONFIG}/passwords/accesslist.txt is used  |
+|  localtime     |                                 The .proc.localtime value of the parent launching process                                 |
+|      p         |                                             0W - a random port will be chosen                                             |
+|     qcmd       |                                                          q                                                                |
+
+The .sys.bgkill function is passed a single argument: the `` `procname`` of the process to
+be terminated, as a string.
+
+If the default value of "0W" is used for the port, a random port will be chosen
+on which to launch the process. In this case the process will neeed to register
+itself with discovery in order for other processes to be able to connect to it.
+This is a standard behaviour for TorQ processes on startup (for more information
+see [Connection Management](http://aquaqanalytics.github.io/TorQ/conn/#connections)).
+
 <a name="api"></a>
 
 Full API
@@ -1588,4 +1628,5 @@ One final important variable is ```.grafana.del```, this dictates the delimeter
 between options in the drop down menus. This has significant repercussions if 
 one of your columns includes full stops, eg. email adresses. As a result we have 
 left this as definable so that the user can alter this to a non-disruptive value 
-for their data eg./.
+for their data eg./.
+

tests/bglaunchprocess/process.csv

@@ -0,0 +1,2 @@
+host,port,proctype,procname,U,localtime,g,T,w,load,startwithall,extras,qcmd
+localhost,{KDBBASEPORT}+1,discovery,discovery1,,1,0,,,${KDBCODE}/processes/discovery.q,1,,q

tests/bglaunchprocess/run.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+#path to test directory
+testpath=${KDBTESTS}/bglaunchprocess/
+
+#start procs
+${TORQHOME}/torq.sh start all -csv ${testpath}/process.csv
+
+#Start test proc
+/usr/bin/rlwrap q ${TORQHOME}/torq.q \
+	-proctype test -procname test1 \
+	-test ${testpath} \
+	-load ${TORQHOME}/code/common/bglaunchutils.q ${testpath}/settings.q     \
+	-procfile ${testpath}/process.csv -debug
+
+#Shut down procs
+${TORQHOME}/torq.sh stop all -csv ${testpath}/process.csv

tests/bglaunchprocess/settings.q

@@ -0,0 +1,7 @@
+/variables and fns to be used during the unit tests:
+/let the test port_no be 7124
+
+input1:`procname`proctype`U`localtime`p`T`g`w`qcmd`custom`load!("test2";"test";"${KDBAPPCONFIG}/passwords/accesslist.txt";"0";"7124";"180";"1";"1000";"q";"custom_arg";"${TORQHOME}/tests/bglaunchprocess/settings.q");
+input2:`procname`proctype`load!("test3";"test";"${TORQHOME}/tests/bglaunchprocess/settings.q");
+.servers.startup[];
+

tests/bglaunchprocess/test.csv

@@ -0,0 +1,27 @@
+action,ms,bytes,lang,code,repeat,minver,comment
+comment,0,,"Test 1 - Launch and then kill a process with all possible cmd line args"
+run,0,0,q,.sys.bglaunch input1,,,"Start a new process with extra command line args"
+run,0,0,q,.os.sleep 2,,,"Wait for process to start"
+run,0,0,q,h:hopen `::7124:admin:admin,,,"Attempt to connect to the new process"
+true,0,0,q,`test2=@[h;".proc.procname";`],,,"Check that connection has been made"
+true,0,0,q,11011111010b~value input1 in' value key[input1]#.Q.opt (h)".z.X",,,"Check that the cmd line args are as expected"
+run,0,0,q,pid1:h".z.i",,,"Find the PID of the launched process"
+run,0,0,q,.sys.bgkill "test2",,,"Kill the process"
+run,0,0,q,.os.sleep 2,,,"Wait for process to be terminated"
+true,0,0,q,@[h;".proc.procname";{1b}],,,"Check that the handle to the process is no longer open"
+true,0,0,q,not pid1 in "I"$1 _ .proc.sys "ps -u $USER | awk '{print $1}'",,,"Check that the PID of the old process is no longer in use"
+
+comment,0,,"Test 2 - Launch and then kill a process with no extra cmd line args, to test defaulting"
+run,0,0,q,.sys.bglaunch input2,,,"Start a new process with no extra command line args, hence relying on defaulting in cloudutils.q"
+run,0,0,q,.os.sleep 2,,,"Wait for process to start"
+run,0,0,q,proc_port:first exec hpup from .servers.querydiscovery`test where procname=`test3,,,"Find process port from discovery services "
+run,0,0,q,"h2:hopen `$string[proc_port],"":admin:admin""",,,"Attempt to connect to the process"
+true,0,0,q,`test3=@[h2;".proc.procname";`],,,"Check that connection has been made"
+true,0,0,q,110b~(value input2)in' value key[input2]#.Q.opt h2".z.X",,,"Check that correct process name and type are present in the cmd line args of the process"
+true,0,0,q,"all (getenv[`KDBAPPCONFIG],""/passwords/accesslist.txt"";string .proc.localtime;""0W"")in' value `U`localtime`p#.Q.opt h2"".z.X""",,,"Check that the expected default parameters for the U,localtime and p flags are present in the cmd line args"
+run,0,0,q,pid2:h2".z.i",,,"Find the PID of the launched process"
+run,0,0,q,.sys.bgkill "test3",,,"Kill the process"
+run,0,0,q,.os.sleep 2,,,"Wait for process to be terminated"
+true,0,0,q,@[h2;".proc.procname";{1b}],,,"Check that the handle to the process is no longer open"
+true,0,0,q,not pid2 in "I"$1 _ .proc.sys "ps -u $USER | awk '{print $1}'",,,"Check that the PID of the old process is no longer in use"
+