Added support for running simultaneous independent microservices.

This involved:
 - configurable socket names (via launch, webserver, and host)
 - unique logs
 - unique killme scripts
 - independent out and live directories

Author: stevehoover

Diff for: .gitignore

@@ -2,7 +2,7 @@
 **/webserver/*.pyc
 **/webserver/ec2_time_bombs/
 **/apps/*/live/
-**/apps/*/build/webserver.log
+**/apps/*/build/log/
 apps/mandelbrot/build/video
 apps/mandelbrot/build/cast
 **.tfstate

Diff for: doc/GettingStarted.md

@@ -92,7 +92,7 @@ make copy_app APP_NAME=toy
 
 ### Web Client Application
 
-The web client is defined by the HTML, CSS, and JS content in `<repo>/apps/toy/client/`. In this case `<repo>/apps/toy/client/html/index.html` simply links to `<repo>/framework/client/html/testbench.html` to utilize the default test bench provided by the framework that you accessed for `vadd`. If you would like to modify the web client code, you can copy `<repo>/framework/client/*/testbench.*` into `<repo>/apps/toy/client` as a simple starting point. To develop a real application, you may have your own thoughts about the framework you would like to use. You can use what you like. Develop in a separate repo, or keep it in the same repo so it is consistently version controlled. If you would like `make launch` to start your web serer, provide `LAUNCH_W=<cmd>` in the application `Makefile`.
+The web client is defined by the HTML, CSS, and JS content in `<repo>/apps/toy/client/`. In this case `<repo>/apps/toy/client/html/index.html` simply links to `<repo>/framework/client/html/testbench.html` to utilize the default test bench provided by the framework that you accessed for `vadd`. If you would like to modify the web client code, you can copy `<repo>/framework/client/*/testbench.*` into `<repo>/apps/toy/client` as a simple starting point. To develop a real application, you may have your own thoughts about the framework you would like to use. You can use what you like. Develop in a separate repo, or keep it in the same repo so it is consistently version controlled.
 
 
 ### Custom Kernel

Diff for: framework/build/Makefile

@@ -105,6 +105,8 @@
 #     WAVES=[true] or default to false behavior. True to generate waveforms. (xocc )
 #     VALGRIND=[true] or default to false behavior. True to use Valgrind to identify memory leaks in the host application.
 #     NOHUP=true: Can be used with 'launch' target to launch in background and stay running after the shell exits. (This is implied by 'make live').
+#     LAUNCH_ID: Used by launch, live, and dead targets. If unassigned, these targets assume a single running microservice.
+#                A unique identifier can be provided in this variable for these targets to enable unique instances.
 #   Eg:
 #     make host TARGET=hw_emu
 #
@@ -348,7 +350,7 @@ endif
 endif
 
 WEBSERVER_PY ?= $(shell if [[ -e "../webserver/$(KERNEL_NAME)_server.py" ]]; then echo "../webserver/$(KERNEL_NAME)_server.py"; else echo "$(FRAMEWORK_DIR)/webserver/default_server.py"; fi)
-LAUNCH_W ?=python3 $(WEBSERVER_PY) --port=<<PORT>> $(WEBSERVER_ARGS)
+LAUNCH_W ?=python3 $(WEBSERVER_PY) --port=<<PORT>> --socket=<<SOCKET>> $(WEBSERVER_ARGS)
 
 
 
@@ -439,15 +441,6 @@ BUILD_DIR_NAME = prebuilt
 OUT_DIR_NAME = prebuilt_out
 endif
 
-# 'live' target.
-ifeq ($(MAKECMDGOALS),live)
-NOHUP=true
-PORT=80
-HOST_EXE_PATH=../live/$(BUILD_TARGET)/$(HOST_EXE)
-else
-HOST_EXE_PATH=$(BUILD_DIR)/$(HOST_EXE)
-endif
-
 #Assign DEST_DIR as <BUILD_TARGET>/<TARGET_DEVICE>, or just <BUILD_TARGET> for sw build (w/ no target device).
 ifneq ($(USE_XILINX),true)
 DEST_DIR=../$(OUT_DIR_NAME)/$(BUILD_TARGET)
@@ -471,16 +464,61 @@ $(error XILINX_XRT is not set. Please source the SDx settings64.{csh,sh} first)
 endif
 endif
 
-# If project has a launch command, use it, otherwise, use launch from framework.
-LAUNCH_CMD_PARTIAL :=$(shell if [[ -e ./launch ]]; then echo ./launch; else echo $(FRAMEWORK_DIR)/build/launch; fi)
-ifdef LAUNCH_W
-LAUNCH_CMD_PARTIAL :=$(LAUNCH_CMD_PARTIAL) -w '$(LAUNCH_W)'
+
+# For instance-specific targets (launch/live/dead)
+
+# Prevent conflicts between launch and live targets.
+ifeq ($(MAKECMDGOALS), launch)
+DEFAULT_SOCKET=SOCKET
+DEFAULT_KILLME=./kill_launch
+else
+DEFAULT_SOCKET=LIVE_SOCKET
+DEFAULT_KILLME=./killme
+endif
+
+# Interpret the LAUNCH_ID
+# KILLME: The name of the killme file for this microservice.
+# WEBSERVER_LOG: The path to the log file for this microservice (./log/*).
+# LAUNCH_DIR: directory from which microservice is launched and where is creates files.
+ifdef LAUNCH_ID
+LAUNCH_LOCAL_DIR=id-$(LAUNCH_ID)/
+LAUNCH_DIR=../live/$(BUILD_TARGET)/id-$(LAUNCH_ID)
+KILLME=./killme-$(LAUNCH_ID)
+WEBSERVER_LOG=log/id-$(LAUNCH_ID).log
+LAUNCH_ID_ARG_STR=$(SPACE)LAUNCH_ID=$(LAUNCH_ID)
+SOCKET=SOCKET-$(LAUNCH_ID)
+else
+LAUNCH_ID=.
+LAUNCH_LOCAL_DIR=
+LAUNCH_DIR=../live/$(BUILD_TARGET)
+KILLME=$(DEFAULT_KILLME)
+WEBSERVER_LOG=log/webserver.log
+LAUNCH_ID_ARG_STR=LAUNCH_ID=
+SOCKET=$(DEFAULT_SOCKET)
+endif
+
+# 'live' target.
+ifeq ($(MAKECMDGOALS),live)
+NOHUP=true
+PORT=80
+HOST_EXE_PATH=$(LAUNCH_DIR)/$(HOST_EXE)
+else
+HOST_EXE_PATH=$(BUILD_DIR)/$(HOST_EXE)
+endif
+
+ifeq ($(MAKECMDGOALS),dead)
+ALREADY_DEAD=else echo "There doesn't appear to be anything to kill.";
 endif
+
+
+# If project has a launch command, use it, otherwise, use launch from framework.
+LAUNCH_CMD_PARTIAL :=$(shell if [[ -e ./launch ]]; then echo ./launch; else echo $(FRAMEWORK_DIR)/build/launch; fi) -w '$(LAUNCH_W)'
+LAUNCH_ARGS=-p $(PORT) -s $(SOCKET) -k '$(KILLME)' $(BUILD_TARGET) '$(HOST_CMD)'
 ifdef NOHUP
-# Run in background to continue if the shell exits; log to webserver.log and cut off stdin to detach from the launching process's stdin (so ssh/etc can exit).
-LAUNCH_CMD=nohup $(LAUNCH_CMD_PARTIAL) -p $(PORT) $(BUILD_TARGET) '$(HOST_CMD)' &> webserver.log < /dev/null &
+# Run in background to continue if the shell exits; log output and cut off stdin to detach from the launching process's stdin (so ssh/etc can exit).
+LAUNCH_CMD=nohup $(LAUNCH_CMD_PARTIAL) $(LAUNCH_ARGS) &> $(WEBSERVER_LOG) < /dev/null &
 else
-LAUNCH_CMD=$(LAUNCH_CMD_PARTIAL) -p $(PORT) $(BUILD_TARGET) '$(HOST_CMD)'
+LAUNCH_CMD=$(LAUNCH_CMD_PARTIAL) $(LAUNCH_ARGS)
 endif
 
 
@@ -503,17 +541,18 @@ else
 VALGRIND_PREFIX=
 endif
 
+HOST_ARGS=-s $(SOCKET)
 ifneq ($(USE_XILINX),true)
 BUILD_TARGETS=$(BUILD_DIR)/$(HOST_EXE)
-HOST_CMD=$(VALGRIND_PREFIX) $(HOST_EXE_PATH)
+HOST_CMD=$(VALGRIND_PREFIX) $(HOST_EXE_PATH) $(HOST_ARGS)
 endif
 ifeq ($(BUILD_TARGET),hw_emu)
 BUILD_TARGETS=$(BUILD_DIR)/$(HOST_EXE) $(HOST_XCLBIN)
-HOST_CMD=export XCL_EMULATION_MODE=$(BUILD_TARGET) && $(XILINX_SDX)/bin/emconfigutil --od $(DEST_DIR) --nd 1  --platform $(AWS_PLATFORM) && $(VALGRIND_PREFIX) $(HOST_EXE_PATH) $(HOST_XCLBIN)
+HOST_CMD=export XCL_EMULATION_MODE=$(BUILD_TARGET) && $(XILINX_SDX)/bin/emconfigutil --od $(DEST_DIR) --nd 1  --platform $(AWS_PLATFORM) && $(VALGRIND_PREFIX) $(HOST_EXE_PATH) $(HOST_ARGS) $(HOST_XCLBIN)
 endif
 ifeq ($(BUILD_TARGET),hw)
 BUILD_TARGETS=$(BUILD_DIR)/$(HOST_EXE) $(HOST_XCLBIN)
-HOST_CMD=$(VALGRIND_PREFIX) $(HOST_EXE_PATH) $(HOST_XCLBIN)
+HOST_CMD=$(VALGRIND_PREFIX) $(HOST_EXE_PATH) $(HOST_ARGS) $(HOST_XCLBIN)
 endif
 
 
@@ -648,7 +687,7 @@ endif
 	mv ../out/sv/$*/out/*.sv ../out/sv
 	# Vivado requires includes to be of .vh files, so change file name.
 	mv ../out/sv/$*_gen.sv ../out/sv/$*_gen.vh
-	cd ../out/sv && sed -i s/$*_gen\\.sv/$*_gen\\.vh/ $*.sv
+	cd ../out/sv && sed -i -e s/$*_gen\\.sv/$*_gen\\.vh/ $*.sv
 
 
 
@@ -812,40 +851,44 @@ endif
 
 # For production use of port 80.
 # Run is done in its own directory to avoid socket collision with development.
-# ../live/$(BUILD_TARGET)/live indicates that the server is live.
-# ../live/$(BUILD_TARGET)/dead indicates that the server is dead.
+# $(LAUNCH_DIR)/live indicates that the server is live.
+# $(LAUNCH_DIR)/dead indicates that the server is dead.
 
 .PHONY: live dead
 
-live: ../live/$(BUILD_TARGET)/live
-../live/$(BUILD_TARGET)/live: ../live/$(BUILD_TARGET)/dead $(BUILD_TARGETS)
-	cp $(BUILD_DIR)/$(HOST_EXE) ../live/$(BUILD_TARGET)
+# TODO: "live" target runs in its own directory. We've since added support for LAUNCH_ID, which provides a generic separation
+#       of microservices. This could be used instead.
+live: $(LAUNCH_DIR)/live
+$(LAUNCH_DIR)/live: $(LAUNCH_DIR)/dead $(BUILD_TARGETS)
+	@# Copy executables to launch dir to avoid impact from active development. Not sure how necessary this is.
+	@cp $(BUILD_DIR)/$(HOST_EXE) $(LAUNCH_DIR)
 ifeq ($(USE_XILINX),true)
-	cp $(HOST_XCLBIN) ../live/$(BUILD_TARGET)
+	@cp $(HOST_XCLBIN) $(LAUNCH_DIR)
 endif
-	# TODO: What about copying the launch script? It this is changed, will that affect the running server?
-	# TODO: Not sure it's necessary to set make vars. These might pass through as environment vars.
+	@# TODO: What about copying the launch script? If this is changed, will that affect the running server?
+	@# TODO: Not sure it's necessary to set make vars. These might pass through as environment vars.
 	@echo "Launching production server in the background"
 	$(call with_secret,$(LAUNCH_PASSWORD) $(LAUNCH_CMD))
-	-rm ../live/$(BUILD_TARGET)/dead
-	touch ../live/$(BUILD_TARGET)/live
-	@echo "Went live!!!   (Stop with 'make dead' or restart with 'make live' again.)"
+	-rm $(LAUNCH_DIR)/dead
+	touch $(LAUNCH_DIR)/live
+	@echo "Went live!!!   (Stop with 'make$(LAUNCH_ID_ARG_STR) dead' or restart by reexecuting this command.)"
 
-dead: ../live/$(BUILD_TARGET)/dead
-../live/$(BUILD_TARGET)/dead:
-	if [[ -e killme ]]; then source killme > /dev/null 2>&1 && echo "Giving web server time to exit gracefully." && sleep 7; fi
-	@mkdir -p ../live/$(BUILD_TARGET)
-	rm -rf ../live/$(BUILD_TARGET)/*
-	@touch ../live/$(BUILD_TARGET)/dead
+dead: $(LAUNCH_DIR)/dead
+$(LAUNCH_DIR)/dead:
+	if [[ -e $(KILLME) ]]; then source $(KILLME) > /dev/null 2>&1 && echo "Giving web server time to exit gracefully." && sleep 7; $(ALREADY_DEAD) fi
+	@mkdir -p $(LAUNCH_DIR) log
+	rm -rf $(LAUNCH_DIR)/*
+	@touch $(LAUNCH_DIR)/dead
 
 
 
 PHONY: build launch
 build: $(BUILD_TARGETS)
 
-LAUNCH_CHECK=@if [ -e killme ]; then echo "Error: There appears to already be an application running. Kill it with <Ctrl-C> or 'source killme', or, if not running, 'rm killme', and try again." && false; fi
+LAUNCH_CHECK=@if [ -e $(KILLME) ]; then echo "Error: There appears to already be an application running. Kill it with <Ctrl-C> or 'source $(KILLME)', or, if not running, 'rm $(KILLME)', and try again." && false; fi
 launch: $(BUILD_TARGETS)
 	$(LAUNCH_CHECK)
+	@mkdir -p $(LAUNCH_DIR)
 	$(call with_secret,$(LAUNCH_PASSWORD) $(LAUNCH_CMD))
 
 # An un-documented target to launch the web server and open Chrome to test it.

Diff for: framework/build/launch

@@ -56,7 +56,10 @@
 #   -c <compile-command>: The compile command used to rebuilt when upgraded.
 #   -w <web-server-command>: The command to launch the web server.
 #             '<<PORT>>' will be substituted with the port number;
+#             '<<SOCKET>>' will be substituted with the socket number;
 #             '<<PASSWORD>>' will be substituted with $LAUNCH_PASSWORD;
+#   -k <killme-file>: The "killme" file to create (which, when source'd will kill the microservice).
+#   -s <socket-file>: The file to use for the socket.
 #   -o: Launch webserver in one-shot mode, where it will kill the launch process when the websocket is closed (browser tab is closed).
 
 # Must be launched from the projects 'build' directory because:
@@ -77,18 +80,22 @@ SCRIPT_CMD="launch $*"
 PORT="8888"
 HOST_PID=""
 COMPILE_CMD=""
-LAUNCH_WEBSERVER_CMD='python3 ../../../framework/webserver/default_server.py --port=<<PORT>>'
+LAUNCH_WEBSERVER_CMD='python3 ../../../framework/webserver/default_server.py --port=<<PORT>> --socket=<<SOCKET>>'
+KILLME="killme"
+SOCKET="SOCKET"
 ONESHOT_ARG=' '
 
 # Read command line options
 
-while getopts "p:h:c:w:o" opt
+while getopts "p:h:c:w:k:s:o" opt
   do
     case "${opt}" in
       p) PORT=${OPTARG};;
       h) HOST_PID=${OPTARG};;
       c) COMPILE_CMD="${OPTARG}";;
       w) LAUNCH_WEBSERVER_CMD="${OPTARG}";;
+      k) KILLME="${OPTARG}";;
+      s) SOCKET="${OPTARG}";;
       o) ONESHOT_ARG=" --oneshot=$$"
     esac
   done
@@ -163,6 +170,7 @@ launch () {
 
   # Substitute <<PORT>>.
   CMD=${LAUNCH_WEBSERVER_CMD/<<PORT>>/$PORT}
+  CMD=${CMD/<<SOCKET>>/$SOCKET}
   # Launch.
   echo "Launching web server as: $CMD"
   # Substitute <<PASSWORD>>.
@@ -176,7 +184,7 @@ launch () {
   export SERVER_PID=$!
 
   # Create a script to teardown the application by sending SIGINT to this process.
-  echo "kill $$ || echo 'Webserver process $$ no longer running.'" > killme   # TODO: Would be good to confirm that this is the right process.
+  echo "kill $$ || echo 'Webserver process $$ no longer running.'" > "$KILLME"   # TODO: Would be good to confirm that this is the right process.
 }
 
 # Tear down.
@@ -190,7 +198,7 @@ teardown () {
       kill $HOST_PID
     fi
     kill $SERVER_PID
-    rm SOCKET   # Needed? Doesn't even work.
+    rm "$SOCKET"   # Needed? Doesn't even work.
   else
     # Processes are invoked through sudo and bash and don't seem to be killed with their parent.
     HOST_PID2=`ps --ppid $HOST_PID -o pid=`
@@ -205,7 +213,7 @@ teardown () {
 
   # Done with "killme" script, created by this script.
   # This script might currently be in use, but it seems to be okay to delete it?
-  rm killme
+  rm "$KILLME"
 
   # Wait for child processes to finish first (to avoid output after exit)
   wait $HOST_PID >& /dev/null

Diff for: framework/host/server_main.c

@@ -59,13 +59,26 @@ int HostApp::server_main(int argc, char const *argv[], const char *kernel_name)
 {
 //TODO else ifndef OPENCL
 #ifdef OPENCL
-  if (argc != 2) {
-    printf("Usage: %s xclbin\n", argv[0]);
+  string opencl_arg_str = " xclbin";
+  int opencl_arg_cnt = 1;
+#else
+  string opencl_arg_str = "";
+  int opencl_arg_cnt = 0;
+#endif
+  // Poor-mans arg parsing.
+  int argn = 1;
+  if (strcmp(argv[1], "-s") == 0) {
+    socket_filename = argv[2];
+    argn += 2;
+  }
+  if (argc != argn + opencl_arg_cnt) {
+    printf("Usage: %s [-s socket] %s\n", argv[0], opencl_arg_str.c_str());
     return EXIT_FAILURE;
   }
 
+#ifdef OPENCL
   // Name of the .xclbin binary file and the name of the Kernel passed as arguments
-  const char *xclbin = argv[1];
+  const char *xclbin = argv[argn + opencl_arg_cnt - 1];
 #endif
 
   // Socket-related variables
@@ -95,8 +108,8 @@ int HostApp::server_main(int argc, char const *argv[], const char *kernel_name)
   }
 
   address.sun_family = AF_UNIX;
-  unlink(SOCKET);
-  strncpy(address.sun_path, SOCKET, sizeof(address.sun_path)-1);
+  unlink(socket_filename.c_str());
+  strncpy(address.sun_path, socket_filename.c_str(), sizeof(address.sun_path)-1);
 
   // Binding to the UNIX SOCKET
   if (bind(server_fd, (struct sockaddr *)&address,

Diff for: framework/host/server_main.h

@@ -103,6 +103,9 @@ class HostApp {
   ostream & cout_line() {return(cout << "C++: ");}
   ostream & cerr_line() {return(cerr << "C++ Error: ");}
 
+  // The default body of the main function for the server.
+  // argv if OPENCL:
+  //   [-s socket-name] [xclbin-name-if-OPENCL]
   int server_main(int argc, char const *argv[], const char *kernel_name);
 
   // Main method for processing traffic from/to the client.
@@ -131,6 +134,7 @@ class HostApp {
   static const int verbosity = 0; // 0: no debug messages; 10: all debug messages.
 
 protected:
+  string socket_filename = "SOCKET"; // The name of the socket file.
   int socket;  // The ID of the socket connected to the web server.
 
   /*

Diff for: framework/terraform/static_f1_reboot.sh

@@ -6,7 +6,7 @@
 # Running via ssh establishes the proper environment.
 # TODO: Look into EC2 user_data or cloud-init to replace this mechanism.
 source /home/centos/server_config.sh
-WEBSERVER_LOG="/home/centos/src/project_data/repo/apps/$KERNEL_NAME/build/webserver.log"
+WEBSERVER_LOG="/home/centos/src/project_data/repo/apps/$KERNEL_NAME/build/log/webserver.log"
 PASSWORD_ARG=$(if [[ -n "$ADMIN_PWD"        ]]; then echo PASSWORD=$ADMIN_PWD;        fi)
 PREBUILT_ARG=$(if [[ -n "$USE_PREBUILT_AFI" ]]; then echo PREBUILT=$USE_PREBUILT_AFI; fi)
 

Diff for: framework/webserver/server.py

@@ -510,11 +510,12 @@ def EC2Args():
     #           by passing flags=None and params=<value of args after command-line processing>.
     #           Implicit params are:
     #              "port" (8888): Socket on which web server will listen.
+    #              "socket" ("SOCKET"): Socket file name.
     # Return: {dict} The parameters/arguments. When a command-line arg is not given, it will have default value. A flag will not exist if not given, or have "" value if given.
     @staticmethod
     def commandLineArgs(flags=[], params={}):
         # Apply implicit args. ("password" is from EC2Args(), but the Makefile will provide it from configuration parameters, whether used or not.)
-        ret = {"port": 8888, "password": None, "oneshot": None, "ssl_crt_file": None, "ssl_key_file": None}
+        ret = {"port": 8888, "socket": "SOCKET", "password": None, "oneshot": None, "ssl_crt_file": None, "ssl_key_file": None}
         ret.update(params)
         arg_list = flags
         # Apply implicit flags (currently none).
@@ -524,7 +525,7 @@ def commandLineArgs(flags=[], params={}):
         try:
             opts, remaining = getopt.getopt(sys.argv[1:], "", arg_list)
         except getopt.GetoptError:
-            print('Usage: %s [--port #] [--instance i-#] [--ec2_time_bomb_timeout <sec>] [--password <password>] [--profile <aws-profile>] [--ssl_crt_file <ssl-crt-file> --ssl_key_file <ssl-key_file>]' % (sys.argv[0]))
+            print('Usage: %s [--port #] [--socket socket-file] [--instance i-#] [--ec2_time_bomb_timeout <sec>] [--password <password>] [--profile <aws-profile>] [--ssl_crt_file <ssl-crt-file> --ssl_key_file <ssl-key_file>]' % (sys.argv[0]))
             sys.exit(2)
         # Strip leading dashes.
         for opt, arg in opts:
@@ -539,12 +540,13 @@ def __init__(self, routes, args):
         self.args = args
 
         self.port = int(self.args['port'])
+        self.socket_filename = self.args['socket']
 
         FPGAServerApplication.application = self
 
         super(FPGAServerApplication, self).__init__(routes)
 
-        self.socket = Socket()
+        self.socket = Socket(self.socket_filename)
 
         # Launch server (with SSL or not)
         self.use_ssl = self.args['ssl_key_file'] != None and self.args['ssl_crt_file'] != None