Compiler interface

bin/vamosc

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+BASEPATH=$(dirname $(readlink -f $0))/../..
+python3 $BASEPATH/shamon/compiler/vamosc.py $@

compiler/README.md

@@ -7,6 +7,12 @@ Python 3 (we use Python 3.8.9).
 
 ## Usage
 
+To see all available options write:
+
+```bash
+python main.py -h
+```
+
 ### Generating a Monitor From a Vamos Specification
 To compile a VAMOS specification into `C` code, specify the path where the VAMOS specification is (i.e `INPUT_FILE`), and the path to write the `C` code of the monitor.
 
@@ -26,13 +32,23 @@ java -jar tessla.jar compile-rust  <TESSLA_SPECIFICATION_FILE> --project-dir <OU
 2. Run the compiler over this `OUTPUT_DIR`. Specify an `OUTPUT_FILE` where the final monitor is going to be written and a VAMOS specification file:
 
 ```bash
-python3 compiler/main.py -o <OUTPUT_FILE> --with-tessla <VAMOS_SPECIFICATION_FILE> --dir <OUTPUT_DIR>
+pythonmain.py -o <OUTPUT_FILE> --with-tessla <VAMOS_SPECIFICATION_FILE> --dir <OUTPUT_DIR>
 ```
 
 The compiler will update the `Cargo.toml` file and generate a Foreign Function Interface (FFI) so that the final monitor can make use of Tessla-generated code.
 
 3. Generate a static library by running `cargo build` inside `OUTPUT_DIR`. This will generate the file `<OUTPUT_DIR>/target/debug/libmonitor.a`, which you have to link when compiling the final monitor.
 
 
+### Generating and Running an Executable from either a VAMOS or TESSLA specification
+
+`vamosc.py` can be used similarly but it also containts other flags and options to generate and run an executable. It compiles a VAMOS/TESSLA specification and generates the following:
+
+1. A compile script that links all the libraries to be able to compile the C file of the monitor into an executable. The path of this file is specified by the flag  `--compilescript` or `-s`. Other flags to customize the generated compile script are available (try `python vamosc.py -h` to see all available options).
+
+2. An executable, for which you can specify the path through `--executable` or `e`.
+
+If the flag `--legacy-mode` or for short `-lg` is passed then it behaves the same way as `main.py`
+
 
 

compiler/main.py

@@ -16,7 +16,7 @@
 					action="store_true")
 parser.add_argument("-d", "--dir",
 					help="Directory where the library generated by Tessla is (not needed if --with-tessla flag is present).")
-parser.add_argument("-b", "--bufsize")
+parser.add_argument("-b", "--bufsize", help="Is the value used to replace @BUFSIZE in a VAMOS specification")
 
 args = parser.parse_args()
 bufsize = args.bufsize
@@ -79,7 +79,7 @@
 	if not os.path.isdir(args.dir):
 		raise Exception(f"{output_path} directory does not exist!")
 
-	update_toml(args.dir)
+	update_toml(args.dir, None)
 
 	# BEGIN writing c-file interface
 	c_interface = get_c_interface(components, ast, streams_to_events_map, stream_types, arbiter_event_source,
@@ -89,7 +89,7 @@
 	file.close()
 
 	# BEGIN writing rust file
-	program = get_rust_file(streams_to_events_map, arbiter_event_source)
+	program = get_rust_file(streams_to_events_map, arbiter_event_source, None)
 	file = open(f"{args.dir}/src/monitor.rs", "r")
 	lines = file.readlines()
 	file.close()

compiler/tessla_utils.py

@@ -2,16 +2,17 @@
 from operator import pos
 from cfile_utils import *
 import os
+import shutil
 
-state_type = '''State<
+old_state_type = '''State<
 (),
 fn(TesslaValue<(TesslaInt, TesslaInt, TesslaInt)>, i64) -> Result<(), ()>,
 fn(TesslaValue<(TesslaInt, TesslaInt, TesslaInt)>, i64) -> Result<(), ()>,
 fn(TesslaOption<TesslaValue<(TesslaInt, TesslaInt)>>, i64) -> Result<(), ()>,
 fn(TesslaOption<TesslaValue<(TesslaInt, TesslaInt)>>, i64) -> Result<(), ()>,
 >'''
 
-def update_toml(output_dir: str) -> None:
+def update_toml(output_dir: str, vendor) -> None:
     toml_file_path = f"{output_dir}/Cargo.toml"
     current_part = ""
     sections = dict()
@@ -34,6 +35,11 @@ def update_toml(output_dir: str) -> None:
 
         sections["[lib]"].add('crate-type = [\"staticlib\"]')
 
+        if vendor is not None:
+            shutil.copy(vendor, f"{output_dir}/vendor", dirs_exist_ok=True)
+            sections["[source.crates-io]"]={'replace-with = "vendored-sources"'}
+            sections["[source.vendored-sources]"]={'directory = "vendor"'}
+
         answer = ""
         for (section, lib) in sections.items():
             if answer == "":
@@ -70,7 +76,7 @@ def get_rust_file_args(args):
     return args_result, value_args
 
 
-def get_rust_code(possible_events):
+def get_rust_code(possible_events, state_type):
     answer = ""
 
     for (event_name, data) in possible_events.items():
@@ -82,15 +88,66 @@ def get_rust_code(possible_events):
             Value_args = f"Value(({Value_args}))"
             answer+=f'''
 #[no_mangle]
-extern "C" fn RUST_FFI_{event_name} (mut bs : &mut {state_type}, {args} ts : c_long) {"{"}
+extern "C" fn RUST_FFI_{event_name} (bs : &mut {state_type}, {args} ts : c_long) {"{"}
     bs.step(ts.into(), false).expect("Step failed");
     bs.set_{event_name}({Value_args});
     bs.flush().expect("Flush failed");
 {"}"}
 '''
     return answer
 
-def get_rust_file(mapping, arbiter_event_source) -> str:
+def try_parse_state_type(lines) -> str:
+    if lines is None:
+        return old_state_type
+    found=0
+    bracketcount=0
+    buffer=""
+    ret=old_state_type
+    for line in lines:
+        if found==0:
+            if line.strip().startswith("impl Default"):
+                buffer=buffer+line.strip()[12:].strip()
+                found=1
+                continue
+        if found==1:
+            buffer=buffer+line.strip()
+            if(buffer.startswith("for State<")):
+                bracketcount=1
+                ret=buffer[4:]
+                buffer=buffer[10:]
+                found=2
+                while bracketcount>0 and len(buffer)>0:
+                    if buffer[0:2]=="->":
+                        ret=ret+buffer[0]
+                        buffer=buffer[1:]
+                    elif buffer[0]=="<":
+                        bracketcount=bracketcount+1
+                    elif buffer[0]==">":
+                        bracketcount=bracketcount-1
+                    ret=ret+buffer[0]
+                    buffer=buffer[1:]
+                continue
+            elif len(buffer)>=10:
+                found=0
+                buffer=""
+        if found==2:
+            buffer=buffer+line.strip()
+            while bracketcount>0 and len(buffer)>0:
+                if buffer[0:2]=="->":
+                    ret=ret+buffer[0]
+                    buffer=buffer[1:]
+                elif buffer[0]=="<":
+                    bracketcount=bracketcount+1
+                elif buffer[0]==">":
+                    bracketcount=bracketcount-1
+                ret=ret+buffer[0]
+                buffer=buffer[1:]
+            if(bracketcount==0):
+                break
+    return ret
+
+def get_rust_file(mapping, arbiter_event_source, origlines) -> str:
+    state_type = try_parse_state_type(origlines)
     possible_events = mapping[arbiter_event_source]
     return f'''
 #[no_mangle]
@@ -99,7 +156,7 @@ def get_rust_file(mapping, arbiter_event_source) -> str:
     Box::new(State::default())
 {"}"}
 
-{get_rust_code(possible_events)}
+{get_rust_code(possible_events, state_type)}
 '''
 
 def received_events_declare_args(event_name, data):

compiler/vamosc.py

@@ -0,0 +1,198 @@
+import sys
+
+from parser import parse_program
+from type_checker import TypeChecker
+import argparse
+from cfile_utils import get_c_program
+from utils import *
+import os
+from tessla_utils import get_rust_file, get_c_interface, update_toml
+from pathlib import Path
+import shutil
+import subprocess
+
+parser = argparse.ArgumentParser(prog="vamosc")
+parser.add_argument("inputfile", type=str, help="VAMOS program to compile.")
+parser.add_argument("-lg", "--legacy-mode", action='store_true', help="Legacy mode: only generate C file with monitor code (plus TeSSLa interface where applicable).")
+parser.add_argument("-o", "--out", help="Output file for monitor C code.",default="vamos_monitor.c")
+parser.add_argument("-e", "--executable", help="Path of the executable to generate.", default="monitor")
+parser.add_argument("-l", "--link", help="Describes list of external libraries that the C compiler should link against", nargs="*")
+parser.add_argument("-ll", "--link-lookup", help="Lists libraries to be linked against via -l parameters", nargs="*")
+parser.add_argument("-t", "--with-tessla",
+					help="Tessla specification to be used as the higher-level monitor. Tessla input streams must match events of arbiter's output stream.")
+parser.add_argument("-d", "--dir",
+					help="Directory where the library generated by Tessla is (not needed if --with-tessla flag is present).", default="vamos_tesslamon")
+parser.add_argument("-b", "--bufsize", help="Is the value used to replace @BUFSIZE in VAMOS specification")
+parser.add_argument("-s", "--compilescript", help="Path to the compile script that should be generated.",default="vamos_compile.sh")
+parser.add_argument("-v", "--verbose", action='store_true')
+parser.add_argument("-g", "--debug", action='store_true')
+parser.add_argument("--crate-local-vendor", help="Path to local copies of Rust dependencies to prevent fetching them from online repositories")
+
+args = parser.parse_args()
+bufsize = args.bufsize
+
+input_file = args.inputfile  # second argument should be input file
+parsed_args_file = replace_cmd_args(open(input_file).readlines(), bufsize)
+file = " ".join(parsed_args_file)
+
+
+# Type checker initialization
+TypeChecker.clean_checker()
+TypeChecker.add_reserved_keywords()
+
+# Parser
+ast = parse_program(file)
+assert (ast[0] == "main_program")
+components = dict()
+get_components_dict(ast[1], components)
+
+if "stream_type" in components.keys():
+	TypeChecker.get_stream_types_data(components["stream_type"])
+
+if "stream_processor" in components.keys():
+	TypeChecker.get_stream_processors_data(components["stream_processor"])
+for event_source in components["event_source"]:
+	TypeChecker.insert_event_source_data(event_source)
+TypeChecker.get_stream_events(components["stream_type"])
+
+if "buff_group_def" in components.keys():
+	for buff_group in components["buff_group_def"]:
+		TypeChecker.add_buffer_group_data(buff_group)
+
+if "match_fun_def" in components.keys():
+	for match_fun in components["match_fun_def"]:
+		TypeChecker.add_match_fun_data(match_fun)
+
+# TypeChecker.check_arbiter(ast[-2])
+# TypeChecker.check_monitor(ast[-1])
+#
+# Produce C file
+
+#
+streams_to_events_map = get_stream_to_events_mapping(components["stream_type"], TypeChecker.stream_processors_data)
+
+stream_types: Dict[str, Tuple[str, str]] = get_stream_types(components["event_source"])
+arbiter_event_source = get_arbiter_event_source(ast[2])
+existing_buffers = get_existing_buffers(TypeChecker)
+
+TypeChecker.arbiter_output_type = arbiter_event_source
+
+#if args.out is None:
+#	print("provide the path of the file where the C program must be written.")
+#	exit(1)
+#else:
+output_path = os.path.abspath(args.out)
+cscript_path = args.compilescript
+executable_path = os.path.abspath(args.executable)
+#if 
+
+ownpath = str(Path(os.path.abspath(__file__)).absolute().parent.parent)
+
+if args.with_tessla is not None:
+	if args.dir is None:
+		print("ERROR: Must provide the directory path where Tessla files are located")
+		exit(1)
+	if os.path.exists(args.dir):
+		shutil.rmtree(args.dir)
+
+	if args.legacy_mode is not True:
+		subprocess.call(['java', '-jar', f'{ownpath}/compiler/tessla-rust.jar', 'compile-rust', '-p', args.dir, args.with_tessla])
+
+	update_toml(args.dir, args.crate_local_vendor)
+
+	# BEGIN writing c-file interface
+	c_interface = get_c_interface(components, ast, streams_to_events_map, stream_types, arbiter_event_source,
+								  existing_buffers)
+	file = open(output_path, "w")
+	file.write(c_interface)
+	file.close()
+
+
+	file = open(f"{args.dir}/src/monitor.rs", "r")
+	lines = file.readlines()
+	file.close()
+	# BEGIN writing rust file
+	program = get_rust_file(streams_to_events_map, arbiter_event_source, lines)
+
+	file = open(f"{args.dir}/src/monitor.rs", "w")
+	extern_keyword_found = False
+	is_there_prev_compilation = False
+	for line in lines:
+		if "#[no_mangle]" in line:
+			print("Code from previous compilation found. Removing it...")
+			is_there_prev_compilation = True
+			break
+
+	for line in lines:
+		file.write(line)
+		if (not is_there_prev_compilation) and ("extern crate tessla_stdlib;" in line):
+			assert(not extern_keyword_found)
+			extern_keyword_found = True
+			file.write("use std::os::raw::c_int;\n")
+			file.write("use std::os::raw::c_long;\n")
+		if "#[no_mangle]" in line:
+			break
+	file.write(program)
+	file.close()
+	if args.legacy_mode is True:
+		print("DO NOT FORGET to add target/debug/libmonitor.a to the build file of your monitor")
+else:
+
+	program = get_c_program(components, ast, streams_to_events_map, stream_types, arbiter_event_source,
+							existing_buffers)
+	output_file = open(output_path, "w")
+	output_file.write(program)
+	output_file.close()
+
+if args.legacy_mode is not True:
+	output_file = open(cscript_path, "w")
+	output_file.write("#!/usr/bin/env bash\n\n")
+	if(args.verbose):
+		output_file.write("set -x\n\n")
+	else:
+		output_file.write("set +x\n\n")
+
+	output_file.write(f'VAMOSDIR="{ownpath}"\n')
+
+	output_file.write('CC=clang\n')
+	output_file.write('CPPFLAGS="-D_POSIX_C_SOURCE=200809L -I$VAMOSDIR/gen -I$VAMOSDIR\\\n')
+	output_file.write('          -I$VAMOSDIR/streams -I$VAMOSDIR/core -I$VAMOSDIR/shmbuf"\n\n')
+
+	if(args.debug):
+		output_file.write('LTOFLAGS=""\n')
+		output_file.write('CFLAGS="-g -O0"\n\n')
+	else:
+		output_file.write('CFLAGS="-g3 -O3 -fPIC -std=c11"\n')
+		#output_file.write('LTOFLAGS="-fno-lto -fno-fat-lto-objects"\n')
+		output_file.write('CPPFLAGS="$CPPFLAGS -DNDEBUG"\n\n')
+
+	linklibraries=["$VAMOSDIR/core/libshamon-arbiter.a","$VAMOSDIR/core/libshamon-stream.a","$VAMOSDIR/shmbuf/libshamon-shmbuf.a","$VAMOSDIR/core/libshamon-parallel-queue.a","$VAMOSDIR/core/libshamon-ringbuf.a","$VAMOSDIR/core/libshamon-event.a","$VAMOSDIR/core/libshamon-source.a","$VAMOSDIR/core/libshamon-signature.a","$VAMOSDIR/core/libshamon-list.a","$VAMOSDIR/core/libshamon-utils.a","$VAMOSDIR/core/libshamon-monitor-buffer.a","$VAMOSDIR/streams/libshamon-streams.a", "$VAMOSDIR/compiler/cfiles/compiler_utils.o"]
+
+	if args.link is not None:
+		linklibraries=linklibraries + [os.path.abspath(p) for p in args.link]
+	if args.link_lookup is not None:
+		linklibraries=linklibraries + [f"-l{p}" for p in args.link_lookup]
+
+	if(args.with_tessla):
+		linklibraries.append(f"{os.path.abspath(args.dir)}/target/release/libmonitor.a")
+
+	output_file.write(f'LIBRARIES="{linklibraries[0]}')
+	for llib in linklibraries[1:] :
+		output_file.write(f'\\\n          {llib}')
+	output_file.write('"\n\n')
+
+	output_file.write(f'EXECUTABLEPATH="{executable_path}"\n')
+	output_file.write(f'CFILEPATH="{output_path}"\n\n')
+
+	if args.with_tessla is not None:
+		output_file.write('CURPATH=${pwd}\n')
+		output_file.write(f'cd {os.path.abspath(args.dir)}\n')
+		output_file.write("cargo build --release\n")
+		output_file.write(f'cd $CURPATH\n\n')
+
+	output_file.write('test -z $CC && CC=cc\n')
+	output_file.write('${CC} $CFLAGS $LTOFLAGS $CPPFLAGS -o $EXECUTABLEPATH $CFILEPATH $@ $LIBRARIES $LDFLAGS\n')
+
+	output_file.close()
+
+	subprocess.call(["bash", cscript_path])

install.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+SHAMONDIR=$(readlink -f $(dirname $0))
+echo $SHAMONDIR
+
+for FILE in bin/*; do
+  echo $SHAMONDIR/$FILE;
+  sed "s|\$BASEPATH/shamon|$SHAMONDIR|" $FILE > tmpfile
+  sudo chmod +xxx tmpfile
+  sudo mv tmpfile /usr/$FILE
+done