Binary and unary operations are hooked

.gitignore

@@ -0,0 +1,7 @@
+myenv
+*.swp
+*~
+iids.json
+__pycache__
+workspace.code-workspace
+*.egg-info

build/lib/dynapyt/instrument/AccessedNamesProvider.py

@@ -0,0 +1,19 @@
+import libcst as cst
+
+
+class IsParamProvider(cst.BatchableMetadataProvider[list]):
+    """
+    Marks Name nodes found as a parameter to a function.
+    """
+    def __init__(self) -> None:
+        super().__init__()
+        self.is_param = False
+
+    def visit_Param(self, node: cst.Param) -> None:
+        # Mark the child Name node as a parameter
+        self.set_metadata(node.name, True)
+
+    def visit_Name(self, node: cst.Name) -> None:
+        # Mark all other Name nodes as not parameters
+        if not self.get_metadata(type(self), node, False):
+            self.set_metadata(node, False)

build/lib/dynapyt/instrument/CodeInstrumenter.py

@@ -0,0 +1,127 @@
+import libcst as cst
+from libcst.metadata import ParentNodeProvider, PositionProvider
+import libcst.matchers as matchers
+import libcst.helpers as helpers
+
+
+class CodeInstrumenter(cst.CSTTransformer):
+
+    METADATA_DEPENDENCIES = (ParentNodeProvider, PositionProvider,)
+
+    def __init__(self, file_path, iids, selected_hooks):
+        self.file_path = file_path
+        self.iids = iids
+        self.name_stack = []
+        self.selected_hooks = selected_hooks
+
+    def __create_iid(self, node):
+        location = self.get_metadata(PositionProvider, node)
+        line = location.start.line
+        column = location.start.column
+        iid = self.iids.new(self.file_path, line, column)
+        return iid
+
+    def __create_import(self, name):
+        module_name = cst.Attribute(value= cst.Name(value='dynapyt'), attr=cst.Name(value="runtime"))
+        fct_name = cst.Name(value=name)
+        imp_alias = cst.ImportAlias(name=fct_name)
+        imp = cst.ImportFrom(module=module_name, names=[imp_alias])
+        stmt = cst.SimpleStatementLine(body=[imp])
+        return stmt
+
+    def __wrap_in_lambda(self, node):
+        used_names = set(map(lambda x: x.value, matchers.findall(node, matchers.Name())))
+        parameters = []
+        for n in used_names:
+            parameters.append(cst.Param(name=cst.Name(value=n), default=cst.Name(value=n)))
+        lambda_expr = cst.Lambda(params=cst.Parameters(params=parameters), body=node)
+        return lambda_expr
+
+    # add import of our runtime library to the file
+    def leave_Module(self, original_node, updated_node):
+        imports_index = -1
+        for i in range(len(updated_node.body)):
+            if isinstance(updated_node.body[i].body, (tuple, list)):
+                if isinstance(updated_node.body[i].body[0], (cst.Import, cst.ImportFrom)):
+                    imports_index = i
+                elif i == 0:
+                    continue
+                else:
+                    break
+            else:
+                break
+        dynapyt_imports = [cst.Newline(value='\n')]
+        if 'assignment' in self.selected_hooks:
+            dynapyt_imports.append(self.__create_import("_assign_"))
+        if 'expression' in self.selected_hooks:
+            dynapyt_imports.append(self.__create_import("_expr_"))
+        if 'binary_operation' in self.selected_hooks:
+            dynapyt_imports.append(self.__create_import("_binary_op_"))
+        if 'unary_operation' in self.selected_hooks:
+            dynapyt_imports.append(self.__create_import("_unary_op_"))
+        dynapyt_imports.append(cst.Newline(value='\n'))
+        new_body = list(updated_node.body[:imports_index+1]) + dynapyt_imports + list(updated_node.body[imports_index+1:])
+        return updated_node.with_changes(body=new_body)
+
+    def leave_BinaryOperation(self, original_node, updated_node):
+        if 'binary_operation' not in self.selected_hooks:
+            return updated_node
+        callee_name = cst.Name(value="_binary_op_")
+        iid = self.__create_iid(original_node)
+        iid_arg = cst.Arg(value=cst.Integer(value=str(iid)))
+        left_arg = cst.Arg(updated_node.left)
+        operator_name = type(original_node.operator).__name__
+        operator_arg = cst.Arg(cst.SimpleString(value=f'"{operator_name}"'))
+        right_arg = cst.Arg(updated_node.right)
+        val_arg = cst.Arg(self.__wrap_in_lambda(original_node))
+        call = cst.Call(func=callee_name, args=[
+                        iid_arg, left_arg, operator_arg, right_arg, val_arg])
+        return call
+
+    def leave_Assign(self, original_node, updated_node):
+        if 'assignment' not in self.selected_hooks:
+            return updated_node
+        callee_name = cst.Name(value="_assign_")
+        iid = self.__create_iid(original_node)
+        iid_arg = cst.Arg(value=cst.Integer(value=str(iid)))
+        left_arg = cst.Arg(value=cst.SimpleString('\"' + cst.Module([]).code_for_node(updated_node) + '\"'))
+        right_arg = cst.Arg(value=self.__wrap_in_lambda(updated_node))
+        call = cst.Call(func=callee_name, args=[iid_arg, left_arg, right_arg])
+        new_node = cst.SimpleStatementLine(body=[cst.Expr(value=call)])
+        print(new_node)
+        return new_node
+
+    def leave_Expr(self, original_node, updated_node):
+        if 'expression' not in self.selected_hooks:
+            return updated_node
+        callee_name = cst.Name(value="_expr_")
+        iid = self.__create_iid(original_node)
+        iid_arg = cst.Arg(value=cst.Integer(value=str(iid)))
+        val_arg = cst.Arg(original_node)
+        call = cst.Call(func=callee_name, args=[iid_arg, val_arg])
+        return updated_node.with_changes(value=call)
+
+    def leave_FunctionDef(self, original_node, updated_node):
+        if 'function_def' not in self.selected_hooks:
+            return updated_node
+        callee_name = cst.Name(value="_func_entry_")
+        iid = self.__create_iid(original_node)
+        iid_arg = cst.Arg(value=cst.Integer(value=str(iid)))
+        entry_stmt = cst.Expr(cst.Call(func=callee_name, args=[iid_arg]))
+        new_body = updated_node.body.with_changes(body=[cst.SimpleStatementLine([entry_stmt])]+list(updated_node.body.body))
+        new_node = updated_node
+        return new_node.with_changes(body=new_body)
+
+    def leave_UnaryOperation(self, original_node, updated_node):
+        if 'unary_operation' not in self.selected_hooks:
+            return updated_node
+        callee_name = cst.Name(value="_unary_op_")
+        iid = self.__create_iid(original_node)
+        iid_arg = cst.Arg(value=cst.Integer(value=str(iid)))
+        operator_name = type(original_node.operator).__name__
+        operator_arg = cst.Arg(cst.SimpleString(value=f'"{operator_name}"'))
+        right_arg = cst.Arg(updated_node.expression)
+        val_arg = cst.Arg(self.__wrap_in_lambda(original_node))
+        call = cst.Call(func=callee_name, args=[
+                        iid_arg, operator_arg, right_arg, val_arg])
+        return call

build/lib/dynapyt/instrument/instrument.py

@@ -0,0 +1,60 @@
+import argparse
+import libcst as cst
+from CodeInstrumenter import CodeInstrumenter
+from IIDs import IIDs
+import re
+from shutil import copyfile
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    "--files", help="Python files to instrument or .txt file with all file paths", nargs="+")
+parser.add_argument(
+    "--iids", help="JSON file with instruction IDs (will create iids.json if nothing given)")
+
+
+def gather_files(files_arg):
+    if len(files_arg) == 1 and files_arg[0].endswith('.txt'):
+        files = []
+        with open(files_arg[0]) as fp:
+            for line in fp.readlines():
+                files.append(line.rstrip())
+    else:
+        for f in files_arg:
+            if not f.endswith('.py'):
+                raise Exception(f'Incorrect argument, expected .py file: {f}')
+        files = files_arg
+    return files
+
+
+def instrument_file(file_path, iids, selected_hooks):
+    with open(file_path, 'r') as file:
+        src = file.read()
+
+    if 'DYNAPYT: DO NOT INSTRUMENT' in src:
+        print(f'{file_path} is already instrumented -- skipping it')
+        return
+
+    ast = cst.parse_module(src)
+    ast_wrapper = cst.metadata.MetadataWrapper(ast)
+
+    instrumented_code = CodeInstrumenter(file_path, iids, selected_hooks)
+    instrumented_ast = ast_wrapper.visit(instrumented_code)
+
+    copied_file_path = re.sub(r'\.py$', '.py.orig', file_path)
+    copyfile(file_path, copied_file_path)
+
+    rewritten_code = '# DYNAPYT: DO NOT INSTRUMENT\n\n' + instrumented_ast.code
+    with open(file_path, 'w') as file:
+        file.write(rewritten_code)
+    print(f'Done with {file_path}')
+
+
+if __name__ == '__main__':
+    args = parser.parse_args()
+    files = gather_files(args.files)
+    iids = IIDs(args.iids)
+    selected_hooks = ['unary_operation', 'binary_operation']
+    for file_path in files:
+        instrument_file(file_path, iids, selected_hooks)
+    iids.store()

build/lib/dynapyt/runtime.py

@@ -0,0 +1,14 @@
+def _assign_(iid, left, right):
+    if iid < 100:
+        temp = 10
+    else:
+        temp = 20
+    temp += iid
+    res = right()
+    return res
+
+def _binary_op_(iid, left, opr, right, val):
+    return val()
+
+def _unary_op_(iid, opr, right, val):
+    return val()

build/lib/dynapyt/test.py

@@ -0,0 +1,2 @@
+def test():
+    print("hello")

build/lib/evaluation/__init__.py

@@ -0,0 +1 @@
+#

build/lib/evaluation/instrument_tracer.py

@@ -0,0 +1,88 @@
+import argparse
+from os import path
+import libcst as cst
+import re
+from shutil import copyfile
+
+import libcst as cst
+from libcst.metadata import ParentNodeProvider, PositionProvider
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    "--files", help="Python files to instrument or .txt file with all file paths", nargs="+")
+
+
+class CodeInstrumenter(cst.CSTTransformer):
+
+    METADATA_DEPENDENCIES = (ParentNodeProvider, PositionProvider,)
+
+    def __create_import(self, name):
+        # module_name = cst.Attribute(value=cst.Name(value=cst.Name(cst.Attribute(value=cst.Name("dynapyt")), attr=cst.Name("evaluation"))), attr=cst.Name(value="instrument_tracer"))
+        module_name = cst.Attribute(value=cst.Name('evaluation'), attr=cst.Name('trc'))
+        fct_name = cst.Name(value=name)
+        imp_alias = cst.ImportAlias(name=fct_name)
+        imp = cst.ImportFrom(module=module_name, names=[imp_alias])
+        stmt = cst.SimpleStatementLine(body=[imp])
+        return stmt
+
+    def leave_Module(self, original_node, updated_node):
+        import_assign = self.__create_import("_trace_opcodes_")
+        callee = cst.Name(value='settrace')
+        arg = cst.Arg(cst.Name(value='_trace_opcodes_'))
+        call_trc = cst.Call(func=callee, args=[arg])
+
+        module_name = value=cst.Name('sys')
+        fct_name = cst.Name(value='settrace')
+        imp_alias = cst.ImportAlias(name=fct_name)
+        imp = cst.ImportFrom(module=module_name, names=[imp_alias])
+        stmt = cst.SimpleStatementLine(body=[imp])
+
+        func = cst.FunctionDef(name=cst.Name('_dynapyt_tracer_run_'), params=cst.Parameters(params=[]), body=cst.IndentedBlock(body=updated_node.body))
+        func_call = cst.SimpleStatementLine(body=[cst.Expr(cst.Call(func=cst.Name('_dynapyt_tracer_run_'), args=[]))])
+
+        new_body = [import_assign, stmt, cst.Newline(value='\n'), func, cst.SimpleStatementLine(body=[cst.Expr(call_trc)]), func_call]
+        return updated_node.with_changes(body=new_body)
+
+def gather_files(files_arg):
+    if len(files_arg) == 1 and files_arg[0].endswith('.txt'):
+        files = []
+        with open(files_arg[0]) as fp:
+            for line in fp.readlines():
+                files.append(line.rstrip())
+    else:
+        for f in files_arg:
+            if not f.endswith('.py'):
+                raise Exception(f'Incorrect argument, expected .py file: {f}')
+        files = files_arg
+    return files
+
+
+def instrument_file(file_path):
+    with open(file_path, 'r') as file:
+        src = file.read()
+
+    if 'tracer: DO NOT INSTRUMENT' in src:
+        print(f'{file_path} is already instrumented -- skipping it')
+        return
+
+    ast = cst.parse_module(src)
+    ast_wrapper = cst.metadata.MetadataWrapper(ast)
+
+    instrumented_code = CodeInstrumenter()
+    instrumented_ast = ast_wrapper.visit(instrumented_code)
+    # print(instrumented_ast)
+
+    copied_file_path = re.sub(r'\.py$', '.py.orig', file_path)
+    copyfile(file_path, copied_file_path)
+
+    rewritten_code = '# tracer: DO NOT INSTRUMENT\n\n' + instrumented_ast.code
+    with open(file_path, 'w') as file:
+        file.write(rewritten_code)
+
+
+if __name__ == '__main__':
+    args = parser.parse_args()
+    files = gather_files(args.files)
+    for file_path in files:
+        instrument_file(file_path)

build/lib/evaluation/trc.py

@@ -0,0 +1,10 @@
+import opcode
+def _trace_opcodes_(frame, event, arg=None):
+    if event == 'opcode':
+        code = frame.f_code
+        offset = frame.f_lasti
+        print(f"{opcode.opname[code.co_code[offset]]:<18} | {frame.f_lineno}")
+    else:
+        print('x')
+        frame.f_trace_opcodes = True
+    return _trace_opcodes_