pass e2e

README.md

@@ -9,6 +9,27 @@ Parrot is a distributed, multi-tenant serving system for **LLM-based Application
 - Performance objective deduction and DAG-aware scheduling.
 - Sharing common prompt prefix between requests with optimized attention kernel, Context-aware scheduling.
 
+We also provide a Python friendly frontend for users to program:
+
+```python
+from parrot import P # 'P' is the alias of pfunc lang.
+
+@P.semantic_function()
+def tell_me_a_joke(topic: P.Input, joke: P.Output):
+    """Tell the me a joke about {{topic}}: {{joke}}."""
+
+@P.native_function()
+def format_joke(joke: P.Input, formatted_joke: P.Output):
+    ret = "Here is the joke for you\n---\n" + joke # Directly use string built-in methods
+    formatted_joke.set(ret) # Use `set` to assign value to output
+
+def main(): # Orchestrator function
+    joke = tell_me_a_joke(topic="chicken")
+    joke1 = format_joke(joke)
+    joke_str = joke1.get()
+    print(joke_str)
+```
+
 ## What's LLM Applications?
 
 The powerful language understanding capability of large language models (LLMs) has enabled a new application paradigm, where one or multiple application entities, known as AI agents or co-pilots, communicate with LLMs via natural language, known as “prompts”, to accomplish a task collaboratively. 

docs/user_docs/pfunc.md

@@ -55,7 +55,8 @@ from parrot import P # 'P' is the alias of pfunc lang.
 
 @P.native_function() # Here we can add some arguments
 def str_concat(a: P.Input, b: P.Input, c: P.Output):
-    c = a + b # Directly use the string grammar in Semantic Variable
+    c_str = a + b # Directly use the string grammar in Semantic Variable
+    c.set(c_str) # Use `set` method to assign value
 
 async def main():
     # Create two SVs
@@ -67,5 +68,5 @@ async def main():
 ```
 
 - Native functions are cached: If you call the same Native Function a second time (based on the function name), you can omit the code part of the function. Parrot will automatically use the cached function code.
-- Don't use `return` in the function body because we include return values (outputs) in the parameter list.
+- Don't use `return` in the function body because we include return values (outputs) in the parameter list. Use `set` to assign values to outputs.
 - Currently there is no sandbox / execution environment in the server side. So it's easy to inject malicious code into the system through native functions.

examples/chatbot.py

@@ -8,6 +8,7 @@
 # 2023.9.26: Fixed
 # 2023.10.23: TODO: Support stateful call in V2
 # 2023.10.31: Implemented.
+# TODO: Support stateful call in V3
 
 from parrot import P
 

examples/readme_example.py

@@ -0,0 +1,32 @@
+# Copyright (c) 2024 by Microsoft Corporation.
+# Author: Chaofan Lin ([email protected])
+
+from parrot import P
+
+vm = P.VirtualMachine(
+    core_http_addr="http://localhost:9000",
+    mode="debug",
+)
+
+
+@P.semantic_function()
+def tell_me_a_joke(topic: P.Input, joke: P.Output):
+    """Tell the me a joke about {{topic}}: {{joke}}."""
+
+
+@P.native_function()
+def format_joke(joke: P.Input, formatted_joke: P.Output):
+    ret = (
+        "Here is the joke for you\n---\n" + joke
+    )  # Directly use string built-in methods
+    formatted_joke.set(ret)  # Use `set` to assign value to output
+
+
+def main():  # Orchestrator function
+    joke = tell_me_a_joke(topic="chicken")
+    joke1 = format_joke(joke)
+    joke_str = joke1.get()
+    print(joke_str)
+
+
+vm.run(main)

parrot/frontend/pfunc/function.py

@@ -280,7 +280,7 @@ def _submit_semantic_call(self, call: "SemanticCall") -> List:
                 "VM environment is not set. Not submit the Call. Return Call instead. "
                 "(Please run a Parrot function under a VM context.)"
             )
-            return {}
+            return []
 
     async def _asubmit_semantic_call(self, call: "SemanticCall") -> List:
         if self._has_vm_env():
@@ -294,7 +294,7 @@ async def _asubmit_semantic_call(self, call: "SemanticCall") -> List:
                 "VM environment is not set. Not submit the Call. Return Call instead. "
                 "(Please run a Parrot function under a VM context.)"
             )
-            return {}
+            return []
 
     # ---------- Call Methods ----------
 
@@ -483,6 +483,34 @@ def __init__(
             # This will generate a register warning if the VM environment is not set.
             self._register_function()
 
+    # ---------- VM Env Methods ----------
+
+    def _submit_native_call(self, call: "PyNativeCall") -> List:
+        if self._has_vm_env():
+            return BasicFunction._virtual_machine_env.submit_py_native_call_handler(
+                call
+            )
+        else:
+            logger.warning(
+                "VM environment is not set. Not submit the Call. Return Call instead. "
+                "(Please run a Parrot function under a VM context.)"
+            )
+            return []
+
+    async def _asubmit_native_call(self, call: "PyNativeCall") -> List:
+        if self._has_vm_env():
+            return (
+                await BasicFunction._virtual_machine_env.asubmit_py_native_call_handler(
+                    call
+                )
+            )
+        else:
+            logger.warning(
+                "VM environment is not set. Not submit the Call. Return Call instead. "
+                "(Please run a Parrot function under a VM context.)"
+            )
+            return []
+
     def __call__(
         self,
         *args: List[Any],
@@ -517,14 +545,11 @@ def _call_func(
     ) -> Union[SemanticVariable, Tuple[SemanticVariable, ...], "PyNativeCall"]:
         call = PyNativeCall(self, *args, **kwargs)
 
-        if BasicFunction._virtual_machine_env is not None:
-            BasicFunction._virtual_machine_env.submit_call_handler(call)
-        else:
-            logger.warning(
-                "VM environment is not set. Not submit the Call. Return Call instead. "
-                "(Please run a Parrot function under a VM context.)"
-            )
+        param_info = self._submit_native_call(call)
+        if not self._has_vm_env():
             return call
+        else:
+            call.update_var_ids(param_info)
 
         # Unpack the output SemanticVariables
         if len(call.output_vars) == 1:
@@ -535,18 +560,14 @@ async def _acall_func(
         self,
         *args: List[Any],
         **kwargs: Dict[str, Any],
-    ) -> Union[SemanticVariable, Tuple[SemanticVariable, ...], "SemanticCall"]:
+    ) -> Union[SemanticVariable, Tuple[SemanticVariable, ...], "PyNativeCall"]:
         call = PyNativeCall(self, *args, **kwargs)
 
-        if BasicFunction._virtual_machine_env is not None:
-            # Different from _call_func, we use asubmit_call_handler here.
-            await BasicFunction._virtual_machine_env.asubmit_call_handler(call)
-        else:
-            logger.warning(
-                "VM environment is not set. Not submit the Call. Return Call instead. "
-                "(Please run a Parrot function under a VM context.)"
-            )
+        param_info = self._submit_native_call(call)
+        if not self._has_vm_env():
             return call
+        else:
+            call.update_var_ids(param_info)
 
         # Unpack the output SemanticVariables
         if len(call.output_vars) == 1:

parrot/frontend/pfunc/vm.py

@@ -18,6 +18,8 @@
     remove_session,
     submit_semantic_call,
     asubmit_semantic_call,
+    submit_py_native_call,
+    asubmit_py_native_call,
     register_semantic_variable,
     set_semantic_variable,
     get_semantic_variable,
@@ -31,6 +33,7 @@
 from .function import (
     BasicFunction,
     PyNativeFunction,
+    PyNativeCall,
     SemanticFunction,
     SemanticCall,
     ParamType,
@@ -201,7 +204,7 @@ def submit_semantic_call_handler(self, call: SemanticCall) -> List:
         return resp.param_info
 
     async def asubmit_semantic_call_handler(self, call: SemanticCall) -> List:
-        """Submit a call to the ServeCore.
+        """(Async) Submit a SemanticCall to the ServeCore.
 
         Args:
             call: SemanticCall. The call to be submitted.
@@ -223,6 +226,52 @@ async def asubmit_semantic_call_handler(self, call: SemanticCall) -> List:
 
         return resp.param_info
 
+    def submit_py_native_call_handler(self, call: PyNativeCall) -> List:
+        """Submit a PyNativeCall to the ServeCore.
+
+        Args:
+            call: PyNativeCall. The call to be submitted.
+
+        Returns:
+            Dict. The "param info" returned by the ServeCore.
+        """
+
+        logger.info(
+            f"VM (session_id={self._get_session_id_str()}) submits PyNativeCall: {call.func.name}"
+        )
+
+        resp = submit_py_native_call(
+            http_addr=self.core_http_addr,
+            session_id=self.session_id,
+            session_auth=self._session_auth,
+            payload=call.to_request_payload(),
+        )
+
+        return resp.param_info
+
+    async def asubmit_py_native_call_handler(self, call: PyNativeCall) -> List:
+        """(Async) Submit a PyNativeCall to the ServeCore.
+
+        Args:
+            call: PyNativeCall. The call to be submitted.
+
+        Returns:
+            Dict. The "param info" returned by the ServeCore.
+        """
+
+        logger.info(
+            f"VM (session_id={self._get_session_id_str()}) submits PyNativeCall: {call.func.name}"
+        )
+
+        resp = await asubmit_py_native_call(
+            http_addr=self.core_http_addr,
+            session_id=self.session_id,
+            session_auth=self._session_auth,
+            payload=call.to_request_payload(),
+        )
+
+        return resp.param_info
+
     # ---------- Public Methods ----------
 
     @property

parrot/serve/session/session.py

@@ -145,7 +145,9 @@ def _add_py_native_request(self, request_id: int, request_payload: Dict) -> List
         func_node = NativeFuncNode(native_func=request)
 
         # Assign Semantic Variables to the Func Node.
-        self.var_mgr.create_vars_for_pynative_func(func_node)
+        self.var_mgr.create_vars_for_pynative_func(
+            session_id=self.session_id, native_func_node=func_node
+        )
 
         # Add the request to the executor.
         self.native_executor.add_native_func(func_node)

parrot/serve/variable_manager.py

@@ -381,7 +381,7 @@ def create_vars_for_pynative_func(
             else:
                 lvar = self._get_local_var_by_id(
                     session_id=session_id,
-                    var_name=key,
+                    var_id=param.var_id,
                 )
 
             if param.is_output: