add "processed_content" to ToolResult struct from function results (#192

)

- added test
- documentation updates
- LLMChain.execute_tool_call supports Elixir functions returning {:ok, "llm result text", elixir_data_to_keep}

lib/chains/llm_chain.ex

@@ -735,6 +735,17 @@ defmodule LangChain.Chains.LLMChain do
       if verbose, do: IO.inspect(function.name, label: "EXECUTING FUNCTION")
 
       case Function.execute(function, call.arguments, context) do
+        {:ok, llm_result, processed_result} ->
+          if verbose, do: IO.inspect(llm_result, label: "FUNCTION RESULT")
+          # successful execution and storage of processed_content.
+          ToolResult.new!(%{
+            tool_call_id: call.call_id,
+            content: llm_result,
+            processed_content: processed_result,
+            name: function.name,
+            display_text: function.display_text
+          })
+
         {:ok, result} ->
           if verbose, do: IO.inspect(result, label: "FUNCTION RESULT")
           # successful execution.

lib/function.ex

@@ -18,9 +18,9 @@ defmodule LangChain.Function do
     passed to the function. (Use if greater control or unsupported features are
     needed.)
   * `function` - An Elixir function to execute when an LLM requests to execute
-    the function. The function should return `{:ok, "and a text response"}`,
-    `{:error, "and text explanation of the error"}` or just plain `"text
-    response"`, which is returned to the LLM.
+    the function. The function should return `{:ok, "text for LLM"}`, `{:ok,
+    "text for LLM", processed_content}`, `{:error, "and text explanation of the
+    error"}` or just plain `"text response"`, which is returned to the LLM.
   * `async` - Boolean value that flags if this can function can be executed
     asynchronously, potentially concurrently with other calls to the same
     function. Defaults to `true`.
@@ -60,8 +60,8 @@ defmodule LangChain.Function do
   `LangChain.Chains.LLMChain`. This is whatever context data is needed for the
   function to do it's work.
 
-  Context examples may be user_id, account_id, account struct, billing level,
-  etc.
+  Context examples could be data like user_id, account_id, account struct,
+  billing level, etc.
 
   ## Function Parameters
 
@@ -111,6 +111,58 @@ defmodule LangChain.Function do
 
   The `LangChain.FunctionParam` is nestable allowing for arrays of object and
   objects with nested objects.
+
+  ## Example that also stores the Elixir result
+
+  Sometimes we want to process a `ToolCall` from the LLM and keep the processed
+  Elixir data for ourselves. This is particularly useful when using an LLM to
+  perform structured data extraction. Our Elixir function may even process that
+  data into a newly created Ecto Schema database entry. The result of the
+  `ToolCall` that goes back to the LLM must be in a string form. That typically
+  means returning a JSON string of the result data.
+
+  To make it easier to process the data, return a string response to the LLM,
+  but **keep** the original Elixir data as well, our Elixir function can return
+  a 3-tuple result.
+
+      Function.new!(%{name: "create_invoice",
+        parameters: [
+          FunctionParam.new!(%{name: "vendor_name", type: :string, required: true})
+          FunctionParam.new!(%{name: "total_amount", type: :string, required: true})
+        ],
+        function: &execute_create_invoice/2
+      })
+
+      # ...
+
+      def execute_create_invoice(args, %{account_id: account_id} = _context) do
+        case MyApp.Invoices.create_invoice(account_id, args) do
+          {:ok, invoice} ->
+            {:ok, "SUCCESS", invoice}
+
+          {:error, changeset} ->
+            {:error, "ERROR: " <> LangChain.Utils.changeset_error_to_string(changeset)}
+        end
+      end
+
+  In this example, the `LangChain.Function` is tied to the
+  `MyApp.Invoices.create_invoice/2` function in our application.
+
+  The Elixir function returns a 3-tuple result. The `"SUCCESS"` is returned to
+  the LLM. In our scenario, we don't care to return a JSON version of the
+  invoice. The important part is we return the actual
+  `%MyApp.Invoices.Invoice{}` struct in the tuple. This is stored on the
+  `LangChain.ToolResult`'s `processed_content` field.
+
+  This is really helpful when all we want is the final, fully processed Elixir
+  result. This pairs well with the `LLMChain.run(chain, mode: :until_success)`.
+  This is when we want the LLM to perform some data extraction and it should be
+  re-run until it succeeds and we have our final, processed result in the
+  `ToolResult`.
+
+  Note: The LLM may issue one or more `ToolCall`s in a single assistant message.
+  Each Elixir function's `ToolResult` may contain a `processed_content`.
+
   """
   use Ecto.Schema
   import Ecto.Changeset
@@ -202,6 +254,10 @@ defmodule LangChain.Function do
     try do
       # execute the function and normalize the results. Want :ok/:error tuples
       case fun.(arguments, context) do
+        {:ok, llm_result, processed_content} ->
+          # successful execution with additional processed_content.
+          {:ok, llm_result, processed_content}
+
         {:ok, result} ->
           # successful execution.
           {:ok, result}

lib/message.ex

@@ -36,6 +36,21 @@ defmodule LangChain.Message do
   support it, and they may require using specific models trained for it. See the
   documentation for the LLM or service for details on their level of support.
 
+  ## Processed Content
+
+  The `processed_content` field is a handy place to store the results of
+  processing a message and needing to hold on to the processed value and store
+  it with the message.
+
+  This is particularly helpful for a `LangChain.MessageProcessors.JsonProcessor`
+  that can process an assistant message and store the processed value on the
+  message itself.
+
+  It is intended for assistant messages when a message processor is applied.
+  This contains the results of the processing. This allows the `content` to
+  reflect what was actually returned from the LLM so it can easily be sent back
+  to the LLM as a part of the entire conversation.
+
   ## Examples
 
   A basic system message example:
@@ -71,10 +86,6 @@ defmodule LangChain.Message do
   embedded_schema do
     # Message content that the LLM sees.
     field :content, :any, virtual: true
-    # For assistant messages when message_processors are applied. This contains
-    # the results of the processing. This allows the `content` to reflect what
-    # was actually returned for when we send it back to the LLM as a historical
-    # message.
     field :processed_content, :any, virtual: true
     field :index, :integer
     field :status, Ecto.Enum, values: [:complete, :cancelled, :length], default: :complete

lib/message/tool_result.ex

@@ -3,6 +3,18 @@ defmodule LangChain.Message.ToolResult do
   Represents a the result of running a requested tool. The LLM's requests a tool
   use through a `ToolCall`. A `ToolResult` returns the answer or result from the
   application back to the AI.
+
+  ## Content
+  The `content` is a string that gets returned to the LLM as the result.
+
+  ## Processed Content
+  The `processed_content` field is optional. When you want to keep the results
+  of the Elixir function call as a native Elixir data structure,
+  `processed_content` can hold it.
+
+  To do this, the Elixir function's result should be a `{:ok, "String response
+  for LLM", native_elixir_data}`. See `LangChain.Function` for details and
+  examples.
   """
   use Ecto.Schema
   import Ecto.Changeset
@@ -20,6 +32,8 @@ defmodule LangChain.Message.ToolResult do
     field :name, :string
     # the content returned to the LLM/AI.
     field :content, :string
+    # optional stored results of tool result
+    field :processed_content, :any, virtual: true
     # Text to display in a UI for the result. Optional.
     field :display_text, :string
     # flag if the result is an error
@@ -28,7 +42,15 @@ defmodule LangChain.Message.ToolResult do
 
   @type t :: %ToolResult{}
 
-  @update_fields [:type, :tool_call_id, :name, :content, :display_text, :is_error]
+  @update_fields [
+    :type,
+    :tool_call_id,
+    :name,
+    :content,
+    :processed_content,
+    :display_text,
+    :is_error
+  ]
   @create_fields @update_fields
   @required_fields [:type, :tool_call_id, :content]
 

lib/message_processors/json_processor.ex

@@ -5,7 +5,7 @@ defmodule LangChain.MessageProcessors.JsonProcessor do
 
   When successful, the assistant message's JSON contents are processed into a
   map and set on `processed_content`. No additional validation or processing of
-  the data is done in by this processor.
+  the data is done by this processor.
 
   When JSON data is expected but not received, or the received JSON is invalid
   or incomplete, a new user `Message` struct is returned with a text error

test/chains/llm_chain_test.exs

@@ -53,6 +53,21 @@ defmodule LangChain.Chains.LLMChainTest do
         end
       })
 
+    get_date =
+      Function.new!(%{
+        name: "get_date",
+        description: "Returns the date as YYYY-MM-DD",
+        function: fn _args, _context ->
+          # return as a formatted string and the date struct
+          date = Date.new!(2024, 11, 1)
+
+          # Format the date as YYYY-MM-DD
+          formatted_date = Calendar.strftime(date, "%Y-%m-%d")
+
+          {:ok, formatted_date, date}
+        end
+      })
+
     # on setup, delete the Process dictionary key for each test run
     Process.delete(:test_func_failed_once)
 
@@ -85,7 +100,8 @@ defmodule LangChain.Chains.LLMChainTest do
       greet: greet,
       sync: sync,
       fail_func: fail_func,
-      fail_once: fail_once
+      fail_once: fail_once,
+      get_date: get_date
     }
   end
 
@@ -1622,6 +1638,20 @@ defmodule LangChain.Chains.LLMChainTest do
       # resets the current_failure_count after processing successfully
       assert updated_chain.current_failure_count == 0
     end
+
+    test "supports returning processed_content to ToolResult", %{chain: chain, get_date: get_date} do
+      chain =
+        chain
+        |> LLMChain.add_tools(get_date)
+        |> LLMChain.add_message(new_function_call!("call_fake123", "get_date", "{}"))
+
+      updated_chain = LLMChain.execute_tool_calls(chain)
+      # get the 1 expected tool result
+      %Message{role: :tool, tool_results: [%ToolResult{} = result]} = updated_chain.last_message
+      assert result.name == "get_date"
+      assert result.content == "2024-11-01"
+      assert result.processed_content == ~D[2024-11-01]
+    end
   end
 
   describe "add_callback/2" do