From ce0d4d73061153a7a1653ae7989c99d0a3f47fcd Mon Sep 17 00:00:00 2001
From: Eth3rnit3 <eth3rnit3@gmail.com>
Date: Tue, 3 Dec 2024 22:13:41 +0100
Subject: [PATCH 01/20] Feat handle image_url from tools output

---
 lib/langchain/assistant.rb                 | 16 ++++++++++++++-
 spec/langchain/assistant/assistant_spec.rb | 23 +++++++++++++++++++++-
 2 files changed, 37 insertions(+), 2 deletions(-)

diff --git a/lib/langchain/assistant.rb b/lib/langchain/assistant.rb
index 25411fff..9280273f 100644
--- a/lib/langchain/assistant.rb
+++ b/lib/langchain/assistant.rb
@@ -177,7 +177,21 @@ def add_message_and_run!(content: nil, image_url: nil)
     # @return [Array<Langchain::Message>] The messages
     def submit_tool_output(tool_call_id:, output:)
       # TODO: Validate that `tool_call_id` is valid by scanning messages and checking if this tool call ID was invoked
-      add_message(role: @llm_adapter.tool_role, content: output, tool_call_id: tool_call_id)
+      content, image_url = parse_tool_output(output: output)
+      add_message(role: @llm_adapter.tool_role, content: content, tool_call_id: tool_call_id, image_url: image_url)
+    end
+
+    def parse_tool_output(output:)
+      image_url = nil
+      content   = nil
+      if output.is_a?(Hash)
+        image_url = output[:image_url]
+        content   = output[:content]
+      else
+        content = output
+      end
+      
+      [content, image_url]
     end
 
     # Delete all messages
diff --git a/spec/langchain/assistant/assistant_spec.rb b/spec/langchain/assistant/assistant_spec.rb
index 432d4df9..1a2fbb66 100644
--- a/spec/langchain/assistant/assistant_spec.rb
+++ b/spec/langchain/assistant/assistant_spec.rb
@@ -201,6 +201,13 @@
         expect(subject.messages.last.role).to eq("tool")
         expect(subject.messages.last.content).to eq("bar")
       end
+
+      it "adds an image to the message" do
+        subject.submit_tool_output(tool_call_id: "123", output: { image_url: "https://example.com/image.jpg", content: "Hello" })
+        expect(subject.messages.last.role).to eq("tool")
+        expect(subject.messages.last.content).to eq("Hello")
+        expect(subject.messages.last.image_url).to eq("https://example.com/image.jpg")
+      end
     end
 
     describe "#run" do
@@ -568,6 +575,13 @@
         expect(subject.messages.last.role).to eq("tool")
         expect(subject.messages.last.content).to eq("bar")
       end
+
+      it "adds an image to the message" do
+        subject.submit_tool_output(tool_call_id: "123", output: { image_url: "https://example.com/image.jpg", content: "Hello" })
+        expect(subject.messages.last.role).to eq("tool")
+        expect(subject.messages.last.content).to eq("Hello")
+        expect(subject.messages.last.image_url).to eq("https://example.com/image.jpg")
+      end
     end
 
     describe "#run" do
@@ -916,12 +930,19 @@
       end
     end
 
-    describe "submit_tool_output" do
+    describe "#submit_tool_output" do
       it "adds a message to the thread" do
         subject.submit_tool_output(tool_call_id: "123", output: "bar")
         expect(subject.messages.last.role).to eq("function")
         expect(subject.messages.last.content).to eq("bar")
       end
+
+      it "does not add image to the message" do
+        subject.submit_tool_output(tool_call_id: "123", output: { image_url: "https://example.com/image.jpg", content: "Hello" })
+        expect(subject.messages.last.role).to eq("function")
+        expect(subject.messages.last.content).to eq("Hello")
+        expect(subject.messages.last.image_url).to be_nil
+      end
     end
 
     describe "#run" do

From eb9584b1ea17b53cd37f2eaab4b9b048d203b76b Mon Sep 17 00:00:00 2001
From: Eth3rnit3 <eth3rnit3@gmail.com>
Date: Tue, 3 Dec 2024 22:33:47 +0100
Subject: [PATCH 02/20] Update assistant.rb

---
 lib/langchain/assistant.rb | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/lib/langchain/assistant.rb b/lib/langchain/assistant.rb
index 9280273f..41e9a8ea 100644
--- a/lib/langchain/assistant.rb
+++ b/lib/langchain/assistant.rb
@@ -181,12 +181,19 @@ def submit_tool_output(tool_call_id:, output:)
       add_message(role: @llm_adapter.tool_role, content: content, tool_call_id: tool_call_id, image_url: image_url)
     end
 
+    # Parses the output of a tool call.
+    # If the output is a hash, it extracts `:content` and `:image_url`.
+    # Otherwise, treats the output as plain content.
+    #
+    # @param output [Hash, String] The tool's output
+    # @return [Array<String, String>] Parsed content and image URL
+
     def parse_tool_output(output:)
       image_url = nil
       content   = nil
-      if output.is_a?(Hash)
-        image_url = output[:image_url]
-        content   = output[:content]
+      if output.is_a?(Hash) && output.key?(:image_url)
+        image_url = output.delete(:image_url)
+        content   = output[:content] || output
       else
         content = output
       end

From f8e4fcf6c3b3c2c572922402e2574639c679400f Mon Sep 17 00:00:00 2001
From: Eth3rnit3 <eth3rnit3@gmail.com>
Date: Wed, 4 Dec 2024 23:35:41 +0100
Subject: [PATCH 03/20] better handling of image_url

---
 lib/langchain/assistant.rb                    | 30 ++++---------------
 .../assistant/messages/anthropic_message.rb   |  2 +-
 .../assistant/messages/mistral_ai_message.rb  |  2 +-
 .../assistant/messages/openai_message.rb      |  2 +-
 spec/langchain/assistant/assistant_spec.rb    | 18 +++++------
 .../messages/anthropic_message_spec.rb        |  7 ++++-
 .../messages/mistral_ai_message_spec.rb       |  2 +-
 7 files changed, 24 insertions(+), 39 deletions(-)

diff --git a/lib/langchain/assistant.rb b/lib/langchain/assistant.rb
index 41e9a8ea..5395fae1 100644
--- a/lib/langchain/assistant.rb
+++ b/lib/langchain/assistant.rb
@@ -173,34 +173,14 @@ def add_message_and_run!(content: nil, image_url: nil)
     # Submit tool output
     #
     # @param tool_call_id [String] The ID of the tool call to submit output for
-    # @param output [String] The output of the tool
+    # @param content [String] The content of the tool call
+    # @param image_url [String] The image URL of the tool call
     # @return [Array<Langchain::Message>] The messages
-    def submit_tool_output(tool_call_id:, output:)
+    def submit_tool_output(tool_call_id:, content:, image_url: nil)
       # TODO: Validate that `tool_call_id` is valid by scanning messages and checking if this tool call ID was invoked
-      content, image_url = parse_tool_output(output: output)
       add_message(role: @llm_adapter.tool_role, content: content, tool_call_id: tool_call_id, image_url: image_url)
     end
 
-    # Parses the output of a tool call.
-    # If the output is a hash, it extracts `:content` and `:image_url`.
-    # Otherwise, treats the output as plain content.
-    #
-    # @param output [Hash, String] The tool's output
-    # @return [Array<String, String>] Parsed content and image URL
-
-    def parse_tool_output(output:)
-      image_url = nil
-      content   = nil
-      if output.is_a?(Hash) && output.key?(:image_url)
-        image_url = output.delete(:image_url)
-        content   = output[:content] || output
-      else
-        content = output
-      end
-      
-      [content, image_url]
-    end
-
     # Delete all messages
     #
     # @return [Array] Empty messages array
@@ -392,9 +372,9 @@ def run_tool(tool_call)
 
       # Call the callback if set
       tool_execution_callback.call(tool_call_id, tool_name, method_name, tool_arguments) if tool_execution_callback # rubocop:disable Style/SafeNavigation
-      output = tool_instance.send(method_name, **tool_arguments)
+      content, image_url = tool_instance.send(method_name, **tool_arguments)
 
-      submit_tool_output(tool_call_id: tool_call_id, output: output)
+      submit_tool_output(tool_call_id: tool_call_id, content: content, image_url: image_url)
     end
 
     # Build a message
diff --git a/lib/langchain/assistant/messages/anthropic_message.rb b/lib/langchain/assistant/messages/anthropic_message.rb
index 70f38209..1285cfeb 100644
--- a/lib/langchain/assistant/messages/anthropic_message.rb
+++ b/lib/langchain/assistant/messages/anthropic_message.rb
@@ -76,7 +76,7 @@ def tool_hash
               {
                 type: "tool_result",
                 tool_use_id: tool_call_id,
-                content: content
+                content: build_content_array
               }
             ]
           }
diff --git a/lib/langchain/assistant/messages/mistral_ai_message.rb b/lib/langchain/assistant/messages/mistral_ai_message.rb
index 2c081d94..ba3de404 100644
--- a/lib/langchain/assistant/messages/mistral_ai_message.rb
+++ b/lib/langchain/assistant/messages/mistral_ai_message.rb
@@ -102,7 +102,7 @@ def system_hash
         def tool_hash
           {
             role: "tool",
-            content: content,
+            content: build_content_array,
             tool_call_id: tool_call_id
           }
         end
diff --git a/lib/langchain/assistant/messages/openai_message.rb b/lib/langchain/assistant/messages/openai_message.rb
index 44e57a34..e19b1246 100644
--- a/lib/langchain/assistant/messages/openai_message.rb
+++ b/lib/langchain/assistant/messages/openai_message.rb
@@ -117,7 +117,7 @@ def tool_hash
           {
             role: "tool",
             tool_call_id: tool_call_id,
-            content: build_content_array
+            content: build_content_array # Using image_url with tools is not supported by OpenAI (Image URLs are only allowed for messages with role 'user', but this message with role 'tool' contains an image URL.)
           }
         end
 
diff --git a/spec/langchain/assistant/assistant_spec.rb b/spec/langchain/assistant/assistant_spec.rb
index 1a2fbb66..e5e96640 100644
--- a/spec/langchain/assistant/assistant_spec.rb
+++ b/spec/langchain/assistant/assistant_spec.rb
@@ -197,13 +197,13 @@
 
     describe "#submit_tool_output" do
       it "adds a message to the thread" do
-        subject.submit_tool_output(tool_call_id: "123", output: "bar")
+        subject.submit_tool_output(tool_call_id: "123", content: "bar")
         expect(subject.messages.last.role).to eq("tool")
         expect(subject.messages.last.content).to eq("bar")
       end
 
       it "adds an image to the message" do
-        subject.submit_tool_output(tool_call_id: "123", output: { image_url: "https://example.com/image.jpg", content: "Hello" })
+        subject.submit_tool_output(tool_call_id: "123", image_url: "https://example.com/image.jpg", content: "Hello")
         expect(subject.messages.last.role).to eq("tool")
         expect(subject.messages.last.content).to eq("Hello")
         expect(subject.messages.last.image_url).to eq("https://example.com/image.jpg")
@@ -571,13 +571,13 @@
 
     describe "#submit_tool_output" do
       it "adds a message to the thread" do
-        subject.submit_tool_output(tool_call_id: "123", output: "bar")
+        subject.submit_tool_output(tool_call_id: "123", content: "bar")
         expect(subject.messages.last.role).to eq("tool")
         expect(subject.messages.last.content).to eq("bar")
       end
 
       it "adds an image to the message" do
-        subject.submit_tool_output(tool_call_id: "123", output: { image_url: "https://example.com/image.jpg", content: "Hello" })
+        subject.submit_tool_output(tool_call_id: "123", image_url: "https://example.com/image.jpg", content: "Hello")
         expect(subject.messages.last.role).to eq("tool")
         expect(subject.messages.last.content).to eq("Hello")
         expect(subject.messages.last.image_url).to eq("https://example.com/image.jpg")
@@ -672,7 +672,7 @@
                     "type" => "function"
                   }
                 ]},
-                {content: "4.0", role: "tool", tool_call_id: "call_9TewGANaaIjzY31UCpAAGLeV"}
+                {content: [{type: "text", text: "4.0"}], role: "tool", tool_call_id: "call_9TewGANaaIjzY31UCpAAGLeV"}
               ],
               tools: calculator.class.function_schemas.to_openai_format,
               tool_choice: "auto"
@@ -932,13 +932,13 @@
 
     describe "#submit_tool_output" do
       it "adds a message to the thread" do
-        subject.submit_tool_output(tool_call_id: "123", output: "bar")
+        subject.submit_tool_output(tool_call_id: "123", content: "bar")
         expect(subject.messages.last.role).to eq("function")
         expect(subject.messages.last.content).to eq("bar")
       end
 
       it "does not add image to the message" do
-        subject.submit_tool_output(tool_call_id: "123", output: { image_url: "https://example.com/image.jpg", content: "Hello" })
+        subject.submit_tool_output(tool_call_id: "123", content: "Hello", image_url: "https://example.com/image.jpg")
         expect(subject.messages.last.role).to eq("function")
         expect(subject.messages.last.content).to eq("Hello")
         expect(subject.messages.last.image_url).to be_nil
@@ -1121,7 +1121,7 @@
 
     describe "submit_tool_output" do
       it "adds a message to the thread" do
-        subject.submit_tool_output(tool_call_id: "123", output: "bar")
+        subject.submit_tool_output(tool_call_id: "123", content: "bar")
         expect(subject.messages.last.role).to eq("tool_result")
         expect(subject.messages.last.content).to eq("bar")
       end
@@ -1239,7 +1239,7 @@
                     "input" => {"input" => "2+2"}
                   }
                 ]},
-                {role: "user", content: [{type: "tool_result", tool_use_id: "toolu_014eSx9oBA5DMe8gZqaqcJ3H", content: "4.0"}]}
+                {role: "user", content: [{type: "tool_result", tool_use_id: "toolu_014eSx9oBA5DMe8gZqaqcJ3H", content: [{type: "text", text: "4.0"}]}]}
               ],
               tools: calculator.class.function_schemas.to_anthropic_format,
               tool_choice: {disable_parallel_tool_use: false, type: "auto"},
diff --git a/spec/langchain/assistant/messages/anthropic_message_spec.rb b/spec/langchain/assistant/messages/anthropic_message_spec.rb
index a1f04924..179727fb 100644
--- a/spec/langchain/assistant/messages/anthropic_message_spec.rb
+++ b/spec/langchain/assistant/messages/anthropic_message_spec.rb
@@ -108,7 +108,12 @@
               {
                 type: "tool_result",
                 tool_use_id: "toolu_014eSx9oBA5DMe8gZqaqcJ3H",
-                content: "4.0"
+                content: [
+                  {
+                    type: "text",
+                    text: "4.0"
+                  }
+                ]
               }
             ]
           }
diff --git a/spec/langchain/assistant/messages/mistral_ai_message_spec.rb b/spec/langchain/assistant/messages/mistral_ai_message_spec.rb
index 9f4af833..33211693 100644
--- a/spec/langchain/assistant/messages/mistral_ai_message_spec.rb
+++ b/spec/langchain/assistant/messages/mistral_ai_message_spec.rb
@@ -18,7 +18,7 @@
       let(:message) { described_class.new(role: "tool", content: "Hello, world!", tool_calls: [], tool_call_id: "123") }
 
       it "returns a hash with the tool_call_id key" do
-        expect(message.to_hash).to eq({role: "tool", content: "Hello, world!", tool_call_id: "123"})
+        expect(message.to_hash).to eq({role: "tool", content: [{text: "Hello, world!", type: "text"}], tool_call_id: "123"})
       end
     end
 

From ed045b556cc00c7f557d4623aef5f7cacdd91a22 Mon Sep 17 00:00:00 2001
From: Eth3rnit3 <eth3rnit3@gmail.com>
Date: Thu, 5 Dec 2024 00:12:07 +0100
Subject: [PATCH 04/20] add specs

---
 .../messages/anthropic_message_spec.rb        | 25 +++++++++++++++++++
 .../messages/mistral_ai_message_spec.rb       | 23 ++++++++++++++++-
 .../assistant/messages/openai_message_spec.rb | 15 +++++++++++
 3 files changed, 62 insertions(+), 1 deletion(-)

diff --git a/spec/langchain/assistant/messages/anthropic_message_spec.rb b/spec/langchain/assistant/messages/anthropic_message_spec.rb
index 179727fb..ef8f132a 100644
--- a/spec/langchain/assistant/messages/anthropic_message_spec.rb
+++ b/spec/langchain/assistant/messages/anthropic_message_spec.rb
@@ -119,6 +119,31 @@
           }
         )
       end
+
+      it "returns tool_hash with image_url" do
+        message = described_class.new(role: "tool_result", image_url: "https://example.com/image.jpg")
+        allow(message).to receive(:image).and_return(double(base64: "base64_data", mime_type: "image/jpeg"))
+
+        expect(message.to_hash).to eq(
+          role: "user",
+          content: [
+            {
+              type: "tool_result",
+              tool_use_id: nil,
+              content: [
+                {
+                  type: "image",
+                  source: {
+                    type: "base64",
+                    data: "base64_data",
+                    media_type: "image/jpeg"
+                  }
+                }
+              ]
+            }
+          ]
+        )
+      end
     end
 
     context "when role is user" do
diff --git a/spec/langchain/assistant/messages/mistral_ai_message_spec.rb b/spec/langchain/assistant/messages/mistral_ai_message_spec.rb
index 33211693..e02f4316 100644
--- a/spec/langchain/assistant/messages/mistral_ai_message_spec.rb
+++ b/spec/langchain/assistant/messages/mistral_ai_message_spec.rb
@@ -36,7 +36,7 @@
       end
     end
 
-    context "when image_url is present" do
+    context "when image_url is present in user message" do
       let(:message) { described_class.new(role: "user", content: "Please describe this image", image_url: "https://example.com/image.jpg") }
 
       it "returns a hash with the image_url key" do
@@ -49,5 +49,26 @@
         })
       end
     end
+
+    context "when image_url is present in tool message" do
+      let(:tool_call) {
+        {"id" => "call_9TewGANaaIjzY31UCpAAGLeV",
+         "type" => "function",
+         "function" => {"name" => "dummy_tool__take_photo"}}
+      }
+
+      let(:message) { described_class.new(role: "tool", content: "Hello, world!", image_url: "https://example.com/image.jpg",  tool_calls: [tool_call], tool_call_id: "123") }
+
+      it "returns a hash with the image_url key" do
+        expect(message.to_hash).to eq({
+          role: "tool",
+          content: [
+            {text: "Hello, world!", type: "text"},
+            {image_url: "https://example.com/image.jpg", type: "image_url"}
+          ],
+          tool_call_id: "123"
+        })
+      end
+    end
   end
 end
diff --git a/spec/langchain/assistant/messages/openai_message_spec.rb b/spec/langchain/assistant/messages/openai_message_spec.rb
index 6e74bf68..e089053c 100644
--- a/spec/langchain/assistant/messages/openai_message_spec.rb
+++ b/spec/langchain/assistant/messages/openai_message_spec.rb
@@ -43,6 +43,21 @@
       it "returns a tool_hash" do
         expect(message.to_hash).to eq({role: "tool", content: [{type: "text", text: "Hello, world!"}], tool_call_id: "123"})
       end
+
+      context "when image_url is present" do
+        let(:message) { described_class.new(role: "tool", content: "Hello, world!", image_url: "https://example.com/image.jpg", tool_calls: [], tool_call_id: "123") }
+
+        it "returns a tool_hash with the image_url key" do
+          expect(message.to_hash).to eq({
+            role: "tool",
+            content: [
+              {type: "text", text: "Hello, world!"},
+              {type: "image_url", image_url: {url: "https://example.com/image.jpg"}}
+            ],
+            tool_call_id: "123"
+          })
+        end
+      end
     end
 
     context "when role is assistant" do

From 20ac78ca04ef79821d5999f979d6f41f4f28d42e Mon Sep 17 00:00:00 2001
From: Eth3rnit3 <eth3rnit3@gmail.com>
Date: Sun, 26 Jan 2025 14:49:11 +0100
Subject: [PATCH 05/20] add ToolResponse class helper and use it

---
 lib/langchain/tool/calculator.rb            |  5 +-
 lib/langchain/tool/file_system.rb           | 11 +++--
 lib/langchain/tool/google_search.rb         | 28 +++++------
 lib/langchain/tool/ruby_code_interpreter.rb |  2 +-
 lib/langchain/tool/weather.rb               |  6 +--
 lib/langchain/tool/wikipedia.rb             |  2 +-
 lib/langchain/tool_definition.rb            |  4 ++
 lib/langchain/tool_response.rb              | 52 +++++++++++++++++++++
 8 files changed, 84 insertions(+), 26 deletions(-)
 create mode 100644 lib/langchain/tool_response.rb

diff --git a/lib/langchain/tool/calculator.rb b/lib/langchain/tool/calculator.rb
index a5a8add1..e49c9d1b 100644
--- a/lib/langchain/tool/calculator.rb
+++ b/lib/langchain/tool/calculator.rb
@@ -30,9 +30,10 @@ def initialize
     def execute(input:)
       Langchain.logger.debug("#{self.class} - Executing \"#{input}\"")
 
-      Eqn::Calculator.calc(input)
+      result = Eqn::Calculator.calc(input)
+      tool_response(content: result)
     rescue Eqn::ParseError, Eqn::NoVariableValueError
-      "\"#{input}\" is an invalid mathematical expression"
+      tool_response(content: "\"#{input}\" is an invalid mathematical expression")
     end
   end
 end
diff --git a/lib/langchain/tool/file_system.rb b/lib/langchain/tool/file_system.rb
index 910e6382..472a730f 100644
--- a/lib/langchain/tool/file_system.rb
+++ b/lib/langchain/tool/file_system.rb
@@ -24,21 +24,22 @@ class FileSystem
     end
 
     def list_directory(directory_path:)
-      Dir.entries(directory_path)
+      tool_response(content: Dir.entries(directory_path))
     rescue Errno::ENOENT
-      "No such directory: #{directory_path}"
+      tool_response(content: "No such directory: #{directory_path}")
     end
 
     def read_file(file_path:)
-      File.read(file_path)
+      tool_response(content: File.read(file_path))
     rescue Errno::ENOENT
-      "No such file: #{file_path}"
+      tool_response(content: "No such file: #{file_path}")
     end
 
     def write_to_file(file_path:, content:)
       File.write(file_path, content)
+      tool_response(content: "File written successfully")
     rescue Errno::EACCES
-      "Permission denied: #{file_path}"
+      tool_response(content: "Permission denied: #{file_path}")
     end
   end
 end
diff --git a/lib/langchain/tool/google_search.rb b/lib/langchain/tool/google_search.rb
index e4ddb8f1..a13bfa00 100644
--- a/lib/langchain/tool/google_search.rb
+++ b/lib/langchain/tool/google_search.rb
@@ -44,31 +44,31 @@ def execute(input:)
 
       answer_box = results[:answer_box_list] ? results[:answer_box_list].first : results[:answer_box]
       if answer_box
-        return answer_box[:result] ||
+        return tool_response(content: answer_box[:result] ||
             answer_box[:answer] ||
             answer_box[:snippet] ||
             answer_box[:snippet_highlighted_words] ||
-            answer_box.reject { |_k, v| v.is_a?(Hash) || v.is_a?(Array) || v.start_with?("http") }
+            answer_box.reject { |_k, v| v.is_a?(Hash) || v.is_a?(Array) || v.start_with?("http") })
       elsif (events_results = results[:events_results])
-        return events_results.take(10)
+        return tool_response(content: events_results.take(10))
       elsif (sports_results = results[:sports_results])
-        return sports_results
+        return tool_response(content: sports_results)
       elsif (top_stories = results[:top_stories])
-        return top_stories
+        return tool_response(content: top_stories)
       elsif (news_results = results[:news_results])
-        return news_results
+        return tool_response(content: news_results)
       elsif (jobs_results = results.dig(:jobs_results, :jobs))
-        return jobs_results
+        return tool_response(content: jobs_results)
       elsif (shopping_results = results[:shopping_results]) && shopping_results.first.key?(:title)
-        return shopping_results.take(3)
+        return tool_response(content: shopping_results.take(3))
       elsif (questions_and_answers = results[:questions_and_answers])
-        return questions_and_answers
+        return tool_response(content: questions_and_answers)
       elsif (popular_destinations = results.dig(:popular_destinations, :destinations))
-        return popular_destinations
+        return tool_response(content: popular_destinations)
       elsif (top_sights = results.dig(:top_sights, :sights))
-        return top_sights
+        return tool_response(content: top_sights)
       elsif (images_results = results[:images_results]) && images_results.first.key?(:thumbnail)
-        return images_results.map { |h| h[:thumbnail] }.take(10)
+        return tool_response(content: images_results.map { |h| h[:thumbnail] }.take(10))
       end
 
       snippets = []
@@ -110,8 +110,8 @@ def execute(input:)
         snippets << local_results
       end
 
-      return "No good search result found" if snippets.empty?
-      snippets
+      return tool_response(content: "No good search result found") if snippets.empty?
+      tool_response(content: snippets)
     end
 
     #
diff --git a/lib/langchain/tool/ruby_code_interpreter.rb b/lib/langchain/tool/ruby_code_interpreter.rb
index a1962718..ddc616c6 100644
--- a/lib/langchain/tool/ruby_code_interpreter.rb
+++ b/lib/langchain/tool/ruby_code_interpreter.rb
@@ -31,7 +31,7 @@ def initialize(timeout: 30)
     def execute(input:)
       Langchain.logger.debug("#{self.class} - Executing \"#{input}\"")
 
-      safe_eval(input)
+      tool_response(content: safe_eval(input))
     end
 
     def safe_eval(code)
diff --git a/lib/langchain/tool/weather.rb b/lib/langchain/tool/weather.rb
index 4f58266c..ad9d8dfc 100644
--- a/lib/langchain/tool/weather.rb
+++ b/lib/langchain/tool/weather.rb
@@ -55,15 +55,15 @@ def fetch_current_weather(city:, state_code:, country_code:, units:)
       params = {appid: @api_key, q: [city, state_code, country_code].compact.join(","), units: units}
 
       location_response = send_request(path: "geo/1.0/direct", params: params.except(:units))
-      return location_response if location_response.is_a?(String) # Error occurred
+      return tool_response(content: location_response) if location_response.is_a?(String) # Error occurred
 
       location = location_response.first
-      return "Location not found" unless location
+      return tool_response(content: "Location not found") unless location
 
       params = params.merge(lat: location["lat"], lon: location["lon"]).except(:q)
       weather_data = send_request(path: "data/2.5/weather", params: params)
 
-      parse_weather_response(weather_data, units)
+      tool_response(content: parse_weather_response(weather_data, units))
     end
 
     def send_request(path:, params:)
diff --git a/lib/langchain/tool/wikipedia.rb b/lib/langchain/tool/wikipedia.rb
index 52ffbdf7..fab9f819 100644
--- a/lib/langchain/tool/wikipedia.rb
+++ b/lib/langchain/tool/wikipedia.rb
@@ -33,7 +33,7 @@ def execute(input:)
 
       page = ::Wikipedia.find(input)
       # It would be nice to figure out a way to provide page.content but the LLM token limit is an issue
-      page.summary
+      tool_response(content: page.summary)
     end
   end
 end
diff --git a/lib/langchain/tool_definition.rb b/lib/langchain/tool_definition.rb
index f431f3b9..cadbd11a 100644
--- a/lib/langchain/tool_definition.rb
+++ b/lib/langchain/tool_definition.rb
@@ -61,6 +61,10 @@ def tool_name
       .downcase
   end
 
+  def tool_response(content: nil, image_url: nil)
+    Langchain::ToolResponse.new(content: content, image_url: image_url)
+  end
+
   # Manages schemas for functions
   class FunctionSchemas
     def initialize(tool_name)
diff --git a/lib/langchain/tool_response.rb b/lib/langchain/tool_response.rb
new file mode 100644
index 00000000..88b6b034
--- /dev/null
+++ b/lib/langchain/tool_response.rb
@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Langchain
+  # ToolResponse represents the standardized output of a tool.
+  # It can contain either text content or an image URL.
+  class ToolResponse
+    attr_reader :content, :image_url
+
+    # Initializes a new ToolResponse.
+    #
+    # @param content [String] The text content of the response.
+    # @param image_url [String, nil] Optional URL to an image.
+    def initialize(content: nil, image_url: nil)
+      raise ArgumentError, "Either content or image_url must be provided" if content.nil? && image_url.nil?
+
+      @content = content
+      @image_url = image_url
+    end
+
+    # Wraps a raw response in a ToolResponse if it is not already one.
+    #
+    # @param response [String, ToolResponse] The raw response to wrap.
+    # @return [ToolResponse] The wrapped response.
+    def self.wrap(response)
+      return response if response.is_a?(ToolResponse)
+
+      new(content: response)
+    end
+
+    # Converts the response into a format compatible with the given LLM provider.
+    #
+    # @param provider_class [Class] The provider class handling the response.
+    # @return [Hash] The formatted response for the provider.
+    def to_api_format(provider_class)
+      provider_class.format_tool_response(self)
+    end
+
+    # Checks if the response has an image URL.
+    #
+    # @return [Boolean] True if an image URL is present.
+    def has_image?
+      !image_url.nil?
+    end
+
+    # Checks if the response has text content.
+    #
+    # @return [Boolean] True if text content is present.
+    def has_content?
+      !content.nil?
+    end
+  end
+end

From ff05e98c57c6a52af8f4e7343d2f316ddfa21b95 Mon Sep 17 00:00:00 2001
From: Eth3rnit3 <eth3rnit3@gmail.com>
Date: Sun, 26 Jan 2025 15:06:45 +0100
Subject: [PATCH 06/20] Add helpers for response

---
 lib/langchain/assistant.rb                  | 12 ++++++++++--
 lib/langchain/tool/calculator.rb            |  1 +
 lib/langchain/tool/database.rb              | 16 +++++++++++-----
 lib/langchain/tool/file_system.rb           |  1 +
 lib/langchain/tool/google_search.rb         |  1 +
 lib/langchain/tool/news_retriever.rb        | 10 +++++++---
 lib/langchain/tool/ruby_code_interpreter.rb |  1 +
 lib/langchain/tool/tavily.rb                |  3 ++-
 lib/langchain/tool/vectorsearch.rb          |  4 +++-
 lib/langchain/tool/weather.rb               |  1 +
 lib/langchain/tool/wikipedia.rb             |  1 +
 lib/langchain/tool_helpers.rb               |  9 +++++++++
 12 files changed, 48 insertions(+), 12 deletions(-)
 create mode 100644 lib/langchain/tool_helpers.rb

diff --git a/lib/langchain/assistant.rb b/lib/langchain/assistant.rb
index 5395fae1..3bf0d045 100644
--- a/lib/langchain/assistant.rb
+++ b/lib/langchain/assistant.rb
@@ -372,9 +372,17 @@ def run_tool(tool_call)
 
       # Call the callback if set
       tool_execution_callback.call(tool_call_id, tool_name, method_name, tool_arguments) if tool_execution_callback # rubocop:disable Style/SafeNavigation
-      content, image_url = tool_instance.send(method_name, **tool_arguments)
 
-      submit_tool_output(tool_call_id: tool_call_id, content: content, image_url: image_url)
+      result = tool_instance.send(method_name, **tool_arguments)
+      
+      # Handle both ToolResponse and legacy return values
+      if result.is_a?(ToolResponse)
+        submit_tool_output(tool_call_id: tool_call_id, content: result.content, image_url: result.image_url)
+      else
+        # Legacy support for tools returning [content, image_url]
+        content, image_url = result
+        submit_tool_output(tool_call_id: tool_call_id, content: content, image_url: image_url)
+      end
     end
 
     # Build a message
diff --git a/lib/langchain/tool/calculator.rb b/lib/langchain/tool/calculator.rb
index e49c9d1b..b61bcf86 100644
--- a/lib/langchain/tool/calculator.rb
+++ b/lib/langchain/tool/calculator.rb
@@ -14,6 +14,7 @@ module Langchain::Tool
   class Calculator
     extend Langchain::ToolDefinition
     include Langchain::DependencyHelper
+    include Langchain::ToolHelpers
 
     define_function :execute, description: "Evaluates a pure math expression or if equation contains non-math characters (e.g.: \"12F in Celsius\") then it uses the google search calculator to evaluate the expression" do
       property :input, type: "string", description: "Math expression", required: true
diff --git a/lib/langchain/tool/database.rb b/lib/langchain/tool/database.rb
index 8ff9a98c..be5510b6 100644
--- a/lib/langchain/tool/database.rb
+++ b/lib/langchain/tool/database.rb
@@ -13,6 +13,7 @@ module Langchain::Tool
   class Database
     extend Langchain::ToolDefinition
     include Langchain::DependencyHelper
+    include Langchain::ToolHelpers
 
     define_function :list_tables, description: "Database Tool: Returns a list of tables in the database"
 
@@ -51,7 +52,7 @@ def initialize(connection_string:, tables: [], exclude_tables: [])
     #
     # @return [Array<Symbol>] List of tables in the database
     def list_tables
-      db.tables
+      tool_response(content: db.tables)
     end
 
     # Database Tool: Returns the schema for a list of tables
@@ -63,11 +64,13 @@ def describe_tables(tables: [])
 
       Langchain.logger.debug("#{self.class} - Describing tables: #{tables}")
 
-      tables
+      result = tables
         .map do |table|
           describe_table(table)
         end
         .join("\n")
+
+      tool_response(content: result)
     end
 
     # Database Tool: Returns the database schema
@@ -79,7 +82,8 @@ def dump_schema
       schemas = db.tables.map do |table|
         describe_table(table)
       end
-      schemas.join("\n")
+
+      tool_response(content: schemas.join("\n"))
     end
 
     # Database Tool: Executes a SQL query and returns the results
@@ -89,10 +93,10 @@ def dump_schema
     def execute(input:)
       Langchain.logger.debug("#{self.class} - Executing \"#{input}\"")
 
-      db[input].to_a
+      tool_response(content: db[input].to_a)
     rescue Sequel::DatabaseError => e
       Langchain.logger.error("#{self.class} - #{e.message}")
-      e.message # Return error to LLM
+      tool_response(content: e.message)
     end
 
     private
@@ -127,6 +131,8 @@ def describe_table(table)
         schema << ",\n" unless fk == db.foreign_key_list(table).last
       end
       schema << ");\n"
+
+      tool_response(content: schema)
     end
   end
 end
diff --git a/lib/langchain/tool/file_system.rb b/lib/langchain/tool/file_system.rb
index 472a730f..4fbda92e 100644
--- a/lib/langchain/tool/file_system.rb
+++ b/lib/langchain/tool/file_system.rb
@@ -9,6 +9,7 @@ module Langchain::Tool
   #
   class FileSystem
     extend Langchain::ToolDefinition
+    include Langchain::ToolHelpers
 
     define_function :list_directory, description: "File System Tool: Lists out the content of a specified directory" do
       property :directory_path, type: "string", description: "Directory path to list", required: true
diff --git a/lib/langchain/tool/google_search.rb b/lib/langchain/tool/google_search.rb
index a13bfa00..c7ca8919 100644
--- a/lib/langchain/tool/google_search.rb
+++ b/lib/langchain/tool/google_search.rb
@@ -14,6 +14,7 @@ module Langchain::Tool
   class GoogleSearch
     extend Langchain::ToolDefinition
     include Langchain::DependencyHelper
+    include Langchain::ToolHelpers
 
     define_function :execute, description: "Executes Google Search and returns the result" do
       property :input, type: "string", description: "Search query", required: true
diff --git a/lib/langchain/tool/news_retriever.rb b/lib/langchain/tool/news_retriever.rb
index c82a53c4..055c743c 100644
--- a/lib/langchain/tool/news_retriever.rb
+++ b/lib/langchain/tool/news_retriever.rb
@@ -10,6 +10,7 @@ module Langchain::Tool
   #
   class NewsRetriever
     extend Langchain::ToolDefinition
+    include Langchain::ToolHelpers
 
     define_function :get_everything, description: "News Retriever: Search through millions of articles from over 150,000 large and small news sources and blogs" do
       property :q, type: "string", description: 'Keywords or phrases to search for in the article title and body. Surround phrases with quotes (") for exact match. Alternatively you can use the AND / OR / NOT keywords, and optionally group these with parenthesis. Must be URL-encoded'
@@ -86,7 +87,8 @@ def get_everything(
       params[:pageSize] = page_size if page_size
       params[:page] = page if page
 
-      send_request(path: "everything", params: params)
+      response = send_request(path: "everything", params: params)
+      tool_response(content: response)
     end
 
     # Retrieve top headlines
@@ -117,7 +119,8 @@ def get_top_headlines(
       params[:pageSize] = page_size if page_size
       params[:page] = page if page
 
-      send_request(path: "top-headlines", params: params)
+      response = send_request(path: "top-headlines", params: params)
+      tool_response(content: response)
     end
 
     # Retrieve news sources
@@ -139,7 +142,8 @@ def get_sources(
       params[:category] = category if category
       params[:language] = language if language
 
-      send_request(path: "top-headlines/sources", params: params)
+      response = send_request(path: "top-headlines/sources", params: params)
+      tool_response(content: response)
     end
 
     private
diff --git a/lib/langchain/tool/ruby_code_interpreter.rb b/lib/langchain/tool/ruby_code_interpreter.rb
index ddc616c6..fada47e5 100644
--- a/lib/langchain/tool/ruby_code_interpreter.rb
+++ b/lib/langchain/tool/ruby_code_interpreter.rb
@@ -13,6 +13,7 @@ module Langchain::Tool
   class RubyCodeInterpreter
     extend Langchain::ToolDefinition
     include Langchain::DependencyHelper
+    include Langchain::ToolHelpers
 
     define_function :execute, description: "Executes Ruby code in a sandboxes environment" do
       property :input, type: "string", description: "Ruby code expression", required: true
diff --git a/lib/langchain/tool/tavily.rb b/lib/langchain/tool/tavily.rb
index ff86944b..4986ac55 100644
--- a/lib/langchain/tool/tavily.rb
+++ b/lib/langchain/tool/tavily.rb
@@ -10,6 +10,7 @@ module Langchain::Tool
   #
   class Tavily
     extend Langchain::ToolDefinition
+    include Langchain::ToolHelpers
 
     define_function :search, description: "Tavily Tool: Robust search API" do
       property :query, type: "string", description: "The search query string", required: true
@@ -70,7 +71,7 @@ def search(
       response = Net::HTTP.start(uri.hostname, uri.port, use_ssl: uri.scheme == "https") do |http|
         http.request(request)
       end
-      response.body
+      tool_response(content: response.body)
     end
   end
 end
diff --git a/lib/langchain/tool/vectorsearch.rb b/lib/langchain/tool/vectorsearch.rb
index 929a8803..f81df538 100644
--- a/lib/langchain/tool/vectorsearch.rb
+++ b/lib/langchain/tool/vectorsearch.rb
@@ -14,6 +14,7 @@ module Langchain::Tool
   #
   class Vectorsearch
     extend Langchain::ToolDefinition
+    include Langchain::ToolHelpers
 
     define_function :similarity_search, description: "Vectorsearch: Retrieves relevant document for the query" do
       property :query, type: "string", description: "Query to find similar documents for", required: true
@@ -34,7 +35,8 @@ def initialize(vectorsearch:)
     # @param query [String] The query to search for
     # @param k [Integer] The number of results to return
     def similarity_search(query:, k: 4)
-      vectorsearch.similarity_search(query:, k: 4)
+      result = vectorsearch.similarity_search(query:, k: 4)
+      tool_response(content: result)
     end
   end
 end
diff --git a/lib/langchain/tool/weather.rb b/lib/langchain/tool/weather.rb
index ad9d8dfc..fca9f5d6 100644
--- a/lib/langchain/tool/weather.rb
+++ b/lib/langchain/tool/weather.rb
@@ -16,6 +16,7 @@ module Langchain::Tool
   #
   class Weather
     extend Langchain::ToolDefinition
+    include Langchain::ToolHelpers
 
     define_function :get_current_weather, description: "Returns current weather for a city" do
       property :city,
diff --git a/lib/langchain/tool/wikipedia.rb b/lib/langchain/tool/wikipedia.rb
index fab9f819..c82c5fb3 100644
--- a/lib/langchain/tool/wikipedia.rb
+++ b/lib/langchain/tool/wikipedia.rb
@@ -14,6 +14,7 @@ module Langchain::Tool
   class Wikipedia
     extend Langchain::ToolDefinition
     include Langchain::DependencyHelper
+    include Langchain::ToolHelpers
 
     define_function :execute, description: "Executes Wikipedia API search and returns the answer" do
       property :input, type: "string", description: "Search query", required: true
diff --git a/lib/langchain/tool_helpers.rb b/lib/langchain/tool_helpers.rb
new file mode 100644
index 00000000..e74c54c0
--- /dev/null
+++ b/lib/langchain/tool_helpers.rb
@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Langchain
+  module ToolHelpers
+    def tool_response(content: nil, image_url: nil)
+      Langchain::ToolResponse.new(content: content, image_url: image_url)
+    end
+  end
+end

From 54e0f0eec4db6820dd44139a22849608f7726bd3 Mon Sep 17 00:00:00 2001
From: Eth3rnit3 <eth3rnit3@gmail.com>
Date: Sun, 26 Jan 2025 15:25:57 +0100
Subject: [PATCH 07/20] update response type

---
 lib/langchain/tool/calculator.rb            |  4 ++--
 lib/langchain/tool/database.rb              | 10 +++++-----
 lib/langchain/tool/google_search.rb         |  2 +-
 lib/langchain/tool/news_retriever.rb        |  6 +++---
 lib/langchain/tool/ruby_code_interpreter.rb |  2 +-
 lib/langchain/tool/tavily.rb                |  2 +-
 lib/langchain/tool/vectorsearch.rb          |  1 +
 lib/langchain/tool/wikipedia.rb             |  2 +-
 8 files changed, 15 insertions(+), 14 deletions(-)

diff --git a/lib/langchain/tool/calculator.rb b/lib/langchain/tool/calculator.rb
index b61bcf86..1be3fc96 100644
--- a/lib/langchain/tool/calculator.rb
+++ b/lib/langchain/tool/calculator.rb
@@ -27,12 +27,12 @@ def initialize
     # Evaluates a pure math expression or if equation contains non-math characters (e.g.: "12F in Celsius") then it uses the google search calculator to evaluate the expression
     #
     # @param input [String] math expression
-    # @return [String] Answer
+    # @return [Langchain::Tool::Response] Answer
     def execute(input:)
       Langchain.logger.debug("#{self.class} - Executing \"#{input}\"")
 
       result = Eqn::Calculator.calc(input)
-      tool_response(content: result)
+      tool_response(content: result.to_s)
     rescue Eqn::ParseError, Eqn::NoVariableValueError
       tool_response(content: "\"#{input}\" is an invalid mathematical expression")
     end
diff --git a/lib/langchain/tool/database.rb b/lib/langchain/tool/database.rb
index be5510b6..83bc478b 100644
--- a/lib/langchain/tool/database.rb
+++ b/lib/langchain/tool/database.rb
@@ -50,7 +50,7 @@ def initialize(connection_string:, tables: [], exclude_tables: [])
 
     # Database Tool: Returns a list of tables in the database
     #
-    # @return [Array<Symbol>] List of tables in the database
+    # @return [Langchain::Tool::Response] List of tables in the database
     def list_tables
       tool_response(content: db.tables)
     end
@@ -58,7 +58,7 @@ def list_tables
     # Database Tool: Returns the schema for a list of tables
     #
     # @param tables [Array<String>] The tables to describe.
-    # @return [String] The schema for the tables
+    # @return [Langchain::Tool::Response] The schema for the tables
     def describe_tables(tables: [])
       return "No tables specified" if tables.empty?
 
@@ -75,7 +75,7 @@ def describe_tables(tables: [])
 
     # Database Tool: Returns the database schema
     #
-    # @return [String] Database schema
+    # @return [Langchain::Tool::Response] Database schema
     def dump_schema
       Langchain.logger.debug("#{self.class} - Dumping schema tables and keys")
 
@@ -89,7 +89,7 @@ def dump_schema
     # Database Tool: Executes a SQL query and returns the results
     #
     # @param input [String] SQL query to be executed
-    # @return [Array] Results from the SQL query
+    # @return [Langchain::Tool::Response] Results from the SQL query
     def execute(input:)
       Langchain.logger.debug("#{self.class} - Executing \"#{input}\"")
 
@@ -104,7 +104,7 @@ def execute(input:)
     # Describes a table and its schema
     #
     # @param table [String] The table to describe
-    # @return [String] The schema for the table
+    # @return [Langchain::Tool::Response] The schema for the table
     def describe_table(table)
       # TODO: There's probably a clear way to do all of this below
 
diff --git a/lib/langchain/tool/google_search.rb b/lib/langchain/tool/google_search.rb
index c7ca8919..5c17d2a1 100644
--- a/lib/langchain/tool/google_search.rb
+++ b/lib/langchain/tool/google_search.rb
@@ -37,7 +37,7 @@ def initialize(api_key:)
     # Executes Google Search and returns the result
     #
     # @param input [String] search query
-    # @return [String] Answer
+    # @return [Langchain::Tool::Response] Answer
     def execute(input:)
       Langchain.logger.debug("#{self.class} - Executing \"#{input}\"")
 
diff --git a/lib/langchain/tool/news_retriever.rb b/lib/langchain/tool/news_retriever.rb
index 055c743c..a5908b8a 100644
--- a/lib/langchain/tool/news_retriever.rb
+++ b/lib/langchain/tool/news_retriever.rb
@@ -58,7 +58,7 @@ def initialize(api_key: ENV["NEWS_API_KEY"])
     # @param page_size [Integer] The number of results to return per page. 20 is the API's default, 100 is the maximum. Our default is 5.
     # @param page [Integer] Use this to page through the results.
     #
-    # @return [String] JSON response
+    # @return [Langchain::Tool::Response] JSON response
     def get_everything(
       q: nil,
       search_in: nil,
@@ -100,7 +100,7 @@ def get_everything(
     # @param page_size [Integer] The number of results to return per page. 20 is the API's default, 100 is the maximum. Our default is 5.
     # @param page [Integer] Use this to page through the results.
     #
-    # @return [String] JSON response
+    # @return [Langchain::Tool::Response] JSON response
     def get_top_headlines(
       country: nil,
       category: nil,
@@ -129,7 +129,7 @@ def get_top_headlines(
     # @param language [String] The 2-letter ISO-639-1 code of the language you want to get headlines for. Possible options: ar, de, en, es, fr, he, it, nl, no, pt, ru, se, ud, zh.
     # @param country [String] The 2-letter ISO 3166-1 code of the country you want to get headlines for. Possible options: ae, ar, at, au, be, bg, br, ca, ch, cn, co, cu, cz, de, eg, fr, gb, gr, hk, hu, id, ie, il, in, it, jp, kr, lt, lv, ma, mx, my, ng, nl, no, nz, ph, pl, pt, ro, rs, ru, sa, se, sg, si, sk, th, tr, tw, ua, us, ve, za.
     #
-    # @return [String] JSON response
+    # @return [Langchain::Tool::Response] JSON response
     def get_sources(
       category: nil,
       language: nil,
diff --git a/lib/langchain/tool/ruby_code_interpreter.rb b/lib/langchain/tool/ruby_code_interpreter.rb
index fada47e5..0a01b489 100644
--- a/lib/langchain/tool/ruby_code_interpreter.rb
+++ b/lib/langchain/tool/ruby_code_interpreter.rb
@@ -28,7 +28,7 @@ def initialize(timeout: 30)
     # Executes Ruby code in a sandboxes environment.
     #
     # @param input [String] ruby code expression
-    # @return [String] Answer
+    # @return [Langchain::Tool::Response] Answer
     def execute(input:)
       Langchain.logger.debug("#{self.class} - Executing \"#{input}\"")
 
diff --git a/lib/langchain/tool/tavily.rb b/lib/langchain/tool/tavily.rb
index 4986ac55..0f69856f 100644
--- a/lib/langchain/tool/tavily.rb
+++ b/lib/langchain/tool/tavily.rb
@@ -42,7 +42,7 @@ def initialize(api_key:)
     # @param include_domains [Array<String>] A list of domains to specifically include in the search results. Default is None, which includes all domains.
     # @param exclude_domains [Array<String>] A list of domains to specifically exclude from the search results. Default is None, which doesn't exclude any domains.
     #
-    # @return [String] The search results in JSON format.
+    # @return [Langchain::Tool::Response] The search results in JSON format.
     def search(
       query:,
       search_depth: "basic",
diff --git a/lib/langchain/tool/vectorsearch.rb b/lib/langchain/tool/vectorsearch.rb
index f81df538..6619939d 100644
--- a/lib/langchain/tool/vectorsearch.rb
+++ b/lib/langchain/tool/vectorsearch.rb
@@ -34,6 +34,7 @@ def initialize(vectorsearch:)
     #
     # @param query [String] The query to search for
     # @param k [Integer] The number of results to return
+    # @return [Langchain::Tool::Response] The response from the server
     def similarity_search(query:, k: 4)
       result = vectorsearch.similarity_search(query:, k: 4)
       tool_response(content: result)
diff --git a/lib/langchain/tool/wikipedia.rb b/lib/langchain/tool/wikipedia.rb
index c82c5fb3..45f5424c 100644
--- a/lib/langchain/tool/wikipedia.rb
+++ b/lib/langchain/tool/wikipedia.rb
@@ -28,7 +28,7 @@ def initialize
     # Executes Wikipedia API search and returns the answer
     #
     # @param input [String] search query
-    # @return [String] Answer
+    # @return [Langchain::Tool::Response] Answer
     def execute(input:)
       Langchain.logger.debug("#{self.class} - Executing \"#{input}\"")
 

From 4be896b1fbebe1d9a1cc13d4b8dd4fdb339cbdb1 Mon Sep 17 00:00:00 2001
From: Eth3rnit3 <eth3rnit3@gmail.com>
Date: Sun, 26 Jan 2025 15:27:56 +0100
Subject: [PATCH 08/20] Update tool_response.rb

---
 lib/langchain/tool_response.rb | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/lib/langchain/tool_response.rb b/lib/langchain/tool_response.rb
index 88b6b034..40aa814b 100644
--- a/lib/langchain/tool_response.rb
+++ b/lib/langchain/tool_response.rb
@@ -48,5 +48,17 @@ def has_image?
     def has_content?
       !content.nil?
     end
+
+    def to_s
+      content.to_s
+    end
+
+    def to_str
+      to_s
+    end
+
+    def include?(other)
+      to_s.include?(other)
+    end
   end
 end

From f5eeb407bbfc8e3c0b1579a55774a3353b6f2b7d Mon Sep 17 00:00:00 2001
From: Eth3rnit3 <eth3rnit3@gmail.com>
Date: Sun, 26 Jan 2025 15:40:29 +0100
Subject: [PATCH 09/20] update specs

---
 lib/langchain/tool/calculator.rb          |  2 +-
 lib/langchain/tool_response.rb            | 32 ------------------
 spec/langchain/tool/calculator_spec.rb    | 10 +++---
 spec/langchain/tool/database_spec.rb      | 16 ++++++---
 spec/langchain/tool/file_system_spec.rb   | 18 ++++++----
 spec/langchain/tool/google_search_spec.rb |  7 ++--
 spec/langchain/tool/tavily_spec.rb        | 14 ++++----
 spec/langchain/tool/weather_spec.rb       |  9 +++--
 spec/tool_response_spec.rb                | 41 +++++++++++++++++++++++
 9 files changed, 90 insertions(+), 59 deletions(-)
 create mode 100644 spec/tool_response_spec.rb

diff --git a/lib/langchain/tool/calculator.rb b/lib/langchain/tool/calculator.rb
index 1be3fc96..13e1dae9 100644
--- a/lib/langchain/tool/calculator.rb
+++ b/lib/langchain/tool/calculator.rb
@@ -32,7 +32,7 @@ def execute(input:)
       Langchain.logger.debug("#{self.class} - Executing \"#{input}\"")
 
       result = Eqn::Calculator.calc(input)
-      tool_response(content: result.to_s)
+      tool_response(content: result)
     rescue Eqn::ParseError, Eqn::NoVariableValueError
       tool_response(content: "\"#{input}\" is an invalid mathematical expression")
     end
diff --git a/lib/langchain/tool_response.rb b/lib/langchain/tool_response.rb
index 40aa814b..34267760 100644
--- a/lib/langchain/tool_response.rb
+++ b/lib/langchain/tool_response.rb
@@ -17,38 +17,6 @@ def initialize(content: nil, image_url: nil)
       @image_url = image_url
     end
 
-    # Wraps a raw response in a ToolResponse if it is not already one.
-    #
-    # @param response [String, ToolResponse] The raw response to wrap.
-    # @return [ToolResponse] The wrapped response.
-    def self.wrap(response)
-      return response if response.is_a?(ToolResponse)
-
-      new(content: response)
-    end
-
-    # Converts the response into a format compatible with the given LLM provider.
-    #
-    # @param provider_class [Class] The provider class handling the response.
-    # @return [Hash] The formatted response for the provider.
-    def to_api_format(provider_class)
-      provider_class.format_tool_response(self)
-    end
-
-    # Checks if the response has an image URL.
-    #
-    # @return [Boolean] True if an image URL is present.
-    def has_image?
-      !image_url.nil?
-    end
-
-    # Checks if the response has text content.
-    #
-    # @return [Boolean] True if text content is present.
-    def has_content?
-      !content.nil?
-    end
-
     def to_s
       content.to_s
     end
diff --git a/spec/langchain/tool/calculator_spec.rb b/spec/langchain/tool/calculator_spec.rb
index b75e0af2..63a7159c 100644
--- a/spec/langchain/tool/calculator_spec.rb
+++ b/spec/langchain/tool/calculator_spec.rb
@@ -5,15 +5,17 @@
 RSpec.describe Langchain::Tool::Calculator do
   describe "#execute" do
     it "calculates the result" do
-      expect(subject.execute(input: "2+2")).to eq(4)
+      response = subject.execute(input: "2+2")
+      expect(response).to be_a(Langchain::ToolResponse)
+      expect(response.content).to eq(4)
     end
 
     it "rescue an error and return an explanation" do
       allow(Eqn::Calculator).to receive(:calc).and_raise(Eqn::ParseError)
 
-      expect(
-        subject.execute(input: "two plus two")
-      ).to eq("\"two plus two\" is an invalid mathematical expression")
+      response = subject.execute(input: "two plus two")
+      expect(response).to be_a(Langchain::ToolResponse)
+      expect(response.content).to eq("\"two plus two\" is an invalid mathematical expression")
     end
   end
 end
diff --git a/spec/langchain/tool/database_spec.rb b/spec/langchain/tool/database_spec.rb
index 3642c593..de339634 100644
--- a/spec/langchain/tool/database_spec.rb
+++ b/spec/langchain/tool/database_spec.rb
@@ -23,11 +23,15 @@
     end
 
     it "returns salary and count of users" do
-      expect(subject.execute(input: "SELECT max(salary), count(*) FROM users")).to eq([{count: 101, salary: 23500}])
+      response = subject.execute(input: "SELECT max(salary), count(*) FROM users")
+      expect(response).to be_a(Langchain::ToolResponse)
+      expect(response.content).to eq([{salary: 23500, count: 101}])
     end
 
     it "returns jobs and counts of users" do
-      expect(subject.execute(input: "SELECT job, count(*) FROM users GROUP BY job")).to eq([{count: 5, job: "teacher"}, {count: 98, job: "cook"}])
+      response = subject.execute(input: "SELECT job, count(*) FROM users GROUP BY job")
+      expect(response).to be_a(Langchain::ToolResponse)
+      expect(response.content).to eq([{job: "teacher", count: 5}, {job: "cook", count: 98}])
     end
   end
 
@@ -39,13 +43,17 @@
     end
 
     it "returns the schema" do
-      expect(subject.dump_schema).to eq("CREATE TABLE users(\nid integer PRIMARY KEY,\nname string,\njob string,\nFOREIGN KEY (job) REFERENCES jobs(job));\n")
+      response = subject.dump_schema
+      expect(response).to be_a(Langchain::ToolResponse)
+      expect(response.content).to eq("CREATE TABLE users(\nid integer PRIMARY KEY,\nname string,\njob string,\nFOREIGN KEY (job) REFERENCES jobs(job));\n")
     end
 
     it "does not fail when key is not present" do
       allow(subject.db).to receive(:foreign_key_list).with(:users).and_return([{columns: [:job], table: :jobs, key: nil}])
 
-      expect(subject.dump_schema).to eq("CREATE TABLE users(\nid integer PRIMARY KEY,\nname string,\njob string,\nFOREIGN KEY (job) REFERENCES jobs());\n")
+      response = subject.dump_schema
+      expect(response).to be_a(Langchain::ToolResponse)
+      expect(response.content).to eq("CREATE TABLE users(\nid integer PRIMARY KEY,\nname string,\njob string,\nFOREIGN KEY (job) REFERENCES jobs());\n")
     end
   end
 end
diff --git a/spec/langchain/tool/file_system_spec.rb b/spec/langchain/tool/file_system_spec.rb
index 2d2b6d7c..df2a6f0a 100644
--- a/spec/langchain/tool/file_system_spec.rb
+++ b/spec/langchain/tool/file_system_spec.rb
@@ -10,13 +10,15 @@
     it "lists a directory" do
       allow(Dir).to receive(:entries).with(directory_path).and_return(entries)
       response = subject.list_directory(directory_path: directory_path)
-      expect(response).to eq(entries)
+      expect(response).to be_a(Langchain::ToolResponse)
+      expect(response.content).to eq(entries)
     end
 
     it "returns a no such directory error" do
       allow(Dir).to receive(:entries).with(directory_path).and_raise(Errno::ENOENT)
       response = subject.list_directory(directory_path: directory_path)
-      expect(response).to eq("No such directory: #{directory_path}")
+      expect(response).to be_a(Langchain::ToolResponse)
+      expect(response.content).to eq("No such directory: #{directory_path}")
     end
   end
 
@@ -28,13 +30,15 @@
       it "successfully writes" do
         allow(File).to receive(:write).with(file_path, content)
         response = subject.write_to_file(file_path: file_path, content: content)
-        expect(response).to eq(nil)
+        expect(response).to be_a(Langchain::ToolResponse)
+        expect(response.content).to eq("File written successfully")
       end
 
       it "returns a permission denied error" do
         allow(File).to receive(:write).with(file_path, content).and_raise(Errno::EACCES)
         response = subject.write_to_file(file_path: file_path, content: content)
-        expect(response).to eq("Permission denied: #{file_path}")
+        expect(response).to be_a(Langchain::ToolResponse)
+        expect(response.content).to eq("Permission denied: #{file_path}")
       end
     end
 
@@ -45,13 +49,15 @@
       it "successfully reads" do
         allow(File).to receive(:read).with(file_path).and_return(content)
         response = subject.read_file(file_path: file_path)
-        expect(response).to eq(content)
+        expect(response).to be_a(Langchain::ToolResponse)
+        expect(response.content).to eq(content)
       end
 
       it "returns an error" do
         allow(File).to receive(:read).with(file_path).and_raise(Errno::ENOENT)
         response = subject.read_file(file_path: file_path)
-        expect(response).to eq("No such file: #{file_path}")
+        expect(response).to be_a(Langchain::ToolResponse)
+        expect(response.content).to eq("No such file: #{file_path}")
       end
     end
   end
diff --git a/spec/langchain/tool/google_search_spec.rb b/spec/langchain/tool/google_search_spec.rb
index a447a9c4..bd78036f 100644
--- a/spec/langchain/tool/google_search_spec.rb
+++ b/spec/langchain/tool/google_search_spec.rb
@@ -24,13 +24,16 @@
 
   describe "#execute_search" do
     it "returns the raw hash" do
-      expect(subject.execute_search(input: "how tall is empire state building")).to be_a(Hash)
+      result = subject.execute_search(input: "how tall is empire state building")
+      expect(result).to be_a(Hash)
     end
   end
 
   describe "#execute" do
     it "returns the answer" do
-      expect(subject.execute(input: "how tall is empire state building")).to eq("1,250′, 1,454′ to tip")
+      response = subject.execute(input: "how tall is empire state building")
+      expect(response).to be_a(Langchain::ToolResponse)
+      expect(response.content).to eq("1,250′, 1,454′ to tip")
     end
   end
 end
diff --git a/spec/langchain/tool/tavily_spec.rb b/spec/langchain/tool/tavily_spec.rb
index fac53809..6130bad3 100644
--- a/spec/langchain/tool/tavily_spec.rb
+++ b/spec/langchain/tool/tavily_spec.rb
@@ -11,13 +11,13 @@
     it "returns a response" do
       allow(Net::HTTP).to receive(:start).and_return(double(body: response))
 
-      expect(
-        subject.search(
-          query: "What's the height of Burj Khalifa?",
-          max_results: 1,
-          include_answer: true
-        )
-      ).to eq(response)
+      result = subject.search(
+        query: "What's the height of Burj Khalifa?",
+        max_results: 1,
+        include_answer: true
+      )
+      expect(result).to be_a(Langchain::ToolResponse)
+      expect(result.content).to eq(response)
     end
   end
 end
diff --git a/spec/langchain/tool/weather_spec.rb b/spec/langchain/tool/weather_spec.rb
index e58e2f8d..5c197c36 100644
--- a/spec/langchain/tool/weather_spec.rb
+++ b/spec/langchain/tool/weather_spec.rb
@@ -33,7 +33,8 @@
 
       it "returns the parsed weather data" do
         result = weather_tool.get_current_weather(city: city, state_code: state_code, country_code: country_code)
-        expect(result).to eq({
+        expect(result).to be_a(Langchain::ToolResponse)
+        expect(result.content).to eq({
           temperature: "72 °F",
           humidity: "50%",
           description: "clear sky",
@@ -56,7 +57,8 @@
 
       it "returns an error message" do
         result = weather_tool.get_current_weather(city: city, state_code: state_code)
-        expect(result).to eq("Location not found")
+        expect(result).to be_a(Langchain::ToolResponse)
+        expect(result.content).to eq("Location not found")
       end
     end
 
@@ -67,7 +69,8 @@
 
       it "returns the error message" do
         result = weather_tool.get_current_weather(city: city, state_code: state_code)
-        expect(result).to eq("API request failed: 404 - Not Found")
+        expect(result).to be_a(Langchain::ToolResponse)
+        expect(result.content).to eq("API request failed: 404 - Not Found")
       end
     end
   end
diff --git a/spec/tool_response_spec.rb b/spec/tool_response_spec.rb
new file mode 100644
index 00000000..cf10ce68
--- /dev/null
+++ b/spec/tool_response_spec.rb
@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+RSpec.describe Langchain::ToolResponse do
+  describe "#initialize" do
+    context "with content" do
+      subject(:response) { described_class.new(content: "test content") }
+
+      it "creates a valid instance" do
+        expect(response).to be_a(described_class)
+        expect(response.content).to eq("test content")
+        expect(response.image_url).to be_nil
+      end
+    end
+
+    context "with image_url" do
+      subject(:response) { described_class.new(image_url: "http://example.com/image.jpg") }
+
+      it "creates a valid instance" do
+        expect(response).to be_a(described_class)
+        expect(response.image_url).to eq("http://example.com/image.jpg")
+        expect(response.content).to be_nil
+      end
+    end
+
+    context "with both content and image_url" do
+      subject(:response) { described_class.new(content: "test content", image_url: "http://example.com/image.jpg") }
+
+      it "creates a valid instance" do
+        expect(response).to be_a(described_class)
+        expect(response.content).to eq("test content")
+        expect(response.image_url).to eq("http://example.com/image.jpg")
+      end
+    end
+
+    context "with neither content nor image_url" do
+      it "raises an ArgumentError" do
+        expect { described_class.new }.to raise_error(ArgumentError, "Either content or image_url must be provided")
+      end
+    end
+  end
+end

From 8744c534211c3e2ab61a2676e2c7361ea7a1faec Mon Sep 17 00:00:00 2001
From: Eth3rnit3 <eth3rnit3@gmail.com>
Date: Sun, 26 Jan 2025 15:48:23 +0100
Subject: [PATCH 10/20] Update assistant.rb

---
 lib/langchain/assistant.rb | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/lib/langchain/assistant.rb b/lib/langchain/assistant.rb
index 3bf0d045..0cd0677d 100644
--- a/lib/langchain/assistant.rb
+++ b/lib/langchain/assistant.rb
@@ -173,12 +173,11 @@ def add_message_and_run!(content: nil, image_url: nil)
     # Submit tool output
     #
     # @param tool_call_id [String] The ID of the tool call to submit output for
-    # @param content [String] The content of the tool call
-    # @param image_url [String] The image URL of the tool call
+    # @param output [String] The output of the tool
     # @return [Array<Langchain::Message>] The messages
-    def submit_tool_output(tool_call_id:, content:, image_url: nil)
+    def submit_tool_output(tool_call_id:, output:)
       # TODO: Validate that `tool_call_id` is valid by scanning messages and checking if this tool call ID was invoked
-      add_message(role: @llm_adapter.tool_role, content: content, tool_call_id: tool_call_id, image_url: image_url)
+      add_message(role: @llm_adapter.tool_role, content: output, tool_call_id: tool_call_id)
     end
 
     # Delete all messages
@@ -374,14 +373,15 @@ def run_tool(tool_call)
       tool_execution_callback.call(tool_call_id, tool_name, method_name, tool_arguments) if tool_execution_callback # rubocop:disable Style/SafeNavigation
 
       result = tool_instance.send(method_name, **tool_arguments)
-      
+
       # Handle both ToolResponse and legacy return values
       if result.is_a?(ToolResponse)
-        submit_tool_output(tool_call_id: tool_call_id, content: result.content, image_url: result.image_url)
+        add_message(role: @llm_adapter.tool_role, content: result.content, image_url: result.image_url, tool_call_id: tool_call_id)
       else
         # Legacy support for tools returning [content, image_url]
-        content, image_url = result
-        submit_tool_output(tool_call_id: tool_call_id, content: content, image_url: image_url)
+        output = tool_instance.send(method_name, **tool_arguments)
+
+        submit_tool_output(tool_call_id: tool_call_id, output: output)
       end
     end
 

From a9bf4acf7b8c9fb97a57829a39f3599c21330bfe Mon Sep 17 00:00:00 2001
From: Eth3rnit3 <eth3rnit3@gmail.com>
Date: Sun, 26 Jan 2025 15:59:09 +0100
Subject: [PATCH 11/20] Update assistant_spec.rb

---
 spec/langchain/assistant/assistant_spec.rb | 37 +++++-----------------
 1 file changed, 8 insertions(+), 29 deletions(-)

diff --git a/spec/langchain/assistant/assistant_spec.rb b/spec/langchain/assistant/assistant_spec.rb
index e5e96640..a8140bb1 100644
--- a/spec/langchain/assistant/assistant_spec.rb
+++ b/spec/langchain/assistant/assistant_spec.rb
@@ -197,17 +197,10 @@
 
     describe "#submit_tool_output" do
       it "adds a message to the thread" do
-        subject.submit_tool_output(tool_call_id: "123", content: "bar")
+        subject.submit_tool_output(tool_call_id: "123", output: "bar")
         expect(subject.messages.last.role).to eq("tool")
         expect(subject.messages.last.content).to eq("bar")
       end
-
-      it "adds an image to the message" do
-        subject.submit_tool_output(tool_call_id: "123", image_url: "https://example.com/image.jpg", content: "Hello")
-        expect(subject.messages.last.role).to eq("tool")
-        expect(subject.messages.last.content).to eq("Hello")
-        expect(subject.messages.last.image_url).to eq("https://example.com/image.jpg")
-      end
     end
 
     describe "#run" do
@@ -571,17 +564,10 @@
 
     describe "#submit_tool_output" do
       it "adds a message to the thread" do
-        subject.submit_tool_output(tool_call_id: "123", content: "bar")
+        subject.submit_tool_output(tool_call_id: "123", output: "bar")
         expect(subject.messages.last.role).to eq("tool")
         expect(subject.messages.last.content).to eq("bar")
       end
-
-      it "adds an image to the message" do
-        subject.submit_tool_output(tool_call_id: "123", image_url: "https://example.com/image.jpg", content: "Hello")
-        expect(subject.messages.last.role).to eq("tool")
-        expect(subject.messages.last.content).to eq("Hello")
-        expect(subject.messages.last.image_url).to eq("https://example.com/image.jpg")
-      end
     end
 
     describe "#run" do
@@ -672,7 +658,7 @@
                     "type" => "function"
                   }
                 ]},
-                {content: [{type: "text", text: "4.0"}], role: "tool", tool_call_id: "call_9TewGANaaIjzY31UCpAAGLeV"}
+                {content: "4.0", role: "tool", tool_call_id: "call_9TewGANaaIjzY31UCpAAGLeV"}
               ],
               tools: calculator.class.function_schemas.to_openai_format,
               tool_choice: "auto"
@@ -930,19 +916,12 @@
       end
     end
 
-    describe "#submit_tool_output" do
+    describe "submit_tool_output" do
       it "adds a message to the thread" do
-        subject.submit_tool_output(tool_call_id: "123", content: "bar")
+        subject.submit_tool_output(tool_call_id: "123", output: "bar")
         expect(subject.messages.last.role).to eq("function")
         expect(subject.messages.last.content).to eq("bar")
       end
-
-      it "does not add image to the message" do
-        subject.submit_tool_output(tool_call_id: "123", content: "Hello", image_url: "https://example.com/image.jpg")
-        expect(subject.messages.last.role).to eq("function")
-        expect(subject.messages.last.content).to eq("Hello")
-        expect(subject.messages.last.image_url).to be_nil
-      end
     end
 
     describe "#run" do
@@ -1121,7 +1100,7 @@
 
     describe "submit_tool_output" do
       it "adds a message to the thread" do
-        subject.submit_tool_output(tool_call_id: "123", content: "bar")
+        subject.submit_tool_output(tool_call_id: "123", output: "bar")
         expect(subject.messages.last.role).to eq("tool_result")
         expect(subject.messages.last.content).to eq("bar")
       end
@@ -1239,7 +1218,7 @@
                     "input" => {"input" => "2+2"}
                   }
                 ]},
-                {role: "user", content: [{type: "tool_result", tool_use_id: "toolu_014eSx9oBA5DMe8gZqaqcJ3H", content: [{type: "text", text: "4.0"}]}]}
+                {role: "user", content: [{type: "tool_result", tool_use_id: "toolu_014eSx9oBA5DMe8gZqaqcJ3H", content: "4.0"}]}
               ],
               tools: calculator.class.function_schemas.to_anthropic_format,
               tool_choice: {disable_parallel_tool_use: false, type: "auto"},
@@ -1332,4 +1311,4 @@
       end
     end
   end
-end
+end
\ No newline at end of file

From a88de85683a48906736626422d51bea5e51a53c2 Mon Sep 17 00:00:00 2001
From: Eth3rnit3 <eth3rnit3@gmail.com>
Date: Sun, 26 Jan 2025 16:04:08 +0100
Subject: [PATCH 12/20] Update assistant.rb

---
 lib/langchain/assistant.rb | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/lib/langchain/assistant.rb b/lib/langchain/assistant.rb
index 0cd0677d..57a45136 100644
--- a/lib/langchain/assistant.rb
+++ b/lib/langchain/assistant.rb
@@ -372,15 +372,12 @@ def run_tool(tool_call)
       # Call the callback if set
       tool_execution_callback.call(tool_call_id, tool_name, method_name, tool_arguments) if tool_execution_callback # rubocop:disable Style/SafeNavigation
 
-      result = tool_instance.send(method_name, **tool_arguments)
+      output = tool_instance.send(method_name, **tool_arguments)
 
       # Handle both ToolResponse and legacy return values
-      if result.is_a?(ToolResponse)
-        add_message(role: @llm_adapter.tool_role, content: result.content, image_url: result.image_url, tool_call_id: tool_call_id)
+      if output.is_a?(ToolResponse)
+        add_message(role: @llm_adapter.tool_role, content: output.content, image_url: output.image_url, tool_call_id: tool_call_id)
       else
-        # Legacy support for tools returning [content, image_url]
-        output = tool_instance.send(method_name, **tool_arguments)
-
         submit_tool_output(tool_call_id: tool_call_id, output: output)
       end
     end

From 059a84a296281f5487da8e7c881551db1feb4725 Mon Sep 17 00:00:00 2001
From: Eth3rnit3 <eth3rnit3@gmail.com>
Date: Sun, 26 Jan 2025 16:07:03 +0100
Subject: [PATCH 13/20] reset llm provider changes

---
 lib/langchain/assistant/messages/anthropic_message.rb  | 2 +-
 lib/langchain/assistant/messages/mistral_ai_message.rb | 2 +-
 lib/langchain/assistant/messages/openai_message.rb     | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/langchain/assistant/messages/anthropic_message.rb b/lib/langchain/assistant/messages/anthropic_message.rb
index 1285cfeb..70f38209 100644
--- a/lib/langchain/assistant/messages/anthropic_message.rb
+++ b/lib/langchain/assistant/messages/anthropic_message.rb
@@ -76,7 +76,7 @@ def tool_hash
               {
                 type: "tool_result",
                 tool_use_id: tool_call_id,
-                content: build_content_array
+                content: content
               }
             ]
           }
diff --git a/lib/langchain/assistant/messages/mistral_ai_message.rb b/lib/langchain/assistant/messages/mistral_ai_message.rb
index ba3de404..2c081d94 100644
--- a/lib/langchain/assistant/messages/mistral_ai_message.rb
+++ b/lib/langchain/assistant/messages/mistral_ai_message.rb
@@ -102,7 +102,7 @@ def system_hash
         def tool_hash
           {
             role: "tool",
-            content: build_content_array,
+            content: content,
             tool_call_id: tool_call_id
           }
         end
diff --git a/lib/langchain/assistant/messages/openai_message.rb b/lib/langchain/assistant/messages/openai_message.rb
index e19b1246..44e57a34 100644
--- a/lib/langchain/assistant/messages/openai_message.rb
+++ b/lib/langchain/assistant/messages/openai_message.rb
@@ -117,7 +117,7 @@ def tool_hash
           {
             role: "tool",
             tool_call_id: tool_call_id,
-            content: build_content_array # Using image_url with tools is not supported by OpenAI (Image URLs are only allowed for messages with role 'user', but this message with role 'tool' contains an image URL.)
+            content: build_content_array
           }
         end
 

From 4010d357984d7db5be64ac537c0985442057e993 Mon Sep 17 00:00:00 2001
From: Eth3rnit3 <eth3rnit3@gmail.com>
Date: Sun, 26 Jan 2025 16:08:11 +0100
Subject: [PATCH 14/20] reset default llm provider specs

---
 .../messages/anthropic_message_spec.rb        | 32 +------------------
 .../messages/mistral_ai_message_spec.rb       | 25 ++-------------
 .../assistant/messages/openai_message_spec.rb | 15 ---------
 3 files changed, 3 insertions(+), 69 deletions(-)

diff --git a/spec/langchain/assistant/messages/anthropic_message_spec.rb b/spec/langchain/assistant/messages/anthropic_message_spec.rb
index ef8f132a..a1f04924 100644
--- a/spec/langchain/assistant/messages/anthropic_message_spec.rb
+++ b/spec/langchain/assistant/messages/anthropic_message_spec.rb
@@ -108,42 +108,12 @@
               {
                 type: "tool_result",
                 tool_use_id: "toolu_014eSx9oBA5DMe8gZqaqcJ3H",
-                content: [
-                  {
-                    type: "text",
-                    text: "4.0"
-                  }
-                ]
+                content: "4.0"
               }
             ]
           }
         )
       end
-
-      it "returns tool_hash with image_url" do
-        message = described_class.new(role: "tool_result", image_url: "https://example.com/image.jpg")
-        allow(message).to receive(:image).and_return(double(base64: "base64_data", mime_type: "image/jpeg"))
-
-        expect(message.to_hash).to eq(
-          role: "user",
-          content: [
-            {
-              type: "tool_result",
-              tool_use_id: nil,
-              content: [
-                {
-                  type: "image",
-                  source: {
-                    type: "base64",
-                    data: "base64_data",
-                    media_type: "image/jpeg"
-                  }
-                }
-              ]
-            }
-          ]
-        )
-      end
     end
 
     context "when role is user" do
diff --git a/spec/langchain/assistant/messages/mistral_ai_message_spec.rb b/spec/langchain/assistant/messages/mistral_ai_message_spec.rb
index e02f4316..9f4af833 100644
--- a/spec/langchain/assistant/messages/mistral_ai_message_spec.rb
+++ b/spec/langchain/assistant/messages/mistral_ai_message_spec.rb
@@ -18,7 +18,7 @@
       let(:message) { described_class.new(role: "tool", content: "Hello, world!", tool_calls: [], tool_call_id: "123") }
 
       it "returns a hash with the tool_call_id key" do
-        expect(message.to_hash).to eq({role: "tool", content: [{text: "Hello, world!", type: "text"}], tool_call_id: "123"})
+        expect(message.to_hash).to eq({role: "tool", content: "Hello, world!", tool_call_id: "123"})
       end
     end
 
@@ -36,7 +36,7 @@
       end
     end
 
-    context "when image_url is present in user message" do
+    context "when image_url is present" do
       let(:message) { described_class.new(role: "user", content: "Please describe this image", image_url: "https://example.com/image.jpg") }
 
       it "returns a hash with the image_url key" do
@@ -49,26 +49,5 @@
         })
       end
     end
-
-    context "when image_url is present in tool message" do
-      let(:tool_call) {
-        {"id" => "call_9TewGANaaIjzY31UCpAAGLeV",
-         "type" => "function",
-         "function" => {"name" => "dummy_tool__take_photo"}}
-      }
-
-      let(:message) { described_class.new(role: "tool", content: "Hello, world!", image_url: "https://example.com/image.jpg",  tool_calls: [tool_call], tool_call_id: "123") }
-
-      it "returns a hash with the image_url key" do
-        expect(message.to_hash).to eq({
-          role: "tool",
-          content: [
-            {text: "Hello, world!", type: "text"},
-            {image_url: "https://example.com/image.jpg", type: "image_url"}
-          ],
-          tool_call_id: "123"
-        })
-      end
-    end
   end
 end
diff --git a/spec/langchain/assistant/messages/openai_message_spec.rb b/spec/langchain/assistant/messages/openai_message_spec.rb
index e089053c..6e74bf68 100644
--- a/spec/langchain/assistant/messages/openai_message_spec.rb
+++ b/spec/langchain/assistant/messages/openai_message_spec.rb
@@ -43,21 +43,6 @@
       it "returns a tool_hash" do
         expect(message.to_hash).to eq({role: "tool", content: [{type: "text", text: "Hello, world!"}], tool_call_id: "123"})
       end
-
-      context "when image_url is present" do
-        let(:message) { described_class.new(role: "tool", content: "Hello, world!", image_url: "https://example.com/image.jpg", tool_calls: [], tool_call_id: "123") }
-
-        it "returns a tool_hash with the image_url key" do
-          expect(message.to_hash).to eq({
-            role: "tool",
-            content: [
-              {type: "text", text: "Hello, world!"},
-              {type: "image_url", image_url: {url: "https://example.com/image.jpg"}}
-            ],
-            tool_call_id: "123"
-          })
-        end
-      end
     end
 
     context "when role is assistant" do

From b792232cd1a4e24c5235fe5d454ee6cf50da795f Mon Sep 17 00:00:00 2001
From: Eth3rnit3 <eth3rnit3@gmail.com>
Date: Sun, 26 Jan 2025 16:13:10 +0100
Subject: [PATCH 15/20] clear and add helper documentation

---
 lib/langchain/tool_definition.rb | 4 ----
 lib/langchain/tool_helpers.rb    | 4 ++++
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/lib/langchain/tool_definition.rb b/lib/langchain/tool_definition.rb
index cadbd11a..f431f3b9 100644
--- a/lib/langchain/tool_definition.rb
+++ b/lib/langchain/tool_definition.rb
@@ -61,10 +61,6 @@ def tool_name
       .downcase
   end
 
-  def tool_response(content: nil, image_url: nil)
-    Langchain::ToolResponse.new(content: content, image_url: image_url)
-  end
-
   # Manages schemas for functions
   class FunctionSchemas
     def initialize(tool_name)
diff --git a/lib/langchain/tool_helpers.rb b/lib/langchain/tool_helpers.rb
index e74c54c0..54c09b64 100644
--- a/lib/langchain/tool_helpers.rb
+++ b/lib/langchain/tool_helpers.rb
@@ -2,6 +2,10 @@
 
 module Langchain
   module ToolHelpers
+    # Create a tool response
+    # @param content [String, nil] The content of the tool response
+    # @param image_url [String, nil] The URL of an image
+    # @return [Langchain::ToolResponse] The tool response
     def tool_response(content: nil, image_url: nil)
       Langchain::ToolResponse.new(content: content, image_url: image_url)
     end

From 515e2ea7fdc13283642d22f8e0c336db144f419d Mon Sep 17 00:00:00 2001
From: Eth3rnit3 <eth3rnit3@gmail.com>
Date: Sun, 26 Jan 2025 16:14:23 +0100
Subject: [PATCH 16/20] Update assistant_spec.rb

---
 spec/langchain/assistant/assistant_spec.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/spec/langchain/assistant/assistant_spec.rb b/spec/langchain/assistant/assistant_spec.rb
index a8140bb1..432d4df9 100644
--- a/spec/langchain/assistant/assistant_spec.rb
+++ b/spec/langchain/assistant/assistant_spec.rb
@@ -1311,4 +1311,4 @@
       end
     end
   end
-end
\ No newline at end of file
+end

From bb9b5250db7488b40ea3c4a3fbeada05852bc9cc Mon Sep 17 00:00:00 2001
From: Eth3rnit3 <eth3rnit3@gmail.com>
Date: Sun, 26 Jan 2025 16:20:47 +0100
Subject: [PATCH 17/20] Update assistant_spec.rb

---
 spec/langchain/assistant/assistant_spec.rb | 62 ++++++++++++++++++++++
 1 file changed, 62 insertions(+)

diff --git a/spec/langchain/assistant/assistant_spec.rb b/spec/langchain/assistant/assistant_spec.rb
index 432d4df9..28d3453f 100644
--- a/spec/langchain/assistant/assistant_spec.rb
+++ b/spec/langchain/assistant/assistant_spec.rb
@@ -338,6 +338,68 @@
       end
     end
 
+    describe "#handle_tool_call" do
+      let(:llm) { Langchain::LLM::OpenAI.new(api_key: "123") }
+      let(:calculator) { Langchain::Tool::Calculator.new }
+      let(:assistant) { described_class.new(llm: llm, tools: [calculator]) }
+
+      context "when tool returns a ToolResponse" do
+        let(:tool_call) do
+          {
+            "id" => "call_123",
+            "type" => "function",
+            "function" => {
+              "name" => "langchain_tool_calculator__execute",
+              "arguments" => {input: "2+2"}.to_json
+            }
+          }
+        end
+        let(:tool_response) { Langchain::ToolResponse.new(content: "4", image_url: "http://example.com/image.jpg") }
+
+        before do
+          allow_any_instance_of(Langchain::Tool::Calculator).to receive(:execute).and_return(tool_response)
+        end
+
+        it "adds a message with the ToolResponse content and image_url" do
+          expect {
+            assistant.send(:run_tool, tool_call)
+          }.to change { assistant.messages.count }.by(1)
+
+          last_message = assistant.messages.last
+          expect(last_message.content).to eq("4")
+          expect(last_message.image_url).to eq("http://example.com/image.jpg")
+          expect(last_message.tool_call_id).to eq("call_123")
+        end
+      end
+
+      context "when tool returns a simple value" do
+        let(:tool_call) do
+          {
+            "id" => "call_123",
+            "type" => "function",
+            "function" => {
+              "name" => "langchain_tool_calculator__execute",
+              "arguments" => {input: "2+2"}.to_json
+            }
+          }
+        end
+
+        before do
+          allow_any_instance_of(Langchain::Tool::Calculator).to receive(:execute).and_return("4")
+        end
+
+        it "adds a message with the simple value as content" do
+          expect {
+            assistant.send(:run_tool, tool_call)
+          }.to change { assistant.messages.count }.by(1)
+
+          last_message = assistant.messages.last
+          expect(last_message.content).to eq("4")
+          expect(last_message.tool_call_id).to eq("call_123")
+        end
+      end
+    end
+
     describe "#extract_tool_call_args" do
       let(:tool_call) { {"id" => "call_9TewGANaaIjzY31UCpAAGLeV", "type" => "function", "function" => {"name" => "langchain_tool_calculator__execute", "arguments" => "{\"input\":\"2+2\"}"}} }
 

From c6b8d24a06c7194bd650a0924325499df86b5f08 Mon Sep 17 00:00:00 2001
From: Andrei Bondarev <andrei.bondarev13@gmail.com>
Date: Sun, 26 Jan 2025 12:05:53 -0500
Subject: [PATCH 18/20] Merge Langchain::ToolHelpers module into
 Langchain::ToolDefinition

---
 lib/langchain/tool/calculator.rb            |  1 -
 lib/langchain/tool/database.rb              |  1 -
 lib/langchain/tool/file_system.rb           |  1 -
 lib/langchain/tool/google_search.rb         |  1 -
 lib/langchain/tool/news_retriever.rb        |  1 -
 lib/langchain/tool/ruby_code_interpreter.rb |  1 -
 lib/langchain/tool/tavily.rb                |  1 -
 lib/langchain/tool/vectorsearch.rb          |  1 -
 lib/langchain/tool/weather.rb               |  1 -
 lib/langchain/tool/wikipedia.rb             |  1 -
 lib/langchain/tool_definition.rb            | 14 ++++++++++++++
 lib/langchain/tool_helpers.rb               | 13 -------------
 12 files changed, 14 insertions(+), 23 deletions(-)
 delete mode 100644 lib/langchain/tool_helpers.rb

diff --git a/lib/langchain/tool/calculator.rb b/lib/langchain/tool/calculator.rb
index 13e1dae9..ff5dc55e 100644
--- a/lib/langchain/tool/calculator.rb
+++ b/lib/langchain/tool/calculator.rb
@@ -14,7 +14,6 @@ module Langchain::Tool
   class Calculator
     extend Langchain::ToolDefinition
     include Langchain::DependencyHelper
-    include Langchain::ToolHelpers
 
     define_function :execute, description: "Evaluates a pure math expression or if equation contains non-math characters (e.g.: \"12F in Celsius\") then it uses the google search calculator to evaluate the expression" do
       property :input, type: "string", description: "Math expression", required: true
diff --git a/lib/langchain/tool/database.rb b/lib/langchain/tool/database.rb
index 83bc478b..0580d2ed 100644
--- a/lib/langchain/tool/database.rb
+++ b/lib/langchain/tool/database.rb
@@ -13,7 +13,6 @@ module Langchain::Tool
   class Database
     extend Langchain::ToolDefinition
     include Langchain::DependencyHelper
-    include Langchain::ToolHelpers
 
     define_function :list_tables, description: "Database Tool: Returns a list of tables in the database"
 
diff --git a/lib/langchain/tool/file_system.rb b/lib/langchain/tool/file_system.rb
index 4fbda92e..472a730f 100644
--- a/lib/langchain/tool/file_system.rb
+++ b/lib/langchain/tool/file_system.rb
@@ -9,7 +9,6 @@ module Langchain::Tool
   #
   class FileSystem
     extend Langchain::ToolDefinition
-    include Langchain::ToolHelpers
 
     define_function :list_directory, description: "File System Tool: Lists out the content of a specified directory" do
       property :directory_path, type: "string", description: "Directory path to list", required: true
diff --git a/lib/langchain/tool/google_search.rb b/lib/langchain/tool/google_search.rb
index 5c17d2a1..d3d78fd7 100644
--- a/lib/langchain/tool/google_search.rb
+++ b/lib/langchain/tool/google_search.rb
@@ -14,7 +14,6 @@ module Langchain::Tool
   class GoogleSearch
     extend Langchain::ToolDefinition
     include Langchain::DependencyHelper
-    include Langchain::ToolHelpers
 
     define_function :execute, description: "Executes Google Search and returns the result" do
       property :input, type: "string", description: "Search query", required: true
diff --git a/lib/langchain/tool/news_retriever.rb b/lib/langchain/tool/news_retriever.rb
index a5908b8a..3a7993b5 100644
--- a/lib/langchain/tool/news_retriever.rb
+++ b/lib/langchain/tool/news_retriever.rb
@@ -10,7 +10,6 @@ module Langchain::Tool
   #
   class NewsRetriever
     extend Langchain::ToolDefinition
-    include Langchain::ToolHelpers
 
     define_function :get_everything, description: "News Retriever: Search through millions of articles from over 150,000 large and small news sources and blogs" do
       property :q, type: "string", description: 'Keywords or phrases to search for in the article title and body. Surround phrases with quotes (") for exact match. Alternatively you can use the AND / OR / NOT keywords, and optionally group these with parenthesis. Must be URL-encoded'
diff --git a/lib/langchain/tool/ruby_code_interpreter.rb b/lib/langchain/tool/ruby_code_interpreter.rb
index 0a01b489..26a5960d 100644
--- a/lib/langchain/tool/ruby_code_interpreter.rb
+++ b/lib/langchain/tool/ruby_code_interpreter.rb
@@ -13,7 +13,6 @@ module Langchain::Tool
   class RubyCodeInterpreter
     extend Langchain::ToolDefinition
     include Langchain::DependencyHelper
-    include Langchain::ToolHelpers
 
     define_function :execute, description: "Executes Ruby code in a sandboxes environment" do
       property :input, type: "string", description: "Ruby code expression", required: true
diff --git a/lib/langchain/tool/tavily.rb b/lib/langchain/tool/tavily.rb
index 0f69856f..38ff5c1c 100644
--- a/lib/langchain/tool/tavily.rb
+++ b/lib/langchain/tool/tavily.rb
@@ -10,7 +10,6 @@ module Langchain::Tool
   #
   class Tavily
     extend Langchain::ToolDefinition
-    include Langchain::ToolHelpers
 
     define_function :search, description: "Tavily Tool: Robust search API" do
       property :query, type: "string", description: "The search query string", required: true
diff --git a/lib/langchain/tool/vectorsearch.rb b/lib/langchain/tool/vectorsearch.rb
index 6619939d..347526e0 100644
--- a/lib/langchain/tool/vectorsearch.rb
+++ b/lib/langchain/tool/vectorsearch.rb
@@ -14,7 +14,6 @@ module Langchain::Tool
   #
   class Vectorsearch
     extend Langchain::ToolDefinition
-    include Langchain::ToolHelpers
 
     define_function :similarity_search, description: "Vectorsearch: Retrieves relevant document for the query" do
       property :query, type: "string", description: "Query to find similar documents for", required: true
diff --git a/lib/langchain/tool/weather.rb b/lib/langchain/tool/weather.rb
index fca9f5d6..ad9d8dfc 100644
--- a/lib/langchain/tool/weather.rb
+++ b/lib/langchain/tool/weather.rb
@@ -16,7 +16,6 @@ module Langchain::Tool
   #
   class Weather
     extend Langchain::ToolDefinition
-    include Langchain::ToolHelpers
 
     define_function :get_current_weather, description: "Returns current weather for a city" do
       property :city,
diff --git a/lib/langchain/tool/wikipedia.rb b/lib/langchain/tool/wikipedia.rb
index 45f5424c..d521d367 100644
--- a/lib/langchain/tool/wikipedia.rb
+++ b/lib/langchain/tool/wikipedia.rb
@@ -14,7 +14,6 @@ module Langchain::Tool
   class Wikipedia
     extend Langchain::ToolDefinition
     include Langchain::DependencyHelper
-    include Langchain::ToolHelpers
 
     define_function :execute, description: "Executes Wikipedia API search and returns the answer" do
       property :input, type: "string", description: "Search query", required: true
diff --git a/lib/langchain/tool_definition.rb b/lib/langchain/tool_definition.rb
index f431f3b9..000da594 100644
--- a/lib/langchain/tool_definition.rb
+++ b/lib/langchain/tool_definition.rb
@@ -61,6 +61,20 @@ def tool_name
       .downcase
   end
 
+  def self.extended(base)
+    base.include(InstanceMethods)
+  end
+
+  module InstanceMethods
+    # Create a tool response
+    # @param content [String, nil] The content of the tool response
+    # @param image_url [String, nil] The URL of an image
+    # @return [Langchain::ToolResponse] The tool response
+    def tool_response(content: nil, image_url: nil)
+      Langchain::ToolResponse.new(content: content, image_url: image_url)
+    end
+  end
+
   # Manages schemas for functions
   class FunctionSchemas
     def initialize(tool_name)
diff --git a/lib/langchain/tool_helpers.rb b/lib/langchain/tool_helpers.rb
deleted file mode 100644
index 54c09b64..00000000
--- a/lib/langchain/tool_helpers.rb
+++ /dev/null
@@ -1,13 +0,0 @@
-# frozen_string_literal: true
-
-module Langchain
-  module ToolHelpers
-    # Create a tool response
-    # @param content [String, nil] The content of the tool response
-    # @param image_url [String, nil] The URL of an image
-    # @return [Langchain::ToolResponse] The tool response
-    def tool_response(content: nil, image_url: nil)
-      Langchain::ToolResponse.new(content: content, image_url: image_url)
-    end
-  end
-end

From 345daa666df212a51b19dd73e2378b2dffa2f43e Mon Sep 17 00:00:00 2001
From: Andrei Bondarev <andrei.bondarev13@gmail.com>
Date: Sun, 26 Jan 2025 13:37:13 -0500
Subject: [PATCH 19/20] Fixing specs

---
 .tool-versions                                    | 2 +-
 lib/langchain/tool_response.rb                    | 8 --------
 spec/langchain/tool/ruby_code_interpreter_spec.rb | 8 ++++++--
 spec/langchain/tool/wikipedia_spec.rb             | 3 ++-
 4 files changed, 9 insertions(+), 12 deletions(-)

diff --git a/.tool-versions b/.tool-versions
index 3294aeda..380cf519 100644
--- a/.tool-versions
+++ b/.tool-versions
@@ -1 +1 @@
-ruby 3.3.0
+ruby 3.4
diff --git a/lib/langchain/tool_response.rb b/lib/langchain/tool_response.rb
index 34267760..f0a8c62d 100644
--- a/lib/langchain/tool_response.rb
+++ b/lib/langchain/tool_response.rb
@@ -20,13 +20,5 @@ def initialize(content: nil, image_url: nil)
     def to_s
       content.to_s
     end
-
-    def to_str
-      to_s
-    end
-
-    def include?(other)
-      to_s.include?(other)
-    end
   end
 end
diff --git a/spec/langchain/tool/ruby_code_interpreter_spec.rb b/spec/langchain/tool/ruby_code_interpreter_spec.rb
index a83bc3b6..bd94f98b 100644
--- a/spec/langchain/tool/ruby_code_interpreter_spec.rb
+++ b/spec/langchain/tool/ruby_code_interpreter_spec.rb
@@ -6,7 +6,9 @@
   RSpec.describe Langchain::Tool::RubyCodeInterpreter do
     describe "#execute" do
       it "executes the expression" do
-        expect(subject.execute(input: '"hello world".reverse!')).to eq("dlrow olleh")
+        response = subject.execute(input: '"hello world".reverse!')
+        expect(response).to be_a(Langchain::ToolResponse)
+        expect(response.content).to eq("dlrow olleh")
       end
 
       it "executes a more complicated expression" do
@@ -18,7 +20,9 @@ def reverse(string)
           reverse('hello world')
         CODE
 
-        expect(subject.execute(input: code)).to eq("dlrow olleh")
+        response = subject.execute(input: code)
+        expect(response).to be_a(Langchain::ToolResponse)
+        expect(response.content).to eq("dlrow olleh")
       end
     end
   end
diff --git a/spec/langchain/tool/wikipedia_spec.rb b/spec/langchain/tool/wikipedia_spec.rb
index 15d7ef18..390f160b 100644
--- a/spec/langchain/tool/wikipedia_spec.rb
+++ b/spec/langchain/tool/wikipedia_spec.rb
@@ -13,7 +13,8 @@
     end
 
     it "returns a wikipedia summary" do
-      expect(subject.execute(input: "Ruby")).to include("Ruby is an interpreted, high-level, general-purpose programming language.")
+      response = subject.execute(input: "Ruby")
+      expect(response.content).to include("Ruby is an interpreted, high-level, general-purpose programming language.")
     end
   end
 end

From 44ade45c5ad2232683ffd244ecc303381b5703f8 Mon Sep 17 00:00:00 2001
From: Andrei Bondarev <andrei.bondarev13@gmail.com>
Date: Sun, 26 Jan 2025 13:42:36 -0500
Subject: [PATCH 20/20] CHANGELOG + README entries

---
 CHANGELOG.md | 1 +
 README.md    | 4 ++--
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index cf19b033..32e4bce6 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -10,6 +10,7 @@
 - [SECURITY]: A change which fixes a security vulnerability.
 
 ## [Unreleased]
+- [BREAKING] [https://github.com/patterns-ai-core/langchainrb/pull/894] Tools can now output image_urls, and all tool output must be wrapped by a tool_response() method
 
 ## [0.19.3] - 2025-01-13
 - [BUGFIX] [https://github.com/patterns-ai-core/langchainrb/pull/900] Empty text content should not be set when content is nil when using AnthropicMessage
diff --git a/README.md b/README.md
index f1e2b30b..229b5712 100644
--- a/README.md
+++ b/README.md
@@ -580,11 +580,11 @@ class MovieInfoTool
   end
 
   def search_movie(query:)
-    ...
+    tool_response(...)
   end
 
   def get_movie_details(movie_id:)
-    ...
+    tool_response(...)
   end
 end
 ```