remove mention of binary scores from template

deepset-ai · Mar 22, 2024 · 0a87bf6 · 0a87bf6
1 parent 8213723
commit 0a87bf6
Show file tree

Hide file tree

Showing 2 changed files with 37 additions and 10 deletions.
diff --git a/haystack/components/evaluators/llm_evaluator.py b/haystack/components/evaluators/llm_evaluator.py
@@ -170,23 +170,51 @@ def run(self, **inputs) -> Dict[str, Any]:
 
     def prepare_template(self) -> str:
         """
-        Combine instructions, inputs, outputs, and examples into one prompt template.
+        Combine instructions, inputs, outputs, and examples into one prompt template with the following format:
+        Instructions:
+        <instructions>
+
+        Generate the response in JSON format with the following keys:
+        <list of output keys>
+        Consider the instructions and the examples below to determine those values.
+
+        Examples:
+        <examples>
+
+        Inputs:
+        <inputs>
+        Outputs:
 
         :returns:
             The prompt template.
         """
         inputs_section = (
             "{" + ",".join([f'"{input_socket[0]}": {{{{ {input_socket[0]} }}}}' for input_socket in self.inputs]) + "}"
         )
-        examples_section = ""
+
         if self.examples:
-            for example in self.examples:
-                examples_section += (
-                    "Inputs:\n" + json.dumps(example["inputs"]) + "\nOutputs:\n" + json.dumps(example["outputs"]) + "\n"
+            examples_section = (
+                "Examples:\n"
+                + "\n".join(
+                    [
+                        "Inputs:\n" + json.dumps(example["inputs"]) + "\nOutputs:\n" + json.dumps(example["outputs"])
+                        for example in self.examples
+                    ]
                 )
+                + "\n\n"
+            )
+        else:
+            examples_section = ""
         return (
-            f"Respond only in JSON format with a key {json.dumps(self.outputs)} and a value of either 0 for FALSE "
-            f"or 1 for TRUE.\n{self.instructions}\n{examples_section}Inputs:\n{inputs_section}\nOutputs:\n"
+            f"Instructions:\n"
+            f"{self.instructions}\n\n"
+            f"Generate the response in JSON format with the following keys:\n"
+            f"{json.dumps(self.outputs)}\n"
+            f"Consider the instructions and the examples below to determine those values.\n\n"
+            f"{examples_section}"
+            f"Inputs:\n"
+            f"{inputs_section}\n"
+            f"Outputs:\n"
         )
 
     def to_dict(self) -> Dict[str, Any]:

diff --git a/test/components/evaluators/test_llm_evaluator.py b/test/components/evaluators/test_llm_evaluator.py
@@ -1,4 +1,3 @@
-import os
 from typing import List
 
 import pytest
@@ -162,7 +161,7 @@ def test_prepare_template_wo_examples(self, monkeypatch):
         template = component.prepare_template()
         assert (
             template
-            == 'Respond only in JSON format with a key ["score"] and a value of either 0 for FALSE or 1 for TRUE.\ntest-instruction\nInputs:\n{"responses": {{ responses }}}\nOutputs:\n'
+            == 'Instructions:\ntest-instruction\n\nGenerate the response in JSON format with the following keys:\n["score"]\nConsider the instructions and the examples below to determine those values.\n\nInputs:\n{"responses": {{ responses }}}\nOutputs:\n'
         )
 
     def test_prepare_template_with_examples(self, monkeypatch):
@@ -179,7 +178,7 @@ def test_prepare_template_with_examples(self, monkeypatch):
         template = component.prepare_template()
         assert (
             template
-            == 'Respond only in JSON format with a key ["score"] and a value of either 0 for FALSE or 1 for TRUE.\ntest-instruction\nInputs:\n{"responses": "Damn, this is straight outta hell!!!"}\nOutputs:\n{"score": 1}\nInputs:\n{"responses": "Football is the most popular sport."}\nOutputs:\n{"score": 0}\nInputs:\n{"responses": {{ responses }}}\nOutputs:\n'
+            == 'Instructions:\ntest-instruction\n\nGenerate the response in JSON format with the following keys:\n["score"]\nConsider the instructions and the examples below to determine those values.\n\nExamples:\nInputs:\n{"responses": "Damn, this is straight outta hell!!!"}\nOutputs:\n{"score": 1}\nInputs:\n{"responses": "Football is the most popular sport."}\nOutputs:\n{"score": 0}\n\nInputs:\n{"responses": {{ responses }}}\nOutputs:\n'
         )
 
     def test_invalid_input_parameters(self, monkeypatch):