From 473932e05ff34a5124e94cf0baf796693cc1eece Mon Sep 17 00:00:00 2001 From: Dmytro Nikolaiev Date: Fri, 29 Nov 2024 15:26:57 -0500 Subject: [PATCH 1/9] Initial implementation for YAML --- council/llm/llm_response_parser.py | 95 ++++++++++++++++++++++++++---- 1 file changed, 82 insertions(+), 13 deletions(-) diff --git a/council/llm/llm_response_parser.py b/council/llm/llm_response_parser.py index 8a4eed39..6d04754e 100644 --- a/council/llm/llm_response_parser.py +++ b/council/llm/llm_response_parser.py @@ -1,9 +1,10 @@ +import abc import json import re -from typing import Any, Callable, Dict, Type, TypeVar +from typing import Any, Callable, Dict, Final, Type, TypeVar import yaml -from pydantic import BaseModel, ValidationError +from pydantic import BaseModel, ConfigDict, ValidationError from ..utils import CodeParser from .llm_answer import LLMParsingException @@ -30,10 +31,13 @@ def from_response(response: LLMResponse) -> str: return response.value -class BaseModelResponseParser(BaseModel): +class BaseModelResponseParser(BaseModel, abc.ABC): """Base class for parsing LLM responses into structured data models""" + model_config = ConfigDict(frozen=True) # to preserve field order + @classmethod + @abc.abstractmethod def from_response(cls: Type[T], response: LLMResponse) -> T: """ Parse an LLM response into a structured data model. @@ -88,7 +92,53 @@ def from_response(cls: Type[T], response: LLMResponse) -> T: return cls.create_and_validate(**parsed_blocks) -class YAMLBlockResponseParser(BaseModelResponseParser): +YAML_HINTS: Final[ + str +] = """ +- Make sure you respect YAML syntax, particularly for lists and dictionaries. +- All keys must be present in the response, even when their values are empty. +- For empty values, include empty quotes ("") rather than leaving them blank. +- Always wrap string values in double quotes (") to ensure proper parsing, except when using the YAML pipe operator (|) for multi-line strings. +""" + +YAML_RESPONSE_PARSER_HINTS: Final[str] = "- Provide your response as a parsable YAML." + YAML_HINTS + +YAML_BLOCK_RESPONSE_PARSER_HINTS: Final[str] = "- Provide your response in a single yaml code block." + YAML_HINTS + +T_YAMLResponseParserBase = TypeVar("T_YAMLResponseParserBase", bound="YAMLResponseParserBase") + + +class YAMLResponseParserBase(BaseModelResponseParser, abc.ABC): + @classmethod + def _to_response_template(cls: Type[T]) -> str: + """Generate a YAML response template based on the model's fields and their descriptions.""" + template_parts = [] + + for field_name, field in cls.model_fields.items(): + description = field.description + if description is None: + raise ValueError(f"Description is required for field `{field_name}` in {cls.__name__}") + + is_multiline = "\n" in description + + if field.annotation is str and is_multiline: + template_parts.append(f"{field_name}: |") + for line in description.split("\n"): + template_parts.append(f" {line.strip()}") + else: + template_parts.append(f"{field_name}: {{{{{description}}}}}") # field_name: {{value}} when formatted + + return "\n".join(template_parts) + + @staticmethod + def parse(content: str) -> Dict[str, Any]: + try: + return yaml.safe_load(content) + except yaml.YAMLError as e: + raise LLMParsingException(f"Error while parsing yaml: {e}") + + +class YAMLBlockResponseParser(YAMLResponseParserBase): @classmethod def from_response(cls: Type[T], response: LLMResponse) -> T: @@ -99,26 +149,45 @@ def from_response(cls: Type[T], response: LLMResponse) -> T: if yaml_block is None: raise LLMParsingException("yaml block is not found") - yaml_content = YAMLResponseParser.parse(yaml_block.code) + yaml_content = YAMLResponseParserBase.parse(yaml_block.code) return cls.create_and_validate(**yaml_content) + @classmethod + def to_response_template(cls: Type[T_YAMLResponseParserBase], include_hints: bool = True) -> str: + """ + Generate YAML block response template based on the model's fields and their descriptions. -class YAMLResponseParser(BaseModelResponseParser): + Args: + include_hints: If True, returned template will include universal YAML block formatting hints. + """ + template_parts = [YAML_BLOCK_RESPONSE_PARSER_HINTS] if include_hints else [] + template_parts.extend(["```yaml", cls._to_response_template(), "```"]) + return "\n".join(template_parts) + + +class YAMLResponseParser(YAMLResponseParserBase): @classmethod def from_response(cls: Type[T], response: LLMResponse) -> T: """LLMFunction ResponseParser for response containing raw YAML content""" llm_response = response.value - yaml_content = YAMLResponseParser.parse(llm_response) + yaml_content = YAMLResponseParserBase.parse(llm_response) return cls.create_and_validate(**yaml_content) - @staticmethod - def parse(content: str) -> Dict[str, Any]: - try: - return yaml.safe_load(content) - except yaml.YAMLError as e: - raise LLMParsingException(f"Error while parsing yaml: {e}") + @classmethod + def to_response_template(cls: Type[T_YAMLResponseParserBase], include_hints: bool = True) -> str: + """ + Generate YAML response template based on the model's fields and their descriptions. + + Args: + include_hints: If True, returned template will include universal YAML formatting hints. + """ + template_parts = [YAML_RESPONSE_PARSER_HINTS] if include_hints else [] + template_parts.append(cls._to_response_template()) + if include_hints: + template_parts.extend(["", "Only respond with parsable YAML. Do not output anything else."]) + return "\n".join(template_parts) class JSONBlockResponseParser(BaseModelResponseParser): From 6664009b264f8f7bb82a4cebbd9f356a9f72f3e3 Mon Sep 17 00:00:00 2001 From: Dmytro Nikolaiev Date: Fri, 29 Nov 2024 15:27:02 -0500 Subject: [PATCH 2/9] Initial tests --- ...t_llm_response_parser_response_template.py | 183 ++++++++++++++++++ 1 file changed, 183 insertions(+) create mode 100644 tests/unit/llm/test_llm_response_parser_response_template.py diff --git a/tests/unit/llm/test_llm_response_parser_response_template.py b/tests/unit/llm/test_llm_response_parser_response_template.py new file mode 100644 index 00000000..9912751b --- /dev/null +++ b/tests/unit/llm/test_llm_response_parser_response_template.py @@ -0,0 +1,183 @@ +import unittest +from council.llm.llm_response_parser import YAMLBlockResponseParser, YAMLResponseParser +from pydantic import Field +from typing import Literal, List + + +class MissingDescriptionField(YAMLBlockResponseParser): + number: float = Field(..., description="Number from 1 to 10") + reasoning: str + + +multiline_description = "Carefully\nreason about the number" + + +class YAMLBlockResponse(YAMLBlockResponseParser): + reasoning: str = Field(..., description=multiline_description) + number: float = Field(..., ge=1, le=10, description="Number from 1 to 10") + abc: str = Field(..., description="Not multiline description") + + +class YAMLResponse(YAMLResponseParser): + reasoning: str = Field(..., description=multiline_description) + number: float = Field(..., ge=1, le=10, description="Number from 1 to 10") + abc: str = Field(..., description="Not multiline description") + + +class YAMLBlockResponseReordered(YAMLBlockResponseParser): + number: float = Field(..., ge=1, le=10, description="Number from 1 to 10") + abc: str = Field(..., description="Not multiline description") + reasoning: str = Field(..., description=multiline_description) + + +class YAMLResponseReordered(YAMLResponseParser): + number: float = Field(..., ge=1, le=10, description="Number from 1 to 10") + abc: str = Field(..., description="Not multiline description") + reasoning: str = Field(..., description=multiline_description) + + +class YAMLBlockResponseReorderedAgain(YAMLBlockResponseParser): + abc: str = Field(..., description="Not multiline description") + number: float = Field(..., ge=1, le=10, description="Number from 1 to 10") + reasoning: str = Field(..., description=multiline_description) + + +class YAMLResponseReorderedAgain(YAMLResponseParser): + abc: str = Field(..., description="Not multiline description") + number: float = Field(..., ge=1, le=10, description="Number from 1 to 10") + reasoning: str = Field(..., description=multiline_description) + + +class ComplexResponse(YAMLBlockResponseParser): + mode: Literal["mode_one", "mode_two"] = Field(..., description="Mode of operation, one of `mode_one` or `mode_two`") + pairs: List[YAMLBlockResponse] = Field(..., description="List of number and reasoning pairs") + + +class TestYAMLBlockResponseParserResponseTemplate(unittest.TestCase): + def test_missing_description_field(self): + with self.assertRaises(ValueError) as e: + MissingDescriptionField.to_response_template() + self.assertEqual(str(e.exception), "Description is required for field `reasoning` in MissingDescriptionField") + + def test_number_reasoning_pair_template(self): + template = YAMLBlockResponse.to_response_template(include_hints=False) + self.assertEqual( + template, + """```yaml +reasoning: | + Carefully + reason about the number +number: {{Number from 1 to 10}} +abc: {{Not multiline description}} +```""", + ) + + def test_number_reasoning_pair_reordered_template(self): + template = YAMLBlockResponseReordered.to_response_template(include_hints=False) + self.assertEqual( + template, + """```yaml +number: {{Number from 1 to 10}} +abc: {{Not multiline description}} +reasoning: | + Carefully + reason about the number +```""", + ) + + def test_number_reasoning_pair_reordered_again_template(self): + template = YAMLBlockResponseReorderedAgain.to_response_template(include_hints=False) + self.assertEqual( + template, + """```yaml +abc: {{Not multiline description}} +number: {{Number from 1 to 10}} +reasoning: | + Carefully + reason about the number +```""", + ) + + def test_complex_response_template(self): + template = ComplexResponse.to_response_template(include_hints=False) + # nested objects are not supported yet + self.assertEqual( + template, + """```yaml +mode: {{Mode of operation, one of `mode_one` or `mode_two`}} +pairs: {{List of number and reasoning pairs}} +```""", + ) + + def test_with_hints(self): + template = YAMLBlockResponse.to_response_template(include_hints=True) + self.assertEqual( + template, + """- Provide your response in a single yaml code block. +- Make sure you respect YAML syntax, particularly for lists and dictionaries. +- All keys must be present in the response, even when their values are empty. +- For empty values, include empty quotes ("") rather than leaving them blank. +- Always wrap string values in double quotes (") to ensure proper parsing, except when using the YAML pipe operator (|) for multi-line strings. + +```yaml +reasoning: | + Carefully + reason about the number +number: {{Number from 1 to 10}} +abc: {{Not multiline description}} +```""", + ) + + +class TestYAMLResponseParserResponseTemplate(unittest.TestCase): + def test_number_reasoning_pair_template(self): + template = YAMLResponse.to_response_template(include_hints=False) + self.assertEqual( + template, + """reasoning: | + Carefully + reason about the number +number: {{Number from 1 to 10}} +abc: {{Not multiline description}}""", + ) + + def test_number_reasoning_pair_reordered_template(self): + template = YAMLResponseReordered.to_response_template(include_hints=False) + self.assertEqual( + template, + """number: {{Number from 1 to 10}} +abc: {{Not multiline description}} +reasoning: | + Carefully + reason about the number""", + ) + + def test_number_reasoning_pair_reordered_again_template(self): + template = YAMLResponseReorderedAgain.to_response_template(include_hints=False) + self.assertEqual( + template, + """abc: {{Not multiline description}} +number: {{Number from 1 to 10}} +reasoning: | + Carefully + reason about the number""", + ) + + def test_with_hints(self): + template = YAMLResponse.to_response_template(include_hints=True) + self.assertEqual( + template, + """- Provide your response as a parsable YAML. +- Make sure you respect YAML syntax, particularly for lists and dictionaries. +- All keys must be present in the response, even when their values are empty. +- For empty values, include empty quotes ("") rather than leaving them blank. +- Always wrap string values in double quotes (") to ensure proper parsing, except when using the YAML pipe operator (|) for multi-line strings. + +reasoning: | + Carefully + reason about the number +number: {{Number from 1 to 10}} +abc: {{Not multiline description}} + +Only respond with parsable YAML. Do not output anything else.""", + ) From 72e55cb0131764b51c7ce4d792879746ad198188 Mon Sep 17 00:00:00 2001 From: Dmytro Nikolaiev Date: Fri, 29 Nov 2024 15:38:09 -0500 Subject: [PATCH 3/9] Change {{value}} to # value --- council/llm/llm_response_parser.py | 2 +- ...t_llm_response_parser_response_template.py | 36 +++++++++---------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/council/llm/llm_response_parser.py b/council/llm/llm_response_parser.py index 6d04754e..ff489bb8 100644 --- a/council/llm/llm_response_parser.py +++ b/council/llm/llm_response_parser.py @@ -126,7 +126,7 @@ def _to_response_template(cls: Type[T]) -> str: for line in description.split("\n"): template_parts.append(f" {line.strip()}") else: - template_parts.append(f"{field_name}: {{{{{description}}}}}") # field_name: {{value}} when formatted + template_parts.append(f"{field_name}: # {description}") return "\n".join(template_parts) diff --git a/tests/unit/llm/test_llm_response_parser_response_template.py b/tests/unit/llm/test_llm_response_parser_response_template.py index 9912751b..adc4b4a6 100644 --- a/tests/unit/llm/test_llm_response_parser_response_template.py +++ b/tests/unit/llm/test_llm_response_parser_response_template.py @@ -67,8 +67,8 @@ def test_number_reasoning_pair_template(self): reasoning: | Carefully reason about the number -number: {{Number from 1 to 10}} -abc: {{Not multiline description}} +number: # Number from 1 to 10 +abc: # Not multiline description ```""", ) @@ -77,8 +77,8 @@ def test_number_reasoning_pair_reordered_template(self): self.assertEqual( template, """```yaml -number: {{Number from 1 to 10}} -abc: {{Not multiline description}} +number: # Number from 1 to 10 +abc: # Not multiline description reasoning: | Carefully reason about the number @@ -90,8 +90,8 @@ def test_number_reasoning_pair_reordered_again_template(self): self.assertEqual( template, """```yaml -abc: {{Not multiline description}} -number: {{Number from 1 to 10}} +abc: # Not multiline description +number: # Number from 1 to 10 reasoning: | Carefully reason about the number @@ -104,8 +104,8 @@ def test_complex_response_template(self): self.assertEqual( template, """```yaml -mode: {{Mode of operation, one of `mode_one` or `mode_two`}} -pairs: {{List of number and reasoning pairs}} +mode: # Mode of operation, one of `mode_one` or `mode_two` +pairs: # List of number and reasoning pairs ```""", ) @@ -123,8 +123,8 @@ def test_with_hints(self): reasoning: | Carefully reason about the number -number: {{Number from 1 to 10}} -abc: {{Not multiline description}} +number: # Number from 1 to 10 +abc: # Not multiline description ```""", ) @@ -137,16 +137,16 @@ def test_number_reasoning_pair_template(self): """reasoning: | Carefully reason about the number -number: {{Number from 1 to 10}} -abc: {{Not multiline description}}""", +number: # Number from 1 to 10 +abc: # Not multiline description""", ) def test_number_reasoning_pair_reordered_template(self): template = YAMLResponseReordered.to_response_template(include_hints=False) self.assertEqual( template, - """number: {{Number from 1 to 10}} -abc: {{Not multiline description}} + """number: # Number from 1 to 10 +abc: # Not multiline description reasoning: | Carefully reason about the number""", @@ -156,8 +156,8 @@ def test_number_reasoning_pair_reordered_again_template(self): template = YAMLResponseReorderedAgain.to_response_template(include_hints=False) self.assertEqual( template, - """abc: {{Not multiline description}} -number: {{Number from 1 to 10}} + """abc: # Not multiline description +number: # Number from 1 to 10 reasoning: | Carefully reason about the number""", @@ -176,8 +176,8 @@ def test_with_hints(self): reasoning: | Carefully reason about the number -number: {{Number from 1 to 10}} -abc: {{Not multiline description}} +number: # Number from 1 to 10 +abc: # Not multiline description Only respond with parsable YAML. Do not output anything else.""", ) From f4e159e2b93aa3079323c5d5535433075ffd6ff7 Mon Sep 17 00:00:00 2001 From: Dmytro Nikolaiev Date: Fri, 29 Nov 2024 16:18:36 -0500 Subject: [PATCH 4/9] Implementation for JSON --- council/llm/llm_response_parser.py | 85 ++++++++++++++++++++++++++---- 1 file changed, 74 insertions(+), 11 deletions(-) diff --git a/council/llm/llm_response_parser.py b/council/llm/llm_response_parser.py index ff489bb8..cc1bd909 100644 --- a/council/llm/llm_response_parser.py +++ b/council/llm/llm_response_parser.py @@ -91,7 +91,6 @@ def from_response(cls: Type[T], response: LLMResponse) -> T: return cls.create_and_validate(**parsed_blocks) - YAML_HINTS: Final[ str ] = """ @@ -190,7 +189,52 @@ def to_response_template(cls: Type[T_YAMLResponseParserBase], include_hints: boo return "\n".join(template_parts) -class JSONBlockResponseParser(BaseModelResponseParser): +JSON_HINTS: Final[ + str +] = """ +- Make sure you respect JSON syntax, particularly for lists and dictionaries. +- All keys must be present in the response, even when their values are empty. +- For empty values, include empty quotes ("") rather than leaving them blank. +""" + +JSON_RESPONSE_PARSER_HINTS: Final[str] = "- Provide your response as a parsable JSON." + JSON_HINTS + +JSON_BLOCK_RESPONSE_PARSER_HINTS: Final[str] = "- Provide your response in a single json code block." + JSON_HINTS + +T_JSONResponseParserBase = TypeVar("T_JSONResponseParserBase", bound="JSONResponseParserBase") + + +class JSONResponseParserBase(BaseModelResponseParser, abc.ABC): + @classmethod + def _to_response_template(cls: Type[T]) -> str: + """Generate a JSON response template based on the model's fields and their descriptions.""" + template_dict = {} + + for field_name, field in cls.model_fields.items(): + description = field.description + if description is None: + raise ValueError(f"Description is required for field `{field_name}` in {cls.__name__}") + + is_multiline = "\n" in description + + if field.annotation is str and is_multiline: + # For multiline strings, join the lines with newlines + template_dict[field_name] = "\n".join(line.strip() for line in description.split("\n")) + else: + template_dict[field_name] = description + + # Return formatted JSON with descriptions as values + return json.dumps(template_dict, indent=2) + + @staticmethod + def parse(content: str) -> Dict[str, Any]: + try: + return json.loads(content) + except json.JSONDecodeError as e: + raise LLMParsingException(f"Error while parsing json: {e}") + + +class JSONBlockResponseParser(JSONResponseParserBase): @classmethod def from_response(cls: Type[T], response: LLMResponse) -> T: @@ -201,23 +245,42 @@ def from_response(cls: Type[T], response: LLMResponse) -> T: if json_block is None: raise LLMParsingException("json block is not found") - json_content = JSONResponseParser.parse(json_block.code) + json_content = JSONResponseParserBase.parse(json_block.code) return cls.create_and_validate(**json_content) + @classmethod + def to_response_template(cls: Type[T_JSONResponseParserBase], include_hints: bool = True) -> str: + """ + Generate JSON block response template based on the model's fields and their descriptions. + + Args: + include_hints: If True, returned template will include universal JSON block formatting hints. + """ + template_parts = [JSON_BLOCK_RESPONSE_PARSER_HINTS] if include_hints else [] + template_parts.extend(["```json", cls._to_response_template(), "```"]) + return "\n".join(template_parts) + -class JSONResponseParser(BaseModelResponseParser): +class JSONResponseParser(JSONResponseParserBase): @classmethod def from_response(cls: Type[T], response: LLMResponse) -> T: """LLMFunction ResponseParser for response containing raw JSON content""" llm_response = response.value - json_content = JSONResponseParser.parse(llm_response) + json_content = JSONResponseParserBase.parse(llm_response) return cls.create_and_validate(**json_content) - @staticmethod - def parse(content: str) -> Dict[str, Any]: - try: - return json.loads(content) - except json.JSONDecodeError as e: - raise LLMParsingException(f"Error while parsing json: {e}") + @classmethod + def to_response_template(cls: Type[T_JSONResponseParserBase], include_hints: bool = True) -> str: + """ + Generate JSON response template based on the model's fields and their descriptions. + + Args: + include_hints: If True, returned template will include universal JSON formatting hints. + """ + template_parts = [JSON_RESPONSE_PARSER_HINTS] if include_hints else [] + template_parts.append(cls._to_response_template()) + if include_hints: + template_parts.extend(["", "Only respond with parsable JSON. Do not output anything else."]) + return "\n".join(template_parts) From c1d1ac3c3a2ffc16c49a933af47ab3f800bb85ce Mon Sep 17 00:00:00 2001 From: Dmytro Nikolaiev Date: Fri, 29 Nov 2024 16:25:50 -0500 Subject: [PATCH 5/9] Update hint --- council/llm/llm_response_parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/council/llm/llm_response_parser.py b/council/llm/llm_response_parser.py index cc1bd909..39410b86 100644 --- a/council/llm/llm_response_parser.py +++ b/council/llm/llm_response_parser.py @@ -185,7 +185,7 @@ def to_response_template(cls: Type[T_YAMLResponseParserBase], include_hints: boo template_parts = [YAML_RESPONSE_PARSER_HINTS] if include_hints else [] template_parts.append(cls._to_response_template()) if include_hints: - template_parts.extend(["", "Only respond with parsable YAML. Do not output anything else."]) + template_parts.extend(["", "Only respond with parsable YAML. Do not output anything else. Do not wrap your response in ```yaml```."]) return "\n".join(template_parts) From df0be69313c1c519cce47ff030be5a51c57aa915 Mon Sep 17 00:00:00 2001 From: Dmytro Nikolaiev Date: Fri, 29 Nov 2024 16:26:06 -0500 Subject: [PATCH 6/9] Add integration tests --- ...t_llm_response_parser_response_template.py | 106 ++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100644 tests/integration/llm/test_llm_response_parser_response_template.py diff --git a/tests/integration/llm/test_llm_response_parser_response_template.py b/tests/integration/llm/test_llm_response_parser_response_template.py new file mode 100644 index 00000000..9b7f0dcb --- /dev/null +++ b/tests/integration/llm/test_llm_response_parser_response_template.py @@ -0,0 +1,106 @@ +import unittest + +import dotenv +from typing import Type + +from pydantic import Field + +from council import OpenAILLM +from council.llm.llm_function import LLMFunction, LLMFunctionResponse +from council.llm.llm_response_parser import ( + YAMLBlockResponseParser, + YAMLResponseParser, + JSONBlockResponseParser, + JSONResponseParser, +) +from council.utils import OsEnviron + +SYSTEM_PROMPT = """ +Generate an RPG character. + +# Response template +{response_template} +""" + + +class Character: + name: str = Field(..., description="The name of the character") + power: float = Field(..., ge=0, le=1, description="The power of the character, float from 0 to 1") + role: str = Field(..., description="The role of the character") + + def __str__(self): + return f"Name: {self.name}, Power: {self.power}, Role: {self.role}" + + +class YAMLBlockCharacter(YAMLBlockResponseParser, Character): + pass + + +class YAMLCharacter(YAMLResponseParser, Character): + pass + + +class JSONBlockCharacter(JSONBlockResponseParser, Character): + pass + + +class JSONCharacter(JSONResponseParser, Character): + pass + + +class TestLLMResponseParserResponseTemplate(unittest.TestCase): + """Requires an Azure LLM model deployed""" + + def setUp(self) -> None: + dotenv.load_dotenv() + with OsEnviron("OPENAI_LLM_MODEL", "gpt-4o-mini"): + self.llm = OpenAILLM.from_env() + + def _check_response(self, llm_function_response: LLMFunctionResponse, expected_response_type: Type): + response = llm_function_response.response + self.assertIsInstance(response, expected_response_type) + print("", response, sep="\n") + + self.assertTrue(len(llm_function_response.consumptions) == 8) # no self-correction + + def test_yaml_block_response_parser(self): + llm_func = LLMFunction( + self.llm, + YAMLBlockCharacter.from_response, + system_message=SYSTEM_PROMPT.format(response_template=YAMLBlockCharacter.to_response_template()), + ) + llm_function_response = llm_func.execute_with_llm_response(user_message="Create wise old wizard") + + self._check_response(llm_function_response, YAMLBlockCharacter) + + def test_yaml_response_parser(self): + llm_func = LLMFunction( + self.llm, + YAMLCharacter.from_response, + system_message=SYSTEM_PROMPT.format(response_template=YAMLCharacter.to_response_template()), + ) + llm_function_response = llm_func.execute_with_llm_response(user_message="Create strong warrior") + + self._check_response(llm_function_response, YAMLCharacter) + + def test_json_block_response_parser(self): + llm_func = LLMFunction( + self.llm, + JSONBlockCharacter.from_response, + system_message=SYSTEM_PROMPT.format(response_template=JSONBlockCharacter.to_response_template()), + ) + llm_function_response = llm_func.execute_with_llm_response(user_message="Create kind dwarf") + + self._check_response(llm_function_response, JSONBlockCharacter) + + def test_json_response_parser(self): + llm_func = LLMFunction( + self.llm, + JSONCharacter.from_response, + system_message=SYSTEM_PROMPT.format(response_template=JSONCharacter.to_response_template()), + ) + llm_function_response = llm_func.execute_with_llm_response( + user_message="Create shadow thief", response_format={"type": "json_object"} + ) + + self._check_response(llm_function_response, JSONCharacter) From 23881b628c7806df233788324922a0825f5f6078 Mon Sep 17 00:00:00 2001 From: Dmytro Nikolaiev Date: Fri, 29 Nov 2024 16:56:20 -0500 Subject: [PATCH 7/9] Move hints to yaml and add more unit tests --- council/llm/data/response_hints.yaml | 22 ++ council/llm/llm_response_parser.py | 71 ++++--- ...t_llm_response_parser_response_template.py | 2 +- ...t_llm_response_parser_response_template.py | 191 ++++++++++++++++-- 4 files changed, 236 insertions(+), 50 deletions(-) create mode 100644 council/llm/data/response_hints.yaml diff --git a/council/llm/data/response_hints.yaml b/council/llm/data/response_hints.yaml new file mode 100644 index 00000000..e3ea9d40 --- /dev/null +++ b/council/llm/data/response_hints.yaml @@ -0,0 +1,22 @@ +# YAML +yaml_hints_common: | + - Make sure you respect YAML syntax, particularly for lists and dictionaries. + - All keys must be present in the response, even when their values are empty. + - For empty values, include empty quotes ("") rather than leaving them blank. + - Always wrap string values in double quotes (") to ensure proper parsing, except when using the YAML pipe operator (|) for multi-line strings. +yaml_parser_hints_start: | + - Provide your response as a parsable YAML. +yaml_parser_hints_end: Only respond with parsable YAML. Do not output anything else. Do not wrap your response in ```yaml```. +yaml_block_parser_hints_start: | + - Provide your response in a single yaml code block. + +# JSON +json_hints_common: | + - Make sure you respect JSON syntax, particularly for lists and dictionaries. + - All keys must be present in the response, even when their values are empty. + - For empty values, include empty quotes ("") rather than leaving them blank. +json_parser_hints_start: | + - Provide your response as a parsable JSON. +json_parser_hints_end: Only respond with parsable JSON. Do not output anything else. +json_block_parser_hints_start: | + - Provide your response in a single json code block. diff --git a/council/llm/llm_response_parser.py b/council/llm/llm_response_parser.py index 39410b86..25b679c3 100644 --- a/council/llm/llm_response_parser.py +++ b/council/llm/llm_response_parser.py @@ -1,5 +1,8 @@ +from __future__ import annotations + import abc import json +import os import re from typing import Any, Callable, Dict, Final, Type, TypeVar @@ -16,6 +19,38 @@ T = TypeVar("T", bound="BaseModelResponseParser") +RESPONSE_HINTS_FILE_PATH: Final[str] = os.path.join(os.path.dirname(__file__), "data", "response_hints.yaml") + + +class ResponseHintsHelper: + def __init__(self, hints: Dict[str, str], prefix: str): + self.hints_common = hints[f"{prefix}_hints_common"] + self.parser_hints_start = hints[f"{prefix}_parser_hints_start"] + self.parser_hints_end = hints[f"{prefix}_parser_hints_end"] + self.block_parser_hints_start = hints[f"{prefix}_block_parser_hints_start"] + + @classmethod + def from_yaml(cls, path: str, prefix: str) -> ResponseHintsHelper: + with open(path, "r", encoding="utf-8") as file: + hints = yaml.safe_load(file) + return cls(hints, prefix) + + @property + def parser(self) -> str: + return self.parser_hints_start + self.hints_common + + @property + def block_parser(self) -> str: + return self.block_parser_hints_start + self.hints_common + + @property + def parser_end(self) -> str: + return self.parser_hints_end + + +yaml_response_hints = ResponseHintsHelper.from_yaml(RESPONSE_HINTS_FILE_PATH, prefix="yaml") +json_response_hints = ResponseHintsHelper.from_yaml(RESPONSE_HINTS_FILE_PATH, prefix="json") + class EchoResponseParser: @staticmethod @@ -91,18 +126,6 @@ def from_response(cls: Type[T], response: LLMResponse) -> T: return cls.create_and_validate(**parsed_blocks) -YAML_HINTS: Final[ - str -] = """ -- Make sure you respect YAML syntax, particularly for lists and dictionaries. -- All keys must be present in the response, even when their values are empty. -- For empty values, include empty quotes ("") rather than leaving them blank. -- Always wrap string values in double quotes (") to ensure proper parsing, except when using the YAML pipe operator (|) for multi-line strings. -""" - -YAML_RESPONSE_PARSER_HINTS: Final[str] = "- Provide your response as a parsable YAML." + YAML_HINTS - -YAML_BLOCK_RESPONSE_PARSER_HINTS: Final[str] = "- Provide your response in a single yaml code block." + YAML_HINTS T_YAMLResponseParserBase = TypeVar("T_YAMLResponseParserBase", bound="YAMLResponseParserBase") @@ -159,7 +182,7 @@ def to_response_template(cls: Type[T_YAMLResponseParserBase], include_hints: boo Args: include_hints: If True, returned template will include universal YAML block formatting hints. """ - template_parts = [YAML_BLOCK_RESPONSE_PARSER_HINTS] if include_hints else [] + template_parts = [yaml_response_hints.block_parser] if include_hints else [] template_parts.extend(["```yaml", cls._to_response_template(), "```"]) return "\n".join(template_parts) @@ -182,25 +205,13 @@ def to_response_template(cls: Type[T_YAMLResponseParserBase], include_hints: boo Args: include_hints: If True, returned template will include universal YAML formatting hints. """ - template_parts = [YAML_RESPONSE_PARSER_HINTS] if include_hints else [] + template_parts = [yaml_response_hints.parser] if include_hints else [] template_parts.append(cls._to_response_template()) if include_hints: - template_parts.extend(["", "Only respond with parsable YAML. Do not output anything else. Do not wrap your response in ```yaml```."]) + template_parts.extend(["", yaml_response_hints.parser_end]) return "\n".join(template_parts) -JSON_HINTS: Final[ - str -] = """ -- Make sure you respect JSON syntax, particularly for lists and dictionaries. -- All keys must be present in the response, even when their values are empty. -- For empty values, include empty quotes ("") rather than leaving them blank. -""" - -JSON_RESPONSE_PARSER_HINTS: Final[str] = "- Provide your response as a parsable JSON." + JSON_HINTS - -JSON_BLOCK_RESPONSE_PARSER_HINTS: Final[str] = "- Provide your response in a single json code block." + JSON_HINTS - T_JSONResponseParserBase = TypeVar("T_JSONResponseParserBase", bound="JSONResponseParserBase") @@ -256,7 +267,7 @@ def to_response_template(cls: Type[T_JSONResponseParserBase], include_hints: boo Args: include_hints: If True, returned template will include universal JSON block formatting hints. """ - template_parts = [JSON_BLOCK_RESPONSE_PARSER_HINTS] if include_hints else [] + template_parts = [json_response_hints.block_parser] if include_hints else [] template_parts.extend(["```json", cls._to_response_template(), "```"]) return "\n".join(template_parts) @@ -279,8 +290,8 @@ def to_response_template(cls: Type[T_JSONResponseParserBase], include_hints: boo Args: include_hints: If True, returned template will include universal JSON formatting hints. """ - template_parts = [JSON_RESPONSE_PARSER_HINTS] if include_hints else [] + template_parts = [json_response_hints.parser] if include_hints else [] template_parts.append(cls._to_response_template()) if include_hints: - template_parts.extend(["", "Only respond with parsable JSON. Do not output anything else."]) + template_parts.extend(["", json_response_hints.parser_end]) return "\n".join(template_parts) diff --git a/tests/integration/llm/test_llm_response_parser_response_template.py b/tests/integration/llm/test_llm_response_parser_response_template.py index 9b7f0dcb..11ddd7bd 100644 --- a/tests/integration/llm/test_llm_response_parser_response_template.py +++ b/tests/integration/llm/test_llm_response_parser_response_template.py @@ -25,7 +25,7 @@ class Character: name: str = Field(..., description="The name of the character") - power: float = Field(..., ge=0, le=1, description="The power of the character, float from 0 to 1") + power: float = Field(..., ge=0, le=1, description="The power of the character, float from 0 to 1") role: str = Field(..., description="The role of the character") def __str__(self): diff --git a/tests/unit/llm/test_llm_response_parser_response_template.py b/tests/unit/llm/test_llm_response_parser_response_template.py index adc4b4a6..e3c3a5c5 100644 --- a/tests/unit/llm/test_llm_response_parser_response_template.py +++ b/tests/unit/llm/test_llm_response_parser_response_template.py @@ -1,10 +1,21 @@ import unittest -from council.llm.llm_response_parser import YAMLBlockResponseParser, YAMLResponseParser +from council.llm.llm_response_parser import ( + YAMLBlockResponseParser, + YAMLResponseParser, + JSONBlockResponseParser, + JSONResponseParser, +) from pydantic import Field from typing import Literal, List class MissingDescriptionField(YAMLBlockResponseParser): + _multiline_description = "\n".join( + [ + "You can define multi-line description inside the response class", + "Like that", + ] + ) number: float = Field(..., description="Number from 1 to 10") reasoning: str @@ -12,40 +23,70 @@ class MissingDescriptionField(YAMLBlockResponseParser): multiline_description = "Carefully\nreason about the number" -class YAMLBlockResponse(YAMLBlockResponseParser): +class BaseResponse: reasoning: str = Field(..., description=multiline_description) number: float = Field(..., ge=1, le=10, description="Number from 1 to 10") abc: str = Field(..., description="Not multiline description") -class YAMLResponse(YAMLResponseParser): - reasoning: str = Field(..., description=multiline_description) +class BaseResponseReordered: number: float = Field(..., ge=1, le=10, description="Number from 1 to 10") abc: str = Field(..., description="Not multiline description") + reasoning: str = Field(..., description=multiline_description) -class YAMLBlockResponseReordered(YAMLBlockResponseParser): - number: float = Field(..., ge=1, le=10, description="Number from 1 to 10") +class BaseResponseReorderedAgain: abc: str = Field(..., description="Not multiline description") + number: float = Field(..., ge=1, le=10, description="Number from 1 to 10") reasoning: str = Field(..., description=multiline_description) -class YAMLResponseReordered(YAMLResponseParser): - number: float = Field(..., ge=1, le=10, description="Number from 1 to 10") - abc: str = Field(..., description="Not multiline description") - reasoning: str = Field(..., description=multiline_description) +class YAMLBlockResponse(YAMLBlockResponseParser, BaseResponse): + pass -class YAMLBlockResponseReorderedAgain(YAMLBlockResponseParser): - abc: str = Field(..., description="Not multiline description") - number: float = Field(..., ge=1, le=10, description="Number from 1 to 10") - reasoning: str = Field(..., description=multiline_description) +class YAMLResponse(YAMLResponseParser, BaseResponse): + pass -class YAMLResponseReorderedAgain(YAMLResponseParser): - abc: str = Field(..., description="Not multiline description") - number: float = Field(..., ge=1, le=10, description="Number from 1 to 10") - reasoning: str = Field(..., description=multiline_description) +class JSONBlockResponse(JSONBlockResponseParser, BaseResponse): + pass + + +class JSONResponse(JSONResponseParser, BaseResponse): + pass + + +class YAMLBlockResponseReordered(YAMLBlockResponseParser, BaseResponseReordered): + pass + + +class YAMLResponseReordered(YAMLResponseParser, BaseResponseReordered): + pass + + +class JSONBlockResponseReordered(JSONBlockResponseParser, BaseResponseReordered): + pass + + +class JSONResponseReordered(JSONResponseParser, BaseResponseReordered): + pass + + +class YAMLBlockResponseReorderedAgain(YAMLBlockResponseParser, BaseResponseReorderedAgain): + pass + + +class YAMLResponseReorderedAgain(YAMLResponseParser, BaseResponseReorderedAgain): + pass + + +class JSONBlockResponseReorderedAgain(JSONBlockResponseParser, BaseResponseReorderedAgain): + pass + + +class JSONResponseReorderedAgain(JSONResponseParser, BaseResponseReorderedAgain): + pass class ComplexResponse(YAMLBlockResponseParser): @@ -179,5 +220,117 @@ def test_with_hints(self): number: # Number from 1 to 10 abc: # Not multiline description -Only respond with parsable YAML. Do not output anything else.""", +Only respond with parsable YAML. Do not output anything else. Do not wrap your response in ```yaml```.""", + ) + + +class TestJSONBlockResponseParserResponseTemplate(unittest.TestCase): + def test_number_reasoning_pair_template(self): + template = JSONBlockResponse.to_response_template(include_hints=False) + self.assertEqual( + template, + """```json +{ + "reasoning": "Carefully\\nreason about the number", + "number": "Number from 1 to 10", + "abc": "Not multiline description" +} +```""", + ) + + def test_number_reasoning_pair_reordered_template(self): + template = JSONBlockResponseReordered.to_response_template(include_hints=False) + self.assertEqual( + template, + """```json +{ + "number": "Number from 1 to 10", + "abc": "Not multiline description", + "reasoning": "Carefully\\nreason about the number" +} +```""", + ) + + def test_number_reasoning_pair_reordered_again_template(self): + template = JSONBlockResponseReorderedAgain.to_response_template(include_hints=False) + self.assertEqual( + template, + """```json +{ + "abc": "Not multiline description", + "number": "Number from 1 to 10", + "reasoning": "Carefully\\nreason about the number" +} +```""", + ) + + def test_with_hints(self): + template = JSONBlockResponse.to_response_template(include_hints=True) + self.assertEqual( + template, + """- Provide your response in a single json code block. +- Make sure you respect JSON syntax, particularly for lists and dictionaries. +- All keys must be present in the response, even when their values are empty. +- For empty values, include empty quotes ("") rather than leaving them blank. + +```json +{ + "reasoning": "Carefully\\nreason about the number", + "number": "Number from 1 to 10", + "abc": "Not multiline description" +} +```""", + ) + + +class TestJSONResponseParserResponseTemplate(unittest.TestCase): + def test_number_reasoning_pair_template(self): + template = JSONResponse.to_response_template(include_hints=False) + self.assertEqual( + template, + """{ + "reasoning": "Carefully\\nreason about the number", + "number": "Number from 1 to 10", + "abc": "Not multiline description" +}""", + ) + + def test_number_reasoning_pair_reordered_template(self): + template = JSONResponseReordered.to_response_template(include_hints=False) + self.assertEqual( + template, + """{ + "number": "Number from 1 to 10", + "abc": "Not multiline description", + "reasoning": "Carefully\\nreason about the number" +}""", + ) + + def test_number_reasoning_pair_reordered_again_template(self): + template = JSONResponseReorderedAgain.to_response_template(include_hints=False) + self.assertEqual( + template, + """{ + "abc": "Not multiline description", + "number": "Number from 1 to 10", + "reasoning": "Carefully\\nreason about the number" +}""", + ) + + def test_with_hints(self): + template = JSONResponse.to_response_template(include_hints=True) + self.assertEqual( + template, + """- Provide your response as a parsable JSON. +- Make sure you respect JSON syntax, particularly for lists and dictionaries. +- All keys must be present in the response, even when their values are empty. +- For empty values, include empty quotes ("") rather than leaving them blank. + +{ + "reasoning": "Carefully\\nreason about the number", + "number": "Number from 1 to 10", + "abc": "Not multiline description" +} + +Only respond with parsable JSON. Do not output anything else.""", ) From 8c4ba8e2366c52374feda43b6f99681e54cdadb8 Mon Sep 17 00:00:00 2001 From: Dmytro Nikolaiev Date: Mon, 2 Dec 2024 14:03:07 -0500 Subject: [PATCH 8/9] Clean up --- council/llm/data/response_hints.yaml | 4 +-- council/llm/llm_response_parser.py | 2 -- ...t_llm_response_parser_response_template.py | 2 -- ...t_llm_response_parser_response_template.py | 28 +++++++++---------- 4 files changed, 16 insertions(+), 20 deletions(-) diff --git a/council/llm/data/response_hints.yaml b/council/llm/data/response_hints.yaml index e3ea9d40..2eb5d437 100644 --- a/council/llm/data/response_hints.yaml +++ b/council/llm/data/response_hints.yaml @@ -8,7 +8,7 @@ yaml_parser_hints_start: | - Provide your response as a parsable YAML. yaml_parser_hints_end: Only respond with parsable YAML. Do not output anything else. Do not wrap your response in ```yaml```. yaml_block_parser_hints_start: | - - Provide your response in a single yaml code block. + - Provide your response in a single YAML code block. # JSON json_hints_common: | @@ -19,4 +19,4 @@ json_parser_hints_start: | - Provide your response as a parsable JSON. json_parser_hints_end: Only respond with parsable JSON. Do not output anything else. json_block_parser_hints_start: | - - Provide your response in a single json code block. + - Provide your response in a single JSON code block. diff --git a/council/llm/llm_response_parser.py b/council/llm/llm_response_parser.py index 25b679c3..c7529496 100644 --- a/council/llm/llm_response_parser.py +++ b/council/llm/llm_response_parser.py @@ -229,12 +229,10 @@ def _to_response_template(cls: Type[T]) -> str: is_multiline = "\n" in description if field.annotation is str and is_multiline: - # For multiline strings, join the lines with newlines template_dict[field_name] = "\n".join(line.strip() for line in description.split("\n")) else: template_dict[field_name] = description - # Return formatted JSON with descriptions as values return json.dumps(template_dict, indent=2) @staticmethod diff --git a/tests/integration/llm/test_llm_response_parser_response_template.py b/tests/integration/llm/test_llm_response_parser_response_template.py index 11ddd7bd..bbaddf44 100644 --- a/tests/integration/llm/test_llm_response_parser_response_template.py +++ b/tests/integration/llm/test_llm_response_parser_response_template.py @@ -49,8 +49,6 @@ class JSONCharacter(JSONResponseParser, Character): class TestLLMResponseParserResponseTemplate(unittest.TestCase): - """Requires an Azure LLM model deployed""" - def setUp(self) -> None: dotenv.load_dotenv() with OsEnviron("OPENAI_LLM_MODEL", "gpt-4o-mini"): diff --git a/tests/unit/llm/test_llm_response_parser_response_template.py b/tests/unit/llm/test_llm_response_parser_response_template.py index e3c3a5c5..a44096f8 100644 --- a/tests/unit/llm/test_llm_response_parser_response_template.py +++ b/tests/unit/llm/test_llm_response_parser_response_template.py @@ -100,7 +100,7 @@ def test_missing_description_field(self): MissingDescriptionField.to_response_template() self.assertEqual(str(e.exception), "Description is required for field `reasoning` in MissingDescriptionField") - def test_number_reasoning_pair_template(self): + def test_template(self): template = YAMLBlockResponse.to_response_template(include_hints=False) self.assertEqual( template, @@ -113,7 +113,7 @@ def test_number_reasoning_pair_template(self): ```""", ) - def test_number_reasoning_pair_reordered_template(self): + def test_reordered_template(self): template = YAMLBlockResponseReordered.to_response_template(include_hints=False) self.assertEqual( template, @@ -126,7 +126,7 @@ def test_number_reasoning_pair_reordered_template(self): ```""", ) - def test_number_reasoning_pair_reordered_again_template(self): + def test_reordered_again_template(self): template = YAMLBlockResponseReorderedAgain.to_response_template(include_hints=False) self.assertEqual( template, @@ -154,7 +154,7 @@ def test_with_hints(self): template = YAMLBlockResponse.to_response_template(include_hints=True) self.assertEqual( template, - """- Provide your response in a single yaml code block. + """- Provide your response in a single YAML code block. - Make sure you respect YAML syntax, particularly for lists and dictionaries. - All keys must be present in the response, even when their values are empty. - For empty values, include empty quotes ("") rather than leaving them blank. @@ -171,7 +171,7 @@ def test_with_hints(self): class TestYAMLResponseParserResponseTemplate(unittest.TestCase): - def test_number_reasoning_pair_template(self): + def test_template(self): template = YAMLResponse.to_response_template(include_hints=False) self.assertEqual( template, @@ -182,7 +182,7 @@ def test_number_reasoning_pair_template(self): abc: # Not multiline description""", ) - def test_number_reasoning_pair_reordered_template(self): + def test_reordered_template(self): template = YAMLResponseReordered.to_response_template(include_hints=False) self.assertEqual( template, @@ -193,7 +193,7 @@ def test_number_reasoning_pair_reordered_template(self): reason about the number""", ) - def test_number_reasoning_pair_reordered_again_template(self): + def test_reordered_again_template(self): template = YAMLResponseReorderedAgain.to_response_template(include_hints=False) self.assertEqual( template, @@ -225,7 +225,7 @@ def test_with_hints(self): class TestJSONBlockResponseParserResponseTemplate(unittest.TestCase): - def test_number_reasoning_pair_template(self): + def test_template(self): template = JSONBlockResponse.to_response_template(include_hints=False) self.assertEqual( template, @@ -238,7 +238,7 @@ def test_number_reasoning_pair_template(self): ```""", ) - def test_number_reasoning_pair_reordered_template(self): + def test_reordered_template(self): template = JSONBlockResponseReordered.to_response_template(include_hints=False) self.assertEqual( template, @@ -251,7 +251,7 @@ def test_number_reasoning_pair_reordered_template(self): ```""", ) - def test_number_reasoning_pair_reordered_again_template(self): + def test_reordered_again_template(self): template = JSONBlockResponseReorderedAgain.to_response_template(include_hints=False) self.assertEqual( template, @@ -268,7 +268,7 @@ def test_with_hints(self): template = JSONBlockResponse.to_response_template(include_hints=True) self.assertEqual( template, - """- Provide your response in a single json code block. + """- Provide your response in a single JSON code block. - Make sure you respect JSON syntax, particularly for lists and dictionaries. - All keys must be present in the response, even when their values are empty. - For empty values, include empty quotes ("") rather than leaving them blank. @@ -284,7 +284,7 @@ def test_with_hints(self): class TestJSONResponseParserResponseTemplate(unittest.TestCase): - def test_number_reasoning_pair_template(self): + def test_template(self): template = JSONResponse.to_response_template(include_hints=False) self.assertEqual( template, @@ -295,7 +295,7 @@ def test_number_reasoning_pair_template(self): }""", ) - def test_number_reasoning_pair_reordered_template(self): + def test_reordered_template(self): template = JSONResponseReordered.to_response_template(include_hints=False) self.assertEqual( template, @@ -306,7 +306,7 @@ def test_number_reasoning_pair_reordered_template(self): }""", ) - def test_number_reasoning_pair_reordered_again_template(self): + def test_reordered_again_template(self): template = JSONResponseReorderedAgain.to_response_template(include_hints=False) self.assertEqual( template, From b2afada2bef4db2c1a50826c8c89b04f010920b5 Mon Sep 17 00:00:00 2001 From: Dmytro Nikolaiev Date: Mon, 2 Dec 2024 14:03:13 -0500 Subject: [PATCH 9/9] Docs update --- .../reference/llm/llm_response_parser.md | 54 +++++++++---------- 1 file changed, 26 insertions(+), 28 deletions(-) diff --git a/docs/source/reference/llm/llm_response_parser.md b/docs/source/reference/llm/llm_response_parser.md index 9dd29b12..055b7aed 100644 --- a/docs/source/reference/llm/llm_response_parser.md +++ b/docs/source/reference/llm/llm_response_parser.md @@ -83,7 +83,7 @@ print(response.sql) import os from typing import Literal -# !pip install council-ai==0.0.24 +# !pip install council-ai==0.0.27 from council import OpenAILLM from council.llm.llm_function import LLMFunction @@ -91,22 +91,19 @@ from council.llm.llm_response_parser import YAMLBlockResponseParser from pydantic import Field SYSTEM_PROMPT = """ -Output RPG character info in the following YAML block: +Generate RPG character: -```yaml -character_class: # character's class (Warrior, Mage, Rogue, Bard or Tech Support) -name: # character's name -description: # character's tragic backstory, 50 chars minimum -health: # character's health, integer, from 1 to 100 points -``` +{response_template} """ class RPGCharacterFromYAMLBlock(YAMLBlockResponseParser): - name: str - character_class: Literal["Warrior", "Mage", "Rogue", "Bard", "Tech Support"] - description: str = Field(..., min_length=50) - health: int = Field(..., ge=1, le=100) + character_class: Literal["Warrior", "Mage", "Rogue", "Bard", "Tech Support"] = Field( + ..., description="Character's class (Warrior, Mage, Rogue, Bard or Tech Support)" + ) + name: str = Field(..., min_length=3, description="Character's name") + description: str = Field(..., min_length=50, description="Character's tragic backstory, 50 chars minimum") + health: int = Field(..., ge=1, le=100, description="Character's health, integer, from 1 to 100 points") os.environ["OPENAI_API_KEY"] = "sk-YOUR-KEY-HERE" @@ -114,7 +111,9 @@ os.environ["OPENAI_LLM_MODEL"] = "gpt-4o-mini-2024-07-18" llm = OpenAILLM.from_env() llm_function: LLMFunction[RPGCharacterFromYAMLBlock] = LLMFunction( - llm, RPGCharacterFromYAMLBlock.from_response, SYSTEM_PROMPT + llm, + RPGCharacterFromYAMLBlock.from_response, + SYSTEM_PROMPT.format(response_template=RPGCharacterFromYAMLBlock.to_response_template()), ) character = llm_function.execute(user_message="Create some wise mage") @@ -155,7 +154,7 @@ Usage example with OpenAI json mode: import os from typing import Literal -# !pip install council-ai==0.0.24 +# !pip install council-ai==0.0.27 from council import OpenAILLM from council.llm.llm_function import LLMFunction @@ -163,22 +162,19 @@ from council.llm.llm_response_parser import JSONResponseParser from pydantic import Field SYSTEM_PROMPT = """ -Output RPG character info in the following JSON format: - -{ -character_class: # character's class (Warrior, Mage, Rogue, Bard or Tech Support) -name: # character's name -description: # character's tragic backstory, 50 chars minimum -health: # character's health, integer, from 1 to 100 points -} +Generate RPG character: + +{response_template} """ class RPGCharacterFromJSON(JSONResponseParser): - name: str - character_class: Literal["Warrior", "Mage", "Rogue", "Bard", "Tech Support"] - description: str = Field(..., min_length=50) - health: int = Field(..., ge=1, le=100) + character_class: Literal["Warrior", "Mage", "Rogue", "Bard", "Tech Support"] = Field( + ..., description="Character's class (Warrior, Mage, Rogue, Bard or Tech Support)" + ) + name: str = Field(..., min_length=3, description="Character's name") + description: str = Field(..., min_length=50, description="Character's tragic backstory, 50 chars minimum") + health: int = Field(..., ge=1, le=100, description="Character's health, integer, from 1 to 100 points") os.environ["OPENAI_API_KEY"] = "sk-YOUR-KEY-HERE" @@ -186,11 +182,13 @@ os.environ["OPENAI_LLM_MODEL"] = "gpt-4o-mini-2024-07-18" llm = OpenAILLM.from_env() llm_function: LLMFunction[RPGCharacterFromJSON] = LLMFunction( - llm, RPGCharacterFromJSON.from_response, SYSTEM_PROMPT + llm, + RPGCharacterFromJSON.from_response, + SYSTEM_PROMPT.format(response_template=RPGCharacterFromJSON.to_response_template()), ) character = llm_function.execute( - user_message="Create some wise mage", + user_message="Create some strong warrior", response_format={"type": "json_object"} # using OpenAI's json mode ) print(type(character))