diff --git a/src/llm/output/actions_parser.py b/src/llm/output/actions_parser.py index a117f9c3..fe01c0fa 100644 --- a/src/llm/output/actions_parser.py +++ b/src/llm/output/actions_parser.py @@ -24,3 +24,6 @@ def modify_sentence_content(self, cut_content: sentence_content, last_content: s # if action.is_interrupting: # settings.stop_generation = True return cut_content, last_content + + def get_cut_indicators(self) -> list[str]: + return [":"] diff --git a/src/llm/output/change_character_parser.py b/src/llm/output/change_character_parser.py index 588ed4cd..651a43d8 100644 --- a/src/llm/output/change_character_parser.py +++ b/src/llm/output/change_character_parser.py @@ -53,4 +53,7 @@ def cut_sentence(self, output: str, current_settings: sentence_generation_settin return None, output #There is a ':' in the text, but it doesn't seem to be part of a character change def modify_sentence_content(self, cut_content: sentence_content, last_content: sentence_content | None, settings: sentence_generation_settings) -> tuple[sentence_content | None, sentence_content | None]: - return cut_content, last_content \ No newline at end of file + return cut_content, last_content + + def get_cut_indicators(self) -> list[str]: + return [":"] \ No newline at end of file diff --git a/src/llm/output/narration_parser.py b/src/llm/output/narration_parser.py index ea233e33..795263ba 100644 --- a/src/llm/output/narration_parser.py +++ b/src/llm/output/narration_parser.py @@ -28,6 +28,9 @@ def __init__(self, narration_start_chars: list[str] = ["*","(","["], narration_e else: self.__start_speech_reg = never_match_anything_regex self.__end_speech_reg = never_match_anything_regex + + def get_cut_indicators(self) -> list[str]: + return self.__narration_start_chars + self.__narration_end_chars + self.__speech_start_chars + self.__speech_end_chars def cut_sentence(self, output: str, current_settings: sentence_generation_settings) -> tuple[sentence_content | None, str]: output = output.lstrip() diff --git a/src/llm/output/output_parser.py b/src/llm/output/output_parser.py index a1dbb8dc..2b9aeed1 100644 --- a/src/llm/output/output_parser.py +++ b/src/llm/output/output_parser.py @@ -70,3 +70,6 @@ def cut_sentence(self, output: str, current_settings: sentence_generation_settin @abstractmethod def modify_sentence_content(self, cut_content: sentence_content, last_content: sentence_content | None, settings: sentence_generation_settings) -> tuple[sentence_content | None, sentence_content | None]: return cut_content, last_content + + def get_cut_indicators(self) -> list[str]: + return [] diff --git a/src/llm/output/sentence_accumulator.py b/src/llm/output/sentence_accumulator.py new file mode 100644 index 00000000..b408440b --- /dev/null +++ b/src/llm/output/sentence_accumulator.py @@ -0,0 +1,43 @@ +import re + + +class sentence_accumulator: + """Accumulates the token-wise output of an LLM into raw sentences. + """ + def __init__(self, cut_indicators: list[str]) -> None: + self.__cut_indicators = cut_indicators + self.__unprocessed_llm_output: str = "" + base_regex_def = "^.*?[{sentence_end_chars}]+" + self.__sentence_end_reg = re.compile(base_regex_def.format(sentence_end_chars = "\\" + "\\".join(cut_indicators))) + self.__unparseable: str = "" + self.__prepared_match: str = "" + + def has_next_sentence(self) -> bool: + if len(self.__prepared_match) > 0: + return True + + match = self.__sentence_end_reg.match(self.__unprocessed_llm_output) + if not match: + return False + else: + self.__prepared_match = match.group() + self.__unprocessed_llm_output = self.__unprocessed_llm_output.removeprefix(self.__prepared_match) + return True + + def get_next_sentence(self) -> str: + result = self.__unparseable + self.__prepared_match + self.__unparseable = "" + self.__prepared_match = "" + return result + + def accumulate(self, llm_output: str): + llm_output = llm_output.replace('\r\n', ' ') + llm_output = llm_output.replace('\n', ' ') + self.__unprocessed_llm_output += llm_output + + def refuse(self, refused_text: str): + self.__unparseable = refused_text + + + + diff --git a/src/llm/output/sentence_end_parser.py b/src/llm/output/sentence_end_parser.py index ecd0116c..8b52d614 100644 --- a/src/llm/output/sentence_end_parser.py +++ b/src/llm/output/sentence_end_parser.py @@ -22,4 +22,7 @@ def cut_sentence(self, output: str, current_settings: sentence_generation_settin return sentence_content(current_settings.current_speaker, matched_text, current_settings.sentence_type, False), rest def modify_sentence_content(self, cut_content: sentence_content, last_content: sentence_content | None, settings: sentence_generation_settings) -> tuple[sentence_content | None, sentence_content | None]: - return cut_content, last_content \ No newline at end of file + return cut_content, last_content + + def get_cut_indicators(self) -> list[str]: + return self.__end_of_sentence_chars \ No newline at end of file diff --git a/src/output_manager.py b/src/output_manager.py index c0a44fe8..8b07fef7 100644 --- a/src/output_manager.py +++ b/src/output_manager.py @@ -5,6 +5,7 @@ import time import unicodedata from openai import APIConnectionError +from src.llm.output.sentence_accumulator import sentence_accumulator from src.config.definitions.llm_definitions import NarrationHandlingEnum from src.llm.output.max_count_sentences_parser import max_count_sentences_parser from src.llm.output.sentence_length_parser import sentence_length_parser @@ -143,6 +144,13 @@ async def process_response(self, active_character: Character, blocking_queue: se sentence_length_parser(self.__config.number_words_tts), max_count_sentences_parser(self.__config.max_response_sentences, not characters.contains_player_character()) ]) + + cut_indicators: set[str] = set() + for parser in parser_chain: + indicators = parser.get_cut_indicators() + for i in indicators: + cut_indicators.add(i) + accumulator: sentence_accumulator = sentence_accumulator(list(cut_indicators)) try: current_sentence: str = '' @@ -160,26 +168,31 @@ async def process_response(self, active_character: Character, blocking_queue: se logging.log(self.loglevel, f"LLM took {round(time.time() - start_time, 5)} seconds to respond") first_token = False - current_sentence += content raw_response += content - parsed_sentence: sentence_content | None = None - # Apply parsers - for parser in parser_chain: - if not parsed_sentence: # Try to extract a complete sentence - parsed_sentence, current_sentence = parser.cut_sentence(current_sentence, settings) - if parsed_sentence: # Apply modifications if we already have a sentence - parsed_sentence, pending_sentence = parser.modify_sentence_content(parsed_sentence, pending_sentence, settings) + accumulator.accumulate(content) + while accumulator.has_next_sentence(): + current_sentence = accumulator.get_next_sentence() + # current_sentence += content + parsed_sentence: sentence_content | None = None + # Apply parsers + for parser in parser_chain: + if not parsed_sentence: # Try to extract a complete sentence + parsed_sentence, current_sentence = parser.cut_sentence(current_sentence, settings) + if parsed_sentence: # Apply modifications if we already have a sentence + parsed_sentence, pending_sentence = parser.modify_sentence_content(parsed_sentence, pending_sentence, settings) + if settings.stop_generation: + break if settings.stop_generation: break + accumulator.refuse(current_sentence) + # Process sentences from the parser chain + if parsed_sentence: + if not self.__config.narration_handling == NarrationHandlingEnum.CUT_NARRATIONS or parsed_sentence.sentence_type != SentenceTypeEnum.NARRATION: + new_sentence = self.generate_sentence(parsed_sentence) + blocking_queue.put(new_sentence) + parsed_sentence = None if settings.stop_generation: - break - - # Process sentences from the parser chain - if parsed_sentence: - if not self.__config.narration_handling == NarrationHandlingEnum.CUT_NARRATIONS or parsed_sentence.sentence_type != SentenceTypeEnum.NARRATION: - new_sentence = self.generate_sentence(parsed_sentence) - blocking_queue.put(new_sentence) - parsed_sentence = None + break break #if the streaming_call() completed without exception, break the while loop except Exception as e: