From 227aaf5b1005ef1ee6f2f535b890c5d29d434349 Mon Sep 17 00:00:00 2001 From: JarbasAI <33701864+JarbasAl@users.noreply.github.com> Date: Wed, 16 Oct 2024 10:32:48 +0100 Subject: [PATCH] feat!:pipeline factory (#15) * fix:standardize_lang * fix:standardize_lang * feat:pipeline plugin factory * feat:pipeline plugin factory * feat:pipeline plugin factory * feat:pipeline plugin factory * feat:pipeline plugin factory * feat:pipeline plugin factory * Update ovos_padatious/opm.py Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> * feat:pipeline plugin factory * feat:pipeline plugin factory --------- Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- ovos_padatious/opm.py | 148 ++++++++++++++++++++++++++++++++++-------- requirements.txt | 2 +- 2 files changed, 121 insertions(+), 29 deletions(-) diff --git a/ovos_padatious/opm.py b/ovos_padatious/opm.py index b93fd1b..158cbc7 100644 --- a/ovos_padatious/opm.py +++ b/ovos_padatious/opm.py @@ -16,26 +16,30 @@ from functools import lru_cache from os.path import expanduser, isfile from threading import Event -from typing import List, Optional +from typing import Optional, Dict, List, Union +from langcodes import closest_match +from ovos_bus_client.client import MessageBusClient from ovos_bus_client.message import Message from ovos_bus_client.session import SessionManager, Session from ovos_config.config import Configuration from ovos_config.meta import get_xdg_base -from ovos_padatious import IntentContainer as PadatiousIntentContainer -from ovos_padatious.match_data import MatchData as PadatiousIntent +from ovos_plugin_manager.templates.pipeline import ConfidenceMatcherPipeline, IntentHandlerMatch, IntentMatch from ovos_utils import flatten_list +from ovos_utils.fakebus import FakeBus from ovos_utils.lang import standardize_lang_tag -from ovos_utils.log import LOG +from ovos_utils.log import LOG, deprecated, log_deprecation from ovos_utils.xdg_utils import xdg_data_home -from ovos_plugin_manager.templates.pipeline import PipelinePlugin, IntentMatch -from langcodes import closest_match + +from ovos_padatious import IntentContainer as PadatiousIntentContainer +from ovos_padatious.match_data import MatchData as PadatiousIntent class PadatiousMatcher: """Matcher class to avoid redundancy in padatious intent matching.""" - def __init__(self, service): + @deprecated("PadatiousMatcher class is deprecated!", "2.0.0") + def __init__(self, service: 'PadatiousPipeline'): self.service = service def _match_level(self, utterances, limit, lang=None, message: Optional[Message] = None) -> Optional[IntentMatch]: @@ -46,16 +50,8 @@ def _match_level(self, utterances, limit, lang=None, message: Optional[Message] with optional normalized version. limit (float): required confidence level. """ - LOG.debug(f'Padatious Matching confidence > {limit}') - # call flatten in case someone is sending the old style list of tuples - utterances = flatten_list(utterances) - lang = standardize_lang_tag(lang or self.service.lang) - padatious_intent = self.service.calc_intent(utterances, lang, message) - if padatious_intent is not None and padatious_intent.conf > limit: - skill_id = padatious_intent.name.split(':')[0] - return IntentMatch( - 'Padatious', padatious_intent.name, - padatious_intent.matches, skill_id, padatious_intent.sent) + m: IntentHandlerMatch = self.service._match_level(utterances, limit, lang, message) + return IntentMatch("Padatious", m.match_type, m.match_data, m.skill_id, m.utterance) def match_high(self, utterances, lang=None, message=None) -> Optional[IntentMatch]: """Intent matcher for high confidence. @@ -85,13 +81,13 @@ def match_low(self, utterances, lang=None, message=None) -> Optional[IntentMatch return self._match_level(utterances, self.service.conf_low, lang, message) -class PadatiousPipeline(PipelinePlugin): +class PadatiousPipeline(ConfidenceMatcherPipeline): """Service class for padatious intent matching.""" - def __init__(self, bus, config): - super().__init__(config) - self.padatious_config = config - self.bus = bus + def __init__(self, bus: Optional[Union[MessageBusClient, FakeBus]] = None, + config: Optional[Dict] = None): + + super().__init__(bus, config) core_config = Configuration() self.lang = standardize_lang_tag(core_config.get("lang", "en-US")) @@ -100,11 +96,11 @@ def __init__(self, bus, config): if self.lang not in langs: langs.append(self.lang) - self.conf_high = self.padatious_config.get("conf_high") or 0.95 - self.conf_med = self.padatious_config.get("conf_med") or 0.8 - self.conf_low = self.padatious_config.get("conf_low") or 0.5 + self.conf_high = self.config.get("conf_high") or 0.95 + self.conf_med = self.config.get("conf_med") or 0.8 + self.conf_low = self.config.get("conf_low") or 0.5 - intent_cache = expanduser(self.padatious_config.get('intent_cache') or + intent_cache = expanduser(self.config.get('intent_cache') or f"{xdg_data_home()}/{get_xdg_base()}/intent_cache") self.containers = {lang: PadatiousIntentContainer(f"{intent_cache}/{lang}") for lang in langs} @@ -123,6 +119,65 @@ def __init__(self, bus, config): self.max_words = 50 # if an utterance contains more words than this, don't attempt to match LOG.debug('Loaded Padatious intent parser.') + @property + def padatious_config(self) -> Dict: + log_deprecation("self.padatious_config is deprecated, access self.config directly instead", "2.0.0") + return self.config + + @padatious_config.setter + def padatious_config(self, val): + log_deprecation("self.padatious_config is deprecated, access self.config directly instead", "2.0.0") + self.config = val + + def _match_level(self, utterances, limit, lang=None, message: Optional[Message] = None) -> Optional[ + IntentHandlerMatch]: + """Match intent and make sure a certain level of confidence is reached. + + Args: + utterances (list of tuples): Utterances to parse, originals paired + with optional normalized version. + limit (float): required confidence level. + """ + LOG.debug(f'Padatious Matching confidence > {limit}') + # call flatten in case someone is sending the old style list of tuples + utterances = flatten_list(utterances) + lang = standardize_lang_tag(lang or self.lang) + padatious_intent = self.calc_intent(utterances, lang, message) + if padatious_intent is not None and padatious_intent.conf > limit: + skill_id = padatious_intent.name.split(':')[0] + return IntentHandlerMatch( + match_type=padatious_intent.name, + match_data=padatious_intent.matches, + skill_id=skill_id, + utterance=padatious_intent.sent) + + def match_high(self, utterances: List[str], lang: str, message: Message) -> Optional[IntentHandlerMatch]: + """Intent matcher for high confidence. + + Args: + utterances (list of tuples): Utterances to parse, originals paired + with optional normalized version. + """ + return self._match_level(utterances, self.conf_high, lang, message) + + def match_medium(self, utterances: List[str], lang: str, message: Message) -> Optional[IntentHandlerMatch]: + """Intent matcher for medium confidence. + + Args: + utterances (list of tuples): Utterances to parse, originals paired + with optional normalized version. + """ + return self._match_level(utterances, self.conf_med, lang, message) + + def match_low(self, utterances: List[str], lang: str, message: Message) -> Optional[IntentHandlerMatch]: + """Intent matcher for low confidence. + + Args: + utterances (list of tuples): Utterances to parse, originals paired + with optional normalized version. + """ + return self._match_level(utterances, self.conf_low, lang, message) + def train(self, message=None): """Perform padatious training. @@ -130,7 +185,7 @@ def train(self, message=None): message (Message): optional triggering message """ self.finished_training_event.clear() - padatious_single_thread = self.padatious_config.get('single_thread', False) + padatious_single_thread = self.config.get('single_thread', False) if message is None: single_thread = padatious_single_thread else: @@ -232,7 +287,7 @@ def register_entity(self, message): self._register_object(message, 'entity', self.containers[lang].add_entity) - def calc_intent(self, utterances: List[str], lang: str = None, + def calc_intent(self, utterances: Union[str, List[str]], lang: Optional[str] = None, message: Optional[Message] = None) -> Optional[PadatiousIntent]: """ Get the best intent match for the given list of utterances. Utilizes a @@ -280,10 +335,47 @@ def _get_closest_lang(self, lang: str) -> Optional[str]: def shutdown(self): self.bus.remove('padatious:register_intent', self.register_intent) self.bus.remove('padatious:register_entity', self.register_entity) + self.bus.remove('intent.service.padatious.get', self.handle_get_padatious) + self.bus.remove('intent.service.padatious.manifest.get', self.handle_padatious_manifest) + self.bus.remove('intent.service.padatious.entities.manifest.get', self.handle_entity_manifest) self.bus.remove('detach_intent', self.handle_detach_intent) self.bus.remove('detach_skill', self.handle_detach_skill) self.bus.remove('mycroft.skills.initialized', self.train) + def handle_get_padatious(self, message): + """messagebus handler for perfoming padatious parsing. + + Args: + message (Message): message triggering the method + """ + utterance = message.data["utterance"] + lang = message.data.get("lang", self.lang) + intent = self.calc_intent(utterance, lang=lang) + if intent: + intent = intent.__dict__ + self.bus.emit(message.reply("intent.service.padatious.reply", + {"intent": intent})) + + def handle_padatious_manifest(self, message): + """Messagebus handler returning the registered padatious intents. + + Args: + message (Message): message triggering the method + """ + self.bus.emit(message.reply( + "intent.service.padatious.manifest", + {"intents": self.registered_intents})) + + def handle_entity_manifest(self, message): + """Messagebus handler returning the registered padatious entities. + + Args: + message (Message): message triggering the method + """ + self.bus.emit(message.reply( + "intent.service.padatious.entities.manifest", + {"entities": self.registered_entities})) + @lru_cache(maxsize=3) # repeat calls under different conf levels wont re-run code def _calc_padatious_intent(utt: str, diff --git a/requirements.txt b/requirements.txt index ae04446..1cde00f 100755 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ fann2>=1.0.7, < 1.1.0 xxhash -ovos-plugin-manager>=0.0.26 +ovos-plugin-manager>=0.5.0,<1.0.0 ovos-workshop>=0.1.7,<2.0.0 ovos-utils>=0.3.4,<1.0.0 langcodes \ No newline at end of file