Skip to content

Commit

Permalink
feat!:pipeline factory (#15)
Browse files Browse the repository at this point in the history 
* fix:standardize_lang

* fix:standardize_lang

* feat:pipeline plugin factory

* feat:pipeline plugin factory

* feat:pipeline plugin factory

* feat:pipeline plugin factory

* feat:pipeline plugin factory

* feat:pipeline plugin factory

* Update ovos_padatious/opm.py

Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>

* feat:pipeline plugin factory

* feat:pipeline plugin factory

---------

Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>
  • Loading branch information
@JarbasAl @coderabbitai
JarbasAl and coderabbitai[bot] authored Oct 16, 2024
1 parent c751fe6 commit 227aaf5
Show file tree
Hide file tree
Showing 2 changed files with 121 additions and 29 deletions.
148 changes: 120 additions & 28 deletions ovos_padatious/opm.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,26 +16,30 @@
from functools import lru_cache
from os.path import expanduser, isfile
from threading import Event
from typing import List, Optional
from typing import Optional, Dict, List, Union

from langcodes import closest_match
from ovos_bus_client.client import MessageBusClient
from ovos_bus_client.message import Message
from ovos_bus_client.session import SessionManager, Session
from ovos_config.config import Configuration
from ovos_config.meta import get_xdg_base
from ovos_padatious import IntentContainer as PadatiousIntentContainer
from ovos_padatious.match_data import MatchData as PadatiousIntent
from ovos_plugin_manager.templates.pipeline import ConfidenceMatcherPipeline, IntentHandlerMatch, IntentMatch
from ovos_utils import flatten_list
from ovos_utils.fakebus import FakeBus
from ovos_utils.lang import standardize_lang_tag
from ovos_utils.log import LOG
from ovos_utils.log import LOG, deprecated, log_deprecation
from ovos_utils.xdg_utils import xdg_data_home
from ovos_plugin_manager.templates.pipeline import PipelinePlugin, IntentMatch
from langcodes import closest_match

from ovos_padatious import IntentContainer as PadatiousIntentContainer
from ovos_padatious.match_data import MatchData as PadatiousIntent


class PadatiousMatcher:
"""Matcher class to avoid redundancy in padatious intent matching."""

def __init__(self, service):
@deprecated("PadatiousMatcher class is deprecated!", "2.0.0")
def __init__(self, service: 'PadatiousPipeline'):
self.service = service

def _match_level(self, utterances, limit, lang=None, message: Optional[Message] = None) -> Optional[IntentMatch]:
Expand All @@ -46,16 +50,8 @@ def _match_level(self, utterances, limit, lang=None, message: Optional[Message]
with optional normalized version.
limit (float): required confidence level.
"""
LOG.debug(f'Padatious Matching confidence > {limit}')
# call flatten in case someone is sending the old style list of tuples
utterances = flatten_list(utterances)
lang = standardize_lang_tag(lang or self.service.lang)
padatious_intent = self.service.calc_intent(utterances, lang, message)
if padatious_intent is not None and padatious_intent.conf > limit:
skill_id = padatious_intent.name.split(':')[0]
return IntentMatch(
'Padatious', padatious_intent.name,
padatious_intent.matches, skill_id, padatious_intent.sent)
m: IntentHandlerMatch = self.service._match_level(utterances, limit, lang, message)
return IntentMatch("Padatious", m.match_type, m.match_data, m.skill_id, m.utterance)

def match_high(self, utterances, lang=None, message=None) -> Optional[IntentMatch]:
"""Intent matcher for high confidence.
Expand Down Expand Up @@ -85,13 +81,13 @@ def match_low(self, utterances, lang=None, message=None) -> Optional[IntentMatch
return self._match_level(utterances, self.service.conf_low, lang, message)


class PadatiousPipeline(PipelinePlugin):
class PadatiousPipeline(ConfidenceMatcherPipeline):
"""Service class for padatious intent matching."""

def __init__(self, bus, config):
super().__init__(config)
self.padatious_config = config
self.bus = bus
def __init__(self, bus: Optional[Union[MessageBusClient, FakeBus]] = None,
config: Optional[Dict] = None):

super().__init__(bus, config)

core_config = Configuration()
self.lang = standardize_lang_tag(core_config.get("lang", "en-US"))
Expand All @@ -100,11 +96,11 @@ def __init__(self, bus, config):
if self.lang not in langs:
langs.append(self.lang)

self.conf_high = self.padatious_config.get("conf_high") or 0.95
self.conf_med = self.padatious_config.get("conf_med") or 0.8
self.conf_low = self.padatious_config.get("conf_low") or 0.5
self.conf_high = self.config.get("conf_high") or 0.95
self.conf_med = self.config.get("conf_med") or 0.8
self.conf_low = self.config.get("conf_low") or 0.5

intent_cache = expanduser(self.padatious_config.get('intent_cache') or
intent_cache = expanduser(self.config.get('intent_cache') or
f"{xdg_data_home()}/{get_xdg_base()}/intent_cache")
self.containers = {lang: PadatiousIntentContainer(f"{intent_cache}/{lang}")
for lang in langs}
Expand All @@ -123,14 +119,73 @@ def __init__(self, bus, config):
self.max_words = 50 # if an utterance contains more words than this, don't attempt to match
LOG.debug('Loaded Padatious intent parser.')

@property
def padatious_config(self) -> Dict:
log_deprecation("self.padatious_config is deprecated, access self.config directly instead", "2.0.0")
return self.config

@padatious_config.setter
def padatious_config(self, val):
log_deprecation("self.padatious_config is deprecated, access self.config directly instead", "2.0.0")
self.config = val

def _match_level(self, utterances, limit, lang=None, message: Optional[Message] = None) -> Optional[
IntentHandlerMatch]:
"""Match intent and make sure a certain level of confidence is reached.
Args:
utterances (list of tuples): Utterances to parse, originals paired
with optional normalized version.
limit (float): required confidence level.
"""
LOG.debug(f'Padatious Matching confidence > {limit}')
# call flatten in case someone is sending the old style list of tuples
utterances = flatten_list(utterances)
lang = standardize_lang_tag(lang or self.lang)
padatious_intent = self.calc_intent(utterances, lang, message)
if padatious_intent is not None and padatious_intent.conf > limit:
skill_id = padatious_intent.name.split(':')[0]
return IntentHandlerMatch(
match_type=padatious_intent.name,
match_data=padatious_intent.matches,
skill_id=skill_id,
utterance=padatious_intent.sent)

def match_high(self, utterances: List[str], lang: str, message: Message) -> Optional[IntentHandlerMatch]:
"""Intent matcher for high confidence.
Args:
utterances (list of tuples): Utterances to parse, originals paired
with optional normalized version.
"""
return self._match_level(utterances, self.conf_high, lang, message)

def match_medium(self, utterances: List[str], lang: str, message: Message) -> Optional[IntentHandlerMatch]:
"""Intent matcher for medium confidence.
Args:
utterances (list of tuples): Utterances to parse, originals paired
with optional normalized version.
"""
return self._match_level(utterances, self.conf_med, lang, message)

def match_low(self, utterances: List[str], lang: str, message: Message) -> Optional[IntentHandlerMatch]:
"""Intent matcher for low confidence.
Args:
utterances (list of tuples): Utterances to parse, originals paired
with optional normalized version.
"""
return self._match_level(utterances, self.conf_low, lang, message)

def train(self, message=None):
"""Perform padatious training.
Args:
message (Message): optional triggering message
"""
self.finished_training_event.clear()
padatious_single_thread = self.padatious_config.get('single_thread', False)
padatious_single_thread = self.config.get('single_thread', False)
if message is None:
single_thread = padatious_single_thread
else:
Expand Down Expand Up @@ -232,7 +287,7 @@ def register_entity(self, message):
self._register_object(message, 'entity',
self.containers[lang].add_entity)

def calc_intent(self, utterances: List[str], lang: str = None,
def calc_intent(self, utterances: Union[str, List[str]], lang: Optional[str] = None,
message: Optional[Message] = None) -> Optional[PadatiousIntent]:
"""
Get the best intent match for the given list of utterances. Utilizes a
Expand Down Expand Up @@ -280,10 +335,47 @@ def _get_closest_lang(self, lang: str) -> Optional[str]:
def shutdown(self):
self.bus.remove('padatious:register_intent', self.register_intent)
self.bus.remove('padatious:register_entity', self.register_entity)
self.bus.remove('intent.service.padatious.get', self.handle_get_padatious)
self.bus.remove('intent.service.padatious.manifest.get', self.handle_padatious_manifest)
self.bus.remove('intent.service.padatious.entities.manifest.get', self.handle_entity_manifest)
self.bus.remove('detach_intent', self.handle_detach_intent)
self.bus.remove('detach_skill', self.handle_detach_skill)
self.bus.remove('mycroft.skills.initialized', self.train)

def handle_get_padatious(self, message):
"""messagebus handler for perfoming padatious parsing.
Args:
message (Message): message triggering the method
"""
utterance = message.data["utterance"]
lang = message.data.get("lang", self.lang)
intent = self.calc_intent(utterance, lang=lang)
if intent:
intent = intent.__dict__
self.bus.emit(message.reply("intent.service.padatious.reply",
{"intent": intent}))

def handle_padatious_manifest(self, message):
"""Messagebus handler returning the registered padatious intents.
Args:
message (Message): message triggering the method
"""
self.bus.emit(message.reply(
"intent.service.padatious.manifest",
{"intents": self.registered_intents}))

def handle_entity_manifest(self, message):
"""Messagebus handler returning the registered padatious entities.
Args:
message (Message): message triggering the method
"""
self.bus.emit(message.reply(
"intent.service.padatious.entities.manifest",
{"entities": self.registered_entities}))


@lru_cache(maxsize=3) # repeat calls under different conf levels wont re-run code
def _calc_padatious_intent(utt: str,
Expand Down
2 changes: 1 addition & 1 deletion requirements.txt
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
fann2>=1.0.7, < 1.1.0
xxhash
ovos-plugin-manager>=0.0.26
ovos-plugin-manager>=0.5.0,<1.0.0
ovos-workshop>=0.1.7,<2.0.0
ovos-utils>=0.3.4,<1.0.0
langcodes

0 comments on commit 227aaf5

Please sign in to comment.