From c9f352e1668e711ef4c531479b83d49a16dc8960 Mon Sep 17 00:00:00 2001 From: JarbasAi Date: Sat, 30 Sep 2023 17:42:32 +0100 Subject: [PATCH] unittests/fallbackV1 unittests for compat with FallbackSkillV1 requires OpenVoiceOS/OVOS-workshop/pull/135 --- .github/workflows/coverage.yml | 1 + .github/workflows/unit_tests.yml | 1 + requirements/requirements.txt | 2 +- .../skill-ovos-fallback-unknownv1/__init__.py | 12 ++ .../locale/en-us/unknown.dialog | 9 + .../skill-ovos-fallback-unknownv1/setup.py | 39 ++++ test/end2end/session/test_fallback_v1.py | 168 ++++++++++++++++++ 7 files changed, 231 insertions(+), 1 deletion(-) create mode 100644 test/end2end/session/skill-ovos-fallback-unknownv1/__init__.py create mode 100755 test/end2end/session/skill-ovos-fallback-unknownv1/locale/en-us/unknown.dialog create mode 100755 test/end2end/session/skill-ovos-fallback-unknownv1/setup.py create mode 100644 test/end2end/session/test_fallback_v1.py diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index ff2fcf8316a9..d254e04e5572 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -30,6 +30,7 @@ jobs: pip install ./test/unittests/common_query/ovos_tskill_fakewiki pip install ./test/end2end/session/skill-ovos-hello-world pip install ./test/end2end/session/skill-ovos-fallback-unknown + pip install ./test/end2end/session/skill-ovos-fallback-unknownv1 - name: Generate coverage report run: | pytest --cov=ovos_core --cov-report xml test/unittests diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 53e2335ecc85..55dce5ac7240 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -57,6 +57,7 @@ jobs: pip install ./test/unittests/common_query/ovos_tskill_fakewiki pip install ./test/end2end/session/skill-ovos-hello-world pip install ./test/end2end/session/skill-ovos-fallback-unknown + pip install ./test/end2end/session/skill-ovos-fallback-unknownv1 - name: Run unittests run: | pytest --cov=ovos_core --cov-report xml test/unittests diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 3545ba19a179..84b3f5e6db36 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -12,7 +12,7 @@ ovos-plugin-manager<0.1.0, >=0.0.24a9 ovos-config~=0.0,>=0.0.11a13 ovos-lingua-franca>=0.4.7 ovos-backend-client>=0.1.0a12 -ovos-workshop<0.1.0, >=0.0.13a6 +ovos-workshop<0.1.0, >=0.0.13a7 # provides plugins and classic machine learning framework ovos-classifiers<0.1.0, >=0.0.0a37 diff --git a/test/end2end/session/skill-ovos-fallback-unknownv1/__init__.py b/test/end2end/session/skill-ovos-fallback-unknownv1/__init__.py new file mode 100644 index 000000000000..0b9382fe4186 --- /dev/null +++ b/test/end2end/session/skill-ovos-fallback-unknownv1/__init__.py @@ -0,0 +1,12 @@ +from ovos_workshop.skills.fallback import FallbackSkillV1 +from ovos_workshop.decorators import fallback_handler + + +# explicitly use class with compat for older cores +# this is usually auto detected, just done here for unittests +class UnknownSkill(FallbackSkillV1): + + @fallback_handler(priority=100) + def handle_fallback(self, message): + self.speak_dialog('unknown') + return True diff --git a/test/end2end/session/skill-ovos-fallback-unknownv1/locale/en-us/unknown.dialog b/test/end2end/session/skill-ovos-fallback-unknownv1/locale/en-us/unknown.dialog new file mode 100755 index 000000000000..3f31d7bbc9aa --- /dev/null +++ b/test/end2end/session/skill-ovos-fallback-unknownv1/locale/en-us/unknown.dialog @@ -0,0 +1,9 @@ +I'm sorry, I don't understand. +I don't know what that means. +I don't understand, but I'm learning new things everyday. +Sorry, I didn't catch that. +Sorry, I don't understand. +I don't understand. +I'm not sure I understood you. +You might have to say that a different way. +Please rephrase your request. diff --git a/test/end2end/session/skill-ovos-fallback-unknownv1/setup.py b/test/end2end/session/skill-ovos-fallback-unknownv1/setup.py new file mode 100755 index 000000000000..a6505f19bd30 --- /dev/null +++ b/test/end2end/session/skill-ovos-fallback-unknownv1/setup.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 +from setuptools import setup +from os import walk, path + +URL = "https://github.com/OpenVoiceOS/skill-ovos-fallback-unknownv1" +SKILL_CLAZZ = "UnknownSkill" # needs to match __init__.py class name +PYPI_NAME = "ovos-skill-fallback-unknown-v1" # pip install PYPI_NAME + +# below derived from github url to ensure standard skill_id +SKILL_AUTHOR, SKILL_NAME = URL.split(".com/")[-1].split("/") +SKILL_PKG = SKILL_NAME.lower().replace('-', '_') +PLUGIN_ENTRY_POINT = f'{SKILL_NAME.lower()}.{SKILL_AUTHOR.lower()}={SKILL_PKG}:{SKILL_CLAZZ}' +# skill_id=package_name:SkillClass + + +def find_resource_files(): + resource_base_dirs = ("locale", "ui", "vocab", "dialog", "regex", "skill") + base_dir = path.dirname(__file__) + package_data = ["*.json"] + for res in resource_base_dirs: + if path.isdir(path.join(base_dir, res)): + for (directory, _, files) in walk(path.join(base_dir, res)): + if files: + package_data.append( + path.join(directory.replace(base_dir, "").lstrip('/'), + '*')) + return package_data + + +setup( + name=PYPI_NAME, + version="0.0.0", + package_dir={SKILL_PKG: ""}, + package_data={SKILL_PKG: find_resource_files()}, + packages=[SKILL_PKG], + include_package_data=True, + keywords='ovos skill plugin', + entry_points={'ovos.plugin.skill': PLUGIN_ENTRY_POINT} +) diff --git a/test/end2end/session/test_fallback_v1.py b/test/end2end/session/test_fallback_v1.py new file mode 100644 index 000000000000..ea0491a54ba5 --- /dev/null +++ b/test/end2end/session/test_fallback_v1.py @@ -0,0 +1,168 @@ +from time import sleep +from unittest import TestCase + +from ovos_bus_client.message import Message +from ovos_bus_client.session import SessionManager, Session +from .minicroft import get_minicroft + + +class TestFallback(TestCase): + + def setUp(self): + self.skill_id = "skill-ovos-fallback-unknownv1.openvoiceos" + self.core = get_minicroft(self.skill_id) + + def test_fallback_v1(self): + SessionManager.sessions = {} + SessionManager.default_session = SessionManager.sessions["default"] = Session("default") + SessionManager.default_session.lang = "en-us" + messages = [] + + def new_msg(msg): + nonlocal messages + m = Message.deserialize(msg) + if m.msg_type in ["ovos.skills.settings_changed"]: + return # skip these, only happen in 1st run + messages.append(m) + print(len(messages), msg) + + def wait_for_n_messages(n): + nonlocal messages + while len(messages) < n: + sleep(0.1) + + self.core.bus.on("message", new_msg) + + utt = Message("recognizer_loop:utterance", + {"utterances": ["invalid"]}, + {"session": SessionManager.default_session.serialize(), # explicit default sess + "x": "xx"}) + self.core.bus.emit(utt) + + # confirm all expected messages are sent + expected_messages = [ + "recognizer_loop:utterance", + # Converse + "skill.converse.ping", + "skill.converse.pong", + # FallbackV1 - high prio + "mycroft.skills.fallback", + "mycroft.skill.handler.start", + "mycroft.skill.handler.complete", + "mycroft.skills.fallback.response", + # FallbackV1 - medium prio + "mycroft.skills.fallback", + "mycroft.skill.handler.start", + "mycroft.skill.handler.complete", + "mycroft.skills.fallback.response", + # FallbackV1 - low prio -> skill selected + "mycroft.skills.fallback", + "mycroft.skill.handler.start", + "enclosure.active_skill", + "speak", + "intent.service.skills.activate", + "intent.service.skills.activated", + f"{self.skill_id}.activate", + # backwards compat activation for older cores + "active_skill_request", + "intent.service.skills.activated", + f"{self.skill_id}.activate", + # report handling + "mycroft.skill.handler.complete", + "mycroft.skills.fallback.response", + # update default sess + "ovos.session.update_default" + ] + wait_for_n_messages(len(expected_messages)) + + self.assertEqual(len(expected_messages), len(messages)) + + mtypes = [m.msg_type for m in messages] + for m in expected_messages: + self.assertTrue(m in mtypes) + + # verify that contexts are kept around + for m in messages: + self.assertEqual(m.context["session"]["session_id"], "default") + self.assertEqual(m.context["x"], "xx") + # verify active skills is empty until "intent.service.skills.activated" + for m in messages[:16]: + self.assertEqual(m.context["session"]["session_id"], "default") + self.assertEqual(m.context["session"]["active_skills"], []) + + # verify converse ping/pong answer from skill + self.assertEqual(messages[1].msg_type, "skill.converse.ping") + self.assertEqual(messages[2].msg_type, "skill.converse.pong") + self.assertEqual(messages[2].data["skill_id"], self.skill_id) + self.assertEqual(messages[2].context["skill_id"], self.skill_id) + self.assertFalse(messages[2].data["can_handle"]) + + # high prio fallback + self.assertEqual(messages[3].msg_type, "mycroft.skills.fallback") + self.assertEqual(messages[3].data["fallback_range"], [0, 5]) + self.assertEqual(messages[4].msg_type, "mycroft.skill.handler.start") + self.assertEqual(messages[4].data["handler"], "fallback") + self.assertEqual(messages[5].msg_type, "mycroft.skill.handler.complete") + self.assertEqual(messages[5].data["handler"], "fallback") + self.assertEqual(messages[6].msg_type, "mycroft.skills.fallback.response") + self.assertFalse(messages[6].data["handled"]) + + # medium prio fallback + self.assertEqual(messages[7].msg_type, "mycroft.skills.fallback") + self.assertEqual(messages[7].data["fallback_range"], [5, 90]) + self.assertEqual(messages[8].msg_type, "mycroft.skill.handler.start") + self.assertEqual(messages[8].data["handler"], "fallback") + self.assertEqual(messages[9].msg_type, "mycroft.skill.handler.complete") + self.assertEqual(messages[9].data["handler"], "fallback") + self.assertEqual(messages[10].msg_type, "mycroft.skills.fallback.response") + self.assertFalse(messages[10].data["handled"]) + + # low prio fallback + self.assertEqual(messages[11].msg_type, "mycroft.skills.fallback") + self.assertEqual(messages[11].data["fallback_range"], [90, 101]) + self.assertEqual(messages[12].msg_type, "mycroft.skill.handler.start") + self.assertEqual(messages[12].data["handler"], "fallback") + + # skill execution + self.assertEqual(messages[13].msg_type, "enclosure.active_skill") + self.assertEqual(messages[13].data["skill_id"], self.skill_id) + self.assertEqual(messages[14].msg_type, "speak") + self.assertEqual(messages[14].data["meta"]["dialog"], "unknown") + self.assertEqual(messages[14].data["meta"]["skill"], self.skill_id) + + # skill making itself active + self.assertEqual(messages[15].msg_type, "intent.service.skills.activate") + self.assertEqual(messages[15].data["skill_id"], self.skill_id) + self.assertEqual(messages[16].msg_type, "intent.service.skills.activated") + self.assertEqual(messages[16].data["skill_id"], self.skill_id) + self.assertEqual(messages[17].msg_type, f"{self.skill_id}.activate") + # skill making itself active again - backwards compat namespace + self.assertEqual(messages[18].msg_type, "active_skill_request") + self.assertEqual(messages[18].data["skill_id"], self.skill_id) + self.assertEqual(messages[19].msg_type, "intent.service.skills.activated") + self.assertEqual(messages[19].data["skill_id"], self.skill_id) + self.assertEqual(messages[20].msg_type, f"{self.skill_id}.activate") + + # fallback execution response + self.assertEqual(messages[21].msg_type, "mycroft.skill.handler.complete") + self.assertEqual(messages[21].data["handler"], "fallback") + self.assertEqual(messages[22].msg_type, "mycroft.skills.fallback.response") + self.assertTrue(messages[22].data["handled"]) + + # verify default session is now updated + self.assertEqual(messages[23].msg_type, "ovos.session.update_default") + self.assertEqual(messages[23].data["session_data"]["session_id"], "default") + + # test second message with no session resumes default active skills + messages = [] + utt = Message("recognizer_loop:utterance", + {"utterances": ["invalid"]}) + self.core.bus.emit(utt) + wait_for_n_messages(len(expected_messages)) + self.assertEqual(len(expected_messages), len(messages)) + + # verify that contexts are kept around + for m in messages[1:]: + self.assertEqual(m.context["session"]["session_id"], "default") + self.assertEqual(m.context["session"]["active_skills"][0][0], self.skill_id) +