diff --git a/.github/workflows/propose_release.yml b/.github/workflows/propose_release.yml index 81dfe43..306662c 100644 --- a/.github/workflows/propose_release.yml +++ b/.github/workflows/propose_release.yml @@ -16,6 +16,7 @@ jobs: branch: dev release_type: ${{ inputs.release_type }} update_changelog: True + version_file: "chatbot_core/version.py" pull_changes: uses: neongeckocom/.github/.github/workflows/pull_master.yml@master needs: update_version diff --git a/.github/workflows/publish_test_build.yml b/.github/workflows/publish_test_build.yml index 2721648..46582c7 100644 --- a/.github/workflows/publish_test_build.yml +++ b/.github/workflows/publish_test_build.yml @@ -13,5 +13,5 @@ jobs: uses: neongeckocom/.github/.github/workflows/publish_alpha_release.yml@master secrets: inherit with: - version_file: "version.py" + version_file: "chatbot_core/version.py" publish_prerelease: true diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 5e6ef21..010c14d 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -7,6 +7,33 @@ on: jobs: unit_tests: + strategy: + matrix: + python-version: [ 3.7, 3.8, 3.9, '3.10', '3.11'] + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v2 + - name: Set up python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install .[lang] -r requirements/test_requirements.txt + env: + GITHUB_TOKEN: ${{secrets.neon_token}} + - name: Test with pytest + run: | + pip install pytest pytest-timeout pytest-cov + pytest tests/units --doctest-modules --junitxml=tests/unit-test-results.xml + - name: Upload pytest unit test results + uses: actions/upload-artifact@v2 + with: + name: unit-test-results + path: tests/unit-test-results.xml + integration_tests: strategy: matrix: python-version: [ 3.7, 3.8, 3.9 ] @@ -23,16 +50,15 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -r requirements/test_requirements.txt - pip install . + pip install . -r requirements/test_requirements.txt env: GITHUB_TOKEN: ${{secrets.neon_token}} - name: Test with pytest run: | pip install pytest pytest-timeout pytest-cov - pytest tests/chatbot_core_tests.py --doctest-modules --junitxml=tests/test-results.xml - - name: Upload pytest test results + pytest tests/integration --doctest-modules --junitxml=tests/integration-test-results.xml + - name: Upload integration test results uses: actions/upload-artifact@v2 with: - name: test-results - path: tests/test-results.xml \ No newline at end of file + name: integration-test-results + path: tests/integration-test-results.xml \ No newline at end of file diff --git a/chatbot_core/__init__.py b/chatbot_core/__init__.py index 11a306e..5057561 100644 --- a/chatbot_core/__init__.py +++ b/chatbot_core/__init__.py @@ -16,9 +16,51 @@ # Specialized conversational reconveyance options from Conversation Processing Intelligence Corp. # US Patents 2008-2021: US7424516, US20140161250, US20140177813, US8638908, US8068604, US8553852, US10530923, US10530924 # China Patent: CN102017585 - Europe Patent: EU2156652 - Patents Pending -from chatbot_core.utils import generate_random_response, clean_up_bot,\ - grammar_check, ConversationControls, ConversationState -from chatbot_core.utils.version_utils import get_class -from chatbot_core.version_mapping import CHATBOT_VERSIONS -ChatBot = get_class(CHATBOT_VERSIONS) +from ovos_utils.log import LOG, log_deprecation +from os import environ +from neon_utils.decorators import module_property + + +@module_property +def _ChatBot(): + LOG.debug(f"Getting class for {environ.get('CHATBOT_VERSION')}") + from chatbot_core.utils.version_utils import get_class + return get_class() + + +@module_property +def _ConversationControls(): + log_deprecation("import from `chatbot_core.utils.enum` directly", + "3.0.0") + from chatbot_core.utils.enum import ConversationControls + return ConversationControls + + +@module_property +def _ConversationState(): + log_deprecation("import from `chatbot_core.utils.enum` directly", + "3.0.0") + from chatbot_core.utils.enum import ConversationState + return ConversationState + + +def generate_random_response(*args, **kwargs): + log_deprecation("import from `chatbot_core.utils.bot_utils` directly", + "3.0.0") + from chatbot_core.utils.bot_utils import generate_random_response + return generate_random_response(*args, **kwargs) + + +def clean_up_bot(*args, **kwargs): + log_deprecation("import from `chatbot_core.utils.bot_utils` directly", + "3.0.0") + from chatbot_core.utils.bot_utils import clean_up_bot + return clean_up_bot(*args, **kwargs) + + +def grammar_check(*args, **kwargs): + log_deprecation("import from `chatbot_core.utils.bot_utils` directly", + "3.0.0") + from chatbot_core.utils.bot_utils import grammar_check + return grammar_check(*args, **kwargs) diff --git a/chatbot_core/chatbot_abc.py b/chatbot_core/chatbot_abc.py index f0212d0..eda2a57 100644 --- a/chatbot_core/chatbot_abc.py +++ b/chatbot_core/chatbot_abc.py @@ -18,14 +18,14 @@ # China Patent: CN102017585 - Europe Patent: EU2156652 - Patents Pending import random +import time from abc import ABC, abstractmethod from queue import Queue from typing import Optional -from chatbot_core.utils import * - -LOG = make_logger("chatbot") +from neon_utils.log_utils import init_log +from ovos_utils.log import LOG class ChatBotABC(ABC): @@ -38,10 +38,7 @@ def __init__(self): @property def log(self): if not self.__log: - global LOG - self.__log = make_logger(self.__class__.__name__) - self.__log.setLevel(LOG.level) - LOG = self.__log + self.__log = init_log(log_name=self.__class__.__name__) return self.__log @abstractmethod diff --git a/chatbot_core/cli.py b/chatbot_core/cli.py new file mode 100644 index 0000000..8caa9bb --- /dev/null +++ b/chatbot_core/cli.py @@ -0,0 +1,199 @@ +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System +# +# Copyright 2008-2021 Neongecko.com Inc. | All Rights Reserved +# +# Notice of License - Duplicating this Notice of License near the start of any file containing +# a derivative of this software is a condition of license for this software. +# Friendly Licensing: +# No charge, open source royalty free use of the Neon AI software source and object is offered for +# educational users, noncommercial enthusiasts, Public Benefit Corporations (and LLCs) and +# Social Purpose Corporations (and LLCs). Developers can contact developers@neon.ai +# For commercial licensing, distribution of derivative works or redistribution please contact licenses@neon.ai +# Distributed on an "AS IS” basis without warranties or conditions of any kind, either express or implied. +# Trademarks of Neongecko: Neon AI(TM), Neon Assist (TM), Neon Communicator(TM), Klat(TM) +# Authors: Guy Daniels, Daniel McKnight, Regina Bloomstine, Elon Gasper, Richard Leeds +# +# Specialized conversational reconveyance options from Conversation Processing Intelligence Corp. +# US Patents 2008-2021: US7424516, US20140161250, US20140177813, US8638908, US8068604, US8553852, US10530923, US10530924 +# China Patent: CN102017585 - Europe Patent: EU2156652 - Patents Pending + +import os +import argparse +import click + +from os.path import expanduser, relpath +from ovos_utils.log import LOG, log_deprecation + + +@click.group("chatbots", cls=click.Group, + no_args_is_help=True, invoke_without_command=True, + help="Chatbot Core Commands\n\n" + "See also: chatbots COMMAND --help") +@click.option("--version", "-v", is_flag=True, required=False, + help="Print the current version") +def chatbot_core_cli(version: bool = False): + if version: + from chatbot_core.version import __version__ + click.echo(f"chatbot-core version {__version__}") + + +@chatbot_core_cli.command(help="Start an MQ chatbot") +@click.argument("bot_entrypoint") +def start_mq_bot(bot_entrypoint): + os.environ['CHATBOT_VERSION'] = 'v2' + from chatbot_core.utils.bot_utils import run_mq_bot + run_mq_bot(bot_entrypoint) + + +@chatbot_core_cli.command(help="Start a local debug session") +@click.option("--bot-dir", default=None, help="Path to legacy chatbots directory") +def debug_bots(bot_dir): + from chatbot_core.utils.bot_utils import debug_bots + bot_dir = expanduser(relpath(bot_dir)) if bot_dir else None + debug_bots(bot_dir) + + +# Below are deprecated entrypoints +def cli_start_mq_bot(): + """ + Entrypoint to start an MQ chatbot + """ + os.environ['CHATBOT_VERSION'] = 'v2' + from chatbot_core.utils.bot_utils import run_mq_bot + log_deprecation("Use `chatbots start-mq-bot`", "3.0.0") + parser = argparse.ArgumentParser(description="Start a chatbot") + parser.add_argument("--bot", dest="bot_name", + help="Chatbot entrypoint name", type=str) + args = parser.parse_args() + run_mq_bot(args.bot_name) + + +def cli_start_bots(): + """ + Entry Point to start bots from a Console Script + """ + from chatbot_core.utils.bot_utils import SERVER, start_bots + log_deprecation("This CLI command is deprecated", "3.0.0") + + parser = argparse.ArgumentParser(description="Start some chatbots") + parser.add_argument("--domain", dest="domain", default="chatbotsforum.org", + help="Domain to connect to " + "(default: chatbotsforum.org)", type=str) + parser.add_argument("--dir", dest="bot_dir", + help="Path to chatbots (default: ./)", type=str) + parser.add_argument("--bot", dest="bot_name", + help="Optional bot name to run a single bot only", + type=str) + parser.add_argument("--credentials", dest="cred_file", + help="Optional path to YAML credentials", type=str) + parser.add_argument("--username", dest="username", + help="Klat username for a single bot", type=str) + parser.add_argument("--password", dest="password", + help="Klat password for a single bot", type=str) + parser.add_argument("--server", dest="server", default=SERVER, + help=f"Klat server (default: {SERVER})", type=str) + parser.add_argument("--debug", dest="debug", action='store_true', + help="Enable more verbose log output") + parser.add_argument("--bot-names", dest="bot-names", + help="comma separated list of bots to run", type=str) + parser.add_argument("--exclude", dest="exclude", + help="comma separated list of bots to exclude " + "from running", type=str) + parser.add_argument("--handle-restart", dest="handle_restart", + default=False, + help="True to handle server emit to restart bots", + type=bool) + args = parser.parse_args() + + if args.debug: + LOG.set_level("DEBUG") + + if args.exclude: + excluded_bots = [name.strip() for name in args.exclude.split(",")] + else: + excluded_bots = None + LOG.debug(args) + start_bots(args.domain, args.bot_dir, args.username, args.password, + args.server, args.cred_file, args.bot_name, + excluded_bots, args.handle_restart) + + +def cli_stop_bots(): + """ + Stops all start-klat-bot instances + """ + from time import sleep + log_deprecation("This CLI command is deprecated", "3.0.0") + + parser = argparse.ArgumentParser(description="Stop some chatbots") + parser.add_argument("--server", dest="server", default="", + help=f"Klat server (default: None)", type=str) + args = parser.parse_args() + if args.server: + server_to_stop = args.server + else: + server_to_stop = None + import psutil + + procs = {p.pid: p.info for p in psutil.process_iter(['name'])} + for pid, name in procs.items(): + if name.get("name") == "start-klat-bots" and \ + (not server_to_stop or f"--server={server_to_stop}" in + psutil.Process(pid).cmdline()): + LOG.info(f"Terminating {pid}") + psutil.Process(pid).terminate() + sleep(1) + if psutil.pid_exists(pid) and psutil.Process(pid).is_running(): + LOG.error(f"Process {pid} not terminated!!") + psutil.Process(pid).kill() + + +def cli_start_prompter(): + """ + Entry Point to start a prompter bot + """ + log_deprecation("This CLI command is deprecated", "3.0.0") + + from chatbot_core.utils.bot_utils import SERVER, start_bots + parser = argparse.ArgumentParser(description="Start a prompter chatbot") + parser.add_argument("--bot", dest="bot_name", + help="Optional bot name to run a single bot only", type=str) + parser.add_argument("--dir", dest="bot_dir", + help="Path to chatbots (default: ./)", type=str) + parser.add_argument("--username", dest="username", + help="Klat username for a single bot", type=str) + parser.add_argument("--password", dest="password", + help="Klat password for a single bot", type=str) + parser.add_argument("--server", dest="server", default=SERVER, + help=f"Klat server (default: {SERVER})", type=str) + parser.add_argument("--debug", dest="debug", action='store_true', + help="Enable more verbose log output") + parser.add_argument("--handle-restart", dest="handle_restart", default=False, + help="True to handle server emit to restart bots", type=bool) + args = parser.parse_args() + + if args.debug: + LOG.level = "DEBUG" + LOG.debug(args) + start_bots("chatbotsforum.org", args.bot_dir, args.username, args.password, + args.server, None, args.bot_name, None, args.handle_restart, + True) + + +def cli_debug_bots(): + """ + Debug bots in the passed directory + :param bot_dir: directory containing the bot to test + """ + import sys + from chatbot_core.utils.bot_utils import debug_bots + + log_deprecation("Use `chatbots debug-bots`", "3.0.0") + + bot_dir = os.getcwd() + # Try handling passed directory + if len(sys.argv) > 1: + arg_dir = os.path.expanduser(sys.argv[1]) + bot_dir = arg_dir if os.path.exists(arg_dir) else bot_dir + + debug_bots(bot_dir) diff --git a/chatbot_core/neon.py b/chatbot_core/neon.py index 283edd6..73b6009 100644 --- a/chatbot_core/neon.py +++ b/chatbot_core/neon.py @@ -16,14 +16,16 @@ # Specialized conversational reconveyance options from Conversation Processing Intelligence Corp. # US Patents 2008-2021: US7424516, US20140161250, US20140177813, US8638908, US8068604, US8553852, US10530923, US10530924 # China Patent: CN102017585 - Europe Patent: EU2156652 - Patents Pending + import os import time -from typing import Optional -from mycroft_bus_client import Message, MessageBusClient -from neon_utils import LOG +from typing import Optional +from ovos_bus_client import Message, MessageBusClient +from ovos_utils.log import LOG -from chatbot_core.utils import BotTypes, init_message_bus +from chatbot_core.utils.enum import BotTypes +from chatbot_core.utils.bot_utils import init_message_bus from chatbot_core import ChatBot if os.environ.get('CHATBOT_VERSION', 'v1') == 'v2': diff --git a/chatbot_core/parlai.py b/chatbot_core/parlai.py index 08b3f49..104936a 100644 --- a/chatbot_core/parlai.py +++ b/chatbot_core/parlai.py @@ -16,10 +16,10 @@ # Specialized conversational reconveyance options from Conversation Processing Intelligence Corp. # US Patents 2008-2021: US7424516, US20140161250, US20140177813, US8638908, US8068604, US8553852, US10530923, US10530924 # China Patent: CN102017585 - Europe Patent: EU2156652 - Patents Pending + from abc import abstractmethod from threading import Event, Thread - -import spacy +from ovos_utils.log import LOG class ParlaiBot: @@ -35,6 +35,7 @@ def __init__(self, interactive_script, response_timeout=25, :param done_string: string that signals about episode done :param exit_string: string that signals about the finish """ + import spacy self.nlp_engine = spacy.load("en_core_web_sm") self.agent_id = 'local_agent' @@ -58,7 +59,7 @@ def observe(self, msg): if msg['id'] != 'context': self.event.set() self.current_response = msg["text"] - self.log.debug(f'[OUT]: {self.current_response}') + LOG.debug(f'[OUT]: {self.current_response}') def act(self): """ @@ -68,7 +69,7 @@ def act(self): # save the current shout locally and clear the attribute to prevent parley() without incoming shout reply_text = self.current_shout self.current_shout = '' - self.log.debug(f'CURRENT SHOUT {reply_text}') + LOG.debug(f'CURRENT SHOUT {reply_text}') # check for episode done if self.done_string in reply_text: raise StopIteration diff --git a/chatbot_core/utils/__init__.py b/chatbot_core/utils/__init__.py index a4d68cf..fac5763 100644 --- a/chatbot_core/utils/__init__.py +++ b/chatbot_core/utils/__init__.py @@ -17,10 +17,39 @@ # US Patents 2008-2021: US7424516, US20140161250, US20140177813, US8638908, US8068604, US8553852, US10530923, US10530924 # China Patent: CN102017585 - Europe Patent: EU2156652 - Patents Pending +from ovos_utils.log import log_deprecation -from chatbot_core.utils.cache import FIFOCache -from chatbot_core.utils.logger import make_logger -from chatbot_core.utils.enum import ConversationState, ConversationControls, BotTypes -from chatbot_core.utils.bot_utils import * -from chatbot_core.utils.string_utils import remove_prefix -from chatbot_core.utils.conversation_utils import create_conversation_cycle + +def create_conversation_cycle(): + from chatbot_core.utils.conversation_utils import create_conversation_cycle + log_deprecation("import from `chatbot_core.utils.conversation_utils`", + "2.3.1") + return create_conversation_cycle() + + +def find_closest_answer(*args, **kwargs): + from chatbot_core.utils.bot_utils import find_closest_answer + log_deprecation("import from `chatbot_core.utils.bot_utils`", + "2.3.1") + return find_closest_answer(*args, **kwargs) + + +def clean_up_bot(*args, **kwargs): + from chatbot_core.utils.bot_utils import clean_up_bot + log_deprecation("import from `chatbot_core.utils.bot_utils`", + "2.3.1") + return clean_up_bot(*args, **kwargs) + + +def get_bots_in_dir(*args, **kwargs): + from chatbot_core.utils.bot_utils import get_bots_in_dir + log_deprecation("import from `chatbot_core.utils.bot_utils`", + "2.3.1") + return get_bots_in_dir(*args, **kwargs) + + +def start_bots(*args, **kwargs): + from chatbot_core.utils.bot_utils import start_bots + log_deprecation("import from `chatbot_core.utils.bot_utils`", + "2.3.1") + return start_bots(*args, **kwargs) diff --git a/chatbot_core/utils/bot_utils.py b/chatbot_core/utils/bot_utils.py index c025abc..b45f726 100644 --- a/chatbot_core/utils/bot_utils.py +++ b/chatbot_core/utils/bot_utils.py @@ -20,44 +20,23 @@ import inspect import logging import os -import pkgutil -import socket import random -import jellyfish -# import psutil import time +import sys +import yaml -# from socketio import Client +from typing import Optional, Callable, Dict from multiprocessing import Process, Event, synchronize from threading import Thread, current_thread - -import pkg_resources -from mycroft_bus_client import Message, MessageBusClient - -from nltk.translate.bleu_score import sentence_bleu -from nltk import word_tokenize - -import sys - -# import chatbots.bots - +from ovos_bus_client import Message, MessageBusClient from datetime import datetime - -import yaml +from ovos_utils.xdg_utils import xdg_config_home from klat_connector import start_socket +from ovos_utils.log import LOG, log_deprecation +from neon_utils.net_utils import get_ip_address -from ovos_utils.log import LOG - -# Causes circular imports -# from chatbot_core import ChatBot -# from chatbot_core.v2 import ChatBot as ChatBotV2 - -def get_ip_address(): - s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - s.connect(("8.8.8.8", 80)) - ip_addr = s.getsockname()[0] - s.close() - return ip_addr +from chatbot_core.chatbot_abc import ChatBotABC +from chatbot_core.v2 import ChatBot as ChatBotV2 ip = get_ip_address() @@ -75,16 +54,25 @@ def get_ip_address(): runner = Event() -def _threaded_start_bot(bot, addr: str, port: int, domain: str, user: str, password: str, - event: synchronize.Event, is_prompter: bool): +def get_ip_address(): + log_deprecation('Import from `neon_utils.net_utils` directly', "2.3.0") + return get_ip_address() + + +def _threaded_start_bot(bot, addr: str, port: int, domain: str, user: str, + password: str, event: synchronize.Event, + is_prompter: bool): """ Helper function for _start_bot """ + # TODO: Deprecate if len(inspect.signature(bot).parameters) == 6: - instance = bot(start_socket(addr, port), domain, user, password, True, is_prompter) + instance = bot(start_socket(addr, port), domain, user, password, True, + is_prompter) elif len(inspect.signature(bot).parameters) == 5: if is_prompter: - LOG.error(f"v2 Bot found, prompter functionality will not be enabled! {bot}") + LOG.error(f"v2 Bot found, prompter functionality will " + f"not be enabled! {bot}") instance = bot(start_socket(addr, port), domain, user, password, True) else: LOG.error(f"Bot params unknown: {inspect.signature(bot).parameters}") @@ -99,7 +87,8 @@ def _threaded_start_bot(bot, addr: str, port: int, domain: str, user: str, passw event.clear() -def _start_bot(bot, addr: str, port: int, domain: str, user: str, password: str, is_prompter: bool = False)\ +def _start_bot(bot, addr: str, port: int, domain: str, user: str, + password: str, is_prompter: bool = False)\ -> (Process, synchronize.Event): """ Creates a thread and starts the passed bot with passed parameters @@ -112,9 +101,12 @@ def _start_bot(bot, addr: str, port: int, domain: str, user: str, password: str, :param is_prompter: True if bot is to generate prompts for the Proctor :returns: Process bot instance is attached to """ + # TODO: Deprecate event = Event() event.set() - thread = Process(target=_threaded_start_bot, args=(bot, addr, port, domain, user, password, event, is_prompter)) + thread = Process(target=_threaded_start_bot, + args=(bot, addr, port, domain, user, password, event, + is_prompter)) thread.daemon = True thread.start() while event.is_set(): @@ -122,23 +114,27 @@ def _start_bot(bot, addr: str, port: int, domain: str, user: str, password: str, return thread, event -def get_bots_in_dir(bot_path: str, names_to_consider: str = os.environ.get("bot-names", False)) -> dict: +def get_bots_in_dir(bot_path: str, + names_to_consider: str = None) -> dict: """ - Gets all ChatBots in the given directory, imports them, and returns a dict of their names to modules. + Gets all ChatBots in the given directory, imports them, and returns a + dict of their names to modules. :param bot_path: absolute file path containing bots :param names_to_consider: limit imported instances to certain list :return: dict of bot name:ChatBot object """ - from chatbot_core import ChatBot - + names_to_consider = names_to_consider or os.environ.get("bot-names") + import pkgutil + log_deprecation("This functionality is deprecated. Bots should specify a " + "`neon.plugin.chatbot` entrypoint", "3.0.0") bots = {} try: # Make sure we have a path and not a filename - bot_path = bot_path if os.path.isdir(bot_path) else os.path.dirname(bot_path) + bot_path = bot_path if os.path.isdir(bot_path) else \ + os.path.dirname(bot_path) # Get all bots in the requested directory bot_names = [name for _, name, _ in pkgutil.iter_modules([bot_path])] - # TODO: Above fails to import more bots if one fails (bad dependencies, etc) DM # only specified bot names if names_to_consider: bot_names = list(set(bot_names) & set(names_to_consider.split(','))) @@ -148,8 +144,9 @@ def get_bots_in_dir(bot_path: str, names_to_consider: str = os.environ.get("bot- for mod in bot_names: module = __import__(mod) for name, obj in inspect.getmembers(module, inspect.isclass): - # TODO: Why are facilitators not subclassed ChatBots? DM - if name != "ChatBot" and (issubclass(obj, ChatBot) or (mod in name and isinstance(obj, type))): + if name != "ChatBot" and (issubclass(obj, ChatBotABC) or + (mod in name and + isinstance(obj, type))): bots[mod] = obj LOG.debug(bots) except Exception as e: @@ -159,10 +156,14 @@ def get_bots_in_dir(bot_path: str, names_to_consider: str = os.environ.get("bot- def load_credentials_yml(cred_file: str) -> dict: """ - Loads a credentials yml file and returns a dictionary of parsed credentials per-module + Loads a credentials yml file and returns a dictionary of parsed credentials + per-module. :param cred_file: Input yml file containing credentials for bot modules :return: dict of bot modules to usernames and passwords """ + log_deprecation(f"This functionality is deprecated. Configuration should " + f"be specified in `{xdg_config_home()}/neon/chatbots.yaml`", + "3.0.0") with open(cred_file, 'r') as f: credentials_dict = yaml.safe_load(f) return credentials_dict @@ -170,6 +171,7 @@ def load_credentials_yml(cred_file: str) -> dict: def _start_bot_processes(bots_to_start: dict, username: str, password: str, credentials: dict, server: str, domain: str) -> list: + # TODO: Deprecate processes = [] # Start Proctor first if in the list of bots to start @@ -177,8 +179,10 @@ def _start_bot_processes(bots_to_start: dict, username: str, password: str, bot = bots_to_start.get("Proctor") try: user = username or credentials.get("Proctor", {}).get("username") - password = password or credentials.get("Proctor", {}).get("password") - process, event = _start_bot(bot, server, 8888, domain, user, password, False) + password = password or credentials.get("Proctor", + {}).get("password") + process, event = _start_bot(bot, server, 8888, domain, user, + password, False) processes.append(process) except Exception as e: LOG.error(e) @@ -191,7 +195,8 @@ def _start_bot_processes(bots_to_start: dict, username: str, password: str, try: user = username or credentials.get(name, {}).get("username") password = password or credentials.get(name, {}).get("password") - process, event = _start_bot(bot, server, 8888, domain, user, password, False) + process, event = _start_bot(bot, server, 8888, domain, user, + password, False) processes.append(process) except Exception as e: LOG.error(name) @@ -200,11 +205,12 @@ def _start_bot_processes(bots_to_start: dict, username: str, password: str, return processes -def start_bots(domain: str = None, bot_dir: str = None, username: str = None, password: str = None, server: str = None, - cred_file: str = None, bot_name: str = None, excluded_bots: list = None, handle_restart: bool = False, - is_prompter: bool = False): +def start_bots(domain: str = None, bot_dir: str = None, username: str = None, + password: str = None, server: str = None, cred_file: str = None, + bot_name: str = None, excluded_bots: list = None, + handle_restart: bool = False, is_prompter: bool = False): """ - Start all of the bots in the given bot_dir and connect them to the given domain + Start all the bots in the given bot_dir and connect them to the given domain :param domain: Domain to put bots in :param bot_dir: Path containing bots to start :param username: Username to login with (or bot name if not defined) @@ -213,9 +219,14 @@ def start_bots(domain: str = None, bot_dir: str = None, username: str = None, pa :param cred_file: Path to a credentials yml file :param bot_name: Optional name of the bot to start (None for all bots) :param excluded_bots: Optional list of bots to exclude from launching - :param handle_restart: If true, listens for a restart message from the server to restart chatbots - :param is_prompter: If true, bot sends prompts to the Proctor and handles responses - """ + :param handle_restart: If true, listens for a restart message from the + server to restart chatbots + :param is_prompter: If true, bot sends prompts to the Proctor and handles + responses + """ + log_deprecation("This method is deprecated. Bots should be loaded by " + "entrypoints.", "3.0.0") + # TODO: Method for loading v1 bots by entrypoints # global active_server global runner domain = domain or "chatbotsforum.org" @@ -231,10 +242,13 @@ def start_bots(domain: str = None, bot_dir: str = None, username: str = None, pa LOG.info(f"No bots in: {bot_dir}") for d in os.listdir(bot_dir): try: - if str(d) not in ("__pycache__", "tests", "venv", "torchmoji", "ParlAI") and not d.startswith(".") \ + if str(d) not in ("__pycache__", "tests", "venv", "torchmoji", + "ParlAI") and not d.startswith(".") \ and os.path.isdir(os.path.join(bot_dir, d)): LOG.info(f"Found bots dir {d}") - bots_to_start = {**get_bots_in_dir(os.path.join(bot_dir, d)), **bots_to_start} + bots_to_start = {**get_bots_in_dir(os.path.join(bot_dir, + d)), + **bots_to_start} except Exception as e: LOG.error(e) @@ -246,7 +260,8 @@ def start_bots(domain: str = None, bot_dir: str = None, username: str = None, pa # Load credentials if cred_file: cred_file = os.path.expanduser(cred_file) - if not os.path.isfile(cred_file) and os.path.isfile(os.path.join(os.getcwd(), cred_file)): + if not os.path.isfile(cred_file) and \ + os.path.isfile(os.path.join(os.getcwd(), cred_file)): cred_file = os.path.join(os.getcwd(), cred_file) elif not os.path.isfile(cred_file): cred_file = None @@ -268,9 +283,12 @@ def start_bots(domain: str = None, bot_dir: str = None, username: str = None, pa if bot: bots_to_start = {bot_name: bot} try: - user = username or credentials.get(bot_name, {}).get("username") - password = password or credentials.get(bot_name, {}).get("password") - p, _ = _start_bot(bot, server, 8888, domain, user, password, is_prompter) + user = username or credentials.get(bot_name, + {}).get("username") + password = password or credentials.get(bot_name, + {}).get("password") + p, _ = _start_bot(bot, server, 8888, domain, user, password, + is_prompter) processes.append(p) # bot(start_socket(server, 8888), domain, user, password, True) except Exception as e: @@ -286,9 +304,12 @@ def start_bots(domain: str = None, bot_dir: str = None, username: str = None, pa if name in bots_to_start.keys(): bots_to_start.pop(name) - processes = _start_bot_processes(bots_to_start, username, password, credentials, server, domain) + processes = _start_bot_processes(bots_to_start, username, password, + credentials, server, domain) if handle_restart: + log_deprecation("Messagebus connections to Neon Core will be " + "deprecated", "3.0.0") LOG.info(f"Setting restart listener for {server}") _listen_for_restart_chatbots(server) try: @@ -307,165 +328,47 @@ def start_bots(domain: str = None, bot_dir: str = None, username: str = None, pa p.terminate() time.sleep(1) if p.is_alive(): - LOG.warning(f"Process {p.pid} not terminated!! Killing..") + LOG.warning(f"Process {p.pid} not terminated! " + f"Killing..") p.kill() time.sleep(1) if p.is_alive(): - LOG.error(f"Process {p.pid} still alive!!") + LOG.error(f"Process {p.pid} still alive!") except Exception as e: LOG.error(e) p.kill() LOG.debug(f"Processes ended") - processes = _start_bot_processes(bots_to_start, username, password, credentials, server, domain) + processes = _start_bot_processes(bots_to_start, username, password, + credentials, server, domain) except KeyboardInterrupt: LOG.info("exiting") for p in processes: p.join() -def cli_start_mq_bot(): - """ - Entrypoint to start an MQ chatbot - """ - import argparse - - parser = argparse.ArgumentParser(description="Start a chatbot") - parser.add_argument("--bot", dest="bot_name", - help="Chatbot entrypoint name", type=str) - args = parser.parse_args() - run_mq_bot(args.bot_name) - - -def cli_start_bots(): - """ - Entry Point to start bots from a Console Script - """ - import argparse - - parser = argparse.ArgumentParser(description="Start some chatbots") - parser.add_argument("--domain", dest="domain", default="chatbotsforum.org", - help="Domain to connect to (default: chatbotsforum.org)", type=str) - parser.add_argument("--dir", dest="bot_dir", - help="Path to chatbots (default: ./)", type=str) - parser.add_argument("--bot", dest="bot_name", - help="Optional bot name to run a single bot only", type=str) - parser.add_argument("--credentials", dest="cred_file", - help="Optional path to YAML credentials", type=str) - parser.add_argument("--username", dest="username", - help="Klat username for a single bot", type=str) - parser.add_argument("--password", dest="password", - help="Klat password for a single bot", type=str) - parser.add_argument("--server", dest="server", default=SERVER, - help=f"Klat server (default: {SERVER})", type=str) - parser.add_argument("--debug", dest="debug", action='store_true', - help="Enable more verbose log output") - parser.add_argument("--bot-names", dest="bot-names", - help="comma separated list of bots to include in running", type=str) - parser.add_argument("--exclude", dest="exclude", - help="comma separated list of bots to exclude from running", type=str) - parser.add_argument("--handle-restart", dest="handle_restart", default=False, - help="True to handle server emit to restart bots", type=bool) - args = parser.parse_args() - - if args.debug: - logging.getLogger("chatbots").setLevel(logging.DEBUG) - logging.getLogger("chatbot").setLevel(logging.DEBUG) - else: - logging.getLogger("chatbots").setLevel(logging.INFO) - logging.getLogger("chatbot").setLevel(logging.INFO) - - if args.exclude: - excluded_bots = [name.strip() for name in args.exclude.split(",")] - else: - excluded_bots = None - LOG.debug(args) - start_bots(args.domain, args.bot_dir, args.username, args.password, args.server, args.cred_file, args.bot_name, - excluded_bots, args.handle_restart) - - -def cli_stop_bots(): - """ - Stops all start-klat-bot instances - """ - import argparse - - parser = argparse.ArgumentParser(description="Stop some chatbots") - parser.add_argument("--server", dest="server", default="", - help=f"Klat server (default: None)", type=str) - args = parser.parse_args() - if args.server: - server_to_stop = args.server - else: - server_to_stop = None - import psutil - - procs = {p.pid: p.info for p in psutil.process_iter(['name'])} - for pid, name in procs.items(): - if name.get("name") == "start-klat-bots" and \ - (not server_to_stop or f"--server={server_to_stop}" in psutil.Process(pid).cmdline()): - LOG.info(f"Terminating {pid}") - psutil.Process(pid).terminate() - time.sleep(1) - if psutil.pid_exists(pid) and psutil.Process(pid).is_running(): - LOG.error(f"Process {pid} not terminated!!") - psutil.Process(pid).kill() - - -def cli_start_prompter(): - """ - Entry Point to start a prompter bot - """ - import argparse - - parser = argparse.ArgumentParser(description="Start a prompter chatbot") - parser.add_argument("--bot", dest="bot_name", - help="Optional bot name to run a single bot only", type=str) - parser.add_argument("--dir", dest="bot_dir", - help="Path to chatbots (default: ./)", type=str) - parser.add_argument("--username", dest="username", - help="Klat username for a single bot", type=str) - parser.add_argument("--password", dest="password", - help="Klat password for a single bot", type=str) - parser.add_argument("--server", dest="server", default=SERVER, - help=f"Klat server (default: {SERVER})", type=str) - parser.add_argument("--debug", dest="debug", action='store_true', - help="Enable more verbose log output") - parser.add_argument("--handle-restart", dest="handle_restart", default=False, - help="True to handle server emit to restart bots", type=bool) - args = parser.parse_args() - - if args.debug: - logging.getLogger("chatbots").setLevel(logging.DEBUG) - logging.getLogger("chatbot").setLevel(logging.DEBUG) - else: - logging.getLogger("chatbots").setLevel(logging.INFO) - logging.getLogger("chatbot").setLevel(logging.INFO) - LOG.debug(args) - start_bots("chatbotsforum.org", args.bot_dir, args.username, args.password, args.server, None, args.bot_name, - None, args.handle_restart, True) - - -def debug_bots(bot_dir: str = os.getcwd()): +def debug_bots(bot_dir: str = None): """ Debug bots in the passed directory :param bot_dir: directory containing the bot to test """ - # TODO: Generalize this to testing different modules? Leave one method for selecting a bot and then create an - # options menu for this interactive testing, along with automated discusser and appraiser testing. - # Automated testing could use pre-built response objects, or run n other bots and handle their outputs offline - - # Try handling passed directory - if len(sys.argv) > 1: - arg_dir = os.path.expanduser(sys.argv[1]) - bot_dir = arg_dir if os.path.exists(arg_dir) else bot_dir - - logging.getLogger("chatbots").setLevel(logging.WARNING) - logging.getLogger("klat_connector").setLevel(logging.WARNING) - - subminds = get_bots_in_dir(bot_dir) - + # TODO: Generalize this to testing different modules? Leave one method for + # selecting a bot and then create an options menu for this interactive + # testing, along with automated discusser and appraiser testing. + # Automated testing could use pre-built response objects, or run n + # other bots and handle their outputs offline + from klat_connector.mach_server import MachKlatServer + server = MachKlatServer() + + if bot_dir: + log_deprecation("Bots should be installed so they may be accessed by " + "entrypoint. Specifying a local directory will no " + "longer be supported", "2.3.1") + subminds = get_bots_in_dir(bot_dir) + else: + subminds = _find_bot_modules() # Options to exit the interactive shell - stop_triggers = ["bye", "see you later", "until next time", "have a great day", "goodbye"] + stop_triggers = ["bye", "see you later", "until next time", + "have a great day", "goodbye"] running = True while running: try: @@ -473,36 +376,40 @@ def debug_bots(bot_dir: str = os.getcwd()): f'Please choose a bot to talk to') bot_name = input('[In]: ') if bot_name in subminds: - bot = subminds[bot_name](start_socket("2222.us", 8888), None, None, None, on_server=False) + bot = subminds[bot_name](start_socket("0.0.0.0", 8888), None, + None, None, on_server=False) while running: utterance = input('[In]: ') - response = bot.ask_chatbot('Tester', utterance, datetime.now().strftime("%I:%M:%S %p")) + response = bot.ask_chatbot('Tester', utterance, + datetime.now().strftime( + "%I:%M:%S %p")) print(f'[Out]: {response}') if utterance.lower() in stop_triggers: running = False - LOG.warning("STOP RUNNING") + LOG.debug("STOP RUNNING") else: - print(f'BOTS: {subminds.keys()}.\n' - f'This bot does not exist. Please choose a valid bot to talk to') + print(f'BOTS: {subminds.keys()}.\nThis bot does not exist.' + f' Please choose a valid bot to talk to') except KeyboardInterrupt: running = False LOG.warning("STOP RUNNING") except EOFError: running = False - LOG.warning("Still Running") - LOG.warning("Done Running") + server.shutdown_server() + LOG.info("Done Running") -def clean_up_bot(bot): +def clean_up_bot(bot: ChatBotABC): """ Performs any standard cleanup for a bot on destroy :param bot: ChatBot instance to clean up """ - from chatbot_core import ChatBot + from chatbot_core.v1 import ChatBot as V1 - if not isinstance(bot, ChatBot): - raise TypeError - bot.socket.disconnect() + if not isinstance(bot, ChatBotABC): + raise TypeError(f"Expected ChatBot, got: {type(bot)}") + if isinstance(bot, V1): + bot.socket.disconnect() if hasattr(bot, "shout_queue"): bot.shout_queue.put(None) if hasattr(bot, "shout_thread"): @@ -517,6 +424,7 @@ def _restart_chatbots(message: Message): Messagebus handler to restart chatbots on a server :param message: Message associated with request """ + # TODO: Deprecate in 3.0.0 global runner LOG.debug(f"Restart received: {message.data} | {message.context}") runner.set() @@ -527,6 +435,7 @@ def _listen_for_restart_chatbots(server: str): Registers a messagebus listener to restart chatbots for the given server :param server: base url of the klat server messagebus to listen to """ + # TODO: Deprecate in 3.0.0 if server == "2222.us": host = "64.34.186.120" elif server == "5555.us": @@ -548,11 +457,14 @@ def init_message_bus(bus_config: dict = None) -> (Thread, MessageBusClient): :param bus_config: messagebus configuration to use :return: Thread, messagebus object """ + log_deprecation("Messagebus connections to Neon Core will be deprecated", + "3.0.0") bus_config = bus_config or {"host": "167.172.112.7", "port": 8181, "ssl": False, "route": "/core"} - bus = MessageBusClient(bus_config["host"], bus_config["port"], bus_config["route"], bus_config["ssl"]) + bus = MessageBusClient(bus_config["host"], bus_config["port"], + bus_config["route"], bus_config["ssl"]) t = bus.run_in_thread() bus.connected_event.wait(10) LOG.info(f"Connected to Messagebus at: {bus_config['host']}") @@ -561,71 +473,95 @@ def init_message_bus(bus_config: dict = None) -> (Thread, MessageBusClient): def generate_random_response(from_iterable: iter): """ - Generates some random bot response from the given options or the default list - - :param from_iterable: source iterable to get random value from + Generates some random bot response from the given options + :param from_iterable: source iterable to get random value from """ + log_deprecation("Use `random.choice` directly", "3.0.0") return random.choice(from_iterable) -def find_closest_answer(algorithm: str = 'random', sentence: str = None, options: dict = None): +def find_closest_answer(algorithm: str = 'random', sentence: str = None, + options: dict = None) -> Optional[str]: """ - Handles an incoming shout into the current conversation - :param algorithm: algorithm considered - :param sentence: base sentence - :param options: options to pick best one from + Determines which option is most similar to an input sentence + :param algorithm: algorithm considered + :param sentence: base sentence + :param options: dict of option ID to response for comparison with `sentence` + :returns: `option` ID with a value closest to `sentence`, None if requested + algorithm fails """ if not sentence: LOG.warning('Empty sentence supplied') - return sentence + return None if not options or len(options.keys()) == 0: LOG.warning('No options provided') - return sentence + return None if algorithm == 'random': - closest_answer = random.choice(options) + closest_answer = random.choice(list(options.keys())) elif algorithm == 'bleu_score': + try: + import nltk + nltk.download('punkt') + from nltk import word_tokenize + from nltk.translate.bleu_score import sentence_bleu + except ImportError: + LOG.warning("`nltk` not installed. install " + "`chatbot-core[lang]` to install NLU packages.") + return None bleu_scores = [] response_tokenized = word_tokenize(sentence.lower()) for option in options.keys(): opinion_tokenized = word_tokenize(options[option].lower()) if len(opinion_tokenized) > 0: if min(len(response_tokenized), len(opinion_tokenized)) < 4: - weighting = 1.0 / min(len(response_tokenized), len(opinion_tokenized)) - weights = tuple([weighting] * min(len(response_tokenized), len(opinion_tokenized))) + weighting = 1.0 / min(len(response_tokenized), + len(opinion_tokenized)) + weights = tuple([weighting] * min(len(response_tokenized), + len(opinion_tokenized))) else: weights = (0.25, 0.25, 0.25, 0.25) - bleu_scores.append( - (option, sentence_bleu([response_tokenized], opinion_tokenized, weights=weights))) - max_score = max([x[1] for x in bleu_scores]) if len(bleu_scores) > 0 else 0 - closest_answer = random.choice(list(filter(lambda x: x[1] == max_score, bleu_scores)))[0] + bleu_scores.append((option, sentence_bleu([response_tokenized], + opinion_tokenized, + weights=weights))) + max_score = max([x[1] for x in + bleu_scores]) if len(bleu_scores) > 0 else 0 + closest_answer = random.choice(list(filter(lambda x: x[1] == max_score, + bleu_scores)))[0] LOG.info(f'Closest answer is {closest_answer}') elif algorithm == 'damerau_levenshtein_distance': closest_distance = None closest_answer = None try: + import jellyfish for option in options.items(): - distance = jellyfish.damerau_levenshtein_distance(option[1], sentence) + distance = jellyfish.damerau_levenshtein_distance(option[1], + sentence) if not closest_distance or closest_distance > distance: closest_distance = distance closest_answer = option[0] LOG.info(f'Closest answer is {closest_answer}') + except ImportError: + jellyfish = None + LOG.warning("`jellyfish` not installed. install " + "`chatbot-core[lang]` to install NLU packages.") except Exception as e: LOG.error(e) else: LOG.error(f'Unknown algorithm supplied:{algorithm}') - return sentence + return None return closest_answer -def grammar_check(func): +def grammar_check(func: Callable): """ - Checks grammar for output of passed function + Decorator to add spelling/grammar checks to a function's output :param func: function to consider """ try: from autocorrect import Speller spell = Speller() except ImportError: + Speller = None LOG.error("autocorrect module not available. Install " "`chatbot-core[extra-lgpl]` to use autocorrect.") spell = None @@ -642,18 +578,23 @@ def wrapper(*args, **kwargs): return wrapper -def _find_bot_modules(): +def _find_bot_modules() -> Dict[str, type(ChatBotABC)]: + """ + Method for locating all installed chatbots by entrypoint. + """ try: from importlib_metadata import entry_points bot_entrypoints = entry_points(group="neon.plugin.chatbot") except ImportError: + entry_points = None + import pkg_resources bot_entrypoints = pkg_resources.iter_entry_points("neon.plugin.chatbot") return {entry.name: entry.load() for entry in bot_entrypoints} def run_mq_bot(chatbot_name: str, vhost: str = '/chatbots', - run_kwargs: dict = None, init_kwargs: dict = None): + run_kwargs: dict = None, init_kwargs: dict = None) -> ChatBotV2: """ Get an initialized MQ Chatbot instance @param chatbot_name: chatbot entrypoint name and configuration key @@ -676,10 +617,7 @@ def run_mq_bot(chatbot_name: str, vhost: str = '/chatbots', raise RuntimeError(f"Requested bot `{chatbot_name}` not found in: " f"{list(bots.keys())}") bot = clazz(service_name=chatbot_name, vhost=vhost, **init_kwargs) + LOG.info(f"Starting {chatbot_name}") bot.run(**run_kwargs) + LOG.info(f"Started {chatbot_name}") return bot - - -if __name__ == "__main__": - start_bots("chatbotsforum.org", "~/PycharmProjects/chatbots", "Prompter", "n30nn30n", "2222.us", None, "BLENDER", - None, True, True) diff --git a/chatbot_core/utils/conversation_utils.py b/chatbot_core/utils/conversation_utils.py index 9997ad4..e5370b5 100644 --- a/chatbot_core/utils/conversation_utils.py +++ b/chatbot_core/utils/conversation_utils.py @@ -1,9 +1,28 @@ +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System +# +# Copyright 2008-2021 Neongecko.com Inc. | All Rights Reserved +# +# Notice of License - Duplicating this Notice of License near the start of any file containing +# a derivative of this software is a condition of license for this software. +# Friendly Licensing: +# No charge, open source royalty free use of the Neon AI software source and object is offered for +# educational users, noncommercial enthusiasts, Public Benefit Corporations (and LLCs) and +# Social Purpose Corporations (and LLCs). Developers can contact developers@neon.ai +# For commercial licensing, distribution of derivative works or redistribution please contact licenses@neon.ai +# Distributed on an "AS IS” basis without warranties or conditions of any kind, either express or implied. +# Trademarks of Neongecko: Neon AI(TM), Neon Assist (TM), Neon Communicator(TM), Klat(TM) +# Authors: Guy Daniels, Daniel McKnight, Regina Bloomstine, Elon Gasper, Richard Leeds +# +# Specialized conversational reconveyance options from Conversation Processing Intelligence Corp. +# US Patents 2008-2021: US7424516, US20140161250, US20140177813, US8638908, US8068604, US8553852, US10530923, US10530924 +# China Patent: CN102017585 - Europe Patent: EU2156652 - Patents Pending + from itertools import cycle def create_conversation_cycle() -> cycle: """Cycle through conversation states""" - from chatbot_core.utils import ConversationState + from chatbot_core.utils.enum import ConversationState return cycle([ConversationState.RESP, ConversationState.DISC, diff --git a/chatbot_core/utils/enum.py b/chatbot_core/utils/enum.py index 93ccafe..14c4155 100644 --- a/chatbot_core/utils/enum.py +++ b/chatbot_core/utils/enum.py @@ -1,4 +1,23 @@ -from enum import IntEnum, Enum +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System +# +# Copyright 2008-2021 Neongecko.com Inc. | All Rights Reserved +# +# Notice of License - Duplicating this Notice of License near the start of any file containing +# a derivative of this software is a condition of license for this software. +# Friendly Licensing: +# No charge, open source royalty free use of the Neon AI software source and object is offered for +# educational users, noncommercial enthusiasts, Public Benefit Corporations (and LLCs) and +# Social Purpose Corporations (and LLCs). Developers can contact developers@neon.ai +# For commercial licensing, distribution of derivative works or redistribution please contact licenses@neon.ai +# Distributed on an "AS IS” basis without warranties or conditions of any kind, either express or implied. +# Trademarks of Neongecko: Neon AI(TM), Neon Assist (TM), Neon Communicator(TM), Klat(TM) +# Authors: Guy Daniels, Daniel McKnight, Regina Bloomstine, Elon Gasper, Richard Leeds +# +# Specialized conversational reconveyance options from Conversation Processing Intelligence Corp. +# US Patents 2008-2021: US7424516, US20140161250, US20140177813, US8638908, US8068604, US8553852, US10530923, US10530924 +# China Patent: CN102017585 - Europe Patent: EU2156652 - Patents Pending + +from enum import IntEnum class ConversationControls: diff --git a/chatbot_core/utils/logger.py b/chatbot_core/utils/logger.py index af7f9d5..7922aaf 100644 --- a/chatbot_core/utils/logger.py +++ b/chatbot_core/utils/logger.py @@ -17,23 +17,19 @@ # US Patents 2008-2021: US7424516, US20140161250, US20140177813, US8638908, US8068604, US8553852, US10530923, US10530924 # China Patent: CN102017585 - Europe Patent: EU2156652 - Patents Pending -from ovos_utils.log import LOG +from ovos_utils.log import LOG, log_deprecation LOG.name = "chatbots" -# fmt = '%(asctime)s - %(levelname)-8s - %(name)s:%(filename)s:%(module)s:%(funcName)s:%(lineno)d - %(message)s' -# logging.basicConfig(level=logging.DEBUG, format=fmt, datefmt='%Y-%m-%d:%H:%M:%S') -# LOG = logging.getLogger("chatbots") -# logging.getLogger("socketio.client").setLevel(logging.WARNING) -# logging.getLogger("engineio.client").setLevel(logging.WARNING) -# logging.getLogger("urllib3").setLevel(logging.WARNING) - -def make_logger(name, level=None): +def make_logger(name, level: str = None): """ Create a logger with the specified name (used to create bot loggers) """ - import logging - level = level or logging.DEBUG - logger = logging.getLogger(name) - logger.setLevel(level) - return logging.getLogger(name) + log_deprecation("Call `neon_utils.log_utils.init_log` directly", "3.0.0") + from neon_utils.log_utils import init_log + init_log(log_name=name) + if level: + LOG.set_level(level) + LOG.warning(f"Log level should be set in configuration") + return LOG + diff --git a/chatbot_core/utils/string_utils.py b/chatbot_core/utils/string_utils.py index 3275297..5a37bf7 100644 --- a/chatbot_core/utils/string_utils.py +++ b/chatbot_core/utils/string_utils.py @@ -1,3 +1,22 @@ +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System +# +# Copyright 2008-2021 Neongecko.com Inc. | All Rights Reserved +# +# Notice of License - Duplicating this Notice of License near the start of any file containing +# a derivative of this software is a condition of license for this software. +# Friendly Licensing: +# No charge, open source royalty free use of the Neon AI software source and object is offered for +# educational users, noncommercial enthusiasts, Public Benefit Corporations (and LLCs) and +# Social Purpose Corporations (and LLCs). Developers can contact developers@neon.ai +# For commercial licensing, distribution of derivative works or redistribution please contact licenses@neon.ai +# Distributed on an "AS IS” basis without warranties or conditions of any kind, either express or implied. +# Trademarks of Neongecko: Neon AI(TM), Neon Assist (TM), Neon Communicator(TM), Klat(TM) +# Authors: Guy Daniels, Daniel McKnight, Regina Bloomstine, Elon Gasper, Richard Leeds +# +# Specialized conversational reconveyance options from Conversation Processing Intelligence Corp. +# US Patents 2008-2021: US7424516, US20140161250, US20140177813, US8638908, US8068604, US8553852, US10530923, US10530924 +# China Patent: CN102017585 - Europe Patent: EU2156652 - Patents Pending + def remove_prefix(prefixed_string: str, prefix: str): """ Removes the specified prefix from the string @@ -6,5 +25,5 @@ def remove_prefix(prefixed_string: str, prefix: str): :return: string with prefix removed """ if prefixed_string.startswith(prefix): - return prefixed_string[len(prefix):] + return prefixed_string[len(prefix):].lstrip() return prefixed_string diff --git a/chatbot_core/utils/version_utils.py b/chatbot_core/utils/version_utils.py index 265a06f..f2ec767 100644 --- a/chatbot_core/utils/version_utils.py +++ b/chatbot_core/utils/version_utils.py @@ -1,15 +1,41 @@ +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System +# +# Copyright 2008-2021 Neongecko.com Inc. | All Rights Reserved +# +# Notice of License - Duplicating this Notice of License near the start of any file containing +# a derivative of this software is a condition of license for this software. +# Friendly Licensing: +# No charge, open source royalty free use of the Neon AI software source and object is offered for +# educational users, noncommercial enthusiasts, Public Benefit Corporations (and LLCs) and +# Social Purpose Corporations (and LLCs). Developers can contact developers@neon.ai +# For commercial licensing, distribution of derivative works or redistribution please contact licenses@neon.ai +# Distributed on an "AS IS” basis without warranties or conditions of any kind, either express or implied. +# Trademarks of Neongecko: Neon AI(TM), Neon Assist (TM), Neon Communicator(TM), Klat(TM) +# Authors: Guy Daniels, Daniel McKnight, Regina Bloomstine, Elon Gasper, Richard Leeds +# +# Specialized conversational reconveyance options from Conversation Processing Intelligence Corp. +# US Patents 2008-2021: US7424516, US20140161250, US20140177813, US8638908, US8068604, US8553852, US10530923, US10530924 +# China Patent: CN102017585 - Europe Patent: EU2156652 - Patents Pending + import os -from typing import Type -version = os.environ.get('CHATBOT_VERSION', 'v1').lower() +from typing import Optional +from chatbot_core.chatbot_abc import ChatBotABC -def get_class(version_class_mapping: dict) -> Type: +def get_class() -> Optional[type(ChatBotABC)]: """ - Get class matching current CHATBOT_VERSION - - :param version_class_mapping: mapping between version and class instance + Get class matching current CHATBOT_VERSION - :returns Class instance matching current version if any + :returns Class instance matching current version if any """ - return version_class_mapping.get(version, None) + from chatbot_core.v1 import ChatBot as ChatBot_v1 + from chatbot_core.v2 import ChatBot as ChatBot_v2 + + version = os.environ.get('CHATBOT_VERSION', 'v1').lower() + + chatbot_versions = { + 'v1': ChatBot_v1, + 'v2': ChatBot_v2 + } + return chatbot_versions.get(version, None) diff --git a/chatbot_core/v1/__init__.py b/chatbot_core/v1/__init__.py index 9d8dc75..6930dc6 100644 --- a/chatbot_core/v1/__init__.py +++ b/chatbot_core/v1/__init__.py @@ -19,42 +19,25 @@ import random import re -from abc import abstractmethod -from queue import Queue -from typing import Optional import time -# import sys -from copy import deepcopy +from copy import deepcopy from engineio.socket import Socket -# import threading -from threading import Thread, Event - +from threading import Thread from klat_connector.klat_api import KlatApi from klat_connector import start_socket -from chatbot_core.utils import init_message_bus, make_logger, ConversationState,ConversationControls,\ - remove_prefix, generate_random_response, BotTypes -from chatbot_core.chatbot_abc import ChatBotABC -from chatbot_core.utils.logger import LOG -from mycroft_bus_client import Message, MessageBusClient -from nltk.translate.bleu_score import sentence_bleu -from nltk import word_tokenize -import jellyfish -import spacy - -try: - from autocorrect import Speller +from ovos_utils.log import LOG - spell = Speller() -except ImportError: - LOG.error("autocorrect module not available. Install " - "`chatbot-core[extra-lgpl]` to use autocorrect.") - spell = None +from chatbot_core.utils.enum import ConversationState, ConversationControls, BotTypes +from chatbot_core.utils.string_utils import remove_prefix +from chatbot_core.chatbot_abc import ChatBotABC class ChatBot(KlatApi, ChatBotABC): def __init__(self, *args, **kwargs): - socket, domain, username, password, on_server, is_prompter = self.parse_init(*args, **kwargs) + socket, domain, username, password, on_server, is_prompter = \ + self.parse_init(*args, **kwargs) + LOG.info(f"Starting {username}") socket = socket or start_socket() init_nick = "Prompter" if is_prompter else "" KlatApi.__init__(self, socket, domain, init_nick) @@ -427,7 +410,7 @@ def propose_response(self, shout: str): # Generate a random response if none is provided if shout == self.active_prompt: self.log.info(f"Pick random response for {self.nick}") - shout = generate_random_response(self.fallback_responses) + shout = random.choice(self.fallback_responses) if not shout: if self.bot_type == BotTypes.SUBMIND: @@ -642,10 +625,11 @@ def _send_first_prompt(self): Sends an initial prompt to the proctor for a prompter bot """ self.log.debug(f"{self.nick} sending initial prompt!") - self.send_shout("@Proctor hello!", self.get_private_conversation(["Proctor"]), "Private") + self.send_shout("@Proctor hello!", + self.get_private_conversation(["Proctor"]), "Private") def exit(self): - from chatbot_core.utils import clean_up_bot + from chatbot_core.utils.bot_utils import clean_up_bot # import sys # self.socket.disconnect() while not self.shout_queue.empty(): diff --git a/chatbot_core/v2/__init__.py b/chatbot_core/v2/__init__.py index 827568f..ff2be6c 100644 --- a/chatbot_core/v2/__init__.py +++ b/chatbot_core/v2/__init__.py @@ -16,18 +16,17 @@ # Specialized conversational reconveyance options from Conversation Processing Intelligence Corp. # US Patents 2008-2021: US7424516, US20140161250, US20140177813, US8638908, US8068604, US8553852, US10530923, US10530924 # China Patent: CN102017585 - Europe Patent: EU2156652 - Patents Pending + import os import time from neon_mq_connector.utils import RepeatingTimer from neon_mq_connector.utils.rabbit_utils import create_mq_callback - from klat_connector.mq_klat_api import KlatAPIMQ from pika.exchange_type import ExchangeType -from chatbot_core import ConversationState +from chatbot_core.utils.enum import ConversationState, BotTypes from chatbot_core.chatbot_abc import ChatBotABC -from chatbot_core.utils import BotTypes class ChatBot(KlatAPIMQ, ChatBotABC): diff --git a/version.py b/chatbot_core/version.py similarity index 100% rename from version.py rename to chatbot_core/version.py diff --git a/chatbot_core/version_mapping.py b/chatbot_core/version_mapping.py deleted file mode 100644 index 6d9da63..0000000 --- a/chatbot_core/version_mapping.py +++ /dev/null @@ -1,7 +0,0 @@ -from chatbot_core.v1 import ChatBot as ChatBot_v1 -from chatbot_core.v2 import ChatBot as ChatBot_v2 - -CHATBOT_VERSIONS = { - 'v1': ChatBot_v1, - 'v2': ChatBot_v2 -} diff --git a/requirements/extra-lang.txt b/requirements/extra-lang.txt new file mode 100644 index 0000000..f5dfa2c --- /dev/null +++ b/requirements/extra-lang.txt @@ -0,0 +1,3 @@ +jellyfish~=0.8 +nltk~=3.5 +spacy~=2.3.1 \ No newline at end of file diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 0653aeb..07d17a3 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -1,8 +1,5 @@ -mycroft-messagebus-client +ovos-bus-client~=0.0.5 klat-connector>=0.6.2a15 -psutil~=5.7.3 -setuptools~=50.3.2 -jellyfish~=0.8.2 -nltk~=3.5 -spacy~=2.3.1 -neon_utils~=1.0 \ No newline at end of file +psutil~=5.7 +neon_utils[network]~=1.0 +click~=8.0 \ No newline at end of file diff --git a/setup.py b/setup.py index 207baae..16b680a 100644 --- a/setup.py +++ b/setup.py @@ -45,7 +45,7 @@ def get_requirements(requirements_filename: str): with open(path.join(BASE_PATH, "README.md"), "r") as f: long_description = f.read() -with open(path.join(BASE_PATH, "version.py"), "r", encoding="utf-8") as v: +with open(path.join(BASE_PATH, "chatbot_core", "version.py"), "r", encoding="utf-8") as v: for line in v.readlines(): if line.startswith("__version__"): if '"' in line: @@ -68,11 +68,13 @@ def get_requirements(requirements_filename: str): "Operating System :: OS Independent" ], python_requires='>=3.6', - entry_points={'console_scripts': ["start-klat-bots=chatbot_core.utils:cli_start_bots", - "stop-klat-bots=chatbot_core.utils:cli_stop_bots", - "debug-klat-bots=chatbot_core.utils:debug_bots", - "start-klat-prompter=chatbot_core.utils:cli_start_prompter", - "start-mq-bot=chatbot_core.utils.bot_utils:cli_start_mq_bot"]}, + entry_points={'console_scripts': ["chatbots=chatbot_core.cli:chatbot_core_cli", + "start-klat-bots=chatbot_core.cli:cli_start_bots", + "stop-klat-bots=chatbot_core.cli:cli_stop_bots", + "debug-klat-bots=chatbot_core.cli:cli_debug_bots", + "start-klat-prompter=chatbot_core.cli:cli_start_prompter", + "start-mq-bot=chatbot_core.cli:cli_start_mq_bot"]}, install_requires=get_requirements("requirements.txt"), - extras_requires={"extra-lgpl": get_requirements("extra-lgpl.txt")} + extras_require={"lgpl": get_requirements("extra-lgpl.txt"), + "lang": get_requirements("extra-lang.txt")} ) diff --git a/tests/__init__.py b/tests/__init__.py index e69de29..97331b9 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -0,0 +1,18 @@ +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System +# +# Copyright 2008-2021 Neongecko.com Inc. | All Rights Reserved +# +# Notice of License - Duplicating this Notice of License near the start of any file containing +# a derivative of this software is a condition of license for this software. +# Friendly Licensing: +# No charge, open source royalty free use of the Neon AI software source and object is offered for +# educational users, noncommercial enthusiasts, Public Benefit Corporations (and LLCs) and +# Social Purpose Corporations (and LLCs). Developers can contact developers@neon.ai +# For commercial licensing, distribution of derivative works or redistribution please contact licenses@neon.ai +# Distributed on an "AS IS” basis without warranties or conditions of any kind, either express or implied. +# Trademarks of Neongecko: Neon AI(TM), Neon Assist (TM), Neon Communicator(TM), Klat(TM) +# Authors: Guy Daniels, Daniel McKnight, Regina Bloomstine, Elon Gasper, Richard Leeds +# +# Specialized conversational reconveyance options from Conversation Processing Intelligence Corp. +# US Patents 2008-2021: US7424516, US20140161250, US20140177813, US8638908, US8068604, US8553852, US10530923, US10530924 +# China Patent: CN102017585 - Europe Patent: EU2156652 - Patents Pending diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py new file mode 100644 index 0000000..97331b9 --- /dev/null +++ b/tests/integration/__init__.py @@ -0,0 +1,18 @@ +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System +# +# Copyright 2008-2021 Neongecko.com Inc. | All Rights Reserved +# +# Notice of License - Duplicating this Notice of License near the start of any file containing +# a derivative of this software is a condition of license for this software. +# Friendly Licensing: +# No charge, open source royalty free use of the Neon AI software source and object is offered for +# educational users, noncommercial enthusiasts, Public Benefit Corporations (and LLCs) and +# Social Purpose Corporations (and LLCs). Developers can contact developers@neon.ai +# For commercial licensing, distribution of derivative works or redistribution please contact licenses@neon.ai +# Distributed on an "AS IS” basis without warranties or conditions of any kind, either express or implied. +# Trademarks of Neongecko: Neon AI(TM), Neon Assist (TM), Neon Communicator(TM), Klat(TM) +# Authors: Guy Daniels, Daniel McKnight, Regina Bloomstine, Elon Gasper, Richard Leeds +# +# Specialized conversational reconveyance options from Conversation Processing Intelligence Corp. +# US Patents 2008-2021: US7424516, US20140161250, US20140177813, US8638908, US8068604, US8553852, US10530923, US10530924 +# China Patent: CN102017585 - Europe Patent: EU2156652 - Patents Pending diff --git a/tests/chatbot_objects.py b/tests/integration/chatbot_objects.py similarity index 98% rename from tests/chatbot_objects.py rename to tests/integration/chatbot_objects.py index 3b4884e..cea3c5c 100644 --- a/tests/chatbot_objects.py +++ b/tests/integration/chatbot_objects.py @@ -19,14 +19,10 @@ import random import copy -# import jellyfish -import sys -import os - -# Required for pytest on GitHub -sys.path.append(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))) -from chatbot_core import ChatBot -from chatbot_core.utils import LOG + +from ovos_utils.log import LOG + +from chatbot_core.v1 import ChatBot class V2Bot(ChatBot): diff --git a/tests/chatbot_core_tests.py b/tests/integration/test_chatbot_core.py similarity index 79% rename from tests/chatbot_core_tests.py rename to tests/integration/test_chatbot_core.py index df8eb66..d7fcdc6 100644 --- a/tests/chatbot_core_tests.py +++ b/tests/integration/test_chatbot_core.py @@ -17,28 +17,23 @@ # US Patents 2008-2021: US7424516, US20140161250, US20140177813, US8638908, US8068604, US8553852, US10530923, US10530924 # China Patent: CN102017585 - Europe Patent: EU2156652 - Patents Pending -from datetime import datetime import unittest -import sys -import os import pytest -import time +from datetime import datetime from klat_connector import start_socket from klat_connector.mach_server import MachKlatServer -# Required for pytest on GitHub -sys.path.append(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))) -from chatbot_core import ConversationControls, ConversationState -from chatbot_core.utils import clean_up_bot -from tests.chatbot_objects import * +from chatbot_core.utils.enum import ConversationControls, ConversationState +from chatbot_core.utils.bot_utils import clean_up_bot +from .chatbot_objects import * SERVER = "0.0.0.0" @pytest.mark.timeout(timeout=300, method='signal') -class ChatbotCoreTests(unittest.TestCase): +class TestChatbotCore(unittest.TestCase): @classmethod def setUpClass(cls) -> None: @@ -195,71 +190,11 @@ def test_15_histories_length(self): # from chatbot_core.utils import get_bots_in_dir # get_bots_in_dir("/home/d_mcknight/PycharmProjects/chatbots/bots/ELIZA") - @pytest.mark.timeout(30) - def test_start_base_bot(self): - from chatbot_core.utils.bot_utils import _start_bot - from multiprocessing import Process, synchronize - - t, e = _start_bot(ChatBot, SERVER, 8888, "Private", "testrunner", "testpassword") - self.assertIsInstance(t, Process) - self.assertIsInstance(e, synchronize.Event) - # self.assertFalse(e.is_set()) - e.set() - timeout = time.time() + 10 - while e.is_set() and time.time() < timeout: - print("...") - time.sleep(2) - self.assertFalse(e.is_set()) - t.terminate() - self.assertFalse(t.is_alive()) - print("Joining...") - t.join() - - @pytest.mark.timeout(30) - def test_start_v2_bot(self): - from chatbot_core.utils.bot_utils import _start_bot - from multiprocessing import Process, synchronize - - t, e = _start_bot(V2Bot, SERVER, 8888, "Private", "testrunner", "testpassword") - self.assertIsInstance(t, Process) - self.assertIsInstance(e, synchronize.Event) - # self.assertFalse(e.is_set()) - e.set() - timeout = time.time() + 10 - while e.is_set() and time.time() < timeout: - print("...") - time.sleep(2) - self.assertFalse(e.is_set()) - t.terminate() - self.assertFalse(t.is_alive()) - print("Joining...") - t.join() - - @pytest.mark.timeout(30) - def test_start_v3_bot(self): - from chatbot_core.utils.bot_utils import _start_bot - from multiprocessing import Process, synchronize - - t, e = _start_bot(V3Bot, SERVER, 8888, "Private", "testrunner", "testpassword") - self.assertIsInstance(t, Process) - self.assertIsInstance(e, synchronize.Event) - # self.assertFalse(e.is_set()) - e.set() - timeout = time.time() + 10 - while e.is_set() and time.time() < timeout: - print("...") - time.sleep(2) - self.assertFalse(e.is_set()) - t.terminate() - self.assertFalse(t.is_alive()) - print("Joining...") - t.join() - @pytest.mark.timeout(30) def test_messagebus_connection(self): - from chatbot_core.utils import init_message_bus + from chatbot_core.utils.bot_utils import init_message_bus from threading import Thread - from mycroft_bus_client import MessageBusClient + from ovos_bus_client.client import MessageBusClient t, bus = init_message_bus() self.assertIsInstance(t, Thread) diff --git a/version_bump.py b/tests/units/test_base_classes.py similarity index 61% rename from version_bump.py rename to tests/units/test_base_classes.py index a55f158..17e5265 100644 --- a/version_bump.py +++ b/tests/units/test_base_classes.py @@ -17,29 +17,33 @@ # US Patents 2008-2021: US7424516, US20140161250, US20140177813, US8638908, US8068604, US8553852, US10530923, US10530924 # China Patent: CN102017585 - Europe Patent: EU2156652 - Patents Pending -import fileinput -from os.path import join, dirname - -with open(join(dirname(__file__), "version.py"), "r", encoding="utf-8") as v: - for line in v.readlines(): - if line.startswith("__version__"): - if '"' in line: - version = line.split('"')[1] - else: - version = line.split("'")[1] - -if "a" not in version: - parts = version.split('.') - parts[-1] = str(int(parts[-1]) + 1) - version = '.'.join(parts) - version = f"{version}a0" -else: - post = version.split("a")[1] - new_post = int(post) + 1 - version = version.replace(f"a{post}", f"a{new_post}") - -for line in fileinput.input(join(dirname(__file__), "version.py"), inplace=True): - if line.startswith("__version__"): - print(f"__version__ = \"{version}\"") - else: - print(line.rstrip('\n')) +import unittest + + +class ChatBotV1Tests(unittest.TestCase): + from chatbot_core.v1 import ChatBot + # TODO + + +class ChatBotV2Tests(unittest.TestCase): + from chatbot_core.v2 import ChatBot + # TODO + + +class ChatBotABCTests(unittest.TestCase): + from chatbot_core.chatbot_abc import ChatBotABC + # TODO + + +class NeonTests(unittest.TestCase): + from chatbot_core.neon import NeonBot + # TODO + + +class ParlaiTests(unittest.TestCase): + from chatbot_core.parlai import ParlaiBot + # TODO + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/units/test_init.py b/tests/units/test_init.py new file mode 100644 index 0000000..98dd917 --- /dev/null +++ b/tests/units/test_init.py @@ -0,0 +1,80 @@ +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System +# +# Copyright 2008-2021 Neongecko.com Inc. | All Rights Reserved +# +# Notice of License - Duplicating this Notice of License near the start of any file containing +# a derivative of this software is a condition of license for this software. +# Friendly Licensing: +# No charge, open source royalty free use of the Neon AI software source and object is offered for +# educational users, noncommercial enthusiasts, Public Benefit Corporations (and LLCs) and +# Social Purpose Corporations (and LLCs). Developers can contact developers@neon.ai +# For commercial licensing, distribution of derivative works or redistribution please contact licenses@neon.ai +# Distributed on an "AS IS” basis without warranties or conditions of any kind, either express or implied. +# Trademarks of Neongecko: Neon AI(TM), Neon Assist (TM), Neon Communicator(TM), Klat(TM) +# Authors: Guy Daniels, Daniel McKnight, Regina Bloomstine, Elon Gasper, Richard Leeds +# +# Specialized conversational reconveyance options from Conversation Processing Intelligence Corp. +# US Patents 2008-2021: US7424516, US20140161250, US20140177813, US8638908, US8068604, US8553852, US10530923, US10530924 +# China Patent: CN102017585 - Europe Patent: EU2156652 - Patents Pending + +import os +import unittest +from unittest.mock import patch + + +class ChatbotCoreTests(unittest.TestCase): + def test_ChatBot_class(self): + from chatbot_core.v1 import ChatBot as V1 + from chatbot_core.v2 import ChatBot as V2 + + # Test valid v2 spec + os.environ["CHATBOT_VERSION"] = "v2" + from chatbot_core import ChatBot + self.assertEqual(ChatBot, V2, "Expected v2") + # sys.modules.pop("chatbot_core") + + # Test valid v1 spec + os.environ["CHATBOT_VERSION"] = "v1" + from chatbot_core import ChatBot + self.assertEqual(ChatBot, V1, "Expected v1") + # sys.modules.pop("chatbot_core") + + # Test default + os.environ.pop("CHATBOT_VERSION") + from chatbot_core import ChatBot + self.assertEqual(ChatBot, V1, "Expected v1") + + # Test invalid + os.environ["CHATBOT_VERSION"] = '3' + from chatbot_core import ChatBot + self.assertIsNone(ChatBot) + + @patch("chatbot_core.utils.bot_utils.grammar_check") + @patch("chatbot_core.utils.bot_utils.clean_up_bot") + @patch("chatbot_core.utils.bot_utils.generate_random_response") + def test_backwards_compat(self, random_response, clean_up, grammar): + import chatbot_core + + from chatbot_core.utils.enum import ConversationControls + self.assertEqual(chatbot_core.ConversationControls, + ConversationControls) + + from chatbot_core.utils.enum import ConversationState + self.assertEqual(chatbot_core.ConversationState, ConversationState) + + random_response.assert_not_called() + clean_up.assert_not_called() + grammar.assert_not_called() + + chatbot_core.generate_random_response("one", test=1) + random_response.assert_called_once_with("one", test=1) + + chatbot_core.clean_up_bot("two", test=2) + clean_up.assert_called_once_with("two", test=2) + + chatbot_core.grammar_check("three", test=3) + grammar.assert_called_once_with("three", test=3) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/units/test_utils.py b/tests/units/test_utils.py new file mode 100644 index 0000000..f9a06f7 --- /dev/null +++ b/tests/units/test_utils.py @@ -0,0 +1,191 @@ +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System +# +# Copyright 2008-2021 Neongecko.com Inc. | All Rights Reserved +# +# Notice of License - Duplicating this Notice of License near the start of any file containing +# a derivative of this software is a condition of license for this software. +# Friendly Licensing: +# No charge, open source royalty free use of the Neon AI software source and object is offered for +# educational users, noncommercial enthusiasts, Public Benefit Corporations (and LLCs) and +# Social Purpose Corporations (and LLCs). Developers can contact developers@neon.ai +# For commercial licensing, distribution of derivative works or redistribution please contact licenses@neon.ai +# Distributed on an "AS IS” basis without warranties or conditions of any kind, either express or implied. +# Trademarks of Neongecko: Neon AI(TM), Neon Assist (TM), Neon Communicator(TM), Klat(TM) +# Authors: Guy Daniels, Daniel McKnight, Regina Bloomstine, Elon Gasper, Richard Leeds +# +# Specialized conversational reconveyance options from Conversation Processing Intelligence Corp. +# US Patents 2008-2021: US7424516, US20140161250, US20140177813, US8638908, US8068604, US8553852, US10530923, US10530924 +# China Patent: CN102017585 - Europe Patent: EU2156652 - Patents Pending +import os +import unittest + + +class BotUtilsTests(unittest.TestCase): + def test_start_bots(self): + from chatbot_core.utils.bot_utils import start_bots + # Deprecated method + + def test_generate_random_response(self): + from chatbot_core.utils.bot_utils import generate_random_response + # Deprecated method + + def test_debug_bots(self): + from chatbot_core.utils.bot_utils import debug_bots + # TODO + + def test_clean_up_bot(self): + from chatbot_core.utils.bot_utils import clean_up_bot + # TODO + + def test_find_closest_answer(self): + from chatbot_core.utils.bot_utils import find_closest_answer + test_sentence = "This is a statement about testing your code." + options = {'1': "testing is good", + '2': "This is a statement about your code", + '3': "This is a statement about testing nothing"} + + random_resp = find_closest_answer('random', test_sentence, options) + self.assertIn(random_resp, options.keys()) + + bleu_response = find_closest_answer('bleu_score', test_sentence, + options) + self.assertEqual(bleu_response, '3', bleu_response) + + damerau_resp = find_closest_answer('damerau_levenshtein_distance', + test_sentence, options) + self.assertEqual(damerau_resp, '2', damerau_resp) + + self.assertIsNone(find_closest_answer(options=options)) + self.assertIsNone(find_closest_answer(sentence=test_sentence)) + self.assertIsNone(find_closest_answer("", test_sentence, options)) + self.assertIsNone(find_closest_answer("invalid", test_sentence, + options)) + + def test_grammar_check(self): + from chatbot_core.utils.bot_utils import grammar_check + # TODO + + def test_find_bot_modules(self): + from chatbot_core.utils.bot_utils import _find_bot_modules + # TODO + + def test_run_mq_bot(self): + from chatbot_core.utils.bot_utils import run_mq_bot + # TODO + + +class CacheTests(unittest.TestCase): + from chatbot_core.utils.cache import FIFOCache + cache_size = 3 + cache = FIFOCache(capacity=cache_size) + + def test_init(self): + from collections import OrderedDict + self.assertIsInstance(self.cache.cache, OrderedDict) + self.assertEqual(self.cache.capacity, self.cache_size) + + def test_put_get(self): + num_put = 10 + for i in range(num_put): + self.cache.put(str(i), str(i)) + self.assertEqual(len(self.cache.cache), self.cache.capacity) + for i in range(num_put): + if i < num_put - self.cache_size: + self.assertIsNone(self.cache.get(str(i)), self.cache.cache) + else: + self.assertEqual(self.cache.get(str(i)), str(i), + self.cache.cache) + + +class TestConversationUtils(unittest.TestCase): + def test_create_conversation_cycle(self): + from chatbot_core.utils.conversation_utils import create_conversation_cycle + from chatbot_core.utils.enum import ConversationState + convo = create_conversation_cycle() + self.assertEqual(next(convo), ConversationState.RESP) + self.assertEqual(next(convo), ConversationState.DISC) + self.assertEqual(next(convo), ConversationState.VOTE) + self.assertEqual(next(convo), ConversationState.PICK) + self.assertEqual(next(convo), ConversationState.IDLE) + self.assertEqual(next(convo), ConversationState.RESP) + + +class TestEnum(unittest.TestCase): + def test_conversation_controls(self): + from chatbot_core.utils.enum import ConversationControls + for c in (ConversationControls.RESP, ConversationControls.DISC, + ConversationControls.VOTE, ConversationControls.PICK, + ConversationControls.NEXT, ConversationControls.HIST, + ConversationControls.WAIT): + self.assertIsInstance(c, str) + + def test_conversation_state(self): + from chatbot_core.utils.enum import ConversationState + for state in ConversationState: + self.assertIsInstance(state.value, int) + + def test_bot_types(self): + from chatbot_core.utils.enum import BotTypes + for b in (BotTypes.PROCTOR, BotTypes.SUBMIND, BotTypes.OBSERVER): + self.assertIsInstance(b, str) + + def test_conversation_state_announcements(self): + from chatbot_core.utils.enum import ConversationState, \ + CONVERSATION_STATE_ANNOUNCEMENTS + for state in (ConversationState.RESP, ConversationState.DISC, + ConversationState.VOTE, ConversationState.PICK): + self.assertIsInstance(CONVERSATION_STATE_ANNOUNCEMENTS[state], str) + + +class LoggerTests(unittest.TestCase): + def test_make_logger(self): + from chatbot_core.utils.logger import make_logger + from ovos_utils.log import LOG + + # Simple named log + log = make_logger("test") + self.assertEqual(log, LOG) + self.assertEqual(LOG.name, "test") + + # Named log with level override + log = make_logger("test_2", "ERROR") + self.assertEqual(log, LOG) + self.assertEqual(LOG.name, "test_2") + self.assertEqual(LOG.level, "ERROR") + + +class StringUtilsTests(unittest.TestCase): + def test_remove_prefix(self): + from chatbot_core.utils.string_utils import remove_prefix + test_string = "This is a test..." + self.assertEqual(remove_prefix(f"@bot {test_string}", "@bot"), + test_string) + self.assertEqual(remove_prefix(f" {test_string}", '@'), + f" {test_string}") + self.assertEqual(remove_prefix(f"{test_string}{test_string}", + test_string), test_string) + + +class VersionUtilsTests(unittest.TestCase): + def test_get_class(self): + from chatbot_core.utils.version_utils import get_class + from chatbot_core.v1 import ChatBot as V1 + from chatbot_core.v2 import ChatBot as V2 + + # Explicit valid option + os.environ["CHATBOT_VERSION"] = "v1" + self.assertEqual(get_class(), V1) + os.environ["CHATBOT_VERSION"] = "v2" + self.assertEqual(get_class(), V2) + + # Default returns v1 + os.environ.pop("CHATBOT_VERSION") + self.assertEqual(get_class(), V1) + + # Invalid config returns None + os.environ["CHATBOT_VERSION"] = "0" + self.assertIsNone(get_class()) + + +if __name__ == '__main__': + unittest.main()