Refactoring and Unit Tests

.github/workflows/propose_release.yml

@@ -16,6 +16,7 @@ jobs:
       branch: dev
       release_type: ${{ inputs.release_type }}
       update_changelog: True
+      version_file: "chatbot_core/version.py"
   pull_changes:
     uses: neongeckocom/.github/.github/workflows/pull_master.yml@master
     needs: update_version

.github/workflows/publish_test_build.yml

@@ -13,5 +13,5 @@ jobs:
     uses: neongeckocom/.github/.github/workflows/publish_alpha_release.yml@master
     secrets: inherit
     with:
-      version_file: "version.py"
+      version_file: "chatbot_core/version.py"
       publish_prerelease: true

.github/workflows/unit_tests.yml

@@ -7,6 +7,33 @@ on:
 
 jobs:
   unit_tests:
+    strategy:
+      matrix:
+        python-version: [ 3.7, 3.8, 3.9, '3.10', '3.11']
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install .[lang] -r requirements/test_requirements.txt
+        env:
+          GITHUB_TOKEN: ${{secrets.neon_token}}
+      - name: Test with pytest
+        run: |
+          pip install pytest pytest-timeout pytest-cov
+          pytest tests/units --doctest-modules --junitxml=tests/unit-test-results.xml
+      - name: Upload pytest unit test results
+        uses: actions/upload-artifact@v2
+        with:
+          name: unit-test-results
+          path: tests/unit-test-results.xml
+  integration_tests:
     strategy:
       matrix:
         python-version: [ 3.7, 3.8, 3.9 ]
@@ -23,16 +50,15 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install -r requirements/test_requirements.txt
-          pip install .
+          pip install . -r requirements/test_requirements.txt
         env:
           GITHUB_TOKEN: ${{secrets.neon_token}}
       - name: Test with pytest
         run: |
           pip install pytest pytest-timeout pytest-cov
-          pytest tests/chatbot_core_tests.py --doctest-modules --junitxml=tests/test-results.xml
-      - name: Upload pytest test results
+          pytest tests/integration --doctest-modules --junitxml=tests/integration-test-results.xml
+      - name: Upload integration test results
         uses: actions/upload-artifact@v2
         with:
-          name: test-results
-          path: tests/test-results.xml
+          name: integration-test-results
+          path: tests/integration-test-results.xml

chatbot_core/__init__.py

@@ -16,9 +16,51 @@
 # Specialized conversational reconveyance options from Conversation Processing Intelligence Corp.
 # US Patents 2008-2021: US7424516, US20140161250, US20140177813, US8638908, US8068604, US8553852, US10530923, US10530924
 # China Patent: CN102017585  -  Europe Patent: EU2156652  -  Patents Pending
-from chatbot_core.utils import generate_random_response, clean_up_bot,\
-    grammar_check, ConversationControls, ConversationState
-from chatbot_core.utils.version_utils import get_class
-from chatbot_core.version_mapping import CHATBOT_VERSIONS
 
-ChatBot = get_class(CHATBOT_VERSIONS)
+from ovos_utils.log import LOG, log_deprecation
+from os import environ
+from neon_utils.decorators import module_property
+
+
+@module_property
+def _ChatBot():
+    LOG.debug(f"Getting class for {environ.get('CHATBOT_VERSION')}")
+    from chatbot_core.utils.version_utils import get_class
+    return get_class()
+
+
+@module_property
+def _ConversationControls():
+    log_deprecation("import from `chatbot_core.utils.enum` directly",
+                    "3.0.0")
+    from chatbot_core.utils.enum import ConversationControls
+    return ConversationControls
+
+
+@module_property
+def _ConversationState():
+    log_deprecation("import from `chatbot_core.utils.enum` directly",
+                    "3.0.0")
+    from chatbot_core.utils.enum import ConversationState
+    return ConversationState
+
+
+def generate_random_response(*args, **kwargs):
+    log_deprecation("import from `chatbot_core.utils.bot_utils` directly",
+                    "3.0.0")
+    from chatbot_core.utils.bot_utils import generate_random_response
+    return generate_random_response(*args, **kwargs)
+
+
+def clean_up_bot(*args, **kwargs):
+    log_deprecation("import from `chatbot_core.utils.bot_utils` directly",
+                    "3.0.0")
+    from chatbot_core.utils.bot_utils import clean_up_bot
+    return clean_up_bot(*args, **kwargs)
+
+
+def grammar_check(*args, **kwargs):
+    log_deprecation("import from `chatbot_core.utils.bot_utils` directly",
+                    "3.0.0")
+    from chatbot_core.utils.bot_utils import grammar_check
+    return grammar_check(*args, **kwargs)

chatbot_core/chatbot_abc.py

@@ -18,14 +18,14 @@
 # China Patent: CN102017585  -  Europe Patent: EU2156652  -  Patents Pending
 
 import random
+import time
 
 from abc import ABC, abstractmethod
 from queue import Queue
 from typing import Optional
 
-from chatbot_core.utils import *
-
-LOG = make_logger("chatbot")
+from neon_utils.log_utils import init_log
+from ovos_utils.log import LOG
 
 
 class ChatBotABC(ABC):
@@ -38,10 +38,7 @@ def __init__(self):
     @property
     def log(self):
         if not self.__log:
-            global LOG
-            self.__log = make_logger(self.__class__.__name__)
-            self.__log.setLevel(LOG.level)
-            LOG = self.__log
+            self.__log = init_log(log_name=self.__class__.__name__)
         return self.__log
 
     @abstractmethod

chatbot_core/cli.py

@@ -0,0 +1,199 @@
+# NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System
+#
+# Copyright 2008-2021 Neongecko.com Inc. | All Rights Reserved
+#
+# Notice of License - Duplicating this Notice of License near the start of any file containing
+# a derivative of this software is a condition of license for this software.
+# Friendly Licensing:
+# No charge, open source royalty free use of the Neon AI software source and object is offered for
+# educational users, noncommercial enthusiasts, Public Benefit Corporations (and LLCs) and
+# Social Purpose Corporations (and LLCs). Developers can contact [email protected]
+# For commercial licensing, distribution of derivative works or redistribution please contact [email protected]
+# Distributed on an "AS IS” basis without warranties or conditions of any kind, either express or implied.
+# Trademarks of Neongecko: Neon AI(TM), Neon Assist (TM), Neon Communicator(TM), Klat(TM)
+# Authors: Guy Daniels, Daniel McKnight, Regina Bloomstine, Elon Gasper, Richard Leeds
+#
+# Specialized conversational reconveyance options from Conversation Processing Intelligence Corp.
+# US Patents 2008-2021: US7424516, US20140161250, US20140177813, US8638908, US8068604, US8553852, US10530923, US10530924
+# China Patent: CN102017585  -  Europe Patent: EU2156652  -  Patents Pending
+
+import os
+import argparse
+import click
+
+from os.path import expanduser, relpath
+from ovos_utils.log import LOG, log_deprecation
+
+
+@click.group("chatbots", cls=click.Group,
+             no_args_is_help=True, invoke_without_command=True,
+             help="Chatbot Core Commands\n\n"
+                  "See also: chatbots COMMAND --help")
+@click.option("--version", "-v", is_flag=True, required=False,
+              help="Print the current version")
+def chatbot_core_cli(version: bool = False):
+    if version:
+        from chatbot_core.version import __version__
+        click.echo(f"chatbot-core version {__version__}")
+
+
+@chatbot_core_cli.command(help="Start an MQ chatbot")
+@click.argument("bot_entrypoint")
+def start_mq_bot(bot_entrypoint):
+    os.environ['CHATBOT_VERSION'] = 'v2'
+    from chatbot_core.utils.bot_utils import run_mq_bot
+    run_mq_bot(bot_entrypoint)
+
+
+@chatbot_core_cli.command(help="Start a local debug session")
+@click.option("--bot-dir", default=None, help="Path to legacy chatbots directory")
+def debug_bots(bot_dir):
+    from chatbot_core.utils.bot_utils import debug_bots
+    bot_dir = expanduser(relpath(bot_dir)) if bot_dir else None
+    debug_bots(bot_dir)
+
+
+# Below are deprecated entrypoints
+def cli_start_mq_bot():
+    """
+    Entrypoint to start an MQ chatbot
+    """
+    os.environ['CHATBOT_VERSION'] = 'v2'
+    from chatbot_core.utils.bot_utils import run_mq_bot
+    log_deprecation("Use `chatbots start-mq-bot`", "3.0.0")
+    parser = argparse.ArgumentParser(description="Start a chatbot")
+    parser.add_argument("--bot", dest="bot_name",
+                        help="Chatbot entrypoint name", type=str)
+    args = parser.parse_args()
+    run_mq_bot(args.bot_name)
+
+
+def cli_start_bots():
+    """
+    Entry Point to start bots from a Console Script
+    """
+    from chatbot_core.utils.bot_utils import SERVER, start_bots
+    log_deprecation("This CLI command is deprecated", "3.0.0")
+
+    parser = argparse.ArgumentParser(description="Start some chatbots")
+    parser.add_argument("--domain", dest="domain", default="chatbotsforum.org",
+                        help="Domain to connect to "
+                             "(default: chatbotsforum.org)", type=str)
+    parser.add_argument("--dir", dest="bot_dir",
+                        help="Path to chatbots (default: ./)", type=str)
+    parser.add_argument("--bot", dest="bot_name",
+                        help="Optional bot name to run a single bot only",
+                        type=str)
+    parser.add_argument("--credentials", dest="cred_file",
+                        help="Optional path to YAML credentials", type=str)
+    parser.add_argument("--username", dest="username",
+                        help="Klat username for a single bot", type=str)
+    parser.add_argument("--password", dest="password",
+                        help="Klat password for a single bot", type=str)
+    parser.add_argument("--server", dest="server", default=SERVER,
+                        help=f"Klat server (default: {SERVER})", type=str)
+    parser.add_argument("--debug", dest="debug", action='store_true',
+                        help="Enable more verbose log output")
+    parser.add_argument("--bot-names", dest="bot-names",
+                        help="comma separated list of bots to run", type=str)
+    parser.add_argument("--exclude", dest="exclude",
+                        help="comma separated list of bots to exclude "
+                             "from running", type=str)
+    parser.add_argument("--handle-restart", dest="handle_restart",
+                        default=False,
+                        help="True to handle server emit to restart bots",
+                        type=bool)
+    args = parser.parse_args()
+
+    if args.debug:
+        LOG.set_level("DEBUG")
+
+    if args.exclude:
+        excluded_bots = [name.strip() for name in args.exclude.split(",")]
+    else:
+        excluded_bots = None
+    LOG.debug(args)
+    start_bots(args.domain, args.bot_dir, args.username, args.password,
+               args.server, args.cred_file, args.bot_name,
+               excluded_bots, args.handle_restart)
+
+
+def cli_stop_bots():
+    """
+    Stops all start-klat-bot instances
+    """
+    from time import sleep
+    log_deprecation("This CLI command is deprecated", "3.0.0")
+
+    parser = argparse.ArgumentParser(description="Stop some chatbots")
+    parser.add_argument("--server", dest="server", default="",
+                        help=f"Klat server (default: None)", type=str)
+    args = parser.parse_args()
+    if args.server:
+        server_to_stop = args.server
+    else:
+        server_to_stop = None
+    import psutil
+
+    procs = {p.pid: p.info for p in psutil.process_iter(['name'])}
+    for pid, name in procs.items():
+        if name.get("name") == "start-klat-bots" and \
+                (not server_to_stop or f"--server={server_to_stop}" in
+                 psutil.Process(pid).cmdline()):
+            LOG.info(f"Terminating {pid}")
+            psutil.Process(pid).terminate()
+            sleep(1)
+            if psutil.pid_exists(pid) and psutil.Process(pid).is_running():
+                LOG.error(f"Process {pid} not terminated!!")
+                psutil.Process(pid).kill()
+
+
+def cli_start_prompter():
+    """
+    Entry Point to start a prompter bot
+    """
+    log_deprecation("This CLI command is deprecated", "3.0.0")
+
+    from chatbot_core.utils.bot_utils import SERVER, start_bots
+    parser = argparse.ArgumentParser(description="Start a prompter chatbot")
+    parser.add_argument("--bot", dest="bot_name",
+                        help="Optional bot name to run a single bot only", type=str)
+    parser.add_argument("--dir", dest="bot_dir",
+                        help="Path to chatbots (default: ./)", type=str)
+    parser.add_argument("--username", dest="username",
+                        help="Klat username for a single bot", type=str)
+    parser.add_argument("--password", dest="password",
+                        help="Klat password for a single bot", type=str)
+    parser.add_argument("--server", dest="server", default=SERVER,
+                        help=f"Klat server (default: {SERVER})", type=str)
+    parser.add_argument("--debug", dest="debug", action='store_true',
+                        help="Enable more verbose log output")
+    parser.add_argument("--handle-restart", dest="handle_restart", default=False,
+                        help="True to handle server emit to restart bots", type=bool)
+    args = parser.parse_args()
+
+    if args.debug:
+        LOG.level = "DEBUG"
+    LOG.debug(args)
+    start_bots("chatbotsforum.org", args.bot_dir, args.username, args.password,
+               args.server, None, args.bot_name, None, args.handle_restart,
+               True)
+
+
+def cli_debug_bots():
+    """
+    Debug bots in the passed directory
+    :param bot_dir: directory containing the bot to test
+    """
+    import sys
+    from chatbot_core.utils.bot_utils import debug_bots
+
+    log_deprecation("Use `chatbots debug-bots`", "3.0.0")
+
+    bot_dir = os.getcwd()
+    # Try handling passed directory
+    if len(sys.argv) > 1:
+        arg_dir = os.path.expanduser(sys.argv[1])
+        bot_dir = arg_dir if os.path.exists(arg_dir) else bot_dir
+
+    debug_bots(bot_dir)

chatbot_core/neon.py

@@ -16,14 +16,16 @@
 # Specialized conversational reconveyance options from Conversation Processing Intelligence Corp.
 # US Patents 2008-2021: US7424516, US20140161250, US20140177813, US8638908, US8068604, US8553852, US10530923, US10530924
 # China Patent: CN102017585  -  Europe Patent: EU2156652  -  Patents Pending
+
 import os
 import time
-from typing import Optional
 
-from mycroft_bus_client import Message, MessageBusClient
-from neon_utils import LOG
+from typing import Optional
+from ovos_bus_client import Message, MessageBusClient
+from ovos_utils.log import LOG
 
-from chatbot_core.utils import BotTypes, init_message_bus
+from chatbot_core.utils.enum import BotTypes
+from chatbot_core.utils.bot_utils import init_message_bus
 from chatbot_core import ChatBot
 
 if os.environ.get('CHATBOT_VERSION', 'v1') == 'v2':