Added MQTT functionality to Rhasspy

kentloving · Jan 13, 2019 · c69c43e · c69c43e
1 parent 01a5933
commit c69c43e
Show file tree

Hide file tree

Showing 17 changed files with 392 additions and 65 deletions.
diff --git a/rhasspy/config.json b/rhasspy/config.json
@@ -1,7 +1,7 @@
 {
     "name": "Rhasspy Assistant",
     "slug": "rhasspy",
-    "version": "1.16",
+    "version": "1.17",
     "description": "Offline voice assistant for Home Assistant",
     "startup": "application",
     "boot": "auto",

diff --git a/rhasspy/dist/css/app.a705a611.css → rhasspy/dist/css/app.9d30c60f.css b/rhasspy/dist/css/app.a705a611.css → rhasspy/dist/css/app.9d30c60f.css
diff --git a/rhasspy/dist/index.html b/rhasspy/dist/index.html
@@ -1 +1 @@
-<!DOCTYPE html><html lang=en><head><meta charset=utf-8><meta http-equiv=X-UA-Compatible content="IE=edge"><meta name=viewport content="width=device-width,initial-scale=1"><link rel=icon href=/img/favicon.png><link rel=stylesheet href=/css/bootstrap.min.css><link rel=stylesheet href=/css/fontawesome-all.min.css><link rel=stylesheet href=/css/main.css><title>Rhasspy Voice Assistant</title><link href=/css/app.a705a611.css rel=preload as=style><link href=/js/app.44125db6.js rel=preload as=script><link href=/js/chunk-vendors.3d86561a.js rel=preload as=script><link href=/css/app.a705a611.css rel=stylesheet></head><body><noscript><strong>Please enable Javascript to continue.</strong></noscript><div id=app></div><script src=/js/jquery-3.3.1.slim.min.js></script><script src=/js/axios.min.js></script><script src=/js/vue-axios.min.js></script><script src=/js/bootstrap.min.js></script><script src=/js/chunk-vendors.3d86561a.js></script><script src=/js/app.44125db6.js></script></body></html>
+<!DOCTYPE html><html lang=en><head><meta charset=utf-8><meta http-equiv=X-UA-Compatible content="IE=edge"><meta name=viewport content="width=device-width,initial-scale=1"><link rel=icon href=/img/favicon.png><link rel=stylesheet href=/css/bootstrap.min.css><link rel=stylesheet href=/css/fontawesome-all.min.css><link rel=stylesheet href=/css/main.css><title>Rhasspy Voice Assistant</title><link href=/css/app.9d30c60f.css rel=preload as=style><link href=/js/app.810952c9.js rel=preload as=script><link href=/js/chunk-vendors.b44528b4.js rel=preload as=script><link href=/css/app.9d30c60f.css rel=stylesheet></head><body><noscript><strong>Please enable Javascript to continue.</strong></noscript><div id=app></div><script src=/js/jquery-3.3.1.slim.min.js></script><script src=/js/axios.min.js></script><script src=/js/vue-axios.min.js></script><script src=/js/bootstrap.min.js></script><script src=/js/chunk-vendors.b44528b4.js></script><script src=/js/app.810952c9.js></script></body></html>
diff --git a/rhasspy/dist/js/app.44125db6.js b/rhasspy/dist/js/app.44125db6.js
diff --git a/rhasspy/dist/js/app.44125db6.js.map b/rhasspy/dist/js/app.44125db6.js.map
diff --git a/rhasspy/dist/js/app.810952c9.js b/rhasspy/dist/js/app.810952c9.js
diff --git a/rhasspy/dist/js/app.810952c9.js.map b/rhasspy/dist/js/app.810952c9.js.map
diff --git a/rhasspy/dist/js/chunk-vendors.3d86561a.js.map b/rhasspy/dist/js/chunk-vendors.3d86561a.js.map
diff --git a/rhasspy/dist/js/chunk-vendors.3d86561a.js → rhasspy/dist/js/chunk-vendors.b44528b4.js b/rhasspy/dist/js/chunk-vendors.3d86561a.js → rhasspy/dist/js/chunk-vendors.b44528b4.js
diff --git a/rhasspy/dist/js/chunk-vendors.b44528b4.js.map b/rhasspy/dist/js/chunk-vendors.b44528b4.js.map
diff --git a/rhasspy/profiles/defaults.json b/rhasspy/profiles/defaults.json
@@ -26,9 +26,18 @@
     "microphone": {
         "system": "pyaudio"
     },
+    "mqtt": {
+        "enabled": false,
+        "host": "localhost",
+        "password": "",
+        "port": 1883,
+        "site_id": "default",
+        "username": "",
+        "reconnect_sec": 5
+    },
     "rhasspy": {
         "default_profile": "en",
-        "listen_on_start": false,
+        "listen_on_start": true,
         "preload_profile": true
     },
     "sounds": {
@@ -95,14 +104,9 @@
             "mllr_matrix": "wake_mllr",
             "threshold": 1e-30
         },
+        "hermes": {
+            "wakeword_id": "default"
+        },
         "system": "pocketsphinx"
-    },
-    "mqtt": {
-        "enabled": false,
-        "siteId": "default",
-        "host": "localhost",
-        "port": 1883,
-        "username": "",
-        "password": ""
     }
 }
diff --git a/rhasspy/rhasspy/__init__.py b/rhasspy/rhasspy/__init__.py
@@ -524,7 +524,7 @@ def test_wake(core, profile, args):
 
     # Instantiate wake listener
     wake_listener = PocketsphinxWakeListener(
-        audio_recorder=None, profile=profile, detected_callback=None)
+        core, audio_recorder=None, profile=profile, detected_callback=None)
 
     wake_listener.preload()
 
@@ -546,6 +546,7 @@ def test_wake(core, profile, args):
 
         done_event = threading.Event()
         audio_recorder = WavAudioRecorder(
+            core,
             wav_path,
             lambda wp: done_event.set())
 

diff --git a/rhasspy/rhasspy/audio_player.py b/rhasspy/rhasspy/audio_player.py
@@ -10,8 +10,9 @@
 
 class AudioPlayer:
     '''Base class for WAV based audio players'''
-    def __init__(self, device=None):
+    def __init__(self, core, device=None):
         '''Optional audio device handled by sub-classes.'''
+        self.core = core
         self.device = device
 
     def play_file(self, path: str):
@@ -57,3 +58,20 @@ def play_data(self, wav_data: bytes):
 
         # Play data
         subprocess.run(aplay_cmd, input=wav_data)
+
+# -----------------------------------------------------------------------------
+# MQTT audio player for Snips.AI Hermes Protocol
+# https://docs.snips.ai/ressources/hermes-protocol
+# -----------------------------------------------------------------------------
+
+class HeremesAudioPlayer(AudioPlayer):
+    '''Sends audio data over MQTT via Hermes protocol'''
+    def play_file(self, path: str):
+        if not os.path.exists(path):
+            return
+
+        with open(path, 'r') as wav_file:
+            self.play_data(wav_file.read())
+
+    def play_data(self, wav_data: bytes):
+        self.core.get_mqtt_client().play_bytes(wav_data)
diff --git a/rhasspy/rhasspy/audio_recorder.py b/rhasspy/rhasspy/audio_recorder.py
@@ -18,7 +18,8 @@
 
 class AudioRecorder:
     '''Base class for microphone audio recorders'''
-    def __init__(self, device=None):
+    def __init__(self, core, device=None):
+        self.core = core
         self.device = device
         self._is_recording = False
 
@@ -66,9 +67,9 @@ def is_recording(self):
 
 class PyAudioRecorder(AudioRecorder):
     '''Records from microphone using pyaudio'''
-    def __init__(self, device=None, frames_per_buffer=480):
+    def __init__(self, core, device=None, frames_per_buffer=480):
         # Frames per buffer is set to 30 ms for webrtcvad
-        AudioRecorder.__init__(self, device)
+        AudioRecorder.__init__(self, core, device)
         self.audio = None
         self.mic = None
         self.frames_per_buffer = frames_per_buffer
@@ -205,9 +206,9 @@ def get_microphones(self) -> Dict[Any, Any]:
 class ARecordAudioRecorder(AudioRecorder):
     '''Records from microphone using arecord'''
 
-    def __init__(self, device=None, chunk_size=480*2):
+    def __init__(self, core, device=None, chunk_size=480*2):
         # Chunk size is set to 30 ms for webrtcvad
-        AudioRecorder.__init__(self, device)
+        AudioRecorder.__init__(self, core, device)
 
         self.record_proc = None
         self.chunk_size = chunk_size
@@ -347,6 +348,7 @@ def get_microphones(self) -> Dict[Any, Any]:
                 name = line.strip()
 
         return mics
+
 # -----------------------------------------------------------------------------
 # WAV based audio "recorder"
 # -----------------------------------------------------------------------------
@@ -355,12 +357,13 @@ class WavAudioRecorder(AudioRecorder):
     '''Pushes WAV data out instead of data from a microphone.'''
 
     def __init__(self,
+                 core,
                  wav_path: str,
                  end_of_file_callback: Optional[Callable[[str], None]]=None,
                  chunk_size=480):
 
         # Chunk size set to 30 ms for webrtcvad
-        AudioRecorder.__init__(self, device=None)
+        AudioRecorder.__init__(self, core, device=None)
         self.wav_path = wav_path
         self.chunk_size = chunk_size
         self.end_of_file_callback = end_of_file_callback
@@ -471,3 +474,124 @@ def stop_all(self) -> None:
 
     def get_queue(self) -> Queue:
         return self.queue
+
+# -----------------------------------------------------------------------------
+# MQTT based audio "recorder" for Snips.AI Hermes Protocol
+# https://docs.snips.ai/ressources/hermes-protocol
+# -----------------------------------------------------------------------------
+
+class HermesAudioRecorder(AudioRecorder):
+    '''Receives audio data from MQTT via Hermes protocol.'''
+
+    def __init__(self, core, chunk_size=480*2):
+
+        # Chunk size set to 30 ms for webrtcvad
+        AudioRecorder.__init__(self, core, device=None)
+        self.chunk_size = chunk_size
+
+        self.buffer = bytes()
+        self.buffer_users = 0
+
+        self.queue = Queue()
+        self.queue_users = 0
+
+    # -------------------------------------------------------------------------
+
+    def start_recording(self, start_buffer: bool, start_queue: bool, device: Any=None):
+        # Allow multiple "users" to listen for audio data
+        if start_buffer:
+            self.buffer_users += 1
+
+        if start_queue:
+            self.queue_users += 1
+
+        if not self.is_recording:
+            # Reset
+            self.buffer = bytes()
+
+            # Clear queue
+            while not self.queue.empty():
+                self.queue.get_nowait()
+
+            def process_data():
+                with wave.open(self.wav_path, 'rb') as wav_file:
+                    rate, width, channels = wav_file.getframerate(), wav_file.getsampwidth(), wav_file.getnchannels()
+                    if (rate != 16000) or (width != 2) or (channels != 1):
+                        audio_data = SpeechDecoder.convert_wav(wav_file.read())
+                    else:
+                        # Use original data
+                        audio_data = wav_file.readframes(wav_file.getnframes())
+
+                i = 0
+                while (i+self.chunk_size) < len(audio_data):
+                    data = audio_data[i:i+self.chunk_size]
+                    i += self.chunk_size
+
+                    if self.buffer_users > 0:
+                        self.buffer += data
+
+                    if self.queue_users > 0:
+                        self.queue.put(data)
+
+                if self.end_of_file_callback is not None:
+                    self.end_of_file_callback(self.wav_path)
+
+            # Start recording
+            self._is_recording = True
+            self.record_thread = threading.Thread(target=process_data, daemon=True)
+            self.record_thread.start()
+
+            logger.debug('Recording from microphone')
+
+    # -------------------------------------------------------------------------
+
+    def stop_recording(self, stop_buffer: bool, stop_queue: bool) -> bytes:
+        if stop_buffer:
+            self.buffer_users = max(0, self.buffer_users - 1)
+
+        if stop_queue:
+            self.queue_users = max(0, self.queue_users - 1)
+
+        # Only stop if all "users" have disconnected
+        if self.is_recording and (self.buffer_users <= 0) and (self.queue_users <= 0):
+            # Shut down audio system
+            self._is_recording = False
+            self.record_thread.join()
+
+            logger.debug('Stopped recording from microphone')
+
+            # Write final empty buffer
+            self.queue.put(bytes())
+
+        if stop_buffer:
+            # Return WAV data
+            with io.BytesIO() as wav_buffer:
+                with wave.open(wav_buffer, mode='wb') as wav_file:
+                    wav_file.setframerate(16000)
+                    wav_file.setsampwidth(2)
+                    wav_file.setnchannels(1)
+                    wav_file.writeframesraw(self.buffer)
+
+                return wav_buffer.getvalue()
+
+        # Empty buffer
+        return bytes()
+
+    # -------------------------------------------------------------------------
+
+    def stop_all(self) -> None:
+        if self.is_recording:
+            self._is_recording = False
+            self.record_thread.join()
+
+            if self.queue_users > 0:
+                # Write final empty buffer
+                self.queue.put(bytes())
+
+            self.buffer_users = 0
+            self.queue_users = 0
+
+    # -------------------------------------------------------------------------
+
+    def get_queue(self) -> Queue:
+        return self.queue
diff --git a/rhasspy/rhasspy/core.py b/rhasspy/rhasspy/core.py
@@ -95,9 +95,17 @@ def get_default(self, path: str, default=None):
     def get_audio_player(self) -> AudioPlayer:
         '''Gets the shared audio player'''
         if self.audio_player is None:
-            from audio_player import APlayAudioPlayer
-            device = self.get_default('sounds.aplay.device')
-            self.audio_player = APlayAudioPlayer(device)
+            system = self.get_default('sounds.system', 'dummy')
+            assert system in ['aplay', 'hermes', 'dummy'], 'Unknown sound system: %s' % system
+            if system == 'aplay':
+                from audio_player import APlayAudioPlayer
+                device = self.get_default('sounds.aplay.device')
+                self.audio_player = APlayAudioPlayer(self, device)
+            elif system == 'heremes':
+                from audio_player import HeremesAudioPlayer
+                self.audio_player = HeremesAudioPlayer(self)
+            elif system == 'dummy':
+                self.audio_player = AudioPlayer(self)
 
         return self.audio_player
 
@@ -108,17 +116,21 @@ def get_audio_recorder(self) -> AudioRecorder:
         if self.audio_recorder is None:
             # Determine which microphone system to use
             system = self.default_profile.get('microphone.system')
-            assert system in ['arecord', 'pyaudio'], 'Unknown microphone system: %s' % system
+            assert system in ['arecord', 'pyaudio', 'hermes'], 'Unknown microphone system: %s' % system
             if system == 'arecord':
                 from audio_recorder import ARecordAudioRecorder
                 device = self.get_default('microphone.arecord.device')
-                self.audio_recorder = ARecordAudioRecorder(device)
+                self.audio_recorder = ARecordAudioRecorder(self, device)
                 logger.debug('Using arecord for microphone')
             elif system == 'pyaudio':
                 from audio_recorder import PyAudioRecorder
                 device = self.get_default('microphone.pyaudio.device')
-                self.audio_recorder = PyAudioRecorder(device)
+                self.audio_recorder = PyAudioRecorder(self, device)
                 logger.debug('Using PyAudio for microphone')
+            elif system == 'hermes':
+                from audio_recorder import HermesAudioRecorder
+                self.audio_recorder = HermesAudioRecorder(self, device)
+                logger.debug('Using Hermes for microphone')
 
         return self.audio_recorder
 
@@ -236,20 +248,25 @@ def get_wake_listener(self, profile_name: str,
             callback = callback or self._handle_wake
             profile = self.profiles[profile_name]
             system = profile.get('wake.system')
-            assert system in ['dummy', 'pocketsphinx', 'nanomsg'], 'Invalid wake system: %s' % system
+            assert system in ['dummy', 'pocketsphinx', 'nanomsg', 'hermes'], 'Invalid wake system: %s' % system
             if system == 'pocketsphinx':
                 # Use pocketsphinx locally
                 from wake import PocketsphinxWakeListener
                 wake = PocketsphinxWakeListener(
-                    self.get_audio_recorder(), profile, callback)
+                    self, self.get_audio_recorder(), profile, callback)
             elif system == 'nanomsg':
                 # Use remote system via nanomsg
                 from wake import NanomsgWakeListener
                 wake = NanomsgWakeListener(
-                    self.get_audio_recorder(), profile, callback)
+                    self, self.get_audio_recorder(), profile, callback)
+            elif system == 'hermes':
+                # Use remote system via MQTT
+                from wake import HermesWakeListener
+                wake = HermesWakeListener(
+                    self, self.get_audio_recorder(), profile)
             elif system == 'dummy':
                 # Does nothing
-                wake = WakeListener(self.get_audio_recorder(), profile)
+                wake = WakeListener(self, self.get_audio_recorder(), profile)
 
             self.wake_listeners[profile_name] = wake
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		<!DOCTYPE html><html lang=en><head><meta charset=utf-8><meta http-equiv=X-UA-Compatible content="IE=edge"><meta name=viewport content="width=device-width,initial-scale=1"><link rel=icon href=/img/favicon.png><link rel=stylesheet href=/css/bootstrap.min.css><link rel=stylesheet href=/css/fontawesome-all.min.css><link rel=stylesheet href=/css/main.css><title>Rhasspy Voice Assistant</title><link href=/css/app.a705a611.css rel=preload as=style><link href=/js/app.44125db6.js rel=preload as=script><link href=/js/chunk-vendors.3d86561a.js rel=preload as=script><link href=/css/app.a705a611.css rel=stylesheet></head><body><noscript><strong>Please enable Javascript to continue.</strong></noscript><div id=app></div><script src=/js/jquery-3.3.1.slim.min.js></script><script src=/js/axios.min.js></script><script src=/js/vue-axios.min.js></script><script src=/js/bootstrap.min.js></script><script src=/js/chunk-vendors.3d86561a.js></script><script src=/js/app.44125db6.js></script></body></html>
		<!DOCTYPE html><html lang=en><head><meta charset=utf-8><meta http-equiv=X-UA-Compatible content="IE=edge"><meta name=viewport content="width=device-width,initial-scale=1"><link rel=icon href=/img/favicon.png><link rel=stylesheet href=/css/bootstrap.min.css><link rel=stylesheet href=/css/fontawesome-all.min.css><link rel=stylesheet href=/css/main.css><title>Rhasspy Voice Assistant</title><link href=/css/app.9d30c60f.css rel=preload as=style><link href=/js/app.810952c9.js rel=preload as=script><link href=/js/chunk-vendors.b44528b4.js rel=preload as=script><link href=/css/app.9d30c60f.css rel=stylesheet></head><body><noscript><strong>Please enable Javascript to continue.</strong></noscript><div id=app></div><script src=/js/jquery-3.3.1.slim.min.js></script><script src=/js/axios.min.js></script><script src=/js/vue-axios.min.js></script><script src=/js/bootstrap.min.js></script><script src=/js/chunk-vendors.b44528b4.js></script><script src=/js/app.810952c9.js></script></body></html>