uni-bremen-agst · SarahAugustinowski · Jan 12, 2025
diff --git a/.gitattributes b/.gitattributes
@@ -37,3 +37,5 @@ Assets/Plugins/CC_Unity_Tools.meta filter=lfs diff=lfs merge=lfs -text
 Assets/Resources/Materials/CC4/** filter=lfs diff=lfs merge=lfs -text
 Assets/Resources/Materials/CC4.meta filter=lfs diff=lfs merge=lfs -text
 Assets/StreamingAssets/HelpSystem/Videos/** filter=lfs diff=lfs merge=lfs -text
+Assets/StreamingAssets/Whisper/** filter=lfs diff=lfs merge=lfs -text
+Assets/StreamingAssets/Rasa/** filter=lfs diff=lfs merge=lfs -text
diff --git a/Assets/Plugins/Dissonance/Core/NAudio/WaveFileWriter.cs b/Assets/Plugins/Dissonance/Core/NAudio/WaveFileWriter.cs
diff --git a/Assets/SEE/Controls/Actions/ActionStateType.cs b/Assets/SEE/Controls/Actions/ActionStateType.cs
@@ -1,4 +1,5 @@
 using SEE.Utils.History;
+using System.Collections.Generic;
 using UnityEngine;
 
 namespace SEE.Controls.Actions
@@ -9,6 +10,9 @@ namespace SEE.Controls.Actions
     /// </summary>
     public class ActionStateType : AbstractActionStateType
     {
+        // A static dictionary to hold all registered ActionStateType instances by name
+        private static readonly Dictionary<string, ActionStateType> actionStateTypes = new Dictionary<string, ActionStateType>();
+
         /// <summary>
         /// Delegate to be called to create a new instance of this kind of action.
         /// May be null if none needs to be created (in which case this delegate will not be called).
@@ -33,6 +37,21 @@ public ActionStateType(string name, string description,
             : base(name, description, color, icon, parent, register)
         {
             CreateReversible = createReversible;
+
+            if (register && !actionStateTypes.ContainsKey(name))
+            {
+                actionStateTypes[name] = this;
+            }
+        }
+
+        public static ActionStateType GetActionStateTypeByName(string name)
+        {
+            if (actionStateTypes.TryGetValue(name, out var actionStateType))
+            {
+                return actionStateType;
+            }
+
+            return null; // Return null if not found
         }
 
         #region Equality & Comparators

diff --git a/Assets/SEE/Controls/KeyActions/KeyAction.cs b/Assets/SEE/Controls/KeyActions/KeyAction.cs
@@ -230,6 +230,10 @@ internal enum KeyAction
         /// <summary>
         /// Opens/closes the drawable manager view.
         /// </summary>
-        DrawableManagerView
+        DrawableManagerView,
+        /// <summary>
+        /// This is to load the graphdatabase for RASA
+        /// </summary>
+        LoadDB
     }
 }
diff --git a/Assets/SEE/Controls/KeyActions/KeyBindings.cs b/Assets/SEE/Controls/KeyActions/KeyBindings.cs
@@ -128,6 +128,9 @@ static KeyBindings()
         {
             // Note: The order of the key actions is important. They are displayed to the
             // user in the order of appearance here.
+            // only for testing
+            Register(KeyAction.LoadDB, KeyCode.F3, "Loads the Database for testing",
+                     KeyActionCategory.General, "Load database"); // TODO should be done via UI
 
             // General
             Register(KeyAction.Help, KeyCode.H, "Help",

diff --git a/Assets/SEE/Controls/SEEInput.cs b/Assets/SEE/Controls/SEEInput.cs
@@ -3,6 +3,7 @@
 using UnityEngine;
 using SEE.GO;
 using SEE.XR;
+using SEE.Controls.VoiceActions;
 
 namespace SEE.Controls
 {
@@ -46,8 +47,15 @@ private static void ResetKeyboardShortcutsEnabled()
         /// <returns>true if the user requests this action and <see cref="KeyboardShortcutsEnabled"/></returns>
         public static bool Help()
         {
-            return KeyboardShortcutsEnabled
-                && KeyBindings.IsDown(KeyAction.Help);
+            return (KeyboardShortcutsEnabled
+                && KeyBindings.IsDown(KeyAction.Help)) || VoiceBindings.isSaid(VoiceAction.Help);
+        }
+
+        //only for testing purpose
+        public static bool LoadDB()
+        {
+            return (KeyboardShortcutsEnabled
+                && KeyBindings.IsDown(KeyAction.LoadDB));
         }
 
         /// <summary>
@@ -56,8 +64,8 @@ public static bool Help()
         /// <returns>true if the user requests this action and <see cref="KeyboardShortcutsEnabled"/></returns>
         public static bool ToggleVoiceControl()
         {
-            return KeyboardShortcutsEnabled
-                && KeyBindings.IsDown(KeyAction.ToggleVoiceControl);
+            return (KeyboardShortcutsEnabled
+                && KeyBindings.IsPressed(KeyAction.ToggleVoiceControl));
         }
 
         /// <summary>
@@ -66,8 +74,8 @@ public static bool ToggleVoiceControl()
         /// <returns>true if the user requests this action and <see cref="KeyboardShortcutsEnabled"/></returns>
         public static bool ToggleMenu()
         {
-            return KeyboardShortcutsEnabled
-                && KeyBindings.IsDown(KeyAction.ToggleMenu);
+            return (KeyboardShortcutsEnabled
+                && KeyBindings.IsDown(KeyAction.ToggleMenu)) || VoiceBindings.isSaid(VoiceAction.ToggleMenu);
         }
 
         /// <summary>
@@ -76,7 +84,8 @@ public static bool ToggleMenu()
         /// <returns>true if the user requests this action and <see cref="KeyboardShortcutsEnabled"/></returns>
         public static bool ToggleSettings()
         {
-            return KeyboardShortcutsEnabled && KeyBindings.IsDown(KeyAction.ToggleSettings);
+            return (KeyboardShortcutsEnabled
+                && KeyBindings.IsDown(KeyAction.ToggleSettings)) || VoiceBindings.isSaid(VoiceAction.ToggleSettings);
         }
 
         /// <summary>
@@ -85,7 +94,8 @@ public static bool ToggleSettings()
         /// <returns>true if the user requests this action and <see cref="KeyboardShortcutsEnabled"/></returns>
         public static bool ToggleBrowser()
         {
-            return KeyboardShortcutsEnabled && KeyBindings.IsDown(KeyAction.ToggleBrowser);
+            return (KeyboardShortcutsEnabled
+                && KeyBindings.IsDown(KeyAction.ToggleBrowser)) || VoiceBindings.isSaid(VoiceAction.ToggleBrowser);
         }
 
         /// <summary>
@@ -94,7 +104,18 @@ public static bool ToggleBrowser()
         /// <returns>true if the user requests this action and <see cref="KeyboardShortcutsEnabled"/></returns>
         public static bool ToggleMirror()
         {
-            return KeyboardShortcutsEnabled && KeyBindings.IsDown(KeyAction.ToggleMirror);
+            return (KeyboardShortcutsEnabled
+                && KeyBindings.IsDown(KeyAction.ToggleMirror)) || VoiceBindings.isSaid(VoiceAction.ToggleMirror);
+        }
+
+        /// <summary>
+        /// Opens/closes the search menu.
+        /// </summary>
+        /// <returns>true if the user requests this action and <see cref="KeyboardShortcutsEnabled"/></returns>
+        internal static bool ToggleSearch()
+        {
+            return (KeyboardShortcutsEnabled
+                && KeyBindings.IsDown(KeyAction.ToggleSettings)) || VoiceBindings.isSaid(VoiceAction.ToggleSettings);
         }
 
         /// <summary>
@@ -125,10 +146,10 @@ public static bool Undo()
             }
 #if UNITY_EDITOR == false
             // Ctrl keys are not available when running the game in the editor
-            if (Input.GetKey(KeyCode.LeftControl) || Input.GetKey(KeyCode.RightControl))
+            if (Input.GetKey(KeyCode.LeftControl) || Input.GetKey(KeyCode.RightControl)  || VoiceBindings.isSaid(VoiceAction.Undo))
             {
 #endif
-            return KeyboardShortcutsEnabled && KeyBindings.IsDown(KeyAction.Undo);
+            return KeyboardShortcutsEnabled && (KeyBindings.IsDown(KeyAction.Undo) || VoiceBindings.isSaid(VoiceAction.Undo));
 #if UNITY_EDITOR == false
             }
             else
@@ -152,10 +173,10 @@ public static bool Redo()
             }
 #if UNITY_EDITOR == false
             // Ctrl keys are not available when running the game in the editor
-            if (Input.GetKey(KeyCode.LeftControl) || Input.GetKey(KeyCode.RightControl))
+            if (Input.GetKey(KeyCode.LeftControl) || Input.GetKey(KeyCode.RightControl) || VoiceBindings.isSaid(VoiceAction.Redo))
             {
 #endif
-            return KeyboardShortcutsEnabled && KeyBindings.IsDown(KeyAction.Redo);
+            return KeyboardShortcutsEnabled && (KeyBindings.IsDown(KeyAction.Redo) || VoiceBindings.isSaid(VoiceAction.Redo));
 #if UNITY_EDITOR == false
             }
             else
@@ -172,7 +193,8 @@ public static bool Redo()
         /// <returns>true if the user wants to toggle the run-time configuration menu</returns>
         internal static bool ToggleConfigMenu()
         {
-            return KeyboardShortcutsEnabled & KeyBindings.IsDown(KeyAction.ConfigMenu);
+            return (KeyboardShortcutsEnabled
+                && KeyBindings.IsDown(KeyAction.ConfigMenu)) || VoiceBindings.isSaid(VoiceAction.ConfigMenu);
         }
 
         /// <summary>
@@ -181,7 +203,8 @@ internal static bool ToggleConfigMenu()
         /// <returns>true if the user requests this action and <see cref="KeyboardShortcutsEnabled"/></returns>
         internal static bool ToggleEdges()
         {
-            return KeyboardShortcutsEnabled && KeyBindings.IsDown(KeyAction.ToggleEdges);
+            return (KeyboardShortcutsEnabled
+                && KeyBindings.IsDown(KeyAction.ToggleEdges)) || VoiceBindings.isSaid(VoiceAction.ToggleEdges);
         }
 
         /// <summary>
@@ -190,7 +213,8 @@ internal static bool ToggleEdges()
         /// <returns>true if the user requests this action and <see cref="KeyboardShortcutsEnabled"/></returns>
         internal static bool ToggleTreeView()
         {
-            return KeyboardShortcutsEnabled && KeyBindings.IsDown(KeyAction.TreeView);
+            return (KeyboardShortcutsEnabled
+                && KeyBindings.IsDown(KeyAction.TreeView)) || VoiceBindings.isSaid(VoiceAction.TreeView);
         }
 
         #endregion
@@ -229,7 +253,8 @@ public static bool TogglePathPlaying()
         /// <returns>true if the user requests this action and <see cref="KeyboardShortcutsEnabled"/></returns>
         public static bool ToggleMetricCharts()
         {
-            return KeyboardShortcutsEnabled && KeyBindings.IsDown(KeyAction.ToggleCharts);
+            return (KeyboardShortcutsEnabled
+                && KeyBindings.IsDown(KeyAction.ToggleCharts)) || VoiceBindings.isSaid(VoiceAction.ToggleCharts);
         }
 
         /// <summary>
@@ -253,7 +278,7 @@ public static bool ToggleMetricHoveringSelection()
         /// <returns>true if the user starts the mouse interaction to open the context menu</returns>
         internal static bool OpenContextMenuStart()
         {
-            return Input.GetMouseButtonDown(rightMouseButton) && !Raycasting.IsMouseOverGUI();
+            return (Input.GetMouseButtonDown(rightMouseButton) || VoiceBindings.isSaid(VoiceAction.OpenContextMenu)) && !Raycasting.IsMouseOverGUI();
         }
 
         /// <summary>
@@ -295,7 +320,8 @@ public static bool ToggleCameraLock()
         /// <returns>true if the user requests this action and <see cref="KeyboardShortcutsEnabled"/></returns>
         public static bool Cancel()
         {
-            return KeyboardShortcutsEnabled && KeyBindings.IsDown(KeyAction.Cancel);
+            return (KeyboardShortcutsEnabled
+                && KeyBindings.IsDown(KeyAction.Cancel)) || VoiceBindings.isSaid(VoiceAction.Cancel);
         }
 
         /// <summary>
@@ -438,7 +464,8 @@ public static bool RotateCamera()
         /// <returns>true if the user wishes to point</returns>
         public static bool TogglePointing()
         {
-            return KeyboardShortcutsEnabled && KeyBindings.IsDown(KeyAction.Pointing);
+            return (KeyboardShortcutsEnabled
+                && KeyBindings.IsDown(KeyAction.Pointing)) || VoiceBindings.isSaid(VoiceAction.Pointing);
         }
 
         #endregion
@@ -484,15 +511,15 @@ public static bool ToggleEvolutionCanvases()
         /// <returns>true if the user requests this action and <see cref="KeyboardShortcutsEnabled"/></returns>
         public static bool Previous()
         {
-            return KeyboardShortcutsEnabled && KeyBindings.IsDown(KeyAction.Previous);
+            return (KeyboardShortcutsEnabled && KeyBindings.IsDown(KeyAction.Previous)) || VoiceBindings.isSaid(VoiceAction.Previous);
         }
         /// <summary>
         /// The next revision is to be shown.
         /// </summary>
         /// <returns>true if the user requests this action and <see cref="KeyboardShortcutsEnabled"/></returns>
         public static bool Next()
         {
-            return KeyboardShortcutsEnabled && KeyBindings.IsDown(KeyAction.Next);
+            return (KeyboardShortcutsEnabled && KeyBindings.IsDown(KeyAction.Next)) || VoiceBindings.isSaid(VoiceAction.Next);
         }
         /// <summary>
         /// Toggles auto play of the animation.
@@ -606,7 +633,8 @@ public static bool Select()
         /// <returns>true if the user requests this action and <see cref="KeyboardShortcutsEnabled"/></returns>
         public static bool OpenTextChat()
         {
-            return KeyboardShortcutsEnabled && KeyBindings.IsDown(KeyAction.ToggleTextChat);
+            return (KeyboardShortcutsEnabled
+                && KeyBindings.IsDown(KeyAction.ToggleTextChat)) || VoiceBindings.isSaid(VoiceAction.ToggleTextChat);
         }
 
         /// <summary>
@@ -630,7 +658,8 @@ public static bool ToggleVoiceChat()
         /// <returns>True if the user wants to close all notifications.</returns>
         public static bool CloseAllNotifications()
         {
-            return KeyboardShortcutsEnabled && KeyBindings.IsDown(KeyAction.CloseNotifications);
+            return (KeyboardShortcutsEnabled
+                && KeyBindings.IsDown(KeyAction.CloseNotifications)) || VoiceBindings.isSaid(VoiceAction.CloseNotifications);
         }
 
         #endregion
@@ -642,7 +671,8 @@ public static bool CloseAllNotifications()
         /// <returns>True if the user wants to turn the FaceCam on or off.</returns>
         internal static bool ToggleFaceCam()
         {
-            return KeyboardShortcutsEnabled && KeyBindings.IsDown(KeyAction.ToggleFaceCam);
+            return (KeyboardShortcutsEnabled
+                && KeyBindings.IsDown(KeyAction.ToggleFaceCam)) || VoiceBindings.isSaid(VoiceAction.ToggleFaceCam);
         }
 
         /// <summary>
@@ -651,7 +681,8 @@ internal static bool ToggleFaceCam()
         /// <returns>True if the user wants to switch the position of the FaceCam.</returns>
         internal static bool ToggleFaceCamPosition()
         {
-            return KeyboardShortcutsEnabled && KeyBindings.IsDown(KeyAction.ToggleFaceCamPosition);
+            return (KeyboardShortcutsEnabled
+                && KeyBindings.IsDown(KeyAction.ToggleFaceCamPosition)) || VoiceBindings.isSaid(VoiceAction.ToggleFaceCamPosition);
         }
         #endregion
         //----------------------------------------------------
@@ -726,7 +757,8 @@ internal static bool MoveObjectBackward()
         /// <returns>True if the user wants to toggle the drawable manager menu.</returns>
         internal static bool ToggleDrawableManagerView()
         {
-            return KeyboardShortcutsEnabled && KeyBindings.IsDown(KeyAction.DrawableManagerView);
+            return (KeyboardShortcutsEnabled
+                && KeyBindings.IsDown(KeyAction.DrawableManagerView)) || VoiceBindings.isSaid(VoiceAction.DrawableManagerView);
         }
         #endregion
     }

diff --git a/Assets/SEE/Controls/VoiceActions.meta b/Assets/SEE/Controls/VoiceActions.meta
@@ -0,0 +1,8 @@
+fileFormatVersion: 2
+guid: b5fa85a285d6248459c1848b50b75d64
+folderAsset: yes
+DefaultImporter:
+  externalObjects: {}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: