Merge pull request #10 from agencyenterprise/feature/conversational-mode

Add support for conversational mode

examples/simple/src/App.tsx

@@ -2,7 +2,16 @@ import { Avatar, useAvatar } from 'alpha-ai-avatar-sdk-react';
 import { Button } from './Button';
 
 export function App() {
-  const { room, isConnected, connect, say, stop, switchAvatar } = useAvatar();
+  const {
+    room,
+    isConnected,
+    connect,
+    disconnect,
+    say,
+    stop,
+    switchAvatar,
+    enableMicrophone,
+  } = useAvatar();
 
   return (
     <div
@@ -12,7 +21,7 @@ export function App() {
         flexDirection: 'column',
         gap: '20px',
       }}>
-      <Avatar style={{ borderRadius: '20px', width: 250, height: 250 }} />
+      <Avatar style={{ borderRadius: '20px', width: 512, height: 512 }} />
 
       <div style={{ display: 'flex', gap: '10px' }}>
         {room ? (
@@ -21,6 +30,10 @@ export function App() {
               <Button onClick={() => say('Hello, how are you?')}>Say</Button>
               <Button onClick={stop}>Stop Avatar</Button>
               <Button onClick={() => switchAvatar(4)}>Switch Avatar</Button>
+              <Button onClick={() => enableMicrophone()}>
+                Enable microphone
+              </Button>
+              <Button onClick={() => disconnect()}>Disconnect</Button>
             </>
           ) : (
             <p>Connecting...</p>

examples/simple/src/index.tsx

@@ -4,7 +4,16 @@ import { App } from './App';
 
 import { AvatarClient, AvatarProvider } from 'alpha-ai-avatar-sdk-react';
 
-const client = new AvatarClient({ apiKey: 'API_KEY' });
+const client = new AvatarClient({
+  apiKey: 'API_KEY',
+  conversational: true,
+  initialPrompt: [
+    {
+      role: 'system',
+      content: 'Act like Albert Einstein',
+    },
+  ],
+});
 
 const root = ReactDOM.createRoot(
   document.getElementById('root') as HTMLElement,

package.json

@@ -1,6 +1,6 @@
 {
   "name": "alpha-ai-avatar-sdk-react",
-  "version": "0.0.7",
+  "version": "0.0.11",
   "description": "Alpha AI Avatar SDK (React)",
   "main": "index.js",
   "module": "index.esm.js",

src/contexts/AvatarContext.tsx

@@ -1,27 +1,14 @@
 import { Room, RoomEvent } from 'livekit-client';
 import { ReactNode, createContext, useState } from 'react';
 import { AvatarClient } from '../core/AvatarClient';
-
-enum MessageState {
-  Idle = 0,
-  Loading = 1,
-  Speaking = 2,
-  Active = 3,
-}
-
-enum MessageType {
-  Transcript = 0,
-  State = 1,
-  Error = 2,
-}
-
-type Message = {
-  data: {
-    message: string;
-    state: MessageState;
-  };
-  type: MessageType;
-};
+import {
+  MessageState,
+  MessageType,
+  ParsedMessage,
+  Prompt,
+  ChatMessage,
+  TranscriberStatus,
+} from '../core/types';
 
 type SayOptions = {
   voiceName?: string;
@@ -40,24 +27,38 @@ type SayOptions = {
 export type AvatarContextType = {
   client: AvatarClient;
   room?: Room;
+  messages: ChatMessage[];
   isConnected: boolean;
   isAvatarSpeaking: boolean;
-  connect: (avatarId?: number) => Promise<void>;
+  transcriberStatus: TranscriberStatus;
+  connect: (
+    avatarId?: number,
+    conversational?: boolean,
+    initialPrompt?: Prompt[],
+  ) => Promise<void>;
   say: (message: string, options?: SayOptions) => Promise<void>;
   stop: () => Promise<void>;
   switchAvatar: (avatarId: number) => Promise<void>;
+  enableMicrophone: () => Promise<void>;
+  disableMicrophone: () => Promise<void>;
+  clearMessages: () => void;
   disconnect: () => Promise<void>;
 };
 
 const AvatarContext = createContext<AvatarContextType>({
   client: new AvatarClient({ apiKey: '' }),
   room: undefined,
+  messages: [],
   isConnected: false,
   isAvatarSpeaking: false,
+  transcriberStatus: TranscriberStatus.Closed,
   connect: () => Promise.resolve(),
   say: () => Promise.resolve(),
   stop: () => Promise.resolve(),
   switchAvatar: () => Promise.resolve(),
+  enableMicrophone: () => Promise.resolve(),
+  disableMicrophone: () => Promise.resolve(),
+  clearMessages: () => {},
   disconnect: () => Promise.resolve(),
 });
 
@@ -70,29 +71,45 @@ function AvatarProvider({ children, client }: AvatarProviderProps) {
   const [room, setRoom] = useState<Room>();
   const [isConnected, setIsConnected] = useState(false);
   const [isAvatarSpeaking, setIsAvatarSpeaking] = useState(false);
+  const [transcriberStatus, setTranscriberStatus] = useState<TranscriberStatus>(
+    TranscriberStatus.Closed,
+  );
+  const [messages, setMessages] = useState<ChatMessage[]>([]);
 
   function handleDataReceived(data: Uint8Array) {
-    const parsedMessage: Message = JSON.parse(new TextDecoder().decode(data));
-
-    if (parsedMessage.type === MessageType.State) {
-      if (parsedMessage.data.state === MessageState.Speaking) {
-        setIsAvatarSpeaking(true);
-      } else {
-        setIsAvatarSpeaking(false);
-      }
-    }
-
-    if (parsedMessage.type === MessageType.Error) {
-      throw new Error('Error from server');
+    const message: ParsedMessage = JSON.parse(new TextDecoder().decode(data));
+
+    switch (message.type) {
+      case MessageType.State:
+        setIsAvatarSpeaking(message.data.state === MessageState.Speaking);
+        break;
+      case MessageType.Transcript:
+        onTranscriptionHandler(message.data);
+        break;
+      case MessageType.TranscriberState:
+        setTranscriberStatus(message.data.status);
+        break;
+      case MessageType.Error:
+        throw new Error('Error from server');
     }
   }
 
-  async function connect(avatarId?: number) {
+  async function connect(
+    avatarId?: number,
+    conversational?: boolean,
+    initialPrompt?: any,
+  ) {
     if (room && room.state !== 'disconnected') {
       return;
     }
 
     const newRoom = new Room({ adaptiveStream: true });
+    const { token, serverUrl } = await client.connect(
+      avatarId,
+      conversational,
+      initialPrompt,
+    );
+    newRoom.prepareConnection(token, serverUrl);
 
     newRoom
       .on(RoomEvent.Connected, () => {
@@ -104,9 +121,6 @@ function AvatarProvider({ children, client }: AvatarProviderProps) {
       });
 
     setRoom(newRoom);
-
-    const { token, serverUrl } = await client.connect(avatarId);
-
     newRoom.connect(serverUrl, token);
   }
 
@@ -129,6 +143,59 @@ function AvatarProvider({ children, client }: AvatarProviderProps) {
     await connect(avatarId);
   }
 
+  async function enableMicrophone() {
+    try {
+      await navigator.mediaDevices.getUserMedia({ audio: true });
+      if (isConnected) {
+        room?.localParticipant?.setMicrophoneEnabled(true);
+      }
+    } catch (error) {
+      console.error('Error enabling conversational mode:', error);
+    }
+  }
+
+  async function disableMicrophone() {
+    if (isConnected) {
+      room?.localParticipant?.setMicrophoneEnabled(false);
+    }
+  }
+
+  function onTranscriptionHandler({
+    role,
+    message,
+    isFinal,
+  }: {
+    role: string;
+    message: string;
+    isFinal: boolean;
+  }) {
+    setMessages((prevMessages) => {
+      const lastIndex = prevMessages.length - 1;
+      const lastMessage = prevMessages[lastIndex];
+
+      if (role === lastMessage?.role) {
+        prevMessages = prevMessages.slice(0, lastIndex);
+
+        if (role === 'assistant' && !isFinal) {
+          message = lastMessage.content + message;
+        }
+      }
+
+      return [
+        ...prevMessages,
+        {
+          role,
+          content: message,
+          isFinal,
+        },
+      ];
+    });
+  }
+
+  function clearMessages() {
+    setMessages([]);
+  }
+
   async function disconnect() {
     await room?.disconnect();
     setRoom(undefined);
@@ -139,12 +206,17 @@ function AvatarProvider({ children, client }: AvatarProviderProps) {
       value={{
         client,
         room,
+        messages,
         isConnected,
         isAvatarSpeaking,
+        transcriberStatus,
         connect,
         say,
         stop,
         switchAvatar,
+        enableMicrophone,
+        disableMicrophone,
+        clearMessages,
         disconnect,
       }}>
       {children}

src/core/AvatarClient.ts

@@ -1,17 +1,33 @@
 import { HTTPClient } from './HTTPClient';
-import { AvatarClientConfig, CreateRoomResponse, GetAvatarsResponse, GetSupportedVoicesResponse } from './types';
+import {
+  AvatarClientConfig,
+  CreateRoomResponse,
+  GetAvatarsResponse,
+  GetSupportedVoicesResponse,
+  Prompt,
+} from './types';
 
 export class AvatarClient extends HTTPClient {
   private avatarId?: number;
+  private conversational: boolean = false;
+  private initialPrompt?: Prompt[];
 
   constructor(config: AvatarClientConfig) {
     super(config.baseUrl ?? 'https://avatar.alpha.school', config.apiKey);
     this.avatarId = config.avatarId;
+    this.conversational = config.conversational ?? false;
+    this.initialPrompt = config.initialPrompt;
   }
 
-  connect(avatarId?: number) {
+  connect(
+    avatarId?: number,
+    conversational?: boolean,
+    initialPrompt?: Prompt[],
+  ) {
     return this.post<CreateRoomResponse>('/rooms', {
       avatarId: avatarId ?? this.avatarId,
+      conversational: conversational ?? this.conversational,
+      initialPrompt: initialPrompt ?? this.initialPrompt,
     });
   }