feat: basic livekit setup

.env.example

@@ -2,3 +2,18 @@
 # https://app.supabase.com/project/_/settings/api
 NEXT_PUBLIC_SUPABASE_URL=your-project-url
 NEXT_PUBLIC_SUPABASE_ANON_KEY=your-anon-key
+
+# Signin and go to https://docs.livekit.io/agents/quickstart/
+LIVEKIT_API_KEY=""
+LIVEKIT_API_SECRET=""
+LIVEKIT_URL=""
+NEXT_PUBLIC_LIVEKIT_URL=""
+
+OPENAI_API_KEY=""
+OPENAI_ORG_ID=""
+
+ELEVEN_API_KEY=""
+DEEPGRAM_API_KEY=""
+
+SUPABASE_URL=""
+SUPABASE_KEY=""

package.json

@@ -10,11 +10,18 @@
     "tauri": "tauri"
   },
   "dependencies": {
+    "@heroicons/react": "^2.1.3",
+    "@livekit/components-react": "^2.0.3",
+    "@radix-ui/react-dropdown-menu": "^2.0.6",
     "@radix-ui/react-icons": "^1.3.0",
+    "@radix-ui/react-menubar": "^1.0.4",
+    "@radix-ui/react-separator": "^1.0.3",
     "@radix-ui/react-slot": "^1.0.2",
     "@tauri-apps/api": "^1.5.3",
     "class-variance-authority": "^0.7.0",
     "clsx": "^2.1.0",
+    "framer-motion": "^11.0.22",
+    "livekit-client": "^2.0.10",
     "lucide-react": "^0.363.0",
     "react": "^18.2.0",
     "react-dom": "^18.2.0",

server/livekit_agents/agent.py

@@ -159,12 +159,12 @@ def reprompt(self, data, msg:str) -> str:
         for match in data:
             print("data", data)
             print("match", match)
-            contextText += f"Title of Demo Day Submission: {match["title"]}\
+            contextText += f'Title of Demo Day Submission: {match["title"]}\
                 Niche: {match["niche"]}\
                 Summary: {match["description"]} \
                 Full Description: {match["youtube_transcript"]} \
                 Social: {match["social"]} \
-                Buildspace Season: {match["season"]}"
+                Buildspace Season: {match["season"]}'
 
         contextText+=f"\nuser's message: {msg}"
 

src-tauri/Cargo.toml

@@ -14,6 +14,7 @@ tauri-build = { version = "1", features = [] }
 tauri = { version = "1", features = [ "shell-all", "system-tray"] }
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
+dotenvy = "0.15.7"
 
 [features]
 # This feature is used for production builds or when a dev server is not specified, DO NOT REMOVE!!

src-tauri/src/main.rs

@@ -8,8 +8,8 @@ use tauri::{
 
 // Learn more about Tauri commands at https://tauri.app/v1/guides/features/command
 #[tauri::command]
-fn greet(name: &str) -> String {
-    format!("Hello, {}! You've been greeted from Rust!", name)
+fn get_env(name: &str) -> String {
+    std::env::var(String::from(name)).unwrap_or(String::from(""))
 }
 
 const links: [(&str, &str, &str); 2] = [
@@ -19,6 +19,8 @@ const links: [(&str, &str, &str); 2] = [
 ];
 
 fn main() {
+    dotenvy::dotenv().expect(".env file not found");
+
     let sub_menu_github = {
         let mut menu = SystemTrayMenu::new();
         for (id, label, _url) in
@@ -41,7 +43,7 @@ fn main() {
     let tray = SystemTray::new().with_menu(tray_menu);
 
     let mut app = tauri::Builder::default()
-        .invoke_handler(tauri::generate_handler![greet])
+        .invoke_handler(tauri::generate_handler![get_env])
         .system_tray(tray)
         .on_system_tray_event(on_system_tray_event)
         .build(tauri::generate_context!())

src-tauri/tauri.conf.json

@@ -6,7 +6,7 @@
     "distDir": "../dist"
   },
   "package": {
-    "productName": "os1",
+    "productName": "OS1",
     "version": "0.0.0"
   },
   "tauri": {
@@ -25,7 +25,7 @@
     },
     "windows": [
       {
-        "title": "os1",
+        "title": "OS1",
         "width": 800,
         "height": 600
       }

src/App.tsx

@@ -1,45 +1,135 @@
-import { useState } from "react";
-// import reactLogo from "./assets/react.svg";
+import { useCallback, useEffect, useMemo, useState } from "react";
 import { invoke } from "@tauri-apps/api/tauri";
-import { Input } from "./components/ui/input";
-import { Button } from "./components/ui/button";
-import { WelcomePage } from "./components/welcome";
+import { DynamicIsland, IslandState, IslandStates } from "./components/DynamicIsland";
+import { LiveKitRoom, RoomAudioRenderer, StartAudio, useToken } from "@livekit/components-react";
+import Playground, { PlaygroundOutputs } from './components/Playground';
+import { PlaygroundToast, ToastType } from './components/PlaygroundToast';
+import { generateRandomAlphanumeric } from './utils/livekit';
+import { AnimatePresence, motion } from "framer-motion";
+import { CallNavBar } from './components/CallNavbar';
+import { useAppConfig } from './hooks/useAppConfig';
+import { WelcomePage } from "./components/Welcome";
+import { tw } from "./utils/tw";
 
-function App() {
-  const [greetMsg, setGreetMsg] = useState("");
-  const [name, setName] = useState("");
+export default function App() {
+  const [toastMessage, setToastMessage] = useState<{
+    message: string;
+    type: ToastType;
+  } | null>(null);
+  const [shouldConnect, setShouldConnect] = useState(false);
+  const [roomName] = useState(createRoomName());
+  const [liveKitUrl, setLiveKitUrl] = useState<string>();
+
+  const [state, setState] = useState<IslandState>(IslandStates[0])
   const onboarding = false
 
-  const greet=async()=> {
-    setGreetMsg(await invoke("greet", { name }));
+  const tokenOptions = useMemo(() => {
+    return {
+      userInfo: { identity: generateRandomAlphanumeric(16) },
+    };
+  }, []);
+
+  // const token = useToken('/api/get-participant-token', roomName, tokenOptions);
+  const token = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE3MTE1ODUwNDEsImlzcyI6IkFQSUJ2NmdzQU5lclhaWCIsIm5iZiI6MTcxMTU4NDE0MSwic3ViIjoiY2hhZCIsInZpZGVvIjp7ImNhblB1Ymxpc2giOnRydWUsImNhblB1Ymxpc2hEYXRhIjp0cnVlLCJjYW5TdWJzY3JpYmUiOnRydWUsInJvb20iOiJjaGFkIiwicm9vbUpvaW4iOnRydWV9fQ.gpPZPvtzCn9Yh8JGhz-Yub98WyWQ_1KxyQAgkFVJRcA"
+
+  const get_env=async(name: string) => {
+    return await invoke("get_env", { name });
   }
 
+  useEffect(()=>{
+    get_env("NEXT_PUBLIC_LIVEKIT_URL").then((livekiturl) => {
+      setLiveKitUrl(livekiturl as string)
+      handleConnect(true)
+    })
+  },[])
+
+  const appConfig = useAppConfig();
 
+  const outputs = [
+    appConfig?.outputs.audio && PlaygroundOutputs.Audio,
+    appConfig?.outputs.video && PlaygroundOutputs.Video,
+    appConfig?.outputs.chat && PlaygroundOutputs.Chat,
+  ].filter((item) => typeof item !== 'boolean') as PlaygroundOutputs[];
+
+  const handleConnect = useCallback((connect: boolean, opts?: { url: string; token: string }) => {
+    if (connect && opts) {
+      setLiveKitUrl(opts.url);
+    }
+    setShouldConnect(connect);
+  }, []);
+
   if (onboarding){
     return <WelcomePage/>
   }
+
   return (
-    <div className="h-dvh w-full flex flex-col items-center justify-center">
-      <div className="space-y-4">
-        <h1 className="text-3xl font-semibold">OS1</h1>
-        <form
-          onSubmit={(e) => {
-            e.preventDefault();
-            greet();
+    <main className="h-dvh w-full flex flex-col items-center justify-center">
+      <AnimatePresence>
+        {toastMessage && (
+          <motion.div
+            className="left-0 right-0 top-0 absolute z-10"
+            initial={{ opacity: 0, translateY: -50 }}
+            animate={{ opacity: 1, translateY: 0 }}
+            exit={{ opacity: 0, translateY: -50 }}
+          >
+            <PlaygroundToast
+              message={
+                toastMessage.message === 'Permission denied'
+                  ? 'Please enable your microphone so i can hear you. 😊'
+                  : toastMessage.message
+              }
+              type={toastMessage.type}
+              onDismiss={() => setToastMessage(null)}
+            />
+          </motion.div>
+        )}
+      </AnimatePresence>
+
+      <DynamicIsland state={state} />
+
+      {liveKitUrl && (
+        <LiveKitRoom
+          className="flex flex-col h-full w-full"
+          serverUrl={liveKitUrl}
+          token={token}
+          audio={appConfig.inputs.mic}
+          video={false}
+          connect={shouldConnect}
+          onError={(e) => {
+            setToastMessage({ message: e.message, type: 'error' });
+            console.error(e);
           }}
         >
-          <Input
-            id="greet-input"
-            className="text-foreground"
-            onChange={(e) => setName(e.currentTarget.value)}
-            placeholder="Enter a name..."
+          <Playground
+            outputs={outputs}
+            themeColor={appConfig.theme_color}
+            videoFit={appConfig.video_fit}
+          />
+          <RoomAudioRenderer />
+          <StartAudio label="Click to enable audio playback" />
+          <CallNavBar
+            className="border-none bg-transparent [&>*:second-child]:bg-white [&>*:second-child]:rounded-full [&>*:second-child]:px-0 [&>*:second-child]:py-0 fixed bottom-6 mx-auto self-center"
           />
-          <Button type="submit" className="mt-4">Greet</Button>
-        </form>
-        <p>{greetMsg}</p>
+        </LiveKitRoom>
+      )}
+
+      <div className="flex space-x-3">
+        {IslandStates.map((curr_state)=>(
+          <button
+              key={curr_state}
+              className={tw("lowercase border ring-offset-1 px-3 py-2 rounded-3xl ring-1 ring-neutral-400",
+                curr_state===state && "bg-black text-white"
+              )}
+              onClick={()=> setState(curr_state)}
+          >
+            {curr_state.replace("_", " ")}
+          </button>
+        ))}
       </div>
-    </div>
+    </main>
   );
 }
 
-export default App;
+const createRoomName = () => {
+  return [generateRandomAlphanumeric(4), generateRandomAlphanumeric(4)].join('-');
+};

src/components/AgentMultibandAudioVisualizer.tsx

@@ -0,0 +1,105 @@
+import { AgentState } from '../utils/types';
+import { useEffect, useState } from 'react';
+
+type AgentMultibandAudioVisualizerProps = {
+  state: AgentState;
+  barWidth: number;
+  minBarHeight: number;
+  maxBarHeight: number;
+  accentColor: string;
+  accentShade?: number;
+  frequencies: Float32Array[];
+  borderRadius: number;
+  gap: number;
+};
+
+export const AgentMultibandAudioVisualizer = ({
+  state,
+  barWidth,
+  minBarHeight,
+  maxBarHeight,
+  accentColor,
+  accentShade,
+  frequencies,
+  borderRadius,
+  gap,
+}: AgentMultibandAudioVisualizerProps) => {
+  const summedFrequencies = frequencies.map((bandFrequencies) => {
+    const sum = bandFrequencies.reduce((a, b) => a + b, 0);
+    return Math.sqrt(sum / bandFrequencies.length);
+  });
+
+  const [thinkingIndex, setThinkingIndex] = useState(Math.floor(summedFrequencies.length / 2));
+  const [thinkingDirection, setThinkingDirection] = useState<'left' | 'right'>('right');
+
+  useEffect(() => {
+    if (state !== 'thinking') {
+      setThinkingIndex(Math.floor(summedFrequencies.length / 2));
+      return;
+    }
+    const timeout = setTimeout(() => {
+      if (thinkingDirection === 'right') {
+        if (thinkingIndex === summedFrequencies.length - 1) {
+          setThinkingDirection('left');
+          setThinkingIndex((prev) => prev - 1);
+        } else {
+          setThinkingIndex((prev) => prev + 1);
+        }
+      } else {
+        if (thinkingIndex === 0) {
+          setThinkingDirection('right');
+          setThinkingIndex((prev) => prev + 1);
+        } else {
+          setThinkingIndex((prev) => prev - 1);
+        }
+      }
+    }, 200);
+
+    return () => clearTimeout(timeout);
+  }, [state, summedFrequencies.length, thinkingDirection, thinkingIndex]);
+
+  return (
+    <div
+      className={`flex flex-row items-center`}
+      style={{
+        gap: gap + 'px',
+      }}
+    >
+      {summedFrequencies.map((frequency, index) => {
+        const isCenter = index === Math.floor(summedFrequencies.length / 2);
+
+        let color = `${accentColor}-${accentShade}`;
+        let shadow = `shadow-lg-${accentColor}`;
+        let transform;
+
+        if (state === 'listening' || state === 'idle') {
+          color = isCenter ? `${accentColor}-${accentShade}` : 'gray-950';
+          shadow = !isCenter ? '' : shadow;
+          transform = !isCenter ? 'scale(1.0)' : 'scale(1.2)';
+        } else if (state === 'speaking') {
+          color = `${accentColor}${accentShade ? '-' + accentShade : ''}`;
+        } else if (state === 'thinking') {
+          color = index === thinkingIndex ? `${accentColor}-${accentShade}` : 'gray-950';
+          shadow = '';
+          transform = thinkingIndex !== index ? 'scale(1)' : 'scale(1.1)';
+        }
+
+        return (
+          <div
+            className={`bg-${color} bg-red-900 ${shadow} ${
+              isCenter && state === 'listening' ? 'animate-pulse' : ''
+            }`}
+            key={'frequency-' + index}
+            style={{
+              height: minBarHeight + frequency * (maxBarHeight - minBarHeight) + 'px',
+              borderRadius: borderRadius + 'px',
+              width: barWidth + 'px',
+              transition: 'background-color 0.35s ease-out, transform 0.25s ease-out',
+              transform: transform,
+            }}
+          ></div>
+        );
+      })}
+    </div>
+  );
+};