conditionally add Trulience avatars to TEN UI

playground/README.md

@@ -38,3 +38,15 @@ pnpm install
 # run
 pnpm dev
 ```
+
+## Avatars
+
+### Trulience
+Add the following or equivalent lines to client .env     
+
+```bash
+NEXT_PUBLIC_trulienceSDK=https://trulience.com/sdk/trulience.sdk.js
+NEXT_PUBLIC_trulienceAvatarToken=
+NEXT_PUBLIC_trulienceAvatarId=8848447098800980663
+NEXT_PUBLIC_trulienceAnimationURL=https://trulience.com
+```

playground/package.json

@@ -44,7 +44,8 @@
     "sonner": "^1.7.0",
     "tailwind-merge": "^2.5.4",
     "tailwindcss-animate": "^1.0.7",
-    "zod": "^3.23.8"
+    "zod": "^3.23.8",
+    "trulience-sdk": "https://trulience.com/home/assets/trulience-sdk-1.0.11.tar.gz"
   },
   "devDependencies": {
     "@minko-fe/postcss-pxtoviewport": "^1.3.2",

playground/src/components/Agent/AvatarTrulience.tsx

@@ -0,0 +1,93 @@
+"use client"
+
+import React, { useEffect, useMemo, useRef, useState } from "react"
+import { useAppSelector } from "@/common"
+import { TrulienceAvatar } from "trulience-sdk"
+import { IMicrophoneAudioTrack } from "agora-rtc-sdk-ng"
+
+interface AvatarProps {
+  audioTrack?: IMicrophoneAudioTrack
+}
+
+export default function Avatar({ audioTrack }: AvatarProps) {
+  const agentConnected = useAppSelector((state) => state.global.agentConnected)
+  const trulienceAvatarRef = useRef<TrulienceAvatar>(null)
+
+  // Track loading progress
+  const [loadProgress, setLoadProgress] = useState(0)
+
+  // Resolve the final avatar ID from URL param or environment variable
+  const finalAvatarId = useMemo(() => {
+    const urlParams = new URLSearchParams(window.location.search)
+    const avatarIdFromURL = urlParams.get("avatarId")
+    return avatarIdFromURL || process.env.NEXT_PUBLIC_trulienceAvatarId || ""
+  }, [])
+
+  // Define any event callbacks that you need
+  const eventCallbacks = useMemo(() => {
+    return {
+      "auth-success": (resp: string) => {
+        console.log("Trulience Avatar auth-success:", resp)
+      },
+      "websocket-connect": (resp: string) => {
+        console.log("Trulience Avatar websocket-connect:", resp)
+      },
+      "load-progress": (details: Record<string, any>) => {
+        console.log("Trulience Avatar load-progress:", details.progress)
+        setLoadProgress(details.progress)
+      },
+    }
+  }, [])
+
+  // Create the Trulience Avatar instance only once
+  const trulienceAvatarInstance = useMemo(() => {
+    return (
+      <TrulienceAvatar
+        url={process.env.NEXT_PUBLIC_trulienceSDK}
+        ref={trulienceAvatarRef}
+        avatarId={finalAvatarId}
+        token={process.env.NEXT_PUBLIC_trulienceAvatarToken}
+        eventCallbacks={eventCallbacks}
+        width="100%"
+        height="100%"
+      />
+    )
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [])
+
+  // Update the Avatar’s audio stream whenever audioTrack or agentConnected changes
+  useEffect(() => {
+    if (trulienceAvatarRef.current) {
+      if (audioTrack && agentConnected) {
+        const stream = new MediaStream([audioTrack.getMediaStreamTrack()])
+        trulienceAvatarRef.current.setMediaStream(null)
+        trulienceAvatarRef.current.setMediaStream(stream)
+        console.warn("[TrulienceAvatar] MediaStream set:", stream)
+      } else if (!agentConnected) {
+        const trulienceObj = trulienceAvatarRef.current.getTrulienceObject()
+        trulienceObj?.sendMessageToAvatar("<trl-stop-background-audio immediate='true' />")
+        trulienceObj?.sendMessageToAvatar("<trl-content position='DefaultCenter' />")
+      }
+    }
+
+    // Cleanup: unset media stream
+    return () => {
+      trulienceAvatarRef.current?.setMediaStream(null)
+    }
+  }, [audioTrack, agentConnected])
+
+  return (
+    <div className="relative my-3 h-60 w-full">
+      {/* Render the TrulienceAvatar */}
+      {trulienceAvatarInstance}
+
+      {/* Show a loader overlay while progress < 1 */}
+      {loadProgress < 1 && (
+        <div className="absolute inset-0 z-10 flex items-center justify-center bg-black bg-opacity-80">
+          {/* a simple Tailwind spinner */}
+          <div className="h-10 w-10 animate-spin rounded-full border-4 border-blue-500 border-t-transparent" />
+        </div>
+      )}
+    </div>
+  )
+}

playground/src/components/Agent/Camera.tsx

@@ -105,7 +105,7 @@ export default function VideoBlock(props: {
       onVideoSourceChange={onVideoSourceChange}
       select={videoSourceType === VideoSourceType.CAMERA ? <CamSelect videoTrack={cameraTrack} /> : <div className="w-[180px]" />}
     >
-      <div className="my-3 h-52 w-full overflow-hidden rounded-lg">
+      <div className="my-3 h-60 w-full overflow-hidden rounded-lg">
         <LocalStreamPlayer videoTrack={videoSourceType === VideoSourceType.CAMERA ? cameraTrack : screenTrack} />
       </div>
     </VideoDeviceWrapper>

playground/src/components/Agent/Microphone.tsx

@@ -56,12 +56,12 @@ export default function MicrophoneBlock(props: {
       isActive={!audioMute}
       select={<MicrophoneSelect audioTrack={audioTrack} />}
     >
-      <div className="mt-3 flex h-28 flex-col items-center justify-center gap-2.5 self-stretch rounded-md border border-[#272A2F] bg-[#1E2024] p-6 shadow-[0px_2px_2px_0px_rgba(0,0,0,0.25)]">
+      <div className="mt-3 flex h-24 flex-col items-center justify-center gap-2.5 self-stretch rounded-md border border-[#272A2F] bg-[#1E2024] p-2 shadow-[0px_2px_2px_0px_rgba(0,0,0,0.25)]">
         <AudioVisualizer
           type="user"
           barWidth={4}
           minBarHeight={2}
-          maxBarHeight={50}
+          maxBarHeight={40}
           frequencies={subscribedVolumes}
           borderRadius={2}
           gap={4}

playground/src/components/Dynamic/RTCCard.tsx

@@ -14,6 +14,7 @@ import {
 } from "@/store/reducers/global"
 import AgentVoicePresetSelect from "@/components/Agent/VoicePresetSelect"
 import AgentView from "@/components/Agent/View"
+import Avatar from "@/components/Agent/AvatarTrulience"
 import MicrophoneBlock from "@/components/Agent/Microphone"
 import VideoBlock from "@/components/Agent/Camera"
 
@@ -32,6 +33,7 @@ export default function RTCCard(props: { className?: string }) {
   const [screenTrack, setScreenTrack] = React.useState<ILocalVideoTrack>()
   const [remoteuser, setRemoteUser] = React.useState<IRtcUser>()
   const [videoSourceType, setVideoSourceType] = React.useState<VideoSourceType>(VideoSourceType.CAMERA)
+  const useTrulienceAvatar = Boolean(process.env.NEXT_PUBLIC_trulienceAvatarId)
 
   React.useEffect(() => {
     if (!options.channel) {
@@ -85,6 +87,10 @@ export default function RTCCard(props: { className?: string }) {
 
   const onRemoteUserChanged = (user: IRtcUser) => {
     console.log("[rtc] onRemoteUserChanged", user)
+    if (useTrulienceAvatar) {
+      // trulience SDK will play audio in synch with mouth
+      user.audioTrack?.stop();
+    }
     setRemoteUser(user)
   }
 
@@ -114,16 +120,22 @@ export default function RTCCard(props: { className?: string }) {
     setVideoSourceType(value)
   }
 
+
   return (
     <>
       <div className={cn("flex-shrink-0", "overflow-y-auto", className)}>
         <div className="flex h-full w-full flex-col">
           {/* -- Agent */}
           <div className="w-full">
             <div className="flex w-full items-center justify-between p-2">
-              <h2 className="mb-2 text-xl font-semibold">Audio & Video</h2>
+              <h2 className="mb-0 text-l font-semibold">Audio & Video</h2>
             </div>
-            <AgentView audioTrack={remoteuser?.audioTrack} />
+            {/* Conditionally render either Avatar or AgentView */}
+            {useTrulienceAvatar ? (
+              <Avatar audioTrack={remoteuser?.audioTrack} />
+            ) : (
+              <AgentView audioTrack={remoteuser?.audioTrack} />
+            )}
           </div>
 
           {/* -- You */}