Add TTS support to deepgram

.github/ISSUE_TEMPLATE/bug.yaml

@@ -0,0 +1,36 @@
+name: Bug Report
+description: Let us know about an issue so we can fix it
+labels: ["bug"]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Hello! Thanks for taking the time to file a bug report.
+
+        Before creating this issue, we kindly ask that you use the search functionality to see if anyone else has already reported this issue.
+  - type: textarea
+    attributes:
+      label: Describe the bug
+      description: What happened? What were you trying to do? How did it break? Which errors were emitted? Provide as much information as possible.
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      label: Relevant log output
+  - type: textarea
+    attributes:
+      label: Describe your environment
+      description: |
+        What OS are you running on? Are you using the latest version of the Agents framework published on npmjs.com?
+        You can get all of your relevant environment information by running `npx envinfo --system --binaries --npmPackages "@livekit/*"`.
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      label: Minimal reproducible example
+      description: |
+        If possible, provide an example we can run on our end that can reasonably reproduce the issue that you're running into.
+  - type: textarea
+    attributes:
+      label: Additional information
+      description: If you have any other information to provide, such as hunches about where the error could come from, do so here.

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,8 @@
+blank_issues_enabled: false
+contact_links:
+  - name: Slack community chat
+    url: https://livekit.io/join-slack
+    about: Ask questions and discuss with other LiveKit users in real time
+  - name: LiveKit Agents for Python
+    url: https://github.com/livekit/agents
+    about: The canonical version of LiveKit Agents. Direct feature requests here

.github/ISSUE_TEMPLATE/question.yaml

@@ -0,0 +1,16 @@
+name: Question
+description: Ask for help with an issue you're facing
+labels: ["question"]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Hello! Thanks for taking the time to ask a question.
+
+        Before creating this issue, we kindly ask that you use the search functionality to see if anyone else has already asked this question.
+        Feel free to join us in the `#agents` channel on [our Slack](https://livekit.io/join-slack), and ask your question there to get quicker help from us and the community.
+  - type: textarea
+    attributes:
+      label: Your question
+    validations:
+      required: true

README.md

@@ -170,8 +170,8 @@ Refer to [the license](LICENSES/Apache-2.0.txt) for details.
 <br/><table>
 <thead><tr><th colspan="2">LiveKit Ecosystem</th></tr></thead>
 <tbody>
-<tr><td>Realtime SDKs</td><td><a href="https://github.com/livekit/client-sdk-js">Browser</a> · <a href="https://github.com/livekit/client-sdk-swift">iOS/macOS/visionOS</a> · <a href="https://github.com/livekit/client-sdk-android">Android</a> · <a href="https://github.com/livekit/client-sdk-flutter">Flutter</a> · <a href="https://github.com/livekit/client-sdk-react-native">React Native</a> · <a href="https://github.com/livekit/rust-sdks">Rust</a> · <a href="https://github.com/livekit/node-sdks">Node.js</a> · <a href="https://github.com/livekit/python-sdks">Python</a> · <a href="https://github.com/livekit/client-sdk-unity">Unity</a> · <a href="https://github.com/livekit/client-sdk-unity-web">Unity (WebGL)</a></td></tr><tr></tr>
-<tr><td>Server APIs</td><td><a href="https://github.com/livekit/node-sdks">Node.js</a> · <a href="https://github.com/livekit/server-sdk-go">Golang</a> · <a href="https://github.com/livekit/server-sdk-ruby">Ruby</a> · <a href="https://github.com/livekit/server-sdk-kotlin">Java/Kotlin</a> · <a href="https://github.com/livekit/python-sdks">Python</a> · <a href="https://github.com/livekit/rust-sdks">Rust</a> · <a href="https://github.com/agence104/livekit-server-sdk-php">PHP (community)</a></td></tr><tr></tr>
+<tr><td>LiveKit SDKs</td><td><a href="https://github.com/livekit/client-sdk-js">Browser</a> · <a href="https://github.com/livekit/client-sdk-swift">iOS/macOS/visionOS</a> · <a href="https://github.com/livekit/client-sdk-android">Android</a> · <a href="https://github.com/livekit/client-sdk-flutter">Flutter</a> · <a href="https://github.com/livekit/client-sdk-react-native">React Native</a> · <a href="https://github.com/livekit/rust-sdks">Rust</a> · <a href="https://github.com/livekit/node-sdks">Node.js</a> · <a href="https://github.com/livekit/python-sdks">Python</a> · <a href="https://github.com/livekit/client-sdk-unity">Unity</a> · <a href="https://github.com/livekit/client-sdk-unity-web">Unity (WebGL)</a></td></tr><tr></tr>
+<tr><td>Server APIs</td><td><a href="https://github.com/livekit/node-sdks">Node.js</a> · <a href="https://github.com/livekit/server-sdk-go">Golang</a> · <a href="https://github.com/livekit/server-sdk-ruby">Ruby</a> · <a href="https://github.com/livekit/server-sdk-kotlin">Java/Kotlin</a> · <a href="https://github.com/livekit/python-sdks">Python</a> · <a href="https://github.com/livekit/rust-sdks">Rust</a> · <a href="https://github.com/agence104/livekit-server-sdk-php">PHP (community)</a> · <a href="https://github.com/pabloFuente/livekit-server-sdk-dotnet">.NET (community)</a></td></tr><tr></tr>
 <tr><td>UI Components</td><td><a href="https://github.com/livekit/components-js">React</a> · <a href="https://github.com/livekit/components-android">Android Compose</a> · <a href="https://github.com/livekit/components-swift">SwiftUI</a></td></tr><tr></tr>
 <tr><td>Agents Frameworks</td><td><a href="https://github.com/livekit/agents">Python</a> · <b>Node.js</b> · <a href="https://github.com/livekit/agent-playground">Playground</a></td></tr><tr></tr>
 <tr><td>Services</td><td><a href="https://github.com/livekit/livekit">LiveKit server</a> · <a href="https://github.com/livekit/egress">Egress</a> · <a href="https://github.com/livekit/ingress">Ingress</a> · <a href="https://github.com/livekit/sip">SIP</a></td></tr><tr></tr>

agents/CHANGELOG.md

@@ -1,5 +1,37 @@
 # @livekit/agents
 
+## 0.6.4
+
+### Patch Changes
+
+- fix(proc): clearer errors on crash - [#271](https://github.com/livekit/agents-js/pull/271) ([@nbsp](https://github.com/nbsp))
+
+- fix(metrics): remove ErrorOptions from MultimodalError - [#278](https://github.com/livekit/agents-js/pull/278) ([@nbsp](https://github.com/nbsp))
+
+## 0.6.3
+
+### Patch Changes
+
+- fix LLM retries breaking on VoicePipelineAgent - [#265](https://github.com/livekit/agents-js/pull/265) ([@nbsp](https://github.com/nbsp))
+
+## 0.6.2
+
+### Patch Changes
+
+- fix(pipeline): double LLM replies - [#259](https://github.com/livekit/agents-js/pull/259) ([@nbsp](https://github.com/nbsp))
+
+- update rtc-node to 0.13.2 to fix issue with e2ee - [#258](https://github.com/livekit/agents-js/pull/258) ([@nbsp](https://github.com/nbsp))
+
+## 0.6.1
+
+### Patch Changes
+
+- remove incorrect !-assertion in queue implementation leading to race conditions - [#240](https://github.com/livekit/agents-js/pull/240) ([@nbsp](https://github.com/nbsp))
+
+- chore(bump): rtc-node -> 0.13.1 - [#247](https://github.com/livekit/agents-js/pull/247) ([@nbsp](https://github.com/nbsp))
+
+- re-request audio response in multimodal agent when text is given - [#243](https://github.com/livekit/agents-js/pull/243) ([@nbsp](https://github.com/nbsp))
+
 ## 0.6.0
 
 ### Minor Changes

agents/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@livekit/agents",
-  "version": "0.6.0",
+  "version": "0.6.4",
   "description": "LiveKit Agents - Node.js",
   "main": "dist/index.js",
   "require": "dist/index.cjs",

plugins/cartesia/CHANGELOG.md

@@ -0,0 +1,21 @@
+# @livekit/agents-plugin-cartesia
+
+## 0.1.1
+
+### Patch Changes
+
+- update rtc-node to 0.13.2 to fix issue with e2ee - [#258](https://github.com/livekit/agents-js/pull/258) ([@nbsp](https://github.com/nbsp))
+
+- Updated dependencies [[`dedb1cf139c8af4ce8709c86440c818157f5b475`](https://github.com/livekit/agents-js/commit/dedb1cf139c8af4ce8709c86440c818157f5b475), [`f3258b948539406213c15f8e817449b2588cde84`](https://github.com/livekit/agents-js/commit/f3258b948539406213c15f8e817449b2588cde84)]:
+  - @livekit/[email protected]
+
+## 0.1.0
+
+### Minor Changes
+
+- plugins(cartesia): init with TTS - [#217](https://github.com/livekit/agents-js/pull/217) ([@nbsp](https://github.com/nbsp))
+
+### Patch Changes
+
+- Updated dependencies [[`4a66a82fc2fd0a25e30bdaa0bd095804c65ee101`](https://github.com/livekit/agents-js/commit/4a66a82fc2fd0a25e30bdaa0bd095804c65ee101), [`01aaa85445bbb8f30afe9c16360afb5a45c38e9e`](https://github.com/livekit/agents-js/commit/01aaa85445bbb8f30afe9c16360afb5a45c38e9e), [`4b7504654c73d9111d39e90d325d5f660b2c8ad9`](https://github.com/livekit/agents-js/commit/4b7504654c73d9111d39e90d325d5f660b2c8ad9)]:
+  - @livekit/[email protected]

plugins/cartesia/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@livekit/agents-plugin-cartesia",
-  "version": "0.0.0",
+  "version": "0.1.1",
   "description": "Cartesia plugin for LiveKit Node Agents",
   "main": "dist/index.js",
   "require": "dist/index.cjs",

plugins/deepgram/CHANGELOG.md

@@ -1,5 +1,22 @@
 # @livekit/agents-plugin-deepgram
 
+## 0.5.3
+
+### Patch Changes
+
+- update rtc-node to 0.13.2 to fix issue with e2ee - [#258](https://github.com/livekit/agents-js/pull/258) ([@nbsp](https://github.com/nbsp))
+
+- update rtc-node - [#254](https://github.com/livekit/agents-js/pull/254) ([@nbsp](https://github.com/nbsp))
+
+- Updated dependencies [[`dedb1cf139c8af4ce8709c86440c818157f5b475`](https://github.com/livekit/agents-js/commit/dedb1cf139c8af4ce8709c86440c818157f5b475), [`f3258b948539406213c15f8e817449b2588cde84`](https://github.com/livekit/agents-js/commit/f3258b948539406213c15f8e817449b2588cde84)]:
+  - @livekit/agents@0.6.2
+
+## 0.5.2
+
+### Patch Changes
+
+- update rtc-node - [#255](https://github.com/livekit/agents-js/pull/255) ([@nbsp](https://github.com/nbsp))
+
 ## 0.5.1
 
 ### Patch Changes

plugins/deepgram/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@livekit/agents-plugin-deepgram",
-  "version": "0.5.1",
+  "version": "0.5.3",
   "description": "Deepgram plugin for LiveKit Agents for Node.js",
   "main": "dist/index.js",
   "require": "dist/index.cjs",
@@ -40,6 +40,7 @@
     "typescript": "^5.0.0"
   },
   "dependencies": {
+    "@livekit/agents-plugin-openai": "^0.8.1",
     "ws": "^8.16.0"
   },
   "peerDependencies": {

plugins/deepgram/src/index.ts

@@ -3,3 +3,4 @@
 // SPDX-License-Identifier: Apache-2.0
 
 export * from './stt.js';
+export * from './tts.js';

plugins/deepgram/src/models.ts

@@ -2,6 +2,22 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 
+export type TTSModels =
+  | 'aura-asteria-en'
+  | 'aura-luna-en'
+  | 'aura-stella-en'
+  | 'aura-athena-en'
+  | 'aura-hera-en'
+  | 'aura-orion-en'
+  | 'aura-arcas-en'
+  | 'aura-perseus-en'
+  | 'aura-angus-en'
+  | 'aura-orpheus-en'
+  | 'aura-helios-en'
+  | 'aura-zeus-en';
+
+export type TTSEncoding = 'linear16' | 'mulaw' | 'alaw' | 'mp3' | 'opus' | 'flac' | 'aac';
+
 export type STTModels =
   | 'nova-general'
   | 'nova-phonecall'

plugins/deepgram/src/tts.spec.ts

@@ -0,0 +1,15 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { STT } from '@livekit/agents-plugin-openai';
+import { tts } from '@livekit/agents-plugins-test';
+import { describe } from 'vitest';
+import { TTS } from './tts.js';
+
+describe(
+  'Deepgram',
+  { timeout: 30_000 }, // For some reason it takes a while before deepgram closes the connection
+  async () => {
+    await tts(new TTS(), new STT());
+  },
+);