feat: Added support for audio transcription on the client side.

ZuoFuhong · Mar 23, 2023 · baf6f4d · baf6f4d
1 parent 8820b78
commit baf6f4d
Show file tree

Hide file tree

Showing 3 changed files with 34 additions and 24 deletions.
diff --git a/frontend/app-im/package.json b/frontend/app-im/package.json
@@ -12,6 +12,7 @@
     "axios": "^1.3.4",
     "core-js": "^3.6.5",
     "element-ui": "^2.4.5",
+    "recorder-core": "^1.2.23020100",
     "vue": "^2.6.11",
     "vue-router": "^3.2.0"
   },

diff --git a/frontend/app-im/src/App.vue b/frontend/app-im/src/App.vue
@@ -110,8 +110,10 @@
 <script>
 import Axios from './utils/axios'
 import Websocket from './utils/websocket'
-import AudioRecorder from './utils/audio'
 import {getUserSettings} from './settings'
+import Recorder from 'recorder-core'
+import 'recorder-core/src/engine/mp3'
+import 'recorder-core/src/engine/mp3-engine'
 import {trim} from './utils/strings'
 
 export default {
@@ -142,6 +144,7 @@ export default {
     await this.loadUserFriends(settings.user_id)
     await this.initWSConn(settings.ws_url)
     await this.initAudioDevice(settings.http_url)
+    this.settings = settings
     // 激活会话
     await this.autoActiveUserSession()
   },
@@ -192,40 +195,41 @@ export default {
       }
     },
     // 初始化音频驱动
-    async initAudioDevice(address) {
-      const that = this
-      let audioDevice = new AudioRecorder()
-      // 注册回调-音频转文本
-      audioDevice.addOnStopCallback(function(blob) {
-        var data = new FormData()
-        data.append('model', 'whisper-1')
-        data.append('file', blob, 'translate_tts.mp3')
-
-        const xhr = new XMLHttpRequest()
-        // xhr.withCredentials = true
-        xhr.addEventListener('readystatechange', function() {
-            if(this.readyState === 4) {
-              const rsp = JSON.parse(this.responseText)
-              // 发送转录的文本
-              that.wsClientSendToUser(rsp.text)
-            }
-        })
-        xhr.open('POST', address + '/audio/transcriptions')
-        xhr.send(data)
+    async initAudioDevice() {
+      var rec = Recorder();
+      rec.open(function(){
+          console.log("open audio device success.")
       })
-      this.audioDevice = audioDevice;
+      this.audioDevice = rec;
     },
     // 录制音频
     audioRecord() {
       if (!this.recording) {
         // 开始录制
         console.log('start audio record')
-        this.audioDevice.startRecord()
+        this.audioDevice.start()
         this.recording = true
       } else {
         // 停止录制
         console.log('stop audio record')
-        this.audioDevice.stopRecord()
+        var that = this;
+        this.audioDevice.stop(function(blob){
+            var data = new FormData()
+            data.append('model', 'whisper-1')
+            data.append('file', blob, 'translate_tts.mp3')
+
+            const xhr = new XMLHttpRequest()
+            // xhr.withCredentials = true
+            xhr.addEventListener('readystatechange', function() {
+                if(this.readyState === 4) {
+                    const rsp = JSON.parse(this.responseText)
+                    // 发送转录的文本
+                    that.wsClientSendToUser(rsp.text)
+                }
+            })
+            xhr.open('POST', that.settings.http_url + '/audio/transcriptions')
+            xhr.send(data)
+        })
         this.recording = false
       }
     },

diff --git a/frontend/app-im/yarn.lock b/frontend/app-im/yarn.lock
@@ -7612,6 +7612,11 @@ readdirp@~3.6.0:
   dependencies:
     picomatch "^2.2.1"
 
+recorder-core@^1.2.23020100:
+  version "1.2.23020100"
+  resolved "https://registry.yarnpkg.com/recorder-core/-/recorder-core-1.2.23020100.tgz#135e1119902bb19027862a579c608b64d6497bac"
+  integrity sha512-HUxAPKmP0Q5Rd0Z57ufKGyJmUAB9cZPfe8vg4mckSDwx1JFJ+tln8i+jnGSCOjlKr6tC2UL4ifcWwa7w0jf5pA==
+
 redent@^3.0.0:
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/redent/-/redent-3.0.0.tgz#e557b7998316bb53c9f1f56fa626352c6963059f"