Microphone: get as far as reading samples

We'll still need to resample them and write them to the buffer.

Author: microbit-matt-hillsdon

src/board/audio/index.ts

@@ -26,6 +26,7 @@ export class BoardAudio {
   speech: BufferedAudio | undefined;
   soundExpression: BufferedAudio | undefined;
   currentSoundExpressionCallback: undefined | (() => void);
+  private stopActiveRecording: (() => void) | undefined;
 
   constructor() {}
 
@@ -155,7 +156,64 @@ export class BoardAudio {
     }
   }
 
+  isRecording(): boolean {
+    return !!this.stopActiveRecording;
+  }
+
+  stopRecording() {
+    if (this.stopActiveRecording) {
+      this.stopActiveRecording();
+    }
+  }
+
+  async startRecording(
+    onChunk: (chunk: Float32Array, sampleRate: number) => void
+  ) {
+    if (!navigator?.mediaDevices?.getUserMedia) {
+      return;
+    }
+    this.stopRecording();
+
+    this.stopActiveRecording = () => {};
+    let micStream: MediaStream | undefined;
+    try {
+      micStream = await navigator.mediaDevices.getUserMedia({
+        video: false,
+        audio: true,
+      });
+    } catch (e) {
+      console.error(e);
+      this.stopRecording();
+      return;
+    }
+
+    const source = this.context!.createMediaStreamSource(micStream);
+    // TODO: wire up microphone sensitivity to this gain node
+    const gain = this.context!.createGain();
+    source.connect(gain);
+    // TODO: consider AudioWorklet - worth it? Browser support?
+    const recorder = this.context!.createScriptProcessor(2048, 1, 1);
+    recorder.onaudioprocess = (e) => {
+      const samples = e.inputBuffer.getChannelData(0);
+      onChunk(samples, this.context!.sampleRate);
+    };
+    gain.connect(recorder);
+    recorder.connect(this.context!.destination);
+
+    this.stopActiveRecording = () => {
+      recorder.disconnect();
+      gain.disconnect();
+      source.disconnect();
+      this.stopActiveRecording = undefined;
+    };
+
+    setTimeout(() => {
+      this.stopRecording();
+    }, 5000);
+  }
+
   boardStopped() {
+    this.stopRecording();
     this.stopOscillator();
     this.speech?.dispose();
     this.soundExpression?.dispose();

src/board/microphone.ts

@@ -17,7 +17,6 @@ export class Microphone {
     150
   );
   private soundLevelCallback: SoundLevelCallback | undefined;
-  private _isRecording: boolean = false;
 
   constructor(
     private element: SVGElement,
@@ -67,54 +66,4 @@ export class Microphone {
   boardStopped() {
     this.microphoneOff();
   }
-
-  isRecording() {
-    return this._isRecording;
-  }
-
-  stopRecording() {
-    // TODO
-  }
-
-  async startRecording(onChunk: (chunk: ArrayBuffer) => void) {
-    if (!navigator?.mediaDevices?.getUserMedia) {
-      return;
-    }
-    if (this.isRecording()) {
-      this.stopRecording();
-      // Wait for it if needed
-    }
-    this._isRecording = true;
-
-    // This might not be the right recording approach as we want 8 bit PCM for AudioFrame
-    // and we're getting a fancy codec.
-    let mediaRecorder: MediaRecorder | undefined;
-    try {
-      const stream = await navigator.mediaDevices.getUserMedia({
-        video: false,
-        audio: true,
-      });
-      mediaRecorder = new MediaRecorder(stream);
-      mediaRecorder.start();
-
-      setTimeout(() => {
-        if (mediaRecorder) {
-          mediaRecorder.stop();
-        }
-      }, 5000);
-
-      mediaRecorder.ondataavailable = async (e: BlobEvent) => {
-        const buffer = await e.data.arrayBuffer();
-        onChunk(buffer);
-      };
-      mediaRecorder.onstop = async () => {
-        this._isRecording = false;
-      };
-    } catch (error) {
-      if (mediaRecorder) {
-        mediaRecorder.stop();
-      }
-      this._isRecording = false;
-    }
-  }
 }

src/board/pins.ts

@@ -9,6 +9,8 @@ export interface Pin {
 
   isTouched(): boolean;
 
+  getAndClearTouches(): number;
+
   boardStopped(): void;
 
   setAnalogPeriodUs(period: number): number;
@@ -44,6 +46,10 @@ abstract class BasePin implements Pin {
     return this.analogPeriodUs;
   }
 
+  getAndClearTouches(): number {
+    return 0;
+  }
+
   isTouched(): boolean {
     return false;
   }

src/demo.html

@@ -95,6 +95,7 @@ <h1>MicroPython-micro:bit simulator example embedding</h1>
               <option value="pin_touches">Pin touches</option>
               <option value="radio">Radio</option>
               <option value="random">Random</option>
+              <option value="record">Record</option>
               <option value="sensors">Sensors</option>
               <option value="sound_effects_builtin">
                 Sound effects (builtin)

src/examples/record.py

@@ -0,0 +1,3 @@
+from microbit import microphone
+
+microphone.record(5000);

src/jshal.js

@@ -286,25 +286,24 @@ mergeInto(LibraryManager.library, {
       value
     );
   },
-
   mp_js_hal_microphone_start_recording: function (
     /** @type {number} */ buf,
     /** @type {number} */ max_len,
     /** @type {number} */ cur_len,
     /** @type {number} */ rate
   ) {
-    Module.board.microphone.startRecording(function (
-      chunk
-    ) /** @type {ArrayBuffer} */
-    {
-      console.log("Chunk", chunk);
+    Module.board.audio.startRecording(function (
+      /** @type {Float32Array} */ chunk,
+      /** @type {number} */ actualSampleRate
+    ) {
+      // TODO: convert from float to int and resample here
     });
   },
   mp_js_hal_microphone_is_recording: function () {
-    return Module.board.microphone.isRecording();
+    return Module.board.audio.isRecording();
   },
   mp_js_hal_microphone_stop_recording: function () {
-    Module.board.microphone.stopRecording();
+    Module.board.audio.stopRecording();
   },
   mp_js_hal_microphone_get_level: function () {
     return Module.board.microphone.soundLevel.value;