diff --git a/src/board/audio/index.ts b/src/board/audio/index.ts index 233ada0..6c6beeb 100644 --- a/src/board/audio/index.ts +++ b/src/board/audio/index.ts @@ -1,3 +1,4 @@ +import { SRC } from "@alexanderolsen/libsamplerate-js/dist/src"; import { replaceBuiltinSound } from "./built-in-sounds"; import { SoundEmojiSynthesizer } from "./sound-emoji-synthesizer"; import { parseSoundEffects } from "./sound-expressions"; @@ -15,7 +16,10 @@ declare global { interface AudioOptions { defaultAudioCallback: () => void; + defaultResampler: SRC; speechAudioCallback: () => void; + speechResampler: SRC; + soundExpressionResampler: SRC; } export class BoardAudio { @@ -38,7 +42,10 @@ export class BoardAudio { initializeCallbacks({ defaultAudioCallback, + defaultResampler, speechAudioCallback, + speechResampler, + soundExpressionResampler, }: AudioOptions) { if (!this.context) { throw new Error("Context must be pre-created from a user event"); @@ -60,16 +67,19 @@ export class BoardAudio { this.default = new BufferedAudio( this.context, this.volumeNode, + defaultResampler, defaultAudioCallback ); this.speech = new BufferedAudio( this.context, this.volumeNode, + speechResampler, speechAudioCallback ); this.soundExpression = new BufferedAudio( this.context, this.volumeNode, + soundExpressionResampler, () => { if (this.currentSoundExpressionCallback) { this.currentSoundExpressionCallback(); @@ -79,12 +89,13 @@ export class BoardAudio { } async createAudioContextFromUserInteraction(): Promise { + // If we set a 44.1kHz rate then we fail to connect to user media on Mac as it selects 48000 + // So we leave it at the default hoping it's most likely to match user media... + // Until there's progress on this there doesn't seem a better way: + // https://bugzilla.mozilla.org/show_bug.cgi?id=1674892 this.context = - this.context ?? - new (window.AudioContext || window.webkitAudioContext)({ - // The highest rate is the sound expression synth. - sampleRate: 44100, - }); + this.context ?? new (window.AudioContext || window.webkitAudioContext)(); + if (this.context.state === "suspended") { return this.context.resume(); } @@ -96,21 +107,16 @@ export class BoardAudio { this.stopSoundExpression(); }; const synth = new SoundEmojiSynthesizer(0, onDone); + this.soundExpression!.setSampleRate(synth.sampleRate); synth.play(soundEffects); const callback = () => { const source = synth.pull(); if (this.context) { - // Use createBuffer instead of new AudioBuffer to support Safari 14.0. - const target = this.context.createBuffer( - 1, - source.length, - synth.sampleRate - ); - const channel = target.getChannelData(0); + const target = new Float32Array(source.length); for (let i = 0; i < source.length; i++) { // Buffer is (0, 1023) we need to map it to (-1, 1) - channel[i] = (source[i] - 512) / 512; + target[i] = (source[i] - 512) / 512; } this.soundExpression!.writeData(target); } @@ -201,6 +207,7 @@ export class BoardAudio { try { micStream = await navigator.mediaDevices.getUserMedia({ video: false, + // It seems Firefox ignores the rate set here audio: true, }); } catch (e) { @@ -271,26 +278,41 @@ class BufferedAudio { constructor( private context: AudioContext, private destination: AudioNode, + private resampler: SRC, private callback: () => void - ) {} + ) { + this.resampler.outputSampleRate = this.context.sampleRate; + } init(sampleRate: number) { // This is called for each new audio source so don't reset nextStartTime // or we start to overlap audio - this.sampleRate = sampleRate; - } - - createBuffer(length: number) { - // Use createBuffer instead of new AudioBuffer to support Safari 14.0. - return this.context.createBuffer(1, length, this.sampleRate); + this.setSampleRate(sampleRate); } setSampleRate(sampleRate: number) { this.sampleRate = sampleRate; + this.resampler.inputSampleRate = sampleRate; } - writeData(buffer: AudioBuffer) { - // Use createBufferSource instead of new AudioBufferSourceNode to support Safari 14.0. + writeData(data: Float32Array) { + let sampleRate = this.sampleRate; + // In practice the supported range is less than the 8k..96k required by the spec + if (sampleRate < 8_000 || sampleRate > 96_000) { + // We need to resample + //sampleRate = this.resampler.outputSampleRate; + //data = this.resampler.full(data); + } + console.log( + "Using actual rate", + sampleRate, + "for requested rate", + this.sampleRate + ); + + // Use createXXX instead to support Safari 14.0. + const buffer = this.context.createBuffer(1, data.length, sampleRate); + buffer.copyToChannel(data, 0); const source = this.context.createBufferSource(); source.buffer = buffer; source.onended = this.callCallback; diff --git a/src/board/conversions.ts b/src/board/conversions.ts index 392d780..f42bca7 100644 --- a/src/board/conversions.ts +++ b/src/board/conversions.ts @@ -98,12 +98,11 @@ export function convertAccelerometerNumberToString(value: number): string { export const convertAudioBuffer = ( heap: Uint8Array, source: number, - target: AudioBuffer + target: Float32Array ) => { - const channel = target.getChannelData(0); - for (let i = 0; i < channel.length; ++i) { + for (let i = 0; i < target.length; ++i) { // Convert from uint8 to -1..+1 float. - channel[i] = (heap[source + i] / 255) * 2 - 1; + target[i] = (heap[source + i] / 255) * 2 - 1; } return target; }; diff --git a/src/board/index.ts b/src/board/index.ts index fa9f543..96d0b7a 100644 --- a/src/board/index.ts +++ b/src/board/index.ts @@ -1,3 +1,4 @@ +import { create as createResampler } from "@alexanderolsen/libsamplerate-js"; import svgText from "../microbit-drawing.svg"; import { Accelerometer } from "./accelerometer"; import { BoardAudio } from "./audio"; @@ -247,10 +248,20 @@ export class Board { noInitialRun: true, instantiateWasm, }); + + // We update the sample rates before use. + const defaultResampler = await createResampler(1, 48000, 48000); + const speechResampler = await createResampler(1, 48000, 48000); + // Probably this one is never used so would be nice to avoid + const soundExpressionResampler = await createResampler(1, 48000, 48000); + const module = new ModuleWrapper(wrapped); this.audio.initializeCallbacks({ defaultAudioCallback: wrapped._microbit_hal_audio_raw_ready_callback, + defaultResampler, speechAudioCallback: wrapped._microbit_hal_audio_speech_ready_callback, + speechResampler, + soundExpressionResampler, }); this.accelerometer.initializeCallbacks( wrapped._microbit_hal_gesture_callback diff --git a/src/jshal.js b/src/jshal.js index c563973..160cc9c 100644 --- a/src/jshal.js +++ b/src/jshal.js @@ -230,8 +230,7 @@ mergeInto(LibraryManager.library, { Module.conversions.convertAudioBuffer( Module.HEAPU8, buf, - // @ts-expect-error - Module.board.audio.default.createBuffer(num_samples) + new Float32Array(num_samples) ) ); }, @@ -246,21 +245,10 @@ mergeInto(LibraryManager.library, { /** @type {number} */ num_samples ) { /** @type {AudioBuffer | undefined} */ let webAudioBuffer; - try { - // @ts-expect-error - webAudioBuffer = Module.board.audio.speech.createBuffer(num_samples); - } catch (e) { - // Swallow error on older Safari to keep the sim in a good state. - // @ts-expect-error - if (e.name === "NotSupportedError") { - return; - } else { - throw e; - } - } + const jsBuf = new Float32Array(num_samples); // @ts-expect-error Module.board.audio.speech.writeData( - Module.conversions.convertAudioBuffer(Module.HEAPU8, buf, webAudioBuffer) + Module.conversions.convertAudioBuffer(Module.HEAPU8, buf, jsBuf) ); },