From 45f9cc48bafca59517892e8cdd274be32810e2f7 Mon Sep 17 00:00:00 2001 From: Carl Woffenden Date: Wed, 16 Oct 2024 11:29:37 +0200 Subject: [PATCH 1/5] Logging and notes for me --- src/audio_worklet.js | 9 ++++++--- src/library_webaudio.js | 3 +++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/audio_worklet.js b/src/audio_worklet.js index 6ed827af22e7..3cb23cbc00fd 100644 --- a/src/audio_worklet.js +++ b/src/audio_worklet.js @@ -64,6 +64,9 @@ function createWasmAudioWorkletProcessor(audioParams) { // Allocate the necessary stack space. inputsPtr = stackAlloc(stackMemoryNeeded); +#if WEBAUDIO_DEBUG + console.log(`WasmAudioWorkletProcessorr::process() ${inputsPtr} (needed: ${stackMemoryNeeded})`); +#endif // Copy input audio descriptor structs and data to Wasm k = inputsPtr >> 2; @@ -115,9 +118,9 @@ function createWasmAudioWorkletProcessor(audioParams) { // (A garbage-free function TypedArray.copy(dstTypedArray, dstOffset, // srcTypedArray, srcOffset, count) would sure be handy.. but web does // not have one, so manually copy all bytes in) - for (i of outputList) { - for (j of i) { - for (k = 0; k < this.samplesPerChannel; ++k) { + for (/*which output*/ i of outputList) { + for (/*which channel Float32Array*/ j of i) { + for (/*channel index*/ k = 0; k < this.samplesPerChannel; ++k) { j[k] = HEAPF32[outputDataPtr++]; } } diff --git a/src/library_webaudio.js b/src/library_webaudio.js index f4269e9759ba..3553a05d5695 100644 --- a/src/library_webaudio.js +++ b/src/library_webaudio.js @@ -161,6 +161,9 @@ let LibraryWebAudio = { #if WEBAUDIO_DEBUG console.log(`emscripten_start_wasm_audio_worklet_thread_async() adding audioworklet.js...`); #endif +#if WEBAUDIO_DEBUG + console.log(`emscripten_start_wasm_audio_worklet_thread_async() stack base/sb: ${stackLowestAddress}, size: ${stackSize} (${stackLowestAddress + stackSize})`); +#endif let audioWorkletCreationFailed = () => { #if WEBAUDIO_DEBUG From 53e20ca296426bdd67da1cf9a5c2df68ab18cc9c Mon Sep 17 00:00:00 2001 From: Carl Woffenden Date: Wed, 16 Oct 2024 16:19:56 +0200 Subject: [PATCH 2/5] Better error message (to see why it fails) --- src/library_webaudio.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/library_webaudio.js b/src/library_webaudio.js index 3553a05d5695..f3b01f633ea5 100644 --- a/src/library_webaudio.js +++ b/src/library_webaudio.js @@ -161,13 +161,13 @@ let LibraryWebAudio = { #if WEBAUDIO_DEBUG console.log(`emscripten_start_wasm_audio_worklet_thread_async() adding audioworklet.js...`); #endif -#if WEBAUDIO_DEBUG - console.log(`emscripten_start_wasm_audio_worklet_thread_async() stack base/sb: ${stackLowestAddress}, size: ${stackSize} (${stackLowestAddress + stackSize})`); -#endif let audioWorkletCreationFailed = () => { #if WEBAUDIO_DEBUG - console.error(`emscripten_start_wasm_audio_worklet_thread_async() addModule() failed!`); + // Note about Cross-Origin here: a lack of Cross-Origin-Opener-Policy and + // Cross-Origin-Embedder-Policy headers to the client request will result + // in the worklet file failing to load. + console.error(`emscripten_start_wasm_audio_worklet_thread_async() addModule() failed! Are the Cross-Origin headers being set?`); #endif {{{ makeDynCall('viip', 'callback') }}}(contextHandle, 0/*EM_FALSE*/, userData); }; From 9b3dbb6a2db9731571194434e2c3dc6281442a4b Mon Sep 17 00:00:00 2001 From: Carl Woffenden Date: Wed, 16 Oct 2024 16:21:01 +0200 Subject: [PATCH 3/5] Create one-time fixed views into the heap We can remove the float-by-float JS copy and replace with this simple TypedArray set() calls. --- src/audio_worklet.js | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/src/audio_worklet.js b/src/audio_worklet.js index 3cb23cbc00fd..dd9a1821939b 100644 --- a/src/audio_worklet.js +++ b/src/audio_worklet.js @@ -37,6 +37,9 @@ function createWasmAudioWorkletProcessor(audioParams) { // 'render quantum size', and this exercise of passing in the value // shouldn't be required (to be verified). this.samplesPerChannel = opts['sc']; + // Typed views of the output buffers on the worklet's stack, which after + // creation should not change (since the stack is passed externally once). + this.outputViews = null; } static get parameterDescriptors() { @@ -64,9 +67,6 @@ function createWasmAudioWorkletProcessor(audioParams) { // Allocate the necessary stack space. inputsPtr = stackAlloc(stackMemoryNeeded); -#if WEBAUDIO_DEBUG - console.log(`WasmAudioWorkletProcessorr::process() ${inputsPtr} (needed: ${stackMemoryNeeded})`); -#endif // Copy input audio descriptor structs and data to Wasm k = inputsPtr >> 2; @@ -97,6 +97,20 @@ function createWasmAudioWorkletProcessor(audioParams) { // Reserve space for the output data dataPtr += bytesPerChannel * i.length; } + if (!this.outputViews) { + this.outputViews = []; + k = outputDataPtr; + for (/*which output*/ i of outputList) { + for (/*which channel*/ j of i) { + this.outputViews.push({ + // dataPtr is the sanity check (to be implemented) + // dataSub is the one-time subarray into the heap + dataPtr: k, + dataSub: HEAPF32.subarray(k, k += this.samplesPerChannel) + }); + } + } + } // Copy parameters descriptor structs and data to Wasm paramsPtr = dataPtr; @@ -115,14 +129,12 @@ function createWasmAudioWorkletProcessor(audioParams) { // Call out to Wasm callback to perform audio processing if (didProduceAudio = this.callbackFunction(numInputs, inputsPtr, numOutputs, outputsPtr, numParams, paramsPtr, this.userData)) { // Read back the produced audio data to all outputs and their channels. - // (A garbage-free function TypedArray.copy(dstTypedArray, dstOffset, - // srcTypedArray, srcOffset, count) would sure be handy.. but web does - // not have one, so manually copy all bytes in) - for (/*which output*/ i of outputList) { - for (/*which channel Float32Array*/ j of i) { - for (/*channel index*/ k = 0; k < this.samplesPerChannel; ++k) { - j[k] = HEAPF32[outputDataPtr++]; - } + // The 'outputViews' are subarray views into the heap, each with the + // correct offset and size to be copied directly into the output. + k = 0; + for (i of outputList) { + for (j of i) { + j.set(this.outputViews[k++].dataSub); } } } From d4680ca6fa76f06b51f1f2b85de4f72f7e5c5ba5 Mon Sep 17 00:00:00 2001 From: Carl Woffenden Date: Thu, 17 Oct 2024 18:15:25 +0200 Subject: [PATCH 4/5] Allow the number of channels to increase (or the audio chain to change) Typed views are recreated if needed but otherwise are reused. --- src/audio_worklet.js | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/src/audio_worklet.js b/src/audio_worklet.js index dd9a1821939b..b55ba8d4347e 100644 --- a/src/audio_worklet.js +++ b/src/audio_worklet.js @@ -38,8 +38,8 @@ function createWasmAudioWorkletProcessor(audioParams) { // shouldn't be required (to be verified). this.samplesPerChannel = opts['sc']; // Typed views of the output buffers on the worklet's stack, which after - // creation should not change (since the stack is passed externally once). - this.outputViews = null; + // creation should only change if the audio chain changes. + this.outputViews = []; } static get parameterDescriptors() { @@ -57,7 +57,7 @@ function createWasmAudioWorkletProcessor(audioParams) { bytesPerChannel = this.samplesPerChannel * 4, stackMemoryNeeded = (numInputs + numOutputs) * {{{ C_STRUCTS.AudioSampleFrame.__size__ }}}, oldStackPtr = stackSave(), - inputsPtr, outputsPtr, outputDataPtr, paramsPtr, + inputsPtr, outputsPtr, outputDataPtr, paramsPtr, requiredViews = 0, didProduceAudio, paramArray; // Calculate how much stack space is needed. @@ -96,18 +96,22 @@ function createWasmAudioWorkletProcessor(audioParams) { k += {{{ C_STRUCTS.AudioSampleFrame.__size__ / 4 }}}; // Reserve space for the output data dataPtr += bytesPerChannel * i.length; + // How many output views are needed in total? + requiredViews += i.length; } - if (!this.outputViews) { + + // Verify we have enough views (it doesn't matter if we have too many, any + // excess won't be accessed) then also verify the views' start address + // hasn't changed. + // TODO: allocate space for outputDataPtr before any inputs? + k = outputDataPtr; + if (this.outputViews.length < requiredViews || (this.outputViews.length && this.outputViews[0].byteOffset != k << 2)) { this.outputViews = []; - k = outputDataPtr; for (/*which output*/ i of outputList) { for (/*which channel*/ j of i) { - this.outputViews.push({ - // dataPtr is the sanity check (to be implemented) - // dataSub is the one-time subarray into the heap - dataPtr: k, - dataSub: HEAPF32.subarray(k, k += this.samplesPerChannel) - }); + this.outputViews.push( + HEAPF32.subarray(k, k += this.samplesPerChannel) + ); } } } @@ -134,7 +138,7 @@ function createWasmAudioWorkletProcessor(audioParams) { k = 0; for (i of outputList) { for (j of i) { - j.set(this.outputViews[k++].dataSub); + j.set(this.outputViews[k++]); } } } From baf7a595de57d4746071082daa8558d74cc887be Mon Sep 17 00:00:00 2001 From: Carl Woffenden Date: Fri, 18 Oct 2024 19:05:02 +0200 Subject: [PATCH 5/5] Work in progress, moved the output buffers first Lots of juggling with the various pointers, and next will be to reduce the code and move all of the output first to stop repeating some of the calculations. Some can also move to the constructor. --- src/audio_worklet.js | 78 ++++++++++++++++++++++++++++---------------- 1 file changed, 50 insertions(+), 28 deletions(-) diff --git a/src/audio_worklet.js b/src/audio_worklet.js index b55ba8d4347e..cfb932578312 100644 --- a/src/audio_worklet.js +++ b/src/audio_worklet.js @@ -33,9 +33,9 @@ function createWasmAudioWorkletProcessor(audioParams) { this.userData = opts['ud']; // Then the samples per channel to process, fixed for the lifetime of the // context that created this processor. Note for when moving to Web Audio - // 1.1: the typed array passed to process() should be the same size as this - // 'render quantum size', and this exercise of passing in the value - // shouldn't be required (to be verified). + // 1.1: the typed array passed to process() should be the same size as + // this 'render quantum size', and this exercise of passing in the value + // shouldn't be required (to be verified with the in/out data lengths). this.samplesPerChannel = opts['sc']; // Typed views of the output buffers on the worklet's stack, which after // creation should only change if the audio chain changes. @@ -47,6 +47,7 @@ function createWasmAudioWorkletProcessor(audioParams) { } process(inputList, outputList, parameters) { + var time = Date.now(); // Marshal all inputs and parameters to the Wasm memory on the thread stack, // then perform the wasm audio worklet call, // and finally marshal audio output data back. @@ -65,12 +66,47 @@ function createWasmAudioWorkletProcessor(audioParams) { for (i of outputList) stackMemoryNeeded += i.length * bytesPerChannel; for (i in parameters) stackMemoryNeeded += parameters[i].byteLength + {{{ C_STRUCTS.AudioParamFrame.__size__ }}}, ++numParams; - // Allocate the necessary stack space. - inputsPtr = stackAlloc(stackMemoryNeeded); + // Allocate the necessary stack space (dataPtr is always in bytes, and + // advances as space for structs as data is taken, but note the switching + // between bytes and indices into the various heaps). + dataPtr = stackAlloc(stackMemoryNeeded); + + // But start the output view allocs first, since once views are created we + // want them to always start from the same address. Even if in the next + // process() the outputList is empty, as soon as there are channels again + // the views' addresses will still be the same (and only if more views are + // required we will recreate them). + outputDataPtr = dataPtr; + for (/*which output*/i of outputList) { +#if ASSERTIONS + for (/*which channel*/ j of i) { + console.assert(j.byteLength === bytesPerChannel, `Unexpected AudioWorklet output buffer size (expected ${bytesPerChannel} got ${j.byteLength})`); + } +#endif + // Keep advancing to make room for the output views + dataPtr += bytesPerChannel * i.length; + // How many output views are needed in total? + requiredViews += i.length; + } + // Verify we have enough views (it doesn't matter if we have too many, any + // excess won't be accessed) then also verify the views' start address + // haven't changed. + if (this.outputViews.length < requiredViews || (this.outputViews.length && this.outputViews[0].byteOffset != outputDataPtr)) { + this.outputViews = []; + k = outputDataPtr >> 2; + for (i of outputList) { + for (j of i) { + this.outputViews.push( + HEAPF32.subarray(k, k += this.samplesPerChannel) + ); + } + } + } // Copy input audio descriptor structs and data to Wasm + inputsPtr = dataPtr; k = inputsPtr >> 2; - dataPtr = inputsPtr + numInputs * {{{ C_STRUCTS.AudioSampleFrame.__size__ }}}; + dataPtr += numInputs * {{{ C_STRUCTS.AudioSampleFrame.__size__ }}}; for (i of inputList) { // Write the AudioSampleFrame struct instance HEAPU32[k + {{{ C_STRUCTS.AudioSampleFrame.numberOfChannels / 4 }}}] = i.length; @@ -85,35 +121,18 @@ function createWasmAudioWorkletProcessor(audioParams) { } // Copy output audio descriptor structs to Wasm + // TODO: now dataPtr tracks the next address, move this above outputsPtr = dataPtr; k = outputsPtr >> 2; - outputDataPtr = (dataPtr += numOutputs * {{{ C_STRUCTS.AudioSampleFrame.__size__ }}}) >> 2; + dataPtr += numOutputs * {{{ C_STRUCTS.AudioSampleFrame.__size__ }}}; for (i of outputList) { // Write the AudioSampleFrame struct instance HEAPU32[k + {{{ C_STRUCTS.AudioSampleFrame.numberOfChannels / 4 }}}] = i.length; HEAPU32[k + {{{ C_STRUCTS.AudioSampleFrame.samplesPerChannel / 4 }}}] = this.samplesPerChannel; - HEAPU32[k + {{{ C_STRUCTS.AudioSampleFrame.data / 4 }}}] = dataPtr; + HEAPU32[k + {{{ C_STRUCTS.AudioSampleFrame.data / 4 }}}] = outputDataPtr; k += {{{ C_STRUCTS.AudioSampleFrame.__size__ / 4 }}}; - // Reserve space for the output data - dataPtr += bytesPerChannel * i.length; - // How many output views are needed in total? - requiredViews += i.length; - } - - // Verify we have enough views (it doesn't matter if we have too many, any - // excess won't be accessed) then also verify the views' start address - // hasn't changed. - // TODO: allocate space for outputDataPtr before any inputs? - k = outputDataPtr; - if (this.outputViews.length < requiredViews || (this.outputViews.length && this.outputViews[0].byteOffset != k << 2)) { - this.outputViews = []; - for (/*which output*/ i of outputList) { - for (/*which channel*/ j of i) { - this.outputViews.push( - HEAPF32.subarray(k, k += this.samplesPerChannel) - ); - } - } + // Advance the output pointer to the next output + outputDataPtr += bytesPerChannel * i.length; } // Copy parameters descriptor structs and data to Wasm @@ -144,6 +163,9 @@ function createWasmAudioWorkletProcessor(audioParams) { } stackRestore(oldStackPtr); + + time = Date.now() - time; + //console.log(time); // Return 'true' to tell the browser to continue running this processor. // (Returning 1 or any other truthy value won't work in Chrome)