Merge pull request #124 from ircam-ismm/feature/audio-worklet-worker

Work in progress for AudioWorkletNode via Worker thread

.github/workflows/build.yml

@@ -84,14 +84,14 @@ jobs:
           # in particular for RPi, do it locally and workaround...
           # ---------------------------------------------------------
 
-    name: stable - ${{ matrix.settings.target }} - node@20
+    name: stable - ${{ matrix.settings.target }} - node@22.1
     runs-on: ${{ matrix.settings.host }}
     steps:
       - uses: actions/checkout@v4
       - name: Setup node
         uses: actions/setup-node@v4
         with:
-          node-version: 20
+          node-version: 22.1
           check-latest: true
           architecture: ${{ matrix.settings.architecture }}
 

.scripts/wpt-harness.mjs

@@ -1,3 +1,5 @@
+import { Blob } from 'node:buffer';
+
 import path from 'path';
 import wptRunner from 'wpt-runner';
 import chalk from 'chalk';
@@ -38,6 +40,13 @@ const rootURL = 'webaudio';
 
 // monkey patch `window` with our web audio API
 const setup = window => {
+  // monkey patch innerText with textContent
+  Object.defineProperty(window.HTMLScriptElement.prototype, 'innerText', {
+    get: function() {
+      return this.textContent;
+    },
+  })
+  // return;
   // This is meant to make some idlharness tests pass:
   // cf. wpt-runnner/testharness/idlharness.js line 1466-1472
   // These tests, which assess the descriptor of the classes according to window,
@@ -75,6 +84,9 @@ const setup = window => {
   window.Promise = Promise;
   window.Event = Event;
   window.EventTarget = EventTarget;
+  window.URL = URL;
+  window.Blob = Blob;
+  window.SharedArrayBuffer = SharedArrayBuffer;
   // @note - adding Function this crashes some tests:
   // the-pannernode-interface/pannernode-setposition-throws.html
   // the-periodicwave-interface/createPeriodicWaveInfiniteValuesThrows.html

Cargo.toml

@@ -10,8 +10,10 @@ version = "0.20.0"
 crate-type = ["cdylib"]
 
 [dependencies]
-napi = {version="2.15", features=["napi9", "tokio_rt"]}
-napi-derive = "2.15"
+crossbeam-channel = "0.5.12"
+napi = { version="2.15", features=["napi9", "tokio_rt"] }
+napi-derive = { version="2.15" }
+thread-priority = "1.1.0"
 web-audio-api = "=0.45.0"
 # web-audio-api = { path = "../web-audio-api-rs" }
 

examples/audio-worklet-shared-array-buffer.mjs

@@ -0,0 +1,43 @@
+import path from 'node:path';
+
+import { AudioContext, AudioWorkletNode } from '../index.mjs';
+import { sleep } from '@ircam/sc-utils';
+
+const latencyHint = process.env.WEB_AUDIO_LATENCY === 'playback' ? 'playback' : 'interactive';
+const audioContext = new AudioContext({ latencyHint });
+
+await audioContext.audioWorklet.addModule(path.join('examples', 'worklets', 'array-source.js'));
+
+// Create a shared float array big enough for 128 floats
+let sharedArray = new SharedArrayBuffer(512);
+let sharedFloats = new Float32Array(sharedArray);
+
+async function runSource() {
+    const src = new AudioWorkletNode(audioContext, 'array-source', {
+        processorOptions: { sharedFloats },
+    });
+    src.connect(audioContext.destination);
+
+    console.log("Sawtooth");
+    for (let i = 0; i < sharedFloats.length; i++) {
+        sharedFloats[i] = -1. + i / 64; // create saw
+    }
+    await sleep(1);
+
+    console.log("Square");
+    for (let i = 0; i < sharedFloats.length; i++) {
+        sharedFloats[i] = i > 64 ? 1 : -1;
+    }
+    await sleep(1);
+
+    src.disconnect();
+
+    // src goes out of scope and is disconnected, so it should be cleaned up
+}
+
+await runSource();
+
+// @todo - this should close the AudioWorkletGlobalScope properly
+// before closing the "real" context
+console.log('closing');
+await audioContext.close();

examples/audio-worklet.mjs

@@ -0,0 +1,60 @@
+import path from 'node:path';
+
+import { AudioContext, OfflineAudioContext, OscillatorNode, AudioWorkletNode } from '../index.mjs';
+import { sleep } from '@ircam/sc-utils';
+
+const latencyHint = process.env.WEB_AUDIO_LATENCY === 'playback' ? 'playback' : 'interactive';
+
+const TEST_ONLINE = false;
+
+const audioContext = TEST_ONLINE
+  ? new AudioContext({ latencyHint })
+  : new OfflineAudioContext(2, 8 * 48000, 48000)
+
+await audioContext.audioWorklet.addModule(path.join('examples', 'worklets', 'bitcrusher.js')); // relative to cwd
+await audioContext.audioWorklet.addModule(path.join('worklets', 'white-noise.js')); // relative path to call site
+
+const sine = new OscillatorNode(audioContext, { type: 'sawtooth', frequency: 5000 });
+const bitCrusher = new AudioWorkletNode(audioContext, 'bitcrusher', {
+  processorOptions: { msg: 'hello world' },
+});
+
+bitCrusher.port.on('message', (event) => console.log('main recv', event));
+bitCrusher.port.postMessage({ hello: 'from main' });
+
+sine
+  .connect(bitCrusher)
+  .connect(audioContext.destination);
+
+const paramBitDepth =  bitCrusher.parameters.get('bitDepth');
+const paramReduction =  bitCrusher.parameters.get('frequencyReduction');
+
+paramBitDepth.setValueAtTime(1, 0);
+
+paramReduction.setValueAtTime(0.01, 0.);
+paramReduction.linearRampToValueAtTime(0.1, 4.);
+paramReduction.exponentialRampToValueAtTime(0.01, 8.);
+
+sine.start();
+sine.stop(8);
+
+const whiteNoise = new AudioWorkletNode(audioContext, 'white-noise');
+whiteNoise.connect(audioContext.destination);
+
+if (TEST_ONLINE) {
+  audioContext.renderCapacity.addEventListener('update', e => {
+    const { timestamp, averageLoad, peakLoad, underrunRatio } = e;
+    console.log('AudioRenderCapacityEvent:', { timestamp, averageLoad, peakLoad, underrunRatio });
+  });
+  audioContext.renderCapacity.start({ updateInterval: 1. });
+
+  await sleep(8);
+  await audioContext.close();
+} else {
+  const buffer = await audioContext.startRendering();
+  const online = new AudioContext();
+  const src = online.createBufferSource();
+  src.buffer = buffer;
+  src.connect(online.destination);
+  src.start();
+}

examples/worklets/array-source.js

@@ -0,0 +1,20 @@
+class ArraySourceProcessor extends AudioWorkletProcessor {
+  constructor(options) {
+    super();
+    this.sharedFloats = options.processorOptions.sharedFloats;
+  }
+
+  process(inputs, outputs, parameters) {
+    const output = outputs[0];
+
+    output.forEach((channel) => {
+      for (let i = 0; i < channel.length; i++) {
+        channel[i] = this.sharedFloats[i];
+      }
+    });
+
+    return true;
+  }
+}
+
+registerProcessor('array-source', ArraySourceProcessor);

examples/worklets/bitcrusher.js

@@ -0,0 +1,77 @@
+class Bitcrusher extends AudioWorkletProcessor {
+    static get parameterDescriptors() {
+        return [{
+            name: 'bitDepth',
+            defaultValue: 12,
+            minValue: 1,
+            maxValue: 16
+        }, {
+            name: 'frequencyReduction',
+            defaultValue: 0.5,
+            minValue: 0,
+            maxValue: 1
+        }];
+    }
+
+    constructor(options) {
+      console.log(`++ in constructor: ${JSON.stringify(options, null, 2)}\n`);
+      // The initial parameter value can be set by passing |options|
+      // to the processor's constructor.
+      super();
+
+      this._phase = 0;
+      this._lastSampleValue = 0;
+      this._msg = options.processorOptions.msg;
+
+      this.port.on('message', event => {
+        console.log(`++ on message: ${JSON.stringify(event, null, 2)}\n`);
+      });
+    }
+
+    process(inputs, outputs, parameters) {
+      const input = inputs[0];
+      const output = outputs[0];
+      const bitDepth = parameters.bitDepth;
+      const frequencyReduction = parameters.frequencyReduction;
+
+      if (bitDepth.length > 1) {
+        for (let channel = 0; channel < output.length; ++channel) {
+          for (let i = 0; i < output[channel].length; ++i) {
+            let step = Math.pow(0.5, bitDepth[i]);
+            // Use modulo for indexing to handle the case where
+            // the length of the frequencyReduction array is 1.
+            this._phase += frequencyReduction[i % frequencyReduction.length];
+            if (this._phase >= 1.0) {
+              this._phase -= 1.0;
+              this._lastSampleValue = step * Math.floor(input[channel][i] / step + 0.5);
+            }
+            output[channel][i] = this._lastSampleValue;
+          }
+        }
+      } else {
+        // Because we know bitDepth is constant for this call,
+        // we can lift the computation of step outside the loop,
+        // saving many operations.
+        const step = Math.pow(0.5, bitDepth[0]);
+        for (let channel = 0; channel < output.length; ++channel) {
+          for (let i = 0; i < output[channel].length; ++i) {
+            this._phase += frequencyReduction[i % frequencyReduction.length];
+            if (this._phase >= 1.0) {
+              this._phase -= 1.0;
+              this._lastSampleValue = step * Math.floor(input[channel][i] / step + 0.5);
+            }
+            output[channel][i] = this._lastSampleValue;
+          }
+        }
+      }
+
+      if (Math.random() < 0.005) {
+          this.port.postMessage({ hello: 'from render', msg: this._msg });
+      }
+
+      // No need to return a value; this node's lifetime is dependent only on its
+      // input connections.
+    }
+}
+
+registerProcessor('bitcrusher', Bitcrusher);

examples/worklets/white-noise.js

@@ -0,0 +1,15 @@
+class WhiteNoiseProcessor extends AudioWorkletProcessor {
+  process(inputs, outputs, parameters) {
+    const output = outputs[0];
+
+    output.forEach((channel) => {
+      for (let i = 0; i < channel.length; i++) {
+        channel[i] = Math.random() * 2 - 1;
+      }
+    });
+
+    return true;
+  }
+}
+
+registerProcessor('white-noise', WhiteNoiseProcessor);

generator/index.mjs

@@ -14,6 +14,7 @@ import { ESLint } from 'eslint';
 // and extended with the list of generatedNodes
 let supportedNodes = [
   'ScriptProcessorNode',
+  'AudioWorkletNode',
 ];
 
 const generatedNodes = [
@@ -276,13 +277,16 @@ async function beautifyAndLint(pathname, code) {
 }
 
 // Generate files that require the list of generated AudioNode
-['index', 'monkey-patch', 'BaseAudioContext'].forEach(src => {
+['indexCjs', 'indexMjs', 'BaseAudioContext'].forEach(src => {
   let input;
   let output;
   // index.tmpl.js generates the ES module re-export
-  if (src === 'index') {
-    input = path.join(jsTemplates, `${src}.tmpl.mjs`);
-    output = path.join(process.cwd(), `${src}.mjs`);
+  if (src === 'indexCjs') {
+    input = path.join(jsTemplates, `index.tmpl.cjs`);
+    output = path.join(process.cwd(), `index.cjs`);
+  } else if (src === 'indexMjs') {
+    input = path.join(jsTemplates, `index.tmpl.mjs`);
+    output = path.join(process.cwd(), `index.mjs`);
   } else {
     input = path.join(jsTemplates, `${src}.tmpl.js`);
     output = path.join(jsOutput, `${src}.js`);