Skip to content

Commit

Permalink
prefetch player
Browse files Browse the repository at this point in the history
shreyas-jadhav committed Oct 2, 2024
1 parent 65e5c13 commit 4bd8af5
Showing 4 changed files with 288 additions and 67 deletions.
4 changes: 2 additions & 2 deletions package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

100 changes: 41 additions & 59 deletions src/players/SpeakerPrefetchPlayer.ts
Original file line number Diff line number Diff line change
@@ -3,11 +3,24 @@ import {
IAudioBufferSourceNode,
IAudioContext,
IGainNode,
IMediaElementAudioSourceNode,
} from "standardized-audio-context";

import { SpeakerConfig } from "../types/roundware";
import { ISpeakerPlayer, SpeakerConstructor } from "../types/speaker";
import { NEARLY_ZERO, speakerLog } from "../utils";

import {
cleanAudioURL,
NEARLY_ZERO,
silenceAudioBase64,
speakerLog,
} from "../utils";

/**
*
* Basic audio utilities for playing speakers audio
* @export
* @class SpeakerPrefetchPlayer
*/
export class SpeakerPrefetchPlayer implements ISpeakerPlayer {
isSafeToPlay: boolean = true;
playing: boolean = false;
@@ -23,6 +36,8 @@ export class SpeakerPrefetchPlayer implements ISpeakerPlayer {
buffer?: IAudioBuffer;

constructor({ audioContext, id, uri, config }: SpeakerConstructor) {
this.log("SpeakerPrefetchPlayer constructor");

this.audio = new Audio();
this.id = id;
this.context = audioContext;
@@ -37,10 +52,10 @@ export class SpeakerPrefetchPlayer implements ISpeakerPlayer {
request.responseType = "arraybuffer";
request.onprogress = (ev) => {
this.loadedPercentage = Number(((ev.loaded / ev.total) * 100).toFixed(2));

this.loadingCallback(this.loadedPercentage);
};
const speakerContext = this;

request.onload = function () {
var audioData = request.response;

@@ -64,88 +79,48 @@ export class SpeakerPrefetchPlayer implements ISpeakerPlayer {
request.send();
}

started = false;
lastStartedAtSeconds = 0;
lastStartedAtTime = 0;
async play(): Promise<boolean> {
if (!this.loaded || !this.started) {
if (!this.loaded || !this.source) {
this.log(`not loaded or started yet`);
this.initializeSource();
return false;
}
if (this.playing) {
this.fade();
return true;
}

this.source.start(0, this.pausedAtSeconds);
this.lastStartedAtSeconds = this.pausedAtSeconds;
this.lastStartedAtTime = this.context.currentTime;
this.log("playing...");

this.gainNode.connect(this.context.destination);
this.playing = true;
return true;
}

replay() {
this.pausedAt = 0;
this.playing = false;
this.initializeSource();
this.timerStart();
}
startedAt = 0;
pausedAt = 0;

get remainingDuration() {
if (!this.buffer) return 0;
return (this.config.length || this.buffer?.duration) - this.pausedAt;
}

endTimeout: NodeJS.Timeout | null = null;

async timerStart() {
if (this.started || !this.buffer) {
return;
}

// see timerStop() note
this.initializeSource();
if (!this.source) return;

// resume audio context if suspended
if (this.context.state !== "running") {
await this.context.resume();
}
// start now will so stay in sync with other speakers, from last paused time
this.source.start(this.context.currentTime, this.pausedAt);

if (this.endTimeout) {
clearTimeout(this.endTimeout);
}
this.endTimeout = setTimeout(() => {
this.endCallback();
this.log(`speaker end`);
}, this.remainingDuration * 1000);
if (!this.source) return;
this.initializeSource();

this.fade();
this.startedAt = this.context.currentTime;
this.started = true;
}

timerStop(): void {
/**
* note: we cant just stop the source and resume
* start() and stop() are allow to called only once
*
* so need to
* note the current time as paused time (incremented with previous paused times)
* destroy current source
* and next time needs to start, create new source
* we can pass the already downloaded buffer
* start with offset as last paused time
*/
this.source?.stop();
this.pausedAt += this.context.currentTime - this.startedAt;
this.log(`next time will start from ${this.pausedAt}`);
this.source = undefined;
this.started = false;
if (this.endTimeout) {
clearTimeout(this.endTimeout);
}
}
timerStop(): void {}

initializeSource() {
if (!this.buffer) return;
@@ -169,24 +144,31 @@ export class SpeakerPrefetchPlayer implements ISpeakerPlayer {
// connect to audio context
this.source.connect(this.gainNode).connect(this.context.destination);

this.started = false;

console.log(`init`);
this.fade();
}

pausedAtSeconds = 0;
pause(): void {
if (!this.playing) return;
this.gainNode.disconnect();
this.source?.stop();

this.playing = false;

const elapsedSeconds = this.context.currentTime - this.lastStartedAtTime;
this.pausedAtSeconds = this.lastStartedAtSeconds + elapsedSeconds;

this.source = undefined;
}
_fadingDestination = 0;
_fading = false;
_fadingTimeout: NodeJS.Timeout | null = null;
fade(toVolume: number = this._fadingDestination, duration: number = 3): void {
if (this._fadingDestination == toVolume && this._fading) return;
this._fadingDestination = toVolume;
if (!this.playing) return;
if (!this.playing) {
this.play();
}

// already at that volume
if (Math.abs(this.volume - this._fadingDestination) < 0.05) return;
230 changes: 230 additions & 0 deletions src/players/SpeakerPrefetchSyncPlayer.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,230 @@
import {
IAudioBuffer,
IAudioBufferSourceNode,
IAudioContext,
IGainNode,
} from "standardized-audio-context";
import { SpeakerConfig } from "../types/roundware";
import { ISpeakerPlayer, SpeakerConstructor } from "../types/speaker";
import { NEARLY_ZERO, speakerLog } from "../utils";

export class SpeakerPrefetchSyncPlayer implements ISpeakerPlayer {
isSafeToPlay: boolean = true;
playing: boolean = false;
loaded = false;
audio: HTMLAudioElement;
source?: IAudioBufferSourceNode<IAudioContext>;
id: number;
gainNode: IGainNode<IAudioContext>;
context: IAudioContext;
config: SpeakerConfig;
loadedPercentage = 0;

buffer?: IAudioBuffer;

constructor({ audioContext, id, uri, config }: SpeakerConstructor) {
this.audio = new Audio();
this.id = id;
this.context = audioContext;
this.config = config;
this.gainNode = audioContext.createGain();
this.gainNode.gain.value = NEARLY_ZERO;

var request = new XMLHttpRequest();

request.open("GET", uri, true);
request.timeout = Infinity;
request.responseType = "arraybuffer";
request.onprogress = (ev) => {
this.loadedPercentage = Number(((ev.loaded / ev.total) * 100).toFixed(2));

this.loadingCallback(this.loadedPercentage);
};
const speakerContext = this;
request.onload = function () {
var audioData = request.response;

audioContext.decodeAudioData(
audioData,
function (buffer) {
speakerContext.buffer = buffer;
// @ts-ignore
global._roundwareTotalAudioBufferSize +=
buffer.length * buffer.numberOfChannels * 4;
speakerContext.loaded = true;
speakerContext.log(`loaded successfully`);
},

function (e) {
speakerContext.log("Error with decoding audio data " + e.message);
}
);
};

request.send();
}

started = false;
async play(): Promise<boolean> {
if (!this.loaded || !this.started) {
this.log(`not loaded or started yet`);
return false;
}
if (this.playing) {
this.fade();
return true;
}

this.gainNode.connect(this.context.destination);
this.playing = true;
return true;
}

replay() {
this.pausedAt = 0;
this.playing = false;
this.initializeSource();
this.timerStart();
}
startedAt = 0;
pausedAt = 0;

get remainingDuration() {
if (!this.buffer) return 0;
return (this.config.length || this.buffer?.duration) - this.pausedAt;
}

endTimeout: NodeJS.Timeout | null = null;

async timerStart() {
if (this.started || !this.buffer) {
return;
}

// see timerStop() note
this.initializeSource();
if (!this.source) return;

// resume audio context if suspended
if (this.context.state !== "running") {
await this.context.resume();
}
// start now will so stay in sync with other speakers, from last paused time
this.source.start(this.context.currentTime, this.pausedAt);

if (this.endTimeout) {
clearTimeout(this.endTimeout);
}
this.endTimeout = setTimeout(() => {
this.endCallback();
this.log(`speaker end`);
}, this.remainingDuration * 1000);

this.fade();
this.startedAt = this.context.currentTime;
this.started = true;
}

timerStop(): void {
/**
* note: we cant just stop the source and resume
* start() and stop() are allow to called only once
*
* so need to
* note the current time as paused time (incremented with previous paused times)
* destroy current source
* and next time needs to start, create new source
* we can pass the already downloaded buffer
* start with offset as last paused time
*/
this.source?.stop();
this.pausedAt += this.context.currentTime - this.startedAt;
this.log(`next time will start from ${this.pausedAt}`);
this.source = undefined;
this.started = false;
if (this.endTimeout) {
clearTimeout(this.endTimeout);
}
}

initializeSource() {
if (!this.buffer) return;
// disconnect previous ones as we are going to create new
this.gainNode.disconnect();
this.source?.disconnect();

// create new source
this.source = this.context.createBufferSource();

// buffer already downloaded from constructor
this.source.buffer = this.buffer;

if (this.config.loop) {
this.source.loop = true;
this.source.loopEnd = this.config.length || this.buffer.duration;
} else {
this.source.loop = false;
}

// connect to audio context
this.source.connect(this.gainNode).connect(this.context.destination);

this.started = false;

console.log(`init`);
this.fade();
}

pause(): void {
if (!this.playing) return;
this.gainNode.disconnect();
this.playing = false;
}
_fadingDestination = 0;
_fading = false;
_fadingTimeout: NodeJS.Timeout | null = null;
fade(toVolume: number = this._fadingDestination, duration: number = 3): void {
if (this._fadingDestination == toVolume && this._fading) return;
this._fadingDestination = toVolume;
if (!this.playing) return;

// already at that volume
if (Math.abs(this.volume - this._fadingDestination) < 0.05) return;
this.log(`startng fade ${this.volume} -> ${this._fadingDestination}`);
this.gainNode.gain.cancelScheduledValues(0);

this.gainNode.gain.exponentialRampToValueAtTime(
this._fadingDestination || NEARLY_ZERO,
this.context.currentTime + duration
);
if (this._fadingTimeout) {
clearTimeout(this._fadingTimeout);
}
this._fadingTimeout = setTimeout(() => {
this._fading = false;
}, duration * 1000);
}
fadeOutAndPause(): void {
if (!this.playing) return;
this.fade(0);
this.log(`fading out and pausing`);
setTimeout(() => {
this.pause();
}, 3000);
}
log(string: string): void {
speakerLog(`${this.id}] ${string}`);
}
get volume() {
return this.gainNode.gain.value;
}
loadingCallback = (number: number) => {};
onLoadingProgress(callback: (newPercent: number) => void): void {
this.loadingCallback = callback;
}
endCallback = () => {};
onEnd(callback: () => void) {
this.endCallback = callback;
console.log(`callback set`);
}
}
21 changes: 15 additions & 6 deletions src/speaker_track.ts
Original file line number Diff line number Diff line change
@@ -13,12 +13,13 @@ import lineToPolygon from "@turf/line-to-polygon";
import pointToLineDistance from "@turf/point-to-line-distance";
import { IAudioContext } from "standardized-audio-context";
import { SpeakerStreamer } from "./players/SpeakerStreamer";
import { SpeakerPrefetchPlayer } from "./players/SpeakerPrefetchPlayer";
import { SpeakerPrefetchSyncPlayer } from "./players/SpeakerPrefetchSyncPlayer";
import { ISpeakerData, ISpeakerPlayer } from "./types/speaker";
import { speakerLog } from "./utils";
import { SpeakerConfig } from "./types/roundware";
import { SpeakerSyncStreamer } from "./players/SpeakerSyncStreamer";
import { Mixer } from "./mixer";
import { SpeakerPrefetchPlayer } from "./players/SpeakerPrefetchPlayer";
const convertLinesToPolygon = (shape: LineString | MultiLineString) =>
lineToPolygon(shape);
const FADE_DURATION_SECONDS = 3;
@@ -126,16 +127,20 @@ export class SpeakerTrack {

let newVolume = this.currentVolume;
if (!listenerPoint) {
this.log("No listener point");
newVolume = this.currentVolume;
} else if (this.attenuationShapeContains(listenerPoint)) {
this.log("In attenuation shape");
newVolume = this.maxVolume;
} else if (this.outerBoundaryContains(listenerPoint)) {
this.log("In outer boundary");
const range = this.maxVolume - this.minVolume;
const volumeGradient =
this.minVolume + range * this.attenuationRatio(listenerPoint);

newVolume = volumeGradient;
} else {
this.log("Outside outer boundary");
newVolume = this.minVolume;
}

@@ -213,11 +218,15 @@ export class SpeakerTrack {
}

initPlayer() {
const Player = this.config.sync
? this.config.prefetch
? SpeakerPrefetchPlayer
: SpeakerSyncStreamer
: SpeakerStreamer;
const Player = (() => {
if (this.config.sync && this.config.prefetch)
return SpeakerPrefetchSyncPlayer;
if (this.config.sync) return SpeakerSyncStreamer;
if (this.config.prefetch) return SpeakerPrefetchPlayer;
return SpeakerStreamer;
})();

console.log(`init player ${this.speakerId}: ${Player.name}`);
this.player = new Player({
audioContext: this.audioContext,
id: this.speakerId,

0 comments on commit 4bd8af5

Please sign in to comment.