From b3c5483a01da30627e5e6989a28db5aeacc461ff Mon Sep 17 00:00:00 2001 From: Joaquin Bartaburu Date: Mon, 9 Dec 2024 16:37:15 -0300 Subject: [PATCH 1/4] feature: added volume control and UI element to test it --- lib/playback/audio.ts | 11 +++++-- lib/playback/backend.ts | 4 +++ lib/playback/index.ts | 9 +++++ web/src/components/volume.tsx | 62 +++++++++++++++++++++++++++-------- web/src/components/watch.tsx | 8 +++-- 5 files changed, 76 insertions(+), 18 deletions(-) diff --git a/lib/playback/audio.ts b/lib/playback/audio.ts index 687885a..a960bc5 100644 --- a/lib/playback/audio.ts +++ b/lib/playback/audio.ts @@ -10,12 +10,15 @@ import workletURL from "./worklet/index.ts?worker&url" export class Audio { context: AudioContext worklet: Promise + volumeNode: GainNode constructor(config: Message.ConfigAudio) { this.context = new AudioContext({ latencyHint: "interactive", sampleRate: config.sampleRate, }) + this.volumeNode = this.context.createGain() + this.volumeNode.gain.value = 1.0 this.worklet = this.load(config) } @@ -36,8 +39,8 @@ export class Audio { } // Connect the worklet to the volume node and then to the speakers - worklet.connect(volume) - volume.connect(this.context.destination) + worklet.connect(this.volumeNode) + this.volumeNode.connect(this.context.destination) worklet.port.postMessage({ config }) @@ -47,4 +50,8 @@ export class Audio { private on(_event: MessageEvent) { // TODO } + + public setVolume(newVolume: number) { + this.volumeNode.gain.setTargetAtTime(newVolume, this.context.currentTime, 0.01) + } } diff --git a/lib/playback/backend.ts b/lib/playback/backend.ts index 5505958..29393e5 100644 --- a/lib/playback/backend.ts +++ b/lib/playback/backend.ts @@ -87,6 +87,10 @@ export default class Backend { this.send({ segment }, segment.stream) } + setVolume(newVolume: number) { + this.#audio?.setVolume(newVolume) + } + async close() { this.#worker.terminate() await this.#audio?.context.close() diff --git a/lib/playback/index.ts b/lib/playback/index.ts index 79c98d3..caedb81 100644 --- a/lib/playback/index.ts +++ b/lib/playback/index.ts @@ -312,6 +312,15 @@ export class Player { } } + async setVolume(newVolume: number) { + this.#backend.setVolume(newVolume) + if (newVolume == 0 && !this.#muted) { + await this.mute(true) + } else if (newVolume > 0 && this.#muted) { + await this.mute(false) + } + } + /* async *timeline() { for (;;) { diff --git a/web/src/components/volume.tsx b/web/src/components/volume.tsx index 6e2b2e1..5415376 100644 --- a/web/src/components/volume.tsx +++ b/web/src/components/volume.tsx @@ -1,29 +1,63 @@ import { createSignal } from "solid-js" -type VolumeButtonProps = { +type VolumeControlProps = { mute: (isMuted: boolean) => void + setVolume: (newVolume: number) => void } -export const VolumeButton = (props: VolumeButtonProps) => { +export const VolumeControl = (props: VolumeControlProps) => { const [isMuted, setIsMuted] = createSignal(false) + const [currentVolume, setCurrentVolume] = createSignal(1) const toggleMute = () => { const newIsMuted = !isMuted() setIsMuted(newIsMuted) - props?.mute(newIsMuted) + props.mute(newIsMuted) + + if (newIsMuted) { + props.setVolume(0) + setCurrentVolume(0) + } else { + props.setVolume(1) + setCurrentVolume(1) + } + } + + const handleVolumeChange = (e: InputEvent & { currentTarget: HTMLInputElement }) => { + const volume = parseFloat(e.currentTarget.value) + if (volume == 0) { + setIsMuted(true) + } else { + setIsMuted(false) + } + setCurrentVolume(volume) + props.setVolume(volume) } return ( - +
+ + + +
) } diff --git a/web/src/components/watch.tsx b/web/src/components/watch.tsx index f054362..0cae9cc 100644 --- a/web/src/components/watch.tsx +++ b/web/src/components/watch.tsx @@ -2,7 +2,7 @@ import { Player } from "@kixelated/moq/playback" import Fail from "./fail" import { createEffect, createMemo, createSignal, onCleanup, Show } from "solid-js" -import { VolumeButton } from "./volume" +import { VolumeControl } from "./volume" import { PlayButton } from "./play-button" import { TrackSelect } from "./track-select" @@ -36,6 +36,10 @@ export default function Watch(props: { name: string }) { player()?.mute(state).catch(setError) } + const setVolume = (newVolume: number) => { + player()?.setVolume(newVolume).catch(setError) + } + const switchTrack = (track: string) => { void player()?.switchTrack(track) } @@ -108,7 +112,7 @@ export default function Watch(props: { name: string }) { >
- +
From 924ce07f5c09fc6ac1c8eb25f6e566ecdf1cb381 Mon Sep 17 00:00:00 2001 From: Joaquin Bartaburu Date: Tue, 10 Dec 2024 14:05:08 -0300 Subject: [PATCH 2/4] fix: restore previous volume level upon unmuting --- web/src/components/volume.tsx | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/web/src/components/volume.tsx b/web/src/components/volume.tsx index 5415376..69955d8 100644 --- a/web/src/components/volume.tsx +++ b/web/src/components/volume.tsx @@ -8,6 +8,7 @@ type VolumeControlProps = { export const VolumeControl = (props: VolumeControlProps) => { const [isMuted, setIsMuted] = createSignal(false) const [currentVolume, setCurrentVolume] = createSignal(1) + const [previousVolume, setPreviousVolume] = createSignal(1) const toggleMute = () => { const newIsMuted = !isMuted() @@ -15,11 +16,13 @@ export const VolumeControl = (props: VolumeControlProps) => { props.mute(newIsMuted) if (newIsMuted) { + setPreviousVolume(currentVolume()) props.setVolume(0) setCurrentVolume(0) } else { - props.setVolume(1) - setCurrentVolume(1) + const restoredVolume = previousVolume() + setCurrentVolume(restoredVolume) + props.setVolume(restoredVolume) } } From 35d75d3d8b3916cd1c26fe4e15a567fc8f0e2446 Mon Sep 17 00:00:00 2001 From: Joaquin Bartaburu Date: Wed, 11 Dec 2024 11:28:50 -0300 Subject: [PATCH 3/4] feat: improved play/pause --- lib/playback/backend.ts | 6 +++++- lib/playback/index.ts | 5 +++-- lib/playback/worker/index.ts | 17 +++++++++++++---- lib/playback/worker/message.ts | 2 +- lib/playback/worker/video.ts | 15 ++++++++++++--- 5 files changed, 34 insertions(+), 11 deletions(-) diff --git a/lib/playback/backend.ts b/lib/playback/backend.ts index 5505958..29cbf5d 100644 --- a/lib/playback/backend.ts +++ b/lib/playback/backend.ts @@ -68,7 +68,11 @@ export default class Backend { } pause() { - this.send({ pause: true }) + this.send({ play: false }) + } + + play() { + this.send({ play: true }) } async mute() { diff --git a/lib/playback/index.ts b/lib/playback/index.ts index 79c98d3..36aac6c 100644 --- a/lib/playback/index.ts +++ b/lib/playback/index.ts @@ -303,12 +303,13 @@ export class Player { this.subscribeFromTrackName(this.#audioTrackName) await this.#backend.unmute() } + this.#backend.play() } else { + this.#paused = true + this.#backend.pause() await this.unsubscribeFromTrack(this.#videoTrackName) await this.unsubscribeFromTrack(this.#audioTrackName) await this.#backend.mute() - this.#backend.pause() - this.#paused = true } } diff --git a/lib/playback/worker/index.ts b/lib/playback/worker/index.ts index a1a7ff9..11d9c10 100644 --- a/lib/playback/worker/index.ts +++ b/lib/playback/worker/index.ts @@ -22,6 +22,7 @@ class Worker { on(e: MessageEvent) { const msg = e.data as Message.ToWorker + console.log("message: ", msg) if (msg.config) { this.#onConfig(msg.config) @@ -30,8 +31,10 @@ class Worker { this.#onInit(msg.init) } else if (msg.segment) { this.#onSegment(msg.segment).catch(console.warn) - } else if (msg.pause) { - this.#onPause(msg.pause) + } else if (msg.play === false) { + this.#onPause(msg.play) + } else if (msg.play === true) { + this.#onPlay(msg.play) } else { throw new Error(`unknown message: + ${JSON.stringify(msg)}`) } @@ -103,11 +106,17 @@ class Worker { await segment.close() } - #onPause(pause: boolean) { - if (this.#video && pause) { + #onPause(play: boolean) { + if (this.#video && !play) { this.#video.pause() } } + + #onPlay(play: boolean) { + if (this.#video && play) { + this.#video.play() + } + } } // Pass all events to the worker diff --git a/lib/playback/worker/message.ts b/lib/playback/worker/message.ts index f0bba26..1d822d4 100644 --- a/lib/playback/worker/message.ts +++ b/lib/playback/worker/message.ts @@ -76,7 +76,7 @@ export interface ToWorker { // Sent on each init/data stream init?: Init segment?: Segment - pause?: boolean + play?: boolean /* // Sent to control playback diff --git a/lib/playback/worker/video.ts b/lib/playback/worker/video.ts index 682b512..5bb015a 100644 --- a/lib/playback/worker/video.ts +++ b/lib/playback/worker/video.ts @@ -27,10 +27,12 @@ export class Renderer { #decoderConfig?: DecoderConfig #waitingForKeyframe: boolean = true + #paused: boolean constructor(config: Message.ConfigVideo, timeline: Component) { this.#canvas = config.canvas this.#timeline = timeline + this.#paused = false this.#queue = new TransformStream({ start: this.#start.bind(this), @@ -41,10 +43,17 @@ export class Renderer { } pause() { - console.log("pause") + this.#paused = true + this.#decoder.flush().catch((err) => { + console.error(err) + }) this.#waitingForKeyframe = true } + play() { + this.#paused = false + } + async #run() { const reader = this.#timeline.frames.pipeThrough(this.#queue).getReader() for (;;) { @@ -74,8 +83,8 @@ export class Renderer { } #transform(frame: Frame) { - if (this.#decoder.state === "closed") { - console.warn("Decoder is closed. Skipping frame.") + if (this.#decoder.state === "closed" || this.#paused) { + console.warn("Decoder is closed or paused. Skipping frame.") return } From 8e04aaf4ec0f8a9e6531dc8e8abca955da4f5572 Mon Sep 17 00:00:00 2001 From: Joaquin Bartaburu Date: Tue, 17 Dec 2024 15:07:09 -0300 Subject: [PATCH 4/4] feat: optimistic play button --- lib/playback/index.ts | 7 ++++--- lib/playback/worker/index.ts | 2 +- lib/playback/worker/video.ts | 1 + web/src/components/watch.tsx | 21 ++++++++++----------- 4 files changed, 16 insertions(+), 15 deletions(-) diff --git a/lib/playback/index.ts b/lib/playback/index.ts index 36aac6c..8a16b96 100644 --- a/lib/playback/index.ts +++ b/lib/playback/index.ts @@ -307,9 +307,10 @@ export class Player { } else { this.#paused = true this.#backend.pause() - await this.unsubscribeFromTrack(this.#videoTrackName) - await this.unsubscribeFromTrack(this.#audioTrackName) - await this.#backend.mute() + const mutePromise = this.#backend.mute() + const audioPromise = this.unsubscribeFromTrack(this.#audioTrackName) + const videoPromise = this.unsubscribeFromTrack(this.#videoTrackName) + await Promise.all([mutePromise, audioPromise, videoPromise]) } } diff --git a/lib/playback/worker/index.ts b/lib/playback/worker/index.ts index 11d9c10..07dd7fe 100644 --- a/lib/playback/worker/index.ts +++ b/lib/playback/worker/index.ts @@ -22,7 +22,7 @@ class Worker { on(e: MessageEvent) { const msg = e.data as Message.ToWorker - console.log("message: ", msg) + // console.log("message: ", msg) if (msg.config) { this.#onConfig(msg.config) diff --git a/lib/playback/worker/video.ts b/lib/playback/worker/video.ts index 5bb015a..b0c3061 100644 --- a/lib/playback/worker/video.ts +++ b/lib/playback/worker/video.ts @@ -58,6 +58,7 @@ export class Renderer { const reader = this.#timeline.frames.pipeThrough(this.#queue).getReader() for (;;) { const { value: frame, done } = await reader.read() + if (this.#paused) continue if (done) break self.requestAnimationFrame(() => { diff --git a/web/src/components/watch.tsx b/web/src/components/watch.tsx index f054362..000aa2f 100644 --- a/web/src/components/watch.tsx +++ b/web/src/components/watch.tsx @@ -5,6 +5,7 @@ import { createEffect, createMemo, createSignal, onCleanup, Show } from "solid-j import { VolumeButton } from "./volume" import { PlayButton } from "./play-button" import { TrackSelect } from "./track-select" +import { promise } from "astro/zod" export default function Watch(props: { name: string }) { // Use query params to allow overriding environment variables. @@ -47,17 +48,15 @@ export default function Watch(props: { name: string }) { const handlePlayPause = () => { const playerInstance = player() if (!playerInstance) return - - if (playerInstance.isPaused()) { - playerInstance - .play() - .then(() => setIsPlaying(true)) - .catch(setError) - } else { - playerInstance - .play() - .then(() => setIsPlaying(false)) - .catch(setError) + try { + void playerInstance.play() + if (playerInstance.isPaused()) { + setIsPlaying(false) + } else { + setIsPlaying(true) + } + } catch (err) { + setError(err instanceof Error ? err : new Error(String(err))) } }