From f218733b0cbbc7a85ca0d734c096f3bc37636652 Mon Sep 17 00:00:00 2001 From: Paul Berberian Date: Thu, 4 May 2023 18:26:05 +0200 Subject: [PATCH] Decide the position and autoplay status of a Reload in the Initializer This commit performs a small modifications so that the `Stream` module, when asking for the current content to be "reloaded" (that is: to replace its MediaSource, generally both for compatibility reasons and to ensure buffers are flushed), no longer needs to either calculate the position to reload to nor if it should auto-play after the reload. This is a simplification of a "reload" event from the point of view of the `Stream`. A "reload" there now is mostly done "in-place" (with a possible time offset to apply, e.g. to re-play the last second after changing the audio track, the `Stream` could be asking for a `timeOffset` of `-1`) and playback characteristics are mostly kept as they were before the reload. The position and playing status is now computed by the `MediaSourceContentInitializer`, which is the actual module doing the reloading logic, based on the position and playing status at the time the reload order was received. This is important in our current Proof-of-Concept of running the RxPlayer in a worker: Calculating the current position and playing status was in the end done synchronously by asking the `HTMLMediaElement` on the page. In a worker, we do not have access to the `HTMLMediaElement`, thus that data cannot be accessed synchronously if the module asking for it is running on the worker (and the `Stream` fully runs in the worker). By keeping such logic closer to the higher level of the RxPlayer's internal architecture (closer to the API, further from the core), we greatly facilitate the possibility of splitting that logic between main thread (`HTMLMediaElement` management) and worker (`Stream`, `MediaSource` management when MSE-in-worker is available). The `MediaSourceContentInitializer` itself is on that matter splitted into two parts: a part in the main thread, the other in the worker. Even if that Proof-of-Concept is not actually merged in the future, this small modification still makes sense, at least to me. --- .../init/media_source_content_initializer.ts | 32 ++++++++-- .../stream/adaptation/adaptation_stream.ts | 31 ++++------ src/core/stream/adaptation/types.ts | 20 ++++--- .../orchestrator/stream_orchestrator.ts | 60 +++++++------------ src/core/stream/period/period_stream.ts | 48 +++++++-------- 5 files changed, 98 insertions(+), 93 deletions(-) diff --git a/src/core/init/media_source_content_initializer.ts b/src/core/init/media_source_content_initializer.ts index cebe87d166..0e425dbea6 100644 --- a/src/core/init/media_source_content_initializer.ts +++ b/src/core/init/media_source_content_initializer.ts @@ -618,19 +618,41 @@ export default class MediaSourceContentInitializer extends ContentInitializer { addedSegment: (value) => self.trigger("addedSegment", value), - needsMediaSourceReload: (value) => onReloadOrder(value), + needsMediaSourceReload: (payload) => { + const lastObservation = streamObserver.getReference().getValue(); + const currentPosition = lastObservation.position.pending ?? + streamObserver.getCurrentTime(); + const isPaused = lastObservation.paused.pending ?? + streamObserver.getIsPaused(); + let position = currentPosition + payload.timeOffset; + if (payload.minimumPosition !== undefined) { + position = Math.max(payload.minimumPosition, position); + } + if (payload.maximumPosition !== undefined) { + position = Math.min(payload.maximumPosition, position); + } + onReloadOrder({ position, autoPlay: !isPaused }); + }, - needsDecipherabilityFlush(value) { + needsDecipherabilityFlush() { const keySystem = getKeySystemConfiguration(mediaElement); if (shouldReloadMediaSourceOnDecipherabilityUpdate(keySystem?.[0])) { - onReloadOrder(value); + const lastObservation = streamObserver.getReference().getValue(); + const position = lastObservation.position.pending ?? + streamObserver.getCurrentTime(); + const isPaused = lastObservation.paused.pending ?? + streamObserver.getIsPaused(); + onReloadOrder({ position, autoPlay: !isPaused }); } else { + const lastObservation = streamObserver.getReference().getValue(); + const position = lastObservation.position.pending ?? + streamObserver.getCurrentTime(); // simple seek close to the current position // to flush the buffers - if (value.position + 0.001 < value.duration) { + if (position + 0.001 < lastObservation.duration) { playbackObserver.setCurrentTime(mediaElement.currentTime + 0.001); } else { - playbackObserver.setCurrentTime(value.position); + playbackObserver.setCurrentTime(position); } } }, diff --git a/src/core/stream/adaptation/adaptation_stream.ts b/src/core/stream/adaptation/adaptation_stream.ts index 695d63c4ca..af26f349d9 100644 --- a/src/core/stream/adaptation/adaptation_stream.ts +++ b/src/core/stream/adaptation/adaptation_stream.ts @@ -182,25 +182,18 @@ export default function AdaptationStream( // the next observation (which may reflect very different playback conditions) // is actually received. return nextTick(() => { - playbackObserver.listen((observation) => { - const { manual: newManual } = estimateRef.getValue(); - if (!newManual) { - return; - } - const currentTime = playbackObserver.getCurrentTime(); - const pos = currentTime + DELTA_POSITION_AFTER_RELOAD.bitrateSwitch; - - // Bind to Period start and end - const position = Math.min(Math.max(period.start, pos), - period.end ?? Infinity); - const autoPlay = !(observation.paused.pending ?? - playbackObserver.getIsPaused()); - return callbacks.waitingMediaSourceReload({ bufferType: adaptation.type, - period, - position, - autoPlay }); - }, { includeLastObservation: true, - clearSignal: repStreamTerminatingCanceller.signal }); + if (repStreamTerminatingCanceller.isUsed()) { + return; + } + const { manual: newManual } = estimateRef.getValue(); + if (!newManual) { + return; + } + const timeOffset = DELTA_POSITION_AFTER_RELOAD.bitrateSwitch; + return callbacks.waitingMediaSourceReload({ bufferType: adaptation.type, + period, + timeOffset, + stayInPeriod: true }); }); } diff --git a/src/core/stream/adaptation/types.ts b/src/core/stream/adaptation/types.ts index 7cca5cc6fd..df381e420b 100644 --- a/src/core/stream/adaptation/types.ts +++ b/src/core/stream/adaptation/types.ts @@ -66,16 +66,22 @@ export interface IWaitingMediaSourceReloadPayload { /** Buffer type concerned. */ bufferType : IBufferType; /** - * The position in seconds and the time at which the MediaSource should be - * reset once it has been reloaded. + * Relative position, compared to the current position, at which we should + * restart playback after reloading. For example `-2` will reload 2 seconds + * before the current position. */ - position : number; + timeOffset : number; /** - * If `true`, we want the HTMLMediaElement to play right after the reload is - * done. - * If `false`, we want to stay in a paused state at that point. + * If `true`, we will control that the position we reload at, after applying + * `timeOffset`, is still part of the Period `period`. + * + * If it isn't we will re-calculate that reloaded position to be: + * - either the Period's start if the calculated position is before the + * Period's start. + * - either the Period'end start if the calculated position is after the + * Period's end. */ - autoPlay : boolean; + stayInPeriod : boolean; } /** Regular playback information needed by the AdaptationStream. */ diff --git a/src/core/stream/orchestrator/stream_orchestrator.ts b/src/core/stream/orchestrator/stream_orchestrator.ts index 5c2bdfc467..2338659e4c 100644 --- a/src/core/stream/orchestrator/stream_orchestrator.ts +++ b/src/core/stream/orchestrator/stream_orchestrator.ts @@ -224,8 +224,11 @@ export default function StreamOrchestrator( callbacks.lockedStream({ bufferType: payload.bufferType, period: payload.period }); } else { - const { position, autoPlay } = payload; - callbacks.needsMediaSourceReload({ position, autoPlay }); + callbacks.needsMediaSourceReload({ + timeOffset: payload.timeOffset, + minimumPosition: payload.stayInPeriod ? payload.period.start : undefined, + maximumPosition: payload.stayInPeriod ? payload.period.end : undefined, + }); } }, periodStreamReady(payload : IPeriodStreamReadyPayload) : void { @@ -345,11 +348,9 @@ export default function StreamOrchestrator( } const observation = playbackObserver.getReference().getValue(); if (needsFlushingAfterClean(observation, undecipherableRanges)) { - const shouldAutoPlay = !(observation.paused.pending ?? - playbackObserver.getIsPaused()); - callbacks.needsDecipherabilityFlush({ position: observation.position.last, - autoPlay: shouldAutoPlay, - duration: observation.duration }); + + // Bind to Period start and end + callbacks.needsDecipherabilityFlush(); if (orchestratorCancelSignal.isCancelled()) { return ; } @@ -578,7 +579,7 @@ export interface IStreamOrchestratorCallbacks * worst cases completely removed and re-created through the "reload" mechanism, * depending on the platform. */ - needsDecipherabilityFlush(payload : INeedsDecipherabilityFlushPayload) : void; + needsDecipherabilityFlush() : void; } /** Payload for the `periodStreamCleared` callback. */ @@ -602,16 +603,23 @@ export interface IPeriodStreamClearedPayload { /** Payload for the `needsMediaSourceReload` callback. */ export interface INeedsMediaSourceReloadPayload { /** - * The position in seconds and the time at which the MediaSource should be - * reset once it has been reloaded. + * Relative position, compared to the current one, at which we should + * restart playback after reloading. For example `-2` will reload 2 seconds + * before the current position. + */ + timeOffset : number; + /** + * If defined and if the new position obtained after relying on + * `timeOffset` is before `minimumPosition`, then we will reload at + * `minimumPosition` instead. */ - position : number; + minimumPosition : number | undefined; /** - * If `true`, we want the HTMLMediaElement to play right after the reload is - * done. - * If `false`, we want to stay in a paused state at that point. + * If defined and if the new position obtained after relying on + * `timeOffset` is after `maximumPosition`, then we will reload at + * `maximumPosition` instead. */ - autoPlay : boolean; + maximumPosition : number | undefined; } /** Payload for the `lockedStream` callback. */ @@ -622,28 +630,6 @@ export interface ILockedStreamPayload { bufferType : IBufferType; } -/** Payload for the `needsDecipherabilityFlush` callback. */ -export interface INeedsDecipherabilityFlushPayload { - /** - * Indicated in the case where the MediaSource has to be reloaded, - * in which case the time of the HTMLMediaElement should be reset to that - * position, in seconds, once reloaded. - */ - position : number; - /** - * If `true`, we want the HTMLMediaElement to play right after the flush is - * done. - * If `false`, we want to stay in a paused state at that point. - */ - autoPlay : boolean; - /** - * The duration (maximum seekable position) of the content. - * This is indicated in the case where a seek has to be performed, to avoid - * seeking too far in the content. - */ - duration : number; -} - /** * Returns `true` if low-level buffers have to be "flushed" after the given * `cleanedRanges` time ranges have been removed from an audio or video diff --git a/src/core/stream/period/period_stream.ts b/src/core/stream/period/period_stream.ts index a06db4fd66..78b3a22c27 100644 --- a/src/core/stream/period/period_stream.ts +++ b/src/core/stream/period/period_stream.ts @@ -134,7 +134,7 @@ export default function PeriodStream( if (segmentBufferStatus.type === "initialized") { log.info(`Stream: Clearing previous ${bufferType} SegmentBuffer`); if (SegmentBuffersStore.isNative(bufferType)) { - return askForMediaSourceReload(0, streamCanceller.signal); + return askForMediaSourceReload(0, true, streamCanceller.signal); } else { const periodEnd = period.end ?? Infinity; if (period.start > periodEnd) { @@ -186,7 +186,9 @@ export default function PeriodStream( if (SegmentBuffersStore.isNative(bufferType) && segmentBuffersStore.getStatus(bufferType).type === "disabled") { - return askForMediaSourceReload(relativePosAfterSwitch, streamCanceller.signal); + return askForMediaSourceReload(relativePosAfterSwitch, + true, + streamCanceller.signal); } log.info(`Stream: Updating ${bufferType} adaptation`, @@ -211,7 +213,9 @@ export default function PeriodStream( playbackInfos, options); if (strategy.type === "needs-reload") { - return askForMediaSourceReload(relativePosAfterSwitch, streamCanceller.signal); + return askForMediaSourceReload(relativePosAfterSwitch, + true, + streamCanceller.signal); } await segmentBuffersStore.waitForUsableBuffers(streamCanceller.signal); @@ -304,22 +308,23 @@ export default function PeriodStream( * Regularly ask to reload the MediaSource on each playback observation * performed by the playback observer. * - * If and only if the Period currently played corresponds to the concerned - * Period, applies an offset to the reloaded position corresponding to - * `deltaPos`. - * This can be useful for example when switching the audio/video tracks, where - * you might want to give back some context if that was the currently played - * track. + * @param {number} timeOffset - Relative position, compared to the current + * playhead, at which we should restart playback after reloading. + * For example `-2` will reload 2 seconds before the current position. + * @param {boolean} stayInPeriod - If `true`, we will control that the position + * we reload at, after applying `timeOffset`, is still part of the Period + * `period`. * - * @param {number} deltaPos - If the concerned Period is playing at the time - * this function is called, we will add this value, in seconds, to the current - * position to indicate the position we should reload at. - * This value allows to give back context (by replaying some media data) after - * a switch. + * If it isn't we will re-calculate that reloaded position to be: + * - either the Period's start if the calculated position is before the + * Period's start. + * - either the Period'end start if the calculated position is after the + * Period's end. * @param {Object} cancelSignal */ function askForMediaSourceReload( - deltaPos : number, + timeOffset : number, + stayInPeriod: boolean, cancelSignal : CancellationSignal ) : void { // We begin by scheduling a micro-task to reduce the possibility of race @@ -329,18 +334,11 @@ export default function PeriodStream( // It can happen when `askForMediaSourceReload` is called as a side-effect of // the same event that triggers the playback observation to be emitted. nextTick(() => { - playbackObserver.listen((observation) => { - const currentTime = playbackObserver.getCurrentTime(); - const pos = currentTime + deltaPos; - - // Bind to Period start and end - const position = Math.min(Math.max(period.start, pos), - period.end ?? Infinity); - const autoPlay = !(observation.paused.pending ?? playbackObserver.getIsPaused()); + playbackObserver.listen(() => { callbacks.waitingMediaSourceReload({ bufferType, period, - position, - autoPlay }); + timeOffset, + stayInPeriod }); }, { includeLastObservation: true, clearSignal: cancelSignal }); }); }