From c6b53e6141482fe3d124d590a03dd3afdd7264fb Mon Sep 17 00:00:00 2001 From: Paul Berberian Date: Tue, 6 Aug 2024 18:18:40 +0200 Subject: [PATCH] [Proposal] Implement DASH Thumbnail tracks Overview ======== This is a feature proposal to add support for DASH thumbnail tracks as specified in the DASH-IF IOP 4.3 6.2.6. Those thumbnail tracks generally allow to provide previews when seeking, and it has been linked as such in our demo page. In a DASH MPD ============= In a DASH MPD (its manifest file), such tracks are set as regular `AdaptationSet`, with an `contentType` attribute set to `"image"` and a specific `EssentialProperty` element. To support multiple thumbnail qualities (e.g. bigger or smaller thumbnails depending on the UI, the usage etc.), multiple `Representation` are also possible. A curiosity is that unlike for "trickmode" tracks (which also fill the role of optionally providing thumbnail previews in the RxPlayer, through our experimental `VideoThumbnailLoader` tool), thumbnail tracks are not linked to any video `AdaptationSet`. So if there's multiple video tracks with different content in it, I'm not sure of how we may be able to choose the right thumbnail track, nor how to communicate it through the API. I guess it could be communicated through a `Subset` element, as defined in the DASH specification to force usage of specific AdaptationSets together, but I never actually encountered this element in the wild and it doesn't seem to be supported by any player. The API ======= Simple solution from other players ---------------------------------- For the API, I saw that most other players do very few things. They generally just synchronously return the metadata on a thumbnail corresponding to a specified timestamp. That metadata includes the thumbnail's URL (e.g. to a jpeg), height and width, but also x and y coordinates as thumbnails are often in image sprites (images including multiple images). It is then the role of the application/UI to load and crop this correctly. This seems acceptable to me, after all UI developers are generally experienced working with images and browsers are also very efficient with it (e.g. doing an `.src = url` vs fetching the jpeg through a fetch request + linking the content to the DOM), but I did want to explore another way for multiple reasons: 1. As the core of the RxPlayer may run in another thread (in what we call "multithreading mode"), and as for now precize manifest information is only available in the WebWorker, we would either have to make such kind of API asynchronous (which makes it much less easy to handle for an application), or to send back the corresponding metadata to main thread (with thus supplementary synchronization complexities). 2. As the thumbnail track is just another AdaptationSet/Representation in the MPD, it may be impacted in the same way by other MPD elements and attributes, like multiple CDNs, content steering... Though thumbnail tracks are much less critical (and they also seem explicitely more limited by the DASH-IF IOP than other media types), I have less confidence on being able to provide a stable API in which the RxPlayer would provide all necessary metadata to the application so it can load and render thumbnails, than just do the loading and thumbnail rendering ourselves. Solution I propose ------------------ So I propose here two APIs: ```ts /** * Get synchronously thumbnail information for the specified time, or * `null` if there's no thumbnail information for that time. * * The returned metadata does not allow an application to load and * render thumbnails, it is mainly meant for an application to check if * thumbnails are available at a particular time and which qualities if * there's multiple ones. */ getThumbnailMetadata({ time }: { time: number }): IThumbnailMetadata[] | null; /** Information returned by the `getThumbnailMetadata` method. */ export interface IThumbnailMetadata { /** Identifier identifying a particular thumbnail track. */ id: string; /** * Width in pixels of the individual thumbnails available in that * thumbnail track. */ width: number | undefined; /** * Height in pixels of the individual thumbnails available in that * thumbnail track. */ height: number | undefined; /** * Expected mime-type of the images in that thumbnail track (e.g. * `image/jpeg` or `image/png`. */ mimeType: string | undefined; } ``` Though with that API, it means that an application continuously has to check if there's thumbnail at each timestamp by calling again and again `getThumbnailMetadata` e.g. as a user moves its mouse on top of the seeking bar. So I'm still unsure with that part, we could also communicate like audio and video tracks per Period and only once. And more importantly the loading and rendering API: ```ts /** * Render inside the given `container` the thumbnail corresponding to the * given time. * * If no thumbnail is available at that time or if the RxPlayer does not succeed * to load or render it, reject the corresponding Promise and remove the * potential previous thumbnail from the container. * * If a new `renderThumbnail` call is made with the same `container` before it * had time to finish, the Promise is also rejected but the previous thumbnail * potentially found in the container is untouched. */ public async renderThumbnail(options: IThumbnailRenderingOptions): Promise; export interface IThumbnailRenderingOptions { /** * HTMLElement inside which the thumbnail should be displayed. * * The resulting thumbnail will fill that container if the thumbnail loading * and rendering operations succeeds. * * If there was already a thumbnail rendering request on that container, the * previous operation is cancelled. */ container: HTMLElement; /** Position, in seconds, for which you want to provide an image thumbnail. */ time: number; /** * If set to `true`, we'll keep the potential previous thumbnail found inside * the container if the current `renderThumbnail` call fail on an error. * We'll still replace it if the new `renderThumbnail` call succeeds (with the * new thumbnail). * * If set to `false`, to `undefined`, or not set, the previous thumbnail * potentially found inside the container will also be removed if the new * new `renderThumbnail` call fails. * * The default behavior (equivalent to `false`) is generally more expected, as * you usually don't want to provide an unrelated preview thumbnail for a * completely different time and prefer to display no thumbnail at all. */ keepPreviousThumbnailOnError?: boolean | undefined; /** * If set, specify from which thumbnail track you want to display the * thumbnail from. That identifier can be obtained from the * `getThumbnailMetadata` call (the `id` property). * * This is mainly useful when encountering multiple thumbnail track qualities. */ thumbnailTrackId?: string | undefined; } ``` Basically this method checks which thumbnail to load, load it and render it inside the given element. For now this is done by going through a Canvas element for easy cropping/resizing. I could also go through an image tag and CSS but I was unsure of how my CSS would interact with outside CSS I do not control, so I chose for now the maybe-less efficient canvas way. As you can see in the method description and in its implementation, there's a lot of added complexities from the fact that we do not control the container element (the application is) and that we're doing the loading ourselves instead of just e.g. the browser through an image tag: - Multiple `renderThumbnail` calls may be performed in a row, in which case we have to cancel the previous requests to avoid rendering thumbnails in the wrong order. - If a new thumbnail request fails, we also have to remove the older thumbnail to avoid having stale data. - Because there's a lot of operations which may take some (still minor) time and as often thumbnails are just present in the same image sprite than the one before, there is a tiny cache implementation which handles just that case: if the previous image sprite already contains the right data, we do not go through the RxPlayer's core code (which may be in another thread) and back. Still, I find the corresponding usage by an application relatively simple and elegant: ```js rxPlayer.renderThumbnail({ time, container }) .then(() => console.log("Thumbnail rendered!")) .catch((err) => { if (err,code !== "ABORTED") { console.warn("Error while loading thumbnails:", err); } ); ``` --- demo/scripts/components/ThumbnailPreview.tsx | 212 +++++++++++++++ demo/scripts/components/VideoThumbnail.tsx | 152 ----------- demo/scripts/contents.ts | 12 + demo/scripts/controllers/ProgressBar.tsx | 28 +- demo/scripts/modules/player/index.ts | 16 ++ demo/styles/style.css | 2 +- src/core/fetchers/index.ts | 12 +- .../fetchers/segment/segment_queue_creator.ts | 10 +- src/core/fetchers/thumbnails/index.ts | 8 + .../fetchers/thumbnails/thumbnail_fetcher.ts | 233 ++++++++++++++++ src/core/main/common/get_thumbnail_data.ts | 43 +++ src/core/main/worker/content_preparer.ts | 13 +- src/core/main/worker/worker_main.ts | 70 ++++- src/default_config.ts | 25 ++ src/main_thread/api/public_api.ts | 123 +++++++-- .../init/directfile_content_initializer.ts | 4 + .../init/media_source_content_initializer.ts | 38 ++- .../init/multi_thread_content_initializer.ts | 96 +++++-- src/main_thread/init/types.ts | 12 + src/main_thread/render_thumbnail.ts | 253 ++++++++++++++++++ .../classes/__tests__/manifest.test.ts | 29 +- src/manifest/classes/__tests__/period.test.ts | 62 ++++- .../classes/__tests__/update_periods.test.ts | 22 +- src/manifest/classes/index.ts | 2 + src/manifest/classes/period.ts | 66 ++++- src/manifest/index.ts | 2 + src/manifest/types.ts | 41 +++ src/multithread_types.ts | 42 ++- .../flatten_overlapping_period.test.ts | 29 +- .../dash/common/infer_adaptation_type.ts | 83 +++++- .../dash/common/parse_adaptation_sets.ts | 89 +++++- .../manifest/dash/common/parse_periods.ts | 6 +- .../dash/common/parse_representations.ts | 1 + .../node_parsers/Representation.ts | 7 + .../node_parsers/Representation.ts | 7 + .../manifest/dash/node_parser_types.ts | 1 + .../ts/generators/Representation.ts | 11 + .../manifest/local/parse_local_manifest.ts | 1 + .../metaplaylist/metaplaylist_parser.ts | 1 + src/parsers/manifest/smooth/create_parser.ts | 1 + src/parsers/manifest/types.ts | 47 ++++ src/public_types.ts | 62 +++++ src/transports/dash/pipelines.ts | 5 + src/transports/dash/thumbnails.ts | 96 +++++++ src/transports/local/pipelines.ts | 9 + src/transports/metaplaylist/pipelines.ts | 9 + src/transports/smooth/pipelines.ts | 7 + src/transports/types.ts | 43 +++ 48 files changed, 1851 insertions(+), 292 deletions(-) create mode 100644 demo/scripts/components/ThumbnailPreview.tsx delete mode 100644 demo/scripts/components/VideoThumbnail.tsx create mode 100644 src/core/fetchers/thumbnails/index.ts create mode 100644 src/core/fetchers/thumbnails/thumbnail_fetcher.ts create mode 100644 src/core/main/common/get_thumbnail_data.ts create mode 100644 src/main_thread/render_thumbnail.ts create mode 100644 src/transports/dash/thumbnails.ts diff --git a/demo/scripts/components/ThumbnailPreview.tsx b/demo/scripts/components/ThumbnailPreview.tsx new file mode 100644 index 0000000000..6fbf5363ed --- /dev/null +++ b/demo/scripts/components/ThumbnailPreview.tsx @@ -0,0 +1,212 @@ +import * as React from "react"; +import useModuleState from "../lib/useModuleState"; +import { IPlayerModule } from "../modules/player"; +import { IThumbnailMetadata } from "../../../src/public_types"; + +const DIV_SPINNER_STYLE = { + backgroundColor: "gray", + position: "absolute", + width: "100%", + height: "100%", + opacity: "50%", + display: "flex", + justifyContent: "center", + alignItems: "center", +} as const; + +const IMG_SPINNER_STYLE = { + width: "50%", + margin: "auto", +} as const; + +export default function ThumbnailPreview({ + xPosition, + time, + player, + showVideoThumbnail, +}: { + player: IPlayerModule; + xPosition: number | null; + time: number; + showVideoThumbnail: boolean; +}): JSX.Element { + const videoThumbnailLoader = useModuleState(player, "videoThumbnailLoader"); + const videoElement = useModuleState(player, "videoThumbnailsElement"); + const imageThumbnailElement = useModuleState(player, "imageThumbnailContainerElement"); + const parentElementRef = React.useRef(null); + const [shouldDisplaySpinner, setShouldDisplaySpinner] = React.useState(true); + const ceiledTime = Math.ceil(time); + + // Insert the div element containing the image thumbnail + React.useEffect(() => { + if (showVideoThumbnail) { + return; + } + + if (parentElementRef.current !== null) { + parentElementRef.current.appendChild(imageThumbnailElement); + } + return () => { + if ( + parentElementRef.current !== null && + parentElementRef.current.contains(imageThumbnailElement) + ) { + parentElementRef.current.removeChild(imageThumbnailElement); + } + }; + }, [showVideoThumbnail]); + + // OR insert the video element containing the thumbnail + React.useEffect(() => { + if (!showVideoThumbnail) { + return; + } + if (videoElement !== null && parentElementRef.current !== null) { + parentElementRef.current.appendChild(videoElement); + } + return () => { + if ( + videoElement !== null && + parentElementRef.current !== null && + parentElementRef.current.contains(videoElement) + ) { + parentElementRef.current.removeChild(videoElement); + } + }; + }, [videoElement, showVideoThumbnail]); + + React.useEffect(() => { + if (!showVideoThumbnail) { + return; + } + player.actions.attachVideoThumbnailLoader(); + return () => { + player.actions.dettachVideoThumbnailLoader(); + }; + }, [showVideoThumbnail]); + + // Change the thumbnail when a new time is wanted + React.useEffect(() => { + let spinnerTimeout: number | null = null; + let loadThumbnailTimeout: number | null = null; + + startSpinnerTimeoutIfNotAlreadyStarted(); + + // load thumbnail after a 40ms timer to avoid doing too many requests + // when the user quickly moves its pointer or whatever is calling this + loadThumbnailTimeout = window.setTimeout(() => { + loadThumbnailTimeout = null; + if (showVideoThumbnail) { + if (videoThumbnailLoader === null) { + return; + } + videoThumbnailLoader + .setTime(ceiledTime) + .then(hideSpinner) + .catch((err) => { + if ( + typeof err === "object" && + err !== null && + (err as Partial>).code === "ABORTED" + ) { + return; + } else { + hideSpinner(); + + // eslint-disable-next-line no-console + console.error("Error while loading thumbnails:", err); + } + }); + } else { + const metadata = player.actions.getThumbnailMetadata(ceiledTime); + const thumbnailTrack = metadata.reduce((acc: IThumbnailMetadata | null, t) => { + if (acc === null || acc.height === undefined) { + return t; + } + if (t.height === undefined) { + return acc; + } + if (acc.height > t.height) { + return t.height > 100 ? t : acc; + } else { + return acc.height > 100 ? acc : t; + } + }, null); + if (thumbnailTrack === null) { + hideSpinner(); + return; + } + player.actions + .renderThumbnail(ceiledTime, thumbnailTrack.id) + .then(hideSpinner) + .catch((err) => { + if ( + typeof err === "object" && + err !== null && + (err as Partial>).code === "ABORTED" + ) { + return; + } else { + hideSpinner(); + // eslint-disable-next-line no-console + console.warn("Error while loading thumbnails:", err); + } + }); + } + }, 30); + + return () => { + if (loadThumbnailTimeout !== null) { + clearTimeout(loadThumbnailTimeout); + } + hideSpinner(); + }; + + /** + * Display a spinner after some delay if `stopSpinnerTimeout` hasn't been + * called since. + * This function allows to schedule a spinner if the request to display a + * thumbnail takes too much time. + */ + function startSpinnerTimeoutIfNotAlreadyStarted() { + if (spinnerTimeout !== null) { + return; + } + + // Wait a little before displaying spinner, to + // be sure loading takes time + spinnerTimeout = window.setTimeout(() => { + spinnerTimeout = null; + setShouldDisplaySpinner(true); + }, 150); + } + + /** + * Hide the spinner if one is active and stop the last started spinner + * timeout. + * Allow to avoid showing a spinner when the thumbnail we were waiting for + * was succesfully loaded. + */ + function hideSpinner() { + if (spinnerTimeout !== null) { + clearTimeout(spinnerTimeout); + spinnerTimeout = null; + } + setShouldDisplaySpinner(false); + } + }, [ceiledTime, videoThumbnailLoader, parentElementRef]); + + return ( +
+ {shouldDisplaySpinner ? ( +
+ +
+ ) : null} +
+ ); +} diff --git a/demo/scripts/components/VideoThumbnail.tsx b/demo/scripts/components/VideoThumbnail.tsx deleted file mode 100644 index 1a4cf94ae6..0000000000 --- a/demo/scripts/components/VideoThumbnail.tsx +++ /dev/null @@ -1,152 +0,0 @@ -import * as React from "react"; -import useModuleState from "../lib/useModuleState"; -import { IPlayerModule } from "../modules/player"; - -const DIV_SPINNER_STYLE = { - backgroundColor: "gray", - position: "absolute", - width: "100%", - height: "100%", - opacity: "50%", - display: "flex", - justifyContent: "center", - alignItems: "center", -} as const; - -const IMG_SPINNER_STYLE = { - width: "50%", - margin: "auto", -} as const; - -export default function VideoThumbnail({ - xPosition, - time, - player, -}: { - player: IPlayerModule; - xPosition: number | null; - time: number; -}): JSX.Element { - const videoThumbnailLoader = useModuleState(player, "videoThumbnailLoader"); - const videoElement = useModuleState(player, "videoThumbnailsElement"); - - React.useEffect(() => { - player.actions.attachVideoThumbnailLoader(); - return () => { - player.actions.dettachVideoThumbnailLoader(); - }; - }, []); - - const elementRef = React.useRef(null); - const [shouldDisplaySpinner, setShouldDisplaySpinner] = React.useState(true); - const roundedTime = Math.round(time); - - // Insert the video element containing the thumbnail when it changes - React.useEffect(() => { - if (videoElement !== null && elementRef.current !== null) { - elementRef.current.appendChild(videoElement); - } - return () => { - if ( - videoElement !== null && - elementRef.current !== null && - elementRef.current.contains(videoElement) - ) { - elementRef.current.removeChild(videoElement); - } - }; - }, [videoElement]); - - // Change the thumbnail when a new time is wanted - React.useEffect(() => { - let spinnerTimeout: number | null = null; - let loadThumbnailTimeout: number | null = null; - - if (videoThumbnailLoader === null) { - return; - } - - startSpinnerTimeoutIfNotAlreadyStarted(); - - if (loadThumbnailTimeout !== null) { - clearTimeout(loadThumbnailTimeout); - } - - // load thumbnail after a 40ms timer to avoid doing too many requests - // when the user quickly moves its pointer or whatever is calling this - loadThumbnailTimeout = window.setTimeout(() => { - loadThumbnailTimeout = null; - videoThumbnailLoader - .setTime(roundedTime) - .then(hideSpinner) - .catch((err) => { - if ( - typeof err === "object" && - err !== null && - (err as Partial>).code === "ABORTED" - ) { - return; - } else { - hideSpinner(); - - // eslint-disable-next-line no-console - console.error("Error while loading thumbnails:", err); - } - }); - }, 40); - return () => { - if (loadThumbnailTimeout !== null) { - clearTimeout(loadThumbnailTimeout); - } - hideSpinner(); - }; - - /** - * Display a spinner after some delay if `stopSpinnerTimeout` hasn't been - * called since. - * This function allows to schedule a spinner if the request to display a - * thumbnail takes too much time. - */ - function startSpinnerTimeoutIfNotAlreadyStarted() { - if (spinnerTimeout !== null) { - return; - } - - // Wait a little before displaying spinner, to - // be sure loading takes time - spinnerTimeout = window.setTimeout(() => { - spinnerTimeout = null; - setShouldDisplaySpinner(true); - }, 150); - } - - /** - * Hide the spinner if one is active and stop the last started spinner - * timeout. - * Allow to avoid showing a spinner when the thumbnail we were waiting for - * was succesfully loaded. - */ - function hideSpinner() { - if (spinnerTimeout !== null) { - clearTimeout(spinnerTimeout); - spinnerTimeout = null; - } - - setShouldDisplaySpinner(false); - } - }, [roundedTime, videoThumbnailLoader]); - - return ( -
- {shouldDisplaySpinner ? ( -
- -
- ) : null} -
- ); -} diff --git a/demo/scripts/contents.ts b/demo/scripts/contents.ts index 39ef0246b5..d4c5b5bb1d 100644 --- a/demo/scripts/contents.ts +++ b/demo/scripts/contents.ts @@ -22,6 +22,12 @@ const DEFAULT_CONTENTS: IDefaultContent[] = [ transport: "dash", live: false, }, + { + name: "Live with thumbnail track", + url: "https://livesim2.dashif.org/livesim2/testpic_2s/Manifest_thumbs.mpd", + transport: "dash", + live: true, + }, { name: "Axinom CMAF multiple Audio and Text tracks Tears of steel", url: "https://media.axprod.net/TestVectors/Cmaf/clear_1080p_h264/manifest.mpd", @@ -64,6 +70,12 @@ const DEFAULT_CONTENTS: IDefaultContent[] = [ transport: "dash", live: true, }, + { + name: "VOD with thumbnail track", + url: "https://dash.akamaized.net/akamai/bbb_30fps/bbb_with_tiled_thumbnails.mpd", + transport: "dash", + live: false, + }, { name: "Super SpeedWay", url: "https://playready.directtaps.net/smoothstreaming/SSWSS720H264/SuperSpeedway_720.ism/Manifest", diff --git a/demo/scripts/controllers/ProgressBar.tsx b/demo/scripts/controllers/ProgressBar.tsx index c891175056..a9bcafb008 100644 --- a/demo/scripts/controllers/ProgressBar.tsx +++ b/demo/scripts/controllers/ProgressBar.tsx @@ -1,7 +1,7 @@ import * as React from "react"; import ProgressbarComponent from "../components/ProgressBar"; import ToolTip from "../components/ToolTip"; -import VideoThumbnail from "../components/VideoThumbnail"; +import ThumbnailPreview from "../components/ThumbnailPreview"; import useModuleState from "../lib/useModuleState"; import type { IPlayerModule } from "../modules/player/index"; @@ -66,23 +66,14 @@ function ProgressBar({ setTimeIndicatorText(""); }, [isLive]); - const showVideoTumbnail = React.useCallback((ts: number, clientX: number): void => { + const showThumbnail = React.useCallback((ts: number, clientX: number): void => { const timestampToMs = ts; setThumbnailIsVisible(true); setTipPosition(clientX); setImageTime(timestampToMs); }, []); - const showThumbnail = React.useCallback( - (ts: number, clientX: number): void => { - if (enableVideoThumbnails) { - showVideoTumbnail(ts, clientX); - } - }, - [showVideoTumbnail, enableVideoThumbnails], - ); - - const hideTumbnail = React.useCallback((): void => { + const hideThumbnail = React.useCallback((): void => { setThumbnailIsVisible(false); setTipPosition(0); setImageTime(null); @@ -98,8 +89,8 @@ function ProgressBar({ const hideToolTips = React.useCallback(() => { hideTimeIndicator(); - hideTumbnail(); - }, [hideTumbnail, hideTimeIndicator]); + hideThumbnail(); + }, [hideThumbnail, hideTimeIndicator]); const onMouseMove = React.useCallback( (position: number, event: React.MouseEvent) => { @@ -127,9 +118,14 @@ function ProgressBar({ let thumbnailElement: JSX.Element | null = null; if (thumbnailIsVisible) { const xThumbnailPosition = tipPosition - toolTipOffset; - if (enableVideoThumbnails && imageTime !== null) { + if (imageTime !== null) { thumbnailElement = ( - + ); } } diff --git a/demo/scripts/modules/player/index.ts b/demo/scripts/modules/player/index.ts index d2c2b2dab0..7e609e7770 100644 --- a/demo/scripts/modules/player/index.ts +++ b/demo/scripts/modules/player/index.ts @@ -40,6 +40,7 @@ import type { ITextTrack, IVideoRepresentation, IVideoTrack, + IThumbnailMetadata, } from "../../../../src/public_types"; RxPlayer.addFeatures([ @@ -127,6 +128,7 @@ export interface IPlayerModuleState { isContentLoaded: boolean; isLive: boolean; isLoading: boolean; + imageThumbnailContainerElement: HTMLElement; isPaused: boolean; isReloading: boolean; isSeeking: boolean; @@ -185,6 +187,7 @@ const PlayerModule = declareModule( isContentLoaded: false, isLive: false, isLoading: false, + imageThumbnailContainerElement: document.createElement("div"), isPaused: false, isReloading: false, isSeeking: false, @@ -339,6 +342,19 @@ const PlayerModule = declareModule( player.unMute(); }, + getThumbnailMetadata(time: number): IThumbnailMetadata[] { + const metadata = player.getThumbnailMetadata({ time }); + return metadata ?? []; + }, + + renderThumbnail(time: number, thumbnailTrackId: string): Promise { + return player.renderThumbnail({ + container: state.get("imageThumbnailContainerElement"), + time, + thumbnailTrackId, + }); + }, + setDefaultVideoRepresentationSwitchingMode( mode: IVideoRepresentationsSwitchingMode, ): void { diff --git a/demo/styles/style.css b/demo/styles/style.css index 7477db48d7..8f61553447 100644 --- a/demo/styles/style.css +++ b/demo/styles/style.css @@ -368,7 +368,7 @@ header .right { } .progress-bar-wrapper:hover { - transform: scaleY(2); + transform: scaleY(2.5); } .progress-bar-current { diff --git a/src/core/fetchers/index.ts b/src/core/fetchers/index.ts index 85c6951f9a..e1bed395e6 100644 --- a/src/core/fetchers/index.ts +++ b/src/core/fetchers/index.ts @@ -14,6 +14,7 @@ * limitations under the License. */ +import CdnPrioritizer from "./cdn_prioritizer"; import type { IManifestFetcherSettings, IManifestFetcherEvent, @@ -22,6 +23,8 @@ import type { import ManifestFetcher from "./manifest"; import type { SegmentQueue, ISegmentQueueCreatorBackoffOptions } from "./segment"; import SegmentQueueCreator from "./segment"; +import createThumbnailFetcher, { getThumbnailFetcherRequestOptions } from "./thumbnails"; +import type { IThumbnailFetcher } from "./thumbnails"; export type { IManifestFetcherSettings, @@ -29,5 +32,12 @@ export type { IManifestRefreshSettings, ISegmentQueueCreatorBackoffOptions, SegmentQueue, + IThumbnailFetcher, +}; +export { + CdnPrioritizer, + ManifestFetcher, + SegmentQueueCreator, + createThumbnailFetcher, + getThumbnailFetcherRequestOptions, }; -export { ManifestFetcher, SegmentQueueCreator }; diff --git a/src/core/fetchers/segment/segment_queue_creator.ts b/src/core/fetchers/segment/segment_queue_creator.ts index 454bad041b..09991b0a2e 100644 --- a/src/core/fetchers/segment/segment_queue_creator.ts +++ b/src/core/fetchers/segment/segment_queue_creator.ts @@ -16,10 +16,9 @@ import config from "../../../config"; import type { ISegmentPipeline, ITransportPipelines } from "../../../transports"; -import type { CancellationSignal } from "../../../utils/task_canceller"; import type CmcdDataBuilder from "../../cmcd"; import type { IBufferType } from "../../segment_sinks"; -import CdnPrioritizer from "../cdn_prioritizer"; +import type CdnPrioritizer from "../cdn_prioritizer"; import applyPrioritizerToSegmentFetcher from "./prioritized_segment_fetcher"; import type { ISegmentFetcherLifecycleCallbacks } from "./segment_fetcher"; import createSegmentFetcher, { getSegmentFetcherRequestOptions } from "./segment_fetcher"; @@ -60,17 +59,16 @@ export default class SegmentQueueCreator { /** * @param {Object} transport + * @param {Object} cdnPrioritizer + * @param {Object|null} cmcdDataBuilder * @param {Object} options - * @param {Object} cancelSignal */ constructor( transport: ITransportPipelines, + cdnPrioritizer: CdnPrioritizer, cmcdDataBuilder: CmcdDataBuilder | null, options: ISegmentQueueCreatorBackoffOptions, - cancelSignal: CancellationSignal, ) { - const cdnPrioritizer = new CdnPrioritizer(cancelSignal); - const { MIN_CANCELABLE_PRIORITY, MAX_HIGH_PRIORITY_LEVEL } = config.getCurrent(); this._transport = transport; this._prioritizer = new TaskPrioritizer({ diff --git a/src/core/fetchers/thumbnails/index.ts b/src/core/fetchers/thumbnails/index.ts new file mode 100644 index 0000000000..5af224a5c6 --- /dev/null +++ b/src/core/fetchers/thumbnails/index.ts @@ -0,0 +1,8 @@ +import createThumbnailFetcher, { + getThumbnailFetcherRequestOptions, +} from "./thumbnail_fetcher"; +import type { IThumbnailFetcher } from "./thumbnail_fetcher"; + +export default createThumbnailFetcher; +export { getThumbnailFetcherRequestOptions }; +export type { IThumbnailFetcher }; diff --git a/src/core/fetchers/thumbnails/thumbnail_fetcher.ts b/src/core/fetchers/thumbnails/thumbnail_fetcher.ts new file mode 100644 index 0000000000..6284c89912 --- /dev/null +++ b/src/core/fetchers/thumbnails/thumbnail_fetcher.ts @@ -0,0 +1,233 @@ +import config from "../../../config"; +import { formatError } from "../../../errors"; +import log from "../../../log"; +import type { ISegment, IThumbnailTrack } from "../../../manifest"; +import type { ICdnMetadata } from "../../../parsers/manifest"; +import type { + IThumbnailLoader, + IThumbnailLoaderOptions, + IThumbnailPipeline, + IThumbnailResponse, +} from "../../../transports"; +import objectAssign from "../../../utils/object_assign"; +import type { CancellationSignal } from "../../../utils/task_canceller"; +import { CancellationError } from "../../../utils/task_canceller"; +import type CdnPrioritizer from "../cdn_prioritizer"; +import errorSelector from "../utils/error_selector"; +import { scheduleRequestWithCdns } from "../utils/schedule_request"; + +/** + * Create an `IThumbnailFetcher` object which will allow to easily fetch and parse + * segments. + * An `IThumbnailFetcher` also implements a retry mechanism, based on the given + * `requestOptions` argument, which may retry a segment request when it fails. + * + * @param {Object} pipeline + * @param {Object|null} cdnPrioritizer + * @returns {Function} + */ +export default function createThumbnailFetcher( + /** The transport-specific logic allowing to load thumbnails. */ + pipeline: IThumbnailPipeline, + /** + * Abstraction allowing to synchronize, update and keep track of the + * priorization of the CDN to use to load any given segment, in cases where + * multiple ones are available. + * + * Can be set to `null` in which case a minimal priorization logic will be used + * instead. + */ + cdnPrioritizer: CdnPrioritizer | null, + // TODO CMCD? +): IThumbnailFetcher { + const { loadThumbnail } = pipeline; + + // TODO short-lived cache? + + /** + * Fetch a specific segment. + * @param {Object} thumbnail + * @param {Object} thumbnailTrack + * @param {Object} requestOptions + * @param {Object} cancellationSignal + * @returns {Promise} + */ + return async function fetchThumbnail( + thumbnail: ISegment, + thumbnailTrack: IThumbnailTrack, + requestOptions: IThumbnailFetcherOptions, + cancellationSignal: CancellationSignal, + ): Promise { + let connectionTimeout; + if ( + requestOptions.connectionTimeout === undefined || + requestOptions.connectionTimeout < 0 + ) { + connectionTimeout = undefined; + } else { + connectionTimeout = requestOptions.connectionTimeout; + } + const pipelineRequestOptions: IThumbnailLoaderOptions = { + timeout: + requestOptions.requestTimeout < 0 ? undefined : requestOptions.requestTimeout, + connectionTimeout, + cmcdPayload: undefined, + }; + + log.debug("TF: Beginning thumbnail request", thumbnail.time); + cancellationSignal.register(onCancellation); + let res; + try { + res = await scheduleRequestWithCdns( + thumbnailTrack.cdnMetadata, + cdnPrioritizer, + callLoaderWithUrl, + objectAssign({ onRetry }, requestOptions), + cancellationSignal, + ); + + if (cancellationSignal.isCancelled()) { + return Promise.reject(cancellationSignal.cancellationError); + } + + log.debug("TF: Thumbnail request ended with success", thumbnail.time); + cancellationSignal.deregister(onCancellation); + } catch (err) { + cancellationSignal.deregister(onCancellation); + if (err instanceof CancellationError) { + log.debug("TF: Thumbnail request aborted", thumbnail.time); + throw err; + } + log.debug("TF: Thumbnail request failed", thumbnail.time); + throw errorSelector(err); + } + + try { + const parsed = pipeline.parseThumbnail(res.responseData, { + thumbnail, + thumbnailTrack, + }); + return parsed; + } catch (error) { + throw formatError(error, { + defaultCode: "PIPELINE_PARSE_ERROR", + defaultReason: "Unknown parsing error", + }); + } + function onCancellation() { + log.debug("TF: Thumbnail request cancelled", thumbnail.time); + } + + /** + * Call a segment loader for the given URL with the right arguments. + * @param {Object|null} cdnMetadata + * @returns {Promise} + */ + function callLoaderWithUrl( + cdnMetadata: ICdnMetadata | null, + ): ReturnType { + return loadThumbnail( + cdnMetadata, + thumbnail, + pipelineRequestOptions, + cancellationSignal, + ); + } + + /** + * Function called when the function request is retried. + * @param {*} err + */ + function onRetry(err: unknown): void { + const formattedErr = errorSelector(err); + log.warn("TF: Thumbnail request retry ", thumbnail.time, formattedErr); + } + }; +} + +/** + * Defines the `IThumbnailFetcher` function which allows to load a single segment. + * + * Loaded data is entirely communicated through callbacks present in the + * `callbacks` arguments. + * + * The returned Promise only gives an indication of if the request ended with + * success or on error. + */ +export type IThumbnailFetcher = ( + /** Actual thumbnail you want to load */ + thumbnail: ISegment, + /** Metadata on the linked thumbnails track. */ + thumbnailTrack: IThumbnailTrack, + /** + * Various tweaking requestOptions allowing to configure the behavior of the returned + * `IThumbnailFetcher` regarding segment requests. + */ + requestOptions: IThumbnailFetcherOptions, + /** CancellationSignal allowing to cancel the request. */ + cancellationSignal: CancellationSignal, +) => Promise; + +/** requestOptions allowing to configure an `IThumbnailFetcher`'s behavior. */ +export interface IThumbnailFetcherOptions { + /** + * Initial delay to wait if a request fails before making a new request, in + * milliseconds. + */ + baseDelay: number; + /** + * Maximum delay to wait if a request fails before making a new request, in + * milliseconds. + */ + maxDelay: number; + /** + * Maximum number of retries to perform on "regular" errors (e.g. due to HTTP + * status, integrity errors, timeouts...). + */ + maxRetry: number; + /** + * Timeout after which request are aborted and, depending on other requestOptions, + * retried. + * To set to `-1` for no timeout. + */ + requestTimeout: number; + /** + * Connection timeout, in milliseconds, after which the request is canceled + * if the responses headers has not being received. + * Do not set or set to "undefined" to disable it. + */ + connectionTimeout: number | undefined; +} + +/** + * @param {Object} baseOptions + * @returns {Object} + */ +export function getThumbnailFetcherRequestOptions({ + maxRetry, + requestTimeout, + connectionTimeout, +}: { + maxRetry?: number | undefined; + requestTimeout?: number | undefined; + connectionTimeout?: number | undefined; +}): IThumbnailFetcherOptions { + const { + DEFAULT_MAX_THUMBNAIL_REQUESTS_RETRY_ON_ERROR, + DEFAULT_THUMBNAIL_REQUEST_TIMEOUT, + DEFAULT_THUMBNAIL_CONNECTION_TIMEOUT, + INITIAL_BACKOFF_DELAY_BASE, + MAX_BACKOFF_DELAY_BASE, + } = config.getCurrent(); + return { + maxRetry: maxRetry ?? DEFAULT_MAX_THUMBNAIL_REQUESTS_RETRY_ON_ERROR, + baseDelay: INITIAL_BACKOFF_DELAY_BASE.REGULAR, + maxDelay: MAX_BACKOFF_DELAY_BASE.REGULAR, + requestTimeout: + requestTimeout === undefined ? DEFAULT_THUMBNAIL_REQUEST_TIMEOUT : requestTimeout, + connectionTimeout: + connectionTimeout === undefined + ? DEFAULT_THUMBNAIL_CONNECTION_TIMEOUT + : connectionTimeout, + }; +} diff --git a/src/core/main/common/get_thumbnail_data.ts b/src/core/main/common/get_thumbnail_data.ts new file mode 100644 index 0000000000..6e13327979 --- /dev/null +++ b/src/core/main/common/get_thumbnail_data.ts @@ -0,0 +1,43 @@ +import type { IManifest } from "../../../manifest"; +import type { IThumbnailResponse } from "../../../transports"; +import arrayFind from "../../../utils/array_find"; +import TaskCanceller from "../../../utils/task_canceller"; +import { getThumbnailFetcherRequestOptions } from "../../fetchers"; +import type { IThumbnailFetcher } from "../../fetchers"; + +/** + * @param {function} fetchThumbnails + * @param {Object} manifest + * @param {string} periodId + * @param {string} thumbnailTrackId + * @param {number} time + * @returns {Promise.} + */ +export default async function getThumbnailData( + fetchThumbnails: IThumbnailFetcher, + manifest: IManifest, + periodId: string, + thumbnailTrackId: string, + time: number, +): Promise { + const period = manifest.getPeriod(periodId); + if (period === undefined) { + throw new Error("Wanted Period not found."); + } + const thumbnailTrack = arrayFind(period.thumbnailTracks, (t) => { + return t.id === thumbnailTrackId; + }); + if (thumbnailTrack === undefined) { + throw new Error("Wanted Period has no thumbnail track."); + } + const wantedThumbnail = thumbnailTrack.index.getSegments(time, 1)[0]; + if (wantedThumbnail === undefined) { + throw new Error("No thumbnail for the given timestamp"); + } + return fetchThumbnails( + wantedThumbnail, + thumbnailTrack, + getThumbnailFetcherRequestOptions({}), + new TaskCanceller().signal, + ); +} diff --git a/src/core/main/worker/content_preparer.ts b/src/core/main/worker/content_preparer.ts index 92a12af54d..9ff6b846f8 100644 --- a/src/core/main/worker/content_preparer.ts +++ b/src/core/main/worker/content_preparer.ts @@ -24,6 +24,9 @@ import createAdaptiveRepresentationSelector from "../../adaptive"; import CmcdDataBuilder from "../../cmcd"; import type { IManifestRefreshSettings } from "../../fetchers"; import { ManifestFetcher, SegmentQueueCreator } from "../../fetchers"; +import CdnPrioritizer from "../../fetchers/cdn_prioritizer"; +import createThumbnailFetcher from "../../fetchers/thumbnails/thumbnail_fetcher"; +import type { IThumbnailFetcher } from "../../fetchers/thumbnails/thumbnail_fetcher"; import SegmentSinksStore from "../../segment_sinks"; import type { INeedsMediaSourceReloadPayload } from "../../stream"; import DecipherabilityFreezeDetector from "../common/DecipherabilityFreezeDetector"; @@ -129,11 +132,16 @@ export default class ContentPreparer { }, ); + const cdnPrioritizer = new CdnPrioritizer(contentCanceller.signal); const segmentQueueCreator = new SegmentQueueCreator( dashPipelines, + cdnPrioritizer, cmcdDataBuilder, context.segmentRetryOptions, - contentCanceller.signal, + ); + const fetchThumbnailData = createThumbnailFetcher( + dashPipelines.thumbnails, + cdnPrioritizer, ); const trackChoiceSetter = new TrackChoiceSetter(); @@ -161,6 +169,7 @@ export default class ContentPreparer { representationEstimator, segmentSinksStore, segmentQueueCreator, + fetchThumbnailData, workerTextSender, trackChoiceSetter, }; @@ -363,6 +372,8 @@ export interface IPreparedContentData { * fetching. */ segmentQueueCreator: SegmentQueueCreator; + /** Allows to load image thumbnails. */ + fetchThumbnailData: IThumbnailFetcher; /** * Allows to store and update the wanted tracks and Representation inside that * track. diff --git a/src/core/main/worker/worker_main.ts b/src/core/main/worker/worker_main.ts index a305e95a9f..d3686e2551 100644 --- a/src/core/main/worker/worker_main.ts +++ b/src/core/main/worker/worker_main.ts @@ -8,6 +8,7 @@ import type { IDiscontinuityUpdateWorkerMessagePayload, IMainThreadMessage, IReferenceUpdateMessage, + IThumbnailDataRequestMainMessage, } from "../../../multithread_types"; import { MainThreadMessageType, WorkerMessageType } from "../../../multithread_types"; import DashFastJsParser from "../../../parsers/manifest/dash/fast-js-parser"; @@ -34,6 +35,7 @@ import type { import StreamOrchestrator from "../../stream"; import createContentTimeBoundariesObserver from "../common/create_content_time_boundaries_observer"; import getBufferedDataPerMediaBuffer from "../common/get_buffered_data_per_media_buffer"; +import getThumbnailData from "../common/get_thumbnail_data"; import ContentPreparer from "./content_preparer"; import { limitVideoResolution, @@ -397,7 +399,12 @@ export default function initializeWorkerMain() { } case MainThreadMessageType.PullSegmentSinkStoreInfos: { - sendSegmentSinksStoreInfos(contentPreparer, msg.value.messageId); + sendSegmentSinksStoreInfos(contentPreparer, msg.value.requestId); + break; + } + + case MainThreadMessageType.ThumbnailDataRequest: { + sendThumbnailData(contentPreparer, msg); break; } @@ -935,7 +942,7 @@ function updateLoggerLevel( */ function sendSegmentSinksStoreInfos( contentPreparer: ContentPreparer, - messageId: number, + requestId: number, ): void { const currentContent = contentPreparer.getCurrentContent(); if (currentContent === null) { @@ -945,6 +952,63 @@ function sendSegmentSinksStoreInfos( sendMessage({ type: WorkerMessageType.SegmentSinkStoreUpdate, contentId: currentContent.contentId, - value: { segmentSinkMetrics: segmentSinksMetrics, messageId }, + value: { segmentSinkMetrics: segmentSinksMetrics, requestId }, }); } + +/** + * Handles thumbnail requests and send back the result to the main thread. + * @param {ContentPreparer} contentPreparer + * @returns {void} + */ +function sendThumbnailData( + contentPreparer: ContentPreparer, + msg: IThumbnailDataRequestMainMessage, +): void { + const preparedContent = contentPreparer.getCurrentContent(); + const respondWithError = (err: unknown) => { + sendMessage({ + type: WorkerMessageType.ThumbnailDataResponse, + contentId: msg.contentId, + value: { + status: "error", + requestId: msg.value.requestId, + error: formatErrorForSender(err), + }, + }); + }; + + if ( + preparedContent === null || + preparedContent.manifest === null || + preparedContent.contentId !== msg.contentId + ) { + return respondWithError(new Error("Content changed")); + } + + getThumbnailData( + preparedContent.fetchThumbnailData, + preparedContent.manifest, + msg.value.periodId, + msg.value.thumbnailTrackId, + msg.value.time, + ).then( + (result) => { + sendMessage( + { + type: WorkerMessageType.ThumbnailDataResponse, + contentId: msg.contentId, + value: { + status: "success", + requestId: msg.value.requestId, + data: result, + }, + }, + [result.data], + ); + }, + (err) => { + return respondWithError(err); + }, + ); +} diff --git a/src/default_config.ts b/src/default_config.ts index d9308ee270..b2e2f23995 100644 --- a/src/default_config.ts +++ b/src/default_config.ts @@ -1183,6 +1183,31 @@ const DEFAULT_CONFIG = { * one. */ DEFAULT_AUDIO_TRACK_SWITCHING_MODE: "seamless" as const, + + /** + * The default number of times a thumbnail request will be re-performed when + * on error which justify a retry. + * + * Note that some errors do not use this counter: + * - if the error is not due to the xhr, no retry will be peformed + * - if the error is an HTTP error code, but not a 500-smthg or a 404, no + * retry will be performed. + * @type Number + */ + DEFAULT_MAX_THUMBNAIL_REQUESTS_RETRY_ON_ERROR: 1, + + /** + * Default time interval after which a thumbnail request will timeout, in ms. + * @type {Number} + */ + DEFAULT_THUMBNAIL_REQUEST_TIMEOUT: 10 * 1000, + + /** + * Default connection time after which a thumbnail request conncection will + * timeout, in ms. + * @type {Number} + */ + DEFAULT_THUMBNAIL_CONNECTION_TIMEOUT: 7 * 1000, }; export type IDefaultConfig = typeof DEFAULT_CONFIG; diff --git a/src/main_thread/api/public_api.ts b/src/main_thread/api/public_api.ts index 4a3940e507..f5cc4b9c29 100644 --- a/src/main_thread/api/public_api.ts +++ b/src/main_thread/api/public_api.ts @@ -60,6 +60,7 @@ import { getMinimumSafePosition, ManifestMetadataFormat, createRepresentationFilterFromFnString, + getPeriodForTime, } from "../../manifest"; import type { IWorkerMessage } from "../../multithread_types"; import { MainThreadMessageType, WorkerMessageType } from "../../multithread_types"; @@ -99,7 +100,10 @@ import type { ITrackType, IModeInformation, IWorkerSettings, + IThumbnailMetadata, + IThumbnailRenderingOptions, } from "../../public_types"; +import type { IThumbnailResponse } from "../../transports"; import arrayFind from "../../utils/array_find"; import arrayIncludes from "../../utils/array_includes"; import assert, { assertUnreachable } from "../../utils/assert"; @@ -121,6 +125,7 @@ import { getKeySystemConfiguration, } from "../decrypt"; import type { ContentInitializer } from "../init"; +import renderThumbnail from "../render_thumbnail"; import type { IMediaElementTracksStore, ITSPeriodObject } from "../tracks_store"; import TracksStore from "../tracks_store"; import type { IParsedLoadVideoOptions, IParsedStartAtOption } from "./option_utils"; @@ -381,14 +386,6 @@ class Player extends EventEmitter { } } - /** - * Function passed from the ContentInitializer that return segment sinks metrics. - * This is used for monitor and debugging. - */ - private _priv_segmentSinkMetricsCallback: - | null - | (() => Promise); - /** * @constructor * @param {Object} options @@ -465,8 +462,6 @@ class Player extends EventEmitter { this._priv_worker = null; - this._priv_segmentSinkMetricsCallback = null; - const onVolumeChange = () => { this.trigger("volumeChange", { volume: videoElement.volume, @@ -741,6 +736,51 @@ class Player extends EventEmitter { }; } + /** + * Returns either an array decribing the various thumbnail tracks that can be + * encountered at the given time, or `null` if no thumbnail track is available + * at that time. + * @param {number} time - The position to check for thumbnail tracks, in + * seconds. + * @returns {Array.|null} + */ + public getThumbnailMetadata({ time }: { time: number }): IThumbnailMetadata[] | null { + if (this._priv_contentInfos === null || this._priv_contentInfos.manifest === null) { + return null; + } + const period = getPeriodForTime(this._priv_contentInfos.manifest, time); + if (period === undefined || period.thumbnailTracks.length === 0) { + return null; + } + return period.thumbnailTracks.map((t) => { + return { + id: t.id, + width: Math.floor(t.width / t.horizontalTiles), + height: Math.floor(t.height / t.verticalTiles), + mimeType: t.mimeType, + }; + }); + } + + /** + * Render inside the given `container` the thumbnail corresponding to the + * given time. + * + * If no thumbnail is available at that time or if the RxPlayer does not succeed + * to load or render it, reject the corresponding Promise and remove the + * potential previous thumbnail from the container. + * + * If a new `renderThumbnail` call is made with the same `container` before it + * had time to finish, the Promise is also rejected but the previous thumbnail + * potentially found in the container is untouched. + * + * @param {Object|undefined} options + * @returns {Promise} + */ + public async renderThumbnail(options: IThumbnailRenderingOptions): Promise { + return renderThumbnail(this._priv_contentInfos, options); + } + /** * From given options, initialize content playback. * @param {Object} options @@ -1012,6 +1052,12 @@ class Player extends EventEmitter { tracksStore: null, mediaElementTracksStore, useWorker, + segmentSinkMetricsCallback: null, + fetchThumbnailDataCallback: null, + thumbnailRequestsInfo: { + pendingRequests: new Map(), + lastResponse: null, + }, }; // Bind events @@ -1030,7 +1076,9 @@ class Player extends EventEmitter { if (contentInfos.tracksStore !== null) { contentInfos.tracksStore.resetPeriodObjects(); } - this._priv_segmentSinkMetricsCallback = null; + if (this._priv_contentInfos !== null) { + this._priv_contentInfos.segmentSinkMetricsCallback = null; + } this._priv_lastAutoPlay = payload.autoPlay; }); initializer.addEventListener("inbandEvents", (inbandEvents) => @@ -1074,7 +1122,10 @@ class Player extends EventEmitter { this._priv_onDecipherabilityUpdate(contentInfos, updates), ); initializer.addEventListener("loaded", (evt) => { - this._priv_segmentSinkMetricsCallback = evt.getSegmentSinkMetrics; + if (this._priv_contentInfos !== null) { + this._priv_contentInfos.segmentSinkMetricsCallback = evt.getSegmentSinkMetrics; + this._priv_contentInfos.fetchThumbnailDataCallback = evt.getThumbnailData; + } }); // Now, that most events are linked, prepare the next content. @@ -2434,11 +2485,7 @@ class Player extends EventEmitter { * @returns */ async __priv_getSegmentSinkMetrics(): Promise { - if (this._priv_segmentSinkMetricsCallback === null) { - return undefined; - } else { - return this._priv_segmentSinkMetricsCallback(); - } + return this._priv_contentInfos?.segmentSinkMetricsCallback?.(); } /** @@ -2506,7 +2553,6 @@ class Player extends EventEmitter { this._priv_contentInfos?.tracksStore?.dispose(); this._priv_contentInfos?.mediaElementTracksStore?.dispose(); this._priv_contentInfos = null; - this._priv_segmentSinkMetricsCallback = null; this._priv_contentEventsMemory = {}; @@ -3348,7 +3394,7 @@ interface IPublicAPIEvent { } /** State linked to a particular contents loaded by the public API. */ -interface IPublicApiContentInfos { +export interface IPublicApiContentInfos { /** * Unique identifier for this `IPublicApiContentInfos` object. * Allows to identify and thus compare this `contentInfos` object with another @@ -3410,6 +3456,45 @@ interface IPublicApiContentInfos { * content. */ useWorker: boolean; + /** + * Function passed from the ContentInitializer that return segment sinks metrics. + * This is used for monitor and debugging. + */ + segmentSinkMetricsCallback: null | (() => Promise); + /** + * Function allowing to retrieve thumbnails from a content. + */ + fetchThumbnailDataCallback: + | null + | (( + periodId: string, + thumbnailTrackId: string, + time: number, + ) => Promise); + /** Metadata related to thumbnail rendering for the current content. */ + thumbnailRequestsInfo: { + /** + * Thumbnail requests that are still pending, identified by the thumbnail + * container. + * The value allows to cancel that task. + */ + pendingRequests: Map; + /** + * Metadata about the last requested thumbnails. + * + * This is an optimization to avoid an unnecessary request and round-trip to + * the core code as many times thumbnail previews asked by applications are + * really close to the last asked one, often in the same thumbnail resource. + */ + lastResponse: { + /** Actual thumbnail data response from core RxPlayer code. */ + response: IThumbnailResponse; + /** The identifier for the Period for which that request was made. */ + periodId: string; + /** The identifier for the thumbnail track for which that request was made. */ + thumbnailTrackId: string; + } | null; + }; } export default Player; diff --git a/src/main_thread/init/directfile_content_initializer.ts b/src/main_thread/init/directfile_content_initializer.ts index 94501cd309..478fc47edb 100644 --- a/src/main_thread/init/directfile_content_initializer.ts +++ b/src/main_thread/init/directfile_content_initializer.ts @@ -231,6 +231,10 @@ export default class DirectFileContentInitializer extends ContentInitializer { stopListening(); this.trigger("loaded", { getSegmentSinkMetrics: null, + getThumbnailData: () => + Promise.reject( + new Error("Thumbnail data not available with directfile contents"), + ), }); } }, diff --git a/src/main_thread/init/media_source_content_initializer.ts b/src/main_thread/init/media_source_content_initializer.ts index e2c004c065..4b0e31e29c 100644 --- a/src/main_thread/init/media_source_content_initializer.ts +++ b/src/main_thread/init/media_source_content_initializer.ts @@ -25,9 +25,15 @@ import type { } from "../../core/adaptive"; import AdaptiveRepresentationSelector from "../../core/adaptive"; import CmcdDataBuilder from "../../core/cmcd"; -import { ManifestFetcher, SegmentQueueCreator } from "../../core/fetchers"; +import { + CdnPrioritizer, + createThumbnailFetcher, + ManifestFetcher, + SegmentQueueCreator, +} from "../../core/fetchers"; import createContentTimeBoundariesObserver from "../../core/main/common/create_content_time_boundaries_observer"; import DecipherabilityFreezeDetector from "../../core/main/common/DecipherabilityFreezeDetector"; +import getThumbnailData from "../../core/main/common/get_thumbnail_data"; import SegmentSinksStore from "../../core/segment_sinks"; import type { IStreamOrchestratorOptions, @@ -49,7 +55,7 @@ import type { IKeySystemOption, IPlayerError, } from "../../public_types"; -import type { ITransportPipelines } from "../../transports"; +import type { IThumbnailResponse, ITransportPipelines } from "../../transports"; import areArraysOfNumbersEqual from "../../utils/are_arrays_of_numbers_equal"; import assert from "../../utils/assert"; import createCancellablePromise from "../../utils/create_cancellable_promise"; @@ -429,11 +435,12 @@ export default class MediaSourceContentInitializer extends ContentInitializer { bufferOptions, ); + const cdnPrioritizer = new CdnPrioritizer(initCanceller.signal); const segmentQueueCreator = new SegmentQueueCreator( transport, + cdnPrioritizer, this._cmcdDataBuilder, segmentRequestOptions, - initCanceller.signal, ); this._refreshManifestCodecSupport(manifest); @@ -478,6 +485,7 @@ export default class MediaSourceContentInitializer extends ContentInitializer { autoPlay: shouldPlay, manifest, representationEstimator, + cdnPrioritizer, segmentQueueCreator, speed, bufferOptions: subBufferOptions, @@ -538,9 +546,11 @@ export default class MediaSourceContentInitializer extends ContentInitializer { mediaSource, playbackObserver, representationEstimator, + cdnPrioritizer, segmentQueueCreator, speed, } = args; + const { transport } = this._settings; const initialPeriod = manifest.getPeriodForTime(initialTime) ?? manifest.getNextPeriod(initialTime); @@ -748,6 +758,23 @@ export default class MediaSourceContentInitializer extends ContentInitializer { resolve(segmentSinksStore.getSegmentSinksMetrics()), ); }, + getThumbnailData: async ( + periodId: string, + thumbnailTrackId: string, + time: number, + ): Promise => { + const fetchThumbnails = createThumbnailFetcher( + transport.thumbnails, + cdnPrioritizer, + ); + return getThumbnailData( + fetchThumbnails, + manifest, + periodId, + thumbnailTrackId, + time, + ); + }, }); } }, @@ -1222,6 +1249,11 @@ interface IBufferingMediaSettings { playbackObserver: IMediaElementPlaybackObserver; /** Estimate the right Representation. */ representationEstimator: IRepresentationEstimator; + /** + * Interface allowing to prioritize CDN between one another depending on past + * performances, content steering, etc. + */ + cdnPrioritizer: CdnPrioritizer; /** Module to facilitate segment fetching. */ segmentQueueCreator: SegmentQueueCreator; /** Last wanted playback rate. */ diff --git a/src/main_thread/init/multi_thread_content_initializer.ts b/src/main_thread/init/multi_thread_content_initializer.ts index de7dc83298..d3bc4179a2 100644 --- a/src/main_thread/init/multi_thread_content_initializer.ts +++ b/src/main_thread/init/multi_thread_content_initializer.ts @@ -40,7 +40,7 @@ import type { IKeySystemOption, IPlayerError, } from "../../public_types"; -import type { ITransportOptions } from "../../transports"; +import type { IThumbnailResponse, ITransportOptions } from "../../transports"; import arrayFind from "../../utils/array_find"; import assert, { assertUnreachable } from "../../utils/assert"; import idGenerator from "../../utils/id_generator"; @@ -99,12 +99,24 @@ export default class MultiThreadContentInitializer extends ContentInitializer { private _currentMediaSourceCanceller: TaskCanceller; /** - * Stores the resolvers and the current messageId that is sent to the web worker to receive segment sink metrics. + * Stores the resolvers and the current requestId that is sent to the web worker to receive segment sink metrics. * The purpose of collecting metrics is for monitoring and debugging. */ - private _segmentMetrics: { - lastMessageId: number; - resolvers: Record void>; + private _awaitingRequests: { + nextRequestId: number; + pendingSinkMetrics: Map< + number /* request id */, + { + resolve: (value: ISegmentSinkMetrics | undefined) => void; + } + >; + pendingThumbnailFetching: Map< + number /* request id */, + { + resolve: (value: IThumbnailResponse) => void; + reject: (error: Error) => void; + } + >; }; /** @@ -119,9 +131,10 @@ export default class MultiThreadContentInitializer extends ContentInitializer { this._currentMediaSourceCanceller = new TaskCanceller(); this._currentMediaSourceCanceller.linkToSignal(this._initCanceller.signal); this._currentContentInfo = null; - this._segmentMetrics = { - lastMessageId: 0, - resolvers: {}, + this._awaitingRequests = { + nextRequestId: 0, + pendingSinkMetrics: new Map(), + pendingThumbnailFetching: new Map(), }; } @@ -1089,15 +1102,37 @@ export default class MultiThreadContentInitializer extends ContentInitializer { if (this._currentContentInfo?.contentId !== msgData.contentId) { return; } - const resolveFn = this._segmentMetrics.resolvers[msgData.value.messageId]; - if (resolveFn !== undefined) { - resolveFn(msgData.value.segmentSinkMetrics); - delete this._segmentMetrics.resolvers[msgData.value.messageId]; + const sinkObj = this._awaitingRequests.pendingSinkMetrics.get( + msgData.value.requestId, + ); + if (sinkObj !== undefined) { + sinkObj.resolve(msgData.value.segmentSinkMetrics); + this._awaitingRequests.pendingSinkMetrics.delete(msgData.value.requestId); } else { log.error("MTCI: Failed to send segment sink store update"); } break; } + case WorkerMessageType.ThumbnailDataResponse: + if (this._currentContentInfo?.contentId !== msgData.contentId) { + return; + } + const tObj = this._awaitingRequests.pendingThumbnailFetching.get( + msgData.value.requestId, + ); + if (tObj !== undefined) { + if (msgData.value.status === "error") { + tObj.reject(formatWorkerError(msgData.value.error)); + } else { + tObj.resolve(msgData.value.data); + } + this._awaitingRequests.pendingThumbnailFetching.delete( + msgData.value.requestId, + ); + } else { + log.error("MTCI: Failed to send segment sink store update"); + } + break; default: assertUnreachable(msgData); } @@ -1528,24 +1563,44 @@ export default class MultiThreadContentInitializer extends ContentInitializer { { clearSignal: cancelSignal, emitCurrentValue: true }, ); - const _getSegmentSinkMetrics: () => Promise< - ISegmentSinkMetrics | undefined - > = async () => { - this._segmentMetrics.lastMessageId++; - const messageId = this._segmentMetrics.lastMessageId; + const _getSegmentSinkMetrics = async (): Promise => { + this._awaitingRequests.nextRequestId++; + const requestId = this._awaitingRequests.nextRequestId; sendMessage(this._settings.worker, { type: MainThreadMessageType.PullSegmentSinkStoreInfos, - value: { messageId }, + value: { requestId }, }); return new Promise((resolve, reject) => { - this._segmentMetrics.resolvers[messageId] = resolve; + this._awaitingRequests.pendingSinkMetrics.set(requestId, { resolve }); const rejectFn = (err: CancellationError) => { - delete this._segmentMetrics.resolvers[messageId]; + this._awaitingRequests.pendingSinkMetrics.delete(requestId); return reject(err); }; cancelSignal.register(rejectFn); }); }; + const _getThumbnailsData = ( + periodId: string, + thumbnailTrackId: string, + time: number, + ): Promise => { + if (this._currentContentInfo === null) { + return Promise.reject(new Error("Cannot fetch thumbnails: No content loaded.")); + } + this._awaitingRequests.nextRequestId++; + const requestId = this._awaitingRequests.nextRequestId; + sendMessage(this._settings.worker, { + type: MainThreadMessageType.ThumbnailDataRequest, + contentId: this._currentContentInfo.contentId, + value: { requestId, periodId, thumbnailTrackId, time }, + }); + return new Promise((resolve, reject) => { + this._awaitingRequests.pendingThumbnailFetching.set(requestId, { + resolve, + reject, + }); + }); + }; /** * Emit a "loaded" events once the initial play has been performed and the * media can begin playback. @@ -1559,6 +1614,7 @@ export default class MultiThreadContentInitializer extends ContentInitializer { stopListening(); this.trigger("loaded", { getSegmentSinkMetrics: _getSegmentSinkMetrics, + getThumbnailData: _getThumbnailsData, }); } }, diff --git a/src/main_thread/init/types.ts b/src/main_thread/init/types.ts index c0911b5b84..73da0bcaba 100644 --- a/src/main_thread/init/types.ts +++ b/src/main_thread/init/types.ts @@ -27,6 +27,7 @@ import type { } from "../../manifest"; import type { IMediaElementPlaybackObserver } from "../../playback_observer"; import type { IPlayerError } from "../../public_types"; +import type { IThumbnailResponse } from "../../transports"; import EventEmitter from "../../utils/event_emitter"; import type SharedReference from "../../utils/reference"; import type { @@ -146,6 +147,17 @@ export interface IContentInitializerEvents { */ loaded: { getSegmentSinkMetrics: null | (() => Promise); + /** + * Fetch the thumbnail data of the given Period for the corresponding time. + * If there's no thumbnail for that Period or if the request fails, reject + * the Promise with a given reason. + * @param {number} time + */ + getThumbnailData: ( + periodId: string, + thumbnailTrackId: string, + time: number, + ) => Promise; }; /** Event emitted when a stream event is encountered. */ streamEvent: IPublicStreamEvent | IPublicNonFiniteStreamEvent; diff --git a/src/main_thread/render_thumbnail.ts b/src/main_thread/render_thumbnail.ts new file mode 100644 index 0000000000..a9b7d355ee --- /dev/null +++ b/src/main_thread/render_thumbnail.ts @@ -0,0 +1,253 @@ +import { formatError } from "../errors"; +import errorMessage from "../errors/error_message"; +import { getPeriodForTime } from "../manifest"; +import type { IThumbnailRenderingOptions } from "../public_types"; +import type { IThumbnailResponse } from "../transports"; +import arrayFind from "../utils/array_find"; +import TaskCanceller from "../utils/task_canceller"; +import type { IPublicApiContentInfos } from "./api/public_api"; + +/** + * Render thumbnail available at `time` in the given `container` (in place of + * a potential previously-rendered thumbnail in that container). + * + * If there is no thumbnail at this time, or if there is but it fails to + * load/render, also removes the previously displayed thumbnail, unless + * `options.keepPreviousThumbnailOnError` is set to `true`. + * + * Returns a Promise which resolves when the thumbnail is rendered successfully, + * rejects if anything prevented a thumbnail to be rendered. + * + * A newer `renderThumbnail` call performed while a previous `renderThumbnail` + * call on the same container did not yet finish will abort that previous call, + * rejecting the old call's returned promise. + * + * You may know if the promise returned by `renderThumbnail` rejected due to it + * being aborted, by checking the `code` property on the rejected error: Error + * due to aborting have their `code` property set to `ABORTED`. + * + * @param {Object} contentInfos + * @param {Object} options + * @returns {Object} + */ +export default async function renderThumbnail( + contentInfos: IPublicApiContentInfos | null, + options: IThumbnailRenderingOptions, +): Promise { + const { time, container } = options; + if ( + contentInfos === null || + contentInfos.fetchThumbnailDataCallback === null || + contentInfos.manifest === null + ) { + return Promise.reject( + new ThumbnailRenderingError( + "NO_CONTENT", + "Cannot get thumbnail: no content loaded", + ), + ); + } + + const { thumbnailRequestsInfo, currentContentCanceller } = contentInfos; + const canceller = new TaskCanceller(); + canceller.linkToSignal(currentContentCanceller.signal); + + let imageUrl: string | undefined; + + const olderTaskSameContainer = thumbnailRequestsInfo.pendingRequests.get(container); + olderTaskSameContainer?.cancel(); + + thumbnailRequestsInfo.pendingRequests.set(container, canceller); + + const onFinished = () => { + canceller.cancel(); + thumbnailRequestsInfo.pendingRequests.delete(container); + + // Let's revoke the URL after a round-trip to the event loop just in case + // to prevent revoking before the browser use it. + // This is normally not necessary, but better safe than sorry. + setTimeout(() => { + if (imageUrl !== undefined) { + URL.revokeObjectURL(imageUrl); + } + }, 0); + }; + + try { + const period = getPeriodForTime(contentInfos.manifest, time); + if (period === undefined) { + throw new ThumbnailRenderingError("NO_THUMBNAIL", "Wanted Period not found."); + } + const thumbnailTracks = period.thumbnailTracks; + const thumbnailTrack = + options.thumbnailTrackId !== undefined + ? arrayFind(thumbnailTracks, (t) => t.id === options.thumbnailTrackId) + : thumbnailTracks[0]; + if (thumbnailTrack === undefined) { + if (options.thumbnailTrackId !== undefined) { + throw new ThumbnailRenderingError( + "NO_THUMBNAIL", + "Given `thumbnailTrackId` not found", + ); + } else { + throw new ThumbnailRenderingError( + "NO_THUMBNAIL", + "Wanted Period has no thumbnail track.", + ); + } + } + + const { lastResponse } = thumbnailRequestsInfo; + let res: IThumbnailResponse | undefined; + if ( + lastResponse !== null && + lastResponse.thumbnailTrackId === thumbnailTrack.id && + lastResponse.periodId === period.id + ) { + const previousThumbs = lastResponse.response.thumbnails; + if ( + previousThumbs.length > 0 && + time >= previousThumbs[0].start && + time < previousThumbs[previousThumbs.length - 1].end + ) { + res = lastResponse.response; + } + } + + if (res === undefined) { + res = await contentInfos.fetchThumbnailDataCallback( + period.id, + thumbnailTrack.id, + time, + ); + thumbnailRequestsInfo.lastResponse = { + response: res, + periodId: period.id, + thumbnailTrackId: thumbnailTrack.id, + }; + } + const canvas = document.createElement("canvas"); + const context = canvas.getContext("2d"); + if (context === null) { + throw new ThumbnailRenderingError( + "RENDERING", + "Cannot display thumbnail: cannot create canvas context", + ); + } + let foundIdx: number | undefined; + for (let i = 0; i < res.thumbnails.length; i++) { + if (res.thumbnails[i].start <= time && res.thumbnails[i].end > time) { + foundIdx = i; + break; + } + } + if (foundIdx === undefined) { + throw new Error("Cannot display thumbnail: time not found in fetched data"); + } + const image = new Image(); + const blob = new Blob([res.data], { type: res.mimeType }); + imageUrl = URL.createObjectURL(blob); + image.src = imageUrl; + canvas.height = res.thumbnails[foundIdx].height; + canvas.width = res.thumbnails[foundIdx].width; + return new Promise((resolve, reject) => { + image.onload = () => { + try { + context.drawImage( + image, + res.thumbnails[foundIdx].offsetX, + res.thumbnails[foundIdx].offsetY, + res.thumbnails[foundIdx].width, + res.thumbnails[foundIdx].height, + 0, + 0, + res.thumbnails[foundIdx].width, + res.thumbnails[foundIdx].height, + ); + canvas.style.width = "100%"; + canvas.style.height = "100%"; + canvas.className = "__rx-thumbnail__"; + clearPreviousThumbnails(); + container.appendChild(canvas); + resolve(); + } catch (srcError) { + reject( + new ThumbnailRenderingError( + "RENDERING", + "Could not draw the image in a canvas", + ), + ); + } + onFinished(); + }; + + image.onerror = () => { + if (options.keepPreviousThumbnailOnError !== true) { + clearPreviousThumbnails(); + } + reject( + new ThumbnailRenderingError( + "RENDERING", + "Could not load the corresponding image in the DOM", + ), + ); + onFinished(); + }; + }); + } catch (srcError) { + if (options.keepPreviousThumbnailOnError !== true) { + clearPreviousThumbnails(); + } + if (srcError !== null && srcError === canceller.signal.cancellationError) { + const error = new ThumbnailRenderingError( + "ABORTED", + "Thumbnail rendering has been aborted", + ); + throw error; + } + const formattedErr = formatError(srcError, { + defaultCode: "NONE", + defaultReason: "Unknown error", + }); + + let returnedError; + if (formattedErr.type === "NETWORK_ERROR") { + returnedError = new ThumbnailRenderingError("LOADING", formattedErr.message); + } else { + returnedError = new ThumbnailRenderingError("NOT_FOUND", formattedErr.message); + } + onFinished(); + throw returnedError; + } + + function clearPreviousThumbnails() { + for (let i = container.children.length - 1; i >= 0; i--) { + const child = container.children[i]; + if (child.className === "__rx-thumbnail__") { + container.removeChild(child); + } + } + } +} + +/** + * Error specifcically defined for the thumbnail rendering API. + * A caller is then supposed to programatically classify the type of error + * by checking the `code` property from such an error. + * @class ThumbnailRenderingError + */ +class ThumbnailRenderingError extends Error { + public readonly name: "ThumbnailRenderingError"; + public readonly code: string; + + /** + * @param {string} code + * @param {string} message + */ + constructor(code: string, message: string) { + super(errorMessage(code, message)); + Object.setPrototypeOf(this, ThumbnailRenderingError.prototype); + this.name = "ThumbnailRenderingError"; + this.code = code; + } +} diff --git a/src/manifest/classes/__tests__/manifest.test.ts b/src/manifest/classes/__tests__/manifest.test.ts index 9cd9b78778..d37f824f34 100644 --- a/src/manifest/classes/__tests__/manifest.test.ts +++ b/src/manifest/classes/__tests__/manifest.test.ts @@ -24,6 +24,7 @@ function generateParsedPeriod( adaptations, duration: end === undefined ? undefined : end - start, streamEvents: [], + thumbnailTracks: [], }; } function generateParsedAudioAdaptation(id: string): IParsedAdaptation { @@ -221,8 +222,8 @@ describe("Manifest - Manifest", () => { it("should expose the adaptations of the first period if set", async () => { const adapP1 = {}; const adapP2 = {}; - const period1 = { id: "0", start: 4, adaptations: adapP1 }; - const period2 = { id: "1", start: 12, adaptations: adapP2 }; + const period1 = { id: "0", start: 4, adaptations: adapP1, thumbnailTracks: [] }; + const period2 = { id: "1", start: 12, adaptations: adapP2, thumbnailTracks: [] }; const simpleFakeManifest = { id: "man", isDynamic: false, @@ -267,8 +268,8 @@ describe("Manifest - Manifest", () => { ); expect(manifest.periods).toEqual([ - { id: "foo0", start: 4, adaptations: adapP1 }, - { id: "foo1", start: 12, adaptations: adapP2 }, + { id: "foo0", start: 4, adaptations: adapP1, thumbnailTracks: [] }, + { id: "foo1", start: 12, adaptations: adapP2, thumbnailTracks: [] }, ]); expect(manifest.adaptations).toBe(adapP1); @@ -412,8 +413,20 @@ describe("Manifest - Manifest", () => { expect(manifest.getMaximumSafePosition()).toEqual(10); expect(manifest.getMinimumSafePosition()).toEqual(5); expect(manifest.periods).toEqual([ - { id: "foo0", adaptations: oldPeriod1.adaptations, start: 4, streamEvents: [] }, - { id: "foo1", adaptations: oldPeriod2.adaptations, start: 12, streamEvents: [] }, + { + id: "foo0", + adaptations: oldPeriod1.adaptations, + start: 4, + streamEvents: [], + thumbnailTracks: [], + }, + { + id: "foo1", + adaptations: oldPeriod2.adaptations, + start: 12, + streamEvents: [], + thumbnailTracks: [], + }, ]); expect(manifest.suggestedPresentationDelay).toEqual(99); expect(manifest.uris).toEqual(["url1", "url2"]); @@ -478,8 +491,8 @@ describe("Manifest - Manifest", () => { isLastPeriodKnown: true, lifetime: 13, periods: [ - { id: "0", start: 4, adaptations: oldPeriod1.adaptations }, - { id: "1", start: 12, adaptations: oldPeriod2.adaptations }, + { id: "0", start: 4, adaptations: oldPeriod1.adaptations, thumbnailTracks: [] }, + { id: "1", start: 12, adaptations: oldPeriod2.adaptations, thumbnailTracks: [] }, ], suggestedPresentationDelay: 99, timeBounds: { diff --git a/src/manifest/classes/__tests__/period.test.ts b/src/manifest/classes/__tests__/period.test.ts index 371cd9c19f..40c86117ed 100644 --- a/src/manifest/classes/__tests__/period.test.ts +++ b/src/manifest/classes/__tests__/period.test.ts @@ -24,7 +24,7 @@ describe("Manifest - Period", () => { })); const Period = (await vi.importActual("../period")).default as typeof IPeriod; - const args = { id: "12", adaptations: {}, start: 0 }; + const args = { id: "12", adaptations: {}, start: 0, thumbnailTracks: [] }; let period: IPeriod | null = null; let errorReceived: unknown = null; const unsupportedAdaptations: Adaptation[] = []; @@ -83,6 +83,7 @@ describe("Manifest - Period", () => { id: "12", adaptations: { foo }, start: 0, + thumbnailTracks: [], } as unknown as IParsedPeriod; let period: IPeriod | null = null; let errorReceived: unknown = null; @@ -131,7 +132,12 @@ describe("Manifest - Period", () => { })); const Period = (await vi.importActual("../period")).default as typeof IPeriod; - const args = { id: "12", adaptations: { video: [], audio: [] }, start: 0 }; + const args = { + id: "12", + thumbnailTracks: [], + adaptations: { video: [], audio: [] }, + start: 0, + }; let period: IPeriod | null = null; let errorReceived: unknown = null; const unsupportedAdaptations: Adaptation[] = []; @@ -223,6 +229,7 @@ describe("Manifest - Period", () => { const args: IParsedPeriod = { id: "12", adaptations: { video, audio }, + thumbnailTracks: [], start: 0, } as unknown as IParsedPeriod; let period: IPeriod | null = null; @@ -324,6 +331,7 @@ describe("Manifest - Period", () => { id: "12", adaptations: { video, audio }, start: 0, + thumbnailTracks: [], }; let period: IPeriod | null = null; let errorReceived: unknown = null; @@ -422,7 +430,12 @@ describe("Manifest - Period", () => { audioAda1, audioAda2, ] as unknown as IParsedAdaptation[]; - const args: IParsedPeriod = { id: "12", adaptations: { video, audio }, start: 0 }; + const args: IParsedPeriod = { + id: "12", + adaptations: { video, audio }, + start: 0, + thumbnailTracks: [], + }; let period: IPeriod | null = null; let errorReceived: unknown = null; const unsupportedAdaptations: Adaptation[] = []; @@ -515,7 +528,12 @@ describe("Manifest - Period", () => { }, }; const audio = [audioAda1, audioAda2] as unknown as IParsedAdaptation[]; - const args = { id: "12", adaptations: { video, audio }, start: 0 }; + const args = { + id: "12", + adaptations: { video, audio }, + start: 0, + thumbnailTracks: [], + }; let period: IPeriod | null = null; let errorReceived: unknown = null; const unsupportedAdaptations: Adaptation[] = []; @@ -580,7 +598,12 @@ describe("Manifest - Period", () => { }, }; const video2 = [videoAda2] as unknown as IParsedAdaptation[]; - const args = { id: "12", adaptations: { video, video2 }, start: 0 }; + const args = { + id: "12", + adaptations: { video, video2 }, + start: 0, + thumbnailTracks: [], + }; const unsupportedAdaptations: Adaptation[] = []; const codecSupportCache = new CodecSupportCache([]); const period = new Period(args, unsupportedAdaptations, codecSupportCache); @@ -626,7 +649,7 @@ describe("Manifest - Period", () => { }; const video = [videoAda1] as unknown as IParsedAdaptation[]; const bar = undefined; - const args = { id: "12", adaptations: { bar, video }, start: 0 }; + const args = { id: "12", thumbnailTracks: [], adaptations: { bar, video }, start: 0 }; const unsupportedAdaptations: Adaptation[] = []; const codecSupportCache = new CodecSupportCache([]); const period = new Period(args, unsupportedAdaptations, codecSupportCache); @@ -682,7 +705,7 @@ describe("Manifest - Period", () => { }, }; const video = [videoAda1, videoAda2] as unknown as IParsedAdaptation[]; - const args = { id: "12", adaptations: { video }, start: 0 }; + const args = { id: "12", adaptations: { video }, start: 0, thumbnailTracks: [] }; const unsupportedAdaptations: Adaptation[] = []; const codecSupportCache = new CodecSupportCache([]); const period = new Period( @@ -746,7 +769,7 @@ describe("Manifest - Period", () => { }; const video = [videoAda1, videoAda2] as unknown as IParsedAdaptation[]; const foo = [fooAda1]; - const args = { id: "12", adaptations: { video, foo }, start: 0 }; + const args = { id: "12", thumbnailTracks: [], adaptations: { video, foo }, start: 0 }; const unsupportedAdaptations: Adaptation[] = []; const codecSupportCache = new CodecSupportCache([]); new Period(args, unsupportedAdaptations, codecSupportCache); @@ -798,7 +821,7 @@ describe("Manifest - Period", () => { }; const video = [videoAda1, videoAda2] as unknown as IParsedAdaptation[]; const foo = [fooAda1]; - const args = { id: "12", adaptations: { video, foo }, start: 0 }; + const args = { id: "12", thumbnailTracks: [], adaptations: { video, foo }, start: 0 }; const unsupportedAdaptations: Adaptation[] = []; const codecSupportCache = new CodecSupportCache([]); new Period(args, unsupportedAdaptations, codecSupportCache); @@ -840,7 +863,7 @@ describe("Manifest - Period", () => { }, }; const video = [videoAda1, videoAda2] as unknown as IParsedAdaptation[]; - const args = { id: "12", adaptations: { video }, start: 72 }; + const args = { id: "12", adaptations: { video }, start: 72, thumbnailTracks: [] }; const unsupportedAdaptations: Adaptation[] = []; const codecSupportCache = new CodecSupportCache([]); const period = new Period(args, unsupportedAdaptations, codecSupportCache); @@ -885,7 +908,13 @@ describe("Manifest - Period", () => { }, }; const video = [videoAda1, videoAda2] as unknown as IParsedAdaptation[]; - const args = { id: "12", adaptations: { video }, start: 0, duration: 12 }; + const args = { + id: "12", + adaptations: { video }, + start: 0, + duration: 12, + thumbnailTracks: [], + }; const unsupportedAdaptations: Adaptation[] = []; const codecSupportCache = new CodecSupportCache([]); const period = new Period(args, unsupportedAdaptations, codecSupportCache); @@ -930,7 +959,13 @@ describe("Manifest - Period", () => { }, }; const video = [videoAda1, videoAda2] as unknown as IParsedAdaptation[]; - const args = { id: "12", adaptations: { video }, start: 50, duration: 12 }; + const args = { + id: "12", + adaptations: { video }, + start: 50, + duration: 12, + thumbnailTracks: [], + }; const unsupportedAdaptations: Adaptation[] = []; const codecSupportCache = new CodecSupportCache([]); const period = new Period(args, unsupportedAdaptations, codecSupportCache); @@ -988,6 +1023,7 @@ describe("Manifest - Period", () => { const args = { id: "12", + thumbnailTracks: [], adaptations: { video, audio }, start: 50, duration: 12, @@ -1050,6 +1086,7 @@ describe("Manifest - Period", () => { const args = { id: "12", + thumbnailTracks: [], adaptations: { video, audio }, start: 50, duration: 12, @@ -1129,6 +1166,7 @@ describe("Manifest - Period", () => { const args = { id: "12", + thumbnailTracks: [], adaptations: { video, audio }, start: 50, duration: 12, diff --git a/src/manifest/classes/__tests__/update_periods.test.ts b/src/manifest/classes/__tests__/update_periods.test.ts index 96861cb99b..6b969015a2 100644 --- a/src/manifest/classes/__tests__/update_periods.test.ts +++ b/src/manifest/classes/__tests__/update_periods.test.ts @@ -32,6 +32,7 @@ function generateFakePeriod({ duration: end === undefined ? undefined : end - (start ?? 0), streamEvents: [], adaptations: {}, + thumbnailTracks: [], refreshCodecSupport() { // noop }, @@ -57,6 +58,7 @@ function generateFakePeriod({ id: id ?? String(start), streamEvents: [], adaptations: {}, + thumbnailTracks: [], }; }, }; @@ -393,7 +395,7 @@ describe("Manifest - updatePeriods", () => { removedPeriods: [], updatedPeriods: [ { - period: { id: "p2", start: 60, streamEvents: [] }, + period: { id: "p2", start: 60, streamEvents: [], thumbnailTracks: [] }, result: fakeUpdatePeriodInPlaceRes, }, ], @@ -434,7 +436,7 @@ describe("Manifest - updatePeriods", () => { removedPeriods: [], updatedPeriods: [ { - period: { id: "p2", start: 60, streamEvents: [] }, + period: { id: "p2", start: 60, streamEvents: [], thumbnailTracks: [] }, result: fakeUpdatePeriodInPlaceRes, }, ], @@ -527,11 +529,11 @@ describe("Manifest - updatePeriods", () => { removedPeriods: [{ id: "p1.5", start: 69, end: 70 }], updatedPeriods: [ { - period: { id: "p1", start: 60, end: 69, streamEvents: [] }, + period: { id: "p1", start: 60, end: 69, streamEvents: [], thumbnailTracks: [] }, result: fakeUpdatePeriodInPlaceRes, }, { - period: { id: "p2", start: 70, streamEvents: [] }, + period: { id: "p2", start: 70, streamEvents: [], thumbnailTracks: [] }, result: fakeUpdatePeriodInPlaceRes, }, ], @@ -805,11 +807,11 @@ describe("Manifest - updatePeriods", () => { removedPeriods: [], updatedPeriods: [ { - period: { id: "p1", start: 60, end: 70, streamEvents: [] }, + period: { id: "p1", start: 60, end: 70, streamEvents: [], thumbnailTracks: [] }, result: fakeUpdatePeriodInPlaceRes, }, { - period: { id: "p2", start: 70, streamEvents: [] }, + period: { id: "p2", start: 70, streamEvents: [], thumbnailTracks: [] }, result: fakeUpdatePeriodInPlaceRes, }, ], @@ -865,11 +867,11 @@ describe("Manifest - updatePeriods", () => { removedPeriods: [{ id: "p2", start: 70, end: 80 }], updatedPeriods: [ { - period: { id: "p1", start: 60, end: 70, streamEvents: [] }, + period: { id: "p1", start: 60, end: 70, streamEvents: [], thumbnailTracks: [] }, result: fakeUpdatePeriodInPlaceRes, }, { - period: { id: "p3", start: 80, streamEvents: [] }, + period: { id: "p3", start: 80, streamEvents: [], thumbnailTracks: [] }, result: fakeUpdatePeriodInPlaceRes, }, ], @@ -928,11 +930,11 @@ describe("Manifest - updatePeriods", () => { ], updatedPeriods: [ { - period: { id: "p1", start: 60, end: 70, streamEvents: [] }, + period: { id: "p1", start: 60, end: 70, streamEvents: [], thumbnailTracks: [] }, result: fakeUpdatePeriodInPlaceRes, }, { - period: { id: "p3", start: 80, end: 90, streamEvents: [] }, + period: { id: "p3", start: 80, end: 90, streamEvents: [], thumbnailTracks: [] }, result: fakeUpdatePeriodInPlaceRes, }, ], diff --git a/src/manifest/classes/index.ts b/src/manifest/classes/index.ts index 4db4d9de1a..c98d80f332 100644 --- a/src/manifest/classes/index.ts +++ b/src/manifest/classes/index.ts @@ -19,6 +19,7 @@ import type { ICodecSupportInfo } from "./codec_support_cache"; import type { IDecipherabilityUpdateElement, IManifestParsingOptions } from "./manifest"; import Manifest from "./manifest"; import Period from "./period"; +import type { IThumbnailTrack } from "./period"; import Representation from "./representation"; import type { IMetaPlaylistPrivateInfos, @@ -42,6 +43,7 @@ export type { IRepresentationIndex, IPrivateInfos, ISegment, + IThumbnailTrack, }; export { areSameContent, diff --git a/src/manifest/classes/period.ts b/src/manifest/classes/period.ts index e31996c27c..05a35b34c4 100644 --- a/src/manifest/classes/period.ts +++ b/src/manifest/classes/period.ts @@ -14,7 +14,11 @@ * limitations under the License. */ import { MediaError } from "../../errors"; -import type { IManifestStreamEvent, IParsedPeriod } from "../../parsers/manifest"; +import type { + ICdnMetadata, + IManifestStreamEvent, + IParsedPeriod, +} from "../../parsers/manifest"; import type { ITrackType, IRepresentationFilter } from "../../public_types"; import arrayFind from "../../utils/array_find"; import isNullOrUndefined from "../../utils/is_null_or_undefined"; @@ -22,6 +26,7 @@ import type { IAdaptationMetadata, IPeriodMetadata } from "../types"; import { getAdaptations, getSupportedAdaptations, periodContainsTime } from "../utils"; import Adaptation from "./adaptation"; import type CodecSupportCache from "./codec_support_cache"; +import type { IRepresentationIndex } from "./representation_index"; /** Structure listing every `Adaptation` in a Period. */ export type IManifestAdaptations = Partial>; @@ -56,6 +61,11 @@ export default class Period implements IPeriodMetadata { /** Array containing every stream event happening on the period */ public streamEvents: IManifestStreamEvent[]; + /** + * If set to an object, this Period has thumbnail tracks. + */ + public thumbnailTracks: IThumbnailTrack[]; + /** * @constructor * @param {Object} args @@ -126,6 +136,16 @@ export default class Period implements IPeriodMetadata { ); } + this.thumbnailTracks = args.thumbnailTracks.map((thumbnailTrack) => ({ + id: thumbnailTrack.id, + mimeType: thumbnailTrack.mimeType, + index: thumbnailTrack.index, + cdnMetadata: thumbnailTrack.cdnMetadata, + height: thumbnailTrack.height, + width: thumbnailTrack.width, + horizontalTiles: thumbnailTrack.horizontalTiles, + verticalTiles: thumbnailTrack.verticalTiles, + })); this.duration = args.duration; this.start = args.start; @@ -278,6 +298,50 @@ export default class Period implements IPeriodMetadata { id: this.id, streamEvents: this.streamEvents, adaptations, + thumbnailTracks: this.thumbnailTracks.map((thumbnailTrack) => ({ + id: thumbnailTrack.id, + mimeType: thumbnailTrack.mimeType, + height: thumbnailTrack.height, + width: thumbnailTrack.width, + horizontalTiles: thumbnailTrack.horizontalTiles, + verticalTiles: thumbnailTrack.verticalTiles, + })), }; } } + +/** + * Metadata on an image thumbnail track associated to a Period. + */ +export interface IThumbnailTrack { + /** Identifier for that thumbnail track. */ + id: string; + /** interface allowing to obtain information on the actual thumbnails. */ + index: IRepresentationIndex; + /** Mime-type for loaded thumbnails, allowing to know their format. */ + mimeType: string; + /** CDN(s) on which the thumbnails may be loaded. */ + cdnMetadata: ICdnMetadata[] | null; + /** + * A loaded thumbnail's height in pixels. Note that there can be multiple actual + * thumbnails per loaded thumbnail resource (see `horizontalTiles` and + * `verticalTiles` properties. + */ + height: number; + /** + * A loaded thumbnail's width in pixels. Note that there can be multiple actual + * thumbnails per loaded thumbnail resource (see `horizontalTiles` and + * `verticalTiles` properties. + */ + width: number; + /** + * Thumbnail tracks are usually grouped together. This is the number of + * images contained horizontally in a whole loaded thumbnail resource. + */ + horizontalTiles: number; + /** + * Thumbnail tracks are usually grouped together. This is the number of + * images contained vertically in a whole loaded thumbnail resource. + */ + verticalTiles: number; +} diff --git a/src/manifest/index.ts b/src/manifest/index.ts index b42643cefd..a366f1d008 100644 --- a/src/manifest/index.ts +++ b/src/manifest/index.ts @@ -9,6 +9,7 @@ import type { IRepresentationIndex, IMetaPlaylistPrivateInfos, IPrivateInfos, + IThumbnailTrack, } from "./classes"; import type Manifest from "./classes"; import { areSameContent, getLoggableSegmentId } from "./classes"; @@ -33,6 +34,7 @@ export type { ISegment, IMetaPlaylistPrivateInfos, IPrivateInfos, + IThumbnailTrack, }; export { areSameContent, getLoggableSegmentId }; export type { diff --git a/src/manifest/types.ts b/src/manifest/types.ts index 0edc8e4140..94497de68f 100644 --- a/src/manifest/types.ts +++ b/src/manifest/types.ts @@ -254,6 +254,47 @@ export interface IPeriodMetadata { adaptations: Partial>; /** Array containing every stream event happening on the period */ streamEvents: IManifestStreamEvent[]; + /** If set to an object, this Period has a thumbnail track. */ + thumbnailTracks: IThumbnailTrackMetadata[]; +} + +/** Describes metadata about a single image thumbnail track. */ +export interface IThumbnailTrackMetadata { + /** Identify that thumbnail track. */ + id: string; + /** Estimated mime-type for the loaded thumbnails (e.g. `"image/jpeg"`). */ + mimeType: string; + /** + * A loaded thumbnail's height in pixels. Note that there can be multiple actual + * thumbnails per loaded thumbnail resource (see `horizontalTiles` and + * `verticalTiles` properties. + */ + height: number; + /** + * A loaded thumbnail's width in pixels. Note that there can be multiple actual + * thumbnails per loaded thumbnail resource (see `horizontalTiles` and + * `verticalTiles` properties. + */ + width: number; + /** + * Thumbnail tracks are usually grouped together. This is the number of + * images contained horizontally in a whole loaded thumbnail resource. + */ + horizontalTiles: number; + /** + * Thumbnail tracks are usually grouped together. This is the number of + * images contained vertically in a whole loaded thumbnail resource. + */ + verticalTiles: number; +} + +export interface ILoadedThumbnailData { + data: BufferSource; + mimeType: string; + start: number; + end?: number | undefined; + height?: number | undefined; + width?: number | undefined; } /** diff --git a/src/multithread_types.ts b/src/multithread_types.ts index f6b5a5132e..e11cda37c6 100644 --- a/src/multithread_types.ts +++ b/src/multithread_types.ts @@ -29,7 +29,7 @@ import type { } from "./mse"; import type { IFreezingStatus, IRebufferingStatus } from "./playback_observer"; import type { ICmcdOptions, ITrackType } from "./public_types"; -import type { ITransportOptions } from "./transports"; +import type { IThumbnailResponse, ITransportOptions } from "./transports"; import type { ILogFormat, ILoggerLevel } from "./utils/logger"; import type { IRange } from "./utils/ranges"; @@ -508,6 +508,18 @@ export interface IRemoveTextDataErrorMessage { }; } +/** Message sent from main thread when it wants to fetch thumbnail data. */ +export interface IThumbnailDataRequestMainMessage { + type: MainThreadMessageType.ThumbnailDataRequest; + contentId: string; + value: { + requestId: number; + periodId: string; + thumbnailTrackId: string; + time: number; + }; +} + /** * Template for a message originating from main thread to update * `SharedReference` objects (a common abstraction of the RxPlayer allowing for @@ -531,7 +543,7 @@ export type IReferenceUpdateMessage = export interface IPullSegmentSinkStoreInfos { type: MainThreadMessageType.PullSegmentSinkStoreInfos; - value: { messageId: number }; + value: { requestId: number }; } export const enum MainThreadMessageType { @@ -555,6 +567,7 @@ export const enum MainThreadMessageType { StopContent = "stop", TrackUpdate = "track-update", PullSegmentSinkStoreInfos = "pull-segment-sink-store-infos", + ThumbnailDataRequest = "thumbnail-request", } export type IMainThreadMessage = @@ -577,7 +590,8 @@ export type IMainThreadMessage = | IPushTextDataErrorMessage | IRemoveTextDataErrorMessage | IMediaSourceReadyStateChangeMainMessage - | IPullSegmentSinkStoreInfos; + | IPullSegmentSinkStoreInfos + | IThumbnailDataRequestMainMessage; export type ISentError = | ISerializedNetworkError @@ -928,10 +942,26 @@ export interface ISegmentSinkStoreUpdateMessage { contentId: string; value: { segmentSinkMetrics: ISegmentSinkMetrics; - messageId: number; + requestId: number; }; } +export interface IThumbnailDataResponseWorkerMessage { + type: WorkerMessageType.ThumbnailDataResponse; + contentId: string; + value: + | { + status: "error"; + requestId: number; + error: ISentError; + } + | { + status: "success"; + requestId: number; + data: IThumbnailResponse; + }; +} + export const enum WorkerMessageType { AbortSourceBuffer = "abort-source-buffer", ActivePeriodChanged = "active-period-changed", @@ -970,6 +1000,7 @@ export const enum WorkerMessageType { UpdatePlaybackRate = "update-playback-rate", Warning = "warning", SegmentSinkStoreUpdate = "segment-sink-store-update", + ThumbnailDataResponse = "thumbnail-response", } export type IWorkerMessage = @@ -1009,4 +1040,5 @@ export type IWorkerMessage = | IUpdateMediaSourceDurationWorkerMessage | IUpdatePlaybackRateWorkerMessage | IWarningWorkerMessage - | ISegmentSinkStoreUpdateMessage; + | ISegmentSinkStoreUpdateMessage + | IThumbnailDataResponseWorkerMessage; diff --git a/src/parsers/manifest/dash/common/__tests__/flatten_overlapping_period.test.ts b/src/parsers/manifest/dash/common/__tests__/flatten_overlapping_period.test.ts index 9098852e64..e2bf13fe8e 100644 --- a/src/parsers/manifest/dash/common/__tests__/flatten_overlapping_period.test.ts +++ b/src/parsers/manifest/dash/common/__tests__/flatten_overlapping_period.test.ts @@ -17,9 +17,9 @@ describe("flattenOverlappingPeriods", function () { const mockLog = vi.spyOn(log, "warn").mockImplementation(vi.fn()); const periods = [ - { id: "1", start: 0, duration: 60, adaptations: {} }, - { id: "2", start: 60, duration: 60, adaptations: {} }, - { id: "3", start: 60, duration: 60, adaptations: {} }, + { id: "1", start: 0, duration: 60, thumbnailTracks: [], adaptations: {} }, + { id: "2", start: 60, duration: 60, thumbnailTracks: [], adaptations: {} }, + { id: "3", start: 60, duration: 60, thumbnailTracks: [], adaptations: {} }, ]; const flattenPeriods = flattenOverlappingPeriods(periods); @@ -42,9 +42,9 @@ describe("flattenOverlappingPeriods", function () { const mockLog = vi.spyOn(log, "warn").mockImplementation(vi.fn()); const periods = [ - { id: "1", start: 0, duration: 60, adaptations: {} }, - { id: "2", start: 60, duration: 60, adaptations: {} }, - { id: "3", start: 90, duration: 60, adaptations: {} }, + { id: "1", start: 0, duration: 60, thumbnailTracks: [], adaptations: {} }, + { id: "2", start: 60, duration: 60, thumbnailTracks: [], adaptations: {} }, + { id: "3", start: 90, duration: 60, thumbnailTracks: [], adaptations: {} }, ]; const flattenPeriods = flattenOverlappingPeriods(periods); @@ -70,9 +70,9 @@ describe("flattenOverlappingPeriods", function () { const mockLog = vi.spyOn(log, "warn").mockImplementation(vi.fn()); const periods = [ - { id: "1", start: 0, duration: 60, adaptations: {} }, - { id: "2", start: 60, duration: 60, adaptations: {} }, - { id: "3", start: 50, duration: 120, adaptations: {} }, + { id: "1", start: 0, duration: 60, thumbnailTracks: [], adaptations: {} }, + { id: "2", start: 60, duration: 60, thumbnailTracks: [], adaptations: {} }, + { id: "3", start: 50, duration: 120, thumbnailTracks: [], adaptations: {} }, ]; const flattenPeriods = flattenOverlappingPeriods(periods); @@ -97,13 +97,16 @@ describe("flattenOverlappingPeriods", function () { it("should keep last announced period from multiple periods with same start and end", function () { const mockLog = vi.spyOn(log, "warn").mockImplementation(vi.fn()); - const periods = [{ id: "1", start: 0, duration: 60, adaptations: {} }]; + const periods = [ + { id: "1", start: 0, duration: 60, thumbnailTracks: [], adaptations: {} }, + ]; for (let i = 1; i <= 100; i++) { periods.push({ id: i.toString(), start: 60, duration: 60, + thumbnailTracks: [], adaptations: {}, }); } @@ -127,9 +130,9 @@ describe("flattenOverlappingPeriods", function () { const mockLog = vi.spyOn(log, "warn").mockImplementation(vi.fn()); const periods = [ - { id: "1", start: 40, duration: 20, adaptations: {} }, - { id: "2", start: 60, duration: 20, adaptations: {} }, - { id: "3", start: 20, duration: 100, adaptations: {} }, + { id: "1", start: 40, duration: 20, thumbnailTracks: [], adaptations: {} }, + { id: "2", start: 60, duration: 20, thumbnailTracks: [], adaptations: {} }, + { id: "3", start: 20, duration: 100, thumbnailTracks: [], adaptations: {} }, ]; const flattenPeriods = flattenOverlappingPeriods(periods); diff --git a/src/parsers/manifest/dash/common/infer_adaptation_type.ts b/src/parsers/manifest/dash/common/infer_adaptation_type.ts index ea725105f2..a33065a48f 100644 --- a/src/parsers/manifest/dash/common/infer_adaptation_type.ts +++ b/src/parsers/manifest/dash/common/infer_adaptation_type.ts @@ -14,10 +14,16 @@ * limitations under the License. */ +import log from "../../../../log"; import { SUPPORTED_ADAPTATIONS_TYPE } from "../../../../manifest"; import arrayFind from "../../../../utils/array_find"; import arrayIncludes from "../../../../utils/array_includes"; -import type { IRepresentationIntermediateRepresentation } from "../node_parser_types"; +import isNonEmptyString from "../../../../utils/is_non_empty_string"; +import isNullOrUndefined from "../../../../utils/is_null_or_undefined"; +import type { + IAdaptationSetIntermediateRepresentation, + IRepresentationIntermediateRepresentation, +} from "../node_parser_types"; /** Different "type" a parsed Adaptation can be. */ type IAdaptationType = "audio" | "video" | "text"; @@ -31,6 +37,56 @@ interface IScheme { value?: string | undefined; } +/** + * From a thumbnail AdaptationSet, returns core information such as the number + * of tiles vertically and horizontally per image. + * + * Returns `null` if the information could not be parsed. + * @param {Object} adaptation + * @returns {Object|null} + */ +export function getThumbnailAdaptationSetInfo( + adaptation: IAdaptationSetIntermediateRepresentation, + representation?: IRepresentationIntermediateRepresentation | undefined, +): { + horizontalTiles: number; + verticalTiles: number; +} | null { + const thumbnailProp = + arrayFind( + adaptation.children.essentialProperties ?? [], + (p) => + p.schemeIdUri === "http://dashif.org/guidelines/thumbnail_tile" || + p.schemeIdUri === "http://dashif.org/thumbnail_tile", + ) ?? + arrayFind( + (representation ?? adaptation.children.representations[0])?.children + .essentialProperties ?? [], + (p) => + p.schemeIdUri === "http://dashif.org/guidelines/thumbnail_tile" || + p.schemeIdUri === "http://dashif.org/thumbnail_tile", + ); + if (thumbnailProp === undefined) { + return null; + } + const tilesRegex = /(\d+)x(\d+)/; + if ( + thumbnailProp === undefined || + thumbnailProp.value === undefined || + !tilesRegex.test(thumbnailProp.value) + ) { + log.warn("DASH: Invalid thumbnails Representation, no tile-related information"); + return null; + } + const match = thumbnailProp.value.match(tilesRegex) as RegExpMatchArray; + const horizontalTiles = parseInt(match[1], 10); + const verticalTiles = parseInt(match[2], 10); + return { + horizontalTiles, + verticalTiles, + }; +} + /** * Infers the type of adaptation from codec and mimetypes found in it. * @@ -42,18 +98,29 @@ interface IScheme { * 3. codec * * Note: This is based on DASH-IF-IOP-v4.0 with some more freedom. + * @param {Object} adaptation * @param {Array.} representations - * @param {string|null} adaptationMimeType - * @param {string|null} adaptationCodecs - * @param {Array.|null} adaptationRoles * @returns {string} - "audio"|"video"|"text"|"metadata"|"unknown" */ export default function inferAdaptationType( + adaptation: IAdaptationSetIntermediateRepresentation, representations: IRepresentationIntermediateRepresentation[], - adaptationMimeType: string | null, - adaptationCodecs: string | null, - adaptationRoles: IScheme[] | null, -): IAdaptationType | undefined { +): IAdaptationType | "thumbnails" | undefined { + if (adaptation.attributes.contentType === "image") { + if (getThumbnailAdaptationSetInfo(adaptation) !== null) { + return "thumbnails"; + } + return undefined; + } + const adaptationMimeType = isNonEmptyString(adaptation.attributes.mimeType) + ? adaptation.attributes.mimeType + : null; + const adaptationCodecs = isNonEmptyString(adaptation.attributes.codecs) + ? adaptation.attributes.codecs + : null; + const adaptationRoles = !isNullOrUndefined(adaptation.children.roles) + ? adaptation.children.roles + : null; function fromMimeType( mimeType: string, roles: IScheme[] | null, diff --git a/src/parsers/manifest/dash/common/parse_adaptation_sets.ts b/src/parsers/manifest/dash/common/parse_adaptation_sets.ts index f7ffeee88d..4409fe9436 100644 --- a/src/parsers/manifest/dash/common/parse_adaptation_sets.ts +++ b/src/parsers/manifest/dash/common/parse_adaptation_sets.ts @@ -23,14 +23,21 @@ import arrayFindIndex from "../../../../utils/array_find_index"; import arrayIncludes from "../../../../utils/array_includes"; import isNonEmptyString from "../../../../utils/is_non_empty_string"; import isNullOrUndefined from "../../../../utils/is_null_or_undefined"; -import type { IParsedAdaptation, IParsedAdaptations } from "../../types"; +import type { + IParsedAdaptation, + IParsedAdaptations, + IParsedRepresentation, + IParsedThumbnailTrack, +} from "../../types"; import type { IAdaptationSetIntermediateRepresentation, ISegmentTemplateIntermediateRepresentation, } from "../node_parser_types"; import attachTrickModeTrack from "./attach_trickmode_track"; import type ContentProtectionParser from "./content_protection_parser"; -import inferAdaptationType from "./infer_adaptation_type"; +import inferAdaptationType, { + getThumbnailAdaptationSetInfo, +} from "./infer_adaptation_type"; import type { IRepresentationContext } from "./parse_representations"; import parseRepresentations from "./parse_representations"; import resolveBaseURLs from "./resolve_base_urls"; @@ -259,11 +266,15 @@ function getAdaptationSetSwitchingIDs( export default function parseAdaptationSets( adaptationsIR: IAdaptationSetIntermediateRepresentation[], context: IAdaptationSetContext, -): IParsedAdaptations { +): { + adaptations: IParsedAdaptations; + thumbnailTracks: IParsedThumbnailTrack[]; +} { const parsedAdaptations: Record< ITrackType, Array<[IParsedAdaptation, IAdaptationSetOrderingData]> > = { video: [], audio: [], text: [] }; + const parsedThumbnailTracks: IParsedThumbnailTrack[] = []; const trickModeAdaptations: Array<{ adaptation: IParsedAdaptation; trickModeAttachedAdaptationIds: string[]; @@ -297,14 +308,7 @@ export default function parseAdaptationSets( (context.availabilityTimeOffset ?? 0); } - const adaptationMimeType = adaptation.attributes.mimeType; - const adaptationCodecs = adaptation.attributes.codecs; - const type = inferAdaptationType( - representationsIR, - isNonEmptyString(adaptationMimeType) ? adaptationMimeType : null, - isNonEmptyString(adaptationCodecs) ? adaptationCodecs : null, - !isNullOrUndefined(adaptationChildren.roles) ? adaptationChildren.roles : null, - ); + const type = inferAdaptationType(adaptation, representationsIR); if (type === undefined) { continue; } @@ -407,6 +411,15 @@ export default function parseAdaptationSets( context.unsafelyBaseOnPreviousPeriod?.getAdaptation(adaptationID) ?? null; const representations = parseRepresentations(representationsIR, adaptation, reprCtxt); + + if (type === "thumbnails") { + const track = createThumbnailTracks(adaptation, representations); + if (track !== null) { + parsedThumbnailTracks.push(...track); + } + continue; + } + const parsedAdaptationSet: IParsedAdaptation = { id: adaptationID, representations, @@ -505,7 +518,10 @@ export default function parseAdaptationSets( ); parsedAdaptations.video.sort(compareAdaptations); attachTrickModeTrack(adaptationsPerType, trickModeAdaptations); - return adaptationsPerType; + return { + adaptations: adaptationsPerType, + thumbnailTracks: parsedThumbnailTracks, + }; } /** Metadata allowing to order AdaptationSets between one another. */ @@ -524,6 +540,55 @@ interface IAdaptationSetOrderingData { indexInMpd: number; } +/** + * From the given attributes, returns a parsed thumbnail track, or null if it + * fails to do so. + * @param {Object} adaptation + * @param {Array.} representations + * @returns {Object|null} + */ +function createThumbnailTracks( + adaptation: IAdaptationSetIntermediateRepresentation, + representations: IParsedRepresentation[], +): IParsedThumbnailTrack[] { + const tracks = []; + for (let i = 0; i < representations.length; i++) { + const representation = representations[i]; + if (representation !== undefined) { + if (representation.mimeType === undefined) { + log.warn("DASH: Invalid thumbnails Representation, no mime-type"); + continue; + } + const tileInfo = getThumbnailAdaptationSetInfo( + adaptation, + adaptation.children.representations[i], + ); + if (tileInfo === null) { + continue; + } + if (representation.height === undefined) { + log.warn("DASH: Invalid thumbnails Representation, no height information"); + continue; + } + if (representation.width === undefined) { + log.warn("DASH: Invalid thumbnails Representation, no width information"); + continue; + } + tracks.push({ + id: representation.id, + cdnMetadata: representation.cdnMetadata, + index: representation.index, + mimeType: representation.mimeType, + height: representation.height, + width: representation.width, + horizontalTiles: tileInfo.horizontalTiles, + verticalTiles: tileInfo.verticalTiles, + }); + } + } + return tracks; +} + /** * Compare groups of parsed AdaptationSet, alongside some ordering metadata, * allowing to easily sort them through JavaScript's `Array.prototype.sort` diff --git a/src/parsers/manifest/dash/common/parse_periods.ts b/src/parsers/manifest/dash/common/parse_periods.ts index ffe26f737c..b5340a389b 100644 --- a/src/parsers/manifest/dash/common/parse_periods.ts +++ b/src/parsers/manifest/dash/common/parse_periods.ts @@ -126,7 +126,10 @@ export default function parsePeriods( start: periodStart, unsafelyBaseOnPreviousPeriod, }; - const adaptations = parseAdaptationSets(periodIR.children.adaptations, adapCtxt); + const { adaptations, thumbnailTracks } = parseAdaptationSets( + periodIR.children.adaptations, + adapCtxt, + ); const namespaces = (context.xmlNamespaces ?? []).concat( periodIR.attributes.namespaces ?? [], @@ -141,6 +144,7 @@ export default function parsePeriods( start: periodStart, end: periodEnd, duration: periodDuration, + thumbnailTracks, adaptations, streamEvents, }; diff --git a/src/parsers/manifest/dash/common/parse_representations.ts b/src/parsers/manifest/dash/common/parse_representations.ts index ce8faaceeb..9e138f0328 100644 --- a/src/parsers/manifest/dash/common/parse_representations.ts +++ b/src/parsers/manifest/dash/common/parse_representations.ts @@ -101,6 +101,7 @@ function getHDRInformation({ /** * Process intermediate representations to create final parsed representations. + * In the same order. * @param {Array.} representationsIR * @param {Object} context * @returns {Array.} diff --git a/src/parsers/manifest/dash/fast-js-parser/node_parsers/Representation.ts b/src/parsers/manifest/dash/fast-js-parser/node_parsers/Representation.ts index 6869a1f6fe..4f0a364ccf 100644 --- a/src/parsers/manifest/dash/fast-js-parser/node_parsers/Representation.ts +++ b/src/parsers/manifest/dash/fast-js-parser/node_parsers/Representation.ts @@ -103,6 +103,13 @@ function parseRepresentationChildren( } break; } + case "EssentialProperty": + if (isNullOrUndefined(children.essentialProperties)) { + children.essentialProperties = [parseScheme(currentElement)]; + } else { + children.essentialProperties.push(parseScheme(currentElement)); + } + break; case "SupplementalProperty": if (isNullOrUndefined(children.supplementalProperties)) { children.supplementalProperties = [parseScheme(currentElement)]; diff --git a/src/parsers/manifest/dash/native-parser/node_parsers/Representation.ts b/src/parsers/manifest/dash/native-parser/node_parsers/Representation.ts index 8a86659dd1..0fb76f5e78 100644 --- a/src/parsers/manifest/dash/native-parser/node_parsers/Representation.ts +++ b/src/parsers/manifest/dash/native-parser/node_parsers/Representation.ts @@ -100,6 +100,13 @@ function parseRepresentationChildren( } break; } + case "EssentialProperty": + if (isNullOrUndefined(children.essentialProperties)) { + children.essentialProperties = [parseScheme(currentElement)]; + } else { + children.essentialProperties.push(parseScheme(currentElement)); + } + break; case "SupplementalProperty": if (isNullOrUndefined(children.supplementalProperties)) { children.supplementalProperties = [parseScheme(currentElement)]; diff --git a/src/parsers/manifest/dash/node_parser_types.ts b/src/parsers/manifest/dash/node_parser_types.ts index 70083f2fff..80a65d62f8 100644 --- a/src/parsers/manifest/dash/node_parser_types.ts +++ b/src/parsers/manifest/dash/node_parser_types.ts @@ -261,6 +261,7 @@ export interface IRepresentationChildren { segmentList?: ISegmentListIntermediateRepresentation; segmentTemplate?: ISegmentTemplateIntermediateRepresentation; supplementalProperties?: IScheme[] | undefined; + essentialProperties?: IScheme[] | undefined; } /* Intermediate representation for A Representation node's attributes. */ diff --git a/src/parsers/manifest/dash/wasm-parser/ts/generators/Representation.ts b/src/parsers/manifest/dash/wasm-parser/ts/generators/Representation.ts index ff8c4ff117..411cd58cca 100644 --- a/src/parsers/manifest/dash/wasm-parser/ts/generators/Representation.ts +++ b/src/parsers/manifest/dash/wasm-parser/ts/generators/Representation.ts @@ -87,6 +87,17 @@ export function generateRepresentationChildrenParser( break; } + case TagName.EssentialProperty: { + const essentialProperty = {}; + if (childrenObj.essentialProperties === undefined) { + childrenObj.essentialProperties = []; + } + childrenObj.essentialProperties.push(essentialProperty); + const attributeParser = generateSchemeAttrParser(essentialProperty, linearMemory); + parsersStack.pushParsers(nodeId, noop, attributeParser); + break; + } + case TagName.SupplementalProperty: { const supplementalProperty = {}; if (childrenObj.supplementalProperties === undefined) { diff --git a/src/parsers/manifest/local/parse_local_manifest.ts b/src/parsers/manifest/local/parse_local_manifest.ts index 2ae6530f27..3e61a05b9f 100644 --- a/src/parsers/manifest/local/parse_local_manifest.ts +++ b/src/parsers/manifest/local/parse_local_manifest.ts @@ -93,6 +93,7 @@ function parsePeriod( start: period.start, end: period.end, duration: period.end - period.start, + thumbnailTracks: [], adaptations: period.adaptations.reduce>>( (acc, ada) => { const type = ada.type; diff --git a/src/parsers/manifest/metaplaylist/metaplaylist_parser.ts b/src/parsers/manifest/metaplaylist/metaplaylist_parser.ts index 582b3a277e..6f61531c2f 100644 --- a/src/parsers/manifest/metaplaylist/metaplaylist_parser.ts +++ b/src/parsers/manifest/metaplaylist/metaplaylist_parser.ts @@ -308,6 +308,7 @@ function createManifest( adaptations, duration: currentPeriod.duration, start: contentOffset + currentPeriod.start, + thumbnailTracks: currentPeriod.thumbnailTracks, }; manifestPeriods.push(newPeriod); } diff --git a/src/parsers/manifest/smooth/create_parser.ts b/src/parsers/manifest/smooth/create_parser.ts index 599e6143d7..1e8f81348b 100644 --- a/src/parsers/manifest/smooth/create_parser.ts +++ b/src/parsers/manifest/smooth/create_parser.ts @@ -681,6 +681,7 @@ function createSmoothStreamingParser( end: periodEnd, id: "gen-smooth-period-0", start: periodStart, + thumbnailTracks: [], }, ], suggestedPresentationDelay, diff --git a/src/parsers/manifest/types.ts b/src/parsers/manifest/types.ts index 629f9d4b20..b1171a932a 100644 --- a/src/parsers/manifest/types.ts +++ b/src/parsers/manifest/types.ts @@ -103,6 +103,52 @@ export interface ICdnMetadata { id?: string | undefined; } +/** Information linked to an image thumbnail track. */ +export interface IParsedThumbnailTrack { + /** Identifier for that thumbnail track. */ + id: string; + /** + * Information on the CDN(s) on which requests should be done to request + * thumbnails. + * + * `null` if there's no CDN involved here (e.g. resources are not + * requested through the network). + * + * An empty array means that no CDN are left to request the resource. As such, + * no resource can be loaded in that situation. + */ + cdnMetadata: ICdnMetadata[] | null; + /** Interface allowing to get timed thumbnail metadata to then be able to fetch them. */ + index: IRepresentationIndex; + /** + * Mimetype of the image thumbnails available here. + * Allows to know the image format (e.g. jpeg, png etc.) + */ + mimeType: string; + /** + * A loaded thumbnail's height in pixels. Note that there can be multiple actual + * thumbnails per loaded thumbnail resource (see `horizontalTiles` and + * `verticalTiles` properties. + */ + height: number; + /** + * A loaded thumbnail's width in pixels. Note that there can be multiple actual + * thumbnails per loaded thumbnail resource (see `horizontalTiles` and + * `verticalTiles` properties. + */ + width: number; + /** + * Thumbnail tracks are usually grouped together. This is the number of + * images contained horizontally in a whole loaded thumbnail resource. + */ + horizontalTiles: number; + /** + * Thumbnail tracks are usually grouped together. This is the number of + * images contained vertically in a whole loaded thumbnail resource. + */ + verticalTiles: number; +} + /** Representation of a "quality" available in an Adaptation. */ export interface IParsedRepresentation { /** Maximum bitrate the Representation is available in, in bits per seconds. */ @@ -269,6 +315,7 @@ export interface IParsedPeriod { * `undefined` if no parsed stream event in manifest. */ streamEvents?: IManifestStreamEvent[] | undefined; + thumbnailTracks: IParsedThumbnailTrack[]; } /** Information on the whole content */ diff --git a/src/public_types.ts b/src/public_types.ts index 8a7e72a9db..79b0ea33cf 100644 --- a/src/public_types.ts +++ b/src/public_types.ts @@ -1233,3 +1233,65 @@ export interface IModeInformation { isDirectFile: boolean; useWorker: boolean; } + +/** Information returned by the `getThumbnailMetadata` method. */ +export interface IThumbnailMetadata { + /** Identifier identifying a particular thumbnail track. */ + id: string; + /** + * Width in pixels of the individual thumbnails available in that + * thumbnail track. + */ + width: number | undefined; + /** + * Height in pixels of the individual thumbnails available in that + * thumbnail track. + */ + height: number | undefined; + /** + * Expected mime-type of the images in that thumbnail track (e.g. + * `image/jpeg` or `image/png`. + */ + mimeType: string | undefined; +} + +/** + * Options that can be provided to the `renderThumbnail` method + */ +export interface IThumbnailRenderingOptions { + /** + * HTMLElement inside which the thumbnail should be displayed. + * + * The resulting thumbnail will fill that container if the thumbnail loading + * and rendering operations succeeds. + * + * If there was already a thumbnail rendering request on that container, the + * previous operation is cancelled. + */ + container: HTMLElement; + /** Position, in seconds, for which you want to provide an image thumbnail. */ + time: number; + /** + * If set to `true`, we'll keep the potential previous thumbnail found inside + * the container if the current `renderThumbnail` call fail on an error. + * We'll still replace it if the new `renderThumbnail` call succeeds (with the + * new thumbnail). + * + * If set to `false`, to `undefined`, or not set, the previous thumbnail + * potentially found inside the container will also be removed if the new + * new `renderThumbnail` call fails. + * + * The default behavior (equivalent to `false`) is generally more expected, as + * you usually don't want to provide an unrelated preview thumbnail for a + * completely different time and prefer to display no thumbnail at all. + */ + keepPreviousThumbnailOnError?: boolean | undefined; + /** + * If set, specify from which thumbnail track you want to display the + * thumbnail from. That identifier can be obtained from the + * `getThumbnailMetadata` call (the `id` property). + * + * This is mainly useful when encountering multiple thumbnail track qualities. + */ + thumbnailTrackId?: string | undefined; +} diff --git a/src/transports/dash/pipelines.ts b/src/transports/dash/pipelines.ts index edf71825bb..0321c5d0ba 100644 --- a/src/transports/dash/pipelines.ts +++ b/src/transports/dash/pipelines.ts @@ -23,6 +23,7 @@ import generateSegmentLoader from "./segment_loader"; import generateAudioVideoSegmentParser from "./segment_parser"; import generateTextTrackLoader from "./text_loader"; import generateTextTrackParser from "./text_parser"; +import { loadThumbnail, parseThumbnail } from "./thumbnails"; /** * Returns pipelines used for DASH streaming. @@ -55,6 +56,10 @@ export default function (options: ITransportOptions): ITransportPipelines { parseSegment: audioVideoSegmentParser, }, text: { loadSegment: textTrackLoader, parseSegment: textTrackParser }, + thumbnails: { + loadThumbnail, + parseThumbnail, + }, }; } diff --git a/src/transports/dash/thumbnails.ts b/src/transports/dash/thumbnails.ts new file mode 100644 index 0000000000..60d508cfdf --- /dev/null +++ b/src/transports/dash/thumbnails.ts @@ -0,0 +1,96 @@ +import type { ISegment } from "../../manifest"; +import type { ICdnMetadata } from "../../parsers/manifest"; +import request from "../../utils/request/xhr"; +import type { CancellationSignal } from "../../utils/task_canceller"; +import type { + IRequestedData, + IThumbnailContext, + IThumbnailLoaderOptions, + IThumbnailResponse, +} from "../types"; +import addQueryString from "../utils/add_query_string"; +import byteRange from "../utils/byte_range"; +import constructSegmentUrl from "./construct_segment_url"; + +/** + * Load thumbnails for DASH content. + * @param {Object|null} wantedCdn + * @param {Object} thumbnail + * @param {Object} options + * @param {Object} cancelSignal + * @returns {Promise} + */ +export async function loadThumbnail( + wantedCdn: ICdnMetadata | null, + thumbnail: ISegment, + options: IThumbnailLoaderOptions, + cancelSignal: CancellationSignal, +): Promise> { + const initialUrl = constructSegmentUrl(wantedCdn, thumbnail); + if (initialUrl === null) { + return Promise.reject(new Error("Cannot load thumbnail: no URL")); + } + const url = + options.cmcdPayload?.type === "query" + ? addQueryString(initialUrl, options.cmcdPayload.value) + : initialUrl; + + const cmcdHeaders = + options.cmcdPayload?.type === "headers" ? options.cmcdPayload.value : undefined; + + let headers; + if (thumbnail.range !== undefined) { + headers = { + ...cmcdHeaders, + Range: byteRange(thumbnail.range), + }; + } else if (cmcdHeaders !== undefined) { + headers = cmcdHeaders; + } + return request({ + url, + responseType: "arraybuffer", + headers, + timeout: options.timeout, + connectionTimeout: options.connectionTimeout, + cancelSignal, + }); +} + +/** + * Parse loaded thumbnail data into exploitable thumbnail data and metadata. + * @param {ArrayBuffer} data - The loaded thumbnail data + * @param {Object} context + * @returns {Object} + */ +export function parseThumbnail( + data: ArrayBuffer, + context: IThumbnailContext, +): IThumbnailResponse { + const { thumbnailTrack, thumbnail: wantedThumbnail } = context; + const height = thumbnailTrack.height / thumbnailTrack.verticalTiles; + const width = thumbnailTrack.width / thumbnailTrack.horizontalTiles; + const thumbnails = []; + const tileDuration = + (wantedThumbnail.end - wantedThumbnail.time) / + (thumbnailTrack.horizontalTiles * thumbnailTrack.verticalTiles); + let start = wantedThumbnail.time; + for (let row = 0; row < thumbnailTrack.verticalTiles; row++) { + for (let column = 0; column < thumbnailTrack.horizontalTiles; column++) { + thumbnails.push({ + start, + end: start + tileDuration, + offsetX: Math.round(column * width), + offsetY: Math.round(row * height), + height: Math.floor(height), + width: Math.floor(width), + }); + start += tileDuration; + } + } + return { + mimeType: thumbnailTrack.mimeType, + data, + thumbnails, + }; +} diff --git a/src/transports/local/pipelines.ts b/src/transports/local/pipelines.ts index d24733ee31..4b3668e7c7 100644 --- a/src/transports/local/pipelines.ts +++ b/src/transports/local/pipelines.ts @@ -93,5 +93,14 @@ export default function getLocalManifestPipelines( audio: segmentPipeline, video: segmentPipeline, text: textTrackPipeline, + thumbnails: { + loadThumbnail: () => + Promise.reject( + new Error("Thumbnail tracks aren't implemented with the local transport"), + ), + parseThumbnail: () => { + throw new Error("Thumbnail tracks aren't implemented with the local transport"); + }, + }, }; } diff --git a/src/transports/metaplaylist/pipelines.ts b/src/transports/metaplaylist/pipelines.ts index 722b21bcd2..5e380c634f 100644 --- a/src/transports/metaplaylist/pipelines.ts +++ b/src/transports/metaplaylist/pipelines.ts @@ -398,5 +398,14 @@ export default function (options: ITransportOptions): ITransportPipelines { audio: audioPipeline, video: videoPipeline, text: textTrackPipeline, + thumbnails: { + loadThumbnail: () => + Promise.reject( + new Error("Thumbnail tracks aren't implemented with MetaPlaylist"), + ), + parseThumbnail: () => { + throw new Error("Thumbnail tracks aren't implemented with MetaPlaylist"); + }, + }, }; } diff --git a/src/transports/smooth/pipelines.ts b/src/transports/smooth/pipelines.ts index 639a5c9cbe..f1168c4680 100644 --- a/src/transports/smooth/pipelines.ts +++ b/src/transports/smooth/pipelines.ts @@ -428,5 +428,12 @@ export default function (transportOptions: ITransportOptions): ITransportPipelin audio: audioVideoPipeline, video: audioVideoPipeline, text: textTrackPipeline, + thumbnails: { + loadThumbnail: () => + Promise.reject(new Error("Thumbnail tracks aren't implemented with smooth")), + parseThumbnail: () => { + throw new Error("Thumbnail tracks aren't implemented with smooth"); + }, + }, }; } diff --git a/src/transports/types.ts b/src/transports/types.ts index 2b7a137ca3..206ae03f28 100644 --- a/src/transports/types.ts +++ b/src/transports/types.ts @@ -16,6 +16,7 @@ import type { IInbandEvent } from "../core/types"; import type { IManifest, ISegment } from "../manifest"; +import type { IThumbnailTrackMetadata } from "../manifest/types"; import type { ICdnMetadata } from "../parsers/manifest"; import type { ITrackType, @@ -62,6 +63,8 @@ export interface ITransportPipelines { >; /** Functions allowing to load an parse text (e.g. subtitles) segments. */ text: ISegmentPipeline; + /** Functions allowing to load image thumbnails. */ + thumbnails: IThumbnailPipeline; } /** Name describing the transport pipeline. */ @@ -309,6 +312,46 @@ export interface IManifestParserOptions { unsafeMode: boolean; } +/** "Pipeline" for image thumbnails. */ +export interface IThumbnailPipeline { + loadThumbnail: IThumbnailLoader; + parseThumbnail: IThumbnailParser; +} + +export type IThumbnailLoader = ( + wantedCdn: ICdnMetadata | null, + thumbnail: ISegment, + options: IThumbnailLoaderOptions, + cancelSignal: CancellationSignal, +) => Promise>; + +export type IThumbnailParser = ( + loadedThumbnail: ArrayBuffer, + context: IThumbnailContext, +) => IThumbnailResponse; + +export interface IThumbnailContext { + /** Metadata about the wanted thumbnail. */ + thumbnail: ISegment; + /** Metadata on the thumbnail track linked to that thumbnail. */ + thumbnailTrack: IThumbnailTrackMetadata; +} + +export interface IThumbnailResponse { + mimeType: string; + data: ArrayBuffer; + thumbnails: Array<{ + height: number; + width: number; + offsetX: number; + offsetY: number; + start: number; + end: number; + }>; +} + +export type IThumbnailLoaderOptions = ISegmentLoaderOptions; + export interface IManifestParserCallbacks { onWarning: (warning: Error) => void;