From 65d557edebd905df5086582226c1eb9a74a03142 Mon Sep 17 00:00:00 2001 From: Florent Bouisset <58945185+Florent-Bouisset@users.noreply.github.com> Date: Mon, 17 Jun 2024 18:28:05 +0200 Subject: [PATCH] feat:(worker) debug element display buffer in worker mode (#1438) * WIP * working POC * add a mechanism to only send data if a debug element is shown * fix a bug in worker mode, same representation was displayed with different colors * remove console.log * use uniqueId rather than serialize * change the behavior so it's main thread that ask for segmentSink infos * WIP * change architecture to be event based, and to not perform complex serialization * rename and fix bugs * clean * let segmentInventory be undefined if there is no segmentInventory for that media type * WIP with async function * clean * reorganiser code * update types and lint * fix a race condition * use correct type name convention * clean * delete __priv_getSegmentSinkContent from public api * factorize code * clean * fix clean up on new content * review * fix integration test for async * fmt * display buffer data in multi_thread mode * reject if cancelled * use correct cancel signal * handle case changing audio or video track --- .../scripts/components/BufferContentGraph.tsx | 29 ++++----- .../controllers/charts/BufferContent.tsx | 9 --- demo/full/scripts/modules/player/events.ts | 39 ++++++------ src/core/main/worker/worker_main.ts | 27 +++++++++ src/core/segment_sinks/inventory/types.ts | 16 ++++- .../segment_sinks/segment_buffers_store.ts | 59 +++++++++++++++++++ src/main_thread/api/debug/buffer_graph.ts | 35 +++++++---- .../debug/modules/segment_buffer_content.ts | 28 ++++++++- src/main_thread/api/public_api.ts | 48 ++++++++------- .../init/directfile_content_initializer.ts | 4 +- .../init/media_source_content_initializer.ts | 9 ++- .../init/multi_thread_content_initializer.ts | 48 ++++++++++++++- src/main_thread/init/types.ts | 12 ++-- src/multithread_types.ts | 23 +++++++- .../scenarios/fast-switching.test.js | 37 ++++++------ tests/utils/checkAfterSleepWithBackoff.js | 5 +- 16 files changed, 314 insertions(+), 114 deletions(-) diff --git a/demo/full/scripts/components/BufferContentGraph.tsx b/demo/full/scripts/components/BufferContentGraph.tsx index 5aef884931..8fe81cedac 100644 --- a/demo/full/scripts/components/BufferContentGraph.tsx +++ b/demo/full/scripts/components/BufferContentGraph.tsx @@ -1,14 +1,9 @@ import * as React from "react"; -import type { - IAudioRepresentation, - IVideoRepresentation, -} from "../../../../src/public_types"; +import type { IVideoRepresentation } from "../../../../src/public_types"; import capitalizeFirstLetter from "../lib/capitalizeFirstLetter"; import shuffleArray from "../lib/shuffleArray"; -import type { IBufferedData } from "../modules/player/index"; import ToolTip from "./ToolTip"; - -type IRepresentation = IAudioRepresentation | IVideoRepresentation; +import { IBufferedChunkSnapshot } from "../../../../src/core/segment_sinks/segment_buffers_store"; const { useEffect, useMemo, useRef, useState } = React; @@ -73,7 +68,7 @@ function paintCurrentPosition( interface IScaledBufferedData { scaledStart: number; scaledEnd: number; - bufferedInfos: IBufferedData; + bufferedInfos: IBufferedChunkSnapshot; } /** @@ -85,7 +80,7 @@ interface IScaledBufferedData { * @returns {Array.} */ function scaleSegments( - bufferedData: IBufferedData[], + bufferedData: IBufferedChunkSnapshot[], minimumPosition: number, maximumPosition: number, ): IScaledBufferedData[] { @@ -128,7 +123,7 @@ export default function BufferContentGraph({ type, // The type of buffer (e.g. "audio", "video" or "text") }: { currentTime: number | undefined; - data: IBufferedData[]; + data: IBufferedChunkSnapshot[]; minimumPosition: number | null | undefined; maximumPosition: number | null | undefined; seek: (pos: number) => void; @@ -139,7 +134,7 @@ export default function BufferContentGraph({ const [tipPosition, setTipPosition] = useState(0); const [tipText, setTipText] = useState(""); const canvasEl = useRef(null); - const representationsEncountered = useRef([]); + const representationsIdEncountered = useRef([]); const usedMaximum = maximumPosition ?? 300; const usedMinimum = minimumPosition ?? 0; const duration = Math.max(usedMaximum - usedMinimum, 0); @@ -153,10 +148,12 @@ export default function BufferContentGraph({ const paintSegment = React.useCallback( (scaledSegment: IScaledBufferedData, canvasCtx: CanvasRenderingContext2D): void => { const representation = scaledSegment.bufferedInfos.infos.representation; - let indexOfRepr = representationsEncountered.current.indexOf(representation); + let indexOfRepr = representationsIdEncountered.current.indexOf( + representation.uniqueId, + ); if (indexOfRepr < 0) { - representationsEncountered.current.push(representation); - indexOfRepr = representationsEncountered.current.length - 1; + representationsIdEncountered.current.push(representation.uniqueId); + indexOfRepr = representationsIdEncountered.current.length - 1; } const colorIndex = indexOfRepr % COLORS.length; const color = randomColors[colorIndex]; @@ -258,7 +255,7 @@ export default function BufferContentGraph({ "\n" + `height: ${rep.height ?? "?"}` + "\n" + - `codec: ${representation.codec ?? "?"}` + + `codec: ${representation.codecs ?? "?"}` + "\n" + `bitrate: ${representation.bitrate ?? "?"}` + "\n"; @@ -270,7 +267,7 @@ export default function BufferContentGraph({ "\n" + `audioDescription: ${String(adaptation.isAudioDescription) ?? false}` + "\n" + - `codec: ${representation.codec ?? "?"}` + + `codec: ${representation.codecs ?? "?"}` + "\n" + `bitrate: ${representation.bitrate ?? "?"}` + "\n"; diff --git a/demo/full/scripts/controllers/charts/BufferContent.tsx b/demo/full/scripts/controllers/charts/BufferContent.tsx index b32136d82a..0ef2ec4351 100644 --- a/demo/full/scripts/controllers/charts/BufferContent.tsx +++ b/demo/full/scripts/controllers/charts/BufferContent.tsx @@ -8,7 +8,6 @@ export default function BufferContentChart({ player }: { player: IPlayerModule } const currentTime = useModuleState(player, "currentTime"); const maximumPosition = useModuleState(player, "maximumPosition"); const minimumPosition = useModuleState(player, "minimumPosition"); - const useWorker = useModuleState(player, "useWorker"); const seek = React.useCallback( (position: number): void => { @@ -17,14 +16,6 @@ export default function BufferContentChart({ player }: { player: IPlayerModule } [player], ); - if (useWorker) { - return ( -
- Unavailable information currently when running in "multithread" mode (in a - WebWorker). -
- ); - } if (bufferedData === null || Object.keys(bufferedData).length === 0) { return
No content yet
; } diff --git a/demo/full/scripts/modules/player/events.ts b/demo/full/scripts/modules/player/events.ts index 313e6980ad..91b7a65559 100644 --- a/demo/full/scripts/modules/player/events.ts +++ b/demo/full/scripts/modules/player/events.ts @@ -90,27 +90,32 @@ function linkPlayerEventsToState( player.removeEventListener("playerStateChange", onStateUpdate); }); - function updateBufferedData(): void { + async function updateBufferedData(): Promise { if (player.getPlayerState() === "STOPPED") { return; } - let audioContent = player.__priv_getSegmentSinkContent("audio"); - if (Array.isArray(audioContent)) { - audioContent = audioContent.slice(); - } - let textContent = player.__priv_getSegmentSinkContent("text"); - if (Array.isArray(textContent)) { - textContent = textContent.slice(); - } - let videoContent = player.__priv_getSegmentSinkContent("video"); - if (Array.isArray(videoContent)) { - videoContent = videoContent.slice(); + try { + const metrics = await player.__priv_getSegmentSinkMetrics(); + let audioContent = metrics?.segmentSinks.audio.segmentInventory ?? null; + if (Array.isArray(audioContent)) { + audioContent = audioContent.slice(); + } + let textContent = metrics?.segmentSinks.text.segmentInventory ?? null; + if (Array.isArray(textContent)) { + textContent = textContent.slice(); + } + let videoContent = metrics?.segmentSinks.video.segmentInventory ?? null; + if (Array.isArray(videoContent)) { + videoContent = videoContent.slice(); + } + state.update("bufferedData", { + audio: audioContent, + video: videoContent, + text: textContent, + }); + } catch (err) { + // Do nothing } - state.update("bufferedData", { - audio: audioContent, - video: videoContent, - text: textContent, - }); } const bufferedDataItv = setInterval(updateBufferedData, BUFFERED_DATA_UPDATES_INTERVAL); diff --git a/src/core/main/worker/worker_main.ts b/src/core/main/worker/worker_main.ts index 35cbec9197..b902a1a749 100644 --- a/src/core/main/worker/worker_main.ts +++ b/src/core/main/worker/worker_main.ts @@ -398,6 +398,11 @@ export default function initializeWorkerMain() { break; } + case MainThreadMessageType.PullSegmentSinkStoreInfos: { + sendSegmentSinksStoreInfos(contentPreparer, msg.value.messageId); + break; + } + default: assertUnreachable(msg); } @@ -900,3 +905,25 @@ function updateLoggerLevel(logLevel: ILoggerLevel, sendBackLogs: boolean): void }); } } + +/** + * Send a message `SegmentSinkStoreUpdate` to the main thread with + * a serialized object that represents the segmentSinksStore state. + * @param {ContentPreparer} contentPreparer + * @returns {void} + */ +function sendSegmentSinksStoreInfos( + contentPreparer: ContentPreparer, + messageId: number, +): void { + const currentContent = contentPreparer.getCurrentContent(); + if (currentContent === null) { + return; + } + const segmentSinksMetrics = currentContent.segmentSinksStore.getSegmentSinksMetrics(); + sendMessage({ + type: WorkerMessageType.SegmentSinkStoreUpdate, + contentId: currentContent.contentId, + value: { segmentSinkMetrics: segmentSinksMetrics, messageId }, + }); +} diff --git a/src/core/segment_sinks/inventory/types.ts b/src/core/segment_sinks/inventory/types.ts index 0b4d5b1270..ae1625c3d3 100644 --- a/src/core/segment_sinks/inventory/types.ts +++ b/src/core/segment_sinks/inventory/types.ts @@ -14,7 +14,15 @@ * limitations under the License. */ -import type { IAdaptation, ISegment, IPeriod, IRepresentation } from "../../../manifest"; +import type { + IAdaptation, + ISegment, + IPeriod, + IRepresentation, + IAdaptationMetadata, + IPeriodMetadata, + IRepresentationMetadata, +} from "../../../manifest"; /** Content information for a single buffered chunk */ export interface IChunkContext { @@ -27,3 +35,9 @@ export interface IChunkContext { /** Segment this chunk is related to. */ segment: ISegment; } + +export interface IChunkContextSnapshot { + adaptation: IAdaptationMetadata; + period: IPeriodMetadata; + representation: IRepresentationMetadata; +} diff --git a/src/core/segment_sinks/segment_buffers_store.ts b/src/core/segment_sinks/segment_buffers_store.ts index 96dc0669e3..9602ec5bc0 100644 --- a/src/core/segment_sinks/segment_buffers_store.ts +++ b/src/core/segment_sinks/segment_buffers_store.ts @@ -25,12 +25,38 @@ import type { IBufferType, SegmentSink } from "./implementations"; import { AudioVideoSegmentSink } from "./implementations"; import type { ITextDisplayerInterface } from "./implementations/text"; import TextSegmentSink from "./implementations/text"; +import type { IBufferedChunk } from "./inventory/segment_inventory"; +import type { IChunkContext, IChunkContextSnapshot } from "./inventory/types"; const POSSIBLE_BUFFER_TYPES: IBufferType[] = ["audio", "video", "text"]; /** Types of "native" media buffers (i.e. which rely on a SourceBuffer) */ type INativeMediaBufferType = "audio" | "video"; +/** + * Interface containing metadata of a buffered chunk. + * The metadata is serializable and does not contain references to JS objects + * that are not serializable, such as Map or class instances. + */ +export interface IBufferedChunkSnapshot extends Omit { + infos: IChunkContextSnapshot; +} + +/** + * Interface representing metrics for segment sinks. + * The metrics include information on the buffer type, codec, and segment inventory, + * and are categorized by segment type (audio, video, text). + */ +export interface ISegmentSinkMetrics { + segmentSinks: Record<"audio" | "video" | "text", ISegmentSinkMetricForType>; +} + +interface ISegmentSinkMetricForType { + bufferType: IBufferType; + codec: string | undefined; + segmentInventory: IBufferedChunkSnapshot[] | undefined; +} + /** * Allows to easily create and dispose SegmentSinks, which are interfaces to * push and remove segments. @@ -347,6 +373,31 @@ export default class SegmentSinksStore { } return true; } + + private createSegmentSinkMetricsForType( + bufferType: IBufferType, + ): ISegmentSinkMetricForType { + return { + bufferType, + codec: this._initializedSegmentSinks[bufferType]?.codec, + segmentInventory: this._initializedSegmentSinks[bufferType] + ?.getLastKnownInventory() + .map((chunk) => ({ + ...chunk, + infos: getChunkContextSnapshot(chunk.infos), + })), + }; + } + + public getSegmentSinksMetrics(): ISegmentSinkMetrics { + return { + segmentSinks: { + audio: this.createSegmentSinkMetricsForType("audio"), + video: this.createSegmentSinkMetricsForType("video"), + text: this.createSegmentSinkMetricsForType("text"), + }, + }; + } } /** @@ -361,3 +412,11 @@ function shouldHaveNativeBuffer( ): bufferType is INativeMediaBufferType { return bufferType === "audio" || bufferType === "video"; } + +function getChunkContextSnapshot(context: IChunkContext): IChunkContextSnapshot { + return { + adaptation: context.adaptation.getMetadataSnapshot(), + period: context.period.getMetadataSnapshot(), + representation: context.representation.getMetadataSnapshot(), + }; +} diff --git a/src/main_thread/api/debug/buffer_graph.ts b/src/main_thread/api/debug/buffer_graph.ts index c6c5f8b405..13428f51f4 100644 --- a/src/main_thread/api/debug/buffer_graph.ts +++ b/src/main_thread/api/debug/buffer_graph.ts @@ -1,5 +1,5 @@ -import type { IBufferedChunk } from "../../../core/types"; -import type { IRepresentation } from "../../../manifest"; +import type { IBufferedChunkSnapshot } from "../../../core/segment_sinks/segment_buffers_store"; +import type { IRepresentationMetadata } from "../../../manifest"; const BUFFER_WIDTH_IN_SECONDS = 30 * 60; @@ -19,7 +19,7 @@ const COLORS = [ export interface ISegmentSinkGrapUpdateData { currentTime: number; - inventory: IBufferedChunk[]; + inventory: IBufferedChunkSnapshot[]; width: number; height: number; minimumPosition: number | undefined; @@ -27,8 +27,8 @@ export interface ISegmentSinkGrapUpdateData { } export default class SegmentSinkGraph { - /** Link buffered Representation to their corresponding color. */ - private readonly _colorMap: WeakMap; + /** Link buffered Representation's uniqueId to their corresponding color. */ + private readonly _colorMap: Map; /** Current amount of colors chosen to represent the various Representation. */ private _currNbColors: number; @@ -39,7 +39,7 @@ export default class SegmentSinkGraph { private readonly _canvasCtxt: CanvasRenderingContext2D | null; constructor(canvasElt: HTMLCanvasElement) { - this._colorMap = new WeakMap(); + this._colorMap = new Map(); this._currNbColors = 0; this._canvasElt = canvasElt; this._canvasCtxt = this._canvasElt.getContext("2d"); @@ -53,6 +53,19 @@ export default class SegmentSinkGraph { } public update(data: ISegmentSinkGrapUpdateData): void { + // Following logic clear the colorMap entries if they are not used anymore + // to prevent memory usage. + const representationStillInUse: Set = new Set(); + data.inventory.forEach((chunk) => { + representationStillInUse.add(chunk.infos.representation.uniqueId); + }); + + this._colorMap.forEach((representationId) => { + if (!representationStillInUse.has(representationId)) { + this._colorMap.delete(representationId); + } + }); + if (this._canvasCtxt === null) { return; } @@ -149,14 +162,14 @@ export default class SegmentSinkGraph { this._canvasCtxt.fillRect(Math.ceil(startX), 0, Math.ceil(endX - startX), height); } - private _getColorForRepresentation(representation: IRepresentation): string { - const color = this._colorMap.get(representation); + private _getColorForRepresentation(representation: IRepresentationMetadata): string { + const color = this._colorMap.get(representation.uniqueId); if (color !== undefined) { return color; } const newColor = COLORS[this._currNbColors % COLORS.length]; this._currNbColors++; - this._colorMap.set(representation, newColor); + this._colorMap.set(representation.uniqueId, newColor); return newColor; } } @@ -203,7 +216,7 @@ function paintCurrentPosition( * @returns {Array.} */ function scaleSegments( - bufferedData: IBufferedChunk[], + bufferedData: IBufferedChunkSnapshot[], minimumPosition: number, maximumPosition: number, ): IScaledChunk[] { @@ -227,5 +240,5 @@ function scaleSegments( interface IScaledChunk { scaledStart: number; scaledEnd: number; - info: IBufferedChunk; + info: IBufferedChunkSnapshot; } diff --git a/src/main_thread/api/debug/modules/segment_buffer_content.ts b/src/main_thread/api/debug/modules/segment_buffer_content.ts index ceeb816e1e..2bc6a72ad7 100644 --- a/src/main_thread/api/debug/modules/segment_buffer_content.ts +++ b/src/main_thread/api/debug/modules/segment_buffer_content.ts @@ -1,3 +1,4 @@ +import type { ISegmentSinkMetrics } from "../../../../core/segment_sinks/segment_buffers_store"; import type { IBufferType } from "../../../../core/types"; import type { IAdaptationMetadata, @@ -34,6 +35,17 @@ export default function createSegmentSinkGraph( cancelSignal.register(() => { clearInterval(intervalId); }); + + let bufferMetrics: ISegmentSinkMetrics | null = null; + instance + .__priv_getSegmentSinkMetrics() + .then((metrics) => { + bufferMetrics = metrics ?? null; + }) + .catch(() => { + // Do nothing + }); + bufferGraphWrapper.appendChild(bufferTitle); bufferGraphWrapper.appendChild(canvasElt); bufferGraphWrapper.appendChild(currentRangeRepInfoElt); @@ -50,9 +62,21 @@ export default function createSegmentSinkGraph( clearInterval(intervalId); return; } + instance + .__priv_getSegmentSinkMetrics() + .then((metrics) => { + bufferMetrics = metrics ?? null; + updateBufferMetrics(); + }) + .catch(() => { + // DO nothing + }); + } + + function updateBufferMetrics() { const showAllInfo = isExtendedMode(parentElt); - const inventory = instance.__priv_getSegmentSinkContent(bufferType); - if (inventory === null) { + const inventory = bufferMetrics?.segmentSinks[bufferType].segmentInventory; + if (bufferMetrics === null || inventory === undefined) { bufferGraphWrapper.style.display = "none"; currentRangeRepInfoElt.innerHTML = ""; loadingRangeRepInfoElt.innerHTML = ""; diff --git a/src/main_thread/api/public_api.ts b/src/main_thread/api/public_api.ts index 5f9ec0050a..11ff24e71a 100644 --- a/src/main_thread/api/public_api.ts +++ b/src/main_thread/api/public_api.ts @@ -31,12 +31,11 @@ import getStartDate from "../../compat/get_start_date"; import hasMseInWorker from "../../compat/has_mse_in_worker"; import hasWorkerApi from "../../compat/has_worker_api"; import isDebugModeEnabled from "../../compat/is_debug_mode_enabled"; +import type { ISegmentSinkMetrics } from "../../core/segment_sinks/segment_buffers_store"; import type { IAdaptationChoice, IInbandEvent, - ISegmentSinksStore, IABRThrottlers, - IBufferedChunk, IBufferType, } from "../../core/types"; import type { IErrorCode, IErrorType } from "../../errors"; @@ -367,6 +366,14 @@ class Player extends EventEmitter { } } + /** + * Function passed from the ContentInitializer that return segment sinks metrics. + * This is used for monitor and debugging. + */ + private _priv_segmentSinkMetricsCallback: + | null + | (() => Promise); + /** * @constructor * @param {Object} options @@ -443,6 +450,8 @@ class Player extends EventEmitter { this._priv_worker = null; + this._priv_segmentSinkMetricsCallback = null; + const onVolumeChange = () => { this.trigger("volumeChange", { volume: videoElement.volume, @@ -972,7 +981,6 @@ class Player extends EventEmitter { defaultAudioTrackSwitchingMode, initializer, isDirectFile, - segmentSinksStore: null, manifest: null, currentPeriod: null, activeAdaptations: null, @@ -995,10 +1003,10 @@ class Player extends EventEmitter { this.trigger("warning", formattedError); }); initializer.addEventListener("reloadingMediaSource", (payload) => { - contentInfos.segmentSinksStore = null; if (contentInfos.tracksStore !== null) { contentInfos.tracksStore.resetPeriodObjects(); } + this._priv_segmentSinkMetricsCallback = null; this._priv_lastAutoPlay = payload.autoPlay; }); initializer.addEventListener("inbandEvents", (inbandEvents) => @@ -1038,7 +1046,7 @@ class Player extends EventEmitter { this._priv_onDecipherabilityUpdate(contentInfos, updates), ); initializer.addEventListener("loaded", (evt) => { - contentInfos.segmentSinksStore = evt.segmentSinksStore; + this._priv_segmentSinkMetricsCallback = evt.getSegmentSinkMetrics; }); // Now, that most events are linked, prepare the next content. @@ -2321,26 +2329,17 @@ class Player extends EventEmitter { // They should not be used by any external code. /** - * /!\ For demo use only! Do not touch! - * - * Returns every chunk buffered for a given buffer type. - * Returns `null` if no SegmentSink was created for this type of buffer. - * @param {string} bufferType - * @returns {Array.|null} + * Used for the display of segmentSink metrics for the debug element + * @param fn + * @param cancellationSignal + * @returns */ - __priv_getSegmentSinkContent(bufferType: IBufferType): IBufferedChunk[] | null { - if ( - this._priv_contentInfos === null || - this._priv_contentInfos.segmentSinksStore === null - ) { - return null; - } - const segmentSinkStatus = - this._priv_contentInfos.segmentSinksStore.getStatus(bufferType); - if (segmentSinkStatus.type === "initialized") { - return segmentSinkStatus.value.getLastKnownInventory(); + async __priv_getSegmentSinkMetrics(): Promise { + if (this._priv_segmentSinkMetricsCallback === null) { + return undefined; + } else { + return this._priv_segmentSinkMetricsCallback(); } - return null; } /** @@ -2408,6 +2407,7 @@ class Player extends EventEmitter { this._priv_contentInfos?.tracksStore?.dispose(); this._priv_contentInfos?.mediaElementTracksStore?.dispose(); this._priv_contentInfos = null; + this._priv_segmentSinkMetricsCallback = null; this._priv_contentEventsMemory = {}; @@ -3273,8 +3273,6 @@ interface IPublicApiContentInfos { activeRepresentations: { [periodId: string]: Partial>; } | null; - /** Keep information on the active SegmentSinks. */ - segmentSinksStore: ISegmentSinksStore | null; /** * TracksStore instance linked to the current content. * `null` if no content has been loaded or if the current content loaded diff --git a/src/main_thread/init/directfile_content_initializer.ts b/src/main_thread/init/directfile_content_initializer.ts index 2806168107..7975daad91 100644 --- a/src/main_thread/init/directfile_content_initializer.ts +++ b/src/main_thread/init/directfile_content_initializer.ts @@ -237,7 +237,9 @@ export default class DirectFileContentInitializer extends ContentInitializer { (isLoaded, stopListening) => { if (isLoaded) { stopListening(); - this.trigger("loaded", { segmentSinksStore: null }); + this.trigger("loaded", { + getSegmentSinkMetrics: null, + }); } }, { emitCurrentValue: true, clearSignal: cancelSignal }, diff --git a/src/main_thread/init/media_source_content_initializer.ts b/src/main_thread/init/media_source_content_initializer.ts index c9aadddf90..4693a479fd 100644 --- a/src/main_thread/init/media_source_content_initializer.ts +++ b/src/main_thread/init/media_source_content_initializer.ts @@ -513,6 +513,7 @@ export default class MediaSourceContentInitializer extends ContentInitializer { mediaElement.nodeName === "VIDEO", textDisplayerInterface, ); + cancelSignal.register(() => { segmentSinksStore.disposeAll(); }); @@ -669,7 +670,13 @@ export default class MediaSourceContentInitializer extends ContentInitializer { (isLoaded, stopListening) => { if (isLoaded) { stopListening(); - this.trigger("loaded", { segmentSinksStore }); + this.trigger("loaded", { + getSegmentSinkMetrics: async () => { + return new Promise((resolve) => + resolve(segmentSinksStore.getSegmentSinksMetrics()), + ); + }, + }); } }, { emitCurrentValue: true, clearSignal: cancelSignal }, diff --git a/src/main_thread/init/multi_thread_content_initializer.ts b/src/main_thread/init/multi_thread_content_initializer.ts index d9e1f9a608..4f0d63758d 100644 --- a/src/main_thread/init/multi_thread_content_initializer.ts +++ b/src/main_thread/init/multi_thread_content_initializer.ts @@ -1,6 +1,7 @@ import isCodecSupported from "../../compat/is_codec_supported"; import mayMediaElementFailOnUndecipherableData from "../../compat/may_media_element_fail_on_undecipherable_data"; import shouldReloadMediaSourceOnDecipherabilityUpdate from "../../compat/should_reload_media_source_on_decipherability_update"; +import type { ISegmentSinkMetrics } from "../../core/segment_sinks/segment_buffers_store"; import type { IAdaptiveRepresentationSelectorArguments, IAdaptationChoice, @@ -91,6 +92,14 @@ export default class MultiThreadContentInitializer extends ContentInitializer { */ private _currentMediaSourceCanceller: TaskCanceller; + /** + * Stores the resolvers and the current messageId that is sent to the web worker to receive segment sink metrics. + * The purpose of collecting metrics is for monitoring and debugging. + */ + private _segmentMetrics: { + lastMessageId: number; + resolvers: Record void>; + }; /** * Create a new `MultiThreadContentInitializer`, associated to the given * settings. @@ -103,6 +112,10 @@ export default class MultiThreadContentInitializer extends ContentInitializer { this._currentMediaSourceCanceller = new TaskCanceller(); this._currentMediaSourceCanceller.linkToSignal(this._initCanceller.signal); this._currentContentInfo = null; + this._segmentMetrics = { + lastMessageId: 0, + resolvers: {}, + }; } /** @@ -1071,6 +1084,19 @@ export default class MultiThreadContentInitializer extends ContentInitializer { // Should already be handled by the API break; + case WorkerMessageType.SegmentSinkStoreUpdate: { + if (this._currentContentInfo?.contentId !== msgData.contentId) { + return; + } + const resolveFn = this._segmentMetrics.resolvers[msgData.value.messageId]; + if (resolveFn !== undefined) { + resolveFn(msgData.value.segmentSinkMetrics); + delete this._segmentMetrics.resolvers[msgData.value.messageId]; + } else { + log.error("MTCI: Failed to send segment sink store update"); + } + break; + } default: assertUnreachable(msgData); } @@ -1454,6 +1480,24 @@ export default class MultiThreadContentInitializer extends ContentInitializer { { clearSignal: cancelSignal, emitCurrentValue: true }, ); + const _getSegmentSinkMetrics: () => Promise< + ISegmentSinkMetrics | undefined + > = async () => { + this._segmentMetrics.lastMessageId++; + const messageId = this._segmentMetrics.lastMessageId; + sendMessage(this._settings.worker, { + type: MainThreadMessageType.PullSegmentSinkStoreInfos, + value: { messageId }, + }); + return new Promise((resolve, reject) => { + this._segmentMetrics.resolvers[messageId] = resolve; + const rejectFn = (err: CancellationError) => { + delete this._segmentMetrics.resolvers[messageId]; + return reject(err); + }; + cancelSignal.register(rejectFn); + }); + }; /** * Emit a "loaded" events once the initial play has been performed and the * media can begin playback. @@ -1465,7 +1509,9 @@ export default class MultiThreadContentInitializer extends ContentInitializer { (isLoaded, stopListening) => { if (isLoaded) { stopListening(); - this.trigger("loaded", { segmentSinksStore: null }); + this.trigger("loaded", { + getSegmentSinkMetrics: _getSegmentSinkMetrics, + }); } }, { emitCurrentValue: true, clearSignal: cancelSignal }, diff --git a/src/main_thread/init/types.ts b/src/main_thread/init/types.ts index 595fde07a7..6b1d1f0567 100644 --- a/src/main_thread/init/types.ts +++ b/src/main_thread/init/types.ts @@ -14,12 +14,8 @@ * limitations under the License. */ -import type { - ISegmentSinksStore, - IBufferType, - IAdaptationChoice, - IInbandEvent, -} from "../../core/types"; +import type { ISegmentSinkMetrics } from "../../core/segment_sinks/segment_buffers_store"; +import type { IBufferType, IAdaptationChoice, IInbandEvent } from "../../core/types"; import type { IPeriodsUpdateResult, IAdaptationMetadata, @@ -143,7 +139,9 @@ export interface IContentInitializerEvents { * Event sent just as the content is considered as "loaded". * From this point on, the user can reliably play/pause/resume the stream. */ - loaded: { segmentSinksStore: ISegmentSinksStore | null }; + loaded: { + getSegmentSinkMetrics: null | (() => Promise); + }; /** Event emitted when a stream event is encountered. */ streamEvent: IPublicStreamEvent | IPublicNonFiniteStreamEvent; streamEventSkip: IPublicStreamEvent | IPublicNonFiniteStreamEvent; diff --git a/src/multithread_types.ts b/src/multithread_types.ts index a94d1ef32d..1938065a79 100644 --- a/src/multithread_types.ts +++ b/src/multithread_types.ts @@ -4,6 +4,7 @@ * multithread situation. */ +import type { ISegmentSinkMetrics } from "./core/segment_sinks/segment_buffers_store"; import type { IResolutionInfo, IManifestFetcherSettings, @@ -515,6 +516,11 @@ export type IReferenceUpdateMessage = | IReferenceUpdate<"limitVideoResolution", IResolutionInfo> | IReferenceUpdate<"throttleVideoBitrate", number>; +export interface IPullSegmentSinkStoreInfos { + type: MainThreadMessageType.PullSegmentSinkStoreInfos; + value: { messageId: number }; +} + export const enum MainThreadMessageType { Init = "init", PushTextDataSuccess = "add-text-success", @@ -535,6 +541,7 @@ export const enum MainThreadMessageType { StartPreparedContent = "start", StopContent = "stop", TrackUpdate = "track-update", + PullSegmentSinkStoreInfos = "pull-segment-sink-store-infos", } export type IMainThreadMessage = @@ -556,7 +563,8 @@ export type IMainThreadMessage = | IRemoveTextDataSuccessMessage | IPushTextDataErrorMessage | IRemoveTextDataErrorMessage - | IMediaSourceReadyStateChangeMainMessage; + | IMediaSourceReadyStateChangeMainMessage + | IPullSegmentSinkStoreInfos; export type ISentError = | ISerializedNetworkError @@ -902,6 +910,15 @@ export interface IDiscontinuityTimeInfo { end: number | null; } +export interface ISegmentSinkStoreUpdateMessage { + type: WorkerMessageType.SegmentSinkStoreUpdate; + contentId: string; + value: { + segmentSinkMetrics: ISegmentSinkMetrics; + messageId: number; + }; +} + export const enum WorkerMessageType { AbortSourceBuffer = "abort-source-buffer", ActivePeriodChanged = "active-period-changed", @@ -939,6 +956,7 @@ export const enum WorkerMessageType { UpdateMediaSourceDuration = "update-media-source-duration", UpdatePlaybackRate = "update-playback-rate", Warning = "warning", + SegmentSinkStoreUpdate = "segment-sink-store-update", } export type IWorkerMessage = @@ -977,4 +995,5 @@ export type IWorkerMessage = | IStopTextDisplayerWorkerMessage | IUpdateMediaSourceDurationWorkerMessage | IUpdatePlaybackRateWorkerMessage - | IWarningWorkerMessage; + | IWarningWorkerMessage + | ISegmentSinkStoreUpdateMessage; diff --git a/tests/integration/scenarios/fast-switching.test.js b/tests/integration/scenarios/fast-switching.test.js index 0468ed1caa..d563d551f4 100644 --- a/tests/integration/scenarios/fast-switching.test.js +++ b/tests/integration/scenarios/fast-switching.test.js @@ -30,16 +30,15 @@ describe("Fast-switching", function () { player.setWantedBufferAhead(30); lockHighestBitrates(player, "lazy"); - await checkAfterSleepWithBackoff({}, () => { - const videoSegmentBuffered = player - .__priv_getSegmentSinkContent("video") - .map(({ infos }) => { + await checkAfterSleepWithBackoff({}, async () => { + const metrics = await player.__priv_getSegmentSinkMetrics(); + const videoSegmentBuffered = metrics.segmentSinks.video.segmentInventory.map( + ({ infos }) => { return { bitrate: infos.representation.bitrate, - time: infos.segment.time, - end: infos.segment.end, }; - }); + }, + ); expect(videoSegmentBuffered.length).to.be.at.least(3); expect(videoSegmentBuffered[1].bitrate).to.equal(1996000); expect(videoSegmentBuffered[2].bitrate).to.equal(1996000); @@ -62,16 +61,15 @@ describe("Fast-switching", function () { player.setWantedBufferAhead(30); lockHighestBitrates(player, "lazy"); - await checkAfterSleepWithBackoff({}, () => { - const videoSegmentBuffered = player - .__priv_getSegmentSinkContent("video") - .map(({ infos }) => { + await checkAfterSleepWithBackoff({}, async () => { + const metrics = await player.__priv_getSegmentSinkMetrics(); + const videoSegmentBuffered = metrics.segmentSinks.video.segmentInventory.map( + ({ infos }) => { return { bitrate: infos.representation.bitrate, - time: infos.segment.time, - end: infos.segment.end, }; - }); + }, + ); expect(videoSegmentBuffered.length).to.be.at.least(3); expect(videoSegmentBuffered[1].bitrate).to.equal(1996000); expect(videoSegmentBuffered[2].bitrate).to.equal(1996000); @@ -94,15 +92,14 @@ describe("Fast-switching", function () { player.setWantedBufferAhead(30); lockHighestBitrates(player, "lazy"); await sleep(1000); - const videoSegmentBuffered = player - .__priv_getSegmentSinkContent("video") - .map(({ infos }) => { + const metrics = await player.__priv_getSegmentSinkMetrics(); + const videoSegmentBuffered = metrics.segmentSinks.video.segmentInventory.map( + ({ infos }) => { return { bitrate: infos.representation.bitrate, - time: infos.segment.time, - end: infos.segment.end, }; - }); + }, + ); expect(videoSegmentBuffered.length).to.be.at.least(3); expect(videoSegmentBuffered[0].bitrate).to.equal(400000); expect(videoSegmentBuffered[1].bitrate).to.equal(400000); diff --git a/tests/utils/checkAfterSleepWithBackoff.js b/tests/utils/checkAfterSleepWithBackoff.js index f9f55c41cc..d2a749a1df 100644 --- a/tests/utils/checkAfterSleepWithBackoff.js +++ b/tests/utils/checkAfterSleepWithBackoff.js @@ -36,7 +36,10 @@ export async function checkAfterSleepWithBackoff(configuration, checks) { let sleepTime = minTimeMs ?? 0; try { await sleep(sleepTime); - checkFn(); + const result = checkFn(); + if (result instanceof Promise) { + await result; + } } catch (err) { onFailure(); const usedMax = maxTimeMs ?? 4000;