feat:(worker) debug element display buffer in worker mode (#1438)

* WIP

* working POC

* add a mechanism to only send data if a debug element is shown

* fix a bug in worker mode, same representation was displayed with different colors

* remove console.log

* use uniqueId rather than serialize

* change the behavior so it's main thread that ask for segmentSink infos

* WIP

* change architecture to be event based, and to not perform complex serialization

* rename and fix bugs

* clean

* let segmentInventory be undefined if there is no segmentInventory for that media type

* WIP with async function

* clean

* reorganiser code

* update types and lint

* fix a race condition

* use correct type name convention

* clean

* delete __priv_getSegmentSinkContent from public api

* factorize code

* clean

* fix clean up on new content

* review

* fix integration test for async

* fmt

* display buffer data in multi_thread mode

* reject if cancelled

* use correct cancel signal

* handle case changing audio or video track

demo/full/scripts/components/BufferContentGraph.tsx

@@ -1,14 +1,9 @@
 import * as React from "react";
-import type {
-  IAudioRepresentation,
-  IVideoRepresentation,
-} from "../../../../src/public_types";
+import type { IVideoRepresentation } from "../../../../src/public_types";
 import capitalizeFirstLetter from "../lib/capitalizeFirstLetter";
 import shuffleArray from "../lib/shuffleArray";
-import type { IBufferedData } from "../modules/player/index";
 import ToolTip from "./ToolTip";
-
-type IRepresentation = IAudioRepresentation | IVideoRepresentation;
+import { IBufferedChunkSnapshot } from "../../../../src/core/segment_sinks/segment_buffers_store";
 
 const { useEffect, useMemo, useRef, useState } = React;
 
@@ -73,7 +68,7 @@ function paintCurrentPosition(
 interface IScaledBufferedData {
   scaledStart: number;
   scaledEnd: number;
-  bufferedInfos: IBufferedData;
+  bufferedInfos: IBufferedChunkSnapshot;
 }
 
 /**
@@ -85,7 +80,7 @@ interface IScaledBufferedData {
  * @returns {Array.<Object>}
  */
 function scaleSegments(
-  bufferedData: IBufferedData[],
+  bufferedData: IBufferedChunkSnapshot[],
   minimumPosition: number,
   maximumPosition: number,
 ): IScaledBufferedData[] {
@@ -128,7 +123,7 @@ export default function BufferContentGraph({
   type, // The type of buffer (e.g. "audio", "video" or "text")
 }: {
   currentTime: number | undefined;
-  data: IBufferedData[];
+  data: IBufferedChunkSnapshot[];
   minimumPosition: number | null | undefined;
   maximumPosition: number | null | undefined;
   seek: (pos: number) => void;
@@ -139,7 +134,7 @@ export default function BufferContentGraph({
   const [tipPosition, setTipPosition] = useState(0);
   const [tipText, setTipText] = useState("");
   const canvasEl = useRef<HTMLCanvasElement>(null);
-  const representationsEncountered = useRef<IRepresentation[]>([]);
+  const representationsIdEncountered = useRef<string[]>([]);
   const usedMaximum = maximumPosition ?? 300;
   const usedMinimum = minimumPosition ?? 0;
   const duration = Math.max(usedMaximum - usedMinimum, 0);
@@ -153,10 +148,12 @@ export default function BufferContentGraph({
   const paintSegment = React.useCallback(
     (scaledSegment: IScaledBufferedData, canvasCtx: CanvasRenderingContext2D): void => {
       const representation = scaledSegment.bufferedInfos.infos.representation;
-      let indexOfRepr = representationsEncountered.current.indexOf(representation);
+      let indexOfRepr = representationsIdEncountered.current.indexOf(
+        representation.uniqueId,
+      );
       if (indexOfRepr < 0) {
-        representationsEncountered.current.push(representation);
-        indexOfRepr = representationsEncountered.current.length - 1;
+        representationsIdEncountered.current.push(representation.uniqueId);
+        indexOfRepr = representationsIdEncountered.current.length - 1;
       }
       const colorIndex = indexOfRepr % COLORS.length;
       const color = randomColors[colorIndex];
@@ -258,7 +255,7 @@ export default function BufferContentGraph({
                 "\n" +
                 `height: ${rep.height ?? "?"}` +
                 "\n" +
-                `codec: ${representation.codec ?? "?"}` +
+                `codec: ${representation.codecs ?? "?"}` +
                 "\n" +
                 `bitrate: ${representation.bitrate ?? "?"}` +
                 "\n";
@@ -270,7 +267,7 @@ export default function BufferContentGraph({
                 "\n" +
                 `audioDescription: ${String(adaptation.isAudioDescription) ?? false}` +
                 "\n" +
-                `codec: ${representation.codec ?? "?"}` +
+                `codec: ${representation.codecs ?? "?"}` +
                 "\n" +
                 `bitrate: ${representation.bitrate ?? "?"}` +
                 "\n";

demo/full/scripts/controllers/charts/BufferContent.tsx

@@ -8,7 +8,6 @@ export default function BufferContentChart({ player }: { player: IPlayerModule }
   const currentTime = useModuleState(player, "currentTime");
   const maximumPosition = useModuleState(player, "maximumPosition");
   const minimumPosition = useModuleState(player, "minimumPosition");
-  const useWorker = useModuleState(player, "useWorker");
 
   const seek = React.useCallback(
     (position: number): void => {
@@ -17,14 +16,6 @@ export default function BufferContentChart({ player }: { player: IPlayerModule }
     [player],
   );
 
-  if (useWorker) {
-    return (
-      <div className="buffer-content-no-content">
-        Unavailable information currently when running in "multithread" mode (in a
-        WebWorker).
-      </div>
-    );
-  }
   if (bufferedData === null || Object.keys(bufferedData).length === 0) {
     return <div className="buffer-content-no-content"> No content yet </div>;
   }

demo/full/scripts/modules/player/events.ts

@@ -90,27 +90,32 @@ function linkPlayerEventsToState(
     player.removeEventListener("playerStateChange", onStateUpdate);
   });
 
-  function updateBufferedData(): void {
+  async function updateBufferedData(): Promise<void> {
     if (player.getPlayerState() === "STOPPED") {
       return;
     }
-    let audioContent = player.__priv_getSegmentSinkContent("audio");
-    if (Array.isArray(audioContent)) {
-      audioContent = audioContent.slice();
-    }
-    let textContent = player.__priv_getSegmentSinkContent("text");
-    if (Array.isArray(textContent)) {
-      textContent = textContent.slice();
-    }
-    let videoContent = player.__priv_getSegmentSinkContent("video");
-    if (Array.isArray(videoContent)) {
-      videoContent = videoContent.slice();
+    try {
+      const metrics = await player.__priv_getSegmentSinkMetrics();
+      let audioContent = metrics?.segmentSinks.audio.segmentInventory ?? null;
+      if (Array.isArray(audioContent)) {
+        audioContent = audioContent.slice();
+      }
+      let textContent = metrics?.segmentSinks.text.segmentInventory ?? null;
+      if (Array.isArray(textContent)) {
+        textContent = textContent.slice();
+      }
+      let videoContent = metrics?.segmentSinks.video.segmentInventory ?? null;
+      if (Array.isArray(videoContent)) {
+        videoContent = videoContent.slice();
+      }
+      state.update("bufferedData", {
+        audio: audioContent,
+        video: videoContent,
+        text: textContent,
+      });
+    } catch (err) {
+      // Do nothing
     }
-    state.update("bufferedData", {
-      audio: audioContent,
-      video: videoContent,
-      text: textContent,
-    });
   }
 
   const bufferedDataItv = setInterval(updateBufferedData, BUFFERED_DATA_UPDATES_INTERVAL);

src/core/main/worker/worker_main.ts

@@ -398,6 +398,11 @@ export default function initializeWorkerMain() {
         break;
       }
 
+      case MainThreadMessageType.PullSegmentSinkStoreInfos: {
+        sendSegmentSinksStoreInfos(contentPreparer, msg.value.messageId);
+        break;
+      }
+
       default:
         assertUnreachable(msg);
     }
@@ -900,3 +905,25 @@ function updateLoggerLevel(logLevel: ILoggerLevel, sendBackLogs: boolean): void
     });
   }
 }
+
+/**
+ * Send a message `SegmentSinkStoreUpdate` to the main thread with
+ * a serialized object that represents the segmentSinksStore state.
+ * @param {ContentPreparer} contentPreparer
+ * @returns {void}
+ */
+function sendSegmentSinksStoreInfos(
+  contentPreparer: ContentPreparer,
+  messageId: number,
+): void {
+  const currentContent = contentPreparer.getCurrentContent();
+  if (currentContent === null) {
+    return;
+  }
+  const segmentSinksMetrics = currentContent.segmentSinksStore.getSegmentSinksMetrics();
+  sendMessage({
+    type: WorkerMessageType.SegmentSinkStoreUpdate,
+    contentId: currentContent.contentId,
+    value: { segmentSinkMetrics: segmentSinksMetrics, messageId },
+  });
+}

src/core/segment_sinks/inventory/types.ts

@@ -14,7 +14,15 @@
  * limitations under the License.
  */
 
-import type { IAdaptation, ISegment, IPeriod, IRepresentation } from "../../../manifest";
+import type {
+  IAdaptation,
+  ISegment,
+  IPeriod,
+  IRepresentation,
+  IAdaptationMetadata,
+  IPeriodMetadata,
+  IRepresentationMetadata,
+} from "../../../manifest";
 
 /** Content information for a single buffered chunk */
 export interface IChunkContext {
@@ -27,3 +35,9 @@ export interface IChunkContext {
   /** Segment this chunk is related to. */
   segment: ISegment;
 }
+
+export interface IChunkContextSnapshot {
+  adaptation: IAdaptationMetadata;
+  period: IPeriodMetadata;
+  representation: IRepresentationMetadata;
+}

src/core/segment_sinks/segment_buffers_store.ts

@@ -25,12 +25,38 @@ import type { IBufferType, SegmentSink } from "./implementations";
 import { AudioVideoSegmentSink } from "./implementations";
 import type { ITextDisplayerInterface } from "./implementations/text";
 import TextSegmentSink from "./implementations/text";
+import type { IBufferedChunk } from "./inventory/segment_inventory";
+import type { IChunkContext, IChunkContextSnapshot } from "./inventory/types";
 
 const POSSIBLE_BUFFER_TYPES: IBufferType[] = ["audio", "video", "text"];
 
 /** Types of "native" media buffers (i.e. which rely on a SourceBuffer) */
 type INativeMediaBufferType = "audio" | "video";
 
+/**
+ * Interface containing metadata of a buffered chunk.
+ * The metadata is serializable and does not contain references to JS objects
+ * that are not serializable, such as Map or class instances.
+ */
+export interface IBufferedChunkSnapshot extends Omit<IBufferedChunk, "infos"> {
+  infos: IChunkContextSnapshot;
+}
+
+/**
+ * Interface representing metrics for segment sinks.
+ * The metrics include information on the buffer type, codec, and segment inventory,
+ * and are categorized by segment type (audio, video, text).
+ */
+export interface ISegmentSinkMetrics {
+  segmentSinks: Record<"audio" | "video" | "text", ISegmentSinkMetricForType>;
+}
+
+interface ISegmentSinkMetricForType {
+  bufferType: IBufferType;
+  codec: string | undefined;
+  segmentInventory: IBufferedChunkSnapshot[] | undefined;
+}
+
 /**
  * Allows to easily create and dispose SegmentSinks, which are interfaces to
  * push and remove segments.
@@ -347,6 +373,31 @@ export default class SegmentSinksStore {
     }
     return true;
   }
+
+  private createSegmentSinkMetricsForType(
+    bufferType: IBufferType,
+  ): ISegmentSinkMetricForType {
+    return {
+      bufferType,
+      codec: this._initializedSegmentSinks[bufferType]?.codec,
+      segmentInventory: this._initializedSegmentSinks[bufferType]
+        ?.getLastKnownInventory()
+        .map((chunk) => ({
+          ...chunk,
+          infos: getChunkContextSnapshot(chunk.infos),
+        })),
+    };
+  }
+
+  public getSegmentSinksMetrics(): ISegmentSinkMetrics {
+    return {
+      segmentSinks: {
+        audio: this.createSegmentSinkMetricsForType("audio"),
+        video: this.createSegmentSinkMetricsForType("video"),
+        text: this.createSegmentSinkMetricsForType("text"),
+      },
+    };
+  }
 }
 
 /**
@@ -361,3 +412,11 @@ function shouldHaveNativeBuffer(
 ): bufferType is INativeMediaBufferType {
   return bufferType === "audio" || bufferType === "video";
 }
+
+function getChunkContextSnapshot(context: IChunkContext): IChunkContextSnapshot {
+  return {
+    adaptation: context.adaptation.getMetadataSnapshot(),
+    period: context.period.getMetadataSnapshot(),
+    representation: context.representation.getMetadataSnapshot(),
+  };
+}

src/main_thread/api/debug/buffer_graph.ts

@@ -1,5 +1,5 @@
-import type { IBufferedChunk } from "../../../core/types";
-import type { IRepresentation } from "../../../manifest";
+import type { IBufferedChunkSnapshot } from "../../../core/segment_sinks/segment_buffers_store";
+import type { IRepresentationMetadata } from "../../../manifest";
 
 const BUFFER_WIDTH_IN_SECONDS = 30 * 60;
 
@@ -19,16 +19,16 @@ const COLORS = [
 
 export interface ISegmentSinkGrapUpdateData {
   currentTime: number;
-  inventory: IBufferedChunk[];
+  inventory: IBufferedChunkSnapshot[];
   width: number;
   height: number;
   minimumPosition: number | undefined;
   maximumPosition: number | undefined;
 }
 
 export default class SegmentSinkGraph {
-  /** Link buffered Representation to their corresponding color. */
-  private readonly _colorMap: WeakMap<IRepresentation, string>;
+  /** Link buffered Representation's uniqueId to their corresponding color. */
+  private readonly _colorMap: Map<string, string>;
 
   /** Current amount of colors chosen to represent the various Representation. */
   private _currNbColors: number;
@@ -39,7 +39,7 @@ export default class SegmentSinkGraph {
   private readonly _canvasCtxt: CanvasRenderingContext2D | null;
 
   constructor(canvasElt: HTMLCanvasElement) {
-    this._colorMap = new WeakMap();
+    this._colorMap = new Map();
     this._currNbColors = 0;
     this._canvasElt = canvasElt;
     this._canvasCtxt = this._canvasElt.getContext("2d");
@@ -53,6 +53,19 @@ export default class SegmentSinkGraph {
   }
 
   public update(data: ISegmentSinkGrapUpdateData): void {
+    // Following logic clear the colorMap entries if they are not used anymore
+    // to prevent memory usage.
+    const representationStillInUse: Set<string> = new Set();
+    data.inventory.forEach((chunk) => {
+      representationStillInUse.add(chunk.infos.representation.uniqueId);
+    });
+
+    this._colorMap.forEach((representationId) => {
+      if (!representationStillInUse.has(representationId)) {
+        this._colorMap.delete(representationId);
+      }
+    });
+
     if (this._canvasCtxt === null) {
       return;
     }
@@ -149,14 +162,14 @@ export default class SegmentSinkGraph {
     this._canvasCtxt.fillRect(Math.ceil(startX), 0, Math.ceil(endX - startX), height);
   }
 
-  private _getColorForRepresentation(representation: IRepresentation): string {
-    const color = this._colorMap.get(representation);
+  private _getColorForRepresentation(representation: IRepresentationMetadata): string {
+    const color = this._colorMap.get(representation.uniqueId);
     if (color !== undefined) {
       return color;
     }
     const newColor = COLORS[this._currNbColors % COLORS.length];
     this._currNbColors++;
-    this._colorMap.set(representation, newColor);
+    this._colorMap.set(representation.uniqueId, newColor);
     return newColor;
   }
 }
@@ -203,7 +216,7 @@ function paintCurrentPosition(
  * @returns {Array.<Object>}
  */
 function scaleSegments(
-  bufferedData: IBufferedChunk[],
+  bufferedData: IBufferedChunkSnapshot[],
   minimumPosition: number,
   maximumPosition: number,
 ): IScaledChunk[] {
@@ -227,5 +240,5 @@ function scaleSegments(
 interface IScaledChunk {
   scaledStart: number;
   scaledEnd: number;
-  info: IBufferedChunk;
+  info: IBufferedChunkSnapshot;
 }