feat: LCEVC Integration (shaka-project#4050)

Integration of MPEG-5 Part-2 LCEVC into Shaka Player. A config must be enabled and a canvas element must be provided. The Shaka Player UI will automatically provide an appropriate canvas. Co-authored-by: Vinod Balakrishnan <[email protected]> Co-authored-by: Vinod <[email protected]> Co-authored-by: Vinod <[email protected]>
joeyparrish · Oct 3, 2022 · 284ea63 · 284ea63
1 parent 3582f0a
commit 284ea63
Show file tree

Hide file tree

Showing 27 changed files with 751 additions and 10 deletions.
diff --git a/AUTHORS b/AUTHORS
@@ -80,5 +80,6 @@ uStudio Inc. <*@ustudio.com>
 Verizon Digital Media Services <*@verizondigitalmedia.com>
 ViacomCBS <*@viacomcbs.com>
 Vincent Valot <[email protected]>
+V-Nova Limited <*@v-nova.com>
 Wayne Morgan <[email protected]>
 Raymond Cheng <[email protected]>
diff --git a/CONTRIBUTORS b/CONTRIBUTORS
@@ -119,6 +119,7 @@ Toshihiro Suzuki <[email protected]>
 Vasanth Polipelli <[email protected]>
 Vignesh Venkatasubramanian <[email protected]>
 Vincent Valot <[email protected]>
+Vinod Balakrishnan <[email protected]>
 Wayne Morgan <[email protected]>
 Yohann Connell <[email protected]>
 Raymond Cheng <[email protected]>
diff --git a/build/types/complete b/build/types/complete
@@ -8,3 +8,4 @@
 +@polyfill
 +@text
 +@ui
++@lcevc
diff --git a/build/types/lcevc b/build/types/lcevc
@@ -0,0 +1,3 @@
+# LCEVC library.
+
++../../lib/lcevc/lcevc_dil.js
diff --git a/demo/common/assets.js b/demo/common/assets.js
@@ -40,6 +40,7 @@ shakaAssets.Source = {
   APPLE: shakaDemo.MessageIds.APPLE,
   IRT: shakaDemo.MessageIds.IRT,
   MICROSOFT: shakaDemo.MessageIds.MICROSOFT,
+  VNOVA: shakaDemo.MessageIds.VNOVA,
 };
 
 
@@ -1283,5 +1284,32 @@ shakaAssets.testAssets = [
       .addFeature(shakaAssets.Feature.HIGH_DEFINITION)
       .addLicenseServer('com.microsoft.playready', 'http://test.playready.microsoft.com/service/rightsmanager.asmx?cfg=(persist:false,ck:W31bfVt9W31bfVt9W31bfQ==,ckt:aescbc)'),
   // }}}
+
+  // MPEG-5 LCEVC assets {{{
+  /* LCEVC Enabled Content with LCEVC Encoded Stream */
+  new ShakaDemoAssetInfo(
+      /* name= */ 'Big Buck Bunny (LCEVC H264)',
+      /* iconUri= */ 'https://storage.googleapis.com/shaka-asset-icons/big_buck_bunny.png',
+      /* manifestUri= */ 'https://dyctis843rxh5.cloudfront.net/vnIAZIaowG1K7qOt/master.m3u8',
+      /* source= */ shakaAssets.Source.VNOVA)
+      .addFeature(shakaAssets.Feature.HLS)
+      .addFeature(shakaAssets.Feature.HIGH_DEFINITION)
+      .addFeature(shakaAssets.Feature.MP4)
+      .addFeature(shakaAssets.Feature.WEBM)
+      .addFeature(shakaAssets.Feature.OFFLINE)
+      .addDescription('H264 HLS stream with LCEVC enhancement')
+      .markAsFeatured('Big Buck Bunny (LCEVC H264)')
+      .setExtraConfig({
+        streaming: {
+          useNativeHlsOnSafari: false,
+          forceTransmuxTS: true,
+        },
+        lcevc: {
+          dynamicPerformanceScaling: true,
+          logLevel: 0,
+          drawLogo: false,
+        },
+      }),
+  // }}}
 ];
 /* eslint-enable max-len */
diff --git a/demo/common/message_ids.js b/demo/common/message_ids.js
@@ -206,6 +206,10 @@ shakaDemo.MessageIds = {
   INACCURATE_MANIFEST_TOLERANCE: 'DEMO_INACCURATE_MANIFEST_TOLERANCE',
   INITIAL_SEGMENT_LIMIT: 'DEMO_INITIAL_SEGMENT_LIMIT',
   LANGUAGE_SECTION_HEADER: 'DEMO_LANGUAGE_SECTION_HEADER',
+  LCEVC_DRAW_LOGO: 'DEMO_LCEVC_DRAW_LOGO',
+  LCEVC_DYNAMIC_PERFORMANCE_SCALING: 'DEMO_LCEVC_DYNAMIC_PERFORMANCE_SCALING',
+  LCEVC_LOG_LEVEL: 'DEMO_LCEVC_LOG_LEVEL',
+  LCEVC_SECTION_HEADER: 'DEMO_LCEVC_SECTION_HEADER',
   LOG_LEVEL: 'DEMO_LOG_LEVEL',
   LOG_LEVEL_DEBUG: 'DEMO_LOG_LEVEL_DEBUG',
   LOG_LEVEL_INFO: 'DEMO_LOG_LEVEL_INFO',
@@ -269,6 +273,7 @@ shakaDemo.MessageIds = {
   USE_NATIVE_HLS_SAFARI: 'DEMO_USE_NATIVE_HLS_SAFARI',
   USE_PERSISTENT_LICENSES: 'DEMO_USE_PERSISTENT_LICENSES',
   VIDEO_ROBUSTNESS: 'DEMO_VIDEO_ROBUSTNESS',
+  VNOVA: 'DEMO_VNOVA',
   XLINK_FAIL_GRACEFULLY: 'DEMO_XLINK_FAIL_GRACEFULLY',
 };
 /* eslint-enable max-len */
diff --git a/demo/config.js b/demo/config.js
@@ -97,6 +97,7 @@ shakaDemo.Config = class {
     this.addRetrictionsSection_('',
         shakaDemo.MessageIds.RESTRICTIONS_SECTION_HEADER);
     this.addCmcdSection_();
+    this.addLcevcSection_();
   }
 
   /**
@@ -294,6 +295,17 @@ shakaDemo.Config = class {
         .addBoolInput_(MessageIds.USE_HEADERS, 'cmcd.useHeaders');
   }
 
+  /** @private */
+  addLcevcSection_() {
+    const MessageIds = shakaDemo.MessageIds;
+    const docLink = this.resolveExternLink_('.LcevcConfiguration');
+    this.addSection_(MessageIds.LCEVC_SECTION_HEADER, docLink)
+        .addBoolInput_(MessageIds.LCEVC_DYNAMIC_PERFORMANCE_SCALING,
+            'lcevc.dynamicPerformanceScaling')
+        .addNumberInput_(MessageIds.LCEVC_LOG_LEVEL, 'lcevc.logLevel')
+        .addBoolInput_(MessageIds.LCEVC_DRAW_LOGO, 'lcevc.drawLogo');
+  }
+
   /**
    * @param {string} category
    * @param {!shakaDemo.MessageIds} sectionName

diff --git a/demo/index.html b/demo/index.html
@@ -38,6 +38,8 @@
     <script defer src="https://www.gstatic.com/cv/js/sender/v1/cast_sender.js"></script>
     <!-- transmuxing support is enabled by including this: -->
     <script defer src="../node_modules/mux.js/dist/mux.min.js"></script>
+    <!-- MPEG-5 Part2 LCEVC support is enabled by including this: -->
+    <script defer src="../node_modules/lcevc_dil.js/dist/lcevc_dil.min.js"></script>
     <!-- MDL is enabled by including this: -->
     <script defer src="../node_modules/material-design-lite/dist/material.min.js"></script>
     <!-- MDL modal dialogs are enabled by including these: -->

diff --git a/demo/locales/en.json b/demo/locales/en.json
@@ -115,6 +115,10 @@
   "DEMO_IRT": "IRT",
   "DEMO_JSDELIVR": "jsDelivr",
   "DEMO_LANGUAGE_SECTION_HEADER": "Language",
+  "DEMO_LCEVC_DRAW_LOGO": "Draw LCEVC Logo",
+  "DEMO_LCEVC_DYNAMIC_PERFORMANCE_SCALING": "LCEVC Dynamic Performance scaling",
+  "DEMO_LCEVC_LOG_LEVEL": "LCEVC Log Level",
+  "DEMO_LCEVC_SECTION_HEADER": "MPEG-5 Part-2 LCEVC",
   "DEMO_LIBRARIES": "Google Hosted Libraries",
   "DEMO_LICENSE": "Apache License",
   "DEMO_LICENSE_CERTIFICATE_URL": "Custom License Certificate URL",
@@ -243,6 +247,7 @@
   "DEMO_VISUALIZER_AUTO_SCREENSHOT_TOGGLE": "Take Screenshot On Stall",
   "DEMO_VISUALIZER_BUTTON": "Buffer Visualizer",
   "DEMO_VISUALIZER_SCREENSHOT_BUTTON": "Take Screenshot",
+  "DEMO_VNOVA": "V-Nova",
   "DEMO_VOD": "VOD",
   "DEMO_WEBM": "WebM",
   "DEMO_WIDEVINE": "Widevine DRM",

diff --git a/demo/locales/source.json b/demo/locales/source.json
@@ -463,6 +463,22 @@
     "description": "The header for a section of configuration values.",
     "message": "Language"
   },
+  "DEMO_LCEVC_DRAW_LOGO": {
+    "description": "LCEVC Watermark on the top left hand corner of the canvas.",
+    "message": "Draw [PROPER_NAME:LCEVC] Logo"
+  },
+  "DEMO_LCEVC_DYNAMIC_PERFORMANCE_SCALING": {
+    "description": "Dynamic Performance scaling for enabling/disabling LCEVC decoding based on decode performance.",
+    "message": "[PROPER_NAME:LCEVC] [JARGON:Dynamic Performance scaling]"
+  },
+  "DEMO_LCEVC_LOG_LEVEL": {
+    "description": "Log Level for LCEVC Lib (0-5)",
+    "message": "[PROPER_NAME:LCEVC] Log Level"
+  },
+  "DEMO_LCEVC_SECTION_HEADER": {
+    "description": "The header for a section of configuration values.",
+    "message": "[PROPER_NAME:MPEG-5 Part-2 LCEVC]"
+  },
   "DEMO_LIBRARIES": {
     "description": "A link in the footer, to Shaka Player in the Google Hosted Libraries CDN.",
     "message": "[PROPER_NAME:Google Hosted Libraries]"
@@ -975,6 +991,10 @@
     "description": "A button that takes a screenshot of the current visualizer state.",
     "message": "Take Screenshot"
   },
+  "DEMO_VNOVA": {
+    "description": "Text that describes an asset that comes from V-Nova.",
+    "message": "[PROPER_NAME:V-Nova]"
+  },
   "DEMO_VOD": {
     "description": "Text that describes an asset that is a VOD (Video On Delivery).",
     "message": "[JARGON:VOD]"

diff --git a/docs/design/lcevc-architecture.png b/docs/design/lcevc-architecture.png
diff --git a/docs/design/lcevc-demo.png b/docs/design/lcevc-demo.png
diff --git a/docs/design/lcevc-integration.md b/docs/design/lcevc-integration.md
@@ -0,0 +1,72 @@
+
+# Shaka Player LCEVC integration
+
+# Introduction
+
+This article describes the V-Nova LCEVC Shaka Player integration.
+
+# LCEVC Integration
+
+## Adding V-Nova required files
+### Importing DIL - Decoder Integration Layer
+
+V-Nova LCEVC DIL Libraries are included using the same approach that the other external libraries are currently using. The necessary V-Nova LCEVC DIL files need to be imported in the HTML page that is going to be used by Shaka Player to decode LCEVC. Checks are inplace that make sure the necessary objects are available.
+
+Npm package : <https://www.npmjs.com/package/lcevc_dil.js>
+
+```javascript
+  <!-- MPEG-5 Part2 LCEVC support is enabled by including this: -->
+  <script defer src="../node_modules/lcevc_dil.js/dist/lcevc_dil.min.js"></script>
+```
+
+To allow the Closure compiler to use the objects and methods that are exported by the DIL.js a new `extern` is created.
+
+### Defining an Extern for LCEVC
+
+`externs/lcevc.js` exposes the functions from the LCEVC DIL library required for LCEVC Decoding.
+
+## Integration point
+
+### The shaka.lcevc.Dil class - (DIL : Decoder Integration Layer)
+
+The main logic of the LCEVC integration is located in the `lib/lcevc_dil.min.js` file. In this file the shaka.lcevc.Dil is exported to be used in the project. This class is in charge of creating the Dil object using the mentioned externs, checking if LCEVC DIL library is available, etc.
+
+### Modifications in the player
+
+The shaka.Player class, defined in the `lib/player.js` file, is the main player object for Shaka Player. There is a setter function for setting up a `canvas` element that is received from the user.
+If shaka.ui is used the `canvas` is created in line with the video element in the same container overlaying the video element. If user provides a custom canvas using the setter function, The user is responsible for placing the canvas element in the desired position and resizing it.
+
+`shaka.externs.LcevcConfiguration` is added to the `playerConfiguration` that is used as configuration for the LCEVC DIL Library.
+
+The Dil object is created in the `onLoad_()` event that is triggered when a new video is loaded in Shaka Player. Attaching to a media element is defined as:
+
+-   Registering error listeners to the media element.
+-   Catching the video element for use outside of the load graph.
+
+The Dil object is created only if LCEVC is supported (LCEVC libs are loaded on the page) and also when it was not already created in another `onLoad_()` event execution.
+
+
+### Feeding the Dil
+
+The logic that Shaka Player uses to communicate with the Media Source Extensions (MSE) is located in the `media/media_source_engine.js` file.
+
+![image.png](lcevc-architecture.png)
+
+ `append_()` function that is used to feed the MSE Source Buffer is intercepted and modified to pass the video buffers to the LCEVC DIL Libraries before appending to the MSE Source Buffers. 
+
+## Demo page
+
+The relevant libraries are added in the Demo Page like so: 
+
+```javascript
+  <!-- MPEG-5 Part2 LCEVC support is enabled by including this: -->
+  <script defer src="../node_modules/lcevc_dil.js/dist/lcevc_dil.min.js"></script>
+```
+
+And a new video sample with enhancement data is added to the `demo/common/assets.js` file under a new source `MPEG-5 Part 2 LCEVC`.
+
+After these changes the demo page looks like :
+
+![image.png](lcevc-demo.png)
+
+          
diff --git a/externs/lcevc.js b/externs/lcevc.js
@@ -0,0 +1,81 @@
+/**
+* @fileoverview Externs for LcevcDil
+* compiler.
+*
+* @externs
+*/
+
+// This empty namespace is declared to check if LcevcDil libraries are loaded.
+var libDPIModule = {};
+var LcevcDil = {};
+
+/**
+* LCEVC DIL constructor
+* @constructor
+*/
+LcevcDil.LcevcDIL = class {
+  /**
+   * @param {HTMLVideoElement} media
+   * @param {HTMLCanvasElement} canvas
+   * @param {shaka.extern.LcevcConfiguration} dilConfig
+   */
+  constructor(media, canvas, dilConfig) {
+  }
+
+  /**
+   * Append the video buffers before they are appended to
+   * Media Source Extensions SourceBuffer. Here the lcevc data
+   * will be parsed and managed to enahnce frames based on timestamps.
+   *
+   * @param {!BufferSource} data Video Buffer Data.
+   * @param {string} type Type of Video Buffer Data.
+   * @param {number} variantId Variant that the fragment belongs to.
+   */
+  appendBuffer(data, type, variantId) {}
+
+  /**
+   * Set current variant as variantId to the LCEVC decoder
+   * @param {!number} variantId
+   * @param {!boolean} autoBufferSwitch is lcevcDil mode that switches variant
+   * when the downloaded buffer from last variant has finished playing and
+   * buffers from the new variant starts to play.
+   */
+  setLevelSwitching(variantId, autoBufferSwitch) {}
+
+  /**
+   * Set container Format for LCEVC Data Parsing.
+   * @param {!number} containerFormat container type of the stream.
+   */
+  setContainerFormat(containerFormat) {}
+
+  /**
+   * Close LCEVC DIL
+   */
+  close() {}
+};
+
+/**
+ * LCEVC Support Check
+ */
+LcevcDil.SupportObject = {
+
+  /**
+   * Check if canvas has WebGL support
+   * @param {HTMLCanvasElement} canvas
+   * @return {boolean} true if requirements are met.
+   */
+  webGLSupport(canvas) {},
+
+};
+
+/**
+ * LCEVC Support Checklist Result
+ * @type {boolean}
+ */
+LcevcDil.SupportObject.SupportStatus;
+
+/**
+ * LCEVC Support CheckList Error if any.
+ * @type {string}
+ */
+LcevcDil.SupportObject.SupportError;
diff --git a/externs/shaka/player.js b/externs/shaka/player.js
@@ -1155,6 +1155,45 @@ shaka.extern.AdvancedAbrConfiguration;
  */
 shaka.extern.CmcdConfiguration;
 
+/**
+ * @typedef {{
+ *   dynamicPerformanceScaling: boolean,
+ *   logLevel: number,
+ *   drawLogo: boolean
+ * }}
+ *
+ * @description
+ *   Decoding for MPEG-5 Part2 LCEVC.
+ *
+ * @property {boolean} dynamicPerformanceScaling
+ *   If <code>true</code>, LCEVC Dynamic Performance Scaling or dps is enabled
+ *   to be triggered, when the system is not able to decode frames within a
+ *   specific tolerance of the fps of the video and disables LCEVC decoding
+ *   for some time. The base video will be shown upscaled to target resolution.
+ *   If it is triggered again within a short period of time, the disabled
+ *   time will be higher and if it is triggered three times in a row the LCEVC
+ *   decoding will be disabled for that playback session.
+ *   If dynamicPerformanceScaling is false, LCEVC decode will be forced
+ *   and will drop frames appropriately if performance is sub optimal.
+ *   Defaults to <code>true</code>.
+ * @property {number} logLevel
+ *   Loglevel 0-5 for logging.
+ *   NONE = 0
+ *   ERROR = 1
+ *   WARNING = 2
+ *   INFO = 3
+ *   DEBUG = 4
+ *   VERBOSE = 5
+ *   Defaults to <code>0</code>.
+ * @property {boolean} drawLogo
+ *   If <code>true</code>, LCEVC Logo is placed on the top left hand corner
+ *   which only appears when the LCEVC enahanced Frames are being rendered.
+ *   Defaults to true for the lib but is forced to false in this integration
+ *   unless explicitly set to true through config.
+ *   Defaults to <code>false</code>.
+ * @exportDoc
+ */
+shaka.extern.LcevcConfiguration;
 
 /**
  * @typedef {{
@@ -1206,6 +1245,7 @@ shaka.extern.OfflineConfiguration;
  *   abrFactory: shaka.extern.AbrManager.Factory,
  *   abr: shaka.extern.AbrConfiguration,
  *   cmcd: shaka.extern.CmcdConfiguration,
+ *   lcevc: shaka.extern.LcevcConfiguration,
  *   offline: shaka.extern.OfflineConfiguration,
  *   preferredAudioLanguage: string,
  *   preferredTextLanguage: string,
@@ -1238,6 +1278,9 @@ shaka.extern.OfflineConfiguration;
  *   ABR configuration and settings.
  * @property {shaka.extern.CmcdConfiguration} cmcd
  *   CMCD configuration and settings. (Common Media Client Data)
+ * @property {shaka.extern.LcevcConfiguration} lcevc
+ *   MPEG-5 LCEVC configuration and settings.
+ *   (Low Complexity Enhancement Video Codec)
  * @property {shaka.extern.OfflineConfiguration} offline
  *   Offline configuration and settings.
  * @property {string} preferredAudioLanguage

diff --git a/lib/cast/cast_utils.js b/lib/cast/cast_utils.js
@@ -399,6 +399,7 @@ shaka.cast.CastUtils.PlayerVoidMethods = [
  */
 shaka.cast.CastUtils.PlayerPromiseMethods = [
   'attach',
+  'attachCanvas',
   'detach',
   // The manifestFactory parameter of load is not supported.
   'load',