diff --git a/AUTHORS b/AUTHORS index 07e71ce452..d01130be27 100644 --- a/AUTHORS +++ b/AUTHORS @@ -80,5 +80,6 @@ uStudio Inc. <*@ustudio.com> Verizon Digital Media Services <*@verizondigitalmedia.com> ViacomCBS <*@viacomcbs.com> Vincent Valot +V-Nova Limited <*@v-nova.com> Wayne Morgan Raymond Cheng diff --git a/CONTRIBUTORS b/CONTRIBUTORS index c1a467f74e..2b985b4ef0 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -119,6 +119,7 @@ Toshihiro Suzuki Vasanth Polipelli Vignesh Venkatasubramanian Vincent Valot +Vinod Balakrishnan Wayne Morgan Yohann Connell Raymond Cheng diff --git a/build/types/complete b/build/types/complete index f2476defe2..c549a23680 100644 --- a/build/types/complete +++ b/build/types/complete @@ -8,3 +8,4 @@ +@polyfill +@text +@ui ++@lcevc diff --git a/build/types/lcevc b/build/types/lcevc new file mode 100644 index 0000000000..df4a50a936 --- /dev/null +++ b/build/types/lcevc @@ -0,0 +1,3 @@ +# LCEVC library. + ++../../lib/lcevc/lcevc_dil.js diff --git a/demo/common/assets.js b/demo/common/assets.js index a1d97e1671..46d14a405f 100644 --- a/demo/common/assets.js +++ b/demo/common/assets.js @@ -40,6 +40,7 @@ shakaAssets.Source = { APPLE: shakaDemo.MessageIds.APPLE, IRT: shakaDemo.MessageIds.IRT, MICROSOFT: shakaDemo.MessageIds.MICROSOFT, + VNOVA: shakaDemo.MessageIds.VNOVA, }; @@ -1283,5 +1284,32 @@ shakaAssets.testAssets = [ .addFeature(shakaAssets.Feature.HIGH_DEFINITION) .addLicenseServer('com.microsoft.playready', 'http://test.playready.microsoft.com/service/rightsmanager.asmx?cfg=(persist:false,ck:W31bfVt9W31bfVt9W31bfQ==,ckt:aescbc)'), // }}} + + // MPEG-5 LCEVC assets {{{ + /* LCEVC Enabled Content with LCEVC Encoded Stream */ + new ShakaDemoAssetInfo( + /* name= */ 'Big Buck Bunny (LCEVC H264)', + /* iconUri= */ 'https://storage.googleapis.com/shaka-asset-icons/big_buck_bunny.png', + /* manifestUri= */ 'https://dyctis843rxh5.cloudfront.net/vnIAZIaowG1K7qOt/master.m3u8', + /* source= */ shakaAssets.Source.VNOVA) + .addFeature(shakaAssets.Feature.HLS) + .addFeature(shakaAssets.Feature.HIGH_DEFINITION) + .addFeature(shakaAssets.Feature.MP4) + .addFeature(shakaAssets.Feature.WEBM) + .addFeature(shakaAssets.Feature.OFFLINE) + .addDescription('H264 HLS stream with LCEVC enhancement') + .markAsFeatured('Big Buck Bunny (LCEVC H264)') + .setExtraConfig({ + streaming: { + useNativeHlsOnSafari: false, + forceTransmuxTS: true, + }, + lcevc: { + dynamicPerformanceScaling: true, + logLevel: 0, + drawLogo: false, + }, + }), + // }}} ]; /* eslint-enable max-len */ diff --git a/demo/common/message_ids.js b/demo/common/message_ids.js index 84a4cb7ea9..6f57cc69e9 100644 --- a/demo/common/message_ids.js +++ b/demo/common/message_ids.js @@ -206,6 +206,10 @@ shakaDemo.MessageIds = { INACCURATE_MANIFEST_TOLERANCE: 'DEMO_INACCURATE_MANIFEST_TOLERANCE', INITIAL_SEGMENT_LIMIT: 'DEMO_INITIAL_SEGMENT_LIMIT', LANGUAGE_SECTION_HEADER: 'DEMO_LANGUAGE_SECTION_HEADER', + LCEVC_DRAW_LOGO: 'DEMO_LCEVC_DRAW_LOGO', + LCEVC_DYNAMIC_PERFORMANCE_SCALING: 'DEMO_LCEVC_DYNAMIC_PERFORMANCE_SCALING', + LCEVC_LOG_LEVEL: 'DEMO_LCEVC_LOG_LEVEL', + LCEVC_SECTION_HEADER: 'DEMO_LCEVC_SECTION_HEADER', LOG_LEVEL: 'DEMO_LOG_LEVEL', LOG_LEVEL_DEBUG: 'DEMO_LOG_LEVEL_DEBUG', LOG_LEVEL_INFO: 'DEMO_LOG_LEVEL_INFO', @@ -269,6 +273,7 @@ shakaDemo.MessageIds = { USE_NATIVE_HLS_SAFARI: 'DEMO_USE_NATIVE_HLS_SAFARI', USE_PERSISTENT_LICENSES: 'DEMO_USE_PERSISTENT_LICENSES', VIDEO_ROBUSTNESS: 'DEMO_VIDEO_ROBUSTNESS', + VNOVA: 'DEMO_VNOVA', XLINK_FAIL_GRACEFULLY: 'DEMO_XLINK_FAIL_GRACEFULLY', }; /* eslint-enable max-len */ diff --git a/demo/config.js b/demo/config.js index e9d41ed6b3..d6e424346f 100644 --- a/demo/config.js +++ b/demo/config.js @@ -97,6 +97,7 @@ shakaDemo.Config = class { this.addRetrictionsSection_('', shakaDemo.MessageIds.RESTRICTIONS_SECTION_HEADER); this.addCmcdSection_(); + this.addLcevcSection_(); } /** @@ -294,6 +295,17 @@ shakaDemo.Config = class { .addBoolInput_(MessageIds.USE_HEADERS, 'cmcd.useHeaders'); } + /** @private */ + addLcevcSection_() { + const MessageIds = shakaDemo.MessageIds; + const docLink = this.resolveExternLink_('.LcevcConfiguration'); + this.addSection_(MessageIds.LCEVC_SECTION_HEADER, docLink) + .addBoolInput_(MessageIds.LCEVC_DYNAMIC_PERFORMANCE_SCALING, + 'lcevc.dynamicPerformanceScaling') + .addNumberInput_(MessageIds.LCEVC_LOG_LEVEL, 'lcevc.logLevel') + .addBoolInput_(MessageIds.LCEVC_DRAW_LOGO, 'lcevc.drawLogo'); + } + /** * @param {string} category * @param {!shakaDemo.MessageIds} sectionName diff --git a/demo/index.html b/demo/index.html index b34d581c02..db4a3a342b 100644 --- a/demo/index.html +++ b/demo/index.html @@ -38,6 +38,8 @@ + + diff --git a/demo/locales/en.json b/demo/locales/en.json index 976c822c8a..321434d534 100644 --- a/demo/locales/en.json +++ b/demo/locales/en.json @@ -115,6 +115,10 @@ "DEMO_IRT": "IRT", "DEMO_JSDELIVR": "jsDelivr", "DEMO_LANGUAGE_SECTION_HEADER": "Language", + "DEMO_LCEVC_DRAW_LOGO": "Draw LCEVC Logo", + "DEMO_LCEVC_DYNAMIC_PERFORMANCE_SCALING": "LCEVC Dynamic Performance scaling", + "DEMO_LCEVC_LOG_LEVEL": "LCEVC Log Level", + "DEMO_LCEVC_SECTION_HEADER": "MPEG-5 Part-2 LCEVC", "DEMO_LIBRARIES": "Google Hosted Libraries", "DEMO_LICENSE": "Apache License", "DEMO_LICENSE_CERTIFICATE_URL": "Custom License Certificate URL", @@ -243,6 +247,7 @@ "DEMO_VISUALIZER_AUTO_SCREENSHOT_TOGGLE": "Take Screenshot On Stall", "DEMO_VISUALIZER_BUTTON": "Buffer Visualizer", "DEMO_VISUALIZER_SCREENSHOT_BUTTON": "Take Screenshot", + "DEMO_VNOVA": "V-Nova", "DEMO_VOD": "VOD", "DEMO_WEBM": "WebM", "DEMO_WIDEVINE": "Widevine DRM", diff --git a/demo/locales/source.json b/demo/locales/source.json index fede82ad59..62219ead8e 100644 --- a/demo/locales/source.json +++ b/demo/locales/source.json @@ -463,6 +463,22 @@ "description": "The header for a section of configuration values.", "message": "Language" }, + "DEMO_LCEVC_DRAW_LOGO": { + "description": "LCEVC Watermark on the top left hand corner of the canvas.", + "message": "Draw [PROPER_NAME:LCEVC] Logo" + }, + "DEMO_LCEVC_DYNAMIC_PERFORMANCE_SCALING": { + "description": "Dynamic Performance scaling for enabling/disabling LCEVC decoding based on decode performance.", + "message": "[PROPER_NAME:LCEVC] [JARGON:Dynamic Performance scaling]" + }, + "DEMO_LCEVC_LOG_LEVEL": { + "description": "Log Level for LCEVC Lib (0-5)", + "message": "[PROPER_NAME:LCEVC] Log Level" + }, + "DEMO_LCEVC_SECTION_HEADER": { + "description": "The header for a section of configuration values.", + "message": "[PROPER_NAME:MPEG-5 Part-2 LCEVC]" + }, "DEMO_LIBRARIES": { "description": "A link in the footer, to Shaka Player in the Google Hosted Libraries CDN.", "message": "[PROPER_NAME:Google Hosted Libraries]" @@ -975,6 +991,10 @@ "description": "A button that takes a screenshot of the current visualizer state.", "message": "Take Screenshot" }, + "DEMO_VNOVA": { + "description": "Text that describes an asset that comes from V-Nova.", + "message": "[PROPER_NAME:V-Nova]" + }, "DEMO_VOD": { "description": "Text that describes an asset that is a VOD (Video On Delivery).", "message": "[JARGON:VOD]" diff --git a/docs/design/lcevc-architecture.png b/docs/design/lcevc-architecture.png new file mode 100644 index 0000000000..3979707b0b Binary files /dev/null and b/docs/design/lcevc-architecture.png differ diff --git a/docs/design/lcevc-demo.png b/docs/design/lcevc-demo.png new file mode 100644 index 0000000000..0d33e198f3 Binary files /dev/null and b/docs/design/lcevc-demo.png differ diff --git a/docs/design/lcevc-integration.md b/docs/design/lcevc-integration.md new file mode 100644 index 0000000000..fb7685dfc2 --- /dev/null +++ b/docs/design/lcevc-integration.md @@ -0,0 +1,72 @@ + +# Shaka Player LCEVC integration + +# Introduction + +This article describes the V-Nova LCEVC Shaka Player integration. + +# LCEVC Integration + +## Adding V-Nova required files +### Importing DIL - Decoder Integration Layer + +V-Nova LCEVC DIL Libraries are included using the same approach that the other external libraries are currently using. The necessary V-Nova LCEVC DIL files need to be imported in the HTML page that is going to be used by Shaka Player to decode LCEVC. Checks are inplace that make sure the necessary objects are available. + +Npm package : + +```javascript + + +``` + +To allow the Closure compiler to use the objects and methods that are exported by the DIL.js a new `extern` is created. + +### Defining an Extern for LCEVC + +`externs/lcevc.js` exposes the functions from the LCEVC DIL library required for LCEVC Decoding. + +## Integration point + +### The shaka.lcevc.Dil class - (DIL : Decoder Integration Layer) + +The main logic of the LCEVC integration is located in the `lib/lcevc_dil.min.js` file. In this file the shaka.lcevc.Dil is exported to be used in the project. This class is in charge of creating the Dil object using the mentioned externs, checking if LCEVC DIL library is available, etc. + +### Modifications in the player + +The shaka.Player class, defined in the `lib/player.js` file, is the main player object for Shaka Player. There is a setter function for setting up a `canvas` element that is received from the user. +If shaka.ui is used the `canvas` is created in line with the video element in the same container overlaying the video element. If user provides a custom canvas using the setter function, The user is responsible for placing the canvas element in the desired position and resizing it. + +`shaka.externs.LcevcConfiguration` is added to the `playerConfiguration` that is used as configuration for the LCEVC DIL Library. + +The Dil object is created in the `onLoad_()` event that is triggered when a new video is loaded in Shaka Player. Attaching to a media element is defined as: + +- Registering error listeners to the media element. +- Catching the video element for use outside of the load graph. + +The Dil object is created only if LCEVC is supported (LCEVC libs are loaded on the page) and also when it was not already created in another `onLoad_()` event execution. + + +### Feeding the Dil + +The logic that Shaka Player uses to communicate with the Media Source Extensions (MSE) is located in the `media/media_source_engine.js` file. + +![image.png](lcevc-architecture.png) + + `append_()` function that is used to feed the MSE Source Buffer is intercepted and modified to pass the video buffers to the LCEVC DIL Libraries before appending to the MSE Source Buffers. + +## Demo page + +The relevant libraries are added in the Demo Page like so: + +```javascript + + +``` + +And a new video sample with enhancement data is added to the `demo/common/assets.js` file under a new source `MPEG-5 Part 2 LCEVC`. + +After these changes the demo page looks like : + +![image.png](lcevc-demo.png) + + \ No newline at end of file diff --git a/externs/lcevc.js b/externs/lcevc.js new file mode 100644 index 0000000000..fa42c7bd03 --- /dev/null +++ b/externs/lcevc.js @@ -0,0 +1,81 @@ +/** +* @fileoverview Externs for LcevcDil +* compiler. +* +* @externs +*/ + +// This empty namespace is declared to check if LcevcDil libraries are loaded. +var libDPIModule = {}; +var LcevcDil = {}; + +/** +* LCEVC DIL constructor +* @constructor +*/ +LcevcDil.LcevcDIL = class { + /** + * @param {HTMLVideoElement} media + * @param {HTMLCanvasElement} canvas + * @param {shaka.extern.LcevcConfiguration} dilConfig + */ + constructor(media, canvas, dilConfig) { + } + + /** + * Append the video buffers before they are appended to + * Media Source Extensions SourceBuffer. Here the lcevc data + * will be parsed and managed to enahnce frames based on timestamps. + * + * @param {!BufferSource} data Video Buffer Data. + * @param {string} type Type of Video Buffer Data. + * @param {number} variantId Variant that the fragment belongs to. + */ + appendBuffer(data, type, variantId) {} + + /** + * Set current variant as variantId to the LCEVC decoder + * @param {!number} variantId + * @param {!boolean} autoBufferSwitch is lcevcDil mode that switches variant + * when the downloaded buffer from last variant has finished playing and + * buffers from the new variant starts to play. + */ + setLevelSwitching(variantId, autoBufferSwitch) {} + + /** + * Set container Format for LCEVC Data Parsing. + * @param {!number} containerFormat container type of the stream. + */ + setContainerFormat(containerFormat) {} + + /** + * Close LCEVC DIL + */ + close() {} +}; + +/** + * LCEVC Support Check + */ +LcevcDil.SupportObject = { + + /** + * Check if canvas has WebGL support + * @param {HTMLCanvasElement} canvas + * @return {boolean} true if requirements are met. + */ + webGLSupport(canvas) {}, + +}; + +/** + * LCEVC Support Checklist Result + * @type {boolean} + */ +LcevcDil.SupportObject.SupportStatus; + +/** + * LCEVC Support CheckList Error if any. + * @type {string} + */ +LcevcDil.SupportObject.SupportError; diff --git a/externs/shaka/player.js b/externs/shaka/player.js index d3cdae9275..a8efdad7f1 100644 --- a/externs/shaka/player.js +++ b/externs/shaka/player.js @@ -1155,6 +1155,45 @@ shaka.extern.AdvancedAbrConfiguration; */ shaka.extern.CmcdConfiguration; +/** + * @typedef {{ + * dynamicPerformanceScaling: boolean, + * logLevel: number, + * drawLogo: boolean + * }} + * + * @description + * Decoding for MPEG-5 Part2 LCEVC. + * + * @property {boolean} dynamicPerformanceScaling + * If true, LCEVC Dynamic Performance Scaling or dps is enabled + * to be triggered, when the system is not able to decode frames within a + * specific tolerance of the fps of the video and disables LCEVC decoding + * for some time. The base video will be shown upscaled to target resolution. + * If it is triggered again within a short period of time, the disabled + * time will be higher and if it is triggered three times in a row the LCEVC + * decoding will be disabled for that playback session. + * If dynamicPerformanceScaling is false, LCEVC decode will be forced + * and will drop frames appropriately if performance is sub optimal. + * Defaults to true. + * @property {number} logLevel + * Loglevel 0-5 for logging. + * NONE = 0 + * ERROR = 1 + * WARNING = 2 + * INFO = 3 + * DEBUG = 4 + * VERBOSE = 5 + * Defaults to 0. + * @property {boolean} drawLogo + * If true, LCEVC Logo is placed on the top left hand corner + * which only appears when the LCEVC enahanced Frames are being rendered. + * Defaults to true for the lib but is forced to false in this integration + * unless explicitly set to true through config. + * Defaults to false. + * @exportDoc + */ +shaka.extern.LcevcConfiguration; /** * @typedef {{ @@ -1206,6 +1245,7 @@ shaka.extern.OfflineConfiguration; * abrFactory: shaka.extern.AbrManager.Factory, * abr: shaka.extern.AbrConfiguration, * cmcd: shaka.extern.CmcdConfiguration, + * lcevc: shaka.extern.LcevcConfiguration, * offline: shaka.extern.OfflineConfiguration, * preferredAudioLanguage: string, * preferredTextLanguage: string, @@ -1238,6 +1278,9 @@ shaka.extern.OfflineConfiguration; * ABR configuration and settings. * @property {shaka.extern.CmcdConfiguration} cmcd * CMCD configuration and settings. (Common Media Client Data) + * @property {shaka.extern.LcevcConfiguration} lcevc + * MPEG-5 LCEVC configuration and settings. + * (Low Complexity Enhancement Video Codec) * @property {shaka.extern.OfflineConfiguration} offline * Offline configuration and settings. * @property {string} preferredAudioLanguage diff --git a/lib/cast/cast_utils.js b/lib/cast/cast_utils.js index 0ebcf89f1d..3eb3e47fc0 100644 --- a/lib/cast/cast_utils.js +++ b/lib/cast/cast_utils.js @@ -399,6 +399,7 @@ shaka.cast.CastUtils.PlayerVoidMethods = [ */ shaka.cast.CastUtils.PlayerPromiseMethods = [ 'attach', + 'attachCanvas', 'detach', // The manifestFactory parameter of load is not supported. 'load', diff --git a/lib/lcevc/lcevc_dil.js b/lib/lcevc/lcevc_dil.js new file mode 100644 index 0000000000..e9f4805c6c --- /dev/null +++ b/lib/lcevc/lcevc_dil.js @@ -0,0 +1,152 @@ +/*! @license + * Shaka Player + * Copyright 2016 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +goog.provide('shaka.lcevc.Dil'); +goog.require('shaka.log'); +goog.require('shaka.util.IReleasable'); + +/** + * @summary + * lcevcDil - (MPEG-5 Part 2 LCEVC - Decoder Integration Layer) provides + * all the operations related to the enhancement and rendering + * of LCEVC enabled streams and on to a canvas. + * @implements {shaka.util.IReleasable} + * @export + */ +shaka.lcevc.Dil = class { + /** + * @param {HTMLVideoElement} media The video element that will be attached to + * LCEVC Dil for input. + * @param {HTMLCanvasElement} canvas The canvas element that will be attached + * to LCEVC Dil to render the enhanced frames. + * @param {shaka.extern.LcevcConfiguration} dilConfig The LCEVC DIL + * config object to initialize the LCEVC DIL. + */ + constructor(media, canvas, dilConfig) { + /** @private {?LcevcDil.LcevcDIL} */ + this.dil_ = null; + + /** @private {number} */ + this.variantId_ = -1; + + /** @private {HTMLVideoElement} */ + this.media_ = media; + + /** @private {HTMLCanvasElement} */ + this.canvas_ = canvas; + + /** @private {shaka.extern.LcevcConfiguration} */ + this.dilConfig_ = dilConfig; + + this.create_(); + } + + /** + * Append data to the LCEVC Dil. + * @param {BufferSource} data + */ + appendBuffer(data) { + if (this.dil_) { + this.dil_.appendBuffer(data, 'video', this.variantId_); + } + } + + /** + * Hide the canvas specifically in the case of a DRM Content + */ + hideCanvas() { + if (this.dil_) { + this.canvas_.classList.add('shaka-hidden'); + } + } + + /** + * Create LCEVC Dil. + * @private + */ + create_() { + if (this.isSupported_() && !this.dil_) { + if (LcevcDil.SupportObject.webGLSupport(this.canvas_)) { + // Make sure the canvas is not hidden from a previous playback session. + this.canvas_.classList.remove('shaka-hidden'); + this.dil_ = new LcevcDil.LcevcDil( + this.media_, + this.canvas_, + this.dilConfig_); + } + } + } + + /** + * Close LCEVC Dil. + * @override + * @export + */ + release() { + if (this.dil_) { + this.dil_.close(); + this.dil_ = null; + } + } + + /** + * Check if the LCEVC Dil lib is present and is supported by the browser. + * @return {boolean} + * @private + */ + isSupported_() { + if (typeof libDPIModule === 'undefined') { + shaka.log.alwaysWarn( + 'Could not Find LCEVC Library dependencies on this page'); + } + + if (typeof LcevcDil === 'undefined') { + shaka.log.alwaysWarn('Could not Find LCEVC Library on this page'); + } else { + if (!LcevcDil.SupportObject.SupportStatus) { + shaka.log.alwaysWarn(LcevcDil.SupportObject.SupportError); + } + } + + return typeof LcevcDil !== 'undefined' && + typeof libDPIModule !== 'undefined' && + this.canvas_ instanceof HTMLCanvasElement && + LcevcDil.SupportObject.SupportStatus; + } + + /** + * Update current active variant + * @param {shaka.extern.Track} track + */ + updateVariant(track) { + let containerFormat = shaka.lcevc.Dil.ContainerFormat.MPEG2_TS; + switch (track.mimeType) { + case 'video/webm': { + containerFormat = shaka.lcevc.Dil.ContainerFormat.WEBM; + break; + } + case 'video/mp4': { + containerFormat = shaka.lcevc.Dil.ContainerFormat.MP4; + break; + } + } + if (this.dil_) { + this.variantId_ = track.id; + this.dil_.setLevelSwitching(track.id, true); + this.dil_.setContainerFormat(containerFormat); + } + } +}; + +/** + * Container Formats. + * @const @enum {number} + */ +shaka.lcevc.Dil.ContainerFormat = { + MPEG2_TS: 0, + WEBM: 1, + MP4: 2, +}; diff --git a/lib/media/media_source_engine.js b/lib/media/media_source_engine.js index a8df88bd68..9e2373585f 100644 --- a/lib/media/media_source_engine.js +++ b/lib/media/media_source_engine.js @@ -23,6 +23,7 @@ goog.require('shaka.util.ManifestParserUtils'); goog.require('shaka.util.MimeUtils'); goog.require('shaka.util.Platform'); goog.require('shaka.util.PublicPromise'); +goog.require('shaka.lcevc.Dil'); /** @@ -48,8 +49,11 @@ shaka.media.MediaSourceEngine = class { * MediaSourceEngine is destroyed, it will destroy the displayer. * @param {!function(!Array., number, ?number)=} * onMetadata + * @param {?shaka.lcevc.Dil} [lcevcDil] Optional - LCEVC Dil Object + * */ - constructor(video, closedCaptionParser, textDisplayer, onMetadata) { + constructor(video, closedCaptionParser, textDisplayer, + onMetadata, lcevcDil) { /** @private {HTMLMediaElement} */ this.video_ = video; @@ -84,6 +88,9 @@ shaka.media.MediaSourceEngine = class { number, ?number)} */ this.onMetadata_ = onMetadata || onMetadataNoOp; + /** @private {?shaka.lcevc.Dil} */ + this.lcevcDil_ = lcevcDil || null; + /** * @private {!Object.>} @@ -309,6 +316,9 @@ shaka.media.MediaSourceEngine = class { } } this.queues_ = {}; + + // This object is owned by Player + this.lcevcDil_ = null; } /** @@ -902,6 +912,15 @@ shaka.media.MediaSourceEngine = class { * @private */ append_(contentType, data) { + const ContentType = shaka.util.ManifestParserUtils.ContentType; + + // Append only video data to the LCEVC Dil. + if (contentType == ContentType.VIDEO && this.lcevcDil_) { + // Append video buffers to the LCEVC Dil for parsing and storing + // of LCEVC data. + this.lcevcDil_.appendBuffer(data); + } + // This will trigger an 'updateend' event. this.sourceBuffers_[contentType].appendBuffer(data); } @@ -1267,6 +1286,14 @@ shaka.media.MediaSourceEngine = class { return segment; } + + /** + * Update LCEVC DIL object when ready for LCEVC Decodes + * @param {?shaka.lcevc.Dil} lcevcDil + */ + updateLcevcDil(lcevcDil) { + this.lcevcDil_ = lcevcDil; + } }; diff --git a/lib/player.js b/lib/player.js index c9a8eeebfc..62c0d4725f 100644 --- a/lib/player.js +++ b/lib/player.js @@ -57,6 +57,7 @@ goog.require('shaka.util.PublicPromise'); goog.require('shaka.util.Stats'); goog.require('shaka.util.StreamUtils'); goog.require('shaka.util.Timer'); +goog.require('shaka.lcevc.Dil'); goog.requireType('shaka.media.IClosedCaptionParser'); goog.requireType('shaka.media.PresentationTimeline'); goog.requireType('shaka.routing.Node'); @@ -522,6 +523,15 @@ shaka.Player = class extends shaka.util.FakeEventTarget { /** @private {shaka.util.CmcdManager} */ this.cmcdManager_ = null; + // This is the canvas element that will be used for rendering LCEVC + // enhanced frames. + /** @private {?HTMLCanvasElement} */ + this.lcevcCanvas_ = null; + + // This is the LCEVC Dil object to decode LCEVC. + /** @private {?shaka.lcevc.Dil} */ + this.lcevcDil_ = null; + /** @private {shaka.media.QualityObserver} */ this.qualityObserver_ = null; @@ -743,6 +753,62 @@ shaka.Player = class extends shaka.util.FakeEventTarget { } } + /** + * Create a shaka.lcevc.Dil object + * @param {shaka.extern.LcevcConfiguration} config + * @private + */ + createLcevcDil_(config) { + if (this.lcevcDil_ == null) { + this.lcevcDil_ = new shaka.lcevc.Dil( + /** @type {HTMLVideoElement} */ (this.video_), + this.lcevcCanvas_, + config, + ); + if (this.mediaSourceEngine_) { + this.mediaSourceEngine_.updateLcevcDil(this.lcevcDil_); + } + } + } + + /** + * Close a shaka.lcevc.Dil object if present and hide the canvas. + * @private + */ + closeLcevcDil_() { + if (this.lcevcDil_ != null) { + this.lcevcDil_.release(); + this.lcevcDil_ = null; + } + } + + /** + * Setup shaka.lcevc.Dil object + * @param {?shaka.extern.PlayerConfiguration} config + * @private + */ + setupLcevc_(config) { + const tracks = this.getVariantTracks(); + if (tracks && tracks[0] && + tracks[0].videoMimeType == + shaka.Player.SRC_EQUAL_EXTENSIONS_TO_MIME_TYPES_['ts']) { + const edge = shaka.util.Platform.isEdge() || + shaka.util.Platform.isLegacyEdge(); + if (edge) { + if (!config.streaming.forceTransmuxTS) { + // If forceTransmux is disabled for Microsoft Edge, LCEVC data + // is stripped out in case of a MPEG-2 TS container. + // Hence the warning for Microsoft Edge when playing content with + // MPEG-2 TS container. + shaka.log.alwaysWarn('LCEVC Warning: For MPEG-2 TS decoding '+ + 'the config.streaming.forceTransmux must be enabled.'); + } + } + } + this.closeLcevcDil_(); + this.createLcevcDil_(config.lcevc); + } + /** * @param {!shaka.util.FakeEvent.EventName} name * @param {Map.=} data @@ -765,6 +831,9 @@ shaka.Player = class extends shaka.util.FakeEventTarget { return; } + // If LCEVC Dil exists close it. + this.closeLcevcDil_(); + // Mark as "dead". This should stop external-facing calls from changing our // internal state any more. This will stop calls to |attach|, |detach|, etc. // from interrupting our final move to the detached state. @@ -1002,6 +1071,18 @@ shaka.Player = class extends shaka.util.FakeEventTarget { return this.wrapWalkerListenersWithPromise_(events); } + + /** + * Calling attachCanvas will tell the player to set canvas + * element for LCEVC decoding. + * + * @param {HTMLCanvasElement} canvas + * @export + */ + attachCanvas(canvas) { + this.lcevcCanvas_ = canvas; + } + /** * Tell the player to stop using its current media element. If the player is: *
    @@ -1073,6 +1154,9 @@ shaka.Player = class extends shaka.util.FakeEventTarget { initializeMediaSource = false; } + // If LCEVC Dil exists close it. + this.closeLcevcDil_(); + // Since we are going either to attached or detached (through unloaded), we // can't allow it to be interrupted or else we could lose track of what // media element we are suppose to use. @@ -1611,7 +1695,8 @@ shaka.Player = class extends shaka.util.FakeEventTarget { textDisplayer, (metadata, offset, endTime) => { this.processTimedMetadataMediaSrc_(metadata, offset, endTime); - }); + }, + this.lcevcDil_); mediaSourceEngine.configure(this.config_.mediaSource); const {segmentRelativeVttTiming} = this.config_.manifest; mediaSourceEngine.setSegmentRelativeVttTiming(segmentRelativeVttTiming); @@ -1883,6 +1968,15 @@ shaka.Player = class extends shaka.util.FakeEventTarget { const now = Date.now() / 1000; const delta = now - startTime; this.stats_.setDrmTime(delta); + // LCEVC data by itself is not encrypted in DRM protected streams and + // can therefore be accessed and decoded as normal. However, the LCEVC + // decoder needs access to the VideoElement output in order to apply + // the enhancement. In DRM contexts where the browser CDM restricts + // access from our decoder, the enhancement cannot be applied and + // therefore the LCEVC output canvas is hidden accordingly. + if (this.lcevcDil_) { + this.lcevcDil_.hideCanvas(); + } } }, }); @@ -1967,6 +2061,10 @@ shaka.Player = class extends shaka.util.FakeEventTarget { this.loadEventManager_.listen(mediaElement, 'ended', updateStateHistory); this.loadEventManager_.listen(mediaElement, 'ratechange', onRateChange); + // Check status of the LCEVC DIL Object and reset or + // create or close based on config + this.setupLcevc_(this.config_); + const abrFactory = this.config_.abrFactory; if (!this.abrManager_ || this.abrManagerFactory_ != abrFactory) { this.abrManagerFactory_ = abrFactory; @@ -2915,13 +3013,18 @@ shaka.Player = class extends shaka.util.FakeEventTarget { * @param {!shaka.extern.TextDisplayer} textDisplayer * @param {!function(!Array., number, ?number)} * onMetadata + * @param {shaka.lcevc.Dil} lcevcDil * * @return {!shaka.media.MediaSourceEngine} */ createMediaSourceEngine(mediaElement, closedCaptionsParser, textDisplayer, - onMetadata) { + onMetadata, lcevcDil) { return new shaka.media.MediaSourceEngine( - mediaElement, closedCaptionsParser, textDisplayer, onMetadata); + mediaElement, + closedCaptionsParser, + textDisplayer, + onMetadata, + lcevcDil); } /** @@ -5239,7 +5342,6 @@ shaka.Player = class extends shaka.util.FakeEventTarget { } } } - this.checkRestrictedVariants_(manifest); } @@ -5749,6 +5851,9 @@ shaka.Player = class extends shaka.util.FakeEventTarget { const data = new Map() .set('oldTrack', from) .set('newTrack', to); + if (this.lcevcDil_) { + this.lcevcDil_.updateVariant(to); + } const event = this.makeEvent_(shaka.util.FakeEvent.EventName.Adaptation, data); this.delayDispatchEvent_(event); @@ -5777,6 +5882,10 @@ shaka.Player = class extends shaka.util.FakeEventTarget { const data = new Map() .set('oldTrack', from) .set('newTrack', to); + if (this.lcevcDil_) { + this.lcevcDil_.updateVariant(to); + } + const event = this.makeEvent_(shaka.util.FakeEvent.EventName.VariantChanged, data); this.delayDispatchEvent_(event); diff --git a/lib/util/dom_utils.js b/lib/util/dom_utils.js index 17b78752a0..1018a5f902 100644 --- a/lib/util/dom_utils.js +++ b/lib/util/dom_utils.js @@ -78,7 +78,6 @@ shaka.util.Dom = class { return shaka.util.Dom.asHTMLElement(elements[0]); } - /** * Remove all of the child nodes of an element. * @param {!Element} element diff --git a/lib/util/player_configuration.js b/lib/util/player_configuration.js index fcffb12d59..e6246ff1c2 100644 --- a/lib/util/player_configuration.js +++ b/lib/util/player_configuration.js @@ -258,6 +258,12 @@ shaka.util.PlayerConfiguration = class { useHeaders: false, }; + const lcevc = { + dynamicPerformanceScaling: true, + logLevel: 0, + drawLogo: false, + }; + const mediaSource = { sourceBufferExtraFeatures: '', }; @@ -299,6 +305,7 @@ shaka.util.PlayerConfiguration = class { playRangeEnd: Infinity, textDisplayFactory: () => null, cmcd: cmcd, + lcevc: lcevc, }; // Add this callback so that we can reference the preferred audio language diff --git a/package-lock.json b/package-lock.json index 7af0edf2cd..b49564ac35 100644 --- a/package-lock.json +++ b/package-lock.json @@ -47,6 +47,7 @@ "karma-sourcemap-loader": "^0.3.8", "karma-spec-reporter": "^0.0.34", "karma-webdriver-launcher": "^1.0.8", + "lcevc_dil.js": "^1.1.20", "less": "https://gitpkg.now.sh/joeyparrish/less.js/packages/less?28c63a43", "less-plugin-clean-css": "github:austingardner/less-plugin-clean-css#4e9e77bf", "material-design-lite": "^1.3.0", @@ -4444,6 +4445,23 @@ "omggif": "^1.0.10" } }, + "node_modules/git-rev-sync": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/git-rev-sync/-/git-rev-sync-3.0.2.tgz", + "integrity": "sha512-Nd5RiYpyncjLv0j6IONy0lGzAqdRXUaBctuGBbrEA2m6Bn4iDrN/9MeQTXuiquw8AEKL9D2BW0nw5m/lQvxqnQ==", + "dev": true, + "dependencies": { + "escape-string-regexp": "1.0.5", + "graceful-fs": "4.1.15", + "shelljs": "0.8.5" + } + }, + "node_modules/git-rev-sync/node_modules/graceful-fs": { + "version": "4.1.15", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.15.tgz", + "integrity": "sha512-6uHUhOPEBgQ24HM+r6b/QwWfZq+yiFcipKFrOFiBEnWdy5sdzYoi+pJeQaPI5qOLRFqWmAXUPQNsielzdLoecA==", + "dev": true + }, "node_modules/glob": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.0.tgz", @@ -4979,6 +4997,15 @@ "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", "dev": true }, + "node_modules/interpret": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/interpret/-/interpret-1.4.0.tgz", + "integrity": "sha512-agE4QfB2Lkp9uICn7BAqoscw4SZP9kTE2hxiFI3jBPmXJfdqiahTbUuKGsMoN2GtqL9AxhYioAcVvgsb1HvRbA==", + "dev": true, + "engines": { + "node": ">= 0.10" + } + }, "node_modules/is-arrayish": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", @@ -5648,6 +5675,15 @@ "safe-buffer": "~5.1.0" } }, + "node_modules/lcevc_dil.js": { + "version": "1.1.20", + "resolved": "https://registry.npmjs.org/lcevc_dil.js/-/lcevc_dil.js-1.1.20.tgz", + "integrity": "sha512-SAAWfAkfVIhyYm1Jgq9EuPWXT9Gw+/kjCdtEEa9cWah01n3K3DpVxqdruitt6ZSu2uurmNPU2wPfqmfFLF97Og==", + "dev": true, + "dependencies": { + "git-rev-sync": "^3.0.1" + } + }, "node_modules/less": { "version": "4.1.2", "resolved": "https://gitpkg.now.sh/joeyparrish/less.js/packages/less?28c63a43", @@ -6989,6 +7025,18 @@ "node": ">=8.10.0" } }, + "node_modules/rechoir": { + "version": "0.6.2", + "resolved": "https://registry.npmjs.org/rechoir/-/rechoir-0.6.2.tgz", + "integrity": "sha1-hSBLVNuoLVdC4oyWdW70OvUOM4Q=", + "dev": true, + "dependencies": { + "resolve": "^1.1.6" + }, + "engines": { + "node": ">= 0.10" + } + }, "node_modules/redent": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/redent/-/redent-3.0.0.tgz", @@ -7311,6 +7359,23 @@ "node": ">=8" } }, + "node_modules/shelljs": { + "version": "0.8.5", + "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.8.5.tgz", + "integrity": "sha512-TiwcRcrkhHvbrZbnRcFYMLl30Dfov3HKqzp5tO5b4pt6G/SezKcYhmDg15zXVBswHmctSAQKznqNW2LO5tTDow==", + "dev": true, + "dependencies": { + "glob": "^7.0.0", + "interpret": "^1.0.0", + "rechoir": "^0.6.2" + }, + "bin": { + "shjs": "bin/shjs" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/signal-exit": { "version": "3.0.7", "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", @@ -11870,6 +11935,25 @@ "omggif": "^1.0.10" } }, + "git-rev-sync": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/git-rev-sync/-/git-rev-sync-3.0.2.tgz", + "integrity": "sha512-Nd5RiYpyncjLv0j6IONy0lGzAqdRXUaBctuGBbrEA2m6Bn4iDrN/9MeQTXuiquw8AEKL9D2BW0nw5m/lQvxqnQ==", + "dev": true, + "requires": { + "escape-string-regexp": "1.0.5", + "graceful-fs": "4.1.15", + "shelljs": "0.8.5" + }, + "dependencies": { + "graceful-fs": { + "version": "4.1.15", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.15.tgz", + "integrity": "sha512-6uHUhOPEBgQ24HM+r6b/QwWfZq+yiFcipKFrOFiBEnWdy5sdzYoi+pJeQaPI5qOLRFqWmAXUPQNsielzdLoecA==", + "dev": true + } + } + }, "glob": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.0.tgz", @@ -12263,6 +12347,12 @@ "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", "dev": true }, + "interpret": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/interpret/-/interpret-1.4.0.tgz", + "integrity": "sha512-agE4QfB2Lkp9uICn7BAqoscw4SZP9kTE2hxiFI3jBPmXJfdqiahTbUuKGsMoN2GtqL9AxhYioAcVvgsb1HvRbA==", + "dev": true + }, "is-arrayish": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", @@ -12805,6 +12895,15 @@ } } }, + "lcevc_dil.js": { + "version": "1.1.20", + "resolved": "https://registry.npmjs.org/lcevc_dil.js/-/lcevc_dil.js-1.1.20.tgz", + "integrity": "sha512-SAAWfAkfVIhyYm1Jgq9EuPWXT9Gw+/kjCdtEEa9cWah01n3K3DpVxqdruitt6ZSu2uurmNPU2wPfqmfFLF97Og==", + "dev": true, + "requires": { + "git-rev-sync": "^3.0.1" + } + }, "less": { "version": "https://gitpkg.now.sh/joeyparrish/less.js/packages/less?28c63a43", "integrity": "sha512-i5qb64PowTQ7eCLt7rlRcsNwFK9M5htl1ioP3Y/7WdU5NNpjpE8JgbUInlA97w4Vqqcd7PdCTgSBPLrs+Yvxag==", @@ -13803,6 +13902,15 @@ "picomatch": "^2.2.1" } }, + "rechoir": { + "version": "0.6.2", + "resolved": "https://registry.npmjs.org/rechoir/-/rechoir-0.6.2.tgz", + "integrity": "sha1-hSBLVNuoLVdC4oyWdW70OvUOM4Q=", + "dev": true, + "requires": { + "resolve": "^1.1.6" + } + }, "redent": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/redent/-/redent-3.0.0.tgz", @@ -14053,6 +14161,17 @@ "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", "dev": true }, + "shelljs": { + "version": "0.8.5", + "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.8.5.tgz", + "integrity": "sha512-TiwcRcrkhHvbrZbnRcFYMLl30Dfov3HKqzp5tO5b4pt6G/SezKcYhmDg15zXVBswHmctSAQKznqNW2LO5tTDow==", + "dev": true, + "requires": { + "glob": "^7.0.0", + "interpret": "^1.0.0", + "rechoir": "^0.6.2" + } + }, "signal-exit": { "version": "3.0.7", "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", diff --git a/package.json b/package.json index a5a6118ec4..b8e27a0dff 100644 --- a/package.json +++ b/package.json @@ -46,6 +46,7 @@ "karma-sourcemap-loader": "^0.3.8", "karma-spec-reporter": "^0.0.34", "karma-webdriver-launcher": "^1.0.8", + "lcevc_dil.js": "^1.1.20", "less": "https://gitpkg.now.sh/joeyparrish/less.js/packages/less?28c63a43", "less-plugin-clean-css": "github:austingardner/less-plugin-clean-css#4e9e77bf", "material-design-lite": "^1.3.0", diff --git a/test/player_unit.js b/test/player_unit.js index 5007a645c3..ba48ff5f5a 100644 --- a/test/player_unit.js +++ b/test/player_unit.js @@ -131,6 +131,8 @@ describe('Player', () => { getUseEmbeddedText: jasmine.createSpy('getUseEmbeddedText'), setSegmentRelativeVttTiming: jasmine.createSpy('setSegmentRelativeVttTiming'), + updateLcevcDil: + jasmine.createSpy('updateLcevcDil'), getTextDisplayer: () => textDisplayer, getBufferedInfo: () => bufferedInfo, ended: jasmine.createSpy('ended').and.returnValue(false), diff --git a/test/test/util/fake_media_source_engine.js b/test/test/util/fake_media_source_engine.js index 360d398308..40224f0943 100644 --- a/test/test/util/fake_media_source_engine.js +++ b/test/test/util/fake_media_source_engine.js @@ -132,6 +132,10 @@ shaka.test.FakeMediaSourceEngine = class { /** @type {!jasmine.Spy} */ this.setSegmentRelativeVttTiming = jasmine.createSpy('setSegmentRelativeVttTiming').and.stub(); + + /** @type {!jasmine.Spy} */ + this.updateLcevcDil = + jasmine.createSpy('updateLcevcDil').and.stub(); } /** @override */ diff --git a/ui/less/containers.less b/ui/less/containers.less index 2a554c9f54..6dbc6aa41b 100644 --- a/ui/less/containers.less +++ b/ui/less/containers.less @@ -98,6 +98,13 @@ } } +/* A container for all canvas for LCEVC decoding + * Sits inside .shaka-video-container, on top of (Z axis) .shaka-video, and + * below (Y axis) .shaka-play-button-container. */ +.shaka-canvas-container { + .overlay-child(); +} + /* Container for controls positioned at the bottom of the video container: * controls button panel and the seek bar. */ .shaka-bottom-controls { diff --git a/ui/ui.js b/ui/ui.js index 5e721cc680..06ba273dd9 100644 --- a/ui/ui.js +++ b/ui/ui.js @@ -290,6 +290,12 @@ shaka.ui.Overlay = class { const videos = document.querySelectorAll( '[data-shaka-player]'); + // Look for elements marked 'data-shaka-player-canvas' + // on the page. These will be used to create our default + // UI. + const canvases = document.querySelectorAll( + '[data-shaka-player-canvas]'); + if (!videos.length && !containers.length) { // No elements have been tagged with shaka attributes. } else if (videos.length && !containers.length) { @@ -307,8 +313,21 @@ shaka.ui.Overlay = class { const videoParent = video.parentElement; videoParent.replaceChild(container, video); container.appendChild(video); - - shaka.ui.Overlay.setupUIandAutoLoad_(container, video); + let currentCanvas = null; + for (const canvas of canvases) { + goog.asserts.assert(canvas.tagName.toLowerCase() == 'canvas', + 'Should be a canvas element!'); + if (canvas.parentElement == container) { + currentCanvas = canvas; + break; + } + } + if (!currentCanvas) { + currentCanvas = document.createElement('canvas'); + currentCanvas.classList.add('shaka-canvas-container'); + container.appendChild(currentCanvas); + } + shaka.ui.Overlay.setupUIandAutoLoad_(container, video, currentCanvas); } } else { for (const container of containers) { @@ -336,9 +355,25 @@ shaka.ui.Overlay = class { container.appendChild(currentVideo); } + let currentCanvas = null; + for (const canvas of canvases) { + goog.asserts.assert(canvas.tagName.toLowerCase() == 'canvas', + 'Should be a canvas element!'); + if (canvas.parentElement == container) { + currentCanvas = canvas; + break; + } + } + if (!currentCanvas) { + currentCanvas = document.createElement('canvas'); + currentCanvas.classList.add('shaka-canvas-container'); + container.appendChild(currentCanvas); + } + try { // eslint-disable-next-line no-await-in-loop - await shaka.ui.Overlay.setupUIandAutoLoad_(container, currentVideo); + await shaka.ui.Overlay.setupUIandAutoLoad_( + container, currentVideo, currentCanvas); } catch (e) { // This can fail if, for example, not every player file has loaded. // Ad-block is a likely cause for this sort of failure. @@ -377,9 +412,10 @@ shaka.ui.Overlay = class { /** * @param {!Element} container * @param {!Element} video + * @param {!Element} canvas * @private */ - static async setupUIandAutoLoad_(container, video) { + static async setupUIandAutoLoad_(container, video, canvas) { // Create the UI const player = new shaka.Player( shaka.util.Dom.asHTMLMediaElement(video)); @@ -387,6 +423,9 @@ shaka.ui.Overlay = class { shaka.util.Dom.asHTMLElement(container), shaka.util.Dom.asHTMLMediaElement(video)); + // Attach Canvas used for LCEVC Decoding + player.attachCanvas(/** @type {HTMLCanvasElement} */(canvas)); + // Get and configure cast app id. let castAppId = '';