Add support to HLS Image Media Playlists

demo/common/message_ids.js

@@ -173,6 +173,7 @@ shakaDemo.MessageIds = {
   IGNORE_DASH_DRM: 'DEMO_IGNORE_DASH_DRM',
   IGNORE_DASH_MAX_SEGMENT_DURATION: 'DEMO_IGNORE_DASH_MAX_SEGMENT_DURATION',
   IGNORE_DASH_SUGGESTED_PRESENTATION_DELAY: 'DEMO_IGNORE_DASH_SUGGESTED_PRESENTATION_DELAY',
+  IGNORE_HLS_IMAGE_FAILURES: 'DEMO_IGNORE_HLS_IMAGE_FAILURES',
   IGNORE_HLS_TEXT_FAILURES: 'DEMO_IGNORE_HLS_TEXT_FAILURES',
   USE_FULL_SEGMENTS_FOR_START_TIME: 'DEMO_USE_FULL_SEGMENTS_FOR_START_TIME',
   IGNORE_MIN_BUFFER_TIME: 'DEMO_IGNORE_MIN_BUFFER_TIME',

demo/config.js

@@ -206,6 +206,8 @@ shakaDemo.Config = class {
             'manifest.dash.ignoreMaxSegmentDuration')
         .addBoolInput_(MessageIds.IGNORE_HLS_TEXT_FAILURES,
             'manifest.hls.ignoreTextStreamFailures')
+        .addBoolInput_(MessageIds.IGNORE_HLS_IMAGE_FAILURES,
+            'manifest.hls.ignoreImageStreamFailures')
         .addBoolInput_(MessageIds.USE_FULL_SEGMENTS_FOR_START_TIME,
             'manifest.hls.useFullSegmentsForStartTime')
         .addNumberInput_(MessageIds.AVAILABILITY_WINDOW_OVERRIDE,

demo/locales/en.json

@@ -85,6 +85,7 @@
   "DEMO_IGNORE_DASH_EMPTY_ADAPTATION_SET": "Ignore empty DASH AdaptationSets",
   "DEMO_IGNORE_DASH_MAX_SEGMENT_DURATION": "Ignore DASH maxSegmentDuration",
   "DEMO_IGNORE_DASH_SUGGESTED_PRESENTATION_DELAY": "Ignore DASH suggestedPresentationDelay",
+  "DEMO_IGNORE_HLS_IMAGE_FAILURES": "Ignore HLS Image Stream Failures",
   "DEMO_IGNORE_HLS_TEXT_FAILURES": "Ignore HLS Text Stream Failures",
   "DEMO_IMA_ASSET_KEY": "Asset key (for LIVE DAI Content)",
   "DEMO_IMA_CONTENT_SRC_ID": "Content source ID (for VOD DAI Content)",

demo/locales/source.json

@@ -347,6 +347,10 @@
     "description": "The name of a configuration value.",
     "message": "Ignore [PROPER_NAME:HLS] Text Stream Failures"
   },
+  "DEMO_IGNORE_HLS_IMAGE_FAILURES": {
+    "description": "The name of a configuration value.",
+    "message": "Ignore [PROPER_NAME:HLS] Image Stream Failures"
+  },
   "DEMO_IMA_ASSET_KEY": {
     "description": "The label on a field that allows users to provide an asset key for a custom asset.",
     "message": "Asset key (for LIVE DAI Content)"

externs/shaka/manifest.js

@@ -328,7 +328,7 @@ shaka.extern.CreateSegmentIndexFunction;
  *   The Stream's label, unique text that should describe the audio/text track.
  * @property {string} type
  *   <i>Required.</i> <br>
- *   Content type (e.g. 'video', 'audio' or 'text')
+ *   Content type (e.g. 'video', 'audio' or 'text', 'image')
  * @property {boolean} primary
  *   <i>Defaults to false.</i> <br>
  *   True indicates that the player should use this Stream over others if user

externs/shaka/player.js

@@ -683,12 +683,16 @@ shaka.extern.DashManifestConfiguration;
 /**
  * @typedef {{
  *   ignoreTextStreamFailures: boolean,
+ *   ignoreImageStreamFailures: boolean,
  *   useFullSegmentsForStartTime: boolean
  * }}
  *
  * @property {boolean} ignoreTextStreamFailures
  *   If <code>true</code>, ignore any errors in a text stream and filter out
  *   those streams.
+ * @property {boolean} ignoreImageStreamFailures
+ *   If <code>true</code>, ignore any errors in a image stream and filter out
+ *   those streams.
  * @property {boolean} useFullSegmentsForStartTime
  *   If <code>true</code>, force HlsParser to use a full segment request for
  *   determining start time in case the server does not support partial requests

lib/hls/hls_parser.js

@@ -92,7 +92,8 @@ shaka.hls.HlsParser = class {
      * timestamps, offsets, and to handle TS rollover.
      *
      * During parsing, used to avoid duplicates in the async methods
-     * createStreamInfoFromMediaTag_ and createStreamInfoFromVariantTag_.
+     * createStreamInfoFromMediaTag_, createStreamInfoFromImageTag_ and
+     * createStreamInfoFromVariantTag_.
      *
      * During parsing of updates, used by getStartTime_ to determine the start
      * time of the first segment from existing segment references.
@@ -403,6 +404,9 @@ shaka.hls.HlsParser = class {
     /** @type {!Array.<!shaka.hls.Tag>} */
     const variantTags = Utils.filterTagsByName(
         playlist.tags, 'EXT-X-STREAM-INF');
+    /** @type {!Array.<!shaka.hls.Tag>} */
+    const imageTags = Utils.filterTagsByName(
+        playlist.tags, 'EXT-X-IMAGE-STREAM-INF');
 
     this.parseCodecs_(variantTags);
 
@@ -439,6 +443,7 @@ shaka.hls.HlsParser = class {
     this.parseClosedCaptions_(mediaTags);
     const variants = await this.createVariantsForTags_(variantTags);
     const textStreams = await this.parseTexts_(mediaTags);
+    const imageStreams = await this.parseImages_(imageTags);
 
     // Make sure that the parser has not been destroyed.
     if (!this.playerInterface_) {
@@ -533,7 +538,7 @@ shaka.hls.HlsParser = class {
       presentationTimeline: this.presentationTimeline_,
       variants,
       textStreams,
-      imageStreams: [],
+      imageStreams,
       offlineSessionIds: [],
       minBufferTime: 0,
     };
@@ -666,6 +671,34 @@ shaka.hls.HlsParser = class {
     return textStreams.filter((s) => s);
   }
 
+  /**
+   * @param {!Array.<!shaka.hls.Tag>} imageTags from the playlist.
+   * @return {!Promise.<!Array.<!shaka.extern.Stream>>}
+   * @private
+   */
+  async parseImages_(imageTags) {
+    // Create image stream for each image tag.
+    const imageStreamPromises = imageTags.map(async (tag) => {
+      const disableThumbnails = this.config_.disableThumbnails;
+      if (disableThumbnails) {
+        return null;
+      }
+      try {
+        const streamInfo = await this.createStreamInfoFromImageTag_(tag);
+        goog.asserts.assert(
+            streamInfo, 'Should always have a streamInfo for image');
+        return streamInfo.stream;
+      } catch (e) {
+        if (this.config_.hls.ignoreImageStreamFailures) {
+          return null;
+        }
+        throw e;
+      }
+    });
+    const imageStreams = await Promise.all(imageStreamPromises);
+    return imageStreams;
+  }
+
   /**
    * @param {!Array.<!shaka.hls.Tag>} mediaTags Media tags from the playlist.
    * @private
@@ -1183,6 +1216,51 @@ shaka.hls.HlsParser = class {
     return streamInfo;
   }
 
+  /**
+   * Parse EXT-X-MEDIA media tag into a Stream object.
+   *
+   * @param {shaka.hls.Tag} tag
+   * @return {!Promise.<?shaka.hls.HlsParser.StreamInfo>}
+   * @private
+   */
+  async createStreamInfoFromImageTag_(tag) {
+    goog.asserts.assert(tag.name == 'EXT-X-IMAGE-STREAM-INF',
+        'Should only be called on image tags!');
+    /** @type {string} */
+    const type = shaka.util.ManifestParserUtils.ContentType.IMAGE;
+
+    const verbatimImagePlaylistUri = this.variableSubstitution_(
+        tag.getRequiredAttrValue('URI'), this.globalVariables_);
+    const codecs = tag.getAttributeValue('CODECS', 'jpeg') || '';
+
+    // Check if the stream has already been created as part of another Variant
+    // and return it if it has.
+    if (this.uriToStreamInfosMap_.has(verbatimImagePlaylistUri)) {
+      return this.uriToStreamInfosMap_.get(verbatimImagePlaylistUri);
+    }
+
+    const language = this.getLanguage_(tag);
+    const name = tag.getAttributeValue('NAME');
+
+    const characteristics = tag.getAttributeValue('CHARACTERISTICS');
+
+    const streamInfo = await this.createStreamInfo_(
+        verbatimImagePlaylistUri, codecs, type, language, /* primary= */ false,
+        name, /* channelsCount= */ null, /* closedCaptions= */ null,
+        characteristics, /* forced= */ false, /* spatialAudio= */ false);
+    if (streamInfo == null) {
+      return null;
+    }
+
+    // TODO: This check is necessary because of the possibility of multiple
+    // calls to createStreamInfoFromImageTag_ before either has resolved.
+    if (this.uriToStreamInfosMap_.has(verbatimImagePlaylistUri)) {
+      return this.uriToStreamInfosMap_.get(verbatimImagePlaylistUri);
+    }
+    this.uriToStreamInfosMap_.set(verbatimImagePlaylistUri, streamInfo);
+    return streamInfo;
+  }
+
   /**
    * Parse an EXT-X-STREAM-INF media tag into a Stream object.
    *
@@ -1255,7 +1333,8 @@ shaka.hls.HlsParser = class {
         response.data, absoluteMediaPlaylistUri);
 
     if (playlist.type != shaka.hls.PlaylistType.MEDIA) {
-      // EXT-X-MEDIA tags should point to media playlists.
+      // EXT-X-MEDIA and EXT-X-IMAGE-STREAM-INF tags should point to media
+      // playlists.
       throw new shaka.util.Error(
           shaka.util.Error.Severity.CRITICAL,
           shaka.util.Error.Category.MANIFEST,
@@ -1618,17 +1697,21 @@ shaka.hls.HlsParser = class {
    * @param {!Map.<string, string>} variables
    * @param {string} absoluteMediaPlaylistUri
    * @return {!shaka.media.SegmentReference}
+   * @param {string} type
    * @private
    */
   createSegmentReference_(
       initSegmentReference, previousReference, hlsSegment, startTime,
-      timestampOffset, variables, absoluteMediaPlaylistUri) {
+      timestampOffset, variables, absoluteMediaPlaylistUri, type) {
     const tags = hlsSegment.tags;
     const absoluteSegmentUri = this.variableSubstitution_(
         hlsSegment.absoluteUri, variables);
     const extinfTag =
         shaka.hls.Utils.getFirstTagWithName(tags, 'EXTINF');
 
+    const tilesTag =
+        shaka.hls.Utils.getFirstTagWithName(tags, 'EXT-X-TILES');
+
     let endTime = 0;
     let startByte = 0;
     let endByte = null;
@@ -1713,6 +1796,15 @@ shaka.hls.HlsParser = class {
       endByte = partialSegmentRefs[partialSegmentRefs.length - 1].endByte;
     }
 
+    let tilesLayout = '';
+    if (type == shaka.util.ManifestParserUtils.ContentType.IMAGE) {
+      // By default in HLS the tilesLayout is 1x1
+      tilesLayout = '1x1';
+      if (tilesTag) {
+        tilesLayout = tilesTag.getAttributeValue('LAYOUT', '1x1');
+      }
+    }
+
     return new shaka.media.SegmentReference(
         startTime,
         endTime,
@@ -1724,6 +1816,7 @@ shaka.hls.HlsParser = class {
         /* appendWindowStart= */ 0,
         /* appendWindowEnd= */ Infinity,
         partialSegmentRefs,
+        tilesLayout,
     );
   }
 
@@ -1819,12 +1912,13 @@ shaka.hls.HlsParser = class {
         initSegmentRef = this.getInitSegmentReference_(
             playlist.absoluteUri, hlsSegments[0].tags, variables);
         goog.asserts.assert(
-            type != shaka.util.ManifestParserUtils.ContentType.TEXT,
+            type != shaka.util.ManifestParserUtils.ContentType.TEXT &&
+            type != shaka.util.ManifestParserUtils.ContentType.IMAGE,
             'Should only get start time from audio or video streams');
         this.playlistStartTime_ = await this.getStartTime_(
             verbatimMediaPlaylistUri, initSegmentRef, mimeType,
             position, /* isDiscontinuity= */ false,
-            hlsSegments[0], variables);
+            hlsSegments[0], variables, type);
       }
       firstStartTime = this.playlistStartTime_;
     }
@@ -1877,7 +1971,7 @@ shaka.hls.HlsParser = class {
         // eslint-disable-next-line no-await-in-loop
         timestampOffset = await this.getTimestampOffset_(
             discontintuitySequenceNum, verbatimMediaPlaylistUri, initSegmentRef,
-            mimeType, position, item, variables, startTime);
+            mimeType, position, item, variables, startTime, type);
       }
 
       // If the stream is low latency and the user has not configured the
@@ -1902,7 +1996,8 @@ shaka.hls.HlsParser = class {
             startTime,
             timestampOffset,
             variables,
-            playlist.absoluteUri);
+            playlist.absoluteUri,
+            type);
 
         references.push(reference);
       } else if (!this.lowLatencyMode_) {
@@ -1932,13 +2027,14 @@ shaka.hls.HlsParser = class {
    * @param {!shaka.hls.Segment} segment
    * @param {!Map.<string, string>} variables
    * @param {number} startTime
+   * @param {string} type
    * @return {!Promise.<number>}
    * @throws {shaka.util.Error}
    * @private
    */
   async getTimestampOffset_(discontintuitySequenceNum,
       verbatimMediaPlaylistUri, initSegmentRef,
-      mimeType, mediaSequenceNumber, segment, variables, startTime) {
+      mimeType, mediaSequenceNumber, segment, variables, startTime, type) {
     let timestampOffset = 0;
     if (this.discontinuityToTso_.has(discontintuitySequenceNum)) {
       timestampOffset =
@@ -1947,7 +2043,7 @@ shaka.hls.HlsParser = class {
       const mediaStartTime = await this.getStartTime_(
           verbatimMediaPlaylistUri, initSegmentRef, mimeType,
           mediaSequenceNumber, /* isDiscontinuity= */ true, segment,
-          variables);
+          variables, type);
       timestampOffset = startTime - mediaStartTime;
       shaka.log.v1('Segment timestampOffset =', timestampOffset);
       this.discontinuityToTso_.set(
@@ -2034,20 +2130,22 @@ shaka.hls.HlsParser = class {
    * @param {boolean} isDiscontinuity
    * @param {!shaka.hls.Segment} segment
    * @param {!Map.<string, string>} variables
+   * @param {string} type
    * @return {!Promise.<number>}
    * @private
    */
   async getStartTime_(
       verbatimMediaPlaylistUri, initSegmentRef, mimeType, mediaSequenceNumber,
-      isDiscontinuity, segment, variables) {
+      isDiscontinuity, segment, variables, type) {
     const segmentRef = this.createSegmentReference_(
         initSegmentRef,
         /* previousReference= */ null,
         segment,
         /* startTime= */ 0,
         /* timestampOffset= */ 0,
         variables,
-        /* absoluteMediaPlaylistUri= */ '');
+        /* absoluteMediaPlaylistUri= */ '',
+        type);
     // If we are updating the manifest, we can usually skip fetching the segment
     // by examining the references we already have.  This won't be possible if
     // there was some kind of lag or delay updating the manifest on the server,
@@ -2409,6 +2507,12 @@ shaka.hls.HlsParser = class {
       }
     }
 
+    if (contentType == ContentType.IMAGE) {
+      if (!codecs || codecs == 'jpeg') {
+        return 'image/jpeg';
+      }
+    }
+
     // If unable to guess mime type, request a segment and try getting it
     // from the response.
     const headRequest = shaka.net.NetworkingEngine.makeRequest(

lib/media/segment_reference.js

@@ -135,12 +135,16 @@ shaka.media.SegmentReference = class {
    *   presentation.  Any content from after this time will be removed by
    *   MediaSource.
    * @param {!Array.<!shaka.media.SegmentReference>=} partialReferences
-       A list of SegmentReferences for the partial segments.
+   *   A list of SegmentReferences for the partial segments.
+   * @param {?string=} tilesLayout
+   *   The value is a grid-item-dimension consisting of two positive decimal
+   *   integers in the format: column-x-row ('4x3'). It describes the
+   *   arrangement of Images in a Grid. The minimum valid LAYOUT is '1x1'.
    */
   constructor(
       startTime, endTime, uris, startByte, endByte, initSegmentReference,
       timestampOffset, appendWindowStart, appendWindowEnd,
-      partialReferences = []) {
+      partialReferences = [], tilesLayout = '') {
     // A preload hinted Partial Segment has the same startTime and endTime.
     goog.asserts.assert(startTime <= endTime,
         'startTime must be less than or equal to endTime');
@@ -176,6 +180,9 @@ shaka.media.SegmentReference = class {
 
     /** @type {!Array.<!shaka.media.SegmentReference>} */
     this.partialReferences = partialReferences;
+
+    /** @type {?string} */
+    this.tilesLayout = tilesLayout;
   }
 
   /**
@@ -250,6 +257,16 @@ shaka.media.SegmentReference = class {
   hasPartialSegments() {
     return this.partialReferences.length > 0;
   }
+
+  /**
+   * Returns the segment's tiles layout. Only defined in image segments.
+   *
+   * @return {?string}
+   * @export
+   */
+  getTilesLayout() {
+    return this.tilesLayout;
+  }
 };
 
 

lib/player.js

@@ -3314,9 +3314,11 @@ shaka.Player = class extends shaka.util.FakeEventTarget {
         return null;
       }
       const reference = imageStream.segmentIndex.get(referencePosition);
+      const tilesLayout =
+          reference.getTilesLayout() || imageStream.tilesLayout;
       // This expression is used to detect one or more numbers (0-9) followed
       // by an x and after one or more numbers (0-9)
-      const match = /(\d+)x(\d+)/.exec(imageStream.tilesLayout);
+      const match = /(\d+)x(\d+)/.exec(tilesLayout);
       if (!match) {
         shaka.log.warning('Tiles layout does not contain a valid format ' +
             ' (columns x rows)');