Add closedCaptions field to stream

Dash and Hls manifests have tags and values to indicate that the content
may have CEA608/708 closed captions embedded in the video content.
Adding a closedCaptions field in Stream to represent that, and adding
parsing closed captions tag for DASH parser.

Reference:
HLS: https://tools.ietf.org/html/draft-pantos-http-live-streaming-23#section-4.3.4.2
DASH: https://dashif.org/wp-content/uploads/2018/04/DASH-IF-IOP-v4.2-clean.pdf

Issue: #1404.

Change-Id: I3c58b6043f7fe294dd642bdada8d2451caec9b55

externs/shaka/manifest.js

@@ -306,7 +306,8 @@ shaka.extern.GetSegmentReferenceFunction;
  *   trickModeVideo: ?shaka.extern.Stream,
  *   containsEmsgBoxes: boolean,
  *   roles: !Array.<string>,
- *   channelsCount: ?number
+ *   channelsCount: ?number,
+ *   closedCaptions: Object.<string, string>
  * }}
  *
  * @description
@@ -403,6 +404,12 @@ shaka.extern.GetSegmentReferenceFunction;
  *   e.g. 'main', 'caption', or 'commentary'.
  * @property {?number} channelsCount
  *   The channel count information for the audio stream.
+ * @property {Object.<string, string>} closedCaptions
+ *   A map containing the description of closed captions, with the caption
+ *   channel number (CC1 | CC2 | CC3 | CC4) as the key and the language code
+ *   as the value. If the channel number is not provided by the description,
+ *   we'll set an 0-based index as the key.
+ *   Example: {'CC1': 'eng'; 'CC3': 'swe'}, or {'1', 'eng'; '2': 'swe'}, etc.
  * @exportDoc
  */
 shaka.extern.Stream;

lib/dash/dash_parser.js

@@ -1006,6 +1006,35 @@ shaka.dash.DashParser.prototype.parseAdaptationSet_ = function(context, elem) {
     }
   });
 
+  const accessibilities = XmlUtils.findChildren(elem, 'Accessibility');
+  const LanguageUtils = shaka.util.LanguageUtils;
+  let closedCaptions = {};
+  let captionId = 0;
+  for (const prop of accessibilities) {
+    let schemeId = prop.getAttribute('schemeIdUri');
+    if (schemeId == 'urn:scte:dash:cc:cea-608:2015' ||
+        schemeId == 'urn:scte:dash:cc:cea-708:2015') {
+      let closedCaptionsValue = prop.getAttribute('value');
+      if (closedCaptionsValue != null) {
+        closedCaptionsValue.split(';').forEach((captionStr) => {
+          // Some closed caption strings have channel number and language, like
+          // "CC1=eng", some may have only language, like "eng".
+          // When the channel number is not provided, use a 0-based index as the
+          // key.
+          let channel;
+          let language;
+          if (captionStr.indexOf('=') == -1) {
+            channel = (captionId++).toString();
+            language = captionStr;
+          } else {
+            [channel, language] = captionStr.split('=');
+          }
+          closedCaptions[channel] = LanguageUtils.normalize(language);
+        });
+      }
+    }
+  }
+
   // According to DASH spec (2014) section 5.8.4.8, "the successful processing
   // of the descriptor is essential to properly use the information in the
   // parent element".  According to DASH IOP v3.3, section 3.3.4, "if the scheme
@@ -1031,7 +1060,7 @@ shaka.dash.DashParser.prototype.parseAdaptationSet_ = function(context, elem) {
   let representations = XmlUtils.findChildren(elem, 'Representation');
   let streams = representations
       .map(this.parseRepresentation_.bind(this, context, contentProtection,
-          kind, language, label, main, roleValues))
+          kind, language, label, main, roleValues, closedCaptions))
       .filter(function(s) { return !!s; });
 
   if (streams.length == 0) {
@@ -1100,14 +1129,16 @@ shaka.dash.DashParser.prototype.parseAdaptationSet_ = function(context, elem) {
  * @param {string} label
  * @param {boolean} isPrimary
  * @param {!Array.<string>} roles
+ * @param {Object.<string, string>} closedCaptions
  * @param {!Element} node
  * @return {?shaka.extern.Stream} The Stream, or null when there is a
  *   non-critical parsing error.
  * @throws shaka.util.Error When there is a parsing error.
  * @private
  */
 shaka.dash.DashParser.prototype.parseRepresentation_ = function(
-    context, contentProtection, kind, language, label, isPrimary, roles, node) {
+    context, contentProtection, kind, language, label, isPrimary, roles,
+    closedCaptions, node) {
   const XmlUtils = shaka.util.XmlUtils;
   const ContentType = shaka.util.ManifestParserUtils.ContentType;
 
@@ -1213,6 +1244,7 @@ shaka.dash.DashParser.prototype.parseRepresentation_ = function(
     containsEmsgBoxes: context.representation.containsEmsgBoxes,
     roles: roles,
     channelsCount: context.representation.numChannels,
+    closedCaptions: closedCaptions,
   };
 };
 

lib/hls/hls_parser.js

@@ -493,6 +493,9 @@ shaka.hls.HlsParser.prototype.createVariantsForTag_ = function(tag, playlist) {
   let bandwidth =
       Number(HlsParser.getRequiredAttributeValue_(tag, 'BANDWIDTH'));
 
+  // TODO: Add parsing for closed captions. Note that the value may be 'NONE'.
+  // let closedCaptions = tag.getAttributeValue('CLOSED-CAPTIONS');
+
   if (resolutionAttr) {
     let resBlocks = resolutionAttr.value.split('x');
     width = resBlocks[0];
@@ -831,8 +834,8 @@ shaka.hls.HlsParser.prototype.createStreamInfoFromMediaTag_ =
   let channelsCount = type == 'audio' ?
       this.getChannelCount_(channelsAttribute) : null;
   let primary = !!defaultAttr || !!autoselectAttr;
-  return this.createStreamInfo_(uri, allCodecs, type,
-      language, primary, name, channelsCount).then(function(streamInfo) {
+  return this.createStreamInfo_(uri, allCodecs, type, language, primary,
+      name, channelsCount).then(function(streamInfo) {
     // TODO: This check is necessary because of the possibility of multiple
     // calls to createStreamInfoFromMediaTag_ before either has resolved.
     if (this.uriToStreamInfosMap_[uri]) {
@@ -885,6 +888,8 @@ shaka.hls.HlsParser.prototype.createStreamInfoFromVariantTag_ =
   if (this.uriToStreamInfosMap_[uri]) {
     return Promise.resolve(this.uriToStreamInfosMap_[uri]);
   }
+  // TODO: Add parsing for closed captions. Note that the value may be 'NONE'.
+  // let closedCaptions = tag.getAttributeValue('CLOSED-CAPTIONS');
 
   return this.createStreamInfo_(uri, allCodecs, type,
                                 /* language */ 'und', /* primary */ false,
@@ -1038,6 +1043,7 @@ shaka.hls.HlsParser.prototype.createStreamInfo_ = function(uri, allCodecs,
       bandwidth: undefined,
       roles: [],
       channelsCount: channelsCount,
+      closedCaptions: {},
     };
 
     this.streamsToIndexMap_[stream.id] = segmentIndex;

lib/offline/manifest_converter.js

@@ -213,6 +213,7 @@ shaka.offline.ManifestConverter = class {
       containsEmsgBoxes: false,
       roles: [],
       channelsCount: null,
+      closedCaptions: {},
     };
 
     if (streamDB.initSegmentKey != null) {

lib/player.js

@@ -2124,6 +2124,7 @@ shaka.Player.prototype.addTextTrack = function(
     containsEmsgBoxes: false,
     roles: [],
     channelsCount: null,
+    closedCaptions: {},
   };
 
   // Add the stream to the loading list to ensure it isn't switched to while it

test/dash/dash_parser_manifest_unit.js

@@ -311,6 +311,61 @@ describe('DashParser Manifest', function() {
         }, 0, null));
   });
 
+
+  it('correctly parses closed captions with channel numbers', async () => {
+    let source = [
+      '<MPD minBufferTime="PT75S">',
+      '  <Period id="1" duration="PT30S">',
+      '    <AdaptationSet mimeType="video/mp4" lang="en" group="1">',
+      '      <Accessibility schemeIdUri="urn:scte:dash:cc:cea-608:2015"',
+      '         value="CC1=eng;CC3=swe"/>',
+      '      <Representation bandwidth="200">',
+      '        <SegmentTemplate media="1.mp4" duration="1" />',
+      '      </Representation>',
+      '    </AdaptationSet>',
+      '  </Period>',
+      '</MPD>',
+    ].join('\n');
+
+    fakeNetEngine.setResponseMapAsText({'dummy://foo': source});
+
+    let manifest = await parser.start('dummy://foo', playerInterface);
+    // First Representation should be dropped.
+    let period = manifest.periods[0];
+    let stream = period.variants[0].video;
+    expect(stream.closedCaptions).toEqual(jasmine.objectContaining({
+      CC1: shaka.util.LanguageUtils.normalize('eng'),
+      CC3: shaka.util.LanguageUtils.normalize('swe'),
+    }));
+  });
+
+
+  it('correctly parses closed captions without channel numbers', async () => {
+    let source = [
+      '<MPD minBufferTime="PT75S">',
+      '  <Period id="1" duration="PT30S">',
+      '    <AdaptationSet mimeType="video/mp4" lang="en" group="1">',
+      '      <Accessibility schemeIdUri="urn:scte:dash:cc:cea-608:2015"',
+      '         value="eng;swe"/>',
+      '      <Representation bandwidth="200">',
+      '        <SegmentTemplate media="1.mp4" duration="1" />',
+      '      </Representation>',
+      '    </AdaptationSet>',
+      '  </Period>',
+      '</MPD>',
+    ].join('\n');
+
+    fakeNetEngine.setResponseMapAsText({'dummy://foo': source});
+
+    let manifest = await parser.start('dummy://foo', playerInterface);
+    let stream = manifest.periods[0].variants[0].video;
+    expect(stream.closedCaptions).toEqual(jasmine.objectContaining({
+      0: shaka.util.LanguageUtils.normalize('eng'),
+      1: shaka.util.LanguageUtils.normalize('swe'),
+    }));
+  });
+
+
   it('correctly parses UTF-8', async () => {
     let source = [
       '<MPD>',

test/offline/manifest_convert_unit.js

@@ -414,6 +414,7 @@ describe('ManifestConverter', function() {
       containsEmsgBoxes: false,
       roles: [],
       channelsCount: null,
+      closedCaptions: {},
     };
 
     expect(stream).toEqual(expectedStream);

test/test/util/manifest_generator.js

@@ -559,6 +559,7 @@ shaka.test.ManifestGenerator.prototype.createStream_ =
     containsEmsgBoxes: false,
     roles: [],
     channelsCount: null,
+    closedCaptions: {},
   };
   return stream;
 };