Skip to content

Commit

Permalink
Refactor Closed Captions Support for HLS
Browse files Browse the repository at this point in the history 
Refactor the closed captions support for HLS, using the same approach as
the closed captions support for Dash.
If closed captions are signaled in the manifest, player will create a
text stream for to represent the closed captions, and text engine will
store and append them. We don't need to set 'useEmbeddedText' value for
closed captions any longer.

Issue #1404

Change-Id: I9a5bf4df7e29d6e6982f29fd5e2df07bc78071d7
  • Loading branch information
@michellezhuogg
michellezhuogg committed Nov 27, 2018
1 parent ad4e099 commit 1afcead
Show file tree
Hide file tree
Showing 13 changed files with 211 additions and 89 deletions.
7 changes: 1 addition & 6 deletions demo/info_section.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -288,12 +288,7 @@ shakaDemo.onTrackSelected_ = function(event) {

player.selectVariantTrack(track, /* clearBuffer */ true);
} else {
// CEA 608/708 captions data is embedded inside the video stream.
if (option.textContent == 'Default Text') {
player.selectEmbeddedTextTrack();
} else {
player.selectTextTrack(track);
}
player.selectTextTrack(track);
}

// Adaptation might have been changed by calling selectTrack().
Expand Down
2 changes: 1 addition & 1 deletion lib/cast/cast_utils.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -133,7 +133,6 @@ shaka.cast.CastUtils.PlayerGetterMethods = {
'getTextLanguagesAndRoles': 2,
'getTextTracks': 2,
'getStats': 5,
'usingEmbeddedTextTrack': 2,
'getVariantTracks': 2,
'isAudioOnly': 10,
'isBuffering': 1,
Expand All @@ -142,6 +141,7 @@ shaka.cast.CastUtils.PlayerGetterMethods = {
'isTextTrackVisible': 1,
'keySystem': 10,
'seekRange': 1,
'usingEmbeddedTextTrack': 2,
};


Expand Down
91 changes: 77 additions & 14 deletions lib/hls/hls_parser.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,12 @@ shaka.hls.HlsParser = function() {

/** @private {!Array.<!Array.<!shaka.media.SegmentReference>>} */
this.segmentsToNotifyByStream_ = [];

/** A map from closed captions' group id, to a map of closed captions info.
* {group id -> {closed captions channel id -> language}}
* @private {Map.<string, Map.<string, string>>}
*/
this.groupIdToClosedCaptionsMap_ = new Map();
};


Expand Down Expand Up @@ -463,11 +469,17 @@ shaka.hls.HlsParser.prototype.createPeriod_ = function(playlist) {
return type == 'SUBTITLES';
}.bind(this));

// TODO: CLOSED-CAPTIONS requires the parsing of CEA-608 from the video.
let textStreamPromises = textStreamTags.map(function(tag) {
return this.createTextStream_(tag, playlist);
}.bind(this));

const closedCaptionsTags = mediaTags.filter((tag) => {
const type = shaka.hls.HlsParser.getRequiredAttributeValue_(tag, 'TYPE');
return type == 'CLOSED-CAPTIONS';
});

this.parseClosedCaptions_(closedCaptionsTags);

return Promise.all(textStreamPromises).then(function(textStreams) {
// Create Variants for every 'EXT-X-STREAM-INF' tag. Do this after text
// streams have been created, so that we can push text codecs found on the
Expand Down Expand Up @@ -521,9 +533,6 @@ shaka.hls.HlsParser.prototype.createVariantsForTag_ = function(tag, playlist) {
let bandwidth =
Number(HlsParser.getRequiredAttributeValue_(tag, 'BANDWIDTH'));

// TODO: Add parsing for closed captions. Note that the value may be 'NONE'.
// let closedCaptions = tag.getAttributeValue('CLOSED-CAPTIONS');

if (resolutionAttr) {
let resBlocks = resolutionAttr.value.split('x');
width = resBlocks[0];
Expand All @@ -534,6 +543,12 @@ shaka.hls.HlsParser.prototype.createVariantsForTag_ = function(tag, playlist) {
// combine with the variant tag (EXT-X-STREAM-INF) we are working on.
let mediaTags = Utils.filterTagsByName(playlist.tags, 'EXT-X-MEDIA');

// Do not create stream info from closed captions media tags, which are
// embedded in video streams.
mediaTags = mediaTags.filter((tag) => {
const type = HlsParser.getRequiredAttributeValue_(tag, 'TYPE');
return type != 'CLOSED-CAPTIONS';
});
let audioGroupId = tag.getAttributeValue('AUDIO');
let videoGroupId = tag.getAttributeValue('VIDEO');
goog.asserts.assert(audioGroupId == null || videoGroupId == null,
Expand Down Expand Up @@ -822,6 +837,43 @@ shaka.hls.HlsParser.prototype.createTextStream_ = function(tag, playlist) {
};


/**
* Parses an EXT-X-MEDIA tag with TYPE="CLOSED-CAPTIONS", add store the values
* into the map of group id to closed captions.
*
* @param {!Array.<shaka.hls.Tag>} tags
* @private
*/
shaka.hls.HlsParser.prototype.parseClosedCaptions_ = function(tags) {
for (const tag of tags) {
goog.asserts.assert(tag.name == 'EXT-X-MEDIA',
'Should only be called on media tags!');
const type = shaka.hls.HlsParser.getRequiredAttributeValue_(tag, 'TYPE');
goog.asserts.assert(type == 'CLOSED-CAPTIONS',
'Should only be called on tags with TYPE="CLOSED-CAPTIONS"!');

const LanguageUtils = shaka.util.LanguageUtils;
const languageValue = tag.getAttributeValue('LANGUAGE') || 'und';
const language = LanguageUtils.normalize(languageValue);

// The GROUP-ID value is a quoted-string that specifies the group to which
// the Rendition belongs.
const groupId =
shaka.hls.HlsParser.getRequiredAttributeValue_(tag, 'GROUP-ID');

// The value of INSTREAM-ID is a quoted-string that specifies a Rendition
// within the segments in the Media Playlist. This attribute is REQUIRED if
// the TYPE attribute is CLOSED-CAPTIONS.
const instreamId =
shaka.hls.HlsParser.getRequiredAttributeValue_(tag, 'INSTREAM-ID');
if (!this.groupIdToClosedCaptionsMap_.get(groupId)) {
this.groupIdToClosedCaptionsMap_.set(groupId, new Map());
}
this.groupIdToClosedCaptionsMap_.get(groupId).set(instreamId, language);
}
};


/**
* Parse EXT-X-MEDIA media tag into a Stream object.
*
Expand Down Expand Up @@ -867,8 +919,8 @@ shaka.hls.HlsParser.prototype.createStreamInfoFromMediaTag_ =
this.getChannelCount_(channelsAttribute) : null;
let primary = !!defaultAttr || !!autoselectAttr;
return this.createStreamInfo_(
verbatimMediaPlaylistUri, allCodecs, type, language, primary,
name, channelsCount).then(function(streamInfo) {
verbatimMediaPlaylistUri, allCodecs, type, language, primary, name,
channelsCount, /* closedCaptions */ null).then(function(streamInfo) {
// TODO: This check is necessary because of the possibility of multiple
// calls to createStreamInfoFromMediaTag_ before either has resolved.
if (this.uriToStreamInfosMap_.has(verbatimMediaPlaylistUri)) {
Expand Down Expand Up @@ -914,6 +966,7 @@ shaka.hls.HlsParser.prototype.createStreamInfoFromVariantTag_ =
function(tag, allCodecs, type) {
goog.asserts.assert(tag.name == 'EXT-X-STREAM-INF',
'Should only be called on media tags!');
const ContentType = shaka.util.ManifestParserUtils.ContentType;

const HlsParser = shaka.hls.HlsParser;
const verbatimMediaPlaylistUri = HlsParser.getRequiredAttributeValue_(
Expand All @@ -923,13 +976,23 @@ shaka.hls.HlsParser.prototype.createStreamInfoFromVariantTag_ =
return Promise.resolve(
this.uriToStreamInfosMap_.get(verbatimMediaPlaylistUri));
}
// TODO: Add parsing for closed captions. Note that the value may be 'NONE'.
// let closedCaptions = tag.getAttributeValue('CLOSED-CAPTIONS');
// The attribute of closed captions is optional, and the value may be 'NONE'.
const closedCaptionsAttr = tag.getAttributeValue('CLOSED-CAPTIONS');

// EXT-X-STREAM-INF tags may have CLOSED-CAPTIONS attributes.
// The value can be either a quoted-string or an enumerated-string with the
// value NONE. If the value is a quoted-string, it MUST match the value of
// the GROUP-ID attribute of an EXT-X-MEDIA tag elsewhere in the Playlist
// whose TYPE attribute is CLOSED-CAPTIONS.
let closedCaptions = null;
if (type == ContentType.VIDEO && closedCaptionsAttr &&
closedCaptionsAttr != 'NONE') {
closedCaptions = this.groupIdToClosedCaptionsMap_.get(closedCaptionsAttr);
}

return this.createStreamInfo_(verbatimMediaPlaylistUri, allCodecs, type,
/* language */ 'und', /* primary */ false,
/* name */ null, /* channelcount */ null).then(
function(streamInfo) {
/* language */ 'und', /* primary */ false, /* name */ null,
/* channelcount */ null, closedCaptions).then(function(streamInfo) {
// TODO: This check is necessary because of the possibility of multiple
// calls to createStreamInfoFromVariantTag_ before either has resolved.
if (this.uriToStreamInfosMap_.has(verbatimMediaPlaylistUri)) {
Expand All @@ -950,13 +1013,14 @@ shaka.hls.HlsParser.prototype.createStreamInfoFromVariantTag_ =
* @param {boolean} primary
* @param {?string} name
* @param {?number} channelsCount
* @param {Map.<string, string>} closedCaptions
* @return {!Promise.<shaka.hls.HlsParser.StreamInfo>}
* @throws shaka.util.Error
* @private
*/
shaka.hls.HlsParser.prototype.createStreamInfo_ = function(
verbatimMediaPlaylistUri, allCodecs, type, language, primary, name,
channelsCount) {
channelsCount, closedCaptions) {
// TODO: Refactor, too many parameters
const Utils = shaka.hls.Utils;
const HlsParser = shaka.hls.HlsParser;
Expand Down Expand Up @@ -1017,7 +1081,6 @@ shaka.hls.HlsParser.prototype.createStreamInfo_ = function(
if (type == ManifestParserUtils.ContentType.TEXT) {
kind = ManifestParserUtils.TextStreamKind.SUBTITLE;
}
// TODO: CLOSED-CAPTIONS requires the parsing of CEA-608 from the video.

let drmTags = [];
playlist.segments.forEach(function(segment) {
Expand Down Expand Up @@ -1087,7 +1150,7 @@ shaka.hls.HlsParser.prototype.createStreamInfo_ = function(
bandwidth: undefined,
roles: [],
channelsCount: channelsCount,
closedCaptions: null,
closedCaptions: closedCaptions,
};

return {
Expand Down
32 changes: 3 additions & 29 deletions lib/media/media_source_engine.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,9 +73,6 @@ shaka.media.MediaSourceEngine = function(video) {
/** @private {!Object.<string, !shaka.media.Transmuxer>} */
this.transmuxers_ = {};

/** @private {boolean} */
this.useEmbeddedText_ = false;

/** @private {muxjs.mp4.CaptionParser} */
this.captionParser_ = null;

Expand Down Expand Up @@ -518,16 +515,16 @@ shaka.media.MediaSourceEngine.prototype.appendBuffer =
} else if (this.transmuxers_[contentType]) {
return this.transmuxers_[contentType].transmux(data).then(
function(transmuxedData) {
// TODO: refactor appending transmuxed cues.
// For HLS CEA-608/708 CLOSED-CAPTIONS, text data is embedded in the
// video stream, so textEngine may not have been initialized.
if (!this.textEngine_) {
this.reinitText('text/vtt');
}
// This doesn't work for native TS support (ex. Edge/Chromecast),
// since no transmuxing is needed for native TS.
if (this.useEmbeddedText_) {
this.textEngine_.appendCues(transmuxedData.cues);
if (transmuxedData.captions) {
this.textEngine_.storeAndAppendClosedCaptions(
transmuxedData.captions, startTime, endTime);
}
return this.enqueueOperation_(contentType,
this.append_.bind(this, contentType, transmuxedData.data.buffer));
Expand Down Expand Up @@ -571,19 +568,6 @@ shaka.media.MediaSourceEngine.prototype.appendBuffer =
};


/**
* Set whether to use embedded text cues.
* Used for CEA 608/708 captions data, which is embedded inside the video
* stream.
*
* @param {boolean} useEmbeddedText
*/
shaka.media.MediaSourceEngine.prototype.setUseEmbeddedText =
function(useEmbeddedText) {
this.useEmbeddedText_ = useEmbeddedText;
};


/**
* Set the selected closed captions Id and language.
*
Expand All @@ -597,16 +581,6 @@ shaka.media.MediaSourceEngine.prototype.setSelectedClosedCaptionId =
};


/**
* Get whether we're using the embedded text cues.
*
* @return {boolean}
*/
shaka.media.MediaSourceEngine.prototype.getUseEmbeddedText = function() {
return this.useEmbeddedText_;
};


/**
* Enqueue an operation to remove data from the SourceBuffer.
*
Expand Down
19 changes: 7 additions & 12 deletions lib/media/transmuxer.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
goog.provide('shaka.media.Transmuxer');

goog.require('goog.asserts');
goog.require('shaka.text.Cue');
goog.require('shaka.util.Error');
goog.require('shaka.util.IDestroyable');
goog.require('shaka.util.ManifestParserUtils');
Expand Down Expand Up @@ -46,8 +45,8 @@ shaka.media.Transmuxer = function() {
/** @private {!Array.<!Uint8Array>} */
this.transmuxedData_ = [];

/** @private {!Array.<shaka.text.Cue>} */
this.cues_ = [];
/** @private {!Array.<muxjs.mp4.ClosedCaption>} */
this.captions_ = [];

/** @private {boolean} */
this.isTransmuxing_ = false;
Expand Down Expand Up @@ -150,15 +149,16 @@ shaka.media.Transmuxer.convertTsCodecs = function(contentType, tsMimeType) {
/**
* Transmux from Transport stream to MP4, using the mux.js library.
* @param {!ArrayBuffer} data
* @return {!Promise.<{data: !Uint8Array, cues: !Array.<!shaka.text.Cue>}>}
* @return {!Promise.<{data: !Uint8Array,
* captions: !Array.<!muxjs.mp4.ClosedCaption>}>}
*/
shaka.media.Transmuxer.prototype.transmux = function(data) {
goog.asserts.assert(!this.isTransmuxing_,
'No transmuxing should be in progress.');
this.isTransmuxing_ = true;
this.transmuxPromise_ = new shaka.util.PublicPromise();
this.transmuxedData_ = [];
this.cues_ = [];
this.captions_ = [];

let dataArray = new Uint8Array(data);
this.muxTransmuxer_.push(dataArray);
Expand Down Expand Up @@ -189,12 +189,7 @@ shaka.media.Transmuxer.prototype.transmux = function(data) {
* @private
*/
shaka.media.Transmuxer.prototype.onTransmuxed_ = function(segment) {
for (let i = 0; i < segment.captions.length; i++) {
let cue = segment.captions[i];
this.cues_.push(
new shaka.text.Cue(cue.startTime, cue.endTime, cue.text));
}

this.captions_ = segment.captions;
let segmentWithInit = new Uint8Array(segment.data.byteLength +
segment.initSegment.byteLength);
segmentWithInit.set(segment.initSegment, 0);
Expand All @@ -211,7 +206,7 @@ shaka.media.Transmuxer.prototype.onTransmuxed_ = function(segment) {
shaka.media.Transmuxer.prototype.onTransmuxDone_ = function() {
let output = {
data: shaka.util.Uint8ArrayUtils.concat.apply(null, this.transmuxedData_),
cues: this.cues_,
captions: this.captions_,
};

this.transmuxPromise_.resolve(output);
Expand Down
Loading

0 comments on commit 1afcead

Please sign in to comment.