From 7812357997aaae1a954cf284c2403cf58b5c4341 Mon Sep 17 00:00:00 2001 From: Yair Ansbacher Date: Wed, 12 Jan 2022 15:48:46 +0200 Subject: [PATCH] feat(FEC-11761): expose stream timed metadata - phase 2 (#623) --- src/engines/html5/html5.js | 54 ++++++++++----- src/event/event-type.js | 4 ++ src/player.js | 1 + src/playkit.js | 6 +- src/track/cue-point.js | 53 -------------- src/track/text-track.js | 4 +- src/track/timed-metadata.js | 93 +++++++++++++++++++++++++ src/utils/binary-search.js | 26 +++++++ src/utils/index.js | 1 + test/src/player.spec.js | 100 +++++++++++++++++++++++++++ test/src/utils/binary-search.spec.js | 19 +++++ 11 files changed, 286 insertions(+), 75 deletions(-) delete mode 100644 src/track/cue-point.js create mode 100644 src/track/timed-metadata.js create mode 100644 src/utils/binary-search.js create mode 100644 test/src/utils/binary-search.spec.js diff --git a/src/engines/html5/html5.js b/src/engines/html5/html5.js index 49bc46723..608e8f07a 100644 --- a/src/engines/html5/html5.js +++ b/src/engines/html5/html5.js @@ -8,6 +8,8 @@ import VideoTrack from '../../track/video-track'; import AudioTrack from '../../track/audio-track'; import PKTextTrack, {getActiveCues} from '../../track/text-track'; import ImageTrack from '../../track/image-track'; +import {Cue} from '../../track/vtt-cue'; +import {createTimedMetadata} from '../../track/timed-metadata'; import * as Utils from '../../utils/util'; import Html5AutoPlayCapability from './capabilities/html5-autoplay'; import Error from '../../error/error'; @@ -1163,27 +1165,45 @@ export default class Html5 extends FakeEventTarget implements IEngine { } _handleMetadataTrackEvents(): void { - const listenToCueChange = track => { - track.mode = PKTextTrack.MODE.HIDDEN; - this._eventManager.listen(track, 'cuechange', () => { - this.dispatchEvent(new FakeEvent(CustomEventType.TIMED_METADATA, {cues: Array.from(track.activeCues), label: track.label})); + const listenToCueChange = metadataTrack => { + metadataTrack.mode = PKTextTrack.MODE.HIDDEN; + this._eventManager.listen(metadataTrack, 'cuechange', () => { + let activeCues = []; + Array.from(this._el.textTracks).forEach((track: TextTrack) => { + if (PKTextTrack.isMetaDataTrack(track)) { + activeCues = activeCues.concat(getActiveCues(track.activeCues)); + } + }); + activeCues = activeCues.sort((a: Cue, b: Cue) => { + return a.startTime - b.startTime; + }); + this.dispatchEvent(new FakeEvent(CustomEventType.TIMED_METADATA, {cues: activeCues})); + this.dispatchEvent( + new FakeEvent(CustomEventType.TIMED_METADATA_CHANGE, { + cues: activeCues.map(cue => createTimedMetadata(cue)) + }) + ); }); }; - const metadataTrack = Array.from(this._el.textTracks).find((track: TextTrack) => PKTextTrack.isMetaDataTrack(track)); - if (metadataTrack) { - listenToCueChange(metadataTrack); - } else { - this._eventManager.listen(this._el.textTracks, 'addtrack', (event: any) => { - if (PKTextTrack.isMetaDataTrack(event.track)) { - listenToCueChange(event.track); + + Array.from(this._el.textTracks).forEach((track: TextTrack) => { + if (PKTextTrack.isMetaDataTrack(track)) { + listenToCueChange(track); + } + }); + + this._eventManager.listen(this._el.textTracks, 'addtrack', (event: any) => { + if (PKTextTrack.isMetaDataTrack(event.track)) { + listenToCueChange(event.track); + } + }); + + this._eventManager.listen(this._el.textTracks, 'change', () => { + Array.from(this._el.textTracks).forEach((track: TextTrack) => { + if (PKTextTrack.isMetaDataTrack(track) && track.mode !== PKTextTrack.MODE.HIDDEN) { + track.mode = PKTextTrack.MODE.HIDDEN; } }); - } - this._eventManager.listen(this._el.textTracks, 'change', () => { - const metadataTrack = Array.from(this._el.textTracks).find((track: TextTrack) => PKTextTrack.isMetaDataTrack(track)); - if (metadataTrack && metadataTrack.mode !== PKTextTrack.MODE.HIDDEN) { - metadataTrack.mode = PKTextTrack.MODE.HIDDEN; - } }); } diff --git a/src/event/event-type.js b/src/event/event-type.js index 974a71af3..9b38df382 100644 --- a/src/event/event-type.js +++ b/src/event/event-type.js @@ -240,6 +240,10 @@ const CustomEventType: PKEventTypes = { * Fired when the timed metadata triggered */ TIMED_METADATA: 'timedmetadata', + /** + * Fired when the timed metadata triggered + */ + TIMED_METADATA_CHANGE: 'timedmetadatachange', /** * Fires when new timed metadata added */ diff --git a/src/player.js b/src/player.js index 802d7d7d0..c9a12787f 100644 --- a/src/player.js +++ b/src/player.js @@ -1921,6 +1921,7 @@ export default class Player extends FakeEventTarget { this._eventManager.listen(this._engine, CustomEventType.TEXT_CUE_CHANGED, (event: FakeEvent) => this._onCueChange(event)); this._eventManager.listen(this._engine, CustomEventType.ABR_MODE_CHANGED, (event: FakeEvent) => this.dispatchEvent(event)); this._eventManager.listen(this._engine, CustomEventType.TIMED_METADATA, (event: FakeEvent) => this.dispatchEvent(event)); + this._eventManager.listen(this._engine, CustomEventType.TIMED_METADATA_CHANGE, (event: FakeEvent) => this.dispatchEvent(event)); this._eventManager.listen(this._engine, CustomEventType.TIMED_METADATA_ADDED, (event: FakeEvent) => this.dispatchEvent(event)); this._eventManager.listen(this._engine, CustomEventType.PLAY_FAILED, (event: FakeEvent) => { this._onPlayFailed(event); diff --git a/src/playkit.js b/src/playkit.js index 258f6c87b..05649792d 100644 --- a/src/playkit.js +++ b/src/playkit.js @@ -11,7 +11,7 @@ import ImageTrack from './track/image-track'; import VideoTrack from './track/video-track'; import AudioTrack from './track/audio-track'; import TextTrack from './track/text-track'; -import {CuePoint, createTextTrackCue} from './track/cue-point'; +import {TimedMetadata, createTextTrackCue, createTimedMetadata} from './track/timed-metadata'; import TextStyle from './track/text-style'; import {Cue} from './track/vtt-cue'; import Env from './utils/env'; @@ -63,8 +63,8 @@ export {BaseMiddleware}; // Export the tracks classes export {Track, VideoTrack, AudioTrack, TextTrack, ImageTrack, TextStyle, Cue}; -// Export the cue point class and function -export {CuePoint, createTextTrackCue}; +// Export the timed metadata class and function +export {TimedMetadata, createTextTrackCue, createTimedMetadata}; // Export utils library export {Utils}; diff --git a/src/track/cue-point.js b/src/track/cue-point.js deleted file mode 100644 index 36eda611a..000000000 --- a/src/track/cue-point.js +++ /dev/null @@ -1,53 +0,0 @@ -//@flow -class CuePoint { - static TYPE: {[type: string]: string}; - - startTime: number; - endTime: number; - id: string; - type: string; - metadata: string | Object; - /** - * @constructor - * @param {number} startTime - start time. - * @param {number} endTime - end time. - * @param {string} id - id. - * @param {string} type - type. - * @param {string|Object} metadata - metadata. - */ - constructor(startTime: number, endTime: number, id: string, type: string, metadata: string | Object) { - this.startTime = startTime; - this.endTime = endTime; - this.id = id; - this.type = type; - this.metadata = metadata; - } -} - -CuePoint.TYPE = { - EMSG: 'emsg', - CUSTOM: 'custom' -}; - -/** - * Create a standard TextTrackCue. - * @param {CuePoint} cuePoint - cue point. - * @returns {TextTrackCue} - the created text track cue - * @private - */ -function createTextTrackCue(cuePoint: CuePoint): TextTrackCue { - const {startTime, endTime, id, type, metadata} = cuePoint; - let cue = {}; - if (window.VTTCue) { - cue = new window.VTTCue(startTime, endTime, ''); - } else if (window.TextTrackCue) { - // IE11 support - cue = new window.TextTrackCue(startTime, endTime, ''); - } - const cueValue = {key: type, data: metadata}; - cue.id = id; - cue.value = cueValue; - return cue; -} - -export {CuePoint, createTextTrackCue}; diff --git a/src/track/text-track.js b/src/track/text-track.js index 466f80634..d43a4535b 100644 --- a/src/track/text-track.js +++ b/src/track/text-track.js @@ -95,11 +95,11 @@ TextTrack.isExternalTrack = (track: any) => { * @returns {void} * @private */ -function getActiveCues(textTrackCueList: TextTrackCueList) { +function getActiveCues(textTrackCueList: TextTrackCueList): Array { let normalizedCues: Array = []; for (let cue of textTrackCueList) { //Normalize cues to be of type of VTT model - if (window.VTTCue && cue instanceof window.VTTCue) { + if ((window.VTTCue && cue instanceof window.VTTCue) || (window.DataCue && cue instanceof window.DataCue)) { normalizedCues.push(cue); } else if (window.TextTrackCue && cue instanceof window.TextTrackCue) { try { diff --git a/src/track/timed-metadata.js b/src/track/timed-metadata.js new file mode 100644 index 000000000..a3cfafd8d --- /dev/null +++ b/src/track/timed-metadata.js @@ -0,0 +1,93 @@ +//@flow +class TimedMetadata { + static TYPE: {[type: string]: string}; + + startTime: number; + endTime: number; + id: string; + type: string; + metadata: string | Object; + /** + * @constructor + * @param {number} startTime - start time. + * @param {number} endTime - end time. + * @param {string} id - id. + * @param {string} type - type. + * @param {any} metadata - metadata. + */ + constructor(startTime: number, endTime: number, id: string, type: string, metadata: any) { + this.startTime = startTime; + this.endTime = endTime; + this.id = id; + this.type = type; + this.metadata = metadata; + } +} + +TimedMetadata.TYPE = { + ID3: 'id3', + EMSG: 'emsg', + CUE_POINT: 'cuepoint', + CUSTOM: 'custom' +}; + +/** + * Create a standard TextTrackCue. + * @param {TimedMetadata} timedMetadata - timed metadata object. + * @returns {TextTrackCue} - the created text track cue + * @private + */ +function createTextTrackCue(timedMetadata: TimedMetadata): TextTrackCue { + const {startTime, endTime, id, type, metadata} = timedMetadata; + let cue = {}; + if (window.VTTCue) { + cue = new window.VTTCue(startTime, endTime, ''); + } else if (window.TextTrackCue) { + // IE11 support + cue = new window.TextTrackCue(startTime, endTime, ''); + } + const cueValue = {key: type, data: metadata}; + cue.id = id; + cue.value = cueValue; + return cue; +} + +/** + * Create a timed metadata object from a standard TextTrackCue. + * @param {TextTrackCue} cue - text track cue. + * @returns {?TimedMetadata} - the created timed metadata object. + * @private + */ +function createTimedMetadata(cue: TextTrackCue): ?TimedMetadata { + if (cue) { + const {startTime, endTime, id} = cue; + const {type, metadata} = _getTypeAndMetadata(cue); + return new TimedMetadata(startTime, endTime, id, type, metadata); + } + return null; +} + +/** + * @param {TextTrackCue} cue - cue + * @return {Object} - type and data + * @private + */ +function _getTypeAndMetadata(cue: TextTrackCue): Object { + const { + type, + value, + track: {label} + } = cue; + const {key, data} = value; + const isId3 = type === 'org.id3' || label === 'id3'; + let timedMetadataType = Object.values(TimedMetadata.TYPE).find(type => type === key); + if (!timedMetadataType) { + timedMetadataType = isId3 ? TimedMetadata.TYPE.ID3 : TimedMetadata.TYPE.CUSTOM; + } + return { + type: timedMetadataType, + metadata: isId3 ? value : data + }; +} + +export {TimedMetadata, createTextTrackCue, createTimedMetadata}; diff --git a/src/utils/binary-search.js b/src/utils/binary-search.js new file mode 100644 index 000000000..c7e2c1764 --- /dev/null +++ b/src/utils/binary-search.js @@ -0,0 +1,26 @@ +//@flow +/** + * @param {Array} list The array to search. + * @param {Function} comparisonFn + * Called and provided a candidate item as the first argument. + * Should return: + * > -1 if the item should be located at a lower index than the provided item. + * > 1 if the item should be located at a higher index than the provided item. + * > 0 if the item is the item you're looking for. + * + * @return {any} The object if it is found or null otherwise. + */ +export function binarySearch(list: Array = [], comparisonFn: Function = () => 1): any { + if (list.length === 0 || (list.length === 1 && comparisonFn(list[0]) !== 0)) { + return null; + } + const mid = Math.floor(list.length / 2); + if (comparisonFn(list[mid]) === 0) { + return list[mid]; + } + if (comparisonFn(list[mid]) > 0) { + return binarySearch(list.slice(0, mid), comparisonFn); + } else { + return binarySearch(list.slice(mid + 1), comparisonFn); + } +} diff --git a/src/utils/index.js b/src/utils/index.js index 89b2aedea..dab5ab10f 100644 --- a/src/utils/index.js +++ b/src/utils/index.js @@ -1,3 +1,4 @@ export * from './util'; export {ResizeWatcher} from './resize-watcher'; export {MultiMap} from './multi-map'; +export {binarySearch} from './binary-search'; diff --git a/test/src/player.spec.js b/test/src/player.spec.js index 71f60fac7..5081a1edd 100644 --- a/test/src/player.spec.js +++ b/test/src/player.spec.js @@ -8,6 +8,7 @@ import Track from '../../src/track/track'; import VideoTrack from '../../src/track/video-track'; import AudioTrack from '../../src/track/audio-track'; import TextTrack from '../../src/track/text-track'; +import {createTextTrackCue, TimedMetadata} from '../../src/track/timed-metadata'; import {createElement, getConfigStructure, removeElement, removeVideoElementsFromTestPage} from './utils/test-utils'; import Locale from '../../src/utils/locale'; import Html5 from '../../src/engines/html5/html5'; @@ -2348,6 +2349,105 @@ describe('Player', function () { player.textStyle = new TextStyle(); }); }); + + describe('timed metadata and timed metadata change', () => { + let config; + let player; + let playerContainer; + + before(() => { + playerContainer = createElement('DIV', targetId); + }); + + beforeEach(() => { + config = getConfigStructure(); + }); + + afterEach(() => { + player.destroy(); + }); + + after(() => { + removeVideoElementsFromTestPage(); + removeElement(targetId); + }); + + it('should dispatch timed metadata and timed metadata change on time', done => { + let onPlaying = () => { + player.removeEventListener(player.Event.PLAYING, onPlaying); + const metadataTrack1 = player.getVideoElement().addTextTrack(TextTrack.KIND.METADATA, 'metadata1'); + const metadataTrack2 = player.getVideoElement().addTextTrack(TextTrack.KIND.METADATA, 'metadata2'); + + const {currentTime, duration} = player; + const timedMetadata1 = new TimedMetadata(currentTime + 1, duration - 2, 'id1', 'type1', {info: 'some_info1'}); + const timedMetadata2 = new TimedMetadata(currentTime + 2, duration - 1, 'id2', 'type2', {info: 'some_info2'}); + const textTrackCue1 = createTextTrackCue(timedMetadata1); + const textTrackCue2 = createTextTrackCue(timedMetadata2); + metadataTrack1.addCue(textTrackCue1); + metadataTrack2.addCue(textTrackCue2); + let eventCounter = -1; + player.addEventListener(player.Event.TIMED_METADATA, e => { + try { + eventCounter++; + switch (eventCounter) { + case 0: + e.payload.cues.length.should.equal(1); + e.payload.cues[0].value.key.should.equal('type1'); + e.payload.cues[0].value.data.should.deep.equal({info: 'some_info1'}); + break; + case 2: + e.payload.cues.length.should.equal(2); + e.payload.cues[0].value.key.should.equal('type1'); + e.payload.cues[0].value.data.should.deep.equal({info: 'some_info1'}); + e.payload.cues[1].value.key.should.equal('type2'); + e.payload.cues[1].value.data.should.deep.equal({info: 'some_info2'}); + break; + case 4: + e.payload.cues.length.should.equal(1); + e.payload.cues[0].value.key.should.equal('type2'); + e.payload.cues[0].value.data.should.deep.equal({info: 'some_info2'}); + break; + case 6: + e.payload.cues.length.should.equal(0); + } + } catch (e) { + done(e); + } + }); + player.addEventListener(player.Event.TIMED_METADATA_CHANGE, e => { + try { + eventCounter++; + switch (eventCounter) { + case 1: + e.payload.cues.length.should.equal(1); + e.payload.cues[0].should.deep.equal({...timedMetadata1, type: 'custom'}); + break; + case 3: + e.payload.cues.length.should.equal(2); + e.payload.cues[0].should.deep.equal({...timedMetadata1, type: 'custom'}); + e.payload.cues[1].should.deep.equal({...timedMetadata2, type: 'custom'}); + break; + case 5: + e.payload.cues.length.should.equal(1); + e.payload.cues[0].should.deep.equal({...timedMetadata2, type: 'custom'}); + break; + case 7: + e.payload.cues.length.should.equal(0); + done(); + } + } catch (e) { + done(e); + } + }); + }; + + player = new Player(config); + player.setSources(sourcesConfig.Mp4); + playerContainer.appendChild(player.getView()); + player.addEventListener(player.Event.PLAYING, onPlaying); + player.play(); + }); + }); }); describe('states', function () { diff --git a/test/src/utils/binary-search.spec.js b/test/src/utils/binary-search.spec.js new file mode 100644 index 000000000..bbd4f4cb4 --- /dev/null +++ b/test/src/utils/binary-search.spec.js @@ -0,0 +1,19 @@ +import {binarySearch} from '../../../src/utils'; + +describe('binarySearch', () => { + it('should find primitives', () => { + binarySearch([1, 2, 3, 4, 7, 8, 9, 10], int => int - 7).should.equal(7); + binarySearch([1, 2, 3, 4, 7, 8, 9, 10], int => int - 2).should.equal(2); + binarySearch([0, 1, 2, 3, 4, 7, 8, 9, 10], int => int - 0).should.equal(0); + binarySearch([-1, 2, 3, 4, 7, 8, 9, 10], int => int + 1).should.equal(-1); + (binarySearch([-1, 2, 3, 4, 7, 8, 9, 10], int => int - 11) === null).should.be.true; + }); + it('should find object', () => { + binarySearch([{num: 0}, {num: 10}, {num: 100}, {num: 1000}], obj => obj.num - 1000).should.deep.equal({num: 1000}); + (binarySearch([{num: 0}, {num: 10}, {num: 100}, {num: 1000}], obj => obj.num - 2000) === null).should.be.true; + }); + it('should has default', () => { + (binarySearch(undefined, int => int - 1) === null).should.be.true; + (binarySearch([-1, 2, 3, 4, 7, 8, 9, 10]) === null).should.be.true; + }); +});