diff --git a/externs/shaka/player.js b/externs/shaka/player.js index 9a56aad2d9..f7a55c951d 100644 --- a/externs/shaka/player.js +++ b/externs/shaka/player.js @@ -1419,7 +1419,8 @@ shaka.extern.LanguageRole; * startTime: number, * duration: number, * uris: !Array., - * width: number + * width: number, + * sprite: boolean * }} * * @property {number} imageHeight @@ -1443,6 +1444,8 @@ shaka.extern.LanguageRole; * given. * @property {number} width * The thumbnail width in px. + * @property {boolean} sprite + * Indicate if the thumbnail is a sprite. * @exportDoc */ shaka.extern.Thumbnail; diff --git a/lib/media/segment_reference.js b/lib/media/segment_reference.js index 8907b4302b..b74a996aa2 100644 --- a/lib/media/segment_reference.js +++ b/lib/media/segment_reference.js @@ -243,6 +243,9 @@ shaka.media.SegmentReference = class { /** @type {?shaka.extern.HlsAes128Key} */ this.hlsAes128Key = hlsAes128Key; + + /** @type {?shaka.media.SegmentReference.ThumbnailSprite} */ + this.thumbnailSprite = null; } /** @@ -357,6 +360,26 @@ shaka.media.SegmentReference = class { markAsUnavailable() { this.status = shaka.media.SegmentReference.Status.UNAVAILABLE; } + + /** + * Set the segment's thumbnail sprite. + * + * @param {shaka.media.SegmentReference.ThumbnailSprite} thumbnailSprite + * @export + */ + setThumbnailSprite(thumbnailSprite) { + this.thumbnailSprite = thumbnailSprite; + } + + /** + * Returns the segment's thumbnail sprite. + * + * @return {?shaka.media.SegmentReference.ThumbnailSprite} + * @export + */ + getThumbnailSprite() { + return this.thumbnailSprite; + } }; @@ -380,3 +403,24 @@ shaka.media.SegmentReference.Status = { * @typedef {shaka.media.InitSegmentReference|shaka.media.SegmentReference} */ shaka.media.AnySegmentReference; + + +/** + * @typedef {{ + * height: number, + * positionX: number, + * positionY: number, + * width: number + * }} + * + * @property {number} height + * The thumbnail height in px. + * @property {number} positionX + * The thumbnail left position in px. + * @property {number} positionY + * The thumbnail top position in px. + * @property {number} width + * The thumbnail width in px. + * @export + */ +shaka.media.SegmentReference.ThumbnailSprite; diff --git a/lib/player.js b/lib/player.js index 2d79ad13a0..40b6defaec 100644 --- a/lib/player.js +++ b/lib/player.js @@ -3843,8 +3843,8 @@ shaka.Player = class extends shaka.util.FakeEventTarget { const fullImageHeight = imageStream.height || 0; const columns = parseInt(match[1], 10); const rows = parseInt(match[2], 10); - const width = fullImageWidth / columns; - const height = fullImageHeight / rows; + let width = fullImageWidth / columns; + let height = fullImageHeight / rows; const totalImages = columns * rows; const segmentDuration = reference.trueEndTime - reference.startTime; const thumbnailDuration = @@ -3870,6 +3870,15 @@ shaka.Player = class extends shaka.util.FakeEventTarget { positionX = (thumbnailPosition % columns) * width; positionY = Math.floor(thumbnailPosition / columns) * height; } + let sprite = false; + const thumbnailSprite = reference.getThumbnailSprite(); + if (thumbnailSprite) { + sprite = true; + height = thumbnailSprite.height; + positionX = thumbnailSprite.positionX; + positionY = thumbnailSprite.positionY; + width = thumbnailSprite.width; + } return { imageHeight: fullImageHeight, imageWidth: fullImageWidth, @@ -3880,6 +3889,7 @@ shaka.Player = class extends shaka.util.FakeEventTarget { duration: thumbnailDuration, uris: reference.getUris(), width: width, + sprite: sprite, }; } return null; @@ -4839,11 +4849,7 @@ shaka.Player = class extends shaka.util.FakeEventTarget { for (const cue of cues) { const imageUri = shaka.util.ManifestParserUtils.resolveUris( [uri], [cue.payload])[0]; - if (imageUri.includes('#xywh')) { - shaka.log.alwaysWarn('Unsupported image uri', imageUri); - continue; - } - references.push(new shaka.media.SegmentReference( + const reference = new shaka.media.SegmentReference( cue.startTime, cue.endTime, () => [imageUri], @@ -4853,7 +4859,19 @@ shaka.Player = class extends shaka.util.FakeEventTarget { /* timestampOffset= */ 0, /* appendWindowStart= */ 0, /* appendWindowEnd= */ Infinity, - )); + ); + if (imageUri.includes('#xywh')) { + const spriteInfo = imageUri.split('#xywh=')[1].split(','); + if (spriteInfo.length === 4) { + reference.setThumbnailSprite({ + height: parseInt(spriteInfo[3], 10), + positionX: parseInt(spriteInfo[0], 10), + positionY: parseInt(spriteInfo[1], 10), + width: parseInt(spriteInfo[2], 10), + }); + } + } + references.push(reference); } /** @type {shaka.extern.Stream} */ diff --git a/test/player_integration.js b/test/player_integration.js index 7109fff5db..c68949bd9d 100644 --- a/test/player_integration.js +++ b/test/player_integration.js @@ -262,49 +262,6 @@ describe('Player', () => { expect(cues.length).toBeGreaterThan(0); }); - it('skip appended thumbnails for external thumbnails with sprites', - async () => { - await player.load('test:sintel_no_text_compiled'); - const locationUri = new goog.Uri(location.href); - const partialUri = - new goog.Uri('/base/test/test/assets/thumbnails-sprites.vtt'); - const absoluteUri = locationUri.resolve(partialUri); - const newTrack = - await player.addThumbnailsTrack(absoluteUri.toString()); - - expect(player.getImageTracks()).toEqual([newTrack]); - - const thumbnail1 = await player.getThumbnails(newTrack.id, 0); - expect(thumbnail1).toBe(null); - const thumbnail2 = await player.getThumbnails(newTrack.id, 10); - expect(thumbnail2).toBe(null); - const thumbnail3 = await player.getThumbnails(newTrack.id, 40); - expect(thumbnail3).toBe(null); - }); - - it('appends thumbnails for external thumbnails without sprites', - async () => { - await player.load('test:sintel_no_text_compiled'); - const locationUri = new goog.Uri(location.href); - const partialUri = - new goog.Uri('/base/test/test/assets/thumbnails.vtt'); - const absoluteUri = locationUri.resolve(partialUri); - const newTrack = - await player.addThumbnailsTrack(absoluteUri.toString()); - - expect(player.getImageTracks()).toEqual([newTrack]); - - const thumbnail1 = await player.getThumbnails(newTrack.id, 0); - expect(thumbnail1.startTime).toBe(0); - expect(thumbnail1.duration).toBe(5); - const thumbnail2 = await player.getThumbnails(newTrack.id, 10); - expect(thumbnail2.startTime).toBe(5); - expect(thumbnail2.duration).toBe(25); - const thumbnail3 = await player.getThumbnails(newTrack.id, 40); - expect(thumbnail3.startTime).toBe(30); - expect(thumbnail3.duration).toBe(30); - }); - // https://github.com/shaka-project/shaka-player/issues/2553 it('does not change the selected track', async () => { player.configure('streaming.alwaysStreamText', false); @@ -1277,4 +1234,64 @@ describe('Player', () => { await expectAsync(player.load('test:sintel-hls-clearkey')) .toBeRejectedWith(expectedError); }); + + describe('addThumbnailsTrack', () => { + it('appends thumbnails for external thumbnails with sprites', + async () => { + await player.load('test:sintel_no_text_compiled'); + const locationUri = new goog.Uri(location.href); + const partialUri = + new goog.Uri('/base/test/test/assets/thumbnails-sprites.vtt'); + const absoluteUri = locationUri.resolve(partialUri); + const newTrack = + await player.addThumbnailsTrack(absoluteUri.toString()); + + expect(player.getImageTracks()).toEqual([newTrack]); + + const thumbnail1 = await player.getThumbnails(newTrack.id, 0); + expect(thumbnail1.startTime).toBe(0); + expect(thumbnail1.duration).toBe(5); + expect(thumbnail1.height).toBe(90); + expect(thumbnail1.positionX).toBe(0); + expect(thumbnail1.positionY).toBe(0); + expect(thumbnail1.width).toBe(160); + const thumbnail2 = await player.getThumbnails(newTrack.id, 10); + expect(thumbnail2.startTime).toBe(5); + expect(thumbnail2.duration).toBe(25); + expect(thumbnail2.height).toBe(90); + expect(thumbnail2.positionX).toBe(160); + expect(thumbnail2.positionY).toBe(0); + expect(thumbnail2.width).toBe(160); + const thumbnail3 = await player.getThumbnails(newTrack.id, 40); + expect(thumbnail3.startTime).toBe(30); + expect(thumbnail3.duration).toBe(30); + expect(thumbnail3.height).toBe(90); + expect(thumbnail3.positionX).toBe(160); + expect(thumbnail3.positionY).toBe(90); + expect(thumbnail3.width).toBe(160); + }); + + it('appends thumbnails for external thumbnails without sprites', + async () => { + await player.load('test:sintel_no_text_compiled'); + const locationUri = new goog.Uri(location.href); + const partialUri = + new goog.Uri('/base/test/test/assets/thumbnails.vtt'); + const absoluteUri = locationUri.resolve(partialUri); + const newTrack = + await player.addThumbnailsTrack(absoluteUri.toString()); + + expect(player.getImageTracks()).toEqual([newTrack]); + + const thumbnail1 = await player.getThumbnails(newTrack.id, 0); + expect(thumbnail1.startTime).toBe(0); + expect(thumbnail1.duration).toBe(5); + const thumbnail2 = await player.getThumbnails(newTrack.id, 10); + expect(thumbnail2.startTime).toBe(5); + expect(thumbnail2.duration).toBe(25); + const thumbnail3 = await player.getThumbnails(newTrack.id, 40); + expect(thumbnail3.startTime).toBe(30); + expect(thumbnail3.duration).toBe(30); + }); + }); // describe('addThumbnailsTrack') }); diff --git a/test/test/assets/thumbnails-sprites.vtt b/test/test/assets/thumbnails-sprites.vtt index 42d2cf0fb5..b335f96544 100644 --- a/test/test/assets/thumbnails-sprites.vtt +++ b/test/test/assets/thumbnails-sprites.vtt @@ -4,7 +4,7 @@ WEBVTT image1.jpg#xywh=0,0,160,90 00:05.000 --> 00:30.000 -image2.jpg#xywh=0,0,160,90 +image2.jpg#xywh=160,0,160,90 00:30.000 --> 01:00.000 -image3.jpg#xywh=0,0,160,90 \ No newline at end of file +image3.jpg#xywh=160,90,160,90 \ No newline at end of file