Skip to content

Commit

Permalink
feat: Add external thumbnails with sprites support (shaka-project#4584)
Browse files Browse the repository at this point in the history 
Continuation of shaka-project#4497
  • Loading branch information
@avelad
avelad authored Oct 18, 2022
1 parent 3b9af2e commit 86cb3e7
Show file tree
Hide file tree
Showing 5 changed files with 136 additions and 54 deletions.
5 changes: 4 additions & 1 deletion externs/shaka/player.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1419,7 +1419,8 @@ shaka.extern.LanguageRole;
* startTime: number,
* duration: number,
* uris: !Array.<string>,
* width: number
* width: number,
* sprite: boolean
* }}
*
* @property {number} imageHeight
Expand All @@ -1443,6 +1444,8 @@ shaka.extern.LanguageRole;
* given.
* @property {number} width
* The thumbnail width in px.
* @property {boolean} sprite
* Indicate if the thumbnail is a sprite.
* @exportDoc
*/
shaka.extern.Thumbnail;
Expand Down
44 changes: 44 additions & 0 deletions lib/media/segment_reference.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -243,6 +243,9 @@ shaka.media.SegmentReference = class {

/** @type {?shaka.extern.HlsAes128Key} */
this.hlsAes128Key = hlsAes128Key;

/** @type {?shaka.media.SegmentReference.ThumbnailSprite} */
this.thumbnailSprite = null;
}

/**
Expand Down Expand Up @@ -357,6 +360,26 @@ shaka.media.SegmentReference = class {
markAsUnavailable() {
this.status = shaka.media.SegmentReference.Status.UNAVAILABLE;
}

/**
* Set the segment's thumbnail sprite.
*
* @param {shaka.media.SegmentReference.ThumbnailSprite} thumbnailSprite
* @export
*/
setThumbnailSprite(thumbnailSprite) {
this.thumbnailSprite = thumbnailSprite;
}

/**
* Returns the segment's thumbnail sprite.
*
* @return {?shaka.media.SegmentReference.ThumbnailSprite}
* @export
*/
getThumbnailSprite() {
return this.thumbnailSprite;
}
};


Expand All @@ -380,3 +403,24 @@ shaka.media.SegmentReference.Status = {
* @typedef {shaka.media.InitSegmentReference|shaka.media.SegmentReference}
*/
shaka.media.AnySegmentReference;


/**
* @typedef {{
* height: number,
* positionX: number,
* positionY: number,
* width: number
* }}
*
* @property {number} height
* The thumbnail height in px.
* @property {number} positionX
* The thumbnail left position in px.
* @property {number} positionY
* The thumbnail top position in px.
* @property {number} width
* The thumbnail width in px.
* @export
*/
shaka.media.SegmentReference.ThumbnailSprite;
34 changes: 26 additions & 8 deletions lib/player.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3843,8 +3843,8 @@ shaka.Player = class extends shaka.util.FakeEventTarget {
const fullImageHeight = imageStream.height || 0;
const columns = parseInt(match[1], 10);
const rows = parseInt(match[2], 10);
const width = fullImageWidth / columns;
const height = fullImageHeight / rows;
let width = fullImageWidth / columns;
let height = fullImageHeight / rows;
const totalImages = columns * rows;
const segmentDuration = reference.trueEndTime - reference.startTime;
const thumbnailDuration =
Expand All @@ -3870,6 +3870,15 @@ shaka.Player = class extends shaka.util.FakeEventTarget {
positionX = (thumbnailPosition % columns) * width;
positionY = Math.floor(thumbnailPosition / columns) * height;
}
let sprite = false;
const thumbnailSprite = reference.getThumbnailSprite();
if (thumbnailSprite) {
sprite = true;
height = thumbnailSprite.height;
positionX = thumbnailSprite.positionX;
positionY = thumbnailSprite.positionY;
width = thumbnailSprite.width;
}
return {
imageHeight: fullImageHeight,
imageWidth: fullImageWidth,
Expand All @@ -3880,6 +3889,7 @@ shaka.Player = class extends shaka.util.FakeEventTarget {
duration: thumbnailDuration,
uris: reference.getUris(),
width: width,
sprite: sprite,
};
}
return null;
Expand Down Expand Up @@ -4839,11 +4849,7 @@ shaka.Player = class extends shaka.util.FakeEventTarget {
for (const cue of cues) {
const imageUri = shaka.util.ManifestParserUtils.resolveUris(
[uri], [cue.payload])[0];
if (imageUri.includes('#xywh')) {
shaka.log.alwaysWarn('Unsupported image uri', imageUri);
continue;
}
references.push(new shaka.media.SegmentReference(
const reference = new shaka.media.SegmentReference(
cue.startTime,
cue.endTime,
() => [imageUri],
Expand All @@ -4853,7 +4859,19 @@ shaka.Player = class extends shaka.util.FakeEventTarget {
/* timestampOffset= */ 0,
/* appendWindowStart= */ 0,
/* appendWindowEnd= */ Infinity,
));
);
if (imageUri.includes('#xywh')) {
const spriteInfo = imageUri.split('#xywh=')[1].split(',');
if (spriteInfo.length === 4) {
reference.setThumbnailSprite({
height: parseInt(spriteInfo[3], 10),
positionX: parseInt(spriteInfo[0], 10),
positionY: parseInt(spriteInfo[1], 10),
width: parseInt(spriteInfo[2], 10),
});
}
}
references.push(reference);
}

/** @type {shaka.extern.Stream} */
Expand Down
103 changes: 60 additions & 43 deletions test/player_integration.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -262,49 +262,6 @@ describe('Player', () => {
expect(cues.length).toBeGreaterThan(0);
});

it('skip appended thumbnails for external thumbnails with sprites',
async () => {
await player.load('test:sintel_no_text_compiled');
const locationUri = new goog.Uri(location.href);
const partialUri =
new goog.Uri('/base/test/test/assets/thumbnails-sprites.vtt');
const absoluteUri = locationUri.resolve(partialUri);
const newTrack =
await player.addThumbnailsTrack(absoluteUri.toString());

expect(player.getImageTracks()).toEqual([newTrack]);

const thumbnail1 = await player.getThumbnails(newTrack.id, 0);
expect(thumbnail1).toBe(null);
const thumbnail2 = await player.getThumbnails(newTrack.id, 10);
expect(thumbnail2).toBe(null);
const thumbnail3 = await player.getThumbnails(newTrack.id, 40);
expect(thumbnail3).toBe(null);
});

it('appends thumbnails for external thumbnails without sprites',
async () => {
await player.load('test:sintel_no_text_compiled');
const locationUri = new goog.Uri(location.href);
const partialUri =
new goog.Uri('/base/test/test/assets/thumbnails.vtt');
const absoluteUri = locationUri.resolve(partialUri);
const newTrack =
await player.addThumbnailsTrack(absoluteUri.toString());

expect(player.getImageTracks()).toEqual([newTrack]);

const thumbnail1 = await player.getThumbnails(newTrack.id, 0);
expect(thumbnail1.startTime).toBe(0);
expect(thumbnail1.duration).toBe(5);
const thumbnail2 = await player.getThumbnails(newTrack.id, 10);
expect(thumbnail2.startTime).toBe(5);
expect(thumbnail2.duration).toBe(25);
const thumbnail3 = await player.getThumbnails(newTrack.id, 40);
expect(thumbnail3.startTime).toBe(30);
expect(thumbnail3.duration).toBe(30);
});

// https://github.com/shaka-project/shaka-player/issues/2553
it('does not change the selected track', async () => {
player.configure('streaming.alwaysStreamText', false);
Expand Down Expand Up @@ -1277,4 +1234,64 @@ describe('Player', () => {
await expectAsync(player.load('test:sintel-hls-clearkey'))
.toBeRejectedWith(expectedError);
});

describe('addThumbnailsTrack', () => {
it('appends thumbnails for external thumbnails with sprites',
async () => {
await player.load('test:sintel_no_text_compiled');
const locationUri = new goog.Uri(location.href);
const partialUri =
new goog.Uri('/base/test/test/assets/thumbnails-sprites.vtt');
const absoluteUri = locationUri.resolve(partialUri);
const newTrack =
await player.addThumbnailsTrack(absoluteUri.toString());

expect(player.getImageTracks()).toEqual([newTrack]);

const thumbnail1 = await player.getThumbnails(newTrack.id, 0);
expect(thumbnail1.startTime).toBe(0);
expect(thumbnail1.duration).toBe(5);
expect(thumbnail1.height).toBe(90);
expect(thumbnail1.positionX).toBe(0);
expect(thumbnail1.positionY).toBe(0);
expect(thumbnail1.width).toBe(160);
const thumbnail2 = await player.getThumbnails(newTrack.id, 10);
expect(thumbnail2.startTime).toBe(5);
expect(thumbnail2.duration).toBe(25);
expect(thumbnail2.height).toBe(90);
expect(thumbnail2.positionX).toBe(160);
expect(thumbnail2.positionY).toBe(0);
expect(thumbnail2.width).toBe(160);
const thumbnail3 = await player.getThumbnails(newTrack.id, 40);
expect(thumbnail3.startTime).toBe(30);
expect(thumbnail3.duration).toBe(30);
expect(thumbnail3.height).toBe(90);
expect(thumbnail3.positionX).toBe(160);
expect(thumbnail3.positionY).toBe(90);
expect(thumbnail3.width).toBe(160);
});

it('appends thumbnails for external thumbnails without sprites',
async () => {
await player.load('test:sintel_no_text_compiled');
const locationUri = new goog.Uri(location.href);
const partialUri =
new goog.Uri('/base/test/test/assets/thumbnails.vtt');
const absoluteUri = locationUri.resolve(partialUri);
const newTrack =
await player.addThumbnailsTrack(absoluteUri.toString());

expect(player.getImageTracks()).toEqual([newTrack]);

const thumbnail1 = await player.getThumbnails(newTrack.id, 0);
expect(thumbnail1.startTime).toBe(0);
expect(thumbnail1.duration).toBe(5);
const thumbnail2 = await player.getThumbnails(newTrack.id, 10);
expect(thumbnail2.startTime).toBe(5);
expect(thumbnail2.duration).toBe(25);
const thumbnail3 = await player.getThumbnails(newTrack.id, 40);
expect(thumbnail3.startTime).toBe(30);
expect(thumbnail3.duration).toBe(30);
});
}); // describe('addThumbnailsTrack')
});
4 changes: 2 additions & 2 deletions test/test/assets/thumbnails-sprites.vtt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ WEBVTT
image1.jpg#xywh=0,0,160,90

00:05.000 --> 00:30.000
image2.jpg#xywh=0,0,160,90
image2.jpg#xywh=160,0,160,90

00:30.000 --> 01:00.000
image3.jpg#xywh=0,0,160,90
image3.jpg#xywh=160,90,160,90

0 comments on commit 86cb3e7

Please sign in to comment.