Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: Add external thumbnails with sprites support #4584

Merged
merged 2 commits into from
Oct 18, 2022
Merged

feat: Add external thumbnails with sprites support #4584

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
78848d6
feat: Add external thumbnails with sprites support
avelad Oct 17, 2022
3ba2c9a
Merge branch 'main' into thumbnails-sprites
avelad Oct 17, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion externs/shaka/player.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1419,7 +1419,8 @@ shaka.extern.LanguageRole;
* startTime: number,
* duration: number,
* uris: !Array.<string>,
* width: number
* width: number,
* sprite: boolean
* }}
*
* @property {number} imageHeight
Expand All @@ -1443,6 +1444,8 @@ shaka.extern.LanguageRole;
* given.
* @property {number} width
* The thumbnail width in px.
* @property {boolean} sprite
* Indicate if the thumbnail is a sprite.
* @exportDoc
*/
shaka.extern.Thumbnail;
Expand Down
44 changes: 44 additions & 0 deletions lib/media/segment_reference.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -243,6 +243,9 @@ shaka.media.SegmentReference = class {

/** @type {?shaka.extern.HlsAes128Key} */
this.hlsAes128Key = hlsAes128Key;

/** @type {?shaka.media.SegmentReference.ThumbnailSprite} */
this.thumbnailSprite = null;
}

/**
Expand Down Expand Up @@ -357,6 +360,26 @@ shaka.media.SegmentReference = class {
markAsUnavailable() {
this.status = shaka.media.SegmentReference.Status.UNAVAILABLE;
}

/**
* Set the segment's thumbnail sprite.
*
* @param {shaka.media.SegmentReference.ThumbnailSprite} thumbnailSprite
* @export
*/
setThumbnailSprite(thumbnailSprite) {
this.thumbnailSprite = thumbnailSprite;
}

/**
* Returns the segment's thumbnail sprite.
*
* @return {?shaka.media.SegmentReference.ThumbnailSprite}
* @export
*/
getThumbnailSprite() {
return this.thumbnailSprite;
}
};


Expand All @@ -380,3 +403,24 @@ shaka.media.SegmentReference.Status = {
* @typedef {shaka.media.InitSegmentReference|shaka.media.SegmentReference}
*/
shaka.media.AnySegmentReference;


/**
* @typedef {{
* height: number,
* positionX: number,
* positionY: number,
* width: number
* }}
*
* @property {number} height
* The thumbnail height in px.
* @property {number} positionX
* The thumbnail left position in px.
* @property {number} positionY
* The thumbnail top position in px.
* @property {number} width
* The thumbnail width in px.
* @export
*/
shaka.media.SegmentReference.ThumbnailSprite;
34 changes: 26 additions & 8 deletions lib/player.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3843,8 +3843,8 @@ shaka.Player = class extends shaka.util.FakeEventTarget {
const fullImageHeight = imageStream.height || 0;
const columns = parseInt(match[1], 10);
const rows = parseInt(match[2], 10);
const width = fullImageWidth / columns;
const height = fullImageHeight / rows;
let width = fullImageWidth / columns;
let height = fullImageHeight / rows;
const totalImages = columns * rows;
const segmentDuration = reference.trueEndTime - reference.startTime;
const thumbnailDuration =
Expand All @@ -3870,6 +3870,15 @@ shaka.Player = class extends shaka.util.FakeEventTarget {
positionX = (thumbnailPosition % columns) * width;
positionY = Math.floor(thumbnailPosition / columns) * height;
}
let sprite = false;
const thumbnailSprite = reference.getThumbnailSprite();
if (thumbnailSprite) {
sprite = true;
height = thumbnailSprite.height;
positionX = thumbnailSprite.positionX;
positionY = thumbnailSprite.positionY;
width = thumbnailSprite.width;
}
return {
imageHeight: fullImageHeight,
imageWidth: fullImageWidth,
Expand All @@ -3880,6 +3889,7 @@ shaka.Player = class extends shaka.util.FakeEventTarget {
duration: thumbnailDuration,
uris: reference.getUris(),
width: width,
sprite: sprite,
};
}
return null;
Expand Down Expand Up @@ -4839,11 +4849,7 @@ shaka.Player = class extends shaka.util.FakeEventTarget {
for (const cue of cues) {
const imageUri = shaka.util.ManifestParserUtils.resolveUris(
[uri], [cue.payload])[0];
if (imageUri.includes('#xywh')) {
shaka.log.alwaysWarn('Unsupported image uri', imageUri);
continue;
}
references.push(new shaka.media.SegmentReference(
const reference = new shaka.media.SegmentReference(
cue.startTime,
cue.endTime,
() => [imageUri],
Expand All @@ -4853,7 +4859,19 @@ shaka.Player = class extends shaka.util.FakeEventTarget {
/* timestampOffset= */ 0,
/* appendWindowStart= */ 0,
/* appendWindowEnd= */ Infinity,
));
);
if (imageUri.includes('#xywh')) {
const spriteInfo = imageUri.split('#xywh=')[1].split(',');
if (spriteInfo.length === 4) {
reference.setThumbnailSprite({
height: parseInt(spriteInfo[3], 10),
positionX: parseInt(spriteInfo[0], 10),
positionY: parseInt(spriteInfo[1], 10),
width: parseInt(spriteInfo[2], 10),
});
}
}
references.push(reference);
}

/** @type {shaka.extern.Stream} */
Expand Down
103 changes: 60 additions & 43 deletions test/player_integration.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -262,49 +262,6 @@ describe('Player', () => {
expect(cues.length).toBeGreaterThan(0);
});

it('skip appended thumbnails for external thumbnails with sprites',
async () => {
await player.load('test:sintel_no_text_compiled');
const locationUri = new goog.Uri(location.href);
const partialUri =
new goog.Uri('/base/test/test/assets/thumbnails-sprites.vtt');
const absoluteUri = locationUri.resolve(partialUri);
const newTrack =
await player.addThumbnailsTrack(absoluteUri.toString());

expect(player.getImageTracks()).toEqual([newTrack]);

const thumbnail1 = await player.getThumbnails(newTrack.id, 0);
expect(thumbnail1).toBe(null);
const thumbnail2 = await player.getThumbnails(newTrack.id, 10);
expect(thumbnail2).toBe(null);
const thumbnail3 = await player.getThumbnails(newTrack.id, 40);
expect(thumbnail3).toBe(null);
});

it('appends thumbnails for external thumbnails without sprites',
async () => {
await player.load('test:sintel_no_text_compiled');
const locationUri = new goog.Uri(location.href);
const partialUri =
new goog.Uri('/base/test/test/assets/thumbnails.vtt');
const absoluteUri = locationUri.resolve(partialUri);
const newTrack =
await player.addThumbnailsTrack(absoluteUri.toString());

expect(player.getImageTracks()).toEqual([newTrack]);

const thumbnail1 = await player.getThumbnails(newTrack.id, 0);
expect(thumbnail1.startTime).toBe(0);
expect(thumbnail1.duration).toBe(5);
const thumbnail2 = await player.getThumbnails(newTrack.id, 10);
expect(thumbnail2.startTime).toBe(5);
expect(thumbnail2.duration).toBe(25);
const thumbnail3 = await player.getThumbnails(newTrack.id, 40);
expect(thumbnail3.startTime).toBe(30);
expect(thumbnail3.duration).toBe(30);
});

// https://github.com/shaka-project/shaka-player/issues/2553
it('does not change the selected track', async () => {
player.configure('streaming.alwaysStreamText', false);
Expand Down Expand Up @@ -1277,4 +1234,64 @@ describe('Player', () => {
await expectAsync(player.load('test:sintel-hls-clearkey'))
.toBeRejectedWith(expectedError);
});

describe('addThumbnailsTrack', () => {
it('appends thumbnails for external thumbnails with sprites',
async () => {
await player.load('test:sintel_no_text_compiled');
const locationUri = new goog.Uri(location.href);
const partialUri =
new goog.Uri('/base/test/test/assets/thumbnails-sprites.vtt');
const absoluteUri = locationUri.resolve(partialUri);
const newTrack =
await player.addThumbnailsTrack(absoluteUri.toString());

expect(player.getImageTracks()).toEqual([newTrack]);

const thumbnail1 = await player.getThumbnails(newTrack.id, 0);
expect(thumbnail1.startTime).toBe(0);
expect(thumbnail1.duration).toBe(5);
expect(thumbnail1.height).toBe(90);
expect(thumbnail1.positionX).toBe(0);
expect(thumbnail1.positionY).toBe(0);
expect(thumbnail1.width).toBe(160);
const thumbnail2 = await player.getThumbnails(newTrack.id, 10);
expect(thumbnail2.startTime).toBe(5);
expect(thumbnail2.duration).toBe(25);
expect(thumbnail2.height).toBe(90);
expect(thumbnail2.positionX).toBe(160);
expect(thumbnail2.positionY).toBe(0);
expect(thumbnail2.width).toBe(160);
const thumbnail3 = await player.getThumbnails(newTrack.id, 40);
expect(thumbnail3.startTime).toBe(30);
expect(thumbnail3.duration).toBe(30);
expect(thumbnail3.height).toBe(90);
expect(thumbnail3.positionX).toBe(160);
expect(thumbnail3.positionY).toBe(90);
expect(thumbnail3.width).toBe(160);
});

it('appends thumbnails for external thumbnails without sprites',
async () => {
await player.load('test:sintel_no_text_compiled');
const locationUri = new goog.Uri(location.href);
const partialUri =
new goog.Uri('/base/test/test/assets/thumbnails.vtt');
const absoluteUri = locationUri.resolve(partialUri);
const newTrack =
await player.addThumbnailsTrack(absoluteUri.toString());

expect(player.getImageTracks()).toEqual([newTrack]);

const thumbnail1 = await player.getThumbnails(newTrack.id, 0);
expect(thumbnail1.startTime).toBe(0);
expect(thumbnail1.duration).toBe(5);
const thumbnail2 = await player.getThumbnails(newTrack.id, 10);
expect(thumbnail2.startTime).toBe(5);
expect(thumbnail2.duration).toBe(25);
const thumbnail3 = await player.getThumbnails(newTrack.id, 40);
expect(thumbnail3.startTime).toBe(30);
expect(thumbnail3.duration).toBe(30);
});
}); // describe('addThumbnailsTrack')
});
4 changes: 2 additions & 2 deletions test/test/assets/thumbnails-sprites.vtt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ WEBVTT
image1.jpg#xywh=0,0,160,90

00:05.000 --> 00:30.000
image2.jpg#xywh=0,0,160,90
image2.jpg#xywh=160,0,160,90

00:30.000 --> 01:00.000
image3.jpg#xywh=0,0,160,90
image3.jpg#xywh=160,90,160,90