Skip to content

Commit

Permalink
Add Error Handling Integration Tests (WIP)
Browse files Browse the repository at this point in the history
  • Loading branch information
@robwalch
robwalch committed Feb 12, 2023
1 parent 3e578db commit 510f954
Show file tree
Hide file tree
Showing 32 changed files with 1,883 additions and 431 deletions.
704 changes: 685 additions & 19 deletions api-extractor/report/hls.js.api.md
View file

Large diffs are not rendered by default.

16 changes: 9 additions & 7 deletions src/config.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import EMEController, {
} from './controller/eme-controller';
import CMCDController from './controller/cmcd-controller';
import ContentSteeringController from './controller/content-steering-controller';
import ErrorController from './controller/error-controller';
import XhrLoader from './utils/xhr-loader';
import FetchLoader, { fetchSupported } from './utils/fetch-loader';
import Cues from './utils/cues';
Expand Down Expand Up @@ -235,7 +236,7 @@ export type HlsConfig = {
cmcdController?: typeof CMCDController;
// Content Steering
contentSteeringController?: typeof ContentSteeringController;

errorController: typeof ErrorController;
abrController: typeof AbrController;
bufferController: typeof BufferController;
capLevelController: typeof CapLevelController;
Expand Down Expand Up @@ -299,13 +300,13 @@ export const hlsDefaultConfig: HlsConfig = {
manifestLoadingMaxRetryTimeout: 64000, // used by playlist-loader
startLevel: undefined, // used by level-controller
levelLoadingTimeOut: 10000, // used by playlist-loader
levelLoadingMaxRetry: 4, // used by playlist-loader
levelLoadingRetryDelay: 1000, // used by playlist-loader
levelLoadingMaxRetryTimeout: 64000, // used by playlist-loader
levelLoadingMaxRetry: 4, // used by playlist/track controllers
levelLoadingRetryDelay: 1000, // used by playlist/track controllers
levelLoadingMaxRetryTimeout: 64000, // used by playlist/track controllers
fragLoadingTimeOut: 20000, // used by fragment-loader
fragLoadingMaxRetry: 6, // used by fragment-loader
fragLoadingRetryDelay: 1000, // used by fragment-loader
fragLoadingMaxRetryTimeout: 64000, // used by fragment-loader
fragLoadingMaxRetry: 6, // used by stream controllers
fragLoadingRetryDelay: 1000, // used by stream controllers
fragLoadingMaxRetryTimeout: 64000, // used by stream controllers
startFragPrefetch: false, // used by stream-controller
fpsDroppedMonitoringPeriod: 5000, // used by fps-controller
fpsDroppedMonitoringThreshold: 0.2, // used by fps-controller
Expand Down Expand Up @@ -366,6 +367,7 @@ export const hlsDefaultConfig: HlsConfig = {
contentSteeringController: __USE_CONTENT_STEERING__
? ContentSteeringController
: undefined,
errorController: ErrorController,
};

function timelineConfig(): TimelineControllerConfig {
Expand Down
7 changes: 5 additions & 2 deletions src/controller/audio-track-controller.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -146,7 +146,6 @@ class AudioTrackController extends BasePlaylistController {
}

protected onError(event: Events.ERROR, data: ErrorData): void {
super.onError(event, data);
if (data.fatal || !data.context) {
return;
}
Expand Down Expand Up @@ -217,12 +216,16 @@ class AudioTrackController extends BasePlaylistController {
if (trackId !== -1) {
this.setAudioTrack(trackId);
} else {
this.warn(`No track found for running audio group-ID: ${this.groupId}`);
const error = new Error(
`No track found for running audio group-ID: ${this.groupId}`
);
this.warn(error.message);

this.hls.trigger(Events.ERROR, {
type: ErrorTypes.MEDIA_ERROR,
details: ErrorDetails.AUDIO_TRACK_LOAD_ERROR,
fatal: true,
error,
});
}
}
Expand Down
18 changes: 6 additions & 12 deletions src/controller/base-playlist-controller.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ import type {
} from '../types/events';
import { ErrorData } from '../types/events';
import { Events } from '../events';
import { ErrorTypes } from '../errors';
import { ErrorDetails, ErrorTypes } from '../errors';

export default class BasePlaylistController implements NetworkComponentAPI {
protected hls: Hls;
Expand All @@ -35,16 +35,6 @@ export default class BasePlaylistController implements NetworkComponentAPI {
this.hls = this.log = this.warn = null;
}

protected onError(event: Events.ERROR, data: ErrorData): void {
if (
data.fatal &&
(data.type === ErrorTypes.NETWORK_ERROR ||
data.type === ErrorTypes.KEY_SYSTEM_ERROR)
) {
this.stopLoad();
}
}

protected clearTimer(): void {
clearTimeout(this.timer);
this.timer = -1;
Expand Down Expand Up @@ -318,7 +308,11 @@ export default class BasePlaylistController implements NetworkComponentAPI {
// exponential backoff capped to max retry timeout
const delay = Math.min(
Math.pow(2, this.retryCount) * config.levelLoadingRetryDelay,
config.levelLoadingMaxRetryTimeout
errorEvent.details === ErrorDetails.LEVEL_LOAD_TIMEOUT ||
errorEvent.details === ErrorDetails.AUDIO_TRACK_LOAD_TIMEOUT ||
errorEvent.details === ErrorDetails.SUBTITLE_TRACK_LOAD_TIMEOUT
? 0
: config.levelLoadingMaxRetryTimeout
);
// Schedule level/track reload
this.timer = self.setTimeout(() => this.loadPlaylist(), delay);
Expand Down
95 changes: 51 additions & 44 deletions src/controller/base-stream-controller.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -624,14 +624,9 @@ export default class BaseStreamController
);
this.nextLoadPosition = part.start + part.duration;
this.state = State.FRAG_LOADING;
this.hls.trigger(Events.FRAG_LOADING, {
frag,
part,
targetBufferTime,
});
this.throwIfFragContextChanged('FRAG_LOADING parts');
let result: Promise<PartsLoadedData | FragLoadedData | null>;
if (keyLoadingPromise) {
return keyLoadingPromise
result = keyLoadingPromise
.then((keyLoadedData) => {
if (
!keyLoadedData ||
Expand All @@ -647,14 +642,21 @@ export default class BaseStreamController
);
})
.catch((error) => this.handleFragLoadError(error));
} else {
result = this.doFragPartsLoad(
frag,
part,
level,
progressCallback
).catch((error: LoadError) => this.handleFragLoadError(error));
}

return this.doFragPartsLoad(
this.hls.trigger(Events.FRAG_LOADING, {
frag,
part,
level,
progressCallback
).catch((error: LoadError) => this.handleFragLoadError(error));
targetBufferTime,
});
this.throwIfFragContextChanged('FRAG_LOADING parts');
return result;
} else if (
!frag.url ||
this.loadedEndOfParts(partList, targetBufferTime)
Expand All @@ -677,38 +679,40 @@ export default class BaseStreamController
this.nextLoadPosition = frag.start + frag.duration;
}
this.state = State.FRAG_LOADING;
this.hls.trigger(Events.FRAG_LOADING, { frag, targetBufferTime });
this.throwIfFragContextChanged('FRAG_LOADING');

// Load key before streaming fragment data
const dataOnProgress = this.config.progressive;
let result: Promise<PartsLoadedData | FragLoadedData | null>;
if (dataOnProgress && keyLoadingPromise) {
return keyLoadingPromise
result = keyLoadingPromise
.then((keyLoadedData) => {
if (!keyLoadedData || this.fragContextChanged(keyLoadedData?.frag)) {
return null;
}
return this.fragmentLoader.load(frag, progressCallback);
})
.catch((error) => this.handleFragLoadError(error));
} else {
// load unencrypted fragment data with progress event,
// or handle fragment result after key and fragment are finished loading
result = Promise.all([
this.fragmentLoader.load(
frag,
dataOnProgress ? progressCallback : undefined
),
keyLoadingPromise,
])
.then(([fragLoadedData]) => {
if (!dataOnProgress && fragLoadedData && progressCallback) {
progressCallback(fragLoadedData);
}
return fragLoadedData;
})
.catch((error) => this.handleFragLoadError(error));
}

// load unencrypted fragment data with progress event,
// or handle fragment result after key and fragment are finished loading
return Promise.all([
this.fragmentLoader.load(
frag,
dataOnProgress ? progressCallback : undefined
),
keyLoadingPromise,
])
.then(([fragLoadedData]) => {
if (!dataOnProgress && fragLoadedData && progressCallback) {
progressCallback(fragLoadedData);
}
return fragLoadedData;
})
.catch((error) => this.handleFragLoadError(error));
this.hls.trigger(Events.FRAG_LOADING, { frag, targetBufferTime });
this.throwIfFragContextChanged('FRAG_LOADING');
return result;
}

private throwIfFragContextChanged(context: string): void | never {
Expand Down Expand Up @@ -768,6 +772,7 @@ export default class BaseStreamController
type: ErrorTypes.OTHER_ERROR,
details: ErrorDetails.INTERNAL_EXCEPTION,
err: error,
error,
fatal: true,
});
}
Expand Down Expand Up @@ -1360,14 +1365,11 @@ export default class BaseStreamController
if (!frag || frag.type !== filterType) {
return;
}
const fragCurrent = this.fragCurrent;
console.assert(
fragCurrent &&
frag.sn === fragCurrent.sn &&
frag.level === fragCurrent.level &&
frag.urlId === fragCurrent.urlId,
'Frag load error must match current frag to retry'
);
if (this.fragContextChanged(frag)) {
this.warn('Frag load error must match current frag to retry');
return;
}

// keep retrying until the limit will be reached
if (this.fragLoadError + 1 <= config.fragLoadingMaxRetry) {
if (!this.loadedmetadata) {
Expand All @@ -1377,13 +1379,16 @@ export default class BaseStreamController
// exponential backoff capped to config.fragLoadingMaxRetryTimeout
const delay = Math.min(
Math.pow(2, this.fragLoadError) * config.fragLoadingRetryDelay,
config.fragLoadingMaxRetryTimeout
data.details === ErrorDetails.FRAG_LOAD_TIMEOUT ||
data.details === ErrorDetails.KEY_LOAD_TIMEOUT
? 0
: config.fragLoadingMaxRetryTimeout
);
this.fragLoadError++;
this.warn(
`Fragment ${frag.sn} of ${filterType} ${frag.level} failed to load, retrying in ${delay}ms`
`Fragment ${frag.sn} of ${filterType} ${frag.level} errored with ${data.details}, retrying ${this.fragLoadError}/${config.fragLoadingMaxRetry} in ${delay}ms`
);
this.retryDate = self.performance.now() + delay;
this.fragLoadError++;
this.state = State.FRAG_LOADING_WAITING_RETRY;
} else if (data.levelRetry) {
if (filterType === PlaylistLevelType.AUDIO) {
Expand All @@ -1393,14 +1398,16 @@ export default class BaseStreamController
// Fragment errors that result in a level switch or redundant fail-over
// should reset the stream controller state to idle
this.fragLoadError = 0;
if (!this.loadedmetadata) {
this.startFragRequested = false;
}
this.state = State.IDLE;
} else {
logger.error(
`${data.details} reaches max retry, redispatch as fatal ...`
);
// switch error to fatal
data.fatal = true;
this.hls.stopLoad();
this.state = State.ERROR;
}
}
Expand Down
28 changes: 15 additions & 13 deletions src/controller/buffer-controller.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -415,6 +415,7 @@ export default class BufferController implements ComponentAPI {
parent: frag.type,
details: ErrorDetails.BUFFER_APPEND_ERROR,
err,
error: err,
fatal: false,
};

Expand All @@ -433,7 +434,6 @@ export default class BufferController implements ComponentAPI {
`[buffer-controller]: Failed ${hls.config.appendErrorMaxRetry} times to append segment in sourceBuffer`
);
event.fatal = true;
hls.stopLoad();
}
}
hls.trigger(Events.ERROR, event);
Expand Down Expand Up @@ -717,18 +717,23 @@ export default class BufferController implements ComponentAPI {
this.pendingTracks = {};
// append any pending segments now !
const buffers = this.getSourceBufferTypes();
if (buffers.length === 0) {
if (buffers.length) {
this.hls.trigger(Events.BUFFER_CREATED, { tracks: this.tracks });
buffers.forEach((type: SourceBufferName) => {
operationQueue.executeNext(type);
});
} else {
const error = new Error(
'could not create source buffer for media codec(s)'
);
this.hls.trigger(Events.ERROR, {
type: ErrorTypes.MEDIA_ERROR,
details: ErrorDetails.BUFFER_INCOMPATIBLE_CODECS_ERROR,
fatal: true,
reason: 'could not create source buffer for media codec(s)',
error,
reason: error.message,
});
return;
}
buffers.forEach((type: SourceBufferName) => {
operationQueue.executeNext(type);
});
}
}

Expand All @@ -737,7 +742,6 @@ export default class BufferController implements ComponentAPI {
if (!mediaSource) {
throw Error('createSourceBuffers called when mediaSource was null');
}
let tracksCreated = 0;
for (const trackName in tracks) {
if (!sourceBuffer[trackName]) {
const track = tracks[trackName as keyof TrackSet];
Expand Down Expand Up @@ -765,7 +769,6 @@ export default class BufferController implements ComponentAPI {
metadata: track.metadata,
id: track.id,
};
tracksCreated++;
} catch (err) {
logger.error(
`[buffer-controller]: error while trying to add sourceBuffer: ${err.message}`
Expand All @@ -780,9 +783,6 @@ export default class BufferController implements ComponentAPI {
}
}
}
if (tracksCreated) {
this.hls.trigger(Events.BUFFER_CREATED, { tracks: this.tracks });
}
}

// Keep as arrow functions so that we can directly reference these functions directly as event listeners
Expand Down Expand Up @@ -833,12 +833,14 @@ export default class BufferController implements ComponentAPI {
}

private _onSBUpdateError(type: SourceBufferName, event: Event) {
logger.error(`[buffer-controller]: ${type} SourceBuffer error`, event);
const error = new Error(`${type} SourceBuffer error`);
logger.error(`[buffer-controller]: ${error}`, event);
// according to http://www.w3.org/TR/media-source/#sourcebuffer-append-error
// SourceBuffer errors are not necessarily fatal; if so, the HTMLMediaElement will fire an error event
this.hls.trigger(Events.ERROR, {
type: ErrorTypes.MEDIA_ERROR,
details: ErrorDetails.BUFFER_APPENDING_ERROR,
error,
fatal: false,
});
// updateend is always fired after error, so we'll allow that to shift the current operation off of the queue
Expand Down
Loading

0 comments on commit 510f954

Please sign in to comment.