Skip to content

Commit

Permalink
chore: refactor transcode config routing (immich-app#9800)
Browse files Browse the repository at this point in the history 
* chore: refactor transcode config

* rename parameter

* handle no /dev/dri

* prefer undefined
  • Loading branch information
@mertalev
mertalev authored May 27, 2024
1 parent 21bd20f commit dca420e
Show file tree
Hide file tree
Showing 4 changed files with 115 additions and 123 deletions.
6 changes: 3 additions & 3 deletions server/src/interfaces/media.interface.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ export interface VideoInfo {
audioStreams: AudioStreamInfo[];
}

export interface TranscodeOptions {
export interface TranscodeCommand {
inputOptions: string[];
outputOptions: string[];
twoPass: boolean;
Expand All @@ -67,7 +67,7 @@ export interface BitrateDistribution {
}

export interface VideoCodecSWConfig {
getOptions(target: TranscodeTarget, videoStream: VideoStreamInfo, audioStream: AudioStreamInfo): TranscodeOptions;
getCommand(target: TranscodeTarget, videoStream: VideoStreamInfo, audioStream: AudioStreamInfo): TranscodeCommand;
}

export interface VideoCodecHWConfig extends VideoCodecSWConfig {
Expand All @@ -83,5 +83,5 @@ export interface IMediaRepository {

// video
probe(input: string): Promise<VideoInfo>;
transcode(input: string, output: string | Writable, options: TranscodeOptions): Promise<void>;
transcode(input: string, output: string | Writable, command: TranscodeCommand): Promise<void>;
}
6 changes: 3 additions & 3 deletions server/src/repositories/media.repository.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import {
IMediaRepository,
ImageDimensions,
ThumbnailOptions,
TranscodeOptions,
TranscodeCommand,
VideoInfo,
} from 'src/interfaces/media.interface';
import { Instrumentation } from 'src/utils/instrumentation';
Expand Down Expand Up @@ -97,7 +97,7 @@ export class MediaRepository implements IMediaRepository {
};
}

transcode(input: string, output: string | Writable, options: TranscodeOptions): Promise<void> {
transcode(input: string, output: string | Writable, options: TranscodeCommand): Promise<void> {
if (!options.twoPass) {
return new Promise((resolve, reject) => {
this.configureFfmpegCall(input, output, options).on('error', reject).on('end', resolve).run();
Expand Down Expand Up @@ -150,7 +150,7 @@ export class MediaRepository implements IMediaRepository {
return { width, height };
}

private configureFfmpegCall(input: string, output: string | Writable, options: TranscodeOptions) {
private configureFfmpegCall(input: string, output: string | Writable, options: TranscodeCommand) {
return ffmpeg(input, { niceness: 10 })
.inputOptions(options.inputOptions)
.outputOptions(options.outputOptions)
Expand Down
148 changes: 37 additions & 111 deletions server/src/services/media.service.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,34 +27,21 @@ import {
QueueName,
} from 'src/interfaces/job.interface';
import { ILoggerRepository } from 'src/interfaces/logger.interface';
import { AudioStreamInfo, IMediaRepository, VideoCodecHWConfig, VideoStreamInfo } from 'src/interfaces/media.interface';
import { AudioStreamInfo, IMediaRepository, VideoStreamInfo } from 'src/interfaces/media.interface';
import { IMoveRepository } from 'src/interfaces/move.interface';
import { IPersonRepository } from 'src/interfaces/person.interface';
import { IStorageRepository } from 'src/interfaces/storage.interface';
import { ISystemMetadataRepository } from 'src/interfaces/system-metadata.interface';
import {
AV1Config,
H264Config,
HEVCConfig,
NvencHwDecodeConfig,
NvencSwDecodeConfig,
QsvHwDecodeConfig,
QsvSwDecodeConfig,
RkmppHwDecodeConfig,
RkmppSwDecodeConfig,
ThumbnailConfig,
VAAPIConfig,
VP9Config,
} from 'src/utils/media';
import { BaseConfig, ThumbnailConfig } from 'src/utils/media';
import { mimeTypes } from 'src/utils/mime-types';
import { usePagination } from 'src/utils/pagination';

@Injectable()
export class MediaService {
private configCore: SystemConfigCore;
private storageCore: StorageCore;
private openCL: boolean | null = null;
private devices: string[] | null = null;
private maliOpenCL?: boolean;
private devices?: string[];

constructor(
@Inject(IAssetRepository) private assetRepository: IAssetRepository,
Expand Down Expand Up @@ -232,8 +219,8 @@ export class MediaService {
return;
}
const mainAudioStream = this.getMainStream(audioStreams);
const config = { ...ffmpeg, targetResolution: size.toString() };
const options = new ThumbnailConfig(config).getOptions(TranscodeTarget.VIDEO, mainVideoStream, mainAudioStream);
const config = ThumbnailConfig.create({ ...ffmpeg, targetResolution: size.toString() });
const options = config.getCommand(TranscodeTarget.VIDEO, mainVideoStream, mainAudioStream);
await this.mediaRepository.transcode(asset.originalPath, path, options);
break;
}
Expand Down Expand Up @@ -331,8 +318,8 @@ export class MediaService {
return JobStatus.FAILED;
}

const { ffmpeg: config } = await this.configCore.getConfig();
const target = this.getTranscodeTarget(config, mainVideoStream, mainAudioStream);
const { ffmpeg } = await this.configCore.getConfig();
const target = this.getTranscodeTarget(ffmpeg, mainVideoStream, mainAudioStream);
if (target === TranscodeTarget.NONE) {
if (asset.encodedVideoPath) {
this.logger.log(`Transcoded video exists for asset ${asset.id}, but is no longer required. Deleting...`);
Expand All @@ -343,30 +330,28 @@ export class MediaService {
return JobStatus.SKIPPED;
}

let transcodeOptions;
let command;
try {
transcodeOptions = await this.getCodecConfig(config).then((c) =>
c.getOptions(target, mainVideoStream, mainAudioStream),
);
const config = BaseConfig.create(ffmpeg, await this.getDevices(), await this.hasMaliOpenCL());
command = config.getCommand(target, mainVideoStream, mainAudioStream);
} catch (error) {
this.logger.error(`An error occurred while configuring transcoding options: ${error}`);
return JobStatus.FAILED;
}

this.logger.log(`Started encoding video ${asset.id} ${JSON.stringify(transcodeOptions)}`);
this.logger.log(`Started encoding video ${asset.id} ${JSON.stringify(command)}`);
try {
await this.mediaRepository.transcode(input, output, transcodeOptions);
await this.mediaRepository.transcode(input, output, command);
} catch (error) {
this.logger.error(error);
if (config.accel !== TranscodeHWAccel.DISABLED) {
if (ffmpeg.accel !== TranscodeHWAccel.DISABLED) {
this.logger.error(
`Error occurred during transcoding. Retrying with ${config.accel.toUpperCase()} acceleration disabled.`,
`Error occurred during transcoding. Retrying with ${ffmpeg.accel.toUpperCase()} acceleration disabled.`,
);
}
transcodeOptions = await this.getCodecConfig({ ...config, accel: TranscodeHWAccel.DISABLED }).then((c) =>
c.getOptions(target, mainVideoStream, mainAudioStream),
);
await this.mediaRepository.transcode(input, output, transcodeOptions);
const config = BaseConfig.create({ ...ffmpeg, accel: TranscodeHWAccel.DISABLED });
command = config.getCommand(target, mainVideoStream, mainAudioStream);
await this.mediaRepository.transcode(input, output, command);
}

this.logger.log(`Successfully encoded ${asset.id}`);
Expand All @@ -382,10 +367,10 @@ export class MediaService {

private getTranscodeTarget(
config: SystemConfigFFmpegDto,
videoStream: VideoStreamInfo | null,
audioStream: AudioStreamInfo | null,
videoStream?: VideoStreamInfo,
audioStream?: AudioStreamInfo,
): TranscodeTarget {
if (videoStream == null && audioStream == null) {
if (!videoStream && !audioStream) {
return TranscodeTarget.NONE;
}

Expand All @@ -407,8 +392,8 @@ export class MediaService {
return TranscodeTarget.NONE;
}

private isAudioTranscodeRequired(ffmpegConfig: SystemConfigFFmpegDto, stream: AudioStreamInfo | null): boolean {
if (stream == null) {
private isAudioTranscodeRequired(ffmpegConfig: SystemConfigFFmpegDto, stream?: AudioStreamInfo): boolean {
if (!stream) {
return false;
}

Expand All @@ -430,8 +415,8 @@ export class MediaService {
}
}

private isVideoTranscodeRequired(ffmpegConfig: SystemConfigFFmpegDto, stream: VideoStreamInfo | null): boolean {
if (stream == null) {
private isVideoTranscodeRequired(ffmpegConfig: SystemConfigFFmpegDto, stream?: VideoStreamInfo): boolean {
if (!stream) {
return false;
}

Expand Down Expand Up @@ -465,70 +450,6 @@ export class MediaService {
}
}

async getCodecConfig(config: SystemConfigFFmpegDto) {
if (config.accel === TranscodeHWAccel.DISABLED) {
return this.getSWCodecConfig(config);
}
return this.getHWCodecConfig(config);
}

private getSWCodecConfig(config: SystemConfigFFmpegDto) {
switch (config.targetVideoCodec) {
case VideoCodec.H264: {
return new H264Config(config);
}
case VideoCodec.HEVC: {
return new HEVCConfig(config);
}
case VideoCodec.VP9: {
return new VP9Config(config);
}
case VideoCodec.AV1: {
return new AV1Config(config);
}
default: {
throw new UnsupportedMediaTypeException(`Codec '${config.targetVideoCodec}' is unsupported`);
}
}
}

private async getHWCodecConfig(config: SystemConfigFFmpegDto) {
let handler: VideoCodecHWConfig;
switch (config.accel) {
case TranscodeHWAccel.NVENC: {
handler = config.accelDecode ? new NvencHwDecodeConfig(config) : new NvencSwDecodeConfig(config);
break;
}
case TranscodeHWAccel.QSV: {
handler = config.accelDecode
? new QsvHwDecodeConfig(config, await this.getDevices())
: new QsvSwDecodeConfig(config, await this.getDevices());
break;
}
case TranscodeHWAccel.VAAPI: {
handler = new VAAPIConfig(config, await this.getDevices());
break;
}
case TranscodeHWAccel.RKMPP: {
handler =
config.accelDecode && (await this.hasOpenCL())
? new RkmppHwDecodeConfig(config, await this.getDevices())
: new RkmppSwDecodeConfig(config, await this.getDevices());
break;
}
default: {
throw new UnsupportedMediaTypeException(`${config.accel.toUpperCase()} acceleration is unsupported`);
}
}
if (!handler.getSupportedCodecs().includes(config.targetVideoCodec)) {
throw new UnsupportedMediaTypeException(
`${config.accel.toUpperCase()} acceleration does not support codec '${config.targetVideoCodec.toUpperCase()}'. Supported codecs: ${handler.getSupportedCodecs()}`,
);
}

return handler;
}

isSRGB(asset: AssetEntity): boolean {
const { colorspace, profileDescription, bitsPerSample } = asset.exifInfo ?? {};
if (colorspace || profileDescription) {
Expand Down Expand Up @@ -567,24 +488,29 @@ export class MediaService {

private async getDevices() {
if (!this.devices) {
this.devices = await this.storageRepository.readdir('/dev/dri');
try {
this.devices = await this.storageRepository.readdir('/dev/dri');
} catch {
this.logger.debug('No devices found in /dev/dri.');
this.devices = [];
}
}

return this.devices;
}

private async hasOpenCL() {
if (this.openCL === null) {
private async hasMaliOpenCL() {
if (this.maliOpenCL === undefined) {
try {
const maliIcdStat = await this.storageRepository.stat('/etc/OpenCL/vendors/mali.icd');
const maliDeviceStat = await this.storageRepository.stat('/dev/mali0');
this.openCL = maliIcdStat.isFile() && maliDeviceStat.isCharacterDevice();
this.maliOpenCL = maliIcdStat.isFile() && maliDeviceStat.isCharacterDevice();
} catch {
this.logger.warn('OpenCL not available for transcoding, using CPU instead.');
this.openCL = false;
this.logger.debug('OpenCL not available for transcoding, using CPU decoding instead.');
this.maliOpenCL = false;
}
}

return this.openCL;
return this.maliOpenCL;
}
}
78 changes: 72 additions & 6 deletions server/src/utils/media.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,22 +3,84 @@ import { SystemConfigFFmpegDto } from 'src/dtos/system-config.dto';
import {
AudioStreamInfo,
BitrateDistribution,
TranscodeOptions,
TranscodeCommand,
VideoCodecHWConfig,
VideoCodecSWConfig,
VideoStreamInfo,
} from 'src/interfaces/media.interface';

class BaseConfig implements VideoCodecSWConfig {
presets = ['veryslow', 'slower', 'slow', 'medium', 'fast', 'faster', 'veryfast', 'superfast', 'ultrafast'];
constructor(protected config: SystemConfigFFmpegDto) {}
export class BaseConfig implements VideoCodecSWConfig {
readonly presets = ['veryslow', 'slower', 'slow', 'medium', 'fast', 'faster', 'veryfast', 'superfast', 'ultrafast'];
protected constructor(protected config: SystemConfigFFmpegDto) {}

getOptions(target: TranscodeTarget, videoStream: VideoStreamInfo, audioStream?: AudioStreamInfo) {
static create(config: SystemConfigFFmpegDto, devices: string[] = [], hasMaliOpenCL = false): VideoCodecSWConfig {
if (config.accel === TranscodeHWAccel.DISABLED) {
return this.getSWCodecConfig(config);
}
return this.getHWCodecConfig(config, devices, hasMaliOpenCL);
}

private static getSWCodecConfig(config: SystemConfigFFmpegDto) {
switch (config.targetVideoCodec) {
case VideoCodec.H264: {
return new H264Config(config);
}
case VideoCodec.HEVC: {
return new HEVCConfig(config);
}
case VideoCodec.VP9: {
return new VP9Config(config);
}
case VideoCodec.AV1: {
return new AV1Config(config);
}
default: {
throw new Error(`Codec '${config.targetVideoCodec}' is unsupported`);
}
}
}

private static getHWCodecConfig(config: SystemConfigFFmpegDto, devices: string[] = [], hasMaliOpenCL = false) {
let handler: VideoCodecHWConfig;
switch (config.accel) {
case TranscodeHWAccel.NVENC: {
handler = config.accelDecode ? new NvencHwDecodeConfig(config) : new NvencSwDecodeConfig(config);
break;
}
case TranscodeHWAccel.QSV: {
handler = config.accelDecode ? new QsvHwDecodeConfig(config, devices) : new QsvSwDecodeConfig(config, devices);
break;
}
case TranscodeHWAccel.VAAPI: {
handler = new VAAPIConfig(config, devices);
break;
}
case TranscodeHWAccel.RKMPP: {
handler =
config.accelDecode && hasMaliOpenCL
? new RkmppHwDecodeConfig(config, devices)
: new RkmppSwDecodeConfig(config, devices);
break;
}
default: {
throw new Error(`${config.accel.toUpperCase()} acceleration is unsupported`);
}
}
if (!handler.getSupportedCodecs().includes(config.targetVideoCodec)) {
throw new Error(
`${config.accel.toUpperCase()} acceleration does not support codec '${config.targetVideoCodec.toUpperCase()}'. Supported codecs: ${handler.getSupportedCodecs()}`,
);
}

return handler;
}

getCommand(target: TranscodeTarget, videoStream: VideoStreamInfo, audioStream?: AudioStreamInfo) {
const options = {
inputOptions: this.getBaseInputOptions(videoStream),
outputOptions: [...this.getBaseOutputOptions(target, videoStream, audioStream), '-v verbose'],
twoPass: this.eligibleForTwoPass(),
} as TranscodeOptions;
} as TranscodeCommand;
if ([TranscodeTarget.ALL, TranscodeTarget.VIDEO].includes(target)) {
const filters = this.getFilterOptions(videoStream);
if (filters.length > 0) {
Expand Down Expand Up @@ -318,6 +380,10 @@ export class BaseHWConfig extends BaseConfig implements VideoCodecHWConfig {
}

export class ThumbnailConfig extends BaseConfig {
static create(config: SystemConfigFFmpegDto): VideoCodecSWConfig {
return new ThumbnailConfig(config);
}

getBaseInputOptions(): string[] {
return ['-skip_frame nokey', '-sws_flags accurate_rnd+full_chroma_int'];
}
Expand Down

0 comments on commit dca420e

Please sign in to comment.