import { ToneMapping, TranscodeHWAccel, VideoCodec } from '@app/infra/entities'; import { SystemConfigFFmpegDto } from '../system-config/dto'; import { BitrateDistribution, TranscodeOptions, VideoCodecHWConfig, VideoCodecSWConfig, VideoStreamInfo, } from './media.repository'; class BaseConfig implements VideoCodecSWConfig { constructor(protected config: SystemConfigFFmpegDto) {} getOptions(stream: VideoStreamInfo) { const options = { inputOptions: this.getBaseInputOptions(), outputOptions: this.getBaseOutputOptions().concat('-v verbose'), twoPass: this.eligibleForTwoPass(), } as TranscodeOptions; const filters = this.getFilterOptions(stream); if (filters.length > 0) { options.outputOptions.push(`-vf ${filters.join(',')}`); } options.outputOptions.push(...this.getPresetOptions()); options.outputOptions.push(...this.getThreadOptions()); options.outputOptions.push(...this.getBitrateOptions()); return options; } getBaseInputOptions(): string[] { return []; } getBaseOutputOptions() { return [ `-acodec ${this.config.targetAudioCodec}`, // Makes a second pass moving the moov atom to the // beginning of the file for improved playback speed. '-movflags faststart', '-fps_mode passthrough', ]; } getFilterOptions(stream: VideoStreamInfo) { const options = []; if (this.shouldScale(stream)) { options.push(`scale=${this.getScaling(stream)}`); } if (this.shouldToneMap(stream)) { options.push(...this.getToneMapping()); } options.push('format=yuv420p'); return options; } getPresetOptions() { return [`-preset ${this.config.preset}`]; } getBitrateOptions() { const bitrates = this.getBitrateDistribution(); if (this.eligibleForTwoPass()) { return [ `-b:v ${bitrates.target}${bitrates.unit}`, `-minrate ${bitrates.min}${bitrates.unit}`, `-maxrate ${bitrates.max}${bitrates.unit}`, ]; } else if (bitrates.max > 0) { // -bufsize is the peak possible bitrate at any moment, while -maxrate is the max rolling average bitrate return [ `-crf ${this.config.crf}`, `-maxrate ${bitrates.max}${bitrates.unit}`, `-bufsize ${bitrates.max * 2}${bitrates.unit}`, ]; } else { return [`-crf ${this.config.crf}`]; } } getThreadOptions(): Array { if (this.config.threads <= 0) { return []; } return [`-threads ${this.config.threads}`]; } eligibleForTwoPass() { if (!this.config.twoPass || this.config.accel !== TranscodeHWAccel.DISABLED) { return false; } return this.isBitrateConstrained() || this.config.targetVideoCodec === VideoCodec.VP9; } getBitrateDistribution() { const max = this.getMaxBitrateValue(); const target = Math.ceil(max / 1.45); // recommended by https://developers.google.com/media/vp9/settings/vod const min = target / 2; const unit = this.getBitrateUnit(); return { max, target, min, unit } as BitrateDistribution; } getTargetResolution(stream: VideoStreamInfo) { if (this.config.targetResolution === 'original') { return Math.min(stream.height, stream.width); } return Number.parseInt(this.config.targetResolution); } shouldScale(stream: VideoStreamInfo) { return Math.min(stream.height, stream.width) > this.getTargetResolution(stream); } shouldToneMap(stream: VideoStreamInfo) { return stream.isHDR && this.config.tonemap !== ToneMapping.DISABLED; } getScaling(stream: VideoStreamInfo) { const targetResolution = this.getTargetResolution(stream); const mult = this.config.accel === TranscodeHWAccel.QSV ? 1 : 2; // QSV doesn't support scaling numbers below -1 return this.isVideoVertical(stream) ? `${targetResolution}:-${mult}` : `-${mult}:${targetResolution}`; } isVideoRotated(stream: VideoStreamInfo) { return Math.abs(stream.rotation) === 90; } isVideoVertical(stream: VideoStreamInfo) { return stream.height > stream.width || this.isVideoRotated(stream); } isBitrateConstrained() { return this.getMaxBitrateValue() > 0; } getBitrateUnit() { const maxBitrate = this.getMaxBitrateValue(); return this.config.maxBitrate.trim().substring(maxBitrate.toString().length); // use inputted unit if provided } getMaxBitrateValue() { return Number.parseInt(this.config.maxBitrate) || 0; } getPresetIndex() { const presets = ['veryslow', 'slower', 'slow', 'medium', 'fast', 'faster', 'veryfast', 'superfast', 'ultrafast']; return presets.indexOf(this.config.preset); } getColors() { return { primaries: 'bt709', transfer: 'bt709', matrix: 'bt709', }; } getToneMapping() { const colors = this.getColors(); // npl stands for nominal peak luminance // lower npl values result in brighter output (compensating for dimmer screens) // since hable already outputs a darker image, we use a lower npl value for it const npl = this.config.tonemap === ToneMapping.HABLE ? 100 : 250; return [ `zscale=t=linear:npl=${npl}`, `tonemap=${this.config.tonemap}:desat=0`, `zscale=p=${colors.primaries}:t=${colors.transfer}:m=${colors.matrix}:range=pc`, ]; } } export class BaseHWConfig extends BaseConfig implements VideoCodecHWConfig { protected devices: string[]; constructor(protected config: SystemConfigFFmpegDto, devices: string[] = []) { super(config); this.devices = this.validateDevices(devices); } getSupportedCodecs() { return [VideoCodec.H264, VideoCodec.HEVC, VideoCodec.VP9]; } validateDevices(devices: string[]) { return devices .filter((device) => device.startsWith('renderD') || device.startsWith('card')) .sort((a, b) => { // order GPU devices first if (a.startsWith('card') && b.startsWith('renderD')) { return -1; } if (a.startsWith('renderD') && b.startsWith('card')) { return 1; } return -a.localeCompare(b); }); } } export class ThumbnailConfig extends BaseConfig { getBaseOutputOptions() { return ['-ss 00:00:00.000', '-frames:v 1']; } getPresetOptions() { return []; } getBitrateOptions() { return []; } getScaling(stream: VideoStreamInfo) { let options = super.getScaling(stream); if (!this.shouldToneMap(stream)) { options += ':out_color_matrix=bt601:out_range=pc'; } return options; } getColors() { return { // jpeg and webp only support bt.601, so we need to convert to that directly when tone-mapping to avoid color shifts primaries: 'bt470bg', transfer: '601', matrix: 'bt470bg', }; } } export class H264Config extends BaseConfig { getBaseOutputOptions() { return [`-vcodec ${this.config.targetVideoCodec}`, ...super.getBaseOutputOptions()]; } getThreadOptions() { if (this.config.threads <= 0) { return []; } return [ ...super.getThreadOptions(), '-x264-params "pools=none"', `-x264-params "frame-threads=${this.config.threads}"`, ]; } } export class HEVCConfig extends BaseConfig { getBaseOutputOptions() { return [`-vcodec ${this.config.targetVideoCodec}`, ...super.getBaseOutputOptions()]; } getThreadOptions() { if (this.config.threads <= 0) { return []; } return [ ...super.getThreadOptions(), '-x265-params "pools=none"', `-x265-params "frame-threads=${this.config.threads}"`, ]; } } export class VP9Config extends BaseConfig { getBaseOutputOptions() { return [`-vcodec ${this.config.targetVideoCodec}`, ...super.getBaseOutputOptions()]; } getPresetOptions() { const speed = Math.min(this.getPresetIndex(), 5); // values over 5 require realtime mode, which is its own can of worms since it overrides -crf and -threads if (speed >= 0) { return [`-cpu-used ${speed}`]; } return []; } getBitrateOptions() { const bitrates = this.getBitrateDistribution(); if (this.eligibleForTwoPass()) { return [ `-b:v ${bitrates.target}${bitrates.unit}`, `-minrate ${bitrates.min}${bitrates.unit}`, `-maxrate ${bitrates.max}${bitrates.unit}`, ]; } return [`-crf ${this.config.crf}`, `-b:v ${bitrates.max}${bitrates.unit}`]; } getThreadOptions() { return ['-row-mt 1', ...super.getThreadOptions()]; } } export class NVENCConfig extends BaseHWConfig { getSupportedCodecs() { return [VideoCodec.H264, VideoCodec.HEVC]; } getBaseInputOptions() { return ['-init_hw_device cuda=cuda:0', '-filter_hw_device cuda']; } getBaseOutputOptions() { return [ `-vcodec ${this.config.targetVideoCodec}_nvenc`, // below settings recommended from https://docs.nvidia.com/video-technologies/video-codec-sdk/12.0/ffmpeg-with-nvidia-gpu/index.html#command-line-for-latency-tolerant-high-quality-transcoding '-tune hq', '-qmin 0', '-g 250', '-bf 3', '-b_ref_mode middle', '-temporal-aq 1', '-rc-lookahead 20', '-i_qfactor 0.75', '-b_qfactor 1.1', ...super.getBaseOutputOptions(), ]; } getFilterOptions(stream: VideoStreamInfo) { const options = this.shouldToneMap(stream) ? this.getToneMapping() : []; options.push('format=nv12', 'hwupload_cuda'); if (this.shouldScale(stream)) { options.push(`scale_cuda=${this.getScaling(stream)}`); } return options; } getPresetOptions() { let presetIndex = this.getPresetIndex(); if (presetIndex < 0) { return []; } presetIndex = 7 - Math.min(6, presetIndex); // map to p1-p7; p7 is the highest quality, so reverse index return [`-preset p${presetIndex}`]; } getBitrateOptions() { const bitrates = this.getBitrateDistribution(); if (bitrates.max > 0 && this.config.twoPass) { return [ `-b:v ${bitrates.target}${bitrates.unit}`, `-maxrate ${bitrates.max}${bitrates.unit}`, `-bufsize ${bitrates.target}${bitrates.unit}`, '-multipass 2', ]; } else if (bitrates.max > 0) { return [ `-cq:v ${this.config.crf}`, `-maxrate ${bitrates.max}${bitrates.unit}`, `-bufsize ${bitrates.target}${bitrates.unit}`, ]; } else { return [`-cq:v ${this.config.crf}`]; } } getThreadOptions() { return []; } } export class QSVConfig extends BaseHWConfig { getBaseInputOptions() { if (!this.devices.length) { throw Error('No QSV device found'); } return ['-init_hw_device qsv=hw', '-filter_hw_device hw']; } getBaseOutputOptions() { // recommended from https://github.com/intel/media-delivery/blob/master/doc/benchmarks/intel-iris-xe-max-graphics/intel-iris-xe-max-graphics.md const options = [ `-vcodec ${this.config.targetVideoCodec}_qsv`, '-g 256', '-extbrc 1', '-refs 5', '-bf 7', ...super.getBaseOutputOptions(), ]; // VP9 requires enabling low power mode https://git.ffmpeg.org/gitweb/ffmpeg.git/commit/33583803e107b6d532def0f9d949364b01b6ad5a if (this.config.targetVideoCodec === VideoCodec.VP9) { options.push('-low_power 1'); } return options; } getFilterOptions(stream: VideoStreamInfo) { const options = this.shouldToneMap(stream) ? this.getToneMapping() : []; options.push('format=nv12', 'hwupload=extra_hw_frames=64'); if (this.shouldScale(stream)) { options.push(`scale_qsv=${this.getScaling(stream)}`); } return options; } getPresetOptions() { let presetIndex = this.getPresetIndex(); if (presetIndex < 0) { return []; } presetIndex = Math.min(6, presetIndex) + 1; // 1 to 7 return [`-preset ${presetIndex}`]; } getBitrateOptions() { const options = []; if (this.config.targetVideoCodec !== VideoCodec.VP9) { options.push(`-global_quality ${this.config.crf}`); } else { options.push(`-q:v ${this.config.crf}`); } const bitrates = this.getBitrateDistribution(); if (bitrates.max > 0) { options.push(`-maxrate ${bitrates.max}${bitrates.unit}`); options.push(`-bufsize ${bitrates.max * 2}${bitrates.unit}`); } return options; } } export class VAAPIConfig extends BaseHWConfig { getBaseInputOptions() { if (this.devices.length === 0) { throw Error('No VAAPI device found'); } return [`-init_hw_device vaapi=accel:/dev/dri/${this.devices[0]}`, '-filter_hw_device accel']; } getBaseOutputOptions() { return [`-vcodec ${this.config.targetVideoCodec}_vaapi`, ...super.getBaseOutputOptions()]; } getFilterOptions(stream: VideoStreamInfo) { const options = this.shouldToneMap(stream) ? this.getToneMapping() : []; options.push('format=nv12', 'hwupload'); if (this.shouldScale(stream)) { options.push(`scale_vaapi=${this.getScaling(stream)}`); } return options; } getPresetOptions() { let presetIndex = this.getPresetIndex(); if (presetIndex < 0) { return []; } presetIndex = Math.min(6, presetIndex) + 1; // 1 to 7 return [`-compression_level ${presetIndex}`]; } getBitrateOptions() { const bitrates = this.getBitrateDistribution(); // VAAPI doesn't allow setting both quality and max bitrate if (bitrates.max > 0) { return [ `-b:v ${bitrates.target}${bitrates.unit}`, `-maxrate ${bitrates.max}${bitrates.unit}`, `-minrate ${bitrates.min}${bitrates.unit}`, '-rc_mode 3', ]; // variable bitrate } else { return [`-qp ${this.config.crf}`, `-global_quality ${this.config.crf}`, '-rc_mode 1']; // constant quality } } }