helpy4/immich
1
0
Fork 0
mirror of https://github.com/immich-app/immich synced 2026-08-15 13:03:57 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge 2bc11a9003 into 63b90ec43d

Browse source
This commit is contained in:
Raanelom 2026-08-06 13:34:04 +02:00 • committed by GitHub
commit 25c0c37aaa
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
34 changed files with 1535 additions and 11 deletions
Download patch file Download diff file Expand all files Collapse all files

41
open-api/immich-openapi-specs.json
View file

@ -19348,7 +19348,9 @@
"IntegrityChecksumFiles",
"IntegrityChecksumFilesRefresh",
"IntegrityDeleteReportType",
"IntegrityDeleteReports"
"IntegrityDeleteReports",
"FrameSamplingQueueAll",
"FrameSampling"
],
"type": "string"
},
@ -25603,6 +25605,9 @@
"ffmpeg": {
"$ref": "#/components/schemas/SystemConfigFFmpegDto"
},
"frameSampling": {
"$ref": "#/components/schemas/SystemConfigFrameSamplingDto"
},
"image": {
"$ref": "#/components/schemas/SystemConfigImageDto"
},
@ -25667,6 +25672,7 @@
"required": [
"backup",
"ffmpeg",
"frameSampling",
"image",
"integrityChecks",
"job",
@ -25859,6 +25865,39 @@
],
"type": "object"
},
"SystemConfigFrameSamplingDto": {
"properties": {
"enabled": {
"description": "Enable frame sampling",
"type": "boolean"
},
"frameInterval": {
"description": "Seconds between sampled frames",
"format": "double",
"minimum": 0.01,
"type": "number"
},
"qp": {
"description": "Target quality (CRF-equivalent) used for the all-intra frame encode",
"maximum": 51,
"minimum": 0,
"type": "integer"
},
"targetResolution": {
"description": "Target short-side resolution (px) of extracted frames",
"maximum": 9007199254740991,
"minimum": 1,
"type": "integer"
}
},
"required": [
"enabled",
"frameInterval",
"qp",
"targetResolution"
],
"type": "object"
},
"SystemConfigGeneratedFullsizeImageDto": {
"properties": {
"enabled": {

15
packages/sdk/src/fetch-client.ts
View file

@ -2346,6 +2346,16 @@ export type SystemConfigFFmpegDto = {
/** Two pass */
twoPass: boolean;
};
export type SystemConfigFrameSamplingDto = {
/** Enable frame sampling */
enabled: boolean;
/** Seconds between sampled frames */
frameInterval: number;
/** Target quality (CRF-equivalent) used for the all-intra frame encode */
qp: number;
/** Target short-side resolution (px) of extracted frames */
targetResolution: number;
};
export type SystemConfigGeneratedFullsizeImageDto = {
/** Enabled */
enabled: boolean;
@ -2618,6 +2628,7 @@ export type SystemConfigUserDto = {
export type SystemConfigDto = {
backup: SystemConfigBackupsDto;
ffmpeg: SystemConfigFFmpegDto;
frameSampling: SystemConfigFrameSamplingDto;
image: SystemConfigImageDto;
integrityChecks: SystemConfigIntegrityChecks;
job: SystemConfigJobDto;
@ -7499,7 +7510,9 @@ export enum JobName {
IntegrityChecksumFiles = "IntegrityChecksumFiles",
IntegrityChecksumFilesRefresh = "IntegrityChecksumFilesRefresh",
IntegrityDeleteReportType = "IntegrityDeleteReportType",
IntegrityDeleteReports = "IntegrityDeleteReports"
IntegrityDeleteReports = "IntegrityDeleteReports",
FrameSamplingQueueAll = "FrameSamplingQueueAll",
FrameSampling = "FrameSampling"
}
export enum SearchSuggestionType {
Country = "country",

12
server/src/config.ts
View file

@ -214,6 +214,12 @@ export type SystemConfig = {
user: {
deleteDelay: number;
};
frameSampling: {
enabled: boolean;
targetResolution: number;
qp: number;
frameInterval: number;
};
};
export type MachineLearningConfig = SystemConfig['machineLearning'];
@ -446,4 +452,10 @@ export const defaults = Object.freeze<SystemConfig>({
user: {
deleteDelay: 7,
},
frameSampling: {
enabled: false,
targetResolution: 640,
qp: 34,
frameInterval: 1,
},
});

7
server/src/cores/storage.core.ts
View file

@ -130,6 +130,10 @@ export class StorageCore {
return StorageCore.getNestedPath(StorageFolder.EncodedVideo, asset.ownerId, `${asset.id}.mp4`);
}
static getVideoFrameArtifactPath(asset: ThumbnailPathEntity) {
return StorageCore.getNestedPath(StorageFolder.EncodedVideo, asset.ownerId, `${asset.id}.m4s`);
}
static getHlsSessionFolder({ ownerId, sessionId }: HlsSessionFolder) {
return StorageCore.getNestedPath(StorageFolder.EncodedVideo, ownerId, sessionId);
}
@ -329,7 +333,8 @@ export class StorageCore {
case AssetFileType.Thumbnail:
case AssetFileType.Preview:
case AssetFileType.Sidecar:
case AssetPathType.EncodedVideo: {
case AssetPathType.EncodedVideo:
case AssetFileType.SampledVideo: {
return this.assetRepository.upsertFile({ assetId: id, type: pathType as AssetFileType, path: newPath });
}

10
server/src/dtos/system-config.dto.ts
View file

@ -406,6 +406,15 @@ const SystemConfigUserSchema = z
})
.meta({ id: 'SystemConfigUserDto' });
const SystemConfigFrameSamplingSchema = z
.object({
enabled: configBool.describe('Enable frame sampling'),
targetResolution: z.int().min(1).describe('Target short-side resolution (px) of extracted frames'),
qp: z.int().min(0).max(51).describe('Target quality (CRF-equivalent) used for the all-intra frame encode'),
frameInterval: z.number().meta({ format: 'double' }).min(0.01).describe('Seconds between sampled frames'),
})
.meta({ id: 'SystemConfigFrameSamplingDto' });
export const SystemConfigSchema = z
.object({
backup: SystemConfigBackupsSchema,
@ -430,6 +439,7 @@ export const SystemConfigSchema = z
server: SystemConfigServerSchema,
user: SystemConfigUserSchema,
integrityChecks: SystemConfigIntegrityChecksSchema,
frameSampling: SystemConfigFrameSamplingSchema,
})
.describe('System configuration')
.meta({ id: 'SystemConfigDto' });

5
server/src/enum.ts
View file

@ -60,6 +60,7 @@ export enum AssetFileType {
Thumbnail = 'thumbnail',
Sidecar = 'sidecar',
EncodedVideo = 'encoded_video',
SampledVideo = 'sampled_video',
}
export enum AlbumUserRole {
@ -915,6 +916,10 @@ export enum JobName {
IntegrityChecksumFilesRefresh = 'IntegrityChecksumFilesRefresh',
IntegrityDeleteReportType = 'IntegrityDeleteReportType',
IntegrityDeleteReports = 'IntegrityDeleteReports',
// Frame sampling
FrameSamplingQueueAll = 'FrameSamplingQueueAll',
FrameSampling = 'FrameSampling',
}
export const JobNameSchema = z.enum(JobName).describe('Job name').meta({ id: 'JobName' });

70
server/src/queries/asset.job.repository.sql
View file

@ -698,6 +698,76 @@ where
"asset"."id" = $1
and "asset"."type" = 'VIDEO'
-- AssetJobRepository.streamForFrameSampling
select
"asset"."id"
from
"asset"
where
"asset"."type" = 'VIDEO'
and not exists (
select
"asset_file"."id"
from
"asset_file"
where
"asset_file"."assetId" = "asset"."id"
and "asset_file"."type" = 'sampled_video'
)
and "asset"."visibility" != 'hidden'
and "asset"."deletedAt" is null
-- AssetJobRepository.getForFrameSampling
select
"asset"."id",
"asset"."ownerId",
"asset"."originalPath",
(
select
to_json(obj)
from
(
select
"asset_video"."index",
"asset_video"."codecName",
"asset_video"."profile",
"asset_video"."level",
"asset_video"."bitrate",
"asset_exif"."exifImageWidth" as "width",
"asset_exif"."exifImageHeight" as "height",
"asset_video"."pixelFormat",
"asset_video"."frameCount",
"asset_exif"."fps" as "frameRate",
"asset_video"."timeBase",
case
when "asset_exif"."orientation" = '6' then -90
when "asset_exif"."orientation" = '8' then 90
when "asset_exif"."orientation" = '3' then 180
else 0
end as "rotation",
"asset_video"."colorPrimaries",
"asset_video"."colorMatrix",
"asset_video"."colorTransfer",
"asset_video"."dvProfile",
"asset_video"."dvLevel",
"asset_video"."dvBlSignalCompatibilityId"
from
(
select
1
) as "dummy"
where
"asset_video"."assetId" is not null
) as obj
) as "videoStream"
from
"asset"
inner join "asset_exif" on "asset"."id" = "asset_exif"."assetId"
inner join "asset_video" on "asset_video"."assetId" = "asset"."id"
where
"asset"."id" = $1
and "asset"."type" = 'VIDEO'
-- AssetJobRepository.streamForMetadataExtraction
select
"asset"."id"

23
server/src/queries/video.frame.repository.sql Normal file
View file

@ -0,0 +1,23 @@
-- NOTE: This file is auto generated by ./sql-generator
-- VideoFrameRepository.upsertFrames
insert into
"video_frames" ("0", "assetId")
values
($1, $2)
on conflict ("assetId") do update
set
"0" = $3
-- VideoFrameRepository.deleteFrames
delete from "video_frames"
where
"assetId" = $1
-- VideoFrameRepository.getFrames
select
*
from
"video_frames"
where
"assetId" = $1

38
server/src/repositories/asset-job.repository.ts
View file

@ -352,6 +352,44 @@ export class AssetJobRepository {
.executeTakeFirst();
}
@GenerateSql({ params: [undefined, DummyValue.STRING], stream: true })
streamForFrameSampling(force: boolean | undefined) {
return this.db
.selectFrom('asset')
.select(['asset.id'])
.where('asset.type', '=', sql.lit(AssetType.Video))
.$if(!force, (qb) =>
qb
.where((eb) =>
eb.not(
eb.exists(
eb
.selectFrom('asset_file')
.select('asset_file.id')
.whereRef('asset_file.assetId', '=', 'asset.id')
.where('asset_file.type', '=', sql.lit(AssetFileType.SampledVideo)),
),
),
)
.where('asset.visibility', '!=', sql.lit(AssetVisibility.Hidden)),
)
.where('asset.deletedAt', 'is', null)
.stream();
}
@GenerateSql({ params: [DummyValue.UUID] })
getForFrameSampling(id: string) {
return this.db
.selectFrom('asset')
.innerJoin('asset_exif', 'asset.id', 'asset_exif.assetId')
.innerJoin('asset_video', 'asset_video.assetId', 'asset.id')
.select(['asset.id', 'asset.ownerId', 'asset.originalPath'])
.select((eb) => withVideoStream(eb).$notNull().as('videoStream'))
.where('asset.id', '=', id)
.where('asset.type', '=', sql.lit(AssetType.Video))
.executeTakeFirst();
}
@GenerateSql({ params: [], stream: true })
streamForMetadataExtraction(force?: boolean) {
return this.db

2
server/src/repositories/index.ts
View file

@ -47,6 +47,7 @@ import { TelemetryRepository } from 'src/repositories/telemetry.repository';
import { TrashRepository } from 'src/repositories/trash.repository';
import { UserRepository } from 'src/repositories/user.repository';
import { VersionHistoryRepository } from 'src/repositories/version-history.repository';
import { VideoFrameRepository } from 'src/repositories/video-frames.repository';
import { VideoStreamRepository } from 'src/repositories/video-stream.repository';
import { ViewRepository } from 'src/repositories/view-repository';
import { WebsocketRepository } from 'src/repositories/websocket.repository';
@ -103,6 +104,7 @@ export const repositories = [
UserRepository,
ViewRepository,
VersionHistoryRepository,
VideoFrameRepository,
VideoStreamRepository,
WebsocketRepository,
WorkflowRepository,

97
server/src/repositories/media.repository.spec.ts
View file

@ -1,3 +1,6 @@
import { spawn } from 'node:child_process';
import { EventEmitter } from 'node:events';
import fs from 'node:fs/promises';
import sharp from 'sharp';
import { AssetFace } from 'src/database';
import { AssetEditAction, MirrorAxis } from 'src/dtos/editing.dto';
@ -8,6 +11,12 @@ import { BoundingBox } from 'src/repositories/machine-learning.repository';
import { MediaRepository } from 'src/repositories/media.repository';
import { checkFaceVisibility, checkOcrVisibility } from 'src/utils/editor';
import { automock } from 'test/utils';
import { MockInstance, vitest } from 'vitest';
vitest.mock('node:child_process', async (importOriginal) => {
const actual = await importOriginal<typeof import('node:child_process')>();
return { ...actual, spawn: vitest.fn() };
});
const getPixelColor = async (buffer: Buffer, x: number, y: number) => {
const metadata = await sharp(buffer).metadata();
@ -664,4 +673,92 @@ describe(MediaRepository.name, () => {
});
});
});
describe('frameSampling', () => {
class FakeFfmpegProcess extends EventEmitter {
stderr = Object.assign(new EventEmitter(), { setEncoding: vitest.fn() });
}
const playlist = [
'#EXTM3U',
'#EXT-X-MAP:URI="asset.m4s",BYTERANGE="813@0"',
'#EXTINF:1.000000,',
'#EXT-X-BYTERANGE:3843@813',
'asset.m4s',
'#EXT-X-ENDLIST',
].join('\n');
const scores = ['frame:0 pts:0 pts_time:0', 'lavfi.scd.mafd=1.234', 'lavfi.scd.score=1.234'].join('\n');
let fakeProcess: FakeFfmpegProcess;
let readFileSpy: MockInstance<typeof fs.readFile>;
beforeEach(() => {
fakeProcess = new FakeFfmpegProcess();
vitest.mocked(spawn).mockReturnValue(fakeProcess as unknown as ReturnType<typeof spawn>);
readFileSpy = vitest.spyOn(fs, 'readFile').mockImplementation((path) => {
return Promise.resolve(String(path).endsWith('scores.txt') ? scores : playlist);
}) as unknown as MockInstance<typeof fs.readFile>;
});
afterEach(() => {
readFileSpy.mockRestore();
});
it('spawns ffmpeg with the given command and resolves with the parsed byte ranges and scores', async () => {
const promise = sut.sampleFrames(['-i', 'input.mp4'], {
playlistPath: '/tmp/frames.m3u8',
scoresPath: '/tmp/scores.txt',
});
fakeProcess.emit('close', 0);
await expect(promise).resolves.toEqual({
byteRanges: [{ byteOffset: 813, byteSize: 3843 }],
intervalChanges: [1.234],
});
expect(spawn).toHaveBeenCalledWith('ffmpeg', ['-i', 'input.mp4'], { stdio: ['ignore', 'ignore', 'pipe'] });
expect(fs.readFile).toHaveBeenCalledWith('/tmp/frames.m3u8', 'utf8');
expect(fs.readFile).toHaveBeenCalledWith('/tmp/scores.txt', 'utf8');
});
it('rejects with the collected stderr when ffmpeg exits with a non-zero code', async () => {
const promise = sut.sampleFrames(['-i', 'input.mp4'], {
playlistPath: '/tmp/frames.m3u8',
scoresPath: '/tmp/scores.txt',
});
fakeProcess.stderr.emit('data', 'something went wrong\n');
fakeProcess.emit('close', 1);
await expect(promise).rejects.toThrowError('ffmpeg exited with code 1: something went wrong');
expect(fs.readFile).not.toHaveBeenCalled();
});
it('rejects when the process emits an error', async () => {
const promise = sut.sampleFrames(['-i', 'input.mp4'], {
playlistPath: '/tmp/frames.m3u8',
scoresPath: '/tmp/scores.txt',
});
const error = new Error('spawn ffmpeg ENOENT');
fakeProcess.emit('error', error);
await expect(promise).rejects.toThrow(error);
});
it('rejects when the output artifacts cannot be read', async () => {
readFileSpy.mockRejectedValue(new Error('ENOENT: no such file or directory'));
const promise = sut.sampleFrames(['-i', 'input.mp4'], {
playlistPath: '/tmp/frames.m3u8',
scoresPath: '/tmp/scores.txt',
});
fakeProcess.emit('close', 0);
await expect(promise).rejects.toThrowError('ENOENT: no such file or directory');
});
});
});

29
server/src/repositories/media.repository.ts
View file

@ -35,6 +35,7 @@ import {
VideoInfo,
VideoPacketInfo,
} from 'src/types';
import { parseByteRangePlaylist, parseIntervalChangeScores } from 'src/utils/frame-sampling';
import { handlePromiseError } from 'src/utils/misc';
import { createAffineMatrix } from 'src/utils/transform';
@ -556,4 +557,32 @@ export class MediaRepository {
const median = history.sort((a, b) => a - b)[1];
return outputFrames + median;
}
async sampleFrames(
command: string[],
artifacts: { playlistPath: string; scoresPath: string },
): Promise<{ byteRanges: ReturnType<typeof parseByteRangePlaylist>['byteRanges']; intervalChanges: number[] }> {
return new Promise((resolve, reject) => {
const ffmpeg = spawn('ffmpeg', command, { stdio: ['ignore', 'ignore', 'pipe'] });
let stderr = '';
ffmpeg.stderr.setEncoding('utf8');
ffmpeg.stderr.on('data', (chunk: string) => (stderr += chunk));
ffmpeg.on('error', reject);
ffmpeg.on('close', (code) => {
if (code !== 0) {
return reject(new Error(`ffmpeg exited with code ${code}: ${stderr.trim()}`));
}
Promise.all([fs.readFile(artifacts.playlistPath, 'utf8'), fs.readFile(artifacts.scoresPath, 'utf8')])
.then(([playlist, scores]) =>
resolve({
byteRanges: parseByteRangePlaylist(playlist).byteRanges,
intervalChanges: parseIntervalChangeScores(scores),
}),
)
.catch(reject);
});
});
}
}

5
server/src/repositories/storage.repository.ts
View file

@ -13,6 +13,7 @@ import {
watch,
} from 'node:fs';
import fs from 'node:fs/promises';
import { tmpdir } from 'node:os';
import path from 'node:path';
import { PassThrough, Readable, Writable } from 'node:stream';
import { createGunzip, createGzip } from 'node:zlib';
@ -209,6 +210,10 @@ export class StorageRepository {
return existsSync(filepath);
}
async mkdtemp(prefix: string): Promise<string> {
return fs.mkdtemp(path.join(tmpdir(), `${prefix}-`));
}
async checkDiskUsage(folder: string): Promise<DiskUsage> {
const stats = await fs.statfs(folder);
return {

32
server/src/repositories/video-frames.repository.ts Normal file
View file

@ -0,0 +1,32 @@
import { Injectable } from '@nestjs/common';
import { Insertable, Kysely } from 'kysely';
import { InjectKysely } from 'nestjs-kysely';
import { DummyValue, GenerateSql } from 'src/decorators';
import { DB } from 'src/schema';
import { VideoFramesTable } from 'src/schema/tables/video-frames.table';
export type VideoFramesInsert = Omit<Insertable<VideoFramesTable>, 'assetId'>;
@Injectable()
export class VideoFrameRepository {
constructor(@InjectKysely() private db: Kysely<DB>) {}
@GenerateSql({ params: [DummyValue.UUID, [{ byteOffset: [0], byteSize: [0], intervalChange: [0] }]] })
async upsertFrames(assetId: string, frames: VideoFramesInsert) {
return this.db
.insertInto('video_frames')
.values({ assetId, ...frames })
.onConflict((oc) => oc.column('assetId').doUpdateSet({ ...frames }))
.execute();
}
@GenerateSql({ params: [DummyValue.UUID] })
async deleteFrames(assetId: string) {
await this.db.deleteFrom('video_frames').where('assetId', '=', assetId).execute();
}
@GenerateSql({ params: [DummyValue.UUID] })
getFrames(assetId: string) {
return this.db.selectFrom('video_frames').selectAll().where('assetId', '=', assetId).executeTakeFirst();
}
}

4
server/src/schema/index.ts
View file

@ -81,6 +81,7 @@ import { UserMetadataAuditTable } from 'src/schema/tables/user-metadata-audit.ta
import { UserMetadataTable } from 'src/schema/tables/user-metadata.table';
import { UserTable } from 'src/schema/tables/user.table';
import { VersionHistoryTable } from 'src/schema/tables/version-history.table';
import { VideoFramesTable } from 'src/schema/tables/video-frames.table';
import {
VideoStreamSegmentTable,
VideoStreamSessionTable,
@ -149,6 +150,7 @@ export class ImmichDatabase {
VideoStreamSessionTable,
VideoStreamVariantTable,
VideoStreamSegmentTable,
VideoFramesTable,
PluginTable,
PluginMethodTable,
WorkflowTable,
@ -268,6 +270,8 @@ export interface DB {
version_history: VersionHistoryTable;
video_frames: VideoFramesTable;
video_stream_session: VideoStreamSessionTable;
video_stream_variant: VideoStreamVariantTable;
video_stream_segment: VideoStreamSegmentTable;

17
server/src/schema/migrations/1785139023817-AddVideoFramesTable.ts Normal file
View file

@ -0,0 +1,17 @@
import { Kysely, sql } from 'kysely';
export async function up(db: Kysely<any>): Promise<void> {
await sql`CREATE TABLE "video_frames" (
"assetId" uuid NOT NULL,
"byteOffset" bigint[] NOT NULL,
"byteSize" integer[] NOT NULL,
"intervalChange" real[] NOT NULL,
"createdAt" timestamp with time zone NOT NULL DEFAULT now(),
CONSTRAINT "video_frames_assetId_fkey" FOREIGN KEY ("assetId") REFERENCES "asset" ("id") ON UPDATE NO ACTION ON DELETE CASCADE,
CONSTRAINT "video_frames_pkey" PRIMARY KEY ("assetId")
);`.execute(db);
}
export async function down(db: Kysely<any>): Promise<void> {
await sql`DROP TABLE "video_frames";`.execute(db);
}

24
server/src/schema/tables/video-frames.table.ts Normal file
View file

@ -0,0 +1,24 @@
import { Column, CreateDateColumn, ForeignKeyColumn, Generated, Table, Timestamp } from '@immich/sql-tools';
import { AssetTable } from 'src/schema/tables/asset.table';
@Table('video_frames')
export class VideoFramesTable {
@ForeignKeyColumn(() => AssetTable, {
onDelete: 'CASCADE',
primary: true,
})
assetId!: string;
@Column({ type: 'bigint', array: true })
byteOffset!: number[];
@Column({ type: 'integer', array: true })
byteSize!: number[];
// Mean absolute frame difference (mafd) score for this frame
@Column({ type: 'real', array: true })
intervalChange!: number[];
@CreateDateColumn()
createdAt!: Generated<Timestamp>;
}

5
server/src/services/base.service.ts
View file

@ -54,6 +54,7 @@ import { TelemetryRepository } from 'src/repositories/telemetry.repository';
import { TrashRepository } from 'src/repositories/trash.repository';
import { UserRepository } from 'src/repositories/user.repository';
import { VersionHistoryRepository } from 'src/repositories/version-history.repository';
import { VideoFrameRepository } from 'src/repositories/video-frames.repository';
import { VideoStreamRepository } from 'src/repositories/video-stream.repository';
import { ViewRepository } from 'src/repositories/view-repository';
import { WebsocketRepository } from 'src/repositories/websocket.repository';
@ -113,6 +114,7 @@ export const BASE_SERVICE_DEPENDENCIES = [
TrashRepository,
UserRepository,
VersionHistoryRepository,
VideoFrameRepository,
VideoStreamRepository,
ViewRepository,
WebsocketRepository,
@ -173,6 +175,7 @@ export class BaseService {
protected trashRepository: TrashRepository,
protected userRepository: UserRepository,
protected versionRepository: VersionHistoryRepository,
protected videoFrameRepository: VideoFrameRepository,
protected videoStreamRepository: VideoStreamRepository,
protected viewRepository: ViewRepository,
protected websocketRepository: WebsocketRepository,
@ -242,6 +245,8 @@ export class BaseService {
ctx.trashRepository,
ctx.userRepository,
ctx.versionRepository,
ctx.videoFrameRepository,
ctx.videoStreamRepository,
ctx.viewRepository,
ctx.websocketRepository,
ctx.workflowRepository,

8
server/src/services/job.service.spec.ts
View file

@ -70,7 +70,13 @@ describe(JobService.name, () => {
},
{
item: { name: JobName.AssetGenerateThumbnails, data: { id: 'asset-1', source: 'upload' } },
jobs: [JobName.SmartSearch, JobName.AssetDetectFaces, JobName.Ocr, JobName.AssetEncodeVideo],
jobs: [
JobName.SmartSearch,
JobName.AssetDetectFaces,
JobName.Ocr,
JobName.AssetEncodeVideo,
JobName.FrameSampling,
],
stub: [AssetFactory.create({ id: 'asset-1', type: AssetType.Video })],
},
{

5
server/src/services/job.service.ts
View file

@ -185,7 +185,10 @@ export class JobService extends BaseService {
];
if (asset.type === AssetType.Video) {
jobs.push({ name: JobName.AssetEncodeVideo, data: item.data });
jobs.push(
{ name: JobName.AssetEncodeVideo, data: item.data },
{ name: JobName.FrameSampling, data: item.data },
);
}
await this.jobRepository.queueAll(jobs);

133
server/src/services/media.service.spec.ts
View file

@ -4276,4 +4276,137 @@ describe(MediaService.name, () => {
expect(mocks.job.queue).not.toHaveBeenCalled();
});
});
describe('frameSampling', () => {
let mocks: ServiceMocks;
const frameSamplingConfig: SystemConfig['frameSampling'] = {
enabled: true,
targetResolution: 640,
qp: 34,
frameInterval: 1,
};
beforeEach(() => {
({ sut, mocks } = newTestService(MediaService));
mocks.systemMetadata.get.mockResolvedValue({ frameSampling: frameSamplingConfig });
mocks.videoFrame.upsertFrames.mockResolvedValue(void 0 as never);
});
it('should work', () => {
expect(sut).toBeDefined();
});
describe('handleQueueGenerateVideoFrames', () => {
it('should skip if disabled', async () => {
mocks.systemMetadata.get.mockResolvedValue({
frameSampling: { ...frameSamplingConfig, enabled: false },
});
await expect(sut.handleQueueSampledFrames({})).resolves.toEqual(JobStatus.Skipped);
expect(mocks.assetJob.streamForFrameSampling).not.toHaveBeenCalled();
});
it('should queue all eligible video assets', async () => {
const asset = AssetFactory.create({ type: AssetType.Video });
mocks.assetJob.streamForFrameSampling.mockReturnValue(makeStream([asset]));
await expect(sut.handleQueueSampledFrames({ force: true })).resolves.toEqual(JobStatus.Success);
expect(mocks.assetJob.streamForFrameSampling).toHaveBeenCalledWith(true);
expect(mocks.job.queueAll).toHaveBeenCalledWith([{ name: JobName.FrameSampling, data: { id: asset.id } }]);
});
});
describe('handleGenerateVideoFrames', () => {
const asset = {
...AssetFactory.create({ id: 'asset-1', type: AssetType.Video, originalPath: '/original/path.ext' }),
videoStream: probeStub.videoStreamH264.videoStream,
};
beforeEach(() => {
mocks.assetJob.getForFrameSampling.mockResolvedValue(asset);
mocks.storage.mkdtemp.mockResolvedValue(`/tmp/immich-video-frames-${asset.id}`);
});
it('should skip if disabled', async () => {
mocks.systemMetadata.get.mockResolvedValue({
frameSampling: { ...frameSamplingConfig, enabled: false },
});
await expect(sut.handleSampledFrames({ id: asset.id })).resolves.toEqual(JobStatus.Skipped);
expect(mocks.assetJob.getForFrameSampling).not.toHaveBeenCalled();
});
it('should fail if asset could not be found', async () => {
mocks.assetJob.getForFrameSampling.mockResolvedValue(void 0);
await expect(sut.handleSampledFrames({ id: asset.id })).resolves.toEqual(JobStatus.Failed);
});
it('should extract frames and persist them', async () => {
mocks.media.sampleFrames.mockResolvedValue({
byteRanges: [
{ byteOffset: 813, byteSize: 3843 },
{ byteOffset: 4656, byteSize: 3238 },
],
intervalChanges: [0, 2.516],
});
await expect(sut.handleSampledFrames({ id: asset.id })).resolves.toEqual(JobStatus.Success);
expect(mocks.storage.mkdirSync).toHaveBeenCalled();
expect(mocks.media.sampleFrames).toHaveBeenCalledWith(expect.any(Array), {
playlistPath: expect.stringContaining('frames.m3u8'),
scoresPath: expect.stringContaining('scores.txt'),
});
expect(mocks.videoFrame.upsertFrames).toHaveBeenCalledWith(asset.id, {
byteOffset: [813, 4656],
byteSize: [3843, 3238],
intervalChange: [0, 2.516],
});
expect(mocks.storage.mkdtemp).toHaveBeenCalledTimes(1);
expect(mocks.storage.unlinkDir).toHaveBeenCalledTimes(1);
});
it('should default a missing interval change to 0', async () => {
mocks.media.sampleFrames.mockResolvedValue({
byteRanges: [{ byteOffset: 0, byteSize: 100 }],
intervalChanges: [],
});
await expect(sut.handleSampledFrames({ id: asset.id })).resolves.toEqual(JobStatus.Success);
expect(mocks.videoFrame.upsertFrames).toHaveBeenCalledWith(asset.id, {
byteOffset: [0],
byteSize: [100],
intervalChange: [0],
});
});
it('should fail and clean up the temp dir if ffmpeg fails', async () => {
mocks.media.sampleFrames.mockRejectedValue(new Error('ffmpeg exited with code 1: boom'));
await expect(sut.handleSampledFrames({ id: asset.id })).resolves.toEqual(JobStatus.Failed);
expect(mocks.videoFrame.upsertFrames).not.toHaveBeenCalled();
expect(mocks.storage.unlinkDir).toHaveBeenCalledTimes(1);
});
it('should fail and clean up the temp dir if no frames were extracted', async () => {
mocks.media.sampleFrames.mockResolvedValue({ byteRanges: [], intervalChanges: [] });
await expect(sut.handleSampledFrames({ id: asset.id })).resolves.toEqual(JobStatus.Failed);
expect(mocks.videoFrame.upsertFrames).not.toHaveBeenCalled();
expect(mocks.storage.unlinkDir).toHaveBeenCalledTimes(1);
});
});
});
});

103
server/src/services/media.service.ts
View file

@ -1,4 +1,5 @@
import { Injectable } from '@nestjs/common';
import { join } from 'node:path';
import { SystemConfig } from 'src/config';
import { FACE_THUMBNAIL_SIZE, JOBS_ASSET_PAGINATION_SIZE } from 'src/constants';
import { ImagePathOptions, StorageCore, ThumbnailPathEntity } from 'src/cores/storage.core';
@ -12,6 +13,7 @@ import {
AssetVisibility,
AudioCodec,
Colorspace,
CQMode,
ImageFormat,
ImmichWorker,
JobName,
@ -251,6 +253,107 @@ export class MediaService extends BaseService {
return JobStatus.Success;
}
@OnJob({ name: JobName.FrameSamplingQueueAll, queue: QueueName.VideoConversion })
async handleQueueSampledFrames({ force }: JobOf<JobName.FrameSamplingQueueAll>): Promise<JobStatus> {
const { frameSampling } = await this.getConfig({ withCache: true });
if (!frameSampling.enabled) {
return JobStatus.Skipped;
}
let jobs: JobItem[] = [];
const queueAll = async () => {
await this.jobRepository.queueAll(jobs);
jobs = [];
};
for await (const asset of this.assetJobRepository.streamForFrameSampling(force)) {
jobs.push({ name: JobName.FrameSampling, data: { id: asset.id } });
if (jobs.length >= JOBS_ASSET_PAGINATION_SIZE) {
await queueAll();
}
}
await queueAll();
return JobStatus.Success;
}
@OnJob({ name: JobName.FrameSampling, queue: QueueName.VideoConversion })
async handleSampledFrames({ id }: JobOf<JobName.FrameSampling>): Promise<JobStatus> {
const { frameSampling, ffmpeg } = await this.getConfig({ withCache: true });
if (!frameSampling.enabled) {
return JobStatus.Skipped;
}
const asset = await this.assetJobRepository.getForFrameSampling(id);
if (!asset) {
return JobStatus.Failed;
}
const artifactPath = StorageCore.getVideoFrameArtifactPath(asset);
this.storageCore.ensureFolders(artifactPath);
const tempDir = await this.storageRepository.mkdtemp(`immich-video-frames-${asset.id}`);
const playlistPath = join(tempDir, 'frames.m3u8');
const scoresPath = join(tempDir, 'scores.txt');
try {
const overrideConfig: SystemConfigFFmpegDto = {
...ffmpeg,
targetVideoCodec: VideoCodec.H264,
targetResolution: String(frameSampling.targetResolution),
crf: frameSampling.qp,
cqMode: CQMode.Icq,
maxBitrate: '0',
};
const config = BaseConfig.create(overrideConfig, this.videoInterfaces, { strictGop: true, lowLatency: false });
const command = config.getFrameSamplingCommand(
{
inputPath: asset.originalPath,
segmentFilename: artifactPath,
playlistFilename: playlistPath,
scoresFilename: scoresPath,
frameInterval: frameSampling.frameInterval,
},
asset.videoStream,
);
let result;
try {
result = await this.mediaRepository.sampleFrames(command, { playlistPath, scoresPath });
} catch (error) {
this.logger.error(`Failed to generate video frames for asset ${asset.id}: ${error}`);
return JobStatus.Failed;
}
const { byteRanges, intervalChanges } = result;
if (byteRanges.length === 0) {
this.logger.warn(`No frames extracted for video ${asset.id}`);
return JobStatus.Failed;
}
const frames = {
byteOffset: byteRanges.map((range) => range.byteOffset),
byteSize: byteRanges.map((range) => range.byteSize),
intervalChange: byteRanges.map((_, frameIndex) => intervalChanges[frameIndex] ?? 0),
};
await this.videoFrameRepository.upsertFrames(asset.id, frames);
await this.assetRepository.upsertFile({
assetId: asset.id,
type: AssetFileType.SampledVideo,
path: artifactPath,
isEdited: false,
});
this.logger.log(`Extracted ${frames.byteOffset.length} frame(s) for video ${asset.id}`);
return JobStatus.Success;
} finally {
await this.storageRepository.unlinkDir(tempDir, { recursive: true, force: true });
}
}
private async extractImage(originalPath: string, minSize: number) {
let extracted = await this.mediaRepository.extract(originalPath);
if (extracted && !(await this.shouldUseExtractedImage(extracted.buffer, minSize))) {

6
server/src/services/system-config.service.spec.ts
View file

@ -251,6 +251,12 @@ const updatedConfig = Object.freeze<SystemConfig>({
albumUpdateTemplate: '',
},
},
frameSampling: {
enabled: false,
targetResolution: 640,
qp: 34,
frameInterval: 1,
},
});
describe(SystemConfigService.name, () => {

18
server/src/types.ts
View file

@ -181,6 +181,14 @@ export interface HlsCommandOptions {
totalDuration: number;
}
export interface FrameSamplingOptions {
inputPath: string;
segmentFilename: string;
playlistFilename: string;
scoresFilename: string;
frameInterval: number;
}
export interface BitrateDistribution {
max: number;
target: number;
@ -203,6 +211,10 @@ export interface VideoCodecSWConfig {
getHlsCommand(options: HlsCommandOptions, video: VideoStreamInfo, audio?: AudioStreamInfo): string[];
}
export interface FrameSamplingConfig {
getFrameSamplingCommand(options: FrameSamplingOptions, video: VideoStreamInfo): string[];
}
export interface ProbeOptions {
countFrames: boolean;
}
@ -472,7 +484,11 @@ export type JobItem =
| { name: JobName.IntegrityDeleteReports; data: IIntegrityDeleteReportsJob }
// Editor
| { name: JobName.AssetEditThumbnailGeneration; data: IEntityJob };
| { name: JobName.AssetEditThumbnailGeneration; data: IEntityJob }
// Frame sampling
| { name: JobName.FrameSamplingQueueAll; data: IBaseJob }
| { name: JobName.FrameSampling; data: IEntityJob };
export type VectorExtension = (typeof VECTOR_EXTENSIONS)[number];

67
server/src/utils/frame-sampling.spec.ts Normal file
View file

@ -0,0 +1,67 @@
import { parseByteRangePlaylist, parseIntervalChangeScores } from 'src/utils/frame-sampling';
import { describe, expect, it } from 'vitest';
describe('parseByteRangePlaylist', () => {
it('parses each sampled frame byte range from a valid playlist', () => {
const playlist = [
'#EXTM3U',
'#EXT-X-VERSION:7',
'#EXT-X-MAP:URI="asset.m4s",BYTERANGE="813@0"',
'#EXTINF:1.000000,',
'#EXT-X-BYTERANGE:3843@813',
'asset.m4s',
'#EXTINF:1.000000,',
'#EXT-X-BYTERANGE:3238@4656',
'asset.m4s',
'#EXT-X-ENDLIST',
].join('\n');
expect(parseByteRangePlaylist(playlist)).toEqual({
byteRanges: [
{ byteOffset: 813, byteSize: 3843 },
{ byteOffset: 4656, byteSize: 3238 },
],
});
});
it('does not mistake the EXT-X-MAP init segment byte range for a sampled frame', () => {
const playlist = ['#EXTM3U', '#EXT-X-MAP:URI="asset.m4s",BYTERANGE="813@0"', '#EXT-X-ENDLIST'].join('\n');
expect(parseByteRangePlaylist(playlist)).toEqual({ byteRanges: [] });
});
it('returns an empty array when no frames were extracted', () => {
expect(parseByteRangePlaylist('')).toEqual({ byteRanges: [] });
});
it('parses a single frame', () => {
const playlist = ['#EXTM3U', '#EXT-X-BYTERANGE:100@0', 'asset.m4s', '#EXT-X-ENDLIST'].join('\n');
expect(parseByteRangePlaylist(playlist)).toEqual({ byteRanges: [{ byteOffset: 0, byteSize: 100 }] });
});
});
describe('parseIntervalChangeScores', () => {
it('parses the mafd score for each frame', () => {
const scores = [
'frame:0 pts:0 pts_time:0',
'lavfi.scd.mafd=0.000',
'lavfi.scd.score=0.000',
'frame:1 pts:1 pts_time:1',
'lavfi.scd.mafd=2.516',
'lavfi.scd.score=2.516',
].join('\n');
expect(parseIntervalChangeScores(scores)).toEqual([0, 2.516]);
});
it('ignores the score line, only reading mafd', () => {
const scores = ['lavfi.scd.mafd=12.345', 'lavfi.scd.score=99.999'].join('\n');
expect(parseIntervalChangeScores(scores)).toEqual([12.345]);
});
it('returns an empty array when there are no scores', () => {
expect(parseIntervalChangeScores('')).toEqual([]);
});
});

40
server/src/utils/frame-sampling.ts Normal file
View file

@ -0,0 +1,40 @@
/**
* Parses the byte-range fMP4 HLS playlist (`.m3u8`) describing a single `EXT-X-MAP`
* init segment followed by one `EXT-X-BYTERANGE` entry per sampled frame, e.g.:
*
* ```
* #EXT-X-MAP:URI="asset.m4s",BYTERANGE="813@0"
* #EXTINF:1.000000,
* #EXT-X-BYTERANGE:3843@813
* asset.m4s
* ```
*/
export function parseByteRangePlaylist(content: string): {
byteRanges: { byteOffset: number; byteSize: number }[];
} {
const byteRanges: { byteOffset: number; byteSize: number }[] = [];
const byteRangeRegex = /^#EXT-X-BYTERANGE:(\d+)@(\d+)/gm;
for (const match of content.matchAll(byteRangeRegex)) {
byteRanges.push({ byteSize: Math.trunc(Number(match[1])), byteOffset: Math.trunc(Number(match[2])) });
}
return { byteRanges };
}
/**
* Parses the raw scdet `mafd` (mean absolute frame difference) interval-change score for each sampled frame:
*
* ```
* frame:0 pts:0 pts_time:0
* lavfi.scd.mafd=0.000
* lavfi.scd.score=0.000
* ```
*/
export function parseIntervalChangeScores(content: string): number[] {
const scores: number[] = [];
const mafdRegex = /lavfi\.scd\.mafd=([\d.]+)/g;
for (const match of content.matchAll(mafdRegex)) {
scores.push(Number(match[1]));
}
return scores;
}

550
server/src/utils/media.spec.ts Normal file
View file

@ -0,0 +1,550 @@
import { defaults } from 'src/config';
import { SystemConfigFFmpegDto } from 'src/dtos/system-config.dto';
import { ColorTransfer, CQMode, TranscodeHardwareAcceleration, VideoCodec } from 'src/enum';
import { VideoInterfaces, VideoStreamInfo } from 'src/types';
import { BaseConfig } from 'src/utils/media';
import { probeStub } from 'test/fixtures/media.stub';
import { describe, expect, it } from 'vitest';
const inputPath = '/original/asset.mp4';
const artifactPath = '/artifacts/asset.m4s';
const playlistPath = '/tmp/frames.m3u8';
const scoresPath = '/tmp/scores.txt';
const targetResolution = 640;
const qp = 34;
// Satisfies BaseHWConfig.validateDevices() (accepts renderD*/card* prefixes) and auto-resolves to
// /dev/dri/renderD128 since preferredHwDevice is 'auto'.
const videoInterfaces: VideoInterfaces = { dri: ['renderD128'], mali: false };
// 1920x1080, yuv420p, Bt709 (SDR), H.264 - see test/fixtures/media.stub.ts:379
const sdrVideoStream = probeStub.videoStreamH264.videoStream;
// Same stream, but with HDR10 color characteristics, to exercise the tonemap branches.
const hdrVideoStream: VideoStreamInfo = {
...sdrVideoStream,
colorTransfer: ColorTransfer.Smpte2084,
pixelFormat: 'yuv420p10le',
};
const lowResVideoStream: VideoStreamInfo = { ...sdrVideoStream, width: 426, height: 240 };
const lowResOddVideoStream: VideoStreamInfo = { ...sdrVideoStream, width: 427, height: 241 };
const hlsMuxerTail = [
'-an',
'-f',
'hls',
'-hls_segment_type',
'fmp4',
'-hls_flags',
'single_file',
'-hls_time',
'0',
'-hls_list_size',
'0',
'-hls_segment_filename',
artifactPath,
playlistPath,
'-map',
'[scored]',
'-f',
'null',
'-',
];
const getFrameSamplingCommand = (
ffmpegOverrides: Partial<SystemConfigFFmpegDto>,
videoStream: VideoStreamInfo = sdrVideoStream,
) => {
const ffmpeg: SystemConfigFFmpegDto = { ...defaults.ffmpeg, ...ffmpegOverrides };
const overrideConfig: SystemConfigFFmpegDto = {
...ffmpeg,
targetVideoCodec: VideoCodec.H264,
targetResolution: String(targetResolution),
crf: qp,
cqMode: CQMode.Icq,
maxBitrate: '0',
};
const config = BaseConfig.create(overrideConfig, videoInterfaces, { strictGop: true, lowLatency: false });
return config.getFrameSamplingCommand(
{
inputPath,
segmentFilename: artifactPath,
playlistFilename: playlistPath,
scoresFilename: scoresPath,
frameInterval: 1,
},
videoStream,
);
};
describe('BaseConfig', () => {
describe('getFrameSamplingCommand', () => {
it('builds the SW (CPU) command when hardware acceleration is disabled', () => {
const args = getFrameSamplingCommand({ accel: TranscodeHardwareAcceleration.Disabled });
expect(args).toEqual([
'-nostdin',
'-nostats',
'-v',
'verbose',
'-i',
inputPath,
'-filter_complex',
`[0:v]scale=-2:640,fps=1,split[enc][an];[an]scdet=threshold=100,metadata=print:file=${scoresPath}[scored]`,
'-map',
'[enc]',
'-c:v',
'h264',
'-g',
'1',
'-bf',
'0',
'-crf',
'34',
'-sc_threshold:v',
'0',
...hlsMuxerTail,
]);
});
it('does not upscale a source already smaller than targetResolution (SW path)', () => {
const args = getFrameSamplingCommand({ accel: TranscodeHardwareAcceleration.Disabled }, lowResVideoStream);
expect(args).toEqual([
'-nostdin',
'-nostats',
'-v',
'verbose',
'-i',
inputPath,
'-filter_complex',
`[0:v]fps=1,split[enc][an];[an]scdet=threshold=100,metadata=print:file=${scoresPath}[scored]`,
'-map',
'[enc]',
'-c:v',
'h264',
'-g',
'1',
'-bf',
'0',
'-crf',
'34',
'-sc_threshold:v',
'0',
...hlsMuxerTail,
]);
});
it('still scales an odd-dimensioned low-res source for encoder parity (SW path)', () => {
const args = getFrameSamplingCommand({ accel: TranscodeHardwareAcceleration.Disabled }, lowResOddVideoStream);
expect(args).toEqual([
'-nostdin',
'-nostats',
'-v',
'verbose',
'-i',
inputPath,
'-filter_complex',
`[0:v]scale=-2:640,fps=1,split[enc][an];[an]scdet=threshold=100,metadata=print:file=${scoresPath}[scored]`,
'-map',
'[enc]',
'-c:v',
'h264',
'-g',
'1',
'-bf',
'0',
'-crf',
'34',
'-sc_threshold:v',
'0',
...hlsMuxerTail,
]);
});
it('does not upscale a source already smaller than targetResolution (VAAPI HW path, for parity with SW)', () => {
const args = getFrameSamplingCommand(
{ accel: TranscodeHardwareAcceleration.Vaapi, accelDecode: true },
lowResVideoStream,
);
expect(args).toEqual([
'-nostdin',
'-nostats',
'-v',
'verbose',
'-hwaccel',
'vaapi',
'-hwaccel_output_format',
'vaapi',
'-noautorotate',
'-hwaccel_device',
'/dev/dri/renderD128',
'-threads',
'1',
'-i',
inputPath,
'-filter_complex',
`[0:v]fps=1,split[enc][an];[an]hwdownload,format=nv12,scdet=threshold=100,metadata=print:file=${scoresPath}[scored]`,
'-map',
'[enc]',
'-c:v',
'h264_vaapi',
'-g',
'1',
'-bf',
'0',
'-global_quality:v',
'34',
'-rc_mode',
'4',
'-idr_interval',
'0',
'-low_power',
'1',
...hlsMuxerTail,
]);
});
it('builds the VAAPI command with hardware decoding', () => {
const args = getFrameSamplingCommand({ accel: TranscodeHardwareAcceleration.Vaapi, accelDecode: true });
expect(args).toEqual([
'-nostdin',
'-nostats',
'-v',
'verbose',
'-hwaccel',
'vaapi',
'-hwaccel_output_format',
'vaapi',
'-noautorotate',
'-hwaccel_device',
'/dev/dri/renderD128',
'-threads',
'1',
'-i',
inputPath,
'-filter_complex',
`[0:v]scale_vaapi=-2:640:mode=hq:out_range=pc,fps=1,split[enc][an];[an]hwdownload,format=nv12,scdet=threshold=100,metadata=print:file=${scoresPath}[scored]`,
'-map',
'[enc]',
'-c:v',
'h264_vaapi',
'-g',
'1',
'-bf',
'0',
'-global_quality:v',
'34',
'-rc_mode',
'4',
'-idr_interval',
'0',
'-low_power',
'1',
...hlsMuxerTail,
]);
});
it('builds the VAAPI command with software decoding', () => {
const args = getFrameSamplingCommand({ accel: TranscodeHardwareAcceleration.Vaapi, accelDecode: false });
expect(args).toEqual([
'-nostdin',
'-nostats',
'-v',
'verbose',
'-init_hw_device',
'vaapi=accel:/dev/dri/renderD128',
'-filter_hw_device',
'accel',
'-i',
inputPath,
'-filter_complex',
`[0:v]hwupload=extra_hw_frames=64,scale_vaapi=-2:640:mode=hq:out_range=pc:format=nv12,fps=1,split[enc][an];[an]hwdownload,format=nv12,scdet=threshold=100,metadata=print:file=${scoresPath}[scored]`,
'-map',
'[enc]',
'-c:v',
'h264_vaapi',
'-g',
'1',
'-bf',
'0',
'-global_quality:v',
'34',
'-rc_mode',
'4',
'-idr_interval',
'0',
'-low_power',
'1',
...hlsMuxerTail,
]);
});
it('builds the QSV command with hardware decoding', () => {
const args = getFrameSamplingCommand({ accel: TranscodeHardwareAcceleration.Qsv, accelDecode: true });
expect(args).toEqual([
'-nostdin',
'-nostats',
'-v',
'verbose',
'-hwaccel',
'qsv',
'-hwaccel_output_format',
'qsv',
'-async_depth',
'4',
'-noautorotate',
'-qsv_device',
'/dev/dri/renderD128',
'-threads',
'1',
'-i',
inputPath,
'-filter_complex',
`[0:v]scale_qsv=-1:640:async_depth=4:mode=hq,fps=1,split[enc][an];[an]hwdownload,format=nv12,scdet=threshold=100,metadata=print:file=${scoresPath}[scored]`,
'-map',
'[enc]',
'-c:v',
'h264_qsv',
'-g',
'1',
'-bf',
'0',
'-global_quality:v',
'34',
'-idr_interval',
'0',
'-low_power',
'1',
...hlsMuxerTail,
]);
});
it('builds the QSV command with software decoding', () => {
const args = getFrameSamplingCommand({ accel: TranscodeHardwareAcceleration.Qsv, accelDecode: false });
expect(args).toEqual([
'-nostdin',
'-nostats',
'-v',
'verbose',
'-init_hw_device',
'qsv=hw,child_device=/dev/dri/renderD128',
'-filter_hw_device',
'hw',
'-i',
inputPath,
'-filter_complex',
`[0:v]hwupload=extra_hw_frames=64,scale_qsv=-1:640:mode=hq:format=nv12,fps=1,split[enc][an];[an]hwdownload,format=nv12,scdet=threshold=100,metadata=print:file=${scoresPath}[scored]`,
'-map',
'[enc]',
'-c:v',
'h264_qsv',
'-g',
'1',
'-bf',
'0',
'-global_quality:v',
'34',
'-idr_interval',
'0',
'-low_power',
'1',
...hlsMuxerTail,
]);
});
it('builds the NVENC command with hardware decoding, without -low_power (VAAPI/QSV-only flag)', () => {
const args = getFrameSamplingCommand({ accel: TranscodeHardwareAcceleration.Nvenc, accelDecode: true });
expect(args).toEqual([
'-nostdin',
'-nostats',
'-v',
'verbose',
'-hwaccel',
'cuda',
'-hwaccel_output_format',
'cuda',
'-noautorotate',
'-threads',
'1',
'-i',
inputPath,
'-filter_complex',
`[0:v]scale_cuda=-2:640:format=nv12,fps=1,split[enc][an];[an]hwdownload,format=nv12,scdet=threshold=100,metadata=print:file=${scoresPath}[scored]`,
'-map',
'[enc]',
'-c:v',
'h264_nvenc',
'-g',
'1',
'-bf',
'0',
'-cq:v',
'34',
'-forced-idr',
'1',
...hlsMuxerTail,
]);
});
it('builds the NVENC command with software decoding, without -low_power (VAAPI/QSV-only flag)', () => {
const args = getFrameSamplingCommand({ accel: TranscodeHardwareAcceleration.Nvenc, accelDecode: false });
expect(args).toEqual([
'-nostdin',
'-nostats',
'-v',
'verbose',
'-init_hw_device',
'cuda=cuda:0',
'-filter_hw_device',
'cuda',
'-i',
inputPath,
'-filter_complex',
`[0:v]hwupload_cuda,scale_cuda=-2:640:format=nv12,fps=1,split[enc][an];[an]hwdownload,format=nv12,scdet=threshold=100,metadata=print:file=${scoresPath}[scored]`,
'-map',
'[enc]',
'-c:v',
'h264_nvenc',
'-g',
'1',
'-bf',
'0',
'-cq:v',
'34',
'-forced-idr',
'1',
...hlsMuxerTail,
]);
});
it('builds the RKMPP command with hardware decoding, without an IDR flag or -low_power', () => {
const args = getFrameSamplingCommand({ accel: TranscodeHardwareAcceleration.Rkmpp, accelDecode: true });
expect(args).toEqual([
'-nostdin',
'-nostats',
'-v',
'verbose',
'-hwaccel',
'rkmpp',
'-hwaccel_output_format',
'drm_prime',
'-afbc',
'rga',
'-noautorotate',
'-i',
inputPath,
'-filter_complex',
`[0:v]scale_rkrga=-2:640:format=nv12:afbc=1:async_depth=4,fps=1,split[enc][an];[an]hwdownload,format=nv12,scdet=threshold=100,metadata=print:file=${scoresPath}[scored]`,
'-map',
'[enc]',
'-c:v',
'h264_rkmpp',
'-g',
'1',
'-bf',
'0',
'-rc_mode',
'CQP',
'-qp_init',
'34',
...hlsMuxerTail,
]);
});
it('builds the VAAPI command with a tonemap filter chain for HDR source content', () => {
const args = getFrameSamplingCommand(
{ accel: TranscodeHardwareAcceleration.Vaapi, accelDecode: true },
hdrVideoStream,
);
expect(args).toEqual([
'-nostdin',
'-nostats',
'-v',
'verbose',
'-hwaccel',
'vaapi',
'-hwaccel_output_format',
'vaapi',
'-noautorotate',
'-hwaccel_device',
'/dev/dri/renderD128',
'-threads',
'1',
'-i',
inputPath,
'-filter_complex',
'[0:v]scale_vaapi=-2:640:mode=hq:out_range=pc,hwmap=derive_device=opencl,' +
'tonemap_opencl=desat=0:format=nv12:matrix=bt709:primaries=bt709:transfer=bt709:range=pc:tonemap=hable:tonemap_mode=lum:peak=100,' +
`hwmap=derive_device=vaapi:reverse=1,format=vaapi,fps=1,split[enc][an];[an]hwdownload,format=nv12,scdet=threshold=100,metadata=print:file=${scoresPath}[scored]`,
'-map',
'[enc]',
'-c:v',
'h264_vaapi',
'-g',
'1',
'-bf',
'0',
'-global_quality:v',
'34',
'-rc_mode',
'4',
'-idr_interval',
'0',
'-low_power',
'1',
...hlsMuxerTail,
]);
});
it('builds the NVENC command with a single tonemap_cuda filter for HDR source content', () => {
const args = getFrameSamplingCommand(
{ accel: TranscodeHardwareAcceleration.Nvenc, accelDecode: true },
hdrVideoStream,
);
expect(args).toEqual([
'-nostdin',
'-nostats',
'-v',
'verbose',
'-hwaccel',
'cuda',
'-hwaccel_output_format',
'cuda',
'-noautorotate',
'-threads',
'1',
'-i',
inputPath,
'-filter_complex',
'[0:v]scale_cuda=-2:640,tonemap_cuda=desat=0:matrix=bt709:primaries=bt709:range=pc:tonemap=hable:tonemap_mode=lum:transfer=bt709:peak=100:format=nv12,' +
`fps=1,split[enc][an];[an]hwdownload,format=nv12,scdet=threshold=100,metadata=print:file=${scoresPath}[scored]`,
'-map',
'[enc]',
'-c:v',
'h264_nvenc',
'-g',
'1',
'-bf',
'0',
'-cq:v',
'34',
'-forced-idr',
'1',
...hlsMuxerTail,
]);
});
});
});

78
server/src/utils/media.ts
View file

@ -13,6 +13,8 @@ import {
import {
AudioStreamInfo,
BitrateDistribution,
FrameSamplingConfig,
FrameSamplingOptions,
HlsCommandOptions,
TranscodeCommand,
VideoCodecSWConfig,
@ -59,7 +61,7 @@ export const getCodecString = (codec: VideoCodec, width: number, height: number,
}
};
export class BaseConfig implements VideoCodecSWConfig {
export class BaseConfig implements VideoCodecSWConfig, FrameSamplingConfig {
readonly presets = ['veryslow', 'slower', 'slow', 'medium', 'fast', 'faster', 'veryfast', 'superfast', 'ultrafast'];
protected constructor(
protected config: SystemConfigFFmpegDto,
@ -73,7 +75,10 @@ export class BaseConfig implements VideoCodecSWConfig {
return BaseConfig.getHWCodecConfig(config, interfaces, tune);
}
private static getSWCodecConfig(config: SystemConfigFFmpegDto, tune?: VideoTuning): VideoCodecSWConfig {
private static getSWCodecConfig(
config: SystemConfigFFmpegDto,
tune?: VideoTuning,
): VideoCodecSWConfig & FrameSamplingConfig {
switch (config.targetVideoCodec) {
case VideoCodec.H264: {
return new H264Config(config, tune);
@ -100,7 +105,7 @@ export class BaseConfig implements VideoCodecSWConfig {
);
}
let handler: VideoCodecSWConfig;
let handler: VideoCodecSWConfig & FrameSamplingConfig;
switch (config.accel) {
case TranscodeHardwareAcceleration.Nvenc: {
handler = config.accelDecode
@ -211,6 +216,62 @@ export class BaseConfig implements VideoCodecSWConfig {
return args;
}
getFrameSamplingCommand(options: FrameSamplingOptions, video: VideoStreamInfo): string[] {
const fps = 1 / options.frameInterval;
const scoreFilterPrefix =
this.config.accel === TranscodeHardwareAcceleration.Disabled ? '' : 'hwdownload,format=nv12,';
const filterComplex = [
`[0:v]${[...this.getFilterOptions(video), `fps=${fps}`].join(',')},split[enc][an]`,
`[an]${scoreFilterPrefix}scdet=threshold=100,metadata=print:file=${options.scoresFilename}[scored]`,
].join(';');
return [
'-nostdin',
'-nostats',
'-v',
'verbose',
...this.getBaseInputOptions(video),
'-i',
options.inputPath,
'-filter_complex',
filterComplex,
'-map',
'[enc]',
'-c:v',
this.getVideoCodec(),
'-g',
'1',
'-bf',
'0',
...this.getBitrateOptions(),
...this.getEncoderOptions(),
...this.getFrameSamplingEncoderOptions(),
'-an',
'-f',
'hls',
'-hls_segment_type',
'fmp4',
'-hls_flags',
'single_file',
'-hls_time',
'0',
'-hls_list_size',
'0',
'-hls_segment_filename',
options.segmentFilename,
options.playlistFilename,
'-map',
'[scored]',
'-f',
'null',
'-',
];
}
protected getFrameSamplingEncoderOptions(): string[] {
return [];
}
// eslint-disable-next-line @typescript-eslint/no-unused-vars
getBaseInputOptions(videoStream: VideoStreamInfo, format?: VideoFormat): string[] {
return this.getInputThreadOptions();
@ -471,7 +532,7 @@ export class BaseHWConfig extends BaseConfig {
}
export class ThumbnailConfig extends BaseConfig {
static create(config: SystemConfigFFmpegDto): VideoCodecSWConfig {
static create(config: SystemConfigFFmpegDto): VideoCodecSWConfig & FrameSamplingConfig {
return new ThumbnailConfig(config);
}
@ -534,7 +595,6 @@ export class ThumbnailConfig extends BaseConfig {
return super.getScaling(videoStream) + ':flags=lanczos+accurate_rnd+full_chroma_int:out_range=pc';
}
}
export class H264Config extends BaseConfig {
getEncoderOptions(): string[] {
const out = this.getOutputThreadOptions();
@ -857,6 +917,10 @@ export class QsvSwDecodeConfig extends BaseHWConfig {
}
return out;
}
protected getFrameSamplingEncoderOptions(): string[] {
return ['-low_power', '1'];
}
}
export class QsvHwDecodeConfig extends QsvSwDecodeConfig {
@ -981,6 +1045,10 @@ export class VaapiSwDecodeConfig extends BaseHWConfig {
}
return out;
}
protected getFrameSamplingEncoderOptions(): string[] {
return ['-low_power', '1'];
}
}
export class VaapiHwDecodeConfig extends VaapiSwDecodeConfig {

2
server/test/medium.factory.ts
View file

@ -57,6 +57,7 @@ import { TagRepository } from 'src/repositories/tag.repository';
import { TelemetryRepository } from 'src/repositories/telemetry.repository';
import { UserRepository } from 'src/repositories/user.repository';
import { VersionHistoryRepository } from 'src/repositories/version-history.repository';
import { VideoFrameRepository } from 'src/repositories/video-frames.repository';
import { WorkflowRepository } from 'src/repositories/workflow.repository';
import { DB } from 'src/schema';
import { AlbumTable } from 'src/schema/tables/album.table';
@ -456,6 +457,7 @@ const newRealRepository = <T>(key: ClassConstructor<T>, db: Kysely<DB>): T => {
case SystemMetadataRepository:
case UserRepository:
case VersionHistoryRepository:
case VideoFrameRepository:
case WorkflowRepository: {
return new key(db);
}

93
server/test/medium/specs/repositories/video-frames.repository.spec.ts Normal file
View file

@ -0,0 +1,93 @@
import { Kysely } from 'kysely';
import { AssetType } from 'src/enum';
import { AssetJobRepository } from 'src/repositories/asset-job.repository';
import { AssetRepository } from 'src/repositories/asset.repository';
import { LoggingRepository } from 'src/repositories/logging.repository';
import { VideoFrameRepository } from 'src/repositories/video-frames.repository';
import { DB } from 'src/schema';
import { BaseService } from 'src/services/base.service';
import { newMediumService } from 'test/medium.factory';
import { getKyselyDB } from 'test/utils';
const consume = async <T>(generator: AsyncIterableIterator<T>) => {
const values: T[] = await Array.fromAsync(generator);
return values;
};
let defaultDatabase: Kysely<DB>;
const setup = (db?: Kysely<DB>) => {
const { ctx } = newMediumService(BaseService, {
database: db || defaultDatabase,
real: [],
mock: [LoggingRepository],
});
return { ctx, sut: ctx.get(VideoFrameRepository) };
};
const setupAssetJob = (db?: Kysely<DB>) => {
const { ctx } = newMediumService(BaseService, {
database: db || defaultDatabase,
real: [],
mock: [LoggingRepository],
});
return { ctx, sut: ctx.get(AssetJobRepository), videoFrameRepository: ctx.get(VideoFrameRepository) };
};
beforeAll(async () => {
defaultDatabase = await getKyselyDB();
});
describe(VideoFrameRepository.name, () => {
describe('cascade delete', () => {
it('should remove video_frames rows when the parent asset is deleted', async () => {
const { ctx, sut } = setup();
const assetRepository = ctx.get(AssetRepository);
const { user } = await ctx.newUser();
const { asset } = await ctx.newAsset({ ownerId: user.id, type: AssetType.Video });
await sut.upsertFrames(asset.id, { byteOffset: [813, 4656], byteSize: [3843, 3238], intervalChange: [0, 2.516] });
await assetRepository.remove({ id: asset.id });
await expect(sut.getFrames(asset.id)).resolves.toBeUndefined();
});
});
describe('upsertFrames', () => {
it('should atomically replace existing frames (delete-then-insert)', async () => {
const { ctx, sut } = setup();
const { user } = await ctx.newUser();
const { asset } = await ctx.newAsset({ ownerId: user.id, type: AssetType.Video });
await sut.upsertFrames(asset.id, {
byteOffset: [0, 100],
byteSize: [100, 200],
intervalChange: [0, 1.5],
});
const frames = await sut.getFrames(asset.id);
expect(frames?.byteOffset).toEqual([0, 100]);
await sut.upsertFrames(asset.id, { byteOffset: [999], byteSize: [50], intervalChange: [3.2] });
const framesUpdated = await sut.getFrames(asset.id);
expect(framesUpdated?.byteOffset).toEqual([999]);
});
});
});
describe(`${AssetJobRepository.name}.streamForFrameSampling`, () => {
it('should yield a video asset with no extraction record yet', async () => {
const { ctx, sut } = setupAssetJob();
const { user } = await ctx.newUser();
const { asset } = await ctx.newAsset({ ownerId: user.id, type: AssetType.Video });
const results = await consume(sut.streamForFrameSampling(false));
expect(results).toEqual(expect.arrayContaining([expect.objectContaining({ id: asset.id })]));
});
});

1
server/test/repositories/media.repository.mock.ts
View file

@ -21,5 +21,6 @@ export const newMediaRepositoryMock = (): Mocked<RepositoryInterface<MediaReposi
}),
transcode: vitest.fn(),
getImageMetadata: vitest.fn(),
sampleFrames: vitest.fn().mockResolvedValue({ byteRanges: [], intervalChanges: [] }),
};
};

1
server/test/repositories/storage.repository.mock.ts
View file

@ -58,6 +58,7 @@ export const newStorageRepositoryMock = (): Mocked<RepositoryInterface<StorageRe
createFile: vitest.fn(),
createWriteStream: vitest.fn(),
createOrOverwriteFile: vitest.fn(),
mkdtemp: vitest.fn(),
existsSync: vitest.fn(),
overwriteFile: vitest.fn(),
unlink: vitest.fn(),

4
server/test/utils.ts
View file

@ -66,6 +66,7 @@ import { TelemetryRepository } from 'src/repositories/telemetry.repository';
import { TrashRepository } from 'src/repositories/trash.repository';
import { UserRepository } from 'src/repositories/user.repository';
import { VersionHistoryRepository } from 'src/repositories/version-history.repository';
import { VideoFrameRepository } from 'src/repositories/video-frames.repository';
import { VideoStreamRepository } from 'src/repositories/video-stream.repository';
import { ViewRepository } from 'src/repositories/view-repository';
import { WebsocketRepository } from 'src/repositories/websocket.repository';
@ -276,6 +277,7 @@ export type ServiceOverrides = {
trash: TrashRepository;
user: UserRepository;
versionHistory: VersionHistoryRepository;
videoFrame: VideoFrameRepository;
videoStream: VideoStreamRepository;
view: ViewRepository;
websocket: WebsocketRepository;
@ -362,6 +364,7 @@ export const getMocks = () => {
trash: automock(TrashRepository),
user: automock(UserRepository, { strict: false }),
versionHistory: automock(VersionHistoryRepository),
videoFrame: automock(VideoFrameRepository),
videoStream: automock(VideoStreamRepository, { strict: false }),
view: automock(ViewRepository),
// eslint-disable-next-line no-sparse-arrays
@ -428,6 +431,7 @@ export const newTestService = <T extends BaseService>(
overrides.trash || (mocks.trash as As<TrashRepository>),
overrides.user || (mocks.user as As<UserRepository>),
overrides.versionHistory || (mocks.versionHistory as As<VersionHistoryRepository>),
overrides.videoFrame || (mocks.videoFrame as As<VideoFrameRepository>),
overrides.videoStream || (mocks.videoStream as As<VideoStreamRepository>),
overrides.view || (mocks.view as As<ViewRepository>),
overrides.websocket || (mocks.websocket as As<WebsocketRepository>),

1
web/src/lib/services/queue.service.ts
View file

@ -17,6 +17,7 @@ import {
mdiFileCheckOutline,
mdiFileJpgBox,
mdiFileXmlBox,
mdiFilmstripBoxMultiple,
mdiFolderMove,
mdiImageSearch,
mdiLibraryShelves,