helpy4/immich
1
0
Fork 0
mirror of https://github.com/immich-app/immich synced 2025-11-14 17:36:12 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat(server): Enqueue jobs in bulk (#5974)

Browse source 
* feat(server): Enqueue jobs in bulk

The Job Repository now has a `queueAll` method, that enqueues messages
in bulk (using BullMQ's
[`addBulk`](https://docs.bullmq.io/guide/queues/adding-bulks)),
improving performance when many jobs must be enqueued within the same
operation.

Primary change is in `src/domain/job/job.service.ts`, and other services
have been refactored to use `queueAll` when useful.

As a simple local benchmark, triggering a full thumbnail generation
process over a library of ~1,200 assets and ~350 faces went from
**~600ms** to **~250ms**.

* fix: Review feedback
This commit is contained in:
Michael Manganiello 2024-01-01 15:45:42 -05:00 • committed by GitHub
parent 7dd88c4114
commit 4a5b8c3770
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
20 changed files with 323 additions and 227 deletions
Download patch file Download diff file Expand all files Collapse all files

27
server/src/domain/asset/asset.service.ts
View file

@ -21,6 +21,7 @@ import {
IStorageRepository,
ISystemConfigRepository,
ImmichReadStream,
JobItem,
TimeBucketOptions,
} from '../repositories';
import { StorageCore, StorageFolder } from '../storage';
@ -449,9 +450,9 @@ export class AssetService {
);
for await (const assets of assetPagination) {
for (const asset of assets) {
await this.jobRepository.queue({ name: JobName.ASSET_DELETION, data: { id: asset.id } });
}
await this.jobRepository.queueAll(
assets.map((asset) => ({ name: JobName.ASSET_DELETION, data: { id: asset.id } })),
);
}
return true;
@ -504,9 +505,7 @@ export class AssetService {
await this.access.requirePermission(auth, Permission.ASSET_DELETE, ids);
if (force) {
for (const id of ids) {
await this.jobRepository.queue({ name: JobName.ASSET_DELETION, data: { id } });
}
await this.jobRepository.queueAll(ids.map((id) => ({ name: JobName.ASSET_DELETION, data: { id } })));
} else {
await this.assetRepository.softDeleteAll(ids);
this.communicationRepository.send(ClientEvent.ASSET_TRASH, auth.user.id, ids);
@ -529,9 +528,9 @@ export class AssetService {
if (action == TrashAction.EMPTY_ALL) {
for await (const assets of assetPagination) {
for (const asset of assets) {
await this.jobRepository.queue({ name: JobName.ASSET_DELETION, data: { id: asset.id } });
}
await this.jobRepository.queueAll(
assets.map((asset) => ({ name: JobName.ASSET_DELETION, data: { id: asset.id } })),
);
}
return;
}
@ -566,21 +565,25 @@ export class AssetService {
async run(auth: AuthDto, dto: AssetJobsDto) {
await this.access.requirePermission(auth, Permission.ASSET_UPDATE, dto.assetIds);
const jobs: JobItem[] = [];
for (const id of dto.assetIds) {
switch (dto.name) {
case AssetJobName.REFRESH_METADATA:
await this.jobRepository.queue({ name: JobName.METADATA_EXTRACTION, data: { id } });
jobs.push({ name: JobName.METADATA_EXTRACTION, data: { id } });
break;
case AssetJobName.REGENERATE_THUMBNAIL:
await this.jobRepository.queue({ name: JobName.GENERATE_JPEG_THUMBNAIL, data: { id } });
jobs.push({ name: JobName.GENERATE_JPEG_THUMBNAIL, data: { id } });
break;
case AssetJobName.TRANSCODE_VIDEO:
await this.jobRepository.queue({ name: JobName.VIDEO_CONVERSION, data: { id } });
jobs.push({ name: JobName.VIDEO_CONVERSION, data: { id } });
break;
}
}
await this.jobRepository.queueAll(jobs);
}
private async updateMetadata(dto: ISidecarWriteJob) {