helpy4/immich
1
0
Fork 0
mirror of https://github.com/immich-app/immich synced 2025-11-14 17:36:12 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat(server): Enqueue jobs in bulk (#5974)

Browse source 
* feat(server): Enqueue jobs in bulk

The Job Repository now has a `queueAll` method, that enqueues messages
in bulk (using BullMQ's
[`addBulk`](https://docs.bullmq.io/guide/queues/adding-bulks)),
improving performance when many jobs must be enqueued within the same
operation.

Primary change is in `src/domain/job/job.service.ts`, and other services
have been refactored to use `queueAll` when useful.

As a simple local benchmark, triggering a full thumbnail generation
process over a library of ~1,200 assets and ~350 faces went from
**~600ms** to **~250ms**.

* fix: Review feedback
This commit is contained in:
Michael Manganiello 2024-01-01 15:45:42 -05:00 • committed by GitHub
parent 7dd88c4114
commit 4a5b8c3770
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
20 changed files with 323 additions and 227 deletions
Download patch file Download diff file Expand all files Collapse all files

77
server/src/domain/library/library.service.spec.ts
View file

@ -135,18 +135,16 @@ describe(LibraryService.name, () => {
await sut.handleQueueAssetRefresh(mockLibraryJob);
expect(jobMock.queue.mock.calls).toEqual([
[
{
name: JobName.LIBRARY_SCAN_ASSET,
data: {
id: libraryStub.externalLibrary1.id,
ownerId: libraryStub.externalLibrary1.owner.id,
assetPath: '/data/user1/photo.jpg',
force: false,
},
expect(jobMock.queueAll).toHaveBeenCalledWith([
{
name: JobName.LIBRARY_SCAN_ASSET,
data: {
id: libraryStub.externalLibrary1.id,
ownerId: libraryStub.externalLibrary1.owner.id,
assetPath: '/data/user1/photo.jpg',
force: false,
},
],
},
]);
});
@ -420,6 +418,7 @@ describe(LibraryService.name, () => {
await expect(sut.handleAssetRefresh(mockLibraryJob)).resolves.toBe(true);
expect(jobMock.queue).not.toHaveBeenCalled();
expect(jobMock.queueAll).not.toHaveBeenCalled();
});
it('should import an asset when mtime differs from db asset', async () => {
@ -468,6 +467,7 @@ describe(LibraryService.name, () => {
expect(assetMock.save).toHaveBeenCalledWith({ id: assetStub.image.id, isOffline: true });
expect(jobMock.queue).not.toHaveBeenCalled();
expect(jobMock.queueAll).not.toHaveBeenCalled();
});
it('should online a previously-offline asset', async () => {
@ -607,6 +607,7 @@ describe(LibraryService.name, () => {
);
expect(jobMock.queue).not.toHaveBeenCalled();
expect(jobMock.queueAll).not.toHaveBeenCalled();
expect(libraryMock.softDelete).not.toHaveBeenCalled();
});
@ -953,9 +954,9 @@ describe(LibraryService.name, () => {
libraryMock.getAllDeleted.mockResolvedValue([libraryStub.uploadLibrary1, libraryStub.externalLibrary1]);
await expect(sut.handleQueueCleanup()).resolves.toBe(true);
expect(jobMock.queue.mock.calls).toEqual([
[{ name: JobName.LIBRARY_DELETE, data: { id: libraryStub.uploadLibrary1.id } }],
[{ name: JobName.LIBRARY_DELETE, data: { id: libraryStub.externalLibrary1.id } }],
expect(jobMock.queueAll).toHaveBeenCalledWith([
{ name: JobName.LIBRARY_DELETE, data: { id: libraryStub.uploadLibrary1.id } },
{ name: JobName.LIBRARY_DELETE, data: { id: libraryStub.externalLibrary1.id } },
]);
});
});
@ -1101,16 +1102,16 @@ describe(LibraryService.name, () => {
data: {},
},
],
[
{
name: JobName.LIBRARY_SCAN,
data: {
id: libraryStub.externalLibrary1.id,
refreshModifiedFiles: true,
refreshAllFiles: false,
},
]);
expect(jobMock.queueAll).toHaveBeenCalledWith([
{
name: JobName.LIBRARY_SCAN,
data: {
id: libraryStub.externalLibrary1.id,
refreshModifiedFiles: true,
refreshAllFiles: false,
},
],
},
]);
});
@ -1126,16 +1127,16 @@ describe(LibraryService.name, () => {
data: {},
},
],
[
{
name: JobName.LIBRARY_SCAN,
data: {
id: libraryStub.externalLibrary1.id,
refreshModifiedFiles: false,
refreshAllFiles: true,
},
]);
expect(jobMock.queueAll).toHaveBeenCalledWith([
{
name: JobName.LIBRARY_SCAN,
data: {
id: libraryStub.externalLibrary1.id,
refreshModifiedFiles: false,
refreshAllFiles: true,
},
],
},
]);
});
});
@ -1147,13 +1148,11 @@ describe(LibraryService.name, () => {
await expect(sut.handleOfflineRemoval({ id: libraryStub.externalLibrary1.id })).resolves.toBe(true);
expect(jobMock.queue.mock.calls).toEqual([
[
{
name: JobName.ASSET_DELETION,
data: { id: assetStub.image1.id, fromExternal: true },
},
],
expect(jobMock.queueAll).toHaveBeenCalledWith([
{
name: JobName.ASSET_DELETION,
data: { id: assetStub.image1.id, fromExternal: true },
},
]);
});
});