mirror of
https://github.com/immich-app/immich
synced 2025-11-14 17:36:12 +00:00
chore(server): Move library watcher to microservices (#7533)
* move watcher init to micro
* document watcher recovery
* chore: fix lint
* add try lock
* use global library watch lock
* fix: ensure lock stays on
* fix: mocks
* unit test for library watch lock
* move statement to correct test
* fix: correct return type of try lock
* fix: tests
* add library teardown
* add chokidar error handler
* make event strings an enum
* wait for event refactor
* refactor event type mocks
* expect correct error
* don't release lock in teardown
* chore: lint
* use enum
* fix mock
* fix lint
* fix watcher await
* remove await
* simplify typing
* remove async
* Revert "remove async"
This reverts commit 84ab5abac4.
* can now change watch settings at runtime
* fix lint
* only watch libraries if enabled
---------
Co-authored-by: mertalev <101130780+mertalev@users.noreply.github.com>
Co-authored-by: Alex Tran <alex.tran1502@gmail.com>
This commit is contained in:
Jonathan Jogenfors
GitHub
parent
3278dcbcbe
commit
4cb0f37918
14 changed files with 149 additions and 81 deletions
|
|
@ -9,11 +9,11 @@ import {
|
|||
newAccessRepositoryMock,
|
||||
newAssetRepositoryMock,
|
||||
newCryptoRepositoryMock,
|
||||
newDatabaseRepositoryMock,
|
||||
newJobRepositoryMock,
|
||||
newLibraryRepositoryMock,
|
||||
newStorageRepositoryMock,
|
||||
newSystemConfigRepositoryMock,
|
||||
newUserRepositoryMock,
|
||||
systemConfigStub,
|
||||
userStub,
|
||||
} from '@test';
|
||||
|
|
@ -23,11 +23,12 @@ import { ILibraryFileJob, ILibraryRefreshJob, JobName } from '../job';
|
|||
import {
|
||||
IAssetRepository,
|
||||
ICryptoRepository,
|
||||
IDatabaseRepository,
|
||||
IJobRepository,
|
||||
ILibraryRepository,
|
||||
IStorageRepository,
|
||||
ISystemConfigRepository,
|
||||
IUserRepository,
|
||||
StorageEventType,
|
||||
} from '../repositories';
|
||||
import { SystemConfigCore } from '../system-config/system-config.core';
|
||||
import { mapLibrary } from './library.dto';
|
||||
|
|
@ -40,20 +41,20 @@ describe(LibraryService.name, () => {
|
|||
let assetMock: jest.Mocked<IAssetRepository>;
|
||||
let configMock: jest.Mocked<ISystemConfigRepository>;
|
||||
let cryptoMock: jest.Mocked<ICryptoRepository>;
|
||||
let userMock: jest.Mocked<IUserRepository>;
|
||||
let jobMock: jest.Mocked<IJobRepository>;
|
||||
let libraryMock: jest.Mocked<ILibraryRepository>;
|
||||
let storageMock: jest.Mocked<IStorageRepository>;
|
||||
let databaseMock: jest.Mocked<IDatabaseRepository>;
|
||||
|
||||
beforeEach(() => {
|
||||
accessMock = newAccessRepositoryMock();
|
||||
configMock = newSystemConfigRepositoryMock();
|
||||
libraryMock = newLibraryRepositoryMock();
|
||||
userMock = newUserRepositoryMock();
|
||||
assetMock = newAssetRepositoryMock();
|
||||
jobMock = newJobRepositoryMock();
|
||||
cryptoMock = newCryptoRepositoryMock();
|
||||
storageMock = newStorageRepositoryMock();
|
||||
databaseMock = newDatabaseRepositoryMock();
|
||||
|
||||
// Always validate owner access for library.
|
||||
accessMock.library.checkOwnerAccess.mockImplementation((_, libraryIds) => Promise.resolve(libraryIds));
|
||||
|
|
@ -66,8 +67,10 @@ describe(LibraryService.name, () => {
|
|||
jobMock,
|
||||
libraryMock,
|
||||
storageMock,
|
||||
userMock,
|
||||
databaseMock,
|
||||
);
|
||||
|
||||
databaseMock.tryLock.mockResolvedValue(true);
|
||||
});
|
||||
|
||||
it('should work', () => {
|
||||
|
|
@ -125,13 +128,22 @@ describe(LibraryService.name, () => {
|
|||
);
|
||||
});
|
||||
|
||||
it('should not initialize when watching is disabled', async () => {
|
||||
it('should not initialize watcher when watching is disabled', async () => {
|
||||
configMock.load.mockResolvedValue(systemConfigStub.libraryWatchDisabled);
|
||||
|
||||
await sut.init();
|
||||
|
||||
expect(storageMock.watch).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should not initialize watcher when lock is taken', async () => {
|
||||
configMock.load.mockResolvedValue(systemConfigStub.libraryWatchEnabled);
|
||||
databaseMock.tryLock.mockResolvedValue(false);
|
||||
|
||||
await sut.init();
|
||||
|
||||
expect(storageMock.watch).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('handleQueueAssetRefresh', () => {
|
||||
|
|
@ -146,7 +158,6 @@ describe(LibraryService.name, () => {
|
|||
storageMock.crawl.mockResolvedValue(['/data/user1/photo.jpg']);
|
||||
assetMock.getPathsNotInLibrary.mockResolvedValue(['/data/user1/photo.jpg']);
|
||||
assetMock.getByLibraryId.mockResolvedValue([]);
|
||||
userMock.get.mockResolvedValue(userStub.admin);
|
||||
|
||||
await sut.handleQueueAssetRefresh(mockLibraryJob);
|
||||
|
||||
|
|
@ -173,7 +184,6 @@ describe(LibraryService.name, () => {
|
|||
libraryMock.get.mockResolvedValue(libraryStub.externalLibrary1);
|
||||
storageMock.crawl.mockResolvedValue(['/data/user1/photo.jpg']);
|
||||
assetMock.getByLibraryId.mockResolvedValue([]);
|
||||
userMock.get.mockResolvedValue(userStub.admin);
|
||||
|
||||
await sut.handleQueueAssetRefresh(mockLibraryJob);
|
||||
|
||||
|
|
@ -224,7 +234,6 @@ describe(LibraryService.name, () => {
|
|||
libraryMock.get.mockResolvedValue(libraryStub.externalLibraryWithImportPaths1);
|
||||
storageMock.crawl.mockResolvedValue([]);
|
||||
assetMock.getByLibraryId.mockResolvedValue([]);
|
||||
userMock.get.mockResolvedValue(userStub.externalPathRoot);
|
||||
|
||||
await sut.handleQueueAssetRefresh(mockLibraryJob);
|
||||
|
||||
|
|
@ -240,7 +249,6 @@ describe(LibraryService.name, () => {
|
|||
|
||||
beforeEach(() => {
|
||||
mockUser = userStub.admin;
|
||||
userMock.get.mockResolvedValue(mockUser);
|
||||
|
||||
storageMock.stat.mockResolvedValue({
|
||||
size: 100,
|
||||
|
|
@ -1167,7 +1175,9 @@ describe(LibraryService.name, () => {
|
|||
it('should handle a new file event', async () => {
|
||||
libraryMock.get.mockResolvedValue(libraryStub.externalLibraryWithImportPaths1);
|
||||
libraryMock.getAll.mockResolvedValue([libraryStub.externalLibraryWithImportPaths1]);
|
||||
storageMock.watch.mockImplementation(makeMockWatcher({ items: [{ event: 'add', value: '/foo/photo.jpg' }] }));
|
||||
storageMock.watch.mockImplementation(
|
||||
makeMockWatcher({ items: [{ event: StorageEventType.ADD, value: '/foo/photo.jpg' }] }),
|
||||
);
|
||||
|
||||
await sut.watchAll();
|
||||
|
||||
|
|
@ -1188,7 +1198,7 @@ describe(LibraryService.name, () => {
|
|||
libraryMock.get.mockResolvedValue(libraryStub.externalLibraryWithImportPaths1);
|
||||
libraryMock.getAll.mockResolvedValue([libraryStub.externalLibraryWithImportPaths1]);
|
||||
storageMock.watch.mockImplementation(
|
||||
makeMockWatcher({ items: [{ event: 'change', value: '/foo/photo.jpg' }] }),
|
||||
makeMockWatcher({ items: [{ event: StorageEventType.CHANGE, value: '/foo/photo.jpg' }] }),
|
||||
);
|
||||
|
||||
await sut.watchAll();
|
||||
|
|
@ -1211,7 +1221,7 @@ describe(LibraryService.name, () => {
|
|||
libraryMock.getAll.mockResolvedValue([libraryStub.externalLibraryWithImportPaths1]);
|
||||
assetMock.getByLibraryIdAndOriginalPath.mockResolvedValue(assetStub.external);
|
||||
storageMock.watch.mockImplementation(
|
||||
makeMockWatcher({ items: [{ event: 'unlink', value: '/foo/photo.jpg' }] }),
|
||||
makeMockWatcher({ items: [{ event: StorageEventType.UNLINK, value: '/foo/photo.jpg' }] }),
|
||||
);
|
||||
|
||||
await sut.watchAll();
|
||||
|
|
@ -1225,17 +1235,19 @@ describe(LibraryService.name, () => {
|
|||
libraryMock.getAll.mockResolvedValue([libraryStub.externalLibraryWithImportPaths1]);
|
||||
storageMock.watch.mockImplementation(
|
||||
makeMockWatcher({
|
||||
items: [{ event: 'error', value: 'Error!' }],
|
||||
items: [{ event: StorageEventType.ERROR, value: 'Error!' }],
|
||||
}),
|
||||
);
|
||||
|
||||
await sut.watchAll();
|
||||
await expect(sut.watchAll()).rejects.toThrow('Error!');
|
||||
});
|
||||
|
||||
it('should ignore unknown extensions', async () => {
|
||||
libraryMock.get.mockResolvedValue(libraryStub.externalLibraryWithImportPaths1);
|
||||
libraryMock.getAll.mockResolvedValue([libraryStub.externalLibraryWithImportPaths1]);
|
||||
storageMock.watch.mockImplementation(makeMockWatcher({ items: [{ event: 'add', value: '/foo/photo.jpg' }] }));
|
||||
storageMock.watch.mockImplementation(
|
||||
makeMockWatcher({ items: [{ event: StorageEventType.ADD, value: '/foo/photo.jpg' }] }),
|
||||
);
|
||||
|
||||
await sut.watchAll();
|
||||
|
||||
|
|
@ -1245,7 +1257,9 @@ describe(LibraryService.name, () => {
|
|||
it('should ignore excluded paths', async () => {
|
||||
libraryMock.get.mockResolvedValue(libraryStub.patternPath);
|
||||
libraryMock.getAll.mockResolvedValue([libraryStub.patternPath]);
|
||||
storageMock.watch.mockImplementation(makeMockWatcher({ items: [{ event: 'add', value: '/dir1/photo.txt' }] }));
|
||||
storageMock.watch.mockImplementation(
|
||||
makeMockWatcher({ items: [{ event: StorageEventType.ADD, value: '/dir1/photo.txt' }] }),
|
||||
);
|
||||
|
||||
await sut.watchAll();
|
||||
|
||||
|
|
@ -1255,7 +1269,9 @@ describe(LibraryService.name, () => {
|
|||
it('should ignore excluded paths without case sensitivity', async () => {
|
||||
libraryMock.get.mockResolvedValue(libraryStub.patternPath);
|
||||
libraryMock.getAll.mockResolvedValue([libraryStub.patternPath]);
|
||||
storageMock.watch.mockImplementation(makeMockWatcher({ items: [{ event: 'add', value: '/DIR1/photo.txt' }] }));
|
||||
storageMock.watch.mockImplementation(
|
||||
makeMockWatcher({ items: [{ event: StorageEventType.ADD, value: '/DIR1/photo.txt' }] }),
|
||||
);
|
||||
|
||||
await sut.watchAll();
|
||||
|
||||
|
|
@ -1264,7 +1280,7 @@ describe(LibraryService.name, () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe('tearDown', () => {
|
||||
describe('teardown', () => {
|
||||
it('should tear down all watchers', async () => {
|
||||
libraryMock.getAll.mockResolvedValue([
|
||||
libraryStub.externalLibraryWithImportPaths1,
|
||||
|
|
@ -1286,7 +1302,7 @@ describe(LibraryService.name, () => {
|
|||
storageMock.watch.mockImplementation(makeMockWatcher({ close: mockClose }));
|
||||
|
||||
await sut.init();
|
||||
await sut.unwatchAll();
|
||||
await sut.teardown();
|
||||
|
||||
expect(mockClose).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue