mirror of
https://github.com/immich-app/immich
synced 2025-11-14 17:36:12 +00:00
fix(server): don't crash when refreshing large libraries (#7934)
* add job to check for offline files * fix lint * only check for offline when using checkForOffline * improve tests * remove old test * wip * remove trie * refactor batches * also check offline status * fix spelling * don't do offline scan * rename scan to check * fix job statuses * fix lint * cleanup * add test * open-api * fix test * fix spinner * reset text * don't double batch * fix comments from mert * remove tries * fix tests * fix e2e * fix test * fix test * add tests * fix lint * fix e2e * interweave scans * fix errors * fix messages * fix test * add mock * fix sql * fix e2e * use library batch size * save -> update * add file extensions * update specs * test for import paths * check import paths when testing offline * fix lint * normalize import path * remove console logs * decrease batch size to 1000 * add test for import path * add test for already-online assets * fix merge * fix lint * add library job back * add offline job to correct queue * library spec compiles now * move one test to new e2e * fix comments * fix comments * fix lint * refactor path validation * fix loop bug * remove logging * expect responses * fix asset mock * take the straightforward approach * use generator correctly * fix vitest on file edit * bump vitest to 1.6.0 * test for offline check * add e2e tests for offlining assets depending on import path * cleanup e2e test after finish * cleanup library service * paginate the walk generator * fix tests * fix typo * refactoring handleOfflineCheck * better testing of handleOfflineCheck * fix lint * handle large library deletions * dont check if library is deleted * fix mock * add a 100k page size to library * fix loading animation * better log messages * Better logging for offline asset removal * fix sql and tests * fix number format * Remove submodule * fix format * chore: cleanup * chore: fix tests --------- Co-authored-by: Alex <alex.tran1502@gmail.com> Co-authored-by: Jason Rasmussen <jason@rasm.me>
This commit is contained in:
Jonathan Jogenfors
GitHub
parent
5811025ebd
commit
363c558db7
18 changed files with 364 additions and 282 deletions
|
|
@ -8,7 +8,15 @@ import { AssetType } from 'src/enum';
|
|||
import { IAssetRepository } from 'src/interfaces/asset.interface';
|
||||
import { ICryptoRepository } from 'src/interfaces/crypto.interface';
|
||||
import { IDatabaseRepository } from 'src/interfaces/database.interface';
|
||||
import { IJobRepository, ILibraryFileJob, ILibraryRefreshJob, JobName, JobStatus } from 'src/interfaces/job.interface';
|
||||
import {
|
||||
IJobRepository,
|
||||
ILibraryFileJob,
|
||||
ILibraryOfflineJob,
|
||||
ILibraryRefreshJob,
|
||||
JobName,
|
||||
JOBS_LIBRARY_PAGINATION_SIZE,
|
||||
JobStatus,
|
||||
} from 'src/interfaces/job.interface';
|
||||
import { ILibraryRepository } from 'src/interfaces/library.interface';
|
||||
import { ILoggerRepository } from 'src/interfaces/logger.interface';
|
||||
import { IStorageRepository } from 'src/interfaces/storage.interface';
|
||||
|
|
@ -154,17 +162,19 @@ describe(LibraryService.name, () => {
|
|||
});
|
||||
|
||||
describe('handleQueueAssetRefresh', () => {
|
||||
it('should queue new assets', async () => {
|
||||
it('should queue refresh of a new asset', async () => {
|
||||
const mockLibraryJob: ILibraryRefreshJob = {
|
||||
id: libraryStub.externalLibrary1.id,
|
||||
refreshModifiedFiles: false,
|
||||
refreshAllFiles: false,
|
||||
};
|
||||
|
||||
assetMock.getWith.mockResolvedValue({ items: [], hasNextPage: false });
|
||||
|
||||
libraryMock.get.mockResolvedValue(libraryStub.externalLibrary1);
|
||||
// eslint-disable-next-line @typescript-eslint/require-await
|
||||
storageMock.walk.mockImplementation(async function* generator() {
|
||||
yield '/data/user1/photo.jpg';
|
||||
yield ['/data/user1/photo.jpg'];
|
||||
});
|
||||
assetMock.getExternalLibraryAssetPaths.mockResolvedValue({ items: [], hasNextPage: false });
|
||||
|
||||
|
|
@ -183,6 +193,44 @@ describe(LibraryService.name, () => {
|
|||
]);
|
||||
});
|
||||
|
||||
it('should queue offline check of existing online assets', async () => {
|
||||
const mockLibraryJob: ILibraryRefreshJob = {
|
||||
id: libraryStub.externalLibrary1.id,
|
||||
refreshModifiedFiles: false,
|
||||
refreshAllFiles: false,
|
||||
};
|
||||
|
||||
assetMock.getWith.mockResolvedValue({ items: [], hasNextPage: false });
|
||||
libraryMock.get.mockResolvedValue(libraryStub.externalLibrary1);
|
||||
storageMock.walk.mockImplementation(async function* generator() {});
|
||||
assetMock.getWith.mockResolvedValue({ items: [assetStub.external], hasNextPage: false });
|
||||
|
||||
await sut.handleQueueAssetRefresh(mockLibraryJob);
|
||||
|
||||
expect(jobMock.queueAll).toHaveBeenCalledWith([
|
||||
{
|
||||
name: JobName.LIBRARY_CHECK_OFFLINE,
|
||||
data: {
|
||||
id: assetStub.external.id,
|
||||
importPaths: libraryStub.externalLibrary1.importPaths,
|
||||
exclusionPatterns: [],
|
||||
},
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("should fail when library can't be found", async () => {
|
||||
const mockLibraryJob: ILibraryRefreshJob = {
|
||||
id: libraryStub.externalLibrary1.id,
|
||||
refreshModifiedFiles: false,
|
||||
refreshAllFiles: false,
|
||||
};
|
||||
|
||||
libraryMock.get.mockResolvedValue(null);
|
||||
|
||||
await expect(sut.handleQueueAssetRefresh(mockLibraryJob)).resolves.toBe(JobStatus.SKIPPED);
|
||||
});
|
||||
|
||||
it('should force queue new assets', async () => {
|
||||
const mockLibraryJob: ILibraryRefreshJob = {
|
||||
id: libraryStub.externalLibrary1.id,
|
||||
|
|
@ -190,10 +238,11 @@ describe(LibraryService.name, () => {
|
|||
refreshAllFiles: true,
|
||||
};
|
||||
|
||||
assetMock.getWith.mockResolvedValue({ items: [], hasNextPage: false });
|
||||
libraryMock.get.mockResolvedValue(libraryStub.externalLibrary1);
|
||||
// eslint-disable-next-line @typescript-eslint/require-await
|
||||
storageMock.walk.mockImplementation(async function* generator() {
|
||||
yield '/data/user1/photo.jpg';
|
||||
yield ['/data/user1/photo.jpg'];
|
||||
});
|
||||
assetMock.getExternalLibraryAssetPaths.mockResolvedValue({ items: [], hasNextPage: false });
|
||||
|
||||
|
|
@ -225,6 +274,8 @@ describe(LibraryService.name, () => {
|
|||
|
||||
storageMock.checkFileExists.mockResolvedValue(true);
|
||||
|
||||
assetMock.getWith.mockResolvedValue({ items: [], hasNextPage: false });
|
||||
|
||||
const mockLibraryJob: ILibraryRefreshJob = {
|
||||
id: libraryStub.externalLibraryWithImportPaths1.id,
|
||||
refreshModifiedFiles: false,
|
||||
|
|
@ -239,51 +290,78 @@ describe(LibraryService.name, () => {
|
|||
expect(storageMock.walk).toHaveBeenCalledWith({
|
||||
pathsToCrawl: [libraryStub.externalLibraryWithImportPaths1.importPaths[1]],
|
||||
exclusionPatterns: [],
|
||||
includeHidden: false,
|
||||
take: JOBS_LIBRARY_PAGINATION_SIZE,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it('should set missing assets offline', async () => {
|
||||
const mockLibraryJob: ILibraryRefreshJob = {
|
||||
id: libraryStub.externalLibrary1.id,
|
||||
refreshModifiedFiles: false,
|
||||
refreshAllFiles: false,
|
||||
describe('handleOfflineCheck', () => {
|
||||
it('should skip missing assets', async () => {
|
||||
const mockAssetJob: ILibraryOfflineJob = {
|
||||
id: assetStub.external.id,
|
||||
importPaths: ['/'],
|
||||
};
|
||||
|
||||
libraryMock.get.mockResolvedValue(libraryStub.externalLibrary1);
|
||||
assetMock.getExternalLibraryAssetPaths.mockResolvedValue({
|
||||
items: [assetStub.external],
|
||||
hasNextPage: false,
|
||||
});
|
||||
assetMock.getById.mockResolvedValue(null);
|
||||
|
||||
await sut.handleQueueAssetRefresh(mockLibraryJob);
|
||||
await expect(sut.handleOfflineCheck(mockAssetJob)).resolves.toBe(JobStatus.SKIPPED);
|
||||
|
||||
expect(assetMock.updateAll).toHaveBeenCalledWith([assetStub.image.id], { isOffline: true });
|
||||
expect(assetMock.updateAll).not.toHaveBeenCalledWith(expect.anything(), { isOffline: false });
|
||||
expect(jobMock.queueAll).not.toHaveBeenCalled();
|
||||
expect(assetMock.update).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should set crawled assets that were previously offline back online', async () => {
|
||||
const mockLibraryJob: ILibraryRefreshJob = {
|
||||
id: libraryStub.externalLibrary1.id,
|
||||
refreshModifiedFiles: false,
|
||||
refreshAllFiles: false,
|
||||
it('should do nothing with already-offline assets', async () => {
|
||||
const mockAssetJob: ILibraryOfflineJob = {
|
||||
id: assetStub.external.id,
|
||||
importPaths: ['/'],
|
||||
};
|
||||
|
||||
libraryMock.get.mockResolvedValue(libraryStub.externalLibrary1);
|
||||
// eslint-disable-next-line @typescript-eslint/require-await
|
||||
storageMock.walk.mockImplementation(async function* generator() {
|
||||
yield assetStub.externalOffline.originalPath;
|
||||
});
|
||||
assetMock.getExternalLibraryAssetPaths.mockResolvedValue({
|
||||
items: [assetStub.externalOffline],
|
||||
hasNextPage: false,
|
||||
});
|
||||
assetMock.getById.mockResolvedValue(assetStub.offline);
|
||||
|
||||
await sut.handleQueueAssetRefresh(mockLibraryJob);
|
||||
await expect(sut.handleOfflineCheck(mockAssetJob)).resolves.toBe(JobStatus.SUCCESS);
|
||||
|
||||
expect(assetMock.updateAll).toHaveBeenCalledWith([assetStub.externalOffline.id], { isOffline: false });
|
||||
expect(assetMock.updateAll).not.toHaveBeenCalledWith(expect.anything(), { isOffline: true });
|
||||
expect(jobMock.queueAll).not.toHaveBeenCalled();
|
||||
expect(assetMock.update).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should offline assets no longer on disk or matching exclusion pattern', async () => {
|
||||
const mockAssetJob: ILibraryOfflineJob = {
|
||||
id: assetStub.external.id,
|
||||
importPaths: ['/'],
|
||||
};
|
||||
|
||||
assetMock.getById.mockResolvedValue(assetStub.external);
|
||||
|
||||
await expect(sut.handleOfflineCheck(mockAssetJob)).resolves.toBe(JobStatus.SUCCESS);
|
||||
|
||||
expect(assetMock.update).toHaveBeenCalledWith({ id: assetStub.external.id, isOffline: true });
|
||||
});
|
||||
|
||||
it('should set assets outside of import paths as offline', async () => {
|
||||
const mockAssetJob: ILibraryOfflineJob = {
|
||||
id: assetStub.external.id,
|
||||
importPaths: ['/data/user2'],
|
||||
};
|
||||
|
||||
assetMock.getById.mockResolvedValue(assetStub.external);
|
||||
storageMock.checkFileExists.mockResolvedValue(true);
|
||||
|
||||
await expect(sut.handleOfflineCheck(mockAssetJob)).resolves.toBe(JobStatus.SUCCESS);
|
||||
|
||||
expect(assetMock.update).toHaveBeenCalledWith({ id: assetStub.external.id, isOffline: true });
|
||||
});
|
||||
|
||||
it('should do nothing with online assets', async () => {
|
||||
const mockAssetJob: ILibraryOfflineJob = {
|
||||
id: assetStub.external.id,
|
||||
importPaths: ['/'],
|
||||
};
|
||||
|
||||
assetMock.getById.mockResolvedValue(assetStub.external);
|
||||
storageMock.checkFileExists.mockResolvedValue(true);
|
||||
|
||||
await expect(sut.handleOfflineCheck(mockAssetJob)).resolves.toBe(JobStatus.SUCCESS);
|
||||
|
||||
expect(assetMock.update).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
|
|
@ -1115,18 +1193,9 @@ describe(LibraryService.name, () => {
|
|||
});
|
||||
|
||||
describe('handleDeleteLibrary', () => {
|
||||
it('should not delete a nonexistent library', async () => {
|
||||
libraryMock.get.mockResolvedValue(null);
|
||||
|
||||
libraryMock.getAssetIds.mockResolvedValue([]);
|
||||
libraryMock.delete.mockImplementation(async () => {});
|
||||
|
||||
await expect(sut.handleDeleteLibrary({ id: libraryStub.externalLibrary1.id })).resolves.toBe(JobStatus.FAILED);
|
||||
});
|
||||
|
||||
it('should delete an empty library', async () => {
|
||||
libraryMock.get.mockResolvedValue(libraryStub.externalLibrary1);
|
||||
libraryMock.getAssetIds.mockResolvedValue([]);
|
||||
assetMock.getAll.mockResolvedValue({ items: [], hasNextPage: false });
|
||||
libraryMock.delete.mockImplementation(async () => {});
|
||||
|
||||
await expect(sut.handleDeleteLibrary({ id: libraryStub.externalLibrary1.id })).resolves.toBe(JobStatus.SUCCESS);
|
||||
|
|
@ -1134,7 +1203,7 @@ describe(LibraryService.name, () => {
|
|||
|
||||
it('should delete a library with assets', async () => {
|
||||
libraryMock.get.mockResolvedValue(libraryStub.externalLibrary1);
|
||||
libraryMock.getAssetIds.mockResolvedValue([assetStub.image1.id]);
|
||||
assetMock.getAll.mockResolvedValue({ items: [assetStub.image1], hasNextPage: false });
|
||||
libraryMock.delete.mockImplementation(async () => {});
|
||||
|
||||
assetMock.getById.mockResolvedValue(assetStub.image1);
|
||||
|
|
@ -1273,7 +1342,7 @@ describe(LibraryService.name, () => {
|
|||
assetMock.getWith.mockResolvedValue({ items: [assetStub.image1], hasNextPage: false });
|
||||
assetMock.getById.mockResolvedValue(assetStub.image1);
|
||||
|
||||
await expect(sut.handleOfflineRemoval({ id: libraryStub.externalLibrary1.id })).resolves.toBe(JobStatus.SUCCESS);
|
||||
await expect(sut.handleRemoveOffline({ id: libraryStub.externalLibrary1.id })).resolves.toBe(JobStatus.SUCCESS);
|
||||
|
||||
expect(jobMock.queueAll).toHaveBeenCalledWith([
|
||||
{ name: JobName.ASSET_DELETION, data: { id: assetStub.image1.id, deleteOnDisk: false } },
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue