mirror of
https://github.com/immich-app/immich
synced 2025-11-14 17:36:12 +00:00
refactor(server): jobs and processors (#1787)
* refactor: jobs and processors * refactor: storage migration processor * fix: tests * fix: code warning * chore: ignore coverage from infra * fix: sync move asset logic between job core and asset core * refactor: move error handling inside of catch * refactor(server): job core into dedicated service calls * refactor: smart info * fix: tests * chore: smart info tests * refactor: use asset repository * refactor: thumbnail processor * chore: coverage reqs
This commit is contained in:
Jason Rasmussen
GitHub
parent
71d8567f18
commit
6c7679714b
108 changed files with 1645 additions and 1072 deletions
|
|
@ -23,6 +23,7 @@ export interface IAssetRepository {
|
|||
asset: Omit<AssetEntity, 'id' | 'createdAt' | 'updatedAt' | 'ownerId' | 'livePhotoVideoId'>,
|
||||
): Promise<AssetEntity>;
|
||||
remove(asset: AssetEntity): Promise<void>;
|
||||
save(asset: Partial<AssetEntity>): Promise<AssetEntity>;
|
||||
|
||||
update(userId: string, asset: AssetEntity, dto: UpdateAssetDto): Promise<AssetEntity>;
|
||||
getAll(): Promise<AssetEntity[]>;
|
||||
|
|
@ -292,6 +293,11 @@ export class AssetRepository implements IAssetRepository {
|
|||
await this.assetRepository.remove(asset);
|
||||
}
|
||||
|
||||
async save(asset: Partial<AssetEntity>): Promise<AssetEntity> {
|
||||
const { id } = await this.assetRepository.save(asset);
|
||||
return this.assetRepository.findOneOrFail({ where: { id } });
|
||||
}
|
||||
|
||||
/**
|
||||
* Update asset
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -1,15 +1,29 @@
|
|||
import { AuthUserDto, IJobRepository, JobName } from '@app/domain';
|
||||
import { AssetEntity, UserEntity } from '@app/infra/db/entities';
|
||||
import { StorageService } from '@app/storage';
|
||||
import {
|
||||
AuthUserDto,
|
||||
IJobRepository,
|
||||
IStorageRepository,
|
||||
ISystemConfigRepository,
|
||||
JobName,
|
||||
StorageTemplateCore,
|
||||
} from '@app/domain';
|
||||
import { AssetEntity, SystemConfig, UserEntity } from '@app/infra/db/entities';
|
||||
import { Logger } from '@nestjs/common';
|
||||
import { IAssetRepository } from './asset-repository';
|
||||
import { CreateAssetDto, UploadFile } from './dto/create-asset.dto';
|
||||
|
||||
export class AssetCore {
|
||||
private templateCore: StorageTemplateCore;
|
||||
private logger = new Logger(AssetCore.name);
|
||||
|
||||
constructor(
|
||||
private repository: IAssetRepository,
|
||||
private jobRepository: IJobRepository,
|
||||
private storageService: StorageService,
|
||||
) {}
|
||||
configRepository: ISystemConfigRepository,
|
||||
config: SystemConfig,
|
||||
private storageRepository: IStorageRepository,
|
||||
) {
|
||||
this.templateCore = new StorageTemplateCore(configRepository, config, storageRepository);
|
||||
}
|
||||
|
||||
async create(
|
||||
authUser: AuthUserDto,
|
||||
|
|
@ -42,10 +56,31 @@ export class AssetCore {
|
|||
sharedLinks: [],
|
||||
});
|
||||
|
||||
asset = await this.storageService.moveAsset(asset, file.originalName);
|
||||
asset = await this.moveAsset(asset, file.originalName);
|
||||
|
||||
await this.jobRepository.add({ name: JobName.ASSET_UPLOADED, data: { asset, fileName: file.originalName } });
|
||||
await this.jobRepository.queue({ name: JobName.ASSET_UPLOADED, data: { asset, fileName: file.originalName } });
|
||||
|
||||
return asset;
|
||||
}
|
||||
|
||||
async moveAsset(asset: AssetEntity, originalName: string) {
|
||||
const destination = await this.templateCore.getTemplatePath(asset, originalName);
|
||||
if (asset.originalPath !== destination) {
|
||||
const source = asset.originalPath;
|
||||
|
||||
try {
|
||||
await this.storageRepository.moveFile(asset.originalPath, destination);
|
||||
try {
|
||||
await this.repository.save({ id: asset.id, originalPath: destination });
|
||||
asset.originalPath = destination;
|
||||
} catch (error: any) {
|
||||
this.logger.warn('Unable to save new originalPath to database, undoing move', error?.stack);
|
||||
await this.storageRepository.moveFile(destination, source);
|
||||
}
|
||||
} catch (error: any) {
|
||||
this.logger.error(`Problem applying storage template`, error?.stack, { id: asset.id, source, destination });
|
||||
}
|
||||
}
|
||||
return asset;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,12 +3,10 @@ import { AssetService } from './asset.service';
|
|||
import { AssetController } from './asset.controller';
|
||||
import { TypeOrmModule } from '@nestjs/typeorm';
|
||||
import { AssetEntity } from '@app/infra';
|
||||
import { CommunicationModule } from '../communication/communication.module';
|
||||
import { AssetRepository, IAssetRepository } from './asset-repository';
|
||||
import { DownloadModule } from '../../modules/download/download.module';
|
||||
import { TagModule } from '../tag/tag.module';
|
||||
import { AlbumModule } from '../album/album.module';
|
||||
import { StorageModule } from '@app/storage';
|
||||
|
||||
const ASSET_REPOSITORY_PROVIDER = {
|
||||
provide: IAssetRepository,
|
||||
|
|
@ -17,11 +15,10 @@ const ASSET_REPOSITORY_PROVIDER = {
|
|||
|
||||
@Module({
|
||||
imports: [
|
||||
//
|
||||
TypeOrmModule.forFeature([AssetEntity]),
|
||||
CommunicationModule,
|
||||
DownloadModule,
|
||||
TagModule,
|
||||
StorageModule,
|
||||
AlbumModule,
|
||||
],
|
||||
controllers: [AssetController],
|
||||
|
|
|
|||
|
|
@ -8,19 +8,30 @@ import { TimeGroupEnum } from './dto/get-asset-count-by-time-bucket.dto';
|
|||
import { AssetCountByUserIdResponseDto } from './response-dto/asset-count-by-user-id-response.dto';
|
||||
import { DownloadService } from '../../modules/download/download.service';
|
||||
import { AlbumRepository, IAlbumRepository } from '../album/album-repository';
|
||||
import { StorageService } from '@app/storage';
|
||||
import { ICryptoRepository, IJobRepository, ISharedLinkRepository, IStorageRepository, JobName } from '@app/domain';
|
||||
import {
|
||||
ICryptoRepository,
|
||||
IJobRepository,
|
||||
ISharedLinkRepository,
|
||||
IStorageRepository,
|
||||
ISystemConfigRepository,
|
||||
JobName,
|
||||
} from '@app/domain';
|
||||
import {
|
||||
assetEntityStub,
|
||||
authStub,
|
||||
fileStub,
|
||||
newCryptoRepositoryMock,
|
||||
newJobRepositoryMock,
|
||||
newSharedLinkRepositoryMock,
|
||||
newStorageRepositoryMock,
|
||||
newSystemConfigRepositoryMock,
|
||||
sharedLinkResponseStub,
|
||||
sharedLinkStub,
|
||||
systemConfigStub,
|
||||
} from '@app/domain/../test';
|
||||
import { CreateAssetsShareLinkDto } from './dto/create-asset-shared-link.dto';
|
||||
import { BadRequestException, ForbiddenException } from '@nestjs/common';
|
||||
import { when } from 'jest-when';
|
||||
|
||||
const _getCreateAssetDto = (): CreateAssetDto => {
|
||||
const createAssetDto = new CreateAssetDto();
|
||||
|
|
@ -109,8 +120,8 @@ describe('AssetService', () => {
|
|||
let assetRepositoryMock: jest.Mocked<IAssetRepository>;
|
||||
let albumRepositoryMock: jest.Mocked<IAlbumRepository>;
|
||||
let downloadServiceMock: jest.Mocked<Partial<DownloadService>>;
|
||||
let storageServiceMock: jest.Mocked<StorageService>;
|
||||
let sharedLinkRepositoryMock: jest.Mocked<ISharedLinkRepository>;
|
||||
let configMock: jest.Mocked<ISystemConfigRepository>;
|
||||
let cryptoMock: jest.Mocked<ICryptoRepository>;
|
||||
let jobMock: jest.Mocked<IJobRepository>;
|
||||
let storageMock: jest.Mocked<IStorageRepository>;
|
||||
|
|
@ -120,6 +131,7 @@ describe('AssetService', () => {
|
|||
get: jest.fn(),
|
||||
create: jest.fn(),
|
||||
remove: jest.fn(),
|
||||
save: jest.fn(),
|
||||
|
||||
update: jest.fn(),
|
||||
getAll: jest.fn(),
|
||||
|
|
@ -150,13 +162,9 @@ describe('AssetService', () => {
|
|||
downloadArchive: jest.fn(),
|
||||
};
|
||||
|
||||
storageServiceMock = {
|
||||
moveAsset: jest.fn(),
|
||||
removeEmptyDirectories: jest.fn(),
|
||||
} as unknown as jest.Mocked<StorageService>;
|
||||
|
||||
sharedLinkRepositoryMock = newSharedLinkRepositoryMock();
|
||||
jobMock = newJobRepositoryMock();
|
||||
configMock = newSystemConfigRepositoryMock();
|
||||
cryptoMock = newCryptoRepositoryMock();
|
||||
storageMock = newStorageRepositoryMock();
|
||||
|
||||
|
|
@ -165,12 +173,20 @@ describe('AssetService', () => {
|
|||
albumRepositoryMock,
|
||||
a,
|
||||
downloadServiceMock as DownloadService,
|
||||
storageServiceMock,
|
||||
sharedLinkRepositoryMock,
|
||||
jobMock,
|
||||
configMock,
|
||||
systemConfigStub.defaults,
|
||||
cryptoMock,
|
||||
storageMock,
|
||||
);
|
||||
|
||||
when(assetRepositoryMock.get)
|
||||
.calledWith(assetEntityStub.livePhotoStillAsset.id)
|
||||
.mockResolvedValue(assetEntityStub.livePhotoStillAsset);
|
||||
when(assetRepositoryMock.get)
|
||||
.calledWith(assetEntityStub.livePhotoMotionAsset.id)
|
||||
.mockResolvedValue(assetEntityStub.livePhotoMotionAsset);
|
||||
});
|
||||
|
||||
describe('createAssetsSharedLink', () => {
|
||||
|
|
@ -255,10 +271,16 @@ describe('AssetService', () => {
|
|||
};
|
||||
const dto = _getCreateAssetDto();
|
||||
|
||||
assetRepositoryMock.create.mockImplementation(() => Promise.resolve(assetEntity));
|
||||
storageServiceMock.moveAsset.mockResolvedValue({ ...assetEntity, originalPath: 'fake_new_path/asset_123.jpeg' });
|
||||
assetRepositoryMock.create.mockResolvedValue(assetEntity);
|
||||
assetRepositoryMock.save.mockResolvedValue(assetEntity);
|
||||
|
||||
await expect(sut.uploadFile(authStub.user1, dto, file)).resolves.toEqual({ duplicate: false, id: 'id_1' });
|
||||
|
||||
expect(assetRepositoryMock.create).toHaveBeenCalled();
|
||||
expect(assetRepositoryMock.save).toHaveBeenCalledWith({
|
||||
id: 'id_1',
|
||||
originalPath: 'upload/user_id_1/2022/2022-06-19/asset_1.jpeg',
|
||||
});
|
||||
});
|
||||
|
||||
it('should handle a duplicate', async () => {
|
||||
|
|
@ -277,59 +299,43 @@ describe('AssetService', () => {
|
|||
|
||||
await expect(sut.uploadFile(authStub.user1, dto, file)).resolves.toEqual({ duplicate: true, id: 'id_1' });
|
||||
|
||||
expect(jobMock.add).toHaveBeenCalledWith({
|
||||
name: JobName.DELETE_FILE_ON_DISK,
|
||||
data: { assets: [{ originalPath: 'fake_path/asset_1.jpeg', resizePath: null }] },
|
||||
expect(jobMock.queue).toHaveBeenCalledWith({
|
||||
name: JobName.DELETE_FILES,
|
||||
data: { files: ['fake_path/asset_1.jpeg', undefined] },
|
||||
});
|
||||
expect(storageServiceMock.moveAsset).not.toHaveBeenCalled();
|
||||
expect(storageMock.moveFile).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should handle a live photo', async () => {
|
||||
const file = {
|
||||
originalPath: 'fake_path/asset_1.jpeg',
|
||||
mimeType: 'image/jpeg',
|
||||
checksum: Buffer.from('file hash', 'utf8'),
|
||||
originalName: 'asset_1.jpeg',
|
||||
};
|
||||
const asset = {
|
||||
id: 'live-photo-asset',
|
||||
originalPath: file.originalPath,
|
||||
ownerId: authStub.user1.id,
|
||||
type: AssetType.IMAGE,
|
||||
isVisible: true,
|
||||
} as AssetEntity;
|
||||
|
||||
const livePhotoFile = {
|
||||
originalPath: 'fake_path/asset_1.mp4',
|
||||
mimeType: 'image/jpeg',
|
||||
checksum: Buffer.from('live photo file hash', 'utf8'),
|
||||
originalName: 'asset_1.jpeg',
|
||||
};
|
||||
|
||||
const livePhotoAsset = {
|
||||
id: 'live-photo-motion',
|
||||
originalPath: livePhotoFile.originalPath,
|
||||
ownerId: authStub.user1.id,
|
||||
type: AssetType.VIDEO,
|
||||
isVisible: false,
|
||||
} as AssetEntity;
|
||||
|
||||
const dto = _getCreateAssetDto();
|
||||
const error = new QueryFailedError('', [], '');
|
||||
(error as any).constraint = 'UQ_userid_checksum';
|
||||
|
||||
assetRepositoryMock.create.mockResolvedValueOnce(livePhotoAsset);
|
||||
assetRepositoryMock.create.mockResolvedValueOnce(asset);
|
||||
storageServiceMock.moveAsset.mockImplementation((asset) => Promise.resolve(asset));
|
||||
assetRepositoryMock.create.mockResolvedValueOnce(assetEntityStub.livePhotoMotionAsset);
|
||||
assetRepositoryMock.save.mockResolvedValueOnce(assetEntityStub.livePhotoMotionAsset);
|
||||
assetRepositoryMock.create.mockResolvedValueOnce(assetEntityStub.livePhotoStillAsset);
|
||||
assetRepositoryMock.save.mockResolvedValueOnce(assetEntityStub.livePhotoStillAsset);
|
||||
|
||||
await expect(sut.uploadFile(authStub.user1, dto, file, livePhotoFile)).resolves.toEqual({
|
||||
await expect(
|
||||
sut.uploadFile(authStub.user1, dto, fileStub.livePhotoStill, fileStub.livePhotoMotion),
|
||||
).resolves.toEqual({
|
||||
duplicate: false,
|
||||
id: 'live-photo-asset',
|
||||
id: 'live-photo-still-asset',
|
||||
});
|
||||
|
||||
expect(jobMock.add.mock.calls).toEqual([
|
||||
[{ name: JobName.ASSET_UPLOADED, data: { asset: livePhotoAsset, fileName: file.originalName } }],
|
||||
[{ name: JobName.ASSET_UPLOADED, data: { asset, fileName: file.originalName } }],
|
||||
expect(jobMock.queue.mock.calls).toEqual([
|
||||
[
|
||||
{
|
||||
name: JobName.ASSET_UPLOADED,
|
||||
data: { asset: assetEntityStub.livePhotoMotionAsset, fileName: 'asset_1.mp4' },
|
||||
},
|
||||
],
|
||||
[
|
||||
{
|
||||
name: JobName.ASSET_UPLOADED,
|
||||
data: { asset: assetEntityStub.livePhotoStillAsset, fileName: 'asset_1.jpeg' },
|
||||
},
|
||||
],
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
|
@ -383,7 +389,7 @@ describe('AssetService', () => {
|
|||
{ id: 'asset1', status: 'FAILED' },
|
||||
]);
|
||||
|
||||
expect(jobMock.add).not.toHaveBeenCalled();
|
||||
expect(jobMock.queue).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should return failed status a delete fails', async () => {
|
||||
|
|
@ -394,35 +400,66 @@ describe('AssetService', () => {
|
|||
{ id: 'asset1', status: 'FAILED' },
|
||||
]);
|
||||
|
||||
expect(jobMock.add).not.toHaveBeenCalled();
|
||||
expect(jobMock.queue).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should delete a live photo', async () => {
|
||||
assetRepositoryMock.get.mockResolvedValueOnce({ id: 'asset1', livePhotoVideoId: 'live-photo' } as AssetEntity);
|
||||
assetRepositoryMock.get.mockResolvedValueOnce({ id: 'live-photo' } as AssetEntity);
|
||||
|
||||
await expect(sut.deleteAll(authStub.user1, { ids: ['asset1'] })).resolves.toEqual([
|
||||
{ id: 'asset1', status: 'SUCCESS' },
|
||||
{ id: 'live-photo', status: 'SUCCESS' },
|
||||
await expect(sut.deleteAll(authStub.user1, { ids: [assetEntityStub.livePhotoStillAsset.id] })).resolves.toEqual([
|
||||
{ id: assetEntityStub.livePhotoStillAsset.id, status: 'SUCCESS' },
|
||||
{ id: assetEntityStub.livePhotoMotionAsset.id, status: 'SUCCESS' },
|
||||
]);
|
||||
|
||||
expect(jobMock.add).toHaveBeenCalledWith({
|
||||
name: JobName.DELETE_FILE_ON_DISK,
|
||||
data: { assets: [{ id: 'asset1', livePhotoVideoId: 'live-photo' }, { id: 'live-photo' }] },
|
||||
expect(jobMock.queue).toHaveBeenCalledWith({
|
||||
name: JobName.DELETE_FILES,
|
||||
data: {
|
||||
files: ['fake_path/asset_1.jpeg', undefined, undefined, 'fake_path/asset_1.mp4', undefined, undefined],
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('should delete a batch of assets', async () => {
|
||||
assetRepositoryMock.get.mockImplementation((id) => Promise.resolve({ id } as AssetEntity));
|
||||
assetRepositoryMock.remove.mockImplementation(() => Promise.resolve());
|
||||
const asset1 = {
|
||||
id: 'asset1',
|
||||
originalPath: 'original-path-1',
|
||||
resizePath: 'resize-path-1',
|
||||
webpPath: 'web-path-1',
|
||||
};
|
||||
|
||||
const asset2 = {
|
||||
id: 'asset2',
|
||||
originalPath: 'original-path-2',
|
||||
resizePath: 'resize-path-2',
|
||||
webpPath: 'web-path-2',
|
||||
};
|
||||
|
||||
when(assetRepositoryMock.get)
|
||||
.calledWith(asset1.id)
|
||||
.mockResolvedValue(asset1 as AssetEntity);
|
||||
when(assetRepositoryMock.get)
|
||||
.calledWith(asset2.id)
|
||||
.mockResolvedValue(asset2 as AssetEntity);
|
||||
|
||||
await expect(sut.deleteAll(authStub.user1, { ids: ['asset1', 'asset2'] })).resolves.toEqual([
|
||||
{ id: 'asset1', status: 'SUCCESS' },
|
||||
{ id: 'asset2', status: 'SUCCESS' },
|
||||
]);
|
||||
|
||||
expect(jobMock.add.mock.calls).toEqual([
|
||||
[{ name: JobName.DELETE_FILE_ON_DISK, data: { assets: [{ id: 'asset1' }, { id: 'asset2' }] } }],
|
||||
expect(jobMock.queue.mock.calls).toEqual([
|
||||
[
|
||||
{
|
||||
name: JobName.DELETE_FILES,
|
||||
data: {
|
||||
files: [
|
||||
'original-path-1',
|
||||
'web-path-1',
|
||||
'resize-path-1',
|
||||
'original-path-2',
|
||||
'web-path-2',
|
||||
'resize-path-2',
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ import {
|
|||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
import { QueryFailedError, Repository } from 'typeorm';
|
||||
import { AuthUserDto } from '../../decorators/auth-user.decorator';
|
||||
import { AssetEntity, AssetType, SharedLinkType } from '@app/infra';
|
||||
import { AssetEntity, AssetType, SharedLinkType, SystemConfig } from '@app/infra';
|
||||
import { constants, createReadStream, ReadStream, stat } from 'fs';
|
||||
import { ServeFileDto } from './dto/serve-file.dto';
|
||||
import { Response as Res } from 'express';
|
||||
|
|
@ -25,7 +25,9 @@ import { CuratedObjectsResponseDto } from './response-dto/curated-objects-respon
|
|||
import {
|
||||
AssetResponseDto,
|
||||
ImmichReadStream,
|
||||
INITIAL_SYSTEM_CONFIG,
|
||||
IStorageRepository,
|
||||
ISystemConfigRepository,
|
||||
JobName,
|
||||
mapAsset,
|
||||
mapAssetWithoutExif,
|
||||
|
|
@ -52,7 +54,6 @@ import { ICryptoRepository, IJobRepository } from '@app/domain';
|
|||
import { DownloadService } from '../../modules/download/download.service';
|
||||
import { DownloadDto } from './dto/download-library.dto';
|
||||
import { IAlbumRepository } from '../album/album-repository';
|
||||
import { StorageService } from '@app/storage';
|
||||
import { ShareCore } from '@app/domain';
|
||||
import { ISharedLinkRepository } from '@app/domain';
|
||||
import { DownloadFilesDto } from './dto/download-files.dto';
|
||||
|
|
@ -61,6 +62,8 @@ import { mapSharedLink, SharedLinkResponseDto } from '@app/domain';
|
|||
import { AssetSearchDto } from './dto/asset-search.dto';
|
||||
import { AddAssetsDto } from '../album/dto/add-assets.dto';
|
||||
import { RemoveAssetsDto } from '../album/dto/remove-assets.dto';
|
||||
import path from 'path';
|
||||
import { getFileNameWithoutExtension } from '../../utils/file-name.util';
|
||||
|
||||
const fileInfo = promisify(stat);
|
||||
|
||||
|
|
@ -76,13 +79,14 @@ export class AssetService {
|
|||
@InjectRepository(AssetEntity)
|
||||
private assetRepository: Repository<AssetEntity>,
|
||||
private downloadService: DownloadService,
|
||||
storageService: StorageService,
|
||||
@Inject(ISharedLinkRepository) sharedLinkRepository: ISharedLinkRepository,
|
||||
@Inject(IJobRepository) private jobRepository: IJobRepository,
|
||||
@Inject(ISystemConfigRepository) configRepository: ISystemConfigRepository,
|
||||
@Inject(INITIAL_SYSTEM_CONFIG) config: SystemConfig,
|
||||
@Inject(ICryptoRepository) cryptoRepository: ICryptoRepository,
|
||||
@Inject(IStorageRepository) private storage: IStorageRepository,
|
||||
@Inject(IStorageRepository) private storageRepository: IStorageRepository,
|
||||
) {
|
||||
this.assetCore = new AssetCore(_assetRepository, jobRepository, storageService);
|
||||
this.assetCore = new AssetCore(_assetRepository, jobRepository, configRepository, config, storageRepository);
|
||||
this.shareCore = new ShareCore(sharedLinkRepository, cryptoRepository);
|
||||
}
|
||||
|
||||
|
|
@ -93,7 +97,10 @@ export class AssetService {
|
|||
livePhotoFile?: UploadFile,
|
||||
): Promise<AssetFileUploadResponseDto> {
|
||||
if (livePhotoFile) {
|
||||
livePhotoFile.originalName = file.originalName;
|
||||
livePhotoFile = {
|
||||
...livePhotoFile,
|
||||
originalName: getFileNameWithoutExtension(file.originalName) + path.extname(livePhotoFile.originalName),
|
||||
};
|
||||
}
|
||||
|
||||
let livePhotoAsset: AssetEntity | null = null;
|
||||
|
|
@ -109,16 +116,9 @@ export class AssetService {
|
|||
return { id: asset.id, duplicate: false };
|
||||
} catch (error: any) {
|
||||
// clean up files
|
||||
await this.jobRepository.add({
|
||||
name: JobName.DELETE_FILE_ON_DISK,
|
||||
data: {
|
||||
assets: [
|
||||
{
|
||||
originalPath: file.originalPath,
|
||||
resizePath: livePhotoFile?.originalPath || null,
|
||||
} as AssetEntity,
|
||||
],
|
||||
},
|
||||
await this.jobRepository.queue({
|
||||
name: JobName.DELETE_FILES,
|
||||
data: { files: [file.originalPath, livePhotoFile?.originalPath] },
|
||||
});
|
||||
|
||||
// handle duplicates with a success response
|
||||
|
|
@ -204,7 +204,7 @@ export class AssetService {
|
|||
try {
|
||||
const asset = await this._assetRepository.get(assetId);
|
||||
if (asset && asset.originalPath && asset.mimeType) {
|
||||
return this.storage.createReadStream(asset.originalPath, asset.mimeType);
|
||||
return this.storageRepository.createReadStream(asset.originalPath, asset.mimeType);
|
||||
}
|
||||
} catch (e) {
|
||||
Logger.error(`Error download asset ${e}`, 'downloadFile');
|
||||
|
|
@ -412,7 +412,7 @@ export class AssetService {
|
|||
}
|
||||
|
||||
public async deleteAll(authUser: AuthUserDto, dto: DeleteAssetDto): Promise<DeleteAssetResponseDto[]> {
|
||||
const deleteQueue: AssetEntity[] = [];
|
||||
const deleteQueue: Array<string | null> = [];
|
||||
const result: DeleteAssetResponseDto[] = [];
|
||||
|
||||
const ids = dto.ids.slice();
|
||||
|
|
@ -427,7 +427,7 @@ export class AssetService {
|
|||
await this._assetRepository.remove(asset);
|
||||
|
||||
result.push({ id, status: DeleteAssetStatusEnum.SUCCESS });
|
||||
deleteQueue.push(asset as any);
|
||||
deleteQueue.push(asset.originalPath, asset.webpPath, asset.resizePath);
|
||||
|
||||
// TODO refactor this to use cascades
|
||||
if (asset.livePhotoVideoId && !ids.includes(asset.livePhotoVideoId)) {
|
||||
|
|
@ -439,7 +439,7 @@ export class AssetService {
|
|||
}
|
||||
|
||||
if (deleteQueue.length > 0) {
|
||||
await this.jobRepository.add({ name: JobName.DELETE_FILE_ON_DISK, data: { assets: deleteQueue } });
|
||||
await this.jobRepository.queue({ name: JobName.DELETE_FILES, data: { files: deleteQueue } });
|
||||
}
|
||||
|
||||
return result;
|
||||
|
|
|
|||
|
|
@ -1,34 +0,0 @@
|
|||
import { Logger } from '@nestjs/common';
|
||||
import { OnGatewayConnection, OnGatewayDisconnect, WebSocketGateway, WebSocketServer } from '@nestjs/websockets';
|
||||
import { Server, Socket } from 'socket.io';
|
||||
import { AuthService } from '@app/domain';
|
||||
|
||||
@WebSocketGateway({ cors: true })
|
||||
export class CommunicationGateway implements OnGatewayConnection, OnGatewayDisconnect {
|
||||
private logger = new Logger(CommunicationGateway.name);
|
||||
|
||||
constructor(private authService: AuthService) {}
|
||||
|
||||
@WebSocketServer() server!: Server;
|
||||
|
||||
handleDisconnect(client: Socket) {
|
||||
client.leave(client.nsp.name);
|
||||
this.logger.log(`Client ${client.id} disconnected from Websocket`);
|
||||
}
|
||||
|
||||
async handleConnection(client: Socket) {
|
||||
try {
|
||||
this.logger.log(`New websocket connection: ${client.id}`);
|
||||
const user = await this.authService.validate(client.request.headers, {});
|
||||
if (user) {
|
||||
client.join(user.id);
|
||||
} else {
|
||||
client.emit('error', 'unauthorized');
|
||||
client.disconnect();
|
||||
}
|
||||
} catch (e) {
|
||||
client.emit('error', 'unauthorized');
|
||||
client.disconnect();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,8 +0,0 @@
|
|||
import { Module } from '@nestjs/common';
|
||||
import { CommunicationGateway } from './communication.gateway';
|
||||
|
||||
@Module({
|
||||
providers: [CommunicationGateway],
|
||||
exports: [CommunicationGateway],
|
||||
})
|
||||
export class CommunicationModule {}
|
||||
|
|
@ -48,14 +48,14 @@ export class JobService {
|
|||
? await this._assetRepository.getAllVideos()
|
||||
: await this._assetRepository.getAssetWithNoEncodedVideo();
|
||||
for (const asset of assets) {
|
||||
await this.jobRepository.add({ name: JobName.VIDEO_CONVERSION, data: { asset } });
|
||||
await this.jobRepository.queue({ name: JobName.VIDEO_CONVERSION, data: { asset } });
|
||||
}
|
||||
|
||||
return assets.length;
|
||||
}
|
||||
|
||||
case QueueName.CONFIG:
|
||||
await this.jobRepository.add({ name: JobName.TEMPLATE_MIGRATION });
|
||||
case QueueName.STORAGE_TEMPLATE_MIGRATION:
|
||||
await this.jobRepository.queue({ name: JobName.STORAGE_TEMPLATE_MIGRATION });
|
||||
return 1;
|
||||
|
||||
case QueueName.MACHINE_LEARNING: {
|
||||
|
|
@ -68,8 +68,8 @@ export class JobService {
|
|||
: await this._assetRepository.getAssetWithNoSmartInfo();
|
||||
|
||||
for (const asset of assets) {
|
||||
await this.jobRepository.add({ name: JobName.IMAGE_TAGGING, data: { asset } });
|
||||
await this.jobRepository.add({ name: JobName.OBJECT_DETECTION, data: { asset } });
|
||||
await this.jobRepository.queue({ name: JobName.IMAGE_TAGGING, data: { asset } });
|
||||
await this.jobRepository.queue({ name: JobName.OBJECT_DETECTION, data: { asset } });
|
||||
}
|
||||
return assets.length;
|
||||
}
|
||||
|
|
@ -81,7 +81,7 @@ export class JobService {
|
|||
|
||||
for (const asset of assets) {
|
||||
if (asset.type === AssetType.VIDEO) {
|
||||
await this.jobRepository.add({
|
||||
await this.jobRepository.queue({
|
||||
name: JobName.EXTRACT_VIDEO_METADATA,
|
||||
data: {
|
||||
asset,
|
||||
|
|
@ -89,7 +89,7 @@ export class JobService {
|
|||
},
|
||||
});
|
||||
} else {
|
||||
await this.jobRepository.add({
|
||||
await this.jobRepository.queue({
|
||||
name: JobName.EXIF_EXTRACTION,
|
||||
data: {
|
||||
asset,
|
||||
|
|
@ -107,7 +107,7 @@ export class JobService {
|
|||
: await this._assetRepository.getAssetWithNoThumbnail();
|
||||
|
||||
for (const asset of assets) {
|
||||
await this.jobRepository.add({ name: JobName.GENERATE_JPEG_THUMBNAIL, data: { asset } });
|
||||
await this.jobRepository.queue({ name: JobName.GENERATE_JPEG_THUMBNAIL, data: { asset } });
|
||||
}
|
||||
return assets.length;
|
||||
}
|
||||
|
|
@ -129,7 +129,7 @@ export class JobService {
|
|||
return QueueName.VIDEO_CONVERSION;
|
||||
|
||||
case JobId.STORAGE_TEMPLATE_MIGRATION:
|
||||
return QueueName.CONFIG;
|
||||
return QueueName.STORAGE_TEMPLATE_MIGRATION;
|
||||
|
||||
case JobId.MACHINE_LEARNING:
|
||||
return QueueName.MACHINE_LEARNING;
|
||||
|
|
|
|||
|
|
@ -3,7 +3,6 @@ import { Module } from '@nestjs/common';
|
|||
import { AssetModule } from './api-v1/asset/asset.module';
|
||||
import { ConfigModule } from '@nestjs/config';
|
||||
import { ServerInfoModule } from './api-v1/server-info/server-info.module';
|
||||
import { CommunicationModule } from './api-v1/communication/communication.module';
|
||||
import { AlbumModule } from './api-v1/album/album.module';
|
||||
import { AppController } from './app.controller';
|
||||
import { ScheduleModule } from '@nestjs/schedule';
|
||||
|
|
@ -36,8 +35,6 @@ import { AuthGuard } from './middlewares/auth.guard';
|
|||
|
||||
ServerInfoModule,
|
||||
|
||||
CommunicationModule,
|
||||
|
||||
AlbumModule,
|
||||
|
||||
ScheduleModule.forRoot(),
|
||||
|
|
|
|||
|
|
@ -1,26 +1,13 @@
|
|||
import { Inject, Injectable } from '@nestjs/common';
|
||||
import { UserService } from '@app/domain';
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { Cron, CronExpression } from '@nestjs/schedule';
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
import { IsNull, Not, Repository } from 'typeorm';
|
||||
import { UserEntity } from '@app/infra';
|
||||
import { userUtils } from '@app/common';
|
||||
import { IJobRepository, JobName } from '@app/domain';
|
||||
|
||||
@Injectable()
|
||||
export class ScheduleTasksService {
|
||||
constructor(
|
||||
@InjectRepository(UserEntity)
|
||||
private userRepository: Repository<UserEntity>,
|
||||
constructor(private userService: UserService) {}
|
||||
|
||||
@Inject(IJobRepository) private jobRepository: IJobRepository,
|
||||
) {}
|
||||
@Cron(CronExpression.EVERY_DAY_AT_11PM)
|
||||
async deleteUserAndRelatedAssets() {
|
||||
const usersToDelete = await this.userRepository.find({ withDeleted: true, where: { deletedAt: Not(IsNull()) } });
|
||||
for (const user of usersToDelete) {
|
||||
if (userUtils.isReadyForDeletion(user)) {
|
||||
await this.jobRepository.add({ name: JobName.USER_DELETION, data: { user } });
|
||||
}
|
||||
}
|
||||
async onUserDeleteCheck() {
|
||||
await this.userService.handleUserDeleteCheck();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,7 +9,6 @@
|
|||
},
|
||||
"moduleNameMapper": {
|
||||
"^@app/common": "<rootDir>../../../libs/common/src",
|
||||
"^@app/storage(|/.*)$": "<rootDir>../../../libs/storage/src/$1",
|
||||
"^@app/infra(|/.*)$": "<rootDir>../../../libs/infra/src/$1",
|
||||
"^@app/domain(|/.*)$": "<rootDir>../../../libs/domain/src/$1"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,56 +1,30 @@
|
|||
import { immichAppConfig } from '@app/common/config';
|
||||
import {
|
||||
AssetEntity,
|
||||
ExifEntity,
|
||||
SmartInfoEntity,
|
||||
UserEntity,
|
||||
APIKeyEntity,
|
||||
InfraModule,
|
||||
UserTokenEntity,
|
||||
AlbumEntity,
|
||||
} from '@app/infra';
|
||||
import { StorageModule } from '@app/storage';
|
||||
import { DomainModule } from '@app/domain';
|
||||
import { ExifEntity, InfraModule } from '@app/infra';
|
||||
import { Module } from '@nestjs/common';
|
||||
import { ConfigModule } from '@nestjs/config';
|
||||
import { TypeOrmModule } from '@nestjs/typeorm';
|
||||
import { CommunicationModule } from '../../immich/src/api-v1/communication/communication.module';
|
||||
import { AssetUploadedProcessor } from './processors/asset-uploaded.processor';
|
||||
import { MachineLearningProcessor } from './processors/machine-learning.processor';
|
||||
import {
|
||||
BackgroundTaskProcessor,
|
||||
MachineLearningProcessor,
|
||||
StorageTemplateMigrationProcessor,
|
||||
ThumbnailGeneratorProcessor,
|
||||
} from './processors';
|
||||
import { MetadataExtractionProcessor } from './processors/metadata-extraction.processor';
|
||||
import { StorageMigrationProcessor } from './processors/storage-migration.processor';
|
||||
import { ThumbnailGeneratorProcessor } from './processors/thumbnail.processor';
|
||||
import { UserDeletionProcessor } from './processors/user-deletion.processor';
|
||||
import { VideoTranscodeProcessor } from './processors/video-transcode.processor';
|
||||
import { BackgroundTaskProcessor } from './processors/background-task.processor';
|
||||
import { DomainModule } from '@app/domain';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
ConfigModule.forRoot(immichAppConfig),
|
||||
DomainModule.register({
|
||||
imports: [InfraModule],
|
||||
}),
|
||||
TypeOrmModule.forFeature([
|
||||
UserEntity,
|
||||
ExifEntity,
|
||||
AssetEntity,
|
||||
SmartInfoEntity,
|
||||
APIKeyEntity,
|
||||
UserTokenEntity,
|
||||
AlbumEntity,
|
||||
]),
|
||||
StorageModule,
|
||||
CommunicationModule,
|
||||
DomainModule.register({ imports: [InfraModule] }),
|
||||
TypeOrmModule.forFeature([ExifEntity]),
|
||||
],
|
||||
controllers: [],
|
||||
providers: [
|
||||
AssetUploadedProcessor,
|
||||
ThumbnailGeneratorProcessor,
|
||||
MetadataExtractionProcessor,
|
||||
VideoTranscodeProcessor,
|
||||
MachineLearningProcessor,
|
||||
UserDeletionProcessor,
|
||||
StorageMigrationProcessor,
|
||||
StorageTemplateMigrationProcessor,
|
||||
BackgroundTaskProcessor,
|
||||
],
|
||||
})
|
||||
|
|
|
|||
87
server/apps/microservices/src/processors.ts
Normal file
87
server/apps/microservices/src/processors.ts
Normal file
|
|
@ -0,0 +1,87 @@
|
|||
import {
|
||||
AssetService,
|
||||
IAssetJob,
|
||||
IAssetUploadedJob,
|
||||
IDeleteFilesJob,
|
||||
IUserDeletionJob,
|
||||
JobName,
|
||||
MediaService,
|
||||
QueueName,
|
||||
SmartInfoService,
|
||||
StorageService,
|
||||
StorageTemplateService,
|
||||
SystemConfigService,
|
||||
UserService,
|
||||
} from '@app/domain';
|
||||
import { Process, Processor } from '@nestjs/bull';
|
||||
import { Job } from 'bull';
|
||||
|
||||
@Processor(QueueName.BACKGROUND_TASK)
|
||||
export class BackgroundTaskProcessor {
|
||||
constructor(
|
||||
private assetService: AssetService,
|
||||
private storageService: StorageService,
|
||||
private systemConfigService: SystemConfigService,
|
||||
private userService: UserService,
|
||||
) {}
|
||||
|
||||
@Process(JobName.ASSET_UPLOADED)
|
||||
async onAssetUpload(job: Job<IAssetUploadedJob>) {
|
||||
await this.assetService.handleAssetUpload(job.data);
|
||||
}
|
||||
|
||||
@Process(JobName.DELETE_FILES)
|
||||
async onDeleteFile(job: Job<IDeleteFilesJob>) {
|
||||
await this.storageService.handleDeleteFiles(job.data);
|
||||
}
|
||||
|
||||
@Process(JobName.SYSTEM_CONFIG_CHANGE)
|
||||
async onSystemConfigChange() {
|
||||
await this.systemConfigService.refreshConfig();
|
||||
}
|
||||
|
||||
@Process(JobName.USER_DELETION)
|
||||
async onUserDelete(job: Job<IUserDeletionJob>) {
|
||||
await this.userService.handleUserDelete(job.data);
|
||||
}
|
||||
}
|
||||
|
||||
@Processor(QueueName.MACHINE_LEARNING)
|
||||
export class MachineLearningProcessor {
|
||||
constructor(private smartInfoService: SmartInfoService) {}
|
||||
|
||||
@Process({ name: JobName.IMAGE_TAGGING, concurrency: 2 })
|
||||
async onTagImage(job: Job<IAssetJob>) {
|
||||
await this.smartInfoService.handleTagImage(job.data);
|
||||
}
|
||||
|
||||
@Process({ name: JobName.OBJECT_DETECTION, concurrency: 2 })
|
||||
async onDetectObject(job: Job<IAssetJob>) {
|
||||
await this.smartInfoService.handleDetectObjects(job.data);
|
||||
}
|
||||
}
|
||||
|
||||
@Processor(QueueName.STORAGE_TEMPLATE_MIGRATION)
|
||||
export class StorageTemplateMigrationProcessor {
|
||||
constructor(private storageTemplateService: StorageTemplateService) {}
|
||||
|
||||
@Process({ name: JobName.STORAGE_TEMPLATE_MIGRATION })
|
||||
async onTemplateMigration() {
|
||||
await this.storageTemplateService.handleTemplateMigration();
|
||||
}
|
||||
}
|
||||
|
||||
@Processor(QueueName.THUMBNAIL_GENERATION)
|
||||
export class ThumbnailGeneratorProcessor {
|
||||
constructor(private mediaService: MediaService) {}
|
||||
|
||||
@Process({ name: JobName.GENERATE_JPEG_THUMBNAIL, concurrency: 3 })
|
||||
async handleGenerateJpegThumbnail(job: Job<IAssetJob>) {
|
||||
await this.mediaService.handleGenerateJpegThumbnail(job.data);
|
||||
}
|
||||
|
||||
@Process({ name: JobName.GENERATE_WEBP_THUMBNAIL, concurrency: 3 })
|
||||
async handleGenerateWepbThumbnail(job: Job<IAssetJob>) {
|
||||
await this.mediaService.handleGenerateWepbThumbnail(job.data);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,13 +0,0 @@
|
|||
import { IAssetUploadedJob, JobName, JobService, QueueName } from '@app/domain';
|
||||
import { Process, Processor } from '@nestjs/bull';
|
||||
import { Job } from 'bull';
|
||||
|
||||
@Processor(QueueName.ASSET_UPLOADED)
|
||||
export class AssetUploadedProcessor {
|
||||
constructor(private jobService: JobService) {}
|
||||
|
||||
@Process(JobName.ASSET_UPLOADED)
|
||||
async processUploadedVideo(job: Job<IAssetUploadedJob>) {
|
||||
await this.jobService.handleUploadedAsset(job);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,17 +0,0 @@
|
|||
import { assetUtils } from '@app/common/utils';
|
||||
import { Process, Processor } from '@nestjs/bull';
|
||||
import { Job } from 'bull';
|
||||
import { JobName, QueueName } from '@app/domain';
|
||||
import { AssetEntity } from '@app/infra/db/entities';
|
||||
|
||||
@Processor(QueueName.BACKGROUND_TASK)
|
||||
export class BackgroundTaskProcessor {
|
||||
@Process(JobName.DELETE_FILE_ON_DISK)
|
||||
async deleteFileOnDisk(job: Job<{ assets: AssetEntity[] }>) {
|
||||
const { assets } = job.data;
|
||||
|
||||
for (const asset of assets) {
|
||||
assetUtils.deleteFiles(asset);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,68 +0,0 @@
|
|||
import { AssetEntity } from '@app/infra';
|
||||
import { SmartInfoEntity } from '@app/infra';
|
||||
import { QueueName, JobName } from '@app/domain';
|
||||
import { IMachineLearningJob } from '@app/domain';
|
||||
import { Process, Processor } from '@nestjs/bull';
|
||||
import { Logger } from '@nestjs/common';
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
import axios from 'axios';
|
||||
import { Job } from 'bull';
|
||||
import { Repository } from 'typeorm';
|
||||
import { MACHINE_LEARNING_ENABLED, MACHINE_LEARNING_URL } from '@app/common';
|
||||
|
||||
@Processor(QueueName.MACHINE_LEARNING)
|
||||
export class MachineLearningProcessor {
|
||||
constructor(
|
||||
@InjectRepository(SmartInfoEntity)
|
||||
private smartInfoRepository: Repository<SmartInfoEntity>,
|
||||
) {}
|
||||
|
||||
@Process({ name: JobName.IMAGE_TAGGING, concurrency: 2 })
|
||||
async tagImage(job: Job<IMachineLearningJob>) {
|
||||
if (!MACHINE_LEARNING_ENABLED) {
|
||||
return;
|
||||
}
|
||||
|
||||
const { asset } = job.data;
|
||||
|
||||
const res = await axios.post(MACHINE_LEARNING_URL + '/image-classifier/tag-image', {
|
||||
thumbnailPath: asset.resizePath,
|
||||
});
|
||||
|
||||
if (res.status == 201 && res.data.length > 0) {
|
||||
const smartInfo = new SmartInfoEntity();
|
||||
smartInfo.assetId = asset.id;
|
||||
smartInfo.tags = [...res.data];
|
||||
await this.smartInfoRepository.upsert(smartInfo, {
|
||||
conflictPaths: ['assetId'],
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@Process({ name: JobName.OBJECT_DETECTION, concurrency: 2 })
|
||||
async detectObject(job: Job<IMachineLearningJob>) {
|
||||
if (!MACHINE_LEARNING_ENABLED) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const { asset }: { asset: AssetEntity } = job.data;
|
||||
|
||||
const res = await axios.post(MACHINE_LEARNING_URL + '/object-detection/detect-object', {
|
||||
thumbnailPath: asset.resizePath,
|
||||
});
|
||||
|
||||
if (res.status == 201 && res.data.length > 0) {
|
||||
const smartInfo = new SmartInfoEntity();
|
||||
smartInfo.assetId = asset.id;
|
||||
smartInfo.objects = [...res.data];
|
||||
|
||||
await this.smartInfoRepository.upsert(smartInfo, {
|
||||
conflictPaths: ['assetId'],
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
Logger.error(`Failed to trigger object detection pipe line ${String(error)}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,13 +1,7 @@
|
|||
import { AssetEntity, AssetType, ExifEntity } from '@app/infra';
|
||||
import {
|
||||
IExifExtractionProcessor,
|
||||
IReverseGeocodingProcessor,
|
||||
IVideoLengthExtractionProcessor,
|
||||
QueueName,
|
||||
JobName,
|
||||
} from '@app/domain';
|
||||
import { IReverseGeocodingJob, IAssetUploadedJob, QueueName, JobName, IAssetRepository } from '@app/domain';
|
||||
import { Process, Processor } from '@nestjs/bull';
|
||||
import { Logger } from '@nestjs/common';
|
||||
import { Inject, Logger } from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
import { Job } from 'bull';
|
||||
|
|
@ -19,7 +13,6 @@ import geocoder, { InitOptions } from 'local-reverse-geocoder';
|
|||
import { getName } from 'i18n-iso-countries';
|
||||
import fs from 'node:fs';
|
||||
import { ExifDateTime, exiftool, Tags } from 'exiftool-vendored';
|
||||
import { IsNull, Not } from 'typeorm';
|
||||
|
||||
interface ImmichTags extends Tags {
|
||||
ContentIdentifier?: string;
|
||||
|
|
@ -79,9 +72,7 @@ export class MetadataExtractionProcessor {
|
|||
private logger = new Logger(MetadataExtractionProcessor.name);
|
||||
private isGeocodeInitialized = false;
|
||||
constructor(
|
||||
@InjectRepository(AssetEntity)
|
||||
private assetRepository: Repository<AssetEntity>,
|
||||
|
||||
@Inject(IAssetRepository) private assetRepository: IAssetRepository,
|
||||
@InjectRepository(ExifEntity)
|
||||
private exifRepository: Repository<ExifEntity>,
|
||||
|
||||
|
|
@ -141,7 +132,7 @@ export class MetadataExtractionProcessor {
|
|||
}
|
||||
|
||||
@Process(JobName.EXIF_EXTRACTION)
|
||||
async extractExifInfo(job: Job<IExifExtractionProcessor>) {
|
||||
async extractExifInfo(job: Job<IAssetUploadedJob>) {
|
||||
try {
|
||||
const { asset, fileName }: { asset: AssetEntity; fileName: string } = job.data;
|
||||
const exifData = await exiftool.read<ImmichTags>(asset.originalPath).catch((e) => {
|
||||
|
|
@ -190,22 +181,14 @@ export class MetadataExtractionProcessor {
|
|||
});
|
||||
|
||||
if (newExif.livePhotoCID && !asset.livePhotoVideoId) {
|
||||
const motionAsset = await this.assetRepository.findOne({
|
||||
where: {
|
||||
id: Not(asset.id),
|
||||
type: AssetType.VIDEO,
|
||||
exifInfo: {
|
||||
livePhotoCID: newExif.livePhotoCID,
|
||||
},
|
||||
},
|
||||
relations: {
|
||||
exifInfo: true,
|
||||
},
|
||||
});
|
||||
|
||||
const motionAsset = await this.assetRepository.findLivePhotoMatch(
|
||||
newExif.livePhotoCID,
|
||||
AssetType.VIDEO,
|
||||
asset.id,
|
||||
);
|
||||
if (motionAsset) {
|
||||
await this.assetRepository.update(asset.id, { livePhotoVideoId: motionAsset.id });
|
||||
await this.assetRepository.update(motionAsset.id, { isVisible: false });
|
||||
await this.assetRepository.save({ id: asset.id, livePhotoVideoId: motionAsset.id });
|
||||
await this.assetRepository.save({ id: motionAsset.id, isVisible: false });
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -249,7 +232,7 @@ export class MetadataExtractionProcessor {
|
|||
}
|
||||
|
||||
@Process({ name: JobName.REVERSE_GEOCODING })
|
||||
async reverseGeocoding(job: Job<IReverseGeocodingProcessor>) {
|
||||
async reverseGeocoding(job: Job<IReverseGeocodingJob>) {
|
||||
if (this.isGeocodeInitialized) {
|
||||
const { latitude, longitude } = job.data;
|
||||
const { country, state, city } = await this.reverseGeocodeExif(latitude, longitude);
|
||||
|
|
@ -258,7 +241,7 @@ export class MetadataExtractionProcessor {
|
|||
}
|
||||
|
||||
@Process({ name: JobName.EXTRACT_VIDEO_METADATA, concurrency: 2 })
|
||||
async extractVideoMetadata(job: Job<IVideoLengthExtractionProcessor>) {
|
||||
async extractVideoMetadata(job: Job<IAssetUploadedJob>) {
|
||||
const { asset, fileName } = job.data;
|
||||
|
||||
if (!asset.isVisible) {
|
||||
|
|
@ -309,20 +292,14 @@ export class MetadataExtractionProcessor {
|
|||
newExif.livePhotoCID = exifData?.ContentIdentifier || null;
|
||||
|
||||
if (newExif.livePhotoCID) {
|
||||
const photoAsset = await this.assetRepository.findOne({
|
||||
where: {
|
||||
id: Not(asset.id),
|
||||
type: AssetType.IMAGE,
|
||||
livePhotoVideoId: IsNull(),
|
||||
exifInfo: {
|
||||
livePhotoCID: newExif.livePhotoCID,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const photoAsset = await this.assetRepository.findLivePhotoMatch(
|
||||
newExif.livePhotoCID,
|
||||
AssetType.IMAGE,
|
||||
asset.id,
|
||||
);
|
||||
if (photoAsset) {
|
||||
await this.assetRepository.update(photoAsset.id, { livePhotoVideoId: asset.id });
|
||||
await this.assetRepository.update(asset.id, { isVisible: false });
|
||||
await this.assetRepository.save({ id: photoAsset.id, livePhotoVideoId: asset.id });
|
||||
await this.assetRepository.save({ id: asset.id, isVisible: false });
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -378,7 +355,7 @@ export class MetadataExtractionProcessor {
|
|||
}
|
||||
|
||||
await this.exifRepository.upsert(newExif, { conflictPaths: ['assetId'] });
|
||||
await this.assetRepository.update({ id: asset.id }, { duration: durationString, fileCreatedAt });
|
||||
await this.assetRepository.save({ id: asset.id, duration: durationString, fileCreatedAt });
|
||||
} catch (err) {
|
||||
``;
|
||||
// do nothing
|
||||
|
|
|
|||
|
|
@ -1,61 +0,0 @@
|
|||
import { APP_UPLOAD_LOCATION } from '@app/common';
|
||||
import { AssetEntity } from '@app/infra';
|
||||
import { SystemConfigService } from '@app/domain';
|
||||
import { QueueName, JobName } from '@app/domain';
|
||||
import { StorageService } from '@app/storage';
|
||||
import { Process, Processor } from '@nestjs/bull';
|
||||
import { Logger } from '@nestjs/common';
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
import { Repository } from 'typeorm';
|
||||
|
||||
@Processor(QueueName.CONFIG)
|
||||
export class StorageMigrationProcessor {
|
||||
readonly logger: Logger = new Logger(StorageMigrationProcessor.name);
|
||||
|
||||
constructor(
|
||||
private storageService: StorageService,
|
||||
private systemConfigService: SystemConfigService,
|
||||
|
||||
@InjectRepository(AssetEntity)
|
||||
private assetRepository: Repository<AssetEntity>,
|
||||
) {}
|
||||
|
||||
/**
|
||||
* Migration process when a new user set a new storage template.
|
||||
* @param job
|
||||
*/
|
||||
@Process({ name: JobName.TEMPLATE_MIGRATION, concurrency: 100 })
|
||||
async templateMigration() {
|
||||
console.time('migrating-time');
|
||||
const assets = await this.assetRepository.find({
|
||||
relations: ['exifInfo'],
|
||||
});
|
||||
|
||||
const livePhotoMap: Record<string, AssetEntity> = {};
|
||||
|
||||
for (const asset of assets) {
|
||||
if (asset.livePhotoVideoId) {
|
||||
livePhotoMap[asset.livePhotoVideoId] = asset;
|
||||
}
|
||||
}
|
||||
|
||||
for (const asset of assets) {
|
||||
const livePhotoParentAsset = livePhotoMap[asset.id];
|
||||
const filename = asset.exifInfo?.imageName || livePhotoParentAsset?.exifInfo?.imageName || asset.id;
|
||||
await this.storageService.moveAsset(asset, filename);
|
||||
}
|
||||
|
||||
await this.storageService.removeEmptyDirectories(APP_UPLOAD_LOCATION);
|
||||
console.timeEnd('migrating-time');
|
||||
}
|
||||
|
||||
/**
|
||||
* Update config when a new storage template is set.
|
||||
* This is to ensure the synchronization between processes.
|
||||
* @param job
|
||||
*/
|
||||
@Process({ name: JobName.CONFIG_CHANGE, concurrency: 1 })
|
||||
async updateTemplate() {
|
||||
await this.systemConfigService.refreshConfig();
|
||||
}
|
||||
}
|
||||
|
|
@ -1,130 +0,0 @@
|
|||
import { APP_UPLOAD_LOCATION } from '@app/common';
|
||||
import { AssetEntity, AssetType } from '@app/infra';
|
||||
import { WebpGeneratorProcessor, JpegGeneratorProcessor, QueueName, JobName } from '@app/domain';
|
||||
import { InjectQueue, Process, Processor } from '@nestjs/bull';
|
||||
import { Logger } from '@nestjs/common';
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
import { mapAsset } from '@app/domain';
|
||||
import { Job, Queue } from 'bull';
|
||||
import ffmpeg from 'fluent-ffmpeg';
|
||||
import { existsSync, mkdirSync } from 'node:fs';
|
||||
import sanitize from 'sanitize-filename';
|
||||
import sharp from 'sharp';
|
||||
import { Repository } from 'typeorm/repository/Repository';
|
||||
import { join } from 'path';
|
||||
import { CommunicationGateway } from 'apps/immich/src/api-v1/communication/communication.gateway';
|
||||
import { IMachineLearningJob } from '@app/domain';
|
||||
import { exiftool } from 'exiftool-vendored';
|
||||
|
||||
@Processor(QueueName.THUMBNAIL_GENERATION)
|
||||
export class ThumbnailGeneratorProcessor {
|
||||
readonly logger: Logger = new Logger(ThumbnailGeneratorProcessor.name);
|
||||
|
||||
constructor(
|
||||
@InjectRepository(AssetEntity)
|
||||
private assetRepository: Repository<AssetEntity>,
|
||||
|
||||
@InjectQueue(QueueName.THUMBNAIL_GENERATION)
|
||||
private thumbnailGeneratorQueue: Queue,
|
||||
|
||||
private wsCommunicationGateway: CommunicationGateway,
|
||||
|
||||
@InjectQueue(QueueName.MACHINE_LEARNING)
|
||||
private machineLearningQueue: Queue<IMachineLearningJob>,
|
||||
) {}
|
||||
|
||||
@Process({ name: JobName.GENERATE_JPEG_THUMBNAIL, concurrency: 3 })
|
||||
async generateJPEGThumbnail(job: Job<JpegGeneratorProcessor>) {
|
||||
const basePath = APP_UPLOAD_LOCATION;
|
||||
|
||||
const { asset } = job.data;
|
||||
const sanitizedDeviceId = sanitize(String(asset.deviceId));
|
||||
|
||||
const resizePath = join(basePath, asset.ownerId, 'thumb', sanitizedDeviceId);
|
||||
|
||||
if (!existsSync(resizePath)) {
|
||||
mkdirSync(resizePath, { recursive: true });
|
||||
}
|
||||
|
||||
const jpegThumbnailPath = join(resizePath, `${asset.id}.jpeg`);
|
||||
|
||||
if (asset.type == AssetType.IMAGE) {
|
||||
try {
|
||||
await sharp(asset.originalPath, { failOnError: false })
|
||||
.resize(1440, 1440, { fit: 'outside', withoutEnlargement: true })
|
||||
.jpeg()
|
||||
.rotate()
|
||||
.toFile(jpegThumbnailPath)
|
||||
.catch(() => {
|
||||
this.logger.warn(
|
||||
'Failed to generate jpeg thumbnail for asset: ' +
|
||||
asset.id +
|
||||
' using sharp, failing over to exiftool-vendored',
|
||||
);
|
||||
return exiftool.extractThumbnail(asset.originalPath, jpegThumbnailPath);
|
||||
});
|
||||
await this.assetRepository.update({ id: asset.id }, { resizePath: jpegThumbnailPath });
|
||||
} catch (error: any) {
|
||||
this.logger.error('Failed to generate jpeg thumbnail for asset: ' + asset.id, error.stack);
|
||||
}
|
||||
|
||||
// Update resize path to send to generate webp queue
|
||||
asset.resizePath = jpegThumbnailPath;
|
||||
|
||||
await this.thumbnailGeneratorQueue.add(JobName.GENERATE_WEBP_THUMBNAIL, { asset });
|
||||
await this.machineLearningQueue.add(JobName.IMAGE_TAGGING, { asset });
|
||||
await this.machineLearningQueue.add(JobName.OBJECT_DETECTION, { asset });
|
||||
|
||||
this.wsCommunicationGateway.server.to(asset.ownerId).emit('on_upload_success', JSON.stringify(mapAsset(asset)));
|
||||
}
|
||||
|
||||
if (asset.type == AssetType.VIDEO) {
|
||||
await new Promise((resolve, reject) => {
|
||||
ffmpeg(asset.originalPath)
|
||||
.outputOptions(['-ss 00:00:00.000', '-frames:v 1'])
|
||||
.output(jpegThumbnailPath)
|
||||
.on('start', () => {
|
||||
Logger.log('Start Generating Video Thumbnail', 'generateJPEGThumbnail');
|
||||
})
|
||||
.on('error', (error) => {
|
||||
Logger.error(`Cannot Generate Video Thumbnail ${error}`, 'generateJPEGThumbnail');
|
||||
reject(error);
|
||||
})
|
||||
.on('end', async () => {
|
||||
Logger.log(`Generating Video Thumbnail Success ${asset.id}`, 'generateJPEGThumbnail');
|
||||
resolve(asset);
|
||||
})
|
||||
.run();
|
||||
});
|
||||
|
||||
await this.assetRepository.update({ id: asset.id }, { resizePath: jpegThumbnailPath });
|
||||
|
||||
// Update resize path to send to generate webp queue
|
||||
asset.resizePath = jpegThumbnailPath;
|
||||
|
||||
await this.thumbnailGeneratorQueue.add(JobName.GENERATE_WEBP_THUMBNAIL, { asset });
|
||||
await this.machineLearningQueue.add(JobName.IMAGE_TAGGING, { asset });
|
||||
await this.machineLearningQueue.add(JobName.OBJECT_DETECTION, { asset });
|
||||
|
||||
this.wsCommunicationGateway.server.to(asset.ownerId).emit('on_upload_success', JSON.stringify(mapAsset(asset)));
|
||||
}
|
||||
}
|
||||
|
||||
@Process({ name: JobName.GENERATE_WEBP_THUMBNAIL, concurrency: 3 })
|
||||
async generateWepbThumbnail(job: Job<WebpGeneratorProcessor>) {
|
||||
const { asset } = job.data;
|
||||
|
||||
if (!asset.resizePath) {
|
||||
return;
|
||||
}
|
||||
|
||||
const webpPath = asset.resizePath.replace('jpeg', 'webp');
|
||||
|
||||
try {
|
||||
await sharp(asset.resizePath, { failOnError: false }).resize(250).webp().rotate().toFile(webpPath);
|
||||
await this.assetRepository.update({ id: asset.id }, { webpPath: webpPath });
|
||||
} catch (error: any) {
|
||||
this.logger.error('Failed to generate webp thumbnail for asset: ' + asset.id, error.stack);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,72 +0,0 @@
|
|||
import { APP_UPLOAD_LOCATION, userUtils } from '@app/common';
|
||||
import { AlbumEntity, APIKeyEntity, AssetEntity, UserEntity, UserTokenEntity } from '@app/infra';
|
||||
import { QueueName, JobName } from '@app/domain';
|
||||
import { IUserDeletionJob } from '@app/domain';
|
||||
import { Process, Processor } from '@nestjs/bull';
|
||||
import { Logger } from '@nestjs/common';
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
import { Job } from 'bull';
|
||||
import { join } from 'path';
|
||||
import fs from 'fs';
|
||||
import { Repository } from 'typeorm';
|
||||
|
||||
@Processor(QueueName.USER_DELETION)
|
||||
export class UserDeletionProcessor {
|
||||
private logger = new Logger(UserDeletionProcessor.name);
|
||||
|
||||
constructor(
|
||||
@InjectRepository(UserEntity)
|
||||
private userRepository: Repository<UserEntity>,
|
||||
|
||||
@InjectRepository(AssetEntity)
|
||||
private assetRepository: Repository<AssetEntity>,
|
||||
|
||||
@InjectRepository(APIKeyEntity)
|
||||
private apiKeyRepository: Repository<APIKeyEntity>,
|
||||
|
||||
@InjectRepository(UserTokenEntity)
|
||||
private userTokenRepository: Repository<UserTokenEntity>,
|
||||
|
||||
@InjectRepository(AlbumEntity)
|
||||
private albumRepository: Repository<AlbumEntity>,
|
||||
) {}
|
||||
|
||||
@Process(JobName.USER_DELETION)
|
||||
async processUserDeletion(job: Job<IUserDeletionJob>) {
|
||||
const { user } = job.data;
|
||||
|
||||
// just for extra protection here
|
||||
if (!userUtils.isReadyForDeletion(user)) {
|
||||
this.logger.warn(`Skipped user that was not ready for deletion: id=${user.id}`);
|
||||
return;
|
||||
}
|
||||
|
||||
this.logger.log(`Deleting user: ${user.id}`);
|
||||
|
||||
try {
|
||||
const basePath = APP_UPLOAD_LOCATION;
|
||||
const userAssetDir = join(basePath, user.id);
|
||||
this.logger.warn(`Removing user from filesystem: ${userAssetDir}`);
|
||||
fs.rmSync(userAssetDir, { recursive: true, force: true });
|
||||
|
||||
this.logger.warn(`Removing user from database: ${user.id}`);
|
||||
const userTokens = await this.userTokenRepository.find({
|
||||
where: { user: { id: user.id } },
|
||||
relations: { user: true },
|
||||
withDeleted: true,
|
||||
});
|
||||
await this.userTokenRepository.remove(userTokens);
|
||||
|
||||
const albums = await this.albumRepository.find({ where: { ownerId: user.id } });
|
||||
await this.albumRepository.remove(albums);
|
||||
|
||||
await this.apiKeyRepository.delete({ userId: user.id });
|
||||
await this.assetRepository.delete({ ownerId: user.id });
|
||||
await this.userRepository.remove(user);
|
||||
} catch (error: any) {
|
||||
this.logger.error(`Failed to remove user`);
|
||||
this.logger.error(error, error?.stack);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,25 +1,22 @@
|
|||
import { APP_UPLOAD_LOCATION } from '@app/common/constants';
|
||||
import { AssetEntity } from '@app/infra';
|
||||
import { IVideoConversionProcessor, JobName, QueueName, SystemConfigService } from '@app/domain';
|
||||
import { IAssetJob, IAssetRepository, JobName, QueueName, SystemConfigService } from '@app/domain';
|
||||
import { Process, Processor } from '@nestjs/bull';
|
||||
import { Logger } from '@nestjs/common';
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
import { Inject, Logger } from '@nestjs/common';
|
||||
import { Job } from 'bull';
|
||||
import ffmpeg, { FfprobeData } from 'fluent-ffmpeg';
|
||||
import { existsSync, mkdirSync } from 'fs';
|
||||
import { Repository } from 'typeorm';
|
||||
|
||||
@Processor(QueueName.VIDEO_CONVERSION)
|
||||
export class VideoTranscodeProcessor {
|
||||
readonly logger = new Logger(VideoTranscodeProcessor.name);
|
||||
constructor(
|
||||
@InjectRepository(AssetEntity)
|
||||
private assetRepository: Repository<AssetEntity>,
|
||||
@Inject(IAssetRepository) private assetRepository: IAssetRepository,
|
||||
private systemConfigService: SystemConfigService,
|
||||
) {}
|
||||
|
||||
@Process({ name: JobName.VIDEO_CONVERSION, concurrency: 2 })
|
||||
async videoConversion(job: Job<IVideoConversionProcessor>) {
|
||||
async videoConversion(job: Job<IAssetJob>) {
|
||||
const { asset } = job.data;
|
||||
const basePath = APP_UPLOAD_LOCATION;
|
||||
const encodedVideoPath = `${basePath}/${asset.ownerId}/encoded-video`;
|
||||
|
|
@ -93,7 +90,7 @@ export class VideoTranscodeProcessor {
|
|||
})
|
||||
.on('end', async () => {
|
||||
this.logger.log(`Converting Success ${asset.id}`);
|
||||
await this.assetRepository.update({ id: asset.id }, { encodedVideoPath: savedEncodedPath });
|
||||
await this.assetRepository.save({ id: asset.id, encodedVideoPath: savedEncodedPath });
|
||||
resolve();
|
||||
})
|
||||
.run();
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue