helpy4/immich
1
0
Fork 0
mirror of https://github.com/immich-app/immich synced 2025-11-14 17:36:12 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

refactor(server): jobs (#2023)

Browse source 
* refactor: job to domain

* chore: regenerate open api

* chore: tests

* fix: missing breaks

* fix: get asset with missing exif data

---------

Co-authored-by: Alex Tran <alex.tran1502@gmail.com>
This commit is contained in:
Jason Rasmussen 2023-03-20 11:55:28 -04:00 • committed by GitHub
parent db6b14361d
commit 386eef046d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
68 changed files with 1355 additions and 907 deletions
Download patch file Download diff file Expand all files Collapse all files

43
server/apps/immich/src/api-v1/asset/asset-repository.ts
View file

@ -38,10 +38,6 @@ export interface IAssetRepository {
getAssetCountByUserId(userId: string): Promise<AssetCountByUserIdResponseDto>;
getAssetByTimeBucket(userId: string, getAssetByTimeBucketDto: GetAssetByTimeBucketDto): Promise<AssetEntity[]>;
getAssetByChecksum(userId: string, checksum: Buffer): Promise<AssetEntity>;
getAssetWithNoThumbnail(): Promise<AssetEntity[]>;
getAssetWithNoEncodedVideo(): Promise<AssetEntity[]>;
getAssetWithNoEXIF(): Promise<AssetEntity[]>;
getAssetWithNoSmartInfo(): Promise<AssetEntity[]>;
getExistingAssets(
userId: string,
checkDuplicateAssetDto: CheckExistingAssetsDto,
@ -76,45 +72,6 @@ export class AssetRepository implements IAssetRepository {
});
}
async getAssetWithNoSmartInfo(): Promise<AssetEntity[]> {
return await this.assetRepository
.createQueryBuilder('asset')
.leftJoinAndSelect('asset.smartInfo', 'si')
.where('asset.resizePath IS NOT NULL')
.andWhere('si.assetId IS NULL')
.andWhere('asset.isVisible = true')
.getMany();
}
async getAssetWithNoThumbnail(): Promise<AssetEntity[]> {
return await this.assetRepository.find({
where: [
{ resizePath: IsNull(), isVisible: true },
{ resizePath: '', isVisible: true },
{ webpPath: IsNull(), isVisible: true },
{ webpPath: '', isVisible: true },
],
});
}
async getAssetWithNoEncodedVideo(): Promise<AssetEntity[]> {
return await this.assetRepository.find({
where: [
{ type: AssetType.VIDEO, encodedVideoPath: IsNull() },
{ type: AssetType.VIDEO, encodedVideoPath: '' },
],
});
}
async getAssetWithNoEXIF(): Promise<AssetEntity[]> {
return await this.assetRepository
.createQueryBuilder('asset')
.leftJoinAndSelect('asset.exifInfo', 'ei')
.where('ei."assetId" IS NULL')
.andWhere('asset.isVisible = true')
.getMany();
}
async getAssetCountByUserId(ownerId: string): Promise<AssetCountByUserIdResponseDto> {
// Get asset count by AssetType
const items = await this.assetRepository

4
server/apps/immich/src/api-v1/asset/asset.service.spec.ts
View file

@ -146,10 +146,6 @@ describe('AssetService', () => {
getAssetByTimeBucket: jest.fn(),
getAssetByChecksum: jest.fn(),
getAssetCountByUserId: jest.fn(),
getAssetWithNoEXIF: jest.fn(),
getAssetWithNoThumbnail: jest.fn(),
getAssetWithNoSmartInfo: jest.fn(),
getAssetWithNoEncodedVideo: jest.fn(),
getExistingAssets: jest.fn(),
countByIdAndUser: jest.fn(),
};

2
server/apps/immich/src/api-v1/asset/asset.service.ts
View file

@ -63,7 +63,7 @@ import { AssetSearchDto } from './dto/asset-search.dto';
import { AddAssetsDto } from '../album/dto/add-assets.dto';
import { RemoveAssetsDto } from '../album/dto/remove-assets.dto';
import path from 'path';
import { getFileNameWithoutExtension } from '../../utils/file-name.util';
import { getFileNameWithoutExtension } from '@app/domain';
const fileInfo = promisify(stat);

23
server/apps/immich/src/api-v1/job/dto/get-job.dto.ts
View file

@ -1,23 +0,0 @@
import { ApiProperty } from '@nestjs/swagger';
import { IsEnum, IsNotEmpty } from 'class-validator';
export enum JobId {
THUMBNAIL_GENERATION = 'thumbnail-generation',
METADATA_EXTRACTION = 'metadata-extraction',
VIDEO_CONVERSION = 'video-conversion',
MACHINE_LEARNING = 'machine-learning',
STORAGE_TEMPLATE_MIGRATION = 'storage-template-migration',
}
export class GetJobDto {
@IsNotEmpty()
@IsEnum(JobId, {
message: `params must be one of ${Object.values(JobId).join()}`,
})
@ApiProperty({
type: String,
enum: JobId,
enumName: 'JobId',
})
jobId!: JobId;
}

16
server/apps/immich/src/api-v1/job/dto/job-command.dto.ts
View file

@ -1,16 +0,0 @@
import { ApiProperty } from '@nestjs/swagger';
import { IsBoolean, IsIn, IsNotEmpty, IsOptional } from 'class-validator';
export class JobCommandDto {
@IsNotEmpty()
@IsIn(['start', 'stop'])
@ApiProperty({
enum: ['start', 'stop'],
enumName: 'JobCommand',
})
command!: string;
@IsOptional()
@IsBoolean()
includeAllAssets!: boolean;
}

33
server/apps/immich/src/api-v1/job/job.controller.ts
View file

@ -1,33 +0,0 @@
import { Body, Controller, Get, Param, Put, ValidationPipe } from '@nestjs/common';
import { ApiTags } from '@nestjs/swagger';
import { Authenticated } from '../../decorators/authenticated.decorator';
import { AllJobStatusResponseDto } from './response-dto/all-job-status-response.dto';
import { GetJobDto } from './dto/get-job.dto';
import { JobService } from './job.service';
import { JobCommandDto } from './dto/job-command.dto';
@Authenticated({ admin: true })
@ApiTags('Job')
@Controller('jobs')
export class JobController {
constructor(private readonly jobService: JobService) {}
@Get()
getAllJobsStatus(): Promise<AllJobStatusResponseDto> {
return this.jobService.getAllJobsStatus();
}
@Put('/:jobId')
async sendJobCommand(
@Param(ValidationPipe) params: GetJobDto,
@Body(ValidationPipe) dto: JobCommandDto,
): Promise<number> {
if (dto.command === 'start') {
return await this.jobService.start(params.jobId, dto.includeAllAssets);
}
if (dto.command === 'stop') {
return await this.jobService.stop(params.jobId);
}
return 0;
}
}

11
server/apps/immich/src/api-v1/job/job.module.ts
View file

@ -1,11 +0,0 @@
import { Module } from '@nestjs/common';
import { JobService } from './job.service';
import { JobController } from './job.controller';
import { AssetModule } from '../asset/asset.module';
@Module({
imports: [AssetModule],
controllers: [JobController],
providers: [JobService],
})
export class JobModule {}

142
server/apps/immich/src/api-v1/job/job.service.ts
View file

@ -1,142 +0,0 @@
import { JobName, IJobRepository, QueueName } from '@app/domain';
import { BadRequestException, Inject, Injectable } from '@nestjs/common';
import { AllJobStatusResponseDto } from './response-dto/all-job-status-response.dto';
import { IAssetRepository } from '../asset/asset-repository';
import { AssetType } from '@app/infra';
import { JobId } from './dto/get-job.dto';
import { MACHINE_LEARNING_ENABLED } from '@app/common';
import { getFileNameWithoutExtension } from '../../utils/file-name.util';
const jobIds = Object.values(JobId) as JobId[];
@Injectable()
export class JobService {
constructor(
@Inject(IAssetRepository) private _assetRepository: IAssetRepository,
@Inject(IJobRepository) private jobRepository: IJobRepository,
) {
for (const jobId of jobIds) {
this.jobRepository.empty(this.asQueueName(jobId));
}
}
start(jobId: JobId, includeAllAssets: boolean): Promise<number> {
return this.run(this.asQueueName(jobId), includeAllAssets);
}
async stop(jobId: JobId): Promise<number> {
await this.jobRepository.empty(this.asQueueName(jobId));
return 0;
}
async getAllJobsStatus(): Promise<AllJobStatusResponseDto> {
const response = new AllJobStatusResponseDto();
for (const jobId of jobIds) {
response[jobId] = await this.jobRepository.getJobCounts(this.asQueueName(jobId));
}
return response;
}
private async run(name: QueueName, includeAllAssets: boolean): Promise<number> {
const isActive = await this.jobRepository.isActive(name);
if (isActive) {
throw new BadRequestException(`Job is already running`);
}
switch (name) {
case QueueName.VIDEO_CONVERSION: {
const assets = includeAllAssets
? await this._assetRepository.getAllVideos()
: await this._assetRepository.getAssetWithNoEncodedVideo();
for (const asset of assets) {
await this.jobRepository.queue({ name: JobName.VIDEO_CONVERSION, data: { asset } });
}
return assets.length;
}
case QueueName.STORAGE_TEMPLATE_MIGRATION:
await this.jobRepository.queue({ name: JobName.STORAGE_TEMPLATE_MIGRATION });
return 1;
case QueueName.MACHINE_LEARNING: {
if (!MACHINE_LEARNING_ENABLED) {
throw new BadRequestException('Machine learning is not enabled.');
}
const assets = includeAllAssets
? await this._assetRepository.getAll()
: await this._assetRepository.getAssetWithNoSmartInfo();
for (const asset of assets) {
await this.jobRepository.queue({ name: JobName.IMAGE_TAGGING, data: { asset } });
await this.jobRepository.queue({ name: JobName.OBJECT_DETECTION, data: { asset } });
await this.jobRepository.queue({ name: JobName.ENCODE_CLIP, data: { asset } });
}
return assets.length;
}
case QueueName.METADATA_EXTRACTION: {
const assets = includeAllAssets
? await this._assetRepository.getAll()
: await this._assetRepository.getAssetWithNoEXIF();
for (const asset of assets) {
if (asset.type === AssetType.VIDEO) {
await this.jobRepository.queue({
name: JobName.EXTRACT_VIDEO_METADATA,
data: {
asset,
fileName: asset.exifInfo?.imageName ?? getFileNameWithoutExtension(asset.originalPath),
},
});
} else {
await this.jobRepository.queue({
name: JobName.EXIF_EXTRACTION,
data: {
asset,
fileName: asset.exifInfo?.imageName ?? getFileNameWithoutExtension(asset.originalPath),
},
});
}
}
return assets.length;
}
case QueueName.THUMBNAIL_GENERATION: {
const assets = includeAllAssets
? await this._assetRepository.getAll()
: await this._assetRepository.getAssetWithNoThumbnail();
for (const asset of assets) {
await this.jobRepository.queue({ name: JobName.GENERATE_JPEG_THUMBNAIL, data: { asset } });
}
return assets.length;
}
default:
return 0;
}
}
private asQueueName(jobId: JobId) {
switch (jobId) {
case JobId.THUMBNAIL_GENERATION:
return QueueName.THUMBNAIL_GENERATION;
case JobId.METADATA_EXTRACTION:
return QueueName.METADATA_EXTRACTION;
case JobId.VIDEO_CONVERSION:
return QueueName.VIDEO_CONVERSION;
case JobId.STORAGE_TEMPLATE_MIGRATION:
return QueueName.STORAGE_TEMPLATE_MIGRATION;
case JobId.MACHINE_LEARNING:
return QueueName.MACHINE_LEARNING;
default:
throw new BadRequestException(`Invalid job id: ${jobId}`);
}
}
}

32
server/apps/immich/src/api-v1/job/response-dto/all-job-status-response.dto.ts
View file

@ -1,32 +0,0 @@
import { ApiProperty } from '@nestjs/swagger';
import { JobId } from '../dto/get-job.dto';
export class JobCounts {
@ApiProperty({ type: 'integer' })
active!: number;
@ApiProperty({ type: 'integer' })
completed!: number;
@ApiProperty({ type: 'integer' })
failed!: number;
@ApiProperty({ type: 'integer' })
delayed!: number;
@ApiProperty({ type: 'integer' })
waiting!: number;
}
export class AllJobStatusResponseDto {
@ApiProperty({ type: JobCounts })
[JobId.THUMBNAIL_GENERATION]!: JobCounts;
@ApiProperty({ type: JobCounts })
[JobId.METADATA_EXTRACTION]!: JobCounts;
@ApiProperty({ type: JobCounts })
[JobId.VIDEO_CONVERSION]!: JobCounts;
@ApiProperty({ type: JobCounts })
[JobId.MACHINE_LEARNING]!: JobCounts;
@ApiProperty({ type: JobCounts })
[JobId.STORAGE_TEMPLATE_MIGRATION]!: JobCounts;
}

5
server/apps/immich/src/app.module.ts
View file

@ -7,7 +7,6 @@ import { AlbumModule } from './api-v1/album/album.module';
import { AppController } from './app.controller';
import { ScheduleModule } from '@nestjs/schedule';
import { ScheduleTasksModule } from './modules/schedule-tasks/schedule-tasks.module';
import { JobModule } from './api-v1/job/job.module';
import { TagModule } from './api-v1/tag/tag.module';
import { DomainModule, SearchService } from '@app/domain';
import { InfraModule } from '@app/infra';
@ -15,6 +14,7 @@ import {
APIKeyController,
AuthController,
DeviceInfoController,
JobController,
OAuthController,
SearchController,
ShareController,
@ -42,8 +42,6 @@ import { AuthGuard } from './middlewares/auth.guard';
ScheduleTasksModule,
JobModule,
TagModule,
],
controllers: [
@ -51,6 +49,7 @@ import { AuthGuard } from './middlewares/auth.guard';
APIKeyController,
AuthController,
DeviceInfoController,
JobController,
OAuthController,
SearchController,
ShareController,

1
server/apps/immich/src/controllers/index.ts
View file

@ -1,6 +1,7 @@
export * from './api-key.controller';
export * from './auth.controller';
export * from './device-info.controller';
export * from './job.controller';
export * from './oauth.controller';
export * from './search.controller';
export * from './share.controller';

21
server/apps/immich/src/controllers/job.controller.ts Normal file
View file

@ -0,0 +1,21 @@
import { AllJobStatusResponseDto, JobCommandDto, JobIdDto, JobService } from '@app/domain';
import { Body, Controller, Get, Param, Put, ValidationPipe } from '@nestjs/common';
import { ApiTags } from '@nestjs/swagger';
import { Authenticated } from '../decorators/authenticated.decorator';
@Authenticated({ admin: true })
@ApiTags('Job')
@Controller('jobs')
export class JobController {
constructor(private readonly jobService: JobService) {}
@Get()
getAllJobsStatus(): Promise<AllJobStatusResponseDto> {
return this.jobService.getAllJobsStatus();
}
@Put('/:jobId')
sendJobCommand(@Param(ValidationPipe) { jobId }: JobIdDto, @Body(ValidationPipe) dto: JobCommandDto): Promise<void> {
return this.jobService.handleCommand(jobId, dto);
}
}

5
server/apps/immich/src/utils/file-name.util.ts
View file

@ -1,5 +0,0 @@
import { basename, extname } from 'node:path';
export function getFileNameWithoutExtension(path: string): string {
return basename(path, extname(path));
}

6
server/apps/microservices/src/microservices.module.ts
View file

@ -6,7 +6,8 @@ import { ConfigModule } from '@nestjs/config';
import { TypeOrmModule } from '@nestjs/typeorm';
import {
BackgroundTaskProcessor,
MachineLearningProcessor,
ClipEncodingProcessor,
ObjectTaggingProcessor,
SearchIndexProcessor,
StorageTemplateMigrationProcessor,
ThumbnailGeneratorProcessor,
@ -24,7 +25,8 @@ import { VideoTranscodeProcessor } from './processors/video-transcode.processor'
ThumbnailGeneratorProcessor,
MetadataExtractionProcessor,
VideoTranscodeProcessor,
MachineLearningProcessor,
ObjectTaggingProcessor,
ClipEncodingProcessor,
StorageTemplateMigrationProcessor,
BackgroundTaskProcessor,
SearchIndexProcessor,

35
server/apps/microservices/src/processors.ts
View file

@ -2,6 +2,7 @@ import {
AssetService,
IAssetJob,
IAssetUploadedJob,
IBaseJob,
IBulkEntityJob,
IDeleteFilesJob,
IUserDeletionJob,
@ -48,20 +49,35 @@ export class BackgroundTaskProcessor {
}
}
@Processor(QueueName.MACHINE_LEARNING)
export class MachineLearningProcessor {
@Processor(QueueName.OBJECT_TAGGING)
export class ObjectTaggingProcessor {
constructor(private smartInfoService: SmartInfoService) {}
@Process({ name: JobName.IMAGE_TAGGING, concurrency: 1 })
async onTagImage(job: Job<IAssetJob>) {
await this.smartInfoService.handleTagImage(job.data);
@Process({ name: JobName.QUEUE_OBJECT_TAGGING, concurrency: 1 })
async onQueueObjectTagging(job: Job<IBaseJob>) {
await this.smartInfoService.handleQueueObjectTagging(job.data);
}
@Process({ name: JobName.OBJECT_DETECTION, concurrency: 1 })
async onDetectObject(job: Job<IAssetJob>) {
@Process({ name: JobName.DETECT_OBJECTS, concurrency: 1 })
async onDetectObjects(job: Job<IAssetJob>) {
await this.smartInfoService.handleDetectObjects(job.data);
}
@Process({ name: JobName.CLASSIFY_IMAGE, concurrency: 1 })
async onClassifyImage(job: Job<IAssetJob>) {
await this.smartInfoService.handleClassifyImage(job.data);
}
}
@Processor(QueueName.CLIP_ENCODING)
export class ClipEncodingProcessor {
constructor(private smartInfoService: SmartInfoService) {}
@Process({ name: JobName.QUEUE_ENCODE_CLIP, concurrency: 1 })
async onQueueClipEncoding(job: Job<IBaseJob>) {
await this.smartInfoService.handleQueueEncodeClip(job.data);
}
@Process({ name: JobName.ENCODE_CLIP, concurrency: 1 })
async onEncodeClip(job: Job<IAssetJob>) {
await this.smartInfoService.handleEncodeClip(job.data);
@ -117,6 +133,11 @@ export class StorageTemplateMigrationProcessor {
export class ThumbnailGeneratorProcessor {
constructor(private mediaService: MediaService) {}
@Process({ name: JobName.QUEUE_GENERATE_THUMBNAILS, concurrency: 1 })
async handleQueueGenerateThumbnails(job: Job<IBaseJob>) {
await this.mediaService.handleQueueGenerateThumbnails(job.data);
}
@Process({ name: JobName.GENERATE_JPEG_THUMBNAIL, concurrency: 3 })
async handleGenerateJpegThumbnail(job: Job<IAssetJob>) {
await this.mediaService.handleGenerateJpegThumbnail(job.data);

25
server/apps/microservices/src/processors/metadata-extraction.processor.ts
View file

@ -1,11 +1,14 @@
import {
AssetCore,
getFileNameWithoutExtension,
IAssetRepository,
IAssetUploadedJob,
IBaseJob,
IJobRepository,
IReverseGeocodingJob,
JobName,
QueueName,
WithoutProperty,
} from '@app/domain';
import { AssetEntity, AssetType, ExifEntity } from '@app/infra';
import { Process, Processor } from '@nestjs/bull';
@ -85,8 +88,8 @@ export class MetadataExtractionProcessor {
private assetCore: AssetCore;
constructor(
@Inject(IAssetRepository) assetRepository: IAssetRepository,
@Inject(IJobRepository) jobRepository: IJobRepository,
@Inject(IAssetRepository) private assetRepository: IAssetRepository,
@Inject(IJobRepository) private jobRepository: IJobRepository,
@InjectRepository(ExifEntity)
private exifRepository: Repository<ExifEntity>,
@ -148,6 +151,24 @@ export class MetadataExtractionProcessor {
return { country, state, city };
}
@Process(JobName.QUEUE_METADATA_EXTRACTION)
async handleQueueMetadataExtraction(job: Job<IBaseJob>) {
try {
const { force } = job.data;
const assets = force
? await this.assetRepository.getAll()
: await this.assetRepository.getWithout(WithoutProperty.EXIF);
for (const asset of assets) {
const fileName = asset.exifInfo?.imageName ?? getFileNameWithoutExtension(asset.originalPath);
const name = asset.type === AssetType.VIDEO ? JobName.EXTRACT_VIDEO_METADATA : JobName.EXIF_EXTRACTION;
await this.jobRepository.queue({ name, data: { asset, fileName } });
}
} catch (error: any) {
this.logger.error(`Unable to queue metadata extraction`, error?.stack);
}
}
@Process(JobName.EXIF_EXTRACTION)
async extractExifInfo(job: Job<IAssetUploadedJob>) {
try {

31
server/apps/microservices/src/processors/video-transcode.processor.ts
View file

@ -1,6 +1,15 @@
import { APP_UPLOAD_LOCATION } from '@app/common/constants';
import { AssetEntity } from '@app/infra';
import { IAssetJob, IAssetRepository, JobName, QueueName, SystemConfigService } from '@app/domain';
import { AssetEntity, AssetType } from '@app/infra';
import {
IAssetJob,
IAssetRepository,
IBaseJob,
IJobRepository,
JobName,
QueueName,
SystemConfigService,
WithoutProperty,
} from '@app/domain';
import { Process, Processor } from '@nestjs/bull';
import { Inject, Logger } from '@nestjs/common';
import { Job } from 'bull';
@ -12,11 +21,27 @@ export class VideoTranscodeProcessor {
readonly logger = new Logger(VideoTranscodeProcessor.name);
constructor(
@Inject(IAssetRepository) private assetRepository: IAssetRepository,
@Inject(IJobRepository) private jobRepository: IJobRepository,
private systemConfigService: SystemConfigService,
) {}
@Process({ name: JobName.QUEUE_VIDEO_CONVERSION, concurrency: 1 })
async handleQueueVideoConversion(job: Job<IBaseJob>): Promise<void> {
try {
const { force } = job.data;
const assets = force
? await this.assetRepository.getAll({ type: AssetType.VIDEO })
: await this.assetRepository.getWithout(WithoutProperty.ENCODED_VIDEO);
for (const asset of assets) {
await this.jobRepository.queue({ name: JobName.VIDEO_CONVERSION, data: { asset } });
}
} catch (error: any) {
this.logger.error('Failed to queue video conversions', error.stack);
}
}
@Process({ name: JobName.VIDEO_CONVERSION, concurrency: 2 })
async videoConversion(job: Job<IAssetJob>) {
async handleVideoConversion(job: Job<IAssetJob>) {
const { asset } = job.data;
const basePath = APP_UPLOAD_LOCATION;
const encodedVideoPath = `${basePath}/${asset.ownerId}/encoded-video`;