feat: nightly tasks (#19879)

server/src/config.ts

@@ -121,6 +121,14 @@ export interface SystemConfig {
   newVersionCheck: {
     enabled: boolean;
   };
+  nightlyTasks: {
+    startTime: string;
+    databaseCleanup: boolean;
+    missingThumbnails: boolean;
+    clusterNewFaces: boolean;
+    generateMemories: boolean;
+    syncQuotaUsage: boolean;
+  };
   trash: {
     enabled: boolean;
     days: number;
@@ -298,6 +306,14 @@ export const defaults = Object.freeze<SystemConfig>({
   newVersionCheck: {
     enabled: true,
   },
+  nightlyTasks: {
+    startTime: '00:00',
+    databaseCleanup: true,
+    generateMemories: true,
+    syncQuotaUsage: true,
+    missingThumbnails: true,
+    clusterNewFaces: true,
+  },
   trash: {
     enabled: true,
     days: 30,

server/src/controllers/system-config.controller.spec.ts

@@ -0,0 +1,74 @@
+import _ from 'lodash';
+import { defaults } from 'src/config';
+import { SystemConfigController } from 'src/controllers/system-config.controller';
+import { StorageTemplateService } from 'src/services/storage-template.service';
+import { SystemConfigService } from 'src/services/system-config.service';
+import request from 'supertest';
+import { errorDto } from 'test/medium/responses';
+import { ControllerContext, controllerSetup, mockBaseService } from 'test/utils';
+
+describe(SystemConfigController.name, () => {
+  let ctx: ControllerContext;
+  const systemConfigService = mockBaseService(SystemConfigService);
+  const templateService = mockBaseService(StorageTemplateService);
+
+  beforeAll(async () => {
+    ctx = await controllerSetup(SystemConfigController, [
+      { provide: SystemConfigService, useValue: systemConfigService },
+      { provide: StorageTemplateService, useValue: templateService },
+    ]);
+    return () => ctx.close();
+  });
+
+  beforeEach(() => {
+    systemConfigService.resetAllMocks();
+    templateService.resetAllMocks();
+    ctx.reset();
+  });
+
+  describe('GET /system-config', () => {
+    it('should be an authenticated route', async () => {
+      await request(ctx.getHttpServer()).get('/system-config');
+      expect(ctx.authenticate).toHaveBeenCalled();
+    });
+  });
+
+  describe('GET /system-config/defaults', () => {
+    it('should be an authenticated route', async () => {
+      await request(ctx.getHttpServer()).get('/system-config/defaults');
+      expect(ctx.authenticate).toHaveBeenCalled();
+    });
+  });
+
+  describe('PUT /system-config', () => {
+    it('should be an authenticated route', async () => {
+      await request(ctx.getHttpServer()).put('/system-config');
+      expect(ctx.authenticate).toHaveBeenCalled();
+    });
+
+    describe('nightlyTasks', () => {
+      it('should validate nightly jobs start time', async () => {
+        const config = _.cloneDeep(defaults);
+        config.nightlyTasks.startTime = 'invalid';
+        const { status, body } = await request(ctx.getHttpServer()).put('/system-config').send(config);
+        expect(status).toBe(400);
+        expect(body).toEqual(errorDto.badRequest(['nightlyTasks.startTime must be in HH:mm format']));
+      });
+
+      it('should accept a valid time', async () => {
+        const config = _.cloneDeep(defaults);
+        config.nightlyTasks.startTime = '05:05';
+        const { status } = await request(ctx.getHttpServer()).put('/system-config').send(config);
+        expect(status).toBe(200);
+      });
+
+      it('should validate a boolean field', async () => {
+        const config = _.cloneDeep(defaults);
+        (config.nightlyTasks.databaseCleanup as any) = 'invalid';
+        const { status, body } = await request(ctx.getHttpServer()).put('/system-config').send(config);
+        expect(status).toBe(400);
+        expect(body).toEqual(errorDto.badRequest(['nightlyTasks.databaseCleanup must be a boolean value']));
+      });
+    });
+  });
+});

server/src/dtos/system-config.dto.ts

@@ -34,7 +34,7 @@ import {
   VideoContainer,
 } from 'src/enum';
 import { ConcurrentQueueName } from 'src/types';
-import { IsCronExpression, Optional, ValidateBoolean } from 'src/validation';
+import { IsCronExpression, IsDateStringFormat, Optional, ValidateBoolean } from 'src/validation';
 
 const isLibraryScanEnabled = (config: SystemConfigLibraryScanDto) => config.enabled;
 const isOAuthEnabled = (config: SystemConfigOAuthDto) => config.enabled;
@@ -329,6 +329,26 @@ class SystemConfigNewVersionCheckDto {
   enabled!: boolean;
 }
 
+class SystemConfigNightlyTasksDto {
+  @IsDateStringFormat('HH:mm', { message: 'startTime must be in HH:mm format' })
+  startTime!: string;
+
+  @ValidateBoolean()
+  databaseCleanup!: boolean;
+
+  @ValidateBoolean()
+  missingThumbnails!: boolean;
+
+  @ValidateBoolean()
+  clusterNewFaces!: boolean;
+
+  @ValidateBoolean()
+  generateMemories!: boolean;
+
+  @ValidateBoolean()
+  syncQuotaUsage!: boolean;
+}
+
 class SystemConfigOAuthDto {
   @ValidateBoolean()
   autoLaunch!: boolean;
@@ -638,6 +658,11 @@ export class SystemConfigDto implements SystemConfig {
   @IsObject()
   newVersionCheck!: SystemConfigNewVersionCheckDto;
 
+  @Type(() => SystemConfigNightlyTasksDto)
+  @ValidateNested()
+  @IsObject()
+  nightlyTasks!: SystemConfigNightlyTasksDto;
+
   @Type(() => SystemConfigOAuthDto)
   @ValidateNested()
   @IsObject()

server/src/enum.ts

@@ -567,6 +567,7 @@ export enum DatabaseLock {
   VersionHistory = 500,
   CLIPDimSize = 512,
   Library = 1337,
+  NightlyJobs = 600,
   GetSystemConfig = 69,
   BackupDatabase = 42,
   MemoryCreation = 777,
@@ -684,3 +685,8 @@ export enum AssetVisibility {
   HIDDEN = 'hidden',
   LOCKED = 'locked',
 }
+
+export enum CronJob {
+  LibraryScan = 'LibraryScan',
+  NightlyJobs = 'NightlyJobs',
+}

server/src/services/api.service.ts

@@ -1,5 +1,5 @@
 import { Injectable } from '@nestjs/common';
-import { Cron, CronExpression, Interval } from '@nestjs/schedule';
+import { Interval } from '@nestjs/schedule';
 import { NextFunction, Request, Response } from 'express';
 import { readFileSync } from 'node:fs';
 import sanitizeHtml from 'sanitize-html';
@@ -54,11 +54,6 @@ export class ApiService {
     await this.versionService.handleQueueVersionCheck();
   }
 
-  @Cron(CronExpression.EVERY_DAY_AT_MIDNIGHT)
-  async onNightlyJob() {
-    await this.jobService.handleNightlyJobs();
-  }
-
   ssr(excludePaths: string[]) {
     const { resourcePaths } = this.configRepository.getEnv();
 

server/src/services/job.service.spec.ts

@@ -41,12 +41,12 @@ describe(JobService.name, () => {
         { name: JobName.USER_DELETE_CHECK },
         { name: JobName.PERSON_CLEANUP },
         { name: JobName.MEMORIES_CLEANUP },
-        { name: JobName.MEMORIES_CREATE },
-        { name: JobName.QUEUE_GENERATE_THUMBNAILS, data: { force: false } },
-        { name: JobName.CLEAN_OLD_AUDIT_LOGS },
-        { name: JobName.USER_SYNC_USAGE },
-        { name: JobName.QUEUE_FACIAL_RECOGNITION, data: { force: false, nightly: true } },
         { name: JobName.CLEAN_OLD_SESSION_TOKENS },
+        { name: JobName.CLEAN_OLD_AUDIT_LOGS },
+        { name: JobName.MEMORIES_CREATE },
+        { name: JobName.USER_SYNC_USAGE },
+        { name: JobName.QUEUE_GENERATE_THUMBNAILS, data: { force: false } },
+        { name: JobName.QUEUE_FACIAL_RECOGNITION, data: { force: false, nightly: true } },
       ]);
     });
   });

server/src/services/job.service.ts

@@ -1,6 +1,7 @@
 import { BadRequestException, Injectable } from '@nestjs/common';
 import { ClassConstructor } from 'class-transformer';
 import { snakeCase } from 'lodash';
+import { SystemConfig } from 'src/config';
 import { OnEvent } from 'src/decorators';
 import { mapAsset } from 'src/dtos/asset-response.dto';
 import { AllJobStatusResponseDto, JobCommandDto, JobCreateDto, JobStatusDto } from 'src/dtos/job.dto';
@@ -8,6 +9,8 @@ import {
   AssetType,
   AssetVisibility,
   BootstrapEventPriority,
+  CronJob,
+  DatabaseLock,
   ImmichWorker,
   JobCommand,
   JobName,
@@ -20,6 +23,7 @@ import { ArgOf, ArgsOf } from 'src/repositories/event.repository';
 import { BaseService } from 'src/services/base.service';
 import { ConcurrentQueueName, JobItem } from 'src/types';
 import { hexOrBufferToBase64 } from 'src/utils/bytes';
+import { handlePromiseError } from 'src/utils/misc';
 
 const asJobItem = (dto: JobCreateDto): JobItem => {
   switch (dto.name) {
@@ -53,12 +57,59 @@ const asJobItem = (dto: JobCreateDto): JobItem => {
   }
 };
 
+const asNightlyTasksCron = (config: SystemConfig) => {
+  const [hours, minutes] = config.nightlyTasks.startTime.split(':').map(Number);
+  return `${minutes} ${hours} * * *`;
+};
+
 @Injectable()
 export class JobService extends BaseService {
   private services: ClassConstructor<unknown>[] = [];
+  private nightlyJobsLock = false;
 
-  @OnEvent({ name: 'config.init', workers: [ImmichWorker.MICROSERVICES] })
-  onConfigInit({ newConfig: config }: ArgOf<'config.init'>) {
+  @OnEvent({ name: 'config.init' })
+  async onConfigInit({ newConfig: config }: ArgOf<'config.init'>) {
+    if (this.worker === ImmichWorker.MICROSERVICES) {
+      this.updateQueueConcurrency(config);
+      return;
+    }
+
+    this.nightlyJobsLock = await this.databaseRepository.tryLock(DatabaseLock.NightlyJobs);
+    if (this.nightlyJobsLock) {
+      const cronExpression = asNightlyTasksCron(config);
+      this.logger.debug(`Scheduling nightly jobs for ${cronExpression}`);
+      this.cronRepository.create({
+        name: CronJob.NightlyJobs,
+        expression: cronExpression,
+        start: true,
+        onTick: () => handlePromiseError(this.handleNightlyJobs(), this.logger),
+      });
+    }
+  }
+
+  @OnEvent({ name: 'config.update', server: true })
+  onConfigUpdate({ newConfig: config }: ArgOf<'config.update'>) {
+    if (this.worker === ImmichWorker.MICROSERVICES) {
+      this.updateQueueConcurrency(config);
+      return;
+    }
+
+    if (this.nightlyJobsLock) {
+      const cronExpression = asNightlyTasksCron(config);
+      this.logger.debug(`Scheduling nightly jobs for ${cronExpression}`);
+      this.cronRepository.update({ name: CronJob.NightlyJobs, expression: cronExpression, start: true });
+    }
+  }
+
+  @OnEvent({ name: 'app.bootstrap', priority: BootstrapEventPriority.JobService })
+  onBootstrap() {
+    this.jobRepository.setup(this.services);
+    if (this.worker === ImmichWorker.MICROSERVICES) {
+      this.jobRepository.startWorkers();
+    }
+  }
+
+  private updateQueueConcurrency(config: SystemConfig) {
     this.logger.debug(`Updating queue concurrency settings`);
     for (const queueName of Object.values(QueueName)) {
       let concurrency = 1;
@@ -70,19 +121,6 @@ export class JobService extends BaseService {
     }
   }
 
-  @OnEvent({ name: 'config.update', server: true, workers: [ImmichWorker.MICROSERVICES] })
-  onConfigUpdate({ newConfig: config }: ArgOf<'config.update'>) {
-    this.onConfigInit({ newConfig: config });
-  }
-
-  @OnEvent({ name: 'app.bootstrap', priority: BootstrapEventPriority.JobService })
-  onBootstrap() {
-    this.jobRepository.setup(this.services);
-    if (this.worker === ImmichWorker.MICROSERVICES) {
-      this.jobRepository.startWorkers();
-    }
-  }
-
   setServices(services: ClassConstructor<unknown>[]) {
     this.services = services;
   }
@@ -233,18 +271,37 @@ export class JobService extends BaseService {
   }
 
   async handleNightlyJobs() {
-    await this.jobRepository.queueAll([
-      { name: JobName.ASSET_DELETION_CHECK },
-      { name: JobName.USER_DELETE_CHECK },
-      { name: JobName.PERSON_CLEANUP },
-      { name: JobName.MEMORIES_CLEANUP },
-      { name: JobName.MEMORIES_CREATE },
-      { name: JobName.QUEUE_GENERATE_THUMBNAILS, data: { force: false } },
-      { name: JobName.CLEAN_OLD_AUDIT_LOGS },
-      { name: JobName.USER_SYNC_USAGE },
-      { name: JobName.QUEUE_FACIAL_RECOGNITION, data: { force: false, nightly: true } },
-      { name: JobName.CLEAN_OLD_SESSION_TOKENS },
-    ]);
+    const config = await this.getConfig({ withCache: false });
+    const jobs: JobItem[] = [];
+
+    if (config.nightlyTasks.databaseCleanup) {
+      jobs.push(
+        { name: JobName.ASSET_DELETION_CHECK },
+        { name: JobName.USER_DELETE_CHECK },
+        { name: JobName.PERSON_CLEANUP },
+        { name: JobName.MEMORIES_CLEANUP },
+        { name: JobName.CLEAN_OLD_SESSION_TOKENS },
+        { name: JobName.CLEAN_OLD_AUDIT_LOGS },
+      );
+    }
+
+    if (config.nightlyTasks.generateMemories) {
+      jobs.push({ name: JobName.MEMORIES_CREATE });
+    }
+
+    if (config.nightlyTasks.syncQuotaUsage) {
+      jobs.push({ name: JobName.USER_SYNC_USAGE });
+    }
+
+    if (config.nightlyTasks.missingThumbnails) {
+      jobs.push({ name: JobName.QUEUE_GENERATE_THUMBNAILS, data: { force: false } });
+    }
+
+    if (config.nightlyTasks.clusterNewFaces) {
+      jobs.push({ name: JobName.QUEUE_FACIAL_RECOGNITION, data: { force: false, nightly: true } });
+    }
+
+    await this.jobRepository.queueAll(jobs);
   }
 
   /**

server/src/services/library.service.spec.ts

@@ -3,7 +3,7 @@ import { Stats } from 'node:fs';
 import { defaults, SystemConfig } from 'src/config';
 import { JOBS_LIBRARY_PAGINATION_SIZE } from 'src/constants';
 import { mapLibrary } from 'src/dtos/library.dto';
-import { AssetType, ImmichWorker, JobName, JobStatus } from 'src/enum';
+import { AssetType, CronJob, ImmichWorker, JobName, JobStatus } from 'src/enum';
 import { LibraryService } from 'src/services/library.service';
 import { ILibraryBulkIdsJob, ILibraryFileJob } from 'src/types';
 import { assetStub } from 'test/fixtures/asset.stub';
@@ -56,7 +56,11 @@ describe(LibraryService.name, () => {
         } as SystemConfig,
       });
 
-      expect(mocks.cron.update).toHaveBeenCalledWith({ name: 'libraryScan', expression: '0 1 * * *', start: true });
+      expect(mocks.cron.update).toHaveBeenCalledWith({
+        name: CronJob.LibraryScan,
+        expression: '0 1 * * *',
+        start: true,
+      });
     });
 
     it('should initialize watcher for all external libraries', async () => {
@@ -128,7 +132,7 @@ describe(LibraryService.name, () => {
       });
 
       expect(mocks.cron.update).toHaveBeenCalledWith({
-        name: 'libraryScan',
+        name: CronJob.LibraryScan,
         expression: systemConfigStub.libraryScan.library.scan.cronExpression,
         start: systemConfigStub.libraryScan.library.scan.enabled,
       });
@@ -149,7 +153,7 @@ describe(LibraryService.name, () => {
       });
 
       expect(mocks.cron.update).toHaveBeenCalledWith({
-        name: 'libraryScan',
+        name: CronJob.LibraryScan,
         expression: systemConfigStub.libraryScan.library.scan.cronExpression,
         start: systemConfigStub.libraryScan.library.scan.enabled,
       });

server/src/services/library.service.ts

@@ -17,7 +17,7 @@ import {
   ValidateLibraryImportPathResponseDto,
   ValidateLibraryResponseDto,
 } from 'src/dtos/library.dto';
-import { AssetStatus, AssetType, DatabaseLock, ImmichWorker, JobName, JobStatus, QueueName } from 'src/enum';
+import { AssetStatus, AssetType, CronJob, DatabaseLock, ImmichWorker, JobName, JobStatus, QueueName } from 'src/enum';
 import { ArgOf } from 'src/repositories/event.repository';
 import { AssetSyncResult } from 'src/repositories/library.repository';
 import { AssetTable } from 'src/schema/tables/asset.table';
@@ -45,7 +45,7 @@ export class LibraryService extends BaseService {
 
     if (this.lock) {
       this.cronRepository.create({
-        name: 'libraryScan',
+        name: CronJob.LibraryScan,
         expression: scan.cronExpression,
         onTick: () =>
           handlePromiseError(this.jobRepository.queue({ name: JobName.LIBRARY_QUEUE_SCAN_ALL }), this.logger),
@@ -65,7 +65,7 @@ export class LibraryService extends BaseService {
     }
 
     this.cronRepository.update({
-      name: 'libraryScan',
+      name: CronJob.LibraryScan,
       expression: library.scan.cronExpression,
       start: library.scan.enabled,
     });

server/src/services/system-config.service.spec.ts

@@ -103,6 +103,14 @@ const updatedConfig = Object.freeze<SystemConfig>({
     lightStyle: 'https://tiles.immich.cloud/v1/style/light.json',
     darkStyle: 'https://tiles.immich.cloud/v1/style/dark.json',
   },
+  nightlyTasks: {
+    startTime: '00:00',
+    databaseCleanup: true,
+    clusterNewFaces: true,
+    missingThumbnails: true,
+    generateMemories: true,
+    syncQuotaUsage: true,
+  },
   reverseGeocoding: {
     enabled: true,
   },