From ae2abb3cfe41704c66c78266b49786048e33ce7e Mon Sep 17 00:00:00 2001 From: mertalev <101130780+mertalev@users.noreply.github.com> Date: Thu, 9 Oct 2025 16:38:21 -0400 Subject: [PATCH] configurable cleanup --- mobile/openapi/README.md | 1 + mobile/openapi/lib/api.dart | 1 + mobile/openapi/lib/api_client.dart | 2 + .../system_config_nightly_tasks_dto.dart | 10 +- ...tem_config_remove_partial_uploads_dto.dart | 108 ++++++++++++++++++ open-api/immich-openapi-specs.json | 20 ++++ open-api/typescript-sdk/src/fetch-client.ts | 5 + server/src/config.ts | 8 ++ server/src/dtos/system-config.dto.ts | 16 +++ server/src/services/asset-upload.service.ts | 4 +- server/src/services/job.service.spec.ts | 1 + server/src/services/job.service.ts | 4 + .../services/system-config.service.spec.ts | 4 + server/src/types.ts | 2 +- 14 files changed, 182 insertions(+), 4 deletions(-) create mode 100644 mobile/openapi/lib/model/system_config_remove_partial_uploads_dto.dart diff --git a/mobile/openapi/README.md b/mobile/openapi/README.md index e9e5782a00..cc4e695503 100644 --- a/mobile/openapi/README.md +++ b/mobile/openapi/README.md @@ -545,6 +545,7 @@ Class | Method | HTTP request | Description - [SystemConfigNotificationsDto](doc//SystemConfigNotificationsDto.md) - [SystemConfigOAuthDto](doc//SystemConfigOAuthDto.md) - [SystemConfigPasswordLoginDto](doc//SystemConfigPasswordLoginDto.md) + - [SystemConfigRemovePartialUploadsDto](doc//SystemConfigRemovePartialUploadsDto.md) - [SystemConfigReverseGeocodingDto](doc//SystemConfigReverseGeocodingDto.md) - [SystemConfigServerDto](doc//SystemConfigServerDto.md) - [SystemConfigSmtpDto](doc//SystemConfigSmtpDto.md) diff --git a/mobile/openapi/lib/api.dart b/mobile/openapi/lib/api.dart index c488f05ff9..4ca8de5f5e 100644 --- a/mobile/openapi/lib/api.dart +++ b/mobile/openapi/lib/api.dart @@ -312,6 +312,7 @@ part 'model/system_config_nightly_tasks_dto.dart'; part 'model/system_config_notifications_dto.dart'; part 'model/system_config_o_auth_dto.dart'; part 'model/system_config_password_login_dto.dart'; +part 'model/system_config_remove_partial_uploads_dto.dart'; part 'model/system_config_reverse_geocoding_dto.dart'; part 'model/system_config_server_dto.dart'; part 'model/system_config_smtp_dto.dart'; diff --git a/mobile/openapi/lib/api_client.dart b/mobile/openapi/lib/api_client.dart index 5ad8bc664c..88cc79d5d7 100644 --- a/mobile/openapi/lib/api_client.dart +++ b/mobile/openapi/lib/api_client.dart @@ -676,6 +676,8 @@ class ApiClient { return SystemConfigOAuthDto.fromJson(value); case 'SystemConfigPasswordLoginDto': return SystemConfigPasswordLoginDto.fromJson(value); + case 'SystemConfigRemovePartialUploadsDto': + return SystemConfigRemovePartialUploadsDto.fromJson(value); case 'SystemConfigReverseGeocodingDto': return SystemConfigReverseGeocodingDto.fromJson(value); case 'SystemConfigServerDto': diff --git a/mobile/openapi/lib/model/system_config_nightly_tasks_dto.dart b/mobile/openapi/lib/model/system_config_nightly_tasks_dto.dart index ab7b4b37c2..c925909994 100644 --- a/mobile/openapi/lib/model/system_config_nightly_tasks_dto.dart +++ b/mobile/openapi/lib/model/system_config_nightly_tasks_dto.dart @@ -17,6 +17,7 @@ class SystemConfigNightlyTasksDto { required this.databaseCleanup, required this.generateMemories, required this.missingThumbnails, + required this.removeStaleUploads, required this.startTime, required this.syncQuotaUsage, }); @@ -29,6 +30,8 @@ class SystemConfigNightlyTasksDto { bool missingThumbnails; + SystemConfigRemovePartialUploadsDto removeStaleUploads; + String startTime; bool syncQuotaUsage; @@ -39,6 +42,7 @@ class SystemConfigNightlyTasksDto { other.databaseCleanup == databaseCleanup && other.generateMemories == generateMemories && other.missingThumbnails == missingThumbnails && + other.removeStaleUploads == removeStaleUploads && other.startTime == startTime && other.syncQuotaUsage == syncQuotaUsage; @@ -49,11 +53,12 @@ class SystemConfigNightlyTasksDto { (databaseCleanup.hashCode) + (generateMemories.hashCode) + (missingThumbnails.hashCode) + + (removeStaleUploads.hashCode) + (startTime.hashCode) + (syncQuotaUsage.hashCode); @override - String toString() => 'SystemConfigNightlyTasksDto[clusterNewFaces=$clusterNewFaces, databaseCleanup=$databaseCleanup, generateMemories=$generateMemories, missingThumbnails=$missingThumbnails, startTime=$startTime, syncQuotaUsage=$syncQuotaUsage]'; + String toString() => 'SystemConfigNightlyTasksDto[clusterNewFaces=$clusterNewFaces, databaseCleanup=$databaseCleanup, generateMemories=$generateMemories, missingThumbnails=$missingThumbnails, removeStaleUploads=$removeStaleUploads, startTime=$startTime, syncQuotaUsage=$syncQuotaUsage]'; Map toJson() { final json = {}; @@ -61,6 +66,7 @@ class SystemConfigNightlyTasksDto { json[r'databaseCleanup'] = this.databaseCleanup; json[r'generateMemories'] = this.generateMemories; json[r'missingThumbnails'] = this.missingThumbnails; + json[r'removeStaleUploads'] = this.removeStaleUploads; json[r'startTime'] = this.startTime; json[r'syncQuotaUsage'] = this.syncQuotaUsage; return json; @@ -79,6 +85,7 @@ class SystemConfigNightlyTasksDto { databaseCleanup: mapValueOfType(json, r'databaseCleanup')!, generateMemories: mapValueOfType(json, r'generateMemories')!, missingThumbnails: mapValueOfType(json, r'missingThumbnails')!, + removeStaleUploads: SystemConfigRemovePartialUploadsDto.fromJson(json[r'removeStaleUploads'])!, startTime: mapValueOfType(json, r'startTime')!, syncQuotaUsage: mapValueOfType(json, r'syncQuotaUsage')!, ); @@ -132,6 +139,7 @@ class SystemConfigNightlyTasksDto { 'databaseCleanup', 'generateMemories', 'missingThumbnails', + 'removeStaleUploads', 'startTime', 'syncQuotaUsage', }; diff --git a/mobile/openapi/lib/model/system_config_remove_partial_uploads_dto.dart b/mobile/openapi/lib/model/system_config_remove_partial_uploads_dto.dart new file mode 100644 index 0000000000..8a14dbe6ca --- /dev/null +++ b/mobile/openapi/lib/model/system_config_remove_partial_uploads_dto.dart @@ -0,0 +1,108 @@ +// +// AUTO-GENERATED FILE, DO NOT MODIFY! +// +// @dart=2.18 + +// ignore_for_file: unused_element, unused_import +// ignore_for_file: always_put_required_named_parameters_first +// ignore_for_file: constant_identifier_names +// ignore_for_file: lines_longer_than_80_chars + +part of openapi.api; + +class SystemConfigRemovePartialUploadsDto { + /// Returns a new [SystemConfigRemovePartialUploadsDto] instance. + SystemConfigRemovePartialUploadsDto({ + required this.enabled, + required this.hoursAgo, + }); + + bool enabled; + + /// Minimum value: 1 + int hoursAgo; + + @override + bool operator ==(Object other) => identical(this, other) || other is SystemConfigRemovePartialUploadsDto && + other.enabled == enabled && + other.hoursAgo == hoursAgo; + + @override + int get hashCode => + // ignore: unnecessary_parenthesis + (enabled.hashCode) + + (hoursAgo.hashCode); + + @override + String toString() => 'SystemConfigRemovePartialUploadsDto[enabled=$enabled, hoursAgo=$hoursAgo]'; + + Map toJson() { + final json = {}; + json[r'enabled'] = this.enabled; + json[r'hoursAgo'] = this.hoursAgo; + return json; + } + + /// Returns a new [SystemConfigRemovePartialUploadsDto] instance and imports its values from + /// [value] if it's a [Map], null otherwise. + // ignore: prefer_constructors_over_static_methods + static SystemConfigRemovePartialUploadsDto? fromJson(dynamic value) { + upgradeDto(value, "SystemConfigRemovePartialUploadsDto"); + if (value is Map) { + final json = value.cast(); + + return SystemConfigRemovePartialUploadsDto( + enabled: mapValueOfType(json, r'enabled')!, + hoursAgo: mapValueOfType(json, r'hoursAgo')!, + ); + } + return null; + } + + static List listFromJson(dynamic json, {bool growable = false,}) { + final result = []; + if (json is List && json.isNotEmpty) { + for (final row in json) { + final value = SystemConfigRemovePartialUploadsDto.fromJson(row); + if (value != null) { + result.add(value); + } + } + } + return result.toList(growable: growable); + } + + static Map mapFromJson(dynamic json) { + final map = {}; + if (json is Map && json.isNotEmpty) { + json = json.cast(); // ignore: parameter_assignments + for (final entry in json.entries) { + final value = SystemConfigRemovePartialUploadsDto.fromJson(entry.value); + if (value != null) { + map[entry.key] = value; + } + } + } + return map; + } + + // maps a json object with a list of SystemConfigRemovePartialUploadsDto-objects as value to a dart map + static Map> mapListFromJson(dynamic json, {bool growable = false,}) { + final map = >{}; + if (json is Map && json.isNotEmpty) { + // ignore: parameter_assignments + json = json.cast(); + for (final entry in json.entries) { + map[entry.key] = SystemConfigRemovePartialUploadsDto.listFromJson(entry.value, growable: growable,); + } + } + return map; + } + + /// The list of required keys that must be present in a JSON. + static const requiredKeys = { + 'enabled', + 'hoursAgo', + }; +} + diff --git a/open-api/immich-openapi-specs.json b/open-api/immich-openapi-specs.json index 43ea99c7fe..d176c15d4a 100644 --- a/open-api/immich-openapi-specs.json +++ b/open-api/immich-openapi-specs.json @@ -16824,6 +16824,9 @@ "missingThumbnails": { "type": "boolean" }, + "removeStaleUploads": { + "$ref": "#/components/schemas/SystemConfigRemovePartialUploadsDto" + }, "startTime": { "type": "string" }, @@ -16836,6 +16839,7 @@ "databaseCleanup", "generateMemories", "missingThumbnails", + "removeStaleUploads", "startTime", "syncQuotaUsage" ], @@ -16951,6 +16955,22 @@ ], "type": "object" }, + "SystemConfigRemovePartialUploadsDto": { + "properties": { + "enabled": { + "type": "boolean" + }, + "hoursAgo": { + "minimum": 1, + "type": "integer" + } + }, + "required": [ + "enabled", + "hoursAgo" + ], + "type": "object" + }, "SystemConfigReverseGeocodingDto": { "properties": { "enabled": { diff --git a/open-api/typescript-sdk/src/fetch-client.ts b/open-api/typescript-sdk/src/fetch-client.ts index 8f75f1c981..5075a2b3b2 100644 --- a/open-api/typescript-sdk/src/fetch-client.ts +++ b/open-api/typescript-sdk/src/fetch-client.ts @@ -1425,11 +1425,16 @@ export type SystemConfigMetadataDto = { export type SystemConfigNewVersionCheckDto = { enabled: boolean; }; +export type SystemConfigRemovePartialUploadsDto = { + enabled: boolean; + hoursAgo: number; +}; export type SystemConfigNightlyTasksDto = { clusterNewFaces: boolean; databaseCleanup: boolean; generateMemories: boolean; missingThumbnails: boolean; + removeStaleUploads: SystemConfigRemovePartialUploadsDto; startTime: string; syncQuotaUsage: boolean; }; diff --git a/server/src/config.ts b/server/src/config.ts index 66c03450fa..071980d6ef 100644 --- a/server/src/config.ts +++ b/server/src/config.ts @@ -133,6 +133,10 @@ export interface SystemConfig { clusterNewFaces: boolean; generateMemories: boolean; syncQuotaUsage: boolean; + removeStaleUploads: { + enabled: boolean; + hoursAgo: number; + }; }; trash: { enabled: boolean; @@ -325,6 +329,10 @@ export const defaults = Object.freeze({ syncQuotaUsage: true, missingThumbnails: true, clusterNewFaces: true, + removeStaleUploads: { + enabled: true, + hoursAgo: 72, + }, }, trash: { enabled: true, diff --git a/server/src/dtos/system-config.dto.ts b/server/src/dtos/system-config.dto.ts index 1facc6c331..8fc7baaa79 100644 --- a/server/src/dtos/system-config.dto.ts +++ b/server/src/dtos/system-config.dto.ts @@ -326,6 +326,17 @@ class SystemConfigNewVersionCheckDto { enabled!: boolean; } +class SystemConfigRemovePartialUploadsDto { + @ValidateBoolean() + enabled!: boolean; + + @IsInt() + @Min(1) + @Type(() => Number) + @ApiProperty({ type: 'integer' }) + hoursAgo!: number; +} + class SystemConfigNightlyTasksDto { @IsDateStringFormat('HH:mm', { message: 'startTime must be in HH:mm format' }) startTime!: string; @@ -344,6 +355,11 @@ class SystemConfigNightlyTasksDto { @ValidateBoolean() syncQuotaUsage!: boolean; + + @Type(() => SystemConfigRemovePartialUploadsDto) + @ValidateNested() + @IsObject() + removeStaleUploads!: SystemConfigRemovePartialUploadsDto; } class SystemConfigOAuthDto { diff --git a/server/src/services/asset-upload.service.ts b/server/src/services/asset-upload.service.ts index 26e5f92012..f097592775 100644 --- a/server/src/services/asset-upload.service.ts +++ b/server/src/services/asset-upload.service.ts @@ -202,8 +202,8 @@ export class AssetUploadService extends BaseService { @OnJob({ name: JobName.PartialAssetCleanupQueueAll, queue: QueueName.BackgroundTask }) async removeStaleUploads(): Promise { - // TODO: make this configurable - const createdBefore = DateTime.now().minus({ days: 7 }).toJSDate(); + const config = await this.getConfig({ withCache: false }); + const createdBefore = DateTime.now().minus({ hours: config.nightlyTasks.removeStaleUploads.hoursAgo }).toJSDate(); let jobs: JobItem[] = []; const assets = this.assetJobRepository.streamForPartialAssetCleanupJob(createdBefore); for await (const asset of assets) { diff --git a/server/src/services/job.service.spec.ts b/server/src/services/job.service.spec.ts index 6b85cdff4d..32f40c605d 100644 --- a/server/src/services/job.service.spec.ts +++ b/server/src/services/job.service.spec.ts @@ -48,6 +48,7 @@ describe(JobService.name, () => { { name: JobName.UserSyncUsage }, { name: JobName.AssetGenerateThumbnailsQueueAll, data: { force: false } }, { name: JobName.FacialRecognitionQueueAll, data: { force: false, nightly: true } }, + { name: JobName.PartialAssetCleanupQueueAll }, ]); }); }); diff --git a/server/src/services/job.service.ts b/server/src/services/job.service.ts index dc48c03bd1..8ffb8b204d 100644 --- a/server/src/services/job.service.ts +++ b/server/src/services/job.service.ts @@ -302,6 +302,10 @@ export class JobService extends BaseService { jobs.push({ name: JobName.FacialRecognitionQueueAll, data: { force: false, nightly: true } }); } + if (config.nightlyTasks.removeStaleUploads.enabled) { + jobs.push({ name: JobName.PartialAssetCleanupQueueAll }); + } + await this.jobRepository.queueAll(jobs); } diff --git a/server/src/services/system-config.service.spec.ts b/server/src/services/system-config.service.spec.ts index 5a9c7f4df3..4163b99ff6 100644 --- a/server/src/services/system-config.service.spec.ts +++ b/server/src/services/system-config.service.spec.ts @@ -115,6 +115,10 @@ const updatedConfig = Object.freeze({ missingThumbnails: true, generateMemories: true, syncQuotaUsage: true, + removeStaleUploads: { + enabled: true, + hoursAgo: 72, + }, }, reverseGeocoding: { enabled: true, diff --git a/server/src/types.ts b/server/src/types.ts index ce321eab8e..cb378b6724 100644 --- a/server/src/types.ts +++ b/server/src/types.ts @@ -353,7 +353,7 @@ export type JobItem = | { name: JobName.AssetDelete; data: IAssetDeleteJob } | { name: JobName.AssetDeleteCheck; data?: IBaseJob } | { name: JobName.PartialAssetCleanup; data: IEntityJob } - | { name: JobName.PartialAssetCleanupQueueAll; data: IBaseJob } + | { name: JobName.PartialAssetCleanupQueueAll; data?: IBaseJob } // Library Management | { name: JobName.LibrarySyncFiles; data: ILibraryFileJob }