configurable cleanup

mobile/openapi/README.md

@@ -545,6 +545,7 @@ Class | Method | HTTP request | Description
  - [SystemConfigNotificationsDto](doc//SystemConfigNotificationsDto.md)
  - [SystemConfigOAuthDto](doc//SystemConfigOAuthDto.md)
  - [SystemConfigPasswordLoginDto](doc//SystemConfigPasswordLoginDto.md)
+ - [SystemConfigRemovePartialUploadsDto](doc//SystemConfigRemovePartialUploadsDto.md)
  - [SystemConfigReverseGeocodingDto](doc//SystemConfigReverseGeocodingDto.md)
  - [SystemConfigServerDto](doc//SystemConfigServerDto.md)
  - [SystemConfigSmtpDto](doc//SystemConfigSmtpDto.md)

mobile/openapi/lib/api.dart

@@ -312,6 +312,7 @@ part 'model/system_config_nightly_tasks_dto.dart';
 part 'model/system_config_notifications_dto.dart';
 part 'model/system_config_o_auth_dto.dart';
 part 'model/system_config_password_login_dto.dart';
+part 'model/system_config_remove_partial_uploads_dto.dart';
 part 'model/system_config_reverse_geocoding_dto.dart';
 part 'model/system_config_server_dto.dart';
 part 'model/system_config_smtp_dto.dart';

mobile/openapi/lib/api_client.dart

@@ -676,6 +676,8 @@ class ApiClient {
           return SystemConfigOAuthDto.fromJson(value);
         case 'SystemConfigPasswordLoginDto':
           return SystemConfigPasswordLoginDto.fromJson(value);
+        case 'SystemConfigRemovePartialUploadsDto':
+          return SystemConfigRemovePartialUploadsDto.fromJson(value);
         case 'SystemConfigReverseGeocodingDto':
           return SystemConfigReverseGeocodingDto.fromJson(value);
         case 'SystemConfigServerDto':

mobile/openapi/lib/model/system_config_nightly_tasks_dto.dart

@@ -17,6 +17,7 @@ class SystemConfigNightlyTasksDto {
     required this.databaseCleanup,
     required this.generateMemories,
     required this.missingThumbnails,
+    required this.removeStaleUploads,
     required this.startTime,
     required this.syncQuotaUsage,
   });
@@ -29,6 +30,8 @@ class SystemConfigNightlyTasksDto {
 
   bool missingThumbnails;
 
+  SystemConfigRemovePartialUploadsDto removeStaleUploads;
+
   String startTime;
 
   bool syncQuotaUsage;
@@ -39,6 +42,7 @@ class SystemConfigNightlyTasksDto {
     other.databaseCleanup == databaseCleanup &&
     other.generateMemories == generateMemories &&
     other.missingThumbnails == missingThumbnails &&
+    other.removeStaleUploads == removeStaleUploads &&
     other.startTime == startTime &&
     other.syncQuotaUsage == syncQuotaUsage;
 
@@ -49,11 +53,12 @@ class SystemConfigNightlyTasksDto {
     (databaseCleanup.hashCode) +
     (generateMemories.hashCode) +
     (missingThumbnails.hashCode) +
+    (removeStaleUploads.hashCode) +
     (startTime.hashCode) +
     (syncQuotaUsage.hashCode);
 
   @override
-  String toString() => 'SystemConfigNightlyTasksDto[clusterNewFaces=$clusterNewFaces, databaseCleanup=$databaseCleanup, generateMemories=$generateMemories, missingThumbnails=$missingThumbnails, startTime=$startTime, syncQuotaUsage=$syncQuotaUsage]';
+  String toString() => 'SystemConfigNightlyTasksDto[clusterNewFaces=$clusterNewFaces, databaseCleanup=$databaseCleanup, generateMemories=$generateMemories, missingThumbnails=$missingThumbnails, removeStaleUploads=$removeStaleUploads, startTime=$startTime, syncQuotaUsage=$syncQuotaUsage]';
 
   Map<String, dynamic> toJson() {
     final json = <String, dynamic>{};
@@ -61,6 +66,7 @@ class SystemConfigNightlyTasksDto {
       json[r'databaseCleanup'] = this.databaseCleanup;
       json[r'generateMemories'] = this.generateMemories;
       json[r'missingThumbnails'] = this.missingThumbnails;
+      json[r'removeStaleUploads'] = this.removeStaleUploads;
       json[r'startTime'] = this.startTime;
       json[r'syncQuotaUsage'] = this.syncQuotaUsage;
     return json;
@@ -79,6 +85,7 @@ class SystemConfigNightlyTasksDto {
         databaseCleanup: mapValueOfType<bool>(json, r'databaseCleanup')!,
         generateMemories: mapValueOfType<bool>(json, r'generateMemories')!,
         missingThumbnails: mapValueOfType<bool>(json, r'missingThumbnails')!,
+        removeStaleUploads: SystemConfigRemovePartialUploadsDto.fromJson(json[r'removeStaleUploads'])!,
         startTime: mapValueOfType<String>(json, r'startTime')!,
         syncQuotaUsage: mapValueOfType<bool>(json, r'syncQuotaUsage')!,
       );
@@ -132,6 +139,7 @@ class SystemConfigNightlyTasksDto {
     'databaseCleanup',
     'generateMemories',
     'missingThumbnails',
+    'removeStaleUploads',
     'startTime',
     'syncQuotaUsage',
   };

mobile/openapi/lib/model/system_config_remove_partial_uploads_dto.dart

@@ -0,0 +1,108 @@
+//
+// AUTO-GENERATED FILE, DO NOT MODIFY!
+//
+// @dart=2.18
+
+// ignore_for_file: unused_element, unused_import
+// ignore_for_file: always_put_required_named_parameters_first
+// ignore_for_file: constant_identifier_names
+// ignore_for_file: lines_longer_than_80_chars
+
+part of openapi.api;
+
+class SystemConfigRemovePartialUploadsDto {
+  /// Returns a new [SystemConfigRemovePartialUploadsDto] instance.
+  SystemConfigRemovePartialUploadsDto({
+    required this.enabled,
+    required this.hoursAgo,
+  });
+
+  bool enabled;
+
+  /// Minimum value: 1
+  int hoursAgo;
+
+  @override
+  bool operator ==(Object other) => identical(this, other) || other is SystemConfigRemovePartialUploadsDto &&
+    other.enabled == enabled &&
+    other.hoursAgo == hoursAgo;
+
+  @override
+  int get hashCode =>
+    // ignore: unnecessary_parenthesis
+    (enabled.hashCode) +
+    (hoursAgo.hashCode);
+
+  @override
+  String toString() => 'SystemConfigRemovePartialUploadsDto[enabled=$enabled, hoursAgo=$hoursAgo]';
+
+  Map<String, dynamic> toJson() {
+    final json = <String, dynamic>{};
+      json[r'enabled'] = this.enabled;
+      json[r'hoursAgo'] = this.hoursAgo;
+    return json;
+  }
+
+  /// Returns a new [SystemConfigRemovePartialUploadsDto] instance and imports its values from
+  /// [value] if it's a [Map], null otherwise.
+  // ignore: prefer_constructors_over_static_methods
+  static SystemConfigRemovePartialUploadsDto? fromJson(dynamic value) {
+    upgradeDto(value, "SystemConfigRemovePartialUploadsDto");
+    if (value is Map) {
+      final json = value.cast<String, dynamic>();
+
+      return SystemConfigRemovePartialUploadsDto(
+        enabled: mapValueOfType<bool>(json, r'enabled')!,
+        hoursAgo: mapValueOfType<int>(json, r'hoursAgo')!,
+      );
+    }
+    return null;
+  }
+
+  static List<SystemConfigRemovePartialUploadsDto> listFromJson(dynamic json, {bool growable = false,}) {
+    final result = <SystemConfigRemovePartialUploadsDto>[];
+    if (json is List && json.isNotEmpty) {
+      for (final row in json) {
+        final value = SystemConfigRemovePartialUploadsDto.fromJson(row);
+        if (value != null) {
+          result.add(value);
+        }
+      }
+    }
+    return result.toList(growable: growable);
+  }
+
+  static Map<String, SystemConfigRemovePartialUploadsDto> mapFromJson(dynamic json) {
+    final map = <String, SystemConfigRemovePartialUploadsDto>{};
+    if (json is Map && json.isNotEmpty) {
+      json = json.cast<String, dynamic>(); // ignore: parameter_assignments
+      for (final entry in json.entries) {
+        final value = SystemConfigRemovePartialUploadsDto.fromJson(entry.value);
+        if (value != null) {
+          map[entry.key] = value;
+        }
+      }
+    }
+    return map;
+  }
+
+  // maps a json object with a list of SystemConfigRemovePartialUploadsDto-objects as value to a dart map
+  static Map<String, List<SystemConfigRemovePartialUploadsDto>> mapListFromJson(dynamic json, {bool growable = false,}) {
+    final map = <String, List<SystemConfigRemovePartialUploadsDto>>{};
+    if (json is Map && json.isNotEmpty) {
+      // ignore: parameter_assignments
+      json = json.cast<String, dynamic>();
+      for (final entry in json.entries) {
+        map[entry.key] = SystemConfigRemovePartialUploadsDto.listFromJson(entry.value, growable: growable,);
+      }
+    }
+    return map;
+  }
+
+  /// The list of required keys that must be present in a JSON.
+  static const requiredKeys = <String>{
+    'enabled',
+    'hoursAgo',
+  };
+}
+

open-api/immich-openapi-specs.json

@@ -16824,6 +16824,9 @@
           "missingThumbnails": {
             "type": "boolean"
           },
+          "removeStaleUploads": {
+            "$ref": "#/components/schemas/SystemConfigRemovePartialUploadsDto"
+          },
           "startTime": {
             "type": "string"
           },
@@ -16836,6 +16839,7 @@
           "databaseCleanup",
           "generateMemories",
           "missingThumbnails",
+          "removeStaleUploads",
           "startTime",
           "syncQuotaUsage"
         ],
@@ -16951,6 +16955,22 @@
         ],
         "type": "object"
       },
+      "SystemConfigRemovePartialUploadsDto": {
+        "properties": {
+          "enabled": {
+            "type": "boolean"
+          },
+          "hoursAgo": {
+            "minimum": 1,
+            "type": "integer"
+          }
+        },
+        "required": [
+          "enabled",
+          "hoursAgo"
+        ],
+        "type": "object"
+      },
       "SystemConfigReverseGeocodingDto": {
         "properties": {
           "enabled": {

open-api/typescript-sdk/src/fetch-client.ts

@@ -1425,11 +1425,16 @@ export type SystemConfigMetadataDto = {
 export type SystemConfigNewVersionCheckDto = {
     enabled: boolean;
 };
+export type SystemConfigRemovePartialUploadsDto = {
+    enabled: boolean;
+    hoursAgo: number;
+};
 export type SystemConfigNightlyTasksDto = {
     clusterNewFaces: boolean;
     databaseCleanup: boolean;
     generateMemories: boolean;
     missingThumbnails: boolean;
+    removeStaleUploads: SystemConfigRemovePartialUploadsDto;
     startTime: string;
     syncQuotaUsage: boolean;
 };

server/src/config.ts

@@ -133,6 +133,10 @@ export interface SystemConfig {
     clusterNewFaces: boolean;
     generateMemories: boolean;
     syncQuotaUsage: boolean;
+    removeStaleUploads: {
+      enabled: boolean;
+      hoursAgo: number;
+    };
   };
   trash: {
     enabled: boolean;
@@ -325,6 +329,10 @@ export const defaults = Object.freeze<SystemConfig>({
     syncQuotaUsage: true,
     missingThumbnails: true,
     clusterNewFaces: true,
+    removeStaleUploads: {
+      enabled: true,
+      hoursAgo: 72,
+    },
   },
   trash: {
     enabled: true,

server/src/dtos/system-config.dto.ts

@@ -326,6 +326,17 @@ class SystemConfigNewVersionCheckDto {
   enabled!: boolean;
 }
 
+class SystemConfigRemovePartialUploadsDto {
+  @ValidateBoolean()
+  enabled!: boolean;
+
+  @IsInt()
+  @Min(1)
+  @Type(() => Number)
+  @ApiProperty({ type: 'integer' })
+  hoursAgo!: number;
+}
+
 class SystemConfigNightlyTasksDto {
   @IsDateStringFormat('HH:mm', { message: 'startTime must be in HH:mm format' })
   startTime!: string;
@@ -344,6 +355,11 @@ class SystemConfigNightlyTasksDto {
 
   @ValidateBoolean()
   syncQuotaUsage!: boolean;
+
+  @Type(() => SystemConfigRemovePartialUploadsDto)
+  @ValidateNested()
+  @IsObject()
+  removeStaleUploads!: SystemConfigRemovePartialUploadsDto;
 }
 
 class SystemConfigOAuthDto {

server/src/services/asset-upload.service.ts

@@ -202,8 +202,8 @@ export class AssetUploadService extends BaseService {
 
   @OnJob({ name: JobName.PartialAssetCleanupQueueAll, queue: QueueName.BackgroundTask })
   async removeStaleUploads(): Promise<void> {
-    // TODO: make this configurable
-    const createdBefore = DateTime.now().minus({ days: 7 }).toJSDate();
+    const config = await this.getConfig({ withCache: false });
+    const createdBefore = DateTime.now().minus({ hours: config.nightlyTasks.removeStaleUploads.hoursAgo }).toJSDate();
     let jobs: JobItem[] = [];
     const assets = this.assetJobRepository.streamForPartialAssetCleanupJob(createdBefore);
     for await (const asset of assets) {

server/src/services/job.service.spec.ts

@@ -48,6 +48,7 @@ describe(JobService.name, () => {
         { name: JobName.UserSyncUsage },
         { name: JobName.AssetGenerateThumbnailsQueueAll, data: { force: false } },
         { name: JobName.FacialRecognitionQueueAll, data: { force: false, nightly: true } },
+        { name: JobName.PartialAssetCleanupQueueAll },
       ]);
     });
   });

server/src/services/job.service.ts

@@ -302,6 +302,10 @@ export class JobService extends BaseService {
       jobs.push({ name: JobName.FacialRecognitionQueueAll, data: { force: false, nightly: true } });
     }
 
+    if (config.nightlyTasks.removeStaleUploads.enabled) {
+      jobs.push({ name: JobName.PartialAssetCleanupQueueAll });
+    }
+
     await this.jobRepository.queueAll(jobs);
   }
 

server/src/services/system-config.service.spec.ts

@@ -115,6 +115,10 @@ const updatedConfig = Object.freeze<SystemConfig>({
     missingThumbnails: true,
     generateMemories: true,
     syncQuotaUsage: true,
+    removeStaleUploads: {
+      enabled: true,
+      hoursAgo: 72,
+    },
   },
   reverseGeocoding: {
     enabled: true,

server/src/types.ts

@@ -353,7 +353,7 @@ export type JobItem =
   | { name: JobName.AssetDelete; data: IAssetDeleteJob }
   | { name: JobName.AssetDeleteCheck; data?: IBaseJob }
   | { name: JobName.PartialAssetCleanup; data: IEntityJob }
-  | { name: JobName.PartialAssetCleanupQueueAll; data: IBaseJob }
+  | { name: JobName.PartialAssetCleanupQueueAll; data?: IBaseJob }
 
   // Library Management
   | { name: JobName.LibrarySyncFiles; data: ILibraryFileJob }