feat(server): asset entity audit (#3824)

* feat(server): audit log

* feedback

* Insert to database

* migration

* test

* controller/repository/service

* test

* module

* feat(server): implement audit endpoint

* directly return changed assets

* add daily cleanup of audit table

* fix tests

* review feedback

* ci

* refactor(server): audit implementation

* chore: open api

---------

Co-authored-by: Alex Tran <alex.tran1502@gmail.com>
Co-authored-by: Fynn Petersen-Frey <zoodyy@users.noreply.github.com>
Co-authored-by: Jason Rasmussen <jrasm91@gmail.com>

mobile/openapi/lib/api.dart

@@ -31,6 +31,7 @@ part 'auth/http_bearer_auth.dart';
 part 'api/api_key_api.dart';
 part 'api/album_api.dart';
 part 'api/asset_api.dart';
+part 'api/audit_api.dart';
 part 'api/authentication_api.dart';
 part 'api/job_api.dart';
 part 'api/o_auth_api.dart';
@@ -66,6 +67,7 @@ part 'model/asset_response_dto.dart';
 part 'model/asset_stats_response_dto.dart';
 part 'model/asset_type_enum.dart';
 part 'model/audio_codec.dart';
+part 'model/audit_deletes_response_dto.dart';
 part 'model/auth_device_response_dto.dart';
 part 'model/bulk_id_response_dto.dart';
 part 'model/bulk_ids_dto.dart';
@@ -86,6 +88,7 @@ part 'model/delete_asset_status.dart';
 part 'model/download_archive_info.dart';
 part 'model/download_info_dto.dart';
 part 'model/download_response_dto.dart';
+part 'model/entity_type.dart';
 part 'model/exif_response_dto.dart';
 part 'model/import_asset_dto.dart';
 part 'model/job_command.dart';

mobile/openapi/lib/api/asset_api.dart

@@ -358,9 +358,11 @@ class AssetApi {
   ///
   /// * [num] skip:
   ///
+  /// * [DateTime] updatedAfter:
+  ///
   /// * [String] ifNoneMatch:
   ///   ETag of data already cached on the client
-  Future<Response> getAllAssetsWithHttpInfo({ String? userId, bool? isFavorite, bool? isArchived, bool? withoutThumbs, num? skip, String? ifNoneMatch, }) async {
+  Future<Response> getAllAssetsWithHttpInfo({ String? userId, bool? isFavorite, bool? isArchived, bool? withoutThumbs, num? skip, DateTime? updatedAfter, String? ifNoneMatch, }) async {
     // ignore: prefer_const_declarations
     final path = r'/asset';
 
@@ -386,6 +388,9 @@ class AssetApi {
     if (skip != null) {
       queryParams.addAll(_queryParams('', 'skip', skip));
     }
+    if (updatedAfter != null) {
+      queryParams.addAll(_queryParams('', 'updatedAfter', updatedAfter));
+    }
 
     if (ifNoneMatch != null) {
       headerParams[r'if-none-match'] = parameterToString(ifNoneMatch);
@@ -420,10 +425,12 @@ class AssetApi {
   ///
   /// * [num] skip:
   ///
+  /// * [DateTime] updatedAfter:
+  ///
   /// * [String] ifNoneMatch:
   ///   ETag of data already cached on the client
-  Future<List<AssetResponseDto>?> getAllAssets({ String? userId, bool? isFavorite, bool? isArchived, bool? withoutThumbs, num? skip, String? ifNoneMatch, }) async {
-    final response = await getAllAssetsWithHttpInfo( userId: userId, isFavorite: isFavorite, isArchived: isArchived, withoutThumbs: withoutThumbs, skip: skip, ifNoneMatch: ifNoneMatch, );
+  Future<List<AssetResponseDto>?> getAllAssets({ String? userId, bool? isFavorite, bool? isArchived, bool? withoutThumbs, num? skip, DateTime? updatedAfter, String? ifNoneMatch, }) async {
+    final response = await getAllAssetsWithHttpInfo( userId: userId, isFavorite: isFavorite, isArchived: isArchived, withoutThumbs: withoutThumbs, skip: skip, updatedAfter: updatedAfter, ifNoneMatch: ifNoneMatch, );
     if (response.statusCode >= HttpStatus.badRequest) {
       throw ApiException(response.statusCode, await _decodeBodyBytes(response));
     }

mobile/openapi/lib/api/audit_api.dart

@@ -0,0 +1,79 @@
+//
+// AUTO-GENERATED FILE, DO NOT MODIFY!
+//
+// @dart=2.12
+
+// ignore_for_file: unused_element, unused_import
+// ignore_for_file: always_put_required_named_parameters_first
+// ignore_for_file: constant_identifier_names
+// ignore_for_file: lines_longer_than_80_chars
+
+part of openapi.api;
+
+
+class AuditApi {
+  AuditApi([ApiClient? apiClient]) : apiClient = apiClient ?? defaultApiClient;
+
+  final ApiClient apiClient;
+
+  /// Performs an HTTP 'GET /audit/deletes' operation and returns the [Response].
+  /// Parameters:
+  ///
+  /// * [EntityType] entityType (required):
+  ///
+  /// * [DateTime] after (required):
+  ///
+  /// * [String] userId:
+  Future<Response> getAuditDeletesWithHttpInfo(EntityType entityType, DateTime after, { String? userId, }) async {
+    // ignore: prefer_const_declarations
+    final path = r'/audit/deletes';
+
+    // ignore: prefer_final_locals
+    Object? postBody;
+
+    final queryParams = <QueryParam>[];
+    final headerParams = <String, String>{};
+    final formParams = <String, String>{};
+
+      queryParams.addAll(_queryParams('', 'entityType', entityType));
+    if (userId != null) {
+      queryParams.addAll(_queryParams('', 'userId', userId));
+    }
+      queryParams.addAll(_queryParams('', 'after', after));
+
+    const contentTypes = <String>[];
+
+
+    return apiClient.invokeAPI(
+      path,
+      'GET',
+      queryParams,
+      postBody,
+      headerParams,
+      formParams,
+      contentTypes.isEmpty ? null : contentTypes.first,
+    );
+  }
+
+  /// Parameters:
+  ///
+  /// * [EntityType] entityType (required):
+  ///
+  /// * [DateTime] after (required):
+  ///
+  /// * [String] userId:
+  Future<AuditDeletesResponseDto?> getAuditDeletes(EntityType entityType, DateTime after, { String? userId, }) async {
+    final response = await getAuditDeletesWithHttpInfo(entityType, after,  userId: userId, );
+    if (response.statusCode >= HttpStatus.badRequest) {
+      throw ApiException(response.statusCode, await _decodeBodyBytes(response));
+    }
+    // When a remote server returns no body with a status of 204, we shall not decode it.
+    // At the time of writing this, `dart:convert` will throw an "Unexpected end of input"
+    // FormatException when trying to decode an empty string.
+    if (response.body.isNotEmpty && response.statusCode != HttpStatus.noContent) {
+      return await apiClient.deserializeAsync(await _decodeBodyBytes(response), 'AuditDeletesResponseDto',) as AuditDeletesResponseDto;
+    
+    }
+    return null;
+  }
+}

mobile/openapi/lib/api_client.dart

@@ -227,6 +227,8 @@ class ApiClient {
           return AssetTypeEnumTypeTransformer().decode(value);
         case 'AudioCodec':
           return AudioCodecTypeTransformer().decode(value);
+        case 'AuditDeletesResponseDto':
+          return AuditDeletesResponseDto.fromJson(value);
         case 'AuthDeviceResponseDto':
           return AuthDeviceResponseDto.fromJson(value);
         case 'BulkIdResponseDto':
@@ -267,6 +269,8 @@ class ApiClient {
           return DownloadInfoDto.fromJson(value);
         case 'DownloadResponseDto':
           return DownloadResponseDto.fromJson(value);
+        case 'EntityType':
+          return EntityTypeTypeTransformer().decode(value);
         case 'ExifResponseDto':
           return ExifResponseDto.fromJson(value);
         case 'ImportAssetDto':

mobile/openapi/lib/api_helper.dart

@@ -67,6 +67,9 @@ String parameterToString(dynamic value) {
   if (value is DeleteAssetStatus) {
     return DeleteAssetStatusTypeTransformer().encode(value).toString();
   }
+  if (value is EntityType) {
+    return EntityTypeTypeTransformer().encode(value).toString();
+  }
   if (value is JobCommand) {
     return JobCommandTypeTransformer().encode(value).toString();
   }

mobile/openapi/lib/model/audit_deletes_response_dto.dart

@@ -0,0 +1,108 @@
+//
+// AUTO-GENERATED FILE, DO NOT MODIFY!
+//
+// @dart=2.12
+
+// ignore_for_file: unused_element, unused_import
+// ignore_for_file: always_put_required_named_parameters_first
+// ignore_for_file: constant_identifier_names
+// ignore_for_file: lines_longer_than_80_chars
+
+part of openapi.api;
+
+class AuditDeletesResponseDto {
+  /// Returns a new [AuditDeletesResponseDto] instance.
+  AuditDeletesResponseDto({
+    this.ids = const [],
+    required this.needsFullSync,
+  });
+
+  List<String> ids;
+
+  bool needsFullSync;
+
+  @override
+  bool operator ==(Object other) => identical(this, other) || other is AuditDeletesResponseDto &&
+     other.ids == ids &&
+     other.needsFullSync == needsFullSync;
+
+  @override
+  int get hashCode =>
+    // ignore: unnecessary_parenthesis
+    (ids.hashCode) +
+    (needsFullSync.hashCode);
+
+  @override
+  String toString() => 'AuditDeletesResponseDto[ids=$ids, needsFullSync=$needsFullSync]';
+
+  Map<String, dynamic> toJson() {
+    final json = <String, dynamic>{};
+      json[r'ids'] = this.ids;
+      json[r'needsFullSync'] = this.needsFullSync;
+    return json;
+  }
+
+  /// Returns a new [AuditDeletesResponseDto] instance and imports its values from
+  /// [value] if it's a [Map], null otherwise.
+  // ignore: prefer_constructors_over_static_methods
+  static AuditDeletesResponseDto? fromJson(dynamic value) {
+    if (value is Map) {
+      final json = value.cast<String, dynamic>();
+
+      return AuditDeletesResponseDto(
+        ids: json[r'ids'] is List
+            ? (json[r'ids'] as List).cast<String>()
+            : const [],
+        needsFullSync: mapValueOfType<bool>(json, r'needsFullSync')!,
+      );
+    }
+    return null;
+  }
+
+  static List<AuditDeletesResponseDto> listFromJson(dynamic json, {bool growable = false,}) {
+    final result = <AuditDeletesResponseDto>[];
+    if (json is List && json.isNotEmpty) {
+      for (final row in json) {
+        final value = AuditDeletesResponseDto.fromJson(row);
+        if (value != null) {
+          result.add(value);
+        }
+      }
+    }
+    return result.toList(growable: growable);
+  }
+
+  static Map<String, AuditDeletesResponseDto> mapFromJson(dynamic json) {
+    final map = <String, AuditDeletesResponseDto>{};
+    if (json is Map && json.isNotEmpty) {
+      json = json.cast<String, dynamic>(); // ignore: parameter_assignments
+      for (final entry in json.entries) {
+        final value = AuditDeletesResponseDto.fromJson(entry.value);
+        if (value != null) {
+          map[entry.key] = value;
+        }
+      }
+    }
+    return map;
+  }
+
+  // maps a json object with a list of AuditDeletesResponseDto-objects as value to a dart map
+  static Map<String, List<AuditDeletesResponseDto>> mapListFromJson(dynamic json, {bool growable = false,}) {
+    final map = <String, List<AuditDeletesResponseDto>>{};
+    if (json is Map && json.isNotEmpty) {
+      // ignore: parameter_assignments
+      json = json.cast<String, dynamic>();
+      for (final entry in json.entries) {
+        map[entry.key] = AuditDeletesResponseDto.listFromJson(entry.value, growable: growable,);
+      }
+    }
+    return map;
+  }
+
+  /// The list of required keys that must be present in a JSON.
+  static const requiredKeys = <String>{
+    'ids',
+    'needsFullSync',
+  };
+}
+

mobile/openapi/lib/model/entity_type.dart

@@ -0,0 +1,85 @@
+//
+// AUTO-GENERATED FILE, DO NOT MODIFY!
+//
+// @dart=2.12
+
+// ignore_for_file: unused_element, unused_import
+// ignore_for_file: always_put_required_named_parameters_first
+// ignore_for_file: constant_identifier_names
+// ignore_for_file: lines_longer_than_80_chars
+
+part of openapi.api;
+
+
+class EntityType {
+  /// Instantiate a new enum with the provided [value].
+  const EntityType._(this.value);
+
+  /// The underlying value of this enum member.
+  final String value;
+
+  @override
+  String toString() => value;
+
+  String toJson() => value;
+
+  static const ASSET = EntityType._(r'ASSET');
+  static const ALBUM = EntityType._(r'ALBUM');
+
+  /// List of all possible values in this [enum][EntityType].
+  static const values = <EntityType>[
+    ASSET,
+    ALBUM,
+  ];
+
+  static EntityType? fromJson(dynamic value) => EntityTypeTypeTransformer().decode(value);
+
+  static List<EntityType>? listFromJson(dynamic json, {bool growable = false,}) {
+    final result = <EntityType>[];
+    if (json is List && json.isNotEmpty) {
+      for (final row in json) {
+        final value = EntityType.fromJson(row);
+        if (value != null) {
+          result.add(value);
+        }
+      }
+    }
+    return result.toList(growable: growable);
+  }
+}
+
+/// Transformation class that can [encode] an instance of [EntityType] to String,
+/// and [decode] dynamic data back to [EntityType].
+class EntityTypeTypeTransformer {
+  factory EntityTypeTypeTransformer() => _instance ??= const EntityTypeTypeTransformer._();
+
+  const EntityTypeTypeTransformer._();
+
+  String encode(EntityType data) => data.value;
+
+  /// Decodes a [dynamic value][data] to a EntityType.
+  ///
+  /// If [allowNull] is true and the [dynamic value][data] cannot be decoded successfully,
+  /// then null is returned. However, if [allowNull] is false and the [dynamic value][data]
+  /// cannot be decoded successfully, then an [UnimplementedError] is thrown.
+  ///
+  /// The [allowNull] is very handy when an API changes and a new enum value is added or removed,
+  /// and users are still using an old app with the old code.
+  EntityType? decode(dynamic data, {bool allowNull = true}) {
+    if (data != null) {
+      switch (data) {
+        case r'ASSET': return EntityType.ASSET;
+        case r'ALBUM': return EntityType.ALBUM;
+        default:
+          if (!allowNull) {
+            throw ArgumentError('Unknown enum value to decode: $data');
+          }
+      }
+    }
+    return null;
+  }
+
+  /// Singleton [EntityTypeTypeTransformer] instance.
+  static EntityTypeTypeTransformer? _instance;
+}
+