feat(mobile) duplicated asset upload handling mechanism (#853)

mobile/lib/modules/backup/services/backup.service.dart

@@ -19,6 +19,8 @@ import 'package:http_parser/http_parser.dart';
 import 'package:path/path.dart' as p;
 import 'package:cancellation_token_http/http.dart' as http;
 
+import '../models/hive_duplicated_assets.model.dart';
+
 final backupServiceProvider = Provider(
   (ref) => BackupService(
     ref.watch(apiServiceProvider),
@@ -41,6 +43,29 @@ class BackupService {
     }
   }
 
+  void _saveDuplicatedAssetIdToLocalStorage(List<String> deviceAssetIds) {
+    HiveDuplicatedAssets duplicatedAssets =
+        Hive.box<HiveDuplicatedAssets>(duplicatedAssetsBox)
+                .get(duplicatedAssetsKey) ??
+            HiveDuplicatedAssets(duplicatedAssetIds: []);
+
+    duplicatedAssets.duplicatedAssetIds =
+        {...duplicatedAssets.duplicatedAssetIds, ...deviceAssetIds}.toList();
+
+    Hive.box<HiveDuplicatedAssets>(duplicatedAssetsBox)
+        .put(duplicatedAssetsKey, duplicatedAssets);
+  }
+
+  /// Get duplicated asset id from Hive storage
+  Set<String> getDuplicatedAssetIds() {
+    HiveDuplicatedAssets duplicatedAssets =
+        Hive.box<HiveDuplicatedAssets>(duplicatedAssetsBox)
+                .get(duplicatedAssetsKey) ??
+            HiveDuplicatedAssets(duplicatedAssetIds: []);
+
+    return duplicatedAssets.duplicatedAssetIds.toSet();
+  }
+
   /// Returns all assets newer than the last successful backup per album
   Future<List<AssetEntity>> buildUploadCandidates(
     HiveBackupAlbums backupAlbums,
@@ -140,34 +165,47 @@ class BackupService {
   Future<List<AssetEntity>> removeAlreadyUploadedAssets(
     List<AssetEntity> candidates,
   ) async {
-    final String deviceId = Hive.box(userInfoBox).get(deviceIdKey);
-    if (candidates.length < 10) {
-      final List<CheckDuplicateAssetResponseDto?> duplicateResponse =
-          await Future.wait(
-        candidates.map(
-          (e) => _apiService.assetApi.checkDuplicateAsset(
-            CheckDuplicateAssetDto(deviceAssetId: e.id, deviceId: deviceId),
-          ),
+    if (candidates.isEmpty) {
+      return candidates;
+    }
+    final Set<String> duplicatedAssetIds = getDuplicatedAssetIds();
+    candidates = duplicatedAssetIds.isEmpty
+        ? candidates
+        : candidates
+            .whereNot((asset) => duplicatedAssetIds.contains(asset.id))
+            .toList();
+    if (candidates.isEmpty) {
+      return candidates;
+    }
+    final Set<String> existing = {};
+    try {
+      final String deviceId = Hive.box(userInfoBox).get(deviceIdKey);
+      final CheckExistingAssetsResponseDto? duplicates =
+          await _apiService.assetApi.checkExistingAssets(
+        CheckExistingAssetsDto(
+          deviceAssetIds: candidates.map((e) => e.id).toList(),
+          deviceId: deviceId,
         ),
       );
-      return candidates
-          .whereIndexed((i, e) => duplicateResponse[i]?.isExist == false)
-          .toList();
-    } else {
-      final List<String>? allAssetsInDatabase = await getDeviceBackupAsset();
-
-      if (allAssetsInDatabase == null) {
-        return candidates;
+      if (duplicates != null) {
+        existing.addAll(duplicates.existingIds);
+      }
+    } on ApiException {
+      // workaround for older server versions or when checking for too many assets at once
+      final List<String>? allAssetsInDatabase = await getDeviceBackupAsset();
+      if (allAssetsInDatabase != null) {
+        existing.addAll(allAssetsInDatabase);
       }
-      final Set<String> inDb = allAssetsInDatabase.toSet();
-      return candidates.whereNot((e) => inDb.contains(e.id)).toList();
     }
+    return existing.isEmpty
+        ? candidates
+        : candidates.whereNot((e) => existing.contains(e.id)).toList();
   }
 
   Future<bool> backupAsset(
     Iterable<AssetEntity> assetList,
     http.CancellationToken cancelToken,
-    Function(String, String) singleAssetDoneCb,
+    Function(String, String, bool) uploadSuccessCb,
     Function(int, int) uploadProgressCb,
     Function(CurrentUploadAsset) setCurrentUploadAssetCb,
     Function(ErrorUploadAsset) errorCb,
@@ -176,6 +214,7 @@ class BackupService {
     String savedEndpoint = Hive.box(userInfoBox).get(serverEndpointKey);
     File? file;
     bool anyErrors = false;
+    final List<String> duplicatedAssetIds = [];
 
     for (var entity in assetList) {
       try {
@@ -235,8 +274,13 @@ class BackupService {
 
           var response = await req.send(cancellationToken: cancelToken);
 
-          if (response.statusCode == 201) {
-            singleAssetDoneCb(entity.id, deviceId);
+          if (response.statusCode == 200) {
+            // asset is a duplicate (already exists on the server)
+            duplicatedAssetIds.add(entity.id);
+            uploadSuccessCb(entity.id, deviceId, true);
+          } else if (response.statusCode == 201) {
+            // stored a new asset on the server
+            uploadSuccessCb(entity.id, deviceId, false);
           } else {
             var data = await response.stream.bytesToString();
             var error = jsonDecode(data);
@@ -260,7 +304,8 @@ class BackupService {
         }
       } on http.CancelledException {
         debugPrint("Backup was cancelled by the user");
-        return false;
+        anyErrors = true;
+        break;
       } catch (e) {
         debugPrint("ERROR backupAsset: ${e.toString()}");
         anyErrors = true;
@@ -271,6 +316,9 @@ class BackupService {
         }
       }
     }
+    if (duplicatedAssetIds.isNotEmpty) {
+      _saveDuplicatedAssetIdToLocalStorage(duplicatedAssetIds);
+    }
     return !anyErrors;
   }