diff --git a/mobile/android/app/src/main/kotlin/app/alextran/immich/sync/Messages.g.kt b/mobile/android/app/src/main/kotlin/app/alextran/immich/sync/Messages.g.kt index fc3cb2b3a9..d6821d1b51 100644 --- a/mobile/android/app/src/main/kotlin/app/alextran/immich/sync/Messages.g.kt +++ b/mobile/android/app/src/main/kotlin/app/alextran/immich/sync/Messages.g.kt @@ -296,7 +296,7 @@ private open class MessagesPigeonCodec : StandardMessageCodec() { /** Generated interface from Pigeon that represents a handler of messages from Flutter. */ interface NativeSyncApi { fun shouldFullSync(): Boolean - fun getMediaChanges(isTrashed: Boolean): SyncDelta + fun getMediaChanges(): SyncDelta fun checkpointSync() fun clearSyncCheckpoint() fun getAssetIdsForAlbum(albumId: String): List @@ -305,7 +305,7 @@ interface NativeSyncApi { fun getAssetsForAlbum(albumId: String, updatedTimeCond: Long?): List fun hashAssets(assetIds: List, allowNetworkAccess: Boolean, callback: (Result>) -> Unit) fun cancelHashing() - fun getTrashedAssetsForAlbum(albumId: String): List + fun getTrashedAssets(albumIds: List, sinceLastCheckpoint: Boolean): Map> companion object { /** The codec used by NativeSyncApi. */ @@ -335,11 +335,9 @@ interface NativeSyncApi { run { val channel = BasicMessageChannel(binaryMessenger, "dev.flutter.pigeon.immich_mobile.NativeSyncApi.getMediaChanges$separatedMessageChannelSuffix", codec, taskQueue) if (api != null) { - channel.setMessageHandler { message, reply -> - val args = message as List - val isTrashedArg = args[0] as Boolean + channel.setMessageHandler { _, reply -> val wrapped: List = try { - listOf(api.getMediaChanges(isTrashedArg)) + listOf(api.getMediaChanges()) } catch (exception: Throwable) { MessagesPigeonUtils.wrapError(exception) } @@ -487,13 +485,14 @@ interface NativeSyncApi { } } run { - val channel = BasicMessageChannel(binaryMessenger, "dev.flutter.pigeon.immich_mobile.NativeSyncApi.getTrashedAssetsForAlbum$separatedMessageChannelSuffix", codec, taskQueue) + val channel = BasicMessageChannel(binaryMessenger, "dev.flutter.pigeon.immich_mobile.NativeSyncApi.getTrashedAssets$separatedMessageChannelSuffix", codec, taskQueue) if (api != null) { channel.setMessageHandler { message, reply -> val args = message as List - val albumIdArg = args[0] as String + val albumIdsArg = args[0] as List + val sinceLastCheckpointArg = args[1] as Boolean val wrapped: List = try { - listOf(api.getTrashedAssetsForAlbum(albumIdArg)) + listOf(api.getTrashedAssets(albumIdsArg, sinceLastCheckpointArg)) } catch (exception: Throwable) { MessagesPigeonUtils.wrapError(exception) } diff --git a/mobile/android/app/src/main/kotlin/app/alextran/immich/sync/MessagesImpl26.kt b/mobile/android/app/src/main/kotlin/app/alextran/immich/sync/MessagesImpl26.kt index 1f42a55706..13961372b9 100644 --- a/mobile/android/app/src/main/kotlin/app/alextran/immich/sync/MessagesImpl26.kt +++ b/mobile/android/app/src/main/kotlin/app/alextran/immich/sync/MessagesImpl26.kt @@ -18,13 +18,14 @@ class NativeSyncApiImpl26(context: Context) : NativeSyncApiImplBase(context), Na // No-op for Android 10 and below } - override fun getMediaChanges(isTrashed: Boolean): SyncDelta { + override fun getMediaChanges(): SyncDelta { throw IllegalStateException("Method not supported on this Android version.") } - override fun getTrashedAssetsForAlbum( - albumId: String, - ): List { + override fun getTrashedAssets( + albumIds: List, + sinceLastCheckpoint: Boolean + ): Map> { throw IllegalStateException("Method not supported on this Android version.") } } diff --git a/mobile/android/app/src/main/kotlin/app/alextran/immich/sync/MessagesImpl30.kt b/mobile/android/app/src/main/kotlin/app/alextran/immich/sync/MessagesImpl30.kt index b43bf7ab72..b676db3c3f 100644 --- a/mobile/android/app/src/main/kotlin/app/alextran/immich/sync/MessagesImpl30.kt +++ b/mobile/android/app/src/main/kotlin/app/alextran/immich/sync/MessagesImpl30.kt @@ -1,23 +1,13 @@ package app.alextran.immich.sync import android.content.ContentResolver -import android.content.ContentUris import android.content.Context -import android.net.Uri import android.os.Build import android.os.Bundle import android.provider.MediaStore import androidx.annotation.RequiresApi import androidx.annotation.RequiresExtension -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.async -import kotlinx.coroutines.awaitAll -import kotlinx.coroutines.ensureActive -import kotlinx.coroutines.launch -import kotlinx.coroutines.sync.withPermit import kotlinx.serialization.json.Json -import kotlin.coroutines.cancellation.CancellationException @RequiresApi(Build.VERSION_CODES.Q) @RequiresExtension(extension = Build.VERSION_CODES.R, version = 1) @@ -59,7 +49,7 @@ class NativeSyncApiImpl30(context: Context) : NativeSyncApiImplBase(context), Na } } - override fun getMediaChanges(isTrashed: Boolean): SyncDelta { + override fun getMediaChanges(): SyncDelta { val genMap = getSavedGenerationMap() val currentVolumes = MediaStore.getExternalVolumeNames(ctx) val changed = mutableListOf() @@ -83,17 +73,8 @@ class NativeSyncApiImpl30(context: Context) : NativeSyncApiImplBase(context), Na storedGen.toString(), storedGen.toString() ) - val cursor = if (isTrashed) { - val queryArgs = Bundle().apply { - putString(ContentResolver.QUERY_ARG_SQL_SELECTION, selection) - putStringArray(ContentResolver.QUERY_ARG_SQL_SELECTION_ARGS, selectionArgs) - putInt(MediaStore.QUERY_ARG_MATCH_TRASHED, MediaStore.MATCH_ONLY) - } - getCursor(volume, queryArgs) - } else { - getCursor(volume, selection, selectionArgs) - } - getAssets(cursor).forEach { + + getAssets(getCursor(volume, selection, selectionArgs)).forEach { when (it) { is AssetResult.ValidAsset -> { changed.add(it.asset) @@ -108,26 +89,81 @@ class NativeSyncApiImpl30(context: Context) : NativeSyncApiImplBase(context), Na return SyncDelta(hasChanges, changed, deleted, assetAlbums) } - override fun getTrashedAssetsForAlbum( - albumId: String - ): List { - val trashed = mutableListOf() +// override fun getTrashedAssetsForAlbum( +// albumId: String +// ): List { +// val trashed = mutableListOf() +// val volumes = MediaStore.getExternalVolumeNames(ctx) +// +// val selection = "$BUCKET_SELECTION AND $MEDIA_SELECTION" +// val selectionArgs = mutableListOf(albumId, *MEDIA_SELECTION_ARGS) +// +// for (volume in volumes) { +// val cursor = getCursor(volume, Bundle().apply { +// putString(ContentResolver.QUERY_ARG_SQL_SELECTION, selection) +// putStringArray(ContentResolver.QUERY_ARG_SQL_SELECTION_ARGS, selectionArgs.toTypedArray()) +// putInt(MediaStore.QUERY_ARG_MATCH_TRASHED, MediaStore.MATCH_ONLY) +// }) +// getAssets(cursor).forEach { res -> +// if (res is AssetResult.ValidAsset) trashed += res.asset +// } +// } +// +// return trashed +// } + + override fun getTrashedAssets( + albumIds: List, + sinceLastCheckpoint: Boolean + ): Map> { + if (albumIds.isEmpty()) return emptyMap() + + val result = LinkedHashMap>(albumIds.size) val volumes = MediaStore.getExternalVolumeNames(ctx) - val selection = "$BUCKET_SELECTION AND $MEDIA_SELECTION" - val selectionArgs = mutableListOf(albumId, *MEDIA_SELECTION_ARGS) + val placeholders = albumIds.joinToString(",") { "?" } + val bucketIn = "(${MediaStore.Files.FileColumns.BUCKET_ID} IN ($placeholders))" + val baseSelection = "$bucketIn AND $MEDIA_SELECTION" + + val baseSelectionArgs = ArrayList(albumIds.size + MEDIA_SELECTION_ARGS.size).apply { + addAll(albumIds) + addAll(MEDIA_SELECTION_ARGS) + } + + val genMap = if (sinceLastCheckpoint) getSavedGenerationMap() else emptyMap() for (volume in volumes) { - val cursor = getCursor(volume, Bundle().apply { + var selection = baseSelection + val selectionArgs = ArrayList(baseSelectionArgs.size + if (sinceLastCheckpoint) 2 else 0).apply { + addAll(baseSelectionArgs) + } + + if (sinceLastCheckpoint) { + val currentGen = MediaStore.getGeneration(ctx, volume) + val storedGen = genMap[volume] ?: 0L + if (currentGen <= storedGen) { + continue + } + selection += " AND (${MediaStore.MediaColumns.GENERATION_MODIFIED} > ? OR ${MediaStore.MediaColumns.GENERATION_ADDED} > ?)" + selectionArgs += storedGen.toString() + selectionArgs += storedGen.toString() + } + + val queryArgs = Bundle().apply { putString(ContentResolver.QUERY_ARG_SQL_SELECTION, selection) putStringArray(ContentResolver.QUERY_ARG_SQL_SELECTION_ARGS, selectionArgs.toTypedArray()) putInt(MediaStore.QUERY_ARG_MATCH_TRASHED, MediaStore.MATCH_ONLY) - }) - getAssets(cursor).forEach { res -> - if (res is AssetResult.ValidAsset) trashed += res.asset + } + + getCursor(volume, queryArgs).use { cursor -> + getAssets(cursor).forEach { res -> + if (res is AssetResult.ValidAsset) { + result.getOrPut(res.albumId) { mutableListOf() }.add(res.asset) + } + } } } - return trashed + return result.mapValues { it.value.toList() } } } diff --git a/mobile/ios/Runner/Sync/Messages.g.swift b/mobile/ios/Runner/Sync/Messages.g.swift index ea0ba53bef..26542e9910 100644 --- a/mobile/ios/Runner/Sync/Messages.g.swift +++ b/mobile/ios/Runner/Sync/Messages.g.swift @@ -355,7 +355,7 @@ class MessagesPigeonCodec: FlutterStandardMessageCodec, @unchecked Sendable { /// Generated protocol from Pigeon that represents a handler of messages from Flutter. protocol NativeSyncApi { func shouldFullSync() throws -> Bool - func getMediaChanges(isTrashed: Bool) throws -> SyncDelta + func getMediaChanges() throws -> SyncDelta func checkpointSync() throws func clearSyncCheckpoint() throws func getAssetIdsForAlbum(albumId: String) throws -> [String] @@ -364,7 +364,7 @@ protocol NativeSyncApi { func getAssetsForAlbum(albumId: String, updatedTimeCond: Int64?) throws -> [PlatformAsset] func hashAssets(assetIds: [String], allowNetworkAccess: Bool, completion: @escaping (Result<[HashResult], Error>) -> Void) func cancelHashing() throws - func getTrashedAssetsForAlbum(albumId: String) throws -> [PlatformAsset] + func getTrashedAssets(albumIds: [String], sinceLastCheckpoint: Bool) throws -> [String: [PlatformAsset]] } /// Generated setup class from Pigeon to handle messages through the `binaryMessenger`. @@ -395,11 +395,9 @@ class NativeSyncApiSetup { ? FlutterBasicMessageChannel(name: "dev.flutter.pigeon.immich_mobile.NativeSyncApi.getMediaChanges\(channelSuffix)", binaryMessenger: binaryMessenger, codec: codec) : FlutterBasicMessageChannel(name: "dev.flutter.pigeon.immich_mobile.NativeSyncApi.getMediaChanges\(channelSuffix)", binaryMessenger: binaryMessenger, codec: codec, taskQueue: taskQueue) if let api = api { - getMediaChangesChannel.setMessageHandler { message, reply in - let args = message as! [Any?] - let isTrashedArg = args[0] as! Bool + getMediaChangesChannel.setMessageHandler { _, reply in do { - let result = try api.getMediaChanges(isTrashed: isTrashedArg) + let result = try api.getMediaChanges() reply(wrapResult(result)) } catch { reply(wrapError(error)) @@ -535,22 +533,23 @@ class NativeSyncApiSetup { } else { cancelHashingChannel.setMessageHandler(nil) } - let getTrashedAssetsForAlbumChannel = taskQueue == nil - ? FlutterBasicMessageChannel(name: "dev.flutter.pigeon.immich_mobile.NativeSyncApi.getTrashedAssetsForAlbum\(channelSuffix)", binaryMessenger: binaryMessenger, codec: codec) - : FlutterBasicMessageChannel(name: "dev.flutter.pigeon.immich_mobile.NativeSyncApi.getTrashedAssetsForAlbum\(channelSuffix)", binaryMessenger: binaryMessenger, codec: codec, taskQueue: taskQueue) + let getTrashedAssetsChannel = taskQueue == nil + ? FlutterBasicMessageChannel(name: "dev.flutter.pigeon.immich_mobile.NativeSyncApi.getTrashedAssets\(channelSuffix)", binaryMessenger: binaryMessenger, codec: codec) + : FlutterBasicMessageChannel(name: "dev.flutter.pigeon.immich_mobile.NativeSyncApi.getTrashedAssets\(channelSuffix)", binaryMessenger: binaryMessenger, codec: codec, taskQueue: taskQueue) if let api = api { - getTrashedAssetsForAlbumChannel.setMessageHandler { message, reply in + getTrashedAssetsChannel.setMessageHandler { message, reply in let args = message as! [Any?] - let albumIdArg = args[0] as! String + let albumIdsArg = args[0] as! [String] + let sinceLastCheckpointArg = args[1] as! Bool do { - let result = try api.getTrashedAssetsForAlbum(albumId: albumIdArg) + let result = try api.getTrashedAssets(albumIds: albumIdsArg, sinceLastCheckpoint: sinceLastCheckpointArg) reply(wrapResult(result)) } catch { reply(wrapError(error)) } } } else { - getTrashedAssetsForAlbumChannel.setMessageHandler(nil) + getTrashedAssetsChannel.setMessageHandler(nil) } } } diff --git a/mobile/lib/domain/services/local_sync.service.dart b/mobile/lib/domain/services/local_sync.service.dart index 5ca5c43e9f..320e01eaa3 100644 --- a/mobile/lib/domain/services/local_sync.service.dart +++ b/mobile/lib/domain/services/local_sync.service.dart @@ -39,9 +39,7 @@ class LocalSyncService { } if (CurrentPlatform.isAndroid) { - final delta = await _nativeSyncApi.getMediaChanges(isTrashed: true); - _log.fine("Delta updated in trash: ${delta.updates.length - delta.updates.length}"); - await _applyTrashDelta(delta); + await _syncTrashedAssets(sinceLastCheckpoint: true); } final delta = await _nativeSyncApi.getMediaChanges(); @@ -98,10 +96,6 @@ class LocalSyncService { try { final Stopwatch stopwatch = Stopwatch()..start(); - if (CurrentPlatform.isAndroid) { - await _syncDeviceTrashSnapshot(); - } - final deviceAlbums = await _nativeSyncApi.getAlbums(); final dbAlbums = await _localAlbumRepository.getAll(sortBy: {SortLocalAlbumsBy.id}); @@ -113,6 +107,9 @@ class LocalSyncService { onlyFirst: removeAlbum, onlySecond: addAlbum, ); + if (CurrentPlatform.isAndroid) { + await _syncTrashedAssets(sinceLastCheckpoint: false); + } await _nativeSyncApi.checkpointSync(); stopwatch.stop(); @@ -293,39 +290,37 @@ class LocalSyncService { return a.name == b.name && a.assetCount == b.assetCount && a.updatedAt.isAtSameMomentAs(b.updatedAt); } - Future _applyTrashDelta(SyncDelta delta) async { - final trashUpdates = delta.updates; - if (trashUpdates.isEmpty) { - return Future.value(); - } - final trashedAssets = delta.toTrashedAssets(); - _log.info("updateLocalTrashChanges trashedAssets: ${trashedAssets.map((e) => e.asset.id)}"); - await _trashedLocalAssetRepository.applyDelta(trashedAssets); - await _applyRemoteRestoreToLocal(); - } - - Future _syncDeviceTrashSnapshot() async { + Future _syncTrashedAssets({required bool sinceLastCheckpoint}) async { final backupAlbums = await _localAlbumRepository.getBackupAlbums(); if (backupAlbums.isEmpty) { - _log.info("syncDeviceTrashSnapshot, No backup albums found"); + _log.info("syncTrashedAssets, No local backup albums found"); return; } - for (final album in backupAlbums) { - _log.info("syncDeviceTrashSnapshot prepare, album: ${album.id}/${album.name}"); - final trashedPlatformAssets = await _nativeSyncApi.getTrashedAssetsForAlbum(album.id); - final trashedAssets = trashedPlatformAssets.toTrashedAssets(album.id); - await _trashedLocalAssetRepository.applyTrashSnapshot(trashedAssets); + final albumIds = backupAlbums.map((e) => e.id).toList(); + final trashedAssetMap = await _nativeSyncApi.getTrashedAssets( + albumIds: albumIds, + sinceLastCheckpoint: sinceLastCheckpoint, + ); + if (trashedAssetMap.isEmpty) { + _log.info("syncTrashedAssets, No trashed assets found ${sinceLastCheckpoint ? "since Last Checkpoint" : ""}"); + } + final trashedAssets = trashedAssetMap.cast>().entries.expand( + (entry) => entry.value.cast().toTrashedAssets(entry.key), + ); + + _log.fine("syncTrashedAssets, trashedAssets: ${trashedAssets.map((e) => e.asset.id)}"); + if (sinceLastCheckpoint) { + await _trashedLocalAssetRepository.applyDelta(trashedAssets); + } else { + await _trashedLocalAssetRepository.applySnapshot(trashedAssets); } - await _applyRemoteRestoreToLocal(); - } - Future _applyRemoteRestoreToLocal() async { final remoteAssetsToRestore = await _trashedLocalAssetRepository.getToRestore(); if (remoteAssetsToRestore.isNotEmpty) { final restoredIds = await _localFilesManager.restoreAssetsFromTrash(remoteAssetsToRestore); await _trashedLocalAssetRepository.applyRestoredAssets(restoredIds); } else { - _log.info("No remote assets found for restoration"); + _log.info("syncTrashedAssets, No remote assets found for restoration"); } } } diff --git a/mobile/lib/infrastructure/repositories/trashed_local_asset.repository.dart b/mobile/lib/infrastructure/repositories/trashed_local_asset.repository.dart index dcc3a03d98..5e794a3f49 100644 --- a/mobile/lib/infrastructure/repositories/trashed_local_asset.repository.dart +++ b/mobile/lib/infrastructure/repositories/trashed_local_asset.repository.dart @@ -59,7 +59,7 @@ class DriftTrashedLocalAssetRepository extends DriftDatabaseRepository { /// Applies resulted snapshot of trashed assets: /// - upserts incoming rows /// - deletes rows that are not present in the snapshot - Future applyTrashSnapshot(Iterable trashedAssets) async { + Future applySnapshot(Iterable trashedAssets) async { if (trashedAssets.isEmpty) { await _db.delete(_db.trashedLocalAssetEntity).go(); return; diff --git a/mobile/lib/platform/native_sync_api.g.dart b/mobile/lib/platform/native_sync_api.g.dart index bb2600b236..1e7dbd30f3 100644 --- a/mobile/lib/platform/native_sync_api.g.dart +++ b/mobile/lib/platform/native_sync_api.g.dart @@ -326,7 +326,7 @@ class NativeSyncApi { } } - Future getMediaChanges({bool isTrashed = false}) async { + Future getMediaChanges() async { final String pigeonVar_channelName = 'dev.flutter.pigeon.immich_mobile.NativeSyncApi.getMediaChanges$pigeonVar_messageChannelSuffix'; final BasicMessageChannel pigeonVar_channel = BasicMessageChannel( @@ -334,7 +334,7 @@ class NativeSyncApi { pigeonChannelCodec, binaryMessenger: pigeonVar_binaryMessenger, ); - final Future pigeonVar_sendFuture = pigeonVar_channel.send([isTrashed]); + final Future pigeonVar_sendFuture = pigeonVar_channel.send(null); final List? pigeonVar_replyList = await pigeonVar_sendFuture as List?; if (pigeonVar_replyList == null) { throw _createConnectionError(pigeonVar_channelName); @@ -563,15 +563,18 @@ class NativeSyncApi { } } - Future> getTrashedAssetsForAlbum(String albumId) async { + Future>> getTrashedAssets({ + required List albumIds, + required bool sinceLastCheckpoint, + }) async { final String pigeonVar_channelName = - 'dev.flutter.pigeon.immich_mobile.NativeSyncApi.getTrashedAssetsForAlbum$pigeonVar_messageChannelSuffix'; + 'dev.flutter.pigeon.immich_mobile.NativeSyncApi.getTrashedAssets$pigeonVar_messageChannelSuffix'; final BasicMessageChannel pigeonVar_channel = BasicMessageChannel( pigeonVar_channelName, pigeonChannelCodec, binaryMessenger: pigeonVar_binaryMessenger, ); - final Future pigeonVar_sendFuture = pigeonVar_channel.send([albumId]); + final Future pigeonVar_sendFuture = pigeonVar_channel.send([albumIds, sinceLastCheckpoint]); final List? pigeonVar_replyList = await pigeonVar_sendFuture as List?; if (pigeonVar_replyList == null) { throw _createConnectionError(pigeonVar_channelName); @@ -587,7 +590,7 @@ class NativeSyncApi { message: 'Host platform returned null value for non-null return value.', ); } else { - return (pigeonVar_replyList[0] as List?)!.cast(); + return (pigeonVar_replyList[0] as Map?)!.cast>(); } } } diff --git a/mobile/pigeon/native_sync_api.dart b/mobile/pigeon/native_sync_api.dart index 2c6c725b39..1223d3057f 100644 --- a/mobile/pigeon/native_sync_api.dart +++ b/mobile/pigeon/native_sync_api.dart @@ -88,7 +88,7 @@ abstract class NativeSyncApi { bool shouldFullSync(); @TaskQueue(type: TaskQueueType.serialBackgroundThread) - SyncDelta getMediaChanges({bool isTrashed = false}); + SyncDelta getMediaChanges(); void checkpointSync(); @@ -113,5 +113,8 @@ abstract class NativeSyncApi { void cancelHashing(); @TaskQueue(type: TaskQueueType.serialBackgroundThread) - List getTrashedAssetsForAlbum(String albumId); + Map> getTrashedAssets({ + required List albumIds, + required bool sinceLastCheckpoint, + }); }