From 532ec10d5f8dacd9bc51ca544ecd99d3a4341ef8 Mon Sep 17 00:00:00 2001 From: shenlong <139912620+shenlong-tanwen@users.noreply.github.com> Date: Thu, 18 Sep 2025 10:12:37 +0530 Subject: [PATCH] refactor: hashing service (#21997) * download only backup selected assets * android impl * fix tests * limit concurrent hashing to 16 * extension cleanup * optimized hashing * hash only selected albums * remove concurrency limit * address review comments * log more info on failure * add native cancellation * small batch size on ios, large on android * fix: get correct resources * cleanup getResource * ios better hash cancellation * handle graceful cancellation android * do not trigger multiple hashing ops * ios: fix circular reference, improve cancellation * kotlin: more cancellation checks * no need to create result * cancel previous task * avoid race condition * ensure cancellation gets called * fix cancellation not happening --------- Co-authored-by: shenlong-tanwen <139912620+shalong-tanwen@users.noreply.github.com> Co-authored-by: mertalev <101130780+mertalev@users.noreply.github.com> Co-authored-by: Alex --- .../app/alextran/immich/sync/Messages.g.kt | 73 ++++++- .../alextran/immich/sync/MessagesImplBase.kt | 96 +++++++-- mobile/ios/.gitignore | 1 - .../Background/BackgroundWorkerApiImpl.swift | 4 +- mobile/ios/Runner/Sync/Messages.g.swift | 74 ++++++- mobile/ios/Runner/Sync/MessagesImpl.swift | 151 ++++++++++---- .../ios/Runner/Sync/PHAssetExtensions.swift | 77 +++++++ .../Sync/PHAssetResourceExtensions.swift | 16 ++ mobile/lib/constants/constants.dart | 4 +- .../services/background_worker.service.dart | 3 +- mobile/lib/domain/services/hash.service.dart | 100 ++++----- .../repositories/local_asset.repository.dart | 8 +- .../drift_backup_album_selection.page.dart | 36 +++- mobile/lib/platform/native_sync_api.g.dart | 75 ++++++- .../infrastructure/sync.provider.dart | 2 - mobile/lib/services/hash.service.dart | 5 +- mobile/pigeon/native_sync_api.dart | 13 +- .../domain/services/hash_service_test.dart | 193 +++++++----------- 18 files changed, 662 insertions(+), 269 deletions(-) create mode 100644 mobile/ios/Runner/Sync/PHAssetExtensions.swift create mode 100644 mobile/ios/Runner/Sync/PHAssetResourceExtensions.swift diff --git a/mobile/android/app/src/main/kotlin/app/alextran/immich/sync/Messages.g.kt b/mobile/android/app/src/main/kotlin/app/alextran/immich/sync/Messages.g.kt index 9c618d9ed0..28400c803f 100644 --- a/mobile/android/app/src/main/kotlin/app/alextran/immich/sync/Messages.g.kt +++ b/mobile/android/app/src/main/kotlin/app/alextran/immich/sync/Messages.g.kt @@ -209,6 +209,40 @@ data class SyncDelta ( override fun hashCode(): Int = toList().hashCode() } + +/** Generated class from Pigeon that represents data sent in messages. */ +data class HashResult ( + val assetId: String, + val error: String? = null, + val hash: String? = null +) + { + companion object { + fun fromList(pigeonVar_list: List): HashResult { + val assetId = pigeonVar_list[0] as String + val error = pigeonVar_list[1] as String? + val hash = pigeonVar_list[2] as String? + return HashResult(assetId, error, hash) + } + } + fun toList(): List { + return listOf( + assetId, + error, + hash, + ) + } + override fun equals(other: Any?): Boolean { + if (other !is HashResult) { + return false + } + if (this === other) { + return true + } + return MessagesPigeonUtils.deepEquals(toList(), other.toList()) } + + override fun hashCode(): Int = toList().hashCode() +} private open class MessagesPigeonCodec : StandardMessageCodec() { override fun readValueOfType(type: Byte, buffer: ByteBuffer): Any? { return when (type) { @@ -227,6 +261,11 @@ private open class MessagesPigeonCodec : StandardMessageCodec() { SyncDelta.fromList(it) } } + 132.toByte() -> { + return (readValue(buffer) as? List)?.let { + HashResult.fromList(it) + } + } else -> super.readValueOfType(type, buffer) } } @@ -244,11 +283,16 @@ private open class MessagesPigeonCodec : StandardMessageCodec() { stream.write(131) writeValue(stream, value.toList()) } + is HashResult -> { + stream.write(132) + writeValue(stream, value.toList()) + } else -> super.writeValue(stream, value) } } } + /** Generated interface from Pigeon that represents a handler of messages from Flutter. */ interface NativeSyncApi { fun shouldFullSync(): Boolean @@ -259,7 +303,8 @@ interface NativeSyncApi { fun getAlbums(): List fun getAssetsCountSince(albumId: String, timestamp: Long): Long fun getAssetsForAlbum(albumId: String, updatedTimeCond: Long?): List - fun hashPaths(paths: List): List + fun hashAssets(assetIds: List, allowNetworkAccess: Boolean, callback: (Result>) -> Unit) + fun cancelHashing() companion object { /** The codec used by NativeSyncApi. */ @@ -402,13 +447,33 @@ interface NativeSyncApi { } } run { - val channel = BasicMessageChannel(binaryMessenger, "dev.flutter.pigeon.immich_mobile.NativeSyncApi.hashPaths$separatedMessageChannelSuffix", codec, taskQueue) + val channel = BasicMessageChannel(binaryMessenger, "dev.flutter.pigeon.immich_mobile.NativeSyncApi.hashAssets$separatedMessageChannelSuffix", codec, taskQueue) if (api != null) { channel.setMessageHandler { message, reply -> val args = message as List - val pathsArg = args[0] as List + val assetIdsArg = args[0] as List + val allowNetworkAccessArg = args[1] as Boolean + api.hashAssets(assetIdsArg, allowNetworkAccessArg) { result: Result> -> + val error = result.exceptionOrNull() + if (error != null) { + reply.reply(MessagesPigeonUtils.wrapError(error)) + } else { + val data = result.getOrNull() + reply.reply(MessagesPigeonUtils.wrapResult(data)) + } + } + } + } else { + channel.setMessageHandler(null) + } + } + run { + val channel = BasicMessageChannel(binaryMessenger, "dev.flutter.pigeon.immich_mobile.NativeSyncApi.cancelHashing$separatedMessageChannelSuffix", codec) + if (api != null) { + channel.setMessageHandler { _, reply -> val wrapped: List = try { - listOf(api.hashPaths(pathsArg)) + api.cancelHashing() + listOf(null) } catch (exception: Throwable) { MessagesPigeonUtils.wrapError(exception) } diff --git a/mobile/android/app/src/main/kotlin/app/alextran/immich/sync/MessagesImplBase.kt b/mobile/android/app/src/main/kotlin/app/alextran/immich/sync/MessagesImplBase.kt index b2ceb8a9f2..868f3c6cdd 100644 --- a/mobile/android/app/src/main/kotlin/app/alextran/immich/sync/MessagesImplBase.kt +++ b/mobile/android/app/src/main/kotlin/app/alextran/immich/sync/MessagesImplBase.kt @@ -1,14 +1,25 @@ package app.alextran.immich.sync import android.annotation.SuppressLint +import android.content.ContentUris import android.content.Context import android.database.Cursor import android.provider.MediaStore -import android.util.Log +import android.util.Base64 import androidx.core.database.getStringOrNull +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.ensureActive +import kotlinx.coroutines.launch +import kotlinx.coroutines.sync.Semaphore +import kotlinx.coroutines.sync.withPermit import java.io.File -import java.io.FileInputStream import java.security.MessageDigest +import kotlin.coroutines.cancellation.CancellationException +import kotlin.coroutines.coroutineContext sealed class AssetResult { data class ValidAsset(val asset: PlatformAsset, val albumId: String) : AssetResult() @@ -19,8 +30,12 @@ sealed class AssetResult { open class NativeSyncApiImplBase(context: Context) { private val ctx: Context = context.applicationContext + private var hashTask: Job? = null + companion object { - private const val TAG = "NativeSyncApiImplBase" + private const val MAX_CONCURRENT_HASH_OPERATIONS = 16 + private val hashSemaphore = Semaphore(MAX_CONCURRENT_HASH_OPERATIONS) + private const val HASHING_CANCELLED_CODE = "HASH_CANCELLED" const val MEDIA_SELECTION = "(${MediaStore.Files.FileColumns.MEDIA_TYPE} = ? OR ${MediaStore.Files.FileColumns.MEDIA_TYPE} = ?)" @@ -215,23 +230,74 @@ open class NativeSyncApiImplBase(context: Context) { .toList() } - fun hashPaths(paths: List): List { - val buffer = ByteArray(HASH_BUFFER_SIZE) - val digest = MessageDigest.getInstance("SHA-1") + fun hashAssets( + assetIds: List, + // allowNetworkAccess is only used on the iOS implementation + @Suppress("UNUSED_PARAMETER") allowNetworkAccess: Boolean, + callback: (Result>) -> Unit + ) { + if (assetIds.isEmpty()) { + callback(Result.success(emptyList())) + return + } - return paths.map { path -> + hashTask?.cancel() + hashTask = CoroutineScope(Dispatchers.IO).launch { try { - FileInputStream(path).use { file -> - var bytesRead: Int - while (file.read(buffer).also { bytesRead = it } > 0) { - digest.update(buffer, 0, bytesRead) + val results = assetIds.map { assetId -> + async { + hashSemaphore.withPermit { + ensureActive() + hashAsset(assetId) + } } - } - digest.digest() + }.awaitAll() + + callback(Result.success(results)) + } catch (e: CancellationException) { + callback( + Result.failure( + FlutterError( + HASHING_CANCELLED_CODE, + "Hashing operation was cancelled", + null + ) + ) + ) } catch (e: Exception) { - Log.w(TAG, "Failed to hash file $path: $e") - null + callback(Result.failure(e)) } } } + + private suspend fun hashAsset(assetId: String): HashResult { + return try { + val assetUri = ContentUris.withAppendedId( + MediaStore.Files.getContentUri(MediaStore.VOLUME_EXTERNAL), + assetId.toLong() + ) + + val digest = MessageDigest.getInstance("SHA-1") + ctx.contentResolver.openInputStream(assetUri)?.use { inputStream -> + var bytesRead: Int + val buffer = ByteArray(HASH_BUFFER_SIZE) + while (inputStream.read(buffer).also { bytesRead = it } > 0) { + coroutineContext.ensureActive() + digest.update(buffer, 0, bytesRead) + } + } ?: return HashResult(assetId, "Cannot open input stream for asset", null) + + val hashString = Base64.encodeToString(digest.digest(), Base64.NO_WRAP) + HashResult(assetId, null, hashString) + } catch (e: SecurityException) { + HashResult(assetId, "Permission denied accessing asset: ${e.message}", null) + } catch (e: Exception) { + HashResult(assetId, "Failed to hash asset: ${e.message}", null) + } + } + + fun cancelHashing() { + hashTask?.cancel() + hashTask = null + } } diff --git a/mobile/ios/.gitignore b/mobile/ios/.gitignore index f312f249a3..e32cadbf68 100644 --- a/mobile/ios/.gitignore +++ b/mobile/ios/.gitignore @@ -4,7 +4,6 @@ *.moved-aside *.pbxuser *.perspectivev3 -**/*sync/ .sconsign.dblite .tags* **/.vagrant/ diff --git a/mobile/ios/Runner/Background/BackgroundWorkerApiImpl.swift b/mobile/ios/Runner/Background/BackgroundWorkerApiImpl.swift index ca2453404c..f7f8f69989 100644 --- a/mobile/ios/Runner/Background/BackgroundWorkerApiImpl.swift +++ b/mobile/ios/Runner/Background/BackgroundWorkerApiImpl.swift @@ -5,7 +5,7 @@ class BackgroundWorkerApiImpl: BackgroundWorkerFgHostApi { func enable() throws { BackgroundWorkerApiImpl.scheduleRefreshWorker() BackgroundWorkerApiImpl.scheduleProcessingWorker() - print("BackgroundUploadImpl:enbale Background worker scheduled") + print("BackgroundWorkerApiImpl:enable Background worker scheduled") } func configure(settings: BackgroundWorkerSettings) throws { @@ -15,7 +15,7 @@ class BackgroundWorkerApiImpl: BackgroundWorkerFgHostApi { func disable() throws { BGTaskScheduler.shared.cancel(taskRequestWithIdentifier: BackgroundWorkerApiImpl.refreshTaskID); BGTaskScheduler.shared.cancel(taskRequestWithIdentifier: BackgroundWorkerApiImpl.processingTaskID); - print("BackgroundUploadImpl:disableUploadWorker Disabled background workers") + print("BackgroundWorkerApiImpl:disableUploadWorker Disabled background workers") } private static let refreshTaskID = "app.alextran.immich.background.refreshUpload" diff --git a/mobile/ios/Runner/Sync/Messages.g.swift b/mobile/ios/Runner/Sync/Messages.g.swift index 19f4384672..305aca5266 100644 --- a/mobile/ios/Runner/Sync/Messages.g.swift +++ b/mobile/ios/Runner/Sync/Messages.g.swift @@ -267,6 +267,39 @@ struct SyncDelta: Hashable { } } +/// Generated class from Pigeon that represents data sent in messages. +struct HashResult: Hashable { + var assetId: String + var error: String? = nil + var hash: String? = nil + + + // swift-format-ignore: AlwaysUseLowerCamelCase + static func fromList(_ pigeonVar_list: [Any?]) -> HashResult? { + let assetId = pigeonVar_list[0] as! String + let error: String? = nilOrValue(pigeonVar_list[1]) + let hash: String? = nilOrValue(pigeonVar_list[2]) + + return HashResult( + assetId: assetId, + error: error, + hash: hash + ) + } + func toList() -> [Any?] { + return [ + assetId, + error, + hash, + ] + } + static func == (lhs: HashResult, rhs: HashResult) -> Bool { + return deepEqualsMessages(lhs.toList(), rhs.toList()) } + func hash(into hasher: inout Hasher) { + deepHashMessages(value: toList(), hasher: &hasher) + } +} + private class MessagesPigeonCodecReader: FlutterStandardReader { override func readValue(ofType type: UInt8) -> Any? { switch type { @@ -276,6 +309,8 @@ private class MessagesPigeonCodecReader: FlutterStandardReader { return PlatformAlbum.fromList(self.readValue() as! [Any?]) case 131: return SyncDelta.fromList(self.readValue() as! [Any?]) + case 132: + return HashResult.fromList(self.readValue() as! [Any?]) default: return super.readValue(ofType: type) } @@ -293,6 +328,9 @@ private class MessagesPigeonCodecWriter: FlutterStandardWriter { } else if let value = value as? SyncDelta { super.writeByte(131) super.writeValue(value.toList()) + } else if let value = value as? HashResult { + super.writeByte(132) + super.writeValue(value.toList()) } else { super.writeValue(value) } @@ -313,6 +351,7 @@ class MessagesPigeonCodec: FlutterStandardMessageCodec, @unchecked Sendable { static let shared = MessagesPigeonCodec(readerWriter: MessagesPigeonCodecReaderWriter()) } + /// Generated protocol from Pigeon that represents a handler of messages from Flutter. protocol NativeSyncApi { func shouldFullSync() throws -> Bool @@ -323,7 +362,8 @@ protocol NativeSyncApi { func getAlbums() throws -> [PlatformAlbum] func getAssetsCountSince(albumId: String, timestamp: Int64) throws -> Int64 func getAssetsForAlbum(albumId: String, updatedTimeCond: Int64?) throws -> [PlatformAsset] - func hashPaths(paths: [String]) throws -> [FlutterStandardTypedData?] + func hashAssets(assetIds: [String], allowNetworkAccess: Bool, completion: @escaping (Result<[HashResult], Error>) -> Void) + func cancelHashing() throws } /// Generated setup class from Pigeon to handle messages through the `binaryMessenger`. @@ -459,22 +499,38 @@ class NativeSyncApiSetup { } else { getAssetsForAlbumChannel.setMessageHandler(nil) } - let hashPathsChannel = taskQueue == nil - ? FlutterBasicMessageChannel(name: "dev.flutter.pigeon.immich_mobile.NativeSyncApi.hashPaths\(channelSuffix)", binaryMessenger: binaryMessenger, codec: codec) - : FlutterBasicMessageChannel(name: "dev.flutter.pigeon.immich_mobile.NativeSyncApi.hashPaths\(channelSuffix)", binaryMessenger: binaryMessenger, codec: codec, taskQueue: taskQueue) + let hashAssetsChannel = taskQueue == nil + ? FlutterBasicMessageChannel(name: "dev.flutter.pigeon.immich_mobile.NativeSyncApi.hashAssets\(channelSuffix)", binaryMessenger: binaryMessenger, codec: codec) + : FlutterBasicMessageChannel(name: "dev.flutter.pigeon.immich_mobile.NativeSyncApi.hashAssets\(channelSuffix)", binaryMessenger: binaryMessenger, codec: codec, taskQueue: taskQueue) if let api = api { - hashPathsChannel.setMessageHandler { message, reply in + hashAssetsChannel.setMessageHandler { message, reply in let args = message as! [Any?] - let pathsArg = args[0] as! [String] + let assetIdsArg = args[0] as! [String] + let allowNetworkAccessArg = args[1] as! Bool + api.hashAssets(assetIds: assetIdsArg, allowNetworkAccess: allowNetworkAccessArg) { result in + switch result { + case .success(let res): + reply(wrapResult(res)) + case .failure(let error): + reply(wrapError(error)) + } + } + } + } else { + hashAssetsChannel.setMessageHandler(nil) + } + let cancelHashingChannel = FlutterBasicMessageChannel(name: "dev.flutter.pigeon.immich_mobile.NativeSyncApi.cancelHashing\(channelSuffix)", binaryMessenger: binaryMessenger, codec: codec) + if let api = api { + cancelHashingChannel.setMessageHandler { _, reply in do { - let result = try api.hashPaths(paths: pathsArg) - reply(wrapResult(result)) + try api.cancelHashing() + reply(wrapResult(nil)) } catch { reply(wrapError(error)) } } } else { - hashPathsChannel.setMessageHandler(nil) + cancelHashingChannel.setMessageHandler(nil) } } } diff --git a/mobile/ios/Runner/Sync/MessagesImpl.swift b/mobile/ios/Runner/Sync/MessagesImpl.swift index 2810dee7c1..bb23bae6b6 100644 --- a/mobile/ios/Runner/Sync/MessagesImpl.swift +++ b/mobile/ios/Runner/Sync/MessagesImpl.swift @@ -17,30 +17,16 @@ struct AssetWrapper: Hashable, Equatable { } } -extension PHAsset { - func toPlatformAsset() -> PlatformAsset { - return PlatformAsset( - id: localIdentifier, - name: title(), - type: Int64(mediaType.rawValue), - createdAt: creationDate.map { Int64($0.timeIntervalSince1970) }, - updatedAt: modificationDate.map { Int64($0.timeIntervalSince1970) }, - width: Int64(pixelWidth), - height: Int64(pixelHeight), - durationInSeconds: Int64(duration), - orientation: 0, - isFavorite: isFavorite - ) - } -} - class NativeSyncApiImpl: NativeSyncApi { private let defaults: UserDefaults private let changeTokenKey = "immich:changeToken" private let albumTypes: [PHAssetCollectionType] = [.album, .smartAlbum] private let recoveredAlbumSubType = 1000000219 - private let hashBufferSize = 2 * 1024 * 1024 + private var hashTask: Task? + private static let hashCancelledCode = "HASH_CANCELLED" + private static let hashCancelled = Result<[HashResult], Error>.failure(PigeonError(code: hashCancelledCode, message: "Hashing cancelled", details: nil)) + init(with defaults: UserDefaults = .standard) { self.defaults = defaults @@ -96,7 +82,7 @@ class NativeSyncApiImpl: NativeSyncApi { let collections = PHAssetCollection.fetchAssetCollections(with: type, subtype: .any, options: nil) for i in 0.. %@ OR modificationDate > %@", date, date) } - + let result = PHAsset.fetchAssets(in: album, options: options) if(result.count == 0) { return [] @@ -267,23 +253,114 @@ class NativeSyncApiImpl: NativeSyncApi { return assets } - func hashPaths(paths: [String]) throws -> [FlutterStandardTypedData?] { - return paths.map { path in - guard let file = FileHandle(forReadingAtPath: path) else { - print("Cannot open file: \(path)") - return nil - } - - var hasher = Insecure.SHA1() - while autoreleasepool(invoking: { - let chunk = file.readData(ofLength: hashBufferSize) - guard !chunk.isEmpty else { return false } - hasher.update(data: chunk) - return true - }) { } - - let digest = hasher.finalize() - return FlutterStandardTypedData(bytes: Data(digest)) + func hashAssets(assetIds: [String], allowNetworkAccess: Bool, completion: @escaping (Result<[HashResult], Error>) -> Void) { + if let prevTask = hashTask { + prevTask.cancel() + hashTask = nil + } + hashTask = Task { [weak self] in + var missingAssetIds = Set(assetIds) + var assets = [PHAsset]() + assets.reserveCapacity(assetIds.count) + PHAsset.fetchAssets(withLocalIdentifiers: assetIds, options: nil).enumerateObjects { (asset, _, stop) in + if Task.isCancelled { + stop.pointee = true + return + } + missingAssetIds.remove(asset.localIdentifier) + assets.append(asset) } + + if Task.isCancelled { + return completion(Self.hashCancelled) + } + + await withTaskGroup(of: HashResult?.self) { taskGroup in + var results = [HashResult]() + results.reserveCapacity(assets.count) + for asset in assets { + if Task.isCancelled { + return completion(Self.hashCancelled) + } + taskGroup.addTask { + guard let self = self else { return nil } + return await self.hashAsset(asset, allowNetworkAccess: allowNetworkAccess) + } + } + + for await result in taskGroup { + guard let result = result else { + return completion(Self.hashCancelled) + } + results.append(result) + } + + for missing in missingAssetIds { + results.append(HashResult(assetId: missing, error: "Asset not found in library", hash: nil)) + } + + completion(.success(results)) + } + } + } + + func cancelHashing() { + hashTask?.cancel() + hashTask = nil + } + + private func hashAsset(_ asset: PHAsset, allowNetworkAccess: Bool) async -> HashResult? { + class RequestRef { + var id: PHAssetResourceDataRequestID? + } + let requestRef = RequestRef() + return await withTaskCancellationHandler(operation: { + if Task.isCancelled { + return nil + } + + guard let resource = asset.getResource() else { + return HashResult(assetId: asset.localIdentifier, error: "Cannot get asset resource", hash: nil) + } + + if Task.isCancelled { + return nil + } + + let options = PHAssetResourceRequestOptions() + options.isNetworkAccessAllowed = allowNetworkAccess + + return await withCheckedContinuation { continuation in + var hasher = Insecure.SHA1() + + requestRef.id = PHAssetResourceManager.default().requestData( + for: resource, + options: options, + dataReceivedHandler: { data in + hasher.update(data: data) + }, + completionHandler: { error in + let result: HashResult? = switch (error) { + case let e as PHPhotosError where e.code == .userCancelled: nil + case let .some(e): HashResult( + assetId: asset.localIdentifier, + error: "Failed to hash asset: \(e.localizedDescription)", + hash: nil + ) + case .none: + HashResult( + assetId: asset.localIdentifier, + error: nil, + hash: Data(hasher.finalize()).base64EncodedString() + ) + } + continuation.resume(returning: result) + } + ) + } + }, onCancel: { + guard let requestId = requestRef.id else { return } + PHAssetResourceManager.default().cancelDataRequest(requestId) + }) } } diff --git a/mobile/ios/Runner/Sync/PHAssetExtensions.swift b/mobile/ios/Runner/Sync/PHAssetExtensions.swift new file mode 100644 index 0000000000..2b1ef6ac88 --- /dev/null +++ b/mobile/ios/Runner/Sync/PHAssetExtensions.swift @@ -0,0 +1,77 @@ +import Photos + +extension PHAsset { + func toPlatformAsset() -> PlatformAsset { + return PlatformAsset( + id: localIdentifier, + name: title, + type: Int64(mediaType.rawValue), + createdAt: creationDate.map { Int64($0.timeIntervalSince1970) }, + updatedAt: modificationDate.map { Int64($0.timeIntervalSince1970) }, + width: Int64(pixelWidth), + height: Int64(pixelHeight), + durationInSeconds: Int64(duration), + orientation: 0, + isFavorite: isFavorite + ) + } + + var title: String { + return filename ?? originalFilename ?? "" + } + + var filename: String? { + return value(forKey: "filename") as? String + } + + // This method is expected to be slow as it goes through the asset resources to fetch the originalFilename + var originalFilename: String? { + return getResource()?.originalFilename + } + + func getResource() -> PHAssetResource? { + let resources = PHAssetResource.assetResources(for: self) + + let filteredResources = resources.filter { $0.isMediaResource && isValidResourceType($0.type) } + + guard !filteredResources.isEmpty else { + return nil + } + + if filteredResources.count == 1 { + return filteredResources.first + } + + if let currentResource = filteredResources.first(where: { $0.isCurrent }) { + return currentResource + } + + if let fullSizeResource = filteredResources.first(where: { isFullSizeResourceType($0.type) }) { + return fullSizeResource + } + + return nil + } + + private func isValidResourceType(_ type: PHAssetResourceType) -> Bool { + switch mediaType { + case .image: + return [.photo, .alternatePhoto, .fullSizePhoto].contains(type) + case .video: + return [.video, .fullSizeVideo, .fullSizePairedVideo].contains(type) + default: + return false + } + } + + private func isFullSizeResourceType(_ type: PHAssetResourceType) -> Bool { + switch mediaType { + case .image: + return type == .fullSizePhoto + case .video: + return type == .fullSizeVideo + default: + return false + } + } +} diff --git a/mobile/ios/Runner/Sync/PHAssetResourceExtensions.swift b/mobile/ios/Runner/Sync/PHAssetResourceExtensions.swift new file mode 100644 index 0000000000..699d55a98d --- /dev/null +++ b/mobile/ios/Runner/Sync/PHAssetResourceExtensions.swift @@ -0,0 +1,16 @@ + +import Photos + +extension PHAssetResource { + var isCurrent: Bool { + return value(forKey: "isCurrent") as? Bool ?? false + } + + var isMediaResource: Bool { + var isMedia = type != .adjustmentData + if #available(iOS 17, *) { + isMedia = isMedia && type != .photoProxy + } + return isMedia + } +} diff --git a/mobile/lib/constants/constants.dart b/mobile/lib/constants/constants.dart index 60c9b92cf9..7429616f14 100644 --- a/mobile/lib/constants/constants.dart +++ b/mobile/lib/constants/constants.dart @@ -1,3 +1,5 @@ +import 'dart:io'; + const int noDbId = -9223372036854775808; // from Isar const double downloadCompleted = -1; const double downloadFailed = -2; @@ -10,7 +12,7 @@ const int kSyncEventBatchSize = 5000; const int kFetchLocalAssetsBatchSize = 40000; // Hash batch limits -const int kBatchHashFileLimit = 256; +final int kBatchHashFileLimit = Platform.isIOS ? 32 : 512; const int kBatchHashSizeLimit = 1024 * 1024 * 1024; // 1GB // Secure storage keys diff --git a/mobile/lib/domain/services/background_worker.service.dart b/mobile/lib/domain/services/background_worker.service.dart index d57fe507d9..78dd1e980f 100644 --- a/mobile/lib/domain/services/background_worker.service.dart +++ b/mobile/lib/domain/services/background_worker.service.dart @@ -184,6 +184,7 @@ class BackgroundWorkerBgService extends BackgroundWorkerFlutterApi { try { final backgroundSyncManager = _ref.read(backgroundSyncProvider); + final nativeSyncApi = _ref.read(nativeSyncApiProvider); _isCleanedUp = true; _ref.dispose(); @@ -199,7 +200,7 @@ class BackgroundWorkerBgService extends BackgroundWorkerFlutterApi { _drift.close(), _driftLogger.close(), backgroundSyncManager.cancel(), - backgroundSyncManager.cancelLocal(), + nativeSyncApi.cancelHashing(), ]; if (_isar.isOpen) { diff --git a/mobile/lib/domain/services/hash.service.dart b/mobile/lib/domain/services/hash.service.dart index 8044b298d3..90f29b8bc1 100644 --- a/mobile/lib/domain/services/hash.service.dart +++ b/mobile/lib/domain/services/hash.service.dart @@ -1,20 +1,18 @@ -import 'dart:convert'; - +import 'package:flutter/services.dart'; import 'package:immich_mobile/constants/constants.dart'; import 'package:immich_mobile/domain/models/album/local_album.model.dart'; import 'package:immich_mobile/domain/models/asset/base_asset.model.dart'; import 'package:immich_mobile/infrastructure/repositories/local_album.repository.dart'; import 'package:immich_mobile/infrastructure/repositories/local_asset.repository.dart'; -import 'package:immich_mobile/infrastructure/repositories/storage.repository.dart'; import 'package:immich_mobile/platform/native_sync_api.g.dart'; import 'package:logging/logging.dart'; +const String _kHashCancelledCode = "HASH_CANCELLED"; + class HashService { - final int batchSizeLimit; - final int batchFileLimit; + final int _batchSize; final DriftLocalAlbumRepository _localAlbumRepository; final DriftLocalAssetRepository _localAssetRepository; - final StorageRepository _storageRepository; final NativeSyncApi _nativeSyncApi; final bool Function()? _cancelChecker; final _log = Logger('HashService'); @@ -22,37 +20,42 @@ class HashService { HashService({ required DriftLocalAlbumRepository localAlbumRepository, required DriftLocalAssetRepository localAssetRepository, - required StorageRepository storageRepository, required NativeSyncApi nativeSyncApi, bool Function()? cancelChecker, - this.batchSizeLimit = kBatchHashSizeLimit, - this.batchFileLimit = kBatchHashFileLimit, + int? batchSize, }) : _localAlbumRepository = localAlbumRepository, _localAssetRepository = localAssetRepository, - _storageRepository = storageRepository, _cancelChecker = cancelChecker, - _nativeSyncApi = nativeSyncApi; + _nativeSyncApi = nativeSyncApi, + _batchSize = batchSize ?? kBatchHashFileLimit; bool get isCancelled => _cancelChecker?.call() ?? false; Future hashAssets() async { _log.info("Starting hashing of assets"); final Stopwatch stopwatch = Stopwatch()..start(); - // Sorted by backupSelection followed by isCloud - final localAlbums = await _localAlbumRepository.getAll( - sortBy: {SortLocalAlbumsBy.backupSelection, SortLocalAlbumsBy.isIosSharedAlbum}, - ); + try { + // Sorted by backupSelection followed by isCloud + final localAlbums = await _localAlbumRepository.getBackupAlbums(); - for (final album in localAlbums) { - if (isCancelled) { - _log.warning("Hashing cancelled. Stopped processing albums."); - break; - } + for (final album in localAlbums) { + if (isCancelled) { + _log.warning("Hashing cancelled. Stopped processing albums."); + break; + } - final assetsToHash = await _localAlbumRepository.getAssetsToHash(album.id); - if (assetsToHash.isNotEmpty) { - await _hashAssets(album, assetsToHash); + final assetsToHash = await _localAlbumRepository.getAssetsToHash(album.id); + if (assetsToHash.isNotEmpty) { + await _hashAssets(album, assetsToHash); + } } + } on PlatformException catch (e) { + if (e.code == _kHashCancelledCode) { + _log.warning("Hashing cancelled by platform"); + return; + } + } catch (e, s) { + _log.severe("Error during hashing", e, s); } stopwatch.stop(); @@ -63,8 +66,7 @@ class HashService { /// with hash for those that were successfully hashed. Hashes are looked up in a table /// [LocalAssetHashEntity] by local id. Only missing entries are newly hashed and added to the DB. Future _hashAssets(LocalAlbum album, List assetsToHash) async { - int bytesProcessed = 0; - final toHash = <_AssetToPath>[]; + final toHash = {}; for (final asset in assetsToHash) { if (isCancelled) { @@ -72,21 +74,10 @@ class HashService { return; } - final file = await _storageRepository.getFileForAsset(asset.id); - if (file == null) { - _log.warning( - "Cannot get file for asset ${asset.id}, name: ${asset.name}, created on: ${asset.createdAt} from album: ${album.name}", - ); - continue; - } - - bytesProcessed += await file.length(); - toHash.add(_AssetToPath(asset: asset, path: file.path)); - - if (toHash.length >= batchFileLimit || bytesProcessed >= batchSizeLimit) { + toHash[asset.id] = asset; + if (toHash.length == _batchSize) { await _processBatch(album, toHash); toHash.clear(); - bytesProcessed = 0; } } @@ -94,33 +85,36 @@ class HashService { } /// Processes a batch of assets. - Future _processBatch(LocalAlbum album, List<_AssetToPath> toHash) async { + Future _processBatch(LocalAlbum album, Map toHash) async { if (toHash.isEmpty) { return; } _log.fine("Hashing ${toHash.length} files"); - final hashed = []; - final hashes = await _nativeSyncApi.hashPaths(toHash.map((e) => e.path).toList()); + final hashed = {}; + final hashResults = await _nativeSyncApi.hashAssets( + toHash.keys.toList(), + allowNetworkAccess: album.backupSelection == BackupSelection.selected, + ); assert( - hashes.length == toHash.length, - "Hashes length does not match toHash length: ${hashes.length} != ${toHash.length}", + hashResults.length == toHash.length, + "Hashes length does not match toHash length: ${hashResults.length} != ${toHash.length}", ); - for (int i = 0; i < hashes.length; i++) { + for (int i = 0; i < hashResults.length; i++) { if (isCancelled) { _log.warning("Hashing cancelled. Stopped processing batch."); return; } - final hash = hashes[i]; - final asset = toHash[i].asset; - if (hash?.length == 20) { - hashed.add(asset.copyWith(checksum: base64.encode(hash!))); + final hashResult = hashResults[i]; + if (hashResult.hash != null) { + hashed[hashResult.assetId] = hashResult.hash!; } else { + final asset = toHash[hashResult.assetId]; _log.warning( - "Failed to hash file for ${asset.id}: ${asset.name} created at ${asset.createdAt} from album: ${album.name}", + "Failed to hash asset with id: ${hashResult.assetId}, name: ${asset?.name}, createdAt: ${asset?.createdAt}, from album: ${album.name}. Error: ${hashResult.error ?? "unknown"}", ); } } @@ -128,13 +122,5 @@ class HashService { _log.fine("Hashed ${hashed.length}/${toHash.length} assets"); await _localAssetRepository.updateHashes(hashed); - await _storageRepository.clearCache(); } } - -class _AssetToPath { - final LocalAsset asset; - final String path; - - const _AssetToPath({required this.asset, required this.path}); -} diff --git a/mobile/lib/infrastructure/repositories/local_asset.repository.dart b/mobile/lib/infrastructure/repositories/local_asset.repository.dart index 05c8e06678..c4f5221966 100644 --- a/mobile/lib/infrastructure/repositories/local_asset.repository.dart +++ b/mobile/lib/infrastructure/repositories/local_asset.repository.dart @@ -36,17 +36,17 @@ class DriftLocalAssetRepository extends DriftDatabaseRepository { Stream watch(String id) => _assetSelectable(id).watchSingleOrNull(); - Future updateHashes(Iterable hashes) { + Future updateHashes(Map hashes) { if (hashes.isEmpty) { return Future.value(); } return _db.batch((batch) async { - for (final asset in hashes) { + for (final entry in hashes.entries) { batch.update( _db.localAssetEntity, - LocalAssetEntityCompanion(checksum: Value(asset.checksum)), - where: (e) => e.id.equals(asset.id), + LocalAssetEntityCompanion(checksum: Value(entry.value)), + where: (e) => e.id.equals(entry.key), ); } }); diff --git a/mobile/lib/pages/backup/drift_backup_album_selection.page.dart b/mobile/lib/pages/backup/drift_backup_album_selection.page.dart index 8cfdd2db95..368341f24a 100644 --- a/mobile/lib/pages/backup/drift_backup_album_selection.page.dart +++ b/mobile/lib/pages/backup/drift_backup_album_selection.page.dart @@ -1,3 +1,4 @@ +import 'dart:async'; import 'dart:io'; import 'package:auto_route/auto_route.dart'; @@ -5,12 +6,14 @@ import 'package:easy_localization/easy_localization.dart'; import 'package:flutter/material.dart'; import 'package:hooks_riverpod/hooks_riverpod.dart'; import 'package:immich_mobile/domain/models/album/local_album.model.dart'; +import 'package:immich_mobile/domain/services/sync_linked_album.service.dart'; import 'package:immich_mobile/extensions/build_context_extensions.dart'; import 'package:immich_mobile/extensions/translate_extensions.dart'; import 'package:immich_mobile/providers/app_settings.provider.dart'; -import 'package:immich_mobile/domain/services/sync_linked_album.service.dart'; +import 'package:immich_mobile/providers/background_sync.provider.dart'; import 'package:immich_mobile/providers/backup/backup_album.provider.dart'; import 'package:immich_mobile/providers/backup/drift_backup.provider.dart'; +import 'package:immich_mobile/providers/infrastructure/platform.provider.dart'; import 'package:immich_mobile/providers/user.provider.dart'; import 'package:immich_mobile/services/app_settings.service.dart'; import 'package:immich_mobile/widgets/backup/drift_album_info_list_tile.dart'; @@ -64,16 +67,6 @@ class _DriftBackupAlbumSelectionPageState extends ConsumerState p.totalCount)); - final totalChanged = currentTotalAssetCount != _initialTotalAssetCount; - final isBackupEnabled = ref.read(appSettingsServiceProvider).getSetting(AppSettingsEnum.enableBackup); - - if (totalChanged && isBackupEnabled) { - await ref.read(driftBackupProvider.notifier).cancel(); - await ref.read(driftBackupProvider.notifier).startBackup(user.id); - } } @override @@ -102,6 +95,27 @@ class _DriftBackupAlbumSelectionPageState extends ConsumerState p.totalCount)); + final totalChanged = currentTotalAssetCount != _initialTotalAssetCount; + final backupNotifier = ref.read(driftBackupProvider.notifier); + final backgroundSync = ref.read(backgroundSyncProvider); + final nativeSync = ref.read(nativeSyncApiProvider); + if (totalChanged) { + // Waits for hashing to be cancelled before starting a new one + unawaited(nativeSync.cancelHashing().whenComplete(() => backgroundSync.hashAssets())); + if (isBackupEnabled) { + unawaited(backupNotifier.cancel().whenComplete(() => backupNotifier.startBackup(user.id))); + } + } + Navigator.of(context).pop(); } }, diff --git a/mobile/lib/platform/native_sync_api.g.dart b/mobile/lib/platform/native_sync_api.g.dart index 6fc96f5046..01237f8c19 100644 --- a/mobile/lib/platform/native_sync_api.g.dart +++ b/mobile/lib/platform/native_sync_api.g.dart @@ -205,6 +205,45 @@ class SyncDelta { int get hashCode => Object.hashAll(_toList()); } +class HashResult { + HashResult({required this.assetId, this.error, this.hash}); + + String assetId; + + String? error; + + String? hash; + + List _toList() { + return [assetId, error, hash]; + } + + Object encode() { + return _toList(); + } + + static HashResult decode(Object result) { + result as List; + return HashResult(assetId: result[0]! as String, error: result[1] as String?, hash: result[2] as String?); + } + + @override + // ignore: avoid_equals_and_hash_code_on_mutable_classes + bool operator ==(Object other) { + if (other is! HashResult || other.runtimeType != runtimeType) { + return false; + } + if (identical(this, other)) { + return true; + } + return _deepEquals(encode(), other.encode()); + } + + @override + // ignore: avoid_equals_and_hash_code_on_mutable_classes + int get hashCode => Object.hashAll(_toList()); +} + class _PigeonCodec extends StandardMessageCodec { const _PigeonCodec(); @override @@ -221,6 +260,9 @@ class _PigeonCodec extends StandardMessageCodec { } else if (value is SyncDelta) { buffer.putUint8(131); writeValue(buffer, value.encode()); + } else if (value is HashResult) { + buffer.putUint8(132); + writeValue(buffer, value.encode()); } else { super.writeValue(buffer, value); } @@ -235,6 +277,8 @@ class _PigeonCodec extends StandardMessageCodec { return PlatformAlbum.decode(readValue(buffer)!); case 131: return SyncDelta.decode(readValue(buffer)!); + case 132: + return HashResult.decode(readValue(buffer)!); default: return super.readValueOfType(type, buffer); } @@ -468,15 +512,15 @@ class NativeSyncApi { } } - Future> hashPaths(List paths) async { + Future> hashAssets(List assetIds, {bool allowNetworkAccess = false}) async { final String pigeonVar_channelName = - 'dev.flutter.pigeon.immich_mobile.NativeSyncApi.hashPaths$pigeonVar_messageChannelSuffix'; + 'dev.flutter.pigeon.immich_mobile.NativeSyncApi.hashAssets$pigeonVar_messageChannelSuffix'; final BasicMessageChannel pigeonVar_channel = BasicMessageChannel( pigeonVar_channelName, pigeonChannelCodec, binaryMessenger: pigeonVar_binaryMessenger, ); - final Future pigeonVar_sendFuture = pigeonVar_channel.send([paths]); + final Future pigeonVar_sendFuture = pigeonVar_channel.send([assetIds, allowNetworkAccess]); final List? pigeonVar_replyList = await pigeonVar_sendFuture as List?; if (pigeonVar_replyList == null) { throw _createConnectionError(pigeonVar_channelName); @@ -492,7 +536,30 @@ class NativeSyncApi { message: 'Host platform returned null value for non-null return value.', ); } else { - return (pigeonVar_replyList[0] as List?)!.cast(); + return (pigeonVar_replyList[0] as List?)!.cast(); + } + } + + Future cancelHashing() async { + final String pigeonVar_channelName = + 'dev.flutter.pigeon.immich_mobile.NativeSyncApi.cancelHashing$pigeonVar_messageChannelSuffix'; + final BasicMessageChannel pigeonVar_channel = BasicMessageChannel( + pigeonVar_channelName, + pigeonChannelCodec, + binaryMessenger: pigeonVar_binaryMessenger, + ); + final Future pigeonVar_sendFuture = pigeonVar_channel.send(null); + final List? pigeonVar_replyList = await pigeonVar_sendFuture as List?; + if (pigeonVar_replyList == null) { + throw _createConnectionError(pigeonVar_channelName); + } else if (pigeonVar_replyList.length > 1) { + throw PlatformException( + code: pigeonVar_replyList[0]! as String, + message: pigeonVar_replyList[1] as String?, + details: pigeonVar_replyList[2], + ); + } else { + return; } } } diff --git a/mobile/lib/providers/infrastructure/sync.provider.dart b/mobile/lib/providers/infrastructure/sync.provider.dart index ddc6eed441..f03754505c 100644 --- a/mobile/lib/providers/infrastructure/sync.provider.dart +++ b/mobile/lib/providers/infrastructure/sync.provider.dart @@ -10,7 +10,6 @@ import 'package:immich_mobile/providers/infrastructure/asset.provider.dart'; import 'package:immich_mobile/providers/infrastructure/cancel.provider.dart'; import 'package:immich_mobile/providers/infrastructure/db.provider.dart'; import 'package:immich_mobile/providers/infrastructure/platform.provider.dart'; -import 'package:immich_mobile/providers/infrastructure/storage.provider.dart'; final syncStreamServiceProvider = Provider( (ref) => SyncStreamService( @@ -35,7 +34,6 @@ final hashServiceProvider = Provider( (ref) => HashService( localAlbumRepository: ref.watch(localAlbumRepository), localAssetRepository: ref.watch(localAssetRepository), - storageRepository: ref.watch(storageRepositoryProvider), nativeSyncApi: ref.watch(nativeSyncApiProvider), ), ); diff --git a/mobile/lib/services/hash.service.dart b/mobile/lib/services/hash.service.dart index 48302be79c..9d1f4e51e8 100644 --- a/mobile/lib/services/hash.service.dart +++ b/mobile/lib/services/hash.service.dart @@ -16,9 +16,10 @@ class HashService { required IsarDeviceAssetRepository deviceAssetRepository, required BackgroundService backgroundService, this.batchSizeLimit = kBatchHashSizeLimit, - this.batchFileLimit = kBatchHashFileLimit, + int? batchFileLimit, }) : _deviceAssetRepository = deviceAssetRepository, - _backgroundService = backgroundService; + _backgroundService = backgroundService, + batchFileLimit = batchFileLimit ?? kBatchHashFileLimit; final IsarDeviceAssetRepository _deviceAssetRepository; final BackgroundService _backgroundService; diff --git a/mobile/pigeon/native_sync_api.dart b/mobile/pigeon/native_sync_api.dart index e84c814c3d..ac08a68ca3 100644 --- a/mobile/pigeon/native_sync_api.dart +++ b/mobile/pigeon/native_sync_api.dart @@ -71,6 +71,14 @@ class SyncDelta { }); } +class HashResult { + final String assetId; + final String? error; + final String? hash; + + const HashResult({required this.assetId, this.error, this.hash}); +} + @HostApi() abstract class NativeSyncApi { bool shouldFullSync(); @@ -94,6 +102,9 @@ abstract class NativeSyncApi { @TaskQueue(type: TaskQueueType.serialBackgroundThread) List getAssetsForAlbum(String albumId, {int? updatedTimeCond}); + @async @TaskQueue(type: TaskQueueType.serialBackgroundThread) - List hashPaths(List paths); + List hashAssets(List assetIds, {bool allowNetworkAccess = false}); + + void cancelHashing(); } diff --git a/mobile/test/domain/services/hash_service_test.dart b/mobile/test/domain/services/hash_service_test.dart index 7969131e7f..20d60b6866 100644 --- a/mobile/test/domain/services/hash_service_test.dart +++ b/mobile/test/domain/services/hash_service_test.dart @@ -1,11 +1,7 @@ -import 'dart:convert'; -import 'dart:io'; -import 'dart:typed_data'; - import 'package:flutter_test/flutter_test.dart'; -import 'package:immich_mobile/domain/models/asset/base_asset.model.dart'; +import 'package:immich_mobile/domain/models/album/local_album.model.dart'; import 'package:immich_mobile/domain/services/hash.service.dart'; -import 'package:immich_mobile/infrastructure/repositories/local_album.repository.dart'; +import 'package:immich_mobile/platform/native_sync_api.g.dart'; import 'package:mocktail/mocktail.dart'; import '../../fixtures/album.stub.dart'; @@ -13,192 +9,137 @@ import '../../fixtures/asset.stub.dart'; import '../../infrastructure/repository.mock.dart'; import '../service.mock.dart'; -class MockFile extends Mock implements File {} - void main() { late HashService sut; late MockLocalAlbumRepository mockAlbumRepo; late MockLocalAssetRepository mockAssetRepo; - late MockStorageRepository mockStorageRepo; late MockNativeSyncApi mockNativeApi; - final sortBy = {SortLocalAlbumsBy.backupSelection, SortLocalAlbumsBy.isIosSharedAlbum}; setUp(() { mockAlbumRepo = MockLocalAlbumRepository(); mockAssetRepo = MockLocalAssetRepository(); - mockStorageRepo = MockStorageRepository(); mockNativeApi = MockNativeSyncApi(); sut = HashService( localAlbumRepository: mockAlbumRepo, localAssetRepository: mockAssetRepo, - storageRepository: mockStorageRepo, nativeSyncApi: mockNativeApi, ); registerFallbackValue(LocalAlbumStub.recent); registerFallbackValue(LocalAssetStub.image1); + registerFallbackValue({}); when(() => mockAssetRepo.updateHashes(any())).thenAnswer((_) async => {}); - when(() => mockStorageRepo.clearCache()).thenAnswer((_) async => {}); }); group('HashService hashAssets', () { test('skips albums with no assets to hash', () async { when( - () => mockAlbumRepo.getAll(sortBy: sortBy), + () => mockAlbumRepo.getBackupAlbums(), ).thenAnswer((_) async => [LocalAlbumStub.recent.copyWith(assetCount: 0)]); when(() => mockAlbumRepo.getAssetsToHash(LocalAlbumStub.recent.id)).thenAnswer((_) async => []); await sut.hashAssets(); - verifyNever(() => mockStorageRepo.getFileForAsset(any())); - verifyNever(() => mockNativeApi.hashPaths(any())); + verifyNever(() => mockNativeApi.hashAssets(any(), allowNetworkAccess: any(named: 'allowNetworkAccess'))); }); }); group('HashService _hashAssets', () { - test('skips assets without files', () async { + test('skips empty batches', () async { final album = LocalAlbumStub.recent; - final asset = LocalAssetStub.image1; - when(() => mockAlbumRepo.getAll(sortBy: sortBy)).thenAnswer((_) async => [album]); - when(() => mockAlbumRepo.getAssetsToHash(album.id)).thenAnswer((_) async => [asset]); - when(() => mockStorageRepo.getFileForAsset(asset.id)).thenAnswer((_) async => null); + when(() => mockAlbumRepo.getBackupAlbums()).thenAnswer((_) async => [album]); + when(() => mockAlbumRepo.getAssetsToHash(album.id)).thenAnswer((_) async => []); await sut.hashAssets(); - verifyNever(() => mockNativeApi.hashPaths(any())); + verifyNever(() => mockNativeApi.hashAssets(any(), allowNetworkAccess: any(named: 'allowNetworkAccess'))); }); test('processes assets when available', () async { final album = LocalAlbumStub.recent; final asset = LocalAssetStub.image1; - final mockFile = MockFile(); - final hash = Uint8List.fromList(List.generate(20, (i) => i)); - when(() => mockFile.length()).thenAnswer((_) async => 1000); - when(() => mockFile.path).thenReturn('image-path'); - - when(() => mockAlbumRepo.getAll(sortBy: sortBy)).thenAnswer((_) async => [album]); + when(() => mockAlbumRepo.getBackupAlbums()).thenAnswer((_) async => [album]); when(() => mockAlbumRepo.getAssetsToHash(album.id)).thenAnswer((_) async => [asset]); - when(() => mockStorageRepo.getFileForAsset(asset.id)).thenAnswer((_) async => mockFile); - when(() => mockNativeApi.hashPaths(['image-path'])).thenAnswer((_) async => [hash]); + when( + () => mockNativeApi.hashAssets([asset.id], allowNetworkAccess: false), + ).thenAnswer((_) async => [HashResult(assetId: asset.id, hash: 'test-hash')]); await sut.hashAssets(); - verify(() => mockNativeApi.hashPaths(['image-path'])).called(1); - final captured = verify(() => mockAssetRepo.updateHashes(captureAny())).captured.first as List; + verify(() => mockNativeApi.hashAssets([asset.id], allowNetworkAccess: false)).called(1); + final captured = verify(() => mockAssetRepo.updateHashes(captureAny())).captured.first as Map; expect(captured.length, 1); - expect(captured[0].checksum, base64.encode(hash)); + expect(captured[asset.id], 'test-hash'); }); test('handles failed hashes', () async { final album = LocalAlbumStub.recent; final asset = LocalAssetStub.image1; - final mockFile = MockFile(); - when(() => mockFile.length()).thenAnswer((_) async => 1000); - when(() => mockFile.path).thenReturn('image-path'); - when(() => mockAlbumRepo.getAll(sortBy: sortBy)).thenAnswer((_) async => [album]); + when(() => mockAlbumRepo.getBackupAlbums()).thenAnswer((_) async => [album]); when(() => mockAlbumRepo.getAssetsToHash(album.id)).thenAnswer((_) async => [asset]); - when(() => mockStorageRepo.getFileForAsset(asset.id)).thenAnswer((_) async => mockFile); - when(() => mockNativeApi.hashPaths(['image-path'])).thenAnswer((_) async => [null]); - when(() => mockAssetRepo.updateHashes(any())).thenAnswer((_) async => {}); + when( + () => mockNativeApi.hashAssets([asset.id], allowNetworkAccess: false), + ).thenAnswer((_) async => [HashResult(assetId: asset.id, error: 'Failed to hash')]); await sut.hashAssets(); - final captured = verify(() => mockAssetRepo.updateHashes(captureAny())).captured.first as List; + final captured = verify(() => mockAssetRepo.updateHashes(captureAny())).captured.first as Map; expect(captured.length, 0); }); - test('handles invalid hash length', () async { + test('handles null hash results', () async { final album = LocalAlbumStub.recent; final asset = LocalAssetStub.image1; - final mockFile = MockFile(); - when(() => mockFile.length()).thenAnswer((_) async => 1000); - when(() => mockFile.path).thenReturn('image-path'); - when(() => mockAlbumRepo.getAll(sortBy: sortBy)).thenAnswer((_) async => [album]); + when(() => mockAlbumRepo.getBackupAlbums()).thenAnswer((_) async => [album]); when(() => mockAlbumRepo.getAssetsToHash(album.id)).thenAnswer((_) async => [asset]); - when(() => mockStorageRepo.getFileForAsset(asset.id)).thenAnswer((_) async => mockFile); - - final invalidHash = Uint8List.fromList([1, 2, 3]); - when(() => mockNativeApi.hashPaths(['image-path'])).thenAnswer((_) async => [invalidHash]); - when(() => mockAssetRepo.updateHashes(any())).thenAnswer((_) async => {}); + when( + () => mockNativeApi.hashAssets([asset.id], allowNetworkAccess: false), + ).thenAnswer((_) async => [HashResult(assetId: asset.id, hash: null)]); await sut.hashAssets(); - final captured = verify(() => mockAssetRepo.updateHashes(captureAny())).captured.first as List; + final captured = verify(() => mockAssetRepo.updateHashes(captureAny())).captured.first as Map; expect(captured.length, 0); }); - test('batches by file count limit', () async { - final sut = HashService( - localAlbumRepository: mockAlbumRepo, - localAssetRepository: mockAssetRepo, - storageRepository: mockStorageRepo, - nativeSyncApi: mockNativeApi, - batchFileLimit: 1, - ); - - final album = LocalAlbumStub.recent; - final asset1 = LocalAssetStub.image1; - final asset2 = LocalAssetStub.image2; - final mockFile1 = MockFile(); - final mockFile2 = MockFile(); - when(() => mockFile1.length()).thenAnswer((_) async => 100); - when(() => mockFile1.path).thenReturn('path-1'); - when(() => mockFile2.length()).thenAnswer((_) async => 100); - when(() => mockFile2.path).thenReturn('path-2'); - - when(() => mockAlbumRepo.getAll(sortBy: sortBy)).thenAnswer((_) async => [album]); - when(() => mockAlbumRepo.getAssetsToHash(album.id)).thenAnswer((_) async => [asset1, asset2]); - when(() => mockStorageRepo.getFileForAsset(asset1.id)).thenAnswer((_) async => mockFile1); - when(() => mockStorageRepo.getFileForAsset(asset2.id)).thenAnswer((_) async => mockFile2); - - final hash = Uint8List.fromList(List.generate(20, (i) => i)); - when(() => mockNativeApi.hashPaths(any())).thenAnswer((_) async => [hash]); - when(() => mockAssetRepo.updateHashes(any())).thenAnswer((_) async => {}); - - await sut.hashAssets(); - - verify(() => mockNativeApi.hashPaths(['path-1'])).called(1); - verify(() => mockNativeApi.hashPaths(['path-2'])).called(1); - verify(() => mockAssetRepo.updateHashes(any())).called(2); - }); - test('batches by size limit', () async { + const batchSize = 2; final sut = HashService( localAlbumRepository: mockAlbumRepo, localAssetRepository: mockAssetRepo, - storageRepository: mockStorageRepo, nativeSyncApi: mockNativeApi, - batchSizeLimit: 80, + batchSize: batchSize, ); final album = LocalAlbumStub.recent; final asset1 = LocalAssetStub.image1; final asset2 = LocalAssetStub.image2; - final mockFile1 = MockFile(); - final mockFile2 = MockFile(); - when(() => mockFile1.length()).thenAnswer((_) async => 100); - when(() => mockFile1.path).thenReturn('path-1'); - when(() => mockFile2.length()).thenAnswer((_) async => 100); - when(() => mockFile2.path).thenReturn('path-2'); + final asset3 = LocalAssetStub.image1.copyWith(id: 'image3', name: 'image3.jpg'); - when(() => mockAlbumRepo.getAll(sortBy: sortBy)).thenAnswer((_) async => [album]); - when(() => mockAlbumRepo.getAssetsToHash(album.id)).thenAnswer((_) async => [asset1, asset2]); - when(() => mockStorageRepo.getFileForAsset(asset1.id)).thenAnswer((_) async => mockFile1); - when(() => mockStorageRepo.getFileForAsset(asset2.id)).thenAnswer((_) async => mockFile2); + final capturedCalls = >[]; - final hash = Uint8List.fromList(List.generate(20, (i) => i)); - when(() => mockNativeApi.hashPaths(any())).thenAnswer((_) async => [hash]); when(() => mockAssetRepo.updateHashes(any())).thenAnswer((_) async => {}); + when(() => mockAlbumRepo.getBackupAlbums()).thenAnswer((_) async => [album]); + when(() => mockAlbumRepo.getAssetsToHash(album.id)).thenAnswer((_) async => [asset1, asset2, asset3]); + when(() => mockNativeApi.hashAssets(any(), allowNetworkAccess: any(named: 'allowNetworkAccess'))).thenAnswer(( + invocation, + ) async { + final assetIds = invocation.positionalArguments[0] as List; + capturedCalls.add(List.from(assetIds)); + return assetIds.map((id) => HashResult(assetId: id, hash: '$id-hash')).toList(); + }); await sut.hashAssets(); - verify(() => mockNativeApi.hashPaths(['path-1'])).called(1); - verify(() => mockNativeApi.hashPaths(['path-2'])).called(1); + expect(capturedCalls.length, 2, reason: 'Should make exactly 2 calls to hashAssets'); + expect(capturedCalls[0], [asset1.id, asset2.id], reason: 'First call should batch the first two assets'); + expect(capturedCalls[1], [asset3.id], reason: 'Second call should have the remaining asset'); + verify(() => mockAssetRepo.updateHashes(any())).called(2); }); @@ -206,27 +147,43 @@ void main() { final album = LocalAlbumStub.recent; final asset1 = LocalAssetStub.image1; final asset2 = LocalAssetStub.image2; - final mockFile1 = MockFile(); - final mockFile2 = MockFile(); - when(() => mockFile1.length()).thenAnswer((_) async => 100); - when(() => mockFile1.path).thenReturn('path-1'); - when(() => mockFile2.length()).thenAnswer((_) async => 100); - when(() => mockFile2.path).thenReturn('path-2'); - when(() => mockAlbumRepo.getAll(sortBy: sortBy)).thenAnswer((_) async => [album]); + when(() => mockAlbumRepo.getBackupAlbums()).thenAnswer((_) async => [album]); when(() => mockAlbumRepo.getAssetsToHash(album.id)).thenAnswer((_) async => [asset1, asset2]); - when(() => mockStorageRepo.getFileForAsset(asset1.id)).thenAnswer((_) async => mockFile1); - when(() => mockStorageRepo.getFileForAsset(asset2.id)).thenAnswer((_) async => mockFile2); - - final validHash = Uint8List.fromList(List.generate(20, (i) => i)); - when(() => mockNativeApi.hashPaths(['path-1', 'path-2'])).thenAnswer((_) async => [validHash, null]); - when(() => mockAssetRepo.updateHashes(any())).thenAnswer((_) async => {}); + when(() => mockNativeApi.hashAssets([asset1.id, asset2.id], allowNetworkAccess: false)).thenAnswer( + (_) async => [ + HashResult(assetId: asset1.id, hash: 'asset1-hash'), + HashResult(assetId: asset2.id, error: 'Failed to hash asset2'), + ], + ); await sut.hashAssets(); - final captured = verify(() => mockAssetRepo.updateHashes(captureAny())).captured.first as List; + final captured = verify(() => mockAssetRepo.updateHashes(captureAny())).captured.first as Map; expect(captured.length, 1); - expect(captured.first.id, asset1.id); + expect(captured[asset1.id], 'asset1-hash'); + }); + + test('uses allowNetworkAccess based on album backup selection', () async { + final selectedAlbum = LocalAlbumStub.recent.copyWith(backupSelection: BackupSelection.selected); + final nonSelectedAlbum = LocalAlbumStub.recent.copyWith(id: 'album2', backupSelection: BackupSelection.excluded); + final asset1 = LocalAssetStub.image1; + final asset2 = LocalAssetStub.image2; + + when(() => mockAlbumRepo.getBackupAlbums()).thenAnswer((_) async => [selectedAlbum, nonSelectedAlbum]); + when(() => mockAlbumRepo.getAssetsToHash(selectedAlbum.id)).thenAnswer((_) async => [asset1]); + when(() => mockAlbumRepo.getAssetsToHash(nonSelectedAlbum.id)).thenAnswer((_) async => [asset2]); + when(() => mockNativeApi.hashAssets(any(), allowNetworkAccess: any(named: 'allowNetworkAccess'))).thenAnswer(( + invocation, + ) async { + final assetIds = invocation.positionalArguments[0] as List; + return assetIds.map((id) => HashResult(assetId: id, hash: '$id-hash')).toList(); + }); + + await sut.hashAssets(); + + verify(() => mockNativeApi.hashAssets([asset1.id], allowNetworkAccess: true)).called(1); + verify(() => mockNativeApi.hashAssets([asset2.id], allowNetworkAccess: false)).called(1); }); }); }