From f931060670ea84d3c22fd8a70fdf3c04cf20765f Mon Sep 17 00:00:00 2001 From: mertalev <101130780+mertalev@users.noreply.github.com> Date: Wed, 13 Aug 2025 00:10:55 -0400 Subject: [PATCH] image provider improvements --- mobile/android/app/CMakeLists.txt | 9 + mobile/android/app/build.gradle | 6 + .../android/app/src/main/cpp/native_buffer.c | 52 ++++ .../app/alextran/immich/AppGlideModule.kt | 10 +- .../alextran/immich/images/Thumbnails.g.kt | 117 +++++++++ .../alextran/immich/images/ThumbnailsImpl.kt | 236 ++++++++++++++++++ mobile/ios/Runner.xcodeproj/project.pbxproj | 16 ++ mobile/ios/Runner/AppDelegate.swift | 1 + mobile/ios/Runner/Images/Thumbnails.g.swift | 119 +++++++++ mobile/ios/Runner/Images/ThumbnailsImpl.swift | 177 +++++++++++++ mobile/ios/Runner/Info.plist | 2 +- mobile/lib/constants/constants.dart | 3 +- .../lib/extensions/datetime_extensions.dart | 10 + .../repositories/asset_media.repository.dart | 221 ++++++++++++++-- mobile/lib/main.dart | 1 + mobile/lib/platform/thumbnail_api.g.dart | 107 ++++++++ .../widgets/images/image_provider.dart | 23 +- .../widgets/images/local_image_provider.dart | 158 +++--------- .../widgets/images/remote_image_provider.dart | 93 +++---- .../widgets/timeline/constants.dart | 2 +- .../infrastructure/platform.provider.dart | 3 + .../photo_view/src/core/photo_view_core.dart | 1 + mobile/makefile | 2 + mobile/pigeon/thumbnail_api.dart | 21 ++ mobile/pubspec.lock | 2 +- mobile/pubspec.yaml | 1 + 26 files changed, 1203 insertions(+), 190 deletions(-) create mode 100644 mobile/android/app/CMakeLists.txt create mode 100644 mobile/android/app/src/main/cpp/native_buffer.c create mode 100644 mobile/android/app/src/main/kotlin/app/alextran/immich/images/Thumbnails.g.kt create mode 100644 mobile/android/app/src/main/kotlin/app/alextran/immich/images/ThumbnailsImpl.kt create mode 100644 mobile/ios/Runner/Images/Thumbnails.g.swift create mode 100644 mobile/ios/Runner/Images/ThumbnailsImpl.swift create mode 100644 mobile/lib/platform/thumbnail_api.g.dart create mode 100644 mobile/pigeon/thumbnail_api.dart diff --git a/mobile/android/app/CMakeLists.txt b/mobile/android/app/CMakeLists.txt new file mode 100644 index 0000000000..5493754141 --- /dev/null +++ b/mobile/android/app/CMakeLists.txt @@ -0,0 +1,9 @@ +cmake_minimum_required(VERSION 3.10.2) +project("native_buffer") + +add_library(native_buffer SHARED + src/main/cpp/native_buffer.c) + +find_library(log-lib log) + +target_link_libraries(native_buffer ${log-lib}) diff --git a/mobile/android/app/build.gradle b/mobile/android/app/build.gradle index 1f0e2e7675..418e9be77e 100644 --- a/mobile/android/app/build.gradle +++ b/mobile/android/app/build.gradle @@ -85,6 +85,12 @@ android { namespace 'app.alextran.immich' } +externalNativeBuild { + cmake { + path "CMakeLists.txt" + } +} + flutter { source '../..' } diff --git a/mobile/android/app/src/main/cpp/native_buffer.c b/mobile/android/app/src/main/cpp/native_buffer.c new file mode 100644 index 0000000000..2f1d76d187 --- /dev/null +++ b/mobile/android/app/src/main/cpp/native_buffer.c @@ -0,0 +1,52 @@ +#include +#include + +JNIEXPORT jlong JNICALL +Java_app_alextran_immich_images_ThumbnailsImpl_00024Companion_allocateNative( + JNIEnv *env, jclass clazz, jint size) +{ + void *ptr = malloc(size); + return (jlong)ptr; +} + +JNIEXPORT jlong JNICALL +Java_app_alextran_immich_images_ThumbnailsImpl_allocateNative( + JNIEnv *env, jclass clazz, jint size) +{ + void *ptr = malloc(size); + return (jlong)ptr; +} + +JNIEXPORT void JNICALL +Java_app_alextran_immich_images_ThumbnailsImpl_00024Companion_freeNative( + JNIEnv *env, jclass clazz, jlong address) +{ + if (address != 0) + { + free((void *)address); + } +} + +JNIEXPORT void JNICALL +Java_app_alextran_immich_images_ThumbnailsImpl_freeNative( + JNIEnv *env, jclass clazz, jlong address) +{ + if (address != 0) + { + free((void *)address); + } +} + +JNIEXPORT jobject JNICALL +Java_app_alextran_immich_images_ThumbnailsImpl_00024Companion_wrapAsBuffer( + JNIEnv *env, jclass clazz, jlong address, jint capacity) +{ + return (*env)->NewDirectByteBuffer(env, (void*)address, capacity); +} + +JNIEXPORT jobject JNICALL +Java_app_alextran_immich_images_ThumbnailsImpl_wrapAsBuffer( + JNIEnv *env, jclass clazz, jlong address, jint capacity) +{ + return (*env)->NewDirectByteBuffer(env, (void*)address, capacity); +} diff --git a/mobile/android/app/src/main/kotlin/app/alextran/immich/AppGlideModule.kt b/mobile/android/app/src/main/kotlin/app/alextran/immich/AppGlideModule.kt index f969b9576f..23555eee81 100644 --- a/mobile/android/app/src/main/kotlin/app/alextran/immich/AppGlideModule.kt +++ b/mobile/android/app/src/main/kotlin/app/alextran/immich/AppGlideModule.kt @@ -4,4 +4,12 @@ import com.bumptech.glide.annotation.GlideModule import com.bumptech.glide.module.AppGlideModule @GlideModule -class AppGlideModule : AppGlideModule() \ No newline at end of file +class AppGlideModule : AppGlideModule() { + override fun applyOptions(context: Context, builder: GlideBuilder) { + super.applyOptions(context, builder) + // disable caching as this is already done on the Flutter side + builder.setMemoryCache(MemoryCacheAdapter()) + builder.setDiskCache(DiskCacheAdapter.Factory()) + builder.setBitmapPool(BitmapPoolAdapter()) + } +} diff --git a/mobile/android/app/src/main/kotlin/app/alextran/immich/images/Thumbnails.g.kt b/mobile/android/app/src/main/kotlin/app/alextran/immich/images/Thumbnails.g.kt new file mode 100644 index 0000000000..4866fb648f --- /dev/null +++ b/mobile/android/app/src/main/kotlin/app/alextran/immich/images/Thumbnails.g.kt @@ -0,0 +1,117 @@ +// Autogenerated from Pigeon (v26.0.0), do not edit directly. +// See also: https://pub.dev/packages/pigeon +@file:Suppress("UNCHECKED_CAST", "ArrayInDataClass") + +package app.alextran.immich.images + +import android.util.Log +import io.flutter.plugin.common.BasicMessageChannel +import io.flutter.plugin.common.BinaryMessenger +import io.flutter.plugin.common.EventChannel +import io.flutter.plugin.common.MessageCodec +import io.flutter.plugin.common.StandardMethodCodec +import io.flutter.plugin.common.StandardMessageCodec +import java.io.ByteArrayOutputStream +import java.nio.ByteBuffer +private object ThumbnailsPigeonUtils { + + fun wrapResult(result: Any?): List { + return listOf(result) + } + + fun wrapError(exception: Throwable): List { + return if (exception is FlutterError) { + listOf( + exception.code, + exception.message, + exception.details + ) + } else { + listOf( + exception.javaClass.simpleName, + exception.toString(), + "Cause: " + exception.cause + ", Stacktrace: " + Log.getStackTraceString(exception) + ) + } + } +} + +/** + * Error class for passing custom error details to Flutter via a thrown PlatformException. + * @property code The error code. + * @property message The error message. + * @property details The error details. Must be a datatype supported by the api codec. + */ +class FlutterError ( + val code: String, + override val message: String? = null, + val details: Any? = null +) : Throwable() +private open class ThumbnailsPigeonCodec : StandardMessageCodec() { + override fun readValueOfType(type: Byte, buffer: ByteBuffer): Any? { + return super.readValueOfType(type, buffer) + } + override fun writeValue(stream: ByteArrayOutputStream, value: Any?) { + super.writeValue(stream, value) + } +} + + +/** Generated interface from Pigeon that represents a handler of messages from Flutter. */ +interface ThumbnailApi { + fun requestImage(assetId: String, requestId: Long, width: Long, height: Long, callback: (Result>) -> Unit) + fun cancelImageRequest(requestId: Long) + + companion object { + /** The codec used by ThumbnailApi. */ + val codec: MessageCodec by lazy { + ThumbnailsPigeonCodec() + } + /** Sets up an instance of `ThumbnailApi` to handle messages through the `binaryMessenger`. */ + @JvmOverloads + fun setUp(binaryMessenger: BinaryMessenger, api: ThumbnailApi?, messageChannelSuffix: String = "") { + val separatedMessageChannelSuffix = if (messageChannelSuffix.isNotEmpty()) ".$messageChannelSuffix" else "" + run { + val channel = BasicMessageChannel(binaryMessenger, "dev.flutter.pigeon.immich_mobile.ThumbnailApi.requestImage$separatedMessageChannelSuffix", codec) + if (api != null) { + channel.setMessageHandler { message, reply -> + val args = message as List + val assetIdArg = args[0] as String + val requestIdArg = args[1] as Long + val widthArg = args[2] as Long + val heightArg = args[3] as Long + api.requestImage(assetIdArg, requestIdArg, widthArg, heightArg) { result: Result> -> + val error = result.exceptionOrNull() + if (error != null) { + reply.reply(ThumbnailsPigeonUtils.wrapError(error)) + } else { + val data = result.getOrNull() + reply.reply(ThumbnailsPigeonUtils.wrapResult(data)) + } + } + } + } else { + channel.setMessageHandler(null) + } + } + run { + val channel = BasicMessageChannel(binaryMessenger, "dev.flutter.pigeon.immich_mobile.ThumbnailApi.cancelImageRequest$separatedMessageChannelSuffix", codec) + if (api != null) { + channel.setMessageHandler { message, reply -> + val args = message as List + val requestIdArg = args[0] as Long + val wrapped: List = try { + api.cancelImageRequest(requestIdArg) + listOf(null) + } catch (exception: Throwable) { + ThumbnailsPigeonUtils.wrapError(exception) + } + reply.reply(wrapped) + } + } else { + channel.setMessageHandler(null) + } + } + } + } +} diff --git a/mobile/android/app/src/main/kotlin/app/alextran/immich/images/ThumbnailsImpl.kt b/mobile/android/app/src/main/kotlin/app/alextran/immich/images/ThumbnailsImpl.kt new file mode 100644 index 0000000000..168e504961 --- /dev/null +++ b/mobile/android/app/src/main/kotlin/app/alextran/immich/images/ThumbnailsImpl.kt @@ -0,0 +1,236 @@ +package app.alextran.immich.images + +import android.content.ContentResolver +import android.content.ContentUris +import android.content.Context +import android.graphics.* +import android.net.Uri +import android.os.Build +import android.os.CancellationSignal +import android.os.OperationCanceledException +import android.provider.MediaStore +import android.provider.MediaStore.Images +import android.provider.MediaStore.Video +import android.util.Size +import java.nio.ByteBuffer +import kotlin.math.* +import java.util.concurrent.Executors +import com.bumptech.glide.Glide +import com.bumptech.glide.Priority +import com.bumptech.glide.load.DecodeFormat +import java.util.HashMap +import java.util.concurrent.CancellationException +import java.util.concurrent.Future + +data class Request( + val requestId: Long, + val taskFuture: Future<*>, + val cancellationSignal: CancellationSignal, + val callback: (Result>) -> Unit +) + +class ThumbnailsImpl(context: Context) : ThumbnailApi { + private val ctx: Context = context.applicationContext + private val resolver: ContentResolver = ctx.contentResolver + private val threadPool = + Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors() / 2 + 1) + private val requestMap = HashMap() + + companion object { + val PROJECTION = arrayOf( + MediaStore.MediaColumns.DATE_MODIFIED, + MediaStore.Files.FileColumns.MEDIA_TYPE, + ) + const val SELECTION = "${MediaStore.MediaColumns._ID} = ?" + val URI: Uri = MediaStore.Files.getContentUri("external") + + const val MEDIA_TYPE_IMAGE = MediaStore.Files.FileColumns.MEDIA_TYPE_IMAGE + const val MEDIA_TYPE_VIDEO = MediaStore.Files.FileColumns.MEDIA_TYPE_VIDEO + val CANCELLED = Result.success>(mapOf()) + val OPTIONS = BitmapFactory.Options().apply { inPreferredConfig = Bitmap.Config.ARGB_8888 } + + init { + System.loadLibrary("native_buffer") + } + + @JvmStatic + external fun allocateNative(size: Int): Long + + @JvmStatic + external fun freeNative(pointer: Long) + + @JvmStatic + external fun wrapAsBuffer(address: Long, capacity: Int): ByteBuffer + } + + override fun requestImage( + assetId: String, + requestId: Long, + width: Long, + height: Long, + callback: (Result>) -> Unit + ) { + val signal = CancellationSignal() + val task = threadPool.submit { + try { + getThumbnailBufferInternal(assetId, width, height, callback, signal) + } catch (e: Exception) { + when (e) { + is OperationCanceledException -> callback(CANCELLED) + is CancellationException -> callback(CANCELLED) + else -> callback(Result.failure(e)) + } + } finally { + requestMap.remove(requestId) + } + } + requestMap[requestId] = Request(requestId, task, signal, callback) + } + + override fun cancelImageRequest(requestId: Long) { + val request = requestMap.remove(requestId) ?: return + request.taskFuture.cancel(false) + request.cancellationSignal.cancel() + if (request.taskFuture.isCancelled) { + request.callback(CANCELLED) + } + } + + private fun getThumbnailBufferInternal( + assetId: String, + width: Long, + height: Long, + callback: (Result>) -> Unit, + signal: CancellationSignal + ) { + signal.throwIfCanceled() + val targetWidth = width.toInt() + val targetHeight = height.toInt() + val id = assetId.toLong() + + val cursor = resolver.query(URI, PROJECTION, SELECTION, arrayOf(assetId), null) + ?: return callback(Result.failure(RuntimeException("Asset not found"))) + + signal.throwIfCanceled() + cursor.use { c -> + if (!c.moveToNext()) { + return callback(Result.failure(RuntimeException("Asset not found"))) + } + + val mediaType = c.getInt(1) + val bitmap = when (mediaType) { + MEDIA_TYPE_IMAGE -> decodeImage(id, targetWidth, targetHeight, signal) + MEDIA_TYPE_VIDEO -> decodeVideoThumbnail(id, targetWidth, targetHeight, signal) + else -> return callback(Result.failure(RuntimeException("Unsupported media type"))) + } + + processBitmap(bitmap, callback, signal) + } + } + + private fun processBitmap( + bitmap: Bitmap, + callback: (Result>) -> Unit, + signal: CancellationSignal + ) { + signal.throwIfCanceled() + val actualWidth = bitmap.width + val actualHeight = bitmap.height + + val size = actualWidth * actualHeight * 4 + val pointer = allocateNative(size) + + try { + signal.throwIfCanceled() + val buffer = wrapAsBuffer(pointer, size) + bitmap.copyPixelsToBuffer(buffer) + signal.throwIfCanceled() + val res = mapOf( + "pointer" to pointer, + "width" to actualWidth.toLong(), + "height" to actualHeight.toLong() + ) + callback(Result.success(res)) + } catch (e: Exception) { + freeNative(pointer) + callback(if (e is OperationCanceledException) CANCELLED else Result.failure(e)) + } + } + + private fun decodeImage( + id: Long, + targetWidth: Int, + targetHeight: Int, + signal: CancellationSignal + ): Bitmap { + signal.throwIfCanceled() + val uri = ContentUris.withAppendedId(Images.Media.EXTERNAL_CONTENT_URI, id) + if (targetHeight > 768 || targetWidth > 768) { + return decodeSource(uri, targetWidth, targetHeight, signal) + } + + return if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { + resolver.loadThumbnail(uri, Size(targetWidth, targetHeight), signal) + } else { + signal.setOnCancelListener { Images.Thumbnails.cancelThumbnailRequest(resolver, id) } + Images.Thumbnails.getThumbnail(resolver, id, Images.Thumbnails.MINI_KIND, OPTIONS) + } + } + + private fun decodeVideoThumbnail( + id: Long, + targetWidth: Int, + targetHeight: Int, + signal: CancellationSignal + ): Bitmap { + signal.throwIfCanceled() + return if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { + val uri = ContentUris.withAppendedId(Video.Media.EXTERNAL_CONTENT_URI, id) + resolver.loadThumbnail(uri, Size(targetWidth, targetHeight), signal) + } else { + signal.setOnCancelListener { Video.Thumbnails.cancelThumbnailRequest(resolver, id) } + Video.Thumbnails.getThumbnail(resolver, id, Video.Thumbnails.MINI_KIND, OPTIONS) + } + } + + private fun decodeSource( + uri: Uri, + targetWidth: Int, + targetHeight: Int, + signal: CancellationSignal + ): Bitmap { + signal.throwIfCanceled() + return if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { + val source = ImageDecoder.createSource(resolver, uri) + signal.throwIfCanceled() + ImageDecoder.decodeBitmap(source) { decoder, info, _ -> + val sampleSize = + getSampleSize(info.size.width, info.size.height, targetWidth, targetHeight) + decoder.setTargetSampleSize(sampleSize) + decoder.allocator = ImageDecoder.ALLOCATOR_SOFTWARE + decoder.setTargetColorSpace(ColorSpace.get(ColorSpace.Named.SRGB)) + } + } else { + val ref = Glide.with(ctx) + .asBitmap() + .priority(Priority.IMMEDIATE) + .load(uri) + .disallowHardwareConfig() + .format(DecodeFormat.PREFER_ARGB_8888) + .submit(targetWidth, targetHeight) + signal.setOnCancelListener { Glide.with(ctx).clear(ref) } + ref.get() + } + } + + private fun getSampleSize(fullWidth: Int, fullHeight: Int, reqWidth: Int, reqHeight: Int): Int { + return 1 shl max( + 0, floor( + min( + log2(fullWidth / (2.0 * reqWidth)), + log2(fullHeight / (2.0 * reqHeight)), + ) + ).toInt() + ) + } +} diff --git a/mobile/ios/Runner.xcodeproj/project.pbxproj b/mobile/ios/Runner.xcodeproj/project.pbxproj index 218a294c33..4557c8b706 100644 --- a/mobile/ios/Runner.xcodeproj/project.pbxproj +++ b/mobile/ios/Runner.xcodeproj/project.pbxproj @@ -24,6 +24,8 @@ FAC6F89B2D287C890078CB2F /* ShareExtension.appex in Embed Foundation Extensions */ = {isa = PBXBuildFile; fileRef = FAC6F8902D287C890078CB2F /* ShareExtension.appex */; settings = {ATTRIBUTES = (RemoveHeadersOnCopy, ); }; }; FAC6F8B72D287F120078CB2F /* ShareViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = FAC6F8B52D287F120078CB2F /* ShareViewController.swift */; }; FAC6F8B92D287F120078CB2F /* MainInterface.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = FAC6F8B32D287F120078CB2F /* MainInterface.storyboard */; }; + FED3B1962E253E9B0030FD97 /* ThumbnailsImpl.swift in Sources */ = {isa = PBXBuildFile; fileRef = FED3B1942E253E9B0030FD97 /* ThumbnailsImpl.swift */; }; + FED3B1972E253E9B0030FD97 /* Thumbnails.g.swift in Sources */ = {isa = PBXBuildFile; fileRef = FED3B1932E253E9B0030FD97 /* Thumbnails.g.swift */; }; /* End PBXBuildFile section */ /* Begin PBXContainerItemProxy section */ @@ -102,6 +104,8 @@ FAC6F8B42D287F120078CB2F /* ShareExtension.entitlements */ = {isa = PBXFileReference; lastKnownFileType = text.plist.entitlements; path = ShareExtension.entitlements; sourceTree = ""; }; FAC6F8B52D287F120078CB2F /* ShareViewController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ShareViewController.swift; sourceTree = ""; }; FAC7416727DB9F5500C668D8 /* RunnerProfile.entitlements */ = {isa = PBXFileReference; lastKnownFileType = text.plist.entitlements; path = RunnerProfile.entitlements; sourceTree = ""; }; + FED3B1932E253E9B0030FD97 /* Thumbnails.g.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Thumbnails.g.swift; sourceTree = ""; }; + FED3B1942E253E9B0030FD97 /* ThumbnailsImpl.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ThumbnailsImpl.swift; sourceTree = ""; }; /* End PBXFileReference section */ /* Begin PBXFileSystemSynchronizedBuildFileExceptionSet section */ @@ -243,6 +247,7 @@ 1498D2331E8E89220040F4C2 /* GeneratedPluginRegistrant.m */, 74858FAE1ED2DC5600515810 /* AppDelegate.swift */, 74858FAD1ED2DC5600515810 /* Runner-Bridging-Header.h */, + FED3B1952E253E9B0030FD97 /* Images */, ); path = Runner; sourceTree = ""; @@ -258,6 +263,15 @@ path = ShareExtension; sourceTree = ""; }; + FED3B1952E253E9B0030FD97 /* Images */ = { + isa = PBXGroup; + children = ( + FED3B1932E253E9B0030FD97 /* Thumbnails.g.swift */, + FED3B1942E253E9B0030FD97 /* ThumbnailsImpl.swift */, + ); + path = Images; + sourceTree = ""; + }; /* End PBXGroup section */ /* Begin PBXNativeTarget section */ @@ -523,6 +537,8 @@ files = ( 65F32F31299BD2F800CE9261 /* BackgroundServicePlugin.swift in Sources */, 74858FAF1ED2DC5600515810 /* AppDelegate.swift in Sources */, + FED3B1962E253E9B0030FD97 /* ThumbnailsImpl.swift in Sources */, + FED3B1972E253E9B0030FD97 /* Thumbnails.g.swift in Sources */, 1498D2341E8E89220040F4C2 /* GeneratedPluginRegistrant.m in Sources */, 65F32F33299D349D00CE9261 /* BackgroundSyncWorker.swift in Sources */, ); diff --git a/mobile/ios/Runner/AppDelegate.swift b/mobile/ios/Runner/AppDelegate.swift index 55d08adc6a..dedda5bd12 100644 --- a/mobile/ios/Runner/AppDelegate.swift +++ b/mobile/ios/Runner/AppDelegate.swift @@ -25,6 +25,7 @@ import UIKit let controller: FlutterViewController = window?.rootViewController as! FlutterViewController NativeSyncApiSetup.setUp(binaryMessenger: controller.binaryMessenger, api: NativeSyncApiImpl()) + ThumbnailApiSetup.setUp(binaryMessenger: controller.binaryMessenger, api: ThumbnailApiImpl()) BackgroundServicePlugin.setPluginRegistrantCallback { registry in if !registry.hasPlugin("org.cocoapods.path-provider-foundation") { diff --git a/mobile/ios/Runner/Images/Thumbnails.g.swift b/mobile/ios/Runner/Images/Thumbnails.g.swift new file mode 100644 index 0000000000..f35ded0775 --- /dev/null +++ b/mobile/ios/Runner/Images/Thumbnails.g.swift @@ -0,0 +1,119 @@ +// Autogenerated from Pigeon (v26.0.0), do not edit directly. +// See also: https://pub.dev/packages/pigeon + +import Foundation + +#if os(iOS) + import Flutter +#elseif os(macOS) + import FlutterMacOS +#else + #error("Unsupported platform.") +#endif + +private func wrapResult(_ result: Any?) -> [Any?] { + return [result] +} + +private func wrapError(_ error: Any) -> [Any?] { + if let pigeonError = error as? PigeonError { + return [ + pigeonError.code, + pigeonError.message, + pigeonError.details, + ] + } + if let flutterError = error as? FlutterError { + return [ + flutterError.code, + flutterError.message, + flutterError.details, + ] + } + return [ + "\(error)", + "\(type(of: error))", + "Stacktrace: \(Thread.callStackSymbols)", + ] +} + +private func isNullish(_ value: Any?) -> Bool { + return value is NSNull || value == nil +} + +private func nilOrValue(_ value: Any?) -> T? { + if value is NSNull { return nil } + return value as! T? +} + + +private class ThumbnailsPigeonCodecReader: FlutterStandardReader { +} + +private class ThumbnailsPigeonCodecWriter: FlutterStandardWriter { +} + +private class ThumbnailsPigeonCodecReaderWriter: FlutterStandardReaderWriter { + override func reader(with data: Data) -> FlutterStandardReader { + return ThumbnailsPigeonCodecReader(data: data) + } + + override func writer(with data: NSMutableData) -> FlutterStandardWriter { + return ThumbnailsPigeonCodecWriter(data: data) + } +} + +class ThumbnailsPigeonCodec: FlutterStandardMessageCodec, @unchecked Sendable { + static let shared = ThumbnailsPigeonCodec(readerWriter: ThumbnailsPigeonCodecReaderWriter()) +} + + +/// Generated protocol from Pigeon that represents a handler of messages from Flutter. +protocol ThumbnailApi { + func requestImage(assetId: String, requestId: Int64, width: Int64, height: Int64, completion: @escaping (Result<[String: Int64], Error>) -> Void) + func cancelImageRequest(requestId: Int64) throws +} + +/// Generated setup class from Pigeon to handle messages through the `binaryMessenger`. +class ThumbnailApiSetup { + static var codec: FlutterStandardMessageCodec { ThumbnailsPigeonCodec.shared } + /// Sets up an instance of `ThumbnailApi` to handle messages through the `binaryMessenger`. + static func setUp(binaryMessenger: FlutterBinaryMessenger, api: ThumbnailApi?, messageChannelSuffix: String = "") { + let channelSuffix = messageChannelSuffix.count > 0 ? ".\(messageChannelSuffix)" : "" + let requestImageChannel = FlutterBasicMessageChannel(name: "dev.flutter.pigeon.immich_mobile.ThumbnailApi.requestImage\(channelSuffix)", binaryMessenger: binaryMessenger, codec: codec) + if let api = api { + requestImageChannel.setMessageHandler { message, reply in + let args = message as! [Any?] + let assetIdArg = args[0] as! String + let requestIdArg = args[1] as! Int64 + let widthArg = args[2] as! Int64 + let heightArg = args[3] as! Int64 + api.requestImage(assetId: assetIdArg, requestId: requestIdArg, width: widthArg, height: heightArg) { result in + switch result { + case .success(let res): + reply(wrapResult(res)) + case .failure(let error): + reply(wrapError(error)) + } + } + } + } else { + requestImageChannel.setMessageHandler(nil) + } + let cancelImageRequestChannel = FlutterBasicMessageChannel(name: "dev.flutter.pigeon.immich_mobile.ThumbnailApi.cancelImageRequest\(channelSuffix)", binaryMessenger: binaryMessenger, codec: codec) + if let api = api { + cancelImageRequestChannel.setMessageHandler { message, reply in + let args = message as! [Any?] + let requestIdArg = args[0] as! Int64 + do { + try api.cancelImageRequest(requestId: requestIdArg) + reply(wrapResult(nil)) + } catch { + reply(wrapError(error)) + } + } + } else { + cancelImageRequestChannel.setMessageHandler(nil) + } + } +} diff --git a/mobile/ios/Runner/Images/ThumbnailsImpl.swift b/mobile/ios/Runner/Images/ThumbnailsImpl.swift new file mode 100644 index 0000000000..e451a8189e --- /dev/null +++ b/mobile/ios/Runner/Images/ThumbnailsImpl.swift @@ -0,0 +1,177 @@ +import CryptoKit +import Flutter +import MobileCoreServices +import Photos + +class Request { + weak var workItem: DispatchWorkItem? + var isCancelled = false + let callback: (Result<[String: Int64], any Error>) -> Void + + init(callback: @escaping (Result<[String: Int64], any Error>) -> Void) { + self.callback = callback + } +} + +class ThumbnailApiImpl: ThumbnailApi { + private static let imageManager = PHImageManager.default() + private static let fetchOptions = { + let fetchOptions = PHFetchOptions() + fetchOptions.fetchLimit = 1 + fetchOptions.wantsIncrementalChangeDetails = false + return fetchOptions + }() + private static let requestOptions = { + let requestOptions = PHImageRequestOptions() + requestOptions.isNetworkAccessAllowed = true + requestOptions.deliveryMode = .highQualityFormat + requestOptions.resizeMode = .fast + requestOptions.isSynchronous = true + requestOptions.version = .current + return requestOptions + }() + + private static let assetQueue = DispatchQueue(label: "thumbnail.assets", qos: .userInitiated) + private static let requestQueue = DispatchQueue(label: "thumbnail.requests", qos: .userInitiated) + private static let cancelQueue = DispatchQueue(label: "thumbnail.cancellation", qos: .default) + private static let processingQueue = DispatchQueue(label: "thumbnail.processing", qos: .userInteractive, attributes: .concurrent) + + private static let rgbColorSpace = CGColorSpaceCreateDeviceRGB() + private static let bitmapInfo = CGBitmapInfo(rawValue: CGImageAlphaInfo.premultipliedLast.rawValue).rawValue + private static var requests = [Int64: Request]() + private static let cancelledResult = Result<[String: Int64], any Error>.success([:]) + private static let concurrencySemaphore = DispatchSemaphore(value: ProcessInfo.processInfo.activeProcessorCount * 2) + private static let assetCache = { + let assetCache = NSCache() + assetCache.countLimit = 10000 + return assetCache + }() + + func requestImage(assetId: String, requestId: Int64, width: Int64, height: Int64, completion: @escaping (Result<[String: Int64], any Error>) -> Void) { + let request = Request(callback: completion) + let item = DispatchWorkItem { + if request.isCancelled { + return completion(Self.cancelledResult) + } + + Self.concurrencySemaphore.wait() + defer { + Self.concurrencySemaphore.signal() + } + + if request.isCancelled { + return completion(Self.cancelledResult) + } + + guard let asset = Self.requestAsset(assetId: assetId) + else { + Self.removeRequest(requestId: requestId) + completion(.failure(PigeonError(code: "", message: "Could not get asset data for \(assetId)", details: nil))) + return + } + + if request.isCancelled { + return completion(Self.cancelledResult) + } + + var image: UIImage? + Self.imageManager.requestImage( + for: asset, + targetSize: CGSize(width: Double(width), height: Double(height)), + contentMode: .aspectFit, + options: Self.requestOptions, + resultHandler: { (_image, info) -> Void in + image = _image + } + ) + + if request.isCancelled { + return completion(Self.cancelledResult) + } + + guard let image = image, + let cgImage = image.cgImage else { + Self.removeRequest(requestId: requestId) + return completion(.failure(PigeonError(code: "", message: "Could not get pixel data for \(assetId)", details: nil))) + } + + let pointer = UnsafeMutableRawPointer.allocate( + byteCount: Int(cgImage.width) * Int(cgImage.height) * 4, + alignment: MemoryLayout.alignment + ) + + if request.isCancelled { + pointer.deallocate() + return completion(Self.cancelledResult) + } + + guard let context = CGContext( + data: pointer, + width: cgImage.width, + height: cgImage.height, + bitsPerComponent: 8, + bytesPerRow: cgImage.width * 4, + space: Self.rgbColorSpace, + bitmapInfo: Self.bitmapInfo + ) else { + pointer.deallocate() + Self.removeRequest(requestId: requestId) + return completion(.failure(PigeonError(code: "", message: "Could not create context for \(assetId)", details: nil))) + } + + if request.isCancelled { + pointer.deallocate() + return completion(Self.cancelledResult) + } + + context.interpolationQuality = .none + context.draw(cgImage, in: CGRect(x: 0, y: 0, width: cgImage.width, height: cgImage.height)) + + if request.isCancelled { + pointer.deallocate() + return completion(Self.cancelledResult) + } + + completion(.success(["pointer": Int64(Int(bitPattern: pointer)), "width": Int64(cgImage.width), "height": Int64(cgImage.height)])) + Self.removeRequest(requestId: requestId) + } + + request.workItem = item + Self.addRequest(requestId: requestId, request: request) + Self.processingQueue.async(execute: item) + } + + func cancelImageRequest(requestId: Int64) { + Self.cancelRequest(requestId: requestId) + } + + private static func addRequest(requestId: Int64, request: Request) -> Void { + requestQueue.sync { requests[requestId] = request } + } + + private static func removeRequest(requestId: Int64) -> Void { + requestQueue.sync { requests[requestId] = nil } + } + + private static func cancelRequest(requestId: Int64) -> Void { + requestQueue.async { + guard let request = requests.removeValue(forKey: requestId) else { return } + request.isCancelled = true + guard let item = request.workItem else { return } + if item.isCancelled { + request.callback(Self.cancelledResult) + } + } + } + + private static func requestAsset(assetId: String) -> PHAsset? { + var asset: PHAsset? + assetQueue.sync { asset = assetCache.object(forKey: assetId as NSString) } + if asset != nil { return asset } + + guard let asset = PHAsset.fetchAssets(withLocalIdentifiers: [assetId], options: Self.fetchOptions).firstObject + else { return nil } + assetQueue.async { assetCache.setObject(asset, forKey: assetId as NSString) } + return asset + } +} diff --git a/mobile/ios/Runner/Info.plist b/mobile/ios/Runner/Info.plist index d3e42b9939..4b93dc820f 100644 --- a/mobile/ios/Runner/Info.plist +++ b/mobile/ios/Runner/Info.plist @@ -184,4 +184,4 @@ We need local network permission to connect to the local server using IP address and allow the casting feature to work - \ No newline at end of file + diff --git a/mobile/lib/constants/constants.dart b/mobile/lib/constants/constants.dart index 616a306d94..63af4ffff8 100644 --- a/mobile/lib/constants/constants.dart +++ b/mobile/lib/constants/constants.dart @@ -26,8 +26,9 @@ const String kDownloadGroupLivePhoto = 'group_livephoto'; // Timeline constants const int kTimelineNoneSegmentSize = 120; -const int kTimelineAssetLoadBatchSize = 256; +const int kTimelineAssetLoadBatchSize = 1024; const int kTimelineAssetLoadOppositeSize = 64; +const int kTimelineImageCacheMemory = 200 * 1024 * 1024; // Widget keys const String appShareGroupId = "group.app.immich.share"; diff --git a/mobile/lib/extensions/datetime_extensions.dart b/mobile/lib/extensions/datetime_extensions.dart index 0bc95565a6..693775d544 100644 --- a/mobile/lib/extensions/datetime_extensions.dart +++ b/mobile/lib/extensions/datetime_extensions.dart @@ -85,3 +85,13 @@ extension DateRangeFormatting on DateTime { } } } + +extension IsSameExtension on DateTime { + bool isSameDay(DateTime other) { + return day == other.day && month == other.month && year == other.year; + } + + bool isSameMonth(DateTime other) { + return month == other.month && year == other.year; + } +} diff --git a/mobile/lib/infrastructure/repositories/asset_media.repository.dart b/mobile/lib/infrastructure/repositories/asset_media.repository.dart index 6c81c7ff7f..2cace0d7ea 100644 --- a/mobile/lib/infrastructure/repositories/asset_media.repository.dart +++ b/mobile/lib/infrastructure/repositories/asset_media.repository.dart @@ -1,18 +1,211 @@ -import 'dart:typed_data'; -import 'dart:ui'; +import 'dart:async'; +import 'dart:ffi'; +import 'dart:io'; +import 'dart:ui' as ui; -import 'package:photo_manager/photo_manager.dart'; +import 'package:flutter/material.dart'; +import 'package:flutter/services.dart'; +import 'package:flutter/foundation.dart'; +import 'package:immich_mobile/providers/image/cache/remote_image_cache_manager.dart'; +import 'package:immich_mobile/providers/infrastructure/platform.provider.dart'; +import 'package:ffi/ffi.dart'; +import 'package:immich_mobile/services/api.service.dart'; +import 'package:logging/logging.dart'; -class AssetMediaRepository { - const AssetMediaRepository(); +abstract class ImageRequest { + static int _nextRequestId = 0; - Future getThumbnail(String id, {int quality = 80, Size size = const Size.square(256)}) => AssetEntity( - id: id, - // The below fields are not used in thumbnailDataWithSize but are required - // to create an AssetEntity instance. It is faster to create a dummy AssetEntity - // instance than to fetch the asset from the device first. - typeInt: AssetType.image.index, - width: size.width.toInt(), - height: size.height.toInt(), - ).thumbnailDataWithSize(ThumbnailSize(size.width.toInt(), size.height.toInt()), quality: quality); + final int requestId = _nextRequestId++; + bool _isCancelled = false; + + get isCancelled => _isCancelled; + + ImageRequest(); + + Future load(ImageDecoderCallback decode, {double scale = 1.0}); + + void cancel() { + if (isCancelled) { + return; + } + _isCancelled = true; + return _onCancelled(); + } + + void _onCancelled(); +} + +class LocalImageRequest extends ImageRequest { + final String localId; + final int width; + final int height; + + LocalImageRequest({required this.localId, required ui.Size size}) + : width = size.width.toInt(), + height = size.height.toInt(); + + @override + Future load(ImageDecoderCallback decode, {double scale = 1.0}) async { + if (_isCancelled) { + return null; + } + + final Map info = await thumbnailApi.requestImage( + localId, + requestId: requestId, + width: width, + height: height, + ); + + final address = info['pointer']; + if (address == null) { + return null; + } + + final pointer = Pointer.fromAddress(address); + try { + if (_isCancelled) { + return null; + } + + final actualWidth = info['width']!; + final actualHeight = info['height']!; + final actualSize = actualWidth * actualHeight * 4; + + final buffer = await ImmutableBuffer.fromUint8List(pointer.asTypedList(actualSize)); + if (_isCancelled) { + return null; + } + + final descriptor = ui.ImageDescriptor.raw( + buffer, + width: actualWidth, + height: actualHeight, + pixelFormat: ui.PixelFormat.rgba8888, + ); + final codec = await descriptor.instantiateCodec(); + if (_isCancelled) { + return null; + } + + final frame = await codec.getNextFrame(); + return ImageInfo(image: frame.image, scale: scale); + } finally { + malloc.free(pointer); + } + } + + @override + Future _onCancelled() { + return thumbnailApi.cancelImageRequest(requestId); + } +} + +class RemoteImageRequest extends ImageRequest { + static final log = Logger('RemoteImageRequest'); + static final cacheManager = RemoteImageCacheManager(); + static final client = HttpClient(); + String uri; + Map headers; + HttpClientRequest? _request; + + RemoteImageRequest({required this.uri, required this.headers}); + + @override + Future load(ImageDecoderCallback decode, {double scale = 1.0}) async { + if (_isCancelled) { + return null; + } + + try { + // The DB calls made by the cache manager are a *massive* bottleneck (6+ seconds) with high concurrency. + // Since it isn't possible to cancel these operations, we only prefer the cache when they can be avoided. + // The DB hit is left as a fallback for offline use. + final cachedFileBuffer = await _loadCachedFile(uri, inMemoryOnly: true); + if (cachedFileBuffer != null) { + return _decodeBuffer(cachedFileBuffer, decode, scale); + } + + final buffer = await _downloadImage(uri); + if (buffer == null || _isCancelled) { + return null; + } + return await _decodeBuffer(buffer, decode, scale); + } catch (e) { + if (e is HttpException && (e.message.endsWith('aborted') || e.message.startsWith('Connection closed'))) { + return null; + } + log.severe('Failed to load remote image', e); + final buffer = await _loadCachedFile(uri, inMemoryOnly: false); + if (buffer != null) { + return _decodeBuffer(buffer, decode, scale); + } + rethrow; + } finally { + _request = null; + } + } + + Future _downloadImage(String url) async { + final request = _request = await client.getUrl(Uri.parse(url)); + if (_isCancelled) { + return null; + } + + final headers = ApiService.getRequestHeaders(); + for (final entry in headers.entries) { + request.headers.set(entry.key, entry.value); + } + final response = await request.close(); + if (_isCancelled) { + return null; + } + + final bytes = await consolidateHttpClientResponseBytes(response); + _cacheFile(url, bytes); + if (_isCancelled) { + return null; + } + return await ImmutableBuffer.fromUint8List(bytes); + } + + Future _cacheFile(String url, Uint8List bytes) async { + try { + await cacheManager.putFile(url, bytes); + } catch (e) { + log.severe('Failed to cache image', e); + } + } + + Future _loadCachedFile(String url, {required bool inMemoryOnly}) async { + if (_isCancelled) { + return null; + } + final file = await (inMemoryOnly ? cacheManager.getFileFromMemory(url) : cacheManager.getFileFromCache(url)); + if (_isCancelled || file == null) { + return null; + } + return await ImmutableBuffer.fromFilePath(file.file.path); + } + + Future _decodeBuffer(ImmutableBuffer buffer, ImageDecoderCallback decode, scale) async { + if (_isCancelled) { + buffer.dispose(); + return null; + } + final codec = await decode(buffer); + if (_isCancelled) { + buffer.dispose(); + codec.dispose(); + return null; + } + final frame = await codec.getNextFrame(); + return ImageInfo(image: frame.image, scale: scale); + } + + @override + void _onCancelled() { + _request?.abort(); + _request = null; + } } diff --git a/mobile/lib/main.dart b/mobile/lib/main.dart index 19ed746833..883840efa6 100644 --- a/mobile/lib/main.dart +++ b/mobile/lib/main.dart @@ -71,6 +71,7 @@ Future initApp() async { } } + PaintingBinding.instance.imageCache.maximumSizeBytes = kTimelineImageCacheMemory; await DynamicTheme.fetchSystemPalette(); final log = Logger("ImmichErrorLogger"); diff --git a/mobile/lib/platform/thumbnail_api.g.dart b/mobile/lib/platform/thumbnail_api.g.dart new file mode 100644 index 0000000000..7a4fee9ba6 --- /dev/null +++ b/mobile/lib/platform/thumbnail_api.g.dart @@ -0,0 +1,107 @@ +// Autogenerated from Pigeon (v26.0.0), do not edit directly. +// See also: https://pub.dev/packages/pigeon +// ignore_for_file: public_member_api_docs, non_constant_identifier_names, avoid_as, unused_import, unnecessary_parenthesis, prefer_null_aware_operators, omit_local_variable_types, unused_shown_name, unnecessary_import, no_leading_underscores_for_local_identifiers + +import 'dart:async'; +import 'dart:typed_data' show Float64List, Int32List, Int64List, Uint8List; + +import 'package:flutter/foundation.dart' show ReadBuffer, WriteBuffer; +import 'package:flutter/services.dart'; + +PlatformException _createConnectionError(String channelName) { + return PlatformException( + code: 'channel-error', + message: 'Unable to establish connection on channel: "$channelName".', + ); +} + +class _PigeonCodec extends StandardMessageCodec { + const _PigeonCodec(); + @override + void writeValue(WriteBuffer buffer, Object? value) { + if (value is int) { + buffer.putUint8(4); + buffer.putInt64(value); + } else { + super.writeValue(buffer, value); + } + } + + @override + Object? readValueOfType(int type, ReadBuffer buffer) { + switch (type) { + default: + return super.readValueOfType(type, buffer); + } + } +} + +class ThumbnailApi { + /// Constructor for [ThumbnailApi]. The [binaryMessenger] named argument is + /// available for dependency injection. If it is left null, the default + /// BinaryMessenger will be used which routes to the host platform. + ThumbnailApi({BinaryMessenger? binaryMessenger, String messageChannelSuffix = ''}) + : pigeonVar_binaryMessenger = binaryMessenger, + pigeonVar_messageChannelSuffix = messageChannelSuffix.isNotEmpty ? '.$messageChannelSuffix' : ''; + final BinaryMessenger? pigeonVar_binaryMessenger; + + static const MessageCodec pigeonChannelCodec = _PigeonCodec(); + + final String pigeonVar_messageChannelSuffix; + + Future> requestImage( + String assetId, { + required int requestId, + required int width, + required int height, + }) async { + final String pigeonVar_channelName = + 'dev.flutter.pigeon.immich_mobile.ThumbnailApi.requestImage$pigeonVar_messageChannelSuffix'; + final BasicMessageChannel pigeonVar_channel = BasicMessageChannel( + pigeonVar_channelName, + pigeonChannelCodec, + binaryMessenger: pigeonVar_binaryMessenger, + ); + final Future pigeonVar_sendFuture = pigeonVar_channel.send([assetId, requestId, width, height]); + final List? pigeonVar_replyList = await pigeonVar_sendFuture as List?; + if (pigeonVar_replyList == null) { + throw _createConnectionError(pigeonVar_channelName); + } else if (pigeonVar_replyList.length > 1) { + throw PlatformException( + code: pigeonVar_replyList[0]! as String, + message: pigeonVar_replyList[1] as String?, + details: pigeonVar_replyList[2], + ); + } else if (pigeonVar_replyList[0] == null) { + throw PlatformException( + code: 'null-error', + message: 'Host platform returned null value for non-null return value.', + ); + } else { + return (pigeonVar_replyList[0] as Map?)!.cast(); + } + } + + Future cancelImageRequest(int requestId) async { + final String pigeonVar_channelName = + 'dev.flutter.pigeon.immich_mobile.ThumbnailApi.cancelImageRequest$pigeonVar_messageChannelSuffix'; + final BasicMessageChannel pigeonVar_channel = BasicMessageChannel( + pigeonVar_channelName, + pigeonChannelCodec, + binaryMessenger: pigeonVar_binaryMessenger, + ); + final Future pigeonVar_sendFuture = pigeonVar_channel.send([requestId]); + final List? pigeonVar_replyList = await pigeonVar_sendFuture as List?; + if (pigeonVar_replyList == null) { + throw _createConnectionError(pigeonVar_channelName); + } else if (pigeonVar_replyList.length > 1) { + throw PlatformException( + code: pigeonVar_replyList[0]! as String, + message: pigeonVar_replyList[1] as String?, + details: pigeonVar_replyList[2], + ); + } else { + return; + } + } +} diff --git a/mobile/lib/presentation/widgets/images/image_provider.dart b/mobile/lib/presentation/widgets/images/image_provider.dart index 19eed71d44..b428937bc0 100644 --- a/mobile/lib/presentation/widgets/images/image_provider.dart +++ b/mobile/lib/presentation/widgets/images/image_provider.dart @@ -5,13 +5,32 @@ import 'package:immich_mobile/domain/services/setting.service.dart'; import 'package:immich_mobile/presentation/widgets/images/local_image_provider.dart'; import 'package:immich_mobile/presentation/widgets/images/remote_image_provider.dart'; import 'package:immich_mobile/presentation/widgets/timeline/constants.dart'; +import 'package:immich_mobile/infrastructure/repositories/asset_media.repository.dart'; + +abstract class CancellableImageProvider { + void cancel(); +} + +mixin class CancellableImageProviderMixin implements CancellableImageProvider { + ImageRequest? request; + + @override + void cancel() { + final request = this.request; + if (request == null) { + return; + } + this.request = null; + return request.cancel(); + } +} ImageProvider getFullImageProvider(BaseAsset asset, {Size size = const Size(1080, 1920)}) { // Create new provider and cache it final ImageProvider provider; if (_shouldUseLocalAsset(asset)) { final id = asset is LocalAsset ? asset.id : (asset as RemoteAsset).localId!; - provider = LocalFullImageProvider(id: id, size: size, type: asset.type, updatedAt: asset.updatedAt); + provider = LocalFullImageProvider(id: id, size: size); } else { final String assetId; if (asset is LocalAsset && asset.hasRemote) { @@ -36,7 +55,7 @@ ImageProvider getThumbnailImageProvider({BaseAsset? asset, String? remoteId, Siz if (_shouldUseLocalAsset(asset!)) { final id = asset is LocalAsset ? asset.id : (asset as RemoteAsset).localId!; - return LocalThumbProvider(id: id, updatedAt: asset.updatedAt, size: size); + return LocalThumbProvider(id: id, size: size); } final String assetId; diff --git a/mobile/lib/presentation/widgets/images/local_image_provider.dart b/mobile/lib/presentation/widgets/images/local_image_provider.dart index 4da4b927f1..5098ab98e7 100644 --- a/mobile/lib/presentation/widgets/images/local_image_provider.dart +++ b/mobile/lib/presentation/widgets/images/local_image_provider.dart @@ -1,37 +1,18 @@ import 'dart:async'; -import 'dart:io'; import 'dart:ui'; import 'package:flutter/foundation.dart'; import 'package:flutter/widgets.dart'; -import 'package:flutter_cache_manager/flutter_cache_manager.dart'; -import 'package:immich_mobile/domain/models/asset/base_asset.model.dart'; -import 'package:immich_mobile/domain/models/setting.model.dart'; -import 'package:immich_mobile/domain/services/setting.service.dart'; -import 'package:immich_mobile/extensions/codec_extensions.dart'; import 'package:immich_mobile/infrastructure/repositories/asset_media.repository.dart'; -import 'package:immich_mobile/infrastructure/repositories/storage.repository.dart'; import 'package:immich_mobile/presentation/widgets/images/image_provider.dart'; import 'package:immich_mobile/presentation/widgets/images/one_frame_multi_image_stream_completer.dart'; import 'package:immich_mobile/presentation/widgets/timeline/constants.dart'; -import 'package:immich_mobile/providers/image/cache/thumbnail_image_cache_manager.dart'; -import 'package:immich_mobile/providers/image/exceptions/image_loading_exception.dart'; -import 'package:logging/logging.dart'; - -class LocalThumbProvider extends ImageProvider { - final AssetMediaRepository _assetMediaRepository = const AssetMediaRepository(); - final CacheManager? cacheManager; +class LocalThumbProvider extends ImageProvider with CancellableImageProviderMixin { final String id; - final DateTime updatedAt; final Size size; - const LocalThumbProvider({ - required this.id, - required this.updatedAt, - this.size = kThumbnailResolution, - this.cacheManager, - }); + LocalThumbProvider({required this.id, this.size = kThumbnailResolution}); @override Future obtainKey(ImageConfiguration configuration) { @@ -40,63 +21,45 @@ class LocalThumbProvider extends ImageProvider { @override ImageStreamCompleter loadImage(LocalThumbProvider key, ImageDecoderCallback decode) { - final cache = cacheManager ?? ThumbnailImageCacheManager(); - return MultiFrameImageStreamCompleter( - codec: _codec(key, cache, decode), - scale: 1.0, + return OneFramePlaceholderImageStreamCompleter( + _codec(key, decode), informationCollector: () => [ DiagnosticsProperty('Id', key.id), - DiagnosticsProperty('Updated at', key.updatedAt), DiagnosticsProperty('Size', key.size), ], ); } - Future _codec(LocalThumbProvider key, CacheManager cache, ImageDecoderCallback decode) async { - final cacheKey = '${key.id}-${key.updatedAt}-${key.size.width}x${key.size.height}'; - - final fileFromCache = await cache.getFileFromCache(cacheKey); - if (fileFromCache != null) { - try { - final buffer = await ImmutableBuffer.fromFilePath(fileFromCache.file.path); - return decode(buffer); - } catch (_) {} + Stream _codec(LocalThumbProvider key, ImageDecoderCallback decode) async* { + final request = this.request = LocalImageRequest(localId: key.id, size: size); + try { + final image = await request.load(decode); + if (image != null) { + yield image; + } + } finally { + this.request = null; } - - final thumbnailBytes = await _assetMediaRepository.getThumbnail(key.id, size: key.size); - if (thumbnailBytes == null) { - PaintingBinding.instance.imageCache.evict(key); - throw StateError("Loading thumb for local photo ${key.id} failed"); - } - - final buffer = await ImmutableBuffer.fromUint8List(thumbnailBytes); - unawaited(cache.putFile(cacheKey, thumbnailBytes)); - return decode(buffer); } @override bool operator ==(Object other) { if (identical(this, other)) return true; if (other is LocalThumbProvider) { - return id == other.id && updatedAt == other.updatedAt; + return id == other.id && size == other.size; } return false; } @override - int get hashCode => id.hashCode ^ updatedAt.hashCode; + int get hashCode => id.hashCode ^ size.hashCode; } -class LocalFullImageProvider extends ImageProvider { - final AssetMediaRepository _assetMediaRepository = const AssetMediaRepository(); - final StorageRepository _storageRepository = const StorageRepository(); - +class LocalFullImageProvider extends ImageProvider with CancellableImageProviderMixin { final String id; final Size size; - final AssetType type; - final DateTime updatedAt; // temporary, only exists to fetch cached thumbnail until local disk cache is removed - const LocalFullImageProvider({required this.id, required this.size, required this.type, required this.updatedAt}); + LocalFullImageProvider({required this.id, required this.size}); @override Future obtainKey(ImageConfiguration configuration) { @@ -107,98 +70,41 @@ class LocalFullImageProvider extends ImageProvider { ImageStreamCompleter loadImage(LocalFullImageProvider key, ImageDecoderCallback decode) { return OneFramePlaceholderImageStreamCompleter( _codec(key, decode), - initialImage: getCachedImage(LocalThumbProvider(id: key.id, updatedAt: key.updatedAt)), + initialImage: getCachedImage(LocalThumbProvider(id: key.id)), informationCollector: () => [ + DiagnosticsProperty('Image provider', this), DiagnosticsProperty('Id', key.id), - DiagnosticsProperty('Updated at', key.updatedAt), DiagnosticsProperty('Size', key.size), ], ); } - // Streams in each stage of the image as we ask for it - Stream _codec(LocalFullImageProvider key, ImageDecoderCallback decode) { - try { - return switch (key.type) { - AssetType.image => _decodeProgressive(key, decode), - AssetType.video => _getThumbnailCodec(key, decode), - _ => throw StateError('Unsupported asset type ${key.type}'), - }; - } catch (error, stack) { - Logger('ImmichLocalImageProvider').severe('Error loading local image ${key.id}', error, stack); - throw const ImageLoadingException('Could not load image from local storage'); - } - } - - Stream _getThumbnailCodec(LocalFullImageProvider key, ImageDecoderCallback decode) async* { - final thumbBytes = await _assetMediaRepository.getThumbnail(key.id, size: key.size); - if (thumbBytes == null) { - throw StateError("Failed to load preview for ${key.id}"); - } - final buffer = await ImmutableBuffer.fromUint8List(thumbBytes); - final codec = await decode(buffer); - yield await codec.getImageInfo(); - } - - Stream _decodeProgressive(LocalFullImageProvider key, ImageDecoderCallback decode) async* { - final file = await _storageRepository.getFileForAsset(key.id); - if (file == null) { - throw StateError("Opening file for asset ${key.id} failed"); - } - - final fileSize = await file.length(); + Stream _codec(LocalFullImageProvider key, ImageDecoderCallback decode) async* { final devicePixelRatio = PlatformDispatcher.instance.views.first.devicePixelRatio; - final isLargeFile = fileSize > 20 * 1024 * 1024; // 20MB - final isHEIC = file.path.toLowerCase().contains(RegExp(r'\.(heic|heif)$')); - final isProgressive = isLargeFile || (isHEIC && !Platform.isIOS); + final request = this.request = LocalImageRequest( + localId: key.id, + size: Size(size.width * devicePixelRatio, size.height * devicePixelRatio), + ); - if (isProgressive) { - try { - final progressiveMultiplier = devicePixelRatio >= 3.0 ? 1.3 : 1.2; - final size = Size( - (key.size.width * progressiveMultiplier).clamp(256, 1024), - (key.size.height * progressiveMultiplier).clamp(256, 1024), - ); - final mediumThumb = await _assetMediaRepository.getThumbnail(key.id, size: size); - if (mediumThumb != null) { - final mediumBuffer = await ImmutableBuffer.fromUint8List(mediumThumb); - final codec = await decode(mediumBuffer); - yield await codec.getImageInfo(); - } - } catch (_) {} - } - - // Load original only when the file is smaller or if the user wants to load original images - // Or load a slightly larger image for progressive loading - if (isProgressive && !(AppSetting.get(Setting.loadOriginal))) { - final progressiveMultiplier = devicePixelRatio >= 3.0 ? 2.0 : 1.6; - final size = Size( - (key.size.width * progressiveMultiplier).clamp(512, 2048), - (key.size.height * progressiveMultiplier).clamp(512, 2048), - ); - final highThumb = await _assetMediaRepository.getThumbnail(key.id, size: size); - if (highThumb != null) { - final highBuffer = await ImmutableBuffer.fromUint8List(highThumb); - final codec = await decode(highBuffer); - yield await codec.getImageInfo(); + try { + final image = await request.load(decode); + if (image != null) { + yield image; } - return; + } finally { + this.request = null; } - - final buffer = await ImmutableBuffer.fromFilePath(file.path); - final codec = await decode(buffer); - yield await codec.getImageInfo(); } @override bool operator ==(Object other) { if (identical(this, other)) return true; if (other is LocalFullImageProvider) { - return id == other.id && size == other.size && type == other.type; + return id == other.id && size == other.size; } return false; } @override - int get hashCode => id.hashCode ^ size.hashCode ^ type.hashCode; + int get hashCode => id.hashCode ^ size.hashCode; } diff --git a/mobile/lib/presentation/widgets/images/remote_image_provider.dart b/mobile/lib/presentation/widgets/images/remote_image_provider.dart index 71c5ad446e..9e918f3f52 100644 --- a/mobile/lib/presentation/widgets/images/remote_image_provider.dart +++ b/mobile/lib/presentation/widgets/images/remote_image_provider.dart @@ -1,23 +1,21 @@ import 'dart:async'; -import 'dart:ui'; import 'package:flutter/foundation.dart'; import 'package:flutter/painting.dart'; import 'package:flutter_cache_manager/flutter_cache_manager.dart'; import 'package:immich_mobile/domain/models/setting.model.dart'; import 'package:immich_mobile/domain/services/setting.service.dart'; -import 'package:immich_mobile/extensions/codec_extensions.dart'; +import 'package:immich_mobile/infrastructure/repositories/asset_media.repository.dart'; import 'package:immich_mobile/presentation/widgets/images/image_provider.dart'; import 'package:immich_mobile/presentation/widgets/images/one_frame_multi_image_stream_completer.dart'; -import 'package:immich_mobile/providers/image/cache/image_loader.dart'; -import 'package:immich_mobile/providers/image/cache/remote_image_cache_manager.dart'; +import 'package:immich_mobile/services/api.service.dart'; import 'package:immich_mobile/utils/image_url_builder.dart'; -class RemoteThumbProvider extends ImageProvider { +class RemoteThumbProvider extends ImageProvider with CancellableImageProviderMixin { final String assetId; final CacheManager? cacheManager; - const RemoteThumbProvider({required this.assetId, this.cacheManager}); + RemoteThumbProvider({required this.assetId, this.cacheManager}); @override Future obtainKey(ImageConfiguration configuration) { @@ -26,12 +24,8 @@ class RemoteThumbProvider extends ImageProvider { @override ImageStreamCompleter loadImage(RemoteThumbProvider key, ImageDecoderCallback decode) { - final cache = cacheManager ?? RemoteImageCacheManager(); - final chunkController = StreamController(); - return MultiFrameImageStreamCompleter( - codec: _codec(key, cache, decode, chunkController), - scale: 1.0, - chunkEvents: chunkController.stream, + return OneFramePlaceholderImageStreamCompleter( + _codec(key, decode), informationCollector: () => [ DiagnosticsProperty('Image provider', this), DiagnosticsProperty('Asset Id', key.assetId), @@ -39,20 +33,17 @@ class RemoteThumbProvider extends ImageProvider { ); } - Future _codec( - RemoteThumbProvider key, - CacheManager cache, - ImageDecoderCallback decode, - StreamController chunkController, - ) async { + Stream _codec(RemoteThumbProvider key, ImageDecoderCallback decode) async* { final preview = getThumbnailUrlForRemoteId(key.assetId); - - return ImageLoader.loadImageFromCache( - preview, - cache: cache, - decode: decode, - chunkEvents: chunkController, - ).whenComplete(chunkController.close); + final request = this.request = RemoteImageRequest(uri: preview, headers: ApiService.getRequestHeaders()); + try { + final image = await request.load(decode); + if (image != null) { + yield image; + } + } finally { + this.request = null; + } } @override @@ -69,11 +60,11 @@ class RemoteThumbProvider extends ImageProvider { int get hashCode => assetId.hashCode; } -class RemoteFullImageProvider extends ImageProvider { +class RemoteFullImageProvider extends ImageProvider with CancellableImageProviderMixin { final String assetId; final CacheManager? cacheManager; - const RemoteFullImageProvider({required this.assetId, this.cacheManager}); + RemoteFullImageProvider({required this.assetId, this.cacheManager}); @override Future obtainKey(ImageConfiguration configuration) { @@ -82,28 +73,44 @@ class RemoteFullImageProvider extends ImageProvider { @override ImageStreamCompleter loadImage(RemoteFullImageProvider key, ImageDecoderCallback decode) { - final cache = cacheManager ?? RemoteImageCacheManager(); return OneFramePlaceholderImageStreamCompleter( - _codec(key, cache, decode), - initialImage: getCachedImage(RemoteThumbProvider(assetId: key.assetId)), + _codec(key, decode), + initialImage: getCachedImage(RemoteThumbProvider(assetId: assetId)), + informationCollector: () => [ + DiagnosticsProperty('Image provider', this), + DiagnosticsProperty('Asset Id', key.assetId), + ], ); } - Stream _codec(RemoteFullImageProvider key, CacheManager cache, ImageDecoderCallback decode) async* { - final codec = await ImageLoader.loadImageFromCache( - getPreviewUrlForRemoteId(key.assetId), - cache: cache, - decode: decode, - ); - yield await codec.getImageInfo(); + Stream _codec(RemoteFullImageProvider key, ImageDecoderCallback decode) async* { + try { + final request = this.request = RemoteImageRequest( + uri: getPreviewUrlForRemoteId(key.assetId), + headers: ApiService.getRequestHeaders(), + ); + final image = await request.load(decode); + if (image == null) { + return; + } + yield image; + } finally { + request = null; + } if (AppSetting.get(Setting.loadOriginal)) { - final codec = await ImageLoader.loadImageFromCache( - getOriginalUrlForRemoteId(key.assetId), - cache: cache, - decode: decode, - ); - yield await codec.getImageInfo(); + try { + final request = this.request = RemoteImageRequest( + uri: getOriginalUrlForRemoteId(key.assetId), + headers: ApiService.getRequestHeaders(), + ); + final image = await request.load(decode); + if (image != null) { + yield image; + } + } finally { + request = null; + } } } diff --git a/mobile/lib/presentation/widgets/timeline/constants.dart b/mobile/lib/presentation/widgets/timeline/constants.dart index e3bb5fe273..242c9bb14e 100644 --- a/mobile/lib/presentation/widgets/timeline/constants.dart +++ b/mobile/lib/presentation/widgets/timeline/constants.dart @@ -2,7 +2,7 @@ import 'dart:ui'; const double kTimelineHeaderExtent = 80.0; const Size kTimelineFixedTileExtent = Size.square(256); -const Size kThumbnailResolution = kTimelineFixedTileExtent; +const Size kThumbnailResolution = Size.square(384); const double kTimelineSpacing = 2.0; const int kTimelineColumnCount = 3; diff --git a/mobile/lib/providers/infrastructure/platform.provider.dart b/mobile/lib/providers/infrastructure/platform.provider.dart index 477046d0bf..6469624c09 100644 --- a/mobile/lib/providers/infrastructure/platform.provider.dart +++ b/mobile/lib/providers/infrastructure/platform.provider.dart @@ -1,4 +1,7 @@ import 'package:hooks_riverpod/hooks_riverpod.dart'; import 'package:immich_mobile/platform/native_sync_api.g.dart'; +import 'package:immich_mobile/platform/thumbnail_api.g.dart'; final nativeSyncApiProvider = Provider((_) => NativeSyncApi()); + +final thumbnailApi = ThumbnailApi(); diff --git a/mobile/lib/widgets/photo_view/src/core/photo_view_core.dart b/mobile/lib/widgets/photo_view/src/core/photo_view_core.dart index 944e5ba7e6..8a76e9b5cf 100644 --- a/mobile/lib/widgets/photo_view/src/core/photo_view_core.dart +++ b/mobile/lib/widgets/photo_view/src/core/photo_view_core.dart @@ -421,6 +421,7 @@ class PhotoViewCoreState extends State filterQuality: widget.filterQuality, width: scaleBoundaries.childSize.width * scale, fit: BoxFit.cover, + isAntiAlias: widget.filterQuality == FilterQuality.high, ); } } diff --git a/mobile/makefile b/mobile/makefile index cfe864a7ee..5a31481f45 100644 --- a/mobile/makefile +++ b/mobile/makefile @@ -7,7 +7,9 @@ build: pigeon: dart run pigeon --input pigeon/native_sync_api.dart + dart run pigeon --input pigeon/thumbnail_api.dart dart format lib/platform/native_sync_api.g.dart + dart format lib/platform/thumbnail_api.g.dart watch: dart run build_runner watch --delete-conflicting-outputs diff --git a/mobile/pigeon/thumbnail_api.dart b/mobile/pigeon/thumbnail_api.dart new file mode 100644 index 0000000000..45b475ec0c --- /dev/null +++ b/mobile/pigeon/thumbnail_api.dart @@ -0,0 +1,21 @@ +import 'package:pigeon/pigeon.dart'; + +@ConfigurePigeon( + PigeonOptions( + dartOut: 'lib/platform/thumbnail_api.g.dart', + swiftOut: 'ios/Runner/Images/Thumbnails.g.swift', + swiftOptions: SwiftOptions(includeErrorClass: false), + kotlinOut: + 'android/app/src/main/kotlin/app/alextran/immich/images/Thumbnails.g.kt', + kotlinOptions: KotlinOptions(package: 'app.alextran.immich.images'), + dartOptions: DartOptions(), + dartPackageName: 'immich_mobile', + ), +) +@HostApi() +abstract class ThumbnailApi { + @async + Map requestImage(String assetId, {required int requestId, required int width, required int height}); + + void cancelImageRequest(int requestId); +} diff --git a/mobile/pubspec.lock b/mobile/pubspec.lock index 929180203b..624ed1fe65 100644 --- a/mobile/pubspec.lock +++ b/mobile/pubspec.lock @@ -514,7 +514,7 @@ packages: source: hosted version: "1.3.3" ffi: - dependency: transitive + dependency: "direct main" description: name: ffi sha256: "289279317b4b16eb2bb7e271abccd4bf84ec9bdcbe999e278a94b804f5630418" diff --git a/mobile/pubspec.yaml b/mobile/pubspec.yaml index f9aa0b19a9..d98bd370b6 100644 --- a/mobile/pubspec.yaml +++ b/mobile/pubspec.yaml @@ -73,6 +73,7 @@ dependencies: wakelock_plus: ^1.2.10 worker_manager: ^7.2.3 scroll_date_picker: ^3.8.0 + ffi: ^2.1.4 native_video_player: git: