diff --git a/mobile/android/app/src/main/kotlin/app/alextran/immich/MainActivity.kt b/mobile/android/app/src/main/kotlin/app/alextran/immich/MainActivity.kt index f9c4ee2a1f..b1a50695a3 100644 --- a/mobile/android/app/src/main/kotlin/app/alextran/immich/MainActivity.kt +++ b/mobile/android/app/src/main/kotlin/app/alextran/immich/MainActivity.kt @@ -3,6 +3,8 @@ package app.alextran.immich import android.os.Build import android.os.ext.SdkExtensions import androidx.annotation.NonNull +import app.alextran.immich.images.ThumbnailApi +import app.alextran.immich.images.ThumbnailsImpl import app.alextran.immich.sync.NativeSyncApi import app.alextran.immich.sync.NativeSyncApiImpl26 import app.alextran.immich.sync.NativeSyncApiImpl30 @@ -22,6 +24,7 @@ class MainActivity : FlutterFragmentActivity() { } else { NativeSyncApiImpl30(this) } - NativeSyncApi.setUp(flutterEngine.dartExecutor.binaryMessenger, nativeSyncApiImpl) + NativeSyncApi.setUp(flutterEngine.dartExecutor.binaryMessenger, nativeSyncApiImpl) + ThumbnailApi.setUp(flutterEngine.dartExecutor.binaryMessenger, ThumbnailsImpl(this)) } } diff --git a/mobile/android/app/src/main/kotlin/app/alextran/immich/images/ThumbHash.java b/mobile/android/app/src/main/kotlin/app/alextran/immich/images/ThumbHash.java index 4f14270ee4..3af76b5763 100644 --- a/mobile/android/app/src/main/kotlin/app/alextran/immich/images/ThumbHash.java +++ b/mobile/android/app/src/main/kotlin/app/alextran/immich/images/ThumbHash.java @@ -9,10 +9,6 @@ import java.nio.ByteBuffer; // modified to use native allocations public final class ThumbHash { - private static native long allocateNative(int size); - - private static native ByteBuffer wrapAsBuffer(long address, int capacity); - /** * Decodes a ThumbHash to an RGBA image. RGB is not be premultiplied by A. * @@ -60,9 +56,8 @@ public final class ThumbHash { int w = Math.round(ratio > 1.0f ? 32.0f : 32.0f * ratio); int h = Math.round(ratio > 1.0f ? 32.0f / ratio : 32.0f); int size = w * h * 4; - long pointer = allocateNative(size); - ByteBuffer buffer = wrapAsBuffer(pointer, size); - byte[] rgba = buffer.array(); + long pointer = ThumbnailsImpl.allocateNative(size); + ByteBuffer rgba = ThumbnailsImpl.wrapAsBuffer(pointer, size); int cx_stop = Math.max(lx, hasAlpha ? 5 : 3); int cy_stop = Math.max(ly, hasAlpha ? 5 : 3); float[] fx = new float[cx_stop]; @@ -106,10 +101,10 @@ public final class ThumbHash { float b = l - 2.0f / 3.0f * p; float r = (3.0f * l - b + q) / 2.0f; float g = r - q; - rgba[i] = (byte) Math.max(0, Math.round(255.0f * Math.min(1, r))); - rgba[i + 1] = (byte) Math.max(0, Math.round(255.0f * Math.min(1, g))); - rgba[i + 2] = (byte) Math.max(0, Math.round(255.0f * Math.min(1, b))); - rgba[i + 3] = (byte) Math.max(0, Math.round(255.0f * Math.min(1, a))); + rgba.put(i, (byte) Math.max(0, Math.round(255.0f * Math.min(1, r)))); + rgba.put(i + 1, (byte) Math.max(0, Math.round(255.0f * Math.min(1, g)))); + rgba.put(i + 2, (byte) Math.max(0, Math.round(255.0f * Math.min(1, b)))); + rgba.put(i + 3, (byte) Math.max(0, Math.round(255.0f * Math.min(1, a)))); } } return new Image(w, h, pointer); diff --git a/mobile/android/app/src/main/kotlin/app/alextran/immich/images/Thumbnails.g.kt b/mobile/android/app/src/main/kotlin/app/alextran/immich/images/Thumbnails.g.kt index 4620240800..e9993a3c45 100644 --- a/mobile/android/app/src/main/kotlin/app/alextran/immich/images/Thumbnails.g.kt +++ b/mobile/android/app/src/main/kotlin/app/alextran/immich/images/Thumbnails.g.kt @@ -59,7 +59,7 @@ private open class ThumbnailsPigeonCodec : StandardMessageCodec() { /** Generated interface from Pigeon that represents a handler of messages from Flutter. */ interface ThumbnailApi { - fun requestImage(assetId: String, requestId: Long, width: Long, height: Long, callback: (Result>) -> Unit) + fun requestImage(assetId: String, requestId: Long, width: Long, height: Long, isVideo: Boolean, callback: (Result>) -> Unit) fun cancelImageRequest(requestId: Long) fun getThumbhash(thumbhash: String, callback: (Result>) -> Unit) @@ -81,7 +81,8 @@ interface ThumbnailApi { val requestIdArg = args[1] as Long val widthArg = args[2] as Long val heightArg = args[3] as Long - api.requestImage(assetIdArg, requestIdArg, widthArg, heightArg) { result: Result> -> + val isVideoArg = args[4] as Boolean + api.requestImage(assetIdArg, requestIdArg, widthArg, heightArg, isVideoArg) { result: Result> -> val error = result.exceptionOrNull() if (error != null) { reply.reply(ThumbnailsPigeonUtils.wrapError(error)) diff --git a/mobile/android/app/src/main/kotlin/app/alextran/immich/images/ThumbnailsImpl.kt b/mobile/android/app/src/main/kotlin/app/alextran/immich/images/ThumbnailsImpl.kt index 80dadbb26d..947fdae810 100644 --- a/mobile/android/app/src/main/kotlin/app/alextran/immich/images/ThumbnailsImpl.kt +++ b/mobile/android/app/src/main/kotlin/app/alextran/immich/images/ThumbnailsImpl.kt @@ -21,6 +21,7 @@ import com.bumptech.glide.load.DecodeFormat import java.util.Base64 import java.util.HashMap import java.util.concurrent.CancellationException +import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.Future data class Request( @@ -33,9 +34,10 @@ data class Request( class ThumbnailsImpl(context: Context) : ThumbnailApi { private val ctx: Context = context.applicationContext private val resolver: ContentResolver = ctx.contentResolver + private val requestThread = Executors.newSingleThreadExecutor() private val threadPool = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors() / 2 + 1) - private val requestMap = HashMap() + private val requestMap = ConcurrentHashMap() companion object { val PROJECTION = arrayOf( @@ -75,9 +77,9 @@ class ThumbnailsImpl(context: Context) : ThumbnailApi { "height" to image.height.toLong() ) callback(Result.success(res)) - } catch (e: Exception) { + } catch (e: Exception) { callback(Result.failure(e)) - } + } } } @@ -86,12 +88,13 @@ class ThumbnailsImpl(context: Context) : ThumbnailApi { requestId: Long, width: Long, height: Long, + isVideo: Boolean, callback: (Result>) -> Unit ) { val signal = CancellationSignal() val task = threadPool.submit { try { - getThumbnailBufferInternal(assetId, width, height, callback, signal) + getThumbnailBufferInternal(assetId, width, height, isVideo, callback, signal) } catch (e: Exception) { when (e) { is OperationCanceledException -> callback(CANCELLED) @@ -102,7 +105,8 @@ class ThumbnailsImpl(context: Context) : ThumbnailApi { requestMap.remove(requestId) } } - requestMap[requestId] = Request(requestId, task, signal, callback) + val request = Request(requestId, task, signal, callback) + requestMap[requestId] = request } override fun cancelImageRequest(requestId: Long) { @@ -110,7 +114,12 @@ class ThumbnailsImpl(context: Context) : ThumbnailApi { request.taskFuture.cancel(false) request.cancellationSignal.cancel() if (request.taskFuture.isCancelled) { - request.callback(CANCELLED) + requestThread.execute { + try { + request.callback(CANCELLED) + } catch (_: Exception) { + } + } } } @@ -118,6 +127,7 @@ class ThumbnailsImpl(context: Context) : ThumbnailApi { assetId: String, width: Long, height: Long, + isVideo: Boolean, callback: (Result>) -> Unit, signal: CancellationSignal ) { @@ -126,24 +136,14 @@ class ThumbnailsImpl(context: Context) : ThumbnailApi { val targetHeight = height.toInt() val id = assetId.toLong() - val cursor = resolver.query(URI, PROJECTION, SELECTION, arrayOf(assetId), null) - ?: return callback(Result.failure(RuntimeException("Asset not found"))) - signal.throwIfCanceled() - cursor.use { c -> - if (!c.moveToNext()) { - return callback(Result.failure(RuntimeException("Asset not found"))) - } - - val mediaType = c.getInt(1) - val bitmap = when (mediaType) { - MEDIA_TYPE_IMAGE -> decodeImage(id, targetWidth, targetHeight, signal) - MEDIA_TYPE_VIDEO -> decodeVideoThumbnail(id, targetWidth, targetHeight, signal) - else -> return callback(Result.failure(RuntimeException("Unsupported media type"))) - } - - processBitmap(bitmap, callback, signal) + val bitmap = if (isVideo) { + decodeVideoThumbnail(id, targetWidth, targetHeight, signal) + } else { + decodeImage(id, targetWidth, targetHeight, signal) } + + processBitmap(bitmap, callback, signal) } private fun processBitmap( @@ -162,6 +162,7 @@ class ThumbnailsImpl(context: Context) : ThumbnailApi { signal.throwIfCanceled() val buffer = wrapAsBuffer(pointer, size) bitmap.copyPixelsToBuffer(buffer) + bitmap.recycle() signal.throwIfCanceled() val res = mapOf( "pointer" to pointer, diff --git a/mobile/ios/Runner/Images/Thumbnails.g.swift b/mobile/ios/Runner/Images/Thumbnails.g.swift index d509ac78da..be40a18b41 100644 --- a/mobile/ios/Runner/Images/Thumbnails.g.swift +++ b/mobile/ios/Runner/Images/Thumbnails.g.swift @@ -70,7 +70,7 @@ class ThumbnailsPigeonCodec: FlutterStandardMessageCodec, @unchecked Sendable { /// Generated protocol from Pigeon that represents a handler of messages from Flutter. protocol ThumbnailApi { - func requestImage(assetId: String, requestId: Int64, width: Int64, height: Int64, completion: @escaping (Result<[String: Int64], Error>) -> Void) + func requestImage(assetId: String, requestId: Int64, width: Int64, height: Int64, isVideo: Bool, completion: @escaping (Result<[String: Int64], Error>) -> Void) func cancelImageRequest(requestId: Int64) throws func getThumbhash(thumbhash: String, completion: @escaping (Result<[String: Int64], Error>) -> Void) } @@ -89,7 +89,8 @@ class ThumbnailApiSetup { let requestIdArg = args[1] as! Int64 let widthArg = args[2] as! Int64 let heightArg = args[3] as! Int64 - api.requestImage(assetId: assetIdArg, requestId: requestIdArg, width: widthArg, height: heightArg) { result in + let isVideoArg = args[4] as! Bool + api.requestImage(assetId: assetIdArg, requestId: requestIdArg, width: widthArg, height: heightArg, isVideo: isVideoArg) { result in switch result { case .success(let res): reply(wrapResult(res)) diff --git a/mobile/lib/constants/constants.dart b/mobile/lib/constants/constants.dart index 63af4ffff8..0cfc0c57e3 100644 --- a/mobile/lib/constants/constants.dart +++ b/mobile/lib/constants/constants.dart @@ -28,7 +28,6 @@ const String kDownloadGroupLivePhoto = 'group_livephoto'; const int kTimelineNoneSegmentSize = 120; const int kTimelineAssetLoadBatchSize = 1024; const int kTimelineAssetLoadOppositeSize = 64; -const int kTimelineImageCacheMemory = 200 * 1024 * 1024; // Widget keys const String appShareGroupId = "group.app.immich.share"; diff --git a/mobile/lib/infrastructure/repositories/asset_media.repository.dart b/mobile/lib/infrastructure/repositories/asset_media.repository.dart index 19f164fd73..0c321f207a 100644 --- a/mobile/lib/infrastructure/repositories/asset_media.repository.dart +++ b/mobile/lib/infrastructure/repositories/asset_media.repository.dart @@ -6,6 +6,7 @@ import 'dart:ui' as ui; import 'package:flutter/material.dart'; import 'package:flutter/services.dart'; import 'package:flutter/foundation.dart'; +import 'package:immich_mobile/domain/models/asset/base_asset.model.dart'; import 'package:immich_mobile/providers/image/cache/remote_image_cache_manager.dart'; import 'package:immich_mobile/providers/infrastructure/platform.provider.dart'; import 'package:ffi/ffi.dart'; @@ -111,8 +112,9 @@ class LocalImageRequest extends ImageRequest { final String localId; final int width; final int height; + final AssetType assetType; - LocalImageRequest({required this.localId, required ui.Size size}) + LocalImageRequest({required this.localId, required ui.Size size, required this.assetType}) : width = size.width.toInt(), height = size.height.toInt(); @@ -131,6 +133,7 @@ class LocalImageRequest extends ImageRequest { requestId: requestId, width: width, height: height, + isVideo: assetType == AssetType.video, ); if (!kReleaseMode) { stopwatch!.stop(); diff --git a/mobile/lib/main.dart b/mobile/lib/main.dart index 883840efa6..19ed746833 100644 --- a/mobile/lib/main.dart +++ b/mobile/lib/main.dart @@ -71,7 +71,6 @@ Future initApp() async { } } - PaintingBinding.instance.imageCache.maximumSizeBytes = kTimelineImageCacheMemory; await DynamicTheme.fetchSystemPalette(); final log = Logger("ImmichErrorLogger"); diff --git a/mobile/lib/platform/thumbnail_api.g.dart b/mobile/lib/platform/thumbnail_api.g.dart index d200dd4106..2b4add7482 100644 --- a/mobile/lib/platform/thumbnail_api.g.dart +++ b/mobile/lib/platform/thumbnail_api.g.dart @@ -54,6 +54,7 @@ class ThumbnailApi { required int requestId, required int width, required int height, + required bool isVideo, }) async { final String pigeonVar_channelName = 'dev.flutter.pigeon.immich_mobile.ThumbnailApi.requestImage$pigeonVar_messageChannelSuffix'; @@ -62,7 +63,13 @@ class ThumbnailApi { pigeonChannelCodec, binaryMessenger: pigeonVar_binaryMessenger, ); - final Future pigeonVar_sendFuture = pigeonVar_channel.send([assetId, requestId, width, height]); + final Future pigeonVar_sendFuture = pigeonVar_channel.send([ + assetId, + requestId, + width, + height, + isVideo, + ]); final List? pigeonVar_replyList = await pigeonVar_sendFuture as List?; if (pigeonVar_replyList == null) { throw _createConnectionError(pigeonVar_channelName); diff --git a/mobile/lib/presentation/widgets/images/image_provider.dart b/mobile/lib/presentation/widgets/images/image_provider.dart index b428937bc0..21ef54e690 100644 --- a/mobile/lib/presentation/widgets/images/image_provider.dart +++ b/mobile/lib/presentation/widgets/images/image_provider.dart @@ -30,7 +30,7 @@ ImageProvider getFullImageProvider(BaseAsset asset, {Size size = const Size(1080 final ImageProvider provider; if (_shouldUseLocalAsset(asset)) { final id = asset is LocalAsset ? asset.id : (asset as RemoteAsset).localId!; - provider = LocalFullImageProvider(id: id, size: size); + provider = LocalFullImageProvider(id: id, size: size, assetType: asset.type); } else { final String assetId; if (asset is LocalAsset && asset.hasRemote) { @@ -55,7 +55,7 @@ ImageProvider getThumbnailImageProvider({BaseAsset? asset, String? remoteId, Siz if (_shouldUseLocalAsset(asset!)) { final id = asset is LocalAsset ? asset.id : (asset as RemoteAsset).localId!; - return LocalThumbProvider(id: id, size: size); + return LocalThumbProvider(id: id, size: size, assetType: asset.type); } final String assetId; diff --git a/mobile/lib/presentation/widgets/images/local_image_provider.dart b/mobile/lib/presentation/widgets/images/local_image_provider.dart index e85033dcad..e676a7eb71 100644 --- a/mobile/lib/presentation/widgets/images/local_image_provider.dart +++ b/mobile/lib/presentation/widgets/images/local_image_provider.dart @@ -3,6 +3,7 @@ import 'dart:ui'; import 'package:flutter/foundation.dart'; import 'package:flutter/widgets.dart'; +import 'package:immich_mobile/domain/models/asset/base_asset.model.dart'; import 'package:immich_mobile/infrastructure/repositories/asset_media.repository.dart'; import 'package:immich_mobile/presentation/widgets/images/image_provider.dart'; import 'package:immich_mobile/presentation/widgets/images/one_frame_multi_image_stream_completer.dart'; @@ -11,8 +12,9 @@ import 'package:immich_mobile/presentation/widgets/timeline/constants.dart'; class LocalThumbProvider extends ImageProvider with CancellableImageProviderMixin { final String id; final Size size; + final AssetType assetType; - LocalThumbProvider({required this.id, this.size = kThumbnailResolution}); + LocalThumbProvider({required this.id, required this.assetType, this.size = kThumbnailResolution}); @override Future obtainKey(ImageConfiguration configuration) { @@ -33,7 +35,7 @@ class LocalThumbProvider extends ImageProvider with Cancella } Stream _codec(LocalThumbProvider key, ImageDecoderCallback decode) async* { - final request = this.request = LocalImageRequest(localId: key.id, size: size); + final request = this.request = LocalImageRequest(localId: key.id, size: size, assetType: key.assetType); try { final image = await request.load(decode); if (image != null) { @@ -60,8 +62,9 @@ class LocalThumbProvider extends ImageProvider with Cancella class LocalFullImageProvider extends ImageProvider with CancellableImageProviderMixin { final String id; final Size size; + final AssetType assetType; - LocalFullImageProvider({required this.id, required this.size}); + LocalFullImageProvider({required this.id, required this.assetType, required this.size}); @override Future obtainKey(ImageConfiguration configuration) { @@ -72,7 +75,7 @@ class LocalFullImageProvider extends ImageProvider with ImageStreamCompleter loadImage(LocalFullImageProvider key, ImageDecoderCallback decode) { final completer = OneFramePlaceholderImageStreamCompleter( _codec(key, decode), - initialImage: getCachedImage(LocalThumbProvider(id: key.id)), + initialImage: getCachedImage(LocalThumbProvider(id: key.id, assetType: key.assetType)), informationCollector: () => [ DiagnosticsProperty('Image provider', this), DiagnosticsProperty('Id', key.id), @@ -88,6 +91,7 @@ class LocalFullImageProvider extends ImageProvider with final request = this.request = LocalImageRequest( localId: key.id, size: Size(size.width * devicePixelRatio, size.height * devicePixelRatio), + assetType: key.assetType, ); try { diff --git a/mobile/lib/presentation/widgets/timeline/constants.dart b/mobile/lib/presentation/widgets/timeline/constants.dart index 68903f0e82..7a9a4c8719 100644 --- a/mobile/lib/presentation/widgets/timeline/constants.dart +++ b/mobile/lib/presentation/widgets/timeline/constants.dart @@ -3,7 +3,7 @@ import 'dart:ui'; const double kTimelineHeaderExtent = 80.0; const double kTimelineFixedTileExtentPixels = 256; const Size kTimelineFixedTileExtent = Size.square(kTimelineFixedTileExtentPixels); -const Size kThumbnailResolution = Size.square(384); +const Size kThumbnailResolution = Size.square(256); const double kTimelineSpacing = 2.0; const int kTimelineColumnCount = 3; diff --git a/mobile/lib/presentation/widgets/timeline/segment_builder.dart b/mobile/lib/presentation/widgets/timeline/segment_builder.dart index 79ffb47e95..a757646cab 100644 --- a/mobile/lib/presentation/widgets/timeline/segment_builder.dart +++ b/mobile/lib/presentation/widgets/timeline/segment_builder.dart @@ -28,7 +28,7 @@ abstract class SegmentBuilder { dimension: size.height, spacing: spacing, textDirection: Directionality.of(context), - children: List.generate(count, (_) => ThumbnailPlaceholder(width: size.width, height: size.height)), + children: List.filled(count, const ThumbnailPlaceholder()), ), ); } diff --git a/mobile/pigeon/thumbnail_api.dart b/mobile/pigeon/thumbnail_api.dart index 5e8421127b..0698e7cdc9 100644 --- a/mobile/pigeon/thumbnail_api.dart +++ b/mobile/pigeon/thumbnail_api.dart @@ -15,7 +15,13 @@ import 'package:pigeon/pigeon.dart'; @HostApi() abstract class ThumbnailApi { @async - Map requestImage(String assetId, {required int requestId, required int width, required int height}); + Map requestImage( + String assetId, { + required int requestId, + required int width, + required int height, + required bool isVideo, + }); void cancelImageRequest(int requestId);