android improvements

mobile/android/app/src/main/kotlin/app/alextran/immich/MainActivity.kt

@@ -3,6 +3,8 @@ package app.alextran.immich
 import android.os.Build
 import android.os.ext.SdkExtensions
 import androidx.annotation.NonNull
+import app.alextran.immich.images.ThumbnailApi
+import app.alextran.immich.images.ThumbnailsImpl
 import app.alextran.immich.sync.NativeSyncApi
 import app.alextran.immich.sync.NativeSyncApiImpl26
 import app.alextran.immich.sync.NativeSyncApiImpl30
@@ -22,6 +24,7 @@ class MainActivity : FlutterFragmentActivity() {
       } else {
         NativeSyncApiImpl30(this)
       }
-    NativeSyncApi.setUp(flutterEngine.dartExecutor.binaryMessenger, nativeSyncApiImpl)
+      NativeSyncApi.setUp(flutterEngine.dartExecutor.binaryMessenger, nativeSyncApiImpl)
+      ThumbnailApi.setUp(flutterEngine.dartExecutor.binaryMessenger, ThumbnailsImpl(this))
   }
 }

mobile/android/app/src/main/kotlin/app/alextran/immich/images/ThumbHash.java

@@ -9,10 +9,6 @@ import java.nio.ByteBuffer;
 
 // modified to use native allocations
 public final class ThumbHash {
-  private static native long allocateNative(int size);
-
-  private static native ByteBuffer wrapAsBuffer(long address, int capacity);
-
   /**
    * Decodes a ThumbHash to an RGBA image. RGB is not be premultiplied by A.
    *
@@ -60,9 +56,8 @@ public final class ThumbHash {
     int w = Math.round(ratio > 1.0f ? 32.0f : 32.0f * ratio);
     int h = Math.round(ratio > 1.0f ? 32.0f / ratio : 32.0f);
     int size = w * h * 4;
-    long pointer = allocateNative(size);
+    long pointer = ThumbnailsImpl.allocateNative(size);
-    ByteBuffer buffer = wrapAsBuffer(pointer, size);
+    ByteBuffer rgba = ThumbnailsImpl.wrapAsBuffer(pointer, size);
-    byte[] rgba = buffer.array();
     int cx_stop = Math.max(lx, hasAlpha ? 5 : 3);
     int cy_stop = Math.max(ly, hasAlpha ? 5 : 3);
     float[] fx = new float[cx_stop];
@@ -106,10 +101,10 @@ public final class ThumbHash {
         float b = l - 2.0f / 3.0f * p;
         float r = (3.0f * l - b + q) / 2.0f;
         float g = r - q;
-        rgba[i] = (byte) Math.max(0, Math.round(255.0f * Math.min(1, r)));
+        rgba.put(i, (byte) Math.max(0, Math.round(255.0f * Math.min(1, r))));
-        rgba[i + 1] = (byte) Math.max(0, Math.round(255.0f * Math.min(1, g)));
+        rgba.put(i + 1, (byte) Math.max(0, Math.round(255.0f * Math.min(1, g))));
-        rgba[i + 2] = (byte) Math.max(0, Math.round(255.0f * Math.min(1, b)));
+        rgba.put(i + 2, (byte) Math.max(0, Math.round(255.0f * Math.min(1, b))));
-        rgba[i + 3] = (byte) Math.max(0, Math.round(255.0f * Math.min(1, a)));
+        rgba.put(i + 3, (byte) Math.max(0, Math.round(255.0f * Math.min(1, a))));
       }
     }
     return new Image(w, h, pointer);

mobile/android/app/src/main/kotlin/app/alextran/immich/images/Thumbnails.g.kt

@@ -59,7 +59,7 @@ private open class ThumbnailsPigeonCodec : StandardMessageCodec() {
 
 /** Generated interface from Pigeon that represents a handler of messages from Flutter. */
 interface ThumbnailApi {
-  fun requestImage(assetId: String, requestId: Long, width: Long, height: Long, callback: (Result<Map<String, Long>>) -> Unit)
+  fun requestImage(assetId: String, requestId: Long, width: Long, height: Long, isVideo: Boolean, callback: (Result<Map<String, Long>>) -> Unit)
   fun cancelImageRequest(requestId: Long)
   fun getThumbhash(thumbhash: String, callback: (Result<Map<String, Long>>) -> Unit)
 
@@ -81,7 +81,8 @@ interface ThumbnailApi {
             val requestIdArg = args[1] as Long
             val widthArg = args[2] as Long
             val heightArg = args[3] as Long
-            api.requestImage(assetIdArg, requestIdArg, widthArg, heightArg) { result: Result<Map<String, Long>> ->
+            val isVideoArg = args[4] as Boolean
+            api.requestImage(assetIdArg, requestIdArg, widthArg, heightArg, isVideoArg) { result: Result<Map<String, Long>> ->
               val error = result.exceptionOrNull()
               if (error != null) {
                 reply.reply(ThumbnailsPigeonUtils.wrapError(error))

mobile/android/app/src/main/kotlin/app/alextran/immich/images/ThumbnailsImpl.kt

@@ -21,6 +21,7 @@ import com.bumptech.glide.load.DecodeFormat
 import java.util.Base64
 import java.util.HashMap
 import java.util.concurrent.CancellationException
+import java.util.concurrent.ConcurrentHashMap
 import java.util.concurrent.Future
 
 data class Request(
@@ -33,9 +34,10 @@ data class Request(
 class ThumbnailsImpl(context: Context) : ThumbnailApi {
     private val ctx: Context = context.applicationContext
     private val resolver: ContentResolver = ctx.contentResolver
+    private val requestThread = Executors.newSingleThreadExecutor()
     private val threadPool =
         Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors() / 2 + 1)
-    private val requestMap = HashMap<Long, Request>()
+    private val requestMap = ConcurrentHashMap<Long, Request>()
 
     companion object {
         val PROJECTION = arrayOf(
@@ -75,9 +77,9 @@ class ThumbnailsImpl(context: Context) : ThumbnailApi {
                     "height" to image.height.toLong()
                 )
                 callback(Result.success(res))
-              } catch (e: Exception) {
+            } catch (e: Exception) {
                 callback(Result.failure(e))
-              }
+            }
         }
     }
 
@@ -86,12 +88,13 @@ class ThumbnailsImpl(context: Context) : ThumbnailApi {
         requestId: Long,
         width: Long,
         height: Long,
+        isVideo: Boolean,
         callback: (Result<Map<String, Long>>) -> Unit
     ) {
         val signal = CancellationSignal()
         val task = threadPool.submit {
             try {
-                getThumbnailBufferInternal(assetId, width, height, callback, signal)
+                getThumbnailBufferInternal(assetId, width, height, isVideo, callback, signal)
             } catch (e: Exception) {
                 when (e) {
                     is OperationCanceledException -> callback(CANCELLED)
@@ -102,7 +105,8 @@ class ThumbnailsImpl(context: Context) : ThumbnailApi {
                 requestMap.remove(requestId)
             }
         }
-        requestMap[requestId] = Request(requestId, task, signal, callback)
+        val request = Request(requestId, task, signal, callback)
+        requestMap[requestId] = request
     }
 
     override fun cancelImageRequest(requestId: Long) {
@@ -110,7 +114,12 @@ class ThumbnailsImpl(context: Context) : ThumbnailApi {
         request.taskFuture.cancel(false)
         request.cancellationSignal.cancel()
         if (request.taskFuture.isCancelled) {
-            request.callback(CANCELLED)
+            requestThread.execute {
+                try {
+                    request.callback(CANCELLED)
+                } catch (_: Exception) {
+                }
+            }
         }
     }
 
@@ -118,6 +127,7 @@ class ThumbnailsImpl(context: Context) : ThumbnailApi {
         assetId: String,
         width: Long,
         height: Long,
+        isVideo: Boolean,
         callback: (Result<Map<String, Long>>) -> Unit,
         signal: CancellationSignal
     ) {
@@ -126,24 +136,14 @@ class ThumbnailsImpl(context: Context) : ThumbnailApi {
         val targetHeight = height.toInt()
         val id = assetId.toLong()
 
-        val cursor = resolver.query(URI, PROJECTION, SELECTION, arrayOf(assetId), null)
-            ?: return callback(Result.failure(RuntimeException("Asset not found")))
-
         signal.throwIfCanceled()
-        cursor.use { c ->
+        val bitmap = if (isVideo) {
-            if (!c.moveToNext()) {
+            decodeVideoThumbnail(id, targetWidth, targetHeight, signal)
-                return callback(Result.failure(RuntimeException("Asset not found")))
+        } else {
-            }
+            decodeImage(id, targetWidth, targetHeight, signal)
-
-            val mediaType = c.getInt(1)
-            val bitmap = when (mediaType) {
-                MEDIA_TYPE_IMAGE -> decodeImage(id, targetWidth, targetHeight, signal)
-                MEDIA_TYPE_VIDEO -> decodeVideoThumbnail(id, targetWidth, targetHeight, signal)
-                else -> return callback(Result.failure(RuntimeException("Unsupported media type")))
-            }
-
-            processBitmap(bitmap, callback, signal)
         }
+
+        processBitmap(bitmap, callback, signal)
     }
 
     private fun processBitmap(
@@ -162,6 +162,7 @@ class ThumbnailsImpl(context: Context) : ThumbnailApi {
             signal.throwIfCanceled()
             val buffer = wrapAsBuffer(pointer, size)
             bitmap.copyPixelsToBuffer(buffer)
+            bitmap.recycle()
             signal.throwIfCanceled()
             val res = mapOf(
                 "pointer" to pointer,

mobile/ios/Runner/Images/Thumbnails.g.swift

@@ -70,7 +70,7 @@ class ThumbnailsPigeonCodec: FlutterStandardMessageCodec, @unchecked Sendable {
 
 /// Generated protocol from Pigeon that represents a handler of messages from Flutter.
 protocol ThumbnailApi {
-  func requestImage(assetId: String, requestId: Int64, width: Int64, height: Int64, completion: @escaping (Result<[String: Int64], Error>) -> Void)
+  func requestImage(assetId: String, requestId: Int64, width: Int64, height: Int64, isVideo: Bool, completion: @escaping (Result<[String: Int64], Error>) -> Void)
   func cancelImageRequest(requestId: Int64) throws
   func getThumbhash(thumbhash: String, completion: @escaping (Result<[String: Int64], Error>) -> Void)
 }
@@ -89,7 +89,8 @@ class ThumbnailApiSetup {
         let requestIdArg = args[1] as! Int64
         let widthArg = args[2] as! Int64
         let heightArg = args[3] as! Int64
-        api.requestImage(assetId: assetIdArg, requestId: requestIdArg, width: widthArg, height: heightArg) { result in
+        let isVideoArg = args[4] as! Bool
+        api.requestImage(assetId: assetIdArg, requestId: requestIdArg, width: widthArg, height: heightArg, isVideo: isVideoArg) { result in
           switch result {
           case .success(let res):
             reply(wrapResult(res))

mobile/lib/constants/constants.dart

@@ -28,7 +28,6 @@ const String kDownloadGroupLivePhoto = 'group_livephoto';
 const int kTimelineNoneSegmentSize = 120;
 const int kTimelineAssetLoadBatchSize = 1024;
 const int kTimelineAssetLoadOppositeSize = 64;
-const int kTimelineImageCacheMemory = 200 * 1024 * 1024;
 
 // Widget keys
 const String appShareGroupId = "group.app.immich.share";

mobile/lib/infrastructure/repositories/asset_media.repository.dart

@@ -6,6 +6,7 @@ import 'dart:ui' as ui;
 import 'package:flutter/material.dart';
 import 'package:flutter/services.dart';
 import 'package:flutter/foundation.dart';
+import 'package:immich_mobile/domain/models/asset/base_asset.model.dart';
 import 'package:immich_mobile/providers/image/cache/remote_image_cache_manager.dart';
 import 'package:immich_mobile/providers/infrastructure/platform.provider.dart';
 import 'package:ffi/ffi.dart';
@@ -111,8 +112,9 @@ class LocalImageRequest extends ImageRequest {
   final String localId;
   final int width;
   final int height;
+  final AssetType assetType;
 
-  LocalImageRequest({required this.localId, required ui.Size size})
+  LocalImageRequest({required this.localId, required ui.Size size, required this.assetType})
     : width = size.width.toInt(),
       height = size.height.toInt();
 
@@ -131,6 +133,7 @@ class LocalImageRequest extends ImageRequest {
       requestId: requestId,
       width: width,
       height: height,
+      isVideo: assetType == AssetType.video,
     );
     if (!kReleaseMode) {
       stopwatch!.stop();

mobile/lib/main.dart

@@ -71,7 +71,6 @@ Future<void> initApp() async {
     }
   }
 
-  PaintingBinding.instance.imageCache.maximumSizeBytes = kTimelineImageCacheMemory;
   await DynamicTheme.fetchSystemPalette();
 
   final log = Logger("ImmichErrorLogger");

mobile/lib/platform/thumbnail_api.g.dart

@@ -54,6 +54,7 @@ class ThumbnailApi {
     required int requestId,
     required int width,
     required int height,
+    required bool isVideo,
   }) async {
     final String pigeonVar_channelName =
         'dev.flutter.pigeon.immich_mobile.ThumbnailApi.requestImage$pigeonVar_messageChannelSuffix';
@@ -62,7 +63,13 @@ class ThumbnailApi {
       pigeonChannelCodec,
       binaryMessenger: pigeonVar_binaryMessenger,
     );
-    final Future<Object?> pigeonVar_sendFuture = pigeonVar_channel.send(<Object?>[assetId, requestId, width, height]);
+    final Future<Object?> pigeonVar_sendFuture = pigeonVar_channel.send(<Object?>[
+      assetId,
+      requestId,
+      width,
+      height,
+      isVideo,
+    ]);
     final List<Object?>? pigeonVar_replyList = await pigeonVar_sendFuture as List<Object?>?;
     if (pigeonVar_replyList == null) {
       throw _createConnectionError(pigeonVar_channelName);

mobile/lib/presentation/widgets/images/image_provider.dart

@@ -30,7 +30,7 @@ ImageProvider getFullImageProvider(BaseAsset asset, {Size size = const Size(1080
   final ImageProvider provider;
   if (_shouldUseLocalAsset(asset)) {
     final id = asset is LocalAsset ? asset.id : (asset as RemoteAsset).localId!;
-    provider = LocalFullImageProvider(id: id, size: size);
+    provider = LocalFullImageProvider(id: id, size: size, assetType: asset.type);
   } else {
     final String assetId;
     if (asset is LocalAsset && asset.hasRemote) {
@@ -55,7 +55,7 @@ ImageProvider getThumbnailImageProvider({BaseAsset? asset, String? remoteId, Siz
 
   if (_shouldUseLocalAsset(asset!)) {
     final id = asset is LocalAsset ? asset.id : (asset as RemoteAsset).localId!;
-    return LocalThumbProvider(id: id, size: size);
+    return LocalThumbProvider(id: id, size: size, assetType: asset.type);
   }
 
   final String assetId;

mobile/lib/presentation/widgets/images/local_image_provider.dart

@@ -3,6 +3,7 @@ import 'dart:ui';
 
 import 'package:flutter/foundation.dart';
 import 'package:flutter/widgets.dart';
+import 'package:immich_mobile/domain/models/asset/base_asset.model.dart';
 import 'package:immich_mobile/infrastructure/repositories/asset_media.repository.dart';
 import 'package:immich_mobile/presentation/widgets/images/image_provider.dart';
 import 'package:immich_mobile/presentation/widgets/images/one_frame_multi_image_stream_completer.dart';
@@ -11,8 +12,9 @@ import 'package:immich_mobile/presentation/widgets/timeline/constants.dart';
 class LocalThumbProvider extends ImageProvider<LocalThumbProvider> with CancellableImageProviderMixin {
   final String id;
   final Size size;
+  final AssetType assetType;
 
-  LocalThumbProvider({required this.id, this.size = kThumbnailResolution});
+  LocalThumbProvider({required this.id, required this.assetType, this.size = kThumbnailResolution});
 
   @override
   Future<LocalThumbProvider> obtainKey(ImageConfiguration configuration) {
@@ -33,7 +35,7 @@ class LocalThumbProvider extends ImageProvider<LocalThumbProvider> with Cancella
   }
 
   Stream<ImageInfo> _codec(LocalThumbProvider key, ImageDecoderCallback decode) async* {
-    final request = this.request = LocalImageRequest(localId: key.id, size: size);
+    final request = this.request = LocalImageRequest(localId: key.id, size: size, assetType: key.assetType);
     try {
       final image = await request.load(decode);
       if (image != null) {
@@ -60,8 +62,9 @@ class LocalThumbProvider extends ImageProvider<LocalThumbProvider> with Cancella
 class LocalFullImageProvider extends ImageProvider<LocalFullImageProvider> with CancellableImageProviderMixin {
   final String id;
   final Size size;
+  final AssetType assetType;
 
-  LocalFullImageProvider({required this.id, required this.size});
+  LocalFullImageProvider({required this.id, required this.assetType, required this.size});
 
   @override
   Future<LocalFullImageProvider> obtainKey(ImageConfiguration configuration) {
@@ -72,7 +75,7 @@ class LocalFullImageProvider extends ImageProvider<LocalFullImageProvider> with
   ImageStreamCompleter loadImage(LocalFullImageProvider key, ImageDecoderCallback decode) {
     final completer = OneFramePlaceholderImageStreamCompleter(
       _codec(key, decode),
-      initialImage: getCachedImage(LocalThumbProvider(id: key.id)),
+      initialImage: getCachedImage(LocalThumbProvider(id: key.id, assetType: key.assetType)),
       informationCollector: () => <DiagnosticsNode>[
         DiagnosticsProperty<ImageProvider>('Image provider', this),
         DiagnosticsProperty<String>('Id', key.id),
@@ -88,6 +91,7 @@ class LocalFullImageProvider extends ImageProvider<LocalFullImageProvider> with
     final request = this.request = LocalImageRequest(
       localId: key.id,
       size: Size(size.width * devicePixelRatio, size.height * devicePixelRatio),
+      assetType: key.assetType,
     );
 
     try {

mobile/lib/presentation/widgets/timeline/constants.dart

@@ -3,7 +3,7 @@ import 'dart:ui';
 const double kTimelineHeaderExtent = 80.0;
 const double kTimelineFixedTileExtentPixels = 256;
 const Size kTimelineFixedTileExtent = Size.square(kTimelineFixedTileExtentPixels);
-const Size kThumbnailResolution = Size.square(384);
+const Size kThumbnailResolution = Size.square(256);
 const double kTimelineSpacing = 2.0;
 const int kTimelineColumnCount = 3;
 

mobile/lib/presentation/widgets/timeline/segment_builder.dart

@@ -28,7 +28,7 @@ abstract class SegmentBuilder {
       dimension: size.height,
       spacing: spacing,
       textDirection: Directionality.of(context),
-      children: List.generate(count, (_) => ThumbnailPlaceholder(width: size.width, height: size.height)),
+      children: List.filled(count, const ThumbnailPlaceholder()),
     ),
   );
 }

mobile/pigeon/thumbnail_api.dart

@@ -15,7 +15,13 @@ import 'package:pigeon/pigeon.dart';
 @HostApi()
 abstract class ThumbnailApi {
   @async
-  Map<String, int> requestImage(String assetId, {required int requestId, required int width, required int height});
+  Map<String, int> requestImage(
+    String assetId, {
+    required int requestId,
+    required int width,
+    required int height,
+    required bool isVideo,
+  });
 
   void cancelImageRequest(int requestId);