From 0d0096d55330f2508ced2fdd89512dd0421c32d8 Mon Sep 17 00:00:00 2001 From: Santo Shakil Date: Thu, 6 Aug 2026 02:02:59 +0600 Subject: [PATCH] heal no-exif dates from mediastore in the v27 migration --- .../alextran/immich/sync/MessagesImpl30.kt | 3 +- .../alextran/immich/sync/MessagesImplBase.kt | 18 +- mobile/ios/Runner/Sync/MessagesImpl.swift | 2 + mobile/lib/main.dart | 4 +- mobile/lib/utils/migration.dart | 78 +++++++-- mobile/pigeon/native_sync_api.dart | 2 + mobile/test/modules/utils/migration_test.dart | 156 +++++++++++++----- 7 files changed, 205 insertions(+), 58 deletions(-) diff --git a/mobile/android/app/src/main/kotlin/app/alextran/immich/sync/MessagesImpl30.kt b/mobile/android/app/src/main/kotlin/app/alextran/immich/sync/MessagesImpl30.kt index 4785b751c0..4b09d97ea9 100644 --- a/mobile/android/app/src/main/kotlin/app/alextran/immich/sync/MessagesImpl30.kt +++ b/mobile/android/app/src/main/kotlin/app/alextran/immich/sync/MessagesImpl30.kt @@ -113,7 +113,8 @@ class NativeSyncApiImpl30(context: Context) : NativeSyncApiImplBase(context), Na putInt(MediaStore.QUERY_ARG_MATCH_TRASHED, MediaStore.MATCH_ONLY) } - getCursor(volume, queryArgs).use { cursor -> + val cursor = getCursor(volume, queryArgs) ?: error("MediaStore trash query failed") + cursor.use { getAssets(cursor).forEach { res -> if (res is AssetResult.ValidAsset) { result.getOrPut(res.albumId) { mutableListOf() }.add(res.asset) diff --git a/mobile/android/app/src/main/kotlin/app/alextran/immich/sync/MessagesImplBase.kt b/mobile/android/app/src/main/kotlin/app/alextran/immich/sync/MessagesImplBase.kt index d6d8e2283d..e4284dd148 100644 --- a/mobile/android/app/src/main/kotlin/app/alextran/immich/sync/MessagesImplBase.kt +++ b/mobile/android/app/src/main/kotlin/app/alextran/immich/sync/MessagesImplBase.kt @@ -1,8 +1,10 @@ package app.alextran.immich.sync +import android.Manifest import android.annotation.SuppressLint import android.content.ContentUris import android.content.Context +import android.content.pm.PackageManager import android.database.Cursor import android.os.Build import android.os.Bundle @@ -10,6 +12,7 @@ import android.os.ext.SdkExtensions import android.provider.MediaStore import android.util.Base64 import android.util.Log +import androidx.core.content.ContextCompat import androidx.core.database.getStringOrNull import app.alextran.immich.core.ImmichPlugin import com.bumptech.glide.Glide @@ -108,6 +111,12 @@ open class NativeSyncApiImplBase(context: Context) : ImmichPlugin(), ActivityAwa SdkExtensions.getExtensionVersion(Build.VERSION_CODES.S) >= 21) } + fun hasMediaReadPermission(): Boolean = + (if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { + arrayOf(Manifest.permission.READ_MEDIA_IMAGES, Manifest.permission.READ_MEDIA_VIDEO) + } else arrayOf(Manifest.permission.READ_EXTERNAL_STORAGE)) + .all { ContextCompat.checkSelfPermission(ctx, it) == PackageManager.PERMISSION_GRANTED } + protected fun getCursor( volume: String, selection: String, @@ -315,13 +324,14 @@ open class NativeSyncApiImplBase(context: Context) : ImmichPlugin(), ActivityAwa val selection = "(${MediaStore.Files.FileColumns.BUCKET_ID} IS NOT NULL) AND $MEDIA_SELECTION" - getCursor( + val cursor = getCursor( MediaStore.VOLUME_EXTERNAL, selection, MEDIA_SELECTION_ARGS, projection, "${MediaStore.Files.FileColumns.DATE_MODIFIED} DESC" - )?.use { cursor -> + ) ?: error("MediaStore album query failed") + cursor.use { val bucketIdColumn = cursor.getColumnIndexOrThrow(MediaStore.Files.FileColumns.BUCKET_ID) val bucketNameColumn = @@ -396,7 +406,9 @@ open class NativeSyncApiImplBase(context: Context) : ImmichPlugin(), ActivityAwa selectionArgs.addAll(listOf(updatedTimeCond.toString(), updatedTimeCond.toString())) } - return getAssets(getCursor(MediaStore.VOLUME_EXTERNAL, selection, selectionArgs.toTypedArray())) + val cursor = getCursor(MediaStore.VOLUME_EXTERNAL, selection, selectionArgs.toTypedArray()) + ?: error("MediaStore asset query failed") + return getAssets(cursor) .mapNotNull { result -> (result as? AssetResult.ValidAsset)?.asset } .toList() } diff --git a/mobile/ios/Runner/Sync/MessagesImpl.swift b/mobile/ios/Runner/Sync/MessagesImpl.swift index ddfd023690..c7de43a439 100644 --- a/mobile/ios/Runner/Sync/MessagesImpl.swift +++ b/mobile/ios/Runner/Sync/MessagesImpl.swift @@ -46,6 +46,8 @@ class NativeSyncApiImpl: ImmichPlugin, NativeSyncApi, FlutterPlugin { init(with defaults: UserDefaults = .standard) { self.defaults = defaults } + + func hasMediaReadPermission() throws -> Bool { PHPhotoLibrary.authorizationStatus(for: .readWrite) == .authorized } @available(iOS 16, *) private func getChangeToken() -> PHPersistentChangeToken? { diff --git a/mobile/lib/main.dart b/mobile/lib/main.dart index 317733f1de..11fec728ba 100644 --- a/mobile/lib/main.dart +++ b/mobile/lib/main.dart @@ -22,6 +22,8 @@ import 'package:immich_mobile/generated/translations.g.dart'; import 'package:immich_mobile/infrastructure/repositories/network.repository.dart'; import 'package:immich_mobile/pages/common/splash_screen.page.dart'; import 'package:immich_mobile/platform/background_worker_lock_api.g.dart'; +import 'package:immich_mobile/platform/native_sync_api.g.dart'; +import 'package:immich_mobile/platform/permission_api.g.dart'; import 'package:immich_mobile/providers/app_life_cycle.provider.dart'; import 'package:immich_mobile/providers/asset_viewer/share_intent_upload.provider.dart'; import 'package:immich_mobile/providers/infrastructure/db.provider.dart'; @@ -56,7 +58,7 @@ void main() async { await initApp(); // Warm-up isolate pool for worker manager await workerManagerPatch.init(dynamicSpawning: true, isolatesCount: max(Platform.numberOfProcessors - 1, 5)); - await migrateDatabaseIfNeeded(drift); + await migrateDatabaseIfNeeded(drift, NativeSyncApi(), PermissionApi()); runApp(ProviderScope(overrides: [driftProvider.overrideWith(driftOverride(drift))], child: const MainWidget())); } catch (error, stack) { diff --git a/mobile/lib/utils/migration.dart b/mobile/lib/utils/migration.dart index c2b1dd0433..a5988c9f4a 100644 --- a/mobile/lib/utils/migration.dart +++ b/mobile/lib/utils/migration.dart @@ -5,6 +5,7 @@ import 'package:collection/collection.dart'; import 'package:drift/drift.dart'; import 'package:flutter/material.dart'; import 'package:immich_mobile/constants/colors.dart'; +import 'package:immich_mobile/constants/constants.dart'; import 'package:immich_mobile/constants/enums.dart'; import 'package:immich_mobile/domain/models/config/app_config.dart'; import 'package:immich_mobile/domain/models/log.model.dart'; @@ -14,16 +15,21 @@ import 'package:immich_mobile/domain/models/timeline.model.dart'; import 'package:immich_mobile/domain/services/feature_message.service.dart'; import 'package:immich_mobile/entities/store.entity.dart'; import 'package:immich_mobile/extensions/platform_extensions.dart'; +import 'package:immich_mobile/infrastructure/entities/local_asset.entity.drift.dart'; import 'package:immich_mobile/infrastructure/entities/settings.entity.drift.dart'; +import 'package:immich_mobile/infrastructure/entities/trashed_local_asset.entity.drift.dart'; import 'package:immich_mobile/infrastructure/repositories/db.repository.dart'; import 'package:immich_mobile/infrastructure/repositories/network.repository.dart'; import 'package:immich_mobile/infrastructure/repositories/settings.repository.dart'; import 'package:immich_mobile/models/auth/auxilary_endpoint.model.dart'; +import 'package:immich_mobile/platform/native_sync_api.g.dart'; +import 'package:immich_mobile/platform/permission_api.g.dart'; import 'package:immich_mobile/providers/album/album_sort_by_options.provider.dart'; +import 'package:immich_mobile/utils/datetime_helpers.dart'; const int targetVersion = 27; -Future migrateDatabaseIfNeeded(Drift drift) async { +Future migrateDatabaseIfNeeded(Drift drift, NativeSyncApi nativeSyncApi, PermissionApi permissionApi) async { final int? storedVersion = Store.tryGet(StoreKey.version); final version = storedVersion ?? targetVersion; @@ -35,7 +41,7 @@ Future migrateDatabaseIfNeeded(Drift drift) async { await _migrateTo26(drift); } - if (version < 27 && !await _migrateTo27(drift)) { + if (version < 27 && !await _migrateTo27(drift, nativeSyncApi, permissionApi)) { await Store.put(StoreKey.version, 26); return; } @@ -48,22 +54,66 @@ Future migrateDatabaseIfNeeded(Drift drift) async { return; } -Future _migrateTo27(Drift drift) async { - // DATE_ADDED can be later than DATE_MODIFIED after a file is copied. +Future _migrateTo27(Drift drift, NativeSyncApi nativeSyncApi, PermissionApi permissionApi) async { if (!CurrentPlatform.isAndroid) { return true; } try { - await drift.customStatement( - "UPDATE local_asset_entity SET created_at = updated_at " - "WHERE julianday(updated_at) > julianday('1970-01-01T00:00:00Z') " - "AND julianday(created_at) > julianday(updated_at)", - ); - await drift.customStatement( - "UPDATE trashed_local_asset_entity SET created_at = updated_at " - "WHERE julianday(updated_at) > julianday('1970-01-01T00:00:00Z') " - "AND julianday(created_at) > julianday(updated_at)", - ); + if (!await nativeSyncApi.hasMediaReadPermission()) { + return false; + } + + final dates = {}; + void addDates(Iterable assets) { + for (final asset in assets) { + dates[asset.id] = tryFromSecondsSinceEpoch(asset.createdAt, isUtc: true) ?? DateTime.timestamp(); + } + } + + for (final album in await nativeSyncApi.getAlbums()) { + addDates(await nativeSyncApi.getAssetsForAlbum(album.id)); + } + if (await permissionApi.hasManageMediaPermission()) { + final trashed = await nativeSyncApi.getTrashedAssets(); + addDates(trashed.values.flattened); + } + + await drift.transaction(() async { + final localDates = { + for (final row in await (drift.selectOnly( + drift.localAssetEntity, + )..addColumns([drift.localAssetEntity.id, drift.localAssetEntity.createdAt])).get()) + row.read(drift.localAssetEntity.id)!: row.read(drift.localAssetEntity.createdAt)!, + }; + final trashedDates = { + for (final row in await (drift.selectOnly( + drift.trashedLocalAssetEntity, + )..addColumns([drift.trashedLocalAssetEntity.id, drift.trashedLocalAssetEntity.createdAt])).get()) + row.read(drift.trashedLocalAssetEntity.id)!: row.read(drift.trashedLocalAssetEntity.createdAt)!, + }; + for (final chunk in dates.entries.slices(kDriftMaxChunk)) { + await drift.batch((batch) { + for (final entry in chunk) { + final localDate = localDates[entry.key]; + if (localDate != null && !localDate.isAtSameMomentAs(entry.value)) { + batch.update( + drift.localAssetEntity, + LocalAssetEntityCompanion(createdAt: Value(entry.value)), + where: (row) => row.id.equals(entry.key), + ); + } + final trashedDate = trashedDates[entry.key]; + if (trashedDate != null && !trashedDate.isAtSameMomentAs(entry.value)) { + batch.update( + drift.trashedLocalAssetEntity, + TrashedLocalAssetEntityCompanion(createdAt: Value(entry.value)), + where: (row) => row.id.equals(entry.key), + ); + } + } + }); + } + }); return true; } catch (_) { return false; diff --git a/mobile/pigeon/native_sync_api.dart b/mobile/pigeon/native_sync_api.dart index 433b154cd1..2e80a18492 100644 --- a/mobile/pigeon/native_sync_api.dart +++ b/mobile/pigeon/native_sync_api.dart @@ -105,6 +105,8 @@ class CloudIdResult { @HostApi() abstract class NativeSyncApi { + bool hasMediaReadPermission(); + @async bool shouldFullSync(); diff --git a/mobile/test/modules/utils/migration_test.dart b/mobile/test/modules/utils/migration_test.dart index dba05dff5f..258024f150 100644 --- a/mobile/test/modules/utils/migration_test.dart +++ b/mobile/test/modules/utils/migration_test.dart @@ -5,14 +5,20 @@ import 'package:immich_mobile/domain/services/store.service.dart'; import 'package:immich_mobile/entities/store.entity.dart'; import 'package:immich_mobile/infrastructure/entities/trashed_local_asset.entity.dart'; import 'package:immich_mobile/infrastructure/repositories/store.repository.dart'; +import 'package:immich_mobile/platform/native_sync_api.g.dart'; +import 'package:immich_mobile/platform/permission_api.g.dart'; import 'package:immich_mobile/utils/migration.dart'; +import 'package:mocktail/mocktail.dart'; import '../../medium/repository_context.dart'; +import '../../service.mocks.dart'; void main() { TestWidgetsFlutterBinding.ensureInitialized(); late MediumRepositoryContext ctx; + late MockNativeSyncApi nativeSyncApi; + late MockPermissionApi permissionApi; setUpAll(() async { debugDefaultTargetPlatformOverride = TargetPlatform.android; @@ -21,10 +27,18 @@ void main() { }); setUp(() async { + nativeSyncApi = MockNativeSyncApi(); + permissionApi = MockPermissionApi(); await Store.clear(); await ctx.db.delete(ctx.db.localAssetEntity).go(); await ctx.db.delete(ctx.db.trashedLocalAssetEntity).go(); - await ctx.db.customStatement('DROP TRIGGER IF EXISTS fail_migration'); + when(() => nativeSyncApi.hasMediaReadPermission()).thenAnswer((_) async => true); + when(() => permissionApi.hasManageMediaPermission()).thenAnswer((_) async => true); + when( + () => nativeSyncApi.getAlbums(), + ).thenAnswer((_) async => [PlatformAlbum(id: 'album', name: 'album', isCloud: false, assetCount: 1)]); + when(() => nativeSyncApi.getAssetsForAlbum('album')).thenAnswer((_) async => []); + when(() => nativeSyncApi.getTrashedAssets()).thenAnswer((_) async => {}); }); tearDownAll(() async { @@ -33,57 +47,121 @@ void main() { await ctx.dispose(); }); - test('stores version 26 when migration 27 fails', () async { - await Store.put(StoreKey.version, 25); - await ctx.newLocalAsset(id: 'asset', createdAt: DateTime(2026), updatedAt: DateTime(2025)); - await ctx.db.customStatement( - "CREATE TRIGGER fail_migration BEFORE UPDATE OF created_at ON local_asset_entity " - "BEGIN SELECT RAISE(FAIL, 'migration failed'); END", - ); - - await migrateDatabaseIfNeeded(ctx.db); - - expect(Store.tryGet(StoreKey.version), 26); - }); - - test('fixes dates in active and trashed assets', () async { - final createdAt = DateTime(2026); - final updatedAt = DateTime(2025); - final epoch = DateTime.fromMillisecondsSinceEpoch(0); + test('heals dates from MediaStore', () async { + final wrongDate = DateTime.utc(2026); + final platformDate = DateTime.utc(2019, 3, 15, 10, 30); await Store.put(StoreKey.version, 26); - await ctx.newLocalAsset(id: 'local', createdAt: createdAt, updatedAt: updatedAt); - await ctx.newLocalAsset(id: 'epoch', createdAt: createdAt, updatedAt: epoch); + await ctx.newLocalAsset(id: 'local', createdAt: wrongDate, updatedAt: platformDate); await ctx.newTrashedLocalAsset( id: 'trashed', albumId: 'album', - createdAt: createdAt, - updatedAt: updatedAt, + createdAt: wrongDate, + updatedAt: platformDate, source: TrashOrigin.localSync, ); - await ctx.newTrashedLocalAsset( - id: 'trashed-epoch', - albumId: 'album', - createdAt: createdAt, - updatedAt: epoch, - source: TrashOrigin.localSync, + when(() => nativeSyncApi.getAssetsForAlbum('album')).thenAnswer((_) async => [_asset('local', platformDate)]); + when(() => nativeSyncApi.getTrashedAssets()).thenAnswer( + (_) async => { + 'album': [_asset('trashed', platformDate)], + }, ); - await migrateDatabaseIfNeeded(ctx.db); + await migrateDatabaseIfNeeded(ctx.db, nativeSyncApi, permissionApi); final local = await (ctx.db.select(ctx.db.localAssetEntity)..where((row) => row.id.equals('local'))).getSingle(); - final unchanged = await (ctx.db.select( - ctx.db.localAssetEntity, - )..where((row) => row.id.equals('epoch'))).getSingle(); final trashed = await (ctx.db.select( ctx.db.trashedLocalAssetEntity, )..where((row) => row.id.equals('trashed'))).getSingle(); - final trashedUnchanged = await (ctx.db.select( - ctx.db.trashedLocalAssetEntity, - )..where((row) => row.id.equals('trashed-epoch'))).getSingle(); - expect(local.createdAt, updatedAt); - expect(unchanged.createdAt, createdAt); - expect(trashed.createdAt, updatedAt); - expect(trashedUnchanged.createdAt, createdAt); + expect(local.createdAt, platformDate); + expect(trashed.createdAt, platformDate); + expect(Store.tryGet(StoreKey.version), 27); + }); + + test('keeps EXIF date without trash access', () async { + final modifiedAt = DateTime.utc(2025); + final takenAt = DateTime.utc(2026); + await Store.put(StoreKey.version, 26); + await ctx.newLocalAsset(id: 'exif', createdAt: takenAt, updatedAt: modifiedAt); + when( + () => nativeSyncApi.getAssetsForAlbum('album'), + ).thenAnswer((_) async => [_asset('exif', takenAt, updatedAt: modifiedAt)]); + when(() => permissionApi.hasManageMediaPermission()).thenAnswer((_) async => false); + + await migrateDatabaseIfNeeded(ctx.db, nativeSyncApi, permissionApi); + + final asset = await (ctx.db.select(ctx.db.localAssetEntity)..where((row) => row.id.equals('exif'))).getSingle(); + expect(asset.createdAt, takenAt); + expect(Store.tryGet(StoreKey.version), 27); + verifyNever(() => nativeSyncApi.getTrashedAssets()); + }); + + test('retries after permission is granted', () async { + final wrongDate = DateTime.utc(2026); + final platformDate = DateTime.utc(2019); + await Store.put(StoreKey.version, 26); + await ctx.newLocalAsset(id: 'local', createdAt: wrongDate, updatedAt: platformDate); + when(() => nativeSyncApi.hasMediaReadPermission()).thenAnswer((_) async => false); + + await migrateDatabaseIfNeeded(ctx.db, nativeSyncApi, permissionApi); + + expect(Store.tryGet(StoreKey.version), 26); + verifyNever(() => nativeSyncApi.getAlbums()); + verifyNever(() => nativeSyncApi.getTrashedAssets()); + var asset = await (ctx.db.select(ctx.db.localAssetEntity)..where((row) => row.id.equals('local'))).getSingle(); + expect(asset.createdAt, wrongDate); + + when(() => nativeSyncApi.hasMediaReadPermission()).thenAnswer((_) async => true); + when(() => nativeSyncApi.getAssetsForAlbum('album')).thenAnswer((_) async => [_asset('local', platformDate)]); + + await migrateDatabaseIfNeeded(ctx.db, nativeSyncApi, permissionApi); + + asset = await (ctx.db.select(ctx.db.localAssetEntity)..where((row) => row.id.equals('local'))).getSingle(); + expect(asset.createdAt, platformDate); + expect(Store.tryGet(StoreKey.version), 27); + }); + + test('keeps version 26 when MediaStore read fails', () async { + final wrongDate = DateTime.utc(2026); + await Store.put(StoreKey.version, 26); + await ctx.newLocalAsset(id: 'local', createdAt: wrongDate); + when(() => nativeSyncApi.getAlbums()).thenThrow(StateError('query failed')); + + await migrateDatabaseIfNeeded(ctx.db, nativeSyncApi, permissionApi); + + final asset = await (ctx.db.select(ctx.db.localAssetEntity)..where((row) => row.id.equals('local'))).getSingle(); + expect(asset.createdAt, wrongDate); + expect(Store.tryGet(StoreKey.version), 26); + }); + + test('heals out-of-range date and completes migration', () async { + final wrongDate = DateTime.utc(2026); + final before = DateTime.timestamp().subtract(const Duration(seconds: 1)); + await Store.put(StoreKey.version, 26); + await ctx.newLocalAsset(id: 'local', createdAt: wrongDate); + when( + () => nativeSyncApi.getAssetsForAlbum('album'), + ).thenAnswer((_) async => [_asset('local', wrongDate, createdAtSeconds: 8640000000001)]); + + await migrateDatabaseIfNeeded(ctx.db, nativeSyncApi, permissionApi); + + final after = DateTime.timestamp().add(const Duration(seconds: 1)); + final asset = await (ctx.db.select(ctx.db.localAssetEntity)..where((row) => row.id.equals('local'))).getSingle(); + expect(asset.createdAt.isBefore(before), isFalse); + expect(asset.createdAt.isAfter(after), isFalse); expect(Store.tryGet(StoreKey.version), 27); }); } + +class MockPermissionApi extends Mock implements PermissionApi {} + +PlatformAsset _asset(String id, DateTime createdAt, {DateTime? updatedAt, int? createdAtSeconds}) => PlatformAsset( + id: id, + name: '$id.jpg', + type: 1, + createdAt: createdAtSeconds ?? createdAt.millisecondsSinceEpoch ~/ 1000, + updatedAt: (updatedAt ?? createdAt).millisecondsSinceEpoch ~/ 1000, + durationMs: 0, + orientation: 0, + isFavorite: false, + playbackStyle: PlatformAssetPlaybackStyle.image, +);