heal each table in one statement, rename the clamp type, read dates as stored

This commit is contained in:
Santo Shakil 2026-08-05 18:22:52 +06:00
parent 194f53beba
commit c51cb9ad63
20 changed files with 38 additions and 83 deletions

View file

@ -1,7 +1,7 @@
import 'package:drift/drift.dart';
import 'package:immich_mobile/infrastructure/entities/person.entity.dart';
import 'package:immich_mobile/infrastructure/entities/remote_asset.entity.dart';
import 'package:immich_mobile/infrastructure/utils/datetime.converter.dart';
import 'package:immich_mobile/infrastructure/utils/datetime_clamp.type.dart';
import 'package:immich_mobile/infrastructure/utils/drift_default.mixin.dart';
@TableIndex.sql('CREATE INDEX IF NOT EXISTS idx_asset_face_person_id ON asset_face_entity (person_id)')

View file

@ -1,6 +1,6 @@
import 'package:drift/drift.dart';
import 'package:immich_mobile/domain/models/user.model.dart';
import 'package:immich_mobile/infrastructure/utils/datetime.converter.dart';
import 'package:immich_mobile/infrastructure/utils/datetime_clamp.type.dart';
import 'package:immich_mobile/infrastructure/utils/drift_default.mixin.dart';
class AuthUserEntity extends Table with DriftDefaultsMixin {

View file

@ -2,7 +2,7 @@ import 'package:drift/drift.dart' hide Query;
import 'package:immich_mobile/domain/models/exif.model.dart' as domain;
import 'package:immich_mobile/infrastructure/entities/exif.entity.drift.dart';
import 'package:immich_mobile/infrastructure/entities/remote_asset.entity.dart';
import 'package:immich_mobile/infrastructure/utils/datetime.converter.dart';
import 'package:immich_mobile/infrastructure/utils/datetime_clamp.type.dart';
import 'package:immich_mobile/infrastructure/utils/drift_default.mixin.dart';
import 'package:immich_mobile/infrastructure/utils/exif.converter.dart';

View file

@ -2,7 +2,7 @@ import 'package:drift/drift.dart';
import 'package:immich_mobile/domain/models/album/local_album.model.dart';
import 'package:immich_mobile/infrastructure/entities/local_album.entity.drift.dart';
import 'package:immich_mobile/infrastructure/entities/remote_album.entity.dart';
import 'package:immich_mobile/infrastructure/utils/datetime.converter.dart';
import 'package:immich_mobile/infrastructure/utils/datetime_clamp.type.dart';
import 'package:immich_mobile/infrastructure/utils/drift_default.mixin.dart';
class LocalAlbumEntity extends Table with DriftDefaultsMixin {

View file

@ -2,7 +2,7 @@ import 'package:drift/drift.dart';
import 'package:immich_mobile/domain/models/asset/base_asset.model.dart';
import 'package:immich_mobile/infrastructure/entities/local_asset.entity.drift.dart';
import 'package:immich_mobile/infrastructure/utils/asset.mixin.dart';
import 'package:immich_mobile/infrastructure/utils/datetime.converter.dart';
import 'package:immich_mobile/infrastructure/utils/datetime_clamp.type.dart';
import 'package:immich_mobile/infrastructure/utils/drift_default.mixin.dart';
@TableIndex.sql('CREATE INDEX IF NOT EXISTS idx_local_asset_checksum ON local_asset_entity (checksum)')

View file

@ -1,7 +1,7 @@
import 'package:drift/drift.dart';
import 'package:immich_mobile/domain/models/log.model.dart' as domain;
import 'package:immich_mobile/infrastructure/entities/log.entity.drift.dart';
import 'package:immich_mobile/infrastructure/utils/datetime.converter.dart';
import 'package:immich_mobile/infrastructure/utils/datetime_clamp.type.dart';
import 'package:immich_mobile/infrastructure/utils/drift_default.mixin.dart';
class LogMessageEntity extends Table with DriftDefaultsMixin {

View file

@ -1,7 +1,7 @@
import 'package:drift/drift.dart';
import 'package:immich_mobile/domain/models/memory.model.dart';
import 'package:immich_mobile/infrastructure/entities/user.entity.dart';
import 'package:immich_mobile/infrastructure/utils/datetime.converter.dart';
import 'package:immich_mobile/infrastructure/utils/datetime_clamp.type.dart';
import 'package:immich_mobile/infrastructure/utils/drift_default.mixin.dart';
class MemoryEntity extends Table with DriftDefaultsMixin {

View file

@ -1,6 +1,6 @@
import 'package:drift/drift.dart';
import 'package:immich_mobile/infrastructure/entities/user.entity.dart';
import 'package:immich_mobile/infrastructure/utils/datetime.converter.dart';
import 'package:immich_mobile/infrastructure/utils/datetime_clamp.type.dart';
import 'package:immich_mobile/infrastructure/utils/drift_default.mixin.dart';
@TableIndex.sql('CREATE INDEX IF NOT EXISTS idx_person_owner_id ON person_entity (owner_id)')

View file

@ -1,7 +1,7 @@
import 'package:drift/drift.dart';
import 'package:immich_mobile/domain/models/album/album.model.dart';
import 'package:immich_mobile/infrastructure/entities/remote_asset.entity.dart';
import 'package:immich_mobile/infrastructure/utils/datetime.converter.dart';
import 'package:immich_mobile/infrastructure/utils/datetime_clamp.type.dart';
import 'package:immich_mobile/infrastructure/utils/drift_default.mixin.dart';
class RemoteAlbumEntity extends Table with DriftDefaultsMixin {

View file

@ -3,7 +3,7 @@ import 'package:immich_mobile/domain/models/asset/base_asset.model.dart';
import 'package:immich_mobile/infrastructure/entities/remote_asset.entity.drift.dart';
import 'package:immich_mobile/infrastructure/entities/user.entity.dart';
import 'package:immich_mobile/infrastructure/utils/asset.mixin.dart';
import 'package:immich_mobile/infrastructure/utils/datetime.converter.dart';
import 'package:immich_mobile/infrastructure/utils/datetime_clamp.type.dart';
import 'package:immich_mobile/infrastructure/utils/drift_default.mixin.dart';
@TableIndex.sql('''

View file

@ -1,6 +1,6 @@
import 'package:drift/drift.dart';
import 'package:immich_mobile/infrastructure/entities/remote_asset.entity.dart';
import 'package:immich_mobile/infrastructure/utils/datetime.converter.dart';
import 'package:immich_mobile/infrastructure/utils/datetime_clamp.type.dart';
import 'package:immich_mobile/infrastructure/utils/drift_default.mixin.dart';
@TableIndex.sql('CREATE INDEX IF NOT EXISTS idx_remote_asset_cloud_id ON remote_asset_cloud_id_entity (cloud_id)')

View file

@ -1,5 +1,5 @@
import 'package:drift/drift.dart';
import 'package:immich_mobile/infrastructure/utils/datetime.converter.dart';
import 'package:immich_mobile/infrastructure/utils/datetime_clamp.type.dart';
import 'package:immich_mobile/infrastructure/utils/drift_default.mixin.dart';
class SettingsEntity extends Table with DriftDefaultsMixin {

View file

@ -1,6 +1,6 @@
import 'package:drift/drift.dart';
import 'package:immich_mobile/infrastructure/entities/user.entity.dart';
import 'package:immich_mobile/infrastructure/utils/datetime.converter.dart';
import 'package:immich_mobile/infrastructure/utils/datetime_clamp.type.dart';
import 'package:immich_mobile/infrastructure/utils/drift_default.mixin.dart';
@TableIndex.sql('CREATE INDEX IF NOT EXISTS idx_stack_primary_asset_id ON stack_entity (primary_asset_id)')

View file

@ -1,6 +1,6 @@
import 'package:drift/drift.dart' hide Index;
import 'package:immich_mobile/domain/models/user.model.dart';
import 'package:immich_mobile/infrastructure/utils/datetime.converter.dart';
import 'package:immich_mobile/infrastructure/utils/datetime_clamp.type.dart';
import 'package:immich_mobile/infrastructure/utils/drift_default.mixin.dart';
class UserEntity extends Table with DriftDefaultsMixin {

View file

@ -347,8 +347,8 @@ class Drift extends $Drift {
);
}
// hardcoded on purpose: datetimes are stored as text, so versioned schema tables type
// these columns String and the set cannot be derived migration_test pins it to drift_schema_v31.json
// every datetime column of the v31 schema, hardcoded: the heal runs once at v31->v32,
// so the set must not follow later schema changes
@visibleForTesting
const healDateTimeColumns = <String, List<String>>{
'auth_user_entity': ['profile_changed_at'],
@ -368,16 +368,21 @@ const healDateTimeColumns = <String, List<String>>{
};
// Rewrites datetime text sqlite date functions cannot handle: signed extended
// years and year 0000 (pre-converter syncs), plus anything later than the safe
// midnight ceiling, which re-overflows sqlite under 'localtime' east of UTC
// years and year 0000 (pre-clamp syncs), plus anything later than the safe
// midnight ceiling, which re-overflows sqlite under 'localtime' east of UTC.
// One statement per table: each column heals only when its own value is out of range
@visibleForTesting
Future<void> healOutOfRangeDateTimes(GeneratedDatabase db) async {
const floor = '0001-01-01T00:00:00.000Z';
const ceiling = '9999-12-31T00:00:00.000Z';
for (final MapEntry(key: table, value: columns) in healDateTimeColumns.entries) {
for (final column in columns) {
await db.customStatement(
"UPDATE $table SET $column = CASE WHEN substr($column, 1, 1) = '-' OR substr($column, 1, 4) = '0000' THEN '0001-01-01T00:00:00.000Z' ELSE '9999-12-31T00:00:00.000Z' END WHERE substr($column, 1, 1) IN ('+', '-') OR substr($column, 1, 4) = '0000' OR $column > '9999-12-31T00:00:00.000Z'",
);
}
String low(String c) => "substr($c, 1, 1) = '-' OR substr($c, 1, 4) = '0000'";
String high(String c) => "substr($c, 1, 1) = '+' OR $c > '$ceiling'";
final assignments = columns.map(
(c) => "$c = CASE WHEN ${low(c)} THEN '$floor' WHEN ${high(c)} THEN '$ceiling' ELSE $c END",
);
final outOfRange = columns.map((c) => '${low(c)} OR ${high(c)}');
await db.customStatement('UPDATE $table SET ${assignments.join(', ')} WHERE ${outOfRange.join(' OR ')}');
}
}

View file

@ -1,6 +1,6 @@
import 'package:drift/drift.dart';
import 'package:immich_mobile/domain/models/asset/base_asset.model.dart';
import 'package:immich_mobile/infrastructure/utils/datetime.converter.dart';
import 'package:immich_mobile/infrastructure/utils/datetime_clamp.type.dart';
import 'package:immich_mobile/infrastructure/utils/drift_default.mixin.dart';
mixin AssetEntityMixin on DriftDefaultsMixin {

View file

@ -2,15 +2,14 @@ import 'package:drift/drift.dart';
// Synced dates can fall outside the year range sqlite date functions handle
// (1..9999), which turns bucket queries into NULLs and crashes the timeline
// (#28524). Clamps every stored datetime into the representable range; anything
// in range passes through untouched.
// (#28524). Clamps on write; the v32 heal migration rewrites the backlog.
const clampedDateTime = DateTimeClampType();
final class DateTimeClampType implements DialectAwareSqlType<DateTime> {
const DateTimeClampType();
@override
DateTime read(SqlTypes types, Object fromSql) => _clampDateTime(types.read(DriftSqlType.dateTime, fromSql)!);
DateTime read(SqlTypes types, Object fromSql) => types.read(DriftSqlType.dateTime, fromSql)!;
@override
Object mapToSqlParameter(GenerationContext context, DateTime value) =>

View file

@ -1,31 +1,24 @@
import 'package:drift/drift.dart';
import 'package:drift/native.dart';
import 'package:flutter_test/flutter_test.dart';
import 'package:immich_mobile/infrastructure/repositories/db.repository.dart';
import 'package:immich_mobile/infrastructure/repositories/logger_db.repository.dart';
import 'package:immich_mobile/infrastructure/utils/datetime.converter.dart';
import 'package:immich_mobile/infrastructure/utils/drift_default.mixin.dart';
import 'package:immich_mobile/infrastructure/utils/datetime_clamp.type.dart';
void main() {
const options = DriftDatabaseOptions(storeDateTimeAsText: true);
final types = options.createTypeMapping(SqlDialect.sqlite);
final ctx = GenerationContext(options, null);
DateTime toSql(DateTime value) => types.read(DriftSqlType.dateTime, clampedDateTime.mapToSqlParameter(ctx, value))!;
DateTime fromSql(DateTime value) => clampedDateTime.read(types, types.mapToSqlVariable(value)!);
test('clamps far-future years to the ceiling in both directions', () {
test('clamps far-future years to the ceiling on write', () {
// the reporter's date from #28524
final poison = DateTime.utc(144769, 11, 18, 12, 38, 32);
final ceiling = DateTime.utc(9999, 12, 31);
expect(toSql(poison), ceiling);
expect(fromSql(poison), ceiling);
expect(toSql(DateTime(144769, 1, 1)), ceiling); // local poison normalizes to utc
});
test('clamps BCE and year zero to the floor in both directions', () {
test('clamps BCE and year zero to the floor on write', () {
final floor = DateTime.utc(1, 1, 1);
expect(toSql(DateTime.utc(-4712, 3, 4, 5, 6, 7)), floor);
expect(fromSql(DateTime.utc(-4712, 3, 4, 5, 6, 7)), floor);
expect(toSql(DateTime.utc(0, 12, 31)), floor);
});
@ -35,10 +28,8 @@ void main() {
final justBefore = DateTime.utc(9999, 12, 30, 23, 59, 59);
final normal = DateTime.utc(2024, 1, 2, 3, 4, 5, 123);
expect(toSql(low), low);
expect(fromSql(low), low);
expect(toSql(justBefore), justBefore);
expect(toSql(normal), normal);
expect(fromSql(normal), normal);
// local datetimes keep their zone flag
final local = DateTime(2024, 6, 15, 10, 30, 25);
expect(toSql(local), local);
@ -50,27 +41,12 @@ void main() {
// bucket query's 'localtime' overflows sqlite's year range on it -> NULL
final ceiling = DateTime.utc(9999, 12, 31);
expect(toSql(DateTime.utc(9999, 12, 31, 23, 59, 59)), ceiling);
expect(fromSql(DateTime.utc(9999, 12, 31, 23, 59, 59)), ceiling);
expect(toSql(DateTime(9999, 12, 31, 23, 59, 59)), ceiling); // local normalizes to utc
});
group('every table uses the datetime guard', () {
void expectGuard(GeneratedDatabase db) {
for (final table in db.allTables) {
expect(table, isA<DriftDefaultsMixin>(), reason: '${table.entityName} bypasses the datetime guard');
}
}
test('main database tables', () {
final db = Drift(NativeDatabase.memory());
addTearDown(db.close);
expectGuard(db);
});
test('logs database table', () {
final logDb = DriftLogger.fromExecutor(NativeDatabase.memory());
addTearDown(logDb.close);
expectGuard(logDb);
});
test('reads pass stored values through untouched', () {
// writes clamp and the v32 migration heals the backlog, so reads never clamp
final stored = types.mapToSqlVariable(DateTime.utc(2024, 1, 2, 3, 4, 5, 123))!;
expect(clampedDateTime.read(types, stored), DateTime.utc(2024, 1, 2, 3, 4, 5, 123));
});
}

View file

@ -1,6 +1,3 @@
import 'dart:convert';
import 'dart:io';
import 'package:drift/drift.dart';
import 'package:flutter_test/flutter_test.dart';
import 'package:immich_mobile/infrastructure/repositories/db.repository.dart';
@ -111,27 +108,5 @@ void main() {
final row = await (ctx.db.select(ctx.db.localAssetEntity)..where((t) => t.id.equals(asset.id))).getSingle();
expect(row.createdAt, local);
});
test('the heal column map matches the datetime columns of the v31 schema json', () async {
final schemaJson = jsonDecode(await File('drift_schemas/main/drift_schema_v31.json').readAsString());
final expected = <String>{
for (final entity in schemaJson['entities'] as List)
if (entity['type'] == 'table')
for (final column in entity['data']['columns'] as List)
if (column['moor_type'] == 'dateTime') '${entity['data']['name']}.${column['name']}',
};
final mapped = <String>{
for (final MapEntry(key: table, value: columns) in healDateTimeColumns.entries)
for (final column in columns) '$table.$column',
};
final missing = expected.difference(mapped);
final extra = mapped.difference(expected);
expect(missing, isEmpty, reason: 'heal map misses v31 datetime columns: ${missing.join(', ')}');
expect(extra, isEmpty, reason: 'heal map has columns that are not v31 datetimes: ${extra.join(', ')}');
final mappedCount = healDateTimeColumns.values.fold(0, (sum, columns) => sum + columns.length);
expect(mappedCount, expected.length, reason: 'heal map count differs from the v31 schema json');
});
});
}