mirror of
https://github.com/immich-app/immich
synced 2026-08-15 13:03:57 +00:00
heal each table in one statement, rename the clamp type, read dates as stored
This commit is contained in:
parent
194f53beba
commit
c51cb9ad63
20 changed files with 38 additions and 83 deletions
|
|
@ -1,7 +1,7 @@
|
|||
import 'package:drift/drift.dart';
|
||||
import 'package:immich_mobile/infrastructure/entities/person.entity.dart';
|
||||
import 'package:immich_mobile/infrastructure/entities/remote_asset.entity.dart';
|
||||
import 'package:immich_mobile/infrastructure/utils/datetime.converter.dart';
|
||||
import 'package:immich_mobile/infrastructure/utils/datetime_clamp.type.dart';
|
||||
import 'package:immich_mobile/infrastructure/utils/drift_default.mixin.dart';
|
||||
|
||||
@TableIndex.sql('CREATE INDEX IF NOT EXISTS idx_asset_face_person_id ON asset_face_entity (person_id)')
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import 'package:drift/drift.dart';
|
||||
import 'package:immich_mobile/domain/models/user.model.dart';
|
||||
import 'package:immich_mobile/infrastructure/utils/datetime.converter.dart';
|
||||
import 'package:immich_mobile/infrastructure/utils/datetime_clamp.type.dart';
|
||||
import 'package:immich_mobile/infrastructure/utils/drift_default.mixin.dart';
|
||||
|
||||
class AuthUserEntity extends Table with DriftDefaultsMixin {
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import 'package:drift/drift.dart' hide Query;
|
|||
import 'package:immich_mobile/domain/models/exif.model.dart' as domain;
|
||||
import 'package:immich_mobile/infrastructure/entities/exif.entity.drift.dart';
|
||||
import 'package:immich_mobile/infrastructure/entities/remote_asset.entity.dart';
|
||||
import 'package:immich_mobile/infrastructure/utils/datetime.converter.dart';
|
||||
import 'package:immich_mobile/infrastructure/utils/datetime_clamp.type.dart';
|
||||
import 'package:immich_mobile/infrastructure/utils/drift_default.mixin.dart';
|
||||
import 'package:immich_mobile/infrastructure/utils/exif.converter.dart';
|
||||
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import 'package:drift/drift.dart';
|
|||
import 'package:immich_mobile/domain/models/album/local_album.model.dart';
|
||||
import 'package:immich_mobile/infrastructure/entities/local_album.entity.drift.dart';
|
||||
import 'package:immich_mobile/infrastructure/entities/remote_album.entity.dart';
|
||||
import 'package:immich_mobile/infrastructure/utils/datetime.converter.dart';
|
||||
import 'package:immich_mobile/infrastructure/utils/datetime_clamp.type.dart';
|
||||
import 'package:immich_mobile/infrastructure/utils/drift_default.mixin.dart';
|
||||
|
||||
class LocalAlbumEntity extends Table with DriftDefaultsMixin {
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import 'package:drift/drift.dart';
|
|||
import 'package:immich_mobile/domain/models/asset/base_asset.model.dart';
|
||||
import 'package:immich_mobile/infrastructure/entities/local_asset.entity.drift.dart';
|
||||
import 'package:immich_mobile/infrastructure/utils/asset.mixin.dart';
|
||||
import 'package:immich_mobile/infrastructure/utils/datetime.converter.dart';
|
||||
import 'package:immich_mobile/infrastructure/utils/datetime_clamp.type.dart';
|
||||
import 'package:immich_mobile/infrastructure/utils/drift_default.mixin.dart';
|
||||
|
||||
@TableIndex.sql('CREATE INDEX IF NOT EXISTS idx_local_asset_checksum ON local_asset_entity (checksum)')
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import 'package:drift/drift.dart';
|
||||
import 'package:immich_mobile/domain/models/log.model.dart' as domain;
|
||||
import 'package:immich_mobile/infrastructure/entities/log.entity.drift.dart';
|
||||
import 'package:immich_mobile/infrastructure/utils/datetime.converter.dart';
|
||||
import 'package:immich_mobile/infrastructure/utils/datetime_clamp.type.dart';
|
||||
import 'package:immich_mobile/infrastructure/utils/drift_default.mixin.dart';
|
||||
|
||||
class LogMessageEntity extends Table with DriftDefaultsMixin {
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import 'package:drift/drift.dart';
|
||||
import 'package:immich_mobile/domain/models/memory.model.dart';
|
||||
import 'package:immich_mobile/infrastructure/entities/user.entity.dart';
|
||||
import 'package:immich_mobile/infrastructure/utils/datetime.converter.dart';
|
||||
import 'package:immich_mobile/infrastructure/utils/datetime_clamp.type.dart';
|
||||
import 'package:immich_mobile/infrastructure/utils/drift_default.mixin.dart';
|
||||
|
||||
class MemoryEntity extends Table with DriftDefaultsMixin {
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import 'package:drift/drift.dart';
|
||||
import 'package:immich_mobile/infrastructure/entities/user.entity.dart';
|
||||
import 'package:immich_mobile/infrastructure/utils/datetime.converter.dart';
|
||||
import 'package:immich_mobile/infrastructure/utils/datetime_clamp.type.dart';
|
||||
import 'package:immich_mobile/infrastructure/utils/drift_default.mixin.dart';
|
||||
|
||||
@TableIndex.sql('CREATE INDEX IF NOT EXISTS idx_person_owner_id ON person_entity (owner_id)')
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import 'package:drift/drift.dart';
|
||||
import 'package:immich_mobile/domain/models/album/album.model.dart';
|
||||
import 'package:immich_mobile/infrastructure/entities/remote_asset.entity.dart';
|
||||
import 'package:immich_mobile/infrastructure/utils/datetime.converter.dart';
|
||||
import 'package:immich_mobile/infrastructure/utils/datetime_clamp.type.dart';
|
||||
import 'package:immich_mobile/infrastructure/utils/drift_default.mixin.dart';
|
||||
|
||||
class RemoteAlbumEntity extends Table with DriftDefaultsMixin {
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ import 'package:immich_mobile/domain/models/asset/base_asset.model.dart';
|
|||
import 'package:immich_mobile/infrastructure/entities/remote_asset.entity.drift.dart';
|
||||
import 'package:immich_mobile/infrastructure/entities/user.entity.dart';
|
||||
import 'package:immich_mobile/infrastructure/utils/asset.mixin.dart';
|
||||
import 'package:immich_mobile/infrastructure/utils/datetime.converter.dart';
|
||||
import 'package:immich_mobile/infrastructure/utils/datetime_clamp.type.dart';
|
||||
import 'package:immich_mobile/infrastructure/utils/drift_default.mixin.dart';
|
||||
|
||||
@TableIndex.sql('''
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import 'package:drift/drift.dart';
|
||||
import 'package:immich_mobile/infrastructure/entities/remote_asset.entity.dart';
|
||||
import 'package:immich_mobile/infrastructure/utils/datetime.converter.dart';
|
||||
import 'package:immich_mobile/infrastructure/utils/datetime_clamp.type.dart';
|
||||
import 'package:immich_mobile/infrastructure/utils/drift_default.mixin.dart';
|
||||
|
||||
@TableIndex.sql('CREATE INDEX IF NOT EXISTS idx_remote_asset_cloud_id ON remote_asset_cloud_id_entity (cloud_id)')
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import 'package:drift/drift.dart';
|
||||
import 'package:immich_mobile/infrastructure/utils/datetime.converter.dart';
|
||||
import 'package:immich_mobile/infrastructure/utils/datetime_clamp.type.dart';
|
||||
import 'package:immich_mobile/infrastructure/utils/drift_default.mixin.dart';
|
||||
|
||||
class SettingsEntity extends Table with DriftDefaultsMixin {
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import 'package:drift/drift.dart';
|
||||
import 'package:immich_mobile/infrastructure/entities/user.entity.dart';
|
||||
import 'package:immich_mobile/infrastructure/utils/datetime.converter.dart';
|
||||
import 'package:immich_mobile/infrastructure/utils/datetime_clamp.type.dart';
|
||||
import 'package:immich_mobile/infrastructure/utils/drift_default.mixin.dart';
|
||||
|
||||
@TableIndex.sql('CREATE INDEX IF NOT EXISTS idx_stack_primary_asset_id ON stack_entity (primary_asset_id)')
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import 'package:drift/drift.dart' hide Index;
|
||||
import 'package:immich_mobile/domain/models/user.model.dart';
|
||||
import 'package:immich_mobile/infrastructure/utils/datetime.converter.dart';
|
||||
import 'package:immich_mobile/infrastructure/utils/datetime_clamp.type.dart';
|
||||
import 'package:immich_mobile/infrastructure/utils/drift_default.mixin.dart';
|
||||
|
||||
class UserEntity extends Table with DriftDefaultsMixin {
|
||||
|
|
|
|||
|
|
@ -347,8 +347,8 @@ class Drift extends $Drift {
|
|||
);
|
||||
}
|
||||
|
||||
// hardcoded on purpose: datetimes are stored as text, so versioned schema tables type
|
||||
// these columns String and the set cannot be derived — migration_test pins it to drift_schema_v31.json
|
||||
// every datetime column of the v31 schema, hardcoded: the heal runs once at v31->v32,
|
||||
// so the set must not follow later schema changes
|
||||
@visibleForTesting
|
||||
const healDateTimeColumns = <String, List<String>>{
|
||||
'auth_user_entity': ['profile_changed_at'],
|
||||
|
|
@ -368,16 +368,21 @@ const healDateTimeColumns = <String, List<String>>{
|
|||
};
|
||||
|
||||
// Rewrites datetime text sqlite date functions cannot handle: signed extended
|
||||
// years and year 0000 (pre-converter syncs), plus anything later than the safe
|
||||
// midnight ceiling, which re-overflows sqlite under 'localtime' east of UTC
|
||||
// years and year 0000 (pre-clamp syncs), plus anything later than the safe
|
||||
// midnight ceiling, which re-overflows sqlite under 'localtime' east of UTC.
|
||||
// One statement per table: each column heals only when its own value is out of range
|
||||
@visibleForTesting
|
||||
Future<void> healOutOfRangeDateTimes(GeneratedDatabase db) async {
|
||||
const floor = '0001-01-01T00:00:00.000Z';
|
||||
const ceiling = '9999-12-31T00:00:00.000Z';
|
||||
for (final MapEntry(key: table, value: columns) in healDateTimeColumns.entries) {
|
||||
for (final column in columns) {
|
||||
await db.customStatement(
|
||||
"UPDATE $table SET $column = CASE WHEN substr($column, 1, 1) = '-' OR substr($column, 1, 4) = '0000' THEN '0001-01-01T00:00:00.000Z' ELSE '9999-12-31T00:00:00.000Z' END WHERE substr($column, 1, 1) IN ('+', '-') OR substr($column, 1, 4) = '0000' OR $column > '9999-12-31T00:00:00.000Z'",
|
||||
);
|
||||
}
|
||||
String low(String c) => "substr($c, 1, 1) = '-' OR substr($c, 1, 4) = '0000'";
|
||||
String high(String c) => "substr($c, 1, 1) = '+' OR $c > '$ceiling'";
|
||||
final assignments = columns.map(
|
||||
(c) => "$c = CASE WHEN ${low(c)} THEN '$floor' WHEN ${high(c)} THEN '$ceiling' ELSE $c END",
|
||||
);
|
||||
final outOfRange = columns.map((c) => '${low(c)} OR ${high(c)}');
|
||||
await db.customStatement('UPDATE $table SET ${assignments.join(', ')} WHERE ${outOfRange.join(' OR ')}');
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import 'package:drift/drift.dart';
|
||||
import 'package:immich_mobile/domain/models/asset/base_asset.model.dart';
|
||||
import 'package:immich_mobile/infrastructure/utils/datetime.converter.dart';
|
||||
import 'package:immich_mobile/infrastructure/utils/datetime_clamp.type.dart';
|
||||
import 'package:immich_mobile/infrastructure/utils/drift_default.mixin.dart';
|
||||
|
||||
mixin AssetEntityMixin on DriftDefaultsMixin {
|
||||
|
|
|
|||
|
|
@ -2,15 +2,14 @@ import 'package:drift/drift.dart';
|
|||
|
||||
// Synced dates can fall outside the year range sqlite date functions handle
|
||||
// (1..9999), which turns bucket queries into NULLs and crashes the timeline
|
||||
// (#28524). Clamps every stored datetime into the representable range; anything
|
||||
// in range passes through untouched.
|
||||
// (#28524). Clamps on write; the v32 heal migration rewrites the backlog.
|
||||
const clampedDateTime = DateTimeClampType();
|
||||
|
||||
final class DateTimeClampType implements DialectAwareSqlType<DateTime> {
|
||||
const DateTimeClampType();
|
||||
|
||||
@override
|
||||
DateTime read(SqlTypes types, Object fromSql) => _clampDateTime(types.read(DriftSqlType.dateTime, fromSql)!);
|
||||
DateTime read(SqlTypes types, Object fromSql) => types.read(DriftSqlType.dateTime, fromSql)!;
|
||||
|
||||
@override
|
||||
Object mapToSqlParameter(GenerationContext context, DateTime value) =>
|
||||
|
|
@ -1,31 +1,24 @@
|
|||
import 'package:drift/drift.dart';
|
||||
import 'package:drift/native.dart';
|
||||
import 'package:flutter_test/flutter_test.dart';
|
||||
import 'package:immich_mobile/infrastructure/repositories/db.repository.dart';
|
||||
import 'package:immich_mobile/infrastructure/repositories/logger_db.repository.dart';
|
||||
import 'package:immich_mobile/infrastructure/utils/datetime.converter.dart';
|
||||
import 'package:immich_mobile/infrastructure/utils/drift_default.mixin.dart';
|
||||
import 'package:immich_mobile/infrastructure/utils/datetime_clamp.type.dart';
|
||||
|
||||
void main() {
|
||||
const options = DriftDatabaseOptions(storeDateTimeAsText: true);
|
||||
final types = options.createTypeMapping(SqlDialect.sqlite);
|
||||
final ctx = GenerationContext(options, null);
|
||||
DateTime toSql(DateTime value) => types.read(DriftSqlType.dateTime, clampedDateTime.mapToSqlParameter(ctx, value))!;
|
||||
DateTime fromSql(DateTime value) => clampedDateTime.read(types, types.mapToSqlVariable(value)!);
|
||||
|
||||
test('clamps far-future years to the ceiling in both directions', () {
|
||||
test('clamps far-future years to the ceiling on write', () {
|
||||
// the reporter's date from #28524
|
||||
final poison = DateTime.utc(144769, 11, 18, 12, 38, 32);
|
||||
final ceiling = DateTime.utc(9999, 12, 31);
|
||||
expect(toSql(poison), ceiling);
|
||||
expect(fromSql(poison), ceiling);
|
||||
expect(toSql(DateTime(144769, 1, 1)), ceiling); // local poison normalizes to utc
|
||||
});
|
||||
|
||||
test('clamps BCE and year zero to the floor in both directions', () {
|
||||
test('clamps BCE and year zero to the floor on write', () {
|
||||
final floor = DateTime.utc(1, 1, 1);
|
||||
expect(toSql(DateTime.utc(-4712, 3, 4, 5, 6, 7)), floor);
|
||||
expect(fromSql(DateTime.utc(-4712, 3, 4, 5, 6, 7)), floor);
|
||||
expect(toSql(DateTime.utc(0, 12, 31)), floor);
|
||||
});
|
||||
|
||||
|
|
@ -35,10 +28,8 @@ void main() {
|
|||
final justBefore = DateTime.utc(9999, 12, 30, 23, 59, 59);
|
||||
final normal = DateTime.utc(2024, 1, 2, 3, 4, 5, 123);
|
||||
expect(toSql(low), low);
|
||||
expect(fromSql(low), low);
|
||||
expect(toSql(justBefore), justBefore);
|
||||
expect(toSql(normal), normal);
|
||||
expect(fromSql(normal), normal);
|
||||
// local datetimes keep their zone flag
|
||||
final local = DateTime(2024, 6, 15, 10, 30, 25);
|
||||
expect(toSql(local), local);
|
||||
|
|
@ -50,27 +41,12 @@ void main() {
|
|||
// bucket query's 'localtime' overflows sqlite's year range on it -> NULL
|
||||
final ceiling = DateTime.utc(9999, 12, 31);
|
||||
expect(toSql(DateTime.utc(9999, 12, 31, 23, 59, 59)), ceiling);
|
||||
expect(fromSql(DateTime.utc(9999, 12, 31, 23, 59, 59)), ceiling);
|
||||
expect(toSql(DateTime(9999, 12, 31, 23, 59, 59)), ceiling); // local normalizes to utc
|
||||
});
|
||||
|
||||
group('every table uses the datetime guard', () {
|
||||
void expectGuard(GeneratedDatabase db) {
|
||||
for (final table in db.allTables) {
|
||||
expect(table, isA<DriftDefaultsMixin>(), reason: '${table.entityName} bypasses the datetime guard');
|
||||
}
|
||||
}
|
||||
|
||||
test('main database tables', () {
|
||||
final db = Drift(NativeDatabase.memory());
|
||||
addTearDown(db.close);
|
||||
expectGuard(db);
|
||||
});
|
||||
|
||||
test('logs database table', () {
|
||||
final logDb = DriftLogger.fromExecutor(NativeDatabase.memory());
|
||||
addTearDown(logDb.close);
|
||||
expectGuard(logDb);
|
||||
});
|
||||
test('reads pass stored values through untouched', () {
|
||||
// writes clamp and the v32 migration heals the backlog, so reads never clamp
|
||||
final stored = types.mapToSqlVariable(DateTime.utc(2024, 1, 2, 3, 4, 5, 123))!;
|
||||
expect(clampedDateTime.read(types, stored), DateTime.utc(2024, 1, 2, 3, 4, 5, 123));
|
||||
});
|
||||
}
|
||||
|
|
@ -1,6 +1,3 @@
|
|||
import 'dart:convert';
|
||||
import 'dart:io';
|
||||
|
||||
import 'package:drift/drift.dart';
|
||||
import 'package:flutter_test/flutter_test.dart';
|
||||
import 'package:immich_mobile/infrastructure/repositories/db.repository.dart';
|
||||
|
|
@ -111,27 +108,5 @@ void main() {
|
|||
final row = await (ctx.db.select(ctx.db.localAssetEntity)..where((t) => t.id.equals(asset.id))).getSingle();
|
||||
expect(row.createdAt, local);
|
||||
});
|
||||
|
||||
test('the heal column map matches the datetime columns of the v31 schema json', () async {
|
||||
final schemaJson = jsonDecode(await File('drift_schemas/main/drift_schema_v31.json').readAsString());
|
||||
final expected = <String>{
|
||||
for (final entity in schemaJson['entities'] as List)
|
||||
if (entity['type'] == 'table')
|
||||
for (final column in entity['data']['columns'] as List)
|
||||
if (column['moor_type'] == 'dateTime') '${entity['data']['name']}.${column['name']}',
|
||||
};
|
||||
|
||||
final mapped = <String>{
|
||||
for (final MapEntry(key: table, value: columns) in healDateTimeColumns.entries)
|
||||
for (final column in columns) '$table.$column',
|
||||
};
|
||||
|
||||
final missing = expected.difference(mapped);
|
||||
final extra = mapped.difference(expected);
|
||||
expect(missing, isEmpty, reason: 'heal map misses v31 datetime columns: ${missing.join(', ')}');
|
||||
expect(extra, isEmpty, reason: 'heal map has columns that are not v31 datetimes: ${extra.join(', ')}');
|
||||
final mappedCount = healDateTimeColumns.values.fold(0, (sum, columns) => sum + columns.length);
|
||||
expect(mappedCount, expected.length, reason: 'heal map count differs from the v31 schema json');
|
||||
});
|
||||
});
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue