mirror of
https://github.com/immich-app/immich
synced 2025-11-07 17:27:20 +00:00
feat: user sync stream (#16862)
* refactor: user entity * chore: rebase fixes * refactor: remove int user Id * refactor: migrate store userId from int to string * refactor: rename uid to id * feat: drift * pr feedback * refactor: move common overrides to mixin * refactor: remove int user Id * refactor: migrate store userId from int to string * refactor: rename uid to id * feat: user & partner sync stream * pr changes * refactor: sync service and add tests * chore: remove generated change * chore: move sync model * rebase: convert string ids to byte uuids * rebase * add processing logs * batch db calls * rewrite isolate manager * rewrite with worker_manager * misc fixes * add sync order test --------- Co-authored-by: shenlong-tanwen <139912620+shalong-tanwen@users.noreply.github.com> Co-authored-by: Alex <alex.tran1502@gmail.com>
This commit is contained in:
shenlong
GitHub
parent
067338b0ed
commit
81ed54aa61
28 changed files with 1065 additions and 117 deletions
|
|
@ -1,37 +1,36 @@
|
|||
import 'dart:async';
|
||||
import 'dart:convert';
|
||||
|
||||
import 'package:flutter/foundation.dart';
|
||||
import 'package:immich_mobile/domain/interfaces/sync_api.interface.dart';
|
||||
import 'package:immich_mobile/domain/models/sync/sync_event.model.dart';
|
||||
import 'package:immich_mobile/services/api.service.dart';
|
||||
import 'package:openapi/api.dart';
|
||||
import 'package:http/http.dart' as http;
|
||||
import 'package:immich_mobile/constants/constants.dart';
|
||||
import 'package:immich_mobile/domain/interfaces/sync_api.interface.dart';
|
||||
import 'package:immich_mobile/domain/models/sync_event.model.dart';
|
||||
import 'package:immich_mobile/services/api.service.dart';
|
||||
import 'package:logging/logging.dart';
|
||||
import 'package:openapi/api.dart';
|
||||
|
||||
class SyncApiRepository implements ISyncApiRepository {
|
||||
final Logger _logger = Logger('SyncApiRepository');
|
||||
final ApiService _api;
|
||||
const SyncApiRepository(this._api);
|
||||
final int _batchSize;
|
||||
SyncApiRepository(this._api, {int batchSize = kSyncEventBatchSize})
|
||||
: _batchSize = batchSize;
|
||||
|
||||
@override
|
||||
Stream<List<SyncEvent>> watchUserSyncEvent() {
|
||||
return _getSyncStream(
|
||||
SyncStreamDto(types: [SyncRequestType.usersV1]),
|
||||
);
|
||||
Stream<List<SyncEvent>> getSyncEvents(List<SyncRequestType> type) {
|
||||
return _getSyncStream(SyncStreamDto(types: type));
|
||||
}
|
||||
|
||||
@override
|
||||
Future<void> ack(String data) {
|
||||
return _api.syncApi.sendSyncAck(SyncAckSetDto(acks: [data]));
|
||||
Future<void> ack(List<String> data) {
|
||||
return _api.syncApi.sendSyncAck(SyncAckSetDto(acks: data));
|
||||
}
|
||||
|
||||
Stream<List<SyncEvent>> _getSyncStream(
|
||||
SyncStreamDto dto, {
|
||||
int batchSize = 5000,
|
||||
}) async* {
|
||||
Stream<List<SyncEvent>> _getSyncStream(SyncStreamDto dto) async* {
|
||||
final client = http.Client();
|
||||
final endpoint = "${_api.apiClient.basePath}/sync/stream";
|
||||
|
||||
final headers = <String, String>{
|
||||
final headers = {
|
||||
'Content-Type': 'application/json',
|
||||
'Accept': 'application/jsonlines+json',
|
||||
};
|
||||
|
|
@ -61,52 +60,54 @@ class SyncApiRepository implements ISyncApiRepository {
|
|||
|
||||
await for (final chunk in response.stream.transform(utf8.decoder)) {
|
||||
previousChunk += chunk;
|
||||
final parts = previousChunk.split('\n');
|
||||
final parts = previousChunk.toString().split('\n');
|
||||
previousChunk = parts.removeLast();
|
||||
lines.addAll(parts);
|
||||
|
||||
if (lines.length < batchSize) {
|
||||
if (lines.length < _batchSize) {
|
||||
continue;
|
||||
}
|
||||
|
||||
yield await compute(_parseSyncResponse, lines);
|
||||
yield _parseSyncResponse(lines);
|
||||
lines.clear();
|
||||
}
|
||||
} finally {
|
||||
if (lines.isNotEmpty) {
|
||||
yield await compute(_parseSyncResponse, lines);
|
||||
yield _parseSyncResponse(lines);
|
||||
}
|
||||
client.close();
|
||||
}
|
||||
}
|
||||
|
||||
List<SyncEvent> _parseSyncResponse(List<String> lines) {
|
||||
final List<SyncEvent> data = [];
|
||||
|
||||
for (final line in lines) {
|
||||
try {
|
||||
final jsonData = jsonDecode(line);
|
||||
final type = SyncEntityType.fromJson(jsonData['type'])!;
|
||||
final dataJson = jsonData['data'];
|
||||
final ack = jsonData['ack'];
|
||||
final converter = _kResponseMap[type];
|
||||
if (converter == null) {
|
||||
_logger.warning("[_parseSyncResponse] Unknown type $type");
|
||||
continue;
|
||||
}
|
||||
|
||||
data.add(SyncEvent(type: type, data: converter(dataJson), ack: ack));
|
||||
} catch (error, stack) {
|
||||
_logger.severe("[_parseSyncResponse] Error parsing json", error, stack);
|
||||
}
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
}
|
||||
|
||||
// ignore: avoid-dynamic
|
||||
const _kResponseMap = <SyncEntityType, Function(dynamic)>{
|
||||
SyncEntityType.userV1: SyncUserV1.fromJson,
|
||||
SyncEntityType.userDeleteV1: SyncUserDeleteV1.fromJson,
|
||||
SyncEntityType.partnerV1: SyncPartnerV1.fromJson,
|
||||
SyncEntityType.partnerDeleteV1: SyncPartnerDeleteV1.fromJson,
|
||||
};
|
||||
|
||||
// Need to be outside of the class to be able to use compute
|
||||
List<SyncEvent> _parseSyncResponse(List<String> lines) {
|
||||
final List<SyncEvent> data = [];
|
||||
|
||||
for (var line in lines) {
|
||||
try {
|
||||
final jsonData = jsonDecode(line);
|
||||
final type = SyncEntityType.fromJson(jsonData['type'])!;
|
||||
final dataJson = jsonData['data'];
|
||||
final ack = jsonData['ack'];
|
||||
final converter = _kResponseMap[type];
|
||||
if (converter == null) {
|
||||
debugPrint("[_parseSyncReponse] Unknown type $type");
|
||||
continue;
|
||||
}
|
||||
|
||||
data.add(SyncEvent(data: converter(dataJson), ack: ack));
|
||||
} catch (error, stack) {
|
||||
debugPrint("[_parseSyncReponse] Error parsing json $error $stack");
|
||||
}
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,104 @@
|
|||
import 'package:drift/drift.dart';
|
||||
import 'package:immich_mobile/domain/interfaces/sync_stream.interface.dart';
|
||||
import 'package:immich_mobile/extensions/string_extensions.dart';
|
||||
import 'package:immich_mobile/infrastructure/entities/partner.entity.drift.dart';
|
||||
import 'package:immich_mobile/infrastructure/entities/user.entity.drift.dart';
|
||||
import 'package:immich_mobile/infrastructure/repositories/db.repository.dart';
|
||||
import 'package:logging/logging.dart';
|
||||
import 'package:openapi/api.dart';
|
||||
|
||||
class DriftSyncStreamRepository extends DriftDatabaseRepository
|
||||
implements ISyncStreamRepository {
|
||||
final Logger _logger = Logger('DriftSyncStreamRepository');
|
||||
final Drift _db;
|
||||
|
||||
DriftSyncStreamRepository(super.db) : _db = db;
|
||||
|
||||
@override
|
||||
Future<bool> deleteUsersV1(Iterable<SyncUserDeleteV1> data) async {
|
||||
try {
|
||||
await _db.batch((batch) {
|
||||
for (final user in data) {
|
||||
batch.delete(
|
||||
_db.userEntity,
|
||||
UserEntityCompanion(id: Value(user.userId.toUuidByte())),
|
||||
);
|
||||
}
|
||||
});
|
||||
return true;
|
||||
} catch (e, s) {
|
||||
_logger.severe('Error while processing SyncUserDeleteV1', e, s);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
@override
|
||||
Future<bool> updateUsersV1(Iterable<SyncUserV1> data) async {
|
||||
try {
|
||||
await _db.batch((batch) {
|
||||
for (final user in data) {
|
||||
final companion = UserEntityCompanion(
|
||||
name: Value(user.name),
|
||||
email: Value(user.email),
|
||||
);
|
||||
|
||||
batch.insert(
|
||||
_db.userEntity,
|
||||
companion.copyWith(id: Value(user.id.toUuidByte())),
|
||||
onConflict: DoUpdate((_) => companion),
|
||||
);
|
||||
}
|
||||
});
|
||||
return true;
|
||||
} catch (e, s) {
|
||||
_logger.severe('Error while processing SyncUserV1', e, s);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
@override
|
||||
Future<bool> deletePartnerV1(Iterable<SyncPartnerDeleteV1> data) async {
|
||||
try {
|
||||
await _db.batch((batch) {
|
||||
for (final partner in data) {
|
||||
batch.delete(
|
||||
_db.partnerEntity,
|
||||
PartnerEntityCompanion(
|
||||
sharedById: Value(partner.sharedById.toUuidByte()),
|
||||
sharedWithId: Value(partner.sharedWithId.toUuidByte()),
|
||||
),
|
||||
);
|
||||
}
|
||||
});
|
||||
return true;
|
||||
} catch (e, s) {
|
||||
_logger.severe('Error while processing SyncPartnerDeleteV1', e, s);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
@override
|
||||
Future<bool> updatePartnerV1(Iterable<SyncPartnerV1> data) async {
|
||||
try {
|
||||
await _db.batch((batch) {
|
||||
for (final partner in data) {
|
||||
final companion =
|
||||
PartnerEntityCompanion(inTimeline: Value(partner.inTimeline));
|
||||
|
||||
batch.insert(
|
||||
_db.partnerEntity,
|
||||
companion.copyWith(
|
||||
sharedById: Value(partner.sharedById.toUuidByte()),
|
||||
sharedWithId: Value(partner.sharedWithId.toUuidByte()),
|
||||
),
|
||||
onConflict: DoUpdate((_) => companion),
|
||||
);
|
||||
}
|
||||
});
|
||||
return true;
|
||||
} catch (e, s) {
|
||||
_logger.severe('Error while processing SyncPartnerV1', e, s);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue