helpy4/immich
1
0
Fork 0
mirror of https://github.com/immich-app/immich synced 2025-11-14 17:36:12 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat(mobile)!: batched full/initial sync (#4840)

Browse source 
* feat(mobile): batched full/initial sync

* use OptionalBetween

* skip/take as integer

---------

Co-authored-by: Fynn Petersen-Frey <zoodyy@users.noreply.github.com>
Co-authored-by: Alex Tran <alex.tran1502@gmail.com>
This commit is contained in:
Fynn Petersen-Frey 2023-11-06 18:40:43 +01:00 • committed by GitHub
parent 26fd9d7e5f
commit 21f2d3058a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 189 additions and 63 deletions
Download patch file Download diff file Expand all files Collapse all files

35
mobile/lib/shared/services/asset.service.dart
View file

@ -62,20 +62,31 @@ class AssetService {
/// Returns `null` if the server state did not change, else list of assets
Future<List<Asset>?> _getRemoteAssets(User user) async {
const int chunkSize = 5000;
try {
final List<AssetResponseDto>? assets =
await _apiService.assetApi.getAllAssets(
userId: user.id,
);
if (assets == null) {
return null;
} else if (assets.isNotEmpty && assets.first.ownerId != user.id) {
log.warning("Make sure that server and app versions match!"
" The server returned assets for user ${assets.first.ownerId}"
" while requesting assets of user ${user.id}");
return null;
final DateTime now = DateTime.now().toUtc();
final List<Asset> allAssets = [];
for (int i = 0;; i += chunkSize) {
final List<AssetResponseDto>? assets =
await _apiService.assetApi.getAllAssets(
userId: user.id,
// updatedBefore is important! without it we could
// a) get the same Asset multiple times in different versions (when
// the asset is modified while the chunks are loaded from the server)
// b) miss assets when new assets are inserted in between the calls
updatedBefore: now,
skip: i,
take: chunkSize,
);
if (assets == null) {
return null;
}
allAssets.addAll(assets.map(Asset.remote));
if (assets.length < chunkSize) {
break;
}
}
return assets.map(Asset.remote).toList();
return allAssets;
} catch (error, stack) {
log.severe(
'Error while getting remote assets: ${error.toString()}',

12
mobile/lib/shared/services/sync.service.dart
View file

@ -197,7 +197,7 @@ class SyncService {
User user,
FutureOr<List<Asset>?> Function(User user) loadAssets,
) async {
final DateTime now = DateTime.now();
final DateTime now = DateTime.now().toUtc();
final List<Asset>? remote = await loadAssets(user);
if (remote == null) {
return false;
@ -210,6 +210,10 @@ class SyncService {
assert(inDb.isSorted(Asset.compareByChecksum), "inDb not sorted!");
remote.sort(Asset.compareByChecksum);
// filter our duplicates that might be introduced by the chunked retrieval
remote.uniqueConsecutive(compare: Asset.compareByChecksum);
final (toAdd, toUpdate, toRemove) = _diffAssets(remote, inDb, remote: true);
if (toAdd.isEmpty && toUpdate.isEmpty && toRemove.isEmpty) {
await _updateUserAssetsETag(user, now);
@ -759,6 +763,12 @@ class SyncService {
final List<Asset> toAdd = [];
final List<Asset> toUpdate = [];
final List<Asset> toRemove = [];
if (assets.isEmpty || inDb.isEmpty) {
// fast path for trivial cases: halfes memory usage during initial sync
return assets.isEmpty
? (toAdd, toUpdate, inDb) // remove all from DB
: (assets, toUpdate, toRemove); // add all assets
}
diffSortedListsSync(
inDb,
assets,