feat(server): Automatic watching of library folders (#6192)

* feat: initial watch support * allow offline files * chore: ignore query errors when resetting e2e db * revert db query * add savepoint * guard the user query * chore: openapi and db migration * wip * support multiple libraries * fix tests * wip * can now cleanup chokidar watchers * fix unit tests * add library watch queue * add missing init from merge * wip * can now filter file extensions * remove watch api from non job client * Fix e2e test * watch library with updated import path and exclusion pattern * add library watch frontend ui * case sensitive watching extensions * can auto watch libraries * move watcher e2e tests to separate file * don't watch libraries from a queue * use event emitters * shorten e2e test timeout * refactor chokidar code to filesystem provider * expose chokidar parameters to config file * fix storage mock * set default config for library watching * add fs provider mocks * cleanup * add more unit tests for watcher * chore: fix format + sql * add more tests * move unwatch feature back to library service * add file event unit tests * chore: formatting * add documentation * fix e2e tests * chore: fix e2e tests * fix library updating * test cleanup * fix typo * cleanup * fixing as per pr comments * reduce library watch config file * update storage config and mocks * move negative event tests to unit tests * fix library watcher e2e * make watch configuration global * remove the feature flag * refactor watcher teardown * fix microservices init * centralize asset scan job queue * improve docs * add more tests * chore: open api * initialize app service * fix docs * fix library watch feature flag * Update docs/docs/features/libraries.md Co-authored-by: Daniel Dietzler <36593685+danieldietzler@users.noreply.github.com> * fix: import right app service * don't be truthy * fix test speling * stricter library update tests * move fs watcher mock to external file * subscribe to config changes * docker does not need polling * make library watch() private * feat: add configuration ui --------- Co-authored-by: Daniel Dietzler <36593685+danieldietzler@users.noreply.github.com> Co-authored-by: Alex Tran <alex.tran1502@gmail.com>
2025-11-07 17:27:20 +00:00 · 2024-01-31 09:15:54 +01:00 · 2024-01-31 09:15:54 +01:00 · 068e703e88
commit 068e703e88
parent 4079e92bbf
48 changed files with 1613 additions and 113 deletions
--- a/mobile/openapi/lib/model/system_config_library_watch_dto.dart
+++ b/mobile/openapi/lib/model/system_config_library_watch_dto.dart
@ -0,0 +1,114 @@
+//
+// AUTO-GENERATED FILE, DO NOT MODIFY!
+//
+// @dart=2.12
+
+// ignore_for_file: unused_element, unused_import
+// ignore_for_file: always_put_required_named_parameters_first
+// ignore_for_file: constant_identifier_names
+// ignore_for_file: lines_longer_than_80_chars
+
+part of openapi.api;
+
+class SystemConfigLibraryWatchDto {
+  /// Returns a new [SystemConfigLibraryWatchDto] instance.
+  SystemConfigLibraryWatchDto({
+    required this.enabled,
+    required this.interval,
+    required this.usePolling,
+  });
+
+  bool enabled;
+
+  int interval;
+
+  bool usePolling;
+
+  @override
+  bool operator ==(Object other) => identical(this, other) || other is SystemConfigLibraryWatchDto &&
+    other.enabled == enabled &&
+    other.interval == interval &&
+    other.usePolling == usePolling;
+
+  @override
+  int get hashCode =>
+    // ignore: unnecessary_parenthesis
+    (enabled.hashCode) +
+    (interval.hashCode) +
+    (usePolling.hashCode);
+
+  @override
+  String toString() => 'SystemConfigLibraryWatchDto[enabled=$enabled, interval=$interval, usePolling=$usePolling]';
+
+  Map<String, dynamic> toJson() {
+    final json = <String, dynamic>{};
+      json[r'enabled'] = this.enabled;
+      json[r'interval'] = this.interval;
+      json[r'usePolling'] = this.usePolling;
+    return json;
+  }
+
+  /// Returns a new [SystemConfigLibraryWatchDto] instance and imports its values from
+  /// [value] if it's a [Map], null otherwise.
+  // ignore: prefer_constructors_over_static_methods
+  static SystemConfigLibraryWatchDto? fromJson(dynamic value) {
+    if (value is Map) {
+      final json = value.cast<String, dynamic>();
+
+      return SystemConfigLibraryWatchDto(
+        enabled: mapValueOfType<bool>(json, r'enabled')!,
+        interval: mapValueOfType<int>(json, r'interval')!,
+        usePolling: mapValueOfType<bool>(json, r'usePolling')!,
+      );
+    }
+    return null;
+  }
+
+  static List<SystemConfigLibraryWatchDto> listFromJson(dynamic json, {bool growable = false,}) {
+    final result = <SystemConfigLibraryWatchDto>[];
+    if (json is List && json.isNotEmpty) {
+      for (final row in json) {
+        final value = SystemConfigLibraryWatchDto.fromJson(row);
+        if (value != null) {
+          result.add(value);
+        }
+      }
+    }
+    return result.toList(growable: growable);
+  }
+
+  static Map<String, SystemConfigLibraryWatchDto> mapFromJson(dynamic json) {
+    final map = <String, SystemConfigLibraryWatchDto>{};
+    if (json is Map && json.isNotEmpty) {
+      json = json.cast<String, dynamic>(); // ignore: parameter_assignments
+      for (final entry in json.entries) {
+        final value = SystemConfigLibraryWatchDto.fromJson(entry.value);
+        if (value != null) {
+          map[entry.key] = value;
+        }
+      }
+    }
+    return map;
+  }
+
+  // maps a json object with a list of SystemConfigLibraryWatchDto-objects as value to a dart map
+  static Map<String, List<SystemConfigLibraryWatchDto>> mapListFromJson(dynamic json, {bool growable = false,}) {
+    final map = <String, List<SystemConfigLibraryWatchDto>>{};
+    if (json is Map && json.isNotEmpty) {
+      // ignore: parameter_assignments
+      json = json.cast<String, dynamic>();
+      for (final entry in json.entries) {
+        map[entry.key] = SystemConfigLibraryWatchDto.listFromJson(entry.value, growable: growable,);
+      }
+    }
+    return map;
+  }
+
+  /// The list of required keys that must be present in a JSON.
+  static const requiredKeys = <String>{
+    'enabled',
+    'interval',
+    'usePolling',
+  };
+}
+