chore(server): Move library watcher to microservices (#7533)

* move watcher init to micro

* document watcher recovery

* chore: fix lint

* add try lock

* use global library watch lock

* fix: ensure lock stays on

* fix: mocks

* unit test for library watch lock

* move statement to correct test

* fix: correct return type of try lock

* fix: tests

* add library teardown

* add chokidar error handler

* make event strings an enum

* wait for event refactor

* refactor event type mocks

* expect correct error

* don't release lock in teardown

* chore: lint

* use enum

* fix mock

* fix lint

* fix watcher await

* remove await

* simplify typing

* remove async

* Revert "remove async"

This reverts commit 84ab5abac4.

* can now change watch settings at runtime

* fix lint

* only watch libraries if enabled

---------

Co-authored-by: mertalev <101130780+mertalev@users.noreply.github.com>
Co-authored-by: Alex Tran <alex.tran1502@gmail.com>

server/src/domain/library/library.service.ts

@@ -13,14 +13,16 @@ import { handlePromiseError, usePagination, validateCronExpression } from '../do
 import { IBaseJob, IEntityJob, ILibraryFileJob, ILibraryRefreshJob, JOBS_ASSET_PAGINATION_SIZE, JobName } from '../job';
 
 import {
+  DatabaseLock,
   IAccessRepository,
   IAssetRepository,
   ICryptoRepository,
+  IDatabaseRepository,
   IJobRepository,
   ILibraryRepository,
   IStorageRepository,
   ISystemConfigRepository,
-  IUserRepository,
+  StorageEventType,
   WithProperty,
 } from '../repositories';
 import { SystemConfigCore } from '../system-config';
@@ -43,6 +45,7 @@ export class LibraryService extends EventEmitter {
   private access: AccessCore;
   private configCore: SystemConfigCore;
   private watchLibraries = false;
+  private watchLock = false;
   private watchers: Record<string, () => Promise<void>> = {};
 
   constructor(
@@ -53,7 +56,7 @@ export class LibraryService extends EventEmitter {
     @Inject(IJobRepository) private jobRepository: IJobRepository,
     @Inject(ILibraryRepository) private repository: ILibraryRepository,
     @Inject(IStorageRepository) private storageRepository: IStorageRepository,
-    @Inject(IUserRepository) private userRepository: IUserRepository,
+    @Inject(IDatabaseRepository) private databaseRepository: IDatabaseRepository,
   ) {
     super();
     this.access = AccessCore.create(accessRepository);
@@ -68,8 +71,15 @@ export class LibraryService extends EventEmitter {
 
   async init() {
     const config = await this.configCore.getConfig();
+
     const { watch, scan } = config.library;
-    this.watchLibraries = watch.enabled;
+
+    // This ensures that library watching only occurs in one microservice
+    // TODO: we could make the lock be per-library instead of global
+    this.watchLock = await this.databaseRepository.tryLock(DatabaseLock.LibraryWatch);
+
+    this.watchLibraries = this.watchLock && watch.enabled;
+
     this.jobRepository.addCronJob(
       'libraryScan',
       scan.cronExpression,
@@ -89,6 +99,7 @@ export class LibraryService extends EventEmitter {
       this.jobRepository.updateCronJob('libraryScan', library.scan.cronExpression, library.scan.enabled);
 
       if (library.watch.enabled !== this.watchLibraries) {
+        // Watch configuration changed, update accordingly
         this.watchLibraries = library.watch.enabled;
         handlePromiseError(this.watchLibraries ? this.watchAll() : this.unwatchAll(), this.logger);
       }
@@ -134,7 +145,7 @@ export class LibraryService extends EventEmitter {
             if (matcher(path)) {
               await this.scanAssets(library.id, [path], library.ownerId, false);
             }
-            this.emit('add', path);
+            this.emit(StorageEventType.ADD, path);
           };
           return handlePromiseError(handler(), this.logger);
         },
@@ -145,7 +156,7 @@ export class LibraryService extends EventEmitter {
               // Note: if the changed file was not previously imported, it will be imported now.
               await this.scanAssets(library.id, [path], library.ownerId, false);
             }
-            this.emit('change', path);
+            this.emit(StorageEventType.CHANGE, path);
           };
           return handlePromiseError(handler(), this.logger);
         },
@@ -156,13 +167,13 @@ export class LibraryService extends EventEmitter {
             if (asset && matcher(path)) {
               await this.assetRepository.save({ id: asset.id, isOffline: true });
             }
-            this.emit('unlink', path);
+            this.emit(StorageEventType.UNLINK, path);
           };
           return handlePromiseError(handler(), this.logger);
         },
         onError: (error) => {
-          // TODO: should we log, or throw an exception?
           this.logger.error(`Library watcher for library ${library.id} encountered error: ${error}`);
+          this.emit(StorageEventType.ERROR, error);
         },
       },
     );
@@ -180,13 +191,25 @@ export class LibraryService extends EventEmitter {
     }
   }
 
-  async unwatchAll() {
+  async teardown() {
+    await this.unwatchAll();
+  }
+
+  private async unwatchAll() {
+    if (!this.watchLock) {
+      return false;
+    }
+
     for (const id in this.watchers) {
       await this.unwatch(id);
     }
   }
 
   async watchAll() {
+    if (!this.watchLock) {
+      return false;
+    }
+
     const libraries = await this.repository.getAll(false, LibraryType.EXTERNAL);
 
     for (const library of libraries) {
@@ -267,7 +290,7 @@ export class LibraryService extends EventEmitter {
 
     this.logger.log(`Creating ${dto.type} library for user ${auth.user.name}`);
 
-    if (dto.type === LibraryType.EXTERNAL && this.watchLibraries) {
+    if (dto.type === LibraryType.EXTERNAL) {
       await this.watch(library.id);
     }