fix(server): Fix delay with multiple ml servers (#16284)

* Prospective fix for ensuring that known active ML servers are used to reduce search delay.

* Added some logging and renamed backoff const.

* Fix lint issues.

* Update to use env vars for timeouts and updated documentation and strings.

* Fix docs.

* Make counter logic clearer.

* Minor readability improvements.

* Extract  skipUrl logic per feedback, and change log to verbose.

* Make code harder to read.

server/src/constants.ts

@@ -38,6 +38,11 @@ export const ONE_HOUR = Duration.fromObject({ hours: 1 });
 
 export const APP_MEDIA_LOCATION = process.env.IMMICH_MEDIA_LOCATION || './upload';
 
+export const MACHINE_LEARNING_PING_TIMEOUT = Number(process.env.MACHINE_LEARNING_PING_TIMEOUT || 2000);
+export const MACHINE_LEARNING_AVAILABILITY_BACKOFF_TIME = Number(
+  process.env.MACHINE_LEARNING_AVAILABILITY_BACKOFF_TIME || 30_000,
+);
+
 export const citiesFile = 'cities500.txt';
 
 export const MOBILE_REDIRECT = 'app.immich:///oauth-callback';

server/src/repositories/machine-learning.repository.ts

@@ -1,5 +1,6 @@
 import { Injectable } from '@nestjs/common';
 import { readFile } from 'node:fs/promises';
+import { MACHINE_LEARNING_AVAILABILITY_BACKOFF_TIME, MACHINE_LEARNING_PING_TIMEOUT } from 'src/constants';
 import { CLIPConfig } from 'src/dtos/model-config.dto';
 import { LoggingRepository } from 'src/repositories/logging.repository';
 
@@ -55,16 +56,80 @@ export type MachineLearningRequest = ClipVisualRequest | ClipTextualRequest | Fa
 
 @Injectable()
 export class MachineLearningRepository {
+  // Note that deleted URL's are not removed from this map (ie: they're leaked)
+  // Cleaning them up is low priority since there should be very few over a
+  // typical server uptime cycle
+  private urlAvailability: {
+    [url: string]:
+      | {
+          active: boolean;
+          lastChecked: number;
+        }
+      | undefined;
+  };
+
   constructor(private logger: LoggingRepository) {
     this.logger.setContext(MachineLearningRepository.name);
+    this.urlAvailability = {};
+  }
+
+  private setUrlAvailability(url: string, active: boolean) {
+    const current = this.urlAvailability[url];
+    if (current?.active !== active) {
+      this.logger.verbose(`Setting ${url} ML server to ${active ? 'active' : 'inactive'}.`);
+    }
+    this.urlAvailability[url] = {
+      active,
+      lastChecked: Date.now(),
+    };
+  }
+
+  private async checkAvailability(url: string) {
+    let active = false;
+    try {
+      const response = await fetch(new URL('/ping', url), {
+        signal: AbortSignal.timeout(MACHINE_LEARNING_PING_TIMEOUT),
+      });
+      active = response.ok;
+    } catch {}
+    this.setUrlAvailability(url, active);
+    return active;
+  }
+
+  private async shouldSkipUrl(url: string) {
+    const availability = this.urlAvailability[url];
+    if (availability === undefined) {
+      // If this is a new endpoint, then check inline and skip if it fails
+      if (!(await this.checkAvailability(url))) {
+        return true;
+      }
+      return false;
+    }
+    if (!availability.active && Date.now() - availability.lastChecked < MACHINE_LEARNING_AVAILABILITY_BACKOFF_TIME) {
+      // If this is an old inactive endpoint that hasn't been checked in a
+      // while then check but don't wait for the result, just skip it
+      // This avoids delays on every search whilst allowing higher priority
+      // ML servers to recover over time.
+      void this.checkAvailability(url);
+      return true;
+    }
+    return false;
   }
 
   private async predict<T>(urls: string[], payload: ModelPayload, config: MachineLearningRequest): Promise<T> {
     const formData = await this.getFormData(payload, config);
+    let urlCounter = 0;
     for (const url of urls) {
+      urlCounter++;
+      const isLast = urlCounter >= urls.length;
+      if (!isLast && (await this.shouldSkipUrl(url))) {
+        continue;
+      }
+
       try {
         const response = await fetch(new URL('/predict', url), { method: 'POST', body: formData });
         if (response.ok) {
+          this.setUrlAvailability(url, true);
           return response.json();
         }
 
@@ -76,6 +141,7 @@ export class MachineLearningRepository {
           `Machine learning request to "${url}" failed: ${error instanceof Error ? error.message : error}`,
         );
       }
+      this.setUrlAvailability(url, false);
     }
 
     throw new Error(`Machine learning request '${JSON.stringify(config)}' failed for all URLs`);