feat: allow users to re-run facial recognition for their group (#30965)

i18n/en.json

@@ -740,6 +740,8 @@
   "close": "Close",
   "cluster_group": "Cluster group",
   "cluster_group_description": "People are recognized across the photos of everyone in the group",
+  "cluster_group_facial_recognition": "Reset facial recognition",
+  "cluster_group_facial_recognition_prompt": "Resetting facial recognition will delete all people for all users in this group. Already set names and birth dates will be lost. Are you sure you want to continue?",
   "cluster_group_invite_description": "You have been invited to join this cluster group.",
   "collapse": "Collapse",
   "collapse_all": "Collapse all",

mise.lock

@@ -91,6 +91,23 @@ url_api = "https://api.github.com/repos/jellyfin/jellyfin-ffmpeg/releases/assets
 version = "7.1.3-6"
 backend = "github:jellyfin/jellyfin-ffmpeg"
 
+[tools."github:jellyfin/jellyfin-ffmpeg".options]
+asset_pattern = "jellyfin-ffmpeg_*_portable_linuxarm64-gpl.tar.xz"
+
+[tools."github:jellyfin/jellyfin-ffmpeg"."platforms.linux-arm64"]
+checksum = "sha256:bea03c670e8cc5bfe9edc0c5d624d4735421610cef5e808db93e7d8596952886"
+url = "https://github.com/jellyfin/jellyfin-ffmpeg/releases/download/v7.1.3-6/jellyfin-ffmpeg_7.1.3-6_portable_linuxarm64-gpl.tar.xz"
+url_api = "https://api.github.com/repos/jellyfin/jellyfin-ffmpeg/releases/assets/409048876"
+
+[tools."github:jellyfin/jellyfin-ffmpeg"."platforms.linux-arm64-musl"]
+checksum = "sha256:bea03c670e8cc5bfe9edc0c5d624d4735421610cef5e808db93e7d8596952886"
+url = "https://github.com/jellyfin/jellyfin-ffmpeg/releases/download/v7.1.3-6/jellyfin-ffmpeg_7.1.3-6_portable_linuxarm64-gpl.tar.xz"
+url_api = "https://api.github.com/repos/jellyfin/jellyfin-ffmpeg/releases/assets/409048876"
+
+[[tools."github:jellyfin/jellyfin-ffmpeg"]]
+version = "7.1.3-6"
+backend = "github:jellyfin/jellyfin-ffmpeg"
+
 [tools."github:jellyfin/jellyfin-ffmpeg".options]
 asset_pattern = "jellyfin-ffmpeg_*_portable_linux64-gpl.tar.xz"
 
@@ -120,23 +137,6 @@ url_api = "https://api.github.com/repos/jellyfin/jellyfin-ffmpeg/releases/assets
 version = "7.1.3-6"
 backend = "github:jellyfin/jellyfin-ffmpeg"
 
-[tools."github:jellyfin/jellyfin-ffmpeg".options]
-asset_pattern = "jellyfin-ffmpeg_*_portable_linuxarm64-gpl.tar.xz"
-
-[tools."github:jellyfin/jellyfin-ffmpeg"."platforms.linux-arm64"]
-checksum = "sha256:bea03c670e8cc5bfe9edc0c5d624d4735421610cef5e808db93e7d8596952886"
-url = "https://github.com/jellyfin/jellyfin-ffmpeg/releases/download/v7.1.3-6/jellyfin-ffmpeg_7.1.3-6_portable_linuxarm64-gpl.tar.xz"
-url_api = "https://api.github.com/repos/jellyfin/jellyfin-ffmpeg/releases/assets/409048876"
-
-[tools."github:jellyfin/jellyfin-ffmpeg"."platforms.linux-arm64-musl"]
-checksum = "sha256:bea03c670e8cc5bfe9edc0c5d624d4735421610cef5e808db93e7d8596952886"
-url = "https://github.com/jellyfin/jellyfin-ffmpeg/releases/download/v7.1.3-6/jellyfin-ffmpeg_7.1.3-6_portable_linuxarm64-gpl.tar.xz"
-url_api = "https://api.github.com/repos/jellyfin/jellyfin-ffmpeg/releases/assets/409048876"
-
-[[tools."github:jellyfin/jellyfin-ffmpeg"]]
-version = "7.1.3-6"
-backend = "github:jellyfin/jellyfin-ffmpeg"
-
 [tools."github:jellyfin/jellyfin-ffmpeg".options]
 asset_pattern = "jellyfin-ffmpeg_*_portable_macarm64-gpl.tar.xz"
 

open-api/immich-openapi-specs.json

@@ -6298,6 +6298,51 @@
         "x-immich-permission": "clusterGroup.leave"
       }
     },
+    "/cluster-groups/{id}/regenerate-people": {
+      "post": {
+        "description": "Forcefully re-run facial recognition for all faces of users in this group.",
+        "operationId": "clusterGroupRegeneratePeople",
+        "parameters": [
+          {
+            "name": "id",
+            "required": true,
+            "in": "path",
+            "schema": {
+              "format": "uuid",
+              "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-4[0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12})$",
+              "type": "string"
+            }
+          }
+        ],
+        "responses": {
+          "204": {
+            "description": ""
+          }
+        },
+        "security": [
+          {
+            "bearer": []
+          },
+          {
+            "cookie": []
+          },
+          {
+            "api_key": []
+          }
+        ],
+        "summary": "Regenerate people of users in cluster group",
+        "tags": [
+          "Cluster groups"
+        ],
+        "x-immich-history": [
+          {
+            "version": "v3.2.0",
+            "state": "Added"
+          }
+        ],
+        "x-immich-permission": "clusterGroup.read"
+      }
+    },
     "/cluster-groups/{id}/requests": {
       "get": {
         "description": "Retrieve the pending requests for other users to join the cluster group.",

packages/sdk/src/fetch-client.ts

@@ -5016,6 +5016,17 @@ export function leaveClusterGroup({ id }: {
         method: "POST"
     }));
 }
+/**
+ * Regenerate people of users in cluster group
+ */
+export function clusterGroupRegeneratePeople({ id }: {
+    id: string;
+}, opts?: Oazapfts.RequestOpts) {
+    return oazapfts.ok(oazapfts.fetchText(`/cluster-groups/${encodeURIComponent(id)}/regenerate-people`, {
+        ...opts,
+        method: "POST"
+    }));
+}
 /**
  * Retrieve the requests sent by a cluster group
  */

server/src/controllers/cluster-group.controller.ts

@@ -93,6 +93,18 @@ export class ClusterGroupController {
     return value;
   }
 
+  @Post(':id/regenerate-people')
+  @Authenticated({ permission: Permission.ClusterGroupRead })
+  @HttpCode(HttpStatus.NO_CONTENT)
+  @Endpoint({
+    summary: 'Regenerate people of users in cluster group',
+    description: 'Forcefully re-run facial recognition for all faces of users in this group.',
+    history: new HistoryBuilder().added('v3.2.0'),
+  })
+  clusterGroupRegeneratePeople(@Auth() auth: AuthDto, @Param() { id }: UUIDParamDto): Promise<void> {
+    return this.service.regeneratePeople(auth, id);
+  }
+
   @Post(':id/leave')
   @Authenticated({ permission: Permission.ClusterGroupLeave })
   @HttpCode(HttpStatus.NO_CONTENT)

server/src/queries/person.repository.sql

@@ -7,6 +7,18 @@ set
 where
   "asset_face"."personGroupId" = $2
 
+-- PersonRepository.unassignFaces
+update "asset_face"
+set
+  "personGroupId" = $1
+from
+  "asset"
+  inner join "user" on "user"."id" = "asset"."ownerId"
+where
+  "asset_face"."assetId" = "asset"."id"
+  and "asset_face"."sourceType" = $2
+  and "user"."clusterGroupId" = $3
+
 -- PersonRepository.delete
 delete from "person"
 where
@@ -46,6 +58,20 @@ where
       "user"."clusterGroupId" = "cluster_group"."id"
   )
 
+-- PersonRepository.getAllFaces
+select
+  "asset_face".*
+from
+  "asset_face"
+  inner join "asset" on "asset"."id" = "asset_face"."assetId"
+  inner join "user" on "user"."id" = "asset"."ownerId"
+where
+  "asset_face"."personGroupId" is null
+  and "asset_face"."sourceType" = $1
+  and "user"."clusterGroupId" = $2
+  and "asset_face"."deletedAt" is null
+  and "asset_face"."isVisible" is true
+
 -- PersonRepository.getFileSamples
 select
   "ownerId",
@@ -112,12 +138,15 @@ from
   left join "asset_face" on "asset_face"."personGroupId" = "person"."personGroupId"
 where
   "asset_face"."deletedAt" is null
-  and "asset_face"."isVisible" is true
+  and (
+    "asset_face"."isVisible" is null
+    or "asset_face"."isVisible" = $1
+  )
 group by
   "person"."ownerId",
   "person"."personGroupId"
 having
-  count("asset_face"."assetId") = $1
+  count("asset_face"."assetId") = $2
 
 -- PersonRepository.getFaces
 select

server/src/repositories/person.repository.ts

@@ -58,9 +58,10 @@ export interface GetAllFacesOptions {
   personGroupId?: string | null;
   assetId?: string;
   sourceType?: SourceType;
+  clusterGroupId?: string;
 }
 
-export type UnassignFacesOptions = DeleteFacesOptions;
+export type UnassignFacesOptions = DeleteFacesOptions & { clusterGroupId?: string };
 
 export type GetFacesOptions = WithPersonOptions & { isVisible?: boolean };
 
@@ -112,11 +113,17 @@ export class PersonRepository {
     return Number(result.numChangedRows ?? 0);
   }
 
-  async unassignFaces({ sourceType }: UnassignFacesOptions): Promise<void> {
+  @GenerateSql({ params: [{ sourceType: SourceType.MachineLearning, clusterGroupId: DummyValue.UUID }] })
+  async unassignFaces({ sourceType, clusterGroupId }: UnassignFacesOptions): Promise<void> {
     await this.db
       .updateTable('asset_face')
       .set({ personGroupId: null })
+      .from('asset')
+      .whereRef('asset_face.assetId', '=', 'asset.id')
       .where('asset_face.sourceType', '=', sourceType)
+      .$if(!!clusterGroupId, (qb) =>
+        qb.innerJoin('user', 'user.id', 'asset.ownerId').where('user.clusterGroupId', '=', clusterGroupId!),
+      )
       .execute();
   }
 
@@ -179,6 +186,10 @@ export class PersonRepository {
     await this.db.deleteFrom('asset_face').where('asset_face.sourceType', '=', sourceType).execute();
   }
 
+  @GenerateSql({
+    params: [{ personGroupId: null, sourceType: SourceType.MachineLearning, clusterGroupId: DummyValue.UUID }],
+    stream: true,
+  })
   getAllFaces(options: GetAllFacesOptions = {}) {
     return this.db
       .selectFrom('asset_face')
@@ -187,6 +198,12 @@ export class PersonRepository {
       .$if(!!options.personGroupId, (qb) => qb.where('asset_face.personGroupId', '=', options.personGroupId!))
       .$if(!!options.sourceType, (qb) => qb.where('asset_face.sourceType', '=', options.sourceType!))
       .$if(!!options.assetId, (qb) => qb.where('asset_face.assetId', '=', options.assetId!))
+      .$if(!!options.clusterGroupId, (qb) =>
+        qb
+          .innerJoin('asset', 'asset.id', 'asset_face.assetId')
+          .innerJoin('user', 'user.id', 'asset.ownerId')
+          .where('user.clusterGroupId', '=', options.clusterGroupId!),
+      )
       .where('asset_face.deletedAt', 'is', null)
       .where('asset_face.isVisible', 'is', true)
       .stream();
@@ -288,7 +305,7 @@ export class PersonRepository {
       .selectAll('person')
       .leftJoin('asset_face', 'asset_face.personGroupId', 'person.personGroupId')
       .where('asset_face.deletedAt', 'is', null)
-      .where('asset_face.isVisible', 'is', true)
+      .where((eb) => eb.or([eb('asset_face.isVisible', 'is', null), eb('asset_face.isVisible', '=', true)]))
       .having((eb) => eb.fn.count('asset_face.assetId'), '=', 0)
       .groupBy(['person.ownerId', 'person.personGroupId'])
       .execute();

server/src/services/cluster-group.service.ts

@@ -7,7 +7,7 @@ import {
   mapClusterGroupRequest,
 } from 'src/dtos/cluster-group.dto';
 import { mapUser, UserResponseDto } from 'src/dtos/user.dto';
-import { Permission } from 'src/enum';
+import { JobName, Permission } from 'src/enum';
 import { BaseService } from 'src/services/base.service';
 import { findOrFail } from 'src/utils/misc';
 
@@ -72,6 +72,15 @@ export class ClusterGroupService extends BaseService {
     await this.clusterGroupRepository.deleteRequest(id);
   }
 
+  async regeneratePeople(auth: AuthDto, id: string) {
+    await this.requireAccess({ auth, permission: Permission.ClusterGroupRead, ids: [id] });
+
+    await this.jobRepository.queue({
+      name: JobName.FacialRecognitionQueueAll,
+      data: { clusterGroupId: id, force: true },
+    });
+  }
+
   async leave(auth: AuthDto, clusterGroupId: string): Promise<void> {
     await this.requireAccess({ auth, permission: Permission.ClusterGroupLeave, ids: [clusterGroupId] });
 

server/src/services/person.service.spec.ts

@@ -787,7 +787,7 @@ describe(PersonService.name, () => {
 
       await sut.handleQueueRecognizeFaces({ force: true });
 
-      expect(mocks.person.getAllFaces).toHaveBeenCalledWith(undefined);
+      expect(mocks.person.getAllFaces).toHaveBeenCalledWith({});
       expect(mocks.job.queueAll).toHaveBeenCalledWith([
         {
           name: JobName.FacialRecognition,

server/src/services/person.service.ts

@@ -414,7 +414,11 @@ export class PersonService extends BaseService {
   }
 
   @OnJob({ name: JobName.FacialRecognitionQueueAll, queue: QueueName.FacialRecognition })
-  async handleQueueRecognizeFaces({ force, nightly }: JobOf<JobName.FacialRecognitionQueueAll>): Promise<JobStatus> {
+  async handleQueueRecognizeFaces({
+    force,
+    nightly,
+    clusterGroupId,
+  }: JobOf<JobName.FacialRecognitionQueueAll>): Promise<JobStatus> {
     const { machineLearning } = await this.getConfig({ withCache: false });
     if (!isFacialRecognitionEnabled(machineLearning)) {
       return JobStatus.Skipped;
@@ -437,7 +441,8 @@ export class PersonService extends BaseService {
     const { waiting } = await this.jobRepository.getJobCounts(QueueName.FacialRecognition);
 
     if (force) {
-      await this.personRepository.unassignFaces({ sourceType: SourceType.MachineLearning });
+      console.log('unassigning faces');
+      await this.personRepository.unassignFaces({ clusterGroupId, sourceType: SourceType.MachineLearning });
       await this.handlePersonCleanup();
       await this.personRepository.vacuum({ reindexVectors: false });
     } else if (waiting) {
@@ -452,7 +457,9 @@ export class PersonService extends BaseService {
     const lastRun = new Date().toISOString();
 
     const faces = this.personRepository.getAllFaces(
-      force ? undefined : { personGroupId: null, sourceType: SourceType.MachineLearning },
+      force
+        ? { clusterGroupId, sourceType: clusterGroupId ? SourceType.MachineLearning : undefined }
+        : { personGroupId: null, clusterGroupId, sourceType: SourceType.MachineLearning },
     );
     for await (const batch of batched(faces)) {
       await this.jobRepository.queueAll(

server/src/types.ts

@@ -321,6 +321,10 @@ export interface IIntegrityPathWithChecksumJob {
   items: { path: string; reportId: string | null; checksum?: string | null }[];
 }
 
+export interface IFacialRecognitionQueueAll extends INightlyJob {
+  clusterGroupId?: string;
+}
+
 export interface JobCounts {
   active: number;
   completed: number;
@@ -374,7 +378,7 @@ export type JobItem =
   // Facial Recognition
   | { name: JobName.AssetDetectFacesQueueAll; data: IBaseJob }
   | { name: JobName.AssetDetectFaces; data: IEntityJob }
-  | { name: JobName.FacialRecognitionQueueAll; data: INightlyJob }
+  | { name: JobName.FacialRecognitionQueueAll; data: IFacialRecognitionQueueAll }
   | { name: JobName.FacialRecognition; data: IDeferrableJob }
   | { name: JobName.PersonGenerateThumbnail; data: IPersonJob }
 

server/test/medium/specs/services/person.service.spec.ts

@@ -2,14 +2,17 @@ import { Kysely } from 'kysely';
 import { DateTime } from 'luxon';
 import { AssetEditAction, MirrorAxis } from 'src/dtos/editing.dto';
 import { AssetFaceCreateDto } from 'src/dtos/person.dto';
+import { JobName } from 'src/enum';
 import { AccessRepository } from 'src/repositories/access.repository';
 import { AssetEditRepository } from 'src/repositories/asset-edit.repository';
 import { AssetRepository } from 'src/repositories/asset.repository';
+import { ConfigRepository } from 'src/repositories/config.repository';
 import { DatabaseRepository } from 'src/repositories/database.repository';
 import { JobRepository } from 'src/repositories/job.repository';
 import { LoggingRepository } from 'src/repositories/logging.repository';
 import { PersonRepository } from 'src/repositories/person.repository';
 import { StorageRepository } from 'src/repositories/storage.repository';
+import { SystemMetadataRepository } from 'src/repositories/system-metadata.repository';
 import { DB } from 'src/schema';
 import { PersonService } from 'src/services/person.service';
 import { newMediumService } from 'test/medium.factory';
@@ -21,7 +24,15 @@ let defaultDatabase: Kysely<DB>;
 const setup = (db?: Kysely<DB>) => {
   return newMediumService(PersonService, {
     database: db || defaultDatabase,
-    real: [AccessRepository, DatabaseRepository, PersonRepository, AssetRepository, AssetEditRepository],
+    real: [
+      AccessRepository,
+      ConfigRepository,
+      DatabaseRepository,
+      PersonRepository,
+      AssetRepository,
+      AssetEditRepository,
+      SystemMetadataRepository,
+    ],
     mock: [JobRepository, LoggingRepository, StorageRepository],
   });
 };
@@ -88,6 +99,73 @@ describe(PersonService.name, () => {
     });
   });
 
+  describe('handleQueueRecognizeFaces', () => {
+    it('should delete all people and queue faces for recognition', async () => {
+      const { sut, ctx } = setup();
+      const jobRepo = ctx.getMock(JobRepository);
+      ctx.getMock(StorageRepository).unlink.mockResolvedValue();
+      jobRepo.waitForQueueCompletion.mockResolvedValue();
+      jobRepo.getJobCounts.mockResolvedValue({ active: 0, waiting: 0, completed: 0, delayed: 0, failed: 0, paused: 0 });
+      jobRepo.queueAll.mockResolvedValue();
+
+      const { user } = await ctx.newUser();
+      const { user: user1 } = await ctx.newUser();
+      const { asset } = await ctx.newAsset({ ownerId: user.id });
+      const { asset: assetUser1 } = await ctx.newAsset({ ownerId: user1.id });
+      const { person } = await ctx.newPerson({ ownerId: user.id });
+      const { person: personUser1 } = await ctx.newPerson({ ownerId: user1.id });
+      const { assetFace } = await ctx.newAssetFace({ assetId: asset.id, personGroupId: person.personGroupId });
+      const { assetFace: assetFaceUser1 } = await ctx.newAssetFace({
+        assetId: assetUser1.id,
+        personGroupId: personUser1.personGroupId,
+      });
+
+      await sut.handleQueueRecognizeFaces({ force: true });
+
+      await expect(ctx.database.selectFrom('person').selectAll().execute()).resolves.toHaveLength(0);
+      expect(jobRepo.queueAll).toHaveBeenCalledWith(
+        expect.objectContaining([
+          { name: JobName.FacialRecognition, data: { id: assetFace.id, deferred: false } },
+          { name: JobName.FacialRecognition, data: { id: assetFaceUser1.id, deferred: false } },
+        ]),
+      );
+    });
+
+    it('should only delete all people of a specified cluster group and queue their faces for recognition', async () => {
+      const { sut, ctx } = setup();
+      const jobRepo = ctx.getMock(JobRepository);
+      ctx.getMock(StorageRepository).unlink.mockResolvedValue();
+      jobRepo.waitForQueueCompletion.mockResolvedValue();
+      jobRepo.getJobCounts.mockResolvedValue({ active: 0, waiting: 0, completed: 0, delayed: 0, failed: 0, paused: 0 });
+      jobRepo.queueAll.mockResolvedValue();
+
+      const { user } = await ctx.newUser();
+      const { user: user1 } = await ctx.newUser();
+      const { asset } = await ctx.newAsset({ ownerId: user.id });
+      const { asset: assetUser1 } = await ctx.newAsset({ ownerId: user1.id });
+      const { person } = await ctx.newPerson({ ownerId: user.id });
+      const { person: personUser1 } = await ctx.newPerson({ ownerId: user1.id });
+      const { assetFace } = await ctx.newAssetFace({ assetId: asset.id, personGroupId: person.personGroupId });
+      const { assetFace: assetFaceUser1 } = await ctx.newAssetFace({
+        assetId: assetUser1.id,
+        personGroupId: personUser1.personGroupId,
+      });
+
+      await sut.handleQueueRecognizeFaces({ force: true, clusterGroupId: user.clusterGroupId });
+
+      await expect(ctx.database.selectFrom('person').selectAll().execute()).resolves.toHaveLength(1);
+      expect(jobRepo.queueAll).toHaveBeenCalledWith(
+        expect.objectContaining([{ name: JobName.FacialRecognition, data: { id: assetFace.id, deferred: false } }]),
+      );
+      expect(jobRepo.queueAll).not.toHaveBeenCalledWith(
+        expect.objectContaining([
+          { name: JobName.FacialRecognition, data: { id: assetFace.id, deferred: false } },
+          { name: JobName.FacialRecognition, data: { id: assetFaceUser1.id, deferred: false } },
+        ]),
+      );
+    });
+  });
+
   describe('mergePerson', () => {
     it('should merge people of multiple users', async () => {
       const { sut, ctx } = setup();

web/src/routes/(user)/user-settings/SharingSettings.svelte

@@ -8,6 +8,7 @@
   import { handleError } from '$lib/utils/handle-error';
   import {
     acceptClusterGroupRequest,
+    clusterGroupRegeneratePeople,
     createClusterGroupRequest,
     createPartner,
     deleteClusterGroupRequest,
@@ -25,7 +26,7 @@
     type PartnerResponseDto,
     type UserResponseDto,
   } from '@immich/sdk';
-  import { Button, Card, CardBody, Icon, IconButton, modalManager, Text } from '@immich/ui';
+  import { Button, Card, CardBody, HStack, Icon, IconButton, modalManager, Text } from '@immich/ui';
   import { mdiCheck, mdiClose } from '@mdi/js';
   import { onMount } from 'svelte';
   import { t } from 'svelte-i18n';
@@ -222,6 +223,24 @@
       handleError(error, $t('errors.unable_to_update_timeline_display_status'));
     }
   };
+
+  const handleRerunFacialRecognition = async () => {
+    const confirmed = await modalManager.showDialog({
+      title: $t('cluster_group_facial_recognition'),
+      prompt: $t('cluster_group_facial_recognition_prompt'),
+      size: 'medium',
+    });
+
+    if (!confirmed) {
+      return;
+    }
+
+    try {
+      await clusterGroupRegeneratePeople({ id: clusterGroupId });
+    } catch (error) {
+      handleError(error, $t('errors.something_went_wrong'));
+    }
+  };
 </script>
 
 <section class="my-4">
@@ -295,9 +314,12 @@
     </Card>
   {/if}
 
-  <div class="mt-5 flex justify-end">
+  <HStack fullWidth class="mt-5 justify-end">
+    <Button shape="round" size="small" onclick={() => handleRerunFacialRecognition()}
+      >{$t('cluster_group_facial_recognition')}</Button
+    >
     <Button shape="round" size="small" onclick={() => handleAddUsers()}>{$t('add_user')}</Button>
-  </div>
+  </HStack>
 </section>
 
 <section class="my-4">