feat(web): manual face tagging and deletion (#16062)

server/src/controllers/face.controller.ts

@@ -1,7 +1,13 @@
-import { Body, Controller, Get, Param, Put, Query } from '@nestjs/common';
+import { Body, Controller, Delete, Get, Param, Post, Put, Query } from '@nestjs/common';
 import { ApiTags } from '@nestjs/swagger';
 import { AuthDto } from 'src/dtos/auth.dto';
-import { AssetFaceResponseDto, FaceDto, PersonResponseDto } from 'src/dtos/person.dto';
+import {
+  AssetFaceCreateDto,
+  AssetFaceDeleteDto,
+  AssetFaceResponseDto,
+  FaceDto,
+  PersonResponseDto,
+} from 'src/dtos/person.dto';
 import { Permission } from 'src/enum';
 import { Auth, Authenticated } from 'src/middleware/auth.guard';
 import { PersonService } from 'src/services/person.service';
@@ -12,6 +18,12 @@ import { UUIDParamDto } from 'src/validation';
 export class FaceController {
   constructor(private service: PersonService) {}
 
+  @Post()
+  @Authenticated({ permission: Permission.FACE_CREATE })
+  createFace(@Auth() auth: AuthDto, @Body() dto: AssetFaceCreateDto) {
+    return this.service.createFace(auth, dto);
+  }
+
   @Get()
   @Authenticated({ permission: Permission.FACE_READ })
   getFaces(@Auth() auth: AuthDto, @Query() dto: FaceDto): Promise<AssetFaceResponseDto[]> {
@@ -27,4 +39,10 @@ export class FaceController {
   ): Promise<PersonResponseDto> {
     return this.service.reassignFacesById(auth, id, dto);
   }
+
+  @Delete(':id')
+  @Authenticated({ permission: Permission.FACE_DELETE })
+  deleteFace(@Auth() auth: AuthDto, @Param() { id }: UUIDParamDto, @Body() dto: AssetFaceDeleteDto) {
+    return this.service.deleteFace(auth, id, dto);
+  }
 }

server/src/db.d.ts

@@ -88,6 +88,7 @@ export interface AssetFaces {
   boundingBoxX2: Generated<number>;
   boundingBoxY1: Generated<number>;
   boundingBoxY2: Generated<number>;
+  deletedAt: Timestamp | null;
   id: Generated<string>;
   imageHeight: Generated<number>;
   imageWidth: Generated<number>;
@@ -334,6 +335,11 @@ export interface SocketIoAttachments {
   payload: Buffer | null;
 }
 
+export interface SystemConfig {
+  key: string;
+  value: string | null;
+}
+
 export interface SystemMetadata {
   key: string;
   value: Json;
@@ -448,6 +454,7 @@ export interface DB {
   shared_links: SharedLinks;
   smart_search: SmartSearch;
   socket_io_attachments: SocketIoAttachments;
+  system_config: SystemConfig;
   system_metadata: SystemMetadata;
   tag_asset: TagAsset;
   tags: Tags;

server/src/dtos/person.dto.ts

@@ -1,6 +1,6 @@
 import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger';
 import { Type } from 'class-transformer';
-import { IsArray, IsInt, IsNotEmpty, IsString, Max, Min, ValidateNested } from 'class-validator';
+import { IsArray, IsInt, IsNotEmpty, IsNumber, IsString, Max, Min, ValidateNested } from 'class-validator';
 import { DateTime } from 'luxon';
 import { PropertyLifecycle } from 'src/decorators';
 import { AuthDto } from 'src/dtos/auth.dto';
@@ -164,6 +164,43 @@ export class AssetFaceUpdateItem {
   assetId!: string;
 }
 
+export class AssetFaceCreateDto extends AssetFaceUpdateItem {
+  @ApiProperty({ type: 'integer' })
+  @IsNotEmpty()
+  @IsNumber()
+  imageWidth!: number;
+
+  @ApiProperty({ type: 'integer' })
+  @IsNotEmpty()
+  @IsNumber()
+  imageHeight!: number;
+
+  @ApiProperty({ type: 'integer' })
+  @IsNotEmpty()
+  @IsNumber()
+  x!: number;
+
+  @ApiProperty({ type: 'integer' })
+  @IsNotEmpty()
+  @IsNumber()
+  y!: number;
+
+  @ApiProperty({ type: 'integer' })
+  @IsNotEmpty()
+  @IsNumber()
+  width!: number;
+
+  @ApiProperty({ type: 'integer' })
+  @IsNotEmpty()
+  @IsNumber()
+  height!: number;
+}
+
+export class AssetFaceDeleteDto {
+  @IsNotEmpty()
+  force!: boolean;
+}
+
 export class PersonStatisticsResponseDto {
   @ApiProperty({ type: 'integer' })
   assets!: number;

server/src/entities/asset-face.entity.ts

@@ -50,4 +50,7 @@ export class AssetFaceEntity {
     nullable: true,
   })
   person!: PersonEntity | null;
+
+  @Column({ type: 'timestamptz' })
+  deletedAt!: Date | null;
 }

server/src/entities/asset.entity.ts

@@ -202,10 +202,14 @@ export function withSmartSearch<O>(qb: SelectQueryBuilder<DB, 'assets', O>) {
     .select((eb) => eb.fn.toJson(eb.table('smart_search')).as('smartSearch'));
 }
 
-export function withFaces(eb: ExpressionBuilder<DB, 'assets'>) {
-  return jsonArrayFrom(eb.selectFrom('asset_faces').selectAll().whereRef('asset_faces.assetId', '=', 'assets.id')).as(
-    'faces',
-  );
+export function withFaces(eb: ExpressionBuilder<DB, 'assets'>, withDeletedFace?: boolean) {
+  return jsonArrayFrom(
+    eb
+      .selectFrom('asset_faces')
+      .selectAll()
+      .whereRef('asset_faces.assetId', '=', 'assets.id')
+      .$if(!withDeletedFace, (qb) => qb.where('asset_faces.deletedAt', 'is', null)),
+  ).as('faces');
 }
 
 export function withFiles(eb: ExpressionBuilder<DB, 'assets'>, type?: AssetFileType) {
@@ -218,11 +222,12 @@ export function withFiles(eb: ExpressionBuilder<DB, 'assets'>, type?: AssetFileT
   ).as('files');
 }
 
-export function withFacesAndPeople(eb: ExpressionBuilder<DB, 'assets'>) {
+export function withFacesAndPeople(eb: ExpressionBuilder<DB, 'assets'>, withDeletedFace?: boolean) {
   return eb
     .selectFrom('asset_faces')
     .leftJoin('person', 'person.id', 'asset_faces.personId')
     .whereRef('asset_faces.assetId', '=', 'assets.id')
+    .$if(!withDeletedFace, (qb) => qb.where('asset_faces.deletedAt', 'is', null))
     .select((eb) =>
       eb
         .fn('jsonb_agg', [

server/src/migrations/1739466714036-AddDeletedAtColumnToAssetFacesTable.ts

@@ -0,0 +1,17 @@
+import { MigrationInterface, QueryRunner } from 'typeorm';
+
+export class AddDeletedAtColumnToAssetFacesTable1739466714036 implements MigrationInterface {
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`
+            ALTER TABLE asset_faces
+            ADD COLUMN "deletedAt" TIMESTAMP WITH TIME ZONE DEFAULT NULL
+        `);
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`
+            ALTER TABLE asset_faces
+            DROP COLUMN "deletedAt"
+        `);
+  }
+}

server/src/queries/asset.repository.sql

@@ -96,6 +96,7 @@ select
       left join "person" on "person"."id" = "asset_faces"."personId"
     where
       "asset_faces"."assetId" = "assets"."id"
+      and "asset_faces"."deletedAt" is null
   ) as "faces",
   (
     select

server/src/queries/person.repository.sql

@@ -42,6 +42,8 @@ select
 from
   "person"
   left join "asset_faces" on "asset_faces"."personId" = "person"."id"
+where
+  "asset_faces"."deletedAt" is null
 group by
   "person"."id"
 having
@@ -67,6 +69,7 @@ from
   "asset_faces"
 where
   "asset_faces"."assetId" = $1
+  and "asset_faces"."deletedAt" is null
 order by
   "asset_faces"."boundingBoxX1" asc
 
@@ -90,6 +93,7 @@ from
   "asset_faces"
 where
   "asset_faces"."id" = $1
+  and "asset_faces"."deletedAt" is null
 
 -- PersonRepository.getFaceByIdWithAssets
 select
@@ -124,6 +128,7 @@ from
   "asset_faces"
 where
   "asset_faces"."id" = $1
+  and "asset_faces"."deletedAt" is null
 
 -- PersonRepository.reassignFace
 update "asset_faces"
@@ -169,6 +174,8 @@ from
   and "asset_faces"."personId" = $1
   and "assets"."isArchived" = $2
   and "assets"."deletedAt" is null
+where
+  "asset_faces"."deletedAt" is null
 
 -- PersonRepository.getNumberOfPeople
 select
@@ -185,6 +192,7 @@ from
   and "assets"."isArchived" = $2
 where
   "person"."ownerId" = $3
+  and "asset_faces"."deletedAt" is null
 
 -- PersonRepository.refreshFaces
 with
@@ -235,6 +243,7 @@ from
 where
   "asset_faces"."assetId" in ($1)
   and "asset_faces"."personId" in ($2)
+  and "asset_faces"."deletedAt" is null
 
 -- PersonRepository.getRandomFace
 select
@@ -243,9 +252,22 @@ from
   "asset_faces"
 where
   "asset_faces"."personId" = $1
+  and "asset_faces"."deletedAt" is null
 
 -- PersonRepository.getLatestFaceDate
 select
   max("asset_job_status"."facesRecognizedAt")::text as "latestDate"
 from
   "asset_job_status"
+
+-- PersonRepository.deleteAssetFace
+delete from "asset_faces"
+where
+  "asset_faces"."id" = $1
+
+-- PersonRepository.softDeleteAssetFaces
+update "asset_faces"
+set
+  "deletedAt" = $1
+where
+  "asset_faces"."id" = $2

server/src/repositories/asset.repository.ts

@@ -132,7 +132,7 @@ export type AssetPathEntity = Pick<AssetEntity, 'id' | 'originalPath' | 'isOffli
 
 export interface GetByIdsRelations {
   exifInfo?: boolean;
-  faces?: { person?: boolean };
+  faces?: { person?: boolean; withDeleted?: boolean };
   files?: boolean;
   library?: boolean;
   owner?: boolean;
@@ -262,7 +262,11 @@ export class AssetRepository {
       .selectAll('assets')
       .where('assets.id', '=', anyUuid(ids))
       .$if(!!exifInfo, withExif)
-      .$if(!!faces, (qb) => qb.select(faces?.person ? withFacesAndPeople : withFaces))
+      .$if(!!faces, (qb) =>
+        qb.select((eb) =>
+          faces?.person ? withFacesAndPeople(eb, faces.withDeleted) : withFaces(eb, faces?.withDeleted),
+        ),
+      )
       .$if(!!files, (qb) => qb.select(withFiles))
       .$if(!!library, (qb) => qb.select(withLibrary))
       .$if(!!owner, (qb) => qb.select(withOwner))

server/src/repositories/person.repository.ts

@@ -130,6 +130,7 @@ export class PersonRepository {
       .$if(!!options.personId, (qb) => qb.where('asset_faces.personId', '=', options.personId!))
       .$if(!!options.sourceType, (qb) => qb.where('asset_faces.sourceType', '=', options.sourceType!))
       .$if(!!options.assetId, (qb) => qb.where('asset_faces.assetId', '=', options.assetId!))
+      .where('asset_faces.deletedAt', 'is', null)
       .stream() as AsyncIterableIterator<AssetFaceEntity>;
   }
 
@@ -161,6 +162,7 @@ export class PersonRepository {
           .on('assets.deletedAt', 'is', null),
       )
       .where('person.ownerId', '=', userId)
+      .where('asset_faces.deletedAt', 'is', null)
       .orderBy('person.isHidden', 'asc')
       .orderBy('person.isFavorite', 'desc')
       .having((eb) =>
@@ -212,6 +214,7 @@ export class PersonRepository {
       .selectFrom('person')
       .selectAll('person')
       .leftJoin('asset_faces', 'asset_faces.personId', 'person.id')
+      .where('asset_faces.deletedAt', 'is', null)
       .having((eb) => eb.fn.count('asset_faces.assetId'), '=', 0)
       .groupBy('person.id')
       .execute() as Promise<PersonEntity[]>;
@@ -224,6 +227,7 @@ export class PersonRepository {
       .selectAll('asset_faces')
       .select(withPerson)
       .where('asset_faces.assetId', '=', assetId)
+      .where('asset_faces.deletedAt', 'is', null)
       .orderBy('asset_faces.boundingBoxX1', 'asc')
       .execute() as Promise<AssetFaceEntity[]>;
   }
@@ -236,6 +240,7 @@ export class PersonRepository {
       .selectAll('asset_faces')
       .select(withPerson)
       .where('asset_faces.id', '=', id)
+      .where('asset_faces.deletedAt', 'is', null)
       .executeTakeFirstOrThrow() as Promise<AssetFaceEntity>;
   }
 
@@ -253,6 +258,7 @@ export class PersonRepository {
       .select(withAsset)
       .$if(!!relations?.faceSearch, (qb) => qb.select(withFaceSearch))
       .where('asset_faces.id', '=', id)
+      .where('asset_faces.deletedAt', 'is', null)
       .executeTakeFirst() as Promise<AssetFaceEntity | undefined>;
   }
 
@@ -317,6 +323,7 @@ export class PersonRepository {
           .on('assets.deletedAt', 'is', null),
       )
       .select((eb) => eb.fn.count(eb.fn('distinct', ['assets.id'])).as('count'))
+      .where('asset_faces.deletedAt', 'is', null)
       .executeTakeFirst();
 
     return {
@@ -330,6 +337,7 @@ export class PersonRepository {
       .selectFrom('person')
       .innerJoin('asset_faces', 'asset_faces.personId', 'person.id')
       .where('person.ownerId', '=', userId)
+      .where('asset_faces.deletedAt', 'is', null)
       .innerJoin('assets', (join) =>
         join
           .onRef('assets.id', '=', 'asset_faces.assetId')
@@ -434,6 +442,7 @@ export class PersonRepository {
       .select(withPerson)
       .where('asset_faces.assetId', 'in', assetIds)
       .where('asset_faces.personId', 'in', personIds)
+      .where('asset_faces.deletedAt', 'is', null)
       .execute() as Promise<AssetFaceEntity[]>;
   }
 
@@ -443,6 +452,7 @@ export class PersonRepository {
       .selectFrom('asset_faces')
       .selectAll('asset_faces')
       .where('asset_faces.personId', '=', personId)
+      .where('asset_faces.deletedAt', 'is', null)
       .executeTakeFirst() as Promise<AssetFaceEntity | undefined>;
   }
 
@@ -456,6 +466,20 @@ export class PersonRepository {
     return result?.latestDate;
   }
 
+  async createAssetFace(face: Insertable<AssetFaces>): Promise<void> {
+    await this.db.insertInto('asset_faces').values(face).execute();
+  }
+
+  @GenerateSql({ params: [DummyValue.UUID] })
+  async deleteAssetFace(id: string): Promise<void> {
+    await this.db.deleteFrom('asset_faces').where('asset_faces.id', '=', id).execute();
+  }
+
+  @GenerateSql({ params: [DummyValue.UUID] })
+  async softDeleteAssetFaces(id: string): Promise<void> {
+    await this.db.updateTable('asset_faces').set({ deletedAt: new Date() }).where('asset_faces.id', '=', id).execute();
+  }
+
   private async vacuum({ reindexVectors }: { reindexVectors: boolean }): Promise<void> {
     await sql`VACUUM ANALYZE asset_faces, face_search, person`.execute(this.db);
     await sql`REINDEX TABLE asset_faces`.execute(this.db);

server/src/services/person.service.ts

@@ -5,9 +5,13 @@ import { OnJob } from 'src/decorators';
 import { BulkIdErrorReason, BulkIdResponseDto } from 'src/dtos/asset-ids.response.dto';
 import { AuthDto } from 'src/dtos/auth.dto';
 import {
+  AssetFaceCreateDto,
+  AssetFaceDeleteDto,
   AssetFaceResponseDto,
   AssetFaceUpdateDto,
   FaceDto,
+  mapFaces,
+  mapPerson,
   MergePersonDto,
   PeopleResponseDto,
   PeopleUpdateDto,
@@ -16,8 +20,6 @@ import {
   PersonSearchDto,
   PersonStatisticsResponseDto,
   PersonUpdateDto,
-  mapFaces,
-  mapPerson,
 } from 'src/dtos/person.dto';
 import { AssetFaceEntity } from 'src/entities/asset-face.entity';
 import { AssetEntity } from 'src/entities/asset.entity';
@@ -295,7 +297,7 @@ export class PersonService extends BaseService {
       return JobStatus.SKIPPED;
     }
 
-    const relations = { exifInfo: true, faces: { person: false }, files: true };
+    const relations = { exifInfo: true, faces: { person: false, withDeleted: true }, files: true };
     const [asset] = await this.assetRepository.getByIds([id], relations);
     const { previewFile } = getAssetFiles(asset.files);
     if (!asset || !previewFile) {
@@ -717,4 +719,29 @@ export class PersonService extends BaseService {
       height: newHalfSize * 2,
     };
   }
+
+  // TODO return a asset face response
+  async createFace(auth: AuthDto, dto: AssetFaceCreateDto): Promise<void> {
+    await Promise.all([
+      this.requireAccess({ auth, permission: Permission.ASSET_READ, ids: [dto.assetId] }),
+      this.requireAccess({ auth, permission: Permission.PERSON_READ, ids: [dto.personId] }),
+    ]);
+
+    await this.personRepository.createAssetFace({
+      personId: dto.personId,
+      assetId: dto.assetId,
+      imageHeight: dto.imageHeight,
+      imageWidth: dto.imageWidth,
+      boundingBoxX1: dto.x,
+      boundingBoxX2: dto.x + dto.width,
+      boundingBoxY1: dto.y,
+      boundingBoxY2: dto.y + dto.height,
+    });
+  }
+
+  async deleteFace(auth: AuthDto, id: string, dto: AssetFaceDeleteDto): Promise<void> {
+    await this.requireAccess({ auth, permission: Permission.FACE_DELETE, ids: [id] });
+
+    return dto.force ? this.personRepository.deleteAssetFace(id) : this.personRepository.softDeleteAssetFaces(id);
+  }
 }

server/src/utils/access.ts

@@ -217,6 +217,10 @@ const checkOtherAccess = async (access: AccessRepository, request: OtherAccessRe
       return await access.authDevice.checkOwnerAccess(auth.user.id, ids);
     }
 
+    case Permission.FACE_DELETE: {
+      return access.person.checkFaceOwnerAccess(auth.user.id, ids);
+    }
+
     case Permission.TAG_ASSET:
     case Permission.TAG_READ:
     case Permission.TAG_UPDATE: