diff --git a/docs/docs/FAQ.mdx b/docs/docs/FAQ.mdx index 869e84e377..0453ca71f2 100644 --- a/docs/docs/FAQ.mdx +++ b/docs/docs/FAQ.mdx @@ -151,6 +151,10 @@ See [Backup and Restore](/administration/backup-and-restore.md). Yes, it creates new faces and persons from the imported asset metadata. For details see the [feature request #4348](https://github.com/immich-app/immich/discussions/4348) and [PR #6455](https://github.com/immich-app/immich/pull/6455). +### Does Immich write face tags back to metadata? + +Optionally. When "Enable face export" is turned on in the metadata settings, Immich writes the people it knows about to the XMP sidecar of an asset (as `mwg-rs:RegionInfo`) whenever its faces change, for example after naming, merging or reassigning a person. Turning the setting on writes out every asset that already has a named person. Only faces that belong to a named person are written, and the original file is never modified — the regions are written to the sidecar file next to it. + ### Does Immich support the filtering of NSFW images? No, it currently does not. There is an [open feature request on Github](https://github.com/immich-app/immich/discussions/2451). diff --git a/docs/docs/features/xmp-sidecars.md b/docs/docs/features/xmp-sidecars.md index 3536777d8a..51cb443a4e 100644 --- a/docs/docs/features/xmp-sidecars.md +++ b/docs/docs/features/xmp-sidecars.md @@ -11,7 +11,7 @@ Tools like Lightroom, Darktable, digiKam and other applications can also be conf Immich does not support _all_ metadata fields. Below is a table showing what fields Immich can _read_ and _write_. It's important to note that writes do not replace the entire file contents, but are merged together with any existing fields. :::info -Immich automatically queues a Sidecar Write job after editing the description, rating, or updating tags. +Immich automatically queues a Sidecar Write job after editing the description, rating, or updating tags. With _Enable face export_ turned on, changing the faces of an asset — naming, merging, reassigning or deleting a person — queues one as well. ::: | Metadata | Immich writes to XMP | Immich reads from XMP | @@ -21,6 +21,11 @@ Immich automatically queues a Sidecar Write job after editing the description, r | **DateTime** | `exif:DateTimeOriginal`, `photoshop:DateCreated` | In prioritized order:
`exif:SubSecDateTimeOriginal`
`exif:DateTimeOriginal`
`xmp:SubSecCreateDate`
`xmp:CreateDate`
`xmp:CreationDate`
`xmp:MediaCreateDate`
`xmp:SubSecMediaCreateDate`
`xmp:DateTimeCreated` | | **Location** | `exif:GPSLatitude`, `exif:GPSLongitude` | `exif:GPSLatitude`, `exif:GPSLongitude` | | **Tags** | `digiKam:TagsList` | In prioritized order:
`digiKam:TagsList`
`lr:HierarchicalSubject`
`IPTC:Keywords` | +| **Faces** | `mwg-rs:RegionInfo` | `mwg-rs:RegionInfo` | + +:::info +Faces are only read when _Enable face import_ is turned on, and only written when _Enable face export_ is turned on. Both settings are found under Administration > Settings > Metadata Settings and are off by default. Only faces that belong to a named person are written, and the regions replace whichever regions the sidecar held before. Turning face export on writes out every asset that already has a named person, so you do not have to wait for their faces to change. +::: :::note All other fields (e.g. `Creator`, `Source`, IPTC, Lightroom edits) remain in the `.xmp` file and are **not searchable** in Immich. diff --git a/docs/docs/install/config-file.md b/docs/docs/install/config-file.md index 5c34acdd9d..580e528834 100644 --- a/docs/docs/install/config-file.md +++ b/docs/docs/install/config-file.md @@ -159,7 +159,8 @@ The default configuration looks like this: }, "metadata": { "faces": { - "import": false + "import": false, + "export": false } }, "newVersionCheck": { diff --git a/i18n/en.json b/i18n/en.json index 536b7677ac..720fa90256 100644 --- a/i18n/en.json +++ b/i18n/en.json @@ -250,6 +250,8 @@ "memory_generate_job": "Memory generation", "metadata_extraction_job": "Extract metadata", "metadata_extraction_job_description": "Extract metadata information from each asset, such as GPS, faces and resolution", + "metadata_faces_export_setting": "Enable face export", + "metadata_faces_export_setting_description": "Write named people to the sidecar files of your assets when their faces change", "metadata_faces_import_setting": "Enable face import", "metadata_faces_import_setting_description": "Import faces from image EXIF data and sidecar files", "metadata_settings": "Metadata Settings", diff --git a/open-api/immich-openapi-specs.json b/open-api/immich-openapi-specs.json index bc3bf82094..53250662e8 100644 --- a/open-api/immich-openapi-specs.json +++ b/open-api/immich-openapi-specs.json @@ -25849,12 +25849,17 @@ }, "SystemConfigFacesDto": { "properties": { + "export": { + "description": "Export", + "type": "boolean" + }, "import": { "description": "Import", "type": "boolean" } }, "required": [ + "export", "import" ], "type": "object" diff --git a/packages/sdk/src/fetch-client.ts b/packages/sdk/src/fetch-client.ts index 3ac958ce2d..d99c667642 100644 --- a/packages/sdk/src/fetch-client.ts +++ b/packages/sdk/src/fetch-client.ts @@ -2495,6 +2495,8 @@ export type SystemConfigMapDto = { lightStyle: string; }; export type SystemConfigFacesDto = { + /** Export */ + "export": boolean; /** Import */ "import": boolean; }; diff --git a/server/src/config.ts b/server/src/config.ts index 55304080a3..631c66d3e1 100644 --- a/server/src/config.ts +++ b/server/src/config.ts @@ -116,6 +116,7 @@ export type SystemConfig = { metadata: { faces: { import: boolean; + export: boolean; }; }; oauth: { @@ -333,6 +334,7 @@ export const defaults = Object.freeze({ metadata: { faces: { import: false, + export: false, }, }, oauth: { diff --git a/server/src/dtos/system-config.dto.ts b/server/src/dtos/system-config.dto.ts index a50b7abe87..99664f0a72 100644 --- a/server/src/dtos/system-config.dto.ts +++ b/server/src/dtos/system-config.dto.ts @@ -288,7 +288,7 @@ const SystemConfigReverseGeocodingSchema = z .meta({ id: 'SystemConfigReverseGeocodingDto' }); const SystemConfigFacesSchema = z - .object({ import: configBool.describe('Import') }) + .object({ import: configBool.describe('Import'), export: configBool.describe('Export') }) .meta({ id: 'SystemConfigFacesDto' }); const SystemConfigMetadataSchema = z.object({ faces: SystemConfigFacesSchema }).meta({ id: 'SystemConfigMetadataDto' }); diff --git a/server/src/queries/asset.job.repository.sql b/server/src/queries/asset.job.repository.sql index aa04603913..f260146c42 100644 --- a/server/src/queries/asset.job.repository.sql +++ b/server/src/queries/asset.job.repository.sql @@ -47,6 +47,62 @@ where limit $3 +-- AssetJobRepository.getForSidecarWriteJob (with faces) +select + "id", + "originalPath", + ( + select + coalesce(json_agg(agg), '[]') + from + ( + select + "asset_file"."id", + "asset_file"."path", + "asset_file"."type", + "asset_file"."isEdited" + from + "asset_file" + where + "asset_file"."assetId" = "asset"."id" + and "asset_file"."type" = $1 + ) as agg + ) as "files", + ( + select + coalesce(json_agg(agg), '[]') + from + ( + select + "asset_face"."boundingBoxX1", + "asset_face"."boundingBoxY1", + "asset_face"."boundingBoxX2", + "asset_face"."boundingBoxY2", + "asset_face"."imageWidth", + "asset_face"."imageHeight", + "person"."name" + from + "asset_face" + inner join "person" on "person"."id" = "asset_face"."personId" + where + "asset_face"."assetId" = "asset"."id" + and "asset_face"."deletedAt" is null + and "asset_face"."isVisible" = $2 + and "person"."name" != $3 + order by + "asset_face"."boundingBoxX1", + "asset_face"."boundingBoxY1" + ) as agg + ) as "faces", + to_json("asset_exif") as "exifInfo" +from + "asset" + inner join "asset_exif" on "asset"."id" = "asset_exif"."assetId" +where + "asset"."id" = $4::uuid +limit + $5 + -- AssetJobRepository.getForSidecarCheckJob select "id", diff --git a/server/src/queries/person.repository.sql b/server/src/queries/person.repository.sql index a2f3f64442..55363e6d26 100644 --- a/server/src/queries/person.repository.sql +++ b/server/src/queries/person.repository.sql @@ -12,6 +12,26 @@ delete from "person" where "person"."id" in ($1) +-- PersonRepository.streamAssetIdsForPeople +select distinct + "asset_face"."assetId" +from + "asset_face" +where + "asset_face"."personId" in ($1) + and "asset_face"."deletedAt" is null + +-- PersonRepository.streamAssetIdsWithNamedFaces +select distinct + "asset_face"."assetId" +from + "asset_face" + inner join "person" on "person"."id" = "asset_face"."personId" +where + "asset_face"."deletedAt" is null + and "asset_face"."isVisible" = $1 + and "person"."name" != $2 + -- PersonRepository.getFileSamples select "id", @@ -131,6 +151,7 @@ where -- PersonRepository.getFaceForFacialRecognitionJob select "asset_face"."id", + "asset_face"."assetId", "asset_face"."personId", "asset_face"."sourceType", ( diff --git a/server/src/repositories/asset-job.repository.ts b/server/src/repositories/asset-job.repository.ts index bab0c44a41..cf81e9336e 100644 --- a/server/src/repositories/asset-job.repository.ts +++ b/server/src/repositories/asset-job.repository.ts @@ -17,6 +17,7 @@ import { withFaces, withFilePath, withFiles, + withNamedFaces, withVideoFormat, withVideoStream, } from 'src/utils/database'; @@ -37,13 +38,14 @@ export class AssetJobRepository { .executeTakeFirst(); } - @GenerateSql({ params: [DummyValue.UUID] }) - getForSidecarWriteJob(id: string) { + @GenerateSql({ params: [DummyValue.UUID] }, { name: 'with faces', params: [DummyValue.UUID, true] }) + getForSidecarWriteJob(id: string, withFaces = false) { return this.db .selectFrom('asset') .where('asset.id', '=', asUuid(id)) .select(['id', 'originalPath']) .select((eb) => withFiles(eb, AssetFileType.Sidecar)) + .$if(withFaces, (qb) => qb.select((eb) => withNamedFaces(eb))) .$call(withExifInner) .limit(1) .executeTakeFirst(); diff --git a/server/src/repositories/event.repository.ts b/server/src/repositories/event.repository.ts index 416f823952..ed35047f7f 100644 --- a/server/src/repositories/event.repository.ts +++ b/server/src/repositories/event.repository.ts @@ -55,6 +55,12 @@ type EventMap = { AssetTrashAll: [{ assetIds: string[]; userId: string }]; AssetDeleteAll: [{ assetIds: string[]; userId: string }]; AssetRestoreAll: [{ assetIds: string[]; userId: string }]; + /** the faces of these assets were added, removed or assigned to a different person */ + AssetFacesUpdate: [{ assetIds: string[] }]; + + // person events + /** these people were renamed, merged or deleted, which affects every asset they appear in */ + PersonFacesUpdate: [{ personIds: string[] }]; /** a worker receives a job and emits this event to run it */ JobRun: [QueueName, JobItem]; diff --git a/server/src/repositories/metadata.repository.ts b/server/src/repositories/metadata.repository.ts index 1d504f6c71..ae392dd43b 100644 --- a/server/src/repositories/metadata.repository.ts +++ b/server/src/repositories/metadata.repository.ts @@ -1,5 +1,5 @@ import { Injectable } from '@nestjs/common'; -import { BinaryField, DefaultReadTaskOptions, ExifTool, ReadTaskOptions, Tags } from 'exiftool-vendored'; +import { BinaryField, DefaultReadTaskOptions, ExifTool, ReadTaskOptions, Tags, WriteTags } from 'exiftool-vendored'; import geotz from 'geo-tz'; import { LoggingRepository } from 'src/repositories/logging.repository'; import { mimeTypes } from 'src/utils/mime-types'; @@ -123,7 +123,7 @@ export class MetadataRepository { return this.exiftool.extractBinaryTagToBuffer(tagName, path); } - async writeTags(path: string, tags: Partial): Promise { + async writeTags(path: string, tags: WriteTags): Promise { // If exiftool assigns a field with ^= instead of =, empty values will be written too. // Since exiftool-vendored doesn't support an option for this, we append the ^ to the name of the tag instead. // https://exiftool.org/exiftool_pod.html#:~:text=is%20used%20to%20write%20an%20empty%20string diff --git a/server/src/repositories/person.repository.ts b/server/src/repositories/person.repository.ts index 0db03a18c7..9795ad6186 100644 --- a/server/src/repositories/person.repository.ts +++ b/server/src/repositories/person.repository.ts @@ -112,6 +112,30 @@ export class PersonRepository { await this.db.deleteFrom('asset_face').where('asset_face.sourceType', '=', sourceType).execute(); } + @GenerateSql({ params: [[DummyValue.UUID]], stream: true }) + streamAssetIdsForPeople(personIds: string[]) { + return this.db + .selectFrom('asset_face') + .select('asset_face.assetId') + .distinct() + .where('asset_face.personId', 'in', personIds) + .where('asset_face.deletedAt', 'is', null) + .stream(); + } + + @GenerateSql({ params: [], stream: true }) + streamAssetIdsWithNamedFaces() { + return this.db + .selectFrom('asset_face') + .innerJoin('person', 'person.id', 'asset_face.personId') + .select('asset_face.assetId') + .distinct() + .where('asset_face.deletedAt', 'is', null) + .where('asset_face.isVisible', '=', true) + .where('person.name', '!=', '') + .stream(); + } + getAllFaces(options: GetAllFacesOptions = {}) { return this.db .selectFrom('asset_face') @@ -257,7 +281,7 @@ export class PersonRepository { getFaceForFacialRecognitionJob(id: string) { return this.db .selectFrom('asset_face') - .select(['asset_face.id', 'asset_face.personId', 'asset_face.sourceType']) + .select(['asset_face.id', 'asset_face.assetId', 'asset_face.personId', 'asset_face.sourceType']) .select((eb) => jsonObjectFrom( eb diff --git a/server/src/services/metadata.service.spec.ts b/server/src/services/metadata.service.spec.ts index 57c029961e..3b5a0598d0 100644 --- a/server/src/services/metadata.service.spec.ts +++ b/server/src/services/metadata.service.spec.ts @@ -62,6 +62,97 @@ const makeFaceTags = ( }, }); +/** + * The geometry of {@link makeFaceTags} as Immich stores it after importing it for a given exif orientation. Since + * exporting is the inverse of importing, writing these faces has to produce the geometry of `makeFaceTags` again. + */ +const orientationTests = [ + { + description: 'undefined', + orientation: undefined, + expected: { imgW: 1000, imgH: 100, x1: 0, x2: 200, y1: 20, y2: 60 }, + }, + { + description: 'Horizontal = 1', + orientation: ExifOrientation.Horizontal, + expected: { imgW: 1000, imgH: 100, x1: 0, x2: 200, y1: 20, y2: 60 }, + }, + { + description: 'MirrorHorizontal = 2', + orientation: ExifOrientation.MirrorHorizontal, + expected: { imgW: 1000, imgH: 100, x1: 800, x2: 1000, y1: 20, y2: 60 }, + }, + { + description: 'Rotate180 = 3', + orientation: ExifOrientation.Rotate180, + expected: { imgW: 1000, imgH: 100, x1: 800, x2: 1000, y1: 40, y2: 80 }, + }, + { + description: 'MirrorVertical = 4', + orientation: ExifOrientation.MirrorVertical, + expected: { imgW: 1000, imgH: 100, x1: 0, x2: 200, y1: 40, y2: 80 }, + }, + { + description: 'MirrorHorizontalRotate270CW = 5', + orientation: ExifOrientation.MirrorHorizontalRotate270CW, + expected: { imgW: 100, imgH: 1000, x1: 20, x2: 60, y1: 0, y2: 200 }, + }, + { + description: 'Rotate90CW = 6', + orientation: ExifOrientation.Rotate90CW, + expected: { imgW: 100, imgH: 1000, x1: 40, x2: 80, y1: 0, y2: 200 }, + }, + { + description: 'MirrorHorizontalRotate90CW = 7', + orientation: ExifOrientation.MirrorHorizontalRotate90CW, + expected: { imgW: 100, imgH: 1000, x1: 40, x2: 80, y1: 800, y2: 1000 }, + }, + { + description: 'Rotate270CW = 8', + orientation: ExifOrientation.Rotate270CW, + expected: { imgW: 100, imgH: 1000, x1: 20, x2: 60, y1: 800, y2: 1000 }, + }, +]; + +const withExport = (value: boolean) => ({ ...defaults, metadata: { faces: { import: false, export: value } } }); + +const makeFaceAsset = ( + { + orientation, + imgW, + imgH, + x1, + x2, + y1, + y2, + }: { + orientation?: ExifOrientation | string; + imgW: number; + imgH: number; + x1: number; + x2: number; + y1: number; + y2: number; + }, + name = 'Alice', +) => + AssetFactory.from() + .file({ type: AssetFileType.Sidecar }) + .exif({ exifImageWidth: 1000, exifImageHeight: 100, orientation: orientation ? String(orientation) : null }) + .face( + { + imageWidth: imgW, + imageHeight: imgH, + boundingBoxX1: x1, + boundingBoxX2: x2, + boundingBoxY1: y1, + boundingBoxY2: y2, + sourceType: SourceType.MachineLearning, + }, + (face) => face.person({ name }), + ) + .build(); + const emptyPackets = { totalDuration: 0, packetCount: 0, @@ -123,15 +214,42 @@ describe(MetadataService.name, () => { }); describe('onConfigUpdate', () => { - it('should update metadata processing concurrency', () => { + it('should update metadata processing concurrency', async () => { const newConfig = structuredClone(defaults); newConfig.job.metadataExtraction.concurrency = 10; - sut.onConfigUpdate({ oldConfig: defaults, newConfig }); + await sut.onConfigUpdate({ oldConfig: defaults, newConfig }); expect(mocks.metadata.setMaxConcurrency).toHaveBeenCalledWith(newConfig.job.metadataExtraction.concurrency); expect(mocks.metadata.setMaxConcurrency).toHaveBeenCalledTimes(1); }); + + it('should write out the people already known when face export is turned on', async () => { + mocks.person.streamAssetIdsWithNamedFaces.mockReturnValue( + makeStream([{ assetId: 'asset-1' }, { assetId: 'asset-2' }]), + ); + + await sut.onConfigUpdate({ oldConfig: withExport(false), newConfig: withExport(true) }); + + expect(mocks.job.queueAll).toHaveBeenCalledWith([ + { name: JobName.SidecarWrite, data: { id: 'asset-1', faces: true } }, + { name: JobName.SidecarWrite, data: { id: 'asset-2', faces: true } }, + ]); + }); + + it('should not write anything when face export was already on', async () => { + await sut.onConfigUpdate({ oldConfig: withExport(true), newConfig: withExport(true) }); + + expect(mocks.person.streamAssetIdsWithNamedFaces).not.toHaveBeenCalled(); + expect(mocks.job.queueAll).not.toHaveBeenCalled(); + }); + + it('should not write anything when face export is turned off', async () => { + await sut.onConfigUpdate({ oldConfig: withExport(true), newConfig: withExport(false) }); + + expect(mocks.person.streamAssetIdsWithNamedFaces).not.toHaveBeenCalled(); + expect(mocks.job.queueAll).not.toHaveBeenCalled(); + }); }); describe('handleQueueMetadataExtraction', () => { @@ -1481,54 +1599,6 @@ describe(MetadataService.name, () => { }); describe('handleFaceTagOrientation', () => { - const orientationTests = [ - { - description: 'undefined', - orientation: undefined, - expected: { imgW: 1000, imgH: 100, x1: 0, x2: 200, y1: 20, y2: 60 }, - }, - { - description: 'Horizontal = 1', - orientation: ExifOrientation.Horizontal, - expected: { imgW: 1000, imgH: 100, x1: 0, x2: 200, y1: 20, y2: 60 }, - }, - { - description: 'MirrorHorizontal = 2', - orientation: ExifOrientation.MirrorHorizontal, - expected: { imgW: 1000, imgH: 100, x1: 800, x2: 1000, y1: 20, y2: 60 }, - }, - { - description: 'Rotate180 = 3', - orientation: ExifOrientation.Rotate180, - expected: { imgW: 1000, imgH: 100, x1: 800, x2: 1000, y1: 40, y2: 80 }, - }, - { - description: 'MirrorVertical = 4', - orientation: ExifOrientation.MirrorVertical, - expected: { imgW: 1000, imgH: 100, x1: 0, x2: 200, y1: 40, y2: 80 }, - }, - { - description: 'MirrorHorizontalRotate270CW = 5', - orientation: ExifOrientation.MirrorHorizontalRotate270CW, - expected: { imgW: 100, imgH: 1000, x1: 20, x2: 60, y1: 0, y2: 200 }, - }, - { - description: 'Rotate90CW = 6', - orientation: ExifOrientation.Rotate90CW, - expected: { imgW: 100, imgH: 1000, x1: 40, x2: 80, y1: 0, y2: 200 }, - }, - { - description: 'MirrorHorizontalRotate90CW = 7', - orientation: ExifOrientation.MirrorHorizontalRotate90CW, - expected: { imgW: 100, imgH: 1000, x1: 40, x2: 80, y1: 800, y2: 1000 }, - }, - { - description: 'Rotate270CW = 8', - orientation: ExifOrientation.Rotate270CW, - expected: { imgW: 100, imgH: 1000, x1: 20, x2: 60, y1: 800, y2: 1000 }, - }, - ]; - it.each(orientationTests)( 'should transform RegionInfo geometry according to exif orientation $description', async ({ orientation, expected }) => { @@ -2040,6 +2110,261 @@ describe(MetadataService.name, () => { expect(mocks.metadata.writeTags).toHaveBeenCalledWith(asset.files[0].path, { Rating: 0 }); expect(mocks.asset.unlockProperties).toHaveBeenCalledWith(asset.id, ['rating']); }); + + describe('faces', () => { + it('should not write faces when face export is disabled', async () => { + const asset = makeFaceAsset({ + orientation: ExifOrientation.Horizontal, + imgW: 1000, + imgH: 100, + x1: 0, + x2: 200, + y1: 20, + y2: 60, + }); + + mocks.assetJob.getLockedPropertiesForMetadataExtraction.mockResolvedValue([]); + mocks.assetJob.getForSidecarWriteJob.mockResolvedValue(getForSidecarWrite(asset)); + + await expect(sut.handleSidecarWrite({ id: asset.id, faces: true })).resolves.toBe(JobStatus.Skipped); + expect(mocks.metadata.writeTags).not.toHaveBeenCalled(); + // the faces are not even fetched, so an install that does not export them pays nothing + expect(mocks.assetJob.getForSidecarWriteJob).toHaveBeenCalledWith(asset.id, false); + }); + + it('should write named faces as mwg regions', async () => { + const asset = makeFaceAsset({ + orientation: ExifOrientation.Horizontal, + imgW: 1000, + imgH: 100, + x1: 0, + x2: 200, + y1: 20, + y2: 60, + }); + + mocks.systemMetadata.get.mockResolvedValue({ metadata: { faces: { export: true } } }); + mocks.assetJob.getLockedPropertiesForMetadataExtraction.mockResolvedValue([]); + mocks.assetJob.getForSidecarWriteJob.mockResolvedValue(getForSidecarWrite(asset)); + + await expect(sut.handleSidecarWrite({ id: asset.id, faces: true })).resolves.toBe(JobStatus.Success); + expect(mocks.assetJob.getForSidecarWriteJob).toHaveBeenCalledWith(asset.id, true); + expect(mocks.metadata.writeTags).toHaveBeenCalledWith(asset.files[0].path, { + RegionInfo: { + AppliedToDimensions: { W: 1000, H: 100, Unit: 'pixel' }, + RegionList: [ + { + Type: 'Face', + Name: 'Alice', + Area: { X: 0.1, Y: 0.4, W: 0.2, H: 0.4, Unit: 'normalized' }, + }, + ], + }, + }); + }); + + it('should write faces along with other metadata', async () => { + const description = 'this is a description'; + const asset = makeFaceAsset({ + orientation: ExifOrientation.Horizontal, + imgW: 1000, + imgH: 100, + x1: 0, + x2: 200, + y1: 20, + y2: 60, + }); + asset.exifInfo.description = description; + + mocks.systemMetadata.get.mockResolvedValue({ metadata: { faces: { export: true } } }); + mocks.assetJob.getLockedPropertiesForMetadataExtraction.mockResolvedValue(['description']); + mocks.assetJob.getForSidecarWriteJob.mockResolvedValue(getForSidecarWrite(asset)); + + await expect(sut.handleSidecarWrite({ id: asset.id })).resolves.toBe(JobStatus.Success); + expect(mocks.metadata.writeTags).toHaveBeenCalledWith( + asset.files[0].path, + expect.objectContaining({ Description: description, RegionInfo: expect.any(Object) }), + ); + }); + + it('should skip faces without a named person', async () => { + const asset = AssetFactory.from() + .file({ type: AssetFileType.Sidecar }) + .exif({ exifImageWidth: 1000, exifImageHeight: 100 }) + .face({ imageWidth: 1000, imageHeight: 100 }) + .build(); + + mocks.systemMetadata.get.mockResolvedValue({ metadata: { faces: { export: true } } }); + mocks.assetJob.getLockedPropertiesForMetadataExtraction.mockResolvedValue([]); + mocks.assetJob.getForSidecarWriteJob.mockResolvedValue(getForSidecarWrite(asset)); + + await expect(sut.handleSidecarWrite({ id: asset.id })).resolves.toBe(JobStatus.Skipped); + expect(mocks.metadata.writeTags).not.toHaveBeenCalled(); + }); + + it('should remove regions when the last named face was removed', async () => { + const asset = AssetFactory.from().file({ type: AssetFileType.Sidecar }).exif().build(); + + mocks.systemMetadata.get.mockResolvedValue({ metadata: { faces: { export: true } } }); + mocks.assetJob.getLockedPropertiesForMetadataExtraction.mockResolvedValue([]); + mocks.assetJob.getForSidecarWriteJob.mockResolvedValue(getForSidecarWrite(asset)); + mockReadTags(makeFaceTags({ Name: 'Alice' })); + + await expect(sut.handleSidecarWrite({ id: asset.id, faces: true })).resolves.toBe(JobStatus.Success); + expect(mocks.metadata.readTags).toHaveBeenCalledWith(asset.files[0].path); + expect(mocks.metadata.writeTags).toHaveBeenCalledWith(asset.files[0].path, { RegionInfo: null }); + }); + + it('should not touch sidecars that have no regions', async () => { + const asset = AssetFactory.from().file({ type: AssetFileType.Sidecar }).exif().build(); + + mocks.systemMetadata.get.mockResolvedValue({ metadata: { faces: { export: true } } }); + mocks.assetJob.getLockedPropertiesForMetadataExtraction.mockResolvedValue([]); + mocks.assetJob.getForSidecarWriteJob.mockResolvedValue(getForSidecarWrite(asset)); + mockReadTags({}); + + await expect(sut.handleSidecarWrite({ id: asset.id, faces: true })).resolves.toBe(JobStatus.Skipped); + expect(mocks.metadata.writeTags).not.toHaveBeenCalled(); + }); + + it('should not create a sidecar just to remove regions', async () => { + const asset = AssetFactory.from().exif().build(); + + mocks.systemMetadata.get.mockResolvedValue({ metadata: { faces: { export: true } } }); + mocks.assetJob.getLockedPropertiesForMetadataExtraction.mockResolvedValue([]); + mocks.assetJob.getForSidecarWriteJob.mockResolvedValue(getForSidecarWrite(asset)); + + await expect(sut.handleSidecarWrite({ id: asset.id, faces: true })).resolves.toBe(JobStatus.Skipped); + expect(mocks.metadata.readTags).not.toHaveBeenCalled(); + expect(mocks.metadata.writeTags).not.toHaveBeenCalled(); + }); + + it('should leave regions alone when the job was not triggered by a face change', async () => { + const asset = AssetFactory.from().file({ type: AssetFileType.Sidecar }).exif().build(); + + mocks.systemMetadata.get.mockResolvedValue({ metadata: { faces: { export: true } } }); + mocks.assetJob.getLockedPropertiesForMetadataExtraction.mockResolvedValue([]); + mocks.assetJob.getForSidecarWriteJob.mockResolvedValue(getForSidecarWrite(asset)); + + await expect(sut.handleSidecarWrite({ id: asset.id })).resolves.toBe(JobStatus.Skipped); + expect(mocks.metadata.writeTags).not.toHaveBeenCalled(); + }); + + it('should not write faces for assets without known dimensions', async () => { + const asset = AssetFactory.from() + .file({ type: AssetFileType.Sidecar }) + .exif({ exifImageWidth: null, exifImageHeight: null }) + .face({ imageWidth: 1000, imageHeight: 100 }, (face) => face.person({ name: 'Alice' })) + .build(); + + mocks.systemMetadata.get.mockResolvedValue({ metadata: { faces: { export: true } } }); + mocks.assetJob.getLockedPropertiesForMetadataExtraction.mockResolvedValue([]); + mocks.assetJob.getForSidecarWriteJob.mockResolvedValue(getForSidecarWrite(asset)); + + await expect(sut.handleSidecarWrite({ id: asset.id, faces: true })).resolves.toBe(JobStatus.Skipped); + expect(mocks.metadata.writeTags).not.toHaveBeenCalled(); + }); + + it('should not transform geometry for a rotation that is not an exif orientation', async () => { + // the image of such an asset is never rotated, so its faces are already in the coordinate space of the file + const asset = makeFaceAsset({ orientation: '90', imgW: 1000, imgH: 100, x1: 0, x2: 200, y1: 20, y2: 60 }); + + mocks.systemMetadata.get.mockResolvedValue({ metadata: { faces: { export: true } } }); + mocks.assetJob.getLockedPropertiesForMetadataExtraction.mockResolvedValue([]); + mocks.assetJob.getForSidecarWriteJob.mockResolvedValue(getForSidecarWrite(asset)); + + await expect(sut.handleSidecarWrite({ id: asset.id, faces: true })).resolves.toBe(JobStatus.Success); + expect(mocks.metadata.writeTags).toHaveBeenCalledWith(asset.files[0].path, { + RegionInfo: { + AppliedToDimensions: { W: 1000, H: 100, Unit: 'pixel' }, + RegionList: [ + { + Type: 'Face', + Name: 'Alice', + Area: { X: 0.1, Y: 0.4, W: 0.2, H: 0.4, Unit: 'normalized' }, + }, + ], + }, + }); + }); + + it.each(orientationTests)( + 'should write RegionInfo geometry in the coordinate space of the file for exif orientation $description', + async ({ orientation, expected }) => { + const asset = makeFaceAsset({ orientation, ...expected }); + + mocks.systemMetadata.get.mockResolvedValue({ metadata: { faces: { export: true } } }); + mocks.assetJob.getLockedPropertiesForMetadataExtraction.mockResolvedValue([]); + mocks.assetJob.getForSidecarWriteJob.mockResolvedValue(getForSidecarWrite(asset)); + + await expect(sut.handleSidecarWrite({ id: asset.id, faces: true })).resolves.toBe(JobStatus.Success); + + // the geometry of makeFaceTags, i.e. exporting is the exact inverse of importing + expect(mocks.metadata.writeTags).toHaveBeenCalledWith(asset.files[0].path, { + RegionInfo: { + AppliedToDimensions: { W: 1000, H: 100, Unit: 'pixel' }, + RegionList: [ + { + Type: 'Face', + Name: 'Alice', + Area: { X: 0.1, Y: 0.4, W: 0.2, H: 0.4, Unit: 'normalized' }, + }, + ], + }, + }); + }, + ); + }); + }); + + describe('handleAssetFacesUpdate', () => { + it('should do nothing when face export is disabled', async () => { + await sut.handleAssetFacesUpdate({ assetIds: ['asset-1'] }); + expect(mocks.job.queueAll).not.toHaveBeenCalled(); + }); + + it('should queue a sidecar write for every asset', async () => { + mocks.systemMetadata.get.mockResolvedValue({ metadata: { faces: { export: true } } }); + + await sut.handleAssetFacesUpdate({ assetIds: ['asset-1', 'asset-2'] }); + + expect(mocks.job.queueAll).toHaveBeenCalledWith([ + { name: JobName.SidecarWrite, data: { id: 'asset-1', faces: true } }, + { name: JobName.SidecarWrite, data: { id: 'asset-2', faces: true } }, + ]); + }); + }); + + describe('handlePersonFacesUpdate', () => { + it('should do nothing when face export is disabled', async () => { + await sut.handlePersonFacesUpdate({ personIds: ['person-1'] }); + expect(mocks.person.streamAssetIdsForPeople).not.toHaveBeenCalled(); + expect(mocks.job.queueAll).not.toHaveBeenCalled(); + }); + + it('should queue a sidecar write for every asset of the people', async () => { + mocks.systemMetadata.get.mockResolvedValue({ metadata: { faces: { export: true } } }); + mocks.person.streamAssetIdsForPeople.mockReturnValue( + makeStream([{ assetId: 'asset-1' }, { assetId: 'asset-2' }]), + ); + + await sut.handlePersonFacesUpdate({ personIds: ['person-1'] }); + + expect(mocks.person.streamAssetIdsForPeople).toHaveBeenCalledWith(['person-1']); + expect(mocks.job.queueAll).toHaveBeenCalledWith([ + { name: JobName.SidecarWrite, data: { id: 'asset-1', faces: true } }, + { name: JobName.SidecarWrite, data: { id: 'asset-2', faces: true } }, + ]); + }); + + it('should not queue anything when the people have no assets', async () => { + mocks.systemMetadata.get.mockResolvedValue({ metadata: { faces: { export: true } } }); + mocks.person.streamAssetIdsForPeople.mockReturnValue(makeStream([])); + + await sut.handlePersonFacesUpdate({ personIds: ['person-1'] }); + + expect(mocks.job.queueAll).not.toHaveBeenCalled(); + }); }); describe('firstDateTime', () => { diff --git a/server/src/services/metadata.service.ts b/server/src/services/metadata.service.ts index 37dd92e27d..555c102399 100644 --- a/server/src/services/metadata.service.ts +++ b/server/src/services/metadata.service.ts @@ -1,5 +1,5 @@ import { Injectable } from '@nestjs/common'; -import { ContainerDirectoryItem, ExifDateTime, Tags } from 'exiftool-vendored'; +import { ContainerDirectoryItem, ExifDateTime, Struct, WriteTags } from 'exiftool-vendored'; import { Insertable } from 'kysely'; import _ from 'lodash'; import { DateTime, Duration } from 'luxon'; @@ -8,7 +8,7 @@ import { constants } from 'node:fs/promises'; import { join, parse } from 'node:path'; import { JOBS_ASSET_PAGINATION_SIZE } from 'src/constants'; import { StorageCore } from 'src/cores/storage.core'; -import { Asset, AssetFile } from 'src/database'; +import { Asset, AssetFile, Exif } from 'src/database'; import { OnEvent, OnJob } from 'src/decorators'; import { AssetFileType, @@ -35,7 +35,7 @@ import { getAssetFiles } from 'src/utils/asset.util'; import { isAssetChecksumConstraint } from 'src/utils/database'; import { mergeTimeZone } from 'src/utils/date'; import { mimeTypes } from 'src/utils/mime-types'; -import { isFaceImportEnabled } from 'src/utils/misc'; +import { isFaceExportEnabled, isFaceImportEnabled } from 'src/utils/misc'; import { upsertTags } from 'src/utils/tag'; import { Tasks } from 'src/utils/tasks'; @@ -131,6 +131,33 @@ const getLensModel = (exifTags: ImmichTags): string | null => { type ImmichTagsWithFaces = ImmichTags & { RegionInfo: NonNullable }; +type SidecarFace = { + name: string; + boundingBoxX1: number; + boundingBoxY1: number; + boundingBoxX2: number; + boundingBoxY2: number; + imageWidth: number; + imageHeight: number; +}; + +/** normalized region coordinates are written with sub-pixel precision for images up to 1,000,000 pixels wide */ +const roundArea = (value: number | string) => Math.round(Number(value) * 1e6) / 1e6; + +/** + * Only exif orientations are recognized, since ORIENTATION_TO_SHARP_ROTATION is what rotated the image the faces were + * detected on. Anything else (some assets store a rotation in degrees) leaves the image as it is stored, so the faces + * of those assets need no conversion either. + */ +const parseOrientation = (value: string | null): ExifOrientation | undefined => { + const orientation = Number(value); + return Number.isSafeInteger(orientation) && + orientation >= ExifOrientation.Horizontal && + orientation <= ExifOrientation.Rotate270CW + ? (orientation as ExifOrientation) + : undefined; +}; + type Dates = { dateTimeOriginal: Date; localDateTime: Date; @@ -155,8 +182,27 @@ export class MetadataService extends BaseService { } @OnEvent({ name: 'ConfigUpdate', workers: [ImmichWorker.Microservices], server: true }) - onConfigUpdate({ newConfig }: ArgOf<'ConfigUpdate'>) { + async onConfigUpdate({ oldConfig, newConfig }: ArgOf<'ConfigUpdate'>) { this.metadataRepository.setMaxConcurrency(newConfig.job.metadataExtraction.concurrency); + + // Turning face export on would otherwise do nothing until an asset's faces + // happen to change, so write out the people that are already known. + if (!isFaceExportEnabled(oldConfig.metadata) && isFaceExportEnabled(newConfig.metadata)) { + await this.queueSidecarFaceWritesForKnownFaces(); + } + } + + private async queueSidecarFaceWritesForKnownFaces() { + let assetIds: string[] = []; + for await (const { assetId } of this.personRepository.streamAssetIdsWithNamedFaces()) { + assetIds.push(assetId); + if (assetIds.length === JOBS_ASSET_PAGINATION_SIZE) { + await this.queueSidecarFaceWrites(assetIds); + assetIds = []; + } + } + + await this.queueSidecarFaceWrites(assetIds); } private async init() { @@ -487,10 +533,52 @@ export class MetadataService extends BaseService { await this.jobRepository.queue({ name: JobName.SidecarWrite, data: { id: assetId } }); } + @OnEvent({ name: 'AssetFacesUpdate' }) + async handleAssetFacesUpdate({ assetIds }: ArgOf<'AssetFacesUpdate'>) { + const { metadata } = await this.getConfig({ withCache: true }); + if (!isFaceExportEnabled(metadata)) { + return; + } + + await this.queueSidecarFaceWrites(assetIds); + } + + @OnEvent({ name: 'PersonFacesUpdate' }) + async handlePersonFacesUpdate({ personIds }: ArgOf<'PersonFacesUpdate'>) { + const { metadata } = await this.getConfig({ withCache: true }); + if (!isFaceExportEnabled(metadata)) { + return; + } + + let assetIds: string[] = []; + for await (const { assetId } of this.personRepository.streamAssetIdsForPeople(personIds)) { + assetIds.push(assetId); + if (assetIds.length === JOBS_ASSET_PAGINATION_SIZE) { + await this.queueSidecarFaceWrites(assetIds); + assetIds = []; + } + } + + await this.queueSidecarFaceWrites(assetIds); + } + + private async queueSidecarFaceWrites(assetIds: string[]) { + if (assetIds.length === 0) { + return; + } + + await this.jobRepository.queueAll( + assetIds.map((id) => ({ name: JobName.SidecarWrite, data: { id, faces: true } }) as const), + ); + } + @OnJob({ name: JobName.SidecarWrite, queue: QueueName.Sidecar }) async handleSidecarWrite(job: JobOf): Promise { const { id } = job; - const asset = await this.assetJobRepository.getForSidecarWriteJob(id); + const { metadata } = await this.getConfig({ withCache: true }); + const exportFaces = isFaceExportEnabled(metadata); + + const asset = await this.assetJobRepository.getForSidecarWriteJob(id, exportFaces); if (!asset) { return JobStatus.Failed; } @@ -513,8 +601,18 @@ export class MetadataService extends BaseService { lockedProperties, ); + let regionInfo: Struct | null | undefined; + if (exportFaces) { + regionInfo = this.getRegionInfo(asset); + + if (!regionInfo && job.faces) { + // the faces of the asset changed, so regions written earlier have to be removed + regionInfo = (await this.hasSidecarRegions(sidecarFile?.path)) ? null : undefined; + } + } + const exif = _.omitBy( - { + { Description: description, ImageDescription: description, DateTimeOriginal: mergeTimeZone(dateTimeOriginal, timeZone)?.toISO(), @@ -522,6 +620,7 @@ export class MetadataService extends BaseService { GPSLongitude: longitude, Rating: rating, TagsList: tags, + RegionInfo: regionInfo, }, _.isUndefined, ); @@ -907,6 +1006,107 @@ export class MetadataService extends BaseService { }; } + /** + * Every EXIF orientation is its own inverse, except for the two 90° rotations, which invert each other. + */ + private invertOrientation(orientation: ExifOrientation): ExifOrientation { + switch (orientation) { + case ExifOrientation.Rotate90CW: { + return ExifOrientation.Rotate270CW; + } + case ExifOrientation.Rotate270CW: { + return ExifOrientation.Rotate90CW; + } + default: { + return orientation; + } + } + } + + /** + * Immich stores face geometry in the coordinate space of the displayed (orientation corrected) image, while MWG + * regions are relative to the image as it is stored in the file. This is the inverse of {@link orientRegionInfo}. + */ + private unorientRegionInfo( + regionInfo: ImmichTagsWithFaces['RegionInfo'], + orientation: ExifOrientation | undefined, + ): ImmichTagsWithFaces['RegionInfo'] { + return orientation === undefined + ? regionInfo + : this.orientRegionInfo(regionInfo, this.invertOrientation(orientation)); + } + + /** + * Regions are only removed from sidecars that actually have them, so that a sidecar is never created just to hold an + * empty list of regions. + */ + private async hasSidecarRegions(sidecarPath?: string) { + if (!sidecarPath) { + return false; + } + + return this.hasTaggedFaces(await this.metadataRepository.readTags(sidecarPath)); + } + + /** Builds MWG regions for every face of the asset that resolves to a named person. */ + private getRegionInfo(asset: { + id: string; + originalPath: string; + exifInfo: Pick; + // only selected when face export is enabled + faces?: SidecarFace[]; + }): Struct | undefined { + const faces = asset.faces ?? []; + if (faces.length === 0) { + return; + } + + // exif dimensions describe the image as it is stored in the file, which is what AppliedToDimensions refers to + const { exifImageWidth: width, exifImageHeight: height } = asset.exifInfo; + if (!width || !height) { + this.logger.warn( + `Cannot write faces for asset ${asset.id}: ${asset.originalPath}, asset has no known dimensions`, + ); + return; + } + + const AppliedToDimensions = { W: width, H: height, Unit: 'pixel' }; + const regionInfo = { + AppliedToDimensions, + RegionList: faces.map((face) => ({ + Type: 'Face', + Name: face.name, + Area: { + // (X,Y) is the center of the rectangle + X: (face.boundingBoxX1 + face.boundingBoxX2) / 2 / face.imageWidth, + Y: (face.boundingBoxY1 + face.boundingBoxY2) / 2 / face.imageHeight, + W: (face.boundingBoxX2 - face.boundingBoxX1) / face.imageWidth, + H: (face.boundingBoxY2 - face.boundingBoxY1) / face.imageHeight, + Unit: 'normalized', + }, + })), + }; + + // only the areas have to be converted, AppliedToDimensions already refers to the stored image + const { RegionList } = this.unorientRegionInfo(regionInfo, parseOrientation(asset.exifInfo.orientation)); + + // the shape is checked against ImmichTags above, exiftool itself types structs loosely + return { + AppliedToDimensions, + // mirroring an area introduces floating point noise, so keep the written values readable + RegionList: RegionList.map((region) => ({ + ...region, + Area: { + ...region.Area, + X: roundArea(region.Area.X), + Y: roundArea(region.Area.Y), + W: roundArea(region.Area.W), + H: roundArea(region.Area.H), + }, + })), + } as Struct; + } + private async applyTaggedFaces( asset: { id: string; ownerId: string; faces: { id: string; sourceType: SourceType }[]; originalPath: string }, tags: ImmichTags, diff --git a/server/src/services/person.service.spec.ts b/server/src/services/person.service.spec.ts index e6a11786af..2366e051c2 100644 --- a/server/src/services/person.service.spec.ts +++ b/server/src/services/person.service.spec.ts @@ -1318,6 +1318,134 @@ describe(PersonService.name, () => { }); }); + describe('sidecar events', () => { + it('should emit an event when a face is reassigned', async () => { + const face = AssetFaceFactory.create(); + const person = PersonFactory.create(); + + mocks.access.person.checkOwnerAccess.mockResolvedValue(new Set([person.id])); + mocks.access.person.checkFaceOwnerAccess.mockResolvedValue(new Set([face.id])); + mocks.person.getFaceById.mockResolvedValue(getForAssetFace(face)); + mocks.person.reassignFace.mockResolvedValue(1); + mocks.person.getById.mockResolvedValue(person); + + await sut.reassignFacesById(AuthFactory.create(), person.id, { id: face.id }); + + expect(mocks.event.emit).toHaveBeenCalledWith('AssetFacesUpdate', { assetIds: [face.assetId] }); + }); + + it('should emit an event when a face is created', async () => { + const auth = AuthFactory.create(); + const asset = AssetFactory.create(); + const person = PersonFactory.create(); + + mocks.access.asset.checkOwnerAccess.mockResolvedValue(new Set([asset.id])); + mocks.access.person.checkOwnerAccess.mockResolvedValue(new Set([person.id])); + mocks.asset.getById.mockResolvedValue(getForAsset(asset)); + mocks.person.getById.mockResolvedValue(person); + + await sut.createFace(auth, { + assetId: asset.id, + personId: person.id, + imageHeight: 500, + imageWidth: 400, + x: 10, + y: 20, + width: 100, + height: 110, + }); + + expect(mocks.event.emit).toHaveBeenCalledWith('AssetFacesUpdate', { assetIds: [asset.id] }); + }); + + it('should emit an event when a face is deleted', async () => { + const face = AssetFaceFactory.create(); + + mocks.access.person.checkFaceOwnerAccess.mockResolvedValue(new Set([face.id])); + mocks.person.getFaceById.mockResolvedValue(getForAssetFace(face)); + + await sut.deleteFace(AuthFactory.create(), face.id, { force: false }); + + expect(mocks.person.softDeleteAssetFaces).toHaveBeenCalledWith(face.id); + expect(mocks.event.emit).toHaveBeenCalledWith('AssetFacesUpdate', { assetIds: [face.assetId] }); + }); + + it('should emit an event when a person is renamed', async () => { + const auth = AuthFactory.create(); + const person = PersonFactory.create({ name: 'Person 1' }); + + mocks.person.update.mockResolvedValue(person); + mocks.access.person.checkOwnerAccess.mockResolvedValue(new Set([person.id])); + + await sut.update(auth, person.id, { name: 'Person 1' }); + + expect(mocks.event.emit).toHaveBeenCalledWith('PersonFacesUpdate', { personIds: [person.id] }); + }); + + it('should not emit an event when a person is updated without a name', async () => { + const auth = AuthFactory.create(); + const person = PersonFactory.create(); + + mocks.person.update.mockResolvedValue(person); + mocks.access.person.checkOwnerAccess.mockResolvedValue(new Set([person.id])); + + await sut.update(auth, person.id, { isFavorite: true }); + + expect(mocks.event.emit).not.toHaveBeenCalled(); + }); + + it('should emit an event for the primary person when people are merged', async () => { + const auth = AuthFactory.create(); + const [person, mergePerson] = [PersonFactory.create(), PersonFactory.create()]; + + mocks.person.getById.mockResolvedValueOnce(person); + mocks.person.getById.mockResolvedValueOnce(mergePerson); + mocks.access.person.checkOwnerAccess.mockResolvedValueOnce(new Set([person.id])); + mocks.access.person.checkOwnerAccess.mockResolvedValueOnce(new Set([mergePerson.id])); + + await sut.mergePerson(auth, person.id, { ids: [mergePerson.id] }); + + expect(mocks.event.emit).toHaveBeenCalledWith('PersonFacesUpdate', { personIds: [person.id] }); + }); + + it('should emit an event before people are deleted', async () => { + const person = PersonFactory.create(); + + mocks.access.person.checkOwnerAccess.mockResolvedValue(new Set([person.id])); + mocks.person.getForPeopleDelete.mockResolvedValue([person]); + + await sut.deleteAll(AuthFactory.create(), { ids: [person.id] }); + + expect(mocks.event.emit).toHaveBeenCalledWith('PersonFacesUpdate', { personIds: [person.id] }); + expect(mocks.event.emit.mock.invocationCallOrder[0]).toBeLessThan( + mocks.person.delete.mock.invocationCallOrder[0], + ); + }); + + it('should emit an event when facial recognition assigns a face to a person', async () => { + const asset = AssetFactory.create(); + const [noPerson, faceWithPerson] = [ + AssetFaceFactory.create({ assetId: asset.id }), + AssetFaceFactory.from().person().build(), + ]; + + mocks.systemMetadata.get.mockResolvedValue({ machineLearning: { facialRecognition: { minFaces: 1 } } }); + mocks.search.searchFaces.mockResolvedValue([ + { ...noPerson, distance: 0 }, + { ...faceWithPerson, distance: 0.2 }, + ] as FaceSearchResult[]); + mocks.person.getFaceForFacialRecognitionJob.mockResolvedValue(getForFacialRecognitionJob(noPerson, asset)); + + await expect(sut.handleRecognizeFaces({ id: noPerson.id })).resolves.toBe(JobStatus.Success); + + expect(mocks.person.reassignFaces).toHaveBeenCalledWith({ + faceIds: [noPerson.id], + newPersonId: faceWithPerson.person!.id, + }); + expect(mocks.event.emit).toHaveBeenCalledWith('AssetFacesUpdate', { assetIds: [asset.id] }); + }); + }); + describe('getStatistics', () => { it('should get correct number of person', async () => { const auth = AuthFactory.create(); diff --git a/server/src/services/person.service.ts b/server/src/services/person.service.ts index 8e0cd2ff01..0236179342 100644 --- a/server/src/services/person.service.ts +++ b/server/src/services/person.service.ts @@ -82,6 +82,7 @@ export class PersonService extends BaseService { const person = await this.findOrFail(personId); const result: PersonResponseDto[] = []; const changeFeaturePhoto: string[] = []; + const reassignedAssetIds: string[] = []; for (const data of dto.data) { const faces = await this.personRepository.getFacesByIds([{ personId: data.personId, assetId: data.assetId }]); @@ -95,6 +96,7 @@ export class PersonService extends BaseService { } await this.personRepository.reassignFace(face.id, personId); + reassignedAssetIds.push(face.assetId); } result.push(mapPerson(person)); @@ -103,6 +105,9 @@ export class PersonService extends BaseService { // Remove duplicates await this.createNewFeaturePhoto([...new Set(changeFeaturePhoto)]); } + + await this.onFacesUpdate(reassignedAssetIds); + return result; } @@ -120,6 +125,8 @@ export class PersonService extends BaseService { await this.createNewFeaturePhoto([face.person.id]); } + await this.onFacesUpdate([face.assetId]); + return mapPerson(await this.findOrFail(personId)); } @@ -217,6 +224,10 @@ export class PersonService extends BaseService { await this.jobRepository.queue({ name: JobName.PersonGenerateThumbnail, data: { id } }); } + if (name !== undefined) { + await this.onPeopleUpdate([id]); + } + return mapPerson(person); } @@ -247,6 +258,8 @@ export class PersonService extends BaseService { async deleteAll(auth: AuthDto, { ids }: BulkIdsDto): Promise { await this.requireAccess({ auth, permission: Permission.PersonDelete, ids }); const people = await this.personRepository.getForPeopleDelete(ids); + // queue the sidecar writes while the faces of these people can still be resolved + await this.onPeopleUpdate(people.map(({ id }) => id)); await this.removeAllPeople(people); } @@ -535,6 +548,7 @@ export class PersonService extends BaseService { if (personId) { this.logger.debug(`Assigning face ${id} to person ${personId}`); await this.personRepository.reassignFaces({ faceIds: [id], newPersonId: personId }); + await this.onFacesUpdate([face.assetId]); } return JobStatus.Success; @@ -605,6 +619,8 @@ export class PersonService extends BaseService { await this.removeAllPeople([mergePerson]); this.logger.log(`Merged ${mergeName} into ${primaryName}`); + // the faces of the merged person now belong to the primary person, which may also have gained a name + await this.onPeopleUpdate([primaryPerson.id]); results.push({ id: mergeId, success: true }); } catch (error: Error | any) { this.logger.error(`Unable to merge ${mergeId} into ${id}: ${error}`, error?.stack); @@ -693,11 +709,35 @@ export class PersonService extends BaseService { if (!person.faceAssetId) { await this.createNewFeaturePhoto([person.id]); } + + await this.onFacesUpdate([dto.assetId]); } async deleteFace(auth: AuthDto, id: string, dto: AssetFaceDeleteDto): Promise { await this.requireAccess({ auth, permission: Permission.FaceDelete, ids: [id] }); - return dto.force ? this.personRepository.deleteAssetFace(id) : this.personRepository.softDeleteAssetFaces(id); + const face = await this.personRepository.getFaceById(id); + + await (dto.force ? this.personRepository.deleteAssetFace(id) : this.personRepository.softDeleteAssetFaces(id)); + + await this.onFacesUpdate([face.assetId]); + } + + /** the faces of these assets changed, which may have to be reflected in their sidecar files */ + private async onFacesUpdate(assetIds: string[]) { + if (assetIds.length === 0) { + return; + } + + await this.eventRepository.emit('AssetFacesUpdate', { assetIds: [...new Set(assetIds)] }); + } + + /** these people changed, which may have to be reflected in the sidecar files of every asset they appear in */ + private async onPeopleUpdate(personIds: string[]) { + if (personIds.length === 0) { + return; + } + + await this.eventRepository.emit('PersonFacesUpdate', { personIds }); } } diff --git a/server/src/services/system-config.service.spec.ts b/server/src/services/system-config.service.spec.ts index 08851da96a..b80a17bbce 100644 --- a/server/src/services/system-config.service.spec.ts +++ b/server/src/services/system-config.service.spec.ts @@ -104,6 +104,7 @@ const updatedConfig = Object.freeze({ metadata: { faces: { import: false, + export: false, }, }, machineLearning: { diff --git a/server/src/types.ts b/server/src/types.ts index 27995e841f..3bb27409b0 100644 --- a/server/src/types.ts +++ b/server/src/types.ts @@ -269,7 +269,8 @@ export interface IDeleteFilesJob extends IBaseJob { } export interface ISidecarWriteJob extends IEntityJob { - tags?: true; + /** the job was queued because the faces of the asset changed, so stale regions have to be removed as well */ + faces?: true; } export interface IDeferrableJob extends IEntityJob { @@ -397,7 +398,7 @@ export type JobItem = // Sidecar Scanning | { name: JobName.SidecarQueueAll; data: IBaseJob } | { name: JobName.SidecarCheck; data: IEntityJob } - | { name: JobName.SidecarWrite; data: IEntityJob } + | { name: JobName.SidecarWrite; data: ISidecarWriteJob } // Facial Recognition | { name: JobName.AssetDetectFacesQueueAll; data: IBaseJob } diff --git a/server/src/utils/database.ts b/server/src/utils/database.ts index 0f4d8775b6..24e58e4eda 100644 --- a/server/src/utils/database.ts +++ b/server/src/utils/database.ts @@ -218,6 +218,30 @@ export function withFaces(eb: ExpressionBuilder, withHidden?: boole ).as('faces'); } +/** visible faces that resolve to a named person, ordered so that repeated writes produce the same output */ +export function withNamedFaces(eb: ExpressionBuilder) { + return jsonArrayFrom( + eb + .selectFrom('asset_face') + .innerJoin('person', 'person.id', 'asset_face.personId') + .select([ + 'asset_face.boundingBoxX1', + 'asset_face.boundingBoxY1', + 'asset_face.boundingBoxX2', + 'asset_face.boundingBoxY2', + 'asset_face.imageWidth', + 'asset_face.imageHeight', + 'person.name', + ]) + .whereRef('asset_face.assetId', '=', 'asset.id') + .where('asset_face.deletedAt', 'is', null) + .where('asset_face.isVisible', '=', true) + .where('person.name', '!=', '') + .orderBy('asset_face.boundingBoxX1') + .orderBy('asset_face.boundingBoxY1'), + ).as('faces'); +} + export function withFiles(eb: ExpressionBuilder, type?: AssetFileType) { return jsonArrayFrom( eb diff --git a/server/src/utils/misc.ts b/server/src/utils/misc.ts index 0e7a9e6cdc..9993550c37 100644 --- a/server/src/utils/misc.ts +++ b/server/src/utils/misc.ts @@ -104,6 +104,7 @@ export const isFacialRecognitionEnabled = (machineLearning: SystemConfig['machin export const isDuplicateDetectionEnabled = (machineLearning: SystemConfig['machineLearning']) => isSmartSearchEnabled(machineLearning) && machineLearning.duplicateDetection.enabled; export const isFaceImportEnabled = (metadata: SystemConfig['metadata']) => metadata.faces.import; +export const isFaceExportEnabled = (metadata: SystemConfig['metadata']) => metadata.faces.export; export const isConnectionAborted = (error: Error | any) => error.code === 'ECONNABORTED'; diff --git a/server/test/mappers.ts b/server/test/mappers.ts index 40ae78fe26..36e2986ef0 100644 --- a/server/test/mappers.ts +++ b/server/test/mappers.ts @@ -178,6 +178,18 @@ export const getForSidecarWrite = (asset: ReturnType) => originalPath: asset.originalPath, files: asset.files.map((file) => getDehydrated(file)), exifInfo: getDehydrated(asset.exifInfo), + faces: asset.faces + .filter((face) => face.isVisible && !face.deletedAt && face.person?.name) + .sort((a, b) => a.boundingBoxX1 - b.boundingBoxX1 || a.boundingBoxY1 - b.boundingBoxY1) + .map((face) => ({ + boundingBoxX1: face.boundingBoxX1, + boundingBoxY1: face.boundingBoxY1, + boundingBoxX2: face.boundingBoxX2, + boundingBoxY2: face.boundingBoxY2, + imageWidth: face.imageWidth, + imageHeight: face.imageHeight, + name: face.person!.name, + })), }); export const getForAssetDeletion = (asset: ReturnType) => ({ diff --git a/server/test/medium/specs/services/metadata.service.spec.ts b/server/test/medium/specs/services/metadata.service.spec.ts index 37603520f7..c5e9efa76b 100644 --- a/server/test/medium/specs/services/metadata.service.spec.ts +++ b/server/test/medium/specs/services/metadata.service.spec.ts @@ -1,14 +1,20 @@ import { Kysely } from 'kysely'; +import { randomUUID } from 'node:crypto'; import { Stats } from 'node:fs'; -import { writeFile } from 'node:fs/promises'; +import { stat, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; +import { PNG } from 'pngjs'; +import { AssetFileType, JobStatus, SystemMetadataKey } from 'src/enum'; import { AssetJobRepository } from 'src/repositories/asset-job.repository'; import { AssetRepository } from 'src/repositories/asset.repository'; import { ConfigRepository } from 'src/repositories/config.repository'; +import { CryptoRepository } from 'src/repositories/crypto.repository'; import { EventRepository } from 'src/repositories/event.repository'; +import { JobRepository } from 'src/repositories/job.repository'; import { LoggingRepository } from 'src/repositories/logging.repository'; import { MetadataRepository } from 'src/repositories/metadata.repository'; +import { PersonRepository } from 'src/repositories/person.repository'; import { StorageRepository } from 'src/repositories/storage.repository'; import { SystemMetadataRepository } from 'src/repositories/system-metadata.repository'; import { TagRepository } from 'src/repositories/tag.repository'; @@ -37,11 +43,13 @@ const setup = (db?: Kysely) => { AssetRepository, AssetJobRepository, ConfigRepository, + CryptoRepository, MetadataRepository, + PersonRepository, SystemMetadataRepository, TagRepository, ], - mock: [EventRepository, StorageRepository, LoggingRepository], + mock: [EventRepository, JobRepository, StorageRepository, LoggingRepository], }); ctx.getMock(StorageRepository).stat.mockResolvedValue({ @@ -63,6 +71,47 @@ const createTestFile = async (exifData: Record) => { return { filePath }; }; +const enableFaces = async ({ sut, ctx }: ReturnType, faces: { import?: boolean; export?: boolean }) => { + await ctx.get(SystemMetadataRepository).set(SystemMetadataKey.SystemConfig, { metadata: { faces } }); + // the config is cached in module scope, so it has to be reloaded after changing it + await sut.getConfig({ withCache: false }); +}; + +const enableFaceExport = (context: ReturnType) => enableFaces(context, { export: true }); + +/** an image with usable dimensions, unlike the 1x1 pixel of {@link newRandomImage} */ +const newSizedImage = (width: number, height: number) => { + const image = new PNG({ width, height }); + image.data.fill(255); + return PNG.sync.write(image); +}; + +const newAssetWithFace = async (ctx: Awaited>['ctx'], name: string | null) => { + const data = newRandomImage(); + const originalPath = join(tmpdir(), `sidecar-${randomUUID()}.png`); + await writeFile(originalPath, data); + + const { user } = await ctx.newUser(); + const { asset } = await ctx.newAsset({ originalPath, ownerId: user.id }); + await ctx.newExif({ assetId: asset.id, description: '', exifImageWidth: 1000, exifImageHeight: 100 }); + + if (name !== null) { + const { person } = await ctx.newPerson({ ownerId: user.id, name }); + await ctx.newAssetFace({ + assetId: asset.id, + personId: person.id, + imageWidth: 1000, + imageHeight: 100, + boundingBoxX1: 0, + boundingBoxX2: 200, + boundingBoxY1: 20, + boundingBoxY2: 60, + }); + } + + return { asset, sidecarPath: `${originalPath}.xmp` }; +}; + beforeAll(async () => { defaultDatabase = await getKyselyDB(); }); @@ -171,4 +220,108 @@ describe(MetadataService.name, () => { .executeTakeFirstOrThrow(), ).resolves.toEqual({ lensModel: '1.8' }); }); + + describe('handleSidecarWrite', () => { + it('should write named people to the sidecar file', async () => { + const context = setup(); + const { sut, ctx } = context; + await enableFaceExport(context); + const { asset, sidecarPath } = await newAssetWithFace(ctx, 'Alice'); + + await expect(sut.handleSidecarWrite({ id: asset.id, faces: true })).resolves.toBe(JobStatus.Success); + + const tags = await ctx.get(MetadataRepository).readTags(sidecarPath); + expect(tags.RegionInfo).toEqual({ + AppliedToDimensions: { W: 1000, H: 100, Unit: 'pixel' }, + RegionList: [ + { + Type: 'Face', + Name: 'Alice', + Area: { X: 0.1, Y: 0.4, W: 0.2, H: 0.4, Unit: 'normalized' }, + }, + ], + }); + }); + + it('should reach a fixed point when the same faces are imported and exported repeatedly', async () => { + const context = setup(); + const { sut, ctx } = context; + ctx.getMock(EventRepository).emit.mockResolvedValue(); + ctx.getMock(JobRepository).queueAll.mockResolvedValue(); + await enableFaces(context, { import: true, export: true }); + + const originalPath = join(tmpdir(), `roundtrip-${randomUUID()}.png`); + await writeFile(originalPath, newSizedImage(1000, 100)); + const sidecarPath = `${originalPath}.xmp`; + + // regions as another application would have written them + const regionInfo = { + AppliedToDimensions: { W: 1000, H: 100, Unit: 'pixel' }, + RegionList: [ + { Type: 'Face', Name: 'Alice', Area: { X: 0.1, Y: 0.4, W: 0.2, H: 0.4, Unit: 'normalized' } }, + { Type: 'Face', Name: 'Bob', Area: { X: 0.6, Y: 0.5, W: 0.1, H: 0.2, Unit: 'normalized' } }, + ], + }; + await ctx.get(MetadataRepository).writeTags(sidecarPath, { RegionInfo: regionInfo }); + + const { user } = await ctx.newUser(); + const { asset } = await ctx.newAsset({ originalPath, ownerId: user.id }); + await ctx.newAssetFile({ assetId: asset.id, type: AssetFileType.Sidecar, path: sidecarPath }); + + const cycle = async () => { + await sut.handleMetadataExtraction({ id: asset.id }); + await expect(sut.handleSidecarWrite({ id: asset.id, faces: true })).resolves.toBe(JobStatus.Success); + const { RegionInfo } = await ctx.get(MetadataRepository).readTags(sidecarPath); + return RegionInfo; + }; + + const first = await cycle(); + const second = await cycle(); + + // importing quantizes the normalized areas to whole pixels, so the first round trip can move an edge by up to a + // pixel. What matters is that it settles there instead of drifting a little further on every pass. + expect(second).toEqual(first); + + expect(first?.AppliedToDimensions).toEqual({ W: 1000, H: 100, Unit: 'pixel' }); + expect(first?.RegionList.map(({ Name }) => Name)).toEqual(['Alice', 'Bob']); + }); + + it('should not write anything for faces without a named person', async () => { + const context = setup(); + const { sut, ctx } = context; + await enableFaceExport(context); + const { asset, sidecarPath } = await newAssetWithFace(ctx, null); + + await expect(sut.handleSidecarWrite({ id: asset.id, faces: true })).resolves.toBe(JobStatus.Skipped); + await expect(stat(sidecarPath)).rejects.toThrow(); + }); + + it('should remove regions once the named people are gone', async () => { + const context = setup(); + const { sut, ctx } = context; + await enableFaceExport(context); + const { asset, sidecarPath } = await newAssetWithFace(ctx, 'Alice'); + + await sut.handleSidecarWrite({ id: asset.id, faces: true }); + await ctx.database.deleteFrom('asset_face').where('assetId', '=', asset.id).execute(); + await expect(sut.handleSidecarWrite({ id: asset.id, faces: true })).resolves.toBe(JobStatus.Success); + + const tags = await ctx.get(MetadataRepository).readTags(sidecarPath); + expect(tags.RegionInfo?.RegionList).toBeUndefined(); + }); + + it('should keep other sidecar metadata when writing faces', async () => { + const context = setup(); + const { sut, ctx } = context; + await enableFaceExport(context); + const { asset, sidecarPath } = await newAssetWithFace(ctx, 'Alice'); + await ctx.get(MetadataRepository).writeTags(sidecarPath, { Description: 'a description' }); + + await expect(sut.handleSidecarWrite({ id: asset.id, faces: true })).resolves.toBe(JobStatus.Success); + + const tags = await ctx.get(MetadataRepository).readTags(sidecarPath); + expect(tags.Description).toBe('a description'); + expect(tags.RegionInfo).toBeDefined(); + }); + }); }); diff --git a/server/test/medium/specs/services/person.service.spec.ts b/server/test/medium/specs/services/person.service.spec.ts index 39805580f6..5a20f9da5b 100644 --- a/server/test/medium/specs/services/person.service.spec.ts +++ b/server/test/medium/specs/services/person.service.spec.ts @@ -5,6 +5,7 @@ import { AccessRepository } from 'src/repositories/access.repository'; import { AssetEditRepository } from 'src/repositories/asset-edit.repository'; import { AssetRepository } from 'src/repositories/asset.repository'; import { DatabaseRepository } from 'src/repositories/database.repository'; +import { EventRepository } from 'src/repositories/event.repository'; import { JobRepository } from 'src/repositories/job.repository'; import { LoggingRepository } from 'src/repositories/logging.repository'; import { PersonRepository } from 'src/repositories/person.repository'; @@ -18,11 +19,15 @@ import { getKyselyDB } from 'test/utils'; let defaultDatabase: Kysely; const setup = (db?: Kysely) => { - return newMediumService(PersonService, { + const context = newMediumService(PersonService, { database: db || defaultDatabase, real: [AccessRepository, DatabaseRepository, PersonRepository, AssetRepository, AssetEditRepository], - mock: [JobRepository, LoggingRepository, StorageRepository], + mock: [EventRepository, JobRepository, LoggingRepository, StorageRepository], }); + + context.ctx.getMock(EventRepository).emit.mockResolvedValue(); + + return context; }; beforeAll(async () => { diff --git a/web/src/routes/admin/system-settings/MetadataSettings.svelte b/web/src/routes/admin/system-settings/MetadataSettings.svelte index d49292694f..46d7c7395f 100644 --- a/web/src/routes/admin/system-settings/MetadataSettings.svelte +++ b/web/src/routes/admin/system-settings/MetadataSettings.svelte @@ -20,6 +20,13 @@ bind:checked={configToEdit.metadata.faces.import} {disabled} /> + +