diff --git a/docs/docs/FAQ.mdx b/docs/docs/FAQ.mdx
index 869e84e377..0453ca71f2 100644
--- a/docs/docs/FAQ.mdx
+++ b/docs/docs/FAQ.mdx
@@ -151,6 +151,10 @@ See [Backup and Restore](/administration/backup-and-restore.md).
Yes, it creates new faces and persons from the imported asset metadata. For details see the [feature request #4348](https://github.com/immich-app/immich/discussions/4348) and [PR #6455](https://github.com/immich-app/immich/pull/6455).
+### Does Immich write face tags back to metadata?
+
+Optionally. When "Enable face export" is turned on in the metadata settings, Immich writes the people it knows about to the XMP sidecar of an asset (as `mwg-rs:RegionInfo`) whenever its faces change, for example after naming, merging or reassigning a person. Turning the setting on writes out every asset that already has a named person. Only faces that belong to a named person are written, and the original file is never modified — the regions are written to the sidecar file next to it.
+
### Does Immich support the filtering of NSFW images?
No, it currently does not. There is an [open feature request on Github](https://github.com/immich-app/immich/discussions/2451).
diff --git a/docs/docs/features/xmp-sidecars.md b/docs/docs/features/xmp-sidecars.md
index 3536777d8a..51cb443a4e 100644
--- a/docs/docs/features/xmp-sidecars.md
+++ b/docs/docs/features/xmp-sidecars.md
@@ -11,7 +11,7 @@ Tools like Lightroom, Darktable, digiKam and other applications can also be conf
Immich does not support _all_ metadata fields. Below is a table showing what fields Immich can _read_ and _write_. It's important to note that writes do not replace the entire file contents, but are merged together with any existing fields.
:::info
-Immich automatically queues a Sidecar Write job after editing the description, rating, or updating tags.
+Immich automatically queues a Sidecar Write job after editing the description, rating, or updating tags. With _Enable face export_ turned on, changing the faces of an asset — naming, merging, reassigning or deleting a person — queues one as well.
:::
| Metadata | Immich writes to XMP | Immich reads from XMP |
@@ -21,6 +21,11 @@ Immich automatically queues a Sidecar Write job after editing the description, r
| **DateTime** | `exif:DateTimeOriginal`, `photoshop:DateCreated` | In prioritized order:
`exif:SubSecDateTimeOriginal`
`exif:DateTimeOriginal`
`xmp:SubSecCreateDate`
`xmp:CreateDate`
`xmp:CreationDate`
`xmp:MediaCreateDate`
`xmp:SubSecMediaCreateDate`
`xmp:DateTimeCreated` |
| **Location** | `exif:GPSLatitude`, `exif:GPSLongitude` | `exif:GPSLatitude`, `exif:GPSLongitude` |
| **Tags** | `digiKam:TagsList` | In prioritized order:
`digiKam:TagsList`
`lr:HierarchicalSubject`
`IPTC:Keywords` |
+| **Faces** | `mwg-rs:RegionInfo` | `mwg-rs:RegionInfo` |
+
+:::info
+Faces are only read when _Enable face import_ is turned on, and only written when _Enable face export_ is turned on. Both settings are found under Administration > Settings > Metadata Settings and are off by default. Only faces that belong to a named person are written, and the regions replace whichever regions the sidecar held before. Turning face export on writes out every asset that already has a named person, so you do not have to wait for their faces to change.
+:::
:::note
All other fields (e.g. `Creator`, `Source`, IPTC, Lightroom edits) remain in the `.xmp` file and are **not searchable** in Immich.
diff --git a/docs/docs/install/config-file.md b/docs/docs/install/config-file.md
index 5c34acdd9d..580e528834 100644
--- a/docs/docs/install/config-file.md
+++ b/docs/docs/install/config-file.md
@@ -159,7 +159,8 @@ The default configuration looks like this:
},
"metadata": {
"faces": {
- "import": false
+ "import": false,
+ "export": false
}
},
"newVersionCheck": {
diff --git a/i18n/en.json b/i18n/en.json
index 536b7677ac..720fa90256 100644
--- a/i18n/en.json
+++ b/i18n/en.json
@@ -250,6 +250,8 @@
"memory_generate_job": "Memory generation",
"metadata_extraction_job": "Extract metadata",
"metadata_extraction_job_description": "Extract metadata information from each asset, such as GPS, faces and resolution",
+ "metadata_faces_export_setting": "Enable face export",
+ "metadata_faces_export_setting_description": "Write named people to the sidecar files of your assets when their faces change",
"metadata_faces_import_setting": "Enable face import",
"metadata_faces_import_setting_description": "Import faces from image EXIF data and sidecar files",
"metadata_settings": "Metadata Settings",
diff --git a/open-api/immich-openapi-specs.json b/open-api/immich-openapi-specs.json
index bc3bf82094..53250662e8 100644
--- a/open-api/immich-openapi-specs.json
+++ b/open-api/immich-openapi-specs.json
@@ -25849,12 +25849,17 @@
},
"SystemConfigFacesDto": {
"properties": {
+ "export": {
+ "description": "Export",
+ "type": "boolean"
+ },
"import": {
"description": "Import",
"type": "boolean"
}
},
"required": [
+ "export",
"import"
],
"type": "object"
diff --git a/packages/sdk/src/fetch-client.ts b/packages/sdk/src/fetch-client.ts
index 3ac958ce2d..d99c667642 100644
--- a/packages/sdk/src/fetch-client.ts
+++ b/packages/sdk/src/fetch-client.ts
@@ -2495,6 +2495,8 @@ export type SystemConfigMapDto = {
lightStyle: string;
};
export type SystemConfigFacesDto = {
+ /** Export */
+ "export": boolean;
/** Import */
"import": boolean;
};
diff --git a/server/src/config.ts b/server/src/config.ts
index 55304080a3..631c66d3e1 100644
--- a/server/src/config.ts
+++ b/server/src/config.ts
@@ -116,6 +116,7 @@ export type SystemConfig = {
metadata: {
faces: {
import: boolean;
+ export: boolean;
};
};
oauth: {
@@ -333,6 +334,7 @@ export const defaults = Object.freeze({
metadata: {
faces: {
import: false,
+ export: false,
},
},
oauth: {
diff --git a/server/src/dtos/system-config.dto.ts b/server/src/dtos/system-config.dto.ts
index a50b7abe87..99664f0a72 100644
--- a/server/src/dtos/system-config.dto.ts
+++ b/server/src/dtos/system-config.dto.ts
@@ -288,7 +288,7 @@ const SystemConfigReverseGeocodingSchema = z
.meta({ id: 'SystemConfigReverseGeocodingDto' });
const SystemConfigFacesSchema = z
- .object({ import: configBool.describe('Import') })
+ .object({ import: configBool.describe('Import'), export: configBool.describe('Export') })
.meta({ id: 'SystemConfigFacesDto' });
const SystemConfigMetadataSchema = z.object({ faces: SystemConfigFacesSchema }).meta({ id: 'SystemConfigMetadataDto' });
diff --git a/server/src/queries/asset.job.repository.sql b/server/src/queries/asset.job.repository.sql
index aa04603913..f260146c42 100644
--- a/server/src/queries/asset.job.repository.sql
+++ b/server/src/queries/asset.job.repository.sql
@@ -47,6 +47,62 @@ where
limit
$3
+-- AssetJobRepository.getForSidecarWriteJob (with faces)
+select
+ "id",
+ "originalPath",
+ (
+ select
+ coalesce(json_agg(agg), '[]')
+ from
+ (
+ select
+ "asset_file"."id",
+ "asset_file"."path",
+ "asset_file"."type",
+ "asset_file"."isEdited"
+ from
+ "asset_file"
+ where
+ "asset_file"."assetId" = "asset"."id"
+ and "asset_file"."type" = $1
+ ) as agg
+ ) as "files",
+ (
+ select
+ coalesce(json_agg(agg), '[]')
+ from
+ (
+ select
+ "asset_face"."boundingBoxX1",
+ "asset_face"."boundingBoxY1",
+ "asset_face"."boundingBoxX2",
+ "asset_face"."boundingBoxY2",
+ "asset_face"."imageWidth",
+ "asset_face"."imageHeight",
+ "person"."name"
+ from
+ "asset_face"
+ inner join "person" on "person"."id" = "asset_face"."personId"
+ where
+ "asset_face"."assetId" = "asset"."id"
+ and "asset_face"."deletedAt" is null
+ and "asset_face"."isVisible" = $2
+ and "person"."name" != $3
+ order by
+ "asset_face"."boundingBoxX1",
+ "asset_face"."boundingBoxY1"
+ ) as agg
+ ) as "faces",
+ to_json("asset_exif") as "exifInfo"
+from
+ "asset"
+ inner join "asset_exif" on "asset"."id" = "asset_exif"."assetId"
+where
+ "asset"."id" = $4::uuid
+limit
+ $5
+
-- AssetJobRepository.getForSidecarCheckJob
select
"id",
diff --git a/server/src/queries/person.repository.sql b/server/src/queries/person.repository.sql
index a2f3f64442..55363e6d26 100644
--- a/server/src/queries/person.repository.sql
+++ b/server/src/queries/person.repository.sql
@@ -12,6 +12,26 @@ delete from "person"
where
"person"."id" in ($1)
+-- PersonRepository.streamAssetIdsForPeople
+select distinct
+ "asset_face"."assetId"
+from
+ "asset_face"
+where
+ "asset_face"."personId" in ($1)
+ and "asset_face"."deletedAt" is null
+
+-- PersonRepository.streamAssetIdsWithNamedFaces
+select distinct
+ "asset_face"."assetId"
+from
+ "asset_face"
+ inner join "person" on "person"."id" = "asset_face"."personId"
+where
+ "asset_face"."deletedAt" is null
+ and "asset_face"."isVisible" = $1
+ and "person"."name" != $2
+
-- PersonRepository.getFileSamples
select
"id",
@@ -131,6 +151,7 @@ where
-- PersonRepository.getFaceForFacialRecognitionJob
select
"asset_face"."id",
+ "asset_face"."assetId",
"asset_face"."personId",
"asset_face"."sourceType",
(
diff --git a/server/src/repositories/asset-job.repository.ts b/server/src/repositories/asset-job.repository.ts
index bab0c44a41..cf81e9336e 100644
--- a/server/src/repositories/asset-job.repository.ts
+++ b/server/src/repositories/asset-job.repository.ts
@@ -17,6 +17,7 @@ import {
withFaces,
withFilePath,
withFiles,
+ withNamedFaces,
withVideoFormat,
withVideoStream,
} from 'src/utils/database';
@@ -37,13 +38,14 @@ export class AssetJobRepository {
.executeTakeFirst();
}
- @GenerateSql({ params: [DummyValue.UUID] })
- getForSidecarWriteJob(id: string) {
+ @GenerateSql({ params: [DummyValue.UUID] }, { name: 'with faces', params: [DummyValue.UUID, true] })
+ getForSidecarWriteJob(id: string, withFaces = false) {
return this.db
.selectFrom('asset')
.where('asset.id', '=', asUuid(id))
.select(['id', 'originalPath'])
.select((eb) => withFiles(eb, AssetFileType.Sidecar))
+ .$if(withFaces, (qb) => qb.select((eb) => withNamedFaces(eb)))
.$call(withExifInner)
.limit(1)
.executeTakeFirst();
diff --git a/server/src/repositories/event.repository.ts b/server/src/repositories/event.repository.ts
index 416f823952..ed35047f7f 100644
--- a/server/src/repositories/event.repository.ts
+++ b/server/src/repositories/event.repository.ts
@@ -55,6 +55,12 @@ type EventMap = {
AssetTrashAll: [{ assetIds: string[]; userId: string }];
AssetDeleteAll: [{ assetIds: string[]; userId: string }];
AssetRestoreAll: [{ assetIds: string[]; userId: string }];
+ /** the faces of these assets were added, removed or assigned to a different person */
+ AssetFacesUpdate: [{ assetIds: string[] }];
+
+ // person events
+ /** these people were renamed, merged or deleted, which affects every asset they appear in */
+ PersonFacesUpdate: [{ personIds: string[] }];
/** a worker receives a job and emits this event to run it */
JobRun: [QueueName, JobItem];
diff --git a/server/src/repositories/metadata.repository.ts b/server/src/repositories/metadata.repository.ts
index 1d504f6c71..ae392dd43b 100644
--- a/server/src/repositories/metadata.repository.ts
+++ b/server/src/repositories/metadata.repository.ts
@@ -1,5 +1,5 @@
import { Injectable } from '@nestjs/common';
-import { BinaryField, DefaultReadTaskOptions, ExifTool, ReadTaskOptions, Tags } from 'exiftool-vendored';
+import { BinaryField, DefaultReadTaskOptions, ExifTool, ReadTaskOptions, Tags, WriteTags } from 'exiftool-vendored';
import geotz from 'geo-tz';
import { LoggingRepository } from 'src/repositories/logging.repository';
import { mimeTypes } from 'src/utils/mime-types';
@@ -123,7 +123,7 @@ export class MetadataRepository {
return this.exiftool.extractBinaryTagToBuffer(tagName, path);
}
- async writeTags(path: string, tags: Partial): Promise {
+ async writeTags(path: string, tags: WriteTags): Promise {
// If exiftool assigns a field with ^= instead of =, empty values will be written too.
// Since exiftool-vendored doesn't support an option for this, we append the ^ to the name of the tag instead.
// https://exiftool.org/exiftool_pod.html#:~:text=is%20used%20to%20write%20an%20empty%20string
diff --git a/server/src/repositories/person.repository.ts b/server/src/repositories/person.repository.ts
index 0db03a18c7..9795ad6186 100644
--- a/server/src/repositories/person.repository.ts
+++ b/server/src/repositories/person.repository.ts
@@ -112,6 +112,30 @@ export class PersonRepository {
await this.db.deleteFrom('asset_face').where('asset_face.sourceType', '=', sourceType).execute();
}
+ @GenerateSql({ params: [[DummyValue.UUID]], stream: true })
+ streamAssetIdsForPeople(personIds: string[]) {
+ return this.db
+ .selectFrom('asset_face')
+ .select('asset_face.assetId')
+ .distinct()
+ .where('asset_face.personId', 'in', personIds)
+ .where('asset_face.deletedAt', 'is', null)
+ .stream();
+ }
+
+ @GenerateSql({ params: [], stream: true })
+ streamAssetIdsWithNamedFaces() {
+ return this.db
+ .selectFrom('asset_face')
+ .innerJoin('person', 'person.id', 'asset_face.personId')
+ .select('asset_face.assetId')
+ .distinct()
+ .where('asset_face.deletedAt', 'is', null)
+ .where('asset_face.isVisible', '=', true)
+ .where('person.name', '!=', '')
+ .stream();
+ }
+
getAllFaces(options: GetAllFacesOptions = {}) {
return this.db
.selectFrom('asset_face')
@@ -257,7 +281,7 @@ export class PersonRepository {
getFaceForFacialRecognitionJob(id: string) {
return this.db
.selectFrom('asset_face')
- .select(['asset_face.id', 'asset_face.personId', 'asset_face.sourceType'])
+ .select(['asset_face.id', 'asset_face.assetId', 'asset_face.personId', 'asset_face.sourceType'])
.select((eb) =>
jsonObjectFrom(
eb
diff --git a/server/src/services/metadata.service.spec.ts b/server/src/services/metadata.service.spec.ts
index 57c029961e..3b5a0598d0 100644
--- a/server/src/services/metadata.service.spec.ts
+++ b/server/src/services/metadata.service.spec.ts
@@ -62,6 +62,97 @@ const makeFaceTags = (
},
});
+/**
+ * The geometry of {@link makeFaceTags} as Immich stores it after importing it for a given exif orientation. Since
+ * exporting is the inverse of importing, writing these faces has to produce the geometry of `makeFaceTags` again.
+ */
+const orientationTests = [
+ {
+ description: 'undefined',
+ orientation: undefined,
+ expected: { imgW: 1000, imgH: 100, x1: 0, x2: 200, y1: 20, y2: 60 },
+ },
+ {
+ description: 'Horizontal = 1',
+ orientation: ExifOrientation.Horizontal,
+ expected: { imgW: 1000, imgH: 100, x1: 0, x2: 200, y1: 20, y2: 60 },
+ },
+ {
+ description: 'MirrorHorizontal = 2',
+ orientation: ExifOrientation.MirrorHorizontal,
+ expected: { imgW: 1000, imgH: 100, x1: 800, x2: 1000, y1: 20, y2: 60 },
+ },
+ {
+ description: 'Rotate180 = 3',
+ orientation: ExifOrientation.Rotate180,
+ expected: { imgW: 1000, imgH: 100, x1: 800, x2: 1000, y1: 40, y2: 80 },
+ },
+ {
+ description: 'MirrorVertical = 4',
+ orientation: ExifOrientation.MirrorVertical,
+ expected: { imgW: 1000, imgH: 100, x1: 0, x2: 200, y1: 40, y2: 80 },
+ },
+ {
+ description: 'MirrorHorizontalRotate270CW = 5',
+ orientation: ExifOrientation.MirrorHorizontalRotate270CW,
+ expected: { imgW: 100, imgH: 1000, x1: 20, x2: 60, y1: 0, y2: 200 },
+ },
+ {
+ description: 'Rotate90CW = 6',
+ orientation: ExifOrientation.Rotate90CW,
+ expected: { imgW: 100, imgH: 1000, x1: 40, x2: 80, y1: 0, y2: 200 },
+ },
+ {
+ description: 'MirrorHorizontalRotate90CW = 7',
+ orientation: ExifOrientation.MirrorHorizontalRotate90CW,
+ expected: { imgW: 100, imgH: 1000, x1: 40, x2: 80, y1: 800, y2: 1000 },
+ },
+ {
+ description: 'Rotate270CW = 8',
+ orientation: ExifOrientation.Rotate270CW,
+ expected: { imgW: 100, imgH: 1000, x1: 20, x2: 60, y1: 800, y2: 1000 },
+ },
+];
+
+const withExport = (value: boolean) => ({ ...defaults, metadata: { faces: { import: false, export: value } } });
+
+const makeFaceAsset = (
+ {
+ orientation,
+ imgW,
+ imgH,
+ x1,
+ x2,
+ y1,
+ y2,
+ }: {
+ orientation?: ExifOrientation | string;
+ imgW: number;
+ imgH: number;
+ x1: number;
+ x2: number;
+ y1: number;
+ y2: number;
+ },
+ name = 'Alice',
+) =>
+ AssetFactory.from()
+ .file({ type: AssetFileType.Sidecar })
+ .exif({ exifImageWidth: 1000, exifImageHeight: 100, orientation: orientation ? String(orientation) : null })
+ .face(
+ {
+ imageWidth: imgW,
+ imageHeight: imgH,
+ boundingBoxX1: x1,
+ boundingBoxX2: x2,
+ boundingBoxY1: y1,
+ boundingBoxY2: y2,
+ sourceType: SourceType.MachineLearning,
+ },
+ (face) => face.person({ name }),
+ )
+ .build();
+
const emptyPackets = {
totalDuration: 0,
packetCount: 0,
@@ -123,15 +214,42 @@ describe(MetadataService.name, () => {
});
describe('onConfigUpdate', () => {
- it('should update metadata processing concurrency', () => {
+ it('should update metadata processing concurrency', async () => {
const newConfig = structuredClone(defaults);
newConfig.job.metadataExtraction.concurrency = 10;
- sut.onConfigUpdate({ oldConfig: defaults, newConfig });
+ await sut.onConfigUpdate({ oldConfig: defaults, newConfig });
expect(mocks.metadata.setMaxConcurrency).toHaveBeenCalledWith(newConfig.job.metadataExtraction.concurrency);
expect(mocks.metadata.setMaxConcurrency).toHaveBeenCalledTimes(1);
});
+
+ it('should write out the people already known when face export is turned on', async () => {
+ mocks.person.streamAssetIdsWithNamedFaces.mockReturnValue(
+ makeStream([{ assetId: 'asset-1' }, { assetId: 'asset-2' }]),
+ );
+
+ await sut.onConfigUpdate({ oldConfig: withExport(false), newConfig: withExport(true) });
+
+ expect(mocks.job.queueAll).toHaveBeenCalledWith([
+ { name: JobName.SidecarWrite, data: { id: 'asset-1', faces: true } },
+ { name: JobName.SidecarWrite, data: { id: 'asset-2', faces: true } },
+ ]);
+ });
+
+ it('should not write anything when face export was already on', async () => {
+ await sut.onConfigUpdate({ oldConfig: withExport(true), newConfig: withExport(true) });
+
+ expect(mocks.person.streamAssetIdsWithNamedFaces).not.toHaveBeenCalled();
+ expect(mocks.job.queueAll).not.toHaveBeenCalled();
+ });
+
+ it('should not write anything when face export is turned off', async () => {
+ await sut.onConfigUpdate({ oldConfig: withExport(true), newConfig: withExport(false) });
+
+ expect(mocks.person.streamAssetIdsWithNamedFaces).not.toHaveBeenCalled();
+ expect(mocks.job.queueAll).not.toHaveBeenCalled();
+ });
});
describe('handleQueueMetadataExtraction', () => {
@@ -1481,54 +1599,6 @@ describe(MetadataService.name, () => {
});
describe('handleFaceTagOrientation', () => {
- const orientationTests = [
- {
- description: 'undefined',
- orientation: undefined,
- expected: { imgW: 1000, imgH: 100, x1: 0, x2: 200, y1: 20, y2: 60 },
- },
- {
- description: 'Horizontal = 1',
- orientation: ExifOrientation.Horizontal,
- expected: { imgW: 1000, imgH: 100, x1: 0, x2: 200, y1: 20, y2: 60 },
- },
- {
- description: 'MirrorHorizontal = 2',
- orientation: ExifOrientation.MirrorHorizontal,
- expected: { imgW: 1000, imgH: 100, x1: 800, x2: 1000, y1: 20, y2: 60 },
- },
- {
- description: 'Rotate180 = 3',
- orientation: ExifOrientation.Rotate180,
- expected: { imgW: 1000, imgH: 100, x1: 800, x2: 1000, y1: 40, y2: 80 },
- },
- {
- description: 'MirrorVertical = 4',
- orientation: ExifOrientation.MirrorVertical,
- expected: { imgW: 1000, imgH: 100, x1: 0, x2: 200, y1: 40, y2: 80 },
- },
- {
- description: 'MirrorHorizontalRotate270CW = 5',
- orientation: ExifOrientation.MirrorHorizontalRotate270CW,
- expected: { imgW: 100, imgH: 1000, x1: 20, x2: 60, y1: 0, y2: 200 },
- },
- {
- description: 'Rotate90CW = 6',
- orientation: ExifOrientation.Rotate90CW,
- expected: { imgW: 100, imgH: 1000, x1: 40, x2: 80, y1: 0, y2: 200 },
- },
- {
- description: 'MirrorHorizontalRotate90CW = 7',
- orientation: ExifOrientation.MirrorHorizontalRotate90CW,
- expected: { imgW: 100, imgH: 1000, x1: 40, x2: 80, y1: 800, y2: 1000 },
- },
- {
- description: 'Rotate270CW = 8',
- orientation: ExifOrientation.Rotate270CW,
- expected: { imgW: 100, imgH: 1000, x1: 20, x2: 60, y1: 800, y2: 1000 },
- },
- ];
-
it.each(orientationTests)(
'should transform RegionInfo geometry according to exif orientation $description',
async ({ orientation, expected }) => {
@@ -2040,6 +2110,261 @@ describe(MetadataService.name, () => {
expect(mocks.metadata.writeTags).toHaveBeenCalledWith(asset.files[0].path, { Rating: 0 });
expect(mocks.asset.unlockProperties).toHaveBeenCalledWith(asset.id, ['rating']);
});
+
+ describe('faces', () => {
+ it('should not write faces when face export is disabled', async () => {
+ const asset = makeFaceAsset({
+ orientation: ExifOrientation.Horizontal,
+ imgW: 1000,
+ imgH: 100,
+ x1: 0,
+ x2: 200,
+ y1: 20,
+ y2: 60,
+ });
+
+ mocks.assetJob.getLockedPropertiesForMetadataExtraction.mockResolvedValue([]);
+ mocks.assetJob.getForSidecarWriteJob.mockResolvedValue(getForSidecarWrite(asset));
+
+ await expect(sut.handleSidecarWrite({ id: asset.id, faces: true })).resolves.toBe(JobStatus.Skipped);
+ expect(mocks.metadata.writeTags).not.toHaveBeenCalled();
+ // the faces are not even fetched, so an install that does not export them pays nothing
+ expect(mocks.assetJob.getForSidecarWriteJob).toHaveBeenCalledWith(asset.id, false);
+ });
+
+ it('should write named faces as mwg regions', async () => {
+ const asset = makeFaceAsset({
+ orientation: ExifOrientation.Horizontal,
+ imgW: 1000,
+ imgH: 100,
+ x1: 0,
+ x2: 200,
+ y1: 20,
+ y2: 60,
+ });
+
+ mocks.systemMetadata.get.mockResolvedValue({ metadata: { faces: { export: true } } });
+ mocks.assetJob.getLockedPropertiesForMetadataExtraction.mockResolvedValue([]);
+ mocks.assetJob.getForSidecarWriteJob.mockResolvedValue(getForSidecarWrite(asset));
+
+ await expect(sut.handleSidecarWrite({ id: asset.id, faces: true })).resolves.toBe(JobStatus.Success);
+ expect(mocks.assetJob.getForSidecarWriteJob).toHaveBeenCalledWith(asset.id, true);
+ expect(mocks.metadata.writeTags).toHaveBeenCalledWith(asset.files[0].path, {
+ RegionInfo: {
+ AppliedToDimensions: { W: 1000, H: 100, Unit: 'pixel' },
+ RegionList: [
+ {
+ Type: 'Face',
+ Name: 'Alice',
+ Area: { X: 0.1, Y: 0.4, W: 0.2, H: 0.4, Unit: 'normalized' },
+ },
+ ],
+ },
+ });
+ });
+
+ it('should write faces along with other metadata', async () => {
+ const description = 'this is a description';
+ const asset = makeFaceAsset({
+ orientation: ExifOrientation.Horizontal,
+ imgW: 1000,
+ imgH: 100,
+ x1: 0,
+ x2: 200,
+ y1: 20,
+ y2: 60,
+ });
+ asset.exifInfo.description = description;
+
+ mocks.systemMetadata.get.mockResolvedValue({ metadata: { faces: { export: true } } });
+ mocks.assetJob.getLockedPropertiesForMetadataExtraction.mockResolvedValue(['description']);
+ mocks.assetJob.getForSidecarWriteJob.mockResolvedValue(getForSidecarWrite(asset));
+
+ await expect(sut.handleSidecarWrite({ id: asset.id })).resolves.toBe(JobStatus.Success);
+ expect(mocks.metadata.writeTags).toHaveBeenCalledWith(
+ asset.files[0].path,
+ expect.objectContaining({ Description: description, RegionInfo: expect.any(Object) }),
+ );
+ });
+
+ it('should skip faces without a named person', async () => {
+ const asset = AssetFactory.from()
+ .file({ type: AssetFileType.Sidecar })
+ .exif({ exifImageWidth: 1000, exifImageHeight: 100 })
+ .face({ imageWidth: 1000, imageHeight: 100 })
+ .build();
+
+ mocks.systemMetadata.get.mockResolvedValue({ metadata: { faces: { export: true } } });
+ mocks.assetJob.getLockedPropertiesForMetadataExtraction.mockResolvedValue([]);
+ mocks.assetJob.getForSidecarWriteJob.mockResolvedValue(getForSidecarWrite(asset));
+
+ await expect(sut.handleSidecarWrite({ id: asset.id })).resolves.toBe(JobStatus.Skipped);
+ expect(mocks.metadata.writeTags).not.toHaveBeenCalled();
+ });
+
+ it('should remove regions when the last named face was removed', async () => {
+ const asset = AssetFactory.from().file({ type: AssetFileType.Sidecar }).exif().build();
+
+ mocks.systemMetadata.get.mockResolvedValue({ metadata: { faces: { export: true } } });
+ mocks.assetJob.getLockedPropertiesForMetadataExtraction.mockResolvedValue([]);
+ mocks.assetJob.getForSidecarWriteJob.mockResolvedValue(getForSidecarWrite(asset));
+ mockReadTags(makeFaceTags({ Name: 'Alice' }));
+
+ await expect(sut.handleSidecarWrite({ id: asset.id, faces: true })).resolves.toBe(JobStatus.Success);
+ expect(mocks.metadata.readTags).toHaveBeenCalledWith(asset.files[0].path);
+ expect(mocks.metadata.writeTags).toHaveBeenCalledWith(asset.files[0].path, { RegionInfo: null });
+ });
+
+ it('should not touch sidecars that have no regions', async () => {
+ const asset = AssetFactory.from().file({ type: AssetFileType.Sidecar }).exif().build();
+
+ mocks.systemMetadata.get.mockResolvedValue({ metadata: { faces: { export: true } } });
+ mocks.assetJob.getLockedPropertiesForMetadataExtraction.mockResolvedValue([]);
+ mocks.assetJob.getForSidecarWriteJob.mockResolvedValue(getForSidecarWrite(asset));
+ mockReadTags({});
+
+ await expect(sut.handleSidecarWrite({ id: asset.id, faces: true })).resolves.toBe(JobStatus.Skipped);
+ expect(mocks.metadata.writeTags).not.toHaveBeenCalled();
+ });
+
+ it('should not create a sidecar just to remove regions', async () => {
+ const asset = AssetFactory.from().exif().build();
+
+ mocks.systemMetadata.get.mockResolvedValue({ metadata: { faces: { export: true } } });
+ mocks.assetJob.getLockedPropertiesForMetadataExtraction.mockResolvedValue([]);
+ mocks.assetJob.getForSidecarWriteJob.mockResolvedValue(getForSidecarWrite(asset));
+
+ await expect(sut.handleSidecarWrite({ id: asset.id, faces: true })).resolves.toBe(JobStatus.Skipped);
+ expect(mocks.metadata.readTags).not.toHaveBeenCalled();
+ expect(mocks.metadata.writeTags).not.toHaveBeenCalled();
+ });
+
+ it('should leave regions alone when the job was not triggered by a face change', async () => {
+ const asset = AssetFactory.from().file({ type: AssetFileType.Sidecar }).exif().build();
+
+ mocks.systemMetadata.get.mockResolvedValue({ metadata: { faces: { export: true } } });
+ mocks.assetJob.getLockedPropertiesForMetadataExtraction.mockResolvedValue([]);
+ mocks.assetJob.getForSidecarWriteJob.mockResolvedValue(getForSidecarWrite(asset));
+
+ await expect(sut.handleSidecarWrite({ id: asset.id })).resolves.toBe(JobStatus.Skipped);
+ expect(mocks.metadata.writeTags).not.toHaveBeenCalled();
+ });
+
+ it('should not write faces for assets without known dimensions', async () => {
+ const asset = AssetFactory.from()
+ .file({ type: AssetFileType.Sidecar })
+ .exif({ exifImageWidth: null, exifImageHeight: null })
+ .face({ imageWidth: 1000, imageHeight: 100 }, (face) => face.person({ name: 'Alice' }))
+ .build();
+
+ mocks.systemMetadata.get.mockResolvedValue({ metadata: { faces: { export: true } } });
+ mocks.assetJob.getLockedPropertiesForMetadataExtraction.mockResolvedValue([]);
+ mocks.assetJob.getForSidecarWriteJob.mockResolvedValue(getForSidecarWrite(asset));
+
+ await expect(sut.handleSidecarWrite({ id: asset.id, faces: true })).resolves.toBe(JobStatus.Skipped);
+ expect(mocks.metadata.writeTags).not.toHaveBeenCalled();
+ });
+
+ it('should not transform geometry for a rotation that is not an exif orientation', async () => {
+ // the image of such an asset is never rotated, so its faces are already in the coordinate space of the file
+ const asset = makeFaceAsset({ orientation: '90', imgW: 1000, imgH: 100, x1: 0, x2: 200, y1: 20, y2: 60 });
+
+ mocks.systemMetadata.get.mockResolvedValue({ metadata: { faces: { export: true } } });
+ mocks.assetJob.getLockedPropertiesForMetadataExtraction.mockResolvedValue([]);
+ mocks.assetJob.getForSidecarWriteJob.mockResolvedValue(getForSidecarWrite(asset));
+
+ await expect(sut.handleSidecarWrite({ id: asset.id, faces: true })).resolves.toBe(JobStatus.Success);
+ expect(mocks.metadata.writeTags).toHaveBeenCalledWith(asset.files[0].path, {
+ RegionInfo: {
+ AppliedToDimensions: { W: 1000, H: 100, Unit: 'pixel' },
+ RegionList: [
+ {
+ Type: 'Face',
+ Name: 'Alice',
+ Area: { X: 0.1, Y: 0.4, W: 0.2, H: 0.4, Unit: 'normalized' },
+ },
+ ],
+ },
+ });
+ });
+
+ it.each(orientationTests)(
+ 'should write RegionInfo geometry in the coordinate space of the file for exif orientation $description',
+ async ({ orientation, expected }) => {
+ const asset = makeFaceAsset({ orientation, ...expected });
+
+ mocks.systemMetadata.get.mockResolvedValue({ metadata: { faces: { export: true } } });
+ mocks.assetJob.getLockedPropertiesForMetadataExtraction.mockResolvedValue([]);
+ mocks.assetJob.getForSidecarWriteJob.mockResolvedValue(getForSidecarWrite(asset));
+
+ await expect(sut.handleSidecarWrite({ id: asset.id, faces: true })).resolves.toBe(JobStatus.Success);
+
+ // the geometry of makeFaceTags, i.e. exporting is the exact inverse of importing
+ expect(mocks.metadata.writeTags).toHaveBeenCalledWith(asset.files[0].path, {
+ RegionInfo: {
+ AppliedToDimensions: { W: 1000, H: 100, Unit: 'pixel' },
+ RegionList: [
+ {
+ Type: 'Face',
+ Name: 'Alice',
+ Area: { X: 0.1, Y: 0.4, W: 0.2, H: 0.4, Unit: 'normalized' },
+ },
+ ],
+ },
+ });
+ },
+ );
+ });
+ });
+
+ describe('handleAssetFacesUpdate', () => {
+ it('should do nothing when face export is disabled', async () => {
+ await sut.handleAssetFacesUpdate({ assetIds: ['asset-1'] });
+ expect(mocks.job.queueAll).not.toHaveBeenCalled();
+ });
+
+ it('should queue a sidecar write for every asset', async () => {
+ mocks.systemMetadata.get.mockResolvedValue({ metadata: { faces: { export: true } } });
+
+ await sut.handleAssetFacesUpdate({ assetIds: ['asset-1', 'asset-2'] });
+
+ expect(mocks.job.queueAll).toHaveBeenCalledWith([
+ { name: JobName.SidecarWrite, data: { id: 'asset-1', faces: true } },
+ { name: JobName.SidecarWrite, data: { id: 'asset-2', faces: true } },
+ ]);
+ });
+ });
+
+ describe('handlePersonFacesUpdate', () => {
+ it('should do nothing when face export is disabled', async () => {
+ await sut.handlePersonFacesUpdate({ personIds: ['person-1'] });
+ expect(mocks.person.streamAssetIdsForPeople).not.toHaveBeenCalled();
+ expect(mocks.job.queueAll).not.toHaveBeenCalled();
+ });
+
+ it('should queue a sidecar write for every asset of the people', async () => {
+ mocks.systemMetadata.get.mockResolvedValue({ metadata: { faces: { export: true } } });
+ mocks.person.streamAssetIdsForPeople.mockReturnValue(
+ makeStream([{ assetId: 'asset-1' }, { assetId: 'asset-2' }]),
+ );
+
+ await sut.handlePersonFacesUpdate({ personIds: ['person-1'] });
+
+ expect(mocks.person.streamAssetIdsForPeople).toHaveBeenCalledWith(['person-1']);
+ expect(mocks.job.queueAll).toHaveBeenCalledWith([
+ { name: JobName.SidecarWrite, data: { id: 'asset-1', faces: true } },
+ { name: JobName.SidecarWrite, data: { id: 'asset-2', faces: true } },
+ ]);
+ });
+
+ it('should not queue anything when the people have no assets', async () => {
+ mocks.systemMetadata.get.mockResolvedValue({ metadata: { faces: { export: true } } });
+ mocks.person.streamAssetIdsForPeople.mockReturnValue(makeStream([]));
+
+ await sut.handlePersonFacesUpdate({ personIds: ['person-1'] });
+
+ expect(mocks.job.queueAll).not.toHaveBeenCalled();
+ });
});
describe('firstDateTime', () => {
diff --git a/server/src/services/metadata.service.ts b/server/src/services/metadata.service.ts
index 37dd92e27d..555c102399 100644
--- a/server/src/services/metadata.service.ts
+++ b/server/src/services/metadata.service.ts
@@ -1,5 +1,5 @@
import { Injectable } from '@nestjs/common';
-import { ContainerDirectoryItem, ExifDateTime, Tags } from 'exiftool-vendored';
+import { ContainerDirectoryItem, ExifDateTime, Struct, WriteTags } from 'exiftool-vendored';
import { Insertable } from 'kysely';
import _ from 'lodash';
import { DateTime, Duration } from 'luxon';
@@ -8,7 +8,7 @@ import { constants } from 'node:fs/promises';
import { join, parse } from 'node:path';
import { JOBS_ASSET_PAGINATION_SIZE } from 'src/constants';
import { StorageCore } from 'src/cores/storage.core';
-import { Asset, AssetFile } from 'src/database';
+import { Asset, AssetFile, Exif } from 'src/database';
import { OnEvent, OnJob } from 'src/decorators';
import {
AssetFileType,
@@ -35,7 +35,7 @@ import { getAssetFiles } from 'src/utils/asset.util';
import { isAssetChecksumConstraint } from 'src/utils/database';
import { mergeTimeZone } from 'src/utils/date';
import { mimeTypes } from 'src/utils/mime-types';
-import { isFaceImportEnabled } from 'src/utils/misc';
+import { isFaceExportEnabled, isFaceImportEnabled } from 'src/utils/misc';
import { upsertTags } from 'src/utils/tag';
import { Tasks } from 'src/utils/tasks';
@@ -131,6 +131,33 @@ const getLensModel = (exifTags: ImmichTags): string | null => {
type ImmichTagsWithFaces = ImmichTags & { RegionInfo: NonNullable };
+type SidecarFace = {
+ name: string;
+ boundingBoxX1: number;
+ boundingBoxY1: number;
+ boundingBoxX2: number;
+ boundingBoxY2: number;
+ imageWidth: number;
+ imageHeight: number;
+};
+
+/** normalized region coordinates are written with sub-pixel precision for images up to 1,000,000 pixels wide */
+const roundArea = (value: number | string) => Math.round(Number(value) * 1e6) / 1e6;
+
+/**
+ * Only exif orientations are recognized, since ORIENTATION_TO_SHARP_ROTATION is what rotated the image the faces were
+ * detected on. Anything else (some assets store a rotation in degrees) leaves the image as it is stored, so the faces
+ * of those assets need no conversion either.
+ */
+const parseOrientation = (value: string | null): ExifOrientation | undefined => {
+ const orientation = Number(value);
+ return Number.isSafeInteger(orientation) &&
+ orientation >= ExifOrientation.Horizontal &&
+ orientation <= ExifOrientation.Rotate270CW
+ ? (orientation as ExifOrientation)
+ : undefined;
+};
+
type Dates = {
dateTimeOriginal: Date;
localDateTime: Date;
@@ -155,8 +182,27 @@ export class MetadataService extends BaseService {
}
@OnEvent({ name: 'ConfigUpdate', workers: [ImmichWorker.Microservices], server: true })
- onConfigUpdate({ newConfig }: ArgOf<'ConfigUpdate'>) {
+ async onConfigUpdate({ oldConfig, newConfig }: ArgOf<'ConfigUpdate'>) {
this.metadataRepository.setMaxConcurrency(newConfig.job.metadataExtraction.concurrency);
+
+ // Turning face export on would otherwise do nothing until an asset's faces
+ // happen to change, so write out the people that are already known.
+ if (!isFaceExportEnabled(oldConfig.metadata) && isFaceExportEnabled(newConfig.metadata)) {
+ await this.queueSidecarFaceWritesForKnownFaces();
+ }
+ }
+
+ private async queueSidecarFaceWritesForKnownFaces() {
+ let assetIds: string[] = [];
+ for await (const { assetId } of this.personRepository.streamAssetIdsWithNamedFaces()) {
+ assetIds.push(assetId);
+ if (assetIds.length === JOBS_ASSET_PAGINATION_SIZE) {
+ await this.queueSidecarFaceWrites(assetIds);
+ assetIds = [];
+ }
+ }
+
+ await this.queueSidecarFaceWrites(assetIds);
}
private async init() {
@@ -487,10 +533,52 @@ export class MetadataService extends BaseService {
await this.jobRepository.queue({ name: JobName.SidecarWrite, data: { id: assetId } });
}
+ @OnEvent({ name: 'AssetFacesUpdate' })
+ async handleAssetFacesUpdate({ assetIds }: ArgOf<'AssetFacesUpdate'>) {
+ const { metadata } = await this.getConfig({ withCache: true });
+ if (!isFaceExportEnabled(metadata)) {
+ return;
+ }
+
+ await this.queueSidecarFaceWrites(assetIds);
+ }
+
+ @OnEvent({ name: 'PersonFacesUpdate' })
+ async handlePersonFacesUpdate({ personIds }: ArgOf<'PersonFacesUpdate'>) {
+ const { metadata } = await this.getConfig({ withCache: true });
+ if (!isFaceExportEnabled(metadata)) {
+ return;
+ }
+
+ let assetIds: string[] = [];
+ for await (const { assetId } of this.personRepository.streamAssetIdsForPeople(personIds)) {
+ assetIds.push(assetId);
+ if (assetIds.length === JOBS_ASSET_PAGINATION_SIZE) {
+ await this.queueSidecarFaceWrites(assetIds);
+ assetIds = [];
+ }
+ }
+
+ await this.queueSidecarFaceWrites(assetIds);
+ }
+
+ private async queueSidecarFaceWrites(assetIds: string[]) {
+ if (assetIds.length === 0) {
+ return;
+ }
+
+ await this.jobRepository.queueAll(
+ assetIds.map((id) => ({ name: JobName.SidecarWrite, data: { id, faces: true } }) as const),
+ );
+ }
+
@OnJob({ name: JobName.SidecarWrite, queue: QueueName.Sidecar })
async handleSidecarWrite(job: JobOf): Promise {
const { id } = job;
- const asset = await this.assetJobRepository.getForSidecarWriteJob(id);
+ const { metadata } = await this.getConfig({ withCache: true });
+ const exportFaces = isFaceExportEnabled(metadata);
+
+ const asset = await this.assetJobRepository.getForSidecarWriteJob(id, exportFaces);
if (!asset) {
return JobStatus.Failed;
}
@@ -513,8 +601,18 @@ export class MetadataService extends BaseService {
lockedProperties,
);
+ let regionInfo: Struct | null | undefined;
+ if (exportFaces) {
+ regionInfo = this.getRegionInfo(asset);
+
+ if (!regionInfo && job.faces) {
+ // the faces of the asset changed, so regions written earlier have to be removed
+ regionInfo = (await this.hasSidecarRegions(sidecarFile?.path)) ? null : undefined;
+ }
+ }
+
const exif = _.omitBy(
- {
+ {
Description: description,
ImageDescription: description,
DateTimeOriginal: mergeTimeZone(dateTimeOriginal, timeZone)?.toISO(),
@@ -522,6 +620,7 @@ export class MetadataService extends BaseService {
GPSLongitude: longitude,
Rating: rating,
TagsList: tags,
+ RegionInfo: regionInfo,
},
_.isUndefined,
);
@@ -907,6 +1006,107 @@ export class MetadataService extends BaseService {
};
}
+ /**
+ * Every EXIF orientation is its own inverse, except for the two 90° rotations, which invert each other.
+ */
+ private invertOrientation(orientation: ExifOrientation): ExifOrientation {
+ switch (orientation) {
+ case ExifOrientation.Rotate90CW: {
+ return ExifOrientation.Rotate270CW;
+ }
+ case ExifOrientation.Rotate270CW: {
+ return ExifOrientation.Rotate90CW;
+ }
+ default: {
+ return orientation;
+ }
+ }
+ }
+
+ /**
+ * Immich stores face geometry in the coordinate space of the displayed (orientation corrected) image, while MWG
+ * regions are relative to the image as it is stored in the file. This is the inverse of {@link orientRegionInfo}.
+ */
+ private unorientRegionInfo(
+ regionInfo: ImmichTagsWithFaces['RegionInfo'],
+ orientation: ExifOrientation | undefined,
+ ): ImmichTagsWithFaces['RegionInfo'] {
+ return orientation === undefined
+ ? regionInfo
+ : this.orientRegionInfo(regionInfo, this.invertOrientation(orientation));
+ }
+
+ /**
+ * Regions are only removed from sidecars that actually have them, so that a sidecar is never created just to hold an
+ * empty list of regions.
+ */
+ private async hasSidecarRegions(sidecarPath?: string) {
+ if (!sidecarPath) {
+ return false;
+ }
+
+ return this.hasTaggedFaces(await this.metadataRepository.readTags(sidecarPath));
+ }
+
+ /** Builds MWG regions for every face of the asset that resolves to a named person. */
+ private getRegionInfo(asset: {
+ id: string;
+ originalPath: string;
+ exifInfo: Pick;
+ // only selected when face export is enabled
+ faces?: SidecarFace[];
+ }): Struct | undefined {
+ const faces = asset.faces ?? [];
+ if (faces.length === 0) {
+ return;
+ }
+
+ // exif dimensions describe the image as it is stored in the file, which is what AppliedToDimensions refers to
+ const { exifImageWidth: width, exifImageHeight: height } = asset.exifInfo;
+ if (!width || !height) {
+ this.logger.warn(
+ `Cannot write faces for asset ${asset.id}: ${asset.originalPath}, asset has no known dimensions`,
+ );
+ return;
+ }
+
+ const AppliedToDimensions = { W: width, H: height, Unit: 'pixel' };
+ const regionInfo = {
+ AppliedToDimensions,
+ RegionList: faces.map((face) => ({
+ Type: 'Face',
+ Name: face.name,
+ Area: {
+ // (X,Y) is the center of the rectangle
+ X: (face.boundingBoxX1 + face.boundingBoxX2) / 2 / face.imageWidth,
+ Y: (face.boundingBoxY1 + face.boundingBoxY2) / 2 / face.imageHeight,
+ W: (face.boundingBoxX2 - face.boundingBoxX1) / face.imageWidth,
+ H: (face.boundingBoxY2 - face.boundingBoxY1) / face.imageHeight,
+ Unit: 'normalized',
+ },
+ })),
+ };
+
+ // only the areas have to be converted, AppliedToDimensions already refers to the stored image
+ const { RegionList } = this.unorientRegionInfo(regionInfo, parseOrientation(asset.exifInfo.orientation));
+
+ // the shape is checked against ImmichTags above, exiftool itself types structs loosely
+ return {
+ AppliedToDimensions,
+ // mirroring an area introduces floating point noise, so keep the written values readable
+ RegionList: RegionList.map((region) => ({
+ ...region,
+ Area: {
+ ...region.Area,
+ X: roundArea(region.Area.X),
+ Y: roundArea(region.Area.Y),
+ W: roundArea(region.Area.W),
+ H: roundArea(region.Area.H),
+ },
+ })),
+ } as Struct;
+ }
+
private async applyTaggedFaces(
asset: { id: string; ownerId: string; faces: { id: string; sourceType: SourceType }[]; originalPath: string },
tags: ImmichTags,
diff --git a/server/src/services/person.service.spec.ts b/server/src/services/person.service.spec.ts
index e6a11786af..2366e051c2 100644
--- a/server/src/services/person.service.spec.ts
+++ b/server/src/services/person.service.spec.ts
@@ -1318,6 +1318,134 @@ describe(PersonService.name, () => {
});
});
+ describe('sidecar events', () => {
+ it('should emit an event when a face is reassigned', async () => {
+ const face = AssetFaceFactory.create();
+ const person = PersonFactory.create();
+
+ mocks.access.person.checkOwnerAccess.mockResolvedValue(new Set([person.id]));
+ mocks.access.person.checkFaceOwnerAccess.mockResolvedValue(new Set([face.id]));
+ mocks.person.getFaceById.mockResolvedValue(getForAssetFace(face));
+ mocks.person.reassignFace.mockResolvedValue(1);
+ mocks.person.getById.mockResolvedValue(person);
+
+ await sut.reassignFacesById(AuthFactory.create(), person.id, { id: face.id });
+
+ expect(mocks.event.emit).toHaveBeenCalledWith('AssetFacesUpdate', { assetIds: [face.assetId] });
+ });
+
+ it('should emit an event when a face is created', async () => {
+ const auth = AuthFactory.create();
+ const asset = AssetFactory.create();
+ const person = PersonFactory.create();
+
+ mocks.access.asset.checkOwnerAccess.mockResolvedValue(new Set([asset.id]));
+ mocks.access.person.checkOwnerAccess.mockResolvedValue(new Set([person.id]));
+ mocks.asset.getById.mockResolvedValue(getForAsset(asset));
+ mocks.person.getById.mockResolvedValue(person);
+
+ await sut.createFace(auth, {
+ assetId: asset.id,
+ personId: person.id,
+ imageHeight: 500,
+ imageWidth: 400,
+ x: 10,
+ y: 20,
+ width: 100,
+ height: 110,
+ });
+
+ expect(mocks.event.emit).toHaveBeenCalledWith('AssetFacesUpdate', { assetIds: [asset.id] });
+ });
+
+ it('should emit an event when a face is deleted', async () => {
+ const face = AssetFaceFactory.create();
+
+ mocks.access.person.checkFaceOwnerAccess.mockResolvedValue(new Set([face.id]));
+ mocks.person.getFaceById.mockResolvedValue(getForAssetFace(face));
+
+ await sut.deleteFace(AuthFactory.create(), face.id, { force: false });
+
+ expect(mocks.person.softDeleteAssetFaces).toHaveBeenCalledWith(face.id);
+ expect(mocks.event.emit).toHaveBeenCalledWith('AssetFacesUpdate', { assetIds: [face.assetId] });
+ });
+
+ it('should emit an event when a person is renamed', async () => {
+ const auth = AuthFactory.create();
+ const person = PersonFactory.create({ name: 'Person 1' });
+
+ mocks.person.update.mockResolvedValue(person);
+ mocks.access.person.checkOwnerAccess.mockResolvedValue(new Set([person.id]));
+
+ await sut.update(auth, person.id, { name: 'Person 1' });
+
+ expect(mocks.event.emit).toHaveBeenCalledWith('PersonFacesUpdate', { personIds: [person.id] });
+ });
+
+ it('should not emit an event when a person is updated without a name', async () => {
+ const auth = AuthFactory.create();
+ const person = PersonFactory.create();
+
+ mocks.person.update.mockResolvedValue(person);
+ mocks.access.person.checkOwnerAccess.mockResolvedValue(new Set([person.id]));
+
+ await sut.update(auth, person.id, { isFavorite: true });
+
+ expect(mocks.event.emit).not.toHaveBeenCalled();
+ });
+
+ it('should emit an event for the primary person when people are merged', async () => {
+ const auth = AuthFactory.create();
+ const [person, mergePerson] = [PersonFactory.create(), PersonFactory.create()];
+
+ mocks.person.getById.mockResolvedValueOnce(person);
+ mocks.person.getById.mockResolvedValueOnce(mergePerson);
+ mocks.access.person.checkOwnerAccess.mockResolvedValueOnce(new Set([person.id]));
+ mocks.access.person.checkOwnerAccess.mockResolvedValueOnce(new Set([mergePerson.id]));
+
+ await sut.mergePerson(auth, person.id, { ids: [mergePerson.id] });
+
+ expect(mocks.event.emit).toHaveBeenCalledWith('PersonFacesUpdate', { personIds: [person.id] });
+ });
+
+ it('should emit an event before people are deleted', async () => {
+ const person = PersonFactory.create();
+
+ mocks.access.person.checkOwnerAccess.mockResolvedValue(new Set([person.id]));
+ mocks.person.getForPeopleDelete.mockResolvedValue([person]);
+
+ await sut.deleteAll(AuthFactory.create(), { ids: [person.id] });
+
+ expect(mocks.event.emit).toHaveBeenCalledWith('PersonFacesUpdate', { personIds: [person.id] });
+ expect(mocks.event.emit.mock.invocationCallOrder[0]).toBeLessThan(
+ mocks.person.delete.mock.invocationCallOrder[0],
+ );
+ });
+
+ it('should emit an event when facial recognition assigns a face to a person', async () => {
+ const asset = AssetFactory.create();
+ const [noPerson, faceWithPerson] = [
+ AssetFaceFactory.create({ assetId: asset.id }),
+ AssetFaceFactory.from().person().build(),
+ ];
+
+ mocks.systemMetadata.get.mockResolvedValue({ machineLearning: { facialRecognition: { minFaces: 1 } } });
+ mocks.search.searchFaces.mockResolvedValue([
+ { ...noPerson, distance: 0 },
+ { ...faceWithPerson, distance: 0.2 },
+ ] as FaceSearchResult[]);
+ mocks.person.getFaceForFacialRecognitionJob.mockResolvedValue(getForFacialRecognitionJob(noPerson, asset));
+
+ await expect(sut.handleRecognizeFaces({ id: noPerson.id })).resolves.toBe(JobStatus.Success);
+
+ expect(mocks.person.reassignFaces).toHaveBeenCalledWith({
+ faceIds: [noPerson.id],
+ newPersonId: faceWithPerson.person!.id,
+ });
+ expect(mocks.event.emit).toHaveBeenCalledWith('AssetFacesUpdate', { assetIds: [asset.id] });
+ });
+ });
+
describe('getStatistics', () => {
it('should get correct number of person', async () => {
const auth = AuthFactory.create();
diff --git a/server/src/services/person.service.ts b/server/src/services/person.service.ts
index 8e0cd2ff01..0236179342 100644
--- a/server/src/services/person.service.ts
+++ b/server/src/services/person.service.ts
@@ -82,6 +82,7 @@ export class PersonService extends BaseService {
const person = await this.findOrFail(personId);
const result: PersonResponseDto[] = [];
const changeFeaturePhoto: string[] = [];
+ const reassignedAssetIds: string[] = [];
for (const data of dto.data) {
const faces = await this.personRepository.getFacesByIds([{ personId: data.personId, assetId: data.assetId }]);
@@ -95,6 +96,7 @@ export class PersonService extends BaseService {
}
await this.personRepository.reassignFace(face.id, personId);
+ reassignedAssetIds.push(face.assetId);
}
result.push(mapPerson(person));
@@ -103,6 +105,9 @@ export class PersonService extends BaseService {
// Remove duplicates
await this.createNewFeaturePhoto([...new Set(changeFeaturePhoto)]);
}
+
+ await this.onFacesUpdate(reassignedAssetIds);
+
return result;
}
@@ -120,6 +125,8 @@ export class PersonService extends BaseService {
await this.createNewFeaturePhoto([face.person.id]);
}
+ await this.onFacesUpdate([face.assetId]);
+
return mapPerson(await this.findOrFail(personId));
}
@@ -217,6 +224,10 @@ export class PersonService extends BaseService {
await this.jobRepository.queue({ name: JobName.PersonGenerateThumbnail, data: { id } });
}
+ if (name !== undefined) {
+ await this.onPeopleUpdate([id]);
+ }
+
return mapPerson(person);
}
@@ -247,6 +258,8 @@ export class PersonService extends BaseService {
async deleteAll(auth: AuthDto, { ids }: BulkIdsDto): Promise {
await this.requireAccess({ auth, permission: Permission.PersonDelete, ids });
const people = await this.personRepository.getForPeopleDelete(ids);
+ // queue the sidecar writes while the faces of these people can still be resolved
+ await this.onPeopleUpdate(people.map(({ id }) => id));
await this.removeAllPeople(people);
}
@@ -535,6 +548,7 @@ export class PersonService extends BaseService {
if (personId) {
this.logger.debug(`Assigning face ${id} to person ${personId}`);
await this.personRepository.reassignFaces({ faceIds: [id], newPersonId: personId });
+ await this.onFacesUpdate([face.assetId]);
}
return JobStatus.Success;
@@ -605,6 +619,8 @@ export class PersonService extends BaseService {
await this.removeAllPeople([mergePerson]);
this.logger.log(`Merged ${mergeName} into ${primaryName}`);
+ // the faces of the merged person now belong to the primary person, which may also have gained a name
+ await this.onPeopleUpdate([primaryPerson.id]);
results.push({ id: mergeId, success: true });
} catch (error: Error | any) {
this.logger.error(`Unable to merge ${mergeId} into ${id}: ${error}`, error?.stack);
@@ -693,11 +709,35 @@ export class PersonService extends BaseService {
if (!person.faceAssetId) {
await this.createNewFeaturePhoto([person.id]);
}
+
+ await this.onFacesUpdate([dto.assetId]);
}
async deleteFace(auth: AuthDto, id: string, dto: AssetFaceDeleteDto): Promise {
await this.requireAccess({ auth, permission: Permission.FaceDelete, ids: [id] });
- return dto.force ? this.personRepository.deleteAssetFace(id) : this.personRepository.softDeleteAssetFaces(id);
+ const face = await this.personRepository.getFaceById(id);
+
+ await (dto.force ? this.personRepository.deleteAssetFace(id) : this.personRepository.softDeleteAssetFaces(id));
+
+ await this.onFacesUpdate([face.assetId]);
+ }
+
+ /** the faces of these assets changed, which may have to be reflected in their sidecar files */
+ private async onFacesUpdate(assetIds: string[]) {
+ if (assetIds.length === 0) {
+ return;
+ }
+
+ await this.eventRepository.emit('AssetFacesUpdate', { assetIds: [...new Set(assetIds)] });
+ }
+
+ /** these people changed, which may have to be reflected in the sidecar files of every asset they appear in */
+ private async onPeopleUpdate(personIds: string[]) {
+ if (personIds.length === 0) {
+ return;
+ }
+
+ await this.eventRepository.emit('PersonFacesUpdate', { personIds });
}
}
diff --git a/server/src/services/system-config.service.spec.ts b/server/src/services/system-config.service.spec.ts
index 08851da96a..b80a17bbce 100644
--- a/server/src/services/system-config.service.spec.ts
+++ b/server/src/services/system-config.service.spec.ts
@@ -104,6 +104,7 @@ const updatedConfig = Object.freeze({
metadata: {
faces: {
import: false,
+ export: false,
},
},
machineLearning: {
diff --git a/server/src/types.ts b/server/src/types.ts
index 27995e841f..3bb27409b0 100644
--- a/server/src/types.ts
+++ b/server/src/types.ts
@@ -269,7 +269,8 @@ export interface IDeleteFilesJob extends IBaseJob {
}
export interface ISidecarWriteJob extends IEntityJob {
- tags?: true;
+ /** the job was queued because the faces of the asset changed, so stale regions have to be removed as well */
+ faces?: true;
}
export interface IDeferrableJob extends IEntityJob {
@@ -397,7 +398,7 @@ export type JobItem =
// Sidecar Scanning
| { name: JobName.SidecarQueueAll; data: IBaseJob }
| { name: JobName.SidecarCheck; data: IEntityJob }
- | { name: JobName.SidecarWrite; data: IEntityJob }
+ | { name: JobName.SidecarWrite; data: ISidecarWriteJob }
// Facial Recognition
| { name: JobName.AssetDetectFacesQueueAll; data: IBaseJob }
diff --git a/server/src/utils/database.ts b/server/src/utils/database.ts
index 0f4d8775b6..24e58e4eda 100644
--- a/server/src/utils/database.ts
+++ b/server/src/utils/database.ts
@@ -218,6 +218,30 @@ export function withFaces(eb: ExpressionBuilder, withHidden?: boole
).as('faces');
}
+/** visible faces that resolve to a named person, ordered so that repeated writes produce the same output */
+export function withNamedFaces(eb: ExpressionBuilder) {
+ return jsonArrayFrom(
+ eb
+ .selectFrom('asset_face')
+ .innerJoin('person', 'person.id', 'asset_face.personId')
+ .select([
+ 'asset_face.boundingBoxX1',
+ 'asset_face.boundingBoxY1',
+ 'asset_face.boundingBoxX2',
+ 'asset_face.boundingBoxY2',
+ 'asset_face.imageWidth',
+ 'asset_face.imageHeight',
+ 'person.name',
+ ])
+ .whereRef('asset_face.assetId', '=', 'asset.id')
+ .where('asset_face.deletedAt', 'is', null)
+ .where('asset_face.isVisible', '=', true)
+ .where('person.name', '!=', '')
+ .orderBy('asset_face.boundingBoxX1')
+ .orderBy('asset_face.boundingBoxY1'),
+ ).as('faces');
+}
+
export function withFiles(eb: ExpressionBuilder, type?: AssetFileType) {
return jsonArrayFrom(
eb
diff --git a/server/src/utils/misc.ts b/server/src/utils/misc.ts
index 0e7a9e6cdc..9993550c37 100644
--- a/server/src/utils/misc.ts
+++ b/server/src/utils/misc.ts
@@ -104,6 +104,7 @@ export const isFacialRecognitionEnabled = (machineLearning: SystemConfig['machin
export const isDuplicateDetectionEnabled = (machineLearning: SystemConfig['machineLearning']) =>
isSmartSearchEnabled(machineLearning) && machineLearning.duplicateDetection.enabled;
export const isFaceImportEnabled = (metadata: SystemConfig['metadata']) => metadata.faces.import;
+export const isFaceExportEnabled = (metadata: SystemConfig['metadata']) => metadata.faces.export;
export const isConnectionAborted = (error: Error | any) => error.code === 'ECONNABORTED';
diff --git a/server/test/mappers.ts b/server/test/mappers.ts
index 40ae78fe26..36e2986ef0 100644
--- a/server/test/mappers.ts
+++ b/server/test/mappers.ts
@@ -178,6 +178,18 @@ export const getForSidecarWrite = (asset: ReturnType) =>
originalPath: asset.originalPath,
files: asset.files.map((file) => getDehydrated(file)),
exifInfo: getDehydrated(asset.exifInfo),
+ faces: asset.faces
+ .filter((face) => face.isVisible && !face.deletedAt && face.person?.name)
+ .sort((a, b) => a.boundingBoxX1 - b.boundingBoxX1 || a.boundingBoxY1 - b.boundingBoxY1)
+ .map((face) => ({
+ boundingBoxX1: face.boundingBoxX1,
+ boundingBoxY1: face.boundingBoxY1,
+ boundingBoxX2: face.boundingBoxX2,
+ boundingBoxY2: face.boundingBoxY2,
+ imageWidth: face.imageWidth,
+ imageHeight: face.imageHeight,
+ name: face.person!.name,
+ })),
});
export const getForAssetDeletion = (asset: ReturnType) => ({
diff --git a/server/test/medium/specs/services/metadata.service.spec.ts b/server/test/medium/specs/services/metadata.service.spec.ts
index 37603520f7..c5e9efa76b 100644
--- a/server/test/medium/specs/services/metadata.service.spec.ts
+++ b/server/test/medium/specs/services/metadata.service.spec.ts
@@ -1,14 +1,20 @@
import { Kysely } from 'kysely';
+import { randomUUID } from 'node:crypto';
import { Stats } from 'node:fs';
-import { writeFile } from 'node:fs/promises';
+import { stat, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
+import { PNG } from 'pngjs';
+import { AssetFileType, JobStatus, SystemMetadataKey } from 'src/enum';
import { AssetJobRepository } from 'src/repositories/asset-job.repository';
import { AssetRepository } from 'src/repositories/asset.repository';
import { ConfigRepository } from 'src/repositories/config.repository';
+import { CryptoRepository } from 'src/repositories/crypto.repository';
import { EventRepository } from 'src/repositories/event.repository';
+import { JobRepository } from 'src/repositories/job.repository';
import { LoggingRepository } from 'src/repositories/logging.repository';
import { MetadataRepository } from 'src/repositories/metadata.repository';
+import { PersonRepository } from 'src/repositories/person.repository';
import { StorageRepository } from 'src/repositories/storage.repository';
import { SystemMetadataRepository } from 'src/repositories/system-metadata.repository';
import { TagRepository } from 'src/repositories/tag.repository';
@@ -37,11 +43,13 @@ const setup = (db?: Kysely) => {
AssetRepository,
AssetJobRepository,
ConfigRepository,
+ CryptoRepository,
MetadataRepository,
+ PersonRepository,
SystemMetadataRepository,
TagRepository,
],
- mock: [EventRepository, StorageRepository, LoggingRepository],
+ mock: [EventRepository, JobRepository, StorageRepository, LoggingRepository],
});
ctx.getMock(StorageRepository).stat.mockResolvedValue({
@@ -63,6 +71,47 @@ const createTestFile = async (exifData: Record) => {
return { filePath };
};
+const enableFaces = async ({ sut, ctx }: ReturnType, faces: { import?: boolean; export?: boolean }) => {
+ await ctx.get(SystemMetadataRepository).set(SystemMetadataKey.SystemConfig, { metadata: { faces } });
+ // the config is cached in module scope, so it has to be reloaded after changing it
+ await sut.getConfig({ withCache: false });
+};
+
+const enableFaceExport = (context: ReturnType) => enableFaces(context, { export: true });
+
+/** an image with usable dimensions, unlike the 1x1 pixel of {@link newRandomImage} */
+const newSizedImage = (width: number, height: number) => {
+ const image = new PNG({ width, height });
+ image.data.fill(255);
+ return PNG.sync.write(image);
+};
+
+const newAssetWithFace = async (ctx: Awaited>['ctx'], name: string | null) => {
+ const data = newRandomImage();
+ const originalPath = join(tmpdir(), `sidecar-${randomUUID()}.png`);
+ await writeFile(originalPath, data);
+
+ const { user } = await ctx.newUser();
+ const { asset } = await ctx.newAsset({ originalPath, ownerId: user.id });
+ await ctx.newExif({ assetId: asset.id, description: '', exifImageWidth: 1000, exifImageHeight: 100 });
+
+ if (name !== null) {
+ const { person } = await ctx.newPerson({ ownerId: user.id, name });
+ await ctx.newAssetFace({
+ assetId: asset.id,
+ personId: person.id,
+ imageWidth: 1000,
+ imageHeight: 100,
+ boundingBoxX1: 0,
+ boundingBoxX2: 200,
+ boundingBoxY1: 20,
+ boundingBoxY2: 60,
+ });
+ }
+
+ return { asset, sidecarPath: `${originalPath}.xmp` };
+};
+
beforeAll(async () => {
defaultDatabase = await getKyselyDB();
});
@@ -171,4 +220,108 @@ describe(MetadataService.name, () => {
.executeTakeFirstOrThrow(),
).resolves.toEqual({ lensModel: '1.8' });
});
+
+ describe('handleSidecarWrite', () => {
+ it('should write named people to the sidecar file', async () => {
+ const context = setup();
+ const { sut, ctx } = context;
+ await enableFaceExport(context);
+ const { asset, sidecarPath } = await newAssetWithFace(ctx, 'Alice');
+
+ await expect(sut.handleSidecarWrite({ id: asset.id, faces: true })).resolves.toBe(JobStatus.Success);
+
+ const tags = await ctx.get(MetadataRepository).readTags(sidecarPath);
+ expect(tags.RegionInfo).toEqual({
+ AppliedToDimensions: { W: 1000, H: 100, Unit: 'pixel' },
+ RegionList: [
+ {
+ Type: 'Face',
+ Name: 'Alice',
+ Area: { X: 0.1, Y: 0.4, W: 0.2, H: 0.4, Unit: 'normalized' },
+ },
+ ],
+ });
+ });
+
+ it('should reach a fixed point when the same faces are imported and exported repeatedly', async () => {
+ const context = setup();
+ const { sut, ctx } = context;
+ ctx.getMock(EventRepository).emit.mockResolvedValue();
+ ctx.getMock(JobRepository).queueAll.mockResolvedValue();
+ await enableFaces(context, { import: true, export: true });
+
+ const originalPath = join(tmpdir(), `roundtrip-${randomUUID()}.png`);
+ await writeFile(originalPath, newSizedImage(1000, 100));
+ const sidecarPath = `${originalPath}.xmp`;
+
+ // regions as another application would have written them
+ const regionInfo = {
+ AppliedToDimensions: { W: 1000, H: 100, Unit: 'pixel' },
+ RegionList: [
+ { Type: 'Face', Name: 'Alice', Area: { X: 0.1, Y: 0.4, W: 0.2, H: 0.4, Unit: 'normalized' } },
+ { Type: 'Face', Name: 'Bob', Area: { X: 0.6, Y: 0.5, W: 0.1, H: 0.2, Unit: 'normalized' } },
+ ],
+ };
+ await ctx.get(MetadataRepository).writeTags(sidecarPath, { RegionInfo: regionInfo });
+
+ const { user } = await ctx.newUser();
+ const { asset } = await ctx.newAsset({ originalPath, ownerId: user.id });
+ await ctx.newAssetFile({ assetId: asset.id, type: AssetFileType.Sidecar, path: sidecarPath });
+
+ const cycle = async () => {
+ await sut.handleMetadataExtraction({ id: asset.id });
+ await expect(sut.handleSidecarWrite({ id: asset.id, faces: true })).resolves.toBe(JobStatus.Success);
+ const { RegionInfo } = await ctx.get(MetadataRepository).readTags(sidecarPath);
+ return RegionInfo;
+ };
+
+ const first = await cycle();
+ const second = await cycle();
+
+ // importing quantizes the normalized areas to whole pixels, so the first round trip can move an edge by up to a
+ // pixel. What matters is that it settles there instead of drifting a little further on every pass.
+ expect(second).toEqual(first);
+
+ expect(first?.AppliedToDimensions).toEqual({ W: 1000, H: 100, Unit: 'pixel' });
+ expect(first?.RegionList.map(({ Name }) => Name)).toEqual(['Alice', 'Bob']);
+ });
+
+ it('should not write anything for faces without a named person', async () => {
+ const context = setup();
+ const { sut, ctx } = context;
+ await enableFaceExport(context);
+ const { asset, sidecarPath } = await newAssetWithFace(ctx, null);
+
+ await expect(sut.handleSidecarWrite({ id: asset.id, faces: true })).resolves.toBe(JobStatus.Skipped);
+ await expect(stat(sidecarPath)).rejects.toThrow();
+ });
+
+ it('should remove regions once the named people are gone', async () => {
+ const context = setup();
+ const { sut, ctx } = context;
+ await enableFaceExport(context);
+ const { asset, sidecarPath } = await newAssetWithFace(ctx, 'Alice');
+
+ await sut.handleSidecarWrite({ id: asset.id, faces: true });
+ await ctx.database.deleteFrom('asset_face').where('assetId', '=', asset.id).execute();
+ await expect(sut.handleSidecarWrite({ id: asset.id, faces: true })).resolves.toBe(JobStatus.Success);
+
+ const tags = await ctx.get(MetadataRepository).readTags(sidecarPath);
+ expect(tags.RegionInfo?.RegionList).toBeUndefined();
+ });
+
+ it('should keep other sidecar metadata when writing faces', async () => {
+ const context = setup();
+ const { sut, ctx } = context;
+ await enableFaceExport(context);
+ const { asset, sidecarPath } = await newAssetWithFace(ctx, 'Alice');
+ await ctx.get(MetadataRepository).writeTags(sidecarPath, { Description: 'a description' });
+
+ await expect(sut.handleSidecarWrite({ id: asset.id, faces: true })).resolves.toBe(JobStatus.Success);
+
+ const tags = await ctx.get(MetadataRepository).readTags(sidecarPath);
+ expect(tags.Description).toBe('a description');
+ expect(tags.RegionInfo).toBeDefined();
+ });
+ });
});
diff --git a/server/test/medium/specs/services/person.service.spec.ts b/server/test/medium/specs/services/person.service.spec.ts
index 39805580f6..5a20f9da5b 100644
--- a/server/test/medium/specs/services/person.service.spec.ts
+++ b/server/test/medium/specs/services/person.service.spec.ts
@@ -5,6 +5,7 @@ import { AccessRepository } from 'src/repositories/access.repository';
import { AssetEditRepository } from 'src/repositories/asset-edit.repository';
import { AssetRepository } from 'src/repositories/asset.repository';
import { DatabaseRepository } from 'src/repositories/database.repository';
+import { EventRepository } from 'src/repositories/event.repository';
import { JobRepository } from 'src/repositories/job.repository';
import { LoggingRepository } from 'src/repositories/logging.repository';
import { PersonRepository } from 'src/repositories/person.repository';
@@ -18,11 +19,15 @@ import { getKyselyDB } from 'test/utils';
let defaultDatabase: Kysely;
const setup = (db?: Kysely) => {
- return newMediumService(PersonService, {
+ const context = newMediumService(PersonService, {
database: db || defaultDatabase,
real: [AccessRepository, DatabaseRepository, PersonRepository, AssetRepository, AssetEditRepository],
- mock: [JobRepository, LoggingRepository, StorageRepository],
+ mock: [EventRepository, JobRepository, LoggingRepository, StorageRepository],
});
+
+ context.ctx.getMock(EventRepository).emit.mockResolvedValue();
+
+ return context;
};
beforeAll(async () => {
diff --git a/web/src/routes/admin/system-settings/MetadataSettings.svelte b/web/src/routes/admin/system-settings/MetadataSettings.svelte
index d49292694f..46d7c7395f 100644
--- a/web/src/routes/admin/system-settings/MetadataSettings.svelte
+++ b/web/src/routes/admin/system-settings/MetadataSettings.svelte
@@ -20,6 +20,13 @@
bind:checked={configToEdit.metadata.faces.import}
{disabled}
/>
+
+