mirror of
https://github.com/immich-app/immich
synced 2025-11-14 17:36:12 +00:00
feat(server): refresh face detection (#12335)
* refresh faces handle non-ml faces * fix metadata face handling * updated tests * added todo comment
This commit is contained in:
Mert
GitHub
parent
9edc9d6151
commit
2c87683fd4
21 changed files with 409 additions and 152 deletions
|
|
@ -92,8 +92,9 @@ export class AssetIdsDto {
|
|||
}
|
||||
|
||||
export enum AssetJobName {
|
||||
REGENERATE_THUMBNAIL = 'regenerate-thumbnail',
|
||||
REFRESH_FACES = 'refresh-faces',
|
||||
REFRESH_METADATA = 'refresh-metadata',
|
||||
REGENERATE_THUMBNAIL = 'regenerate-thumbnail',
|
||||
TRANSCODE_VIDEO = 'transcode-video',
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ export class JobCommandDto {
|
|||
command!: JobCommand;
|
||||
|
||||
@ValidateBoolean({ optional: true })
|
||||
force!: boolean;
|
||||
force?: boolean; // TODO: this uses undefined as a third state, which should be refactored to be more explicit
|
||||
}
|
||||
|
||||
export class JobCreateDto {
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import { AssetFaceEntity } from 'src/entities/asset-face.entity';
|
||||
import { AssetEntity } from 'src/entities/asset.entity';
|
||||
import { FaceSearchEntity } from 'src/entities/face-search.entity';
|
||||
import { PersonEntity } from 'src/entities/person.entity';
|
||||
import { SourceType } from 'src/enum';
|
||||
import { Paginated, PaginationOptions } from 'src/utils/pagination';
|
||||
|
|
@ -63,7 +64,11 @@ export interface IPersonRepository {
|
|||
delete(entities: PersonEntity[]): Promise<void>;
|
||||
deleteAll(): Promise<void>;
|
||||
deleteFaces(options: DeleteFacesOptions): Promise<void>;
|
||||
replaceFaces(assetId: string, entities: Partial<AssetFaceEntity>[], sourceType?: string): Promise<string[]>;
|
||||
refreshFaces(
|
||||
facesToAdd: Partial<AssetFaceEntity>[],
|
||||
faceIdsToRemove: string[],
|
||||
embeddingsToAdd?: FaceSearchEntity[],
|
||||
): Promise<void>;
|
||||
getAllFaces(pagination: PaginationOptions, options?: FindManyOptions<AssetFaceEntity>): Paginated<AssetFaceEntity>;
|
||||
getFaceById(id: string): Promise<AssetFaceEntity>;
|
||||
getFaceByIdWithAssets(
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import { ChunkedArray, DummyValue, GenerateSql } from 'src/decorators';
|
|||
import { AssetFaceEntity } from 'src/entities/asset-face.entity';
|
||||
import { AssetJobStatusEntity } from 'src/entities/asset-job-status.entity';
|
||||
import { AssetEntity } from 'src/entities/asset.entity';
|
||||
import { FaceSearchEntity } from 'src/entities/face-search.entity';
|
||||
import { PersonEntity } from 'src/entities/person.entity';
|
||||
import { PaginationMode, SourceType } from 'src/enum';
|
||||
import {
|
||||
|
|
@ -31,6 +32,7 @@ export class PersonRepository implements IPersonRepository {
|
|||
@InjectRepository(AssetEntity) private assetRepository: Repository<AssetEntity>,
|
||||
@InjectRepository(PersonEntity) private personRepository: Repository<PersonEntity>,
|
||||
@InjectRepository(AssetFaceEntity) private assetFaceRepository: Repository<AssetFaceEntity>,
|
||||
@InjectRepository(FaceSearchEntity) private faceSearchRepository: Repository<FaceSearchEntity>,
|
||||
@InjectRepository(AssetJobStatusEntity) private jobStatusRepository: Repository<AssetJobStatusEntity>,
|
||||
) {}
|
||||
|
||||
|
|
@ -296,12 +298,31 @@ export class PersonRepository implements IPersonRepository {
|
|||
return res.map((row) => row.id);
|
||||
}
|
||||
|
||||
async replaceFaces(assetId: string, entities: AssetFaceEntity[], sourceType: string): Promise<string[]> {
|
||||
return this.dataSource.transaction(async (manager) => {
|
||||
await manager.delete(AssetFaceEntity, { assetId, sourceType });
|
||||
const assetFaces = await manager.save(AssetFaceEntity, entities);
|
||||
return assetFaces.map(({ id }) => id);
|
||||
});
|
||||
async refreshFaces(
|
||||
facesToAdd: Partial<AssetFaceEntity>[],
|
||||
faceIdsToRemove: string[],
|
||||
embeddingsToAdd?: FaceSearchEntity[],
|
||||
): Promise<void> {
|
||||
const query = this.faceSearchRepository.createQueryBuilder().select('1');
|
||||
if (facesToAdd.length > 0) {
|
||||
const insertCte = this.assetFaceRepository.createQueryBuilder().insert().values(facesToAdd);
|
||||
query.addCommonTableExpression(insertCte, 'added');
|
||||
}
|
||||
|
||||
if (faceIdsToRemove.length > 0) {
|
||||
const deleteCte = this.assetFaceRepository
|
||||
.createQueryBuilder()
|
||||
.delete()
|
||||
.where('id = any(:faceIdsToRemove)', { faceIdsToRemove });
|
||||
query.addCommonTableExpression(deleteCte, 'deleted');
|
||||
}
|
||||
|
||||
if (embeddingsToAdd?.length) {
|
||||
const embeddingCte = this.faceSearchRepository.createQueryBuilder().insert().values(embeddingsToAdd).orIgnore();
|
||||
query.addCommonTableExpression(embeddingCte, 'embeddings');
|
||||
}
|
||||
|
||||
await query.execute();
|
||||
}
|
||||
|
||||
async update(person: Partial<PersonEntity>): Promise<PersonEntity> {
|
||||
|
|
|
|||
|
|
@ -92,9 +92,9 @@ export class AssetService extends BaseService {
|
|||
id,
|
||||
{
|
||||
exifInfo: true,
|
||||
tags: true,
|
||||
sharedLinks: true,
|
||||
smartInfo: true,
|
||||
tags: true,
|
||||
owner: true,
|
||||
faces: {
|
||||
person: true,
|
||||
|
|
@ -290,6 +290,11 @@ export class AssetService extends BaseService {
|
|||
|
||||
for (const id of dto.assetIds) {
|
||||
switch (dto.name) {
|
||||
case AssetJobName.REFRESH_FACES: {
|
||||
jobs.push({ name: JobName.FACE_DETECTION, data: { id } });
|
||||
break;
|
||||
}
|
||||
|
||||
case AssetJobName.REFRESH_METADATA: {
|
||||
jobs.push({ name: JobName.METADATA_EXTRACTION, data: { id } });
|
||||
break;
|
||||
|
|
|
|||
|
|
@ -247,7 +247,7 @@ describe(MetadataService.name, () => {
|
|||
it('should handle an asset that could not be found', async () => {
|
||||
await expect(sut.handleMetadataExtraction({ id: assetStub.image.id })).resolves.toBe(JobStatus.FAILED);
|
||||
|
||||
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.image.id]);
|
||||
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.image.id], { faces: { person: false } });
|
||||
expect(assetMock.upsertExif).not.toHaveBeenCalled();
|
||||
expect(assetMock.update).not.toHaveBeenCalled();
|
||||
});
|
||||
|
|
@ -265,7 +265,7 @@ describe(MetadataService.name, () => {
|
|||
});
|
||||
|
||||
await sut.handleMetadataExtraction({ id: assetStub.image.id });
|
||||
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.sidecar.id]);
|
||||
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.sidecar.id], { faces: { person: false } });
|
||||
expect(assetMock.upsertExif).toHaveBeenCalledWith(expect.objectContaining({ dateTimeOriginal: sidecarDate }));
|
||||
expect(assetMock.update).toHaveBeenCalledWith({
|
||||
id: assetStub.image.id,
|
||||
|
|
@ -280,7 +280,7 @@ describe(MetadataService.name, () => {
|
|||
metadataMock.readTags.mockResolvedValue({ ISO: [160] });
|
||||
|
||||
await sut.handleMetadataExtraction({ id: assetStub.image.id });
|
||||
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.image.id]);
|
||||
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.image.id], { faces: { person: false } });
|
||||
expect(assetMock.upsertExif).toHaveBeenCalledWith(expect.objectContaining({ iso: 160 }));
|
||||
expect(assetMock.update).toHaveBeenCalledWith({
|
||||
id: assetStub.image.id,
|
||||
|
|
@ -300,7 +300,7 @@ describe(MetadataService.name, () => {
|
|||
});
|
||||
|
||||
await sut.handleMetadataExtraction({ id: assetStub.image.id });
|
||||
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.image.id]);
|
||||
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.image.id], { faces: { person: false } });
|
||||
expect(assetMock.upsertExif).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ city: 'City', state: 'State', country: 'Country' }),
|
||||
);
|
||||
|
|
@ -320,7 +320,7 @@ describe(MetadataService.name, () => {
|
|||
});
|
||||
|
||||
await sut.handleMetadataExtraction({ id: assetStub.image.id });
|
||||
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.image.id]);
|
||||
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.image.id], { faces: { person: false } });
|
||||
expect(assetMock.upsertExif).toHaveBeenCalledWith(expect.objectContaining({ latitude: null, longitude: null }));
|
||||
});
|
||||
|
||||
|
|
@ -482,7 +482,9 @@ describe(MetadataService.name, () => {
|
|||
mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
|
||||
|
||||
await sut.handleMetadataExtraction({ id: assetStub.livePhotoMotionAsset.id });
|
||||
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.livePhotoMotionAsset.id]);
|
||||
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.livePhotoMotionAsset.id], {
|
||||
faces: { person: false },
|
||||
});
|
||||
expect(storageMock.createOrOverwriteFile).not.toHaveBeenCalled();
|
||||
expect(jobMock.queue).not.toHaveBeenCalled();
|
||||
expect(jobMock.queueAll).not.toHaveBeenCalled();
|
||||
|
|
@ -508,7 +510,7 @@ describe(MetadataService.name, () => {
|
|||
|
||||
await sut.handleMetadataExtraction({ id: assetStub.video.id });
|
||||
|
||||
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.video.id]);
|
||||
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.video.id], { faces: { person: false } });
|
||||
expect(assetMock.upsertExif).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ orientation: Orientation.Rotate270CW.toString() }),
|
||||
);
|
||||
|
|
@ -536,7 +538,9 @@ describe(MetadataService.name, () => {
|
|||
assetStub.livePhotoWithOriginalFileName.originalPath,
|
||||
'MotionPhotoVideo',
|
||||
);
|
||||
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.livePhotoWithOriginalFileName.id]);
|
||||
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.livePhotoWithOriginalFileName.id], {
|
||||
faces: { person: false },
|
||||
});
|
||||
expect(assetMock.create).toHaveBeenCalledWith({
|
||||
checksum: expect.any(Buffer),
|
||||
deviceAssetId: 'NONE',
|
||||
|
|
@ -579,7 +583,9 @@ describe(MetadataService.name, () => {
|
|||
assetStub.livePhotoWithOriginalFileName.originalPath,
|
||||
'EmbeddedVideoFile',
|
||||
);
|
||||
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.livePhotoWithOriginalFileName.id]);
|
||||
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.livePhotoWithOriginalFileName.id], {
|
||||
faces: { person: false },
|
||||
});
|
||||
expect(assetMock.create).toHaveBeenCalledWith({
|
||||
checksum: expect.any(Buffer),
|
||||
deviceAssetId: 'NONE',
|
||||
|
|
@ -619,7 +625,9 @@ describe(MetadataService.name, () => {
|
|||
storageMock.readFile.mockResolvedValue(video);
|
||||
|
||||
await sut.handleMetadataExtraction({ id: assetStub.livePhotoWithOriginalFileName.id });
|
||||
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.livePhotoWithOriginalFileName.id]);
|
||||
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.livePhotoWithOriginalFileName.id], {
|
||||
faces: { person: false },
|
||||
});
|
||||
expect(storageMock.readFile).toHaveBeenCalledWith(
|
||||
assetStub.livePhotoWithOriginalFileName.originalPath,
|
||||
expect.any(Object),
|
||||
|
|
@ -768,7 +776,7 @@ describe(MetadataService.name, () => {
|
|||
metadataMock.readTags.mockResolvedValue(tags);
|
||||
|
||||
await sut.handleMetadataExtraction({ id: assetStub.image.id });
|
||||
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.image.id]);
|
||||
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.image.id], { faces: { person: false } });
|
||||
expect(assetMock.upsertExif).toHaveBeenCalledWith({
|
||||
assetId: assetStub.image.id,
|
||||
bitsPerSample: expect.any(Number),
|
||||
|
|
@ -826,7 +834,7 @@ describe(MetadataService.name, () => {
|
|||
metadataMock.readTags.mockResolvedValue(tags);
|
||||
|
||||
await sut.handleMetadataExtraction({ id: assetStub.image.id });
|
||||
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.image.id]);
|
||||
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.image.id], { faces: { person: false } });
|
||||
expect(assetMock.upsertExif).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
timeZone: 'UTC+0',
|
||||
|
|
@ -846,7 +854,7 @@ describe(MetadataService.name, () => {
|
|||
|
||||
await sut.handleMetadataExtraction({ id: assetStub.video.id });
|
||||
|
||||
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.video.id]);
|
||||
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.video.id], { faces: { person: false } });
|
||||
expect(assetMock.upsertExif).toHaveBeenCalled();
|
||||
expect(assetMock.update).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
|
|
@ -867,7 +875,7 @@ describe(MetadataService.name, () => {
|
|||
});
|
||||
await sut.handleMetadataExtraction({ id: assetStub.image.id });
|
||||
|
||||
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.image.id]);
|
||||
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.image.id], { faces: { person: false } });
|
||||
expect(assetMock.upsertExif).toHaveBeenCalled();
|
||||
expect(assetMock.update).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
|
|
@ -889,7 +897,7 @@ describe(MetadataService.name, () => {
|
|||
|
||||
await sut.handleMetadataExtraction({ id: assetStub.video.id });
|
||||
|
||||
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.video.id]);
|
||||
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.video.id], { faces: { person: false } });
|
||||
expect(assetMock.upsertExif).toHaveBeenCalled();
|
||||
expect(assetMock.update).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
|
|
@ -911,7 +919,7 @@ describe(MetadataService.name, () => {
|
|||
|
||||
await sut.handleMetadataExtraction({ id: assetStub.video.id });
|
||||
|
||||
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.video.id]);
|
||||
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.video.id], { faces: { person: false } });
|
||||
expect(assetMock.upsertExif).toHaveBeenCalled();
|
||||
expect(assetMock.update).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
|
|
@ -975,11 +983,10 @@ describe(MetadataService.name, () => {
|
|||
metadataMock.readTags.mockResolvedValue(metadataStub.withFaceNoName);
|
||||
personMock.getDistinctNames.mockResolvedValue([]);
|
||||
personMock.createAll.mockResolvedValue([]);
|
||||
personMock.replaceFaces.mockResolvedValue([]);
|
||||
await sut.handleMetadataExtraction({ id: assetStub.image.id });
|
||||
expect(personMock.createAll).toHaveBeenCalledWith([]);
|
||||
expect(personMock.replaceFaces).toHaveBeenCalledWith(assetStub.primaryImage.id, [], SourceType.EXIF);
|
||||
expect(personMock.updateAll).toHaveBeenCalledWith([]);
|
||||
expect(personMock.createAll).not.toHaveBeenCalled();
|
||||
expect(personMock.refreshFaces).not.toHaveBeenCalled();
|
||||
expect(personMock.updateAll).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should skip importing faces with empty name', async () => {
|
||||
|
|
@ -988,11 +995,10 @@ describe(MetadataService.name, () => {
|
|||
metadataMock.readTags.mockResolvedValue(metadataStub.withFaceEmptyName);
|
||||
personMock.getDistinctNames.mockResolvedValue([]);
|
||||
personMock.createAll.mockResolvedValue([]);
|
||||
personMock.replaceFaces.mockResolvedValue([]);
|
||||
await sut.handleMetadataExtraction({ id: assetStub.image.id });
|
||||
expect(personMock.createAll).toHaveBeenCalledWith([]);
|
||||
expect(personMock.replaceFaces).toHaveBeenCalledWith(assetStub.primaryImage.id, [], SourceType.EXIF);
|
||||
expect(personMock.updateAll).toHaveBeenCalledWith([]);
|
||||
expect(personMock.createAll).not.toHaveBeenCalled();
|
||||
expect(personMock.refreshFaces).not.toHaveBeenCalled();
|
||||
expect(personMock.updateAll).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should apply metadata face tags creating new persons', async () => {
|
||||
|
|
@ -1001,14 +1007,12 @@ describe(MetadataService.name, () => {
|
|||
metadataMock.readTags.mockResolvedValue(metadataStub.withFace);
|
||||
personMock.getDistinctNames.mockResolvedValue([]);
|
||||
personMock.createAll.mockResolvedValue([personStub.withName.id]);
|
||||
personMock.replaceFaces.mockResolvedValue(['face-asset-uuid']);
|
||||
personMock.update.mockResolvedValue(personStub.withName);
|
||||
await sut.handleMetadataExtraction({ id: assetStub.primaryImage.id });
|
||||
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.primaryImage.id]);
|
||||
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.primaryImage.id], { faces: { person: false } });
|
||||
expect(personMock.getDistinctNames).toHaveBeenCalledWith(assetStub.primaryImage.ownerId, { withHidden: true });
|
||||
expect(personMock.createAll).toHaveBeenCalledWith([expect.objectContaining({ name: personStub.withName.name })]);
|
||||
expect(personMock.replaceFaces).toHaveBeenCalledWith(
|
||||
assetStub.primaryImage.id,
|
||||
expect(personMock.refreshFaces).toHaveBeenCalledWith(
|
||||
[
|
||||
{
|
||||
id: 'random-uuid',
|
||||
|
|
@ -1023,7 +1027,7 @@ describe(MetadataService.name, () => {
|
|||
sourceType: SourceType.EXIF,
|
||||
},
|
||||
],
|
||||
SourceType.EXIF,
|
||||
[],
|
||||
);
|
||||
expect(personMock.updateAll).toHaveBeenCalledWith([{ id: 'random-uuid', faceAssetId: 'random-uuid' }]);
|
||||
expect(jobMock.queueAll).toHaveBeenCalledWith([
|
||||
|
|
@ -1040,14 +1044,12 @@ describe(MetadataService.name, () => {
|
|||
metadataMock.readTags.mockResolvedValue(metadataStub.withFace);
|
||||
personMock.getDistinctNames.mockResolvedValue([{ id: personStub.withName.id, name: personStub.withName.name }]);
|
||||
personMock.createAll.mockResolvedValue([]);
|
||||
personMock.replaceFaces.mockResolvedValue(['face-asset-uuid']);
|
||||
personMock.update.mockResolvedValue(personStub.withName);
|
||||
await sut.handleMetadataExtraction({ id: assetStub.primaryImage.id });
|
||||
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.primaryImage.id]);
|
||||
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.primaryImage.id], { faces: { person: false } });
|
||||
expect(personMock.getDistinctNames).toHaveBeenCalledWith(assetStub.primaryImage.ownerId, { withHidden: true });
|
||||
expect(personMock.createAll).toHaveBeenCalledWith([]);
|
||||
expect(personMock.replaceFaces).toHaveBeenCalledWith(
|
||||
assetStub.primaryImage.id,
|
||||
expect(personMock.createAll).not.toHaveBeenCalled();
|
||||
expect(personMock.refreshFaces).toHaveBeenCalledWith(
|
||||
[
|
||||
{
|
||||
id: 'random-uuid',
|
||||
|
|
@ -1062,10 +1064,10 @@ describe(MetadataService.name, () => {
|
|||
sourceType: SourceType.EXIF,
|
||||
},
|
||||
],
|
||||
SourceType.EXIF,
|
||||
[],
|
||||
);
|
||||
expect(personMock.updateAll).toHaveBeenCalledWith([]);
|
||||
expect(jobMock.queueAll).toHaveBeenCalledWith([]);
|
||||
expect(personMock.updateAll).not.toHaveBeenCalled();
|
||||
expect(jobMock.queueAll).not.toHaveBeenCalledWith();
|
||||
});
|
||||
|
||||
it('should handle invalid modify date', async () => {
|
||||
|
|
|
|||
|
|
@ -178,7 +178,7 @@ export class MetadataService extends BaseService {
|
|||
|
||||
async handleMetadataExtraction({ id }: IEntityJob): Promise<JobStatus> {
|
||||
const { metadata, reverseGeocoding } = await this.getConfig({ withCache: true });
|
||||
const [asset] = await this.assetRepository.getByIds([id]);
|
||||
const [asset] = await this.assetRepository.getByIds([id], { faces: { person: false } });
|
||||
if (!asset) {
|
||||
return JobStatus.FAILED;
|
||||
}
|
||||
|
|
@ -513,7 +513,7 @@ export class MetadataService extends BaseService {
|
|||
return;
|
||||
}
|
||||
|
||||
const discoveredFaces: Partial<AssetFaceEntity>[] = [];
|
||||
const facesToAdd: Partial<AssetFaceEntity>[] = [];
|
||||
const existingNames = await this.personRepository.getDistinctNames(asset.ownerId, { withHidden: true });
|
||||
const existingNameMap = new Map(existingNames.map(({ id, name }) => [name.toLowerCase(), id]));
|
||||
const missing: Partial<PersonEntity>[] = [];
|
||||
|
|
@ -541,7 +541,7 @@ export class MetadataService extends BaseService {
|
|||
sourceType: SourceType.EXIF,
|
||||
};
|
||||
|
||||
discoveredFaces.push(face);
|
||||
facesToAdd.push(face);
|
||||
if (!existingNameMap.has(loweredName)) {
|
||||
missing.push({ id: personId, ownerId: asset.ownerId, name: region.Name });
|
||||
missingWithFaceAsset.push({ id: personId, faceAssetId: face.id });
|
||||
|
|
@ -550,18 +550,27 @@ export class MetadataService extends BaseService {
|
|||
|
||||
if (missing.length > 0) {
|
||||
this.logger.debug(`Creating missing persons: ${missing.map((p) => `${p.name}/${p.id}`)}`);
|
||||
const newPersonIds = await this.personRepository.createAll(missing);
|
||||
const jobs = newPersonIds.map((id) => ({ name: JobName.GENERATE_PERSON_THUMBNAIL, data: { id } }) as const);
|
||||
await this.jobRepository.queueAll(jobs);
|
||||
}
|
||||
|
||||
const newPersonIds = await this.personRepository.createAll(missing);
|
||||
const facesToRemove = asset.faces.filter((face) => face.sourceType === SourceType.EXIF).map((face) => face.id);
|
||||
if (facesToRemove.length > 0) {
|
||||
this.logger.debug(`Removing ${facesToRemove.length} faces for asset ${asset.id}`);
|
||||
}
|
||||
|
||||
const faceIds = await this.personRepository.replaceFaces(asset.id, discoveredFaces, SourceType.EXIF);
|
||||
this.logger.debug(`Created ${faceIds.length} faces for asset ${asset.id}`);
|
||||
if (facesToAdd.length > 0) {
|
||||
this.logger.debug(`Creating ${facesToAdd} faces from metadata for asset ${asset.id}`);
|
||||
}
|
||||
|
||||
await this.personRepository.updateAll(missingWithFaceAsset);
|
||||
if (facesToRemove.length > 0 || facesToAdd.length > 0) {
|
||||
await this.personRepository.refreshFaces(facesToAdd, facesToRemove);
|
||||
}
|
||||
|
||||
await this.jobRepository.queueAll(
|
||||
newPersonIds.map((id) => ({ name: JobName.GENERATE_PERSON_THUMBNAIL, data: { id } })),
|
||||
);
|
||||
if (missingWithFaceAsset.length > 0) {
|
||||
await this.personRepository.updateAll(missingWithFaceAsset);
|
||||
}
|
||||
}
|
||||
|
||||
private getDates(asset: AssetEntity, exifTags: ImmichTags) {
|
||||
|
|
|
|||
|
|
@ -35,21 +35,33 @@ const responseDto: PersonResponseDto = {
|
|||
|
||||
const statistics = { assets: 3 };
|
||||
|
||||
const faceId = 'face-id';
|
||||
const face = {
|
||||
id: faceId,
|
||||
assetId: 'asset-id',
|
||||
boundingBoxX1: 100,
|
||||
boundingBoxY1: 100,
|
||||
boundingBoxX2: 200,
|
||||
boundingBoxY2: 200,
|
||||
imageHeight: 500,
|
||||
imageWidth: 400,
|
||||
};
|
||||
const faceSearch = { faceId, embedding: [1, 2, 3, 4] };
|
||||
const detectFaceMock: DetectedFaces = {
|
||||
faces: [
|
||||
{
|
||||
boundingBox: {
|
||||
x1: 100,
|
||||
y1: 100,
|
||||
x2: 200,
|
||||
y2: 200,
|
||||
x1: face.boundingBoxX1,
|
||||
y1: face.boundingBoxY1,
|
||||
x2: face.boundingBoxX2,
|
||||
y2: face.boundingBoxY2,
|
||||
},
|
||||
embedding: [1, 2, 3, 4],
|
||||
embedding: faceSearch.embedding,
|
||||
score: 0.2,
|
||||
},
|
||||
],
|
||||
imageHeight: 500,
|
||||
imageWidth: 400,
|
||||
imageHeight: face.imageHeight,
|
||||
imageWidth: face.imageWidth,
|
||||
};
|
||||
|
||||
describe(PersonService.name, () => {
|
||||
|
|
@ -449,7 +461,7 @@ describe(PersonService.name, () => {
|
|||
hasNextPage: false,
|
||||
});
|
||||
|
||||
await sut.handleQueueDetectFaces({});
|
||||
await sut.handleQueueDetectFaces({ force: false });
|
||||
|
||||
expect(assetMock.getWithout).toHaveBeenCalledWith({ skip: 0, take: 1000 }, WithoutProperty.FACES);
|
||||
expect(jobMock.queueAll).toHaveBeenCalledWith([
|
||||
|
|
@ -465,14 +477,13 @@ describe(PersonService.name, () => {
|
|||
items: [assetStub.image],
|
||||
hasNextPage: false,
|
||||
});
|
||||
personMock.getAll.mockResolvedValue({
|
||||
items: [personStub.withName],
|
||||
hasNextPage: false,
|
||||
});
|
||||
personMock.getAllWithoutFaces.mockResolvedValue([]);
|
||||
personMock.getAllWithoutFaces.mockResolvedValue([personStub.withName]);
|
||||
|
||||
await sut.handleQueueDetectFaces({ force: true });
|
||||
|
||||
expect(personMock.deleteFaces).toHaveBeenCalledWith({ sourceType: SourceType.MACHINE_LEARNING });
|
||||
expect(personMock.delete).toHaveBeenCalledWith([personStub.withName]);
|
||||
expect(storageMock.unlink).toHaveBeenCalledWith(personStub.withName.thumbnailPath);
|
||||
expect(assetMock.getAll).toHaveBeenCalled();
|
||||
expect(jobMock.queueAll).toHaveBeenCalledWith([
|
||||
{
|
||||
|
|
@ -482,6 +493,27 @@ describe(PersonService.name, () => {
|
|||
]);
|
||||
});
|
||||
|
||||
it('should refresh all assets', async () => {
|
||||
assetMock.getAll.mockResolvedValue({
|
||||
items: [assetStub.image],
|
||||
hasNextPage: false,
|
||||
});
|
||||
|
||||
await sut.handleQueueDetectFaces({ force: undefined });
|
||||
|
||||
expect(personMock.delete).not.toHaveBeenCalled();
|
||||
expect(personMock.deleteFaces).not.toHaveBeenCalled();
|
||||
expect(storageMock.unlink).not.toHaveBeenCalled();
|
||||
expect(assetMock.getAll).toHaveBeenCalled();
|
||||
expect(jobMock.queueAll).toHaveBeenCalledWith([
|
||||
{
|
||||
name: JobName.FACE_DETECTION,
|
||||
data: { id: assetStub.image.id },
|
||||
},
|
||||
]);
|
||||
expect(jobMock.queue).toHaveBeenCalledWith({ name: JobName.PERSON_CLEANUP });
|
||||
});
|
||||
|
||||
it('should delete existing people and faces if forced', async () => {
|
||||
personMock.getAll.mockResolvedValue({
|
||||
items: [faceStub.face1.person, personStub.randomPerson],
|
||||
|
|
@ -542,7 +574,7 @@ describe(PersonService.name, () => {
|
|||
|
||||
expect(personMock.getAllFaces).toHaveBeenCalledWith(
|
||||
{ skip: 0, take: 1000 },
|
||||
{ where: { personId: IsNull(), sourceType: IsNull() } },
|
||||
{ where: { personId: IsNull(), sourceType: SourceType.MACHINE_LEARNING } },
|
||||
);
|
||||
expect(jobMock.queueAll).toHaveBeenCalledWith([
|
||||
{
|
||||
|
|
@ -663,6 +695,10 @@ describe(PersonService.name, () => {
|
|||
});
|
||||
|
||||
describe('handleDetectFaces', () => {
|
||||
beforeEach(() => {
|
||||
cryptoMock.randomUUID.mockReturnValue(faceId);
|
||||
});
|
||||
|
||||
it('should skip if machine learning is disabled', async () => {
|
||||
systemMock.get.mockResolvedValue(systemConfigStub.machineLearningDisabled);
|
||||
|
||||
|
|
@ -719,27 +755,73 @@ describe(PersonService.name, () => {
|
|||
it('should create a face with no person and queue recognition job', async () => {
|
||||
personMock.createFaces.mockResolvedValue([faceStub.face1.id]);
|
||||
machineLearningMock.detectFaces.mockResolvedValue(detectFaceMock);
|
||||
searchMock.searchFaces.mockResolvedValue([{ face: faceStub.face1, distance: 0.7 }]);
|
||||
assetMock.getByIds.mockResolvedValue([assetStub.image]);
|
||||
const faceId = 'face-id';
|
||||
cryptoMock.randomUUID.mockReturnValue(faceId);
|
||||
const face = {
|
||||
id: faceId,
|
||||
assetId: 'asset-id',
|
||||
boundingBoxX1: 100,
|
||||
boundingBoxY1: 100,
|
||||
boundingBoxX2: 200,
|
||||
boundingBoxY2: 200,
|
||||
imageHeight: 500,
|
||||
imageWidth: 400,
|
||||
faceSearch: { faceId, embedding: [1, 2, 3, 4] },
|
||||
};
|
||||
|
||||
await sut.handleDetectFaces({ id: assetStub.image.id });
|
||||
|
||||
expect(personMock.createFaces).toHaveBeenCalledWith([face]);
|
||||
expect(personMock.refreshFaces).toHaveBeenCalledWith([face], [], [faceSearch]);
|
||||
expect(jobMock.queueAll).toHaveBeenCalledWith([
|
||||
{ name: JobName.FACIAL_RECOGNITION, data: { id: faceStub.face1.id } },
|
||||
{ name: JobName.QUEUE_FACIAL_RECOGNITION, data: { force: false } },
|
||||
{ name: JobName.FACIAL_RECOGNITION, data: { id: faceId } },
|
||||
]);
|
||||
expect(personMock.reassignFace).not.toHaveBeenCalled();
|
||||
expect(personMock.reassignFaces).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should delete an existing face not among the new detected faces', async () => {
|
||||
machineLearningMock.detectFaces.mockResolvedValue({ faces: [], imageHeight: 500, imageWidth: 400 });
|
||||
assetMock.getByIds.mockResolvedValue([{ ...assetStub.image, faces: [faceStub.primaryFace1] }]);
|
||||
|
||||
await sut.handleDetectFaces({ id: assetStub.image.id });
|
||||
|
||||
expect(personMock.refreshFaces).toHaveBeenCalledWith([], [faceStub.primaryFace1.id], []);
|
||||
expect(jobMock.queueAll).not.toHaveBeenCalled();
|
||||
expect(personMock.reassignFace).not.toHaveBeenCalled();
|
||||
expect(personMock.reassignFaces).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should add new face and delete an existing face not among the new detected faces', async () => {
|
||||
personMock.createFaces.mockResolvedValue([faceStub.face1.id]);
|
||||
machineLearningMock.detectFaces.mockResolvedValue(detectFaceMock);
|
||||
assetMock.getByIds.mockResolvedValue([{ ...assetStub.image, faces: [faceStub.primaryFace1] }]);
|
||||
|
||||
await sut.handleDetectFaces({ id: assetStub.image.id });
|
||||
|
||||
expect(personMock.refreshFaces).toHaveBeenCalledWith([face], [faceStub.primaryFace1.id], [faceSearch]);
|
||||
expect(jobMock.queueAll).toHaveBeenCalledWith([
|
||||
{ name: JobName.QUEUE_FACIAL_RECOGNITION, data: { force: false } },
|
||||
{ name: JobName.FACIAL_RECOGNITION, data: { id: faceId } },
|
||||
]);
|
||||
expect(personMock.reassignFace).not.toHaveBeenCalled();
|
||||
expect(personMock.reassignFaces).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should add embedding to matching metadata face', async () => {
|
||||
machineLearningMock.detectFaces.mockResolvedValue(detectFaceMock);
|
||||
assetMock.getByIds.mockResolvedValue([{ ...assetStub.image, faces: [faceStub.fromExif1] }]);
|
||||
|
||||
await sut.handleDetectFaces({ id: assetStub.image.id });
|
||||
|
||||
expect(personMock.refreshFaces).toHaveBeenCalledWith(
|
||||
[],
|
||||
[],
|
||||
[{ faceId: faceStub.fromExif1.id, embedding: faceSearch.embedding }],
|
||||
);
|
||||
expect(jobMock.queueAll).not.toHaveBeenCalled();
|
||||
expect(personMock.reassignFace).not.toHaveBeenCalled();
|
||||
expect(personMock.reassignFaces).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should not add embedding to non-matching metadata face', async () => {
|
||||
machineLearningMock.detectFaces.mockResolvedValue(detectFaceMock);
|
||||
assetMock.getByIds.mockResolvedValue([{ ...assetStub.image, faces: [faceStub.fromExif2] }]);
|
||||
|
||||
await sut.handleDetectFaces({ id: assetStub.image.id });
|
||||
|
||||
expect(personMock.refreshFaces).toHaveBeenCalledWith([face], [], [faceSearch]);
|
||||
expect(jobMock.queueAll).toHaveBeenCalledWith([
|
||||
{ name: JobName.QUEUE_FACIAL_RECOGNITION, data: { force: false } },
|
||||
{ name: JobName.FACIAL_RECOGNITION, data: { id: faceId } },
|
||||
]);
|
||||
expect(personMock.reassignFace).not.toHaveBeenCalled();
|
||||
expect(personMock.reassignFaces).not.toHaveBeenCalled();
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ import {
|
|||
} from 'src/dtos/person.dto';
|
||||
import { AssetFaceEntity } from 'src/entities/asset-face.entity';
|
||||
import { AssetEntity } from 'src/entities/asset.entity';
|
||||
import { FaceSearchEntity } from 'src/entities/face-search.entity';
|
||||
import { PersonEntity } from 'src/entities/person.entity';
|
||||
import {
|
||||
AssetType,
|
||||
|
|
@ -256,14 +257,14 @@ export class PersonService extends BaseService {
|
|||
}
|
||||
|
||||
const assetPagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) => {
|
||||
return force
|
||||
? this.assetRepository.getAll(pagination, {
|
||||
return force === false
|
||||
? this.assetRepository.getWithout(pagination, WithoutProperty.FACES)
|
||||
: this.assetRepository.getAll(pagination, {
|
||||
orderDirection: 'DESC',
|
||||
withFaces: true,
|
||||
withArchived: true,
|
||||
isVisible: true,
|
||||
})
|
||||
: this.assetRepository.getWithout(pagination, WithoutProperty.FACES);
|
||||
});
|
||||
});
|
||||
|
||||
for await (const assets of assetPagination) {
|
||||
|
|
@ -272,6 +273,10 @@ export class PersonService extends BaseService {
|
|||
);
|
||||
}
|
||||
|
||||
if (force === undefined) {
|
||||
await this.jobRepository.queue({ name: JobName.PERSON_CLEANUP });
|
||||
}
|
||||
|
||||
return JobStatus.SUCCESS;
|
||||
}
|
||||
|
||||
|
|
@ -290,11 +295,11 @@ export class PersonService extends BaseService {
|
|||
};
|
||||
const [asset] = await this.assetRepository.getByIds([id], relations);
|
||||
const { previewFile } = getAssetFiles(asset.files);
|
||||
if (!asset || !previewFile || asset.faces?.length > 0) {
|
||||
if (!asset || !previewFile) {
|
||||
return JobStatus.FAILED;
|
||||
}
|
||||
|
||||
if (!asset.isVisible || asset.faces.length > 0) {
|
||||
if (!asset.isVisible) {
|
||||
return JobStatus.SKIPPED;
|
||||
}
|
||||
|
||||
|
|
@ -303,39 +308,82 @@ export class PersonService extends BaseService {
|
|||
previewFile.path,
|
||||
machineLearning.facialRecognition,
|
||||
);
|
||||
|
||||
this.logger.debug(`${faces.length} faces detected in ${previewFile.path}`);
|
||||
|
||||
if (faces.length > 0) {
|
||||
await this.jobRepository.queue({ name: JobName.QUEUE_FACIAL_RECOGNITION, data: { force: false } });
|
||||
const mappedFaces: Partial<AssetFaceEntity>[] = [];
|
||||
for (const face of faces) {
|
||||
const facesToAdd: (Partial<AssetFaceEntity> & { id: string })[] = [];
|
||||
const embeddings: FaceSearchEntity[] = [];
|
||||
const mlFaceIds = new Set<string>();
|
||||
for (const face of asset.faces) {
|
||||
if (face.sourceType === SourceType.MACHINE_LEARNING) {
|
||||
mlFaceIds.add(face.id);
|
||||
}
|
||||
}
|
||||
|
||||
const heightScale = imageHeight / (asset.faces[0]?.imageHeight || 1);
|
||||
const widthScale = imageWidth / (asset.faces[0]?.imageWidth || 1);
|
||||
for (const { boundingBox, embedding } of faces) {
|
||||
const scaledBox = {
|
||||
x1: boundingBox.x1 * widthScale,
|
||||
y1: boundingBox.y1 * heightScale,
|
||||
x2: boundingBox.x2 * widthScale,
|
||||
y2: boundingBox.y2 * heightScale,
|
||||
};
|
||||
const match = asset.faces.find((face) => this.iou(face, scaledBox) > 0.5);
|
||||
|
||||
if (match && !mlFaceIds.delete(match.id)) {
|
||||
embeddings.push({ faceId: match.id, embedding });
|
||||
} else {
|
||||
const faceId = this.cryptoRepository.randomUUID();
|
||||
mappedFaces.push({
|
||||
facesToAdd.push({
|
||||
id: faceId,
|
||||
assetId: asset.id,
|
||||
imageHeight,
|
||||
imageWidth,
|
||||
boundingBoxX1: face.boundingBox.x1,
|
||||
boundingBoxY1: face.boundingBox.y1,
|
||||
boundingBoxX2: face.boundingBox.x2,
|
||||
boundingBoxY2: face.boundingBox.y2,
|
||||
faceSearch: { faceId, embedding: face.embedding },
|
||||
boundingBoxX1: boundingBox.x1,
|
||||
boundingBoxY1: boundingBox.y1,
|
||||
boundingBoxX2: boundingBox.x2,
|
||||
boundingBoxY2: boundingBox.y2,
|
||||
});
|
||||
embeddings.push({ faceId, embedding });
|
||||
}
|
||||
}
|
||||
const faceIdsToRemove = [...mlFaceIds];
|
||||
|
||||
const faceIds = await this.personRepository.createFaces(mappedFaces);
|
||||
await this.jobRepository.queueAll(faceIds.map((id) => ({ name: JobName.FACIAL_RECOGNITION, data: { id } })));
|
||||
if (facesToAdd.length > 0 || faceIdsToRemove.length > 0 || embeddings.length > 0) {
|
||||
await this.personRepository.refreshFaces(facesToAdd, faceIdsToRemove, embeddings);
|
||||
}
|
||||
|
||||
await this.assetRepository.upsertJobStatus({
|
||||
assetId: asset.id,
|
||||
facesRecognizedAt: new Date(),
|
||||
});
|
||||
if (faceIdsToRemove.length > 0) {
|
||||
this.logger.log(`Removed ${faceIdsToRemove.length} faces below detection threshold in asset ${id}`);
|
||||
}
|
||||
|
||||
if (facesToAdd.length > 0) {
|
||||
this.logger.log(`Detected ${facesToAdd.length} new faces in asset ${id}`);
|
||||
const jobs = facesToAdd.map((face) => ({ name: JobName.FACIAL_RECOGNITION, data: { id: face.id } }) as const);
|
||||
await this.jobRepository.queueAll([{ name: JobName.QUEUE_FACIAL_RECOGNITION, data: { force: false } }, ...jobs]);
|
||||
} else if (embeddings.length > 0) {
|
||||
this.logger.log(`Added ${embeddings.length} face embeddings for asset ${id}`);
|
||||
}
|
||||
|
||||
await this.assetRepository.upsertJobStatus({ assetId: asset.id, facesRecognizedAt: new Date() });
|
||||
|
||||
return JobStatus.SUCCESS;
|
||||
}
|
||||
|
||||
private iou(face: AssetFaceEntity, newBox: BoundingBox): number {
|
||||
const x1 = Math.max(face.boundingBoxX1, newBox.x1);
|
||||
const y1 = Math.max(face.boundingBoxY1, newBox.y1);
|
||||
const x2 = Math.min(face.boundingBoxX2, newBox.x2);
|
||||
const y2 = Math.min(face.boundingBoxY2, newBox.y2);
|
||||
|
||||
const intersection = Math.max(0, x2 - x1) * Math.max(0, y2 - y1);
|
||||
const area1 = (face.boundingBoxX2 - face.boundingBoxX1) * (face.boundingBoxY2 - face.boundingBoxY1);
|
||||
const area2 = (newBox.x2 - newBox.x1) * (newBox.y2 - newBox.y1);
|
||||
const union = area1 + area2 - intersection;
|
||||
|
||||
return intersection / union;
|
||||
}
|
||||
|
||||
async handleQueueRecognizeFaces({ force, nightly }: INightlyJob): Promise<JobStatus> {
|
||||
const { machineLearning } = await this.getConfig({ withCache: false });
|
||||
if (!isFacialRecognitionEnabled(machineLearning)) {
|
||||
|
|
@ -371,7 +419,7 @@ export class PersonService extends BaseService {
|
|||
const lastRun = new Date().toISOString();
|
||||
const facePagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) =>
|
||||
this.personRepository.getAllFaces(pagination, {
|
||||
where: force ? undefined : { personId: IsNull(), sourceType: IsNull() },
|
||||
where: force ? undefined : { personId: IsNull(), sourceType: SourceType.MACHINE_LEARNING },
|
||||
}),
|
||||
);
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue