immich/server/src/infra/infra.utils.ts

192 lines
5.5 KiB
TypeScript
Raw Normal View History

import _ from 'lodash';
import { AssetEntity } from 'src/infra/entities/asset.entity';
import { AssetSearchBuilderOptions } from 'src/interfaces/search.repository';
import { Paginated, PaginatedBuilderOptions, PaginationMode, PaginationOptions, PaginationResult } from 'src/utils';
import {
Between,
FindManyOptions,
IsNull,
LessThanOrEqual,
MoreThanOrEqual,
Not,
ObjectLiteral,
Repository,
SelectQueryBuilder,
} from 'typeorm';
2023-12-08 11:15:46 -05:00
/**
* Allows optional values unlike the regular Between and uses MoreThanOrEqual
* or LessThanOrEqual when only one parameter is specified.
*/
export function OptionalBetween<T>(from?: T, to?: T) {
if (from && to) {
return Between(from, to);
} else if (from) {
return MoreThanOrEqual(from);
} else if (to) {
return LessThanOrEqual(to);
}
}
function paginationHelper<Entity extends ObjectLiteral>(items: Entity[], take: number): PaginationResult<Entity> {
const hasNextPage = items.length > take;
items.splice(take);
return { items, hasNextPage };
}
2023-12-08 11:15:46 -05:00
export async function paginate<Entity extends ObjectLiteral>(
repository: Repository<Entity>,
{ take, skip }: PaginationOptions,
feat(server): separate face clustering job (#5598) * separate facial clustering job * update api * fixed some tests * invert clustering * hdbscan * update api * remove commented code * wip dbscan * cleanup removed cluster endpoint remove commented code * fixes updated tests minor fixes and formatting fixed queuing refinements * scale search range based on library size * defer non-core faces * optimizations removed unused query option * assign faces individually for correctness fixed unit tests remove unused method * don't select face embedding update sql linting fixed ml typing * updated job mock * paginate people query * select face embeddings because typeorm * fix setting face detection concurrency * update sql formatting linting * simplify logic remove unused imports * more specific delete signature * more accurate typing for face stubs * add migration formatting * chore: better typing * don't select embedding by default remove unused import * updated sql * use normal try/catch * stricter concurrency typing and enforcement * update api * update job concurrency panel to show disabled queues formatting * check jobId in queueAll fix tests * remove outdated comment * better facial recognition icon * wording wording formatting * fixed tests * fix * formatting & sql * try to fix sql check * more detailed description * update sql * formatting * wording * update `minFaces` description --------- Co-authored-by: Jason Rasmussen <jrasm91@gmail.com> Co-authored-by: Alex Tran <alex.tran1502@gmail.com>
2024-01-18 00:08:48 -05:00
searchOptions?: FindManyOptions<Entity>,
2023-12-08 11:15:46 -05:00
): Paginated<Entity> {
feat(server): separate face clustering job (#5598) * separate facial clustering job * update api * fixed some tests * invert clustering * hdbscan * update api * remove commented code * wip dbscan * cleanup removed cluster endpoint remove commented code * fixes updated tests minor fixes and formatting fixed queuing refinements * scale search range based on library size * defer non-core faces * optimizations removed unused query option * assign faces individually for correctness fixed unit tests remove unused method * don't select face embedding update sql linting fixed ml typing * updated job mock * paginate people query * select face embeddings because typeorm * fix setting face detection concurrency * update sql formatting linting * simplify logic remove unused imports * more specific delete signature * more accurate typing for face stubs * add migration formatting * chore: better typing * don't select embedding by default remove unused import * updated sql * use normal try/catch * stricter concurrency typing and enforcement * update api * update job concurrency panel to show disabled queues formatting * check jobId in queueAll fix tests * remove outdated comment * better facial recognition icon * wording wording formatting * fixed tests * fix * formatting & sql * try to fix sql check * more detailed description * update sql * formatting * wording * update `minFaces` description --------- Co-authored-by: Jason Rasmussen <jrasm91@gmail.com> Co-authored-by: Alex Tran <alex.tran1502@gmail.com>
2024-01-18 00:08:48 -05:00
const items = await repository.find(
_.omitBy(
{
...searchOptions,
// Take one more item to check if there's a next page
take: take + 1,
skip,
feat(server): separate face clustering job (#5598) * separate facial clustering job * update api * fixed some tests * invert clustering * hdbscan * update api * remove commented code * wip dbscan * cleanup removed cluster endpoint remove commented code * fixes updated tests minor fixes and formatting fixed queuing refinements * scale search range based on library size * defer non-core faces * optimizations removed unused query option * assign faces individually for correctness fixed unit tests remove unused method * don't select face embedding update sql linting fixed ml typing * updated job mock * paginate people query * select face embeddings because typeorm * fix setting face detection concurrency * update sql formatting linting * simplify logic remove unused imports * more specific delete signature * more accurate typing for face stubs * add migration formatting * chore: better typing * don't select embedding by default remove unused import * updated sql * use normal try/catch * stricter concurrency typing and enforcement * update api * update job concurrency panel to show disabled queues formatting * check jobId in queueAll fix tests * remove outdated comment * better facial recognition icon * wording wording formatting * fixed tests * fix * formatting & sql * try to fix sql check * more detailed description * update sql * formatting * wording * update `minFaces` description --------- Co-authored-by: Jason Rasmussen <jrasm91@gmail.com> Co-authored-by: Alex Tran <alex.tran1502@gmail.com>
2024-01-18 00:08:48 -05:00
},
_.isUndefined,
),
);
2023-12-08 11:15:46 -05:00
return paginationHelper(items, take);
}
2023-12-08 11:15:46 -05:00
export async function paginatedBuilder<Entity extends ObjectLiteral>(
qb: SelectQueryBuilder<Entity>,
{ take, skip, mode }: PaginatedBuilderOptions,
): Paginated<Entity> {
if (mode === PaginationMode.LIMIT_OFFSET) {
qb.limit(take + 1).offset(skip);
} else {
qb.take(take + 1).skip(skip);
}
const items = await qb.getMany();
return paginationHelper(items, take);
2023-12-08 11:15:46 -05:00
}
export const asVector = (embedding: number[], quote = false) =>
quote ? `'[${embedding.join(',')}]'` : `[${embedding.join(',')}]`;
export function searchAssetBuilder(
builder: SelectQueryBuilder<AssetEntity>,
options: AssetSearchBuilderOptions,
): SelectQueryBuilder<AssetEntity> {
builder.andWhere(
_.omitBy(
{
createdAt: OptionalBetween(options.createdAfter, options.createdBefore),
updatedAt: OptionalBetween(options.updatedAfter, options.updatedBefore),
deletedAt: OptionalBetween(options.trashedAfter, options.trashedBefore),
fileCreatedAt: OptionalBetween(options.takenAfter, options.takenBefore),
},
_.isUndefined,
),
);
const exifInfo = _.omitBy(_.pick(options, ['city', 'country', 'lensModel', 'make', 'model', 'state']), _.isUndefined);
const hasExifQuery = Object.keys(exifInfo).length > 0;
if (options.withExif && !hasExifQuery) {
builder.leftJoinAndSelect(`${builder.alias}.exifInfo`, 'exifInfo');
}
if (hasExifQuery) {
options.withExif
? builder.leftJoinAndSelect(`${builder.alias}.exifInfo`, 'exifInfo')
: builder.leftJoin(`${builder.alias}.exifInfo`, 'exifInfo');
builder.andWhere({ exifInfo });
}
const id = _.pick(options, ['checksum', 'deviceAssetId', 'deviceId', 'id', 'libraryId']);
builder.andWhere(_.omitBy(id, _.isUndefined));
if (options.userIds) {
builder.andWhere(`${builder.alias}.ownerId IN (:...userIds)`, { userIds: options.userIds });
}
const path = _.pick(options, ['encodedVideoPath', 'originalPath', 'resizePath', 'webpPath']);
builder.andWhere(_.omitBy(path, _.isUndefined));
if (options.originalFileName) {
builder.andWhere(`f_unaccent(${builder.alias}.originalFileName) ILIKE f_unaccent(:originalFileName)`, {
originalFileName: `%${options.originalFileName}%`,
});
}
const status = _.pick(options, ['isExternal', 'isFavorite', 'isOffline', 'isReadOnly', 'isVisible', 'type']);
const {
isArchived,
isEncoded,
isMotion,
withArchived,
isNotInAlbum,
withFaces,
withPeople,
withSmartInfo,
personIds,
withExif,
withStacked,
trashedAfter,
trashedBefore,
} = options;
builder.andWhere(
_.omitBy(
{
...status,
isArchived: isArchived ?? (withArchived ? undefined : false),
encodedVideoPath: isEncoded ? Not(IsNull()) : undefined,
livePhotoVideoId: isMotion ? Not(IsNull()) : undefined,
},
_.isUndefined,
),
);
if (isNotInAlbum) {
builder
.leftJoin(`${builder.alias}.albums`, 'albums')
.andWhere('albums.id IS NULL')
.andWhere(`${builder.alias}.isVisible = true`);
}
if (withFaces || withPeople) {
builder.leftJoinAndSelect(`${builder.alias}.faces`, 'faces');
}
if (withPeople) {
builder.leftJoinAndSelect(`${builder.alias}.person`, 'person');
}
if (withSmartInfo) {
builder.leftJoinAndSelect(`${builder.alias}.smartInfo`, 'smartInfo');
}
if (personIds && personIds.length > 0) {
builder
.leftJoin(`${builder.alias}.faces`, 'faces')
.andWhere('faces.personId IN (:...personIds)', { personIds })
.addGroupBy(`${builder.alias}.id`)
.having('COUNT(DISTINCT faces.personId) = :personCount', { personCount: personIds.length });
if (withExif) {
builder.addGroupBy('exifInfo.assetId');
}
}
if (withStacked) {
builder.leftJoinAndSelect(`${builder.alias}.stack`, 'stack').leftJoinAndSelect('stack.assets', 'stackedAssets');
}
const withDeleted = options.withDeleted ?? (trashedAfter !== undefined || trashedBefore !== undefined);
if (withDeleted) {
builder.withDeleted();
}
return builder;
}