immich/server/src/infra/infra.utils.ts

200 lines
6.6 KiB
TypeScript
Raw Normal View History

import { AssetSearchBuilderOptions, Paginated, PaginationOptions } from '@app/domain';
import _ from 'lodash';
import {
Between,
Brackets,
FindManyOptions,
IsNull,
LessThanOrEqual,
MoreThanOrEqual,
Not,
ObjectLiteral,
Repository,
SelectQueryBuilder,
} from 'typeorm';
import { PaginatedBuilderOptions, PaginationMode, PaginationResult, chunks, setUnion } from '../domain/domain.util';
import { AssetEntity } from './entities';
import { DATABASE_PARAMETER_CHUNK_SIZE } from './infra.util';
2023-12-08 11:15:46 -05:00
/**
* Allows optional values unlike the regular Between and uses MoreThanOrEqual
* or LessThanOrEqual when only one parameter is specified.
*/
export function OptionalBetween<T>(from?: T, to?: T) {
if (from && to) {
return Between(from, to);
} else if (from) {
return MoreThanOrEqual(from);
} else if (to) {
return LessThanOrEqual(to);
}
}
export const isValidInteger = (value: number, options: { min?: number; max?: number }): value is number => {
const { min = Number.MIN_SAFE_INTEGER, max = Number.MAX_SAFE_INTEGER } = options;
return Number.isInteger(value) && value >= min && value <= max;
};
function paginationHelper<Entity extends ObjectLiteral>(items: Entity[], take: number): PaginationResult<Entity> {
const hasNextPage = items.length > take;
items.splice(take);
return { items, hasNextPage };
}
2023-12-08 11:15:46 -05:00
export async function paginate<Entity extends ObjectLiteral>(
repository: Repository<Entity>,
{ take, skip }: PaginationOptions,
feat(server): separate face clustering job (#5598) * separate facial clustering job * update api * fixed some tests * invert clustering * hdbscan * update api * remove commented code * wip dbscan * cleanup removed cluster endpoint remove commented code * fixes updated tests minor fixes and formatting fixed queuing refinements * scale search range based on library size * defer non-core faces * optimizations removed unused query option * assign faces individually for correctness fixed unit tests remove unused method * don't select face embedding update sql linting fixed ml typing * updated job mock * paginate people query * select face embeddings because typeorm * fix setting face detection concurrency * update sql formatting linting * simplify logic remove unused imports * more specific delete signature * more accurate typing for face stubs * add migration formatting * chore: better typing * don't select embedding by default remove unused import * updated sql * use normal try/catch * stricter concurrency typing and enforcement * update api * update job concurrency panel to show disabled queues formatting * check jobId in queueAll fix tests * remove outdated comment * better facial recognition icon * wording wording formatting * fixed tests * fix * formatting & sql * try to fix sql check * more detailed description * update sql * formatting * wording * update `minFaces` description --------- Co-authored-by: Jason Rasmussen <jrasm91@gmail.com> Co-authored-by: Alex Tran <alex.tran1502@gmail.com>
2024-01-18 00:08:48 -05:00
searchOptions?: FindManyOptions<Entity>,
2023-12-08 11:15:46 -05:00
): Paginated<Entity> {
feat(server): separate face clustering job (#5598) * separate facial clustering job * update api * fixed some tests * invert clustering * hdbscan * update api * remove commented code * wip dbscan * cleanup removed cluster endpoint remove commented code * fixes updated tests minor fixes and formatting fixed queuing refinements * scale search range based on library size * defer non-core faces * optimizations removed unused query option * assign faces individually for correctness fixed unit tests remove unused method * don't select face embedding update sql linting fixed ml typing * updated job mock * paginate people query * select face embeddings because typeorm * fix setting face detection concurrency * update sql formatting linting * simplify logic remove unused imports * more specific delete signature * more accurate typing for face stubs * add migration formatting * chore: better typing * don't select embedding by default remove unused import * updated sql * use normal try/catch * stricter concurrency typing and enforcement * update api * update job concurrency panel to show disabled queues formatting * check jobId in queueAll fix tests * remove outdated comment * better facial recognition icon * wording wording formatting * fixed tests * fix * formatting & sql * try to fix sql check * more detailed description * update sql * formatting * wording * update `minFaces` description --------- Co-authored-by: Jason Rasmussen <jrasm91@gmail.com> Co-authored-by: Alex Tran <alex.tran1502@gmail.com>
2024-01-18 00:08:48 -05:00
const items = await repository.find(
_.omitBy(
{
...searchOptions,
// Take one more item to check if there's a next page
take: take + 1,
skip,
feat(server): separate face clustering job (#5598) * separate facial clustering job * update api * fixed some tests * invert clustering * hdbscan * update api * remove commented code * wip dbscan * cleanup removed cluster endpoint remove commented code * fixes updated tests minor fixes and formatting fixed queuing refinements * scale search range based on library size * defer non-core faces * optimizations removed unused query option * assign faces individually for correctness fixed unit tests remove unused method * don't select face embedding update sql linting fixed ml typing * updated job mock * paginate people query * select face embeddings because typeorm * fix setting face detection concurrency * update sql formatting linting * simplify logic remove unused imports * more specific delete signature * more accurate typing for face stubs * add migration formatting * chore: better typing * don't select embedding by default remove unused import * updated sql * use normal try/catch * stricter concurrency typing and enforcement * update api * update job concurrency panel to show disabled queues formatting * check jobId in queueAll fix tests * remove outdated comment * better facial recognition icon * wording wording formatting * fixed tests * fix * formatting & sql * try to fix sql check * more detailed description * update sql * formatting * wording * update `minFaces` description --------- Co-authored-by: Jason Rasmussen <jrasm91@gmail.com> Co-authored-by: Alex Tran <alex.tran1502@gmail.com>
2024-01-18 00:08:48 -05:00
},
_.isUndefined,
),
);
2023-12-08 11:15:46 -05:00
return paginationHelper(items, take);
}
2023-12-08 11:15:46 -05:00
export async function paginatedBuilder<Entity extends ObjectLiteral>(
qb: SelectQueryBuilder<Entity>,
{ take, skip, mode }: PaginatedBuilderOptions,
): Paginated<Entity> {
if (mode === PaginationMode.LIMIT_OFFSET) {
qb.limit(take + 1).offset(skip);
} else {
qb.take(take + 1).skip(skip);
}
const items = await qb.getMany();
return paginationHelper(items, take);
2023-12-08 11:15:46 -05:00
}
export const asVector = (embedding: number[], quote = false) =>
quote ? `'[${embedding.join(',')}]'` : `[${embedding.join(',')}]`;
/**
* Wraps a method that takes a collection of parameters and sequentially calls it with chunks of the collection,
* to overcome the maximum number of parameters allowed by the database driver.
*
* @param options.paramIndex The index of the function parameter to chunk. Defaults to 0.
* @param options.flatten Whether to flatten the results. Defaults to false.
*/
export function Chunked(options: { paramIndex?: number; mergeFn?: (results: any) => any } = {}): MethodDecorator {
return (target: any, propertyKey: string | symbol, descriptor: PropertyDescriptor) => {
const originalMethod = descriptor.value;
const parameterIndex = options.paramIndex ?? 0;
descriptor.value = async function (...arguments_: any[]) {
const argument = arguments_[parameterIndex];
// Early return if argument length is less than or equal to the chunk size.
if (
(Array.isArray(argument) && argument.length <= DATABASE_PARAMETER_CHUNK_SIZE) ||
(argument instanceof Set && argument.size <= DATABASE_PARAMETER_CHUNK_SIZE)
) {
return await originalMethod.apply(this, arguments_);
}
return Promise.all(
chunks(argument, DATABASE_PARAMETER_CHUNK_SIZE).map(async (chunk) => {
await Reflect.apply(originalMethod, this, [
...arguments_.slice(0, parameterIndex),
chunk,
...arguments_.slice(parameterIndex + 1),
]);
}),
).then((results) => (options.mergeFn ? options.mergeFn(results) : results));
};
};
}
export function ChunkedArray(options?: { paramIndex?: number }): MethodDecorator {
return Chunked({ ...options, mergeFn: _.flatten });
}
export function ChunkedSet(options?: { paramIndex?: number }): MethodDecorator {
return Chunked({ ...options, mergeFn: setUnion });
}
export function searchAssetBuilder(
builder: SelectQueryBuilder<AssetEntity>,
options: AssetSearchBuilderOptions,
): SelectQueryBuilder<AssetEntity> {
builder.andWhere(
_.omitBy(
{
createdAt: OptionalBetween(options.createdAfter, options.createdBefore),
updatedAt: OptionalBetween(options.updatedAfter, options.updatedBefore),
deletedAt: OptionalBetween(options.trashedAfter, options.trashedBefore),
fileCreatedAt: OptionalBetween(options.takenAfter, options.takenBefore),
},
_.isUndefined,
),
);
const exifInfo = _.omitBy(_.pick(options, ['city', 'country', 'lensModel', 'make', 'model', 'state']), _.isUndefined);
if (Object.keys(exifInfo).length > 0) {
builder.leftJoin(`${builder.alias}.exifInfo`, 'exifInfo');
builder.andWhere({ exifInfo });
}
const id = _.pick(options, ['checksum', 'deviceAssetId', 'deviceId', 'id', 'libraryId', 'ownerId']);
builder.andWhere(_.omitBy(id, _.isUndefined));
const path = _.pick(options, ['encodedVideoPath', 'originalFileName', 'originalPath', 'resizePath', 'webpPath']);
builder.andWhere(_.omitBy(path, _.isUndefined));
const status = _.pick(options, ['isExternal', 'isFavorite', 'isOffline', 'isReadOnly', 'isVisible', 'type']);
const { isArchived, isEncoded, isMotion, withArchived } = options;
builder.andWhere(
_.omitBy(
{
...status,
isArchived: isArchived ?? withArchived,
encodedVideoPath: isEncoded ? Not(IsNull()) : undefined,
livePhotoVideoId: isMotion ? Not(IsNull()) : undefined,
},
_.isUndefined,
),
);
if (options.withExif) {
builder.leftJoinAndSelect(`${builder.alias}.exifInfo`, 'exifInfo');
}
if (options.withFaces || options.withPeople) {
builder.leftJoinAndSelect(`${builder.alias}.faces`, 'faces');
}
if (options.withPeople) {
builder.leftJoinAndSelect(`${builder.alias}.person`, 'person');
}
if (options.withSmartInfo) {
builder.leftJoinAndSelect(`${builder.alias}.smartInfo`, 'smartInfo');
}
if (options.withStacked) {
builder
.leftJoinAndSelect(`${builder.alias}.stack`, 'stack')
.leftJoinAndSelect('stack.assets', 'stackedAssets')
.andWhere(
new Brackets((qb) => qb.where(`stack.primaryAssetId = ${builder.alias}.id`).orWhere('asset.stackId IS NULL')),
);
}
const withDeleted =
options.withDeleted ?? (options.trashedAfter !== undefined || options.trashedBefore !== undefined);
if (withDeleted) {
builder.withDeleted();
}
return builder;
}