helpy4/immich
1
0
Fork 0
mirror of https://github.com/immich-app/immich synced 2025-11-07 17:27:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat(server, web): smart search filtering and pagination (#6525)

Browse source 
* initial pagination impl

* use limit + offset instead of take + skip

* wip web pagination

* working infinite scroll

* update api

* formatting

* fix rebase

* search refactor

* re-add runtime config for vector search

* fix rebase

* fixes

* useless omitBy

* unnecessary handling

* add sql decorator for `searchAssets`

* fixed search builder

* fixed sql

* remove mock method

* linting

* fixed pagination

* fixed unit tests

* formatting

* fix e2e tests

* re-flatten search builder

* refactor endpoints

* clean up dto

* refinements

* don't break everything just yet

* update openapi spec & sql

* update api

* linting

* update sql

* fixes

* optimize web code

* fix typing

* add page limit

* make limit based on asset count

* increase limit

* simpler import
This commit is contained in:
Mert 2024-02-12 20:50:47 -05:00 • committed by GitHub
parent f1e4fdf175
commit e334443919
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
54 changed files with 3993 additions and 790 deletions
Download patch file Download diff file Expand all files Collapse all files

134
server/src/infra/infra.utils.ts
View file

@ -1,7 +1,19 @@
import { Paginated, PaginationOptions } from '@app/domain';
import { AssetSearchBuilderOptions, Paginated, PaginationOptions } from '@app/domain';
import _ from 'lodash';
import { Between, FindManyOptions, LessThanOrEqual, MoreThanOrEqual, ObjectLiteral, Repository } from 'typeorm';
import { chunks, setUnion } from '../domain/domain.util';
import {
Between,
Brackets,
FindManyOptions,
IsNull,
LessThanOrEqual,
MoreThanOrEqual,
Not,
ObjectLiteral,
Repository,
SelectQueryBuilder,
} from 'typeorm';
import { PaginatedBuilderOptions, PaginationMode, PaginationResult, chunks, setUnion } from '../domain/domain.util';
import { AssetEntity } from './entities';
import { DATABASE_PARAMETER_CHUNK_SIZE } from './infra.util';
/**
@ -18,9 +30,21 @@ export function OptionalBetween<T>(from?: T, to?: T) {
}
}
export const isValidInteger = (value: number, options: { min?: number; max?: number }): value is number => {
const { min = Number.MIN_SAFE_INTEGER, max = Number.MAX_SAFE_INTEGER } = options;
return Number.isInteger(value) && value >= min && value <= max;
};
function paginationHelper<Entity extends ObjectLiteral>(items: Entity[], take: number): PaginationResult<Entity> {
const hasNextPage = items.length > take;
items.splice(take);
return { items, hasNextPage };
}
export async function paginate<Entity extends ObjectLiteral>(
repository: Repository<Entity>,
paginationOptions: PaginationOptions,
{ take, skip }: PaginationOptions,
searchOptions?: FindManyOptions<Entity>,
): Paginated<Entity> {
const items = await repository.find(
@ -28,27 +52,33 @@ export async function paginate<Entity extends ObjectLiteral>(
{
...searchOptions,
// Take one more item to check if there's a next page
take: paginationOptions.take + 1,
skip: paginationOptions.skip,
take: take + 1,
skip,
},
_.isUndefined,
),
);
const hasNextPage = items.length > paginationOptions.take;
items.splice(paginationOptions.take);
return paginationHelper(items, take);
}
return { items, hasNextPage };
export async function paginatedBuilder<Entity extends ObjectLiteral>(
qb: SelectQueryBuilder<Entity>,
{ take, skip, mode }: PaginatedBuilderOptions,
): Paginated<Entity> {
if (mode === PaginationMode.LIMIT_OFFSET) {
qb.limit(take + 1).offset(skip);
} else {
qb.take(take + 1).skip(skip);
}
const items = await qb.getMany();
return paginationHelper(items, take);
}
export const asVector = (embedding: number[], quote = false) =>
quote ? `'[${embedding.join(',')}]'` : `[${embedding.join(',')}]`;
export const isValidInteger = (value: number, options: { min?: number; max?: number }): value is number => {
const { min = Number.MIN_SAFE_INTEGER, max = Number.MAX_SAFE_INTEGER } = options;
return Number.isInteger(value) && value >= min && value <= max;
};
/**
* Wraps a method that takes a collection of parameters and sequentially calls it with chunks of the collection,
* to overcome the maximum number of parameters allowed by the database driver.
@ -91,3 +121,79 @@ export function ChunkedArray(options?: { paramIndex?: number }): MethodDecorator
export function ChunkedSet(options?: { paramIndex?: number }): MethodDecorator {
return Chunked({ ...options, mergeFn: setUnion });
}
export function searchAssetBuilder(
builder: SelectQueryBuilder<AssetEntity>,
options: AssetSearchBuilderOptions,
): SelectQueryBuilder<AssetEntity> {
builder.andWhere(
_.omitBy(
{
createdAt: OptionalBetween(options.createdAfter, options.createdBefore),
updatedAt: OptionalBetween(options.updatedAfter, options.updatedBefore),
deletedAt: OptionalBetween(options.trashedAfter, options.trashedBefore),
fileCreatedAt: OptionalBetween(options.takenAfter, options.takenBefore),
},
_.isUndefined,
),
);
const exifInfo = _.omitBy(_.pick(options, ['city', 'country', 'lensModel', 'make', 'model', 'state']), _.isUndefined);
if (Object.keys(exifInfo).length > 0) {
builder.leftJoin(`${builder.alias}.exifInfo`, 'exifInfo');
builder.andWhere({ exifInfo });
}
const id = _.pick(options, ['checksum', 'deviceAssetId', 'deviceId', 'id', 'libraryId', 'ownerId']);
builder.andWhere(_.omitBy(id, _.isUndefined));
const path = _.pick(options, ['encodedVideoPath', 'originalFileName', 'originalPath', 'resizePath', 'webpPath']);
builder.andWhere(_.omitBy(path, _.isUndefined));
const status = _.pick(options, ['isExternal', 'isFavorite', 'isOffline', 'isReadOnly', 'isVisible', 'type']);
const { isArchived, isEncoded, isMotion, withArchived } = options;
builder.andWhere(
_.omitBy(
{
...status,
isArchived: isArchived ?? withArchived,
encodedVideoPath: isEncoded ? Not(IsNull()) : undefined,
livePhotoVideoId: isMotion ? Not(IsNull()) : undefined,
},
_.isUndefined,
),
);
if (options.withExif) {
builder.leftJoinAndSelect(`${builder.alias}.exifInfo`, 'exifInfo');
}
if (options.withFaces || options.withPeople) {
builder.leftJoinAndSelect(`${builder.alias}.faces`, 'faces');
}
if (options.withPeople) {
builder.leftJoinAndSelect(`${builder.alias}.person`, 'person');
}
if (options.withSmartInfo) {
builder.leftJoinAndSelect(`${builder.alias}.smartInfo`, 'smartInfo');
}
if (options.withStacked) {
builder
.leftJoinAndSelect(`${builder.alias}.stack`, 'stack')
.leftJoinAndSelect('stack.assets', 'stackedAssets')
.andWhere(
new Brackets((qb) => qb.where(`stack.primaryAssetId = ${builder.alias}.id`).orWhere('asset.stackId IS NULL')),
);
}
const withDeleted =
options.withDeleted ?? (options.trashedAfter !== undefined || options.trashedBefore !== undefined);
if (withDeleted) {
builder.withDeleted();
}
return builder;
}