immich/server/src/utils/misc.ts

238 lines
6.9 KiB
TypeScript
Raw Normal View History

2024-03-21 08:07:47 -05:00
import { INestApplication } from '@nestjs/common';
import {
DocumentBuilder,
OpenAPIObject,
SwaggerCustomOptions,
SwaggerDocumentOptions,
SwaggerModule,
} from '@nestjs/swagger';
import { ReferenceObject, SchemaObject } from '@nestjs/swagger/dist/interfaces/open-api-spec.interface';
2024-03-21 08:07:47 -05:00
import _ from 'lodash';
import { writeFileSync } from 'node:fs';
import path from 'node:path';
import { SystemConfig } from 'src/config';
import { CLIP_MODEL_INFO, isDev, serverVersion } from 'src/constants';
2024-04-19 11:19:23 -04:00
import { ImmichCookie, ImmichHeader } from 'src/dtos/auth.dto';
import { ILoggerRepository } from 'src/interfaces/logger.interface';
2024-03-21 08:07:47 -05:00
import { Metadata } from 'src/middleware/auth.guard';
2024-03-20 22:15:09 -05:00
/**
* @returns a list of strings representing the keys of the object in dot notation
*/
export const getKeysDeep = (target: unknown, path: string[] = []) => {
if (!target || typeof target !== 'object') {
return [];
}
const obj = target as object;
const properties: string[] = [];
for (const key of Object.keys(obj as object)) {
const value = obj[key as keyof object];
if (value === undefined) {
continue;
}
if (_.isObject(value) && !_.isArray(value) && !_.isDate(value)) {
properties.push(...getKeysDeep(value, [...path, key]));
continue;
}
properties.push([...path, key].join('.'));
}
return properties;
};
export const unsetDeep = (object: unknown, key: string) => {
const parts = key.split('.');
while (parts.length > 0) {
_.unset(object, parts);
parts.pop();
if (!_.isEmpty(_.get(object, parts))) {
break;
}
}
return _.isEmpty(object) ? undefined : object;
};
const isMachineLearningEnabled = (machineLearning: SystemConfig['machineLearning']) => machineLearning.enabled;
export const isSmartSearchEnabled = (machineLearning: SystemConfig['machineLearning']) =>
isMachineLearningEnabled(machineLearning) && machineLearning.clip.enabled;
export const isFacialRecognitionEnabled = (machineLearning: SystemConfig['machineLearning']) =>
isMachineLearningEnabled(machineLearning) && machineLearning.facialRecognition.enabled;
feat(server): near-duplicate detection (#8228) * duplicate detection job, entity, config * queueing * job panel, update api * use embedding in db instead of fetching * disable concurrency * only queue visible assets * handle multiple duplicateIds * update concurrent queue check * add provider * add web placeholder, server endpoint, migration, various fixes * update sql * select embedding by default * rename variable * simplify * remove separate entity, handle re-running with different threshold, set default back to 0.02 * fix tests * add tests * add index to entity * formatting * update asset mock * fix `upsertJobStatus` signature * update sql * formatting * default to 0.03 * optimize clustering * use asset's `duplicateId` if present * update sql * update tests * expose admin setting * refactor * formatting * skip if ml is disabled * debug trash e2e * remove from web * remove from sidebar * test if ml is disabled * update sql * separate duplicate detection from clip in config, disable by default for now * fix doc * lower minimum `maxDistance` * update api * Add and Use Duplicate Detection Feature Flag (#9364) * Add Duplicate Detection Flag * Use Duplicate Detection Flag * Attempt Fixes for Failing Checks * lower minimum `maxDistance` * fix tests --------- Co-authored-by: mertalev <101130780+mertalev@users.noreply.github.com> * chore: fixes and additions after rebase * chore: update api (remove new Role enum) * fix: left join smart search so getAll works without machine learning * test: trash e2e go back to checking length of assets is zero * chore: regen api after rebase * test: fix tests after rebase * redundant join --------- Co-authored-by: Nicholas Flamy <30300649+NicholasFlamy@users.noreply.github.com> Co-authored-by: Zack Pollard <zackpollard@ymail.com> Co-authored-by: Zack Pollard <zack@futo.org>
2024-05-16 13:08:37 -04:00
export const isDuplicateDetectionEnabled = (machineLearning: SystemConfig['machineLearning']) =>
isSmartSearchEnabled(machineLearning) && machineLearning.duplicateDetection.enabled;
2024-03-20 22:15:09 -05:00
export const isConnectionAborted = (error: Error | any) => error.code === 'ECONNABORTED';
export const handlePromiseError = <T>(promise: Promise<T>, logger: ILoggerRepository): void => {
2024-03-20 22:15:09 -05:00
promise.catch((error: Error | any) => logger.error(`Promise error: ${error}`, error?.stack));
};
export interface OpenGraphTags {
title: string;
description: string;
imageUrl?: string;
}
function cleanModelName(modelName: string): string {
const token = modelName.split('/').at(-1);
if (!token) {
throw new Error(`Invalid model name: ${modelName}`);
}
return token.replaceAll(':', '_');
}
export function getCLIPModelInfo(modelName: string) {
const modelInfo = CLIP_MODEL_INFO[cleanModelName(modelName)];
if (!modelInfo) {
throw new Error(`Unknown CLIP model: ${modelName}`);
}
return modelInfo;
}
2024-03-21 08:07:47 -05:00
function sortKeys<T>(target: T): T {
if (!target || typeof target !== 'object' || Array.isArray(target)) {
return target;
}
const result: Partial<T> = {};
const keys = Object.keys(target).sort() as Array<keyof T>;
for (const key of keys) {
result[key] = sortKeys(target[key]);
}
return result as T;
}
export const routeToErrorMessage = (methodName: string) =>
'Failed to ' + methodName.replaceAll(/[A-Z]+/g, (letter) => ` ${letter.toLowerCase()}`);
const isSchema = (schema: string | ReferenceObject | SchemaObject): schema is SchemaObject => {
if (typeof schema === 'string' || '$ref' in schema) {
return false;
}
return true;
};
2024-03-21 08:07:47 -05:00
const patchOpenAPI = (document: OpenAPIObject) => {
document.paths = sortKeys(document.paths);
if (document.components?.schemas) {
const schemas = document.components.schemas as Record<string, SchemaObject>;
document.components.schemas = sortKeys(schemas);
for (const [schemaName, schema] of Object.entries(schemas)) {
2024-03-21 08:07:47 -05:00
if (schema.properties) {
schema.properties = sortKeys(schema.properties);
for (const [key, value] of Object.entries(schema.properties)) {
if (typeof value === 'string') {
continue;
}
if (isSchema(value) && value.type === 'number' && value.format === 'float') {
throw new Error(`Invalid number format: ${schemaName}.${key}=float (use double instead). `);
}
}
if (schema.required) {
schema.required = schema.required.sort();
}
2024-03-21 08:07:47 -05:00
}
}
}
for (const [key, value] of Object.entries(document.paths)) {
const newKey = key.replace('/api/', '/');
delete document.paths[key];
document.paths[newKey] = value;
}
for (const path of Object.values(document.paths)) {
const operations = {
get: path.get,
put: path.put,
post: path.post,
delete: path.delete,
options: path.options,
head: path.head,
patch: path.patch,
trace: path.trace,
};
for (const operation of Object.values(operations)) {
if (!operation) {
continue;
}
if (operation.summary === '') {
delete operation.summary;
}
if (operation.operationId) {
// console.log(`${routeToErrorMessage(operation.operationId).padEnd(40)} (${operation.operationId})`);
}
if (operation.description === '') {
delete operation.description;
}
if (operation.parameters) {
operation.parameters = _.orderBy(operation.parameters, 'name');
}
}
}
return document;
};
export const useSwagger = (app: INestApplication, force = false) => {
2024-03-21 08:07:47 -05:00
const config = new DocumentBuilder()
.setTitle('Immich')
.setDescription('Immich API')
.setVersion(serverVersion.toString())
.addBearerAuth({
type: 'http',
scheme: 'Bearer',
in: 'header',
})
2024-04-19 11:19:23 -04:00
.addCookieAuth(ImmichCookie.ACCESS_TOKEN)
2024-03-21 08:07:47 -05:00
.addApiKey(
{
type: 'apiKey',
in: 'header',
2024-04-19 11:19:23 -04:00
name: ImmichHeader.API_KEY,
2024-03-21 08:07:47 -05:00
},
2024-04-19 11:19:23 -04:00
Metadata.API_KEY_SECURITY,
2024-03-21 08:07:47 -05:00
)
.addServer('/api')
.build();
const options: SwaggerDocumentOptions = {
operationIdFactory: (controllerKey: string, methodKey: string) => methodKey,
};
const specification = SwaggerModule.createDocument(app, config, options);
const customOptions: SwaggerCustomOptions = {
swaggerOptions: {
persistAuthorization: true,
},
customSiteTitle: 'Immich API Documentation',
};
SwaggerModule.setup('doc', app, specification, customOptions);
if (isDev() || force) {
2024-03-21 08:07:47 -05:00
// Generate API Documentation only in development mode
const outputPath = path.resolve(process.cwd(), '../open-api/immich-openapi-specs.json');
writeFileSync(outputPath, JSON.stringify(patchOpenAPI(specification), null, 2), { encoding: 'utf8' });
}
};