mirror of
https://github.com/immich-app/immich
synced 2025-11-14 17:36:12 +00:00
feat: original-sized previews for non-web-friendly images (#14446)
* feat(server): extract full-size previews from RAW images * feat(web): load fullsize preview for RAW images when zoomed in * refactor: tweaks for code review * refactor: rename "converted" preview/assets to "fullsize" * feat(web/server): fullsize preview for non-web-friendly images * feat: tweaks for code review * feat(server): require ASSET_DOWNLOAD premission for fullsize previews * test: fix types and interfaces * chore: gen open-api * feat(server): keep only essential exif in fullsize preview * chore: regen openapi * test: revert unnecessary timeout * feat: move full-size preview config to standalone entry * feat(i18n): update en texts * fix: don't return fullsizePath when disabled * test: full-size previews * test(web): full-size previews * chore: make open-api * feat(server): redirect to preview/original URL when fullsize thumbnail not available * fix(server): delete fullsize preview image on thumbnail regen after fullsize preview turned off * refactor(server): AssetRepository.deleteFiles with Kysely * fix(server): type of MediaRepository.writeExif * minor simplification * minor styling changes and condensed wording * simplify * chore: reuild open-api * test(server): fix media.service tests * test(web): fix photo-viewer test * fix(server): use fullsize image when requested * fix file path extension * formatting * use fullsize when zooming back out or when "display original photos" is enabled * simplify condition --------- Co-authored-by: mertalev <101130780+mertalev@users.noreply.github.com>
This commit is contained in:
Eli Gao
GitHub
parent
a5093a9434
commit
5c80e8734b
33 changed files with 778 additions and 115 deletions
|
|
@ -12,7 +12,7 @@ import {
|
|||
VideoCodec,
|
||||
VideoContainer,
|
||||
} from 'src/enum';
|
||||
import { ConcurrentQueueName, ImageOptions } from 'src/types';
|
||||
import { ConcurrentQueueName, FullsizeImageOptions, ImageOptions } from 'src/types';
|
||||
|
||||
export interface SystemConfig {
|
||||
backup: {
|
||||
|
|
@ -112,6 +112,7 @@ export interface SystemConfig {
|
|||
preview: ImageOptions;
|
||||
colorspace: Colorspace;
|
||||
extractEmbedded: boolean;
|
||||
fullsize: FullsizeImageOptions;
|
||||
};
|
||||
newVersionCheck: {
|
||||
enabled: boolean;
|
||||
|
|
@ -281,6 +282,11 @@ export const defaults = Object.freeze<SystemConfig>({
|
|||
},
|
||||
colorspace: Colorspace.P3,
|
||||
extractEmbedded: false,
|
||||
fullsize: {
|
||||
enabled: false,
|
||||
format: ImageFormat.JPEG,
|
||||
quality: 80,
|
||||
},
|
||||
},
|
||||
newVersionCheck: {
|
||||
enabled: true,
|
||||
|
|
|
|||
|
|
@ -10,12 +10,13 @@ import {
|
|||
Post,
|
||||
Put,
|
||||
Query,
|
||||
Req,
|
||||
Res,
|
||||
UploadedFiles,
|
||||
UseInterceptors,
|
||||
} from '@nestjs/common';
|
||||
import { ApiBody, ApiConsumes, ApiHeader, ApiOperation, ApiTags } from '@nestjs/swagger';
|
||||
import { NextFunction, Response } from 'express';
|
||||
import { NextFunction, Request, Response } from 'express';
|
||||
import { EndpointLifecycle } from 'src/decorators';
|
||||
import {
|
||||
AssetBulkUploadCheckResponseDto,
|
||||
|
|
@ -28,6 +29,7 @@ import {
|
|||
AssetMediaCreateDto,
|
||||
AssetMediaOptionsDto,
|
||||
AssetMediaReplaceDto,
|
||||
AssetMediaSize,
|
||||
CheckExistingAssetsDto,
|
||||
UploadFieldName,
|
||||
} from 'src/dtos/asset-media.dto';
|
||||
|
|
@ -39,7 +41,7 @@ import { FileUploadInterceptor, getFiles } from 'src/middleware/file-upload.inte
|
|||
import { LoggingRepository } from 'src/repositories/logging.repository';
|
||||
import { AssetMediaService } from 'src/services/asset-media.service';
|
||||
import { UploadFiles } from 'src/types';
|
||||
import { sendFile } from 'src/utils/file';
|
||||
import { ImmichFileResponse, sendFile } from 'src/utils/file';
|
||||
import { FileNotEmptyValidator, UUIDParamDto } from 'src/validation';
|
||||
|
||||
@ApiTags('Assets')
|
||||
|
|
@ -123,10 +125,34 @@ export class AssetMediaController {
|
|||
@Auth() auth: AuthDto,
|
||||
@Param() { id }: UUIDParamDto,
|
||||
@Query() dto: AssetMediaOptionsDto,
|
||||
@Req() req: Request,
|
||||
@Res() res: Response,
|
||||
@Next() next: NextFunction,
|
||||
) {
|
||||
await sendFile(res, next, () => this.service.viewThumbnail(auth, id, dto), this.logger);
|
||||
const viewThumbnailRes = await this.service.viewThumbnail(auth, id, dto);
|
||||
|
||||
if (viewThumbnailRes instanceof ImmichFileResponse) {
|
||||
await sendFile(res, next, () => Promise.resolve(viewThumbnailRes), this.logger);
|
||||
} else {
|
||||
// viewThumbnailRes is a AssetMediaRedirectResponse
|
||||
// which redirects to the original asset or a specific size to make better use of caching
|
||||
const { targetSize } = viewThumbnailRes;
|
||||
const [reqPath, reqSearch] = req.url.split('?');
|
||||
let redirPath: string;
|
||||
const redirSearchParams = new URLSearchParams(reqSearch);
|
||||
if (targetSize === 'original') {
|
||||
// relative path to this.downloadAsset
|
||||
redirPath = 'original';
|
||||
redirSearchParams.delete('size');
|
||||
} else if (Object.values(AssetMediaSize).includes(targetSize)) {
|
||||
redirPath = reqPath;
|
||||
redirSearchParams.set('size', targetSize);
|
||||
} else {
|
||||
throw new Error('Invalid targetSize: ' + targetSize);
|
||||
}
|
||||
const finalRedirPath = redirPath + '?' + redirSearchParams.toString();
|
||||
return res.redirect(finalRedirPath);
|
||||
}
|
||||
}
|
||||
|
||||
@Get(':id/video/playback')
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ export interface MoveRequest {
|
|||
};
|
||||
}
|
||||
|
||||
export type GeneratedImageType = AssetPathType.PREVIEW | AssetPathType.THUMBNAIL;
|
||||
export type GeneratedImageType = AssetPathType.PREVIEW | AssetPathType.THUMBNAIL | AssetPathType.FULLSIZE;
|
||||
export type GeneratedAssetType = GeneratedImageType | AssetPathType.ENCODED_VIDEO;
|
||||
|
||||
let instance: StorageCore | null;
|
||||
|
|
@ -277,6 +277,9 @@ export class StorageCore {
|
|||
case AssetPathType.ORIGINAL: {
|
||||
return this.assetRepository.update({ id, originalPath: newPath });
|
||||
}
|
||||
case AssetPathType.FULLSIZE: {
|
||||
return this.assetRepository.upsertFile({ assetId: id, type: AssetFileType.FULLSIZE, path: newPath });
|
||||
}
|
||||
case AssetPathType.PREVIEW: {
|
||||
return this.assetRepository.upsertFile({ assetId: id, type: AssetFileType.PREVIEW, path: newPath });
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,6 +4,11 @@ import { ArrayNotEmpty, IsArray, IsEnum, IsNotEmpty, IsString, ValidateNested }
|
|||
import { Optional, ValidateBoolean, ValidateDate, ValidateUUID } from 'src/validation';
|
||||
|
||||
export enum AssetMediaSize {
|
||||
/**
|
||||
* An full-sized image extracted/converted from non-web-friendly formats like RAW/HIF.
|
||||
* or otherwise the original image itself.
|
||||
*/
|
||||
FULLSIZE = 'fullsize',
|
||||
PREVIEW = 'preview',
|
||||
THUMBNAIL = 'thumbnail',
|
||||
}
|
||||
|
|
|
|||
|
|
@ -531,6 +531,24 @@ class SystemConfigGeneratedImageDto {
|
|||
size!: number;
|
||||
}
|
||||
|
||||
class SystemConfigGeneratedFullsizeImageDto {
|
||||
@IsBoolean()
|
||||
@Type(() => Boolean)
|
||||
@ApiProperty({ type: 'boolean' })
|
||||
enabled!: boolean;
|
||||
|
||||
@IsEnum(ImageFormat)
|
||||
@ApiProperty({ enumName: 'ImageFormat', enum: ImageFormat })
|
||||
format!: ImageFormat;
|
||||
|
||||
@IsInt()
|
||||
@Min(1)
|
||||
@Max(100)
|
||||
@Type(() => Number)
|
||||
@ApiProperty({ type: 'integer' })
|
||||
quality!: number;
|
||||
}
|
||||
|
||||
export class SystemConfigImageDto {
|
||||
@Type(() => SystemConfigGeneratedImageDto)
|
||||
@ValidateNested()
|
||||
|
|
@ -542,6 +560,11 @@ export class SystemConfigImageDto {
|
|||
@IsObject()
|
||||
preview!: SystemConfigGeneratedImageDto;
|
||||
|
||||
@Type(() => SystemConfigGeneratedFullsizeImageDto)
|
||||
@ValidateNested()
|
||||
@IsObject()
|
||||
fullsize!: SystemConfigGeneratedFullsizeImageDto;
|
||||
|
||||
@IsEnum(Colorspace)
|
||||
@ApiProperty({ enumName: 'Colorspace', enum: Colorspace })
|
||||
colorspace!: Colorspace;
|
||||
|
|
|
|||
|
|
@ -33,6 +33,10 @@ export enum AssetType {
|
|||
}
|
||||
|
||||
export enum AssetFileType {
|
||||
/**
|
||||
* An full/large-size image extracted/converted from RAW photos
|
||||
*/
|
||||
FULLSIZE = 'fullsize',
|
||||
PREVIEW = 'preview',
|
||||
THUMBNAIL = 'thumbnail',
|
||||
}
|
||||
|
|
@ -242,6 +246,7 @@ export enum ManualJobName {
|
|||
|
||||
export enum AssetPathType {
|
||||
ORIGINAL = 'original',
|
||||
FULLSIZE = 'fullsize',
|
||||
PREVIEW = 'preview',
|
||||
THUMBNAIL = 'thumbnail',
|
||||
ENCODED_VIDEO = 'encoded_video',
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import { Injectable } from '@nestjs/common';
|
||||
import { Insertable, Kysely, UpdateResult, Updateable, sql } from 'kysely';
|
||||
import { Insertable, Kysely, Selectable, UpdateResult, Updateable, sql } from 'kysely';
|
||||
import { isEmpty, isUndefined, omitBy } from 'lodash';
|
||||
import { InjectKysely } from 'nestjs-kysely';
|
||||
import { AssetFiles, AssetJobStatus, Assets, DB, Exif } from 'src/db';
|
||||
|
|
@ -1036,6 +1036,17 @@ export class AssetRepository {
|
|||
.execute();
|
||||
}
|
||||
|
||||
async deleteFiles(files: Pick<Selectable<AssetFiles>, 'id'>[]): Promise<void> {
|
||||
if (files.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
await this.db
|
||||
.deleteFrom('asset_files')
|
||||
.where('id', '=', anyUuid(files.map((file) => file.id)))
|
||||
.execute();
|
||||
}
|
||||
|
||||
@GenerateSql({
|
||||
params: [{ libraryId: DummyValue.UUID, importPaths: [DummyValue.STRING], exclusionPatterns: [DummyValue.STRING] }],
|
||||
})
|
||||
|
|
|
|||
|
|
@ -1,11 +1,12 @@
|
|||
import { Injectable } from '@nestjs/common';
|
||||
import { exiftool } from 'exiftool-vendored';
|
||||
import { ExifDateTime, exiftool, WriteTags } from 'exiftool-vendored';
|
||||
import ffmpeg, { FfprobeData } from 'fluent-ffmpeg';
|
||||
import { Duration } from 'luxon';
|
||||
import fs from 'node:fs/promises';
|
||||
import { Writable } from 'node:stream';
|
||||
import sharp from 'sharp';
|
||||
import { ORIENTATION_TO_SHARP_ROTATION } from 'src/constants';
|
||||
import { ExifEntity } from 'src/entities/exif.entity';
|
||||
import { Colorspace, LogLevel } from 'src/enum';
|
||||
import { LoggingRepository } from 'src/repositories/logging.repository';
|
||||
import {
|
||||
|
|
@ -43,9 +44,14 @@ export class MediaRepository {
|
|||
|
||||
async extract(input: string, output: string): Promise<boolean> {
|
||||
try {
|
||||
// remove existing output file if it exists
|
||||
// as exiftool-vendored does not support overwriting via "-w!" flag
|
||||
// and throws "1 files could not be read" error when the output file exists
|
||||
await fs.unlink(output).catch(() => null);
|
||||
await exiftool.extractBinaryTag('JpgFromRaw2', input, output);
|
||||
} catch {
|
||||
try {
|
||||
this.logger.debug('Extracting JPEG from RAW image:', input);
|
||||
await exiftool.extractJpgFromRaw(input, output);
|
||||
} catch (error: any) {
|
||||
this.logger.debug('Could not extract JPEG from image, trying preview', error.message);
|
||||
|
|
@ -57,10 +63,47 @@ export class MediaRepository {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
async writeExif(tags: Partial<ExifEntity>, output: string): Promise<boolean> {
|
||||
try {
|
||||
const tagsToWrite: WriteTags = {
|
||||
ExifImageWidth: tags.exifImageWidth,
|
||||
ExifImageHeight: tags.exifImageHeight,
|
||||
DateTimeOriginal: tags.dateTimeOriginal && ExifDateTime.fromMillis(tags.dateTimeOriginal.getTime()),
|
||||
ModifyDate: tags.modifyDate && ExifDateTime.fromMillis(tags.modifyDate.getTime()),
|
||||
TimeZone: tags.timeZone,
|
||||
GPSLatitude: tags.latitude,
|
||||
GPSLongitude: tags.longitude,
|
||||
ProjectionType: tags.projectionType,
|
||||
City: tags.city,
|
||||
Country: tags.country,
|
||||
Make: tags.make,
|
||||
Model: tags.model,
|
||||
LensModel: tags.lensModel,
|
||||
Fnumber: tags.fNumber?.toFixed(1),
|
||||
FocalLength: tags.focalLength?.toFixed(1),
|
||||
ISO: tags.iso,
|
||||
ExposureTime: tags.exposureTime,
|
||||
ProfileDescription: tags.profileDescription,
|
||||
ColorSpace: tags.colorspace,
|
||||
Rating: tags.rating,
|
||||
// specially convert Orientation to numeric Orientation# for exiftool
|
||||
'Orientation#': tags.orientation ? Number(tags.orientation) : undefined,
|
||||
};
|
||||
|
||||
await exiftool.write(output, tagsToWrite, {
|
||||
ignoreMinorErrors: true,
|
||||
writeArgs: ['-overwrite_original'],
|
||||
});
|
||||
return true;
|
||||
} catch (error: any) {
|
||||
this.logger.warn(`Could not write exif data to image: ${error.message}`);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
decodeImage(input: string, options: DecodeToBufferOptions) {
|
||||
return this.getImageDecodingPipeline(input, options).raw().toBuffer({ resolveWithObject: true });
|
||||
}
|
||||
|
|
@ -101,7 +144,10 @@ export class MediaRepository {
|
|||
pipeline = pipeline.extract(options.crop);
|
||||
}
|
||||
|
||||
return pipeline.resize(options.size, options.size, { fit: 'outside', withoutEnlargement: true });
|
||||
if (options.size !== undefined) {
|
||||
pipeline = pipeline.resize(options.size, options.size, { fit: 'outside', withoutEnlargement: true });
|
||||
}
|
||||
return pipeline;
|
||||
}
|
||||
|
||||
async generateThumbhash(input: string | Buffer, options: GenerateThumbhashOptions): Promise<Buffer> {
|
||||
|
|
|
|||
|
|
@ -37,6 +37,10 @@ interface UploadRequest {
|
|||
file: UploadFile;
|
||||
}
|
||||
|
||||
export interface AssetMediaRedirectResponse {
|
||||
targetSize: AssetMediaSize | 'original';
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class AssetMediaService extends BaseService {
|
||||
async getUploadAssetIdByChecksum(auth: AuthDto, checksum?: string): Promise<AssetMediaResponseDto | undefined> {
|
||||
|
|
@ -194,16 +198,31 @@ export class AssetMediaService extends BaseService {
|
|||
});
|
||||
}
|
||||
|
||||
async viewThumbnail(auth: AuthDto, id: string, dto: AssetMediaOptionsDto): Promise<ImmichFileResponse> {
|
||||
async viewThumbnail(
|
||||
auth: AuthDto,
|
||||
id: string,
|
||||
dto: AssetMediaOptionsDto,
|
||||
): Promise<ImmichFileResponse | AssetMediaRedirectResponse> {
|
||||
await this.requireAccess({ auth, permission: Permission.ASSET_VIEW, ids: [id] });
|
||||
|
||||
const asset = await this.findOrFail(id);
|
||||
const size = dto.size ?? AssetMediaSize.THUMBNAIL;
|
||||
|
||||
const { thumbnailFile, previewFile } = getAssetFiles(asset.files);
|
||||
const { thumbnailFile, previewFile, fullsizeFile } = getAssetFiles(asset.files);
|
||||
let filepath = previewFile?.path;
|
||||
if (size === AssetMediaSize.THUMBNAIL && thumbnailFile) {
|
||||
filepath = thumbnailFile.path;
|
||||
} else if (size === AssetMediaSize.FULLSIZE) {
|
||||
if (mimeTypes.isWebSupportedImage(asset.originalPath)) {
|
||||
// use original file for web supported images
|
||||
return { targetSize: 'original' };
|
||||
}
|
||||
if (!fullsizeFile) {
|
||||
// downgrade to preview if fullsize is not available.
|
||||
// e.g. disabled or not yet (re)generated
|
||||
return { targetSize: AssetMediaSize.PREVIEW };
|
||||
}
|
||||
filepath = fullsizeFile.path;
|
||||
}
|
||||
|
||||
if (!filepath) {
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import { OutputInfo } from 'sharp';
|
||||
import { SystemConfig } from 'src/config';
|
||||
import { AssetMediaSize } from 'src/dtos/asset-media.dto';
|
||||
import { AssetEntity } from 'src/entities/asset.entity';
|
||||
import { ExifEntity } from 'src/entities/exif.entity';
|
||||
import {
|
||||
|
|
@ -233,18 +234,26 @@ describe(MediaService.name, () => {
|
|||
});
|
||||
|
||||
await expect(sut.handleAssetMigration({ id: assetStub.image.id })).resolves.toBe(JobStatus.SUCCESS);
|
||||
expect(mocks.move.create).toHaveBeenCalledTimes(2);
|
||||
expect(mocks.move.create).toHaveBeenCalledTimes(3);
|
||||
});
|
||||
});
|
||||
|
||||
describe('handleGenerateThumbnails', () => {
|
||||
let rawBuffer: Buffer;
|
||||
let fullsizeBuffer: Buffer;
|
||||
let rawInfo: RawImageInfo;
|
||||
|
||||
beforeEach(() => {
|
||||
fullsizeBuffer = Buffer.from('embedded image data');
|
||||
rawBuffer = Buffer.from('image data');
|
||||
rawInfo = { width: 100, height: 100, channels: 3 };
|
||||
mocks.media.decodeImage.mockResolvedValue({ data: rawBuffer, info: rawInfo as OutputInfo });
|
||||
mocks.media.decodeImage.mockImplementation((path) =>
|
||||
Promise.resolve(
|
||||
path.includes(AssetMediaSize.FULLSIZE)
|
||||
? { data: fullsizeBuffer, info: rawInfo as OutputInfo }
|
||||
: { data: rawBuffer, info: rawInfo as OutputInfo },
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
it('should skip thumbnail generation if asset not found', async () => {
|
||||
|
|
@ -591,15 +600,13 @@ describe(MediaService.name, () => {
|
|||
|
||||
await sut.handleGenerateThumbnails({ id: assetStub.image.id });
|
||||
|
||||
const extractedPath = mocks.media.extract.mock.calls.at(-1)?.[1].toString();
|
||||
const convertedPath = mocks.media.extract.mock.lastCall?.[1].toString();
|
||||
expect(mocks.media.decodeImage).toHaveBeenCalledOnce();
|
||||
expect(mocks.media.decodeImage).toHaveBeenCalledWith(extractedPath, {
|
||||
expect(mocks.media.decodeImage).toHaveBeenCalledWith(convertedPath, {
|
||||
colorspace: Colorspace.P3,
|
||||
processInvalidImages: false,
|
||||
size: 1440,
|
||||
});
|
||||
expect(extractedPath?.endsWith('.tmp')).toBe(true);
|
||||
expect(mocks.storage.unlink).toHaveBeenCalledWith(extractedPath);
|
||||
});
|
||||
|
||||
it('should resize original image if embedded image is too small', async () => {
|
||||
|
|
@ -610,15 +617,14 @@ describe(MediaService.name, () => {
|
|||
|
||||
await sut.handleGenerateThumbnails({ id: assetStub.image.id });
|
||||
|
||||
expect(mocks.media.decodeImage).toHaveBeenCalledOnce();
|
||||
const extractedPath = mocks.media.extract.mock.lastCall?.[1].toString();
|
||||
expect(extractedPath).toMatch(/-fullsize\.jpeg$/);
|
||||
|
||||
expect(mocks.media.decodeImage).toHaveBeenCalledWith(assetStub.imageDng.originalPath, {
|
||||
colorspace: Colorspace.P3,
|
||||
processInvalidImages: false,
|
||||
size: 1440,
|
||||
});
|
||||
const extractedPath = mocks.media.extract.mock.calls.at(-1)?.[1].toString();
|
||||
expect(extractedPath?.endsWith('.tmp')).toBe(true);
|
||||
expect(mocks.storage.unlink).toHaveBeenCalledWith(extractedPath);
|
||||
});
|
||||
|
||||
it('should resize original image if embedded image not found', async () => {
|
||||
|
|
@ -686,6 +692,159 @@ describe(MediaService.name, () => {
|
|||
expect(mocks.media.getImageDimensions).not.toHaveBeenCalled();
|
||||
vi.unstubAllEnvs();
|
||||
});
|
||||
|
||||
it('should generate full-size preview using embedded JPEG from RAW images when extractEmbedded is true', async () => {
|
||||
mocks.systemMetadata.get.mockResolvedValue({ image: { fullsize: { enabled: true }, extractEmbedded: true } });
|
||||
mocks.media.extract.mockResolvedValue(true);
|
||||
mocks.media.getImageDimensions.mockResolvedValue({ width: 3840, height: 2160 });
|
||||
mocks.asset.getById.mockResolvedValue(assetStub.imageDng);
|
||||
|
||||
await sut.handleGenerateThumbnails({ id: assetStub.image.id });
|
||||
|
||||
const extractedPath = mocks.media.extract.mock.lastCall?.[1].toString();
|
||||
expect(mocks.media.decodeImage).toHaveBeenCalledOnce();
|
||||
expect(mocks.media.decodeImage).toHaveBeenCalledWith(extractedPath, {
|
||||
colorspace: Colorspace.P3,
|
||||
processInvalidImages: false,
|
||||
});
|
||||
|
||||
expect(mocks.media.generateThumbnail).toHaveBeenCalledTimes(2);
|
||||
expect(mocks.media.generateThumbnail).toHaveBeenCalledWith(
|
||||
fullsizeBuffer,
|
||||
{
|
||||
colorspace: Colorspace.P3,
|
||||
format: ImageFormat.JPEG,
|
||||
size: 1440,
|
||||
quality: 80,
|
||||
processInvalidImages: false,
|
||||
raw: rawInfo,
|
||||
},
|
||||
'upload/thumbs/user-id/as/se/asset-id-preview.jpeg',
|
||||
);
|
||||
});
|
||||
|
||||
it('should generate full-size preview directly from RAW images when extractEmbedded is false', async () => {
|
||||
mocks.systemMetadata.get.mockResolvedValue({ image: { fullsize: { enabled: true }, extractEmbedded: false } });
|
||||
mocks.media.extract.mockResolvedValue(true);
|
||||
mocks.media.getImageDimensions.mockResolvedValue({ width: 3840, height: 2160 });
|
||||
mocks.asset.getById.mockResolvedValue(assetStub.imageDng);
|
||||
|
||||
await sut.handleGenerateThumbnails({ id: assetStub.image.id });
|
||||
|
||||
expect(mocks.media.decodeImage).toHaveBeenCalledOnce();
|
||||
expect(mocks.media.decodeImage).toHaveBeenCalledWith(assetStub.imageDng.originalPath, {
|
||||
colorspace: Colorspace.P3,
|
||||
processInvalidImages: false,
|
||||
});
|
||||
|
||||
expect(mocks.media.generateThumbnail).toHaveBeenCalledTimes(3);
|
||||
expect(mocks.media.generateThumbnail).toHaveBeenCalledWith(
|
||||
rawBuffer,
|
||||
{
|
||||
colorspace: Colorspace.P3,
|
||||
format: ImageFormat.JPEG,
|
||||
quality: 80,
|
||||
processInvalidImages: false,
|
||||
raw: rawInfo,
|
||||
},
|
||||
'upload/thumbs/user-id/as/se/asset-id-fullsize.jpeg',
|
||||
);
|
||||
expect(mocks.media.generateThumbnail).toHaveBeenCalledWith(
|
||||
rawBuffer,
|
||||
{
|
||||
colorspace: Colorspace.P3,
|
||||
format: ImageFormat.JPEG,
|
||||
quality: 80,
|
||||
size: 1440,
|
||||
processInvalidImages: false,
|
||||
raw: rawInfo,
|
||||
},
|
||||
'upload/thumbs/user-id/as/se/asset-id-preview.jpeg',
|
||||
);
|
||||
});
|
||||
|
||||
it('should generate full-size preview from non-web-friendly images', async () => {
|
||||
mocks.systemMetadata.get.mockResolvedValue({ image: { fullsize: { enabled: true } } });
|
||||
mocks.media.extract.mockResolvedValue(true);
|
||||
mocks.media.getImageDimensions.mockResolvedValue({ width: 3840, height: 2160 });
|
||||
// HEIF/HIF image taken by cameras are not web-friendly, only has limited support on Safari.
|
||||
mocks.asset.getById.mockResolvedValue(assetStub.imageHif);
|
||||
|
||||
await sut.handleGenerateThumbnails({ id: assetStub.image.id });
|
||||
|
||||
expect(mocks.media.decodeImage).toHaveBeenCalledOnce();
|
||||
expect(mocks.media.decodeImage).toHaveBeenCalledWith(assetStub.imageHif.originalPath, {
|
||||
colorspace: Colorspace.P3,
|
||||
processInvalidImages: false,
|
||||
});
|
||||
|
||||
expect(mocks.media.generateThumbnail).toHaveBeenCalledTimes(3);
|
||||
expect(mocks.media.generateThumbnail).toHaveBeenCalledWith(
|
||||
rawBuffer,
|
||||
{
|
||||
colorspace: Colorspace.P3,
|
||||
format: ImageFormat.JPEG,
|
||||
quality: 80,
|
||||
processInvalidImages: false,
|
||||
raw: rawInfo,
|
||||
},
|
||||
'upload/thumbs/user-id/as/se/asset-id-fullsize.jpeg',
|
||||
);
|
||||
});
|
||||
|
||||
it('should skip generating full-size preview for web-friendly images', async () => {
|
||||
mocks.systemMetadata.get.mockResolvedValue({ image: { fullsize: { enabled: true } } });
|
||||
mocks.media.extract.mockResolvedValue(true);
|
||||
mocks.media.getImageDimensions.mockResolvedValue({ width: 3840, height: 2160 });
|
||||
mocks.asset.getById.mockResolvedValue(assetStub.image);
|
||||
|
||||
await sut.handleGenerateThumbnails({ id: assetStub.image.id });
|
||||
|
||||
expect(mocks.media.decodeImage).toHaveBeenCalledOnce();
|
||||
expect(mocks.media.decodeImage).toHaveBeenCalledWith(assetStub.image.originalPath, {
|
||||
colorspace: Colorspace.SRGB,
|
||||
processInvalidImages: false,
|
||||
size: 1440,
|
||||
});
|
||||
|
||||
expect(mocks.media.generateThumbnail).toHaveBeenCalledTimes(2);
|
||||
expect(mocks.media.generateThumbnail).not.toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
expect.anything(),
|
||||
'upload/thumbs/user-id/as/se/asset-id-fullsize.jpeg',
|
||||
);
|
||||
});
|
||||
|
||||
it('should respect encoding options when generating full-size preview', async () => {
|
||||
mocks.systemMetadata.get.mockResolvedValue({
|
||||
image: { fullsize: { enabled: true, format: ImageFormat.WEBP, quality: 90 } },
|
||||
});
|
||||
mocks.media.extract.mockResolvedValue(true);
|
||||
mocks.media.getImageDimensions.mockResolvedValue({ width: 3840, height: 2160 });
|
||||
// HEIF/HIF image taken by cameras are not web-friendly, only has limited support on Safari.
|
||||
mocks.asset.getById.mockResolvedValue(assetStub.imageHif);
|
||||
|
||||
await sut.handleGenerateThumbnails({ id: assetStub.image.id });
|
||||
|
||||
expect(mocks.media.decodeImage).toHaveBeenCalledOnce();
|
||||
expect(mocks.media.decodeImage).toHaveBeenCalledWith(assetStub.imageHif.originalPath, {
|
||||
colorspace: Colorspace.P3,
|
||||
processInvalidImages: false,
|
||||
});
|
||||
|
||||
expect(mocks.media.generateThumbnail).toHaveBeenCalledTimes(3);
|
||||
expect(mocks.media.generateThumbnail).toHaveBeenCalledWith(
|
||||
rawBuffer,
|
||||
{
|
||||
colorspace: Colorspace.P3,
|
||||
format: ImageFormat.WEBP,
|
||||
quality: 90,
|
||||
processInvalidImages: false,
|
||||
raw: rawInfo,
|
||||
},
|
||||
'upload/thumbs/user-id/as/se/asset-id-fullsize.webp',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('handleQueueVideoConversion', () => {
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
import { Injectable } from '@nestjs/common';
|
||||
import { dirname } from 'node:path';
|
||||
import { JOBS_ASSET_PAGINATION_SIZE } from 'src/constants';
|
||||
import { StorageCore } from 'src/cores/storage.core';
|
||||
import { OnEvent, OnJob } from 'src/decorators';
|
||||
|
|
@ -11,6 +10,7 @@ import {
|
|||
AssetType,
|
||||
AudioCodec,
|
||||
Colorspace,
|
||||
ImageFormat,
|
||||
JobName,
|
||||
JobStatus,
|
||||
LogLevel,
|
||||
|
|
@ -24,7 +24,16 @@ import {
|
|||
} from 'src/enum';
|
||||
import { UpsertFileOptions, WithoutProperty } from 'src/repositories/asset.repository';
|
||||
import { BaseService } from 'src/services/base.service';
|
||||
import { AudioStreamInfo, JobItem, JobOf, VideoFormat, VideoInterfaces, VideoStreamInfo } from 'src/types';
|
||||
import {
|
||||
AudioStreamInfo,
|
||||
DecodeToBufferOptions,
|
||||
GenerateThumbnailOptions,
|
||||
JobItem,
|
||||
JobOf,
|
||||
VideoFormat,
|
||||
VideoInterfaces,
|
||||
VideoStreamInfo,
|
||||
} from 'src/types';
|
||||
import { getAssetFiles } from 'src/utils/asset.util';
|
||||
import { BaseConfig, ThumbnailConfig } from 'src/utils/media';
|
||||
import { mimeTypes } from 'src/utils/mime-types';
|
||||
|
|
@ -131,6 +140,7 @@ export class MediaService extends BaseService {
|
|||
return JobStatus.FAILED;
|
||||
}
|
||||
|
||||
await this.storageCore.moveAssetImage(asset, AssetPathType.FULLSIZE, ImageFormat.JPEG);
|
||||
await this.storageCore.moveAssetImage(asset, AssetPathType.PREVIEW, image.preview.format);
|
||||
await this.storageCore.moveAssetImage(asset, AssetPathType.THUMBNAIL, image.thumbnail.format);
|
||||
await this.storageCore.moveAssetVideo(asset);
|
||||
|
|
@ -151,7 +161,12 @@ export class MediaService extends BaseService {
|
|||
return JobStatus.SKIPPED;
|
||||
}
|
||||
|
||||
let generated: { previewPath: string; thumbnailPath: string; thumbhash: Buffer };
|
||||
let generated: {
|
||||
previewPath: string;
|
||||
thumbnailPath: string;
|
||||
fullsizePath?: string;
|
||||
thumbhash: Buffer;
|
||||
};
|
||||
if (asset.type === AssetType.VIDEO || asset.originalFileName.toLowerCase().endsWith('.gif')) {
|
||||
generated = await this.generateVideoThumbnails(asset);
|
||||
} else if (asset.type === AssetType.IMAGE) {
|
||||
|
|
@ -161,7 +176,7 @@ export class MediaService extends BaseService {
|
|||
return JobStatus.SKIPPED;
|
||||
}
|
||||
|
||||
const { previewFile, thumbnailFile } = getAssetFiles(asset.files);
|
||||
const { previewFile, thumbnailFile, fullsizeFile } = getAssetFiles(asset.files);
|
||||
const toUpsert: UpsertFileOptions[] = [];
|
||||
if (previewFile?.path !== generated.previewPath) {
|
||||
toUpsert.push({ assetId: asset.id, path: generated.previewPath, type: AssetFileType.PREVIEW });
|
||||
|
|
@ -171,11 +186,15 @@ export class MediaService extends BaseService {
|
|||
toUpsert.push({ assetId: asset.id, path: generated.thumbnailPath, type: AssetFileType.THUMBNAIL });
|
||||
}
|
||||
|
||||
if (generated.fullsizePath && fullsizeFile?.path !== generated.fullsizePath) {
|
||||
toUpsert.push({ assetId: asset.id, path: generated.fullsizePath, type: AssetFileType.FULLSIZE });
|
||||
}
|
||||
|
||||
if (toUpsert.length > 0) {
|
||||
await this.assetRepository.upsertFiles(toUpsert);
|
||||
}
|
||||
|
||||
const pathsToDelete = [];
|
||||
const pathsToDelete: string[] = [];
|
||||
if (previewFile && previewFile.path !== generated.previewPath) {
|
||||
this.logger.debug(`Deleting old preview for asset ${asset.id}`);
|
||||
pathsToDelete.push(previewFile.path);
|
||||
|
|
@ -186,6 +205,15 @@ export class MediaService extends BaseService {
|
|||
pathsToDelete.push(thumbnailFile.path);
|
||||
}
|
||||
|
||||
if (fullsizeFile && fullsizeFile.path !== generated.fullsizePath) {
|
||||
this.logger.debug(`Deleting old fullsize preview image for asset ${asset.id}`);
|
||||
pathsToDelete.push(fullsizeFile.path);
|
||||
if (!generated.fullsizePath) {
|
||||
// did not generate a new fullsize image, delete the existing record
|
||||
await this.assetRepository.deleteFiles([fullsizeFile]);
|
||||
}
|
||||
}
|
||||
|
||||
if (pathsToDelete.length > 0) {
|
||||
await Promise.all(pathsToDelete.map((path) => this.storageRepository.unlink(path)));
|
||||
}
|
||||
|
|
@ -205,33 +233,71 @@ export class MediaService extends BaseService {
|
|||
const thumbnailPath = StorageCore.getImagePath(asset, AssetPathType.THUMBNAIL, image.thumbnail.format);
|
||||
this.storageCore.ensureFolders(previewPath);
|
||||
|
||||
const shouldExtract = image.extractEmbedded && mimeTypes.isRaw(asset.originalPath);
|
||||
const extractedPath = StorageCore.getTempPathInDir(dirname(previewPath));
|
||||
const didExtract = shouldExtract && (await this.mediaRepository.extract(asset.originalPath, extractedPath));
|
||||
const processInvalidImages = process.env.IMMICH_PROCESS_INVALID_IMAGES === 'true';
|
||||
const colorspace = this.isSRGB(asset) ? Colorspace.SRGB : image.colorspace;
|
||||
|
||||
try {
|
||||
const useExtracted = didExtract && (await this.shouldUseExtractedImage(extractedPath, image.preview.size));
|
||||
const inputPath = useExtracted ? extractedPath : asset.originalPath;
|
||||
const colorspace = this.isSRGB(asset) ? Colorspace.SRGB : image.colorspace;
|
||||
const processInvalidImages = process.env.IMMICH_PROCESS_INVALID_IMAGES === 'true';
|
||||
// prevents this extra "enabled" from leaking into fullsizeOptions later
|
||||
const { enabled: imageFullsizeEnabled, ...imageFullsizeConfig } = image.fullsize;
|
||||
|
||||
const orientation = useExtracted && asset.exifInfo?.orientation ? Number(asset.exifInfo.orientation) : undefined;
|
||||
const decodeOptions = { colorspace, processInvalidImages, size: image.preview.size, orientation };
|
||||
const { data, info } = await this.mediaRepository.decodeImage(inputPath, decodeOptions);
|
||||
const shouldConvertFullsize = imageFullsizeEnabled && !mimeTypes.isWebSupportedImage(asset.originalFileName);
|
||||
const shouldExtractEmbedded = image.extractEmbedded && mimeTypes.isRaw(asset.originalFileName);
|
||||
const decodeOptions: DecodeToBufferOptions = { colorspace, processInvalidImages, size: image.preview.size };
|
||||
|
||||
const options = { colorspace, processInvalidImages, raw: info };
|
||||
const outputs = await Promise.all([
|
||||
this.mediaRepository.generateThumbnail(data, { ...image.thumbnail, ...options }, thumbnailPath),
|
||||
this.mediaRepository.generateThumbnail(data, { ...image.preview, ...options }, previewPath),
|
||||
this.mediaRepository.generateThumbhash(data, options),
|
||||
]);
|
||||
let useExtracted = false;
|
||||
let decodeInputPath: string = asset.originalPath;
|
||||
// Converted or extracted image from non-web-supported formats (e.g. RAW)
|
||||
let fullsizePath: string | undefined;
|
||||
|
||||
return { previewPath, thumbnailPath, thumbhash: outputs[2] };
|
||||
} finally {
|
||||
if (didExtract) {
|
||||
await this.storageRepository.unlink(extractedPath);
|
||||
if (shouldConvertFullsize) {
|
||||
// unset size to decode fullsize image
|
||||
decodeOptions.size = undefined;
|
||||
fullsizePath = StorageCore.getImagePath(asset, AssetPathType.FULLSIZE, image.fullsize.format);
|
||||
}
|
||||
|
||||
if (shouldExtractEmbedded) {
|
||||
// For RAW files, try extracting embedded preview first
|
||||
// Assume extracted image from RAW always in JPEG format, as implied from the `jpgFromRaw` tag name
|
||||
const extractedPath = StorageCore.getImagePath(asset, AssetPathType.FULLSIZE, ImageFormat.JPEG);
|
||||
const didExtract = await this.mediaRepository.extract(asset.originalPath, extractedPath);
|
||||
useExtracted = didExtract && (await this.shouldUseExtractedImage(extractedPath, image.preview.size));
|
||||
|
||||
if (useExtracted) {
|
||||
if (shouldConvertFullsize) {
|
||||
// skip re-encoding and directly use extracted as fullsize preview
|
||||
// as usually the extracted image is already heavily compressed, no point doing lossy conversion again
|
||||
fullsizePath = extractedPath;
|
||||
}
|
||||
// use this as origin of preview and thumbnail
|
||||
decodeInputPath = extractedPath;
|
||||
if (asset.exifInfo) {
|
||||
// write essential orientation and colorspace EXIF for correct fullsize preview and subsequent processing
|
||||
const exif = { orientation: asset.exifInfo.orientation, colorspace: asset.exifInfo.colorspace };
|
||||
await this.mediaRepository.writeExif(exif, extractedPath);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const { info, data } = await this.mediaRepository.decodeImage(decodeInputPath, decodeOptions);
|
||||
|
||||
const thumbnailOptions = { colorspace, processInvalidImages, raw: info };
|
||||
const promises = [
|
||||
this.mediaRepository.generateThumbhash(data, thumbnailOptions),
|
||||
this.mediaRepository.generateThumbnail(data, { ...image.thumbnail, ...thumbnailOptions }, thumbnailPath),
|
||||
this.mediaRepository.generateThumbnail(data, { ...image.preview, ...thumbnailOptions }, previewPath),
|
||||
];
|
||||
|
||||
// did not extract a usable image from RAW
|
||||
if (fullsizePath && !useExtracted) {
|
||||
const fullsizeOptions: GenerateThumbnailOptions = {
|
||||
...imageFullsizeConfig,
|
||||
...thumbnailOptions,
|
||||
size: undefined,
|
||||
};
|
||||
promises.push(this.mediaRepository.generateThumbnail(data, fullsizeOptions, fullsizePath));
|
||||
}
|
||||
const outputs = await Promise.all(promises);
|
||||
|
||||
return { previewPath, thumbnailPath, fullsizePath, thumbhash: outputs[0] as Buffer };
|
||||
}
|
||||
|
||||
private async generateVideoThumbnails(asset: AssetEntity) {
|
||||
|
|
|
|||
|
|
@ -146,6 +146,7 @@ const updatedConfig = Object.freeze<SystemConfig>({
|
|||
format: ImageFormat.JPEG,
|
||||
quality: 80,
|
||||
},
|
||||
fullsize: { enabled: false, format: ImageFormat.JPEG, quality: 80 },
|
||||
colorspace: Colorspace.P3,
|
||||
extractEmbedded: false,
|
||||
},
|
||||
|
|
|
|||
|
|
@ -58,6 +58,12 @@ export interface CropOptions {
|
|||
height: number;
|
||||
}
|
||||
|
||||
export interface FullsizeImageOptions {
|
||||
format: ImageFormat;
|
||||
quality: number;
|
||||
enabled: boolean;
|
||||
}
|
||||
|
||||
export interface ImageOptions {
|
||||
format: ImageFormat;
|
||||
quality: number;
|
||||
|
|
@ -78,11 +84,11 @@ interface DecodeImageOptions {
|
|||
}
|
||||
|
||||
export interface DecodeToBufferOptions extends DecodeImageOptions {
|
||||
size: number;
|
||||
size?: number;
|
||||
orientation?: ExifOrientation;
|
||||
}
|
||||
|
||||
export type GenerateThumbnailOptions = ImageOptions & DecodeImageOptions;
|
||||
export type GenerateThumbnailOptions = Pick<ImageOptions, 'format' | 'quality'> & DecodeToBufferOptions;
|
||||
|
||||
export type GenerateThumbnailFromBufferOptions = GenerateThumbnailOptions & { raw: RawImageInfo };
|
||||
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ const getFileByType = (files: AssetFileEntity[] | undefined, type: AssetFileType
|
|||
};
|
||||
|
||||
export const getAssetFiles = (files?: AssetFileEntity[]) => ({
|
||||
fullsizeFile: getFileByType(files, AssetFileType.FULLSIZE),
|
||||
previewFile: getFileByType(files, AssetFileType.PREVIEW),
|
||||
thumbnailFile: getFileByType(files, AssetFileType.THUMBNAIL),
|
||||
});
|
||||
|
|
|
|||
|
|
@ -55,6 +55,20 @@ const image: Record<string, string[]> = {
|
|||
'.webp': ['image/webp'],
|
||||
};
|
||||
|
||||
/**
|
||||
* list of supported image extensions from https://developer.mozilla.org/en-US/docs/Web/Media/Formats/Image_types excluding svg
|
||||
* @TODO share with the client
|
||||
* @see {@link web/src/lib/utils/asset-utils.ts#L329}
|
||||
**/
|
||||
const webSupportedImageMimeTypes = new Set([
|
||||
'image/apng',
|
||||
'image/avif',
|
||||
'image/gif',
|
||||
'image/jpeg',
|
||||
'image/png',
|
||||
'image/webp',
|
||||
]);
|
||||
|
||||
const profileExtensions = new Set(['.avif', '.dng', '.heic', '.heif', '.jpeg', '.jpg', '.png', '.webp', '.svg']);
|
||||
const profile: Record<string, string[]> = Object.fromEntries(
|
||||
Object.entries(image).filter(([key]) => profileExtensions.has(key)),
|
||||
|
|
@ -100,6 +114,7 @@ export const mimeTypes = {
|
|||
|
||||
isAsset: (filename: string) => isType(filename, image) || isType(filename, video),
|
||||
isImage: (filename: string) => isType(filename, image),
|
||||
isWebSupportedImage: (filename: string) => webSupportedImageMimeTypes.has(lookup(filename)),
|
||||
isProfile: (filename: string) => isType(filename, profile),
|
||||
isSidecar: (filename: string) => isType(filename, sidecar),
|
||||
isVideo: (filename: string) => isType(filename, video),
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue