feat(web,server): add thumbhash support (#2649)

* add thumbhash: server generation and web impl

* move logic to infra & use byta in db

* remove unnecesary logs

* update generated API and simplify thumbhash gen

* fix check errors

* removed unnecessary library and css tag

* style edits

* syntax mistake

* update server test, change thumbhash job name

* fix tests

* Update server/src/domain/asset/response-dto/asset-response.dto.ts

Co-authored-by: Thomas <9749173+uhthomas@users.noreply.github.com>

* add unit test, change migration date

* change to official thumbhash impl

* update call method to not use eval

* "generate missing" looks for thumbhash

* improve queue & improve syntax

* update syntax again

* update tests

* fix thumbhash generation

* consolidate queueing to avoid duplication

* cover all types of incorrect thumbnail cases

* split out jest tasks

* put back thumbnail duration loading for images without thumbhash

* Remove stray package.json

---------

Co-authored-by: Luke McCarthy <mail@lukehmcc.com>
Co-authored-by: Thomas <9749173+uhthomas@users.noreply.github.com>
Co-authored-by: Alex Tran <alex.tran1502@gmail.com>

server/src/domain/asset/response-dto/asset-response.dto.ts

@@ -16,6 +16,8 @@ export class AssetResponseDto {
   originalPath!: string;
   originalFileName!: string;
   resized!: boolean;
+  /**base64 encoded thumbhash */
+  thumbhash!: string | null;
   fileCreatedAt!: Date;
   fileModifiedAt!: Date;
   updatedAt!: Date;
@@ -42,6 +44,7 @@ export function mapAsset(entity: AssetEntity): AssetResponseDto {
     originalPath: entity.originalPath,
     originalFileName: entity.originalFileName,
     resized: !!entity.resizePath,
+    thumbhash: entity.thumbhash?.toString('base64') ?? null,
     fileCreatedAt: entity.fileCreatedAt,
     fileModifiedAt: entity.fileModifiedAt,
     updatedAt: entity.updatedAt,
@@ -68,6 +71,7 @@ export function mapAssetWithoutExif(entity: AssetEntity): AssetResponseDto {
     originalPath: entity.originalPath,
     originalFileName: entity.originalFileName,
     resized: !!entity.resizePath,
+    thumbhash: entity.thumbhash?.toString('base64') || null,
     fileCreatedAt: entity.fileCreatedAt,
     fileModifiedAt: entity.fileModifiedAt,
     updatedAt: entity.updatedAt,

server/src/domain/job/job.constants.ts

@@ -27,6 +27,7 @@ export enum JobName {
   QUEUE_GENERATE_THUMBNAILS = 'queue-generate-thumbnails',
   GENERATE_JPEG_THUMBNAIL = 'generate-jpeg-thumbnail',
   GENERATE_WEBP_THUMBNAIL = 'generate-webp-thumbnail',
+  GENERATE_THUMBHASH_THUMBNAIL = 'generate-thumbhash-thumbnail',
 
   // metadata
   QUEUE_METADATA_EXTRACTION = 'queue-metadata-extraction',
@@ -92,6 +93,7 @@ export const JOBS_TO_QUEUE: Record<JobName, QueueName> = {
   [JobName.QUEUE_GENERATE_THUMBNAILS]: QueueName.THUMBNAIL_GENERATION,
   [JobName.GENERATE_JPEG_THUMBNAIL]: QueueName.THUMBNAIL_GENERATION,
   [JobName.GENERATE_WEBP_THUMBNAIL]: QueueName.THUMBNAIL_GENERATION,
+  [JobName.GENERATE_THUMBHASH_THUMBNAIL]: QueueName.THUMBNAIL_GENERATION,
 
   // metadata
   [JobName.QUEUE_METADATA_EXTRACTION]: QueueName.METADATA_EXTRACTION,

server/src/domain/job/job.repository.ts

@@ -31,6 +31,7 @@ export type JobItem =
   | { name: JobName.QUEUE_GENERATE_THUMBNAILS; data: IBaseJob }
   | { name: JobName.GENERATE_JPEG_THUMBNAIL; data: IEntityJob }
   | { name: JobName.GENERATE_WEBP_THUMBNAIL; data: IEntityJob }
+  | { name: JobName.GENERATE_THUMBHASH_THUMBNAIL; data: IEntityJob }
 
   // User Deletion
   | { name: JobName.USER_DELETE_CHECK; data?: IBaseJob }

server/src/domain/job/job.service.spec.ts

@@ -261,7 +261,13 @@ describe(JobService.name, () => {
       },
       {
         item: { name: JobName.GENERATE_JPEG_THUMBNAIL, data: { id: 'asset-1' } },
-        jobs: [JobName.GENERATE_WEBP_THUMBNAIL, JobName.CLASSIFY_IMAGE, JobName.ENCODE_CLIP, JobName.RECOGNIZE_FACES],
+        jobs: [
+          JobName.GENERATE_WEBP_THUMBNAIL,
+          JobName.CLASSIFY_IMAGE,
+          JobName.ENCODE_CLIP,
+          JobName.RECOGNIZE_FACES,
+          JobName.GENERATE_THUMBHASH_THUMBNAIL,
+        ],
       },
       {
         item: { name: JobName.CLASSIFY_IMAGE, data: { id: 'asset-1' } },

server/src/domain/job/job.service.ts

@@ -160,6 +160,7 @@ export class JobService {
 
       case JobName.GENERATE_JPEG_THUMBNAIL: {
         await this.jobRepository.queue({ name: JobName.GENERATE_WEBP_THUMBNAIL, data: item.data });
+        await this.jobRepository.queue({ name: JobName.GENERATE_THUMBHASH_THUMBNAIL, data: item.data });
         await this.jobRepository.queue({ name: JobName.CLASSIFY_IMAGE, data: item.data });
         await this.jobRepository.queue({ name: JobName.ENCODE_CLIP, data: item.data });
         await this.jobRepository.queue({ name: JobName.RECOGNIZE_FACES, data: item.data });

server/src/domain/media/media.repository.ts

@@ -47,6 +47,7 @@ export interface IMediaRepository {
   // image
   resize(input: string | Buffer, output: string, options: ResizeOptions): Promise<void>;
   crop(input: string, options: CropOptions): Promise<Buffer>;
+  generateThumbhash(imagePath: string): Promise<Buffer>;
 
   // video
   extractVideoThumbnail(input: string, output: string, size: number): Promise<void>;

server/src/domain/media/media.service.spec.ts

@@ -54,9 +54,9 @@ describe(MediaService.name, () => {
       });
     });
 
-    it('should queue all assets with missing thumbnails', async () => {
+    it('should queue all assets with missing resize path', async () => {
       assetMock.getWithout.mockResolvedValue({
-        items: [assetEntityStub.image],
+        items: [assetEntityStub.noResizePath],
         hasNextPage: false,
       });
 
@@ -69,6 +69,38 @@ describe(MediaService.name, () => {
         data: { id: assetEntityStub.image.id },
       });
     });
+
+    it('should queue all assets with missing webp path', async () => {
+      assetMock.getWithout.mockResolvedValue({
+        items: [assetEntityStub.noWebpPath],
+        hasNextPage: false,
+      });
+
+      await sut.handleQueueGenerateThumbnails({ force: false });
+
+      expect(assetMock.getAll).not.toHaveBeenCalled();
+      expect(assetMock.getWithout).toHaveBeenCalledWith({ skip: 0, take: 1000 }, WithoutProperty.THUMBNAIL);
+      expect(jobMock.queue).toHaveBeenCalledWith({
+        name: JobName.GENERATE_WEBP_THUMBNAIL,
+        data: { id: assetEntityStub.image.id },
+      });
+    });
+
+    it('should queue all assets with missing thumbhash', async () => {
+      assetMock.getWithout.mockResolvedValue({
+        items: [assetEntityStub.noThumbhash],
+        hasNextPage: false,
+      });
+
+      await sut.handleQueueGenerateThumbnails({ force: false });
+
+      expect(assetMock.getAll).not.toHaveBeenCalled();
+      expect(assetMock.getWithout).toHaveBeenCalledWith({ skip: 0, take: 1000 }, WithoutProperty.THUMBNAIL);
+      expect(jobMock.queue).toHaveBeenCalledWith({
+        name: JobName.GENERATE_THUMBHASH_THUMBNAIL,
+        data: { id: assetEntityStub.image.id },
+      });
+    });
   });
 
   describe('handleGenerateJpegThumbnail', () => {
@@ -129,6 +161,25 @@ describe(MediaService.name, () => {
     });
   });
 
+  describe('handleGenerateThumbhashThumbnail', () => {
+    it('should skip thumbhash generation if resize path is missing', async () => {
+      assetMock.getByIds.mockResolvedValue([assetEntityStub.noResizePath]);
+      await sut.handleGenerateThumbhashThumbnail({ id: assetEntityStub.noResizePath.id });
+      expect(mediaMock.generateThumbhash).not.toHaveBeenCalled();
+    });
+
+    it('should generate a thumbhash', async () => {
+      const thumbhashBuffer = Buffer.from('a thumbhash', 'utf8');
+      assetMock.getByIds.mockResolvedValue([assetEntityStub.image]);
+      mediaMock.generateThumbhash.mockResolvedValue(thumbhashBuffer);
+
+      await sut.handleGenerateThumbhashThumbnail({ id: assetEntityStub.image.id });
+
+      expect(mediaMock.generateThumbhash).toHaveBeenCalledWith('/uploads/user-id/thumbs/path.ext');
+      expect(assetMock.save).toHaveBeenCalledWith({ id: 'asset-id', thumbhash: thumbhashBuffer });
+    });
+  });
+
   describe('handleQueueVideoConversion', () => {
     it('should queue all video assets', async () => {
       assetMock.getAll.mockResolvedValue({

server/src/domain/media/media.service.ts

@@ -37,7 +37,16 @@ export class MediaService {
 
     for await (const assets of assetPagination) {
       for (const asset of assets) {
-        await this.jobRepository.queue({ name: JobName.GENERATE_JPEG_THUMBNAIL, data: { id: asset.id } });
+        if (!asset.resizePath || force) {
+          await this.jobRepository.queue({ name: JobName.GENERATE_JPEG_THUMBNAIL, data: { id: asset.id } });
+          continue;
+        }
+        if (!asset.webpPath) {
+          await this.jobRepository.queue({ name: JobName.GENERATE_WEBP_THUMBNAIL, data: { id: asset.id } });
+        }
+        if (!asset.thumbhash) {
+          await this.jobRepository.queue({ name: JobName.GENERATE_THUMBHASH_THUMBNAIL, data: { id: asset.id } });
+        }
       }
     }
 
@@ -87,6 +96,18 @@ export class MediaService {
     return true;
   }
 
+  async handleGenerateThumbhashThumbnail({ id }: IEntityJob): Promise<boolean> {
+    const [asset] = await this.assetRepository.getByIds([id]);
+    if (!asset?.resizePath) {
+      return false;
+    }
+
+    const thumbhash = await this.mediaRepository.generateThumbhash(asset.resizePath);
+    await this.assetRepository.save({ id: asset.id, thumbhash });
+
+    return true;
+  }
+
   async handleQueueVideoConversion(job: IBaseJob) {
     const { force } = job;
 

server/src/immich/api-v1/asset/asset.core.ts

@@ -35,6 +35,7 @@ export class AssetCore {
       livePhotoVideo: livePhotoAssetId != null ? ({ id: livePhotoAssetId } as AssetEntity) : null,
       resizePath: null,
       webpPath: null,
+      thumbhash: null,
       encodedVideoPath: null,
       tags: [],
       sharedLinks: [],

server/src/infra/entities/asset.entity.ts

@@ -51,6 +51,9 @@ export class AssetEntity {
   @Column({ type: 'varchar', nullable: true, default: '' })
   webpPath!: string | null;
 
+  @Column({ type: 'bytea', nullable: true })
+  thumbhash!: Buffer | null;
+
   @Column({ type: 'varchar', nullable: true, default: '' })
   encodedVideoPath!: string | null;
 

server/src/infra/migrations/1686762895180-AddThumbhashColumn.ts

@@ -0,0 +1,13 @@
+import { MigrationInterface, QueryRunner } from 'typeorm';
+
+export class AddThumbhashColumn1685546571785 implements MigrationInterface {
+  name = 'AddThumbhashColumn1686762895180';
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`ALTER TABLE "assets" ADD "thumbhash" bytea NULL`);
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`ALTER TABLE "assets" DROP COLUMN "thumbhash"`);
+  }
+}

server/src/infra/repositories/asset.repository.ts

@@ -135,6 +135,7 @@ export class AssetRepository implements IAssetRepository {
           { resizePath: '', isVisible: true },
           { webpPath: IsNull(), isVisible: true },
           { webpPath: '', isVisible: true },
+          { thumbhash: IsNull(), isVisible: true },
         ];
         break;
 

server/src/infra/repositories/media.repository.ts

@@ -119,4 +119,17 @@ export class MediaRepository implements IMediaRepository {
         .run();
     });
   }
+
+  async generateThumbhash(imagePath: string): Promise<Buffer> {
+    const maxSize = 100;
+
+    const { data, info } = await sharp(imagePath)
+      .resize(maxSize, maxSize, { fit: 'inside', withoutEnlargement: true })
+      .raw()
+      .ensureAlpha()
+      .toBuffer({ resolveWithObject: true });
+
+    const thumbhash = await import('thumbhash');
+    return Buffer.from(thumbhash.rgbaToThumbHash(info.width, info.height, data));
+  }
 }

server/src/microservices/app.service.ts

@@ -61,6 +61,7 @@ export class AppService {
       [JobName.QUEUE_GENERATE_THUMBNAILS]: (data) => this.mediaService.handleQueueGenerateThumbnails(data),
       [JobName.GENERATE_JPEG_THUMBNAIL]: (data) => this.mediaService.handleGenerateJpegThumbnail(data),
       [JobName.GENERATE_WEBP_THUMBNAIL]: (data) => this.mediaService.handleGenerateWepbThumbnail(data),
+      [JobName.GENERATE_THUMBHASH_THUMBNAIL]: (data) => this.mediaService.handleGenerateThumbhashThumbnail(data),
       [JobName.QUEUE_VIDEO_CONVERSION]: (data) => this.mediaService.handleQueueVideoConversion(data),
       [JobName.VIDEO_CONVERSION]: (data) => this.mediaService.handleVideoConversion(data),
       [JobName.QUEUE_METADATA_EXTRACTION]: (data) => this.metadataProcessor.handleQueueMetadataExtraction(data),