feat(server): add checksum algorithm field (#26573)

* feat: add checksum algorithm field

* fix comments

* chore: rename migration

---------

Co-authored-by: Jason Rasmussen <jason@rasm.me>

server/src/database.ts

@@ -5,6 +5,7 @@ import {
   AssetFileType,
   AssetType,
   AssetVisibility,
+  ChecksumAlgorithm,
   MemoryType,
   Permission,
   PluginContext,
@@ -112,6 +113,7 @@ export type Memory = {
 export type Asset = {
   id: string;
   checksum: Buffer<ArrayBufferLike>;
+  checksumAlgorithm: ChecksumAlgorithm;
   deviceAssetId: string;
   deviceId: string;
   fileCreatedAt: Date;
@@ -330,6 +332,7 @@ export const columns = {
   asset: [
     'asset.id',
     'asset.checksum',
+    'asset.checksumAlgorithm',
     'asset.deviceAssetId',
     'asset.deviceId',
     'asset.fileCreatedAt',

server/src/dtos/asset-response.dto.ts

@@ -13,7 +13,7 @@ import {
 } from 'src/dtos/person.dto';
 import { TagResponseDto, mapTag } from 'src/dtos/tag.dto';
 import { UserResponseDto, mapUser } from 'src/dtos/user.dto';
-import { AssetStatus, AssetType, AssetVisibility } from 'src/enum';
+import { AssetStatus, AssetType, AssetVisibility, ChecksumAlgorithm } from 'src/enum';
 import { ImageDimensions, MaybeDehydrated } from 'src/types';
 import { getDimensions } from 'src/utils/asset.util';
 import { hexOrBufferToBase64 } from 'src/utils/bytes';
@@ -148,6 +148,7 @@ export type MapAsset = {
   updateId: string;
   status: AssetStatus;
   checksum: Buffer<ArrayBufferLike>;
+  checksumAlgorithm: ChecksumAlgorithm;
   deviceAssetId: string;
   deviceId: string;
   duplicateId: string | null;

server/src/enum.ts

@@ -37,6 +37,11 @@ export enum AssetType {
   Other = 'OTHER',
 }
 
+export enum ChecksumAlgorithm {
+  sha1File = 'sha1', // sha1 checksum of the whole file contents
+  sha1Path = 'sha1-path', // sha1 checksum of "path:" plus the file path, currently used in external libraries, deprecated
+}
+
 export enum AssetFileType {
   /**
    * An full/large-size image extracted/converted from RAW photos

server/src/queries/asset.job.repository.sql

@@ -249,6 +249,7 @@ where
 select
   "asset"."id",
   "asset"."checksum",
+  "asset"."checksumAlgorithm",
   "asset"."deviceAssetId",
   "asset"."deviceId",
   "asset"."fileCreatedAt",

server/src/schema/enums.ts

@@ -1,5 +1,5 @@
 import { registerEnum } from '@immich/sql-tools';
-import { AssetStatus, AssetVisibility, SourceType } from 'src/enum';
+import { AssetStatus, AssetVisibility, ChecksumAlgorithm, SourceType } from 'src/enum';
 
 export const assets_status_enum = registerEnum({
   name: 'assets_status_enum',
@@ -15,3 +15,8 @@ export const asset_visibility_enum = registerEnum({
   name: 'asset_visibility_enum',
   values: Object.values(AssetVisibility),
 });
+
+export const asset_checksum_algorithm_enum = registerEnum({
+  name: 'asset_checksum_algorithm_enum',
+  values: Object.values(ChecksumAlgorithm),
+});

server/src/schema/migrations/1774548649115-AddChecksumAlgorithm.ts.ts

@@ -0,0 +1,21 @@
+import { Kysely, sql } from 'kysely';
+
+export async function up(db: Kysely<any>): Promise<void> {
+  await sql`CREATE TYPE "asset_checksum_algorithm_enum" AS ENUM ('sha1','sha1-path');`.execute(db);
+  await sql`ALTER TABLE "asset" ADD "checksumAlgorithm" asset_checksum_algorithm_enum;`.execute(db);
+
+  await sql`
+    UPDATE "asset"
+    SET "checksumAlgorithm" = CASE
+      WHEN "isExternal" = true THEN 'sha1-path'::asset_checksum_algorithm_enum
+      ELSE 'sha1'::asset_checksum_algorithm_enum
+    END
+  `.execute(db);
+
+  await sql`ALTER TABLE "asset" ALTER COLUMN "checksumAlgorithm" SET NOT NULL;`.execute(db);
+}
+
+export async function down(db: Kysely<any>): Promise<void> {
+  await sql`ALTER TABLE "asset" DROP COLUMN "checksumAlgorithm";`.execute(db);
+  await sql`DROP TYPE "asset_checksum_algorithm_enum";`.execute(db);
+}

server/src/schema/tables/asset.table.ts

@@ -12,8 +12,8 @@ import {
   UpdateDateColumn,
 } from '@immich/sql-tools';
 import { UpdatedAtTrigger, UpdateIdColumn } from 'src/decorators';
-import { AssetStatus, AssetType, AssetVisibility } from 'src/enum';
+import { AssetStatus, AssetType, AssetVisibility, ChecksumAlgorithm } from 'src/enum';
-import { asset_visibility_enum, assets_status_enum } from 'src/schema/enums';
+import { asset_checksum_algorithm_enum, asset_visibility_enum, assets_status_enum } from 'src/schema/enums';
 import { asset_delete_audit } from 'src/schema/functions';
 import { LibraryTable } from 'src/schema/tables/library.table';
 import { StackTable } from 'src/schema/tables/stack.table';
@@ -95,6 +95,9 @@ export class AssetTable {
   @Column({ type: 'bytea', index: true })
   checksum!: Buffer; // sha1 checksum
 
+  @Column({ enum: asset_checksum_algorithm_enum })
+  checksumAlgorithm!: ChecksumAlgorithm;
+
   @ForeignKeyColumn(() => AssetTable, { nullable: true, onUpdate: 'CASCADE', onDelete: 'SET NULL' })
   livePhotoVideoId!: string | null;
 

server/src/services/asset-media.service.ts

@@ -27,6 +27,7 @@ import {
   AssetStatus,
   AssetVisibility,
   CacheControl,
+  ChecksumAlgorithm,
   JobName,
   Permission,
   StorageFolder,
@@ -425,6 +426,7 @@ export class AssetMediaService extends BaseService {
       deviceId: asset.deviceId,
       type: asset.type,
       checksum: asset.checksum,
+      checksumAlgorithm: asset.checksumAlgorithm,
       fileCreatedAt: asset.fileCreatedAt,
       localDateTime: asset.localDateTime,
       fileModifiedAt: asset.fileModifiedAt,
@@ -446,6 +448,7 @@ export class AssetMediaService extends BaseService {
       libraryId: null,
 
       checksum: file.checksum,
+      checksumAlgorithm: ChecksumAlgorithm.sha1File,
       originalPath: file.originalPath,
 
       deviceAssetId: dto.deviceAssetId,

server/src/services/library.service.ts

@@ -17,7 +17,17 @@ import {
   ValidateLibraryImportPathResponseDto,
   ValidateLibraryResponseDto,
 } from 'src/dtos/library.dto';
-import { AssetStatus, AssetType, CronJob, DatabaseLock, ImmichWorker, JobName, JobStatus, QueueName } from 'src/enum';
+import {
+  AssetStatus,
+  AssetType,
+  ChecksumAlgorithm,
+  CronJob,
+  DatabaseLock,
+  ImmichWorker,
+  JobName,
+  JobStatus,
+  QueueName,
+} from 'src/enum';
 import { ArgOf } from 'src/repositories/event.repository';
 import { AssetSyncResult } from 'src/repositories/library.repository';
 import { AssetTable } from 'src/schema/tables/asset.table';
@@ -400,6 +410,7 @@ export class LibraryService extends BaseService {
       ownerId,
       libraryId,
       checksum: this.cryptoRepository.hashSha1(`path:${assetPath}`),
+      checksumAlgorithm: ChecksumAlgorithm.sha1Path,
       originalPath: assetPath,
 
       fileCreatedAt: stat.mtime,

server/src/services/metadata.service.spec.ts

@@ -7,6 +7,7 @@ import {
   AssetFileType,
   AssetType,
   AssetVisibility,
+  ChecksumAlgorithm,
   ExifOrientation,
   ImmichWorker,
   JobName,
@@ -652,6 +653,7 @@ describe(MetadataService.name, () => {
       expect(mocks.assetJob.getForMetadataExtraction).toHaveBeenCalledWith(asset.id);
       expect(mocks.asset.create).toHaveBeenCalledWith({
         checksum: expect.any(Buffer),
+        checksumAlgorithm: ChecksumAlgorithm.sha1File,
         deviceAssetId: 'NONE',
         deviceId: 'NONE',
         fileCreatedAt: asset.fileCreatedAt,
@@ -705,6 +707,7 @@ describe(MetadataService.name, () => {
       expect(mocks.assetJob.getForMetadataExtraction).toHaveBeenCalledWith(asset.id);
       expect(mocks.asset.create).toHaveBeenCalledWith({
         checksum: expect.any(Buffer),
+        checksumAlgorithm: ChecksumAlgorithm.sha1File,
         deviceAssetId: 'NONE',
         deviceId: 'NONE',
         fileCreatedAt: asset.fileCreatedAt,
@@ -758,6 +761,7 @@ describe(MetadataService.name, () => {
       expect(mocks.storage.readFile).toHaveBeenCalledWith(asset.originalPath, expect.any(Object));
       expect(mocks.asset.create).toHaveBeenCalledWith({
         checksum: expect.any(Buffer),
+        checksumAlgorithm: ChecksumAlgorithm.sha1File,
         deviceAssetId: 'NONE',
         deviceId: 'NONE',
         fileCreatedAt: asset.fileCreatedAt,

server/src/services/metadata.service.ts

@@ -14,6 +14,7 @@ import {
   AssetFileType,
   AssetType,
   AssetVisibility,
+  ChecksumAlgorithm,
   DatabaseLock,
   ExifOrientation,
   ImmichWorker,
@@ -675,6 +676,7 @@ export class MetadataService extends BaseService {
             fileModifiedAt: stats.mtime,
             localDateTime: dates.localDateTime,
             checksum,
+            checksumAlgorithm: ChecksumAlgorithm.sha1File,
             ownerId: asset.ownerId,
             originalPath: StorageCore.getAndroidMotionPath(asset, motionAssetId),
             originalFileName: `${parse(asset.originalFileName).name}.mp4`,

server/test/factories/asset.factory.ts

@@ -1,5 +1,5 @@
 import { Selectable } from 'kysely';
-import { AssetFileType, AssetStatus, AssetType, AssetVisibility } from 'src/enum';
+import { AssetFileType, AssetStatus, AssetType, AssetVisibility, ChecksumAlgorithm } from 'src/enum';
 import { AssetTable } from 'src/schema/tables/asset.table';
 import { StackTable } from 'src/schema/tables/stack.table';
 import { AssetEditFactory } from 'test/factories/asset-edit.factory';
@@ -53,6 +53,7 @@ export class AssetFactory {
       updateId: newUuidV7(),
       status: AssetStatus.Active,
       checksum: newSha1(),
+      checksumAlgorithm: ChecksumAlgorithm.sha1File,
       deviceAssetId: '',
       deviceId: '',
       duplicateId: null,

server/test/mappers.ts

@@ -125,6 +125,7 @@ export const getForMemory = (memory: ReturnType<MemoryFactory['build']>) => ({
 export const getForMetadataExtraction = (asset: ReturnType<AssetFactory['build']>) => ({
   id: asset.id,
   checksum: asset.checksum,
+  checksumAlgorithm: asset.checksumAlgorithm,
   deviceAssetId: asset.deviceAssetId,
   deviceId: asset.deviceId,
   fileCreatedAt: asset.fileCreatedAt,

server/test/medium.factory.ts

@@ -12,6 +12,7 @@ import {
   AlbumUserRole,
   AssetType,
   AssetVisibility,
+  ChecksumAlgorithm,
   MemoryType,
   SourceType,
   SyncEntityType,
@@ -547,6 +548,7 @@ const assetInsert = (asset: Partial<Insertable<AssetTable>> = {}) => {
     deviceId: '',
     originalFileName: '',
     checksum: randomBytes(32),
+    checksumAlgorithm: ChecksumAlgorithm.sha1File,
     type: AssetType.Image,
     originalPath: '/path/to/something.jpg',
     ownerId: 'not-a-valid-uuid',