fix(web): hash large files before upload

pnpm-lock.yaml

@@ -737,6 +737,9 @@ importers:
       happy-dom:
         specifier: ^18.0.1
         version: 18.0.1
+      hash-wasm:
+        specifier: ^4.12.0
+        version: 4.12.0
       intl-messageformat:
         specifier: ^10.7.11
         version: 10.7.16
@@ -6762,6 +6765,9 @@ packages:
     resolution: {integrity: sha512-IrsVwUHhEULx3R8f/aA8AHuEzAorplsab/v8HBzEiIukwq5i/EC+xmOW+HfP1OaDP+2JkgT1yILHN2O3UFIbcA==}
     engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}
 
+  hash-wasm@4.12.0:
+    resolution: {integrity: sha512-+/2B2rYLb48I/evdOIhP+K/DD2ca2fgBjp6O+GBEnCDk2e4rpeXIK8GvIyRPjTezgmWn9gmKwkQjjx6BtqDHVQ==}
+
   hasown@2.0.2:
     resolution: {integrity: sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==}
     engines: {node: '>= 0.4'}
@@ -18623,6 +18629,8 @@ snapshots:
 
   has-yarn@3.0.0: {}
 
+  hash-wasm@4.12.0: {}
+
   hasown@2.0.2:
     dependencies:
       function-bind: 1.1.2

web/package.json

@@ -46,6 +46,7 @@
     "geojson": "^0.5.0",
     "handlebars": "^4.7.8",
     "happy-dom": "^18.0.1",
+    "hash-wasm": "^4.12.0",
     "intl-messageformat": "^10.7.11",
     "justified-layout": "^4.1.0",
     "lodash-es": "^4.17.21",

web/src/lib/components/shared-components/upload-asset-preview.svelte

@@ -92,8 +92,8 @@
 
   {#if uploadAsset.state === UploadState.STARTED}
     <div class="text-black relative mt-[5px] h-[15px] w-full rounded-md bg-gray-300 dark:bg-gray-700">
-      <div class="h-[15px] rounded-md bg-immich-primary transition-all" style={`width: ${uploadAsset.progress}%`}></div>
-      <p class="absolute top-0 h-full w-full text-center text-primary text-[10px]">
+      <div class="h-[15px] rounded-md bg-primary/50 transition-all" style={`width: ${uploadAsset.progress}%`}></div>
+      <p class="absolute top-0 h-full w-full text-center text-dark text-[10px]">
         {#if uploadAsset.message}
           {uploadAsset.message}
         {:else}

web/src/lib/utils/file-uploader.ts

@@ -7,6 +7,7 @@ import { uploadRequest } from '$lib/utils';
 import { addAssetsToAlbum } from '$lib/utils/asset-utils';
 import { ExecutorQueue } from '$lib/utils/executor-queue';
 import { asQueryString } from '$lib/utils/shared-links';
+import { hashFile } from '$lib/utils/sw-messaging';
 import {
   Action,
   AssetMediaStatus,
@@ -154,16 +155,16 @@ async function fileUploader({
     }
 
     let responseData: { id: string; status: AssetMediaStatus; isTrashed?: boolean } | undefined;
-    if (crypto?.subtle?.digest && !authManager.isSharedLink) {
+    if (!authManager.isSharedLink) {
       uploadAssetsStore.updateItem(deviceAssetId, { message: $t('asset_hashing') });
       await tick();
       try {
-        const bytes = await assetFile.arrayBuffer();
-        const hash = await crypto.subtle.digest('SHA-1', bytes);
-        const checksum = Array.from(new Uint8Array(hash))
-          .map((b) => b.toString(16).padStart(2, '0'))
-          .join('');
-
+        const checksum = await hashFile(assetFile, {
+          id: deviceAssetId,
+          onProgress: (progress, total) => {
+            uploadAssetsStore.updateProgress(deviceAssetId, progress, total);
+          },
+        });
         const {
           results: [checkUploadResult],
         } = await checkBulkUpload({ assetBulkUploadCheckDto: { assets: [{ id: assetFile.name, checksum }] } });

web/src/lib/utils/sw-messaging.ts

@@ -1,8 +1,46 @@
+type OnProgress = (progress: number, total: number) => void;
+type Callback = { onChecksum: (checksum: string) => void; onProgress: OnProgress };
+
+const callbacks: Record<string, Callback> = {};
 const broadcast = new BroadcastChannel('immich');
 
-export function cancelImageUrl(url: string) {
+broadcast.addEventListener('message', (event) => {
+  const { type, id, checksum, progress, total } = event.data;
+
+  switch (type) {
+    case 'checksum': {
+      if (id && checksum) {
+        const callback = callbacks[id];
+        callback?.onChecksum(checksum);
+        delete callbacks[id];
+      }
+      break;
+    }
+
+    case 'hash.progress': {
+      if (id && progress && total) {
+        const callback = callbacks[id];
+        callback?.onProgress(progress, total);
+      }
+      break;
+    }
+  }
+});
+
+export const cancelImageUrl = (url: string) => {
   broadcast.postMessage({ type: 'cancel', url });
-}
-export function preloadImageUrl(url: string) {
+};
+export const preloadImageUrl = (url: string) => {
   broadcast.postMessage({ type: 'preload', url });
-}
+};
+
+export const hashFile = (file: File, { id, onProgress }: { id: string; onProgress: OnProgress }): Promise<string> => {
+  return new Promise((onChecksum) => {
+    if (callbacks[id]) {
+      return;
+    }
+
+    callbacks[id] = { onChecksum, onProgress };
+    broadcast.postMessage({ type: 'hash', id, file });
+  });
+};

web/src/service-worker/broadcast-channel.ts

@@ -1,9 +1,13 @@
+import { createSHA1, sha1 } from 'hash-wasm';
 import { handleCancel, handlePreload } from './request';
 
-export const installBroadcastChannelListener = () => {
-  const broadcast = new BroadcastChannel('immich');
-  // eslint-disable-next-line  unicorn/prefer-add-event-listener
-  broadcast.onmessage = (event) => {
+type HashRequest = { id: string; file: File };
+
+const MAX_HASH_FILE_SIZE = 100 * 1024 * 1024; // 100 MiB
+
+const broadcast = new BroadcastChannel('immich');
+
+broadcast.addEventListener('message', async (event) => {
   if (!event.data) {
     return;
   }
@@ -20,6 +24,45 @@ export const installBroadcastChannelListener = () => {
       handleCancel(url);
       break;
     }
+
+    case 'hash': {
+      await handleHash(event.data);
     }
-  };
+  }
+});
+
+const handleHash = async (request: HashRequest) => {
+  const { id, file } = request;
+  const checksum = file.size <= MAX_HASH_FILE_SIZE ? await hashSmallFile(request) : await hashLargeFile(request);
+  broadcast.postMessage({ type: 'checksum', id, checksum });
+};
+
+const hashSmallFile = async ({ file }: HashRequest): Promise<string> => {
+  const buffer = await file.arrayBuffer();
+  return sha1(new Uint8Array(buffer));
+};
+
+const hashLargeFile = async ({ id, file }: HashRequest): Promise<string> => {
+  const sha1 = await createSHA1();
+  const reader = file.stream().getReader();
+  let processedBytes = 0;
+  let lastUpdate = Date.now();
+
+  while (true) {
+    const { done, value } = await reader.read();
+    if (done) {
+      break;
+    }
+    sha1.update(value);
+    processedBytes += value.length;
+
+    broadcast.postMessage({
+      type: 'hash.progress',
+      id,
+      progress: processedBytes,
+      total: file.size,
+    });
+  }
+
+  return sha1.digest('hex');
 };

web/src/service-worker/index.ts

@@ -2,7 +2,7 @@
 /// <reference no-default-lib="true"/>
 /// <reference lib="esnext" />
 /// <reference lib="webworker" />
-import { installBroadcastChannelListener } from './broadcast-channel';
+import './broadcast-channel';
 import { prune } from './cache';
 import { handleRequest } from './request';
 
@@ -36,4 +36,3 @@ const handleFetch = (event: FetchEvent): void => {
 sw.addEventListener('install', handleInstall, { passive: true });
 sw.addEventListener('activate', handleActivate, { passive: true });
 sw.addEventListener('fetch', handleFetch, { passive: true });
-installBroadcastChannelListener();