feat(cli): add --delete-duplicates option (#20035)

* Add --delete-duplicates option to delete local assets that already exist on the server, fixes #12181

* Update docs

* Fix `--delete-duplicates` implying `--delete`

* fix the test, break the english

* format

* also ran the formatter on the e2e folder :)

* early return, fewer allocations

* add back log

---------

Co-authored-by: Robin Jacobs <robin.jacobs@beeline.com>
Co-authored-by: mertalev <101130780+mertalev@users.noreply.github.com>

cli/src/commands/asset.spec.ts

@@ -271,7 +271,7 @@ describe('startWatch', () => {
     });
   });
 
-  it('should filger out ignored patterns', async () => {
+  it('should filter out ignored patterns', async () => {
     const testFilePath = path.join(testFolder, 'test.jpg');
     const ignoredPattern = 'ignored';
     const ignoredFolder = path.join(testFolder, ignoredPattern);

cli/src/commands/asset.ts

@@ -37,6 +37,7 @@ export interface UploadOptionsDto {
   dryRun?: boolean;
   skipHash?: boolean;
   delete?: boolean;
+  deleteDuplicates?: boolean;
   album?: boolean;
   albumName?: string;
   includeHidden?: boolean;
@@ -70,10 +71,8 @@ const uploadBatch = async (files: string[], options: UploadOptionsDto) => {
     console.log(JSON.stringify({ newFiles, duplicates, newAssets }, undefined, 4));
   }
   await updateAlbums([...newAssets, ...duplicates], options);
-  await deleteFiles(
-    newAssets.map(({ filepath }) => filepath),
-    options,
-  );
+
+  await deleteFiles(newAssets, duplicates, options);
 };
 
 export const startWatch = async (
@@ -406,28 +405,46 @@ const uploadFile = async (input: string, stats: Stats): Promise<AssetMediaRespon
   return response.json();
 };
 
-const deleteFiles = async (files: string[], options: UploadOptionsDto): Promise<void> => {
-  if (!options.delete) {
-    return;
+const deleteFiles = async (uploaded: Asset[], duplicates: Asset[], options: UploadOptionsDto): Promise<void> => {
+  let fileCount = 0;
+  if (options.delete) {
+    fileCount += uploaded.length;
+  }
+
+  if (options.deleteDuplicates) {
+    fileCount += duplicates.length;
   }
 
   if (options.dryRun) {
-    console.log(`Would have deleted ${files.length} local asset${s(files.length)}`);
+    console.log(`Would have deleted ${fileCount} local asset${s(fileCount)}`);
+    return;
+  }
+
+  if (fileCount === 0) {
     return;
   }
 
   console.log('Deleting assets that have been uploaded...');
-
   const deletionProgress = new SingleBar(
     { format: 'Deleting local assets | {bar} | {percentage}% | ETA: {eta}s | {value}/{total} assets' },
     Presets.shades_classic,
   );
-  deletionProgress.start(files.length, 0);
+  deletionProgress.start(fileCount, 0);
+
+  const chunkDelete = async (files: Asset[]) => {
+    for (const assetBatch of chunk(files, options.concurrency)) {
+      await Promise.all(assetBatch.map((input: Asset) => unlink(input.filepath)));
+      deletionProgress.update(assetBatch.length);
+    }
+  };
 
   try {
-    for (const assetBatch of chunk(files, options.concurrency)) {
-      await Promise.all(assetBatch.map((input: string) => unlink(input)));
-      deletionProgress.update(assetBatch.length);
+    if (options.delete) {
+      await chunkDelete(uploaded);
+    }
+
+    if (options.deleteDuplicates) {
+      await chunkDelete(duplicates);
     }
   } finally {
     deletionProgress.stop();

cli/src/index.ts

@@ -75,6 +75,11 @@ program
       .default(false),
   )
   .addOption(new Option('--delete', 'Delete local assets after upload').env('IMMICH_DELETE_ASSETS'))
+  .addOption(
+    new Option('--delete-duplicates', 'Delete local assets that are duplicates (already exist on server)').env(
+      'IMMICH_DELETE_DUPLICATES',
+    ),
+  )
   .addOption(new Option('--no-progress', 'Hide progress bars').env('IMMICH_PROGRESS_BAR').default(true))
   .addOption(
     new Option('--watch', 'Watch for changes and upload automatically')