From 817ba5f067acf516c344b281faed742245f9a557 Mon Sep 17 00:00:00 2001 From: Robin Jacobs Date: Sun, 20 Jul 2025 22:20:31 +0200 Subject: [PATCH 1/6] Add --delete-duplicates option to delete local assets that already exist on the server, fixes #12181 --- cli/src/commands/asset.spec.ts | 2 +- cli/src/commands/asset.ts | 13 +- cli/src/index.ts | 5 + e2e/src/cli/specs/upload.e2e-spec.ts | 170 +++++++++++++++++++++++++++ 4 files changed, 188 insertions(+), 2 deletions(-) diff --git a/cli/src/commands/asset.spec.ts b/cli/src/commands/asset.spec.ts index 21137a3296..7dce135985 100644 --- a/cli/src/commands/asset.spec.ts +++ b/cli/src/commands/asset.spec.ts @@ -271,7 +271,7 @@ describe('startWatch', () => { }); }); - it('should filger out ignored patterns', async () => { + it('should filter out ignored patterns', async () => { const testFilePath = path.join(testFolder, 'test.jpg'); const ignoredPattern = 'ignored'; const ignoredFolder = path.join(testFolder, ignoredPattern); diff --git a/cli/src/commands/asset.ts b/cli/src/commands/asset.ts index 2af3cf8d5e..e288d38ce7 100644 --- a/cli/src/commands/asset.ts +++ b/cli/src/commands/asset.ts @@ -37,6 +37,7 @@ export interface UploadOptionsDto { dryRun?: boolean; skipHash?: boolean; delete?: boolean; + deleteDuplicates?: boolean; album?: boolean; albumName?: string; includeHidden?: boolean; @@ -70,10 +71,20 @@ const uploadBatch = async (files: string[], options: UploadOptionsDto) => { console.log(JSON.stringify({ newFiles, duplicates, newAssets }, undefined, 4)); } await updateAlbums([...newAssets, ...duplicates], options); + + // Delete successfully uploaded files await deleteFiles( newAssets.map(({ filepath }) => filepath), options, ); + + // Delete duplicate files if --delete-duplicates flag is set + if (options.deleteDuplicates) { + await deleteFiles( + duplicates.map(({ filepath }) => filepath), + options, + ); + } }; export const startWatch = async ( @@ -407,7 +418,7 @@ const uploadFile = async (input: string, stats: Stats): Promise => { - if (!options.delete) { + if (!options.delete && !options.deleteDuplicates) { return; } diff --git a/cli/src/index.ts b/cli/src/index.ts index a0392186c0..d1b4ab2dde 100644 --- a/cli/src/index.ts +++ b/cli/src/index.ts @@ -74,6 +74,11 @@ program .default(false), ) .addOption(new Option('--delete', 'Delete local assets after upload').env('IMMICH_DELETE_ASSETS')) + .addOption( + new Option('--delete-duplicates', 'Delete local assets that are duplicates (already exist on server)').env( + 'IMMICH_DELETE_DUPLICATES', + ), + ) .addOption(new Option('--no-progress', 'Hide progress bars').env('IMMICH_PROGRESS_BAR').default(true)) .addOption( new Option('--watch', 'Watch for changes and upload automatically') diff --git a/e2e/src/cli/specs/upload.e2e-spec.ts b/e2e/src/cli/specs/upload.e2e-spec.ts index 8249b9b360..3302152d4f 100644 --- a/e2e/src/cli/specs/upload.e2e-spec.ts +++ b/e2e/src/cli/specs/upload.e2e-spec.ts @@ -442,6 +442,176 @@ describe(`immich upload`, () => { }); }); + describe('immich upload --delete-duplicates', () => { + it('should delete local duplicate files', async () => { + const { stderr: firstStderr, stdout: firstStdout, exitCode: firstExitCode } = await immichCli([ + 'upload', + `${testAssetDir}/albums/nature/silver_fir.jpg`, + ]); + expect(firstStderr).toContain('{message}'); + expect(firstStdout.split('\n')).toEqual( + expect.arrayContaining([expect.stringContaining('Successfully uploaded 1 new asset')]), + ); + expect(firstExitCode).toBe(0); + + await mkdir(`/tmp/albums/nature`, { recursive: true }); + await symlink(`${testAssetDir}/albums/nature/silver_fir.jpg`, `/tmp/albums/nature/silver_fir.jpg`); + + // Upload with --delete-duplicates flag + const { stderr, stdout, exitCode } = await immichCli([ + 'upload', + `/tmp/albums/nature/silver_fir.jpg`, + '--delete-duplicates', + ]); + + // Check that the duplicate file was deleted + const files = await readdir(`/tmp/albums/nature`); + await rm(`/tmp/albums/nature`, { recursive: true }); + expect(files.length).toBe(0); + + expect(stdout.split('\n')).toEqual( + expect.arrayContaining([ + expect.stringContaining('Found 0 new files and 1 duplicate'), + expect.stringContaining('All assets were already uploaded, nothing to do'), + ]), + ); + expect(stderr).toContain('{message}'); + expect(exitCode).toBe(0); + + // Verify no new assets were uploaded + const assets = await getAssetStatistics({}, { headers: asKeyAuth(key) }); + expect(assets.total).toBe(1); + }); + + it('should have accurate dry run with --delete-duplicates', async () => { + const { stderr: firstStderr, stdout: firstStdout, exitCode: firstExitCode } = await immichCli([ + 'upload', + `${testAssetDir}/albums/nature/silver_fir.jpg`, + ]); + expect(firstStderr).toContain('{message}'); + expect(firstStdout.split('\n')).toEqual( + expect.arrayContaining([expect.stringContaining('Successfully uploaded 1 new asset')]), + ); + expect(firstExitCode).toBe(0); + + await mkdir(`/tmp/albums/nature`, { recursive: true }); + await symlink(`${testAssetDir}/albums/nature/silver_fir.jpg`, `/tmp/albums/nature/silver_fir.jpg`); + + // Upload with --delete-duplicates and --dry-run flags + const { stderr, stdout, exitCode } = await immichCli([ + 'upload', + `/tmp/albums/nature/silver_fir.jpg`, + '--delete-duplicates', + '--dry-run', + ]); + + // Check that the duplicate file was NOT deleted in dry run mode + const files = await readdir(`/tmp/albums/nature`); + await rm(`/tmp/albums/nature`, { recursive: true }); + expect(files.length).toBe(1); + + expect(stdout.split('\n')).toEqual( + expect.arrayContaining([ + expect.stringContaining('Found 0 new files and 1 duplicate'), + expect.stringContaining('Would have deleted 1 local asset'), + ]), + ); + expect(stderr).toContain('{message}'); + expect(exitCode).toBe(0); + + // Verify no new assets were uploaded + const assets = await getAssetStatistics({}, { headers: asKeyAuth(key) }); + expect(assets.total).toBe(1); + }); + + it('should work with both --delete and --delete-duplicates flags', async () => { + // First, upload a file to create a duplicate on the server + const { stderr: firstStderr, stdout: firstStdout, exitCode: firstExitCode } = await immichCli([ + 'upload', + `${testAssetDir}/albums/nature/silver_fir.jpg`, + ]); + expect(firstStderr).toContain('{message}'); + expect(firstStdout.split('\n')).toEqual( + expect.arrayContaining([expect.stringContaining('Successfully uploaded 1 new asset')]), + ); + expect(firstExitCode).toBe(0); + + // Both new and duplicate files + await mkdir(`/tmp/albums/nature`, { recursive: true }); + await symlink(`${testAssetDir}/albums/nature/silver_fir.jpg`, `/tmp/albums/nature/silver_fir.jpg`); // duplicate + await symlink(`${testAssetDir}/albums/nature/el_torcal_rocks.jpg`, `/tmp/albums/nature/el_torcal_rocks.jpg`); // new + + // Upload with both --delete and --delete-duplicates flags + const { stderr, stdout, exitCode } = await immichCli([ + 'upload', + `/tmp/albums/nature`, + '--delete', + '--delete-duplicates', + ]); + + // Check that both files were deleted (new file due to --delete, duplicate due to --delete-duplicates) + const files = await readdir(`/tmp/albums/nature`); + await rm(`/tmp/albums/nature`, { recursive: true }); + expect(files.length).toBe(0); + + expect(stdout.split('\n')).toEqual( + expect.arrayContaining([ + expect.stringContaining('Found 1 new file and 1 duplicate'), + expect.stringContaining('Successfully uploaded 1 new asset'), + expect.stringContaining('Deleting assets that have been uploaded'), + ]), + ); + expect(stderr).toContain('{message}'); + expect(exitCode).toBe(0); + + // Verify one new asset was uploaded (total should be 2 now) + const assets = await getAssetStatistics({}, { headers: asKeyAuth(key) }); + expect(assets.total).toBe(2); + }); + + it('should only delete duplicates when --delete-duplicates is used without --delete', async () => { + const { stderr: firstStderr, stdout: firstStdout, exitCode: firstExitCode } = await immichCli([ + 'upload', + `${testAssetDir}/albums/nature/silver_fir.jpg`, + ]); + expect(firstStderr).toContain('{message}'); + expect(firstStdout.split('\n')).toEqual( + expect.arrayContaining([expect.stringContaining('Successfully uploaded 1 new asset')]), + ); + expect(firstExitCode).toBe(0); + + // Both new and duplicate files + await mkdir(`/tmp/albums/nature`, { recursive: true }); + await symlink(`${testAssetDir}/albums/nature/silver_fir.jpg`, `/tmp/albums/nature/silver_fir.jpg`); // duplicate + await symlink(`${testAssetDir}/albums/nature/el_torcal_rocks.jpg`, `/tmp/albums/nature/el_torcal_rocks.jpg`); // new + + // Upload with only --delete-duplicates flag + const { stderr, stdout, exitCode } = await immichCli([ + 'upload', + `/tmp/albums/nature`, + '--delete-duplicates', + ]); + + // Check that only the duplicate was deleted, new file should remain + const files = await readdir(`/tmp/albums/nature`); + await rm(`/tmp/albums/nature`, { recursive: true }); + expect(files).toEqual(['el_torcal_rocks.jpg']); + + expect(stdout.split('\n')).toEqual( + expect.arrayContaining([ + expect.stringContaining('Found 1 new file and 1 duplicate'), + expect.stringContaining('Successfully uploaded 1 new asset'), + ]), + ); + expect(stderr).toContain('{message}'); + expect(exitCode).toBe(0); + + // Verify one new asset was uploaded (total should be 2 now) + const assets = await getAssetStatistics({}, { headers: asKeyAuth(key) }); + expect(assets.total).toBe(2); + }); + }); + describe('immich upload --skip-hash', () => { it('should skip hashing', async () => { const filename = `albums/nature/silver_fir.jpg`; From e90a33b97a5a215a27e839afcead35e565e539c8 Mon Sep 17 00:00:00 2001 From: Robin Jacobs Date: Sun, 20 Jul 2025 22:37:24 +0200 Subject: [PATCH 2/6] Update docs --- docs/docs/features/command-line-interface.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/docs/features/command-line-interface.md b/docs/docs/features/command-line-interface.md index 436b499e50..ec8d68d6f7 100644 --- a/docs/docs/features/command-line-interface.md +++ b/docs/docs/features/command-line-interface.md @@ -103,6 +103,7 @@ Options: -c, --concurrency Number of assets to upload at the same time (default: 4, env: IMMICH_UPLOAD_CONCURRENCY) -j, --json-output Output detailed information in json format (default: false, env: IMMICH_JSON_OUTPUT) --delete Delete local assets after upload (env: IMMICH_DELETE_ASSETS) + --delete-duplicates Delete local assets that are duplicates (already exist on server) (env: IMMICH_DELETE_DUPLICATES) --no-progress Hide progress bars (env: IMMICH_PROGRESS_BAR) --watch Watch for changes and upload automatically (default: false, env: IMMICH_WATCH_CHANGES) --help display help for command From 4cb7c578d840db7e2424dc62d48db3a6654ff0bc Mon Sep 17 00:00:00 2001 From: Robin Jacobs Date: Sun, 20 Jul 2025 23:25:52 +0200 Subject: [PATCH 3/6] Fix `--delete-duplicates` implying `--delete` --- cli/src/commands/asset.ts | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/cli/src/commands/asset.ts b/cli/src/commands/asset.ts index e288d38ce7..d2868ca6a5 100644 --- a/cli/src/commands/asset.ts +++ b/cli/src/commands/asset.ts @@ -72,19 +72,11 @@ const uploadBatch = async (files: string[], options: UploadOptionsDto) => { } await updateAlbums([...newAssets, ...duplicates], options); - // Delete successfully uploaded files await deleteFiles( newAssets.map(({ filepath }) => filepath), + duplicates.map(({ filepath }) => filepath), options, ); - - // Delete duplicate files if --delete-duplicates flag is set - if (options.deleteDuplicates) { - await deleteFiles( - duplicates.map(({ filepath }) => filepath), - options, - ); - } }; export const startWatch = async ( @@ -417,11 +409,14 @@ const uploadFile = async (input: string, stats: Stats): Promise => { - if (!options.delete && !options.deleteDuplicates) { - return; - } - +const deleteFiles = async ( + uploaded: string[], + duplicates: string[], + options: UploadOptionsDto): Promise => { + const files: string[] = [ + ...(options.delete ? uploaded : []), + ...(options.deleteDuplicates ? duplicates : []) + ]; if (options.dryRun) { console.log(`Would have deleted ${files.length} local asset${s(files.length)}`); return; From ce836061fa9f46e5f824e73640290f68f7d5f2db Mon Sep 17 00:00:00 2001 From: Robin Jacobs Date: Sun, 20 Jul 2025 23:31:49 +0200 Subject: [PATCH 4/6] fix the test, break the english --- e2e/src/cli/specs/upload.e2e-spec.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/e2e/src/cli/specs/upload.e2e-spec.ts b/e2e/src/cli/specs/upload.e2e-spec.ts index 3302152d4f..92aa12baee 100644 --- a/e2e/src/cli/specs/upload.e2e-spec.ts +++ b/e2e/src/cli/specs/upload.e2e-spec.ts @@ -556,7 +556,7 @@ describe(`immich upload`, () => { expect(stdout.split('\n')).toEqual( expect.arrayContaining([ - expect.stringContaining('Found 1 new file and 1 duplicate'), + expect.stringContaining('Found 1 new files and 1 duplicate'), expect.stringContaining('Successfully uploaded 1 new asset'), expect.stringContaining('Deleting assets that have been uploaded'), ]), @@ -599,7 +599,7 @@ describe(`immich upload`, () => { expect(stdout.split('\n')).toEqual( expect.arrayContaining([ - expect.stringContaining('Found 1 new file and 1 duplicate'), + expect.stringContaining('Found 1 new files and 1 duplicate'), expect.stringContaining('Successfully uploaded 1 new asset'), ]), ); From 5917d24adaa65943c389801b01cd1b73dfc6ae88 Mon Sep 17 00:00:00 2001 From: Robin Jacobs Date: Mon, 13 Oct 2025 23:09:51 +0100 Subject: [PATCH 5/6] format --- cli/src/commands/asset.ts | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/cli/src/commands/asset.ts b/cli/src/commands/asset.ts index d2868ca6a5..6dee5e9d30 100644 --- a/cli/src/commands/asset.ts +++ b/cli/src/commands/asset.ts @@ -409,14 +409,8 @@ const uploadFile = async (input: string, stats: Stats): Promise => { - const files: string[] = [ - ...(options.delete ? uploaded : []), - ...(options.deleteDuplicates ? duplicates : []) - ]; +const deleteFiles = async (uploaded: string[], duplicates: string[], options: UploadOptionsDto): Promise => { + const files: string[] = [...(options.delete ? uploaded : []), ...(options.deleteDuplicates ? duplicates : [])]; if (options.dryRun) { console.log(`Would have deleted ${files.length} local asset${s(files.length)}`); return; From e1351c570a120663c9e5eeab7594d3ccb2649785 Mon Sep 17 00:00:00 2001 From: Robin Jacobs Date: Tue, 14 Oct 2025 16:10:45 +0100 Subject: [PATCH 6/6] also ran the formatter on the e2e folder :) --- e2e/src/cli/specs/upload.e2e-spec.ts | 42 ++++++++++++++-------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/e2e/src/cli/specs/upload.e2e-spec.ts b/e2e/src/cli/specs/upload.e2e-spec.ts index 92aa12baee..b53b4403f8 100644 --- a/e2e/src/cli/specs/upload.e2e-spec.ts +++ b/e2e/src/cli/specs/upload.e2e-spec.ts @@ -444,10 +444,11 @@ describe(`immich upload`, () => { describe('immich upload --delete-duplicates', () => { it('should delete local duplicate files', async () => { - const { stderr: firstStderr, stdout: firstStdout, exitCode: firstExitCode } = await immichCli([ - 'upload', - `${testAssetDir}/albums/nature/silver_fir.jpg`, - ]); + const { + stderr: firstStderr, + stdout: firstStdout, + exitCode: firstExitCode, + } = await immichCli(['upload', `${testAssetDir}/albums/nature/silver_fir.jpg`]); expect(firstStderr).toContain('{message}'); expect(firstStdout.split('\n')).toEqual( expect.arrayContaining([expect.stringContaining('Successfully uploaded 1 new asset')]), @@ -484,10 +485,11 @@ describe(`immich upload`, () => { }); it('should have accurate dry run with --delete-duplicates', async () => { - const { stderr: firstStderr, stdout: firstStdout, exitCode: firstExitCode } = await immichCli([ - 'upload', - `${testAssetDir}/albums/nature/silver_fir.jpg`, - ]); + const { + stderr: firstStderr, + stdout: firstStdout, + exitCode: firstExitCode, + } = await immichCli(['upload', `${testAssetDir}/albums/nature/silver_fir.jpg`]); expect(firstStderr).toContain('{message}'); expect(firstStdout.split('\n')).toEqual( expect.arrayContaining([expect.stringContaining('Successfully uploaded 1 new asset')]), @@ -526,10 +528,11 @@ describe(`immich upload`, () => { it('should work with both --delete and --delete-duplicates flags', async () => { // First, upload a file to create a duplicate on the server - const { stderr: firstStderr, stdout: firstStdout, exitCode: firstExitCode } = await immichCli([ - 'upload', - `${testAssetDir}/albums/nature/silver_fir.jpg`, - ]); + const { + stderr: firstStderr, + stdout: firstStdout, + exitCode: firstExitCode, + } = await immichCli(['upload', `${testAssetDir}/albums/nature/silver_fir.jpg`]); expect(firstStderr).toContain('{message}'); expect(firstStdout.split('\n')).toEqual( expect.arrayContaining([expect.stringContaining('Successfully uploaded 1 new asset')]), @@ -570,10 +573,11 @@ describe(`immich upload`, () => { }); it('should only delete duplicates when --delete-duplicates is used without --delete', async () => { - const { stderr: firstStderr, stdout: firstStdout, exitCode: firstExitCode } = await immichCli([ - 'upload', - `${testAssetDir}/albums/nature/silver_fir.jpg`, - ]); + const { + stderr: firstStderr, + stdout: firstStdout, + exitCode: firstExitCode, + } = await immichCli(['upload', `${testAssetDir}/albums/nature/silver_fir.jpg`]); expect(firstStderr).toContain('{message}'); expect(firstStdout.split('\n')).toEqual( expect.arrayContaining([expect.stringContaining('Successfully uploaded 1 new asset')]), @@ -586,11 +590,7 @@ describe(`immich upload`, () => { await symlink(`${testAssetDir}/albums/nature/el_torcal_rocks.jpg`, `/tmp/albums/nature/el_torcal_rocks.jpg`); // new // Upload with only --delete-duplicates flag - const { stderr, stdout, exitCode } = await immichCli([ - 'upload', - `/tmp/albums/nature`, - '--delete-duplicates', - ]); + const { stderr, stdout, exitCode } = await immichCli(['upload', `/tmp/albums/nature`, '--delete-duplicates']); // Check that only the duplicate was deleted, new file should remain const files = await readdir(`/tmp/albums/nature`);