From 650b0b985fe75786959518faac31331372f0b78e Mon Sep 17 00:00:00 2001 From: NoiceHax Date: Thu, 13 Aug 2026 22:52:26 +0530 Subject: [PATCH] fix(cli): report upload-time duplicates in json output The `duplicates` key of `--json-output` was only populated from `checkForDuplicates`, which asks the server about the checksums of the whole batch at once. That check cannot detect files that are duplicated within the batch itself, so those are handed to `uploadFiles`, which the server then rejects with `AssetMediaStatus.Duplicate`. `uploadFiles` counted them for the "Skipped N duplicate assets" log line but returned them as new assets, so they never reached the json output. It now returns them separately and `uploadBatch` merges them into `duplicates`. Closes #24291 --- packages/cli/src/commands/asset.spec.ts | 54 +++++++++++++++++++------ packages/cli/src/commands/asset.ts | 21 ++++++---- 2 files changed, 55 insertions(+), 20 deletions(-) diff --git a/packages/cli/src/commands/asset.spec.ts b/packages/cli/src/commands/asset.spec.ts index 7e582039ba..7fa99af173 100644 --- a/packages/cli/src/commands/asset.spec.ts +++ b/packages/cli/src/commands/asset.spec.ts @@ -72,12 +72,34 @@ describe('uploadFiles', () => { }; }); - await expect(uploadFiles([testFilePath], { concurrency: 1 })).resolves.toEqual([ - { - filepath: testFilePath, - id: 'fc5621b1-86f6-44a1-9905-403e607df9f5', - }, - ]); + await expect(uploadFiles([testFilePath], { concurrency: 1 })).resolves.toEqual({ + newAssets: [ + { + filepath: testFilePath, + id: 'fc5621b1-86f6-44a1-9905-403e607df9f5', + }, + ], + duplicateAssets: [], + }); + }); + + it('returns duplicate assets when the server rejects the upload as a duplicate', async () => { + fetchMocker.doMockIf(new RegExp(`${baseUrl}/assets$`), function () { + return { + status: 200, + body: JSON.stringify({ id: 'fc5621b1-86f6-44a1-9905-403e607df9f5', status: 'duplicate' }), + }; + }); + + await expect(uploadFiles([testFilePath], { concurrency: 1 })).resolves.toEqual({ + newAssets: [], + duplicateAssets: [ + { + filepath: testFilePath, + id: 'fc5621b1-86f6-44a1-9905-403e607df9f5', + }, + ], + }); }); it('returns new assets when upload file retry is successful', async () => { @@ -94,12 +116,15 @@ describe('uploadFiles', () => { }; }); - await expect(uploadFiles([testFilePath], { concurrency: 1 })).resolves.toEqual([ - { - filepath: testFilePath, - id: 'fc5621b1-86f6-44a1-9905-403e607df9f5', - }, - ]); + await expect(uploadFiles([testFilePath], { concurrency: 1 })).resolves.toEqual({ + newAssets: [ + { + filepath: testFilePath, + id: 'fc5621b1-86f6-44a1-9905-403e607df9f5', + }, + ], + duplicateAssets: [], + }); }); it('returns new assets when upload file retry is failed', async () => { @@ -107,7 +132,10 @@ describe('uploadFiles', () => { throw new Error('Network error'); }); - await expect(uploadFiles([testFilePath], { concurrency: 1 })).resolves.toEqual([]); + await expect(uploadFiles([testFilePath], { concurrency: 1 })).resolves.toEqual({ + newAssets: [], + duplicateAssets: [], + }); }); it('uploads assets with the specified visibility', async () => { diff --git a/packages/cli/src/commands/asset.ts b/packages/cli/src/commands/asset.ts index f1dfdaec8e..9d049ebe03 100644 --- a/packages/cli/src/commands/asset.ts +++ b/packages/cli/src/commands/asset.ts @@ -67,8 +67,10 @@ class UploadFile extends File { } const uploadBatch = async (files: string[], options: UploadOptionsDto) => { - const { newFiles, duplicates } = await checkForDuplicates(files, options); - const newAssets = await uploadFiles(newFiles, options); + const { newFiles, duplicates: existingDuplicates } = await checkForDuplicates(files, options); + const { newAssets, duplicateAssets } = await uploadFiles(newFiles, options); + // the server can still reject an upload as a duplicate, e.g. when the same file is present twice in the batch + const duplicates = [...existingDuplicates, ...duplicateAssets]; if (options.jsonOutput) { console.log(JSON.stringify({ newFiles, duplicates, newAssets }, undefined, 4)); } @@ -309,11 +311,14 @@ export const checkForDuplicates = async (files: string[], { concurrency, skipHas return { newFiles, duplicates }; }; -export const uploadFiles = async (files: string[], options: UploadOptionsDto): Promise => { +export const uploadFiles = async ( + files: string[], + options: UploadOptionsDto, +): Promise<{ newAssets: Asset[]; duplicateAssets: Asset[] }> => { const { dryRun, concurrency, progress } = options; if (files.length === 0) { console.log('All assets were already uploaded, nothing to do.'); - return []; + return { newAssets: [], duplicateAssets: [] }; } // Compute total size first @@ -327,7 +332,7 @@ export const uploadFiles = async (files: string[], options: UploadOptionsDto): P if (dryRun) { console.log(`Would have uploaded ${files.length} asset${s(files.length)} (${byteSize(totalSize)})`); - return files.map((filepath) => ({ id: '', filepath })); + return { newAssets: files.map((filepath) => ({ id: '', filepath })), duplicateAssets: [] }; } let uploadProgress: SingleBar | undefined; @@ -351,6 +356,7 @@ export const uploadFiles = async (files: string[], options: UploadOptionsDto): P let successSize = 0; const newAssets: Asset[] = []; + const duplicateAssets: Asset[] = []; const queue = new Queue( async (filepath: string) => { @@ -360,11 +366,12 @@ export const uploadFiles = async (files: string[], options: UploadOptionsDto): P } const response = await uploadFile(filepath, stats, options); - newAssets.push({ id: response.id, filepath }); if (response.status === AssetMediaStatus.Duplicate) { + duplicateAssets.push({ id: response.id, filepath }); duplicateCount++; duplicateSize += stats.size ?? 0; } else { + newAssets.push({ id: response.id, filepath }); successCount++; successSize += stats.size ?? 0; } @@ -398,7 +405,7 @@ export const uploadFiles = async (files: string[], options: UploadOptionsDto): P } } - return newAssets; + return { newAssets, duplicateAssets }; }; const uploadFile = async (