helpy4/immich
1
0
Fork 0
mirror of https://github.com/immich-app/immich synced 2026-08-15 13:03:57 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix(cli): report upload-time duplicates in json output

Browse source 
The `duplicates` key of `--json-output` was only populated from
`checkForDuplicates`, which asks the server about the checksums of the
whole batch at once. That check cannot detect files that are duplicated
within the batch itself, so those are handed to `uploadFiles`, which the
server then rejects with `AssetMediaStatus.Duplicate`. `uploadFiles`
counted them for the "Skipped N duplicate assets" log line but returned
them as new assets, so they never reached the json output. It now
returns them separately and `uploadBatch` merges them into `duplicates`.

Closes #24291
This commit is contained in:
NoiceHax 2026-08-13 22:52:26 +05:30
parent af33a78d18
commit 650b0b985f
2 changed files with 55 additions and 20 deletions
Download patch file Download diff file Expand all files Collapse all files

54
packages/cli/src/commands/asset.spec.ts
View file

@ -72,12 +72,34 @@ describe('uploadFiles', () => {
};
});
await expect(uploadFiles([testFilePath], { concurrency: 1 })).resolves.toEqual([
{
filepath: testFilePath,
id: 'fc5621b1-86f6-44a1-9905-403e607df9f5',
},
]);
await expect(uploadFiles([testFilePath], { concurrency: 1 })).resolves.toEqual({
newAssets: [
{
filepath: testFilePath,
id: 'fc5621b1-86f6-44a1-9905-403e607df9f5',
},
],
duplicateAssets: [],
});
});
it('returns duplicate assets when the server rejects the upload as a duplicate', async () => {
fetchMocker.doMockIf(new RegExp(`${baseUrl}/assets$`), function () {
return {
status: 200,
body: JSON.stringify({ id: 'fc5621b1-86f6-44a1-9905-403e607df9f5', status: 'duplicate' }),
};
});
await expect(uploadFiles([testFilePath], { concurrency: 1 })).resolves.toEqual({
newAssets: [],
duplicateAssets: [
{
filepath: testFilePath,
id: 'fc5621b1-86f6-44a1-9905-403e607df9f5',
},
],
});
});
it('returns new assets when upload file retry is successful', async () => {
@ -94,12 +116,15 @@ describe('uploadFiles', () => {
};
});
await expect(uploadFiles([testFilePath], { concurrency: 1 })).resolves.toEqual([
{
filepath: testFilePath,
id: 'fc5621b1-86f6-44a1-9905-403e607df9f5',
},
]);
await expect(uploadFiles([testFilePath], { concurrency: 1 })).resolves.toEqual({
newAssets: [
{
filepath: testFilePath,
id: 'fc5621b1-86f6-44a1-9905-403e607df9f5',
},
],
duplicateAssets: [],
});
});
it('returns new assets when upload file retry is failed', async () => {
@ -107,7 +132,10 @@ describe('uploadFiles', () => {
throw new Error('Network error');
});
await expect(uploadFiles([testFilePath], { concurrency: 1 })).resolves.toEqual([]);
await expect(uploadFiles([testFilePath], { concurrency: 1 })).resolves.toEqual({
newAssets: [],
duplicateAssets: [],
});
});
it('uploads assets with the specified visibility', async () => {

21
packages/cli/src/commands/asset.ts
View file

@ -67,8 +67,10 @@ class UploadFile extends File {
}
const uploadBatch = async (files: string[], options: UploadOptionsDto) => {
const { newFiles, duplicates } = await checkForDuplicates(files, options);
const newAssets = await uploadFiles(newFiles, options);
const { newFiles, duplicates: existingDuplicates } = await checkForDuplicates(files, options);
const { newAssets, duplicateAssets } = await uploadFiles(newFiles, options);
// the server can still reject an upload as a duplicate, e.g. when the same file is present twice in the batch
const duplicates = [...existingDuplicates, ...duplicateAssets];
if (options.jsonOutput) {
console.log(JSON.stringify({ newFiles, duplicates, newAssets }, undefined, 4));
}
@ -309,11 +311,14 @@ export const checkForDuplicates = async (files: string[], { concurrency, skipHas
return { newFiles, duplicates };
};
export const uploadFiles = async (files: string[], options: UploadOptionsDto): Promise<Asset[]> => {
export const uploadFiles = async (
files: string[],
options: UploadOptionsDto,
): Promise<{ newAssets: Asset[]; duplicateAssets: Asset[] }> => {
const { dryRun, concurrency, progress } = options;
if (files.length === 0) {
console.log('All assets were already uploaded, nothing to do.');
return [];
return { newAssets: [], duplicateAssets: [] };
}
// Compute total size first
@ -327,7 +332,7 @@ export const uploadFiles = async (files: string[], options: UploadOptionsDto): P
if (dryRun) {
console.log(`Would have uploaded ${files.length} asset${s(files.length)} (${byteSize(totalSize)})`);
return files.map((filepath) => ({ id: '', filepath }));
return { newAssets: files.map((filepath) => ({ id: '', filepath })), duplicateAssets: [] };
}
let uploadProgress: SingleBar | undefined;
@ -351,6 +356,7 @@ export const uploadFiles = async (files: string[], options: UploadOptionsDto): P
let successSize = 0;
const newAssets: Asset[] = [];
const duplicateAssets: Asset[] = [];
const queue = new Queue<string, AssetMediaResponseDto>(
async (filepath: string) => {
@ -360,11 +366,12 @@ export const uploadFiles = async (files: string[], options: UploadOptionsDto): P
}
const response = await uploadFile(filepath, stats, options);
newAssets.push({ id: response.id, filepath });
if (response.status === AssetMediaStatus.Duplicate) {
duplicateAssets.push({ id: response.id, filepath });
duplicateCount++;
duplicateSize += stats.size ?? 0;
} else {
newAssets.push({ id: response.id, filepath });
successCount++;
successSize += stats.size ?? 0;
}
@ -398,7 +405,7 @@ export const uploadFiles = async (files: string[], options: UploadOptionsDto): P
}
}
return newAssets;
return { newAssets, duplicateAssets };
};
const uploadFile = async (