mirror of
https://github.com/immich-app/immich
synced 2025-10-17 18:19:27 +00:00
feat: enhance upload command for large uploads
- Implement persistent hash caching to avoid unnecessary file rehashing - Switch progress bars to show processed file size instead of asset count - Fix concurrency handling based on CPU count
This commit is contained in:
parent
2ce8a31b2f
commit
42a0f9f125
2 changed files with 13 additions and 44 deletions
|
|
@ -121,7 +121,7 @@ const processedAlbumNames = new Set<string>();
|
||||||
export const upload = async (paths: string[], baseOptions: BaseOptions, options: UploadOptionsDto) => {
|
export const upload = async (paths: string[], baseOptions: BaseOptions, options: UploadOptionsDto) => {
|
||||||
// Clear the album names cache at the start of each upload command
|
// Clear the album names cache at the start of each upload command
|
||||||
processedAlbumNames.clear();
|
processedAlbumNames.clear();
|
||||||
|
|
||||||
// Initialize hash cache
|
// Initialize hash cache
|
||||||
const hashCache = new FileHashCache(baseOptions.configDirectory);
|
const hashCache = new FileHashCache(baseOptions.configDirectory);
|
||||||
await hashCache.load();
|
await hashCache.load();
|
||||||
|
|
@ -205,14 +205,14 @@ export const checkForDuplicates = async (
|
||||||
|
|
||||||
if (progress) {
|
if (progress) {
|
||||||
multiBar = new MultiBar(
|
multiBar = new MultiBar(
|
||||||
{
|
{
|
||||||
format: '{message} | {bar} | {percentage}% | ETA: {eta_formatted} | {value}/{total}',
|
format: '{message} | {bar} | {percentage}% | ETA: {eta_formatted} | {value}/{total}',
|
||||||
formatValue: (v: number, options, type) => {
|
formatValue: (v: number, options, type) => {
|
||||||
// Don't format percentage
|
// Don't format percentage
|
||||||
if (type === 'percentage') return v.toString();
|
if (type === 'percentage') return v.toString();
|
||||||
return byteSize(v).toString();
|
return byteSize(v).toString();
|
||||||
},
|
},
|
||||||
etaBuffer: 100, // Increase samples for ETA calculation
|
etaBuffer: 100, // Increase samples for ETA calculation
|
||||||
},
|
},
|
||||||
Presets.shades_classic,
|
Presets.shades_classic,
|
||||||
);
|
);
|
||||||
|
|
@ -228,11 +228,11 @@ export const checkForDuplicates = async (
|
||||||
console.log(`Received ${files.length} files (${byteSize(totalSize)}), hashing...`);
|
console.log(`Received ${files.length} files (${byteSize(totalSize)}), hashing...`);
|
||||||
}
|
}
|
||||||
|
|
||||||
const hashProgressBar = multiBar?.create(totalSize, 0, {
|
const hashProgressBar = multiBar?.create(totalSize, 0, {
|
||||||
message: 'Hashing files '
|
message: 'Hashing files ',
|
||||||
});
|
});
|
||||||
const checkProgressBar = multiBar?.create(totalSize, 0, {
|
const checkProgressBar = multiBar?.create(totalSize, 0, {
|
||||||
message: 'Checking for duplicates'
|
message: 'Checking for duplicates',
|
||||||
});
|
});
|
||||||
|
|
||||||
const newFiles: string[] = [];
|
const newFiles: string[] = [];
|
||||||
|
|
@ -291,7 +291,7 @@ export const checkForDuplicates = async (
|
||||||
if (!stats) {
|
if (!stats) {
|
||||||
throw new Error(`Stats not found for ${filepath}`);
|
throw new Error(`Stats not found for ${filepath}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
const dto = { id: filepath, checksum: await hashCache.getHash(filepath, stats) };
|
const dto = { id: filepath, checksum: await hashCache.getHash(filepath, stats) };
|
||||||
|
|
||||||
results.push(dto);
|
results.push(dto);
|
||||||
|
|
@ -586,46 +586,15 @@ const updateAlbums = async (assets: Asset[], options: UploadOptionsDto) => {
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
const processAlbumName = (name: string, options: UploadOptionsDto): string => {
|
|
||||||
// Only process if formatAlbumNames is enabled
|
|
||||||
if (!options.formatAlbumNames) {
|
|
||||||
return name;
|
|
||||||
}
|
|
||||||
|
|
||||||
const originalName = name;
|
|
||||||
let processedName = name;
|
|
||||||
|
|
||||||
// Remove leading non-alphabetic characters (including numbers but preserving accented letters)
|
|
||||||
processedName = processedName.replace(/^[^\p{Letter}]+/u, '');
|
|
||||||
|
|
||||||
// Replace underscores and dashes with spaces, then handle multiple spaces
|
|
||||||
processedName = processedName
|
|
||||||
.replace(/[_\-]/g, ' ') // Convert underscores and dashes to spaces
|
|
||||||
.replace(/\s+/g, ' ') // Replace multiple spaces with single space
|
|
||||||
.trim(); // Remove leading/trailing spaces
|
|
||||||
|
|
||||||
// Capitalize first letter
|
|
||||||
processedName = processedName.charAt(0).toUpperCase() + processedName.slice(1);
|
|
||||||
|
|
||||||
// Only show message once per unique original name
|
|
||||||
if (originalName !== processedName && !processedAlbumNames.has(originalName)) {
|
|
||||||
processedAlbumNames.add(originalName);
|
|
||||||
console.log(`Album name changed: "${originalName}" → "${processedName}"`);
|
|
||||||
}
|
|
||||||
|
|
||||||
return processedName;
|
|
||||||
};
|
|
||||||
|
|
||||||
// `filepath` valid format:
|
// `filepath` valid format:
|
||||||
// - Windows: `D:\\test\\Filename.txt` or `D:/test/Filename.txt`
|
// - Windows: `D:\\test\\Filename.txt` or `D:/test/Filename.txt`
|
||||||
// - Unix: `/test/Filename.txt`
|
// - Unix: `/test/Filename.txt`
|
||||||
export const getAlbumName = (filepath: string, options: UploadOptionsDto) => {
|
export const getAlbumName = (filepath: string, options: UploadOptionsDto) => {
|
||||||
if (options.albumName) {
|
if (options.albumName) {
|
||||||
return options.albumName; // Don't process custom album names
|
return options.albumName;
|
||||||
}
|
}
|
||||||
|
|
||||||
const dirName = path.basename(path.dirname(filepath));
|
return path.basename(path.dirname(filepath));
|
||||||
return processAlbumName(dirName, options);
|
|
||||||
};
|
};
|
||||||
|
|
||||||
// `
|
// `
|
||||||
|
|
|
||||||
|
|
@ -56,7 +56,7 @@ program
|
||||||
.addOption(
|
.addOption(
|
||||||
new Option('-A, --album-name <n>', 'Add all assets to specified album')
|
new Option('-A, --album-name <n>', 'Add all assets to specified album')
|
||||||
.env('IMMICH_ALBUM_NAME')
|
.env('IMMICH_ALBUM_NAME')
|
||||||
.conflicts('album'),
|
.conflicts('album')
|
||||||
)
|
)
|
||||||
.addOption(
|
.addOption(
|
||||||
new Option('-f, --format-album-names', 'Format album names (remove leading numbers, clean up spaces and dashes)')
|
new Option('-f, --format-album-names', 'Format album names (remove leading numbers, clean up spaces and dashes)')
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue