mirror of
https://github.com/immich-app/immich
synced 2025-11-14 17:36:12 +00:00
feat(server): xmp sidecar metadata (#2466)
* initial commit for XMP sidecar support * Added support for 'missing' metadata files to include those without sidecar files, now detects sidecar files in the filesystem for media already ingested but the sidecar was created afterwards * didn't mean to commit default log level during testing * new sidecar logic for video metadata as well * Added xml mimetype for sidecars only * don't need capture group for this regex * wrong default value reverted * simplified the move here - keep it in the same try catch since the outcome is to move the media back anyway * simplified setter logic Co-authored-by: Jason Rasmussen <jrasm91@gmail.com> * simplified logic per suggestions * sidecar is now its own queue with a discover and sync, updated UI for the new job queueing * queue a sidecar job for every asset based on discovery or sync, though the logic is almost identical aside from linking the sidecar * now queue sidecar jobs for each assset, though logic is mostly the same between discovery and sync * simplified logic of filename extraction and asset instantiation * not sure how that got deleted.. * updated code per suggestions and comments in the PR * stat was not being used, removed the variable set * better type checking, using in-scope variables for exif getter instead of passing in every time * removed commented out test * ran and resolved all lints, formats, checks, and tests * resolved suggested change in PR * made getExifProperty more dynamic with multiple possible args for fallbacks, fixed typo, used generic in function for better type checking * better error handling and moving files back to positions on move or save failure * regenerated api * format fixes * Added XMP documentation * documentation typo * Merged in main * missed merge conflict * more changes due to a merge * Resolving conflicts * added icon for sidecar jobs --------- Co-authored-by: Jason Rasmussen <jrasm91@gmail.com> Co-authored-by: Alex Tran <alex.tran1502@gmail.com>
This commit is contained in:
Alex Phillips
GitHub
parent
1b54c4f8e7
commit
7c1dae918d
35 changed files with 371 additions and 48 deletions
|
|
@ -78,6 +78,7 @@ export class AssetController {
|
|||
[
|
||||
{ name: 'assetData', maxCount: 1 },
|
||||
{ name: 'livePhotoData', maxCount: 1 },
|
||||
{ name: 'sidecarData', maxCount: 1 },
|
||||
],
|
||||
assetUploadOption,
|
||||
),
|
||||
|
|
@ -90,18 +91,24 @@ export class AssetController {
|
|||
async uploadFile(
|
||||
@GetAuthUser() authUser: AuthUserDto,
|
||||
@UploadedFiles(new ParseFilePipe({ validators: [new FileNotEmptyValidator(['assetData'])] }))
|
||||
files: { assetData: ImmichFile[]; livePhotoData?: ImmichFile[] },
|
||||
files: { assetData: ImmichFile[]; livePhotoData?: ImmichFile[]; sidecarData: ImmichFile[] },
|
||||
@Body(new ValidationPipe()) dto: CreateAssetDto,
|
||||
@Response({ passthrough: true }) res: Res,
|
||||
): Promise<AssetFileUploadResponseDto> {
|
||||
const file = mapToUploadFile(files.assetData[0]);
|
||||
const _livePhotoFile = files.livePhotoData?.[0];
|
||||
const _sidecarFile = files.sidecarData?.[0];
|
||||
let livePhotoFile;
|
||||
if (_livePhotoFile) {
|
||||
livePhotoFile = mapToUploadFile(_livePhotoFile);
|
||||
}
|
||||
|
||||
const responseDto = await this.assetService.uploadFile(authUser, dto, file, livePhotoFile);
|
||||
let sidecarFile;
|
||||
if (_sidecarFile) {
|
||||
sidecarFile = mapToUploadFile(_sidecarFile);
|
||||
}
|
||||
|
||||
const responseDto = await this.assetService.uploadFile(authUser, dto, file, livePhotoFile, sidecarFile);
|
||||
if (responseDto.duplicate) {
|
||||
res.status(200);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ export class AssetCore {
|
|||
dto: CreateAssetDto,
|
||||
file: UploadFile,
|
||||
livePhotoAssetId?: string,
|
||||
sidecarFile?: UploadFile,
|
||||
): Promise<AssetEntity> {
|
||||
const asset = await this.repository.create({
|
||||
owner: { id: authUser.id } as UserEntity,
|
||||
|
|
@ -39,6 +40,7 @@ export class AssetCore {
|
|||
sharedLinks: [],
|
||||
originalFileName: parse(file.originalName).name,
|
||||
faces: [],
|
||||
sidecarPath: sidecarFile?.originalPath || null,
|
||||
});
|
||||
|
||||
await this.jobRepository.queue({ name: JobName.ASSET_UPLOADED, data: { asset } });
|
||||
|
|
|
|||
|
|
@ -305,7 +305,7 @@ describe('AssetService', () => {
|
|||
|
||||
expect(jobMock.queue).toHaveBeenCalledWith({
|
||||
name: JobName.DELETE_FILES,
|
||||
data: { files: ['fake_path/asset_1.jpeg', undefined] },
|
||||
data: { files: ['fake_path/asset_1.jpeg', undefined, undefined] },
|
||||
});
|
||||
expect(storageMock.moveFile).not.toHaveBeenCalled();
|
||||
});
|
||||
|
|
@ -413,10 +413,12 @@ describe('AssetService', () => {
|
|||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
'fake_path/asset_1.mp4',
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
],
|
||||
},
|
||||
});
|
||||
|
|
@ -462,10 +464,12 @@ describe('AssetService', () => {
|
|||
'web-path-1',
|
||||
'resize-path-1',
|
||||
undefined,
|
||||
undefined,
|
||||
'original-path-2',
|
||||
'web-path-2',
|
||||
'resize-path-2',
|
||||
'encoded-video-path-2',
|
||||
undefined,
|
||||
],
|
||||
},
|
||||
},
|
||||
|
|
|
|||
|
|
@ -106,6 +106,7 @@ export class AssetService {
|
|||
dto: CreateAssetDto,
|
||||
file: UploadFile,
|
||||
livePhotoFile?: UploadFile,
|
||||
sidecarFile?: UploadFile,
|
||||
): Promise<AssetFileUploadResponseDto> {
|
||||
if (livePhotoFile) {
|
||||
livePhotoFile = {
|
||||
|
|
@ -122,14 +123,14 @@ export class AssetService {
|
|||
livePhotoAsset = await this.assetCore.create(authUser, livePhotoDto, livePhotoFile);
|
||||
}
|
||||
|
||||
const asset = await this.assetCore.create(authUser, dto, file, livePhotoAsset?.id);
|
||||
const asset = await this.assetCore.create(authUser, dto, file, livePhotoAsset?.id, sidecarFile);
|
||||
|
||||
return { id: asset.id, duplicate: false };
|
||||
} catch (error: any) {
|
||||
// clean up files
|
||||
await this.jobRepository.queue({
|
||||
name: JobName.DELETE_FILES,
|
||||
data: { files: [file.originalPath, livePhotoFile?.originalPath] },
|
||||
data: { files: [file.originalPath, livePhotoFile?.originalPath, sidecarFile?.originalPath] },
|
||||
});
|
||||
|
||||
// handle duplicates with a success response
|
||||
|
|
@ -366,7 +367,13 @@ export class AssetService {
|
|||
await this.jobRepository.queue({ name: JobName.SEARCH_REMOVE_ASSET, data: { ids: [id] } });
|
||||
|
||||
result.push({ id, status: DeleteAssetStatusEnum.SUCCESS });
|
||||
deleteQueue.push(asset.originalPath, asset.webpPath, asset.resizePath, asset.encodedVideoPath);
|
||||
deleteQueue.push(
|
||||
asset.originalPath,
|
||||
asset.webpPath,
|
||||
asset.resizePath,
|
||||
asset.encodedVideoPath,
|
||||
asset.sidecarPath,
|
||||
);
|
||||
|
||||
// TODO refactor this to use cascades
|
||||
if (asset.livePhotoVideoId && !ids.includes(asset.livePhotoVideoId)) {
|
||||
|
|
|
|||
|
|
@ -45,6 +45,9 @@ export class CreateAssetDto {
|
|||
|
||||
@ApiProperty({ type: 'string', format: 'binary' })
|
||||
livePhotoData?: any;
|
||||
|
||||
@ApiProperty({ type: 'string', format: 'binary' })
|
||||
sidecarData?: any;
|
||||
}
|
||||
|
||||
export interface UploadFile {
|
||||
|
|
|
|||
|
|
@ -60,6 +60,11 @@ function fileFilter(req: AuthRequest, file: any, cb: any) {
|
|||
) {
|
||||
cb(null, true);
|
||||
} else {
|
||||
// Additionally support XML but only for sidecar files
|
||||
if (file.fieldname == 'sidecarData' && file.mimetype.match(/\/xml$/)) {
|
||||
return cb(null, true);
|
||||
}
|
||||
|
||||
logger.error(`Unsupported file type ${extname(file.originalname)} file MIME type ${file.mimetype}`);
|
||||
cb(new BadRequestException(`Unsupported file type ${extname(file.originalname)}`), false);
|
||||
}
|
||||
|
|
@ -95,6 +100,11 @@ function filename(req: AuthRequest, file: Express.Multer.File, cb: any) {
|
|||
return cb(null, sanitize(livePhotoFileName));
|
||||
}
|
||||
|
||||
if (file.fieldname === 'sidecarData') {
|
||||
const sidecarFileName = `${fileNameUUID}.xmp`;
|
||||
return cb(null, sanitize(sidecarFileName));
|
||||
}
|
||||
|
||||
const fileName = `${fileNameUUID}${req.body['fileExtension']}`;
|
||||
return cb(null, sanitize(fileName));
|
||||
}
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ import {
|
|||
ThumbnailGeneratorProcessor,
|
||||
VideoTranscodeProcessor,
|
||||
} from './processors';
|
||||
import { MetadataExtractionProcessor } from './processors/metadata-extraction.processor';
|
||||
import { MetadataExtractionProcessor, SidecarProcessor } from './processors/metadata-extraction.processor';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
|
|
@ -31,6 +31,7 @@ import { MetadataExtractionProcessor } from './processors/metadata-extraction.pr
|
|||
BackgroundTaskProcessor,
|
||||
SearchIndexProcessor,
|
||||
FacialRecognitionProcessor,
|
||||
SidecarProcessor,
|
||||
],
|
||||
})
|
||||
export class MicroservicesModule {}
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ import {
|
|||
QueueName,
|
||||
usePagination,
|
||||
WithoutProperty,
|
||||
WithProperty,
|
||||
} from '@app/domain';
|
||||
import { AssetEntity, AssetType, ExifEntity } from '@app/infra/entities';
|
||||
import { Process, Processor } from '@nestjs/bull';
|
||||
|
|
@ -98,13 +99,22 @@ export class MetadataExtractionProcessor {
|
|||
let asset = job.data.asset;
|
||||
|
||||
try {
|
||||
const exifData = await exiftool.read<ImmichTags>(asset.originalPath).catch((error: any) => {
|
||||
const mediaExifData = await exiftool.read<ImmichTags>(asset.originalPath).catch((error: any) => {
|
||||
this.logger.warn(
|
||||
`The exifData parsing failed due to ${error} for asset ${asset.id} at ${asset.originalPath}`,
|
||||
error?.stack,
|
||||
);
|
||||
return null;
|
||||
});
|
||||
const sidecarExifData = asset.sidecarPath
|
||||
? await exiftool.read<ImmichTags>(asset.sidecarPath).catch((error: any) => {
|
||||
this.logger.warn(
|
||||
`The exifData parsing failed due to ${error} for asset ${asset.id} at ${asset.originalPath}`,
|
||||
error?.stack,
|
||||
);
|
||||
return null;
|
||||
})
|
||||
: {};
|
||||
|
||||
const exifToDate = (exifDate: string | ExifDateTime | undefined) => {
|
||||
if (!exifDate) return null;
|
||||
|
|
@ -126,31 +136,46 @@ export class MetadataExtractionProcessor {
|
|||
return exifDate.zone ?? null;
|
||||
};
|
||||
|
||||
const timeZone = exifTimeZone(exifData?.DateTimeOriginal ?? exifData?.CreateDate ?? asset.fileCreatedAt);
|
||||
const fileCreatedAt = exifToDate(exifData?.DateTimeOriginal ?? exifData?.CreateDate ?? asset.fileCreatedAt);
|
||||
const fileModifiedAt = exifToDate(exifData?.ModifyDate ?? asset.fileModifiedAt);
|
||||
const getExifProperty = <T extends keyof ImmichTags>(...properties: T[]): any | null => {
|
||||
for (const property of properties) {
|
||||
const value = sidecarExifData?.[property] ?? mediaExifData?.[property];
|
||||
if (value !== null && value !== undefined) {
|
||||
return value;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
};
|
||||
|
||||
const timeZone = exifTimeZone(getExifProperty('DateTimeOriginal', 'CreateDate') ?? asset.fileCreatedAt);
|
||||
const fileCreatedAt = exifToDate(getExifProperty('DateTimeOriginal', 'CreateDate') ?? asset.fileCreatedAt);
|
||||
const fileModifiedAt = exifToDate(getExifProperty('ModifyDate') ?? asset.fileModifiedAt);
|
||||
const fileStats = fs.statSync(asset.originalPath);
|
||||
const fileSizeInBytes = fileStats.size;
|
||||
|
||||
const newExif = new ExifEntity();
|
||||
newExif.assetId = asset.id;
|
||||
newExif.fileSizeInByte = fileSizeInBytes;
|
||||
newExif.make = exifData?.Make || null;
|
||||
newExif.model = exifData?.Model || null;
|
||||
newExif.exifImageHeight = exifData?.ExifImageHeight || exifData?.ImageHeight || null;
|
||||
newExif.exifImageWidth = exifData?.ExifImageWidth || exifData?.ImageWidth || null;
|
||||
newExif.exposureTime = exifData?.ExposureTime || null;
|
||||
newExif.orientation = exifData?.Orientation?.toString() || null;
|
||||
newExif.make = getExifProperty('Make');
|
||||
newExif.model = getExifProperty('Model');
|
||||
newExif.exifImageHeight = getExifProperty('ExifImageHeight', 'ImageHeight');
|
||||
newExif.exifImageWidth = getExifProperty('ExifImageWidth', 'ImageWidth');
|
||||
newExif.exposureTime = getExifProperty('ExposureTime');
|
||||
newExif.orientation = getExifProperty('Orientation')?.toString();
|
||||
newExif.dateTimeOriginal = fileCreatedAt;
|
||||
newExif.modifyDate = fileModifiedAt;
|
||||
newExif.timeZone = timeZone;
|
||||
newExif.lensModel = exifData?.LensModel || null;
|
||||
newExif.fNumber = exifData?.FNumber || null;
|
||||
newExif.focalLength = exifData?.FocalLength ? parseFloat(exifData.FocalLength) : null;
|
||||
newExif.iso = exifData?.ISO || null;
|
||||
newExif.latitude = exifData?.GPSLatitude || null;
|
||||
newExif.longitude = exifData?.GPSLongitude || null;
|
||||
newExif.livePhotoCID = exifData?.MediaGroupUUID || null;
|
||||
newExif.lensModel = getExifProperty('LensModel');
|
||||
newExif.fNumber = getExifProperty('FNumber');
|
||||
const focalLength = getExifProperty('FocalLength');
|
||||
newExif.focalLength = focalLength ? parseFloat(focalLength) : null;
|
||||
// This is unusual - exifData.ISO should return a number, but experienced that sidecar XMP
|
||||
// files MAY return an array of numbers instead.
|
||||
const iso = getExifProperty('ISO');
|
||||
newExif.iso = Array.isArray(iso) ? iso[0] : iso || null;
|
||||
newExif.latitude = getExifProperty('GPSLatitude');
|
||||
newExif.longitude = getExifProperty('GPSLongitude');
|
||||
newExif.livePhotoCID = getExifProperty('MediaGroupUUID');
|
||||
|
||||
if (newExif.livePhotoCID && !asset.livePhotoVideoId) {
|
||||
const motionAsset = await this.assetCore.findLivePhotoMatch({
|
||||
|
|
@ -220,7 +245,7 @@ export class MetadataExtractionProcessor {
|
|||
}
|
||||
}
|
||||
|
||||
const exifData = await exiftool.read<ImmichTags>(asset.originalPath).catch((error: any) => {
|
||||
const exifData = await exiftool.read<ImmichTags>(asset.sidecarPath || asset.originalPath).catch((error: any) => {
|
||||
this.logger.warn(
|
||||
`The exifData parsing failed due to ${error} for asset ${asset.id} at ${asset.originalPath}`,
|
||||
error?.stack,
|
||||
|
|
@ -345,3 +370,83 @@ export class MetadataExtractionProcessor {
|
|||
return Duration.fromObject({ seconds: videoDurationInSecond }).toFormat('hh:mm:ss.SSS');
|
||||
}
|
||||
}
|
||||
|
||||
@Processor(QueueName.SIDECAR)
|
||||
export class SidecarProcessor {
|
||||
private logger = new Logger(SidecarProcessor.name);
|
||||
private assetCore: AssetCore;
|
||||
|
||||
constructor(
|
||||
@Inject(IAssetRepository) private assetRepository: IAssetRepository,
|
||||
@Inject(IJobRepository) private jobRepository: IJobRepository,
|
||||
) {
|
||||
this.assetCore = new AssetCore(assetRepository, jobRepository);
|
||||
}
|
||||
|
||||
@Process(JobName.QUEUE_SIDECAR)
|
||||
async handleQueueSidecar(job: Job<IBaseJob>) {
|
||||
try {
|
||||
const { force } = job.data;
|
||||
const assetPagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) => {
|
||||
return force
|
||||
? this.assetRepository.getWith(pagination, WithProperty.SIDECAR)
|
||||
: this.assetRepository.getWithout(pagination, WithoutProperty.SIDECAR);
|
||||
});
|
||||
|
||||
for await (const assets of assetPagination) {
|
||||
for (const asset of assets) {
|
||||
const name = force ? JobName.SIDECAR_SYNC : JobName.SIDECAR_DISCOVERY;
|
||||
await this.jobRepository.queue({ name, data: { asset } });
|
||||
}
|
||||
}
|
||||
} catch (error: any) {
|
||||
this.logger.error(`Unable to queue sidecar scanning`, error?.stack);
|
||||
}
|
||||
}
|
||||
|
||||
@Process(JobName.SIDECAR_SYNC)
|
||||
async handleSidecarSync(job: Job<IAssetJob>) {
|
||||
const { asset } = job.data;
|
||||
if (!asset.isVisible) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const name = asset.type === AssetType.VIDEO ? JobName.EXTRACT_VIDEO_METADATA : JobName.EXIF_EXTRACTION;
|
||||
await this.jobRepository.queue({ name, data: { asset } });
|
||||
} catch (error: any) {
|
||||
this.logger.error(`Unable to queue metadata extraction`, error?.stack);
|
||||
}
|
||||
}
|
||||
|
||||
@Process(JobName.SIDECAR_DISCOVERY)
|
||||
async handleSidecarDiscovery(job: Job<IAssetJob>) {
|
||||
let { asset } = job.data;
|
||||
if (!asset.isVisible) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (asset.sidecarPath) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
await fs.promises.access(`${asset.originalPath}.xmp`, fs.constants.W_OK);
|
||||
|
||||
try {
|
||||
asset = await this.assetCore.save({ id: asset.id, sidecarPath: `${asset.originalPath}.xmp` });
|
||||
// TODO: optimize to only queue assets with recent xmp changes
|
||||
const name = asset.type === AssetType.VIDEO ? JobName.EXTRACT_VIDEO_METADATA : JobName.EXIF_EXTRACTION;
|
||||
await this.jobRepository.queue({ name, data: { asset } });
|
||||
} catch (error: any) {
|
||||
this.logger.error(`Unable to sync sidecar`, error?.stack);
|
||||
}
|
||||
} catch (error: any) {
|
||||
if (error.code == 'EACCES') {
|
||||
this.logger.error(`Unable to queue metadata extraction, file is not writable`, error?.stack);
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue