feat(server): transcoding improvements (#1370)

* feat: support isEdited flag for SettingSwitch

* feat: add transcodeAll ffmpeg settings for extra transcoding control

* refactor: tidy up and rename current video transcoding code + transcode everything

* feat: better video transcoding with ffprobe

analyses video files to see if they are already in the desired format
allows admin to choose to transcode all videos regardless of the current format

* fix: always serve encoded video if it exists

* feat: change video codec option to a select box, limit options

removed previous video codec config option as it's incompatible with new options
removed mapping for encoder to codec as we now store the codec in the config

* feat: add video conversion job for transcoding previously missed videos

* chore: fix spelling of job messages to pluralise assets

* chore: fix prettier/eslint warnings

* feat: force switch targetAudioCodec default to aac to avoid iOS incompatibility

* chore: lint issues after rebase

server/apps/immich/src/api-v1/asset/asset-repository.ts

@@ -39,6 +39,7 @@ export interface IAssetRepository {
   getAssetByTimeBucket(userId: string, getAssetByTimeBucketDto: GetAssetByTimeBucketDto): Promise<AssetEntity[]>;
   getAssetByChecksum(userId: string, checksum: Buffer): Promise<AssetEntity>;
   getAssetWithNoThumbnail(): Promise<AssetEntity[]>;
+  getAssetWithNoEncodedVideo(): Promise<AssetEntity[]>;
   getAssetWithNoEXIF(): Promise<AssetEntity[]>;
   getAssetWithNoSmartInfo(): Promise<AssetEntity[]>;
   getExistingAssets(
@@ -80,6 +81,15 @@ export class AssetRepository implements IAssetRepository {
     });
   }
 
+  async getAssetWithNoEncodedVideo(): Promise<AssetEntity[]> {
+    return await this.assetRepository.find({
+      where: [
+        { type: AssetType.VIDEO, encodedVideoPath: IsNull() },
+        { type: AssetType.VIDEO, encodedVideoPath: '' },
+      ],
+    });
+  }
+
   async getAssetWithNoEXIF(): Promise<AssetEntity[]> {
     return await this.assetRepository
       .createQueryBuilder('asset')

server/apps/immich/src/api-v1/asset/asset.service.spec.ts

@@ -128,6 +128,7 @@ describe('AssetService', () => {
       getAssetWithNoEXIF: jest.fn(),
       getAssetWithNoThumbnail: jest.fn(),
       getAssetWithNoSmartInfo: jest.fn(),
+      getAssetWithNoEncodedVideo: jest.fn(),
       getExistingAssets: jest.fn(),
       countByIdAndUser: jest.fn(),
     };

server/apps/immich/src/api-v1/asset/asset.service.ts

@@ -37,13 +37,13 @@ import {
 import { GetAssetCountByTimeBucketDto } from './dto/get-asset-count-by-time-bucket.dto';
 import { GetAssetByTimeBucketDto } from './dto/get-asset-by-time-bucket.dto';
 import { AssetCountByUserIdResponseDto } from './response-dto/asset-count-by-user-id-response.dto';
-import { assetUtils, timeUtils } from '@app/common/utils';
+import { timeUtils } from '@app/common/utils';
 import { CheckExistingAssetsDto } from './dto/check-existing-assets.dto';
 import { CheckExistingAssetsResponseDto } from './response-dto/check-existing-assets-response.dto';
 import { UpdateAssetDto } from './dto/update-asset.dto';
 import { AssetFileUploadResponseDto } from './response-dto/asset-file-upload-response.dto';
 import { BackgroundTaskService } from '../../modules/background-task/background-task.service';
-import { IAssetUploadedJob, IVideoTranscodeJob, QueueName, JobName } from '@app/domain';
+import { IAssetUploadedJob, IVideoTranscodeJob, JobName, QueueName } from '@app/domain';
 import { InjectQueue } from '@nestjs/bull';
 import { Queue } from 'bull';
 import { DownloadService } from '../../modules/download/download.service';
@@ -122,7 +122,7 @@ export class AssetService {
 
         await this.storageService.moveAsset(livePhotoAssetEntity, originalAssetData.originalname);
 
-        await this.videoConversionQueue.add(JobName.MP4_CONVERSION, { asset: livePhotoAssetEntity });
+        await this.videoConversionQueue.add(JobName.VIDEO_CONVERSION, { asset: livePhotoAssetEntity });
       }
 
       const assetEntity = await this.createUserAsset(
@@ -456,7 +456,7 @@ export class AssetService {
 
         await fs.access(videoPath, constants.R_OK | constants.W_OK);
 
-        if (query.isWeb && !assetUtils.isWebPlayable(asset.mimeType)) {
+        if (asset.encodedVideoPath) {
           videoPath = asset.encodedVideoPath == '' ? String(asset.originalPath) : String(asset.encodedVideoPath);
           mimeType = asset.encodedVideoPath == '' ? asset.mimeType : 'video/mp4';
         }

server/apps/immich/src/api-v1/job/job.service.ts

@@ -3,8 +3,8 @@ import {
   IMetadataExtractionJob,
   IThumbnailGenerationJob,
   IVideoTranscodeJob,
-  QueueName,
   JobName,
+  QueueName,
 } from '@app/domain';
 import { InjectQueue } from '@nestjs/bull';
 import { Queue } from 'bull';
@@ -53,7 +53,7 @@ export class JobService {
       case JobId.METADATA_EXTRACTION:
         return this.runMetadataExtractionJob();
       case JobId.VIDEO_CONVERSION:
-        return 0;
+        return this.runVideoConversionJob();
       case JobId.MACHINE_LEARNING:
         return this.runMachineLearningPipeline();
       case JobId.STORAGE_TEMPLATE_MIGRATION:
@@ -79,7 +79,6 @@ export class JobService {
     response.videoConversionQueueCount = videoConversionJobCount;
     response.isMachineLearningActive = Boolean(machineLearningJobCount.waiting);
     response.machineLearningQueueCount = machineLearningJobCount;
-
     response.isStorageMigrationActive = Boolean(storageMigrationJobCount.active);
     response.storageMigrationQueueCount = storageMigrationJobCount;
 
@@ -188,6 +187,22 @@ export class JobService {
     return assetWithNoSmartInfo.length;
   }
 
+  private async runVideoConversionJob(): Promise<number> {
+    const jobCount = await this.videoConversionQueue.getJobCounts();
+
+    if (jobCount.waiting > 0) {
+      throw new BadRequestException('Video conversion job is already running');
+    }
+
+    const assetsWithNoConvertedVideo = await this._assetRepository.getAssetWithNoEncodedVideo();
+
+    for (const asset of assetsWithNoConvertedVideo) {
+      await this.videoConversionQueue.add(JobName.VIDEO_CONVERSION, { asset });
+    }
+
+    return assetsWithNoConvertedVideo.length;
+  }
+
   async runStorageMigration() {
     const jobCount = await this.configQueue.getJobCounts();
 

server/apps/immich/src/modules/schedule-tasks/schedule-tasks.service.ts

@@ -69,7 +69,7 @@ export class ScheduleTasksService {
     });
 
     for (const asset of assets) {
-      await this.videoConversionQueue.add(JobName.MP4_CONVERSION, { asset });
+      await this.videoConversionQueue.add(JobName.VIDEO_CONVERSION, { asset });
     }
   }
 

server/apps/microservices/src/processors/asset-uploaded.processor.ts

@@ -40,7 +40,7 @@ export class AssetUploadedProcessor {
 
     // Video Conversion
     if (asset.type == AssetType.VIDEO) {
-      await this.videoConversionQueue.add(JobName.MP4_CONVERSION, { asset });
+      await this.videoConversionQueue.add(JobName.VIDEO_CONVERSION, { asset });
       await this.metadataExtractionQueue.add(JobName.EXTRACT_VIDEO_METADATA, { asset, fileName });
     } else {
       // Extract Metadata/Exif for Images - Currently the EXIF library on the web cannot extract EXIF for video yet

server/apps/microservices/src/processors/video-transcode.processor.ts

@@ -1,14 +1,12 @@
 import { APP_UPLOAD_LOCATION } from '@app/common/constants';
 import { AssetEntity } from '@app/infra';
-import { QueueName, JobName } from '@app/domain';
-import { IMp4ConversionProcessor } from '@app/domain';
+import { IVideoConversionProcessor, JobName, QueueName, SystemConfigService } from '@app/domain';
 import { Process, Processor } from '@nestjs/bull';
 import { Logger } from '@nestjs/common';
 import { InjectRepository } from '@nestjs/typeorm';
 import { Job } from 'bull';
-import ffmpeg from 'fluent-ffmpeg';
+import ffmpeg, { FfprobeData } from 'fluent-ffmpeg';
 import { existsSync, mkdirSync } from 'fs';
-import { SystemConfigService } from '@app/domain';
 import { Repository } from 'typeorm';
 
 @Processor(QueueName.VIDEO_CONVERSION)
@@ -19,24 +17,60 @@ export class VideoTranscodeProcessor {
     private systemConfigService: SystemConfigService,
   ) {}
 
-  @Process({ name: JobName.MP4_CONVERSION, concurrency: 2 })
-  async mp4Conversion(job: Job<IMp4ConversionProcessor>) {
+  @Process({ name: JobName.VIDEO_CONVERSION, concurrency: 2 })
+  async videoConversion(job: Job<IVideoConversionProcessor>) {
     const { asset } = job.data;
 
-    if (asset.mimeType != 'video/mp4') {
-      const basePath = APP_UPLOAD_LOCATION;
-      const encodedVideoPath = `${basePath}/${asset.userId}/encoded-video`;
+    const basePath = APP_UPLOAD_LOCATION;
+    const encodedVideoPath = `${basePath}/${asset.userId}/encoded-video`;
 
-      if (!existsSync(encodedVideoPath)) {
-        mkdirSync(encodedVideoPath, { recursive: true });
-      }
+    if (!existsSync(encodedVideoPath)) {
+      mkdirSync(encodedVideoPath, { recursive: true });
+    }
 
-      const savedEncodedPath = encodedVideoPath + '/' + asset.id + '.mp4';
+    const savedEncodedPath = `${encodedVideoPath}/${asset.id}.mp4`;
 
-      if (asset.encodedVideoPath == '' || !asset.encodedVideoPath) {
-        // Put the processing into its own async function to prevent the job exist right away
-        await this.runFFMPEGPipeLine(asset, savedEncodedPath);
-      }
+    if (!asset.encodedVideoPath) {
+      // Put the processing into its own async function to prevent the job exist right away
+      await this.runVideoEncode(asset, savedEncodedPath);
+    }
+  }
+
+  async runFFProbePipeline(asset: AssetEntity): Promise<FfprobeData> {
+    return new Promise((resolve, reject) => {
+      ffmpeg.ffprobe(asset.originalPath, (err, data) => {
+        if (err || !data) {
+          Logger.error(`Cannot probe video ${err}`, 'mp4Conversion');
+          reject(err);
+        }
+
+        resolve(data);
+      });
+    });
+  }
+
+  async runVideoEncode(asset: AssetEntity, savedEncodedPath: string): Promise<void> {
+    const config = await this.systemConfigService.getConfig();
+
+    if (config.ffmpeg.transcodeAll) {
+      return this.runFFMPEGPipeLine(asset, savedEncodedPath);
+    }
+
+    const videoInfo = await this.runFFProbePipeline(asset);
+
+    const videoStreams = videoInfo.streams.filter((stream) => {
+      return stream.codec_type === 'video';
+    });
+
+    const longestVideoStream = videoStreams.sort((stream1, stream2) => {
+      const stream1Frames = Number.parseInt(stream1.nb_frames ?? '0');
+      const stream2Frames = Number.parseInt(stream2.nb_frames ?? '0');
+      return stream2Frames - stream1Frames;
+    })[0];
+
+    //TODO: If video or audio are already the correct format, don't re-encode, copy the stream
+    if (longestVideoStream.codec_name !== config.ffmpeg.targetVideoCodec) {
+      return this.runFFMPEGPipeLine(asset, savedEncodedPath);
     }
   }