helpy4/immich
1
0
Fork 0
mirror of https://github.com/immich-app/immich synced 2025-11-07 17:27:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

WIP refactor container and queuing system (#206)

Browse source 
* refactor microservices to machine-learning

* Update tGithub issue template with correct task syntax

* Added microservices container

* Communicate between service based on queue system

* added dependency

* Fixed problem with having to import BullQueue into the individual service

* Added todo

* refactor server into monorepo with microservices

* refactor database and entity to library

* added simple migration

* Move migrations and database config to library

* Migration works in library

* Cosmetic change in logging message

* added user dto

* Fixed issue with testing not able to find the shared library

* Clean up library mapping path

* Added webp generator to microservices

* Update Github Action build latest

* Fixed issue NPM cannot install due to conflict witl Bull Queue

* format project with prettier

* Modified docker-compose file

* Add GH Action for Staging build:

* Fixed GH action job name

* Modified GH Action to only build & push latest when pushing to main

* Added Test 2e2 Github Action

* Added Test 2e2 Github Action

* Implemented microservice to extract exif

* Added cronjob to scan and generate webp thumbnail  at midnight

* Refactor to ireduce hit time to database when running microservices

* Added error handling to asset services that handle read file from disk

* Added video transcoding queue to process one video at a time

* Fixed loading spinner on web while loading covering the info panel

* Add mechanism to show new release announcement to web and mobile app (#209)

* Added changelog page

* Fixed issues based on PR comments

* Fixed issue with video transcoding run on the server

* Change entry point content for backward combatibility when starting up server

* Added announcement box

* Added error handling to failed silently when the app version checking is not able to make the request to GITHUB

* Added new version announcement overlay

* Update message

* Added messages

* Added logic to check and show announcement

* Add method to handle saving new version

* Added button to dimiss the acknowledge message

* Up version for deployment to the app store
This commit is contained in:
Alex 2022-06-11 16:12:06 -05:00 • committed by GitHub
parent 397f8c70b4
commit a8220172f8
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
192 changed files with 1823 additions and 2117 deletions
Download patch file Download diff file Expand all files Collapse all files

67
server/apps/microservices/src/processors/asset-uploaded.processor.ts Normal file
View file

@ -0,0 +1,67 @@
import { InjectQueue, OnQueueActive, OnQueueCompleted, OnQueueWaiting, Process, Processor } from '@nestjs/bull';
import { Job, Queue } from 'bull';
import { AssetEntity, AssetType } from '@app/database/entities/asset.entity';
import { InjectRepository } from '@nestjs/typeorm';
import { Repository } from 'typeorm';
import { randomUUID } from 'crypto';
@Processor('asset-uploaded-queue')
export class AssetUploadedProcessor {
constructor(
@InjectQueue('thumbnail-generator-queue')
private thumbnailGeneratorQueue: Queue,
@InjectQueue('metadata-extraction-queue')
private metadataExtractionQueue: Queue,
@InjectQueue('video-conversion-queue')
private videoConversionQueue: Queue,
@InjectRepository(AssetEntity)
private assetRepository: Repository<AssetEntity>,
) {}
/**
* Post processing uploaded asset to perform the following function if missing
* 1. Generate JPEG Thumbnail
* 2. Generate Webp Thumbnail <-> if JPEG thumbnail exist
* 3. EXIF extractor
* 4. Reverse Geocoding
*
* @param job asset-uploaded
*/
@Process('asset-uploaded')
async processUploadedVideo(job: Job) {
const {
asset,
fileName,
fileSize,
hasThumbnail,
}: { asset: AssetEntity; fileName: string; fileSize: number; hasThumbnail: boolean } = job.data;
if (hasThumbnail) {
// The jobs below depends on the existence of jpeg thumbnail
await this.thumbnailGeneratorQueue.add('generate-webp-thumbnail', { asset }, { jobId: randomUUID() });
await this.metadataExtractionQueue.add('tag-image', { asset }, { jobId: randomUUID() });
await this.metadataExtractionQueue.add('detect-object', { asset }, { jobId: randomUUID() });
} else {
// Generate Thumbnail -> Then generate webp, tag image and detect object
}
// Video Conversion
if (asset.type == AssetType.VIDEO) {
await this.videoConversionQueue.add('mp4-conversion', { asset }, { jobId: randomUUID() });
} else {
// Extract Metadata/Exif for Images - Currently the library cannot extract EXIF for video yet
await this.metadataExtractionQueue.add(
'exif-extraction',
{
asset,
fileName,
fileSize,
},
{ jobId: randomUUID() },
);
}
}
}