Implemented image tagging using TensorFlow InceptionV3 (#28)

* Refactor docker-compose to its own folder
* Added FastAPI development environment
* Added support for GPU in docker file
* Added image classification
* creating endpoint for smart Image info
* added logo with white background on ios
* Added endpoint and trigger for image tagging
* Classify image and save into database
* Update readme

server/src/api-v1/asset/entities/asset.entity.ts

@@ -1,5 +1,6 @@
 import { Column, Entity, JoinColumn, OneToOne, PrimaryColumn, PrimaryGeneratedColumn, Unique } from 'typeorm';
 import { ExifEntity } from './exif.entity';
+import { SmartInfoEntity } from './smart-info.entity';
 
 @Entity('assets')
 @Unique(['deviceAssetId', 'userId', 'deviceId'])
@@ -42,6 +43,9 @@ export class AssetEntity {
 
   @OneToOne(() => ExifEntity, (exifEntity) => exifEntity.asset)
   exifInfo: ExifEntity;
+
+  @OneToOne(() => SmartInfoEntity, (smartInfoEntity) => smartInfoEntity.asset)
+  smartInfo: SmartInfoEntity;
 }
 
 export enum AssetType {

server/src/api-v1/asset/entities/smart-info.entity.ts

@@ -0,0 +1,19 @@
+import { Column, Entity, Index, JoinColumn, OneToOne, PrimaryGeneratedColumn } from 'typeorm';
+import { AssetEntity } from './asset.entity';
+
+@Entity('smart_info')
+export class SmartInfoEntity {
+  @PrimaryGeneratedColumn()
+  id: string;
+
+  @Index({ unique: true })
+  @Column({ type: 'uuid' })
+  assetId: string;
+
+  @Column({ type: 'text', array: true, nullable: true })
+  tags: string[];
+
+  @OneToOne(() => AssetEntity, { onDelete: 'CASCADE', nullable: true })
+  @JoinColumn({ name: 'assetId', referencedColumnName: 'id' })
+  asset: SmartInfoEntity;
+}

server/src/config/database.config.ts

@@ -1,12 +1,11 @@
 import { TypeOrmModuleOptions } from '@nestjs/typeorm';
-import dotenv from 'dotenv';
+// import dotenv from 'dotenv';
 
-const result = dotenv.config();
-
-if (result.error) {
-  console.log(result.error);
-}
+// const result = dotenv.config();
 
+// if (result.error) {
+//   console.log(result.error);
+// }
 export const databaseConfig: TypeOrmModuleOptions = {
   type: 'postgres',
   host: 'immich_postgres',
@@ -15,13 +14,10 @@ export const databaseConfig: TypeOrmModuleOptions = {
   password: process.env.DB_PASSWORD,
   database: process.env.DB_DATABASE_NAME,
   entities: [__dirname + '/../**/*.entity.{js,ts}'],
-  synchronize: true,
-  // logging: true,
-  // logger: 'advanced-console',
-  // ssl: process.env.NODE_ENV == 'production',
-  // extra: {
-  //   ssl: {
-  //     rejectUnauthorized: false,
-  //   },
-  // },
+  synchronize: false,
+  migrations: [__dirname + '/../migration/*.js'],
+  cli: {
+    migrationsDir: __dirname + '/../migration',
+  },
+  migrationsRun: true,
 };

server/src/middlewares/redis-io.adapter.middleware.ts

@@ -1,15 +1,23 @@
 import { IoAdapter } from '@nestjs/platform-socket.io';
-import { RedisClient, createClient } from 'redis';
+import { RedisClient } from 'redis';
 import { ServerOptions } from 'socket.io';
-import { createAdapter } from '@socket.io/redis-adapter';
+import { createAdapter } from 'socket.io-redis';
+
+// const pubClient = createClient({ url: 'redis://immich_redis:6379' });
+// const subClient = pubClient.duplicate();
+
+const pubClient = new RedisClient({
+  port: 6379,
+  host: 'immich_redis',
+});
 
-const pubClient = createClient({ url: 'redis://immich_redis:6379' });
 const subClient = pubClient.duplicate();
+const redisAdapter = createAdapter({ pubClient, subClient });
 
 export class RedisIoAdapter extends IoAdapter {
   createIOServer(port: number, options?: ServerOptions): any {
     const server = super.createIOServer(port, options);
-    server.adapter(createAdapter(pubClient, subClient));
+    server.adapter(redisAdapter);
     return server;
   }
 }

server/src/migration/1645130759468-CreateUserTable.ts

@@ -0,0 +1,22 @@
+import { MigrationInterface, QueryRunner } from 'typeorm';
+
+export class CreateUserTable1645130759468 implements MigrationInterface {
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`
+        create table if not exists users
+        (
+            id          uuid      default uuid_generate_v4() not null
+                constraint "PK_a3ffb1c0c8416b9fc6f907b7433"
+                    primary key,
+            email       varchar                              not null,
+            password    varchar                              not null,
+            salt        varchar                              not null,
+            "createdAt" timestamp default now()              not null
+        );
+      `);
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`drop table users`);
+  }
+}

server/src/migration/1645130777674-CreateDeviceInfoTable.ts

@@ -0,0 +1,26 @@
+import { MigrationInterface, QueryRunner } from 'typeorm';
+
+export class CreateDeviceInfoTable1645130777674 implements MigrationInterface {
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`
+        create table if not exists device_info
+        (
+            id                  serial
+                constraint "PK_b1c15a80b0a4e5f4eebadbdd92c"
+                    primary key,
+            "userId"            varchar                 not null,
+            "deviceId"          varchar                 not null,
+            "deviceType"        varchar                 not null,
+            "notificationToken" varchar,
+            "createdAt"         timestamp default now() not null,
+            "isAutoBackup"      boolean   default false not null,
+            constraint "UQ_ebad78f36b10d15fbea8560e107"
+                unique ("userId", "deviceId")
+        );
+      `);
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`drop table device_info`);
+  }
+}

server/src/migration/1645130805273-CreateAssetsTable.ts

@@ -0,0 +1,31 @@
+import { MigrationInterface, QueryRunner } from 'typeorm';
+
+export class CreateAssetsTable1645130805273 implements MigrationInterface {
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`
+        create table if not exists assets
+        (
+            id              uuid    default uuid_generate_v4() not null
+                constraint "PK_da96729a8b113377cfb6a62439c"
+                    primary key,
+            "deviceAssetId" varchar                            not null,
+            "userId"        varchar                            not null,
+            "deviceId"      varchar                            not null,
+            type            varchar                            not null,
+            "originalPath"  varchar                            not null,
+            "resizePath"    varchar,
+            "createdAt"     varchar                            not null,
+            "modifiedAt"    varchar                            not null,
+            "isFavorite"    boolean default false              not null,
+            "mimeType"      varchar,
+            duration        varchar,
+            constraint "UQ_b599ab0bd9574958acb0b30a90e"
+                unique ("deviceAssetId", "userId", "deviceId")
+        );
+      `);
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`drop table assets`);
+  }
+}

server/src/migration/1645130817965-CreateExifTable.ts

@@ -0,0 +1,42 @@
+import { MigrationInterface, QueryRunner } from 'typeorm';
+
+export class CreateExifTable1645130817965 implements MigrationInterface {
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`
+        create table if not exists exif
+        (
+            id                 serial
+                constraint "PK_28663352d85078ad0046dafafaa"
+                    primary key,
+            "assetId"          uuid not null
+                constraint "REL_c0117fdbc50b917ef9067740c4"
+                    unique
+                constraint "FK_c0117fdbc50b917ef9067740c44"
+                    references assets
+                    on delete cascade,
+            make               varchar,
+            model              varchar,
+            "imageName"        varchar,
+            "exifImageWidth"   integer,
+            "exifImageHeight"  integer,
+            "fileSizeInByte"   integer,
+            orientation        varchar,
+            "dateTimeOriginal" timestamp with time zone,
+            "modifyDate"       timestamp with time zone,
+            "lensModel"        varchar,
+            "fNumber"          double precision,
+            "focalLength"      double precision,
+            iso                integer,
+            "exposureTime"     double precision,
+            latitude           double precision,
+            longitude          double precision
+        );
+
+        create unique index if not exists "IDX_c0117fdbc50b917ef9067740c4" on exif ("assetId");
+      `);
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`drop table exif`);
+  }
+}

server/src/migration/1645130870184-CreateSmartInfoTable.ts

@@ -0,0 +1,30 @@
+import { MigrationInterface, QueryRunner } from 'typeorm';
+
+export class CreateSmartInfoTable1645130870184 implements MigrationInterface {
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`
+        create table if not exists smart_info
+        (
+            id        serial
+                constraint "PK_0beace66440e9713f5c40470e46"
+                    primary key,
+            "assetId" uuid not null
+                constraint "UQ_5e3753aadd956110bf3ec0244ac"
+                    unique
+                constraint "FK_5e3753aadd956110bf3ec0244ac"
+                    references assets
+                    on delete cascade,
+            tags      text[]
+        );
+
+        create unique index if not exists "IDX_5e3753aadd956110bf3ec0244a"
+          on smart_info ("assetId");
+      `);
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`
+        drop table smart_info;
+      `);
+  }
+}

server/src/modules/background-task/background-task.module.ts

@@ -3,6 +3,7 @@ import { Module } from '@nestjs/common';
 import { TypeOrmModule } from '@nestjs/typeorm';
 import { AssetEntity } from '../../api-v1/asset/entities/asset.entity';
 import { ExifEntity } from '../../api-v1/asset/entities/exif.entity';
+import { SmartInfoEntity } from '../../api-v1/asset/entities/smart-info.entity';
 import { BackgroundTaskProcessor } from './background-task.processor';
 import { BackgroundTaskService } from './background-task.service';
 
@@ -16,7 +17,7 @@ import { BackgroundTaskService } from './background-task.service';
         removeOnFail: false,
       },
     }),
-    TypeOrmModule.forFeature([AssetEntity, ExifEntity]),
+    TypeOrmModule.forFeature([AssetEntity, ExifEntity, SmartInfoEntity]),
   ],
   providers: [BackgroundTaskService, BackgroundTaskProcessor],
   exports: [BackgroundTaskService],

server/src/modules/background-task/background-task.processor.ts

@@ -9,6 +9,8 @@ import { readFile } from 'fs/promises';
 import fs from 'fs';
 import { Logger } from '@nestjs/common';
 import { ExifEntity } from '../../api-v1/asset/entities/exif.entity';
+import axios from 'axios';
+import { SmartInfoEntity } from '../../api-v1/asset/entities/smart-info.entity';
 
 @Processor('background-task')
 export class BackgroundTaskProcessor {
@@ -16,6 +18,9 @@ export class BackgroundTaskProcessor {
     @InjectRepository(AssetEntity)
     private assetRepository: Repository<AssetEntity>,
 
+    @InjectRepository(SmartInfoEntity)
+    private smartInfoRepository: Repository<SmartInfoEntity>,
+
     @InjectRepository(ExifEntity)
     private exifRepository: Repository<ExifEntity>,
 
@@ -76,4 +81,18 @@ export class BackgroundTaskProcessor {
       });
     });
   }
+
+  @Process('tag-image')
+  async tagImage(job) {
+    const { thumbnailPath, asset }: { thumbnailPath: string; asset: AssetEntity } = job.data;
+    const res = await axios.post('http://immich_tf_fastapi:8000/tagImage', { thumbnail_path: thumbnailPath });
+
+    if (res.status == 200) {
+      const smartInfo = new SmartInfoEntity();
+      smartInfo.assetId = asset.id;
+      smartInfo.tags = [...res.data];
+
+      this.smartInfoRepository.save(smartInfo);
+    }
+  }
 }

server/src/modules/background-task/background-task.service.ts

@@ -32,4 +32,15 @@ export class BackgroundTaskService {
       { jobId: randomUUID() },
     );
   }
+
+  async tagImage(thumbnailPath: string, asset: AssetEntity) {
+    await this.backgroundTaskQueue.add(
+      'tag-image',
+      {
+        thumbnailPath,
+        asset,
+      },
+      { jobId: randomUUID() },
+    );
+  }
 }

server/src/modules/image-optimize/image-optimize.module.ts

@@ -1,20 +1,17 @@
 import { BullModule } from '@nestjs/bull';
 import { Module } from '@nestjs/common';
 import { TypeOrmModule } from '@nestjs/typeorm';
-import { join } from 'path';
-import { AssetModule } from '../../api-v1/asset/asset.module';
-import { AssetService } from '../../api-v1/asset/asset.service';
 import { AssetEntity } from '../../api-v1/asset/entities/asset.entity';
-import { CommunicationGateway } from '../../api-v1/communication/communication.gateway';
 import { CommunicationModule } from '../../api-v1/communication/communication.module';
-import { UserEntity } from '../../api-v1/user/entities/user.entity';
-import { ImmichJwtModule } from '../immich-jwt/immich-jwt.module';
+import { BackgroundTaskModule } from '../background-task/background-task.module';
+import { BackgroundTaskService } from '../background-task/background-task.service';
 import { ImageOptimizeProcessor } from './image-optimize.processor';
 import { AssetOptimizeService } from './image-optimize.service';
 
 @Module({
   imports: [
     CommunicationModule,
+    BackgroundTaskModule,
     BullModule.registerQueue({
       name: 'optimize',
       defaultJobOptions: {
@@ -23,10 +20,17 @@ import { AssetOptimizeService } from './image-optimize.service';
         removeOnFail: false,
       },
     }),
-
+    BullModule.registerQueue({
+      name: 'background-task',
+      defaultJobOptions: {
+        attempts: 3,
+        removeOnComplete: true,
+        removeOnFail: false,
+      },
+    }),
     TypeOrmModule.forFeature([AssetEntity]),
   ],
-  providers: [AssetOptimizeService, ImageOptimizeProcessor],
+  providers: [AssetOptimizeService, ImageOptimizeProcessor, BackgroundTaskService],
   exports: [AssetOptimizeService],
 })
 export class ImageOptimizeModule {}

server/src/modules/image-optimize/image-optimize.processor.ts

@@ -11,6 +11,7 @@ import { APP_UPLOAD_LOCATION } from '../../constants/upload_location.constant';
 import { WebSocketServer } from '@nestjs/websockets';
 import { Socket, Server as SocketIoServer } from 'socket.io';
 import { CommunicationGateway } from '../../api-v1/communication/communication.gateway';
+import { BackgroundTaskService } from '../background-task/background-task.service';
 
 @Processor('optimize')
 export class ImageOptimizeProcessor {
@@ -18,6 +19,8 @@ export class ImageOptimizeProcessor {
     private wsCommunicateionGateway: CommunicationGateway,
     @InjectRepository(AssetEntity)
     private assetRepository: Repository<AssetEntity>,
+
+    private backgroundTaskService: BackgroundTaskService,
   ) {}
 
   @Process('resize-image')
@@ -58,11 +61,15 @@ export class ImageOptimizeProcessor {
             }
 
             const res = await this.assetRepository.update(savedAsset, { resizePath: desitnation });
+
             if (res.affected) {
               this.wsCommunicateionGateway.server
                 .to(savedAsset.userId)
                 .emit('on_upload_success', JSON.stringify(savedAsset));
             }
+
+            // Tag Image
+            this.backgroundTaskService.tagImage(desitnation, savedAsset);
           });
       } else {
         sharp(data)
@@ -79,6 +86,9 @@ export class ImageOptimizeProcessor {
                 .to(savedAsset.userId)
                 .emit('on_upload_success', JSON.stringify(savedAsset));
             }
+
+            // Tag Image
+            this.backgroundTaskService.tagImage(resizePath, savedAsset);
           });
       }
     });
@@ -107,12 +117,18 @@ export class ImageOptimizeProcessor {
         filename: `${filename}.png`,
       })
       .on('end', async (a) => {
+        const thumbnailPath = `${resizeDir}/${filename}.png`;
+
         const res = await this.assetRepository.update(savedAsset, { resizePath: `${resizeDir}/${filename}.png` });
+
         if (res.affected) {
           this.wsCommunicateionGateway.server
             .to(savedAsset.userId)
             .emit('on_upload_success', JSON.stringify(savedAsset));
         }
+
+        // Tag Image
+        this.backgroundTaskService.tagImage(thumbnailPath, savedAsset);
       });
 
     return 'ok';