Added video player in group display, will move to thumbnail for better performance

Makefile

@@ -2,4 +2,4 @@ dev:
 	docker-compose -f ./server/docker-compose.yml up
 
 dev-update:
-	docker-compose -f ./server/docker-compose.yml up --build -V
+	docker-compose -f ./server/docker-compose.yml up --build -V

mobile/android/app/src/main/java/io/flutter/app/FlutterMultiDexApplication.java

@@ -0,0 +1,20 @@
+// Generated file.
+// If you wish to remove Flutter's multidex support, delete this entire file.
+
+package io.flutter.app;
+
+import android.content.Context;
+import androidx.annotation.CallSuper;
+import androidx.multidex.MultiDex;
+
+/**
+ * Extension of {@link io.flutter.app.FlutterApplication}, adding multidex support.
+ */
+public class FlutterMultiDexApplication extends FlutterApplication {
+  @Override
+  @CallSuper
+  protected void attachBaseContext(Context base) {
+    super.attachBaseContext(base);
+    MultiDex.install(this);
+  }
+}

mobile/lib/modules/home/ui/image_grid.dart

@@ -1,6 +1,8 @@
+import 'package:chewie/chewie.dart';
 import 'package:flutter/material.dart';
 import 'package:immich_mobile/modules/home/ui/thumbnail_image.dart';
 import 'package:immich_mobile/shared/models/immich_asset.model.dart';
+import 'package:video_player/video_player.dart';
 
 class ImageGrid extends StatelessWidget {
   final List<ImmichAsset> assetGroup;
@@ -14,9 +16,13 @@ class ImageGrid extends StatelessWidget {
           const SliverGridDelegateWithFixedCrossAxisCount(crossAxisCount: 3, crossAxisSpacing: 5.0, mainAxisSpacing: 5),
       delegate: SliverChildBuilderDelegate(
         (BuildContext context, int index) {
+          var assetType = assetGroup[index].type;
+
           return GestureDetector(
             onTap: () {},
-            child: ThumbnailImage(asset: assetGroup[index]),
+            child: assetType == 'IMAGE'
+                ? ThumbnailImage(asset: assetGroup[index])
+                : VideoThumbnailPlayer(key: Key(assetGroup[index].id), videoAsset: assetGroup[index]),
           );
         },
         childCount: assetGroup.length,
@@ -24,3 +30,56 @@ class ImageGrid extends StatelessWidget {
     );
   }
 }
+
+class VideoThumbnailPlayer extends StatefulWidget {
+  ImmichAsset videoAsset;
+
+  VideoThumbnailPlayer({Key? key, required this.videoAsset}) : super(key: key);
+
+  @override
+  State<VideoThumbnailPlayer> createState() => _VideoThumbnailPlayerState();
+}
+
+class _VideoThumbnailPlayerState extends State<VideoThumbnailPlayer> {
+  late VideoPlayerController videoPlayerController;
+  ChewieController? chewieController;
+
+  @override
+  void initState() {
+    super.initState();
+    initializePlayer();
+  }
+
+  Future<void> initializePlayer() async {
+    videoPlayerController =
+        VideoPlayerController.network('https://flutter.github.io/assets-for-api-docs/assets/videos/butterfly.mp4');
+
+    await Future.wait([
+      videoPlayerController.initialize(),
+    ]);
+    _createChewieController();
+    setState(() {});
+  }
+
+  _createChewieController() {
+    chewieController = ChewieController(
+      showControlsOnInitialize: false,
+      videoPlayerController: videoPlayerController,
+      autoPlay: true,
+      looping: true,
+    );
+  }
+
+  @override
+  Widget build(BuildContext context) {
+    return chewieController != null && chewieController!.videoPlayerController.value.isInitialized
+        ? SizedBox(
+            height: 300,
+            width: 300,
+            child: Chewie(
+              controller: chewieController!,
+            ),
+          )
+        : const Text("Loading Video");
+  }
+}

mobile/lib/shared/models/immich_asset.model.dart

@@ -5,9 +5,7 @@ class ImmichAsset {
   final String deviceAssetId;
   final String userId;
   final String deviceId;
-  final String assetType;
+  final String type;
-  final String localPath;
-  final String remotePath;
   final String createdAt;
   final String modifiedAt;
   final bool isFavorite;
@@ -18,9 +16,7 @@ class ImmichAsset {
     required this.deviceAssetId,
     required this.userId,
     required this.deviceId,
-    required this.assetType,
+    required this.type,
-    required this.localPath,
-    required this.remotePath,
     required this.createdAt,
     required this.modifiedAt,
     required this.isFavorite,
@@ -32,9 +28,7 @@ class ImmichAsset {
     String? deviceAssetId,
     String? userId,
     String? deviceId,
-    String? assetType,
+    String? type,
-    String? localPath,
-    String? remotePath,
     String? createdAt,
     String? modifiedAt,
     bool? isFavorite,
@@ -45,9 +39,7 @@ class ImmichAsset {
       deviceAssetId: deviceAssetId ?? this.deviceAssetId,
       userId: userId ?? this.userId,
       deviceId: deviceId ?? this.deviceId,
-      assetType: assetType ?? this.assetType,
+      type: type ?? this.type,
-      localPath: localPath ?? this.localPath,
-      remotePath: remotePath ?? this.remotePath,
       createdAt: createdAt ?? this.createdAt,
       modifiedAt: modifiedAt ?? this.modifiedAt,
       isFavorite: isFavorite ?? this.isFavorite,
@@ -61,9 +53,7 @@ class ImmichAsset {
       'deviceAssetId': deviceAssetId,
       'userId': userId,
       'deviceId': deviceId,
-      'assetType': assetType,
+      'type': type,
-      'localPath': localPath,
-      'remotePath': remotePath,
       'createdAt': createdAt,
       'modifiedAt': modifiedAt,
       'isFavorite': isFavorite,
@@ -77,9 +67,7 @@ class ImmichAsset {
       deviceAssetId: map['deviceAssetId'] ?? '',
       userId: map['userId'] ?? '',
       deviceId: map['deviceId'] ?? '',
-      assetType: map['assetType'] ?? '',
+      type: map['type'] ?? '',
-      localPath: map['localPath'] ?? '',
-      remotePath: map['remotePath'] ?? '',
       createdAt: map['createdAt'] ?? '',
       modifiedAt: map['modifiedAt'] ?? '',
       isFavorite: map['isFavorite'] ?? false,
@@ -93,7 +81,7 @@ class ImmichAsset {
 
   @override
   String toString() {
-    return 'ImmichAsset(id: $id, deviceAssetId: $deviceAssetId, userId: $userId, deviceId: $deviceId, assetType: $assetType, localPath: $localPath, remotePath: $remotePath, createdAt: $createdAt, modifiedAt: $modifiedAt, isFavorite: $isFavorite, description: $description)';
+    return 'ImmichAsset(id: $id, deviceAssetId: $deviceAssetId, userId: $userId, deviceId: $deviceId, type: $type, createdAt: $createdAt, modifiedAt: $modifiedAt, isFavorite: $isFavorite, description: $description)';
   }
 
   @override
@@ -105,9 +93,7 @@ class ImmichAsset {
         other.deviceAssetId == deviceAssetId &&
         other.userId == userId &&
         other.deviceId == deviceId &&
-        other.assetType == assetType &&
+        other.type == type &&
-        other.localPath == localPath &&
-        other.remotePath == remotePath &&
         other.createdAt == createdAt &&
         other.modifiedAt == modifiedAt &&
         other.isFavorite == isFavorite &&
@@ -120,9 +106,7 @@ class ImmichAsset {
         deviceAssetId.hashCode ^
         userId.hashCode ^
         deviceId.hashCode ^
-        assetType.hashCode ^
+        type.hashCode ^
-        localPath.hashCode ^
-        remotePath.hashCode ^
         createdAt.hashCode ^
         modifiedAt.hashCode ^
         isFavorite.hashCode ^

mobile/pubspec.lock

@@ -155,6 +155,13 @@ packages:
       url: "https://pub.dartlang.org"
     source: hosted
     version: "2.0.1"
+  chewie:
+    dependency: "direct main"
+    description:
+      name: chewie
+      url: "https://pub.dartlang.org"
+    source: hosted
+    version: "1.2.2"
   cli_util:
     dependency: transitive
     description:
@@ -527,6 +534,13 @@ packages:
       url: "https://pub.dartlang.org"
     source: hosted
     version: "1.0.1"
+  nested:
+    dependency: transitive
+    description:
+      name: nested
+      url: "https://pub.dartlang.org"
+    source: hosted
+    version: "1.0.0"
   octo_image:
     dependency: transitive
     description:
@@ -653,6 +667,13 @@ packages:
       url: "https://pub.dartlang.org"
     source: hosted
     version: "4.2.4"
+  provider:
+    dependency: transitive
+    description:
+      name: provider
+      url: "https://pub.dartlang.org"
+    source: hosted
+    version: "5.0.0"
   pub_semver:
     dependency: transitive
     description:
@@ -847,6 +868,41 @@ packages:
       url: "https://pub.dartlang.org"
     source: hosted
     version: "2.1.1"
+  video_player:
+    dependency: "direct main"
+    description:
+      name: video_player
+      url: "https://pub.dartlang.org"
+    source: hosted
+    version: "2.2.18"
+  video_player_android:
+    dependency: transitive
+    description:
+      name: video_player_android
+      url: "https://pub.dartlang.org"
+    source: hosted
+    version: "2.2.17"
+  video_player_avfoundation:
+    dependency: transitive
+    description:
+      name: video_player_avfoundation
+      url: "https://pub.dartlang.org"
+    source: hosted
+    version: "2.2.18"
+  video_player_platform_interface:
+    dependency: transitive
+    description:
+      name: video_player_platform_interface
+      url: "https://pub.dartlang.org"
+    source: hosted
+    version: "5.0.1"
+  video_player_web:
+    dependency: transitive
+    description:
+      name: video_player_web
+      url: "https://pub.dartlang.org"
+    source: hosted
+    version: "2.0.6"
   visibility_detector:
     dependency: "direct main"
     description:
@@ -854,6 +910,41 @@ packages:
       url: "https://pub.dartlang.org"
     source: hosted
     version: "0.2.2"
+  wakelock:
+    dependency: transitive
+    description:
+      name: wakelock
+      url: "https://pub.dartlang.org"
+    source: hosted
+    version: "0.5.6"
+  wakelock_macos:
+    dependency: transitive
+    description:
+      name: wakelock_macos
+      url: "https://pub.dartlang.org"
+    source: hosted
+    version: "0.4.0"
+  wakelock_platform_interface:
+    dependency: transitive
+    description:
+      name: wakelock_platform_interface
+      url: "https://pub.dartlang.org"
+    source: hosted
+    version: "0.3.0"
+  wakelock_web:
+    dependency: transitive
+    description:
+      name: wakelock_web
+      url: "https://pub.dartlang.org"
+    source: hosted
+    version: "0.4.0"
+  wakelock_windows:
+    dependency: transitive
+    description:
+      name: wakelock_windows
+      url: "https://pub.dartlang.org"
+    source: hosted
+    version: "0.2.0"
   watcher:
     dependency: transitive
     description:
@@ -898,4 +989,4 @@ packages:
     version: "3.1.0"
 sdks:
   dart: ">=2.15.1 <3.0.0"
-  flutter: ">=2.5.0"
+  flutter: ">=2.8.0"

mobile/pubspec.yaml

@@ -28,6 +28,8 @@ dependencies:
   visibility_detector: ^0.2.2
   flutter_launcher_icons: "^0.9.2"
   fluttertoast: ^8.0.8
+  video_player: ^2.2.18
+  chewie: ^1.2.2
 
 dev_dependencies:
   flutter_test:

server/Dockerfile

@@ -15,7 +15,8 @@ RUN apt-get update && apt-get install -y --fix-missing --no-install-recommends \
   rsync \
   software-properties-common \
   unzip \
-  wget
+  wget \
+  ffmpeg
 
 # Install NodeJS
 RUN curl --silent --location https://deb.nodesource.com/setup_14.x | bash -
@@ -54,7 +55,8 @@ RUN apt-get update && apt-get install -y --fix-missing --no-install-recommends \
   rsync \
   software-properties-common \
   unzip \
-  wget
+  wget \
+  ffmpeg
 
 # Install NodeJS
 RUN curl --silent --location https://deb.nodesource.com/setup_14.x | bash -

server/Dockerfile-minimal

@@ -1,3 +1,6 @@
+##################################
+# DEVELOPMENT
+##################################
 FROM node:16-bullseye-slim AS development
 
 ARG DEBIAN_FRONTEND=noninteractive
@@ -7,7 +10,7 @@ WORKDIR /usr/src/app
 COPY package.json yarn.lock ./
 
 RUN apt-get update
-RUN apt-get install gcc g++ make cmake python3 python3-pip -y
+RUN apt-get install gcc g++ make cmake python3 python3-pip ffmpeg -y
 
 RUN npm i -g yarn --force
 
@@ -17,6 +20,18 @@ COPY . .
 
 RUN yarn build
 
+# Clean up commands
+RUN apt-get autoremove -y && apt-get clean && \
+  rm -rf /usr/local/src/*
+
+RUN apt-get clean && \
+  rm -rf /var/lib/apt/lists/*
+
+
+
+##################################
+# PRODUCTION
+##################################
 FROM node:16-bullseye-slim as production
 ARG DEBIAN_FRONTEND=noninteractive
 ARG NODE_ENV=production
@@ -27,7 +42,7 @@ WORKDIR /usr/src/app
 COPY package.json yarn.lock ./
 
 RUN apt-get update
-RUN apt-get install gcc g++ make cmake python3 python3-pip -y
+RUN apt-get install gcc g++ make cmake python3 python3-pip ffmpeg -y
 
 RUN npm i -g yarn --force
 
@@ -37,4 +52,12 @@ COPY . .
 
 COPY --from=development /usr/src/app/dist ./dist
 
+# Clean up commands
+RUN apt-get autoremove -y && apt-get clean && \
+  rm -rf /usr/local/src/*
+
+RUN apt-get clean && \
+  rm -rf /var/lib/apt/lists/*
+
+
 CMD ["node", "dist/main"]

server/package.json

@@ -36,12 +36,12 @@
     "@tensorflow/tfjs-converter": "^3.13.0",
     "@tensorflow/tfjs-core": "^3.13.0",
     "@tensorflow/tfjs-node": "^3.13.0",
-    "@types/sharp": "^0.29.5",
     "bcrypt": "^5.0.1",
     "bull": "^4.4.0",
     "class-transformer": "^0.5.1",
     "class-validator": "^0.13.2",
     "dotenv": "^14.2.0",
+    "fluent-ffmpeg": "^2.1.2",
     "joi": "^17.5.0",
     "lodash": "^4.17.21",
     "passport": "^0.5.2",
@@ -61,12 +61,14 @@
     "@types/bcrypt": "^5.0.0",
     "@types/bull": "^3.15.7",
     "@types/express": "^4.17.13",
+    "@types/fluent-ffmpeg": "^2.1.20",
     "@types/imagemin": "^8.0.0",
     "@types/jest": "27.0.2",
     "@types/lodash": "^4.14.178",
     "@types/multer": "^1.4.7",
     "@types/node": "^16.0.0",
     "@types/passport-jwt": "^3.0.6",
+    "@types/sharp": "^0.29.5",
     "@types/supertest": "^2.0.11",
     "@typescript-eslint/eslint-plugin": "^5.0.0",
     "@typescript-eslint/parser": "^5.0.0",

server/src/api-v1/asset/asset.controller.ts

@@ -48,7 +48,7 @@ export class AssetController {
         await this.assetOptimizeService.resizeImage(savedAsset);
       }
 
-      if (savedAsset && savedAsset.type == AssetType.IMAGE) {
+      if (savedAsset && savedAsset.type == AssetType.VIDEO) {
         await this.assetOptimizeService.resizeVideo(savedAsset);
       }
     });

server/src/modules/image-optimize/image-optimize.processor.ts

@@ -6,7 +6,8 @@ import { AssetEntity } from '../../api-v1/asset/entities/asset.entity';
 import sharp from 'sharp';
 import fs, { existsSync, mkdirSync } from 'fs';
 import { ConfigService } from '@nestjs/config';
-import { randomUUID } from 'crypto';
+import ffmpeg from 'fluent-ffmpeg';
+import { Logger } from '@nestjs/common';
 
 @Processor('optimize')
 export class ImageOptimizeProcessor {
@@ -73,30 +74,19 @@ export class ImageOptimizeProcessor {
       mkdirSync(resizeDir, { recursive: true });
     }
 
-    fs.readFile(savedAsset.originalPath, (err, data) => {
+    ffmpeg(savedAsset.originalPath)
-      if (err) {
+      .output(resizePath)
-        console.error('Error Reading File');
+      .noAudio()
-      }
+      .videoCodec('libx264')
-
+      .size('640x?')
-      sharp(data)
+      .aspect('4:3')
-        .resize(512, 512, { fit: 'outside' })
+      .on('error', (e) => {
-        .toFile(resizePath, async (err, info) => {
+        Logger.log(`Error resizing File: ${e}`, 'resizeUploadedVideo');
-          if (err) {
+      })
-            console.error('Error resizing file ', err);
+      .on('end', async () => {
-          }
+        await this.assetRepository.update(savedAsset, { resizePath: resizePath });
-
+      })
-          await this.assetRepository.update(savedAsset, { resizePath: resizePath });
+      .run();
-
-          // Send file to object detection after resizing
-          // const detectionJob = await this.machineLearningQueue.add(
-          //   'object-detection',
-          //   {
-          //     resizePath,
-          //   },
-          //   { jobId: randomUUID() },
-          // );
-        });
-    });
 
     return 'ok';
   }

server/yarn.lock

@@ -1039,6 +1039,13 @@
     "@types/qs" "*"
     "@types/serve-static" "*"
 
+"@types/fluent-ffmpeg@^2.1.20":
+  version "2.1.20"
+  resolved "https://registry.yarnpkg.com/@types/fluent-ffmpeg/-/fluent-ffmpeg-2.1.20.tgz#3b5f42fc8263761d58284fa46ee6759a64ce54ac"
+  integrity sha512-B+OvhCdJ3LgEq2PhvWNOiB/EfwnXLElfMCgc4Z1K5zXgSfo9I6uGKwR/lqmNPFQuebNnes7re3gqkV77SyypLg==
+  dependencies:
+    "@types/node" "*"
+
 "@types/graceful-fs@^4.1.2":
   version "4.1.5"
   resolved "https://registry.npmjs.org/@types/graceful-fs/-/graceful-fs-4.1.5.tgz"
@@ -1739,6 +1746,11 @@ asap@^2.0.0:
   resolved "https://registry.npmjs.org/asap/-/asap-2.0.6.tgz"
   integrity sha1-5QNHYR1+aQlDIIu9r+vLwvuGbUY=
 
+async@>=0.2.9:
+  version "3.2.3"
+  resolved "https://registry.yarnpkg.com/async/-/async-3.2.3.tgz#ac53dafd3f4720ee9e8a160628f18ea91df196c9"
+  integrity sha512-spZRyzKL5l5BZQrr/6m/SqFdBN0q3OCI0f9rjfBzCMBIP4p75P620rR3gTmaksNOhmzgdxcaxdNfMy6anrbM0g==
+
 asynckit@^0.4.0:
   version "0.4.0"
   resolved "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz"
@@ -3094,6 +3106,14 @@ flatted@^3.1.0:
   resolved "https://registry.npmjs.org/flatted/-/flatted-3.2.4.tgz"
   integrity sha512-8/sOawo8tJ4QOBX8YlQBMxL8+RLZfxMQOif9o0KUKTNTjMYElWPE0r/m5VNFxTRd0NSw8qSy8dajrwX4RYI1Hw==
 
+fluent-ffmpeg@^2.1.2:
+  version "2.1.2"
+  resolved "https://registry.yarnpkg.com/fluent-ffmpeg/-/fluent-ffmpeg-2.1.2.tgz#c952de2240f812ebda0aa8006d7776ee2acf7d74"
+  integrity sha1-yVLeIkD4EuvaCqgAbXd27irPfXQ=
+  dependencies:
+    async ">=0.2.9"
+    which "^1.1.1"
+
 follow-redirects@^1.14.4:
   version "1.14.7"
   resolved "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.7.tgz"
@@ -6467,6 +6487,13 @@ whatwg-url@^8.0.0, whatwg-url@^8.5.0:
     tr46 "^2.1.0"
     webidl-conversions "^6.1.0"
 
+which@^1.1.1:
+  version "1.3.1"
+  resolved "https://registry.yarnpkg.com/which/-/which-1.3.1.tgz#a45043d54f5805316da8d62f9f50918d3da70b0a"
+  integrity sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==
+  dependencies:
+    isexe "^2.0.0"
+
 which@^2.0.1:
   version "2.0.2"
   resolved "https://registry.npmjs.org/which/-/which-2.0.2.tgz"