[mob][preview] add support for video streaming

mobile/ios/Podfile.lock

@@ -10,12 +10,12 @@ PODS:
     - Flutter
   - device_info_plus (0.0.1):
     - Flutter
-  - ffmpeg-kit-ios-min (6.0)
-  - ffmpeg_kit_flutter_min (6.0.3):
-    - ffmpeg_kit_flutter_min/min (= 6.0.3)
+  - ffmpeg-kit-ios-full-gpl (6.0)
+  - ffmpeg_kit_flutter_full_gpl (6.0.3):
+    - ffmpeg_kit_flutter_full_gpl/full-gpl (= 6.0.3)
     - Flutter
-  - ffmpeg_kit_flutter_min/min (6.0.3):
-    - ffmpeg-kit-ios-min (= 6.0)
+  - ffmpeg_kit_flutter_full_gpl/full-gpl (6.0.3):
+    - ffmpeg-kit-ios-full-gpl (= 6.0)
     - Flutter
   - file_saver (0.0.1):
     - Flutter
@@ -232,6 +232,8 @@ PODS:
     - Flutter
   - url_launcher_ios (0.0.1):
     - Flutter
+  - video_compress (0.3.0):
+    - Flutter
   - video_player_avfoundation (0.0.1):
     - Flutter
     - FlutterMacOS
@@ -249,7 +251,7 @@ DEPENDENCIES:
   - connectivity_plus (from `.symlinks/plugins/connectivity_plus/darwin`)
   - dart_ui_isolate (from `.symlinks/plugins/dart_ui_isolate/ios`)
   - device_info_plus (from `.symlinks/plugins/device_info_plus/ios`)
-  - ffmpeg_kit_flutter_min (from `.symlinks/plugins/ffmpeg_kit_flutter_min/ios`)
+  - ffmpeg_kit_flutter_full_gpl (from `.symlinks/plugins/ffmpeg_kit_flutter_full_gpl/ios`)
   - file_saver (from `.symlinks/plugins/file_saver/ios`)
   - firebase_core (from `.symlinks/plugins/firebase_core/ios`)
   - firebase_messaging (from `.symlinks/plugins/firebase_messaging/ios`)
@@ -296,14 +298,15 @@ DEPENDENCIES:
   - ua_client_hints (from `.symlinks/plugins/ua_client_hints/ios`)
   - uni_links (from `.symlinks/plugins/uni_links/ios`)
   - url_launcher_ios (from `.symlinks/plugins/url_launcher_ios/ios`)
+  - video_compress (from `.symlinks/plugins/video_compress/ios`)
   - video_player_avfoundation (from `.symlinks/plugins/video_player_avfoundation/darwin`)
   - video_thumbnail (from `.symlinks/plugins/video_thumbnail/ios`)
   - volume_controller (from `.symlinks/plugins/volume_controller/ios`)
   - wakelock_plus (from `.symlinks/plugins/wakelock_plus/ios`)
 
 SPEC REPOS:
   trunk:
-    - ffmpeg-kit-ios-min
+    - ffmpeg-kit-ios-full-gpl
     - Firebase
     - FirebaseCore
     - FirebaseCoreInternal
@@ -335,8 +338,8 @@ EXTERNAL SOURCES:
     :path: ".symlinks/plugins/dart_ui_isolate/ios"
   device_info_plus:
     :path: ".symlinks/plugins/device_info_plus/ios"
-  ffmpeg_kit_flutter_min:
-    :path: ".symlinks/plugins/ffmpeg_kit_flutter_min/ios"
+  ffmpeg_kit_flutter_full_gpl:
+    :path: ".symlinks/plugins/ffmpeg_kit_flutter_full_gpl/ios"
   file_saver:
     :path: ".symlinks/plugins/file_saver/ios"
   firebase_core:
@@ -429,6 +432,8 @@ EXTERNAL SOURCES:
     :path: ".symlinks/plugins/uni_links/ios"
   url_launcher_ios:
     :path: ".symlinks/plugins/url_launcher_ios/ios"
+  video_compress:
+    :path: ".symlinks/plugins/video_compress/ios"
   video_player_avfoundation:
     :path: ".symlinks/plugins/video_player_avfoundation/darwin"
   video_thumbnail:
@@ -444,8 +449,8 @@ SPEC CHECKSUMS:
   connectivity_plus: ddd7f30999e1faaef5967c23d5b6d503d10434db
   dart_ui_isolate: d5bcda83ca4b04f129d70eb90110b7a567aece14
   device_info_plus: c6fb39579d0f423935b0c9ce7ee2f44b71b9fce6
-  ffmpeg-kit-ios-min: 4e9a088f4ee9629435960b9d68e54848975f1931
-  ffmpeg_kit_flutter_min: 5eff47f4965bf9d1150e98961eb6129f5ae3f28c
+  ffmpeg-kit-ios-full-gpl: 80adc341962e55ef709e36baa8ed9a70cf4ea62b
+  ffmpeg_kit_flutter_full_gpl: 8d15c14c0c3aba616fac04fe44b3d27d02e3c330
   file_saver: 503e386464dbe118f630e17b4c2e1190fa0cf808
   Firebase: 98e6bf5278170668a7983e12971a66b2cd57fc8c
   firebase_core: 2bedc3136ec7c7b8561c6123ed0239387b53f2af
@@ -511,11 +516,12 @@ SPEC CHECKSUMS:
   ua_client_hints: 46bb5817a868f9e397c0ba7e3f2f5c5d90c35156
   uni_links: d97da20c7701486ba192624d99bffaaffcfc298a
   url_launcher_ios: 5334b05cef931de560670eeae103fd3e431ac3fe
+  video_compress: fce97e4fb1dfd88175aa07d2ffc8a2f297f87fbe
   video_player_avfoundation: 7c6c11d8470e1675df7397027218274b6d2360b3
   video_thumbnail: c4e2a3c539e247d4de13cd545344fd2d26ffafd1
   volume_controller: 531ddf792994285c9b17f9d8a7e4dcdd29b3eae9
   wakelock_plus: 8b09852c8876491e4b6d179e17dfe2a0b5f60d47
 
 PODFILE CHECKSUM: 20e086e6008977d43a3d40260f3f9bffcac748dd
 
-COCOAPODS: 1.16.2
+COCOAPODS: 1.15.2

mobile/ios/Runner.xcodeproj/project.pbxproj

@@ -338,18 +338,19 @@
 				"${BUILT_PRODUCTS_DIR}/ua_client_hints/ua_client_hints.framework",
 				"${BUILT_PRODUCTS_DIR}/uni_links/uni_links.framework",
 				"${BUILT_PRODUCTS_DIR}/url_launcher_ios/url_launcher_ios.framework",
+				"${BUILT_PRODUCTS_DIR}/video_compress/video_compress.framework",
 				"${BUILT_PRODUCTS_DIR}/video_player_avfoundation/video_player_avfoundation.framework",
 				"${BUILT_PRODUCTS_DIR}/video_thumbnail/video_thumbnail.framework",
 				"${BUILT_PRODUCTS_DIR}/volume_controller/volume_controller.framework",
 				"${BUILT_PRODUCTS_DIR}/wakelock_plus/wakelock_plus.framework",
-				"${PODS_XCFRAMEWORKS_BUILD_DIR}/ffmpeg-kit-ios-min/ffmpegkit.framework/ffmpegkit",
-				"${PODS_XCFRAMEWORKS_BUILD_DIR}/ffmpeg-kit-ios-min/libavcodec.framework/libavcodec",
-				"${PODS_XCFRAMEWORKS_BUILD_DIR}/ffmpeg-kit-ios-min/libavdevice.framework/libavdevice",
-				"${PODS_XCFRAMEWORKS_BUILD_DIR}/ffmpeg-kit-ios-min/libavfilter.framework/libavfilter",
-				"${PODS_XCFRAMEWORKS_BUILD_DIR}/ffmpeg-kit-ios-min/libavformat.framework/libavformat",
-				"${PODS_XCFRAMEWORKS_BUILD_DIR}/ffmpeg-kit-ios-min/libavutil.framework/libavutil",
-				"${PODS_XCFRAMEWORKS_BUILD_DIR}/ffmpeg-kit-ios-min/libswresample.framework/libswresample",
-				"${PODS_XCFRAMEWORKS_BUILD_DIR}/ffmpeg-kit-ios-min/libswscale.framework/libswscale",
+				"${PODS_XCFRAMEWORKS_BUILD_DIR}/ffmpeg-kit-ios-full-gpl/ffmpegkit.framework/ffmpegkit",
+				"${PODS_XCFRAMEWORKS_BUILD_DIR}/ffmpeg-kit-ios-full-gpl/libavcodec.framework/libavcodec",
+				"${PODS_XCFRAMEWORKS_BUILD_DIR}/ffmpeg-kit-ios-full-gpl/libavdevice.framework/libavdevice",
+				"${PODS_XCFRAMEWORKS_BUILD_DIR}/ffmpeg-kit-ios-full-gpl/libavfilter.framework/libavfilter",
+				"${PODS_XCFRAMEWORKS_BUILD_DIR}/ffmpeg-kit-ios-full-gpl/libavformat.framework/libavformat",
+				"${PODS_XCFRAMEWORKS_BUILD_DIR}/ffmpeg-kit-ios-full-gpl/libavutil.framework/libavutil",
+				"${PODS_XCFRAMEWORKS_BUILD_DIR}/ffmpeg-kit-ios-full-gpl/libswresample.framework/libswresample",
+				"${PODS_XCFRAMEWORKS_BUILD_DIR}/ffmpeg-kit-ios-full-gpl/libswscale.framework/libswscale",
 				"${PODS_XCFRAMEWORKS_BUILD_DIR}/media_kit_libs_ios_video/Ass.framework/Ass",
 				"${PODS_XCFRAMEWORKS_BUILD_DIR}/media_kit_libs_ios_video/Avcodec.framework/Avcodec",
 				"${PODS_XCFRAMEWORKS_BUILD_DIR}/media_kit_libs_ios_video/Avfilter.framework/Avfilter",
@@ -433,6 +434,7 @@
 				"${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/ua_client_hints.framework",
 				"${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/uni_links.framework",
 				"${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/url_launcher_ios.framework",
+				"${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/video_compress.framework",
 				"${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/video_player_avfoundation.framework",
 				"${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/video_thumbnail.framework",
 				"${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/volume_controller.framework",

mobile/lib/db/ml/filedata.dart

@@ -26,6 +26,7 @@ extension FileDataTable on MLDataDB {
     );
   }
 
+  // optimize this
   Future<Set<int>> getFileIDsWithFDData() async {
     final db = await MLDataDB.instance.asyncDB;
     final res = await db.execute('SELECT $fileIDColumn FROM $fileDataTable');

mobile/lib/main.dart

@@ -51,6 +51,7 @@ import "package:photos/utils/email_util.dart";
 import 'package:photos/utils/file_uploader.dart';
 import "package:photos/utils/lock_screen_settings.dart";
 import 'package:shared_preferences/shared_preferences.dart';
+import "package:video_player_media_kit/video_player_media_kit.dart";
 
 final _logger = Logger("main");
 
@@ -236,6 +237,10 @@ Future<void> _init(bool isBackground, {String via = ''}) async {
     ServiceLocator.instance
         .init(preferences, NetworkClient.instance.enteDio, packageInfo);
 
+    if (!isBackground && flagService.internalUser) {
+      VideoPlayerMediaKit.ensureInitialized(iOS: true);
+    }
+
     _logger.info("UserService init $tlog");
     await UserService.instance.init();
     _logger.info("UserService init done $tlog");

mobile/lib/models/metadata/file_magic.dart

@@ -14,6 +14,7 @@ const latKey = "lat";
 const longKey = "long";
 const motionVideoIndexKey = "mvi";
 const noThumbKey = "noThumb";
+const previewVersionKey = "previewVersion";
 
 class MagicMetadata {
   // 0 -> visible

mobile/lib/services/preview_video_store.dart

@@ -0,0 +1,214 @@
+import "dart:async";
+import "dart:io";
+
+import "package:dio/dio.dart";
+import "package:encrypt/encrypt.dart";
+import "package:ffmpeg_kit_flutter_full_gpl/ffmpeg_kit.dart";
+import "package:ffmpeg_kit_flutter_full_gpl/return_code.dart";
+import "package:flutter/foundation.dart" hide Key;
+import "package:logging/logging.dart";
+import "package:path_provider/path_provider.dart";
+import "package:photos/core/network/network.dart";
+import "package:photos/models/file/file.dart";
+import "package:photos/models/file/file_type.dart";
+import "package:photos/utils/file_key.dart";
+import "package:photos/utils/file_util.dart";
+import "package:photos/utils/gzip.dart";
+import "package:video_compress/video_compress.dart";
+
+class PreviewVideoStore {
+  PreviewVideoStore._privateConstructor();
+
+  static final PreviewVideoStore instance =
+      PreviewVideoStore._privateConstructor();
+
+  final _logger = Logger("PreviewVideoStore");
+  final _dio = NetworkClient.instance.enteDio;
+
+  Future<void> chunkAndUploadVideo(EnteFile enteFile) async {
+    if (!enteFile.isUploaded) return;
+    final file = await getFile(enteFile, isOrigin: true);
+    if (file == null) return;
+    try {
+      // check if playlist already exist
+      await getPlaylist(enteFile);
+      final resultUrl = await getPreviewUrl(enteFile);
+      debugPrint("previewUrl $resultUrl");
+      return;
+    } catch (e) {
+      _logger.warning("Failed to get playlist for $enteFile", e);
+    }
+    final tmpDirectory = await getApplicationDocumentsDirectory();
+    final prefix = "${tmpDirectory.path}/${enteFile.generatedID}";
+    Directory(prefix).createSync();
+    final mediaInfo = await VideoCompress.compressVideo(
+      file.path,
+      quality: VideoQuality.Res1280x720Quality,
+    );
+    if (mediaInfo?.path == null) return;
+
+    final key = Key.fromLength(16);
+
+    final keyfile = File('$prefix/keyfile.key');
+    keyfile.writeAsBytesSync(key.bytes);
+
+    final keyinfo = File('$prefix/mykey.keyinfo');
+    keyinfo.writeAsStringSync("data:text/plain;base64,${key.base64}\n"
+        "${keyfile.path}\n");
+
+    final session = await FFmpegKit.execute(
+      '-i "${mediaInfo!.path}" '
+      '-c copy -f hls -hls_time 10 -hls_flags single_file '
+      '-hls_list_size 0 -hls_key_info_file ${keyinfo.path} '
+      '$prefix/output.m3u8',
+    );
+
+    final returnCode = await session.getReturnCode();
+
+    if (ReturnCode.isSuccess(returnCode)) {
+      final playlistFile = File("$prefix/output.m3u8");
+      final previewFile = File("$prefix/output.ts");
+      final result = await _uploadPreviewVideo(enteFile, previewFile);
+      final String objectID = result.$1;
+      final objectSize = result.$2;
+      await _reportVideoPreview(
+        enteFile,
+        playlistFile,
+        objectID: objectID,
+        objectSize: objectSize,
+      );
+      _logger.info("Video preview uploaded for $enteFile");
+    } else if (ReturnCode.isCancel(returnCode)) {
+      _logger.warning("FFmpeg command cancelled");
+    } else {
+      _logger.severe("FFmpeg command failed with return code $returnCode");
+      if (kDebugMode) {
+        final output = await session.getOutput();
+        _logger.severe(output);
+      }
+    }
+  }
+
+  Future<void> _reportVideoPreview(
+    EnteFile file,
+    File playlist, {
+    required String objectID,
+    required int objectSize,
+  }) async {
+    _logger.info("Pushing playlist for ${file.uploadedFileID}");
+    try {
+      final encryptionKey = getFileKey(file);
+      final playlistContent = playlist.readAsStringSync();
+      final result = await gzipAndEncryptJson(
+        {
+          "playlist": playlistContent,
+          'type': 'hls_video',
+        },
+        encryptionKey,
+      );
+      final _ = await _dio.put(
+        "/files/video-data",
+        data: {
+          "fileID": file.uploadedFileID!,
+          "objectID": objectID,
+          "objectSize": objectSize,
+          "playlist": result.encData,
+          "playlistHeader": result.header,
+        },
+      );
+    } catch (e, s) {
+      _logger.severe("Failed to report video preview", e, s);
+    }
+  }
+
+  Future<(String, int)> _uploadPreviewVideo(EnteFile file, File preview) async {
+    _logger.info("Pushing preview for $file");
+    try {
+      final response = await _dio.get(
+        "/files/data/preview-upload-url",
+        queryParameters: {
+          "fileID": file.uploadedFileID!,
+          "type": "vid_preview",
+        },
+      );
+      final uploadURL = response.data["url"];
+      final String objectID = response.data["objectID"];
+      final objectSize = preview.lengthSync();
+      final _ = await _dio.put(
+        uploadURL,
+        data: preview.openRead(),
+        options: Options(
+          headers: {
+            Headers.contentLengthHeader: objectSize,
+          },
+        ),
+      );
+      return (objectID, objectSize);
+    } catch (e) {
+      _logger.warning("failed to upload previewVideo", e);
+      rethrow;
+    }
+  }
+
+  Future<File?> getPlaylist(EnteFile file) async {
+    return await _getPlaylist(file);
+  }
+
+  Future<File?> _getPlaylist(EnteFile file) async {
+    _logger.info("Getting playlist for $file");
+    try {
+      final response = await _dio.get(
+        "/files/data/fetch/",
+        queryParameters: {
+          "fileID": file.uploadedFileID,
+          "type": "vid_preview",
+        },
+      );
+      final encryptedData = response.data["data"]["encryptedData"];
+      final header = response.data["data"]["decryptionHeader"];
+      final encryptionKey = getFileKey(file);
+      final playlistData = await decryptAndUnzipJson(
+        encryptionKey,
+        encryptedData: encryptedData,
+        header: header,
+      );
+      final response2 = await _dio.get(
+        "/files/data/preview",
+        queryParameters: {
+          "fileID": file.uploadedFileID,
+          "type": "vid_preview",
+        },
+      );
+      final previewURL = response2.data["url"];
+      // todo: (prateek/neeraj) review this
+      final finalPlaylist = (playlistData["playlist"])
+          .replaceAll('\nvideo.ts', '\n$previewURL')
+          .replaceAll('\noutput.ts', '\n$previewURL');
+      final tempDir = await getTemporaryDirectory();
+      final playlistFile = File("${tempDir.path}/${file.generatedID}.m3u8");
+      await playlistFile.writeAsString(finalPlaylist);
+      _logger.info("Writing playlist to ${playlistFile.path}");
+
+      return playlistFile;
+    } catch (_) {
+      rethrow;
+    }
+  }
+
+  Future<String> getPreviewUrl(EnteFile file) async {
+    try {
+      final response = await _dio.get(
+        "/files/data/preview",
+        queryParameters: {
+          "fileID": file.uploadedFileID,
+          "type":
+              file.fileType == FileType.video ? "vid_preview" : "img_preview",
+        },
+      );
+      return response.data["url"];
+    } catch (e) {
+      _logger.warning("Failed to get preview url", e);
+      rethrow;
+    }
+  }
+}

mobile/lib/services/sync_service.dart

@@ -18,6 +18,7 @@ import 'package:photos/events/sync_status_update_event.dart';
 import 'package:photos/events/trigger_logout_event.dart';
 import 'package:photos/models/backup_status.dart';
 import 'package:photos/models/file/file_type.dart';
+import "package:photos/services/filedata/filedata_service.dart";
 import "package:photos/services/files_service.dart";
 import 'package:photos/services/local_sync_service.dart';
 import 'package:photos/services/notification_service.dart';
@@ -86,6 +87,7 @@ class SyncService {
     _existingSync = Completer<bool>();
     bool successful = false;
     try {
+      FileDataService.instance.syncFDStatus().ignore();
       await _doSync();
       if (_lastSyncStatusEvent != null &&
           _lastSyncStatusEvent!.status !=