From 91646a809b1429cd16c4e1232dac2b618199813f Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Fri, 30 Aug 2024 12:07:06 +0200 Subject: [PATCH 01/24] [mob][photos] Actual logging in ML Computer --- mobile/lib/services/machine_learning/ml_computer.dart | 4 ++++ .../semantic_search/clip/clip_text_encoder.dart | 4 +++- mobile/lib/utils/image_ml_util.dart | 4 ++-- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/mobile/lib/services/machine_learning/ml_computer.dart b/mobile/lib/services/machine_learning/ml_computer.dart index afbd511c62..8f27dfb3d4 100644 --- a/mobile/lib/services/machine_learning/ml_computer.dart +++ b/mobile/lib/services/machine_learning/ml_computer.dart @@ -4,7 +4,9 @@ import 'dart:isolate'; import 'dart:typed_data' show Uint8List; import "package:dart_ui_isolate/dart_ui_isolate.dart"; +import "package:flutter/foundation.dart" show kDebugMode; import "package:logging/logging.dart"; +import "package:photos/core/error-reporting/super_logging.dart"; import "package:photos/models/ml/face/box.dart"; import "package:photos/services/machine_learning/ml_model.dart"; import "package:photos/services/machine_learning/semantic_search/clip/clip_text_encoder.dart"; @@ -59,6 +61,8 @@ class MLComputer { @pragma('vm:entry-point') static void _isolateMain(SendPort mainSendPort) async { + Logger.root.level = kDebugMode ? Level.ALL : Level.INFO; + Logger.root.onRecord.listen(SuperLogging.onLogRecord); final receivePort = ReceivePort(); mainSendPort.send(receivePort.sendPort); diff --git a/mobile/lib/services/machine_learning/semantic_search/clip/clip_text_encoder.dart b/mobile/lib/services/machine_learning/semantic_search/clip/clip_text_encoder.dart index ff75a9028e..cd59fd1aa5 100644 --- a/mobile/lib/services/machine_learning/semantic_search/clip/clip_text_encoder.dart +++ b/mobile/lib/services/machine_learning/semantic_search/clip/clip_text_encoder.dart @@ -56,7 +56,9 @@ class ClipTextEncoder extends MlModel { final embedding = (outputs[0]?.value as List>)[0]; inputOrt.release(); runOptions.release(); - outputs.forEach((element) => element?.release()); + for (var element in outputs) { + element?.release(); + } normalizeEmbedding(embedding); return embedding; } diff --git a/mobile/lib/utils/image_ml_util.dart b/mobile/lib/utils/image_ml_util.dart index ee259dd8af..e8eb3e7312 100644 --- a/mobile/lib/utils/image_ml_util.dart +++ b/mobile/lib/utils/image_ml_util.dart @@ -150,8 +150,8 @@ Future> generateFaceThumbnailsUsingCanvas( await Future.wait(futureFaceThumbnails); return faceThumbnails; } catch (e) { - log('[ImageMlUtils] Error generating face thumbnails: $e'); - log('[ImageMlUtils] cropImage problematic input argument: ${faceBoxes[i]}'); + _logger.severe('Error generating face thumbnails: $e'); + _logger.severe('cropImage problematic input argument: ${faceBoxes[i]}'); return []; } } From 1b1f54feb0cf1472e933bf4915caa96d860cae2b Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Fri, 30 Aug 2024 12:22:58 +0200 Subject: [PATCH 02/24] [mob][photos] Actual logging in cluster isolate --- .../face_clustering_service.dart | 36 ++++++++++--------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/mobile/lib/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart b/mobile/lib/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart index d37a8821f9..66fa306dcd 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart @@ -8,6 +8,7 @@ import "package:flutter/foundation.dart" show kDebugMode; import "package:logging/logging.dart"; import "package:ml_linalg/dtype.dart"; import "package:ml_linalg/vector.dart"; +import "package:photos/core/error-reporting/super_logging.dart"; import "package:photos/generated/protos/ente/common/vector.pb.dart"; import "package:photos/models/base/id.dart"; import "package:photos/services/machine_learning/face_ml/face_clustering/face_db_info_for_clustering.dart"; @@ -118,6 +119,8 @@ class FaceClusteringService { /// The main execution function of the isolate. static void _isolateMain(SendPort mainSendPort) async { + Logger.root.level = kDebugMode ? Level.ALL : Level.INFO; + Logger.root.onRecord.listen(SuperLogging.onLogRecord); final receivePort = ReceivePort(); mainSendPort.send(receivePort.sendPort); @@ -407,6 +410,8 @@ class FaceClusteringService { } } +final _logger = Logger("FaceLinearClustering"); + ClusteringResult _runLinearClustering(Map args) { // final input = args['input'] as Map; final input = args['input'] as Set; @@ -419,8 +424,8 @@ ClusteringResult _runLinearClustering(Map args) { final oldClusterSummaries = args['oldClusterSummaries'] as Map?; - log( - "[ClusterIsolate] ${DateTime.now()} Copied to isolate ${input.length} faces", + _logger.info( + "Copied to isolate ${input.length} faces", ); // Organize everything into a list of FaceInfo objects @@ -470,14 +475,11 @@ ClusteringResult _runLinearClustering(Map args) { } } final alreadyClusteredCount = facesWithClusterID.length; + final newToClusterCount = facesWithoutClusterID.length; final sortedFaceInfos = []; sortedFaceInfos.addAll(facesWithClusterID); sortedFaceInfos.addAll(facesWithoutClusterID); - log( - "[ClusterIsolate] ${DateTime.now()} Clustering ${facesWithoutClusterID.length} new faces without clusterId, and $alreadyClusteredCount faces with clusterId", - ); - // Make sure the first face has a clusterId final int totalFaces = sortedFaceInfos.length; int dynamicThresholdCount = 0; @@ -487,8 +489,8 @@ ClusteringResult _runLinearClustering(Map args) { } // Start actual clustering - log( - "[ClusterIsolate] ${DateTime.now()} Processing $totalFaces faces in total in this round ${offset != null ? "on top of ${offset + facesWithClusterID.length} earlier processed faces" : ""}", + _logger.info( + "[ClusterIsolate] ${DateTime.now()} Processing $totalFaces faces ($newToClusterCount new, $alreadyClusteredCount already done) in total in this round ${offset != null ? "on top of ${offset + facesWithClusterID.length} earlier processed faces" : ""}", ); // set current epoch time as clusterID String clusterID = newClusterID(); @@ -517,7 +519,7 @@ ClusteringResult _runLinearClustering(Map args) { thresholdValue = distanceThreshold; } if (i % 250 == 0) { - log("[ClusterIsolate] ${DateTime.now()} Processed ${offset != null ? i + offset : i} faces"); + _logger.info("Processed ${offset != null ? i + offset : i} faces"); } // WARNING: The loop below is now O(n^2) so be very careful with anything you put in there! for (int j = i - 1; j >= 0; j--) { @@ -536,8 +538,8 @@ ClusteringResult _runLinearClustering(Map args) { if (closestDistance < thresholdValue) { if (sortedFaceInfos[closestIdx].clusterId == null) { // Ideally this should never happen, but just in case log it - log( - " [ClusterIsolate] [WARNING] ${DateTime.now()} Found new cluster $clusterID", + _logger.severe( + "Found new cluster $clusterID, but closest face has no clusterId", ); clusterID = newClusterID(); sortedFaceInfos[closestIdx].clusterId = clusterID; @@ -568,12 +570,12 @@ ClusteringResult _runLinearClustering(Map args) { } stopwatchClustering.stop(); - log( - ' [ClusterIsolate] ${DateTime.now()} Clustering for ${sortedFaceInfos.length} embeddings executed in ${stopwatchClustering.elapsedMilliseconds}ms', + _logger.info( + 'Clustering for ${sortedFaceInfos.length} embeddings executed in ${stopwatchClustering.elapsedMilliseconds}ms', ); if (useDynamicThreshold) { - log( - "[ClusterIsolate] ${DateTime.now()} Dynamic thresholding: $dynamicThresholdCount faces had a low face score or low blur clarity", + _logger.info( + "Dynamic thresholding: $dynamicThresholdCount faces had a low face score or low blur clarity", ); } @@ -838,8 +840,8 @@ Map _updateClusterSummaries({ ); } } - log( - "[ClusterIsolate] ${DateTime.now()} Calculated cluster summaries in ${DateTime.now().difference(calcSummariesStart).inMilliseconds}ms", + _logger.info( + "Calculated cluster summaries in ${DateTime.now().difference(calcSummariesStart).inMilliseconds}ms", ); return newClusterSummaries; From a9bc6502cb8c32695f3c9048285e727cac439fd7 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Fri, 30 Aug 2024 13:58:46 +0200 Subject: [PATCH 03/24] [mob][photos] proper logging in ML indexing isolate --- .../face_detection_service.dart | 28 +++------- .../face_embedding_service.dart | 8 +-- .../face_ml/face_recognition_service.dart | 54 ++++++++++--------- .../machine_learning/ml_computer.dart | 4 +- .../machine_learning/ml_indexing_isolate.dart | 10 ++-- .../clip/clip_image_encoder.dart | 21 ++++++-- .../semantic_search_service.dart | 7 +-- mobile/lib/utils/image_ml_util.dart | 12 +++-- mobile/lib/utils/ml_util.dart | 15 ++++-- 9 files changed, 85 insertions(+), 74 deletions(-) diff --git a/mobile/lib/services/machine_learning/face_ml/face_detection/face_detection_service.dart b/mobile/lib/services/machine_learning/face_ml/face_detection/face_detection_service.dart index c232317b3a..728de95dae 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_detection/face_detection_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_detection/face_detection_service.dart @@ -1,5 +1,4 @@ import "dart:async"; -import "dart:developer" as dev show log; import 'dart:typed_data' show ByteData, Float32List; import 'dart:ui' as ui show Image; @@ -55,9 +54,8 @@ class FaceDetectionService extends MlModel { 'sessionAddress should be valid', ); - final stopwatch = Stopwatch()..start(); + final startTime = DateTime.now(); - final stopwatchPreprocessing = Stopwatch()..start(); final (inputImageList, newSize) = await preprocessImageToFloat32ChannelsFirst( image, @@ -67,17 +65,12 @@ class FaceDetectionService extends MlModel { requiredHeight: kInputHeight, maintainAspectRatio: true, ); - stopwatchPreprocessing.stop(); - dev.log( - 'Face detection image preprocessing is finished, in ${stopwatchPreprocessing.elapsedMilliseconds}ms', - ); + final preprocessingTime = DateTime.now(); _logger.info( - 'Image decoding and preprocessing is finished, in ${stopwatchPreprocessing.elapsedMilliseconds}ms', + 'Face detection preprocessing is finished, in ${preprocessingTime.difference(startTime).inMilliseconds} ms', ); // Run inference - final stopwatchInterpreter = Stopwatch()..start(); - List>>? nestedResults = []; try { if (MlModel.usePlatformPlugin) { @@ -88,23 +81,16 @@ class FaceDetectionService extends MlModel { inputImageList, ); // [1, 25200, 16] } + _logger.info( + 'inference is finished, in ${DateTime.now().difference(preprocessingTime).inMilliseconds} ms', + ); } catch (e, s) { - dev.log('Error while running inference', error: e, stackTrace: s); + _logger.severe('Error while running inference', e, s); throw YOLOFaceInterpreterRunException(); } - stopwatchInterpreter.stop(); try { - _logger.info( - 'interpreter.run is finished, in ${stopwatchInterpreter.elapsedMilliseconds} ms', - ); - final relativeDetections = _yoloPostProcessOutputs(nestedResults!, newSize); - stopwatch.stop(); - _logger.info( - 'predict() face detection executed in ${stopwatch.elapsedMilliseconds}ms', - ); - return relativeDetections; } catch (e, s) { _logger.severe('Error while post processing', e, s); diff --git a/mobile/lib/services/machine_learning/face_ml/face_embedding/face_embedding_service.dart b/mobile/lib/services/machine_learning/face_ml/face_embedding/face_embedding_service.dart index f36a69c752..0853112c44 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_embedding/face_embedding_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_embedding/face_embedding_service.dart @@ -46,9 +46,11 @@ class FaceEmbeddingService extends MlModel { } else { return _runFFIBasedPredict(input, sessionAddress); } - } catch (e) { - _logger.info( - 'MobileFaceNet (PlatformPlugin: $MlModel.usePlatformPlugin)Error while running inference: $e', + } catch (e, s) { + _logger.severe( + 'Error while running inference (PlatformPlugin: ${MlModel.usePlatformPlugin})', + e, + s, ); throw MobileFaceNetInterpreterRunException(); } diff --git a/mobile/lib/services/machine_learning/face_ml/face_recognition_service.dart b/mobile/lib/services/machine_learning/face_ml/face_recognition_service.dart index 8b3bb7eda1..22e3019517 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_recognition_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_recognition_service.dart @@ -1,5 +1,4 @@ import "dart:async" show unawaited; -import "dart:developer" as dev show log; import "dart:typed_data" show ByteData, Float32List; import "dart:ui" show Image; @@ -16,7 +15,7 @@ import "package:photos/services/machine_learning/ml_result.dart"; import "package:photos/utils/image_ml_util.dart"; class FaceRecognitionService { - final _logger = Logger("FaceRecognitionService"); + static final _logger = Logger("FaceRecognitionService"); // Singleton pattern FaceRecognitionService._privateConstructor(); @@ -76,8 +75,6 @@ class FaceRecognitionService { int faceEmbeddingAddress, ) async { final faceResults = []; - - final Stopwatch stopwatch = Stopwatch()..start(); final startTime = DateTime.now(); // Get the faces @@ -89,19 +86,17 @@ class FaceRecognitionService { faceDetectionAddress, faceResults, ); - dev.log( - "${faceDetectionResult.length} faces detected with scores ${faceDetectionResult.map((e) => e.score).toList()}: completed `detectFacesSync` function, in " - "${stopwatch.elapsedMilliseconds} ms"); + final detectFacesTime = DateTime.now(); + final detectFacesMs = detectFacesTime.difference(startTime).inMilliseconds; // If no faces were detected, return a result with no faces. Otherwise, continue. if (faceDetectionResult.isEmpty) { - dev.log( - "No faceDetectionResult, Completed analyzing image with uploadedFileID $enteFileID, in " - "${stopwatch.elapsedMilliseconds} ms"); + _logger.info( + "Finished runFacesPipeline with fileID $enteFileID in $detectFacesMs ms (${faceDetectionResult.length} faces, detectFaces: $detectFacesMs ms)", + ); return []; } - stopwatch.reset(); // Align the faces final Float32List faceAlignmentResult = await _alignFacesSync( image, @@ -109,23 +104,24 @@ class FaceRecognitionService { faceDetectionResult, faceResults, ); - dev.log("Completed `alignFacesSync` function, in " - "${stopwatch.elapsedMilliseconds} ms"); + final alignFacesTime = DateTime.now(); + final alignFacesMs = + alignFacesTime.difference(detectFacesTime).inMilliseconds; - stopwatch.reset(); // Get the embeddings of the faces - final embeddings = await _embedFacesSync( + await _embedFacesSync( faceAlignmentResult, faceEmbeddingAddress, faceResults, ); - dev.log("Completed `embedFacesSync` function, in " - "${stopwatch.elapsedMilliseconds} ms"); - stopwatch.stop(); + final embedFacesTime = DateTime.now(); + final embedFacesMs = + embedFacesTime.difference(alignFacesTime).inMilliseconds; + final totalMs = DateTime.now().difference(startTime).inMilliseconds; - dev.log("Finished faces pipeline (${embeddings.length} faces) with " - "uploadedFileID $enteFileID, in " - "${DateTime.now().difference(startTime).inMilliseconds} ms"); + _logger.info( + "Finished runFacesPipeline with fileID $enteFileID in $totalMs ms (${faceDetectionResult.length} faces, detectFaces: $detectFacesMs ms, alignFaces: $alignFacesMs ms, embedFaces: $embedFacesMs ms)", + ); return faceResults; } @@ -160,8 +156,8 @@ class FaceRecognitionService { return faces; } on YOLOFaceInterpreterRunException { throw CouldNotRunFaceDetector(); - } catch (e) { - dev.log('[SEVERE] Face detection failed: $e'); + } catch (e, s) { + _logger.severe('Face detection failed', e, s); throw GeneralFaceMlException('Face detection failed: $e'); } } @@ -184,6 +180,9 @@ class FaceRecognitionService { // Store the results if (alignmentResults.length != faces.length) { + _logger.severe( + "The amount of alignment results (${alignmentResults.length}) does not match the number of faces (${faces.length})", + ); throw Exception( "The amount of alignment results (${alignmentResults.length}) does not match the number of faces (${faces.length})", ); @@ -195,7 +194,7 @@ class FaceRecognitionService { return alignedFaces; } catch (e, s) { - dev.log('[SEVERE] Face alignment failed: $e $s'); + _logger.severe('Face alignment failed: $e $s'); throw CouldNotWarpAffine(); } } @@ -214,6 +213,9 @@ class FaceRecognitionService { // Store the results if (embeddings.length != faceResults.length) { + _logger.severe( + "The amount of embeddings (${embeddings.length}) does not match the number of faces (${faceResults.length})", + ); throw Exception( "The amount of embeddings (${embeddings.length}) does not match the number of faces (${faceResults.length})", ); @@ -225,8 +227,8 @@ class FaceRecognitionService { return embeddings; } on MobileFaceNetInterpreterRunException { throw CouldNotRunFaceEmbeddor(); - } catch (e) { - dev.log('[SEVERE] Face embedding (batch) failed: $e'); + } catch (e, s) { + _logger.severe('Face embedding (batch) failed', e, s); throw GeneralFaceMlException('Face embedding (batch) failed: $e'); } } diff --git a/mobile/lib/services/machine_learning/ml_computer.dart b/mobile/lib/services/machine_learning/ml_computer.dart index 8f27dfb3d4..78743bbd84 100644 --- a/mobile/lib/services/machine_learning/ml_computer.dart +++ b/mobile/lib/services/machine_learning/ml_computer.dart @@ -102,6 +102,7 @@ class MLComputer { sendPort.send(true); break; case MLComputerOperation.runClipText: + //TODO:lau check logging here final textEmbedding = await ClipTextEncoder.predict(args); sendPort.send(List.from(textEmbedding, growable: false)); break; @@ -199,7 +200,8 @@ class MLComputer { // Load ClipText model final String modelName = ClipTextEncoder.instance.modelName; - final String? modelPath = await ClipTextEncoder.instance.downloadModelSafe(); + final String? modelPath = + await ClipTextEncoder.instance.downloadModelSafe(); if (modelPath == null) { throw Exception("Could not download clip text model, no wifi"); } diff --git a/mobile/lib/services/machine_learning/ml_indexing_isolate.dart b/mobile/lib/services/machine_learning/ml_indexing_isolate.dart index fec1cbf410..66772dd40b 100644 --- a/mobile/lib/services/machine_learning/ml_indexing_isolate.dart +++ b/mobile/lib/services/machine_learning/ml_indexing_isolate.dart @@ -67,9 +67,7 @@ class MLIndexingIsolate { @pragma('vm:entry-point') static void _isolateMain(SendPort mainSendPort) async { Logger.root.level = kDebugMode ? Level.ALL : Level.INFO; - Logger.root.onRecord.listen((LogRecord rec) { - debugPrint('[MLIsolate] ${rec.toPrettyString()}'); - }); + Logger.root.onRecord.listen(SuperLogging.onLogRecord); final receivePort = ReceivePort(); mainSendPort.send(receivePort.sendPort); receivePort.listen((message) async { @@ -81,11 +79,7 @@ class MLIndexingIsolate { try { switch (function) { case MLIndexingOperation.analyzeImage: - final time = DateTime.now(); final MLResult result = await analyzeImageStatic(args); - _logger.info( - "`analyzeImageSync` function executed in ${DateTime.now().difference(time).inMilliseconds} ms", - ); sendPort.send(result.toJsonString()); break; case MLIndexingOperation.loadModels: @@ -93,6 +87,7 @@ class MLIndexingIsolate { final modelPaths = args['modelPaths'] as List; final addresses = []; for (int i = 0; i < modelNames.length; i++) { + // TODO:lau check logging here final int address = await MlModel.loadModel( modelNames[i], modelPaths[i], @@ -102,6 +97,7 @@ class MLIndexingIsolate { sendPort.send(List.from(addresses, growable: false)); break; case MLIndexingOperation.releaseModels: + // TODO:lau check logging here final modelNames = args['modelNames'] as List; final modelAddresses = args['modelAddresses'] as List; for (int i = 0; i < modelNames.length; i++) { diff --git a/mobile/lib/services/machine_learning/semantic_search/clip/clip_image_encoder.dart b/mobile/lib/services/machine_learning/semantic_search/clip/clip_image_encoder.dart index 1b6e8a9ebe..193e9bac14 100644 --- a/mobile/lib/services/machine_learning/semantic_search/clip/clip_image_encoder.dart +++ b/mobile/lib/services/machine_learning/semantic_search/clip/clip_image_encoder.dart @@ -31,14 +31,27 @@ class ClipImageEncoder extends MlModel { static Future> predict( Image image, ByteData imageByteData, - int sessionAddress, - ) async { + int sessionAddress, [ + int? enteFileID, + ]) async { + final startTime = DateTime.now(); final inputList = await preprocessImageClip(image, imageByteData); + final preprocessingTime = DateTime.now(); + final preprocessingMs = + preprocessingTime.difference(startTime).inMilliseconds; + late List result; if (MlModel.usePlatformPlugin) { - return await _runPlatformPluginPredict(inputList); + result = await _runPlatformPluginPredict(inputList); } else { - return _runFFIBasedPredict(inputList, sessionAddress); + result = _runFFIBasedPredict(inputList, sessionAddress); } + final inferTime = DateTime.now(); + final inferenceMs = inferTime.difference(preprocessingTime).inMilliseconds; + final totalMs = inferTime.difference(startTime).inMilliseconds; + _logger.info( + "Clip predict took $totalMs ms${enteFileID != null ? " with fileID $enteFileID" : ""} (nference: $inferenceMs ms, preprocessing: $preprocessingMs ms)", + ); + return result; } static List _runFFIBasedPredict( diff --git a/mobile/lib/services/machine_learning/semantic_search/semantic_search_service.dart b/mobile/lib/services/machine_learning/semantic_search/semantic_search_service.dart index 7beb284df7..067259307f 100644 --- a/mobile/lib/services/machine_learning/semantic_search/semantic_search_service.dart +++ b/mobile/lib/services/machine_learning/semantic_search/semantic_search_service.dart @@ -24,7 +24,7 @@ import "package:photos/services/machine_learning/semantic_search/clip/clip_image import "package:shared_preferences/shared_preferences.dart"; class SemanticSearchService { - final _logger = Logger("SemanticSearchService"); + static final _logger = Logger("SemanticSearchService"); SemanticSearchService._privateConstructor(); static final SemanticSearchService instance = @@ -293,18 +293,15 @@ class SemanticSearchService { ByteData imageByteData, int clipImageAddress, ) async { - final startTime = DateTime.now(); final embedding = await ClipImageEncoder.predict( image, imageByteData, clipImageAddress, + enteFileID, ); final clipResult = ClipResult(fileID: enteFileID, embedding: embedding); - dev.log('Finished running ClipImage for $enteFileID in ' - '${DateTime.now().difference(startTime).inMilliseconds} ms'); - return clipResult; } } diff --git a/mobile/lib/utils/image_ml_util.dart b/mobile/lib/utils/image_ml_util.dart index e8eb3e7312..0de42f0c98 100644 --- a/mobile/lib/utils/image_ml_util.dart +++ b/mobile/lib/utils/image_ml_util.dart @@ -35,7 +35,9 @@ Future<(Image, ByteData)> decodeImageFromPath(String imagePath) async { await HeifConverter.convert(imagePath, format: 'jpg'); if (jpgPath == null) { _logger.severe('Error converting $format to jpg:', e, s); - throw Exception('InvalidImageFormatException: Error decoding image of format $format'); + throw Exception( + 'InvalidImageFormatException: Error decoding image of format $format', + ); } final imageData = await File(jpgPath).readAsBytes(); final image = await decodeImageFromData(imageData); @@ -47,7 +49,9 @@ Future<(Image, ByteData)> decodeImageFromPath(String imagePath) async { e, s, ); - throw Exception('InvalidImageFormatException: Error decoding image of format $format'); + throw Exception( + 'InvalidImageFormatException: Error decoding image of format $format', + ); } } } @@ -278,7 +282,9 @@ Future<(Float32List, List, List, List, Size)> final (alignmentResult, correctlyEstimated) = SimilarityTransform.estimate(face.allKeypoints); if (!correctlyEstimated) { - log('Face alignment failed because not able to estimate SimilarityTransform, for face: $face'); + _logger.severe( + 'Face alignment failed because not able to estimate SimilarityTransform, for face: $face', + ); throw Exception( 'Face alignment failed because not able to estimate SimilarityTransform', ); diff --git a/mobile/lib/utils/ml_util.dart b/mobile/lib/utils/ml_util.dart index 01ddd353a9..817c0bd4ea 100644 --- a/mobile/lib/utils/ml_util.dart +++ b/mobile/lib/utils/ml_util.dart @@ -388,18 +388,20 @@ Future analyzeImageStatic(Map args) async { final int clipImageAddress = args["clipImageAddress"] as int; _logger.info( - "Start analyzing image with uploadedFileID: $enteFileID inside the isolate", + "Start analyzeImageStatic with fileID: $enteFileID (runFaces: $runFaces, runClip: $runClip)", ); - final time = DateTime.now(); + final startTime = DateTime.now(); // Decode the image once to use for both face detection and alignment final (image, imageByteData) = await decodeImageFromPath(imagePath); - _logger.info('Reading and decoding image took ' - '${DateTime.now().difference(time).inMilliseconds} ms'); final decodedImageSize = Dimensions(height: image.height, width: image.width); final result = MLResult.fromEnteFileID(enteFileID); result.decodedImageSize = decodedImageSize; + final decodeTime = DateTime.now(); + _logger.info( + 'Reading and decoding image took ${decodeTime.difference(startTime).inMilliseconds} ms (fileID: $enteFileID)', + ); if (runFaces) { final resultFaces = await FaceRecognitionService.runFacesPipeline( @@ -426,6 +428,11 @@ Future analyzeImageStatic(Map args) async { result.clip = clipResult; } + final endTime = DateTime.now(); + _logger.info( + 'Finished analyzeImageStatic with fileID: $enteFileID, in ${endTime.difference(startTime).inMilliseconds} ms', + ); + return result; } catch (e, s) { _logger.severe("Could not analyze image", e, s); From d40dc0617147f5ac9709ec4ffb0a1d8cb49d48cb Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Sun, 1 Sep 2024 22:53:24 +0200 Subject: [PATCH 04/24] [mob][photos] MVP logs working in isolate --- .../core/error-reporting/isolate_logging.dart | 59 +++++++++++++++++++ .../core/error-reporting/super_logging.dart | 8 ++- .../face_clustering_service.dart | 7 ++- .../machine_learning/ml_computer.dart | 31 +++++++++- .../machine_learning/ml_indexing_isolate.dart | 5 +- .../ui/settings/ml/ml_user_dev_screen.dart | 11 ++++ 6 files changed, 114 insertions(+), 7 deletions(-) create mode 100644 mobile/lib/core/error-reporting/isolate_logging.dart diff --git a/mobile/lib/core/error-reporting/isolate_logging.dart b/mobile/lib/core/error-reporting/isolate_logging.dart new file mode 100644 index 0000000000..0d686178f8 --- /dev/null +++ b/mobile/lib/core/error-reporting/isolate_logging.dart @@ -0,0 +1,59 @@ +import "dart:collection" show Queue; +import "dart:convert" show jsonEncode, jsonDecode; + +import "package:logging/logging.dart"; +import "package:photos/core/error-reporting/super_logging.dart"; + +class IsolateLogString { + final String logString; + final Object? error; + + IsolateLogString(this.logString, this.error); + + String toJsonString() => jsonEncode({ + 'logString': logString, + 'error': error, + }); + + static IsolateLogString fromJsonString(String jsonString) { + final json = jsonDecode(jsonString); + return IsolateLogString( + json['logString'] as String, + json['error'], + ); + } +} + +class IsolateLogger { + final Queue fileQueueEntries = Queue(); + + Future onLogRecordInIsolate(LogRecord rec) async { + final str = "[ISOLATE]" + rec.toPrettyString(); + + // write to stdout + SuperLogging.printLog(str); + + // push to log queue + fileQueueEntries.add(IsolateLogString(str, rec.error != null)); + } + + /// WARNING: only call this from the isolate + Queue getLogStringsAndClear() { + if (fileQueueEntries.isEmpty) return Queue(); + final result = Queue(); + while (fileQueueEntries.isNotEmpty) { + final entry = fileQueueEntries.removeFirst(); + result.add(entry.toJsonString()); + } + return result; + } + + /// WARNING: only call this from the main thread + static void handLogStringsToMainLogger(Queue logs) { + while (logs.isNotEmpty) { + final logString = logs.removeFirst(); + final log = IsolateLogString.fromJsonString(logString); + SuperLogging.saveLogString(log.logString, log.error); + } + } +} diff --git a/mobile/lib/core/error-reporting/super_logging.dart b/mobile/lib/core/error-reporting/super_logging.dart index 2a677b3fd3..f146b1b14c 100644 --- a/mobile/lib/core/error-reporting/super_logging.dart +++ b/mobile/lib/core/error-reporting/super_logging.dart @@ -270,6 +270,10 @@ class SuperLogging { // write to stdout printLog(str); + saveLogString(str, rec.error); + } + + static void saveLogString(String str, Object? error) { // push to log queue if (fileIsEnabled) { fileQueueEntries.add(str + '\n'); @@ -279,8 +283,8 @@ class SuperLogging { } // add error to sentry queue - if (sentryIsEnabled && rec.error != null) { - _sendErrorToSentry(rec.error!, null).ignore(); + if (sentryIsEnabled && error != null) { + _sendErrorToSentry(error, null).ignore(); } } diff --git a/mobile/lib/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart b/mobile/lib/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart index 66fa306dcd..fefe2910d2 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart @@ -4,7 +4,7 @@ import "dart:isolate"; import "dart:typed_data" show Uint8List; import "package:computer/computer.dart"; -import "package:flutter/foundation.dart" show kDebugMode; +import "package:flutter/foundation.dart" show debugPrint, kDebugMode; import "package:logging/logging.dart"; import "package:ml_linalg/dtype.dart"; import "package:ml_linalg/vector.dart"; @@ -120,7 +120,10 @@ class FaceClusteringService { /// The main execution function of the isolate. static void _isolateMain(SendPort mainSendPort) async { Logger.root.level = kDebugMode ? Level.ALL : Level.INFO; - Logger.root.onRecord.listen(SuperLogging.onLogRecord); + // TODO:lau move to right isolate logging + Logger.root.onRecord.listen((LogRecord rec) { + debugPrint('[MLIsolate] ${rec.toPrettyString()}'); + }); final receivePort = ReceivePort(); mainSendPort.send(receivePort.sendPort); diff --git a/mobile/lib/services/machine_learning/ml_computer.dart b/mobile/lib/services/machine_learning/ml_computer.dart index 78743bbd84..f027208758 100644 --- a/mobile/lib/services/machine_learning/ml_computer.dart +++ b/mobile/lib/services/machine_learning/ml_computer.dart @@ -1,4 +1,5 @@ import 'dart:async'; +import "dart:collection" show Queue; import "dart:io" show File; import 'dart:isolate'; import 'dart:typed_data' show Uint8List; @@ -6,7 +7,7 @@ import 'dart:typed_data' show Uint8List; import "package:dart_ui_isolate/dart_ui_isolate.dart"; import "package:flutter/foundation.dart" show kDebugMode; import "package:logging/logging.dart"; -import "package:photos/core/error-reporting/super_logging.dart"; +import "package:photos/core/error-reporting/isolate_logging.dart"; import "package:photos/models/ml/face/box.dart"; import "package:photos/services/machine_learning/ml_model.dart"; import "package:photos/services/machine_learning/semantic_search/clip/clip_text_encoder.dart"; @@ -20,6 +21,7 @@ enum MLComputerOperation { loadModel, initializeClipTokenizer, runClipText, + testLogging, } class MLComputer { @@ -62,7 +64,8 @@ class MLComputer { @pragma('vm:entry-point') static void _isolateMain(SendPort mainSendPort) async { Logger.root.level = kDebugMode ? Level.ALL : Level.INFO; - Logger.root.onRecord.listen(SuperLogging.onLogRecord); + final IsolateLogger isolateLogger = IsolateLogger(); + Logger.root.onRecord.listen(isolateLogger.onLogRecordInIsolate); final receivePort = ReceivePort(); mainSendPort.send(receivePort.sendPort); @@ -106,6 +109,14 @@ class MLComputer { final textEmbedding = await ClipTextEncoder.predict(args); sendPort.send(List.from(textEmbedding, growable: false)); break; + case MLComputerOperation.testLogging: + final logger = Logger('XXX MLComputerTestLogging'); + logger.info("XXX logging from isolate is working!!!"); + final Queue logStrings = + isolateLogger.getLogStringsAndClear(); + final test = [List.from(logStrings)]; + sendPort.send(test); + break; } } catch (e, stackTrace) { sendPort @@ -221,4 +232,20 @@ class MLComputer { } }); } + + Future testLogging() async { + try { + final test = await _runInIsolate( + ( + MLComputerOperation.testLogging, + {}, + ), + ) as List>; + IsolateLogger.handLogStringsToMainLogger(Queue.from(test[0])); + return; + } catch (e, s) { + _logger.severe("XXX Could not test logging in isolate", e, s); + rethrow; + } + } } diff --git a/mobile/lib/services/machine_learning/ml_indexing_isolate.dart b/mobile/lib/services/machine_learning/ml_indexing_isolate.dart index 66772dd40b..996a766d64 100644 --- a/mobile/lib/services/machine_learning/ml_indexing_isolate.dart +++ b/mobile/lib/services/machine_learning/ml_indexing_isolate.dart @@ -67,7 +67,10 @@ class MLIndexingIsolate { @pragma('vm:entry-point') static void _isolateMain(SendPort mainSendPort) async { Logger.root.level = kDebugMode ? Level.ALL : Level.INFO; - Logger.root.onRecord.listen(SuperLogging.onLogRecord); + // TODO:lau move to right isolate logging + Logger.root.onRecord.listen((LogRecord rec) { + debugPrint('[MLIsolate] ${rec.toPrettyString()}'); + }); final receivePort = ReceivePort(); mainSendPort.send(receivePort.sendPort); receivePort.listen((message) async { diff --git a/mobile/lib/ui/settings/ml/ml_user_dev_screen.dart b/mobile/lib/ui/settings/ml/ml_user_dev_screen.dart index 8d70cee21b..7c4d808592 100644 --- a/mobile/lib/ui/settings/ml/ml_user_dev_screen.dart +++ b/mobile/lib/ui/settings/ml/ml_user_dev_screen.dart @@ -3,6 +3,7 @@ import "package:photos/core/event_bus.dart"; import "package:photos/db/ml/clip_db.dart"; import "package:photos/db/ml/db.dart"; import "package:photos/events/people_changed_event.dart"; +import "package:photos/services/machine_learning/ml_computer.dart"; import "package:photos/services/machine_learning/semantic_search/semantic_search_service.dart"; import "package:photos/theme/ente_theme.dart"; import "package:photos/ui/components/buttons/button_widget.dart"; @@ -55,6 +56,16 @@ class MLUserDeveloperOptions extends StatelessWidget { await deleteAllLocalML(context); }, ), + // TODO:lau remove below code + const SizedBox(height: 24), + ButtonWidget( + buttonType: ButtonType.neutral, + labelText: "Log something in isolate", + onTap: () async { + await MLComputer.instance.testLogging(); + showShortToast(context, "Done"); + }, + ), const SafeArea( child: SizedBox( height: 12, From b90a719972c13b479806f66c2101fbc32293a88c Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Mon, 2 Sep 2024 11:24:57 +0200 Subject: [PATCH 05/24] [mob][photos] Cleaner indication of isolate logging --- mobile/lib/core/error-reporting/isolate_logging.dart | 2 +- mobile/lib/core/error-reporting/super_logging.dart | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/mobile/lib/core/error-reporting/isolate_logging.dart b/mobile/lib/core/error-reporting/isolate_logging.dart index 0d686178f8..5521e0748d 100644 --- a/mobile/lib/core/error-reporting/isolate_logging.dart +++ b/mobile/lib/core/error-reporting/isolate_logging.dart @@ -28,7 +28,7 @@ class IsolateLogger { final Queue fileQueueEntries = Queue(); Future onLogRecordInIsolate(LogRecord rec) async { - final str = "[ISOLATE]" + rec.toPrettyString(); + final str = rec.toPrettyString(null, true); // write to stdout SuperLogging.printLog(str); diff --git a/mobile/lib/core/error-reporting/super_logging.dart b/mobile/lib/core/error-reporting/super_logging.dart index f146b1b14c..c71cea8254 100644 --- a/mobile/lib/core/error-reporting/super_logging.dart +++ b/mobile/lib/core/error-reporting/super_logging.dart @@ -38,8 +38,9 @@ extension SuperString on String { } extension SuperLogRecord on LogRecord { - String toPrettyString([String? extraLines]) { - final header = "[$loggerName] [$level] [$time]"; + String toPrettyString([String? extraLines, bool inIsolate = false]) { + final header = + "[$loggerName${inIsolate ? " (in isolate)" : ""}] [$level] [$time]"; var msg = "$header $message"; From dcac2332965bbcb0f7dd99b156158761dc7364a9 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Mon, 2 Sep 2024 11:50:06 +0200 Subject: [PATCH 06/24] [mob][photos] refactor --- .../lib/core/error-reporting/isolate_logging.dart | 4 ++-- .../services/machine_learning/ml_computer.dart | 15 ++++++++------- mobile/lib/ui/settings/ml/ml_user_dev_screen.dart | 4 ++-- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/mobile/lib/core/error-reporting/isolate_logging.dart b/mobile/lib/core/error-reporting/isolate_logging.dart index 5521e0748d..5ac74b6303 100644 --- a/mobile/lib/core/error-reporting/isolate_logging.dart +++ b/mobile/lib/core/error-reporting/isolate_logging.dart @@ -49,9 +49,9 @@ class IsolateLogger { } /// WARNING: only call this from the main thread - static void handLogStringsToMainLogger(Queue logs) { + static void handLogStringsToMainLogger(List logs) { while (logs.isNotEmpty) { - final logString = logs.removeFirst(); + final logString = logs.removeAt(0); final log = IsolateLogString.fromJsonString(logString); SuperLogging.saveLogString(log.logString, log.error); } diff --git a/mobile/lib/services/machine_learning/ml_computer.dart b/mobile/lib/services/machine_learning/ml_computer.dart index f027208758..f0e01d4c7d 100644 --- a/mobile/lib/services/machine_learning/ml_computer.dart +++ b/mobile/lib/services/machine_learning/ml_computer.dart @@ -114,8 +114,8 @@ class MLComputer { logger.info("XXX logging from isolate is working!!!"); final Queue logStrings = isolateLogger.getLogStringsAndClear(); - final test = [List.from(logStrings)]; - sendPort.send(test); + final logs = List.from(logStrings); + sendPort.send({"data": true, "logs": logs}); break; } } catch (e, stackTrace) { @@ -233,16 +233,17 @@ class MLComputer { }); } - Future testLogging() async { + Future testLogging() async { try { - final test = await _runInIsolate( + final result = await _runInIsolate( ( MLComputerOperation.testLogging, {}, ), - ) as List>; - IsolateLogger.handLogStringsToMainLogger(Queue.from(test[0])); - return; + ) as Map; + final logs = result['logs'] as List; + IsolateLogger.handLogStringsToMainLogger(logs); + return result['data'] as bool; } catch (e, s) { _logger.severe("XXX Could not test logging in isolate", e, s); rethrow; diff --git a/mobile/lib/ui/settings/ml/ml_user_dev_screen.dart b/mobile/lib/ui/settings/ml/ml_user_dev_screen.dart index 7c4d808592..1f75c95d5b 100644 --- a/mobile/lib/ui/settings/ml/ml_user_dev_screen.dart +++ b/mobile/lib/ui/settings/ml/ml_user_dev_screen.dart @@ -62,8 +62,8 @@ class MLUserDeveloperOptions extends StatelessWidget { buttonType: ButtonType.neutral, labelText: "Log something in isolate", onTap: () async { - await MLComputer.instance.testLogging(); - showShortToast(context, "Done"); + final boolean = await MLComputer.instance.testLogging(); + showShortToast(context, "Done: $boolean"); }, ), const SafeArea( From ba083fff70b1664456f23bcb43ca9aba97209af6 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Mon, 2 Sep 2024 12:45:08 +0200 Subject: [PATCH 07/24] [mob][photos] exception handlign works in isolate --- mobile/lib/ui/settings/ml/ml_user_dev_screen.dart | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/mobile/lib/ui/settings/ml/ml_user_dev_screen.dart b/mobile/lib/ui/settings/ml/ml_user_dev_screen.dart index 1f75c95d5b..c50ede988f 100644 --- a/mobile/lib/ui/settings/ml/ml_user_dev_screen.dart +++ b/mobile/lib/ui/settings/ml/ml_user_dev_screen.dart @@ -62,8 +62,17 @@ class MLUserDeveloperOptions extends StatelessWidget { buttonType: ButtonType.neutral, labelText: "Log something in isolate", onTap: () async { - final boolean = await MLComputer.instance.testLogging(); - showShortToast(context, "Done: $boolean"); + try { + final boolean = + await MLComputer.instance.testLogging(); + showShortToast(context, "Done: $boolean"); + } catch (e) { + // ignore: unawaited_futures + showGenericErrorDialog( + context: context, + error: e, + ); + } }, ), const SafeArea( From da316fdcfd38b9d9d94bc81380d2a2854266b1dd Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Mon, 2 Sep 2024 12:45:30 +0200 Subject: [PATCH 08/24] [mob][photos] refactor --- .../machine_learning/ml_computer.dart | 112 +++++++++--------- 1 file changed, 59 insertions(+), 53 deletions(-) diff --git a/mobile/lib/services/machine_learning/ml_computer.dart b/mobile/lib/services/machine_learning/ml_computer.dart index f0e01d4c7d..91f49925af 100644 --- a/mobile/lib/services/machine_learning/ml_computer.dart +++ b/mobile/lib/services/machine_learning/ml_computer.dart @@ -1,5 +1,4 @@ import 'dart:async'; -import "dart:collection" show Queue; import "dart:io" show File; import 'dart:isolate'; import 'dart:typed_data' show Uint8List; @@ -75,56 +74,62 @@ class MLComputer { final args = message[1] as Map; final sendPort = message[2] as SendPort; + late final Object data; try { - switch (function) { - case MLComputerOperation.generateFaceThumbnails: - final imagePath = args['imagePath'] as String; - final Uint8List imageData = await File(imagePath).readAsBytes(); - final faceBoxesJson = - args['faceBoxesList'] as List>; - final List faceBoxes = - faceBoxesJson.map((json) => FaceBox.fromJson(json)).toList(); - final List results = - await generateFaceThumbnailsUsingCanvas( - imageData, - faceBoxes, - ); - sendPort.send(List.from(results)); - case MLComputerOperation.loadModel: - final modelName = args['modelName'] as String; - final modelPath = args['modelPath'] as String; - final int address = await MlModel.loadModel( - modelName, - modelPath, - ); - sendPort.send(address); - break; - case MLComputerOperation.initializeClipTokenizer: - final vocabPath = args["vocabPath"] as String; - await ClipTextTokenizer.instance.init(vocabPath); - sendPort.send(true); - break; - case MLComputerOperation.runClipText: - //TODO:lau check logging here - final textEmbedding = await ClipTextEncoder.predict(args); - sendPort.send(List.from(textEmbedding, growable: false)); - break; - case MLComputerOperation.testLogging: - final logger = Logger('XXX MLComputerTestLogging'); - logger.info("XXX logging from isolate is working!!!"); - final Queue logStrings = - isolateLogger.getLogStringsAndClear(); - final logs = List.from(logStrings); - sendPort.send({"data": true, "logs": logs}); - break; - } + data = await cases(function, args); } catch (e, stackTrace) { - sendPort - .send({'error': e.toString(), 'stackTrace': stackTrace.toString()}); + data = { + 'error': e.toString(), + 'stackTrace': stackTrace.toString(), + }; } + final logs = List.from(isolateLogger.getLogStringsAndClear()); + sendPort.send({"data": data, "logs": logs}); }); } + /// WARNING: Only return primitives: https://api.flutter.dev/flutter/dart-isolate/SendPort/send.html + static Future cases( + MLComputerOperation function, + Map args, + ) async { + switch (function) { + case MLComputerOperation.generateFaceThumbnails: + final imagePath = args['imagePath'] as String; + final Uint8List imageData = await File(imagePath).readAsBytes(); + final faceBoxesJson = + args['faceBoxesList'] as List>; + final List faceBoxes = + faceBoxesJson.map((json) => FaceBox.fromJson(json)).toList(); + final List results = await generateFaceThumbnailsUsingCanvas( + imageData, + faceBoxes, + ); + return List.from(results); + case MLComputerOperation.loadModel: + final modelName = args['modelName'] as String; + final modelPath = args['modelPath'] as String; + final int address = await MlModel.loadModel( + modelName, + modelPath, + ); + return address; + case MLComputerOperation.initializeClipTokenizer: + final vocabPath = args["vocabPath"] as String; + await ClipTextTokenizer.instance.init(vocabPath); + return true; + case MLComputerOperation.runClipText: + //TODO:lau check logging here + final textEmbedding = await ClipTextEncoder.predict(args); + return List.from(textEmbedding, growable: false); + case MLComputerOperation.testLogging: + final logger = Logger('XXX MLComputerTestLogging'); + logger.info("XXX logging from isolate is working!!!"); + throw Exception("XXX logging from isolate testing exception handling"); + return true; + } + } + /// The common method to run any operation in the isolate. It sends the [message] to [_isolateMain] and waits for the result. Future _runInIsolate( (MLComputerOperation, Map) message, @@ -137,15 +142,18 @@ class MLComputer { _mainSendPort.send([message.$1.index, message.$2, answerPort.sendPort]); answerPort.listen((receivedMessage) { - if (receivedMessage is Map && receivedMessage.containsKey('error')) { + final logs = receivedMessage['logs'] as List; + IsolateLogger.handLogStringsToMainLogger(logs); + final data = receivedMessage['data']; + if (data is Map && data.containsKey('error')) { // Handle the error - final errorMessage = receivedMessage['error']; - final errorStackTrace = receivedMessage['stackTrace']; + final errorMessage = data['error']; + final errorStackTrace = data['stackTrace']; final exception = Exception(errorMessage); final stackTrace = StackTrace.fromString(errorStackTrace); completer.completeError(exception, stackTrace); } else { - completer.complete(receivedMessage); + completer.complete(data); } }); @@ -240,10 +248,8 @@ class MLComputer { MLComputerOperation.testLogging, {}, ), - ) as Map; - final logs = result['logs'] as List; - IsolateLogger.handLogStringsToMainLogger(logs); - return result['data'] as bool; + ) as bool; + return result; } catch (e, s) { _logger.severe("XXX Could not test logging in isolate", e, s); rethrow; From 163c8161fc593b95117b7fc07a032142b4149b60 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Mon, 2 Sep 2024 15:44:24 +0200 Subject: [PATCH 09/24] [mob][photos] Isolate base mvp --- mobile/lib/services/isolate_functions.dart | 70 +++++++++ mobile/lib/services/isolate_service.dart | 161 +++++++++++++++++++++ 2 files changed, 231 insertions(+) create mode 100644 mobile/lib/services/isolate_functions.dart create mode 100644 mobile/lib/services/isolate_service.dart diff --git a/mobile/lib/services/isolate_functions.dart b/mobile/lib/services/isolate_functions.dart new file mode 100644 index 0000000000..e0e98a03d6 --- /dev/null +++ b/mobile/lib/services/isolate_functions.dart @@ -0,0 +1,70 @@ +import "dart:io" show File; +import 'dart:typed_data' show Uint8List; + +import "package:logging/logging.dart"; +import "package:photos/models/ml/face/box.dart"; +import "package:photos/services/machine_learning/ml_model.dart"; +import "package:photos/services/machine_learning/semantic_search/clip/clip_text_encoder.dart"; +import "package:photos/services/machine_learning/semantic_search/clip/clip_text_tokenizer.dart"; +import "package:photos/utils/image_ml_util.dart"; + +enum IsolateOperation { + /// [MLComputer] + generateFaceThumbnails, + + /// [MLComputer] + loadModel, + + /// [MLComputer] + initializeClipTokenizer, + + /// [MLComputer] + runClipText, + + /// [MLComputer] + testLogging, +} + +/// WARNING: Only return primitives unless you know the method is only going +/// to be used on regular isolates as opposed to DartUI and Flutter isolates +/// https://api.flutter.dev/flutter/dart-isolate/SendPort/send.html +Future isolateFunction( + IsolateOperation function, + Map args, +) async { + switch (function) { + // Cases for MLComputer + case IsolateOperation.generateFaceThumbnails: + final imagePath = args['imagePath'] as String; + final Uint8List imageData = await File(imagePath).readAsBytes(); + final faceBoxesJson = args['faceBoxesList'] as List>; + final List faceBoxes = + faceBoxesJson.map((json) => FaceBox.fromJson(json)).toList(); + final List results = await generateFaceThumbnailsUsingCanvas( + imageData, + faceBoxes, + ); + return List.from(results); + case IsolateOperation.loadModel: + final modelName = args['modelName'] as String; + final modelPath = args['modelPath'] as String; + final int address = await MlModel.loadModel( + modelName, + modelPath, + ); + return address; + case IsolateOperation.initializeClipTokenizer: + final vocabPath = args["vocabPath"] as String; + await ClipTextTokenizer.instance.init(vocabPath); + return true; + case IsolateOperation.runClipText: + //TODO:lau check logging here + final textEmbedding = await ClipTextEncoder.predict(args); + return List.from(textEmbedding, growable: false); + case IsolateOperation.testLogging: + final logger = Logger('XXX MLComputerTestLogging'); + logger.info("XXX logging from isolate is working!!!"); + throw Exception("XXX logging from isolate testing exception handling"); + return true; + } +} diff --git a/mobile/lib/services/isolate_service.dart b/mobile/lib/services/isolate_service.dart new file mode 100644 index 0000000000..98b365c0ae --- /dev/null +++ b/mobile/lib/services/isolate_service.dart @@ -0,0 +1,161 @@ +import 'dart:async'; +import 'dart:isolate'; + +import "package:dart_ui_isolate/dart_ui_isolate.dart"; +import "package:flutter/foundation.dart" show kDebugMode; +import "package:logging/logging.dart"; +import "package:photos/core/error-reporting/isolate_logging.dart"; +import "package:photos/services/isolate_functions.dart"; +import "package:synchronized/synchronized.dart"; + +abstract class SuperIsolate { + static final Logger abstractLogger = Logger('SuperIsolate'); + Logger get logger; + + Timer? _inactivityTimer; + final Duration _inactivityDuration = const Duration(seconds: 120); + int _activeTasks = 0; + + final _initLock = Lock(); + final _functionLock = Lock(); + + bool get isDartUiIsolate; + bool get automaticDispose; + String get isolateName; + + late dynamic _isolate; + late ReceivePort _receivePort; + late SendPort _mainSendPort; + + bool get isIsolateSpawned => _isIsolateSpawned; + bool _isIsolateSpawned = false; + + Future _initIsolate() async { + return _initLock.synchronized(() async { + if (_isIsolateSpawned) return; + + _receivePort = ReceivePort(); + + try { + _isolate = isDartUiIsolate + ? await DartUiIsolate.spawn( + _isolateMain, + _receivePort.sendPort, + ) + : await Isolate.spawn( + _isolateMain, + _receivePort.sendPort, + debugName: isolateName, + ); + _mainSendPort = await _receivePort.first as SendPort; + + if (automaticDispose) _resetInactivityTimer(); + logger.info('initIsolate done'); + _isIsolateSpawned = true; + } catch (e) { + logger.severe('Could not spawn isolate', e); + _isIsolateSpawned = false; + } + }); + } + + @pragma('vm:entry-point') + static void _isolateMain(SendPort mainSendPort) async { + Logger.root.level = kDebugMode ? Level.ALL : Level.INFO; + final IsolateLogger isolateLogger = IsolateLogger(); + Logger.root.onRecord.listen(isolateLogger.onLogRecordInIsolate); + final receivePort = ReceivePort(); + mainSendPort.send(receivePort.sendPort); + + receivePort.listen((message) async { + final functionIndex = message[0] as int; + final function = IsolateOperation.values[functionIndex]; + final args = message[1] as Map; + final sendPort = message[2] as SendPort; + + late final Object data; + try { + data = await isolateFunction(function, args); + } catch (e, stackTrace) { + data = { + 'error': e.toString(), + 'stackTrace': stackTrace.toString(), + }; + } + final logs = List.from(isolateLogger.getLogStringsAndClear()); + sendPort.send({"data": data, "logs": logs}); + }); + } + + /// The common method to run any operation in the isolate. It sends the [message] to [_isolateMain] and waits for the result. + Future _runInIsolate( + (int, Map) message, + ) async { + await _initIsolate(); + return _functionLock.synchronized(() async { + if (automaticDispose) _resetInactivityTimer(); + + if (_postLockStop()) { + return null; + } + + final completer = Completer(); + final answerPort = ReceivePort(); + + _activeTasks++; + _mainSendPort.send([message.$1, message.$2, answerPort.sendPort]); + + answerPort.listen((receivedMessage) { + final logs = receivedMessage['logs'] as List; + IsolateLogger.handLogStringsToMainLogger(logs); + final data = receivedMessage['data']; + if (data is Map && data.containsKey('error')) { + // Handle the error + final errorMessage = data['error']; + final errorStackTrace = data['stackTrace']; + final exception = Exception(errorMessage); + final stackTrace = StackTrace.fromString(errorStackTrace); + completer.completeError(exception, stackTrace); + } else { + completer.complete(data); + } + }); + _activeTasks--; + + return completer.future; + }); + } + + bool _postLockStop() => false; + + /// Resets a timer that kills the isolate after a certain amount of inactivity. + /// + /// Should be called after initialization (e.g. inside `init()`) and after every call to isolate (e.g. inside `_runInIsolate()`) + void _resetInactivityTimer() { + _inactivityTimer?.cancel(); + _inactivityTimer = Timer(_inactivityDuration, () { + if (_activeTasks > 0) { + logger.info('Tasks are still running. Delaying isolate disposal.'); + // Optionally, reschedule the timer to check again later. + _resetInactivityTimer(); + } else { + logger.info( + 'Clustering Isolate has been inactive for ${_inactivityDuration.inSeconds} seconds with no tasks running. Killing isolate.', + ); + _disposeIsolate(); + } + }); + } + + Future _onDispose(); + + void _disposeIsolate() async { + if (!_isIsolateSpawned) return; + logger.info('Disposing isolate'); + await _onDispose(); + _isIsolateSpawned = false; + _isolate.kill(); + _receivePort.close(); + _inactivityTimer?.cancel(); + } +} From 9fe76eb52726550c2486927556ce881e5b6eea50 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Mon, 2 Sep 2024 16:56:04 +0200 Subject: [PATCH 10/24] [mob][photos] Use isolate abstraction on MLComputer --- mobile/lib/services/isolate_functions.dart | 18 +- mobile/lib/services/isolate_service.dart | 18 +- .../machine_learning/ml_computer.dart | 211 +++--------------- 3 files changed, 59 insertions(+), 188 deletions(-) diff --git a/mobile/lib/services/isolate_functions.dart b/mobile/lib/services/isolate_functions.dart index e0e98a03d6..75de71c9d3 100644 --- a/mobile/lib/services/isolate_functions.dart +++ b/mobile/lib/services/isolate_functions.dart @@ -33,7 +33,9 @@ Future isolateFunction( Map args, ) async { switch (function) { - // Cases for MLComputer + /// Cases for MLComputer start here + + /// MLComputer case IsolateOperation.generateFaceThumbnails: final imagePath = args['imagePath'] as String; final Uint8List imageData = await File(imagePath).readAsBytes(); @@ -45,6 +47,8 @@ Future isolateFunction( faceBoxes, ); return List.from(results); + + /// MLComputer case IsolateOperation.loadModel: final modelName = args['modelName'] as String; final modelPath = args['modelPath'] as String; @@ -53,18 +57,28 @@ Future isolateFunction( modelPath, ); return address; + + /// MLComputer case IsolateOperation.initializeClipTokenizer: final vocabPath = args["vocabPath"] as String; await ClipTextTokenizer.instance.init(vocabPath); return true; + + /// MLComputer case IsolateOperation.runClipText: //TODO:lau check logging here final textEmbedding = await ClipTextEncoder.predict(args); return List.from(textEmbedding, growable: false); + + /// MLComputer case IsolateOperation.testLogging: final logger = Logger('XXX MLComputerTestLogging'); logger.info("XXX logging from isolate is working!!!"); - throw Exception("XXX logging from isolate testing exception handling"); + // throw Exception("XXX logging from isolate testing exception handling"); return true; + + /// Cases for MLComputer ends here + + /// } } diff --git a/mobile/lib/services/isolate_service.dart b/mobile/lib/services/isolate_service.dart index 98b365c0ae..4c7227516e 100644 --- a/mobile/lib/services/isolate_service.dart +++ b/mobile/lib/services/isolate_service.dart @@ -9,7 +9,6 @@ import "package:photos/services/isolate_functions.dart"; import "package:synchronized/synchronized.dart"; abstract class SuperIsolate { - static final Logger abstractLogger = Logger('SuperIsolate'); Logger get logger; Timer? _inactivityTimer; @@ -20,7 +19,7 @@ abstract class SuperIsolate { final _functionLock = Lock(); bool get isDartUiIsolate; - bool get automaticDispose; + bool get shouldAutomaticDispose; String get isolateName; late dynamic _isolate; @@ -49,7 +48,7 @@ abstract class SuperIsolate { ); _mainSendPort = await _receivePort.first as SendPort; - if (automaticDispose) _resetInactivityTimer(); + if (shouldAutomaticDispose) _resetInactivityTimer(); logger.info('initIsolate done'); _isIsolateSpawned = true; } catch (e) { @@ -87,13 +86,16 @@ abstract class SuperIsolate { }); } - /// The common method to run any operation in the isolate. It sends the [message] to [_isolateMain] and waits for the result. - Future _runInIsolate( - (int, Map) message, + /// The common method to run any operation in the isolate. + /// It sends the [message] to [_isolateMain] and waits for the result. + /// The actual function executed is [isolateFunction]. + Future runInIsolate( + IsolateOperation operation, + Map args, ) async { await _initIsolate(); return _functionLock.synchronized(() async { - if (automaticDispose) _resetInactivityTimer(); + if (shouldAutomaticDispose) _resetInactivityTimer(); if (_postLockStop()) { return null; @@ -103,7 +105,7 @@ abstract class SuperIsolate { final answerPort = ReceivePort(); _activeTasks++; - _mainSendPort.send([message.$1, message.$2, answerPort.sendPort]); + _mainSendPort.send([operation.index, args, answerPort.sendPort]); answerPort.listen((receivedMessage) { final logs = receivedMessage['logs'] as List; diff --git a/mobile/lib/services/machine_learning/ml_computer.dart b/mobile/lib/services/machine_learning/ml_computer.dart index 91f49925af..7293d336f1 100644 --- a/mobile/lib/services/machine_learning/ml_computer.dart +++ b/mobile/lib/services/machine_learning/ml_computer.dart @@ -1,166 +1,36 @@ import 'dart:async'; -import "dart:io" show File; -import 'dart:isolate'; import 'dart:typed_data' show Uint8List; -import "package:dart_ui_isolate/dart_ui_isolate.dart"; -import "package:flutter/foundation.dart" show kDebugMode; import "package:logging/logging.dart"; -import "package:photos/core/error-reporting/isolate_logging.dart"; import "package:photos/models/ml/face/box.dart"; -import "package:photos/services/machine_learning/ml_model.dart"; +import "package:photos/services/isolate_functions.dart"; +import "package:photos/services/isolate_service.dart"; import "package:photos/services/machine_learning/semantic_search/clip/clip_text_encoder.dart"; -import "package:photos/services/machine_learning/semantic_search/clip/clip_text_tokenizer.dart"; import "package:photos/services/remote_assets_service.dart"; import "package:photos/utils/image_ml_util.dart"; import "package:synchronized/synchronized.dart"; -enum MLComputerOperation { - generateFaceThumbnails, - loadModel, - initializeClipTokenizer, - runClipText, - testLogging, -} - -class MLComputer { +class MLComputer extends SuperIsolate { + @override + Logger get logger => _logger; final _logger = Logger('MLComputer'); - final _initLock = Lock(); - final _functionLock = Lock(); final _initModelLock = Lock(); - late ReceivePort _receivePort = ReceivePort(); - late SendPort _mainSendPort; + @override + bool get isDartUiIsolate => true; - bool isSpawned = false; + @override + String get isolateName => "MLComputerIsolate"; + + @override + bool get shouldAutomaticDispose => false; // Singleton pattern MLComputer._privateConstructor(); static final MLComputer instance = MLComputer._privateConstructor(); factory MLComputer() => instance; - Future _init() async { - return _initLock.synchronized(() async { - if (isSpawned) return; - - _receivePort = ReceivePort(); - - try { - await DartUiIsolate.spawn( - _isolateMain, - _receivePort.sendPort, - ); - _mainSendPort = await _receivePort.first as SendPort; - isSpawned = true; - } catch (e) { - _logger.severe('Could not spawn isolate', e); - isSpawned = false; - } - }); - } - - @pragma('vm:entry-point') - static void _isolateMain(SendPort mainSendPort) async { - Logger.root.level = kDebugMode ? Level.ALL : Level.INFO; - final IsolateLogger isolateLogger = IsolateLogger(); - Logger.root.onRecord.listen(isolateLogger.onLogRecordInIsolate); - final receivePort = ReceivePort(); - mainSendPort.send(receivePort.sendPort); - - receivePort.listen((message) async { - final functionIndex = message[0] as int; - final function = MLComputerOperation.values[functionIndex]; - final args = message[1] as Map; - final sendPort = message[2] as SendPort; - - late final Object data; - try { - data = await cases(function, args); - } catch (e, stackTrace) { - data = { - 'error': e.toString(), - 'stackTrace': stackTrace.toString(), - }; - } - final logs = List.from(isolateLogger.getLogStringsAndClear()); - sendPort.send({"data": data, "logs": logs}); - }); - } - - /// WARNING: Only return primitives: https://api.flutter.dev/flutter/dart-isolate/SendPort/send.html - static Future cases( - MLComputerOperation function, - Map args, - ) async { - switch (function) { - case MLComputerOperation.generateFaceThumbnails: - final imagePath = args['imagePath'] as String; - final Uint8List imageData = await File(imagePath).readAsBytes(); - final faceBoxesJson = - args['faceBoxesList'] as List>; - final List faceBoxes = - faceBoxesJson.map((json) => FaceBox.fromJson(json)).toList(); - final List results = await generateFaceThumbnailsUsingCanvas( - imageData, - faceBoxes, - ); - return List.from(results); - case MLComputerOperation.loadModel: - final modelName = args['modelName'] as String; - final modelPath = args['modelPath'] as String; - final int address = await MlModel.loadModel( - modelName, - modelPath, - ); - return address; - case MLComputerOperation.initializeClipTokenizer: - final vocabPath = args["vocabPath"] as String; - await ClipTextTokenizer.instance.init(vocabPath); - return true; - case MLComputerOperation.runClipText: - //TODO:lau check logging here - final textEmbedding = await ClipTextEncoder.predict(args); - return List.from(textEmbedding, growable: false); - case MLComputerOperation.testLogging: - final logger = Logger('XXX MLComputerTestLogging'); - logger.info("XXX logging from isolate is working!!!"); - throw Exception("XXX logging from isolate testing exception handling"); - return true; - } - } - - /// The common method to run any operation in the isolate. It sends the [message] to [_isolateMain] and waits for the result. - Future _runInIsolate( - (MLComputerOperation, Map) message, - ) async { - await _init(); - return _functionLock.synchronized(() async { - final completer = Completer(); - final answerPort = ReceivePort(); - - _mainSendPort.send([message.$1.index, message.$2, answerPort.sendPort]); - - answerPort.listen((receivedMessage) { - final logs = receivedMessage['logs'] as List; - IsolateLogger.handLogStringsToMainLogger(logs); - final data = receivedMessage['data']; - if (data is Map && data.containsKey('error')) { - // Handle the error - final errorMessage = data['error']; - final errorStackTrace = data['stackTrace']; - final exception = Exception(errorMessage); - final stackTrace = StackTrace.fromString(errorStackTrace); - completer.completeError(exception, stackTrace); - } else { - completer.complete(data); - } - }); - - return completer.future; - }); - } - /// Generates face thumbnails for all [faceBoxes] in [imageData]. /// /// Uses [generateFaceThumbnailsUsingCanvas] inside the isolate. @@ -170,14 +40,12 @@ class MLComputer { ) async { final List> faceBoxesJson = faceBoxes.map((box) => box.toJson()).toList(); - return await _runInIsolate( - ( - MLComputerOperation.generateFaceThumbnails, - { - 'imagePath': imagePath, - 'faceBoxesList': faceBoxesJson, - }, - ), + return await runInIsolate( + IsolateOperation.generateFaceThumbnails, + { + 'imagePath': imagePath, + 'faceBoxesList': faceBoxesJson, + }, ).then((value) => value.cast()); } @@ -185,15 +53,10 @@ class MLComputer { try { await _ensureLoadedClipTextModel(); final int clipAddress = ClipTextEncoder.instance.sessionAddress; - final textEmbedding = await _runInIsolate( - ( - MLComputerOperation.runClipText, - { - "text": query, - "address": clipAddress, - } - ), - ) as List; + final textEmbedding = await runInIsolate(IsolateOperation.runClipText, { + "text": query, + "address": clipAddress, + }) as List; return textEmbedding; } catch (e, s) { _logger.severe("Could not run clip text in isolate", e, s); @@ -210,11 +73,9 @@ class MLComputer { ClipTextEncoder.instance.vocabRemotePath; final String tokenizerVocabPath = await RemoteAssetsService.instance .getAssetPath(tokenizerRemotePath); - await _runInIsolate( - ( - MLComputerOperation.initializeClipTokenizer, - {'vocabPath': tokenizerVocabPath}, - ), + await runInIsolate( + IsolateOperation.initializeClipTokenizer, + {'vocabPath': tokenizerVocabPath}, ); // Load ClipText model @@ -224,14 +85,12 @@ class MLComputer { if (modelPath == null) { throw Exception("Could not download clip text model, no wifi"); } - final address = await _runInIsolate( - ( - MLComputerOperation.loadModel, - { - 'modelName': modelName, - 'modelPath': modelPath, - }, - ), + final address = await runInIsolate( + IsolateOperation.loadModel, + { + 'modelName': modelName, + 'modelPath': modelPath, + }, ) as int; ClipTextEncoder.instance.storeSessionAddress(address); } catch (e, s) { @@ -243,12 +102,8 @@ class MLComputer { Future testLogging() async { try { - final result = await _runInIsolate( - ( - MLComputerOperation.testLogging, - {}, - ), - ) as bool; + final result = + await runInIsolate(IsolateOperation.testLogging, {}) as bool; return result; } catch (e, s) { _logger.severe("XXX Could not test logging in isolate", e, s); From a78831397444dc4e1e486713c93c769091350172 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Mon, 2 Sep 2024 17:06:43 +0200 Subject: [PATCH 11/24] [mob][photos] Extra isolate response check using nanoID --- mobile/lib/models/base/id.dart | 4 ++++ mobile/lib/services/isolate_service.dart | 17 ++++++++++++----- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/mobile/lib/models/base/id.dart b/mobile/lib/models/base/id.dart index 9f226eb90f..b8e5647a0a 100644 --- a/mobile/lib/models/base/id.dart +++ b/mobile/lib/models/base/id.dart @@ -7,3 +7,7 @@ const clusterIDLength = 22; String newClusterID() { return "cluster_${customAlphabet(enteWhiteListedAlphabet, clusterIDLength)}"; } + +String newIsolateTaskID(String task) { + return "${task}_${customAlphabet(enteWhiteListedAlphabet, clusterIDLength)}"; +} diff --git a/mobile/lib/services/isolate_service.dart b/mobile/lib/services/isolate_service.dart index 4c7227516e..31fdb0a8cf 100644 --- a/mobile/lib/services/isolate_service.dart +++ b/mobile/lib/services/isolate_service.dart @@ -5,6 +5,7 @@ import "package:dart_ui_isolate/dart_ui_isolate.dart"; import "package:flutter/foundation.dart" show kDebugMode; import "package:logging/logging.dart"; import "package:photos/core/error-reporting/isolate_logging.dart"; +import "package:photos/models/base/id.dart"; import "package:photos/services/isolate_functions.dart"; import "package:synchronized/synchronized.dart"; @@ -67,10 +68,11 @@ abstract class SuperIsolate { mainSendPort.send(receivePort.sendPort); receivePort.listen((message) async { - final functionIndex = message[0] as int; + final taskID = message[0] as String; + final functionIndex = message[1] as int; final function = IsolateOperation.values[functionIndex]; - final args = message[1] as Map; - final sendPort = message[2] as SendPort; + final args = message[2] as Map; + final sendPort = message[3] as SendPort; late final Object data; try { @@ -82,7 +84,7 @@ abstract class SuperIsolate { }; } final logs = List.from(isolateLogger.getLogStringsAndClear()); - sendPort.send({"data": data, "logs": logs}); + sendPort.send({"taskID": taskID, "data": data, "logs": logs}); }); } @@ -105,9 +107,14 @@ abstract class SuperIsolate { final answerPort = ReceivePort(); _activeTasks++; - _mainSendPort.send([operation.index, args, answerPort.sendPort]); + final taskID = newIsolateTaskID(operation.name); + _mainSendPort.send([taskID, operation.index, args, answerPort.sendPort]); answerPort.listen((receivedMessage) { + if (receivedMessage['taskID'] != taskID) { + logger.severe("Received isolate message with wrong taskID"); + return; + } final logs = receivedMessage['logs'] as List; IsolateLogger.handLogStringsToMainLogger(logs); final data = receivedMessage['data']; From 947bd57b99258426635249a02566931a3f09f7f4 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Mon, 2 Sep 2024 17:16:48 +0200 Subject: [PATCH 12/24] [mob][photos] Small changes --- mobile/lib/services/isolate_service.dart | 12 ++++++------ .../lib/services/machine_learning/ml_computer.dart | 3 +++ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/mobile/lib/services/isolate_service.dart b/mobile/lib/services/isolate_service.dart index 31fdb0a8cf..b4e9cb3c62 100644 --- a/mobile/lib/services/isolate_service.dart +++ b/mobile/lib/services/isolate_service.dart @@ -16,7 +16,7 @@ abstract class SuperIsolate { final Duration _inactivityDuration = const Duration(seconds: 120); int _activeTasks = 0; - final _initLock = Lock(); + final _initIsolateLock = Lock(); final _functionLock = Lock(); bool get isDartUiIsolate; @@ -31,7 +31,7 @@ abstract class SuperIsolate { bool _isIsolateSpawned = false; Future _initIsolate() async { - return _initLock.synchronized(() async { + return _initIsolateLock.synchronized(() async { if (_isIsolateSpawned) return; _receivePort = ReceivePort(); @@ -99,7 +99,7 @@ abstract class SuperIsolate { return _functionLock.synchronized(() async { if (shouldAutomaticDispose) _resetInactivityTimer(); - if (_postLockStop()) { + if (postFunctionlockStop()) { return null; } @@ -135,7 +135,7 @@ abstract class SuperIsolate { }); } - bool _postLockStop() => false; + bool postFunctionlockStop() => false; /// Resets a timer that kills the isolate after a certain amount of inactivity. /// @@ -156,12 +156,12 @@ abstract class SuperIsolate { }); } - Future _onDispose(); + Future onDispose(); void _disposeIsolate() async { if (!_isIsolateSpawned) return; logger.info('Disposing isolate'); - await _onDispose(); + await onDispose(); _isIsolateSpawned = false; _isolate.kill(); _receivePort.close(); diff --git a/mobile/lib/services/machine_learning/ml_computer.dart b/mobile/lib/services/machine_learning/ml_computer.dart index 7293d336f1..465be8f671 100644 --- a/mobile/lib/services/machine_learning/ml_computer.dart +++ b/mobile/lib/services/machine_learning/ml_computer.dart @@ -26,6 +26,9 @@ class MLComputer extends SuperIsolate { @override bool get shouldAutomaticDispose => false; + @override + Future onDispose() async => {}; + // Singleton pattern MLComputer._privateConstructor(); static final MLComputer instance = MLComputer._privateConstructor(); From e1288bfd6110c738d83e534feb5623f0ef113c0c Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Mon, 2 Sep 2024 18:12:11 +0200 Subject: [PATCH 13/24] [mob][photos] Simplify MLIndexingIsolate --- mobile/lib/services/isolate_functions.dart | 48 ++++ mobile/lib/services/isolate_service.dart | 6 +- .../machine_learning/ml_indexing_isolate.dart | 242 ++++-------------- 3 files changed, 95 insertions(+), 201 deletions(-) diff --git a/mobile/lib/services/isolate_functions.dart b/mobile/lib/services/isolate_functions.dart index 75de71c9d3..4224200857 100644 --- a/mobile/lib/services/isolate_functions.dart +++ b/mobile/lib/services/isolate_functions.dart @@ -4,11 +4,22 @@ import 'dart:typed_data' show Uint8List; import "package:logging/logging.dart"; import "package:photos/models/ml/face/box.dart"; import "package:photos/services/machine_learning/ml_model.dart"; +import "package:photos/services/machine_learning/ml_result.dart"; import "package:photos/services/machine_learning/semantic_search/clip/clip_text_encoder.dart"; import "package:photos/services/machine_learning/semantic_search/clip/clip_text_tokenizer.dart"; import "package:photos/utils/image_ml_util.dart"; +import "package:photos/utils/ml_util.dart"; enum IsolateOperation { + /// [MLIndexingIsolate] + analyzeImage, + + /// [MLIndexingIsolate] + loadIndexingModels, + + /// [MLIndexingIsolate] + releaseIndexingModels, + /// [MLComputer] generateFaceThumbnails, @@ -33,6 +44,43 @@ Future isolateFunction( Map args, ) async { switch (function) { + /// Cases for MLIndexingIsolate start here + + /// MLIndexingIsolate + case IsolateOperation.analyzeImage: + final MLResult result = await analyzeImageStatic(args); + return result.toJsonString(); + + /// MLIndexingIsolate + case IsolateOperation.loadIndexingModels: + final modelNames = args['modelNames'] as List; + final modelPaths = args['modelPaths'] as List; + final addresses = []; + for (int i = 0; i < modelNames.length; i++) { + // TODO:lau check logging here + final int address = await MlModel.loadModel( + modelNames[i], + modelPaths[i], + ); + addresses.add(address); + } + return List.from(addresses, growable: false); + + /// MLIndexingIsolate + case IsolateOperation.releaseIndexingModels: + // TODO:lau check logging here + final modelNames = args['modelNames'] as List; + final modelAddresses = args['modelAddresses'] as List; + for (int i = 0; i < modelNames.length; i++) { + await MlModel.releaseModel( + modelNames[i], + modelAddresses[i], + ); + } + return true; + + /// Cases for MLIndexingIsolate stop here + /// Cases for MLComputer start here /// MLComputer diff --git a/mobile/lib/services/isolate_service.dart b/mobile/lib/services/isolate_service.dart index b4e9cb3c62..c02f7b19f2 100644 --- a/mobile/lib/services/isolate_service.dart +++ b/mobile/lib/services/isolate_service.dart @@ -99,7 +99,7 @@ abstract class SuperIsolate { return _functionLock.synchronized(() async { if (shouldAutomaticDispose) _resetInactivityTimer(); - if (postFunctionlockStop()) { + if (postFunctionlockStop(operation)) { return null; } @@ -135,7 +135,7 @@ abstract class SuperIsolate { }); } - bool postFunctionlockStop() => false; + bool postFunctionlockStop(IsolateOperation operation) => false; /// Resets a timer that kills the isolate after a certain amount of inactivity. /// @@ -149,7 +149,7 @@ abstract class SuperIsolate { _resetInactivityTimer(); } else { logger.info( - 'Clustering Isolate has been inactive for ${_inactivityDuration.inSeconds} seconds with no tasks running. Killing isolate.', + 'Isolate has been inactive for ${_inactivityDuration.inSeconds} seconds with no tasks running. Killing isolate.', ); _disposeIsolate(); } diff --git a/mobile/lib/services/machine_learning/ml_indexing_isolate.dart b/mobile/lib/services/machine_learning/ml_indexing_isolate.dart index 996a766d64..e11891ae7a 100644 --- a/mobile/lib/services/machine_learning/ml_indexing_isolate.dart +++ b/mobile/lib/services/machine_learning/ml_indexing_isolate.dart @@ -1,36 +1,43 @@ import "dart:async"; -import "dart:isolate"; -import "package:dart_ui_isolate/dart_ui_isolate.dart"; -import "package:flutter/foundation.dart" show debugPrint, kDebugMode; +import "package:flutter/foundation.dart" show debugPrint; import "package:logging/logging.dart"; -import "package:photos/core/error-reporting/super_logging.dart"; +import "package:photos/services/isolate_functions.dart"; +import "package:photos/services/isolate_service.dart"; import 'package:photos/services/machine_learning/face_ml/face_detection/face_detection_service.dart'; import 'package:photos/services/machine_learning/face_ml/face_embedding/face_embedding_service.dart'; -import "package:photos/services/machine_learning/ml_model.dart"; import "package:photos/services/machine_learning/ml_models_overview.dart"; import 'package:photos/services/machine_learning/ml_result.dart'; import "package:photos/services/machine_learning/semantic_search/clip/clip_image_encoder.dart"; import "package:photos/utils/ml_util.dart"; -import "package:synchronized/synchronized.dart"; -enum MLIndexingOperation { analyzeImage, loadModels, releaseModels } +class MLIndexingIsolate extends SuperIsolate { + @override + Logger get logger => _logger; + final _logger = Logger("MLIndexingIsolate"); -class MLIndexingIsolate { - static final _logger = Logger("MLIndexingIsolate"); + @override + bool get isDartUiIsolate => true; - Timer? _inactivityTimer; - final Duration _inactivityDuration = const Duration(seconds: 120); - int _activeTasks = 0; + @override + String get isolateName => "MLIndexingIsolate"; - final _functionLock = Lock(); - final _initIsolateLock = Lock(); + @override + bool get shouldAutomaticDispose => true; - late DartUiIsolate _isolate; - late ReceivePort _receivePort = ReceivePort(); - late SendPort _mainSendPort; + @override + Future onDispose() async { + await _releaseModels(); + } - bool _isIsolateSpawned = false; + @override + bool postFunctionlockStop(IsolateOperation operation) { + if (operation == IsolateOperation.analyzeImage && + shouldPauseIndexingAndClustering) { + return true; + } + return false; + } bool shouldPauseIndexingAndClustering = false; @@ -39,152 +46,6 @@ class MLIndexingIsolate { static final instance = MLIndexingIsolate._privateConstructor(); factory MLIndexingIsolate() => instance; - Future _initIsolate() async { - return _initIsolateLock.synchronized(() async { - if (_isIsolateSpawned) return; - _logger.info("initIsolate called"); - - _receivePort = ReceivePort(); - - try { - _isolate = await DartUiIsolate.spawn( - _isolateMain, - _receivePort.sendPort, - ); - _mainSendPort = await _receivePort.first as SendPort; - _isIsolateSpawned = true; - - _resetInactivityTimer(); - _logger.info('initIsolate done'); - } catch (e) { - _logger.severe('Could not spawn isolate', e); - _isIsolateSpawned = false; - } - }); - } - - /// The main execution function of the isolate. - @pragma('vm:entry-point') - static void _isolateMain(SendPort mainSendPort) async { - Logger.root.level = kDebugMode ? Level.ALL : Level.INFO; - // TODO:lau move to right isolate logging - Logger.root.onRecord.listen((LogRecord rec) { - debugPrint('[MLIsolate] ${rec.toPrettyString()}'); - }); - final receivePort = ReceivePort(); - mainSendPort.send(receivePort.sendPort); - receivePort.listen((message) async { - final functionIndex = message[0] as int; - final function = MLIndexingOperation.values[functionIndex]; - final args = message[1] as Map; - final sendPort = message[2] as SendPort; - - try { - switch (function) { - case MLIndexingOperation.analyzeImage: - final MLResult result = await analyzeImageStatic(args); - sendPort.send(result.toJsonString()); - break; - case MLIndexingOperation.loadModels: - final modelNames = args['modelNames'] as List; - final modelPaths = args['modelPaths'] as List; - final addresses = []; - for (int i = 0; i < modelNames.length; i++) { - // TODO:lau check logging here - final int address = await MlModel.loadModel( - modelNames[i], - modelPaths[i], - ); - addresses.add(address); - } - sendPort.send(List.from(addresses, growable: false)); - break; - case MLIndexingOperation.releaseModels: - // TODO:lau check logging here - final modelNames = args['modelNames'] as List; - final modelAddresses = args['modelAddresses'] as List; - for (int i = 0; i < modelNames.length; i++) { - await MlModel.releaseModel( - modelNames[i], - modelAddresses[i], - ); - } - sendPort.send(true); - break; - } - } catch (e, s) { - _logger.severe("Error in FaceML isolate", e, s); - sendPort.send({'error': e.toString(), 'stackTrace': s.toString()}); - } - }); - } - - /// The common method to run any operation in the isolate. It sends the [message] to [_isolateMain] and waits for the result. - Future _runInIsolate( - (MLIndexingOperation, Map) message, - ) async { - await _initIsolate(); - return _functionLock.synchronized(() async { - _resetInactivityTimer(); - - if (message.$1 == MLIndexingOperation.analyzeImage && - shouldPauseIndexingAndClustering) { - return null; - } - - final completer = Completer(); - final answerPort = ReceivePort(); - - _activeTasks++; - _mainSendPort.send([message.$1.index, message.$2, answerPort.sendPort]); - - answerPort.listen((receivedMessage) { - if (receivedMessage is Map && receivedMessage.containsKey('error')) { - // Handle the error - final errorMessage = receivedMessage['error']; - final errorStackTrace = receivedMessage['stackTrace']; - final exception = Exception(errorMessage); - final stackTrace = StackTrace.fromString(errorStackTrace); - completer.completeError(exception, stackTrace); - } else { - completer.complete(receivedMessage); - } - }); - _activeTasks--; - - return completer.future; - }); - } - - /// Resets a timer that kills the isolate after a certain amount of inactivity. - /// - /// Should be called after initialization (e.g. inside `init()`) and after every call to isolate (e.g. inside `_runInIsolate()`) - void _resetInactivityTimer() { - _inactivityTimer?.cancel(); - _inactivityTimer = Timer(_inactivityDuration, () { - if (_activeTasks > 0) { - _logger.info('Tasks are still running. Delaying isolate disposal.'); - // Optionally, reschedule the timer to check again later. - _resetInactivityTimer(); - } else { - _logger.info( - 'Clustering Isolate has been inactive for ${_inactivityDuration.inSeconds} seconds with no tasks running. Killing isolate.', - ); - _dispose(); - } - }); - } - - void _dispose() async { - if (!_isIsolateSpawned) return; - _logger.info('Disposing isolate and models'); - await _releaseModels(); - _isIsolateSpawned = false; - _isolate.kill(); - _receivePort.close(); - _inactivityTimer?.cancel(); - } - /// Analyzes the given image data by running the full pipeline for faces, using [_analyzeImageSync] in the isolate. Future analyzeImage( FileMLInstruction instruction, @@ -194,22 +55,16 @@ class MLIndexingIsolate { late MLResult result; try { - final resultJsonString = await _runInIsolate( - ( - MLIndexingOperation.analyzeImage, - { - "enteFileID": instruction.file.uploadedFileID ?? -1, - "filePath": filePath, - "runFaces": instruction.shouldRunFaces, - "runClip": instruction.shouldRunClip, - "faceDetectionAddress": - FaceDetectionService.instance.sessionAddress, - "faceEmbeddingAddress": - FaceEmbeddingService.instance.sessionAddress, - "clipImageAddress": ClipImageEncoder.instance.sessionAddress, - } - ), - ) as String?; + final resultJsonString = + await runInIsolate(IsolateOperation.analyzeImage, { + "enteFileID": instruction.file.uploadedFileID ?? -1, + "filePath": filePath, + "runFaces": instruction.shouldRunFaces, + "runClip": instruction.shouldRunClip, + "faceDetectionAddress": FaceDetectionService.instance.sessionAddress, + "faceEmbeddingAddress": FaceEmbeddingService.instance.sessionAddress, + "clipImageAddress": ClipImageEncoder.instance.sessionAddress, + }) as String?; if (resultJsonString == null) { if (!shouldPauseIndexingAndClustering) { _logger.severe('Analyzing image in isolate is giving back null'); @@ -264,15 +119,11 @@ class MLIndexingIsolate { } try { - final addresses = await _runInIsolate( - ( - MLIndexingOperation.loadModels, - { - "modelNames": modelNames, - "modelPaths": modelPaths, - } - ), - ) as List; + final addresses = + await runInIsolate(IsolateOperation.loadIndexingModels, { + "modelNames": modelNames, + "modelPaths": modelPaths, + }) as List; for (int i = 0; i < models.length; i++) { final model = models[i].model; final address = addresses[i]; @@ -299,15 +150,10 @@ class MLIndexingIsolate { } if (modelNames.isEmpty) return; try { - await _runInIsolate( - ( - MLIndexingOperation.releaseModels, - { - "modelNames": modelNames, - "modelAddresses": modelAddresses, - } - ), - ); + await runInIsolate(IsolateOperation.releaseIndexingModels, { + "modelNames": modelNames, + "modelAddresses": modelAddresses, + }); for (final model in models) { model.model.releaseSessionAddress(); } From b598729f2802f99f601f337595068634cbd5a163 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Mon, 2 Sep 2024 18:33:29 +0200 Subject: [PATCH 14/24] [mob][photos] Simplify face clustering isolate service --- mobile/lib/services/isolate_functions.dart | 15 +- .../face_clustering_service.dart | 190 ++++-------------- 2 files changed, 50 insertions(+), 155 deletions(-) diff --git a/mobile/lib/services/isolate_functions.dart b/mobile/lib/services/isolate_functions.dart index 4224200857..308a5f7b1d 100644 --- a/mobile/lib/services/isolate_functions.dart +++ b/mobile/lib/services/isolate_functions.dart @@ -3,6 +3,7 @@ import 'dart:typed_data' show Uint8List; import "package:logging/logging.dart"; import "package:photos/models/ml/face/box.dart"; +import "package:photos/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart"; import "package:photos/services/machine_learning/ml_model.dart"; import "package:photos/services/machine_learning/ml_result.dart"; import "package:photos/services/machine_learning/semantic_search/clip/clip_text_encoder.dart"; @@ -34,6 +35,9 @@ enum IsolateOperation { /// [MLComputer] testLogging, + + /// [FaceClusteringService] + linearIncrementalClustering } /// WARNING: Only return primitives unless you know the method is only going @@ -125,8 +129,15 @@ Future isolateFunction( // throw Exception("XXX logging from isolate testing exception handling"); return true; - /// Cases for MLComputer ends here + /// Cases for MLComputer end here - /// + /// Cases for FaceClusteringService start here + + /// FaceClusteringService + case IsolateOperation.linearIncrementalClustering: + final ClusteringResult result = runLinearClustering(args); + return result; + + /// Cases for FaceClusteringService end here } } diff --git a/mobile/lib/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart b/mobile/lib/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart index fefe2910d2..c6dbad7446 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart @@ -1,20 +1,19 @@ import "dart:async"; import "dart:developer"; -import "dart:isolate"; import "dart:typed_data" show Uint8List; import "package:computer/computer.dart"; -import "package:flutter/foundation.dart" show debugPrint, kDebugMode; +import "package:flutter/foundation.dart" show kDebugMode; import "package:logging/logging.dart"; import "package:ml_linalg/dtype.dart"; import "package:ml_linalg/vector.dart"; -import "package:photos/core/error-reporting/super_logging.dart"; import "package:photos/generated/protos/ente/common/vector.pb.dart"; import "package:photos/models/base/id.dart"; +import "package:photos/services/isolate_functions.dart"; +import "package:photos/services/isolate_service.dart"; import "package:photos/services/machine_learning/face_ml/face_clustering/face_db_info_for_clustering.dart"; import "package:photos/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart"; import "package:photos/services/machine_learning/ml_result.dart"; -import "package:synchronized/synchronized.dart"; class FaceInfo { final String faceID; @@ -61,26 +60,32 @@ class ClusteringResult { } } -class FaceClusteringService { +class FaceClusteringService extends SuperIsolate { + @override + Logger get logger => _logger; final _logger = Logger("FaceLinearClustering"); + final _computer = Computer.shared(); - Timer? _inactivityTimer; - final Duration _inactivityDuration = const Duration(minutes: 2); - int _activeTasks = 0; - - final _initLock = Lock(); - - late Isolate _isolate; - late ReceivePort _receivePort = ReceivePort(); - late SendPort _mainSendPort; - - bool isSpawned = false; bool isRunning = false; static const kRecommendedDistanceThreshold = 0.24; static const kConservativeDistanceThreshold = 0.16; + @override + bool get isDartUiIsolate => false; + + @override + String get isolateName => "FaceClusteringIsolate"; + + @override + Future onDispose() async { + return; + } + + @override + bool get shouldAutomaticDispose => true; + // singleton pattern FaceClusteringService._privateConstructor(); @@ -89,124 +94,7 @@ class FaceClusteringService { static final instance = FaceClusteringService._privateConstructor(); factory FaceClusteringService() => instance; - Future _initIsolate() async { - return _initLock.synchronized(() async { - if (isSpawned) return; - - _receivePort = ReceivePort(); - - try { - _isolate = await Isolate.spawn( - _isolateMain, - _receivePort.sendPort, - ); - _mainSendPort = await _receivePort.first as SendPort; - isSpawned = true; - - _resetInactivityTimer(); - } catch (e) { - _logger.severe('Could not spawn isolate', e); - isSpawned = false; - } - }); - } - - Future _ensureSpawnedIsolate() async { - if (!isSpawned) { - await _initIsolate(); - } - } - - /// The main execution function of the isolate. - static void _isolateMain(SendPort mainSendPort) async { - Logger.root.level = kDebugMode ? Level.ALL : Level.INFO; - // TODO:lau move to right isolate logging - Logger.root.onRecord.listen((LogRecord rec) { - debugPrint('[MLIsolate] ${rec.toPrettyString()}'); - }); - final receivePort = ReceivePort(); - mainSendPort.send(receivePort.sendPort); - - receivePort.listen((message) async { - final functionIndex = message[0] as int; - final function = ClusterOperation.values[functionIndex]; - final args = message[1] as Map; - final sendPort = message[2] as SendPort; - - try { - switch (function) { - case ClusterOperation.linearIncrementalClustering: - final ClusteringResult result = _runLinearClustering(args); - sendPort.send(result); - break; - } - } catch (e, stackTrace) { - sendPort - .send({'error': e.toString(), 'stackTrace': stackTrace.toString()}); - } - }); - } - - /// The common method to run any operation in the isolate. It sends the [message] to [_isolateMain] and waits for the result. - Future _runInIsolate( - (ClusterOperation, Map) message, - ) async { - await _ensureSpawnedIsolate(); - _resetInactivityTimer(); - final completer = Completer(); - final answerPort = ReceivePort(); - - _activeTasks++; - _mainSendPort.send([message.$1.index, message.$2, answerPort.sendPort]); - - answerPort.listen((receivedMessage) { - if (receivedMessage is Map && receivedMessage.containsKey('error')) { - // Handle the error - final errorMessage = receivedMessage['error']; - final errorStackTrace = receivedMessage['stackTrace']; - final exception = Exception(errorMessage); - final stackTrace = StackTrace.fromString(errorStackTrace); - _activeTasks--; - completer.completeError(exception, stackTrace); - } else { - _activeTasks--; - completer.complete(receivedMessage); - } - }); - - return completer.future; - } - - /// Resets a timer that kills the isolate after a certain amount of inactivity. - /// - /// Should be called after initialization (e.g. inside `init()`) and after every call to isolate (e.g. inside `_runInIsolate()`) - void _resetInactivityTimer() { - _inactivityTimer?.cancel(); - _inactivityTimer = Timer(_inactivityDuration, () { - if (_activeTasks > 0) { - _logger.info('Tasks are still running. Delaying isolate disposal.'); - // Optionally, reschedule the timer to check again later. - _resetInactivityTimer(); - } else { - _logger.info( - 'Clustering Isolate has been inactive for ${_inactivityDuration.inSeconds} seconds with no tasks running. Killing isolate.', - ); - _dispose(); - } - }); - } - - /// Disposes the isolate worker. - void _dispose() { - if (!isSpawned) return; - - isSpawned = false; - _isolate.kill(); - _receivePort.close(); - _inactivityTimer?.cancel(); - } - - /// Runs the clustering algorithm [_runLinearClustering] on the given [input], in an isolate. + /// Runs the clustering algorithm [runLinearClustering] on the given [input], in an isolate. /// /// Returns the clustering result, which is a list of clusters, where each cluster is a list of indices of the dataset. Future predictLinearIsolate( @@ -238,31 +126,27 @@ class FaceClusteringService { final stopwatchClustering = Stopwatch()..start(); // final Map faceIdToCluster = // await _runLinearClusteringInComputer(input); - final ClusteringResult faceIdToCluster = await _runInIsolate( - ( - ClusterOperation.linearIncrementalClustering, - { - 'input': input, - 'fileIDToCreationTime': fileIDToCreationTime, - 'distanceThreshold': distanceThreshold, - 'conservativeDistanceThreshold': conservativeDistanceThreshold, - 'useDynamicThreshold': useDynamicThreshold, - 'offset': offset, - 'oldClusterSummaries': oldClusterSummaries, - } - ), - ); + final ClusteringResult faceIdToCluster = + await runInIsolate(IsolateOperation.linearIncrementalClustering, { + 'input': input, + 'fileIDToCreationTime': fileIDToCreationTime, + 'distanceThreshold': distanceThreshold, + 'conservativeDistanceThreshold': conservativeDistanceThreshold, + 'useDynamicThreshold': useDynamicThreshold, + 'offset': offset, + 'oldClusterSummaries': oldClusterSummaries, + }); // return _runLinearClusteringInComputer(input); _logger.info( 'predictLinear Clustering executed in ${stopwatchClustering.elapsed.inSeconds} seconds', ); - isRunning = false; return faceIdToCluster; } catch (e, stackTrace) { _logger.severe('Error while running clustering', e, stackTrace); - isRunning = false; rethrow; + } finally { + isRunning = false; } } @@ -308,7 +192,7 @@ class FaceClusteringService { } } - /// Runs the clustering algorithm [_runLinearClustering] on the given [input], in computer, without any dynamic thresholding + /// Runs the clustering algorithm [runLinearClustering] on the given [input], in computer, without any dynamic thresholding Future predictLinearComputer( Map input, { Map? fileIDToCreationTime, @@ -344,7 +228,7 @@ class FaceClusteringService { .toSet(); final startTime = DateTime.now(); final faceIdToCluster = await _computer.compute( - _runLinearClustering, + runLinearClustering, param: { "input": clusteringInput, "fileIDToCreationTime": fileIDToCreationTime, @@ -415,7 +299,7 @@ class FaceClusteringService { final _logger = Logger("FaceLinearClustering"); -ClusteringResult _runLinearClustering(Map args) { +ClusteringResult runLinearClustering(Map args) { // final input = args['input'] as Map; final input = args['input'] as Set; final fileIDToCreationTime = args['fileIDToCreationTime'] as Map?; From 2828fdf2c6c23c19718ef6df58ecf12bf7629ac9 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Mon, 2 Sep 2024 18:36:31 +0200 Subject: [PATCH 15/24] [mob][photos] Minor change --- mobile/lib/services/isolate_service.dart | 2 +- .../face_ml/face_clustering/face_clustering_service.dart | 5 ----- mobile/lib/services/machine_learning/ml_computer.dart | 3 --- 3 files changed, 1 insertion(+), 9 deletions(-) diff --git a/mobile/lib/services/isolate_service.dart b/mobile/lib/services/isolate_service.dart index c02f7b19f2..66d41a13a5 100644 --- a/mobile/lib/services/isolate_service.dart +++ b/mobile/lib/services/isolate_service.dart @@ -156,7 +156,7 @@ abstract class SuperIsolate { }); } - Future onDispose(); + Future onDispose() async {} void _disposeIsolate() async { if (!_isIsolateSpawned) return; diff --git a/mobile/lib/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart b/mobile/lib/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart index c6dbad7446..b4abd6cbd4 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart @@ -78,11 +78,6 @@ class FaceClusteringService extends SuperIsolate { @override String get isolateName => "FaceClusteringIsolate"; - @override - Future onDispose() async { - return; - } - @override bool get shouldAutomaticDispose => true; diff --git a/mobile/lib/services/machine_learning/ml_computer.dart b/mobile/lib/services/machine_learning/ml_computer.dart index 465be8f671..7293d336f1 100644 --- a/mobile/lib/services/machine_learning/ml_computer.dart +++ b/mobile/lib/services/machine_learning/ml_computer.dart @@ -26,9 +26,6 @@ class MLComputer extends SuperIsolate { @override bool get shouldAutomaticDispose => false; - @override - Future onDispose() async => {}; - // Singleton pattern MLComputer._privateConstructor(); static final MLComputer instance = MLComputer._privateConstructor(); From e70656b5a33db39c132f1b443e2bf90af999b262 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Tue, 3 Sep 2024 10:20:08 +0200 Subject: [PATCH 16/24] [mob][photos] documentation --- .../face_ml/face_clustering/face_clustering_service.dart | 6 ++---- .../lib/services/machine_learning/ml_indexing_isolate.dart | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/mobile/lib/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart b/mobile/lib/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart index b4abd6cbd4..ebb16f2252 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart @@ -362,13 +362,11 @@ ClusteringResult runLinearClustering(Map args) { sortedFaceInfos.addAll(facesWithClusterID); sortedFaceInfos.addAll(facesWithoutClusterID); - // Make sure the first face has a clusterId - final int totalFaces = sortedFaceInfos.length; - int dynamicThresholdCount = 0; - if (sortedFaceInfos.isEmpty) { return ClusteringResult.empty(); } + final int totalFaces = sortedFaceInfos.length; + int dynamicThresholdCount = 0; // Start actual clustering _logger.info( diff --git a/mobile/lib/services/machine_learning/ml_indexing_isolate.dart b/mobile/lib/services/machine_learning/ml_indexing_isolate.dart index e11891ae7a..6a0d26b45b 100644 --- a/mobile/lib/services/machine_learning/ml_indexing_isolate.dart +++ b/mobile/lib/services/machine_learning/ml_indexing_isolate.dart @@ -46,7 +46,7 @@ class MLIndexingIsolate extends SuperIsolate { static final instance = MLIndexingIsolate._privateConstructor(); factory MLIndexingIsolate() => instance; - /// Analyzes the given image data by running the full pipeline for faces, using [_analyzeImageSync] in the isolate. + /// Analyzes the given image data by running the full pipeline for faces, using [analyzeImageStatic] in the isolate. Future analyzeImage( FileMLInstruction instruction, String filePath, From ab636232ff6fb799fe29ab97b128ae3179b4775d Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Tue, 3 Sep 2024 12:04:35 +0200 Subject: [PATCH 17/24] [mob][photos] Cleaner logging --- .../face_detection_postprocessing.dart | 4 ---- .../face_detection_service.dart | 16 +++++++++------- .../machine_learning/ml_indexing_isolate.dart | 8 +------- .../services/machine_learning/ml_service.dart | 13 ++++--------- .../clip/clip_image_encoder.dart | 19 ++++++++++++++----- .../clip/clip_text_encoder.dart | 17 +++++++++++++---- mobile/lib/utils/image_ml_util.dart | 12 ------------ mobile/lib/utils/ml_util.dart | 17 +++++++++++------ 8 files changed, 52 insertions(+), 54 deletions(-) diff --git a/mobile/lib/services/machine_learning/face_ml/face_detection/face_detection_postprocessing.dart b/mobile/lib/services/machine_learning/face_ml/face_detection/face_detection_postprocessing.dart index 417b99e8e4..a01017559d 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_detection/face_detection_postprocessing.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_detection/face_detection_postprocessing.dart @@ -1,4 +1,3 @@ -import 'dart:developer' as dev show log; import 'dart:math' as math show max, min; import 'package:photos/services/machine_learning/face_ml/face_detection/detection.dart'; @@ -33,9 +32,6 @@ List yoloOnnxFilterExtractDetections( maxScore = result[4]; } } - dev.log( - 'No face detections found above the minScoreSigmoidThreshold of $minScoreSigmoidThreshold. The max score was $maxScore.', - ); } for (final List rawDetection in output) { diff --git a/mobile/lib/services/machine_learning/face_ml/face_detection/face_detection_service.dart b/mobile/lib/services/machine_learning/face_ml/face_detection/face_detection_service.dart index 728de95dae..7622b011d8 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_detection/face_detection_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_detection/face_detection_service.dart @@ -66,9 +66,8 @@ class FaceDetectionService extends MlModel { maintainAspectRatio: true, ); final preprocessingTime = DateTime.now(); - _logger.info( - 'Face detection preprocessing is finished, in ${preprocessingTime.difference(startTime).inMilliseconds} ms', - ); + final preprocessingMs = + preprocessingTime.difference(startTime).inMilliseconds; // Run inference List>>? nestedResults = []; @@ -81,11 +80,14 @@ class FaceDetectionService extends MlModel { inputImageList, ); // [1, 25200, 16] } + final inferenceTime = DateTime.now(); + final inferenceMs = + inferenceTime.difference(preprocessingTime).inMilliseconds; _logger.info( - 'inference is finished, in ${DateTime.now().difference(preprocessingTime).inMilliseconds} ms', + 'Face detection is finished, in ${inferenceTime.difference(startTime).inMilliseconds} ms (preprocessing: $preprocessingMs ms, inference: $inferenceMs ms)', ); } catch (e, s) { - _logger.severe('Error while running inference', e, s); + _logger.severe('Error while running inference (PlatformPlugin: ${MlModel.usePlatformPlugin})', e, s); throw YOLOFaceInterpreterRunException(); } try { @@ -121,9 +123,9 @@ class FaceDetectionService extends MlModel { outputs[0]?.value as List>>; // [1, 25200, 16] inputOrt.release(); runOptions.release(); - outputs.forEach((element) { + for (var element in outputs) { element?.release(); - }); + } return result; } diff --git a/mobile/lib/services/machine_learning/ml_indexing_isolate.dart b/mobile/lib/services/machine_learning/ml_indexing_isolate.dart index 6a0d26b45b..91dae6906b 100644 --- a/mobile/lib/services/machine_learning/ml_indexing_isolate.dart +++ b/mobile/lib/services/machine_learning/ml_indexing_isolate.dart @@ -51,7 +51,6 @@ class MLIndexingIsolate extends SuperIsolate { FileMLInstruction instruction, String filePath, ) async { - final Stopwatch stopwatch = Stopwatch()..start(); late MLResult result; try { @@ -79,15 +78,10 @@ class MLIndexingIsolate extends SuperIsolate { s, ); debugPrint( - "This image with ID ${instruction.file.uploadedFileID} has name ${instruction.file.displayName}.", + "This image with fileID ${instruction.file.uploadedFileID} has name ${instruction.file.displayName}.", ); rethrow; } - stopwatch.stop(); - _logger.info( - "Finished Analyze image with uploadedFileID ${instruction.file.uploadedFileID}, in " - "${stopwatch.elapsedMilliseconds} ms (including time waiting for inference engine availability)", - ); return result; } diff --git a/mobile/lib/services/machine_learning/ml_service.dart b/mobile/lib/services/machine_learning/ml_service.dart index 19ef46f2be..55ce37d37a 100644 --- a/mobile/lib/services/machine_learning/ml_service.dart +++ b/mobile/lib/services/machine_learning/ml_service.dart @@ -385,9 +385,6 @@ class MLService { } Future processImage(FileMLInstruction instruction) async { - _logger.info( - "`processImage` start processing image with uploadedFileID: ${instruction.file.uploadedFileID}", - ); bool actuallyRanML = false; try { @@ -420,7 +417,6 @@ class MLService { if (result.facesRan) { if (result.faces!.isEmpty) { faces.add(Face.empty(result.fileId)); - _logger.info("no face detected, storing empty for ${result.fileId}"); } if (result.faces!.isNotEmpty) { for (int i = 0; i < result.faces!.length; ++i) { @@ -432,7 +428,6 @@ class MLService { ), ); } - _logger.info("storing ${faces.length} faces for ${result.fileId}"); } dataEntity.putFace( RemoteFaceEmbedding( @@ -459,7 +454,7 @@ class MLService { instruction.file, dataEntity, ); - _logger.info("Results for file ${result.fileId} stored on remote"); + _logger.info("ML results for fileID ${result.fileId} stored on remote"); // Storing results locally if (result.facesRan) await MLDataDB.instance.bulkInsertFaces(faces); if (result.clipRan) { @@ -467,7 +462,7 @@ class MLService { result.clip!, ); } - _logger.info("Results for file ${result.fileId} stored locally"); + _logger.info("ML results for fileID ${result.fileId} stored locally"); return actuallyRanML; } catch (e, s) { final String errorString = e.toString(); @@ -480,7 +475,7 @@ class MLService { errorString.contains('FileSizeTooLargeForMobileIndexing'); if (acceptedIssue) { _logger.severe( - '$errorString with ID ${instruction.file.uploadedFileID} (format $format, type $fileType, size $size), storing empty results so indexing does not get stuck', + '$errorString for fileID ${instruction.file.uploadedFileID} (format $format, type $fileType, size $size), storing empty results so indexing does not get stuck', e, s, ); @@ -493,7 +488,7 @@ class MLService { return true; } _logger.severe( - "Failed to index file with ID: ${instruction.file.uploadedFileID} (format $format, type $fileType, size $size). Not storing any results locally, which means it will be automatically retried later.", + "Failed to index file for fileID ${instruction.file.uploadedFileID} (format $format, type $fileType, size $size). Not storing any results locally, which means it will be automatically retried later.", e, s, ); diff --git a/mobile/lib/services/machine_learning/semantic_search/clip/clip_image_encoder.dart b/mobile/lib/services/machine_learning/semantic_search/clip/clip_image_encoder.dart index 193e9bac14..24dd7e7bba 100644 --- a/mobile/lib/services/machine_learning/semantic_search/clip/clip_image_encoder.dart +++ b/mobile/lib/services/machine_learning/semantic_search/clip/clip_image_encoder.dart @@ -40,16 +40,25 @@ class ClipImageEncoder extends MlModel { final preprocessingMs = preprocessingTime.difference(startTime).inMilliseconds; late List result; - if (MlModel.usePlatformPlugin) { - result = await _runPlatformPluginPredict(inputList); - } else { - result = _runFFIBasedPredict(inputList, sessionAddress); + try { + if (MlModel.usePlatformPlugin) { + result = await _runPlatformPluginPredict(inputList); + } else { + result = _runFFIBasedPredict(inputList, sessionAddress); + } + } catch (e, stackTrace) { + _logger.severe( + "Clip image inference failed${enteFileID != null ? " with fileID $enteFileID" : ""} (PlatformPlugin: ${MlModel.usePlatformPlugin})", + e, + stackTrace, + ); + rethrow; } final inferTime = DateTime.now(); final inferenceMs = inferTime.difference(preprocessingTime).inMilliseconds; final totalMs = inferTime.difference(startTime).inMilliseconds; _logger.info( - "Clip predict took $totalMs ms${enteFileID != null ? " with fileID $enteFileID" : ""} (nference: $inferenceMs ms, preprocessing: $preprocessingMs ms)", + "Clip image predict took $totalMs ms${enteFileID != null ? " with fileID $enteFileID" : ""} (inference: $inferenceMs ms, preprocessing: $preprocessingMs ms)", ); return result; } diff --git a/mobile/lib/services/machine_learning/semantic_search/clip/clip_text_encoder.dart b/mobile/lib/services/machine_learning/semantic_search/clip/clip_text_encoder.dart index cd59fd1aa5..8a50b785f1 100644 --- a/mobile/lib/services/machine_learning/semantic_search/clip/clip_text_encoder.dart +++ b/mobile/lib/services/machine_learning/semantic_search/clip/clip_text_encoder.dart @@ -36,10 +36,19 @@ class ClipTextEncoder extends MlModel { final address = args["address"] as int; final List tokenize = await ClipTextTokenizer.instance.tokenize(text); final int32list = Int32List.fromList(tokenize); - if (MlModel.usePlatformPlugin) { - return await _runPlatformPluginPredict(int32list); - } else { - return _runFFIBasedPredict(int32list, address); + try { + if (MlModel.usePlatformPlugin) { + return await _runPlatformPluginPredict(int32list); + } else { + return _runFFIBasedPredict(int32list, address); + } + } catch (e, s) { + _logger.severe( + "Clip text inference failed (PlatformPlugin: ${MlModel.usePlatformPlugin})", + e, + s, + ); + rethrow; } } diff --git a/mobile/lib/utils/image_ml_util.dart b/mobile/lib/utils/image_ml_util.dart index 0de42f0c98..9b68ca9b3a 100644 --- a/mobile/lib/utils/image_ml_util.dart +++ b/mobile/lib/utils/image_ml_util.dart @@ -259,7 +259,6 @@ Future<(Float32List, List, List, List, Size)> int width = 112, int height = 112, }) async { - final stopwatch = Stopwatch()..start(); final Size originalSize = Size(image.width.toDouble(), image.height.toDouble()); @@ -299,31 +298,20 @@ Future<(Float32List, List, List, List, Size)> alignedImageIndex, ); - final blurDetectionStopwatch = Stopwatch()..start(); final faceGrayMatrix = _createGrayscaleIntMatrixFromNormalized2List( alignedImagesFloat32List, alignedImageIndex, ); alignedImageIndex += 3 * width * height; - final grayscalems = blurDetectionStopwatch.elapsedMilliseconds; - log('creating grayscale matrix took $grayscalems ms'); final (isBlur, blurValue) = await BlurDetectionService.predictIsBlurGrayLaplacian( faceGrayMatrix, faceDirection: face.getFaceDirection(), ); - final blurms = blurDetectionStopwatch.elapsedMilliseconds - grayscalems; - log('blur detection took $blurms ms'); - log( - 'total blur detection took ${blurDetectionStopwatch.elapsedMilliseconds} ms', - ); - blurDetectionStopwatch.stop(); isBlurs.add(isBlur); blurValues.add(blurValue); } - stopwatch.stop(); - log("Face Alignment took: ${stopwatch.elapsedMilliseconds} ms"); return ( alignedImagesFloat32List, alignmentResults, diff --git a/mobile/lib/utils/ml_util.dart b/mobile/lib/utils/ml_util.dart index 817c0bd4ea..d43a3ef7d3 100644 --- a/mobile/lib/utils/ml_util.dart +++ b/mobile/lib/utils/ml_util.dart @@ -332,7 +332,7 @@ Future getImagePathForML(EnteFile enteFile) async { imagePath = file?.path; stopwatch.stop(); _logger.info( - "Getting file data for uploadedFileID ${enteFile.uploadedFileID} took ${stopwatch.elapsedMilliseconds} ms", + "Getting file data for fileID ${enteFile.uploadedFileID} took ${stopwatch.elapsedMilliseconds} ms", ); if (imagePath == null) { @@ -388,7 +388,7 @@ Future analyzeImageStatic(Map args) async { final int clipImageAddress = args["clipImageAddress"] as int; _logger.info( - "Start analyzeImageStatic with fileID: $enteFileID (runFaces: $runFaces, runClip: $runClip)", + "Start analyzeImageStatic for fileID $enteFileID (runFaces: $runFaces, runClip: $runClip)", ); final startTime = DateTime.now(); @@ -399,9 +399,7 @@ Future analyzeImageStatic(Map args) async { final result = MLResult.fromEnteFileID(enteFileID); result.decodedImageSize = decodedImageSize; final decodeTime = DateTime.now(); - _logger.info( - 'Reading and decoding image took ${decodeTime.difference(startTime).inMilliseconds} ms (fileID: $enteFileID)', - ); + final decodeMs = decodeTime.difference(startTime).inMilliseconds; if (runFaces) { final resultFaces = await FaceRecognitionService.runFacesPipeline( @@ -417,6 +415,9 @@ Future analyzeImageStatic(Map args) async { result.faces = resultFaces; } } + final facesTime = DateTime.now(); + final facesMs = facesTime.difference(decodeTime).inMilliseconds; + final faceMsString = runFaces ? ", faces: $facesMs ms" : ""; if (runClip) { final clipResult = await SemanticSearchService.runClipImage( @@ -427,10 +428,14 @@ Future analyzeImageStatic(Map args) async { ); result.clip = clipResult; } + final clipTime = DateTime.now(); + final clipMs = clipTime.difference(facesTime).inMilliseconds; + final clipMsString = runClip ? ", clip: $clipMs ms" : ""; final endTime = DateTime.now(); + final totalMs = endTime.difference(startTime).inMilliseconds; _logger.info( - 'Finished analyzeImageStatic with fileID: $enteFileID, in ${endTime.difference(startTime).inMilliseconds} ms', + 'Finished analyzeImageStatic for fileID $enteFileID, in $totalMs ms (decode: $decodeMs ms$faceMsString$clipMsString)', ); return result; From 1433903faceadcef94947a01142b9dd74480d155 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Tue, 3 Sep 2024 12:06:24 +0200 Subject: [PATCH 18/24] [mob][photos] Remove redundant testing code --- mobile/lib/services/isolate_functions.dart | 11 ---------- .../machine_learning/ml_computer.dart | 11 ---------- .../ui/settings/ml/ml_user_dev_screen.dart | 20 ------------------- 3 files changed, 42 deletions(-) diff --git a/mobile/lib/services/isolate_functions.dart b/mobile/lib/services/isolate_functions.dart index 308a5f7b1d..e660351666 100644 --- a/mobile/lib/services/isolate_functions.dart +++ b/mobile/lib/services/isolate_functions.dart @@ -1,7 +1,6 @@ import "dart:io" show File; import 'dart:typed_data' show Uint8List; -import "package:logging/logging.dart"; import "package:photos/models/ml/face/box.dart"; import "package:photos/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart"; import "package:photos/services/machine_learning/ml_model.dart"; @@ -33,9 +32,6 @@ enum IsolateOperation { /// [MLComputer] runClipText, - /// [MLComputer] - testLogging, - /// [FaceClusteringService] linearIncrementalClustering } @@ -122,13 +118,6 @@ Future isolateFunction( final textEmbedding = await ClipTextEncoder.predict(args); return List.from(textEmbedding, growable: false); - /// MLComputer - case IsolateOperation.testLogging: - final logger = Logger('XXX MLComputerTestLogging'); - logger.info("XXX logging from isolate is working!!!"); - // throw Exception("XXX logging from isolate testing exception handling"); - return true; - /// Cases for MLComputer end here /// Cases for FaceClusteringService start here diff --git a/mobile/lib/services/machine_learning/ml_computer.dart b/mobile/lib/services/machine_learning/ml_computer.dart index 7293d336f1..bce51b674a 100644 --- a/mobile/lib/services/machine_learning/ml_computer.dart +++ b/mobile/lib/services/machine_learning/ml_computer.dart @@ -99,15 +99,4 @@ class MLComputer extends SuperIsolate { } }); } - - Future testLogging() async { - try { - final result = - await runInIsolate(IsolateOperation.testLogging, {}) as bool; - return result; - } catch (e, s) { - _logger.severe("XXX Could not test logging in isolate", e, s); - rethrow; - } - } } diff --git a/mobile/lib/ui/settings/ml/ml_user_dev_screen.dart b/mobile/lib/ui/settings/ml/ml_user_dev_screen.dart index c50ede988f..8d70cee21b 100644 --- a/mobile/lib/ui/settings/ml/ml_user_dev_screen.dart +++ b/mobile/lib/ui/settings/ml/ml_user_dev_screen.dart @@ -3,7 +3,6 @@ import "package:photos/core/event_bus.dart"; import "package:photos/db/ml/clip_db.dart"; import "package:photos/db/ml/db.dart"; import "package:photos/events/people_changed_event.dart"; -import "package:photos/services/machine_learning/ml_computer.dart"; import "package:photos/services/machine_learning/semantic_search/semantic_search_service.dart"; import "package:photos/theme/ente_theme.dart"; import "package:photos/ui/components/buttons/button_widget.dart"; @@ -56,25 +55,6 @@ class MLUserDeveloperOptions extends StatelessWidget { await deleteAllLocalML(context); }, ), - // TODO:lau remove below code - const SizedBox(height: 24), - ButtonWidget( - buttonType: ButtonType.neutral, - labelText: "Log something in isolate", - onTap: () async { - try { - final boolean = - await MLComputer.instance.testLogging(); - showShortToast(context, "Done: $boolean"); - } catch (e) { - // ignore: unawaited_futures - showGenericErrorDialog( - context: context, - error: e, - ); - } - }, - ), const SafeArea( child: SizedBox( height: 12, From 9dadb92d8d059f35ad66db063d16b47b3eb04fdd Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Tue, 3 Sep 2024 12:49:45 +0200 Subject: [PATCH 19/24] [mob][photos] Last logging todos --- mobile/lib/services/isolate_functions.dart | 3 - .../services/machine_learning/ml_model.dart | 60 ++++++++++++------- .../clip/clip_text_encoder.dart | 15 ++++- .../clip/clip_text_tokenizer.dart | 4 ++ mobile/lib/utils/image_ml_util.dart | 15 ++--- 5 files changed, 62 insertions(+), 35 deletions(-) diff --git a/mobile/lib/services/isolate_functions.dart b/mobile/lib/services/isolate_functions.dart index e660351666..08624ef206 100644 --- a/mobile/lib/services/isolate_functions.dart +++ b/mobile/lib/services/isolate_functions.dart @@ -57,7 +57,6 @@ Future isolateFunction( final modelPaths = args['modelPaths'] as List; final addresses = []; for (int i = 0; i < modelNames.length; i++) { - // TODO:lau check logging here final int address = await MlModel.loadModel( modelNames[i], modelPaths[i], @@ -68,7 +67,6 @@ Future isolateFunction( /// MLIndexingIsolate case IsolateOperation.releaseIndexingModels: - // TODO:lau check logging here final modelNames = args['modelNames'] as List; final modelAddresses = args['modelAddresses'] as List; for (int i = 0; i < modelNames.length; i++) { @@ -114,7 +112,6 @@ Future isolateFunction( /// MLComputer case IsolateOperation.runClipText: - //TODO:lau check logging here final textEmbedding = await ClipTextEncoder.predict(args); return List.from(textEmbedding, growable: false); diff --git a/mobile/lib/services/machine_learning/ml_model.dart b/mobile/lib/services/machine_learning/ml_model.dart index a3a05e5f14..ec91f0a0df 100644 --- a/mobile/lib/services/machine_learning/ml_model.dart +++ b/mobile/lib/services/machine_learning/ml_model.dart @@ -9,7 +9,7 @@ import "package:photos/utils/network_util.dart"; import "package:synchronized/synchronized.dart"; abstract class MlModel { - static final Logger isolateLogger = Logger("MlModelInIsolate"); + static final Logger isolateLogger = Logger("MlModel"); Logger get logger; String get kModelBucketEndpoint => "https://models.ente.io/"; @@ -95,10 +95,28 @@ abstract class MlModel { String modelName, String modelPath, ) async { - if (usePlatformPlugin) { - return await _loadModelWithPlatformPlugin(modelName, modelPath); - } else { - return await _loadModelWithFFI(modelName, modelPath); + isolateLogger + .info('Start loading $modelName (platformPlugin: $usePlatformPlugin)'); + final time = DateTime.now(); + try { + late int result; + if (usePlatformPlugin) { + result = await _loadModelWithPlatformPlugin(modelName, modelPath); + } else { + result = await _loadModelWithFFI(modelName, modelPath); + } + final timeMs = DateTime.now().difference(time).inMilliseconds; + isolateLogger.info( + "$modelName model loaded in $timeMs ms (platformPlugin: $usePlatformPlugin)", + ); + return result; + } catch (e, s) { + isolateLogger.severe( + "Failed to load model $modelName (platformPlugin: $usePlatformPlugin)", + e, + s, + ); + rethrow; } } @@ -106,18 +124,12 @@ abstract class MlModel { String modelName, String modelPath, ) async { - final startTime = DateTime.now(); - isolateLogger.info('Initializing $modelName with EntePlugin'); final OnnxDart plugin = OnnxDart(); final bool? initResult = await plugin.init(modelName, modelPath); if (initResult == null || !initResult) { isolateLogger.severe("Failed to initialize $modelName with EntePlugin."); throw Exception("Failed to initialize $modelName with EntePlugin."); } - final endTime = DateTime.now(); - isolateLogger.info( - "$modelName loaded via EntePlugin in ${endTime.difference(startTime).inMilliseconds}ms", - ); return 0; } @@ -125,10 +137,8 @@ abstract class MlModel { String modelName, String modelPath, ) async { - isolateLogger.info('Initializing $modelName with FFI'); ONNXEnvFFI.instance.initONNX(modelName); try { - final startTime = DateTime.now(); final sessionOptions = OrtSessionOptions() ..setInterOpNumThreads(1) ..setIntraOpNumThreads(1) @@ -136,21 +146,26 @@ abstract class MlModel { GraphOptimizationLevel.ortEnableAll, ); final session = OrtSession.fromFile(File(modelPath), sessionOptions); - final endTime = DateTime.now(); - isolateLogger.info( - "$modelName loaded with FFI, took: ${endTime.difference(startTime).inMilliseconds}ms", - ); return session.address; - } catch (e) { + } catch (e, s) { + isolateLogger.severe("Failed to load model $modelName with FFI", e, s); rethrow; } } static Future releaseModel(String modelName, int sessionAddress) async { - if (usePlatformPlugin) { - await _releaseModelWithPlatformPlugin(modelName); - } else { - await _releaseModelWithFFI(modelName, sessionAddress); + try { + if (usePlatformPlugin) { + await _releaseModelWithPlatformPlugin(modelName); + } else { + await _releaseModelWithFFI(modelName, sessionAddress); + } + } catch (e, s) { + isolateLogger.severe( + "Failed to release model $modelName (platformPlugin: $usePlatformPlugin)", + e, + s, + ); } } @@ -158,7 +173,6 @@ abstract class MlModel { final OnnxDart plugin = OnnxDart(); final bool? initResult = await plugin.release(modelName); if (initResult == null || !initResult) { - isolateLogger.severe("Failed to release $modelName with PlatformPlugin."); throw Exception("Failed to release $modelName with PlatformPlugin."); } } diff --git a/mobile/lib/services/machine_learning/semantic_search/clip/clip_text_encoder.dart b/mobile/lib/services/machine_learning/semantic_search/clip/clip_text_encoder.dart index 8a50b785f1..792b20fed8 100644 --- a/mobile/lib/services/machine_learning/semantic_search/clip/clip_text_encoder.dart +++ b/mobile/lib/services/machine_learning/semantic_search/clip/clip_text_encoder.dart @@ -34,14 +34,25 @@ class ClipTextEncoder extends MlModel { static Future> predict(Map args) async { final text = args["text"] as String; final address = args["address"] as int; + final startTime = DateTime.now(); final List tokenize = await ClipTextTokenizer.instance.tokenize(text); final int32list = Int32List.fromList(tokenize); + final tokenizeTime = DateTime.now(); + final tokenizeMs = tokenizeTime.difference(startTime).inMilliseconds; try { + late List embedding; if (MlModel.usePlatformPlugin) { - return await _runPlatformPluginPredict(int32list); + embedding = await _runPlatformPluginPredict(int32list); } else { - return _runFFIBasedPredict(int32list, address); + embedding = _runFFIBasedPredict(int32list, address); } + final inferTime = DateTime.now(); + final inferMs = inferTime.difference(tokenizeTime).inMilliseconds; + final totalMs = inferTime.difference(startTime).inMilliseconds; + _logger.info( + "Clip text predict took $totalMs ms (predict: $inferMs ms, tokenize: $tokenizeMs ms) for text: '$text'", + ); + return embedding; } catch (e, s) { _logger.severe( "Clip text inference failed (PlatformPlugin: ${MlModel.usePlatformPlugin})", diff --git a/mobile/lib/services/machine_learning/semantic_search/clip/clip_text_tokenizer.dart b/mobile/lib/services/machine_learning/semantic_search/clip/clip_text_tokenizer.dart index 7ac5c7950d..7bb4e1a0c2 100644 --- a/mobile/lib/services/machine_learning/semantic_search/clip/clip_text_tokenizer.dart +++ b/mobile/lib/services/machine_learning/semantic_search/clip/clip_text_tokenizer.dart @@ -3,9 +3,12 @@ import "dart:io" show File; import "dart:math"; import "package:html_unescape/html_unescape.dart"; +import "package:logging/logging.dart"; import "package:tuple/tuple.dart"; class ClipTextTokenizer { + final _logger = Logger("ClipTextTokenizer"); + static const int totalTokens = 77; late String vocabulary; @@ -75,6 +78,7 @@ class ClipTextTokenizer { sot = encoder['<|startoftext|>']!; eot = encoder['<|endoftext|>']!; + _logger.info("Clip text tokenizer initialized"); _isInitialized = true; } diff --git a/mobile/lib/utils/image_ml_util.dart b/mobile/lib/utils/image_ml_util.dart index 9b68ca9b3a..f2224d2b4b 100644 --- a/mobile/lib/utils/image_ml_util.dart +++ b/mobile/lib/utils/image_ml_util.dart @@ -109,10 +109,9 @@ Future> generateFaceThumbnailsUsingCanvas( Uint8List imageData, List faceBoxes, ) async { - final Image img = await decodeImageFromData(imageData); - int i = 0; - + int i = 0; // Index of the faceBox, initialized here for logging purposes try { + final Image img = await decodeImageFromData(imageData); final futureFaceThumbnails = >[]; for (final faceBox in faceBoxes) { // Note that the faceBox values are relative to the image size, so we need to convert them to absolute values first @@ -153,9 +152,12 @@ Future> generateFaceThumbnailsUsingCanvas( final List faceThumbnails = await Future.wait(futureFaceThumbnails); return faceThumbnails; - } catch (e) { - _logger.severe('Error generating face thumbnails: $e'); - _logger.severe('cropImage problematic input argument: ${faceBoxes[i]}'); + } catch (e, s) { + _logger.severe( + 'Error generating face thumbnails. cropImage problematic input argument: ${faceBoxes[i]}', + e, + s, + ); return []; } } @@ -259,7 +261,6 @@ Future<(Float32List, List, List, List, Size)> int width = 112, int height = 112, }) async { - final Size originalSize = Size(image.width.toDouble(), image.height.toDouble()); From ed3b27475e5f5a747e182017ccee1e1ce6b6cbd5 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Tue, 3 Sep 2024 13:13:56 +0200 Subject: [PATCH 20/24] [mob][photos] Try converting any unsupported format on Android --- mobile/lib/utils/image_ml_util.dart | 35 +++++++++++++---------------- 1 file changed, 15 insertions(+), 20 deletions(-) diff --git a/mobile/lib/utils/image_ml_util.dart b/mobile/lib/utils/image_ml_util.dart index f2224d2b4b..8c68bf9226 100644 --- a/mobile/lib/utils/image_ml_util.dart +++ b/mobile/lib/utils/image_ml_util.dart @@ -29,30 +29,25 @@ Future<(Image, ByteData)> decodeImageFromPath(String imagePath) async { return (image, imageByteData); } catch (e, s) { final format = imagePath.split('.').last; - if ((format == 'heic' || format == 'heif') && Platform.isAndroid) { - _logger.info('Cannot decode $format, converting to JPG format'); + if (Platform.isAndroid) { + _logger.info('Cannot decode $format, converting to JPG on Android'); final String? jpgPath = await HeifConverter.convert(imagePath, format: 'jpg'); - if (jpgPath == null) { - _logger.severe('Error converting $format to jpg:', e, s); - throw Exception( - 'InvalidImageFormatException: Error decoding image of format $format', - ); + if (jpgPath != null) { + final imageData = await File(jpgPath).readAsBytes(); + final image = await decodeImageFromData(imageData); + final ByteData imageByteData = await getByteDataFromImage(image); + return (image, imageByteData); } - final imageData = await File(jpgPath).readAsBytes(); - final image = await decodeImageFromData(imageData); - final ByteData imageByteData = await getByteDataFromImage(image); - return (image, imageByteData); - } else { - _logger.severe( - 'Error decoding image of format $format:', - e, - s, - ); - throw Exception( - 'InvalidImageFormatException: Error decoding image of format $format', - ); } + _logger.severe( + 'Error decoding image of format $format (Android: ${Platform.isAndroid})', + e, + s, + ); + throw Exception( + 'InvalidImageFormatException: Error decoding image of format $format', + ); } } From 4abd131b8aa55bc568868d8ee5d22ded256f0ea7 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Tue, 3 Sep 2024 13:21:50 +0200 Subject: [PATCH 21/24] [mob][photos] Remove redundant logging --- .../semantic_search/clip/clip_image_encoder.dart | 5 ----- 1 file changed, 5 deletions(-) diff --git a/mobile/lib/services/machine_learning/semantic_search/clip/clip_image_encoder.dart b/mobile/lib/services/machine_learning/semantic_search/clip/clip_image_encoder.dart index 24dd7e7bba..c25ef45526 100644 --- a/mobile/lib/services/machine_learning/semantic_search/clip/clip_image_encoder.dart +++ b/mobile/lib/services/machine_learning/semantic_search/clip/clip_image_encoder.dart @@ -4,7 +4,6 @@ import "dart:ui" show Image; import "package:logging/logging.dart"; import "package:onnx_dart/onnx_dart.dart"; import "package:onnxruntime/onnxruntime.dart"; -import "package:photos/extensions/stop_watch.dart"; import "package:photos/services/machine_learning/ml_model.dart"; import "package:photos/utils/image_ml_util.dart"; import "package:photos/utils/ml_util.dart"; @@ -67,7 +66,6 @@ class ClipImageEncoder extends MlModel { Float32List inputList, int sessionAddress, ) { - final w = EnteWatch("ClipImageEncoder._runFFIBasedPredict")..start(); final inputOrt = OrtValueTensor.createTensorWithDataList(inputList, [1, 3, 256, 256]); final inputs = {'input': inputOrt}; @@ -81,14 +79,12 @@ class ClipImageEncoder extends MlModel { element?.release(); } normalizeEmbedding(embedding); - w.stopWithLog("done"); return embedding; } static Future> _runPlatformPluginPredict( Float32List inputImageList, ) async { - final w = EnteWatch("ClipImageEncoder._runEntePlugin")..start(); final OnnxDart plugin = OnnxDart(); final result = await plugin.predict( inputImageList, @@ -96,7 +92,6 @@ class ClipImageEncoder extends MlModel { ); final List embedding = result!.sublist(0, 512); normalizeEmbedding(embedding); - w.stopWithLog("done"); return embedding; } } From dae8fbc9b59465d04cd3a8d7cb31b9a9b7415bc6 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Tue, 3 Sep 2024 13:25:08 +0200 Subject: [PATCH 22/24] [mob][photos] Redundant logging --- .../face_ml/face_filtering/blur_detection_service.dart | 4 ---- 1 file changed, 4 deletions(-) diff --git a/mobile/lib/services/machine_learning/face_ml/face_filtering/blur_detection_service.dart b/mobile/lib/services/machine_learning/face_ml/face_filtering/blur_detection_service.dart index 4e23d36556..1211f4336a 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_filtering/blur_detection_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_filtering/blur_detection_service.dart @@ -1,10 +1,7 @@ -import 'package:logging/logging.dart'; import "package:photos/services/machine_learning/face_ml/face_detection/detection.dart"; import 'package:photos/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart'; class BlurDetectionService { - static final _logger = Logger('BlurDetectionService'); - // singleton pattern BlurDetectionService._privateConstructor(); static final instance = BlurDetectionService._privateConstructor(); @@ -18,7 +15,6 @@ class BlurDetectionService { final List> laplacian = _applyLaplacian(grayImage, faceDirection: faceDirection); final double variance = _calculateVariance(laplacian); - _logger.info('Variance: $variance'); return (variance < threshold, variance); } From 4ee96637de7d66587db76795a67a7614e1b5299d Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Tue, 3 Sep 2024 13:27:30 +0200 Subject: [PATCH 23/24] [mob][photos] logs --- .../src/main/kotlin/io/ente/photos/onnx_dart/OnnxDartPlugin.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobile/plugins/onnx_dart/android/src/main/kotlin/io/ente/photos/onnx_dart/OnnxDartPlugin.kt b/mobile/plugins/onnx_dart/android/src/main/kotlin/io/ente/photos/onnx_dart/OnnxDartPlugin.kt index 5b84568b2e..44b1811c2d 100644 --- a/mobile/plugins/onnx_dart/android/src/main/kotlin/io/ente/photos/onnx_dart/OnnxDartPlugin.kt +++ b/mobile/plugins/onnx_dart/android/src/main/kotlin/io/ente/photos/onnx_dart/OnnxDartPlugin.kt @@ -201,7 +201,7 @@ class OnnxDartPlugin: FlutterPlugin, MethodCallHandler { inputs["input"] = inputTensor } val outputs = session.run(inputs) - Log.d(TAG, "Output shape: ${outputs.size()}") + // Log.d(TAG, "Output shape: ${outputs.size()}") if (modelType == ModelType.YOLOv5Face) { val outputTensor = (outputs[0].value as Array>).get(0) val flatList = outputTensor.flattenToFloatArray() From 13f1309857a49329e068c37a5231d653aa3d6db2 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Tue, 3 Sep 2024 13:36:48 +0200 Subject: [PATCH 24/24] [mob][photos] Logging --- .../face_ml/face_embedding/face_embedding_service.dart | 10 +++------- mobile/lib/utils/image_ml_util.dart | 2 ++ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/mobile/lib/services/machine_learning/face_ml/face_embedding/face_embedding_service.dart b/mobile/lib/services/machine_learning/face_ml/face_embedding/face_embedding_service.dart index 0853112c44..14736aa2c4 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_embedding/face_embedding_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_embedding/face_embedding_service.dart @@ -60,8 +60,6 @@ class FaceEmbeddingService extends MlModel { Float32List input, int sessionAddress, ) { - final stopwatch = Stopwatch()..start(); - _logger.info('MobileFaceNet interpreter.run is called'); final runOptions = OrtRunOptions(); final int numberOfFaces = input.length ~/ (kInputSize * kInputSize * 3); final inputOrt = OrtValueTensor.createTensorWithDataList( @@ -78,11 +76,9 @@ class FaceEmbeddingService extends MlModel { } inputOrt.release(); runOptions.release(); - outputs.forEach((element) => element?.release()); - stopwatch.stop(); - _logger.info( - 'MobileFaceNetFFI interpreter.run is finished, in ${stopwatch.elapsedMilliseconds}ms', - ); + for (var element in outputs) { + element?.release(); + } return embeddings; } diff --git a/mobile/lib/utils/image_ml_util.dart b/mobile/lib/utils/image_ml_util.dart index 8c68bf9226..5713527108 100644 --- a/mobile/lib/utils/image_ml_util.dart +++ b/mobile/lib/utils/image_ml_util.dart @@ -34,11 +34,13 @@ Future<(Image, ByteData)> decodeImageFromPath(String imagePath) async { final String? jpgPath = await HeifConverter.convert(imagePath, format: 'jpg'); if (jpgPath != null) { + _logger.info('Conversion successful, decoding JPG'); final imageData = await File(jpgPath).readAsBytes(); final image = await decodeImageFromData(imageData); final ByteData imageByteData = await getByteDataFromImage(image); return (image, imageByteData); } + _logger.info('Unable to convert $format to JPG'); } _logger.severe( 'Error decoding image of format $format (Android: ${Platform.isAndroid})',