diff --git a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart index 6e2a9c470d..2dab96fb58 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart @@ -3,8 +3,7 @@ import "dart:developer" as dev show log; import "dart:io" show File; import "dart:isolate"; import "dart:math" show min; -import "dart:typed_data" show Uint8List, Float32List, ByteData; -import "dart:ui" show Image; +import "dart:typed_data" show Uint8List, ByteData; import "package:computer/computer.dart"; import "package:dart_ui_isolate/dart_ui_isolate.dart"; @@ -26,7 +25,6 @@ import "package:photos/models/file/file.dart"; import "package:photos/service_locator.dart"; import 'package:photos/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart'; import "package:photos/services/machine_learning/face_ml/face_clustering/face_db_info_for_clustering.dart"; -import 'package:photos/services/machine_learning/face_ml/face_detection/detection.dart'; import 'package:photos/services/machine_learning/face_ml/face_detection/face_detection_service.dart'; import 'package:photos/services/machine_learning/face_ml/face_embedding/face_embedding_service.dart'; import 'package:photos/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart'; @@ -736,178 +734,33 @@ class FaceMlService { final int faceDetectionAddress = args["faceDetectionAddress"] as int; final int faceEmbeddingAddress = args["faceEmbeddingAddress"] as int; - final resultBuilder = FaceMlResult.fromEnteFileID(enteFileID); - dev.log( "Start analyzing image with uploadedFileID: $enteFileID inside the isolate", ); - final stopwatchTotal = Stopwatch()..start(); - final stopwatch = Stopwatch()..start(); + final time = DateTime.now(); // Decode the image once to use for both face detection and alignment final imageData = await File(imagePath).readAsBytes(); final image = await decodeImageFromData(imageData); - final ByteData imgByteData = await getByteDataFromImage(image); + final ByteData imageByteData = await getByteDataFromImage(image); dev.log('Reading and decoding image took ' - '${stopwatch.elapsedMilliseconds} ms'); - stopwatch.reset(); + '${DateTime.now().difference(time).inMilliseconds} ms'); - // Get the faces - final List faceDetectionResult = - await FaceMlService._detectFacesSync( + final resultFaces = await FaceRecognitionService.runFacesPipeline( + enteFileID, image, - imgByteData, + imageByteData, faceDetectionAddress, - resultBuilder: resultBuilder, - ); - - dev.log( - "${faceDetectionResult.length} faces detected with scores ${faceDetectionResult.map((e) => e.score).toList()}: completed `detectFacesSync` function, in " - "${stopwatch.elapsedMilliseconds} ms"); - - // If no faces were detected, return a result with no faces. Otherwise, continue. - if (faceDetectionResult.isEmpty) { - dev.log( - "No faceDetectionResult, Completed analyzing image with uploadedFileID $enteFileID, in " - "${stopwatch.elapsedMilliseconds} ms"); - resultBuilder.noFaceDetected(); - return resultBuilder; - } - - stopwatch.reset(); - // Align the faces - final Float32List faceAlignmentResult = - await FaceMlService._alignFacesSync( - image, - imgByteData, - faceDetectionResult, - resultBuilder: resultBuilder, - ); - - dev.log("Completed `alignFacesSync` function, in " - "${stopwatch.elapsedMilliseconds} ms"); - - stopwatch.reset(); - // Get the embeddings of the faces - final embeddings = await FaceMlService._embedFacesSync( - faceAlignmentResult, faceEmbeddingAddress, - resultBuilder: resultBuilder, ); - dev.log("Completed `embedFacesSync` function, in " - "${stopwatch.elapsedMilliseconds} ms"); - - stopwatch.stop(); - stopwatchTotal.stop(); - dev.log("Finished Analyze image (${embeddings.length} faces) with " - "uploadedFileID $enteFileID, in " - "${stopwatchTotal.elapsedMilliseconds} ms"); - - return resultBuilder; + return resultFaces; } catch (e, s) { dev.log("Could not analyze image: \n e: $e \n s: $s"); rethrow; } } - /// Detects faces in the given image data. - /// - /// `imageData`: The image data to analyze. - /// - /// Returns a list of face detection results. - static Future> _detectFacesSync( - Image image, - ByteData imageByteData, - int interpreterAddress, { - FaceMlResult? resultBuilder, - }) async { - try { - // Get the bounding boxes of the faces - final (List faces, dataSize) = - await FaceDetectionService.predict( - image, - imageByteData, - interpreterAddress, - ); - - // Add detected faces to the resultBuilder - if (resultBuilder != null) { - resultBuilder.addNewlyDetectedFaces(faces, dataSize); - } - - return faces; - } on YOLOFaceInterpreterRunException { - throw CouldNotRunFaceDetector(); - } catch (e) { - dev.log('[SEVERE] Face detection failed: $e'); - throw GeneralFaceMlException('Face detection failed: $e'); - } - } - - /// Aligns multiple faces from the given image data. - /// - /// `imageData`: The image data in [Uint8List] that contains the faces. - /// `faces`: The face detection results in a list of [FaceDetectionAbsolute] for the faces to align. - /// - /// Returns a list of the aligned faces as image data. - static Future _alignFacesSync( - Image image, - ByteData imageByteData, - List faces, { - FaceMlResult? resultBuilder, - }) async { - try { - final stopwatch = Stopwatch()..start(); - final (alignedFaces, alignmentResults, _, blurValues, _) = - await preprocessToMobileFaceNetFloat32List( - image, - imageByteData, - faces, - ); - stopwatch.stop(); - dev.log( - "Face alignment image decoding and processing took ${stopwatch.elapsedMilliseconds} ms", - ); - - if (resultBuilder != null) { - resultBuilder.addAlignmentResults( - alignmentResults, - blurValues, - ); - } - - return alignedFaces; - } catch (e, s) { - dev.log('[SEVERE] Face alignment failed: $e $s'); - throw CouldNotWarpAffine(); - } - } - - static Future>> _embedFacesSync( - Float32List facesList, - int interpreterAddress, { - FaceMlResult? resultBuilder, - }) async { - try { - // Get the embedding of the faces - final List> embeddings = - await FaceEmbeddingService.predict(facesList, interpreterAddress); - - // Add the embeddings to the resultBuilder - if (resultBuilder != null) { - resultBuilder.addEmbeddingsToExistingFaces(embeddings); - } - - return embeddings; - } on MobileFaceNetInterpreterRunException { - throw CouldNotRunFaceEmbeddor(); - } catch (e) { - dev.log('[SEVERE] Face embedding (batch) failed: $e'); - throw GeneralFaceMlException('Face embedding (batch) failed: $e'); - } - } - bool _cannotRunMLFunction({String function = ""}) { if (_isIndexingOrClusteringRunning) { _logger.info( diff --git a/mobile/lib/services/machine_learning/face_ml/face_recognition_service.dart b/mobile/lib/services/machine_learning/face_ml/face_recognition_service.dart index c141c0eb10..e5b687d3dd 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_recognition_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_recognition_service.dart @@ -1,4 +1,7 @@ import "dart:async" show unawaited; +import "dart:developer" as dev show log; +import "dart:typed_data" show ByteData, Float32List; +import "dart:ui" show Image; import "package:logging/logging.dart"; import "package:photos/core/event_bus.dart"; @@ -8,9 +11,15 @@ import "package:photos/extensions/list.dart"; import "package:photos/face/db.dart"; import "package:photos/face/model/face.dart"; import "package:photos/models/ml/ml_versions.dart"; +import "package:photos/services/machine_learning/face_ml/face_detection/detection.dart"; +import "package:photos/services/machine_learning/face_ml/face_detection/face_detection_service.dart"; +import "package:photos/services/machine_learning/face_ml/face_embedding/face_embedding_service.dart"; +import "package:photos/services/machine_learning/face_ml/face_ml_result.dart"; import "package:photos/services/machine_learning/face_ml/person/person_service.dart"; import "package:photos/services/machine_learning/file_ml/file_ml.dart"; import "package:photos/services/machine_learning/file_ml/remote_fileml_service.dart"; +import "package:photos/services/machine_learning/ml_exceptions.dart"; +import "package:photos/utils/image_ml_util.dart"; import "package:photos/utils/local_settings.dart"; import "package:photos/utils/ml_util.dart"; @@ -150,4 +159,149 @@ class FaceRecognitionService { } _logger.info('Fetched $fetchedCount embeddings'); } + + static Future runFacesPipeline( + int enteFileID, + Image image, + ByteData imageByteData, + int faceDetectionAddress, + int faceEmbeddingAddress, + ) async { + final resultBuilder = FaceMlResult.fromEnteFileID(enteFileID); + + final Stopwatch stopwatch = Stopwatch()..start(); + final startTime = DateTime.now(); + + // Get the faces + final List faceDetectionResult = + await _detectFacesSync( + image, + imageByteData, + faceDetectionAddress, + resultBuilder, + ); + dev.log( + "${faceDetectionResult.length} faces detected with scores ${faceDetectionResult.map((e) => e.score).toList()}: completed `detectFacesSync` function, in " + "${stopwatch.elapsedMilliseconds} ms"); + + // If no faces were detected, return a result with no faces. Otherwise, continue. + if (faceDetectionResult.isEmpty) { + dev.log( + "No faceDetectionResult, Completed analyzing image with uploadedFileID $enteFileID, in " + "${stopwatch.elapsedMilliseconds} ms"); + resultBuilder.noFaceDetected(); + return resultBuilder; + } + + stopwatch.reset(); + // Align the faces + final Float32List faceAlignmentResult = await _alignFacesSync( + image, + imageByteData, + faceDetectionResult, + resultBuilder, + ); + dev.log("Completed `alignFacesSync` function, in " + "${stopwatch.elapsedMilliseconds} ms"); + + stopwatch.reset(); + // Get the embeddings of the faces + final embeddings = await _embedFacesSync( + faceAlignmentResult, + faceEmbeddingAddress, + resultBuilder, + ); + dev.log("Completed `embedFacesSync` function, in " + "${stopwatch.elapsedMilliseconds} ms"); + stopwatch.stop(); + + dev.log("Finished faces pipeline (${embeddings.length} faces) with " + "uploadedFileID $enteFileID, in " + "${DateTime.now().difference(startTime).inMilliseconds} ms"); + + return resultBuilder; + } + + /// Runs face recognition on the given image data. + static Future> _detectFacesSync( + Image image, + ByteData imageByteData, + int interpreterAddress, + FaceMlResult resultBuilder, + ) async { + try { + // Get the bounding boxes of the faces + final (List faces, dataSize) = + await FaceDetectionService.predict( + image, + imageByteData, + interpreterAddress, + ); + + // Add detected faces to the resultBuilder + resultBuilder.addNewlyDetectedFaces(faces, dataSize); + + return faces; + } on YOLOFaceInterpreterRunException { + throw CouldNotRunFaceDetector(); + } catch (e) { + dev.log('[SEVERE] Face detection failed: $e'); + throw GeneralFaceMlException('Face detection failed: $e'); + } + } + + /// Aligns multiple faces from the given image data. + /// Returns a list of the aligned faces as image data. + static Future _alignFacesSync( + Image image, + ByteData imageByteData, + List faces, + FaceMlResult resultBuilder, + ) async { + try { + final stopwatch = Stopwatch()..start(); + final (alignedFaces, alignmentResults, _, blurValues, _) = + await preprocessToMobileFaceNetFloat32List( + image, + imageByteData, + faces, + ); + stopwatch.stop(); + dev.log( + "Face alignment image decoding and processing took ${stopwatch.elapsedMilliseconds} ms", + ); + + resultBuilder.addAlignmentResults( + alignmentResults, + blurValues, + ); + + return alignedFaces; + } catch (e, s) { + dev.log('[SEVERE] Face alignment failed: $e $s'); + throw CouldNotWarpAffine(); + } + } + + static Future>> _embedFacesSync( + Float32List facesList, + int interpreterAddress, + FaceMlResult resultBuilder, + ) async { + try { + // Get the embedding of the faces + final List> embeddings = + await FaceEmbeddingService.predict(facesList, interpreterAddress); + + // Add the embeddings to the resultBuilder + resultBuilder.addEmbeddingsToExistingFaces(embeddings); + + return embeddings; + } on MobileFaceNetInterpreterRunException { + throw CouldNotRunFaceEmbeddor(); + } catch (e) { + dev.log('[SEVERE] Face embedding (batch) failed: $e'); + throw GeneralFaceMlException('Face embedding (batch) failed: $e'); + } + } }