From 83d8d7ae7a58f703cc24dd3fb1c02e18bd96b75f Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Tue, 19 Mar 2024 18:24:46 +0530 Subject: [PATCH 01/22] [mob] empty constructor for Face --- mobile/lib/face/model/face.dart | 11 +++++++++++ mobile/lib/services/face_ml/face_ml_service.dart | 8 ++------ .../file_ml/remote_fileml_service.dart | 3 ++- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/mobile/lib/face/model/face.dart b/mobile/lib/face/model/face.dart index 49612c3d73..aa352cc191 100644 --- a/mobile/lib/face/model/face.dart +++ b/mobile/lib/face/model/face.dart @@ -20,6 +20,17 @@ class Face { this.blur, ); + factory Face.empty(int fileID, {bool error = false}) { + return Face( + "$fileID-0", + fileID, + [], + error ? -1.0 : 0.0, + Detection.empty(), + 0.0, + ); + } + factory Face.fromJson(Map json) { return Face( json['faceID'] as String, diff --git a/mobile/lib/services/face_ml/face_ml_service.dart b/mobile/lib/services/face_ml/face_ml_service.dart index 1352823390..d457b8d127 100644 --- a/mobile/lib/services/face_ml/face_ml_service.dart +++ b/mobile/lib/services/face_ml/face_ml_service.dart @@ -461,13 +461,9 @@ class FaceMlService { if (fileMl.faceEmbedding.version != faceMlVersion) continue; if (fileMl.faceEmbedding.faces.isEmpty) { faces.add( - Face( - '${fileMl.fileID}-0', + Face.empty( fileMl.fileID, - [], - (fileMl.faceEmbedding.error ?? false) ? -1.0 : 0.0, - face_detection.Detection.empty(), - 0.0, + error: (fileMl.faceEmbedding.error ?? false), ), ); } else { diff --git a/mobile/lib/services/machine_learning/file_ml/remote_fileml_service.dart b/mobile/lib/services/machine_learning/file_ml/remote_fileml_service.dart index d8b39f0425..95db6c3811 100644 --- a/mobile/lib/services/machine_learning/file_ml/remote_fileml_service.dart +++ b/mobile/lib/services/machine_learning/file_ml/remote_fileml_service.dart @@ -123,7 +123,8 @@ class RemoteFileMLService { } Future> decryptFileMLComputer( - Map args) async { + Map args, + ) async { final result = {}; final inputs = args["inputs"] as List; for (final input in inputs) { From 17fa64aa5e7624aebd243c52fb55a4b6fa5eddb9 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Wed, 20 Mar 2024 10:43:02 +0530 Subject: [PATCH 02/22] Remove unused import --- mobile/lib/ui/settings/debug/face_debug_section_widget.dart | 1 - 1 file changed, 1 deletion(-) diff --git a/mobile/lib/ui/settings/debug/face_debug_section_widget.dart b/mobile/lib/ui/settings/debug/face_debug_section_widget.dart index 68fa8392e9..f94e4ba614 100644 --- a/mobile/lib/ui/settings/debug/face_debug_section_widget.dart +++ b/mobile/lib/ui/settings/debug/face_debug_section_widget.dart @@ -8,7 +8,6 @@ import "package:photos/events/people_changed_event.dart"; import "package:photos/extensions/stop_watch.dart"; import "package:photos/face/db.dart"; import "package:photos/face/model/person.dart"; -import "package:photos/models/ml/ml_versions.dart"; import "package:photos/services/face_ml/face_ml_service.dart"; import "package:photos/services/face_ml/feedback/cluster_feedback.dart"; import 'package:photos/theme/ente_theme.dart'; From eaea4f81b7028895c003fb7e59abd201249a6ce1 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Wed, 20 Mar 2024 11:28:37 +0530 Subject: [PATCH 03/22] [mob] update server on old FaceMlVersion --- mobile/lib/services/face_ml/face_ml_service.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobile/lib/services/face_ml/face_ml_service.dart b/mobile/lib/services/face_ml/face_ml_service.dart index d457b8d127..9b8e550c57 100644 --- a/mobile/lib/services/face_ml/face_ml_service.dart +++ b/mobile/lib/services/face_ml/face_ml_service.dart @@ -458,7 +458,7 @@ class FaceMlService { final List faces = []; final remoteFileIdToVersion = {}; for (FileMl fileMl in res.mlData.values) { - if (fileMl.faceEmbedding.version != faceMlVersion) continue; + if (fileMl.faceEmbedding.version < faceMlVersion) continue; if (fileMl.faceEmbedding.faces.isEmpty) { faces.add( Face.empty( From 2573328c301f1d10a6f20c3dabac5fc97bac6b68 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Wed, 20 Mar 2024 11:28:59 +0530 Subject: [PATCH 04/22] [mob] Minor change --- mobile/lib/services/face_ml/face_ml_service.dart | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/mobile/lib/services/face_ml/face_ml_service.dart b/mobile/lib/services/face_ml/face_ml_service.dart index 9b8e550c57..0fedda70d8 100644 --- a/mobile/lib/services/face_ml/face_ml_service.dart +++ b/mobile/lib/services/face_ml/face_ml_service.dart @@ -558,15 +558,14 @@ class FaceMlService { } for (int i = 0; i < result.faces.length; ++i) { final FaceResult faceRes = result.faces[i]; - final FaceDetectionRelative relativeDetection = faceRes.detection; final detection = face_detection.Detection( box: FaceBox( - xMin: relativeDetection.xMinBox, - yMin: relativeDetection.yMinBox, - width: relativeDetection.width, - height: relativeDetection.height, + xMin: faceRes.detection.xMinBox, + yMin: faceRes.detection.yMinBox, + width: faceRes.detection.width, + height: faceRes.detection.height, ), - landmarks: relativeDetection.allKeypoints + landmarks: faceRes.detection.allKeypoints .map( (keypoint) => Landmark( x: keypoint[0], From af1a6fc9fa5bff83f527efc7498e6ad670345523 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Wed, 20 Mar 2024 12:03:08 +0530 Subject: [PATCH 05/22] [mob] Log actually analyzed count --- .../lib/services/face_ml/face_ml_service.dart | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/mobile/lib/services/face_ml/face_ml_service.dart b/mobile/lib/services/face_ml/face_ml_service.dart index 0fedda70d8..789430386f 100644 --- a/mobile/lib/services/face_ml/face_ml_service.dart +++ b/mobile/lib/services/face_ml/face_ml_service.dart @@ -446,7 +446,7 @@ class FaceMlService { final List> chunks = sortedBylocalID.chunks(kParallelism); outerLoop: for (final chunk in chunks) { - final futures = []; + final futures = >[]; final List fileIds = []; // Try to find embeddings on the remote server for (final f in chunk) { @@ -495,8 +495,12 @@ class FaceMlService { } futures.add(processImage(enteFile)); } - await Future.wait(futures); - fileAnalyzedCount += futures.length; + final awaitedFutures = await Future.wait(futures); + final sumFutures = awaitedFutures.fold( + 0, + (previousValue, element) => previousValue + (element ? 1 : 0), + ); + fileAnalyzedCount += sumFutures; } stopwatch.stop(); @@ -514,7 +518,7 @@ class FaceMlService { } } - Future processImage(EnteFile enteFile) async { + Future processImage(EnteFile enteFile) async { _logger.info( "`indexAllImages()` on file number start processing image with uploadedFileID: ${enteFile.uploadedFileID}", ); @@ -526,10 +530,7 @@ class FaceMlService { // disposeImageIsolateAfterUse: false, ); if (result == null) { - _logger.warning( - "Image not analyzed with uploadedFileID: ${enteFile.uploadedFileID}", - ); - return; + return false; } final List faces = []; if (!result.hasFaces) { @@ -599,12 +600,14 @@ class FaceMlService { ), ); await FaceMLDataDB.instance.bulkInsertFaces(faces); + return true; } catch (e, s) { _logger.severe( "Failed to analyze using FaceML for image with ID: ${enteFile.uploadedFileID}", e, s, ); + return true; } } @@ -1153,6 +1156,9 @@ class FaceMlService { } bool _skipAnalysisEnteFile(EnteFile enteFile, Map indexedFileIds) { + if (isImageIndexRunning == false) { + return true; + } // Skip if the file is not uploaded or not owned by the user if (!enteFile.isUploaded || enteFile.isOwner == false) { return true; From 974b7c7329a6ba754e44770e6162fd5645ff4344 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Wed, 20 Mar 2024 14:15:59 +0530 Subject: [PATCH 06/22] Increase blur threshold --- mobile/lib/services/face_ml/blur_detection/blur_constants.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobile/lib/services/face_ml/blur_detection/blur_constants.dart b/mobile/lib/services/face_ml/blur_detection/blur_constants.dart index 4d770162cc..614096ed41 100644 --- a/mobile/lib/services/face_ml/blur_detection/blur_constants.dart +++ b/mobile/lib/services/face_ml/blur_detection/blur_constants.dart @@ -1,2 +1,2 @@ -const kLaplacianThreshold = 10; +const kLaplacianThreshold = 15; const kLapacianDefault = 10000.0; From 39f16ff517af3bdc08655148dab5bb06144a59b4 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Wed, 20 Mar 2024 14:34:12 +0530 Subject: [PATCH 07/22] Only show high quality faces in file info --- mobile/lib/face/db.dart | 9 ++++++--- mobile/lib/face/db_fields.dart | 2 +- mobile/lib/face/model/face.dart | 6 +++++- .../face_ml/blur_detection/blur_constants.dart | 2 -- .../blur_detection_service.dart | 2 +- .../face_filtering/face_filtering_constants.dart | 8 ++++++++ mobile/lib/services/face_ml/face_ml_result.dart | 2 +- mobile/lib/services/face_ml/face_ml_service.dart | 3 ++- .../lib/ui/viewer/file_details/face_widget.dart | 4 ++++ .../viewer/file_details/faces_item_widget.dart | 16 ++++++++++++++-- mobile/lib/utils/image_ml_util.dart | 2 +- 11 files changed, 43 insertions(+), 13 deletions(-) delete mode 100644 mobile/lib/services/face_ml/blur_detection/blur_constants.dart rename mobile/lib/services/face_ml/{blur_detection => face_filtering}/blur_detection_service.dart (97%) create mode 100644 mobile/lib/services/face_ml/face_filtering/face_filtering_constants.dart diff --git a/mobile/lib/face/db.dart b/mobile/lib/face/db.dart index 0b85e303c6..c8b4f75f71 100644 --- a/mobile/lib/face/db.dart +++ b/mobile/lib/face/db.dart @@ -11,7 +11,7 @@ import "package:photos/face/db_model_mappers.dart"; import "package:photos/face/model/face.dart"; import "package:photos/face/model/person.dart"; import "package:photos/models/file/file.dart"; -import "package:photos/services/face_ml/blur_detection/blur_constants.dart"; +import 'package:photos/services/face_ml/face_filtering/face_filtering_constants.dart'; import 'package:sqflite/sqflite.dart'; /// Stores all data for the ML-related features. The database can be accessed by `MlDataDB.instance.database`. @@ -226,7 +226,7 @@ class FaceMLDataDB { return null; } - Future> getFacesForGivenFileID(int fileUploadID) async { + Future?> getFacesForGivenFileID(int fileUploadID) async { final db = await instance.database; final List> maps = await db.query( facesTable, @@ -246,6 +246,9 @@ class FaceMLDataDB { where: '$fileIDColumn = ?', whereArgs: [fileUploadID], ); + if (maps.isEmpty) { + return null; + } return maps.map((e) => mapRowToFace(e)).toList(); } @@ -338,7 +341,7 @@ class FaceMLDataDB { /// /// Only selects faces with score greater than [minScore] and blur score greater than [minClarity] Future> getFaceEmbeddingMap({ - double minScore = 0.78, + double minScore = kMinFaceScore, int minClarity = kLaplacianThreshold, int maxRows = 20000, }) async { diff --git a/mobile/lib/face/db_fields.dart b/mobile/lib/face/db_fields.dart index a1f185e7ee..8b17195751 100644 --- a/mobile/lib/face/db_fields.dart +++ b/mobile/lib/face/db_fields.dart @@ -1,5 +1,5 @@ // Faces Table Fields & Schema Queries -import "package:photos/services/face_ml/blur_detection/blur_constants.dart"; +import 'package:photos/services/face_ml/face_filtering/face_filtering_constants.dart'; const facesTable = 'faces'; const fileIDColumn = 'file_id'; diff --git a/mobile/lib/face/model/face.dart b/mobile/lib/face/model/face.dart index aa352cc191..c29c03ae11 100644 --- a/mobile/lib/face/model/face.dart +++ b/mobile/lib/face/model/face.dart @@ -1,5 +1,5 @@ import "package:photos/face/model/detection.dart"; -import "package:photos/services/face_ml/blur_detection/blur_constants.dart"; +import 'package:photos/services/face_ml/face_filtering/face_filtering_constants.dart'; class Face { final int fileID; @@ -11,6 +11,10 @@ class Face { bool get isBlurry => blur < kLaplacianThreshold; + bool get hasHighScore => score > kMinFaceScore; + + bool get isHighQuality => (!isBlurry) && hasHighScore; + Face( this.faceID, this.fileID, diff --git a/mobile/lib/services/face_ml/blur_detection/blur_constants.dart b/mobile/lib/services/face_ml/blur_detection/blur_constants.dart deleted file mode 100644 index 614096ed41..0000000000 --- a/mobile/lib/services/face_ml/blur_detection/blur_constants.dart +++ /dev/null @@ -1,2 +0,0 @@ -const kLaplacianThreshold = 15; -const kLapacianDefault = 10000.0; diff --git a/mobile/lib/services/face_ml/blur_detection/blur_detection_service.dart b/mobile/lib/services/face_ml/face_filtering/blur_detection_service.dart similarity index 97% rename from mobile/lib/services/face_ml/blur_detection/blur_detection_service.dart rename to mobile/lib/services/face_ml/face_filtering/blur_detection_service.dart index ff58304683..9c15523648 100644 --- a/mobile/lib/services/face_ml/blur_detection/blur_detection_service.dart +++ b/mobile/lib/services/face_ml/face_filtering/blur_detection_service.dart @@ -1,5 +1,5 @@ import 'package:logging/logging.dart'; -import "package:photos/services/face_ml/blur_detection/blur_constants.dart"; +import 'package:photos/services/face_ml/face_filtering/face_filtering_constants.dart'; class BlurDetectionService { final _logger = Logger('BlurDetectionService'); diff --git a/mobile/lib/services/face_ml/face_filtering/face_filtering_constants.dart b/mobile/lib/services/face_ml/face_filtering/face_filtering_constants.dart new file mode 100644 index 0000000000..478161f22b --- /dev/null +++ b/mobile/lib/services/face_ml/face_filtering/face_filtering_constants.dart @@ -0,0 +1,8 @@ +/// Blur detection threshold +const kLaplacianThreshold = 15; + +/// Default blur value +const kLapacianDefault = 10000.0; + +/// The minimum score for a face to be considered a face +const kMinFaceScore = 0.78; \ No newline at end of file diff --git a/mobile/lib/services/face_ml/face_ml_result.dart b/mobile/lib/services/face_ml/face_ml_result.dart index c770efde7a..3a981b377e 100644 --- a/mobile/lib/services/face_ml/face_ml_result.dart +++ b/mobile/lib/services/face_ml/face_ml_result.dart @@ -6,11 +6,11 @@ import "package:photos/db/ml_data_db.dart"; import "package:photos/models/file/file.dart"; import 'package:photos/models/ml/ml_typedefs.dart'; import "package:photos/models/ml/ml_versions.dart"; -import "package:photos/services/face_ml/blur_detection/blur_constants.dart"; import "package:photos/services/face_ml/face_alignment/alignment_result.dart"; import "package:photos/services/face_ml/face_clustering/cosine_distance.dart"; import "package:photos/services/face_ml/face_detection/detection.dart"; import "package:photos/services/face_ml/face_feedback.dart/cluster_feedback.dart"; +import 'package:photos/services/face_ml/face_filtering/face_filtering_constants.dart'; import "package:photos/services/face_ml/face_ml_methods.dart"; final _logger = Logger('ClusterResult_FaceMlResult'); diff --git a/mobile/lib/services/face_ml/face_ml_service.dart b/mobile/lib/services/face_ml/face_ml_service.dart index 789430386f..db3931ed7d 100644 --- a/mobile/lib/services/face_ml/face_ml_service.dart +++ b/mobile/lib/services/face_ml/face_ml_service.dart @@ -31,6 +31,7 @@ import 'package:photos/services/face_ml/face_detection/yolov5face/onnx_face_dete import "package:photos/services/face_ml/face_detection/yolov5face/yolo_face_detection_exceptions.dart"; import "package:photos/services/face_ml/face_embedding/face_embedding_exceptions.dart"; import 'package:photos/services/face_ml/face_embedding/onnx_face_embedding.dart'; +import "package:photos/services/face_ml/face_filtering/face_filtering_constants.dart"; import "package:photos/services/face_ml/face_ml_exceptions.dart"; import "package:photos/services/face_ml/face_ml_result.dart"; import 'package:photos/services/machine_learning/file_ml/file_ml.dart'; @@ -361,7 +362,7 @@ class FaceMlService { await clusterAllImages(); } - Future clusterAllImages({double minFaceScore = 0.75}) async { + Future clusterAllImages({double minFaceScore = kMinFaceScore}) async { _logger.info("`clusterAllImages()` called"); try { diff --git a/mobile/lib/ui/viewer/file_details/face_widget.dart b/mobile/lib/ui/viewer/file_details/face_widget.dart index a7045f92c2..6dd0b4aebf 100644 --- a/mobile/lib/ui/viewer/file_details/face_widget.dart +++ b/mobile/lib/ui/viewer/file_details/face_widget.dart @@ -88,6 +88,10 @@ class FaceWidget extends StatelessWidget { ), ), const SizedBox(height: 8), + Text( + (face.score).toStringAsFixed(2), + style: Theme.of(context).textTheme.bodySmall, + ), if (person != null) Text( person!.attr.name.trim(), diff --git a/mobile/lib/ui/viewer/file_details/faces_item_widget.dart b/mobile/lib/ui/viewer/file_details/faces_item_widget.dart index fb2d8cbd53..68de105c7b 100644 --- a/mobile/lib/ui/viewer/file_details/faces_item_widget.dart +++ b/mobile/lib/ui/viewer/file_details/faces_item_widget.dart @@ -36,9 +36,18 @@ class FacesItemWidget extends StatelessWidget { ]; } - final List faces = await FaceMLDataDB.instance + final List? faces = await FaceMLDataDB.instance .getFacesForGivenFileID(file.uploadedFileID!); - if (faces.isEmpty || faces.every((face) => face.score < 0.5)) { + if (faces == null) { + return [ + const ChipButtonWidget( + "Image not analyzed", + noChips: true, + ), + ]; + } + if (faces.isEmpty || + faces.every((face) => face.score < 0.75 || face.isBlurry)) { return [ const ChipButtonWidget( "No faces found", @@ -50,6 +59,9 @@ class FacesItemWidget extends StatelessWidget { // Sort the faces by score in descending order, so that the highest scoring face is first. faces.sort((Face a, Face b) => b.score.compareTo(a.score)); + // Remove faces with low scores and blurry faces + faces.removeWhere((face) => face.isHighQuality == false); + // TODO: add deduplication of faces of same person final faceIdsToClusterIds = await FaceMLDataDB.instance .getFaceIdsToClusterIds(faces.map((face) => face.faceID)); diff --git a/mobile/lib/utils/image_ml_util.dart b/mobile/lib/utils/image_ml_util.dart index fd04cc1c14..9b11efa8ad 100644 --- a/mobile/lib/utils/image_ml_util.dart +++ b/mobile/lib/utils/image_ml_util.dart @@ -18,10 +18,10 @@ import 'package:flutter/painting.dart' as paint show decodeImageFromList; import 'package:ml_linalg/linalg.dart'; import "package:photos/face/model/box.dart"; import 'package:photos/models/ml/ml_typedefs.dart'; -import "package:photos/services/face_ml/blur_detection/blur_detection_service.dart"; import "package:photos/services/face_ml/face_alignment/alignment_result.dart"; import "package:photos/services/face_ml/face_alignment/similarity_transform.dart"; import "package:photos/services/face_ml/face_detection/detection.dart"; +import 'package:photos/services/face_ml/face_filtering/blur_detection_service.dart'; /// All of the functions in this file are helper functions for the [ImageMlIsolate] isolate. /// Don't use them outside of the isolate, unless you are okay with UI jank!!!! From a443ac16803352098114e15f1b3e925174a1279f Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Wed, 20 Mar 2024 14:42:42 +0530 Subject: [PATCH 08/22] Better use of constants --- mobile/lib/face/db.dart | 8 ++++---- mobile/lib/face/model/face.dart | 2 +- .../face_ml/face_filtering/face_filtering_constants.dart | 9 +++++++-- mobile/lib/services/face_ml/face_ml_service.dart | 4 +++- 4 files changed, 15 insertions(+), 8 deletions(-) diff --git a/mobile/lib/face/db.dart b/mobile/lib/face/db.dart index c8b4f75f71..bc4b15885b 100644 --- a/mobile/lib/face/db.dart +++ b/mobile/lib/face/db.dart @@ -154,7 +154,7 @@ class FaceMLDataDB { final Map result = {}; final db = await instance.database; final List> maps = await db.rawQuery( - 'SELECT $fileIDColumn, COUNT(*) as count FROM $facesTable where $faceScore > 0.8 GROUP BY $fileIDColumn', + 'SELECT $fileIDColumn, COUNT(*) as count FROM $facesTable where $faceScore > $kMinFaceDetectionScore GROUP BY $fileIDColumn', ); for (final map in maps) { @@ -197,7 +197,7 @@ class FaceMLDataDB { final clusterIDs = cluterRows.map((e) => e[cluserIDColumn] as int).toList(); final List> faceMaps = await db.rawQuery( - 'SELECT * FROM $facesTable where $faceClusterId IN (${clusterIDs.join(",")}) AND $fileIDColumn in (${fileId.join(",")}) AND $faceScore > 0.8 ORDER BY $faceScore DESC', + 'SELECT * FROM $facesTable where $faceClusterId IN (${clusterIDs.join(",")}) AND $fileIDColumn in (${fileId.join(",")}) AND $faceScore > $kMinHighQualityFaceScore ORDER BY $faceScore DESC', ); if (faceMaps.isNotEmpty) { if (avatarFileId != null) { @@ -341,7 +341,7 @@ class FaceMLDataDB { /// /// Only selects faces with score greater than [minScore] and blur score greater than [minClarity] Future> getFaceEmbeddingMap({ - double minScore = kMinFaceScore, + double minScore = kMinHighQualityFaceScore, int minClarity = kLaplacianThreshold, int maxRows = 20000, }) async { @@ -398,7 +398,7 @@ class FaceMLDataDB { facesTable, columns: [faceIDColumn, faceEmbeddingBlob], where: - '$faceScore > 0.8 AND $faceBlur > $kLaplacianThreshold AND $fileIDColumn IN (${fileIDs.join(",")})', + '$faceScore > $kMinHighQualityFaceScore AND $faceBlur > $kLaplacianThreshold AND $fileIDColumn IN (${fileIDs.join(",")})', limit: batchSize, offset: offset, orderBy: '$faceIDColumn DESC', diff --git a/mobile/lib/face/model/face.dart b/mobile/lib/face/model/face.dart index c29c03ae11..af151c2ecf 100644 --- a/mobile/lib/face/model/face.dart +++ b/mobile/lib/face/model/face.dart @@ -11,7 +11,7 @@ class Face { bool get isBlurry => blur < kLaplacianThreshold; - bool get hasHighScore => score > kMinFaceScore; + bool get hasHighScore => score > kMinHighQualityFaceScore; bool get isHighQuality => (!isBlurry) && hasHighScore; diff --git a/mobile/lib/services/face_ml/face_filtering/face_filtering_constants.dart b/mobile/lib/services/face_ml/face_filtering/face_filtering_constants.dart index 478161f22b..a1970fd4d6 100644 --- a/mobile/lib/services/face_ml/face_filtering/face_filtering_constants.dart +++ b/mobile/lib/services/face_ml/face_filtering/face_filtering_constants.dart @@ -1,8 +1,13 @@ +import "package:photos/services/face_ml/face_detection/yolov5face/onnx_face_detection.dart"; + /// Blur detection threshold const kLaplacianThreshold = 15; /// Default blur value const kLapacianDefault = 10000.0; -/// The minimum score for a face to be considered a face -const kMinFaceScore = 0.78; \ No newline at end of file +/// The minimum score for a face to be considered a high quality face for clustering and person detection +const kMinHighQualityFaceScore = 0.78; + +/// The minimum score for a face to be detected, regardless of quality. Use [kMinHighQualityFaceScore] for high quality faces. +const kMinFaceDetectionScore = YoloOnnxFaceDetection.kMinScoreSigmoidThreshold; diff --git a/mobile/lib/services/face_ml/face_ml_service.dart b/mobile/lib/services/face_ml/face_ml_service.dart index db3931ed7d..a3b36e192d 100644 --- a/mobile/lib/services/face_ml/face_ml_service.dart +++ b/mobile/lib/services/face_ml/face_ml_service.dart @@ -362,7 +362,9 @@ class FaceMlService { await clusterAllImages(); } - Future clusterAllImages({double minFaceScore = kMinFaceScore}) async { + Future clusterAllImages({ + double minFaceScore = kMinHighQualityFaceScore, + }) async { _logger.info("`clusterAllImages()` called"); try { From 1819ea834a9c355f849a0eedbc5fce92346349e0 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Wed, 20 Mar 2024 14:44:28 +0530 Subject: [PATCH 09/22] Forgot to delete debug stuff --- mobile/lib/ui/viewer/file_details/face_widget.dart | 4 ---- 1 file changed, 4 deletions(-) diff --git a/mobile/lib/ui/viewer/file_details/face_widget.dart b/mobile/lib/ui/viewer/file_details/face_widget.dart index 6dd0b4aebf..a7045f92c2 100644 --- a/mobile/lib/ui/viewer/file_details/face_widget.dart +++ b/mobile/lib/ui/viewer/file_details/face_widget.dart @@ -88,10 +88,6 @@ class FaceWidget extends StatelessWidget { ), ), const SizedBox(height: 8), - Text( - (face.score).toStringAsFixed(2), - style: Theme.of(context).textTheme.bodySmall, - ), if (person != null) Text( person!.attr.name.trim(), From a4582c0e558c2f4e0a885224eb9265ab288a32d9 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Wed, 20 Mar 2024 15:51:57 +0530 Subject: [PATCH 10/22] Move FaceML inside Machine Learning directory --- mobile/lib/db/ml_data_db.dart | 6 +++--- mobile/lib/face/db.dart | 2 +- mobile/lib/face/db_fields.dart | 2 +- mobile/lib/face/model/face.dart | 2 +- mobile/lib/main.dart | 2 +- .../face_alignment/alignment_result.dart | 0 .../face_alignment/similarity_transform.dart | 2 +- .../face_clustering/cosine_distance.dart | 0 .../linear_clustering_service.dart | 2 +- .../face_ml/face_detection/detection.dart | 0 .../naive_non_max_suppression.dart | 2 +- .../yolov5face/onnx_face_detection.dart | 8 ++++---- .../yolo_face_detection_exceptions.dart | 0 .../yolo_face_detection_options.dart | 0 .../yolo_filter_extract_detections.dart | 2 +- .../yolov5face/yolo_model_config.dart | 4 ++-- .../face_embedding_exceptions.dart | 0 .../face_embedding/face_embedding_options.dart | 0 .../face_embedding/face_embedding_service.dart | 8 ++++---- .../mobilefacenet_model_config.dart | 4 ++-- .../face_embedding/onnx_face_embedding.dart | 2 +- .../face_feedback.dart/cluster_feedback.dart | 6 +++--- .../face_feedback_service.dart | 6 +++--- .../face_ml/face_feedback.dart/feedback.dart | 2 +- .../face_feedback.dart/feedback_types.dart | 0 .../face_filtering/blur_detection_service.dart | 2 +- .../face_filtering_constants.dart | 2 +- .../face_ml/face_ml_exceptions.dart | 0 .../face_ml/face_ml_methods.dart | 2 +- .../face_ml/face_ml_result.dart | 12 ++++++------ .../face_ml/face_ml_service.dart | 18 +++++++++--------- .../face_ml/face_ml_version.dart | 0 .../face_ml/face_search_service.dart | 0 .../face_ml/feedback/cluster_feedback.dart | 2 +- .../face_ml/model_file.dart | 0 .../debug/face_debug_section_widget.dart | 4 ++-- .../actions/file_selection_actions_widget.dart | 2 +- .../viewer/people/add_person_action_sheet.dart | 2 +- .../people/person_cluster_suggestion.dart | 2 +- mobile/lib/utils/image_ml_isolate.dart | 4 ++-- mobile/lib/utils/image_ml_util.dart | 8 ++++---- 41 files changed, 61 insertions(+), 61 deletions(-) rename mobile/lib/services/{ => machine_learning}/face_ml/face_alignment/alignment_result.dart (100%) rename mobile/lib/services/{ => machine_learning}/face_ml/face_alignment/similarity_transform.dart (98%) rename mobile/lib/services/{ => machine_learning}/face_ml/face_clustering/cosine_distance.dart (100%) rename mobile/lib/services/{ => machine_learning}/face_ml/face_clustering/linear_clustering_service.dart (99%) rename mobile/lib/services/{ => machine_learning}/face_ml/face_detection/detection.dart (100%) rename mobile/lib/services/{ => machine_learning}/face_ml/face_detection/naive_non_max_suppression.dart (94%) rename mobile/lib/services/{ => machine_learning}/face_ml/face_detection/yolov5face/onnx_face_detection.dart (98%) rename mobile/lib/services/{ => machine_learning}/face_ml/face_detection/yolov5face/yolo_face_detection_exceptions.dart (100%) rename mobile/lib/services/{ => machine_learning}/face_ml/face_detection/yolov5face/yolo_face_detection_options.dart (100%) rename mobile/lib/services/{ => machine_learning}/face_ml/face_detection/yolov5face/yolo_filter_extract_detections.dart (96%) rename mobile/lib/services/{ => machine_learning}/face_ml/face_detection/yolov5face/yolo_model_config.dart (70%) rename mobile/lib/services/{ => machine_learning}/face_ml/face_embedding/face_embedding_exceptions.dart (100%) rename mobile/lib/services/{ => machine_learning}/face_ml/face_embedding/face_embedding_options.dart (100%) rename mobile/lib/services/{ => machine_learning}/face_ml/face_embedding/face_embedding_service.dart (95%) rename mobile/lib/services/{ => machine_learning}/face_ml/face_embedding/mobilefacenet_model_config.dart (71%) rename mobile/lib/services/{ => machine_learning}/face_ml/face_embedding/onnx_face_embedding.dart (98%) rename mobile/lib/services/{ => machine_learning}/face_ml/face_feedback.dart/cluster_feedback.dart (97%) rename mobile/lib/services/{ => machine_learning}/face_ml/face_feedback.dart/face_feedback_service.dart (98%) rename mobile/lib/services/{ => machine_learning}/face_ml/face_feedback.dart/feedback.dart (90%) rename mobile/lib/services/{ => machine_learning}/face_ml/face_feedback.dart/feedback_types.dart (100%) rename mobile/lib/services/{ => machine_learning}/face_ml/face_filtering/blur_detection_service.dart (96%) rename mobile/lib/services/{ => machine_learning}/face_ml/face_filtering/face_filtering_constants.dart (81%) rename mobile/lib/services/{ => machine_learning}/face_ml/face_ml_exceptions.dart (100%) rename mobile/lib/services/{ => machine_learning}/face_ml/face_ml_methods.dart (97%) rename mobile/lib/services/{ => machine_learning}/face_ml/face_ml_result.dart (97%) rename mobile/lib/services/{ => machine_learning}/face_ml/face_ml_service.dart (97%) rename mobile/lib/services/{ => machine_learning}/face_ml/face_ml_version.dart (100%) rename mobile/lib/services/{ => machine_learning}/face_ml/face_search_service.dart (100%) rename mobile/lib/services/{ => machine_learning}/face_ml/feedback/cluster_feedback.dart (99%) rename mobile/lib/services/{ => machine_learning}/face_ml/model_file.dart (100%) diff --git a/mobile/lib/db/ml_data_db.dart b/mobile/lib/db/ml_data_db.dart index 07150f09b5..46ca06466b 100644 --- a/mobile/lib/db/ml_data_db.dart +++ b/mobile/lib/db/ml_data_db.dart @@ -4,9 +4,9 @@ import 'package:logging/logging.dart'; import 'package:path/path.dart' show join; import 'package:path_provider/path_provider.dart'; import 'package:photos/models/ml/ml_typedefs.dart'; -import "package:photos/services/face_ml/face_feedback.dart/cluster_feedback.dart"; -import "package:photos/services/face_ml/face_feedback.dart/feedback_types.dart"; -import "package:photos/services/face_ml/face_ml_result.dart"; +import 'package:photos/services/machine_learning/face_ml/face_feedback.dart/cluster_feedback.dart'; +import 'package:photos/services/machine_learning/face_ml/face_feedback.dart/feedback_types.dart'; +import 'package:photos/services/machine_learning/face_ml/face_ml_result.dart'; import 'package:sqflite/sqflite.dart'; /// Stores all data for the ML-related features. The database can be accessed by `MlDataDB.instance.database`. diff --git a/mobile/lib/face/db.dart b/mobile/lib/face/db.dart index bc4b15885b..585f7d7bb2 100644 --- a/mobile/lib/face/db.dart +++ b/mobile/lib/face/db.dart @@ -11,7 +11,7 @@ import "package:photos/face/db_model_mappers.dart"; import "package:photos/face/model/face.dart"; import "package:photos/face/model/person.dart"; import "package:photos/models/file/file.dart"; -import 'package:photos/services/face_ml/face_filtering/face_filtering_constants.dart'; +import 'package:photos/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart'; import 'package:sqflite/sqflite.dart'; /// Stores all data for the ML-related features. The database can be accessed by `MlDataDB.instance.database`. diff --git a/mobile/lib/face/db_fields.dart b/mobile/lib/face/db_fields.dart index 8b17195751..e43747f682 100644 --- a/mobile/lib/face/db_fields.dart +++ b/mobile/lib/face/db_fields.dart @@ -1,5 +1,5 @@ // Faces Table Fields & Schema Queries -import 'package:photos/services/face_ml/face_filtering/face_filtering_constants.dart'; +import 'package:photos/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart'; const facesTable = 'faces'; const fileIDColumn = 'file_id'; diff --git a/mobile/lib/face/model/face.dart b/mobile/lib/face/model/face.dart index af151c2ecf..0df0987dff 100644 --- a/mobile/lib/face/model/face.dart +++ b/mobile/lib/face/model/face.dart @@ -1,5 +1,5 @@ import "package:photos/face/model/detection.dart"; -import 'package:photos/services/face_ml/face_filtering/face_filtering_constants.dart'; +import 'package:photos/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart'; class Face { final int fileID; diff --git a/mobile/lib/main.dart b/mobile/lib/main.dart index d800336dfe..cc0bb60b15 100644 --- a/mobile/lib/main.dart +++ b/mobile/lib/main.dart @@ -25,13 +25,13 @@ import 'package:photos/services/app_lifecycle_service.dart'; import 'package:photos/services/billing_service.dart'; import 'package:photos/services/collections_service.dart'; import "package:photos/services/entity_service.dart"; -import "package:photos/services/face_ml/face_ml_service.dart"; import 'package:photos/services/favorites_service.dart'; import 'package:photos/services/feature_flag_service.dart'; import 'package:photos/services/home_widget_service.dart'; import 'package:photos/services/local_file_update_service.dart'; import 'package:photos/services/local_sync_service.dart'; import "package:photos/services/location_service.dart"; +import 'package:photos/services/machine_learning/face_ml/face_ml_service.dart'; import 'package:photos/services/machine_learning/file_ml/remote_fileml_service.dart'; import "package:photos/services/machine_learning/machine_learning_controller.dart"; import 'package:photos/services/machine_learning/semantic_search/semantic_search_service.dart'; diff --git a/mobile/lib/services/face_ml/face_alignment/alignment_result.dart b/mobile/lib/services/machine_learning/face_ml/face_alignment/alignment_result.dart similarity index 100% rename from mobile/lib/services/face_ml/face_alignment/alignment_result.dart rename to mobile/lib/services/machine_learning/face_ml/face_alignment/alignment_result.dart diff --git a/mobile/lib/services/face_ml/face_alignment/similarity_transform.dart b/mobile/lib/services/machine_learning/face_ml/face_alignment/similarity_transform.dart similarity index 98% rename from mobile/lib/services/face_ml/face_alignment/similarity_transform.dart rename to mobile/lib/services/machine_learning/face_ml/face_alignment/similarity_transform.dart index 4ae27794b4..0d8e7ab3ae 100644 --- a/mobile/lib/services/face_ml/face_alignment/similarity_transform.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_alignment/similarity_transform.dart @@ -1,7 +1,7 @@ import 'dart:math' show atan2; import 'package:ml_linalg/linalg.dart'; import 'package:photos/extensions/ml_linalg_extensions.dart'; -import "package:photos/services/face_ml/face_alignment/alignment_result.dart"; +import 'package:photos/services/machine_learning/face_ml/face_alignment/alignment_result.dart'; /// Class to compute the similarity transform between two sets of points. /// diff --git a/mobile/lib/services/face_ml/face_clustering/cosine_distance.dart b/mobile/lib/services/machine_learning/face_ml/face_clustering/cosine_distance.dart similarity index 100% rename from mobile/lib/services/face_ml/face_clustering/cosine_distance.dart rename to mobile/lib/services/machine_learning/face_ml/face_clustering/cosine_distance.dart diff --git a/mobile/lib/services/face_ml/face_clustering/linear_clustering_service.dart b/mobile/lib/services/machine_learning/face_ml/face_clustering/linear_clustering_service.dart similarity index 99% rename from mobile/lib/services/face_ml/face_clustering/linear_clustering_service.dart rename to mobile/lib/services/machine_learning/face_ml/face_clustering/linear_clustering_service.dart index 8e77838592..b9cb5345db 100644 --- a/mobile/lib/services/face_ml/face_clustering/linear_clustering_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_clustering/linear_clustering_service.dart @@ -6,7 +6,7 @@ import "dart:typed_data"; import "package:logging/logging.dart"; import "package:photos/generated/protos/ente/common/vector.pb.dart"; -import "package:photos/services/face_ml/face_clustering/cosine_distance.dart"; +import 'package:photos/services/machine_learning/face_ml/face_clustering/cosine_distance.dart'; import "package:synchronized/synchronized.dart"; class FaceInfo { diff --git a/mobile/lib/services/face_ml/face_detection/detection.dart b/mobile/lib/services/machine_learning/face_ml/face_detection/detection.dart similarity index 100% rename from mobile/lib/services/face_ml/face_detection/detection.dart rename to mobile/lib/services/machine_learning/face_ml/face_detection/detection.dart diff --git a/mobile/lib/services/face_ml/face_detection/naive_non_max_suppression.dart b/mobile/lib/services/machine_learning/face_ml/face_detection/naive_non_max_suppression.dart similarity index 94% rename from mobile/lib/services/face_ml/face_detection/naive_non_max_suppression.dart rename to mobile/lib/services/machine_learning/face_ml/face_detection/naive_non_max_suppression.dart index ca1e4aba5a..624181a669 100644 --- a/mobile/lib/services/face_ml/face_detection/naive_non_max_suppression.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_detection/naive_non_max_suppression.dart @@ -1,6 +1,6 @@ import 'dart:math' as math show max, min; -import "package:photos/services/face_ml/face_detection/detection.dart"; +import 'package:photos/services/machine_learning/face_ml/face_detection/detection.dart'; List naiveNonMaxSuppression({ required List detections, diff --git a/mobile/lib/services/face_ml/face_detection/yolov5face/onnx_face_detection.dart b/mobile/lib/services/machine_learning/face_ml/face_detection/yolov5face/onnx_face_detection.dart similarity index 98% rename from mobile/lib/services/face_ml/face_detection/yolov5face/onnx_face_detection.dart rename to mobile/lib/services/machine_learning/face_ml/face_detection/yolov5face/onnx_face_detection.dart index c67c4a3fc6..a2138fe7a5 100644 --- a/mobile/lib/services/face_ml/face_detection/yolov5face/onnx_face_detection.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_detection/yolov5face/onnx_face_detection.dart @@ -9,10 +9,10 @@ import "package:computer/computer.dart"; import 'package:flutter/material.dart'; import 'package:logging/logging.dart'; import 'package:onnxruntime/onnxruntime.dart'; -import "package:photos/services/face_ml/face_detection/detection.dart"; -import "package:photos/services/face_ml/face_detection/naive_non_max_suppression.dart"; -import "package:photos/services/face_ml/face_detection/yolov5face/yolo_face_detection_exceptions.dart"; -import "package:photos/services/face_ml/face_detection/yolov5face/yolo_filter_extract_detections.dart"; +import 'package:photos/services/machine_learning/face_ml/face_detection/detection.dart'; +import 'package:photos/services/machine_learning/face_ml/face_detection/naive_non_max_suppression.dart'; +import 'package:photos/services/machine_learning/face_ml/face_detection/yolov5face/yolo_face_detection_exceptions.dart'; +import 'package:photos/services/machine_learning/face_ml/face_detection/yolov5face/yolo_filter_extract_detections.dart'; import "package:photos/services/remote_assets_service.dart"; import "package:photos/utils/image_ml_isolate.dart"; import "package:photos/utils/image_ml_util.dart"; diff --git a/mobile/lib/services/face_ml/face_detection/yolov5face/yolo_face_detection_exceptions.dart b/mobile/lib/services/machine_learning/face_ml/face_detection/yolov5face/yolo_face_detection_exceptions.dart similarity index 100% rename from mobile/lib/services/face_ml/face_detection/yolov5face/yolo_face_detection_exceptions.dart rename to mobile/lib/services/machine_learning/face_ml/face_detection/yolov5face/yolo_face_detection_exceptions.dart diff --git a/mobile/lib/services/face_ml/face_detection/yolov5face/yolo_face_detection_options.dart b/mobile/lib/services/machine_learning/face_ml/face_detection/yolov5face/yolo_face_detection_options.dart similarity index 100% rename from mobile/lib/services/face_ml/face_detection/yolov5face/yolo_face_detection_options.dart rename to mobile/lib/services/machine_learning/face_ml/face_detection/yolov5face/yolo_face_detection_options.dart diff --git a/mobile/lib/services/face_ml/face_detection/yolov5face/yolo_filter_extract_detections.dart b/mobile/lib/services/machine_learning/face_ml/face_detection/yolov5face/yolo_filter_extract_detections.dart similarity index 96% rename from mobile/lib/services/face_ml/face_detection/yolov5face/yolo_filter_extract_detections.dart rename to mobile/lib/services/machine_learning/face_ml/face_detection/yolov5face/yolo_filter_extract_detections.dart index 168d06df88..ec546533ab 100644 --- a/mobile/lib/services/face_ml/face_detection/yolov5face/yolo_filter_extract_detections.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_detection/yolov5face/yolo_filter_extract_detections.dart @@ -1,6 +1,6 @@ import 'dart:developer' as dev show log; -import "package:photos/services/face_ml/face_detection/detection.dart"; +import 'package:photos/services/machine_learning/face_ml/face_detection/detection.dart'; List yoloOnnxFilterExtractDetections( double minScoreSigmoidThreshold, diff --git a/mobile/lib/services/face_ml/face_detection/yolov5face/yolo_model_config.dart b/mobile/lib/services/machine_learning/face_ml/face_detection/yolov5face/yolo_model_config.dart similarity index 70% rename from mobile/lib/services/face_ml/face_detection/yolov5face/yolo_model_config.dart rename to mobile/lib/services/machine_learning/face_ml/face_detection/yolov5face/yolo_model_config.dart index c803beffd4..578036bc25 100644 --- a/mobile/lib/services/face_ml/face_detection/yolov5face/yolo_model_config.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_detection/yolov5face/yolo_model_config.dart @@ -1,5 +1,5 @@ -import "package:photos/services/face_ml/face_detection/yolov5face/yolo_face_detection_options.dart"; -import "package:photos/services/face_ml/model_file.dart"; +import 'package:photos/services/machine_learning/face_ml/face_detection/yolov5face/yolo_face_detection_options.dart'; +import 'package:photos/services/machine_learning/face_ml/model_file.dart'; class YOLOModelConfig { final String modelPath; diff --git a/mobile/lib/services/face_ml/face_embedding/face_embedding_exceptions.dart b/mobile/lib/services/machine_learning/face_ml/face_embedding/face_embedding_exceptions.dart similarity index 100% rename from mobile/lib/services/face_ml/face_embedding/face_embedding_exceptions.dart rename to mobile/lib/services/machine_learning/face_ml/face_embedding/face_embedding_exceptions.dart diff --git a/mobile/lib/services/face_ml/face_embedding/face_embedding_options.dart b/mobile/lib/services/machine_learning/face_ml/face_embedding/face_embedding_options.dart similarity index 100% rename from mobile/lib/services/face_ml/face_embedding/face_embedding_options.dart rename to mobile/lib/services/machine_learning/face_ml/face_embedding/face_embedding_options.dart diff --git a/mobile/lib/services/face_ml/face_embedding/face_embedding_service.dart b/mobile/lib/services/machine_learning/face_ml/face_embedding/face_embedding_service.dart similarity index 95% rename from mobile/lib/services/face_ml/face_embedding/face_embedding_service.dart rename to mobile/lib/services/machine_learning/face_ml/face_embedding/face_embedding_service.dart index 2711550bdb..1a3dcf4167 100644 --- a/mobile/lib/services/face_ml/face_embedding/face_embedding_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_embedding/face_embedding_service.dart @@ -6,10 +6,10 @@ import 'dart:typed_data' show Uint8List; import "package:flutter/foundation.dart"; import "package:logging/logging.dart"; import 'package:photos/models/ml/ml_typedefs.dart'; -import "package:photos/services/face_ml/face_detection/detection.dart"; -import "package:photos/services/face_ml/face_embedding/face_embedding_exceptions.dart"; -import "package:photos/services/face_ml/face_embedding/face_embedding_options.dart"; -import "package:photos/services/face_ml/face_embedding/mobilefacenet_model_config.dart"; +import 'package:photos/services/machine_learning/face_ml/face_detection/detection.dart'; +import 'package:photos/services/machine_learning/face_ml/face_embedding/face_embedding_exceptions.dart'; +import 'package:photos/services/machine_learning/face_ml/face_embedding/face_embedding_options.dart'; +import 'package:photos/services/machine_learning/face_ml/face_embedding/mobilefacenet_model_config.dart'; import 'package:photos/utils/image_ml_isolate.dart'; import 'package:photos/utils/image_ml_util.dart'; import 'package:tflite_flutter/tflite_flutter.dart'; diff --git a/mobile/lib/services/face_ml/face_embedding/mobilefacenet_model_config.dart b/mobile/lib/services/machine_learning/face_ml/face_embedding/mobilefacenet_model_config.dart similarity index 71% rename from mobile/lib/services/face_ml/face_embedding/mobilefacenet_model_config.dart rename to mobile/lib/services/machine_learning/face_ml/face_embedding/mobilefacenet_model_config.dart index d55a2d3331..fd45f51674 100644 --- a/mobile/lib/services/face_ml/face_embedding/mobilefacenet_model_config.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_embedding/mobilefacenet_model_config.dart @@ -1,5 +1,5 @@ -import "package:photos/services/face_ml/face_embedding/face_embedding_options.dart"; -import "package:photos/services/face_ml/model_file.dart"; +import 'package:photos/services/machine_learning/face_ml/face_embedding/face_embedding_options.dart'; +import 'package:photos/services/machine_learning/face_ml/model_file.dart'; class MobileFaceNetModelConfig { final String modelPath; diff --git a/mobile/lib/services/face_ml/face_embedding/onnx_face_embedding.dart b/mobile/lib/services/machine_learning/face_ml/face_embedding/onnx_face_embedding.dart similarity index 98% rename from mobile/lib/services/face_ml/face_embedding/onnx_face_embedding.dart rename to mobile/lib/services/machine_learning/face_ml/face_embedding/onnx_face_embedding.dart index f15b25b46c..bdf2ac5cfb 100644 --- a/mobile/lib/services/face_ml/face_embedding/onnx_face_embedding.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_embedding/onnx_face_embedding.dart @@ -5,7 +5,7 @@ import 'dart:typed_data' show Float32List; import 'package:computer/computer.dart'; import 'package:logging/logging.dart'; import 'package:onnxruntime/onnxruntime.dart'; -import "package:photos/services/face_ml/face_detection/detection.dart"; +import 'package:photos/services/machine_learning/face_ml/face_detection/detection.dart'; import "package:photos/services/remote_assets_service.dart"; import "package:photos/utils/image_ml_isolate.dart"; import "package:synchronized/synchronized.dart"; diff --git a/mobile/lib/services/face_ml/face_feedback.dart/cluster_feedback.dart b/mobile/lib/services/machine_learning/face_ml/face_feedback.dart/cluster_feedback.dart similarity index 97% rename from mobile/lib/services/face_ml/face_feedback.dart/cluster_feedback.dart rename to mobile/lib/services/machine_learning/face_ml/face_feedback.dart/cluster_feedback.dart index b99d3950ab..a7da4fa556 100644 --- a/mobile/lib/services/face_ml/face_feedback.dart/cluster_feedback.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_feedback.dart/cluster_feedback.dart @@ -1,8 +1,8 @@ import "dart:convert"; -import "package:photos/services/face_ml/face_clustering/cosine_distance.dart"; -import "package:photos/services/face_ml/face_feedback.dart/feedback.dart"; -import "package:photos/services/face_ml/face_feedback.dart/feedback_types.dart"; +import 'package:photos/services/machine_learning/face_ml/face_clustering/cosine_distance.dart'; +import 'package:photos/services/machine_learning/face_ml/face_feedback.dart/feedback.dart'; +import 'package:photos/services/machine_learning/face_ml/face_feedback.dart/feedback_types.dart'; abstract class ClusterFeedback extends Feedback { static final Map fromJsonStringRegistry = { diff --git a/mobile/lib/services/face_ml/face_feedback.dart/face_feedback_service.dart b/mobile/lib/services/machine_learning/face_ml/face_feedback.dart/face_feedback_service.dart similarity index 98% rename from mobile/lib/services/face_ml/face_feedback.dart/face_feedback_service.dart rename to mobile/lib/services/machine_learning/face_ml/face_feedback.dart/face_feedback_service.dart index 0e95e3d7cc..c94c8c8d85 100644 --- a/mobile/lib/services/face_ml/face_feedback.dart/face_feedback_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_feedback.dart/face_feedback_service.dart @@ -1,8 +1,8 @@ import "package:logging/logging.dart"; import "package:photos/db/ml_data_db.dart"; -import "package:photos/services/face_ml/face_detection/detection.dart"; -import "package:photos/services/face_ml/face_feedback.dart/cluster_feedback.dart"; -import "package:photos/services/face_ml/face_ml_result.dart"; +import 'package:photos/services/machine_learning/face_ml/face_detection/detection.dart'; +import 'package:photos/services/machine_learning/face_ml/face_feedback.dart/cluster_feedback.dart'; +import 'package:photos/services/machine_learning/face_ml/face_ml_result.dart'; class FaceFeedbackService { final _logger = Logger("FaceFeedbackService"); diff --git a/mobile/lib/services/face_ml/face_feedback.dart/feedback.dart b/mobile/lib/services/machine_learning/face_ml/face_feedback.dart/feedback.dart similarity index 90% rename from mobile/lib/services/face_ml/face_feedback.dart/feedback.dart rename to mobile/lib/services/machine_learning/face_ml/face_feedback.dart/feedback.dart index 320ec64e92..8b3eb3c6ad 100644 --- a/mobile/lib/services/face_ml/face_feedback.dart/feedback.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_feedback.dart/feedback.dart @@ -1,5 +1,5 @@ import "package:photos/models/ml/ml_versions.dart"; -import "package:photos/services/face_ml/face_feedback.dart/feedback_types.dart"; +import 'package:photos/services/machine_learning/face_ml/face_feedback.dart/feedback_types.dart'; import "package:uuid/uuid.dart"; abstract class Feedback { diff --git a/mobile/lib/services/face_ml/face_feedback.dart/feedback_types.dart b/mobile/lib/services/machine_learning/face_ml/face_feedback.dart/feedback_types.dart similarity index 100% rename from mobile/lib/services/face_ml/face_feedback.dart/feedback_types.dart rename to mobile/lib/services/machine_learning/face_ml/face_feedback.dart/feedback_types.dart diff --git a/mobile/lib/services/face_ml/face_filtering/blur_detection_service.dart b/mobile/lib/services/machine_learning/face_ml/face_filtering/blur_detection_service.dart similarity index 96% rename from mobile/lib/services/face_ml/face_filtering/blur_detection_service.dart rename to mobile/lib/services/machine_learning/face_ml/face_filtering/blur_detection_service.dart index 9c15523648..43f6b252d2 100644 --- a/mobile/lib/services/face_ml/face_filtering/blur_detection_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_filtering/blur_detection_service.dart @@ -1,5 +1,5 @@ import 'package:logging/logging.dart'; -import 'package:photos/services/face_ml/face_filtering/face_filtering_constants.dart'; +import 'package:photos/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart'; class BlurDetectionService { final _logger = Logger('BlurDetectionService'); diff --git a/mobile/lib/services/face_ml/face_filtering/face_filtering_constants.dart b/mobile/lib/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart similarity index 81% rename from mobile/lib/services/face_ml/face_filtering/face_filtering_constants.dart rename to mobile/lib/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart index a1970fd4d6..6606e858ec 100644 --- a/mobile/lib/services/face_ml/face_filtering/face_filtering_constants.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart @@ -1,4 +1,4 @@ -import "package:photos/services/face_ml/face_detection/yolov5face/onnx_face_detection.dart"; +import 'package:photos/services/machine_learning/face_ml/face_detection/yolov5face/onnx_face_detection.dart'; /// Blur detection threshold const kLaplacianThreshold = 15; diff --git a/mobile/lib/services/face_ml/face_ml_exceptions.dart b/mobile/lib/services/machine_learning/face_ml/face_ml_exceptions.dart similarity index 100% rename from mobile/lib/services/face_ml/face_ml_exceptions.dart rename to mobile/lib/services/machine_learning/face_ml/face_ml_exceptions.dart diff --git a/mobile/lib/services/face_ml/face_ml_methods.dart b/mobile/lib/services/machine_learning/face_ml/face_ml_methods.dart similarity index 97% rename from mobile/lib/services/face_ml/face_ml_methods.dart rename to mobile/lib/services/machine_learning/face_ml/face_ml_methods.dart index a6c967e52f..5745234b58 100644 --- a/mobile/lib/services/face_ml/face_ml_methods.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_ml_methods.dart @@ -1,4 +1,4 @@ -import "package:photos/services/face_ml/face_ml_version.dart"; +import 'package:photos/services/machine_learning/face_ml/face_ml_version.dart'; /// Represents a face detection method with a specific version. class FaceDetectionMethod extends VersionedMethod { diff --git a/mobile/lib/services/face_ml/face_ml_result.dart b/mobile/lib/services/machine_learning/face_ml/face_ml_result.dart similarity index 97% rename from mobile/lib/services/face_ml/face_ml_result.dart rename to mobile/lib/services/machine_learning/face_ml/face_ml_result.dart index 3a981b377e..da0f9f6ed4 100644 --- a/mobile/lib/services/face_ml/face_ml_result.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_ml_result.dart @@ -6,12 +6,12 @@ import "package:photos/db/ml_data_db.dart"; import "package:photos/models/file/file.dart"; import 'package:photos/models/ml/ml_typedefs.dart'; import "package:photos/models/ml/ml_versions.dart"; -import "package:photos/services/face_ml/face_alignment/alignment_result.dart"; -import "package:photos/services/face_ml/face_clustering/cosine_distance.dart"; -import "package:photos/services/face_ml/face_detection/detection.dart"; -import "package:photos/services/face_ml/face_feedback.dart/cluster_feedback.dart"; -import 'package:photos/services/face_ml/face_filtering/face_filtering_constants.dart'; -import "package:photos/services/face_ml/face_ml_methods.dart"; +import 'package:photos/services/machine_learning/face_ml/face_alignment/alignment_result.dart'; +import 'package:photos/services/machine_learning/face_ml/face_clustering/cosine_distance.dart'; +import 'package:photos/services/machine_learning/face_ml/face_detection/detection.dart'; +import 'package:photos/services/machine_learning/face_ml/face_feedback.dart/cluster_feedback.dart'; +import 'package:photos/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart'; +import 'package:photos/services/machine_learning/face_ml/face_ml_methods.dart'; final _logger = Logger('ClusterResult_FaceMlResult'); diff --git a/mobile/lib/services/face_ml/face_ml_service.dart b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart similarity index 97% rename from mobile/lib/services/face_ml/face_ml_service.dart rename to mobile/lib/services/machine_learning/face_ml/face_ml_service.dart index a3b36e192d..34e265dcf0 100644 --- a/mobile/lib/services/face_ml/face_ml_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart @@ -25,15 +25,15 @@ import "package:photos/models/file/extensions/file_props.dart"; import "package:photos/models/file/file.dart"; import "package:photos/models/file/file_type.dart"; import "package:photos/models/ml/ml_versions.dart"; -import "package:photos/services/face_ml/face_clustering/linear_clustering_service.dart"; -import "package:photos/services/face_ml/face_detection/detection.dart"; -import 'package:photos/services/face_ml/face_detection/yolov5face/onnx_face_detection.dart'; -import "package:photos/services/face_ml/face_detection/yolov5face/yolo_face_detection_exceptions.dart"; -import "package:photos/services/face_ml/face_embedding/face_embedding_exceptions.dart"; -import 'package:photos/services/face_ml/face_embedding/onnx_face_embedding.dart'; -import "package:photos/services/face_ml/face_filtering/face_filtering_constants.dart"; -import "package:photos/services/face_ml/face_ml_exceptions.dart"; -import "package:photos/services/face_ml/face_ml_result.dart"; +import 'package:photos/services/machine_learning/face_ml/face_clustering/linear_clustering_service.dart'; +import 'package:photos/services/machine_learning/face_ml/face_detection/detection.dart'; +import 'package:photos/services/machine_learning/face_ml/face_detection/yolov5face/onnx_face_detection.dart'; +import 'package:photos/services/machine_learning/face_ml/face_detection/yolov5face/yolo_face_detection_exceptions.dart'; +import 'package:photos/services/machine_learning/face_ml/face_embedding/face_embedding_exceptions.dart'; +import 'package:photos/services/machine_learning/face_ml/face_embedding/onnx_face_embedding.dart'; +import 'package:photos/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart'; +import 'package:photos/services/machine_learning/face_ml/face_ml_exceptions.dart'; +import 'package:photos/services/machine_learning/face_ml/face_ml_result.dart'; import 'package:photos/services/machine_learning/file_ml/file_ml.dart'; import 'package:photos/services/machine_learning/file_ml/remote_fileml_service.dart'; import "package:photos/services/search_service.dart"; diff --git a/mobile/lib/services/face_ml/face_ml_version.dart b/mobile/lib/services/machine_learning/face_ml/face_ml_version.dart similarity index 100% rename from mobile/lib/services/face_ml/face_ml_version.dart rename to mobile/lib/services/machine_learning/face_ml/face_ml_version.dart diff --git a/mobile/lib/services/face_ml/face_search_service.dart b/mobile/lib/services/machine_learning/face_ml/face_search_service.dart similarity index 100% rename from mobile/lib/services/face_ml/face_search_service.dart rename to mobile/lib/services/machine_learning/face_ml/face_search_service.dart diff --git a/mobile/lib/services/face_ml/feedback/cluster_feedback.dart b/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart similarity index 99% rename from mobile/lib/services/face_ml/feedback/cluster_feedback.dart rename to mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart index 129eee83f3..6a6fc28c0c 100644 --- a/mobile/lib/services/face_ml/feedback/cluster_feedback.dart +++ b/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart @@ -11,7 +11,7 @@ import "package:photos/face/db.dart"; import "package:photos/face/model/person.dart"; import "package:photos/generated/protos/ente/common/vector.pb.dart"; import "package:photos/models/file/file.dart"; -import "package:photos/services/face_ml/face_clustering/cosine_distance.dart"; +import 'package:photos/services/machine_learning/face_ml/face_clustering/cosine_distance.dart'; import "package:photos/services/search_service.dart"; class ClusterFeedbackService { diff --git a/mobile/lib/services/face_ml/model_file.dart b/mobile/lib/services/machine_learning/face_ml/model_file.dart similarity index 100% rename from mobile/lib/services/face_ml/model_file.dart rename to mobile/lib/services/machine_learning/face_ml/model_file.dart diff --git a/mobile/lib/ui/settings/debug/face_debug_section_widget.dart b/mobile/lib/ui/settings/debug/face_debug_section_widget.dart index f94e4ba614..80983309f7 100644 --- a/mobile/lib/ui/settings/debug/face_debug_section_widget.dart +++ b/mobile/lib/ui/settings/debug/face_debug_section_widget.dart @@ -8,8 +8,8 @@ import "package:photos/events/people_changed_event.dart"; import "package:photos/extensions/stop_watch.dart"; import "package:photos/face/db.dart"; import "package:photos/face/model/person.dart"; -import "package:photos/services/face_ml/face_ml_service.dart"; -import "package:photos/services/face_ml/feedback/cluster_feedback.dart"; +import 'package:photos/services/machine_learning/face_ml/face_ml_service.dart'; +import 'package:photos/services/machine_learning/face_ml/feedback/cluster_feedback.dart'; import 'package:photos/theme/ente_theme.dart'; import 'package:photos/ui/components/captioned_text_widget.dart'; import 'package:photos/ui/components/expandable_menu_item_widget.dart'; diff --git a/mobile/lib/ui/viewer/actions/file_selection_actions_widget.dart b/mobile/lib/ui/viewer/actions/file_selection_actions_widget.dart index 3692696739..a87cca795b 100644 --- a/mobile/lib/ui/viewer/actions/file_selection_actions_widget.dart +++ b/mobile/lib/ui/viewer/actions/file_selection_actions_widget.dart @@ -20,8 +20,8 @@ import 'package:photos/models/gallery_type.dart'; import "package:photos/models/metadata/common_keys.dart"; import 'package:photos/models/selected_files.dart'; import 'package:photos/services/collections_service.dart'; -import "package:photos/services/face_ml/feedback/cluster_feedback.dart"; import 'package:photos/services/hidden_service.dart'; +import 'package:photos/services/machine_learning/face_ml/feedback/cluster_feedback.dart'; import "package:photos/theme/colors.dart"; import "package:photos/theme/ente_theme.dart"; import 'package:photos/ui/actions/collection/collection_file_actions.dart'; diff --git a/mobile/lib/ui/viewer/people/add_person_action_sheet.dart b/mobile/lib/ui/viewer/people/add_person_action_sheet.dart index 4a072280fe..935af98801 100644 --- a/mobile/lib/ui/viewer/people/add_person_action_sheet.dart +++ b/mobile/lib/ui/viewer/people/add_person_action_sheet.dart @@ -10,7 +10,7 @@ import "package:photos/events/people_changed_event.dart"; import "package:photos/face/db.dart"; import "package:photos/face/model/person.dart"; import "package:photos/generated/l10n.dart"; -import "package:photos/services/face_ml/feedback/cluster_feedback.dart"; +import 'package:photos/services/machine_learning/face_ml/feedback/cluster_feedback.dart'; import 'package:photos/theme/colors.dart'; import 'package:photos/theme/ente_theme.dart'; import 'package:photos/ui/common/loading_widget.dart'; diff --git a/mobile/lib/ui/viewer/people/person_cluster_suggestion.dart b/mobile/lib/ui/viewer/people/person_cluster_suggestion.dart index 3ec179856e..1bbcb4390b 100644 --- a/mobile/lib/ui/viewer/people/person_cluster_suggestion.dart +++ b/mobile/lib/ui/viewer/people/person_cluster_suggestion.dart @@ -6,7 +6,7 @@ import "package:photos/events/people_changed_event.dart"; import "package:photos/face/db.dart"; import "package:photos/face/model/person.dart"; import "package:photos/models/file/file.dart"; -import "package:photos/services/face_ml/feedback/cluster_feedback.dart"; +import 'package:photos/services/machine_learning/face_ml/feedback/cluster_feedback.dart'; import "package:photos/theme/ente_theme.dart"; import "package:photos/ui/components/buttons/button_widget.dart"; import "package:photos/ui/components/models/button_type.dart"; diff --git a/mobile/lib/utils/image_ml_isolate.dart b/mobile/lib/utils/image_ml_isolate.dart index f9869ef8fd..157615d8e7 100644 --- a/mobile/lib/utils/image_ml_isolate.dart +++ b/mobile/lib/utils/image_ml_isolate.dart @@ -9,8 +9,8 @@ import 'package:flutter_isolate/flutter_isolate.dart'; import "package:logging/logging.dart"; import "package:photos/face/model/box.dart"; import 'package:photos/models/ml/ml_typedefs.dart'; -import "package:photos/services/face_ml/face_alignment/alignment_result.dart"; -import "package:photos/services/face_ml/face_detection/detection.dart"; +import 'package:photos/services/machine_learning/face_ml/face_alignment/alignment_result.dart'; +import 'package:photos/services/machine_learning/face_ml/face_detection/detection.dart'; import "package:photos/utils/image_ml_util.dart"; import "package:synchronized/synchronized.dart"; diff --git a/mobile/lib/utils/image_ml_util.dart b/mobile/lib/utils/image_ml_util.dart index 9b11efa8ad..1ba29df6b0 100644 --- a/mobile/lib/utils/image_ml_util.dart +++ b/mobile/lib/utils/image_ml_util.dart @@ -18,10 +18,10 @@ import 'package:flutter/painting.dart' as paint show decodeImageFromList; import 'package:ml_linalg/linalg.dart'; import "package:photos/face/model/box.dart"; import 'package:photos/models/ml/ml_typedefs.dart'; -import "package:photos/services/face_ml/face_alignment/alignment_result.dart"; -import "package:photos/services/face_ml/face_alignment/similarity_transform.dart"; -import "package:photos/services/face_ml/face_detection/detection.dart"; -import 'package:photos/services/face_ml/face_filtering/blur_detection_service.dart'; +import 'package:photos/services/machine_learning/face_ml/face_alignment/alignment_result.dart'; +import 'package:photos/services/machine_learning/face_ml/face_alignment/similarity_transform.dart'; +import 'package:photos/services/machine_learning/face_ml/face_detection/detection.dart'; +import 'package:photos/services/machine_learning/face_ml/face_filtering/blur_detection_service.dart'; /// All of the functions in this file are helper functions for the [ImageMlIsolate] isolate. /// Don't use them outside of the isolate, unless you are okay with UI jank!!!! From 125a4de66aebb3a7ca5901993355c79bcd10da4b Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Wed, 20 Mar 2024 16:20:55 +0530 Subject: [PATCH 11/22] Deprecate function --- .../lib/services/machine_learning/face_ml/face_ml_service.dart | 1 + 1 file changed, 1 insertion(+) diff --git a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart index 34e265dcf0..b506bd3e14 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart @@ -876,6 +876,7 @@ class FaceMlService { return imagePath; } + @Deprecated('Deprecated in favor of `_getImagePathForML`') Future _getDataForML( EnteFile enteFile, { FileDataForML typeOfData = FileDataForML.fileData, From a9fdee96a8c09325ae05f1e36732a30bdb5080ba Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Thu, 21 Mar 2024 12:40:03 +0530 Subject: [PATCH 12/22] More debug options --- .../face_ml/face_ml_service.dart | 1 + .../debug/face_debug_section_widget.dart | 34 +++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart index b506bd3e14..43c1b958a6 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart @@ -816,6 +816,7 @@ class FaceMlService { e, s, ); + debugPrint("This image with ID ${enteFile.uploadedFileID} has name ${enteFile.displayName}."); final resultBuilder = FaceMlResultBuilder.fromEnteFile(enteFile); return resultBuilder.buildErrorOccurred(); } diff --git a/mobile/lib/ui/settings/debug/face_debug_section_widget.dart b/mobile/lib/ui/settings/debug/face_debug_section_widget.dart index 80983309f7..ce28f0f04a 100644 --- a/mobile/lib/ui/settings/debug/face_debug_section_widget.dart +++ b/mobile/lib/ui/settings/debug/face_debug_section_widget.dart @@ -10,6 +10,7 @@ import "package:photos/face/db.dart"; import "package:photos/face/model/person.dart"; import 'package:photos/services/machine_learning/face_ml/face_ml_service.dart'; import 'package:photos/services/machine_learning/face_ml/feedback/cluster_feedback.dart'; +import "package:photos/services/search_service.dart"; import 'package:photos/theme/ente_theme.dart'; import 'package:photos/ui/components/captioned_text_widget.dart'; import 'package:photos/ui/components/expandable_menu_item_widget.dart'; @@ -92,6 +93,39 @@ class _FaceDebugSectionWidgetState extends State { } }, ), + MenuItemWidget( + captionedTextWidget: const CaptionedTextWidget( + title: "Analyze file ID 25728869", + ), + pressedColor: getEnteColorScheme(context).fillFaint, + trailingIcon: Icons.chevron_right_outlined, + trailingIconIsMuted: true, + onTap: () async { + try { + final enteFile = await SearchService.instance.getAllFiles().then( + (value) => value.firstWhere( + (element) => element.uploadedFileID == 25728869, + ), + ); + _logger.info( + 'File with ID ${enteFile.uploadedFileID} has name ${enteFile.displayName}', + ); + FaceMlService.instance.isImageIndexRunning = true; + final result = await FaceMlService.instance + .analyzeImageInSingleIsolate(enteFile); + if (result != null) { + final resultJson = result.toJsonString(); + _logger.info('result: $resultJson'); + } + FaceMlService.instance.isImageIndexRunning = false; + } catch (e, s) { + _logger.severe('indexing failed ', e, s); + await showGenericErrorDialog(context: context, error: e); + } finally { + FaceMlService.instance.isImageIndexRunning = false; + } + }, + ), MenuItemWidget( captionedTextWidget: const CaptionedTextWidget( title: "Run Clustering", From a2bca84b91ebd99b5492c2def05363f1e661ba57 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Thu, 21 Mar 2024 15:41:34 +0530 Subject: [PATCH 13/22] [mob] Sort clustering on fileCreationTime asc --- mobile/lib/db/files_db.dart | 17 ++++++++ .../linear_clustering_service.dart | 40 ++++++++++++++++--- .../face_ml/face_ml_result.dart | 6 +-- .../face_ml/face_ml_service.dart | 11 ++++- 4 files changed, 63 insertions(+), 11 deletions(-) diff --git a/mobile/lib/db/files_db.dart b/mobile/lib/db/files_db.dart index 39ff832103..ac3c273e00 100644 --- a/mobile/lib/db/files_db.dart +++ b/mobile/lib/db/files_db.dart @@ -1304,6 +1304,23 @@ class FilesDB { return result; } + Future> getFileIDToCreationTime() async { + final db = await instance.database; + final rows = await db.rawQuery( + ''' + SELECT $columnUploadedFileID, $columnCreationTime + FROM $filesTable + WHERE + ($columnUploadedFileID IS NOT NULL AND $columnUploadedFileID IS NOT -1); + ''', + ); + final result = {}; + for (final row in rows) { + result[row[columnUploadedFileID] as int] = row[columnCreationTime] as int; + } + return result; + } + // getCollectionFileFirstOrLast returns the first or last uploaded file in // the collection based on the given collectionID and the order. Future getCollectionFileFirstOrLast( diff --git a/mobile/lib/services/machine_learning/face_ml/face_clustering/linear_clustering_service.dart b/mobile/lib/services/machine_learning/face_ml/face_clustering/linear_clustering_service.dart index b9cb5345db..d82db2b43f 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_clustering/linear_clustering_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_clustering/linear_clustering_service.dart @@ -94,7 +94,12 @@ class FaceLinearClustering { switch (function) { case ClusterOperation.linearIncrementalClustering: final input = args['input'] as Map; - final result = FaceLinearClustering._runLinearClustering(input); + final fileIDToCreationTime = + args['fileIDToCreationTime'] as Map?; + final result = FaceLinearClustering._runLinearClustering( + input, + fileIDToCreationTime: fileIDToCreationTime, + ); sendPort.send(result); break; } @@ -169,8 +174,9 @@ class FaceLinearClustering { /// /// WARNING: Make sure to always input data in the same ordering, otherwise the clustering can less less deterministic. Future?> predict( - Map input, - ) async { + Map input, { + Map? fileIDToCreationTime, + }) async { if (input.isEmpty) { _logger.warning( "Clustering dataset of embeddings is empty, returning empty list.", @@ -192,7 +198,10 @@ class FaceLinearClustering { // final Map faceIdToCluster = // await _runLinearClusteringInComputer(input); final Map faceIdToCluster = await _runInIsolate( - (ClusterOperation.linearIncrementalClustering, {'input': input}), + ( + ClusterOperation.linearIncrementalClustering, + {'input': input, 'fileIDToCreationTime': fileIDToCreationTime} + ), ); // return _runLinearClusteringInComputer(input); _logger.info( @@ -205,8 +214,9 @@ class FaceLinearClustering { } static Map _runLinearClustering( - Map x, - ) { + Map x, { + Map? fileIDToCreationTime, + }) { log( "[ClusterIsolate] ${DateTime.now()} Copied to isolate ${x.length} faces", ); @@ -217,9 +227,27 @@ class FaceLinearClustering { faceID: entry.key, embedding: EVector.fromBuffer(entry.value.$2).values, clusterId: entry.value.$1, + fileCreationTime: + fileIDToCreationTime?[getFileIdFromFaceId(entry.key)], ), ); } + + // Sort the faceInfos based on fileCreationTime, in ascending order, so oldest faces are first + if (fileIDToCreationTime != null) { + faceInfos.sort((a, b) { + if (a.fileCreationTime == null && b.fileCreationTime == null) { + return 0; + } else if (a.fileCreationTime == null) { + return 1; + } else if (b.fileCreationTime == null) { + return -1; + } else { + return a.fileCreationTime!.compareTo(b.fileCreationTime!); + } + }); + } + // Sort the faceInfos such that the ones with null clusterId are at the end faceInfos.sort((a, b) { if (a.clusterId == null && b.clusterId == null) { diff --git a/mobile/lib/services/machine_learning/face_ml/face_ml_result.dart b/mobile/lib/services/machine_learning/face_ml/face_ml_result.dart index da0f9f6ed4..58fc72ac5a 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_ml_result.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_ml_result.dart @@ -37,7 +37,7 @@ class ClusterResult { String get thumbnailFaceId => _thumbnailFaceId; - int get thumbnailFileId => _getFileIdFromFaceId(_thumbnailFaceId); + int get thumbnailFileId => getFileIdFromFaceId(_thumbnailFaceId); /// Sets the thumbnail faceId to the given faceId. /// Throws an exception if the faceId is not in the list of faceIds. @@ -89,7 +89,7 @@ class ClusterResult { int removedCount = 0; for (var i = 0; i < _fileIds.length; i++) { if (_fileIds[i] == fileId) { - assert(_getFileIdFromFaceId(_faceIds[i]) == fileId); + assert(getFileIdFromFaceId(_faceIds[i]) == fileId); _fileIds.removeAt(i); _faceIds.removeAt(i); debugPrint( @@ -748,6 +748,6 @@ class FaceResultBuilder { } } -int _getFileIdFromFaceId(String faceId) { +int getFileIdFromFaceId(String faceId) { return int.parse(faceId.split("_")[0]); } diff --git a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart index 43c1b958a6..c8a09bbff7 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart @@ -13,6 +13,7 @@ import "package:logging/logging.dart"; import "package:onnxruntime/onnxruntime.dart"; import "package:photos/core/configuration.dart"; import "package:photos/core/event_bus.dart"; +import "package:photos/db/files_db.dart"; import "package:photos/db/ml_data_db.dart"; import "package:photos/events/diff_sync_complete_event.dart"; import "package:photos/extensions/list.dart"; @@ -375,9 +376,15 @@ class FaceMlService { ); _logger.info('read embeddings ${faceIdToEmbedding.length} '); + // Read the creation times from Files DB, in a map from fileID to creation time + final fileIDToCreationTime = + await FilesDB.instance.getFileIDToCreationTime(); + // Cluster the embeddings using the linear clustering algorithm, returning a map from faceID to clusterID - final faceIdToCluster = - await FaceLinearClustering.instance.predict(faceIdToEmbedding); + final faceIdToCluster = await FaceLinearClustering.instance.predict( + faceIdToEmbedding, + fileIDToCreationTime: fileIDToCreationTime, + ); if (faceIdToCluster == null) { _logger.warning("faceIdToCluster is null"); return; From fc8122b18eb7d1f1b14a1ec125ab2e99d34baa88 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Thu, 21 Mar 2024 16:52:52 +0530 Subject: [PATCH 14/22] Add indexing debug cooldown --- .../machine_learning/face_ml/face_ml_service.dart | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart index c8a09bbff7..8ed8331781 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart @@ -511,6 +511,14 @@ class FaceMlService { (previousValue, element) => previousValue + (element ? 1 : 0), ); fileAnalyzedCount += sumFutures; + + // TODO: remove this cooldown later. Cooldown of one minute every 400 images + if (fileAnalyzedCount > 400 && fileAnalyzedCount % 400 < kParallelism) { + _logger.info( + "indexAllImages() analyzed $fileAnalyzedCount images, cooldown for 1 minute", + ); + await Future.delayed(const Duration(minutes: 1)); + } } stopwatch.stop(); From 212208ae010584467b62f2507eac0345fca74f7d Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Thu, 21 Mar 2024 16:53:22 +0530 Subject: [PATCH 15/22] Add debugPrint --- .../services/machine_learning/face_ml/face_ml_service.dart | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart index 8ed8331781..4ca13b8d9d 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart @@ -831,7 +831,9 @@ class FaceMlService { e, s, ); - debugPrint("This image with ID ${enteFile.uploadedFileID} has name ${enteFile.displayName}."); + debugPrint( + "This image with ID ${enteFile.uploadedFileID} has name ${enteFile.displayName}.", + ); final resultBuilder = FaceMlResultBuilder.fromEnteFile(enteFile); return resultBuilder.buildErrorOccurred(); } From b5cff212bbe897ad131168ba381e549b8d0dc8a2 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Thu, 21 Mar 2024 16:59:55 +0530 Subject: [PATCH 16/22] Refactor of clustering --- .../linear_clustering_service.dart | 171 ++++++++---------- 1 file changed, 72 insertions(+), 99 deletions(-) diff --git a/mobile/lib/services/machine_learning/face_ml/face_clustering/linear_clustering_service.dart b/mobile/lib/services/machine_learning/face_ml/face_clustering/linear_clustering_service.dart index d82db2b43f..c427cbc930 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_clustering/linear_clustering_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_clustering/linear_clustering_service.dart @@ -7,6 +7,7 @@ import "dart:typed_data"; import "package:logging/logging.dart"; import "package:photos/generated/protos/ente/common/vector.pb.dart"; import 'package:photos/services/machine_learning/face_ml/face_clustering/cosine_distance.dart'; +import "package:photos/services/machine_learning/face_ml/face_ml_result.dart"; import "package:synchronized/synchronized.dart"; class FaceInfo { @@ -15,10 +16,12 @@ class FaceInfo { int? clusterId; String? closestFaceId; int? closestDist; + int? fileCreationTime; FaceInfo({ required this.faceID, required this.embedding, this.clusterId, + this.fileCreationTime, }); } @@ -31,7 +34,6 @@ class FaceLinearClustering { final Duration _inactivityDuration = const Duration(seconds: 30); int _activeTasks = 0; - final _initLock = Lock(); late Isolate _isolate; @@ -151,8 +153,8 @@ class FaceLinearClustering { _resetInactivityTimer(); } else { _logger.info( - 'Clustering Isolate has been inactive for ${_inactivityDuration.inSeconds} seconds with no tasks running. Killing isolate.', - ); + 'Clustering Isolate has been inactive for ${_inactivityDuration.inSeconds} seconds with no tasks running. Killing isolate.', + ); dispose(); } }); @@ -220,6 +222,8 @@ class FaceLinearClustering { log( "[ClusterIsolate] ${DateTime.now()} Copied to isolate ${x.length} faces", ); + + // Organize everything into a list of FaceInfo objects final List faceInfos = []; for (final entry in x.entries) { faceInfos.add( @@ -249,59 +253,61 @@ class FaceLinearClustering { } // Sort the faceInfos such that the ones with null clusterId are at the end - faceInfos.sort((a, b) { - if (a.clusterId == null && b.clusterId == null) { - return 0; - } else if (a.clusterId == null) { - return 1; - } else if (b.clusterId == null) { - return -1; - } else { - return 0; - } - }); - // Count the amount of null values at the end - int nullCount = 0; - for (final faceInfo in faceInfos.reversed) { + final List facesWithClusterID = []; + final List facesWithoutClusterID = []; + for (final FaceInfo faceInfo in faceInfos) { if (faceInfo.clusterId == null) { - nullCount++; + facesWithoutClusterID.add(faceInfo); } else { - break; + facesWithClusterID.add(faceInfo); } } + final sortedFaceInfos = []; + sortedFaceInfos.addAll(facesWithClusterID); + sortedFaceInfos.addAll(facesWithoutClusterID); + log( - "[ClusterIsolate] ${DateTime.now()} Clustering $nullCount new faces without clusterId, and ${faceInfos.length - nullCount} faces with clusterId", + "[ClusterIsolate] ${DateTime.now()} Clustering ${facesWithoutClusterID.length} new faces without clusterId, and ${facesWithClusterID.length} faces with clusterId", ); - for (final clusteredFaceInfo - in faceInfos.sublist(0, faceInfos.length - nullCount)) { - assert(clusteredFaceInfo.clusterId != null); + + // Make sure the first face has a clusterId + final int totalFaces = sortedFaceInfos.length; + int clusterID = 1; + if (sortedFaceInfos.isNotEmpty) { + if (sortedFaceInfos.first.clusterId == null) { + sortedFaceInfos.first.clusterId = clusterID; + } else { + clusterID = sortedFaceInfos.first.clusterId!; + } + } else { + return {}; } - final int totalFaces = faceInfos.length; - int clusterID = 1; - if (faceInfos.isNotEmpty) { - faceInfos.first.clusterId = clusterID; - } + // Start actual clustering log( "[ClusterIsolate] ${DateTime.now()} Processing $totalFaces faces", ); + final Map newFaceIdToCluster = {}; final stopwatchClustering = Stopwatch()..start(); for (int i = 1; i < totalFaces; i++) { // Incremental clustering, so we can skip faces that already have a clusterId - if (faceInfos[i].clusterId != null) { - clusterID = max(clusterID, faceInfos[i].clusterId!); + if (sortedFaceInfos[i].clusterId != null) { + clusterID = max(clusterID, sortedFaceInfos[i].clusterId!); + if (i % 250 == 0) { + log("[ClusterIsolate] ${DateTime.now()} First $i faces already had a clusterID"); + } continue; } - final currentEmbedding = faceInfos[i].embedding; + final currentEmbedding = sortedFaceInfos[i].embedding; int closestIdx = -1; double closestDistance = double.infinity; if (i % 250 == 0) { log("[ClusterIsolate] ${DateTime.now()} Processing $i faces"); } - for (int j = 0; j < i; j++) { + for (int j = i - 1; j >= 0; j--) { final double distance = cosineDistForNormVectors( currentEmbedding, - faceInfos[j].embedding, + sortedFaceInfos[j].embedding, ); if (distance < closestDistance) { closestDistance = distance; @@ -310,42 +316,43 @@ class FaceLinearClustering { } if (closestDistance < recommendedDistanceThreshold) { - if (faceInfos[closestIdx].clusterId == null) { + if (sortedFaceInfos[closestIdx].clusterId == null) { // Ideally this should never happen, but just in case log it log( - " [ClusterIsolate] ${DateTime.now()} Found new cluster $clusterID", + " [ClusterIsolate] [WARNING] ${DateTime.now()} Found new cluster $clusterID", ); clusterID++; - faceInfos[closestIdx].clusterId = clusterID; + sortedFaceInfos[closestIdx].clusterId = clusterID; + newFaceIdToCluster[sortedFaceInfos[closestIdx].faceID] = clusterID; } - faceInfos[i].clusterId = faceInfos[closestIdx].clusterId; + sortedFaceInfos[i].clusterId = sortedFaceInfos[closestIdx].clusterId; + newFaceIdToCluster[sortedFaceInfos[i].faceID] = + sortedFaceInfos[closestIdx].clusterId!; } else { clusterID++; - faceInfos[i].clusterId = clusterID; + sortedFaceInfos[i].clusterId = clusterID; + newFaceIdToCluster[sortedFaceInfos[i].faceID] = clusterID; } } - final Map result = {}; - for (final faceInfo in faceInfos) { - result[faceInfo.faceID] = faceInfo.clusterId!; - } + stopwatchClustering.stop(); log( - ' [ClusterIsolate] ${DateTime.now()} Clustering for ${faceInfos.length} embeddings (${faceInfos[0].embedding.length} size) executed in ${stopwatchClustering.elapsedMilliseconds}ms, clusters $clusterID', + ' [ClusterIsolate] ${DateTime.now()} Clustering for ${sortedFaceInfos.length} embeddings (${sortedFaceInfos[0].embedding.length} size) executed in ${stopwatchClustering.elapsedMilliseconds}ms, clusters $clusterID', ); - // return result; - // NOTe: The main clustering logic is done, the following is just filtering and logging - final input = x; - final faceIdToCluster = result; - stopwatchClustering.reset(); - stopwatchClustering.start(); + // analyze the results + FaceLinearClustering._analyzeClusterResults(sortedFaceInfos); - final Set newFaceIds = {}; - input.forEach((key, value) { - if (value.$1 == null) { - newFaceIds.add(key); - } - }); + return newFaceIdToCluster; + } + + static void _analyzeClusterResults(List sortedFaceInfos) { + final stopwatch = Stopwatch()..start(); + + final Map faceIdToCluster = {}; + for (final faceInfo in sortedFaceInfos) { + faceIdToCluster[faceInfo.faceID] = faceInfo.clusterId!; + } // Find faceIDs that are part of a cluster which is larger than 5 and are new faceIDs final Map clusterIdToSize = {}; @@ -356,12 +363,6 @@ class FaceLinearClustering { clusterIdToSize[value] = 1; } }); - final Map faceIdToClusterFiltered = {}; - for (final entry in faceIdToCluster.entries) { - if (clusterIdToSize[entry.value]! > 0 && newFaceIds.contains(entry.key)) { - faceIdToClusterFiltered[entry.key] = entry.value; - } - } // print top 10 cluster ids and their sizes based on the internal cluster id final clusterIds = faceIdToCluster.values.toSet(); @@ -369,7 +370,7 @@ class FaceLinearClustering { return faceIdToCluster.values.where((id) => id == clusterId).length; }).toList(); clusterSizes.sort(); - // find clusters whose size is graeter than 1 + // find clusters whose size is greater than 1 int oneClusterCount = 0; int moreThan5Count = 0; int moreThan10Count = 0; @@ -377,57 +378,29 @@ class FaceLinearClustering { int moreThan50Count = 0; int moreThan100Count = 0; - // for (int i = 0; i < clusterSizes.length; i++) { - // if (clusterSizes[i] > 100) { - // moreThan100Count++; - // } else if (clusterSizes[i] > 50) { - // moreThan50Count++; - // } else if (clusterSizes[i] > 20) { - // moreThan20Count++; - // } else if (clusterSizes[i] > 10) { - // moreThan10Count++; - // } else if (clusterSizes[i] > 5) { - // moreThan5Count++; - // } else if (clusterSizes[i] == 1) { - // oneClusterCount++; - // } - // } for (int i = 0; i < clusterSizes.length; i++) { if (clusterSizes[i] > 100) { moreThan100Count++; - } - if (clusterSizes[i] > 50) { + } else if (clusterSizes[i] > 50) { moreThan50Count++; - } - if (clusterSizes[i] > 20) { + } else if (clusterSizes[i] > 20) { moreThan20Count++; - } - if (clusterSizes[i] > 10) { + } else if (clusterSizes[i] > 10) { moreThan10Count++; - } - if (clusterSizes[i] > 5) { + } else if (clusterSizes[i] > 5) { moreThan5Count++; - } - if (clusterSizes[i] == 1) { + } else if (clusterSizes[i] == 1) { oneClusterCount++; } } + // print the metrics log( - '[ClusterIsolate] Total clusters ${clusterIds.length}, ' - 'oneClusterCount $oneClusterCount, ' - 'moreThan5Count $moreThan5Count, ' - 'moreThan10Count $moreThan10Count, ' - 'moreThan20Count $moreThan20Count, ' - 'moreThan50Count $moreThan50Count, ' - 'moreThan100Count $moreThan100Count', + "[ClusterIsolate] Total clusters ${clusterIds.length}: \n oneClusterCount $oneClusterCount \n moreThan5Count $moreThan5Count \n moreThan10Count $moreThan10Count \n moreThan20Count $moreThan20Count \n moreThan50Count $moreThan50Count \n moreThan100Count $moreThan100Count", ); - stopwatchClustering.stop(); + stopwatch.stop(); log( - "[ClusterIsolate] Clustering additional steps took ${stopwatchClustering.elapsedMilliseconds} ms", + "[ClusterIsolate] Clustering additional analysis took ${stopwatch.elapsedMilliseconds} ms", ); - - // log('Top clusters count ${clusterSizes.reversed.take(10).toList()}'); - return faceIdToClusterFiltered; } } From f94aa400bf0d6b0c3624add187f330c854d0d299 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Thu, 21 Mar 2024 18:07:12 +0530 Subject: [PATCH 17/22] [mob] Minor changes to clustering --- .../face_ml/face_clustering/linear_clustering_service.dart | 3 ++- .../services/machine_learning/face_ml/face_ml_service.dart | 7 ++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/mobile/lib/services/machine_learning/face_ml/face_clustering/linear_clustering_service.dart b/mobile/lib/services/machine_learning/face_ml/face_clustering/linear_clustering_service.dart index c427cbc930..5e0855eb47 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_clustering/linear_clustering_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_clustering/linear_clustering_service.dart @@ -131,12 +131,13 @@ class FaceLinearClustering { final errorStackTrace = receivedMessage['stackTrace']; final exception = Exception(errorMessage); final stackTrace = StackTrace.fromString(errorStackTrace); + _activeTasks--; completer.completeError(exception, stackTrace); } else { + _activeTasks--; completer.complete(receivedMessage); } }); - _activeTasks--; return completer.future; } diff --git a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart index 4ca13b8d9d..81682e30a4 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart @@ -374,11 +374,16 @@ class FaceMlService { final faceIdToEmbedding = await FaceMLDataDB.instance.getFaceEmbeddingMap( minScore: minFaceScore, ); - _logger.info('read embeddings ${faceIdToEmbedding.length} '); + final gotFaceEmbeddingsTime = DateTime.now(); + _logger.info( + 'read embeddings ${faceIdToEmbedding.length} in ${gotFaceEmbeddingsTime.difference(clusterStartTime).inMilliseconds} ms', + ); // Read the creation times from Files DB, in a map from fileID to creation time final fileIDToCreationTime = await FilesDB.instance.getFileIDToCreationTime(); + _logger.info('read creation times from FilesDB in ' + '${DateTime.now().difference(gotFaceEmbeddingsTime).inMilliseconds} ms'); // Cluster the embeddings using the linear clustering algorithm, returning a map from faceID to clusterID final faceIdToCluster = await FaceLinearClustering.instance.predict( From b8813161a1680b2b8d0e6311e17b4bdbc260968d Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Thu, 21 Mar 2024 18:23:21 +0530 Subject: [PATCH 18/22] Add faces count --- mobile/lib/face/db.dart | 8 ++++++++ .../debug/face_debug_section_widget.dart | 17 +++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/mobile/lib/face/db.dart b/mobile/lib/face/db.dart index 585f7d7bb2..1cbaf165bd 100644 --- a/mobile/lib/face/db.dart +++ b/mobile/lib/face/db.dart @@ -419,6 +419,14 @@ class FaceMLDataDB { return result; } + Future getTotalFaceCount() async { + final db = await instance.database; + final List> maps = await db.rawQuery( + 'SELECT COUNT(*) as count FROM $facesTable WHERE $faceScore > $kMinHighQualityFaceScore AND $faceBlur > $kLaplacianThreshold', + ); + return maps.first['count'] as int; + } + Future resetClusterIDs() async { final db = await instance.database; await db.update( diff --git a/mobile/lib/ui/settings/debug/face_debug_section_widget.dart b/mobile/lib/ui/settings/debug/face_debug_section_widget.dart index ce28f0f04a..ca4d3ddec1 100644 --- a/mobile/lib/ui/settings/debug/face_debug_section_widget.dart +++ b/mobile/lib/ui/settings/debug/face_debug_section_widget.dart @@ -93,6 +93,23 @@ class _FaceDebugSectionWidgetState extends State { } }, ), + MenuItemWidget( + captionedTextWidget: FutureBuilder( + future: FaceMLDataDB.instance.getTotalFaceCount(), + builder: (context, snapshot) { + if (snapshot.hasData) { + return CaptionedTextWidget( + title: "${snapshot.data!} high quality faces", + ); + } + return const SizedBox.shrink(); + }, + ), + pressedColor: getEnteColorScheme(context).fillFaint, + trailingIcon: Icons.chevron_right_outlined, + trailingIconIsMuted: true, + onTap: () async {}, + ), MenuItemWidget( captionedTextWidget: const CaptionedTextWidget( title: "Analyze file ID 25728869", From 005ab0814faab16dbf2d0eb59253bb0add379e24 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Thu, 21 Mar 2024 18:59:45 +0530 Subject: [PATCH 19/22] [mob] Remove empty CTA icon when there are people --- mobile/lib/ui/viewer/search_tab/people_section.dart | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/mobile/lib/ui/viewer/search_tab/people_section.dart b/mobile/lib/ui/viewer/search_tab/people_section.dart index 25ea6bc6ba..e9d24a9e73 100644 --- a/mobile/lib/ui/viewer/search_tab/people_section.dart +++ b/mobile/lib/ui/viewer/search_tab/people_section.dart @@ -170,7 +170,6 @@ class SearchExampleRow extends StatelessWidget { ), ); }); - scrollableExamples.add(SearchSectionCTAIcon(sectionType)); return SizedBox( child: SingleChildScrollView( physics: const BouncingScrollPhysics(), @@ -237,7 +236,9 @@ class SearchExample extends StatelessWidget { child: searchResult.previewThumbnail() != null ? Hero( tag: heroTag, - child: ClipOval( + child: ClipRRect( + borderRadius: + const BorderRadius.all(Radius.elliptical(16, 12)), child: searchResult.type() != ResultType.faces ? ThumbnailWidget( searchResult.previewThumbnail()!, @@ -246,7 +247,9 @@ class SearchExample extends StatelessWidget { : FaceSearchResult(searchResult, heroTag), ), ) - : const ClipOval( + : const ClipRRect( + borderRadius: + BorderRadius.all(Radius.elliptical(16, 12)), child: NoThumbnailWidget( addBorder: false, ), From 85f76497b43c232a78f0aef9b98bd65891e44afa Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Fri, 22 Mar 2024 11:31:33 +0530 Subject: [PATCH 20/22] More debug info --- .../debug/face_debug_section_widget.dart | 73 ++++++++++--------- 1 file changed, 40 insertions(+), 33 deletions(-) diff --git a/mobile/lib/ui/settings/debug/face_debug_section_widget.dart b/mobile/lib/ui/settings/debug/face_debug_section_widget.dart index ca4d3ddec1..3fccae2977 100644 --- a/mobile/lib/ui/settings/debug/face_debug_section_widget.dart +++ b/mobile/lib/ui/settings/debug/face_debug_section_widget.dart @@ -8,9 +8,10 @@ import "package:photos/events/people_changed_event.dart"; import "package:photos/extensions/stop_watch.dart"; import "package:photos/face/db.dart"; import "package:photos/face/model/person.dart"; +import "package:photos/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart"; import 'package:photos/services/machine_learning/face_ml/face_ml_service.dart'; import 'package:photos/services/machine_learning/face_ml/feedback/cluster_feedback.dart'; -import "package:photos/services/search_service.dart"; +// import "package:photos/services/search_service.dart"; import 'package:photos/theme/ente_theme.dart'; import 'package:photos/ui/components/captioned_text_widget.dart'; import 'package:photos/ui/components/expandable_menu_item_widget.dart'; @@ -108,41 +109,47 @@ class _FaceDebugSectionWidgetState extends State { pressedColor: getEnteColorScheme(context).fillFaint, trailingIcon: Icons.chevron_right_outlined, trailingIconIsMuted: true, - onTap: () async {}, - ), - MenuItemWidget( - captionedTextWidget: const CaptionedTextWidget( - title: "Analyze file ID 25728869", - ), - pressedColor: getEnteColorScheme(context).fillFaint, - trailingIcon: Icons.chevron_right_outlined, - trailingIconIsMuted: true, onTap: () async { - try { - final enteFile = await SearchService.instance.getAllFiles().then( - (value) => value.firstWhere( - (element) => element.uploadedFileID == 25728869, - ), - ); - _logger.info( - 'File with ID ${enteFile.uploadedFileID} has name ${enteFile.displayName}', - ); - FaceMlService.instance.isImageIndexRunning = true; - final result = await FaceMlService.instance - .analyzeImageInSingleIsolate(enteFile); - if (result != null) { - final resultJson = result.toJsonString(); - _logger.info('result: $resultJson'); - } - FaceMlService.instance.isImageIndexRunning = false; - } catch (e, s) { - _logger.severe('indexing failed ', e, s); - await showGenericErrorDialog(context: context, error: e); - } finally { - FaceMlService.instance.isImageIndexRunning = false; - } + final faces75 = await FaceMLDataDB.instance + .getTotalFaceCount(minFaceScore: 0.75); + final faces78 = await FaceMLDataDB.instance + .getTotalFaceCount(minFaceScore: kMinHighQualityFaceScore); + showShortToast(context, "Faces75: $faces75, Faces78: $faces78"); }, ), + // MenuItemWidget( + // captionedTextWidget: const CaptionedTextWidget( + // title: "Analyze file ID 25728869", + // ), + // pressedColor: getEnteColorScheme(context).fillFaint, + // trailingIcon: Icons.chevron_right_outlined, + // trailingIconIsMuted: true, + // onTap: () async { + // try { + // final enteFile = await SearchService.instance.getAllFiles().then( + // (value) => value.firstWhere( + // (element) => element.uploadedFileID == 25728869, + // ), + // ); + // _logger.info( + // 'File with ID ${enteFile.uploadedFileID} has name ${enteFile.displayName}', + // ); + // FaceMlService.instance.isImageIndexRunning = true; + // final result = await FaceMlService.instance + // .analyzeImageInSingleIsolate(enteFile); + // if (result != null) { + // final resultJson = result.toJsonString(); + // _logger.info('result: $resultJson'); + // } + // FaceMlService.instance.isImageIndexRunning = false; + // } catch (e, s) { + // _logger.severe('indexing failed ', e, s); + // await showGenericErrorDialog(context: context, error: e); + // } finally { + // FaceMlService.instance.isImageIndexRunning = false; + // } + // }, + // ), MenuItemWidget( captionedTextWidget: const CaptionedTextWidget( title: "Run Clustering", From b1b3bcc5340dc8ea03b496b1fc46edb9f53948be Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Fri, 22 Mar 2024 11:49:23 +0530 Subject: [PATCH 21/22] Support for clustering in buckets --- mobile/lib/face/db.dart | 16 +-- .../face_ml/face_ml_service.dart | 124 +++++++++++++----- 2 files changed, 96 insertions(+), 44 deletions(-) diff --git a/mobile/lib/face/db.dart b/mobile/lib/face/db.dart index 1cbaf165bd..b14a58d2a6 100644 --- a/mobile/lib/face/db.dart +++ b/mobile/lib/face/db.dart @@ -343,15 +343,13 @@ class FaceMLDataDB { Future> getFaceEmbeddingMap({ double minScore = kMinHighQualityFaceScore, int minClarity = kLaplacianThreshold, - int maxRows = 20000, + int maxFaces = 20000, + int offset = 0, + int batchSize = 10000, }) async { _logger.info('reading as float'); final db = await instance.database; - // Define the batch size - const batchSize = 10000; - int offset = 0; - final Map result = {}; while (true) { // Query a batch of rows @@ -373,7 +371,7 @@ class FaceMLDataDB { result[faceID] = (map[faceClusterId] as int?, map[faceEmbeddingBlob] as Uint8List); } - if (result.length >= 20000) { + if (result.length >= maxFaces) { break; } offset += batchSize; @@ -419,10 +417,12 @@ class FaceMLDataDB { return result; } - Future getTotalFaceCount() async { + Future getTotalFaceCount({ + double minFaceScore = kMinHighQualityFaceScore, + }) async { final db = await instance.database; final List> maps = await db.rawQuery( - 'SELECT COUNT(*) as count FROM $facesTable WHERE $faceScore > $kMinHighQualityFaceScore AND $faceBlur > $kLaplacianThreshold', + 'SELECT COUNT(*) as count FROM $facesTable WHERE $faceScore > $minFaceScore AND $faceBlur > $kLaplacianThreshold', ); return maps.first['count'] as int; } diff --git a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart index 81682e30a4..3ef326e7bc 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart @@ -365,48 +365,98 @@ class FaceMlService { Future clusterAllImages({ double minFaceScore = kMinHighQualityFaceScore, + bool clusterInBuckets = false, }) async { _logger.info("`clusterAllImages()` called"); try { - // Read all the embeddings from the database, in a map from faceID to embedding - final clusterStartTime = DateTime.now(); - final faceIdToEmbedding = await FaceMLDataDB.instance.getFaceEmbeddingMap( - minScore: minFaceScore, - ); - final gotFaceEmbeddingsTime = DateTime.now(); - _logger.info( - 'read embeddings ${faceIdToEmbedding.length} in ${gotFaceEmbeddingsTime.difference(clusterStartTime).inMilliseconds} ms', - ); + if (clusterInBuckets) { + // Get a sense of the total number of faces in the database + final int totalFaces = await FaceMLDataDB.instance + .getTotalFaceCount(minFaceScore: minFaceScore); - // Read the creation times from Files DB, in a map from fileID to creation time - final fileIDToCreationTime = - await FilesDB.instance.getFileIDToCreationTime(); - _logger.info('read creation times from FilesDB in ' - '${DateTime.now().difference(gotFaceEmbeddingsTime).inMilliseconds} ms'); + // read the creation times from Files DB, in a map from fileID to creation time + final fileIDToCreationTime = + await FilesDB.instance.getFileIDToCreationTime(); - // Cluster the embeddings using the linear clustering algorithm, returning a map from faceID to clusterID - final faceIdToCluster = await FaceLinearClustering.instance.predict( - faceIdToEmbedding, - fileIDToCreationTime: fileIDToCreationTime, - ); - if (faceIdToCluster == null) { - _logger.warning("faceIdToCluster is null"); - return; + const int bucketSize = 10000; + const int offsetIncrement = 7500; + const int batchSize = 5000; + int offset = 0; + + while (true) { + final faceIdToEmbeddingBucket = + await FaceMLDataDB.instance.getFaceEmbeddingMap( + minScore: minFaceScore, + maxFaces: bucketSize, + offset: offset, + batchSize: batchSize, + ); + if (faceIdToEmbeddingBucket.isEmpty) { + break; + } + if (offset > totalFaces) { + _logger.warning( + 'offset > totalFaces, this should ideally not happen. offset: $offset, totalFaces: $totalFaces', + ); + break; + } + + final faceIdToCluster = await FaceLinearClustering.instance.predict( + faceIdToEmbeddingBucket, + fileIDToCreationTime: fileIDToCreationTime, + ); + if (faceIdToCluster == null) { + _logger.warning("faceIdToCluster is null"); + return; + } + + await FaceMLDataDB.instance + .updatePersonIDForFaceIDIFNotSet(faceIdToCluster); + + offset += offsetIncrement; + } + } else { + // Read all the embeddings from the database, in a map from faceID to embedding + final clusterStartTime = DateTime.now(); + final faceIdToEmbedding = + await FaceMLDataDB.instance.getFaceEmbeddingMap( + minScore: minFaceScore, + ); + final gotFaceEmbeddingsTime = DateTime.now(); + _logger.info( + 'read embeddings ${faceIdToEmbedding.length} in ${gotFaceEmbeddingsTime.difference(clusterStartTime).inMilliseconds} ms', + ); + + // Read the creation times from Files DB, in a map from fileID to creation time + final fileIDToCreationTime = + await FilesDB.instance.getFileIDToCreationTime(); + _logger.info('read creation times from FilesDB in ' + '${DateTime.now().difference(gotFaceEmbeddingsTime).inMilliseconds} ms'); + + // Cluster the embeddings using the linear clustering algorithm, returning a map from faceID to clusterID + final faceIdToCluster = await FaceLinearClustering.instance.predict( + faceIdToEmbedding, + fileIDToCreationTime: fileIDToCreationTime, + ); + if (faceIdToCluster == null) { + _logger.warning("faceIdToCluster is null"); + return; + } + final clusterDoneTime = DateTime.now(); + _logger.info( + 'done with clustering ${faceIdToEmbedding.length} in ${clusterDoneTime.difference(clusterStartTime).inSeconds} seconds ', + ); + + // Store the updated clusterIDs in the database + _logger.info( + 'Updating ${faceIdToCluster.length} FaceIDs with clusterIDs in the DB', + ); + await FaceMLDataDB.instance + .updatePersonIDForFaceIDIFNotSet(faceIdToCluster); + _logger.info('Done updating FaceIDs with clusterIDs in the DB, in ' + '${DateTime.now().difference(clusterDoneTime).inSeconds} seconds'); } - final clusterDoneTime = DateTime.now(); - _logger.info( - 'done with clustering ${faceIdToEmbedding.length} in ${clusterDoneTime.difference(clusterStartTime).inSeconds} seconds ', - ); - - // Store the updated clusterIDs in the database - _logger.info( - 'Updating ${faceIdToCluster.length} FaceIDs with clusterIDs in the DB', - ); - await FaceMLDataDB.instance - .updatePersonIDForFaceIDIFNotSet(faceIdToCluster); - _logger.info('Done updating FaceIDs with clusterIDs in the DB, in ' - '${DateTime.now().difference(clusterDoneTime).inSeconds} seconds'); } catch (e, s) { _logger.severe("`clusterAllImages` failed", e, s); } @@ -522,7 +572,9 @@ class FaceMlService { _logger.info( "indexAllImages() analyzed $fileAnalyzedCount images, cooldown for 1 minute", ); - await Future.delayed(const Duration(minutes: 1)); + await Future.delayed(const Duration(minutes: 1), () { + _logger.info("indexAllImages() cooldown finished"); + }); } } From a09b71cc15cadb4c536febf392d2d8e5da33bb0a Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Sat, 23 Mar 2024 17:02:22 +0530 Subject: [PATCH 22/22] [mob] Faster face cropping method --- .../ui/viewer/file_details/face_widget.dart | 139 +++++++++++++----- .../people/cropped_face_image_view.dart | 117 +++++++++++++++ 2 files changed, 223 insertions(+), 33 deletions(-) create mode 100644 mobile/lib/ui/viewer/people/cropped_face_image_view.dart diff --git a/mobile/lib/ui/viewer/file_details/face_widget.dart b/mobile/lib/ui/viewer/file_details/face_widget.dart index a7045f92c2..42ae6b0a10 100644 --- a/mobile/lib/ui/viewer/file_details/face_widget.dart +++ b/mobile/lib/ui/viewer/file_details/face_widget.dart @@ -1,4 +1,5 @@ import "dart:developer" show log; +import "dart:io" show Platform; import "dart:typed_data"; import "package:flutter/material.dart"; @@ -9,6 +10,7 @@ import 'package:photos/models/file/file.dart'; import "package:photos/services/search_service.dart"; import "package:photos/ui/viewer/file/no_thumbnail_widget.dart"; import "package:photos/ui/viewer/people/cluster_page.dart"; +import "package:photos/ui/viewer/people/cropped_face_image_view.dart"; import "package:photos/ui/viewer/people/people_page.dart"; import "package:photos/utils/face/face_box_crop.dart"; import "package:photos/utils/thumbnail_util.dart"; @@ -29,11 +31,104 @@ class FaceWidget extends StatelessWidget { @override Widget build(BuildContext context) { - return FutureBuilder( - future: getFaceCrop(), - builder: (context, snapshot) { - if (snapshot.hasData) { - final ImageProvider imageProvider = MemoryImage(snapshot.data!); + if (Platform.isIOS) { + return FutureBuilder( + future: getFaceCrop(), + builder: (context, snapshot) { + if (snapshot.hasData) { + final ImageProvider imageProvider = MemoryImage(snapshot.data!); + return GestureDetector( + onTap: () async { + log( + "FaceWidget is tapped, with person $person and clusterID $clusterID", + name: "FaceWidget", + ); + if (person == null && clusterID == null) { + return; + } + if (person != null) { + await Navigator.of(context).push( + MaterialPageRoute( + builder: (context) => PeoplePage( + person: person!, + ), + ), + ); + } else if (clusterID != null) { + final fileIdsToClusterIds = + await FaceMLDataDB.instance.getFileIdToClusterIds(); + final files = await SearchService.instance.getAllFiles(); + final clusterFiles = files + .where( + (file) => + fileIdsToClusterIds[file.uploadedFileID] + ?.contains(clusterID) ?? + false, + ) + .toList(); + await Navigator.of(context).push( + MaterialPageRoute( + builder: (context) => ClusterPage( + clusterFiles, + cluserID: clusterID!, + ), + ), + ); + } + }, + child: Column( + children: [ + ClipRRect( + borderRadius: + const BorderRadius.all(Radius.elliptical(16, 12)), + child: SizedBox( + width: 60, + height: 60, + child: Image( + image: imageProvider, + fit: BoxFit.cover, + ), + ), + ), + const SizedBox(height: 8), + if (person != null) + Text( + person!.attr.name.trim(), + style: Theme.of(context).textTheme.bodySmall, + overflow: TextOverflow.ellipsis, + maxLines: 1, + ), + ], + ), + ); + } else { + if (snapshot.connectionState == ConnectionState.waiting) { + return const ClipRRect( + borderRadius: BorderRadius.all(Radius.elliptical(16, 12)), + child: SizedBox( + width: 60, // Ensure consistent sizing + height: 60, + child: CircularProgressIndicator(), + ), + ); + } + if (snapshot.hasError) { + log('Error getting face: ${snapshot.error}'); + } + return const ClipRRect( + borderRadius: BorderRadius.all(Radius.elliptical(16, 12)), + child: SizedBox( + width: 60, // Ensure consistent sizing + height: 60, + child: NoThumbnailWidget(), + ), + ); + } + }, + ); + } else { + return Builder( + builder: (context) { return GestureDetector( onTap: () async { log( @@ -81,9 +176,9 @@ class FaceWidget extends StatelessWidget { child: SizedBox( width: 60, height: 60, - child: Image( - image: imageProvider, - fit: BoxFit.cover, + child: CroppedFaceImageView( + enteFile: file, + face: face, ), ), ), @@ -98,31 +193,9 @@ class FaceWidget extends StatelessWidget { ], ), ); - } else { - if (snapshot.connectionState == ConnectionState.waiting) { - return const ClipRRect( - borderRadius: BorderRadius.all(Radius.elliptical(16, 12)), - child: SizedBox( - width: 60, // Ensure consistent sizing - height: 60, - child: CircularProgressIndicator(), - ), - ); - } - if (snapshot.hasError) { - log('Error getting face: ${snapshot.error}'); - } - return const ClipRRect( - borderRadius: BorderRadius.all(Radius.elliptical(16, 12)), - child: SizedBox( - width: 60, // Ensure consistent sizing - height: 60, - child: NoThumbnailWidget(), - ), - ); - } - }, - ); + }, + ); + } } Future getFaceCrop() async { diff --git a/mobile/lib/ui/viewer/people/cropped_face_image_view.dart b/mobile/lib/ui/viewer/people/cropped_face_image_view.dart new file mode 100644 index 0000000000..04980098fd --- /dev/null +++ b/mobile/lib/ui/viewer/people/cropped_face_image_view.dart @@ -0,0 +1,117 @@ +import 'dart:developer' show log; +import "dart:io" show File; + +import 'package:flutter/material.dart'; +import "package:photos/face/model/face.dart"; +import "package:photos/models/file/file.dart"; +import "package:photos/ui/viewer/file/thumbnail_widget.dart"; +import "package:photos/utils/file_util.dart"; + +class CroppedFaceInfo { + final Image image; + final double scale; + final double offsetX; + final double offsetY; + + const CroppedFaceInfo({ + required this.image, + required this.scale, + required this.offsetX, + required this.offsetY, + }); +} + +class CroppedFaceImageView extends StatelessWidget { + final EnteFile enteFile; + final Face face; + + const CroppedFaceImageView({ + Key? key, + required this.enteFile, + required this.face, + }) : super(key: key); + + @override + Widget build(BuildContext context) { + return FutureBuilder( + future: getImage(), + builder: (context, snapshot) { + if (snapshot.hasData) { + return LayoutBuilder( + builder: (BuildContext context, BoxConstraints constraints) { + final Image image = snapshot.data!; + + final double viewWidth = constraints.maxWidth; + final double viewHeight = constraints.maxHeight; + + final faceBox = face.detection.box; + + final double relativeFaceCenterX = + faceBox.xMin + faceBox.width / 2; + final double relativeFaceCenterY = + faceBox.yMin + faceBox.height / 2; + + const double desiredFaceHeightRelativeToWidget = 1 / 2; + final double scale = + (1 / faceBox.height) * desiredFaceHeightRelativeToWidget; + + final double widgetCenterX = viewWidth / 2; + final double widgetCenterY = viewHeight / 2; + + final double imageAspectRatio = enteFile.width / enteFile.height; + final double widgetAspectRatio = viewWidth / viewHeight; + final double imageToWidgetRatio = + imageAspectRatio / widgetAspectRatio; + + double offsetX = + (widgetCenterX - relativeFaceCenterX * viewWidth) * scale; + double offsetY = + (widgetCenterY - relativeFaceCenterY * viewHeight) * scale; + + if (imageAspectRatio > widgetAspectRatio) { + // Landscape Image: Adjust offsetX more conservatively + offsetX = offsetX * imageToWidgetRatio; + } else { + // Portrait Image: Adjust offsetY more conservatively + offsetY = offsetY / imageToWidgetRatio; + } + + return ClipRect( + clipBehavior: Clip.antiAlias, + child: Transform.translate( + offset: Offset( + offsetX, + offsetY, + ), + child: Transform.scale( + scale: scale, + child: image, + ), + ), + ); + }, + ); + } else { + if (snapshot.hasError) { + log('Error getting cover face for person: ${snapshot.error}'); + } + return ThumbnailWidget( + enteFile, + ); + } + }, + ); + } + + Future getImage() async { + final File? ioFile = await getFile(enteFile); + if (ioFile == null) { + return null; + } + + final imageData = await ioFile.readAsBytes(); + final image = Image.memory(imageData, fit: BoxFit.cover); + + return image; + } +}