diff --git a/mobile/lib/db/files_db.dart b/mobile/lib/db/files_db.dart index 39ff832103..ac3c273e00 100644 --- a/mobile/lib/db/files_db.dart +++ b/mobile/lib/db/files_db.dart @@ -1304,6 +1304,23 @@ class FilesDB { return result; } + Future> getFileIDToCreationTime() async { + final db = await instance.database; + final rows = await db.rawQuery( + ''' + SELECT $columnUploadedFileID, $columnCreationTime + FROM $filesTable + WHERE + ($columnUploadedFileID IS NOT NULL AND $columnUploadedFileID IS NOT -1); + ''', + ); + final result = {}; + for (final row in rows) { + result[row[columnUploadedFileID] as int] = row[columnCreationTime] as int; + } + return result; + } + // getCollectionFileFirstOrLast returns the first or last uploaded file in // the collection based on the given collectionID and the order. Future getCollectionFileFirstOrLast( diff --git a/mobile/lib/db/ml_data_db.dart b/mobile/lib/db/ml_data_db.dart index 07150f09b5..46ca06466b 100644 --- a/mobile/lib/db/ml_data_db.dart +++ b/mobile/lib/db/ml_data_db.dart @@ -4,9 +4,9 @@ import 'package:logging/logging.dart'; import 'package:path/path.dart' show join; import 'package:path_provider/path_provider.dart'; import 'package:photos/models/ml/ml_typedefs.dart'; -import "package:photos/services/face_ml/face_feedback.dart/cluster_feedback.dart"; -import "package:photos/services/face_ml/face_feedback.dart/feedback_types.dart"; -import "package:photos/services/face_ml/face_ml_result.dart"; +import 'package:photos/services/machine_learning/face_ml/face_feedback.dart/cluster_feedback.dart'; +import 'package:photos/services/machine_learning/face_ml/face_feedback.dart/feedback_types.dart'; +import 'package:photos/services/machine_learning/face_ml/face_ml_result.dart'; import 'package:sqflite/sqflite.dart'; /// Stores all data for the ML-related features. The database can be accessed by `MlDataDB.instance.database`. diff --git a/mobile/lib/face/db.dart b/mobile/lib/face/db.dart index 58a54782a4..ffda695d14 100644 --- a/mobile/lib/face/db.dart +++ b/mobile/lib/face/db.dart @@ -11,7 +11,7 @@ import "package:photos/face/db_model_mappers.dart"; import "package:photos/face/model/face.dart"; import "package:photos/face/model/person.dart"; import "package:photos/models/file/file.dart"; -import "package:photos/services/face_ml/blur_detection/blur_constants.dart"; +import 'package:photos/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart'; import 'package:sqflite/sqflite.dart'; /// Stores all data for the ML-related features. The database can be accessed by `MlDataDB.instance.database`. @@ -185,7 +185,7 @@ class FaceMLDataDB { final Map result = {}; final db = await instance.database; final List> maps = await db.rawQuery( - 'SELECT $fileIDColumn, COUNT(*) as count FROM $facesTable where $faceScore > 0.8 GROUP BY $fileIDColumn', + 'SELECT $fileIDColumn, COUNT(*) as count FROM $facesTable where $faceScore > $kMinFaceDetectionScore GROUP BY $fileIDColumn', ); for (final map in maps) { @@ -228,7 +228,7 @@ class FaceMLDataDB { final clusterIDs = cluterRows.map((e) => e[cluserIDColumn] as int).toList(); final List> faceMaps = await db.rawQuery( - 'SELECT * FROM $facesTable where $faceClusterId IN (${clusterIDs.join(",")}) AND $fileIDColumn in (${fileId.join(",")}) AND $faceScore > 0.8 ORDER BY $faceScore DESC', + 'SELECT * FROM $facesTable where $faceClusterId IN (${clusterIDs.join(",")}) AND $fileIDColumn in (${fileId.join(",")}) AND $faceScore > $kMinHighQualityFaceScore ORDER BY $faceScore DESC', ); if (faceMaps.isNotEmpty) { if (avatarFileId != null) { @@ -257,7 +257,7 @@ class FaceMLDataDB { return null; } - Future> getFacesForGivenFileID(int fileUploadID) async { + Future?> getFacesForGivenFileID(int fileUploadID) async { final db = await instance.database; final List> maps = await db.query( facesTable, @@ -277,6 +277,9 @@ class FaceMLDataDB { where: '$fileIDColumn = ?', whereArgs: [fileUploadID], ); + if (maps.isEmpty) { + return null; + } return maps.map((e) => mapRowToFace(e)).toList(); } @@ -347,17 +350,17 @@ class FaceMLDataDB { /// /// Only selects faces with score greater than [minScore] and blur score greater than [minClarity] Future> getFaceEmbeddingMap({ - double minScore = 0.78, + double minScore = kMinHighQualityFaceScore, int minClarity = kLaplacianThreshold, - int maxRows = 10000, + int maxFaces = 20000, + int offset = 0, + int batchSize = 10000, }) async { - _logger.info('reading as float'); + _logger.info( + 'reading as float offset: $offset, maxFaces: $maxFaces, batchSize: $batchSize', + ); final db = await instance.database; - // Define the batch size - const batchSize = 10000; - int offset = 0; - final Map result = {}; while (true) { // Query a batch of rows @@ -379,7 +382,7 @@ class FaceMLDataDB { result[faceID] = (map[faceClusterId] as int?, map[faceEmbeddingBlob] as Uint8List); } - if (result.length >= maxRows) { + if (result.length >= maxFaces) { break; } offset += batchSize; @@ -404,7 +407,7 @@ class FaceMLDataDB { facesTable, columns: [faceIDColumn, faceEmbeddingBlob], where: - '$faceScore > 0.8 AND $faceBlur > $kLaplacianThreshold AND $fileIDColumn IN (${fileIDs.join(",")})', + '$faceScore > $kMinHighQualityFaceScore AND $faceBlur > $kLaplacianThreshold AND $fileIDColumn IN (${fileIDs.join(",")})', limit: batchSize, offset: offset, orderBy: '$faceIDColumn DESC', @@ -425,6 +428,16 @@ class FaceMLDataDB { return result; } + Future getTotalFaceCount({ + double minFaceScore = kMinHighQualityFaceScore, + }) async { + final db = await instance.database; + final List> maps = await db.rawQuery( + 'SELECT COUNT(*) as count FROM $facesTable WHERE $faceScore > $minFaceScore AND $faceBlur > $kLaplacianThreshold', + ); + return maps.first['count'] as int; + } + Future resetClusterIDs() async { final db = await instance.database; await db.update( diff --git a/mobile/lib/face/db_fields.dart b/mobile/lib/face/db_fields.dart index a1f185e7ee..e43747f682 100644 --- a/mobile/lib/face/db_fields.dart +++ b/mobile/lib/face/db_fields.dart @@ -1,5 +1,5 @@ // Faces Table Fields & Schema Queries -import "package:photos/services/face_ml/blur_detection/blur_constants.dart"; +import 'package:photos/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart'; const facesTable = 'faces'; const fileIDColumn = 'file_id'; diff --git a/mobile/lib/face/model/face.dart b/mobile/lib/face/model/face.dart index 49612c3d73..0df0987dff 100644 --- a/mobile/lib/face/model/face.dart +++ b/mobile/lib/face/model/face.dart @@ -1,5 +1,5 @@ import "package:photos/face/model/detection.dart"; -import "package:photos/services/face_ml/blur_detection/blur_constants.dart"; +import 'package:photos/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart'; class Face { final int fileID; @@ -11,6 +11,10 @@ class Face { bool get isBlurry => blur < kLaplacianThreshold; + bool get hasHighScore => score > kMinHighQualityFaceScore; + + bool get isHighQuality => (!isBlurry) && hasHighScore; + Face( this.faceID, this.fileID, @@ -20,6 +24,17 @@ class Face { this.blur, ); + factory Face.empty(int fileID, {bool error = false}) { + return Face( + "$fileID-0", + fileID, + [], + error ? -1.0 : 0.0, + Detection.empty(), + 0.0, + ); + } + factory Face.fromJson(Map json) { return Face( json['faceID'] as String, diff --git a/mobile/lib/main.dart b/mobile/lib/main.dart index d800336dfe..cc0bb60b15 100644 --- a/mobile/lib/main.dart +++ b/mobile/lib/main.dart @@ -25,13 +25,13 @@ import 'package:photos/services/app_lifecycle_service.dart'; import 'package:photos/services/billing_service.dart'; import 'package:photos/services/collections_service.dart'; import "package:photos/services/entity_service.dart"; -import "package:photos/services/face_ml/face_ml_service.dart"; import 'package:photos/services/favorites_service.dart'; import 'package:photos/services/feature_flag_service.dart'; import 'package:photos/services/home_widget_service.dart'; import 'package:photos/services/local_file_update_service.dart'; import 'package:photos/services/local_sync_service.dart'; import "package:photos/services/location_service.dart"; +import 'package:photos/services/machine_learning/face_ml/face_ml_service.dart'; import 'package:photos/services/machine_learning/file_ml/remote_fileml_service.dart'; import "package:photos/services/machine_learning/machine_learning_controller.dart"; import 'package:photos/services/machine_learning/semantic_search/semantic_search_service.dart'; diff --git a/mobile/lib/services/face_ml/blur_detection/blur_constants.dart b/mobile/lib/services/face_ml/blur_detection/blur_constants.dart deleted file mode 100644 index 4d770162cc..0000000000 --- a/mobile/lib/services/face_ml/blur_detection/blur_constants.dart +++ /dev/null @@ -1,2 +0,0 @@ -const kLaplacianThreshold = 10; -const kLapacianDefault = 10000.0; diff --git a/mobile/lib/services/face_ml/face_ml_service.dart b/mobile/lib/services/face_ml/face_ml_service.dart index dd033da570..162b77e975 100644 --- a/mobile/lib/services/face_ml/face_ml_service.dart +++ b/mobile/lib/services/face_ml/face_ml_service.dart @@ -34,6 +34,7 @@ import "package:photos/services/face_ml/face_embedding/face_embedding_exceptions import 'package:photos/services/face_ml/face_embedding/onnx_face_embedding.dart'; import "package:photos/services/face_ml/face_ml_exceptions.dart"; import "package:photos/services/face_ml/face_ml_result.dart"; +import "package:photos/services/machine_learning/face_ml/face_clustering/linear_clustering_service.dart"; import 'package:photos/services/machine_learning/file_ml/file_ml.dart'; import 'package:photos/services/machine_learning/file_ml/remote_fileml_service.dart'; import "package:photos/services/search_service.dart"; diff --git a/mobile/lib/services/face_ml/face_alignment/alignment_result.dart b/mobile/lib/services/machine_learning/face_ml/face_alignment/alignment_result.dart similarity index 100% rename from mobile/lib/services/face_ml/face_alignment/alignment_result.dart rename to mobile/lib/services/machine_learning/face_ml/face_alignment/alignment_result.dart diff --git a/mobile/lib/services/face_ml/face_alignment/similarity_transform.dart b/mobile/lib/services/machine_learning/face_ml/face_alignment/similarity_transform.dart similarity index 98% rename from mobile/lib/services/face_ml/face_alignment/similarity_transform.dart rename to mobile/lib/services/machine_learning/face_ml/face_alignment/similarity_transform.dart index 4ae27794b4..0d8e7ab3ae 100644 --- a/mobile/lib/services/face_ml/face_alignment/similarity_transform.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_alignment/similarity_transform.dart @@ -1,7 +1,7 @@ import 'dart:math' show atan2; import 'package:ml_linalg/linalg.dart'; import 'package:photos/extensions/ml_linalg_extensions.dart'; -import "package:photos/services/face_ml/face_alignment/alignment_result.dart"; +import 'package:photos/services/machine_learning/face_ml/face_alignment/alignment_result.dart'; /// Class to compute the similarity transform between two sets of points. /// diff --git a/mobile/lib/services/face_ml/face_clustering/cosine_distance.dart b/mobile/lib/services/machine_learning/face_ml/face_clustering/cosine_distance.dart similarity index 100% rename from mobile/lib/services/face_ml/face_clustering/cosine_distance.dart rename to mobile/lib/services/machine_learning/face_ml/face_clustering/cosine_distance.dart diff --git a/mobile/lib/services/face_ml/face_clustering/linear_clustering_service.dart b/mobile/lib/services/machine_learning/face_ml/face_clustering/linear_clustering_service.dart similarity index 64% rename from mobile/lib/services/face_ml/face_clustering/linear_clustering_service.dart rename to mobile/lib/services/machine_learning/face_ml/face_clustering/linear_clustering_service.dart index 2b36623154..5e0855eb47 100644 --- a/mobile/lib/services/face_ml/face_clustering/linear_clustering_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_clustering/linear_clustering_service.dart @@ -6,7 +6,8 @@ import "dart:typed_data"; import "package:logging/logging.dart"; import "package:photos/generated/protos/ente/common/vector.pb.dart"; -import "package:photos/services/face_ml/face_clustering/cosine_distance.dart"; +import 'package:photos/services/machine_learning/face_ml/face_clustering/cosine_distance.dart'; +import "package:photos/services/machine_learning/face_ml/face_ml_result.dart"; import "package:synchronized/synchronized.dart"; class FaceInfo { @@ -15,10 +16,12 @@ class FaceInfo { int? clusterId; String? closestFaceId; int? closestDist; + int? fileCreationTime; FaceInfo({ required this.faceID, required this.embedding, this.clusterId, + this.fileCreationTime, }); } @@ -31,7 +34,6 @@ class FaceLinearClustering { final Duration _inactivityDuration = const Duration(seconds: 30); int _activeTasks = 0; - final _initLock = Lock(); late Isolate _isolate; @@ -94,7 +96,12 @@ class FaceLinearClustering { switch (function) { case ClusterOperation.linearIncrementalClustering: final input = args['input'] as Map; - final result = FaceLinearClustering._runLinearClustering(input); + final fileIDToCreationTime = + args['fileIDToCreationTime'] as Map?; + final result = FaceLinearClustering._runLinearClustering( + input, + fileIDToCreationTime: fileIDToCreationTime, + ); sendPort.send(result); break; } @@ -124,12 +131,13 @@ class FaceLinearClustering { final errorStackTrace = receivedMessage['stackTrace']; final exception = Exception(errorMessage); final stackTrace = StackTrace.fromString(errorStackTrace); + _activeTasks--; completer.completeError(exception, stackTrace); } else { + _activeTasks--; completer.complete(receivedMessage); } }); - _activeTasks--; return completer.future; } @@ -146,8 +154,8 @@ class FaceLinearClustering { _resetInactivityTimer(); } else { _logger.info( - 'Clustering Isolate has been inactive for ${_inactivityDuration.inSeconds} seconds with no tasks running. Killing isolate.', - ); + 'Clustering Isolate has been inactive for ${_inactivityDuration.inSeconds} seconds with no tasks running. Killing isolate.', + ); dispose(); } }); @@ -169,8 +177,9 @@ class FaceLinearClustering { /// /// WARNING: Make sure to always input data in the same ordering, otherwise the clustering can less less deterministic. Future?> predict( - Map input, - ) async { + Map input, { + Map? fileIDToCreationTime, + }) async { if (input.isEmpty) { _logger.warning( "Clustering dataset of embeddings is empty, returning empty list.", @@ -192,7 +201,10 @@ class FaceLinearClustering { // final Map faceIdToCluster = // await _runLinearClusteringInComputer(input); final Map faceIdToCluster = await _runInIsolate( - (ClusterOperation.linearIncrementalClustering, {'input': input}), + ( + ClusterOperation.linearIncrementalClustering, + {'input': input, 'fileIDToCreationTime': fileIDToCreationTime} + ), ); // return _runLinearClusteringInComputer(input); _logger.info( @@ -205,11 +217,14 @@ class FaceLinearClustering { } static Map _runLinearClustering( - Map x, - ) { + Map x, { + Map? fileIDToCreationTime, + }) { log( "[ClusterIsolate] ${DateTime.now()} Copied to isolate ${x.length} faces", ); + + // Organize everything into a list of FaceInfo objects final List faceInfos = []; for (final entry in x.entries) { faceInfos.add( @@ -217,63 +232,83 @@ class FaceLinearClustering { faceID: entry.key, embedding: EVector.fromBuffer(entry.value.$2).values, clusterId: entry.value.$1, + fileCreationTime: + fileIDToCreationTime?[getFileIdFromFaceId(entry.key)], ), ); } - // Sort the faceInfos such that the ones with null clusterId are at the end - faceInfos.sort((a, b) { - if (a.clusterId == null && b.clusterId == null) { - return 0; - } else if (a.clusterId == null) { - return 1; - } else if (b.clusterId == null) { - return -1; - } else { - return 0; - } - }); - // Count the amount of null values at the end - int nullCount = 0; - for (final faceInfo in faceInfos.reversed) { - if (faceInfo.clusterId == null) { - nullCount++; - } else { - break; - } - } - log( - "[ClusterIsolate] ${DateTime.now()} Clustering $nullCount new faces without clusterId, and ${faceInfos.length - nullCount} faces with clusterId", - ); - for (final clusteredFaceInfo - in faceInfos.sublist(0, faceInfos.length - nullCount)) { - assert(clusteredFaceInfo.clusterId != null); + + // Sort the faceInfos based on fileCreationTime, in ascending order, so oldest faces are first + if (fileIDToCreationTime != null) { + faceInfos.sort((a, b) { + if (a.fileCreationTime == null && b.fileCreationTime == null) { + return 0; + } else if (a.fileCreationTime == null) { + return 1; + } else if (b.fileCreationTime == null) { + return -1; + } else { + return a.fileCreationTime!.compareTo(b.fileCreationTime!); + } + }); } - final int totalFaces = faceInfos.length; - int clusterID = 1; - if (faceInfos.isNotEmpty) { - faceInfos.first.clusterId = clusterID; + // Sort the faceInfos such that the ones with null clusterId are at the end + final List facesWithClusterID = []; + final List facesWithoutClusterID = []; + for (final FaceInfo faceInfo in faceInfos) { + if (faceInfo.clusterId == null) { + facesWithoutClusterID.add(faceInfo); + } else { + facesWithClusterID.add(faceInfo); + } } + final sortedFaceInfos = []; + sortedFaceInfos.addAll(facesWithClusterID); + sortedFaceInfos.addAll(facesWithoutClusterID); + + log( + "[ClusterIsolate] ${DateTime.now()} Clustering ${facesWithoutClusterID.length} new faces without clusterId, and ${facesWithClusterID.length} faces with clusterId", + ); + + // Make sure the first face has a clusterId + final int totalFaces = sortedFaceInfos.length; + int clusterID = 1; + if (sortedFaceInfos.isNotEmpty) { + if (sortedFaceInfos.first.clusterId == null) { + sortedFaceInfos.first.clusterId = clusterID; + } else { + clusterID = sortedFaceInfos.first.clusterId!; + } + } else { + return {}; + } + + // Start actual clustering log( "[ClusterIsolate] ${DateTime.now()} Processing $totalFaces faces", ); + final Map newFaceIdToCluster = {}; final stopwatchClustering = Stopwatch()..start(); for (int i = 1; i < totalFaces; i++) { // Incremental clustering, so we can skip faces that already have a clusterId - if (faceInfos[i].clusterId != null) { - clusterID = max(clusterID, faceInfos[i].clusterId!); + if (sortedFaceInfos[i].clusterId != null) { + clusterID = max(clusterID, sortedFaceInfos[i].clusterId!); + if (i % 250 == 0) { + log("[ClusterIsolate] ${DateTime.now()} First $i faces already had a clusterID"); + } continue; } - final currentEmbedding = faceInfos[i].embedding; + final currentEmbedding = sortedFaceInfos[i].embedding; int closestIdx = -1; double closestDistance = double.infinity; if (i % 250 == 0) { log("[ClusterIsolate] ${DateTime.now()} Processing $i faces"); } - for (int j = 0; j < i; j++) { + for (int j = i - 1; j >= 0; j--) { final double distance = cosineDistForNormVectors( currentEmbedding, - faceInfos[j].embedding, + sortedFaceInfos[j].embedding, ); if (distance < closestDistance) { closestDistance = distance; @@ -282,42 +317,43 @@ class FaceLinearClustering { } if (closestDistance < recommendedDistanceThreshold) { - if (faceInfos[closestIdx].clusterId == null) { + if (sortedFaceInfos[closestIdx].clusterId == null) { // Ideally this should never happen, but just in case log it log( - " [ClusterIsolate] ${DateTime.now()} Found new cluster $clusterID", + " [ClusterIsolate] [WARNING] ${DateTime.now()} Found new cluster $clusterID", ); clusterID++; - faceInfos[closestIdx].clusterId = clusterID; + sortedFaceInfos[closestIdx].clusterId = clusterID; + newFaceIdToCluster[sortedFaceInfos[closestIdx].faceID] = clusterID; } - faceInfos[i].clusterId = faceInfos[closestIdx].clusterId; + sortedFaceInfos[i].clusterId = sortedFaceInfos[closestIdx].clusterId; + newFaceIdToCluster[sortedFaceInfos[i].faceID] = + sortedFaceInfos[closestIdx].clusterId!; } else { clusterID++; - faceInfos[i].clusterId = clusterID; + sortedFaceInfos[i].clusterId = clusterID; + newFaceIdToCluster[sortedFaceInfos[i].faceID] = clusterID; } } - final Map result = {}; - for (final faceInfo in faceInfos) { - result[faceInfo.faceID] = faceInfo.clusterId!; - } + stopwatchClustering.stop(); log( - ' [ClusterIsolate] ${DateTime.now()} Clustering for ${faceInfos.length} embeddings (${faceInfos[0].embedding.length} size) executed in ${stopwatchClustering.elapsedMilliseconds}ms, clusters $clusterID', + ' [ClusterIsolate] ${DateTime.now()} Clustering for ${sortedFaceInfos.length} embeddings (${sortedFaceInfos[0].embedding.length} size) executed in ${stopwatchClustering.elapsedMilliseconds}ms, clusters $clusterID', ); - // return result; - // NOTe: The main clustering logic is done, the following is just filtering and logging - final input = x; - final faceIdToCluster = result; - stopwatchClustering.reset(); - stopwatchClustering.start(); + // analyze the results + FaceLinearClustering._analyzeClusterResults(sortedFaceInfos); - final Set newFaceIds = {}; - input.forEach((key, value) { - if (value.$1 == null) { - newFaceIds.add(key); - } - }); + return newFaceIdToCluster; + } + + static void _analyzeClusterResults(List sortedFaceInfos) { + final stopwatch = Stopwatch()..start(); + + final Map faceIdToCluster = {}; + for (final faceInfo in sortedFaceInfos) { + faceIdToCluster[faceInfo.faceID] = faceInfo.clusterId!; + } // Find faceIDs that are part of a cluster which is larger than 5 and are new faceIDs final Map clusterIdToSize = {}; @@ -328,12 +364,6 @@ class FaceLinearClustering { clusterIdToSize[value] = 1; } }); - final Map faceIdToClusterFiltered = {}; - for (final entry in faceIdToCluster.entries) { - if (clusterIdToSize[entry.value]! > 0 && newFaceIds.contains(entry.key)) { - faceIdToClusterFiltered[entry.key] = entry.value; - } - } // print top 10 cluster ids and their sizes based on the internal cluster id final clusterIds = faceIdToCluster.values.toSet(); @@ -341,7 +371,7 @@ class FaceLinearClustering { return faceIdToCluster.values.where((id) => id == clusterId).length; }).toList(); clusterSizes.sort(); - // find clusters whose size is graeter than 1 + // find clusters whose size is greater than 1 int oneClusterCount = 0; int moreThan5Count = 0; int moreThan10Count = 0; @@ -349,43 +379,29 @@ class FaceLinearClustering { int moreThan50Count = 0; int moreThan100Count = 0; - for (int i = 0; i < clusterSizes.length; i++) { if (clusterSizes[i] > 100) { moreThan100Count++; - } - if (clusterSizes[i] > 50) { + } else if (clusterSizes[i] > 50) { moreThan50Count++; - } - if (clusterSizes[i] > 20) { + } else if (clusterSizes[i] > 20) { moreThan20Count++; - } - if (clusterSizes[i] > 10) { + } else if (clusterSizes[i] > 10) { moreThan10Count++; - } - if (clusterSizes[i] > 5) { + } else if (clusterSizes[i] > 5) { moreThan5Count++; - } - if (clusterSizes[i] == 1) { + } else if (clusterSizes[i] == 1) { oneClusterCount++; } } + // print the metrics log( - '[ClusterIsolate] Total clusters ${clusterIds.length}, ' - 'oneClusterCount $oneClusterCount, ' - 'moreThan5Count $moreThan5Count, ' - 'moreThan10Count $moreThan10Count, ' - 'moreThan20Count $moreThan20Count, ' - 'moreThan50Count $moreThan50Count, ' - 'moreThan100Count $moreThan100Count', + "[ClusterIsolate] Total clusters ${clusterIds.length}: \n oneClusterCount $oneClusterCount \n moreThan5Count $moreThan5Count \n moreThan10Count $moreThan10Count \n moreThan20Count $moreThan20Count \n moreThan50Count $moreThan50Count \n moreThan100Count $moreThan100Count", ); - stopwatchClustering.stop(); + stopwatch.stop(); log( - "[ClusterIsolate] Clustering additional steps took ${stopwatchClustering.elapsedMilliseconds} ms", + "[ClusterIsolate] Clustering additional analysis took ${stopwatch.elapsedMilliseconds} ms", ); - - // log('Top clusters count ${clusterSizes.reversed.take(10).toList()}'); - return faceIdToClusterFiltered; } } diff --git a/mobile/lib/services/face_ml/face_detection/detection.dart b/mobile/lib/services/machine_learning/face_ml/face_detection/detection.dart similarity index 100% rename from mobile/lib/services/face_ml/face_detection/detection.dart rename to mobile/lib/services/machine_learning/face_ml/face_detection/detection.dart diff --git a/mobile/lib/services/face_ml/face_detection/naive_non_max_suppression.dart b/mobile/lib/services/machine_learning/face_ml/face_detection/naive_non_max_suppression.dart similarity index 94% rename from mobile/lib/services/face_ml/face_detection/naive_non_max_suppression.dart rename to mobile/lib/services/machine_learning/face_ml/face_detection/naive_non_max_suppression.dart index ca1e4aba5a..624181a669 100644 --- a/mobile/lib/services/face_ml/face_detection/naive_non_max_suppression.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_detection/naive_non_max_suppression.dart @@ -1,6 +1,6 @@ import 'dart:math' as math show max, min; -import "package:photos/services/face_ml/face_detection/detection.dart"; +import 'package:photos/services/machine_learning/face_ml/face_detection/detection.dart'; List naiveNonMaxSuppression({ required List detections, diff --git a/mobile/lib/services/face_ml/face_detection/yolov5face/onnx_face_detection.dart b/mobile/lib/services/machine_learning/face_ml/face_detection/yolov5face/onnx_face_detection.dart similarity index 98% rename from mobile/lib/services/face_ml/face_detection/yolov5face/onnx_face_detection.dart rename to mobile/lib/services/machine_learning/face_ml/face_detection/yolov5face/onnx_face_detection.dart index c67c4a3fc6..a2138fe7a5 100644 --- a/mobile/lib/services/face_ml/face_detection/yolov5face/onnx_face_detection.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_detection/yolov5face/onnx_face_detection.dart @@ -9,10 +9,10 @@ import "package:computer/computer.dart"; import 'package:flutter/material.dart'; import 'package:logging/logging.dart'; import 'package:onnxruntime/onnxruntime.dart'; -import "package:photos/services/face_ml/face_detection/detection.dart"; -import "package:photos/services/face_ml/face_detection/naive_non_max_suppression.dart"; -import "package:photos/services/face_ml/face_detection/yolov5face/yolo_face_detection_exceptions.dart"; -import "package:photos/services/face_ml/face_detection/yolov5face/yolo_filter_extract_detections.dart"; +import 'package:photos/services/machine_learning/face_ml/face_detection/detection.dart'; +import 'package:photos/services/machine_learning/face_ml/face_detection/naive_non_max_suppression.dart'; +import 'package:photos/services/machine_learning/face_ml/face_detection/yolov5face/yolo_face_detection_exceptions.dart'; +import 'package:photos/services/machine_learning/face_ml/face_detection/yolov5face/yolo_filter_extract_detections.dart'; import "package:photos/services/remote_assets_service.dart"; import "package:photos/utils/image_ml_isolate.dart"; import "package:photos/utils/image_ml_util.dart"; diff --git a/mobile/lib/services/face_ml/face_detection/yolov5face/yolo_face_detection_exceptions.dart b/mobile/lib/services/machine_learning/face_ml/face_detection/yolov5face/yolo_face_detection_exceptions.dart similarity index 100% rename from mobile/lib/services/face_ml/face_detection/yolov5face/yolo_face_detection_exceptions.dart rename to mobile/lib/services/machine_learning/face_ml/face_detection/yolov5face/yolo_face_detection_exceptions.dart diff --git a/mobile/lib/services/face_ml/face_detection/yolov5face/yolo_face_detection_options.dart b/mobile/lib/services/machine_learning/face_ml/face_detection/yolov5face/yolo_face_detection_options.dart similarity index 100% rename from mobile/lib/services/face_ml/face_detection/yolov5face/yolo_face_detection_options.dart rename to mobile/lib/services/machine_learning/face_ml/face_detection/yolov5face/yolo_face_detection_options.dart diff --git a/mobile/lib/services/face_ml/face_detection/yolov5face/yolo_filter_extract_detections.dart b/mobile/lib/services/machine_learning/face_ml/face_detection/yolov5face/yolo_filter_extract_detections.dart similarity index 96% rename from mobile/lib/services/face_ml/face_detection/yolov5face/yolo_filter_extract_detections.dart rename to mobile/lib/services/machine_learning/face_ml/face_detection/yolov5face/yolo_filter_extract_detections.dart index 168d06df88..ec546533ab 100644 --- a/mobile/lib/services/face_ml/face_detection/yolov5face/yolo_filter_extract_detections.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_detection/yolov5face/yolo_filter_extract_detections.dart @@ -1,6 +1,6 @@ import 'dart:developer' as dev show log; -import "package:photos/services/face_ml/face_detection/detection.dart"; +import 'package:photos/services/machine_learning/face_ml/face_detection/detection.dart'; List yoloOnnxFilterExtractDetections( double minScoreSigmoidThreshold, diff --git a/mobile/lib/services/face_ml/face_detection/yolov5face/yolo_model_config.dart b/mobile/lib/services/machine_learning/face_ml/face_detection/yolov5face/yolo_model_config.dart similarity index 70% rename from mobile/lib/services/face_ml/face_detection/yolov5face/yolo_model_config.dart rename to mobile/lib/services/machine_learning/face_ml/face_detection/yolov5face/yolo_model_config.dart index c803beffd4..578036bc25 100644 --- a/mobile/lib/services/face_ml/face_detection/yolov5face/yolo_model_config.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_detection/yolov5face/yolo_model_config.dart @@ -1,5 +1,5 @@ -import "package:photos/services/face_ml/face_detection/yolov5face/yolo_face_detection_options.dart"; -import "package:photos/services/face_ml/model_file.dart"; +import 'package:photos/services/machine_learning/face_ml/face_detection/yolov5face/yolo_face_detection_options.dart'; +import 'package:photos/services/machine_learning/face_ml/model_file.dart'; class YOLOModelConfig { final String modelPath; diff --git a/mobile/lib/services/face_ml/face_embedding/face_embedding_exceptions.dart b/mobile/lib/services/machine_learning/face_ml/face_embedding/face_embedding_exceptions.dart similarity index 100% rename from mobile/lib/services/face_ml/face_embedding/face_embedding_exceptions.dart rename to mobile/lib/services/machine_learning/face_ml/face_embedding/face_embedding_exceptions.dart diff --git a/mobile/lib/services/face_ml/face_embedding/face_embedding_options.dart b/mobile/lib/services/machine_learning/face_ml/face_embedding/face_embedding_options.dart similarity index 100% rename from mobile/lib/services/face_ml/face_embedding/face_embedding_options.dart rename to mobile/lib/services/machine_learning/face_ml/face_embedding/face_embedding_options.dart diff --git a/mobile/lib/services/face_ml/face_embedding/face_embedding_service.dart b/mobile/lib/services/machine_learning/face_ml/face_embedding/face_embedding_service.dart similarity index 95% rename from mobile/lib/services/face_ml/face_embedding/face_embedding_service.dart rename to mobile/lib/services/machine_learning/face_ml/face_embedding/face_embedding_service.dart index 2711550bdb..1a3dcf4167 100644 --- a/mobile/lib/services/face_ml/face_embedding/face_embedding_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_embedding/face_embedding_service.dart @@ -6,10 +6,10 @@ import 'dart:typed_data' show Uint8List; import "package:flutter/foundation.dart"; import "package:logging/logging.dart"; import 'package:photos/models/ml/ml_typedefs.dart'; -import "package:photos/services/face_ml/face_detection/detection.dart"; -import "package:photos/services/face_ml/face_embedding/face_embedding_exceptions.dart"; -import "package:photos/services/face_ml/face_embedding/face_embedding_options.dart"; -import "package:photos/services/face_ml/face_embedding/mobilefacenet_model_config.dart"; +import 'package:photos/services/machine_learning/face_ml/face_detection/detection.dart'; +import 'package:photos/services/machine_learning/face_ml/face_embedding/face_embedding_exceptions.dart'; +import 'package:photos/services/machine_learning/face_ml/face_embedding/face_embedding_options.dart'; +import 'package:photos/services/machine_learning/face_ml/face_embedding/mobilefacenet_model_config.dart'; import 'package:photos/utils/image_ml_isolate.dart'; import 'package:photos/utils/image_ml_util.dart'; import 'package:tflite_flutter/tflite_flutter.dart'; diff --git a/mobile/lib/services/face_ml/face_embedding/mobilefacenet_model_config.dart b/mobile/lib/services/machine_learning/face_ml/face_embedding/mobilefacenet_model_config.dart similarity index 71% rename from mobile/lib/services/face_ml/face_embedding/mobilefacenet_model_config.dart rename to mobile/lib/services/machine_learning/face_ml/face_embedding/mobilefacenet_model_config.dart index d55a2d3331..fd45f51674 100644 --- a/mobile/lib/services/face_ml/face_embedding/mobilefacenet_model_config.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_embedding/mobilefacenet_model_config.dart @@ -1,5 +1,5 @@ -import "package:photos/services/face_ml/face_embedding/face_embedding_options.dart"; -import "package:photos/services/face_ml/model_file.dart"; +import 'package:photos/services/machine_learning/face_ml/face_embedding/face_embedding_options.dart'; +import 'package:photos/services/machine_learning/face_ml/model_file.dart'; class MobileFaceNetModelConfig { final String modelPath; diff --git a/mobile/lib/services/face_ml/face_embedding/onnx_face_embedding.dart b/mobile/lib/services/machine_learning/face_ml/face_embedding/onnx_face_embedding.dart similarity index 98% rename from mobile/lib/services/face_ml/face_embedding/onnx_face_embedding.dart rename to mobile/lib/services/machine_learning/face_ml/face_embedding/onnx_face_embedding.dart index f15b25b46c..bdf2ac5cfb 100644 --- a/mobile/lib/services/face_ml/face_embedding/onnx_face_embedding.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_embedding/onnx_face_embedding.dart @@ -5,7 +5,7 @@ import 'dart:typed_data' show Float32List; import 'package:computer/computer.dart'; import 'package:logging/logging.dart'; import 'package:onnxruntime/onnxruntime.dart'; -import "package:photos/services/face_ml/face_detection/detection.dart"; +import 'package:photos/services/machine_learning/face_ml/face_detection/detection.dart'; import "package:photos/services/remote_assets_service.dart"; import "package:photos/utils/image_ml_isolate.dart"; import "package:synchronized/synchronized.dart"; diff --git a/mobile/lib/services/face_ml/face_feedback.dart/cluster_feedback.dart b/mobile/lib/services/machine_learning/face_ml/face_feedback.dart/cluster_feedback.dart similarity index 97% rename from mobile/lib/services/face_ml/face_feedback.dart/cluster_feedback.dart rename to mobile/lib/services/machine_learning/face_ml/face_feedback.dart/cluster_feedback.dart index b99d3950ab..a7da4fa556 100644 --- a/mobile/lib/services/face_ml/face_feedback.dart/cluster_feedback.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_feedback.dart/cluster_feedback.dart @@ -1,8 +1,8 @@ import "dart:convert"; -import "package:photos/services/face_ml/face_clustering/cosine_distance.dart"; -import "package:photos/services/face_ml/face_feedback.dart/feedback.dart"; -import "package:photos/services/face_ml/face_feedback.dart/feedback_types.dart"; +import 'package:photos/services/machine_learning/face_ml/face_clustering/cosine_distance.dart'; +import 'package:photos/services/machine_learning/face_ml/face_feedback.dart/feedback.dart'; +import 'package:photos/services/machine_learning/face_ml/face_feedback.dart/feedback_types.dart'; abstract class ClusterFeedback extends Feedback { static final Map fromJsonStringRegistry = { diff --git a/mobile/lib/services/face_ml/face_feedback.dart/face_feedback_service.dart b/mobile/lib/services/machine_learning/face_ml/face_feedback.dart/face_feedback_service.dart similarity index 98% rename from mobile/lib/services/face_ml/face_feedback.dart/face_feedback_service.dart rename to mobile/lib/services/machine_learning/face_ml/face_feedback.dart/face_feedback_service.dart index 0e95e3d7cc..c94c8c8d85 100644 --- a/mobile/lib/services/face_ml/face_feedback.dart/face_feedback_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_feedback.dart/face_feedback_service.dart @@ -1,8 +1,8 @@ import "package:logging/logging.dart"; import "package:photos/db/ml_data_db.dart"; -import "package:photos/services/face_ml/face_detection/detection.dart"; -import "package:photos/services/face_ml/face_feedback.dart/cluster_feedback.dart"; -import "package:photos/services/face_ml/face_ml_result.dart"; +import 'package:photos/services/machine_learning/face_ml/face_detection/detection.dart'; +import 'package:photos/services/machine_learning/face_ml/face_feedback.dart/cluster_feedback.dart'; +import 'package:photos/services/machine_learning/face_ml/face_ml_result.dart'; class FaceFeedbackService { final _logger = Logger("FaceFeedbackService"); diff --git a/mobile/lib/services/face_ml/face_feedback.dart/feedback.dart b/mobile/lib/services/machine_learning/face_ml/face_feedback.dart/feedback.dart similarity index 90% rename from mobile/lib/services/face_ml/face_feedback.dart/feedback.dart rename to mobile/lib/services/machine_learning/face_ml/face_feedback.dart/feedback.dart index 320ec64e92..8b3eb3c6ad 100644 --- a/mobile/lib/services/face_ml/face_feedback.dart/feedback.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_feedback.dart/feedback.dart @@ -1,5 +1,5 @@ import "package:photos/models/ml/ml_versions.dart"; -import "package:photos/services/face_ml/face_feedback.dart/feedback_types.dart"; +import 'package:photos/services/machine_learning/face_ml/face_feedback.dart/feedback_types.dart'; import "package:uuid/uuid.dart"; abstract class Feedback { diff --git a/mobile/lib/services/face_ml/face_feedback.dart/feedback_types.dart b/mobile/lib/services/machine_learning/face_ml/face_feedback.dart/feedback_types.dart similarity index 100% rename from mobile/lib/services/face_ml/face_feedback.dart/feedback_types.dart rename to mobile/lib/services/machine_learning/face_ml/face_feedback.dart/feedback_types.dart diff --git a/mobile/lib/services/face_ml/blur_detection/blur_detection_service.dart b/mobile/lib/services/machine_learning/face_ml/face_filtering/blur_detection_service.dart similarity index 96% rename from mobile/lib/services/face_ml/blur_detection/blur_detection_service.dart rename to mobile/lib/services/machine_learning/face_ml/face_filtering/blur_detection_service.dart index ff58304683..43f6b252d2 100644 --- a/mobile/lib/services/face_ml/blur_detection/blur_detection_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_filtering/blur_detection_service.dart @@ -1,5 +1,5 @@ import 'package:logging/logging.dart'; -import "package:photos/services/face_ml/blur_detection/blur_constants.dart"; +import 'package:photos/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart'; class BlurDetectionService { final _logger = Logger('BlurDetectionService'); diff --git a/mobile/lib/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart b/mobile/lib/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart new file mode 100644 index 0000000000..6606e858ec --- /dev/null +++ b/mobile/lib/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart @@ -0,0 +1,13 @@ +import 'package:photos/services/machine_learning/face_ml/face_detection/yolov5face/onnx_face_detection.dart'; + +/// Blur detection threshold +const kLaplacianThreshold = 15; + +/// Default blur value +const kLapacianDefault = 10000.0; + +/// The minimum score for a face to be considered a high quality face for clustering and person detection +const kMinHighQualityFaceScore = 0.78; + +/// The minimum score for a face to be detected, regardless of quality. Use [kMinHighQualityFaceScore] for high quality faces. +const kMinFaceDetectionScore = YoloOnnxFaceDetection.kMinScoreSigmoidThreshold; diff --git a/mobile/lib/services/face_ml/face_ml_exceptions.dart b/mobile/lib/services/machine_learning/face_ml/face_ml_exceptions.dart similarity index 100% rename from mobile/lib/services/face_ml/face_ml_exceptions.dart rename to mobile/lib/services/machine_learning/face_ml/face_ml_exceptions.dart diff --git a/mobile/lib/services/face_ml/face_ml_methods.dart b/mobile/lib/services/machine_learning/face_ml/face_ml_methods.dart similarity index 97% rename from mobile/lib/services/face_ml/face_ml_methods.dart rename to mobile/lib/services/machine_learning/face_ml/face_ml_methods.dart index a6c967e52f..5745234b58 100644 --- a/mobile/lib/services/face_ml/face_ml_methods.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_ml_methods.dart @@ -1,4 +1,4 @@ -import "package:photos/services/face_ml/face_ml_version.dart"; +import 'package:photos/services/machine_learning/face_ml/face_ml_version.dart'; /// Represents a face detection method with a specific version. class FaceDetectionMethod extends VersionedMethod { diff --git a/mobile/lib/services/face_ml/face_ml_result.dart b/mobile/lib/services/machine_learning/face_ml/face_ml_result.dart similarity index 96% rename from mobile/lib/services/face_ml/face_ml_result.dart rename to mobile/lib/services/machine_learning/face_ml/face_ml_result.dart index c770efde7a..58fc72ac5a 100644 --- a/mobile/lib/services/face_ml/face_ml_result.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_ml_result.dart @@ -6,12 +6,12 @@ import "package:photos/db/ml_data_db.dart"; import "package:photos/models/file/file.dart"; import 'package:photos/models/ml/ml_typedefs.dart'; import "package:photos/models/ml/ml_versions.dart"; -import "package:photos/services/face_ml/blur_detection/blur_constants.dart"; -import "package:photos/services/face_ml/face_alignment/alignment_result.dart"; -import "package:photos/services/face_ml/face_clustering/cosine_distance.dart"; -import "package:photos/services/face_ml/face_detection/detection.dart"; -import "package:photos/services/face_ml/face_feedback.dart/cluster_feedback.dart"; -import "package:photos/services/face_ml/face_ml_methods.dart"; +import 'package:photos/services/machine_learning/face_ml/face_alignment/alignment_result.dart'; +import 'package:photos/services/machine_learning/face_ml/face_clustering/cosine_distance.dart'; +import 'package:photos/services/machine_learning/face_ml/face_detection/detection.dart'; +import 'package:photos/services/machine_learning/face_ml/face_feedback.dart/cluster_feedback.dart'; +import 'package:photos/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart'; +import 'package:photos/services/machine_learning/face_ml/face_ml_methods.dart'; final _logger = Logger('ClusterResult_FaceMlResult'); @@ -37,7 +37,7 @@ class ClusterResult { String get thumbnailFaceId => _thumbnailFaceId; - int get thumbnailFileId => _getFileIdFromFaceId(_thumbnailFaceId); + int get thumbnailFileId => getFileIdFromFaceId(_thumbnailFaceId); /// Sets the thumbnail faceId to the given faceId. /// Throws an exception if the faceId is not in the list of faceIds. @@ -89,7 +89,7 @@ class ClusterResult { int removedCount = 0; for (var i = 0; i < _fileIds.length; i++) { if (_fileIds[i] == fileId) { - assert(_getFileIdFromFaceId(_faceIds[i]) == fileId); + assert(getFileIdFromFaceId(_faceIds[i]) == fileId); _fileIds.removeAt(i); _faceIds.removeAt(i); debugPrint( @@ -748,6 +748,6 @@ class FaceResultBuilder { } } -int _getFileIdFromFaceId(String faceId) { +int getFileIdFromFaceId(String faceId) { return int.parse(faceId.split("_")[0]); } diff --git a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart new file mode 100644 index 0000000000..d16ae165f2 --- /dev/null +++ b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart @@ -0,0 +1,1291 @@ +import "dart:async"; +import "dart:developer" as dev show log; +import "dart:io" show File; +import "dart:isolate"; +import "dart:typed_data" show Uint8List, Float32List, ByteData; +import "dart:ui" show Image; + +import "package:computer/computer.dart"; +import "package:flutter/foundation.dart" show debugPrint, kDebugMode; +import "package:flutter_image_compress/flutter_image_compress.dart"; +import "package:flutter_isolate/flutter_isolate.dart"; +import "package:logging/logging.dart"; +import "package:onnxruntime/onnxruntime.dart"; +import "package:photos/core/configuration.dart"; +import "package:photos/core/event_bus.dart"; +import "package:photos/db/files_db.dart"; +import "package:photos/db/ml_data_db.dart"; +import "package:photos/events/diff_sync_complete_event.dart"; +import "package:photos/extensions/list.dart"; +import "package:photos/extensions/stop_watch.dart"; +import "package:photos/face/db.dart"; +import "package:photos/face/model/box.dart"; +import "package:photos/face/model/detection.dart" as face_detection; +import "package:photos/face/model/face.dart"; +import "package:photos/face/model/landmark.dart"; +import "package:photos/models/file/extensions/file_props.dart"; +import "package:photos/models/file/file.dart"; +import "package:photos/models/file/file_type.dart"; +import "package:photos/models/ml/ml_versions.dart"; +import 'package:photos/services/machine_learning/face_ml/face_clustering/linear_clustering_service.dart'; +import 'package:photos/services/machine_learning/face_ml/face_detection/detection.dart'; +import 'package:photos/services/machine_learning/face_ml/face_detection/yolov5face/onnx_face_detection.dart'; +import 'package:photos/services/machine_learning/face_ml/face_detection/yolov5face/yolo_face_detection_exceptions.dart'; +import 'package:photos/services/machine_learning/face_ml/face_embedding/face_embedding_exceptions.dart'; +import 'package:photos/services/machine_learning/face_ml/face_embedding/onnx_face_embedding.dart'; +import 'package:photos/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart'; +import 'package:photos/services/machine_learning/face_ml/face_ml_exceptions.dart'; +import 'package:photos/services/machine_learning/face_ml/face_ml_result.dart'; +import 'package:photos/services/machine_learning/file_ml/file_ml.dart'; +import 'package:photos/services/machine_learning/file_ml/remote_fileml_service.dart'; +import "package:photos/services/search_service.dart"; +import "package:photos/utils/file_util.dart"; +import 'package:photos/utils/image_ml_isolate.dart'; +import "package:photos/utils/image_ml_util.dart"; +import "package:photos/utils/local_settings.dart"; +import "package:photos/utils/thumbnail_util.dart"; +import "package:synchronized/synchronized.dart"; + +enum FileDataForML { thumbnailData, fileData, compressedFileData } + +enum FaceMlOperation { analyzeImage } + +/// This class is responsible for running the full face ml pipeline on images. +/// +/// WARNING: For getting the ML results needed for the UI, you should use `FaceSearchService` instead of this class! +/// +/// The pipeline consists of face detection, face alignment and face embedding. +class FaceMlService { + final _logger = Logger("FaceMlService"); + + // Flutter isolate things for running the image ml pipeline + Timer? _inactivityTimer; + final Duration _inactivityDuration = const Duration(seconds: 120); + int _activeTasks = 0; + final _initLockIsolate = Lock(); + late FlutterIsolate _isolate; + late ReceivePort _receivePort = ReceivePort(); + late SendPort _mainSendPort; + + bool isIsolateSpawned = false; + + // singleton pattern + FaceMlService._privateConstructor(); + static final instance = FaceMlService._privateConstructor(); + factory FaceMlService() => instance; + + final _initLock = Lock(); + final _functionLock = Lock(); + + final _computer = Computer.shared(); + + bool isInitialized = false; + bool isImageIndexRunning = false; + int kParallelism = 15; + + Future init({bool initializeImageMlIsolate = false}) async { + return _initLock.synchronized(() async { + if (isInitialized) { + return; + } + _logger.info("init called"); + await _computer.compute(initOrtEnv); + try { + await YoloOnnxFaceDetection.instance.init(); + } catch (e, s) { + _logger.severe("Could not initialize yolo onnx", e, s); + } + if (initializeImageMlIsolate) { + try { + await ImageMlIsolate.instance.init(); + } catch (e, s) { + _logger.severe("Could not initialize image ml isolate", e, s); + } + } + try { + await FaceEmbeddingOnnx.instance.init(); + } catch (e, s) { + _logger.severe("Could not initialize mobilefacenet", e, s); + } + + isInitialized = true; + }); + } + + static void initOrtEnv() async { + OrtEnv.instance.init(); + } + + void listenIndexOnDiffSync() { + Bus.instance.on().listen((event) async { + if (LocalSettings.instance.isFaceIndexingEnabled == false) { + return; + } + // [neeraj] intentional delay in starting indexing on diff sync, this gives time for the user + // to disable face-indexing in case it's causing crash. In the future, we + // should have a better way to handle this. + Future.delayed(const Duration(seconds: 10), () { + unawaited(indexAllImages()); + }); + }); + } + + Future ensureInitialized() async { + if (!isInitialized) { + await init(); + } + } + + Future release() async { + return _initLock.synchronized(() async { + _logger.info("dispose called"); + if (!isInitialized) { + return; + } + try { + await YoloOnnxFaceDetection.instance.release(); + } catch (e, s) { + _logger.severe("Could not dispose yolo onnx", e, s); + } + try { + ImageMlIsolate.instance.dispose(); + } catch (e, s) { + _logger.severe("Could not dispose image ml isolate", e, s); + } + try { + await FaceEmbeddingOnnx.instance.release(); + } catch (e, s) { + _logger.severe("Could not dispose mobilefacenet", e, s); + } + OrtEnv.instance.release(); + isInitialized = false; + }); + } + + Future initIsolate() async { + return _initLockIsolate.synchronized(() async { + if (isIsolateSpawned) return; + _logger.info("initIsolate called"); + + _receivePort = ReceivePort(); + + try { + _isolate = await FlutterIsolate.spawn( + _isolateMain, + _receivePort.sendPort, + ); + _mainSendPort = await _receivePort.first as SendPort; + isIsolateSpawned = true; + + _resetInactivityTimer(); + } catch (e) { + _logger.severe('Could not spawn isolate', e); + isIsolateSpawned = false; + } + }); + } + + Future ensureSpawnedIsolate() async { + if (!isIsolateSpawned) { + await initIsolate(); + } + } + + /// The main execution function of the isolate. + static void _isolateMain(SendPort mainSendPort) async { + final receivePort = ReceivePort(); + mainSendPort.send(receivePort.sendPort); + + receivePort.listen((message) async { + final functionIndex = message[0] as int; + final function = FaceMlOperation.values[functionIndex]; + final args = message[1] as Map; + final sendPort = message[2] as SendPort; + + try { + switch (function) { + case FaceMlOperation.analyzeImage: + final int enteFileID = args["enteFileID"] as int; + final String imagePath = args["filePath"] as String; + final int faceDetectionAddress = + args["faceDetectionAddress"] as int; + final int faceEmbeddingAddress = + args["faceEmbeddingAddress"] as int; + + final resultBuilder = + FaceMlResultBuilder.fromEnteFileID(enteFileID); + + dev.log( + "Start analyzing image with uploadedFileID: $enteFileID inside the isolate", + ); + final stopwatchTotal = Stopwatch()..start(); + final stopwatch = Stopwatch()..start(); + + // Decode the image once to use for both face detection and alignment + final imageData = await File(imagePath).readAsBytes(); + final image = await decodeImageFromData(imageData); + final ByteData imgByteData = await getByteDataFromImage(image); + dev.log('Reading and decoding image took ' + '${stopwatch.elapsedMilliseconds} ms'); + stopwatch.reset(); + + // Get the faces + final List faceDetectionResult = + await FaceMlService.detectFacesSync( + image, + imgByteData, + faceDetectionAddress, + resultBuilder: resultBuilder, + ); + + dev.log( + "${faceDetectionResult.length} faces detected with scores ${faceDetectionResult.map((e) => e.score).toList()}: completed `detectFacesSync` function, in " + "${stopwatch.elapsedMilliseconds} ms"); + + // If no faces were detected, return a result with no faces. Otherwise, continue. + if (faceDetectionResult.isEmpty) { + dev.log( + "No faceDetectionResult, Completed analyzing image with uploadedFileID $enteFileID, in " + "${stopwatch.elapsedMilliseconds} ms"); + sendPort.send(resultBuilder.buildNoFaceDetected().toJsonString()); + break; + } + + stopwatch.reset(); + // Align the faces + final Float32List faceAlignmentResult = + await FaceMlService.alignFacesSync( + image, + imgByteData, + faceDetectionResult, + resultBuilder: resultBuilder, + ); + + dev.log("Completed `alignFacesSync` function, in " + "${stopwatch.elapsedMilliseconds} ms"); + + stopwatch.reset(); + // Get the embeddings of the faces + final embeddings = await FaceMlService.embedFacesSync( + faceAlignmentResult, + faceEmbeddingAddress, + resultBuilder: resultBuilder, + ); + + dev.log("Completed `embedFacesSync` function, in " + "${stopwatch.elapsedMilliseconds} ms"); + + stopwatch.stop(); + stopwatchTotal.stop(); + dev.log("Finished Analyze image (${embeddings.length} faces) with " + "uploadedFileID $enteFileID, in " + "${stopwatchTotal.elapsedMilliseconds} ms"); + + sendPort.send(resultBuilder.build().toJsonString()); + break; + } + } catch (e, stackTrace) { + dev.log( + "[SEVERE] Error in FaceML isolate: $e", + error: e, + stackTrace: stackTrace, + ); + sendPort + .send({'error': e.toString(), 'stackTrace': stackTrace.toString()}); + } + }); + } + + /// The common method to run any operation in the isolate. It sends the [message] to [_isolateMain] and waits for the result. + Future _runInIsolate( + (FaceMlOperation, Map) message, + ) async { + await ensureSpawnedIsolate(); + return _functionLock.synchronized(() async { + _resetInactivityTimer(); + + if (isImageIndexRunning == false) { + return null; + } + + final completer = Completer(); + final answerPort = ReceivePort(); + + _activeTasks++; + _mainSendPort.send([message.$1.index, message.$2, answerPort.sendPort]); + + answerPort.listen((receivedMessage) { + if (receivedMessage is Map && receivedMessage.containsKey('error')) { + // Handle the error + final errorMessage = receivedMessage['error']; + final errorStackTrace = receivedMessage['stackTrace']; + final exception = Exception(errorMessage); + final stackTrace = StackTrace.fromString(errorStackTrace); + completer.completeError(exception, stackTrace); + } else { + completer.complete(receivedMessage); + } + }); + _activeTasks--; + + return completer.future; + }); + } + + /// Resets a timer that kills the isolate after a certain amount of inactivity. + /// + /// Should be called after initialization (e.g. inside `init()`) and after every call to isolate (e.g. inside `_runInIsolate()`) + void _resetInactivityTimer() { + _inactivityTimer?.cancel(); + _inactivityTimer = Timer(_inactivityDuration, () { + if (_activeTasks > 0) { + _logger.info('Tasks are still running. Delaying isolate disposal.'); + // Optionally, reschedule the timer to check again later. + _resetInactivityTimer(); + } else { + _logger.info( + 'Clustering Isolate has been inactive for ${_inactivityDuration.inSeconds} seconds with no tasks running. Killing isolate.', + ); + disposeIsolate(); + } + }); + } + + void disposeIsolate() async { + if (!isIsolateSpawned) return; + await release(); + + isIsolateSpawned = false; + _isolate.kill(); + _receivePort.close(); + _inactivityTimer?.cancel(); + } + + Future indexAndClusterAllImages() async { + // Run the analysis on all images to make sure everything is analyzed + await indexAllImages(); + + // Cluster all the images + await clusterAllImages(); + } + + Future clusterAllImages({ + double minFaceScore = kMinHighQualityFaceScore, + bool clusterInBuckets = false, + }) async { + _logger.info("`clusterAllImages()` called"); + + try { + if (clusterInBuckets) { + // Get a sense of the total number of faces in the database + final int totalFaces = await FaceMLDataDB.instance + .getTotalFaceCount(minFaceScore: minFaceScore); + + // read the creation times from Files DB, in a map from fileID to creation time + final fileIDToCreationTime = + await FilesDB.instance.getFileIDToCreationTime(); + + const int bucketSize = 10000; + const int batchSize = 10000; + const int offsetIncrement = 7500; + int offset = 0; + + while (true) { + final faceIdToEmbeddingBucket = + await FaceMLDataDB.instance.getFaceEmbeddingMap( + minScore: minFaceScore, + maxFaces: bucketSize, + offset: offset, + batchSize: batchSize, + ); + if (faceIdToEmbeddingBucket.isEmpty) { + break; + } + if (offset > totalFaces) { + _logger.warning( + 'offset > totalFaces, this should ideally not happen. offset: $offset, totalFaces: $totalFaces', + ); + break; + } + + final faceIdToCluster = await FaceLinearClustering.instance.predict( + faceIdToEmbeddingBucket, + fileIDToCreationTime: fileIDToCreationTime, + ); + if (faceIdToCluster == null) { + _logger.warning("faceIdToCluster is null"); + return; + } + + await FaceMLDataDB.instance + .updatePersonIDForFaceIDIFNotSet(faceIdToCluster); + if (offset == 0) { + offset += offsetIncrement; + } else { + offset += bucketSize; + } + } + } else { + // Read all the embeddings from the database, in a map from faceID to embedding + final clusterStartTime = DateTime.now(); + final faceIdToEmbedding = + await FaceMLDataDB.instance.getFaceEmbeddingMap( + minScore: minFaceScore, + ); + final gotFaceEmbeddingsTime = DateTime.now(); + _logger.info( + 'read embeddings ${faceIdToEmbedding.length} in ${gotFaceEmbeddingsTime.difference(clusterStartTime).inMilliseconds} ms', + ); + + // Read the creation times from Files DB, in a map from fileID to creation time + final fileIDToCreationTime = + await FilesDB.instance.getFileIDToCreationTime(); + _logger.info('read creation times from FilesDB in ' + '${DateTime.now().difference(gotFaceEmbeddingsTime).inMilliseconds} ms'); + + // Cluster the embeddings using the linear clustering algorithm, returning a map from faceID to clusterID + final faceIdToCluster = await FaceLinearClustering.instance.predict( + faceIdToEmbedding, + fileIDToCreationTime: fileIDToCreationTime, + ); + if (faceIdToCluster == null) { + _logger.warning("faceIdToCluster is null"); + return; + } + final clusterDoneTime = DateTime.now(); + _logger.info( + 'done with clustering ${faceIdToEmbedding.length} in ${clusterDoneTime.difference(clusterStartTime).inSeconds} seconds ', + ); + + // Store the updated clusterIDs in the database + _logger.info( + 'Updating ${faceIdToCluster.length} FaceIDs with clusterIDs in the DB', + ); + await FaceMLDataDB.instance + .updatePersonIDForFaceIDIFNotSet(faceIdToCluster); + _logger.info('Done updating FaceIDs with clusterIDs in the DB, in ' + '${DateTime.now().difference(clusterDoneTime).inSeconds} seconds'); + } + } catch (e, s) { + _logger.severe("`clusterAllImages` failed", e, s); + } + } + + /// Analyzes all the images in the database with the latest ml version and stores the results in the database. + /// + /// This function first checks if the image has already been analyzed with the lastest faceMlVersion and stored in the database. If so, it skips the image. + Future indexAllImages() async { + if (isImageIndexRunning) { + _logger.warning("indexAllImages is already running, skipping"); + return; + } + // verify indexing is enabled + if (LocalSettings.instance.isFaceIndexingEnabled == false) { + _logger.warning("indexAllImages is disabled"); + return; + } + try { + isImageIndexRunning = true; + _logger.info('starting image indexing'); + final List enteFiles = + await SearchService.instance.getAllFiles(); + final Map alreadyIndexedFiles = + await FaceMLDataDB.instance.getIndexedFileIds(); + + // Make sure the image conversion isolate is spawned + // await ImageMlIsolate.instance.ensureSpawned(); + await ensureInitialized(); + + int fileAnalyzedCount = 0; + int fileSkippedCount = 0; + final stopwatch = Stopwatch()..start(); + final List filesWithLocalID = []; + final List filesWithoutLocalID = []; + for (final EnteFile enteFile in enteFiles) { + if (_skipAnalysisEnteFile(enteFile, alreadyIndexedFiles)) { + fileSkippedCount++; + continue; + } + if ((enteFile.localID ?? '').isEmpty) { + filesWithoutLocalID.add(enteFile); + } else { + filesWithLocalID.add(enteFile); + } + } + + // list of files where files with localID are first + final sortedBylocalID = []; + sortedBylocalID.addAll(filesWithLocalID); + sortedBylocalID.addAll(filesWithoutLocalID); + final List> chunks = sortedBylocalID.chunks(kParallelism); + outerLoop: + for (final chunk in chunks) { + final futures = >[]; + final List fileIds = []; + // Try to find embeddings on the remote server + for (final f in chunk) { + fileIds.add(f.uploadedFileID!); + } + try { + final EnteWatch? w = kDebugMode ? EnteWatch("face_em_fetch") : null; + w?.start(); + final res = + await RemoteFileMLService.instance.getFilessEmbedding(fileIds); + w?.logAndReset('fetched ${res.mlData.length} embeddings'); + final List faces = []; + final remoteFileIdToVersion = {}; + for (FileMl fileMl in res.mlData.values) { + if (fileMl.faceEmbedding.version < faceMlVersion) continue; + if (fileMl.faceEmbedding.faces.isEmpty) { + faces.add( + Face.empty( + fileMl.fileID, + error: (fileMl.faceEmbedding.error ?? false), + ), + ); + } else { + faces.addAll(fileMl.faceEmbedding.faces); + } + remoteFileIdToVersion[fileMl.fileID] = fileMl.faceEmbedding.version; + } + await FaceMLDataDB.instance.bulkInsertFaces(faces); + w?.logAndReset('stored embeddings'); + for (final entry in remoteFileIdToVersion.entries) { + alreadyIndexedFiles[entry.key] = entry.value; + } + _logger.info('already indexed files ${remoteFileIdToVersion.length}'); + } catch (e, s) { + _logger.severe("err while getting files embeddings", e, s); + rethrow; + } + + for (final enteFile in chunk) { + if (isImageIndexRunning == false) { + _logger.info("indexAllImages() was paused, stopping"); + break outerLoop; + } + if (_skipAnalysisEnteFile( + enteFile, + alreadyIndexedFiles, + )) { + fileSkippedCount++; + continue; + } + futures.add(processImage(enteFile)); + } + final awaitedFutures = await Future.wait(futures); + final sumFutures = awaitedFutures.fold( + 0, + (previousValue, element) => previousValue + (element ? 1 : 0), + ); + fileAnalyzedCount += sumFutures; + + // TODO: remove this cooldown later. Cooldown of one minute every 400 images + if (fileAnalyzedCount > 400 && fileAnalyzedCount % 400 < kParallelism) { + _logger.info( + "indexAllImages() analyzed $fileAnalyzedCount images, cooldown for 1 minute", + ); + await Future.delayed(const Duration(minutes: 1), () { + _logger.info("indexAllImages() cooldown finished"); + }); + } + } + + stopwatch.stop(); + _logger.info( + "`indexAllImages()` finished. Analyzed $fileAnalyzedCount images, in ${stopwatch.elapsed.inSeconds} seconds (avg of ${stopwatch.elapsed.inSeconds / fileAnalyzedCount} seconds per image, skipped $fileSkippedCount images)", + ); + + // Dispose of all the isolates + // ImageMlIsolate.instance.dispose(); + // await release(); + } catch (e, s) { + _logger.severe("indexAllImages failed", e, s); + } finally { + isImageIndexRunning = false; + } + } + + Future processImage(EnteFile enteFile) async { + _logger.info( + "`indexAllImages()` on file number start processing image with uploadedFileID: ${enteFile.uploadedFileID}", + ); + + try { + final FaceMlResult? result = await analyzeImageInSingleIsolate( + enteFile, + // preferUsingThumbnailForEverything: false, + // disposeImageIsolateAfterUse: false, + ); + if (result == null) { + return false; + } + final List faces = []; + if (!result.hasFaces) { + debugPrint( + 'No faces detected for file with name:${enteFile.displayName}', + ); + faces.add( + Face( + '${result.fileId}-0', + result.fileId, + [], + result.errorOccured ? -1.0 : 0.0, + face_detection.Detection.empty(), + 0.0, + ), + ); + } else { + if (result.faceDetectionImageSize == null || + result.faceAlignmentImageSize == null) { + _logger.severe( + "faceDetectionImageSize or faceDetectionImageSize is null for image with " + "ID: ${enteFile.uploadedFileID}"); + _logger.info( + "Using aligned image size for image with ID: ${enteFile.uploadedFileID}. This size is ${result.faceAlignmentImageSize!.width}x${result.faceAlignmentImageSize!.height} compared to size of ${enteFile.width}x${enteFile.height} in the metadata", + ); + } + for (int i = 0; i < result.faces.length; ++i) { + final FaceResult faceRes = result.faces[i]; + final detection = face_detection.Detection( + box: FaceBox( + xMin: faceRes.detection.xMinBox, + yMin: faceRes.detection.yMinBox, + width: faceRes.detection.width, + height: faceRes.detection.height, + ), + landmarks: faceRes.detection.allKeypoints + .map( + (keypoint) => Landmark( + x: keypoint[0], + y: keypoint[0], + ), + ) + .toList(), + ); + faces.add( + Face( + faceRes.faceId, + result.fileId, + faceRes.embedding, + faceRes.detection.score, + detection, + faceRes.blurValue, + ), + ); + } + } + _logger.info("inserting ${faces.length} faces for ${result.fileId}"); + await RemoteFileMLService.instance.putFileEmbedding( + enteFile, + FileMl( + enteFile.uploadedFileID!, + FaceEmbeddings( + faces, + result.mlVersion, + error: result.errorOccured ? true : null, + ), + ), + ); + await FaceMLDataDB.instance.bulkInsertFaces(faces); + return true; + } catch (e, s) { + _logger.severe( + "Failed to analyze using FaceML for image with ID: ${enteFile.uploadedFileID}", + e, + s, + ); + return true; + } + } + + void pauseIndexing() { + isImageIndexRunning = false; + } + + /// Analyzes the given image data by running the full pipeline using [analyzeImageInComputerAndImageIsolate] and stores the result in the database [MlDataDB]. + /// This function first checks if the image has already been analyzed (with latest ml version) and stored in the database. If so, it returns the stored result. + /// + /// 'enteFile': The ente file to analyze. + /// + /// Returns an immutable [FaceMlResult] instance containing the results of the analysis. The result is also stored in the database. + Future indexImage(EnteFile enteFile) async { + _logger.info( + "`indexImage` called on image with uploadedFileID ${enteFile.uploadedFileID}", + ); + _checkEnteFileForID(enteFile); + + // Check if the image has already been analyzed and stored in the database with the latest ml version + final existingResult = await _checkForExistingUpToDateResult(enteFile); + if (existingResult != null) { + return existingResult; + } + + // If the image has not been analyzed and stored in the database, analyze it and store the result in the database + _logger.info( + "Image with uploadedFileID ${enteFile.uploadedFileID} has not been analyzed and stored in the database. Analyzing it now.", + ); + FaceMlResult result; + try { + result = await analyzeImageInComputerAndImageIsolate(enteFile); + } catch (e, s) { + _logger.severe( + "`indexImage` failed on image with uploadedFileID ${enteFile.uploadedFileID}", + e, + s, + ); + throw GeneralFaceMlException( + "`indexImage` failed on image with uploadedFileID ${enteFile.uploadedFileID}", + ); + } + + // Store the result in the database + await MlDataDB.instance.createFaceMlResult(result); + + return result; + } + + /// Analyzes the given image data by running the full pipeline (face detection, face alignment, face embedding). + /// + /// [enteFile] The ente file to analyze. + /// + /// [preferUsingThumbnailForEverything] If true, the thumbnail will be used for everything (face detection, face alignment, face embedding), and file data will be used only if a thumbnail is unavailable. + /// If false, thumbnail will only be used for detection, and the original image will be used for face alignment and face embedding. + /// + /// Returns an immutable [FaceMlResult] instance containing the results of the analysis. + /// Does not store the result in the database, for that you should use [indexImage]. + /// Throws [CouldNotRetrieveAnyFileData] or [GeneralFaceMlException] if something goes wrong. + /// TODO: improve function such that it only uses full image if it is already on the device, otherwise it uses thumbnail. And make sure to store what is used! + Future analyzeImageInComputerAndImageIsolate( + EnteFile enteFile, { + bool preferUsingThumbnailForEverything = false, + bool disposeImageIsolateAfterUse = true, + }) async { + _checkEnteFileForID(enteFile); + + final String? thumbnailPath = await _getImagePathForML( + enteFile, + typeOfData: FileDataForML.thumbnailData, + ); + String? filePath; + + // // TODO: remove/optimize this later. Not now though: premature optimization + // fileData = + // await _getDataForML(enteFile, typeOfData: FileDataForML.fileData); + + if (thumbnailPath == null) { + filePath = await _getImagePathForML( + enteFile, + typeOfData: FileDataForML.fileData, + ); + if (thumbnailPath == null && filePath == null) { + _logger.severe( + "Failed to get any data for enteFile with uploadedFileID ${enteFile.uploadedFileID}", + ); + throw CouldNotRetrieveAnyFileData(); + } + } + // TODO: use smallData and largeData instead of thumbnailData and fileData again! + final String smallDataPath = thumbnailPath ?? filePath!; + + final resultBuilder = FaceMlResultBuilder.fromEnteFile(enteFile); + + _logger.info( + "Analyzing image with uploadedFileID: ${enteFile.uploadedFileID} ${kDebugMode ? enteFile.displayName : ''}", + ); + final stopwatch = Stopwatch()..start(); + + try { + // Get the faces + final List faceDetectionResult = + await _detectFacesIsolate( + smallDataPath, + resultBuilder: resultBuilder, + ); + + _logger.info("Completed `detectFaces` function"); + + // If no faces were detected, return a result with no faces. Otherwise, continue. + if (faceDetectionResult.isEmpty) { + _logger.info( + "No faceDetectionResult, Completed analyzing image with uploadedFileID ${enteFile.uploadedFileID}, in " + "${stopwatch.elapsedMilliseconds} ms"); + return resultBuilder.buildNoFaceDetected(); + } + + if (!preferUsingThumbnailForEverything) { + filePath ??= await _getImagePathForML( + enteFile, + typeOfData: FileDataForML.fileData, + ); + } + resultBuilder.onlyThumbnailUsed = filePath == null; + final String largeDataPath = filePath ?? thumbnailPath!; + + // Align the faces + final Float32List faceAlignmentResult = await _alignFaces( + largeDataPath, + faceDetectionResult, + resultBuilder: resultBuilder, + ); + + _logger.info("Completed `alignFaces` function"); + + // Get the embeddings of the faces + final embeddings = await _embedFaces( + faceAlignmentResult, + resultBuilder: resultBuilder, + ); + + _logger.info("Completed `embedBatchFaces` function"); + + stopwatch.stop(); + _logger.info("Finished Analyze image (${embeddings.length} faces) with " + "uploadedFileID ${enteFile.uploadedFileID}, in " + "${stopwatch.elapsedMilliseconds} ms"); + + if (disposeImageIsolateAfterUse) { + // Close the image conversion isolate + ImageMlIsolate.instance.dispose(); + } + + return resultBuilder.build(); + } catch (e, s) { + _logger.severe( + "Could not analyze image with ID ${enteFile.uploadedFileID} \n", + e, + s, + ); + // throw GeneralFaceMlException("Could not analyze image"); + return resultBuilder.buildErrorOccurred(); + } + } + + Future analyzeImageInSingleIsolate(EnteFile enteFile) async { + _checkEnteFileForID(enteFile); + await ensureInitialized(); + + final String? filePath = + await _getImagePathForML(enteFile, typeOfData: FileDataForML.fileData); + + if (filePath == null) { + _logger.severe( + "Failed to get any data for enteFile with uploadedFileID ${enteFile.uploadedFileID}", + ); + throw CouldNotRetrieveAnyFileData(); + } + + final Stopwatch stopwatch = Stopwatch()..start(); + late FaceMlResult result; + + try { + final resultJsonString = await _runInIsolate( + ( + FaceMlOperation.analyzeImage, + { + "enteFileID": enteFile.uploadedFileID ?? -1, + "filePath": filePath, + "faceDetectionAddress": + YoloOnnxFaceDetection.instance.sessionAddress, + "faceEmbeddingAddress": FaceEmbeddingOnnx.instance.sessionAddress, + } + ), + ) as String?; + if (resultJsonString == null) { + return null; + } + result = FaceMlResult.fromJsonString(resultJsonString); + } catch (e, s) { + _logger.severe( + "Could not analyze image with ID ${enteFile.uploadedFileID} \n", + e, + s, + ); + debugPrint( + "This image with ID ${enteFile.uploadedFileID} has name ${enteFile.displayName}.", + ); + final resultBuilder = FaceMlResultBuilder.fromEnteFile(enteFile); + return resultBuilder.buildErrorOccurred(); + } + stopwatch.stop(); + _logger.info( + "Finished Analyze image (${result.faces.length} faces) with uploadedFileID ${enteFile.uploadedFileID}, in " + "${stopwatch.elapsedMilliseconds} ms", + ); + + return result; + } + + Future _getImagePathForML( + EnteFile enteFile, { + FileDataForML typeOfData = FileDataForML.fileData, + }) async { + String? imagePath; + + switch (typeOfData) { + case FileDataForML.fileData: + final stopwatch = Stopwatch()..start(); + final File? file = await getFile(enteFile, isOrigin: true); + if (file == null) { + _logger.warning("Could not get file for $enteFile"); + imagePath = null; + break; + } + imagePath = file.path; + stopwatch.stop(); + _logger.info( + "Getting file data for uploadedFileID ${enteFile.uploadedFileID} took ${stopwatch.elapsedMilliseconds} ms", + ); + break; + + case FileDataForML.thumbnailData: + final stopwatch = Stopwatch()..start(); + final File? thumbnail = await getThumbnailForUploadedFile(enteFile); + if (thumbnail == null) { + _logger.warning("Could not get thumbnail for $enteFile"); + imagePath = null; + break; + } + imagePath = thumbnail.path; + stopwatch.stop(); + _logger.info( + "Getting thumbnail data for uploadedFileID ${enteFile.uploadedFileID} took ${stopwatch.elapsedMilliseconds} ms", + ); + break; + + case FileDataForML.compressedFileData: + _logger.warning( + "Getting compressed file data for uploadedFileID ${enteFile.uploadedFileID} is not implemented yet", + ); + imagePath = null; + break; + } + + return imagePath; + } + + @Deprecated('Deprecated in favor of `_getImagePathForML`') + Future _getDataForML( + EnteFile enteFile, { + FileDataForML typeOfData = FileDataForML.fileData, + }) async { + Uint8List? data; + + switch (typeOfData) { + case FileDataForML.fileData: + final stopwatch = Stopwatch()..start(); + final File? actualIoFile = await getFile(enteFile, isOrigin: true); + if (actualIoFile != null) { + data = await actualIoFile.readAsBytes(); + } + stopwatch.stop(); + _logger.info( + "Getting file data for uploadedFileID ${enteFile.uploadedFileID} took ${stopwatch.elapsedMilliseconds} ms", + ); + + break; + + case FileDataForML.thumbnailData: + final stopwatch = Stopwatch()..start(); + data = await getThumbnail(enteFile); + stopwatch.stop(); + _logger.info( + "Getting thumbnail data for uploadedFileID ${enteFile.uploadedFileID} took ${stopwatch.elapsedMilliseconds} ms", + ); + break; + + case FileDataForML.compressedFileData: + final stopwatch = Stopwatch()..start(); + final String tempPath = Configuration.instance.getTempDirectory() + + "${enteFile.uploadedFileID!}"; + final File? actualIoFile = await getFile(enteFile); + if (actualIoFile != null) { + final compressResult = await FlutterImageCompress.compressAndGetFile( + actualIoFile.path, + tempPath + ".jpg", + ); + if (compressResult != null) { + data = await compressResult.readAsBytes(); + } + } + stopwatch.stop(); + _logger.info( + "Getting compressed file data for uploadedFileID ${enteFile.uploadedFileID} took ${stopwatch.elapsedMilliseconds} ms", + ); + break; + } + + return data; + } + + /// Detects faces in the given image data. + /// + /// `imageData`: The image data to analyze. + /// + /// Returns a list of face detection results. + /// + /// Throws [CouldNotInitializeFaceDetector], [CouldNotRunFaceDetector] or [GeneralFaceMlException] if something goes wrong. + Future> _detectFacesIsolate( + String imagePath, + // Uint8List fileData, + { + FaceMlResultBuilder? resultBuilder, + }) async { + try { + // Get the bounding boxes of the faces + final (List faces, dataSize) = + await YoloOnnxFaceDetection.instance.predictInComputer(imagePath); + + // Add detected faces to the resultBuilder + if (resultBuilder != null) { + resultBuilder.addNewlyDetectedFaces(faces, dataSize); + } + + return faces; + } on YOLOInterpreterInitializationException { + throw CouldNotInitializeFaceDetector(); + } on YOLOInterpreterRunException { + throw CouldNotRunFaceDetector(); + } catch (e) { + _logger.severe('Face detection failed: $e'); + throw GeneralFaceMlException('Face detection failed: $e'); + } + } + + /// Detects faces in the given image data. + /// + /// `imageData`: The image data to analyze. + /// + /// Returns a list of face detection results. + /// + /// Throws [CouldNotInitializeFaceDetector], [CouldNotRunFaceDetector] or [GeneralFaceMlException] if something goes wrong. + static Future> detectFacesSync( + Image image, + ByteData imageByteData, + int interpreterAddress, { + FaceMlResultBuilder? resultBuilder, + }) async { + try { + // Get the bounding boxes of the faces + final (List faces, dataSize) = + await YoloOnnxFaceDetection.predictSync( + image, + imageByteData, + interpreterAddress, + ); + + // Add detected faces to the resultBuilder + if (resultBuilder != null) { + resultBuilder.addNewlyDetectedFaces(faces, dataSize); + } + + return faces; + } on YOLOInterpreterInitializationException { + throw CouldNotInitializeFaceDetector(); + } on YOLOInterpreterRunException { + throw CouldNotRunFaceDetector(); + } catch (e) { + dev.log('[SEVERE] Face detection failed: $e'); + throw GeneralFaceMlException('Face detection failed: $e'); + } + } + + /// Aligns multiple faces from the given image data. + /// + /// `imageData`: The image data in [Uint8List] that contains the faces. + /// `faces`: The face detection results in a list of [FaceDetectionAbsolute] for the faces to align. + /// + /// Returns a list of the aligned faces as image data. + /// + /// Throws [CouldNotWarpAffine] or [GeneralFaceMlException] if the face alignment fails. + Future _alignFaces( + String imagePath, + List faces, { + FaceMlResultBuilder? resultBuilder, + }) async { + try { + final (alignedFaces, alignmentResults, _, blurValues, originalImageSize) = + await ImageMlIsolate.instance + .preprocessMobileFaceNetOnnx(imagePath, faces); + + if (resultBuilder != null) { + resultBuilder.addAlignmentResults( + alignmentResults, + blurValues, + originalImageSize, + ); + } + + return alignedFaces; + } catch (e, s) { + _logger.severe('Face alignment failed: $e', e, s); + throw CouldNotWarpAffine(); + } + } + + /// Aligns multiple faces from the given image data. + /// + /// `imageData`: The image data in [Uint8List] that contains the faces. + /// `faces`: The face detection results in a list of [FaceDetectionAbsolute] for the faces to align. + /// + /// Returns a list of the aligned faces as image data. + /// + /// Throws [CouldNotWarpAffine] or [GeneralFaceMlException] if the face alignment fails. + static Future alignFacesSync( + Image image, + ByteData imageByteData, + List faces, { + FaceMlResultBuilder? resultBuilder, + }) async { + try { + final stopwatch = Stopwatch()..start(); + final (alignedFaces, alignmentResults, _, blurValues, originalImageSize) = + await preprocessToMobileFaceNetFloat32List( + image, + imageByteData, + faces, + ); + stopwatch.stop(); + dev.log( + "Face alignment image decoding and processing took ${stopwatch.elapsedMilliseconds} ms", + ); + + if (resultBuilder != null) { + resultBuilder.addAlignmentResults( + alignmentResults, + blurValues, + originalImageSize, + ); + } + + return alignedFaces; + } catch (e, s) { + dev.log('[SEVERE] Face alignment failed: $e $s'); + throw CouldNotWarpAffine(); + } + } + + /// Embeds multiple faces from the given input matrices. + /// + /// `facesMatrices`: The input matrices of the faces to embed. + /// + /// Returns a list of the face embeddings as lists of doubles. + /// + /// Throws [CouldNotInitializeFaceEmbeddor], [CouldNotRunFaceEmbeddor], [InputProblemFaceEmbeddor] or [GeneralFaceMlException] if the face embedding fails. + Future>> _embedFaces( + Float32List facesList, { + FaceMlResultBuilder? resultBuilder, + }) async { + try { + // Get the embedding of the faces + final List> embeddings = + await FaceEmbeddingOnnx.instance.predictInComputer(facesList); + + // Add the embeddings to the resultBuilder + if (resultBuilder != null) { + resultBuilder.addEmbeddingsToExistingFaces(embeddings); + } + + return embeddings; + } on MobileFaceNetInterpreterInitializationException { + throw CouldNotInitializeFaceEmbeddor(); + } on MobileFaceNetInterpreterRunException { + throw CouldNotRunFaceEmbeddor(); + } on MobileFaceNetEmptyInput { + throw InputProblemFaceEmbeddor("Input is empty"); + } on MobileFaceNetWrongInputSize { + throw InputProblemFaceEmbeddor("Input size is wrong"); + } on MobileFaceNetWrongInputRange { + throw InputProblemFaceEmbeddor("Input range is wrong"); + // ignore: avoid_catches_without_on_clauses + } catch (e) { + _logger.severe('Face embedding (batch) failed: $e'); + throw GeneralFaceMlException('Face embedding (batch) failed: $e'); + } + } + + static Future>> embedFacesSync( + Float32List facesList, + int interpreterAddress, { + FaceMlResultBuilder? resultBuilder, + }) async { + try { + // Get the embedding of the faces + final List> embeddings = + await FaceEmbeddingOnnx.predictSync(facesList, interpreterAddress); + + // Add the embeddings to the resultBuilder + if (resultBuilder != null) { + resultBuilder.addEmbeddingsToExistingFaces(embeddings); + } + + return embeddings; + } on MobileFaceNetInterpreterInitializationException { + throw CouldNotInitializeFaceEmbeddor(); + } on MobileFaceNetInterpreterRunException { + throw CouldNotRunFaceEmbeddor(); + } on MobileFaceNetEmptyInput { + throw InputProblemFaceEmbeddor("Input is empty"); + } on MobileFaceNetWrongInputSize { + throw InputProblemFaceEmbeddor("Input size is wrong"); + } on MobileFaceNetWrongInputRange { + throw InputProblemFaceEmbeddor("Input range is wrong"); + // ignore: avoid_catches_without_on_clauses + } catch (e) { + dev.log('[SEVERE] Face embedding (batch) failed: $e'); + throw GeneralFaceMlException('Face embedding (batch) failed: $e'); + } + } + + /// Checks if the ente file to be analyzed actually can be analyzed: it must be uploaded and in the correct format. + void _checkEnteFileForID(EnteFile enteFile) { + if (_skipAnalysisEnteFile(enteFile, {})) { + _logger.severe( + "Skipped analysis of image with enteFile ${enteFile.toString()} because it is the wrong format or has no uploadedFileID", + ); + throw CouldNotRetrieveAnyFileData(); + } + } + + bool _skipAnalysisEnteFile(EnteFile enteFile, Map indexedFileIds) { + if (isImageIndexRunning == false) { + return true; + } + // Skip if the file is not uploaded or not owned by the user + if (!enteFile.isUploaded || enteFile.isOwner == false) { + return true; + } + // Skip if the file is a video + if (enteFile.fileType == FileType.video) { + return true; + } + // I don't know how motionPhotos and livePhotos work, so I'm also just skipping them for now + if (enteFile.fileType == FileType.other) { + return true; + } + // Skip if the file is already analyzed with the latest ml version + final id = enteFile.uploadedFileID!; + + return indexedFileIds.containsKey(id) && + indexedFileIds[id]! >= faceMlVersion; + } + + Future _checkForExistingUpToDateResult( + EnteFile enteFile, + ) async { + // Check if the image has already been analyzed and stored in the database + final existingResult = + await MlDataDB.instance.getFaceMlResult(enteFile.uploadedFileID!); + + // If the image has already been analyzed and stored in the database, return the stored result + if (existingResult != null) { + if (existingResult.mlVersion >= faceMlVersion) { + _logger.info( + "Image with uploadedFileID ${enteFile.uploadedFileID} has already been analyzed and stored in the database with the latest ml version. Returning the stored result.", + ); + return existingResult; + } + } + return null; + } +} diff --git a/mobile/lib/services/face_ml/face_ml_version.dart b/mobile/lib/services/machine_learning/face_ml/face_ml_version.dart similarity index 100% rename from mobile/lib/services/face_ml/face_ml_version.dart rename to mobile/lib/services/machine_learning/face_ml/face_ml_version.dart diff --git a/mobile/lib/services/face_ml/face_search_service.dart b/mobile/lib/services/machine_learning/face_ml/face_search_service.dart similarity index 100% rename from mobile/lib/services/face_ml/face_search_service.dart rename to mobile/lib/services/machine_learning/face_ml/face_search_service.dart diff --git a/mobile/lib/services/face_ml/feedback/cluster_feedback.dart b/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart similarity index 99% rename from mobile/lib/services/face_ml/feedback/cluster_feedback.dart rename to mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart index 129eee83f3..6a6fc28c0c 100644 --- a/mobile/lib/services/face_ml/feedback/cluster_feedback.dart +++ b/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart @@ -11,7 +11,7 @@ import "package:photos/face/db.dart"; import "package:photos/face/model/person.dart"; import "package:photos/generated/protos/ente/common/vector.pb.dart"; import "package:photos/models/file/file.dart"; -import "package:photos/services/face_ml/face_clustering/cosine_distance.dart"; +import 'package:photos/services/machine_learning/face_ml/face_clustering/cosine_distance.dart'; import "package:photos/services/search_service.dart"; class ClusterFeedbackService { diff --git a/mobile/lib/services/face_ml/model_file.dart b/mobile/lib/services/machine_learning/face_ml/model_file.dart similarity index 100% rename from mobile/lib/services/face_ml/model_file.dart rename to mobile/lib/services/machine_learning/face_ml/model_file.dart diff --git a/mobile/lib/services/machine_learning/file_ml/remote_fileml_service.dart b/mobile/lib/services/machine_learning/file_ml/remote_fileml_service.dart index d8b39f0425..95db6c3811 100644 --- a/mobile/lib/services/machine_learning/file_ml/remote_fileml_service.dart +++ b/mobile/lib/services/machine_learning/file_ml/remote_fileml_service.dart @@ -123,7 +123,8 @@ class RemoteFileMLService { } Future> decryptFileMLComputer( - Map args) async { + Map args, + ) async { final result = {}; final inputs = args["inputs"] as List; for (final input in inputs) { diff --git a/mobile/lib/ui/settings/debug/face_debug_section_widget.dart b/mobile/lib/ui/settings/debug/face_debug_section_widget.dart index 68fa8392e9..4ee93b004c 100644 --- a/mobile/lib/ui/settings/debug/face_debug_section_widget.dart +++ b/mobile/lib/ui/settings/debug/face_debug_section_widget.dart @@ -8,9 +8,10 @@ import "package:photos/events/people_changed_event.dart"; import "package:photos/extensions/stop_watch.dart"; import "package:photos/face/db.dart"; import "package:photos/face/model/person.dart"; -import "package:photos/models/ml/ml_versions.dart"; -import "package:photos/services/face_ml/face_ml_service.dart"; -import "package:photos/services/face_ml/feedback/cluster_feedback.dart"; +import "package:photos/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart"; +import 'package:photos/services/machine_learning/face_ml/face_ml_service.dart'; +import 'package:photos/services/machine_learning/face_ml/feedback/cluster_feedback.dart'; +// import "package:photos/services/search_service.dart"; import 'package:photos/theme/ente_theme.dart'; import 'package:photos/ui/components/captioned_text_widget.dart'; import 'package:photos/ui/components/expandable_menu_item_widget.dart'; @@ -93,6 +94,62 @@ class _FaceDebugSectionWidgetState extends State { } }, ), + MenuItemWidget( + captionedTextWidget: FutureBuilder( + future: FaceMLDataDB.instance.getTotalFaceCount(), + builder: (context, snapshot) { + if (snapshot.hasData) { + return CaptionedTextWidget( + title: "${snapshot.data!} high quality faces", + ); + } + return const SizedBox.shrink(); + }, + ), + pressedColor: getEnteColorScheme(context).fillFaint, + trailingIcon: Icons.chevron_right_outlined, + trailingIconIsMuted: true, + onTap: () async { + final faces75 = await FaceMLDataDB.instance + .getTotalFaceCount(minFaceScore: 0.75); + final faces78 = await FaceMLDataDB.instance + .getTotalFaceCount(minFaceScore: kMinHighQualityFaceScore); + showShortToast(context, "Faces75: $faces75, Faces78: $faces78"); + }, + ), + // MenuItemWidget( + // captionedTextWidget: const CaptionedTextWidget( + // title: "Analyze file ID 25728869", + // ), + // pressedColor: getEnteColorScheme(context).fillFaint, + // trailingIcon: Icons.chevron_right_outlined, + // trailingIconIsMuted: true, + // onTap: () async { + // try { + // final enteFile = await SearchService.instance.getAllFiles().then( + // (value) => value.firstWhere( + // (element) => element.uploadedFileID == 25728869, + // ), + // ); + // _logger.info( + // 'File with ID ${enteFile.uploadedFileID} has name ${enteFile.displayName}', + // ); + // FaceMlService.instance.isImageIndexRunning = true; + // final result = await FaceMlService.instance + // .analyzeImageInSingleIsolate(enteFile); + // if (result != null) { + // final resultJson = result.toJsonString(); + // _logger.info('result: $resultJson'); + // } + // FaceMlService.instance.isImageIndexRunning = false; + // } catch (e, s) { + // _logger.severe('indexing failed ', e, s); + // await showGenericErrorDialog(context: context, error: e); + // } finally { + // FaceMlService.instance.isImageIndexRunning = false; + // } + // }, + // ), MenuItemWidget( captionedTextWidget: const CaptionedTextWidget( title: "Run Clustering", @@ -101,7 +158,8 @@ class _FaceDebugSectionWidgetState extends State { trailingIcon: Icons.chevron_right_outlined, trailingIconIsMuted: true, onTap: () async { - await FaceMlService.instance.clusterAllImages(minFaceScore: 0.75); + await FaceMlService.instance + .clusterAllImages(minFaceScore: 0.75, clusterInBuckets: true); Bus.instance.fire(PeopleChangedEvent()); showShortToast(context, "Done"); }, diff --git a/mobile/lib/ui/viewer/actions/file_selection_actions_widget.dart b/mobile/lib/ui/viewer/actions/file_selection_actions_widget.dart index 3692696739..a87cca795b 100644 --- a/mobile/lib/ui/viewer/actions/file_selection_actions_widget.dart +++ b/mobile/lib/ui/viewer/actions/file_selection_actions_widget.dart @@ -20,8 +20,8 @@ import 'package:photos/models/gallery_type.dart'; import "package:photos/models/metadata/common_keys.dart"; import 'package:photos/models/selected_files.dart'; import 'package:photos/services/collections_service.dart'; -import "package:photos/services/face_ml/feedback/cluster_feedback.dart"; import 'package:photos/services/hidden_service.dart'; +import 'package:photos/services/machine_learning/face_ml/feedback/cluster_feedback.dart'; import "package:photos/theme/colors.dart"; import "package:photos/theme/ente_theme.dart"; import 'package:photos/ui/actions/collection/collection_file_actions.dart'; diff --git a/mobile/lib/ui/viewer/file_details/face_widget.dart b/mobile/lib/ui/viewer/file_details/face_widget.dart index a7045f92c2..42ae6b0a10 100644 --- a/mobile/lib/ui/viewer/file_details/face_widget.dart +++ b/mobile/lib/ui/viewer/file_details/face_widget.dart @@ -1,4 +1,5 @@ import "dart:developer" show log; +import "dart:io" show Platform; import "dart:typed_data"; import "package:flutter/material.dart"; @@ -9,6 +10,7 @@ import 'package:photos/models/file/file.dart'; import "package:photos/services/search_service.dart"; import "package:photos/ui/viewer/file/no_thumbnail_widget.dart"; import "package:photos/ui/viewer/people/cluster_page.dart"; +import "package:photos/ui/viewer/people/cropped_face_image_view.dart"; import "package:photos/ui/viewer/people/people_page.dart"; import "package:photos/utils/face/face_box_crop.dart"; import "package:photos/utils/thumbnail_util.dart"; @@ -29,11 +31,104 @@ class FaceWidget extends StatelessWidget { @override Widget build(BuildContext context) { - return FutureBuilder( - future: getFaceCrop(), - builder: (context, snapshot) { - if (snapshot.hasData) { - final ImageProvider imageProvider = MemoryImage(snapshot.data!); + if (Platform.isIOS) { + return FutureBuilder( + future: getFaceCrop(), + builder: (context, snapshot) { + if (snapshot.hasData) { + final ImageProvider imageProvider = MemoryImage(snapshot.data!); + return GestureDetector( + onTap: () async { + log( + "FaceWidget is tapped, with person $person and clusterID $clusterID", + name: "FaceWidget", + ); + if (person == null && clusterID == null) { + return; + } + if (person != null) { + await Navigator.of(context).push( + MaterialPageRoute( + builder: (context) => PeoplePage( + person: person!, + ), + ), + ); + } else if (clusterID != null) { + final fileIdsToClusterIds = + await FaceMLDataDB.instance.getFileIdToClusterIds(); + final files = await SearchService.instance.getAllFiles(); + final clusterFiles = files + .where( + (file) => + fileIdsToClusterIds[file.uploadedFileID] + ?.contains(clusterID) ?? + false, + ) + .toList(); + await Navigator.of(context).push( + MaterialPageRoute( + builder: (context) => ClusterPage( + clusterFiles, + cluserID: clusterID!, + ), + ), + ); + } + }, + child: Column( + children: [ + ClipRRect( + borderRadius: + const BorderRadius.all(Radius.elliptical(16, 12)), + child: SizedBox( + width: 60, + height: 60, + child: Image( + image: imageProvider, + fit: BoxFit.cover, + ), + ), + ), + const SizedBox(height: 8), + if (person != null) + Text( + person!.attr.name.trim(), + style: Theme.of(context).textTheme.bodySmall, + overflow: TextOverflow.ellipsis, + maxLines: 1, + ), + ], + ), + ); + } else { + if (snapshot.connectionState == ConnectionState.waiting) { + return const ClipRRect( + borderRadius: BorderRadius.all(Radius.elliptical(16, 12)), + child: SizedBox( + width: 60, // Ensure consistent sizing + height: 60, + child: CircularProgressIndicator(), + ), + ); + } + if (snapshot.hasError) { + log('Error getting face: ${snapshot.error}'); + } + return const ClipRRect( + borderRadius: BorderRadius.all(Radius.elliptical(16, 12)), + child: SizedBox( + width: 60, // Ensure consistent sizing + height: 60, + child: NoThumbnailWidget(), + ), + ); + } + }, + ); + } else { + return Builder( + builder: (context) { return GestureDetector( onTap: () async { log( @@ -81,9 +176,9 @@ class FaceWidget extends StatelessWidget { child: SizedBox( width: 60, height: 60, - child: Image( - image: imageProvider, - fit: BoxFit.cover, + child: CroppedFaceImageView( + enteFile: file, + face: face, ), ), ), @@ -98,31 +193,9 @@ class FaceWidget extends StatelessWidget { ], ), ); - } else { - if (snapshot.connectionState == ConnectionState.waiting) { - return const ClipRRect( - borderRadius: BorderRadius.all(Radius.elliptical(16, 12)), - child: SizedBox( - width: 60, // Ensure consistent sizing - height: 60, - child: CircularProgressIndicator(), - ), - ); - } - if (snapshot.hasError) { - log('Error getting face: ${snapshot.error}'); - } - return const ClipRRect( - borderRadius: BorderRadius.all(Radius.elliptical(16, 12)), - child: SizedBox( - width: 60, // Ensure consistent sizing - height: 60, - child: NoThumbnailWidget(), - ), - ); - } - }, - ); + }, + ); + } } Future getFaceCrop() async { diff --git a/mobile/lib/ui/viewer/file_details/faces_item_widget.dart b/mobile/lib/ui/viewer/file_details/faces_item_widget.dart index fb2d8cbd53..68de105c7b 100644 --- a/mobile/lib/ui/viewer/file_details/faces_item_widget.dart +++ b/mobile/lib/ui/viewer/file_details/faces_item_widget.dart @@ -36,9 +36,18 @@ class FacesItemWidget extends StatelessWidget { ]; } - final List faces = await FaceMLDataDB.instance + final List? faces = await FaceMLDataDB.instance .getFacesForGivenFileID(file.uploadedFileID!); - if (faces.isEmpty || faces.every((face) => face.score < 0.5)) { + if (faces == null) { + return [ + const ChipButtonWidget( + "Image not analyzed", + noChips: true, + ), + ]; + } + if (faces.isEmpty || + faces.every((face) => face.score < 0.75 || face.isBlurry)) { return [ const ChipButtonWidget( "No faces found", @@ -50,6 +59,9 @@ class FacesItemWidget extends StatelessWidget { // Sort the faces by score in descending order, so that the highest scoring face is first. faces.sort((Face a, Face b) => b.score.compareTo(a.score)); + // Remove faces with low scores and blurry faces + faces.removeWhere((face) => face.isHighQuality == false); + // TODO: add deduplication of faces of same person final faceIdsToClusterIds = await FaceMLDataDB.instance .getFaceIdsToClusterIds(faces.map((face) => face.faceID)); diff --git a/mobile/lib/ui/viewer/people/add_person_action_sheet.dart b/mobile/lib/ui/viewer/people/add_person_action_sheet.dart index 4a072280fe..935af98801 100644 --- a/mobile/lib/ui/viewer/people/add_person_action_sheet.dart +++ b/mobile/lib/ui/viewer/people/add_person_action_sheet.dart @@ -10,7 +10,7 @@ import "package:photos/events/people_changed_event.dart"; import "package:photos/face/db.dart"; import "package:photos/face/model/person.dart"; import "package:photos/generated/l10n.dart"; -import "package:photos/services/face_ml/feedback/cluster_feedback.dart"; +import 'package:photos/services/machine_learning/face_ml/feedback/cluster_feedback.dart'; import 'package:photos/theme/colors.dart'; import 'package:photos/theme/ente_theme.dart'; import 'package:photos/ui/common/loading_widget.dart'; diff --git a/mobile/lib/ui/viewer/people/cropped_face_image_view.dart b/mobile/lib/ui/viewer/people/cropped_face_image_view.dart new file mode 100644 index 0000000000..04980098fd --- /dev/null +++ b/mobile/lib/ui/viewer/people/cropped_face_image_view.dart @@ -0,0 +1,117 @@ +import 'dart:developer' show log; +import "dart:io" show File; + +import 'package:flutter/material.dart'; +import "package:photos/face/model/face.dart"; +import "package:photos/models/file/file.dart"; +import "package:photos/ui/viewer/file/thumbnail_widget.dart"; +import "package:photos/utils/file_util.dart"; + +class CroppedFaceInfo { + final Image image; + final double scale; + final double offsetX; + final double offsetY; + + const CroppedFaceInfo({ + required this.image, + required this.scale, + required this.offsetX, + required this.offsetY, + }); +} + +class CroppedFaceImageView extends StatelessWidget { + final EnteFile enteFile; + final Face face; + + const CroppedFaceImageView({ + Key? key, + required this.enteFile, + required this.face, + }) : super(key: key); + + @override + Widget build(BuildContext context) { + return FutureBuilder( + future: getImage(), + builder: (context, snapshot) { + if (snapshot.hasData) { + return LayoutBuilder( + builder: (BuildContext context, BoxConstraints constraints) { + final Image image = snapshot.data!; + + final double viewWidth = constraints.maxWidth; + final double viewHeight = constraints.maxHeight; + + final faceBox = face.detection.box; + + final double relativeFaceCenterX = + faceBox.xMin + faceBox.width / 2; + final double relativeFaceCenterY = + faceBox.yMin + faceBox.height / 2; + + const double desiredFaceHeightRelativeToWidget = 1 / 2; + final double scale = + (1 / faceBox.height) * desiredFaceHeightRelativeToWidget; + + final double widgetCenterX = viewWidth / 2; + final double widgetCenterY = viewHeight / 2; + + final double imageAspectRatio = enteFile.width / enteFile.height; + final double widgetAspectRatio = viewWidth / viewHeight; + final double imageToWidgetRatio = + imageAspectRatio / widgetAspectRatio; + + double offsetX = + (widgetCenterX - relativeFaceCenterX * viewWidth) * scale; + double offsetY = + (widgetCenterY - relativeFaceCenterY * viewHeight) * scale; + + if (imageAspectRatio > widgetAspectRatio) { + // Landscape Image: Adjust offsetX more conservatively + offsetX = offsetX * imageToWidgetRatio; + } else { + // Portrait Image: Adjust offsetY more conservatively + offsetY = offsetY / imageToWidgetRatio; + } + + return ClipRect( + clipBehavior: Clip.antiAlias, + child: Transform.translate( + offset: Offset( + offsetX, + offsetY, + ), + child: Transform.scale( + scale: scale, + child: image, + ), + ), + ); + }, + ); + } else { + if (snapshot.hasError) { + log('Error getting cover face for person: ${snapshot.error}'); + } + return ThumbnailWidget( + enteFile, + ); + } + }, + ); + } + + Future getImage() async { + final File? ioFile = await getFile(enteFile); + if (ioFile == null) { + return null; + } + + final imageData = await ioFile.readAsBytes(); + final image = Image.memory(imageData, fit: BoxFit.cover); + + return image; + } +} diff --git a/mobile/lib/ui/viewer/people/person_cluster_suggestion.dart b/mobile/lib/ui/viewer/people/person_cluster_suggestion.dart index 3ec179856e..1bbcb4390b 100644 --- a/mobile/lib/ui/viewer/people/person_cluster_suggestion.dart +++ b/mobile/lib/ui/viewer/people/person_cluster_suggestion.dart @@ -6,7 +6,7 @@ import "package:photos/events/people_changed_event.dart"; import "package:photos/face/db.dart"; import "package:photos/face/model/person.dart"; import "package:photos/models/file/file.dart"; -import "package:photos/services/face_ml/feedback/cluster_feedback.dart"; +import 'package:photos/services/machine_learning/face_ml/feedback/cluster_feedback.dart'; import "package:photos/theme/ente_theme.dart"; import "package:photos/ui/components/buttons/button_widget.dart"; import "package:photos/ui/components/models/button_type.dart"; diff --git a/mobile/lib/ui/viewer/search_tab/people_section.dart b/mobile/lib/ui/viewer/search_tab/people_section.dart index 25ea6bc6ba..e9d24a9e73 100644 --- a/mobile/lib/ui/viewer/search_tab/people_section.dart +++ b/mobile/lib/ui/viewer/search_tab/people_section.dart @@ -170,7 +170,6 @@ class SearchExampleRow extends StatelessWidget { ), ); }); - scrollableExamples.add(SearchSectionCTAIcon(sectionType)); return SizedBox( child: SingleChildScrollView( physics: const BouncingScrollPhysics(), @@ -237,7 +236,9 @@ class SearchExample extends StatelessWidget { child: searchResult.previewThumbnail() != null ? Hero( tag: heroTag, - child: ClipOval( + child: ClipRRect( + borderRadius: + const BorderRadius.all(Radius.elliptical(16, 12)), child: searchResult.type() != ResultType.faces ? ThumbnailWidget( searchResult.previewThumbnail()!, @@ -246,7 +247,9 @@ class SearchExample extends StatelessWidget { : FaceSearchResult(searchResult, heroTag), ), ) - : const ClipOval( + : const ClipRRect( + borderRadius: + BorderRadius.all(Radius.elliptical(16, 12)), child: NoThumbnailWidget( addBorder: false, ), diff --git a/mobile/lib/utils/image_ml_isolate.dart b/mobile/lib/utils/image_ml_isolate.dart index f9869ef8fd..157615d8e7 100644 --- a/mobile/lib/utils/image_ml_isolate.dart +++ b/mobile/lib/utils/image_ml_isolate.dart @@ -9,8 +9,8 @@ import 'package:flutter_isolate/flutter_isolate.dart'; import "package:logging/logging.dart"; import "package:photos/face/model/box.dart"; import 'package:photos/models/ml/ml_typedefs.dart'; -import "package:photos/services/face_ml/face_alignment/alignment_result.dart"; -import "package:photos/services/face_ml/face_detection/detection.dart"; +import 'package:photos/services/machine_learning/face_ml/face_alignment/alignment_result.dart'; +import 'package:photos/services/machine_learning/face_ml/face_detection/detection.dart'; import "package:photos/utils/image_ml_util.dart"; import "package:synchronized/synchronized.dart"; diff --git a/mobile/lib/utils/image_ml_util.dart b/mobile/lib/utils/image_ml_util.dart index fd04cc1c14..1ba29df6b0 100644 --- a/mobile/lib/utils/image_ml_util.dart +++ b/mobile/lib/utils/image_ml_util.dart @@ -18,10 +18,10 @@ import 'package:flutter/painting.dart' as paint show decodeImageFromList; import 'package:ml_linalg/linalg.dart'; import "package:photos/face/model/box.dart"; import 'package:photos/models/ml/ml_typedefs.dart'; -import "package:photos/services/face_ml/blur_detection/blur_detection_service.dart"; -import "package:photos/services/face_ml/face_alignment/alignment_result.dart"; -import "package:photos/services/face_ml/face_alignment/similarity_transform.dart"; -import "package:photos/services/face_ml/face_detection/detection.dart"; +import 'package:photos/services/machine_learning/face_ml/face_alignment/alignment_result.dart'; +import 'package:photos/services/machine_learning/face_ml/face_alignment/similarity_transform.dart'; +import 'package:photos/services/machine_learning/face_ml/face_detection/detection.dart'; +import 'package:photos/services/machine_learning/face_ml/face_filtering/blur_detection_service.dart'; /// All of the functions in this file are helper functions for the [ImageMlIsolate] isolate. /// Don't use them outside of the isolate, unless you are okay with UI jank!!!!