From 37ab467da5631e4fce2bdff57534cf3bdc5dd4fe Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Mon, 22 Apr 2024 11:35:40 +0530 Subject: [PATCH 01/69] [mob][photos] Remove blur ranking debug option --- .../debug/face_debug_section_widget.dart | 57 +++++++++---------- 1 file changed, 28 insertions(+), 29 deletions(-) diff --git a/mobile/lib/ui/settings/debug/face_debug_section_widget.dart b/mobile/lib/ui/settings/debug/face_debug_section_widget.dart index f1c835fa26..6ce7e0609b 100644 --- a/mobile/lib/ui/settings/debug/face_debug_section_widget.dart +++ b/mobile/lib/ui/settings/debug/face_debug_section_widget.dart @@ -8,7 +8,6 @@ import "package:photos/events/people_changed_event.dart"; import "package:photos/face/db.dart"; import "package:photos/face/model/person.dart"; import 'package:photos/services/machine_learning/face_ml/face_ml_service.dart'; -import "package:photos/services/machine_learning/face_ml/feedback/cluster_feedback.dart"; import "package:photos/services/machine_learning/face_ml/person/person_service.dart"; import 'package:photos/theme/ente_theme.dart'; import 'package:photos/ui/components/captioned_text_widget.dart'; @@ -284,34 +283,34 @@ class _FaceDebugSectionWidgetState extends State { ); }, ), - sectionOptionSpacing, - MenuItemWidget( - captionedTextWidget: const CaptionedTextWidget( - title: "Rank blurs", - ), - pressedColor: getEnteColorScheme(context).fillFaint, - trailingIcon: Icons.chevron_right_outlined, - trailingIconIsMuted: true, - onTap: () async { - await showChoiceDialog( - context, - title: "Are you sure?", - body: - "This will delete all clusters and put blurry faces in separate clusters per ten points.", - firstButtonLabel: "Yes, confirm", - firstButtonOnTap: () async { - try { - await ClusterFeedbackService.instance - .createFakeClustersByBlurValue(); - showShortToast(context, "Done"); - } catch (e, s) { - _logger.warning('Failed to rank faces on blur values ', e, s); - await showGenericErrorDialog(context: context, error: e); - } - }, - ); - }, - ), + // sectionOptionSpacing, + // MenuItemWidget( + // captionedTextWidget: const CaptionedTextWidget( + // title: "Rank blurs", + // ), + // pressedColor: getEnteColorScheme(context).fillFaint, + // trailingIcon: Icons.chevron_right_outlined, + // trailingIconIsMuted: true, + // onTap: () async { + // await showChoiceDialog( + // context, + // title: "Are you sure?", + // body: + // "This will delete all clusters and put blurry faces in separate clusters per ten points.", + // firstButtonLabel: "Yes, confirm", + // firstButtonOnTap: () async { + // try { + // await ClusterFeedbackService.instance + // .createFakeClustersByBlurValue(); + // showShortToast(context, "Done"); + // } catch (e, s) { + // _logger.warning('Failed to rank faces on blur values ', e, s); + // await showGenericErrorDialog(context: context, error: e); + // } + // }, + // ); + // }, + // ), sectionOptionSpacing, MenuItemWidget( captionedTextWidget: const CaptionedTextWidget( From ae046e33b49f487ca67a566ee27baff6cc1d1419 Mon Sep 17 00:00:00 2001 From: ashilkn Date: Mon, 22 Apr 2024 11:45:54 +0530 Subject: [PATCH 02/69] [mob] fix: face thumbnails getting cropped on the edges because the image uses BoxFit.cover --- mobile/lib/ui/viewer/file_details/face_widget.dart | 12 ++++-------- .../ui/viewer/people/cropped_face_image_view.dart | 6 +++--- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/mobile/lib/ui/viewer/file_details/face_widget.dart b/mobile/lib/ui/viewer/file_details/face_widget.dart index 79637af829..beaff4d16b 100644 --- a/mobile/lib/ui/viewer/file_details/face_widget.dart +++ b/mobile/lib/ui/viewer/file_details/face_widget.dart @@ -48,7 +48,7 @@ class _FaceWidgetState extends State { @override Widget build(BuildContext context) { - if (Platform.isIOS) { + if (Platform.isAndroid) { return FutureBuilder( future: getFaceCrop(), builder: (context, snapshot) { @@ -280,13 +280,9 @@ class _FaceWidgetState extends State { child: ClipRRect( borderRadius: const BorderRadius.all(Radius.elliptical(16, 12)), - child: SizedBox( - width: 60, - height: 60, - child: CroppedFaceImageView( - enteFile: widget.file, - face: widget.face, - ), + child: CroppedFaceImageView( + enteFile: widget.file, + face: widget.face, ), ), ), diff --git a/mobile/lib/ui/viewer/people/cropped_face_image_view.dart b/mobile/lib/ui/viewer/people/cropped_face_image_view.dart index 04980098fd..d4877a2d01 100644 --- a/mobile/lib/ui/viewer/people/cropped_face_image_view.dart +++ b/mobile/lib/ui/viewer/people/cropped_face_image_view.dart @@ -51,7 +51,7 @@ class CroppedFaceImageView extends StatelessWidget { final double relativeFaceCenterY = faceBox.yMin + faceBox.height / 2; - const double desiredFaceHeightRelativeToWidget = 1 / 2; + const double desiredFaceHeightRelativeToWidget = 7 / 10; final double scale = (1 / faceBox.height) * desiredFaceHeightRelativeToWidget; @@ -68,7 +68,7 @@ class CroppedFaceImageView extends StatelessWidget { double offsetY = (widgetCenterY - relativeFaceCenterY * viewHeight) * scale; - if (imageAspectRatio > widgetAspectRatio) { + if (imageAspectRatio < widgetAspectRatio) { // Landscape Image: Adjust offsetX more conservatively offsetX = offsetX * imageToWidgetRatio; } else { @@ -110,7 +110,7 @@ class CroppedFaceImageView extends StatelessWidget { } final imageData = await ioFile.readAsBytes(); - final image = Image.memory(imageData, fit: BoxFit.cover); + final image = Image.memory(imageData, fit: BoxFit.contain); return image; } From c1587cc5ea851d858e4ca638f346bae73a287a80 Mon Sep 17 00:00:00 2001 From: ashilkn Date: Mon, 22 Apr 2024 11:52:39 +0530 Subject: [PATCH 03/69] [mob] Remove redundant clippling and anti aliasing for performance gain --- mobile/lib/ui/viewer/file_details/face_widget.dart | 12 ++++-------- .../ui/viewer/people/cropped_face_image_view.dart | 4 ++-- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/mobile/lib/ui/viewer/file_details/face_widget.dart b/mobile/lib/ui/viewer/file_details/face_widget.dart index beaff4d16b..898dcf8afa 100644 --- a/mobile/lib/ui/viewer/file_details/face_widget.dart +++ b/mobile/lib/ui/viewer/file_details/face_widget.dart @@ -48,7 +48,7 @@ class _FaceWidgetState extends State { @override Widget build(BuildContext context) { - if (Platform.isAndroid) { + if (Platform.isIOS) { return FutureBuilder( future: getFaceCrop(), builder: (context, snapshot) { @@ -277,13 +277,9 @@ class _FaceWidgetState extends State { : BorderSide.none, ), ), - child: ClipRRect( - borderRadius: - const BorderRadius.all(Radius.elliptical(16, 12)), - child: CroppedFaceImageView( - enteFile: widget.file, - face: widget.face, - ), + child: CroppedFaceImageView( + enteFile: widget.file, + face: widget.face, ), ), const SizedBox(height: 8), diff --git a/mobile/lib/ui/viewer/people/cropped_face_image_view.dart b/mobile/lib/ui/viewer/people/cropped_face_image_view.dart index d4877a2d01..2c7768f7fd 100644 --- a/mobile/lib/ui/viewer/people/cropped_face_image_view.dart +++ b/mobile/lib/ui/viewer/people/cropped_face_image_view.dart @@ -76,8 +76,8 @@ class CroppedFaceImageView extends StatelessWidget { offsetY = offsetY / imageToWidgetRatio; } - return ClipRect( - clipBehavior: Clip.antiAlias, + return ClipRRect( + borderRadius: const BorderRadius.all(Radius.elliptical(16, 12)), child: Transform.translate( offset: Offset( offsetX, From fa466d715f7862ed39bf3352217cb82b4e3ff97a Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Mon, 22 Apr 2024 16:40:31 +0530 Subject: [PATCH 04/69] [mob][photos] Improve suggestions by improving speed and preferring big clusters --- .../face_filtering_constants.dart | 3 + .../face_ml/feedback/cluster_feedback.dart | 198 +++++++++--------- mobile/lib/services/search_service.dart | 3 +- 3 files changed, 104 insertions(+), 100 deletions(-) diff --git a/mobile/lib/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart b/mobile/lib/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart index b1f2f60183..0feb275a7b 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart @@ -15,3 +15,6 @@ const kHighQualityFaceScore = 0.90; /// The minimum score for a face to be detected, regardless of quality. Use [kMinimumQualityFaceScore] for high quality faces. const kMinFaceDetectionScore = FaceDetectionService.kMinScoreSigmoidThreshold; + +/// The minimum cluster size for displaying a cluster in the UI +const kMinimumClusterSizeSearchResult = 20; diff --git a/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart b/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart index 95497a90d2..e30dc375fe 100644 --- a/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart +++ b/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart @@ -15,6 +15,7 @@ import "package:photos/generated/protos/ente/common/vector.pb.dart"; import "package:photos/models/file/file.dart"; import 'package:photos/services/machine_learning/face_ml/face_clustering/cosine_distance.dart'; import "package:photos/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart"; +import "package:photos/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart"; import "package:photos/services/machine_learning/face_ml/face_ml_result.dart"; import "package:photos/services/machine_learning/face_ml/person/person_service.dart"; import "package:photos/services/search_service.dart"; @@ -65,7 +66,7 @@ class ClusterFeedbackService { try { // Get the suggestions for the person using centroids and median final List<(int, double, bool)> suggestClusterIds = - await _getSuggestionsUsingMedian(person); + await _getSuggestions(person); // Get the files for the suggestions final Map> fileIdToClusterID = @@ -241,7 +242,7 @@ class ClusterFeedbackService { watch.log('computed avg for ${clusterAvg.length} clusters'); // Find the actual closest clusters for the person - final Map> suggestions = _calcSuggestionsMean( + final List<(int, double)> suggestions = _calcSuggestionsMean( clusterAvg, personClusters, ignoredClusters, @@ -257,21 +258,17 @@ class ClusterFeedbackService { } // log suggestions - for (final entry in suggestions.entries) { - dev.log( - ' ${entry.value.length} suggestion for ${p.data.name} for cluster ID ${entry.key} are suggestions ${entry.value}}', - name: "ClusterFeedbackService", - ); - } + dev.log( + 'suggestions for ${p.data.name} for cluster ID ${p.remoteID} are suggestions $suggestions}', + name: "ClusterFeedbackService", + ); - for (final suggestionsPerCluster in suggestions.values) { - for (final suggestion in suggestionsPerCluster) { - final clusterID = suggestion.$1; - await PersonService.instance.assignClusterToPerson( - personID: p.remoteID, - clusterID: clusterID, - ); - } + for (final suggestion in suggestions) { + final clusterID = suggestion.$1; + await PersonService.instance.assignClusterToPerson( + personID: p.remoteID, + clusterID: clusterID, + ); } Bus.instance.fire(PeopleChangedEvent()); @@ -433,111 +430,77 @@ class ClusterFeedbackService { return; } - /// Returns a map of person's clusterID to map of closest clusterID to with disstance - Future>> getSuggestionsUsingMean( - PersonEntity p, { - double maxClusterDistance = 0.4, - }) async { - // Get all the cluster data - final faceMlDb = FaceMLDataDB.instance; - - final allClusterIdsToCountMap = (await faceMlDb.clusterIdToFaceCount()); - final ignoredClusters = await faceMlDb.getPersonIgnoredClusters(p.remoteID); - final personClusters = await faceMlDb.getPersonClusterIDs(p.remoteID); - dev.log( - 'existing clusters for ${p.data.name} are $personClusters', - name: "ClusterFeedbackService", - ); - - // Get and update the cluster summary to get the avg (centroid) and count - final EnteWatch watch = EnteWatch("ClusterFeedbackService")..start(); - final Map> clusterAvg = await _getUpdateClusterAvg( - allClusterIdsToCountMap, - ignoredClusters, - ); - watch.log('computed avg for ${clusterAvg.length} clusters'); - - // Find the actual closest clusters for the person - final Map> suggestions = _calcSuggestionsMean( - clusterAvg, - personClusters, - ignoredClusters, - maxClusterDistance, - ); - - // log suggestions - for (final entry in suggestions.entries) { - dev.log( - ' ${entry.value.length} suggestion for ${p.data.name} for cluster ID ${entry.key} are suggestions ${entry.value}}', - name: "ClusterFeedbackService", - ); - } - return suggestions; - } - /// Returns a list of suggestions. For each suggestion we return a record consisting of the following elements: /// 1. clusterID: the ID of the cluster /// 2. distance: the distance between the person's cluster and the suggestion /// 3. usedMean: whether the suggestion was found using the mean (true) or the median (false) - Future> _getSuggestionsUsingMedian( + Future> _getSuggestions( PersonEntity p, { int sampleSize = 50, double maxMedianDistance = 0.65, double goodMedianDistance = 0.55, double maxMeanDistance = 0.65, - double goodMeanDistance = 0.4, + double goodMeanDistance = 0.5, }) async { // Get all the cluster data + final startTime = DateTime.now(); final faceMlDb = FaceMLDataDB.instance; // final Map> suggestions = {}; - final allClusterIdsToCountMap = (await faceMlDb.clusterIdToFaceCount()); + final allClusterIdsToCountMap = await faceMlDb.clusterIdToFaceCount(); final ignoredClusters = await faceMlDb.getPersonIgnoredClusters(p.remoteID); final personClusters = await faceMlDb.getPersonClusterIDs(p.remoteID); dev.log( - 'existing clusters for ${p.data.name} are $personClusters', + 'existing clusters for ${p.data.name} are $personClusters, getting all database data took ${DateTime.now().difference(startTime).inMilliseconds} ms', name: "getSuggestionsUsingMedian", ); - // Get and update the cluster summary to get the avg (centroid) and count + // First only do a simple check on the big clusters final EnteWatch watch = EnteWatch("ClusterFeedbackService")..start(); + final Map> clusterAvgBigClusters = + await _getUpdateClusterAvg( + allClusterIdsToCountMap, + ignoredClusters, + minClusterSize: kMinimumClusterSizeSearchResult, + ); + dev.log( + 'computed avg for ${clusterAvgBigClusters.length} clusters, in ${DateTime.now().difference(startTime).inMilliseconds} ms', + ); + final List<(int, double)> suggestionsMeanBigClusters = _calcSuggestionsMean( + clusterAvgBigClusters, + personClusters, + ignoredClusters, + goodMeanDistance, + ); + if (suggestionsMeanBigClusters.isNotEmpty) { + return suggestionsMeanBigClusters + .map((e) => (e.$1, e.$2, true)) + .toList(growable: false); + } + + // Get and update the cluster summary to get the avg (centroid) and count final Map> clusterAvg = await _getUpdateClusterAvg( allClusterIdsToCountMap, ignoredClusters, ); - watch.log('computed avg for ${clusterAvg.length} clusters'); + dev.log( + 'computed avg for ${clusterAvg.length} clusters, in ${DateTime.now().difference(startTime).inMilliseconds} ms', + ); // Find the other cluster candidates based on the mean - final Map> suggestionsMean = _calcSuggestionsMean( + final List<(int, double)> suggestionsMean = _calcSuggestionsMean( clusterAvg, personClusters, ignoredClusters, goodMeanDistance, ); if (suggestionsMean.isNotEmpty) { - final List<(int, double)> suggestClusterIds = []; - for (final List<(int, double)> suggestion in suggestionsMean.values) { - suggestClusterIds.addAll(suggestion); - } - suggestClusterIds.sort( - (a, b) => allClusterIdsToCountMap[b.$1]! - .compareTo(allClusterIdsToCountMap[a.$1]!), - ); - final suggestClusterIdsSizes = suggestClusterIds - .map((e) => allClusterIdsToCountMap[e.$1]!) - .toList(growable: false); - final suggestClusterIdsDistances = - suggestClusterIds.map((e) => e.$2).toList(growable: false); - _logger.info( - "Already found good suggestions using mean: $suggestClusterIds, with sizes $suggestClusterIdsSizes and distances $suggestClusterIdsDistances", - ); - return suggestClusterIds + return suggestionsMean .map((e) => (e.$1, e.$2, true)) .toList(growable: false); } // Find the other cluster candidates based on the median - final Map> moreSuggestionsMean = - _calcSuggestionsMean( + final List<(int, double)> moreSuggestionsMean = _calcSuggestionsMean( clusterAvg, personClusters, ignoredClusters, @@ -549,12 +512,8 @@ class ClusterFeedbackService { return []; } - final List<(int, double)> temp = []; - for (final List<(int, double)> suggestion in moreSuggestionsMean.values) { - temp.addAll(suggestion); - } - temp.sort((a, b) => a.$2.compareTo(b.$2)); - final otherClusterIdsCandidates = temp + moreSuggestionsMean.sort((a, b) => a.$2.compareTo(b.$2)); + final otherClusterIdsCandidates = moreSuggestionsMean .map( (e) => e.$1, ) @@ -655,20 +614,26 @@ class ClusterFeedbackService { int maxClusterInCurrentRun = 500, int maxEmbeddingToRead = 10000, }) async { + final startTime = DateTime.now(); final faceMlDb = FaceMLDataDB.instance; _logger.info( 'start getUpdateClusterAvg for ${allClusterIdsToCountMap.length} clusters, minClusterSize $minClusterSize, maxClusterInCurrentRun $maxClusterInCurrentRun', ); final Map clusterToSummary = - await faceMlDb.getAllClusterSummary(); + await faceMlDb.getAllClusterSummary(minClusterSize); final Map updatesForClusterSummary = {}; final Map> clusterAvg = {}; + dev.log( + 'getUpdateClusterAvg database call for getAllClusterSummary took ${DateTime.now().difference(startTime).inMilliseconds} ms', + ); + final allClusterIds = allClusterIdsToCountMap.keys.toSet(); int ignoredClustersCnt = 0, alreadyUpdatedClustersCnt = 0; int smallerClustersCnt = 0; + final serializationTime = DateTime.now(); for (final id in allClusterIdsToCountMap.keys) { if (ignoredClusters.contains(id)) { allClusterIds.remove(id); @@ -684,9 +649,20 @@ class ClusterFeedbackService { smallerClustersCnt++; } } + dev.log( + 'serialization of embeddings took ${DateTime.now().difference(serializationTime).inMilliseconds} ms', + ); _logger.info( 'Ignored $ignoredClustersCnt clusters, already updated $alreadyUpdatedClustersCnt clusters, $smallerClustersCnt clusters are smaller than $minClusterSize', ); + + if (allClusterIds.isEmpty) { + _logger.info( + 'No clusters to update, getUpdateClusterAvg done in ${DateTime.now().difference(startTime).inMilliseconds} ms', + ); + return clusterAvg; + } + // get clusterIDs sorted by count in descending order final sortedClusterIDs = allClusterIds.toList(); sortedClusterIDs.sort( @@ -760,18 +736,21 @@ class ClusterFeedbackService { await faceMlDb.clusterSummaryUpdate(updatesForClusterSummary); } w?.logAndReset('done computing avg '); - _logger.info('end getUpdateClusterAvg for ${clusterAvg.length} clusters'); + _logger.info( + 'end getUpdateClusterAvg for ${clusterAvg.length} clusters, done in ${DateTime.now().difference(startTime).inMilliseconds} ms', + ); return clusterAvg; } /// Returns a map of person's clusterID to map of closest clusterID to with disstance - Map> _calcSuggestionsMean( + List<(int, double)> _calcSuggestionsMean( Map> clusterAvg, Set personClusters, Set ignoredClusters, - double maxClusterDistance, - ) { + double maxClusterDistance, { + Map? allClusterIdsToCountMap, + }) { final Map> suggestions = {}; for (final otherClusterID in clusterAvg.keys) { // ignore the cluster that belong to the person or is ignored @@ -802,11 +781,32 @@ class ClusterFeedbackService { .add((otherClusterID, minDistance)); } } - for (final entry in suggestions.entries) { - entry.value.sort((a, b) => a.$1.compareTo(b.$1)); - } - return suggestions; + if (suggestions.isNotEmpty) { + final List<(int, double)> suggestClusterIds = []; + for (final List<(int, double)> suggestion in suggestions.values) { + suggestClusterIds.addAll(suggestion); + } + List? suggestClusterIdsSizes; + if (allClusterIdsToCountMap != null) { + suggestClusterIds.sort( + (a, b) => allClusterIdsToCountMap[b.$1]! + .compareTo(allClusterIdsToCountMap[a.$1]!), + ); + suggestClusterIdsSizes = suggestClusterIds + .map((e) => allClusterIdsToCountMap[e.$1]!) + .toList(growable: false); + } + final suggestClusterIdsDistances = + suggestClusterIds.map((e) => e.$2).toList(growable: false); + _logger.info( + "Already found good suggestions using mean: $suggestClusterIds, ${suggestClusterIdsSizes != null ? 'with sizes $suggestClusterIdsSizes' : ''} and distances $suggestClusterIdsDistances", + ); + return suggestClusterIds; + } else { + _logger.info("No suggestions found using mean"); + return <(int, double)>[]; + } } List _randomSampleWithoutReplacement( diff --git a/mobile/lib/services/search_service.dart b/mobile/lib/services/search_service.dart index 3f54187c16..5f04e5ba48 100644 --- a/mobile/lib/services/search_service.dart +++ b/mobile/lib/services/search_service.dart @@ -28,6 +28,7 @@ import "package:photos/models/search/search_constants.dart"; import "package:photos/models/search/search_types.dart"; import 'package:photos/services/collections_service.dart'; import "package:photos/services/location_service.dart"; +import "package:photos/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart"; import "package:photos/services/machine_learning/face_ml/person/person_service.dart"; import 'package:photos/services/machine_learning/semantic_search/semantic_search_service.dart'; import "package:photos/states/location_screen_state.dart"; @@ -824,7 +825,7 @@ class SearchService { "Cluster $clusterId should not have person id ${clusterIDToPersonID[clusterId]}", ); } - if (files.length < 20 && sortedClusterIds.length > 3) { + if (files.length < kMinimumClusterSizeSearchResult && sortedClusterIds.length > 3) { continue; } facesResult.add( From 9cdd4fd7132c657a0cf9ea7e131fd5bf41fb3691 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Mon, 22 Apr 2024 17:24:34 +0530 Subject: [PATCH 05/69] [mob][photos] Face thumbnail generation from widgets --- mobile/lib/ui/viewer/file_details/face_widget.dart | 5 +++-- mobile/lib/ui/viewer/people/cropped_face_image_view.dart | 2 +- mobile/lib/ui/viewer/search/result/person_face_widget.dart | 5 +++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/mobile/lib/ui/viewer/file_details/face_widget.dart b/mobile/lib/ui/viewer/file_details/face_widget.dart index 79637af829..b4ab2318af 100644 --- a/mobile/lib/ui/viewer/file_details/face_widget.dart +++ b/mobile/lib/ui/viewer/file_details/face_widget.dart @@ -1,5 +1,4 @@ import "dart:developer" show log; -import "dart:io" show Platform; import "dart:typed_data"; import "package:flutter/cupertino.dart"; @@ -21,6 +20,8 @@ import "package:photos/utils/face/face_box_crop.dart"; import "package:photos/utils/thumbnail_util.dart"; // import "package:photos/utils/toast_util.dart"; +const useGeneratedFaceCrops = false; + class FaceWidget extends StatefulWidget { final EnteFile file; final Face face; @@ -48,7 +49,7 @@ class _FaceWidgetState extends State { @override Widget build(BuildContext context) { - if (Platform.isIOS) { + if (useGeneratedFaceCrops) { return FutureBuilder( future: getFaceCrop(), builder: (context, snapshot) { diff --git a/mobile/lib/ui/viewer/people/cropped_face_image_view.dart b/mobile/lib/ui/viewer/people/cropped_face_image_view.dart index 04980098fd..79c2b77750 100644 --- a/mobile/lib/ui/viewer/people/cropped_face_image_view.dart +++ b/mobile/lib/ui/viewer/people/cropped_face_image_view.dart @@ -51,7 +51,7 @@ class CroppedFaceImageView extends StatelessWidget { final double relativeFaceCenterY = faceBox.yMin + faceBox.height / 2; - const double desiredFaceHeightRelativeToWidget = 1 / 2; + const double desiredFaceHeightRelativeToWidget = 7 / 10; final double scale = (1 / faceBox.height) * desiredFaceHeightRelativeToWidget; diff --git a/mobile/lib/ui/viewer/search/result/person_face_widget.dart b/mobile/lib/ui/viewer/search/result/person_face_widget.dart index 36f5db8007..f2c96fc7a9 100644 --- a/mobile/lib/ui/viewer/search/result/person_face_widget.dart +++ b/mobile/lib/ui/viewer/search/result/person_face_widget.dart @@ -1,5 +1,5 @@ import "dart:developer"; -import "dart:io"; +// import "dart:io"; import "dart:typed_data"; import 'package:flutter/widgets.dart'; @@ -10,6 +10,7 @@ import "package:photos/face/model/person.dart"; import 'package:photos/models/file/file.dart'; import "package:photos/services/machine_learning/face_ml/person/person_service.dart"; import 'package:photos/ui/viewer/file/thumbnail_widget.dart'; +import "package:photos/ui/viewer/file_details/face_widget.dart"; import "package:photos/ui/viewer/people/cropped_face_image_view.dart"; import "package:photos/utils/face/face_box_crop.dart"; import "package:photos/utils/thumbnail_util.dart"; @@ -34,7 +35,7 @@ class PersonFaceWidget extends StatelessWidget { @override Widget build(BuildContext context) { - if (Platform.isIOS || Platform.isAndroid) { + if (useGeneratedFaceCrops) { return FutureBuilder( future: getFaceCrop(), builder: (context, snapshot) { From f49ede4a741d113e867d7ac9e761487fb6330176 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Mon, 22 Apr 2024 18:07:50 +0530 Subject: [PATCH 06/69] [mob][photos] Small fix in detecting sideways faces --- mobile/lib/face/model/detection.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobile/lib/face/model/detection.dart b/mobile/lib/face/model/detection.dart index 49e8c3652b..9dc828d569 100644 --- a/mobile/lib/face/model/detection.dart +++ b/mobile/lib/face/model/detection.dart @@ -155,7 +155,7 @@ class Detection { (nose[0] < min(leftEye[0], rightEye[0]) - 0.5 * eyeDistanceX) && (nose[0] < min(leftMouth[0], rightMouth[0])); final bool noseStickingOutRight = - (nose[0] > max(leftEye[0], rightEye[0]) - 0.5 * eyeDistanceX) && + (nose[0] > max(leftEye[0], rightEye[0]) + 0.5 * eyeDistanceX) && (nose[0] > max(leftMouth[0], rightMouth[0])); return faceIsUpright && (noseStickingOutLeft || noseStickingOutRight); From 3253a2bf26ea432d06f91819c066f29461dce7d6 Mon Sep 17 00:00:00 2001 From: ashilkn Date: Tue, 23 Apr 2024 08:26:29 +0530 Subject: [PATCH 07/69] [mob] Remove unnecessary LayoutBuilder --- .../people/cropped_face_image_view.dart | 85 +++++++++---------- 1 file changed, 41 insertions(+), 44 deletions(-) diff --git a/mobile/lib/ui/viewer/people/cropped_face_image_view.dart b/mobile/lib/ui/viewer/people/cropped_face_image_view.dart index 2c7768f7fd..3a9053d317 100644 --- a/mobile/lib/ui/viewer/people/cropped_face_image_view.dart +++ b/mobile/lib/ui/viewer/people/cropped_face_image_view.dart @@ -37,59 +37,56 @@ class CroppedFaceImageView extends StatelessWidget { future: getImage(), builder: (context, snapshot) { if (snapshot.hasData) { - return LayoutBuilder( - builder: (BuildContext context, BoxConstraints constraints) { - final Image image = snapshot.data!; + final double imageAspectRatio = enteFile.width / enteFile.height; + final Image image = snapshot.data!; - final double viewWidth = constraints.maxWidth; - final double viewHeight = constraints.maxHeight; + const double viewWidth = 60; + const double viewHeight = 60; - final faceBox = face.detection.box; + final faceBox = face.detection.box; - final double relativeFaceCenterX = - faceBox.xMin + faceBox.width / 2; - final double relativeFaceCenterY = - faceBox.yMin + faceBox.height / 2; + final double relativeFaceCenterX = faceBox.xMin + faceBox.width / 2; + final double relativeFaceCenterY = faceBox.yMin + faceBox.height / 2; - const double desiredFaceHeightRelativeToWidget = 7 / 10; - final double scale = - (1 / faceBox.height) * desiredFaceHeightRelativeToWidget; + const double desiredFaceHeightRelativeToWidget = 8 / 10; + final double scale = + (1 / faceBox.height) * desiredFaceHeightRelativeToWidget; - final double widgetCenterX = viewWidth / 2; - final double widgetCenterY = viewHeight / 2; + const double widgetCenterX = viewWidth / 2; + const double widgetCenterY = viewHeight / 2; - final double imageAspectRatio = enteFile.width / enteFile.height; - final double widgetAspectRatio = viewWidth / viewHeight; - final double imageToWidgetRatio = - imageAspectRatio / widgetAspectRatio; + const double widgetAspectRatio = viewWidth / viewHeight; + final double imageToWidgetRatio = + imageAspectRatio / widgetAspectRatio; - double offsetX = - (widgetCenterX - relativeFaceCenterX * viewWidth) * scale; - double offsetY = - (widgetCenterY - relativeFaceCenterY * viewHeight) * scale; + double offsetX = + (widgetCenterX - relativeFaceCenterX * viewWidth) * scale; + double offsetY = + (widgetCenterY - relativeFaceCenterY * viewHeight) * scale; - if (imageAspectRatio < widgetAspectRatio) { - // Landscape Image: Adjust offsetX more conservatively - offsetX = offsetX * imageToWidgetRatio; - } else { - // Portrait Image: Adjust offsetY more conservatively - offsetY = offsetY / imageToWidgetRatio; - } - - return ClipRRect( - borderRadius: const BorderRadius.all(Radius.elliptical(16, 12)), - child: Transform.translate( - offset: Offset( - offsetX, - offsetY, - ), - child: Transform.scale( - scale: scale, - child: image, - ), + if (imageAspectRatio < widgetAspectRatio) { + // Landscape Image: Adjust offsetX more conservatively + offsetX = offsetX * imageToWidgetRatio; + } else { + // Portrait Image: Adjust offsetY more conservatively + offsetY = offsetY / imageToWidgetRatio; + } + return SizedBox( + width: viewWidth, + height: viewHeight, + child: ClipRRect( + borderRadius: const BorderRadius.all(Radius.elliptical(16, 12)), + child: Transform.translate( + offset: Offset( + offsetX, + offsetY, ), - ); - }, + child: Transform.scale( + scale: scale, + child: image, + ), + ), + ), ); } else { if (snapshot.hasError) { From e875eb138993845e68ee577a174407c649a2b0ad Mon Sep 17 00:00:00 2001 From: ashilkn Date: Tue, 23 Apr 2024 08:27:47 +0530 Subject: [PATCH 08/69] [mob] Remove unnecessary ShapeDecoration and width constrain --- .../lib/ui/viewer/file_details/face_widget.dart | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/mobile/lib/ui/viewer/file_details/face_widget.dart b/mobile/lib/ui/viewer/file_details/face_widget.dart index f828db43a4..3f09ffd986 100644 --- a/mobile/lib/ui/viewer/file_details/face_widget.dart +++ b/mobile/lib/ui/viewer/file_details/face_widget.dart @@ -55,6 +55,7 @@ class _FaceWidgetState extends State { builder: (context, snapshot) { if (snapshot.hasData) { final ImageProvider imageProvider = MemoryImage(snapshot.data!); + return GestureDetector( onTap: () async { if (widget.editMode) return; @@ -263,21 +264,8 @@ class _FaceWidgetState extends State { }, child: Column( children: [ - Container( + SizedBox( height: 60, - width: 60, - decoration: ShapeDecoration( - shape: RoundedRectangleBorder( - borderRadius: - const BorderRadius.all(Radius.elliptical(16, 12)), - side: widget.highlight - ? BorderSide( - color: getEnteColorScheme(context).primary700, - width: 2.0, - ) - : BorderSide.none, - ), - ), child: CroppedFaceImageView( enteFile: widget.file, face: widget.face, From 731610ed94edd9b4ee45fdd850a20003774b4f17 Mon Sep 17 00:00:00 2001 From: ashilkn Date: Tue, 23 Apr 2024 08:30:59 +0530 Subject: [PATCH 09/69] [mob] Refactor --- .../ui/viewer/file_details/face_widget.dart | 9 +- .../people/cropped_face_image_view.dart | 90 +++++++++---------- 2 files changed, 48 insertions(+), 51 deletions(-) diff --git a/mobile/lib/ui/viewer/file_details/face_widget.dart b/mobile/lib/ui/viewer/file_details/face_widget.dart index 3f09ffd986..4357cec76c 100644 --- a/mobile/lib/ui/viewer/file_details/face_widget.dart +++ b/mobile/lib/ui/viewer/file_details/face_widget.dart @@ -264,12 +264,9 @@ class _FaceWidgetState extends State { }, child: Column( children: [ - SizedBox( - height: 60, - child: CroppedFaceImageView( - enteFile: widget.file, - face: widget.face, - ), + CroppedFaceImageView( + enteFile: widget.file, + face: widget.face, ), const SizedBox(height: 8), if (widget.person != null) diff --git a/mobile/lib/ui/viewer/people/cropped_face_image_view.dart b/mobile/lib/ui/viewer/people/cropped_face_image_view.dart index 3a9053d317..9f1717707c 100644 --- a/mobile/lib/ui/viewer/people/cropped_face_image_view.dart +++ b/mobile/lib/ui/viewer/people/cropped_face_image_view.dart @@ -33,48 +33,48 @@ class CroppedFaceImageView extends StatelessWidget { @override Widget build(BuildContext context) { - return FutureBuilder( - future: getImage(), - builder: (context, snapshot) { - if (snapshot.hasData) { - final double imageAspectRatio = enteFile.width / enteFile.height; - final Image image = snapshot.data!; + const double viewWidth = 60; + const double viewHeight = 60; + return SizedBox( + width: viewWidth, + height: viewHeight, + child: FutureBuilder( + future: getImage(), + builder: (context, snapshot) { + if (snapshot.hasData) { + final double imageAspectRatio = enteFile.width / enteFile.height; + final Image image = snapshot.data!; - const double viewWidth = 60; - const double viewHeight = 60; + final faceBox = face.detection.box; - final faceBox = face.detection.box; + final double relativeFaceCenterX = faceBox.xMin + faceBox.width / 2; + final double relativeFaceCenterY = + faceBox.yMin + faceBox.height / 2; - final double relativeFaceCenterX = faceBox.xMin + faceBox.width / 2; - final double relativeFaceCenterY = faceBox.yMin + faceBox.height / 2; + const double desiredFaceHeightRelativeToWidget = 8 / 10; + final double scale = + (1 / faceBox.height) * desiredFaceHeightRelativeToWidget; - const double desiredFaceHeightRelativeToWidget = 8 / 10; - final double scale = - (1 / faceBox.height) * desiredFaceHeightRelativeToWidget; + const double widgetCenterX = viewWidth / 2; + const double widgetCenterY = viewHeight / 2; - const double widgetCenterX = viewWidth / 2; - const double widgetCenterY = viewHeight / 2; + const double widgetAspectRatio = viewWidth / viewHeight; + final double imageToWidgetRatio = + imageAspectRatio / widgetAspectRatio; - const double widgetAspectRatio = viewWidth / viewHeight; - final double imageToWidgetRatio = - imageAspectRatio / widgetAspectRatio; + double offsetX = + (widgetCenterX - relativeFaceCenterX * viewWidth) * scale; + double offsetY = + (widgetCenterY - relativeFaceCenterY * viewHeight) * scale; - double offsetX = - (widgetCenterX - relativeFaceCenterX * viewWidth) * scale; - double offsetY = - (widgetCenterY - relativeFaceCenterY * viewHeight) * scale; - - if (imageAspectRatio < widgetAspectRatio) { - // Landscape Image: Adjust offsetX more conservatively - offsetX = offsetX * imageToWidgetRatio; - } else { - // Portrait Image: Adjust offsetY more conservatively - offsetY = offsetY / imageToWidgetRatio; - } - return SizedBox( - width: viewWidth, - height: viewHeight, - child: ClipRRect( + if (imageAspectRatio < widgetAspectRatio) { + // Landscape Image: Adjust offsetX more conservatively + offsetX = offsetX * imageToWidgetRatio; + } else { + // Portrait Image: Adjust offsetY more conservatively + offsetY = offsetY / imageToWidgetRatio; + } + return ClipRRect( borderRadius: const BorderRadius.all(Radius.elliptical(16, 12)), child: Transform.translate( offset: Offset( @@ -86,17 +86,17 @@ class CroppedFaceImageView extends StatelessWidget { child: image, ), ), - ), - ); - } else { - if (snapshot.hasError) { - log('Error getting cover face for person: ${snapshot.error}'); + ); + } else { + if (snapshot.hasError) { + log('Error getting cover face for person: ${snapshot.error}'); + } + return ThumbnailWidget( + enteFile, + ); } - return ThumbnailWidget( - enteFile, - ); - } - }, + }, + ), ); } From 6a0a9bad1ee0d8d406084bb585dc30b768a5fba4 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Tue, 23 Apr 2024 09:56:46 +0530 Subject: [PATCH 10/69] [mob][photos] Clustering time logs --- .../face_ml/face_clustering/face_clustering_service.dart | 2 +- .../services/machine_learning/face_ml/face_ml_service.dart | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/mobile/lib/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart b/mobile/lib/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart index 80fff99c68..de645c01d8 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart @@ -624,7 +624,7 @@ class FaceClusteringService { } // analyze the results - FaceClusteringService._analyzeClusterResults(sortedFaceInfos); + // FaceClusteringService._analyzeClusterResults(sortedFaceInfos); return ClusteringResult( newFaceIdToCluster: newFaceIdToCluster, diff --git a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart index 3df9b30561..da57ad4e71 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart @@ -295,6 +295,7 @@ class FaceMlService { bool clusterInBuckets = true, }) async { _logger.info("`clusterAllImages()` called"); + final clusterAllImagesTime = DateTime.now(); try { // Get a sense of the total number of faces in the database @@ -408,7 +409,8 @@ class FaceMlService { _logger.info('Done updating FaceIDs with clusterIDs in the DB, in ' '${DateTime.now().difference(clusterDoneTime).inSeconds} seconds'); } - _logger.info('clusterAllImages() finished'); + _logger.info('clusterAllImages() finished, in ' + '${DateTime.now().difference(clusterAllImagesTime).inSeconds} seconds'); } catch (e, s) { _logger.severe("`clusterAllImages` failed", e, s); } From 4c25997bb61281add8a5ac54abd0c61bd35ce4a3 Mon Sep 17 00:00:00 2001 From: ashilkn Date: Tue, 23 Apr 2024 11:15:46 +0530 Subject: [PATCH 11/69] [mob] use layoutBuilder in face thumbnail --- .../ui/viewer/file_details/face_widget.dart | 10 +- .../people/cropped_face_image_view.dart | 112 +++++++++--------- 2 files changed, 64 insertions(+), 58 deletions(-) diff --git a/mobile/lib/ui/viewer/file_details/face_widget.dart b/mobile/lib/ui/viewer/file_details/face_widget.dart index 4357cec76c..da592a150b 100644 --- a/mobile/lib/ui/viewer/file_details/face_widget.dart +++ b/mobile/lib/ui/viewer/file_details/face_widget.dart @@ -264,9 +264,13 @@ class _FaceWidgetState extends State { }, child: Column( children: [ - CroppedFaceImageView( - enteFile: widget.file, - face: widget.face, + SizedBox( + width: 60, + height: 60, + child: CroppedFaceImageView( + enteFile: widget.file, + face: widget.face, + ), ), const SizedBox(height: 8), if (widget.person != null) diff --git a/mobile/lib/ui/viewer/people/cropped_face_image_view.dart b/mobile/lib/ui/viewer/people/cropped_face_image_view.dart index 9f1717707c..4ef3692595 100644 --- a/mobile/lib/ui/viewer/people/cropped_face_image_view.dart +++ b/mobile/lib/ui/viewer/people/cropped_face_image_view.dart @@ -33,70 +33,72 @@ class CroppedFaceImageView extends StatelessWidget { @override Widget build(BuildContext context) { - const double viewWidth = 60; - const double viewHeight = 60; - return SizedBox( - width: viewWidth, - height: viewHeight, - child: FutureBuilder( - future: getImage(), - builder: (context, snapshot) { - if (snapshot.hasData) { - final double imageAspectRatio = enteFile.width / enteFile.height; - final Image image = snapshot.data!; + return FutureBuilder( + future: getImage(), + builder: (context, snapshot) { + if (snapshot.hasData) { + return LayoutBuilder( + builder: ((context, constraints) { + final double imageAspectRatio = enteFile.width / enteFile.height; + final Image image = snapshot.data!; - final faceBox = face.detection.box; + final double viewWidth = constraints.maxWidth; + final double viewHeight = constraints.maxHeight; - final double relativeFaceCenterX = faceBox.xMin + faceBox.width / 2; - final double relativeFaceCenterY = - faceBox.yMin + faceBox.height / 2; + final faceBox = face.detection.box; - const double desiredFaceHeightRelativeToWidget = 8 / 10; - final double scale = - (1 / faceBox.height) * desiredFaceHeightRelativeToWidget; + final double relativeFaceCenterX = + faceBox.xMin + faceBox.width / 2; + final double relativeFaceCenterY = + faceBox.yMin + faceBox.height / 2; - const double widgetCenterX = viewWidth / 2; - const double widgetCenterY = viewHeight / 2; + const double desiredFaceHeightRelativeToWidget = 8 / 10; + final double scale = + (1 / faceBox.height) * desiredFaceHeightRelativeToWidget; - const double widgetAspectRatio = viewWidth / viewHeight; - final double imageToWidgetRatio = - imageAspectRatio / widgetAspectRatio; + final double widgetCenterX = viewWidth / 2; + final double widgetCenterY = viewHeight / 2; - double offsetX = - (widgetCenterX - relativeFaceCenterX * viewWidth) * scale; - double offsetY = - (widgetCenterY - relativeFaceCenterY * viewHeight) * scale; + final double widgetAspectRatio = viewWidth / viewHeight; + final double imageToWidgetRatio = + imageAspectRatio / widgetAspectRatio; - if (imageAspectRatio < widgetAspectRatio) { - // Landscape Image: Adjust offsetX more conservatively - offsetX = offsetX * imageToWidgetRatio; - } else { - // Portrait Image: Adjust offsetY more conservatively - offsetY = offsetY / imageToWidgetRatio; - } - return ClipRRect( - borderRadius: const BorderRadius.all(Radius.elliptical(16, 12)), - child: Transform.translate( - offset: Offset( - offsetX, - offsetY, + double offsetX = + (widgetCenterX - relativeFaceCenterX * viewWidth) * scale; + double offsetY = + (widgetCenterY - relativeFaceCenterY * viewHeight) * scale; + + if (imageAspectRatio < widgetAspectRatio) { + // Landscape Image: Adjust offsetX more conservatively + offsetX = offsetX * imageToWidgetRatio; + } else { + // Portrait Image: Adjust offsetY more conservatively + offsetY = offsetY / imageToWidgetRatio; + } + return ClipRRect( + borderRadius: const BorderRadius.all(Radius.elliptical(16, 12)), + child: Transform.translate( + offset: Offset( + offsetX, + offsetY, + ), + child: Transform.scale( + scale: scale, + child: image, + ), ), - child: Transform.scale( - scale: scale, - child: image, - ), - ), - ); - } else { - if (snapshot.hasError) { - log('Error getting cover face for person: ${snapshot.error}'); - } - return ThumbnailWidget( - enteFile, - ); + ); + }), + ); + } else { + if (snapshot.hasError) { + log('Error getting cover face for person: ${snapshot.error}'); } - }, - ), + return ThumbnailWidget( + enteFile, + ); + } + }, ); } From 9e87b4a2ccf1fc77831d59cabee342b7676d0c26 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Tue, 23 Apr 2024 11:58:39 +0530 Subject: [PATCH 12/69] [mob][photos] Highlight face in cluster --- .../ui/viewer/file_details/face_widget.dart | 57 ++++++++++++++++--- 1 file changed, 50 insertions(+), 7 deletions(-) diff --git a/mobile/lib/ui/viewer/file_details/face_widget.dart b/mobile/lib/ui/viewer/file_details/face_widget.dart index da592a150b..4caf1305f4 100644 --- a/mobile/lib/ui/viewer/file_details/face_widget.dart +++ b/mobile/lib/ui/viewer/file_details/face_widget.dart @@ -264,13 +264,56 @@ class _FaceWidgetState extends State { }, child: Column( children: [ - SizedBox( - width: 60, - height: 60, - child: CroppedFaceImageView( - enteFile: widget.file, - face: widget.face, - ), + Stack( + children: [ + Container( + height: 60, + width: 60, + decoration: ShapeDecoration( + shape: RoundedRectangleBorder( + borderRadius: const BorderRadius.all( + Radius.elliptical(16, 12), + ), + side: widget.highlight + ? BorderSide( + color: getEnteColorScheme(context).primary700, + width: 1.0, + ) + : BorderSide.none, + ), + ), + child: ClipRRect( + borderRadius: + const BorderRadius.all(Radius.elliptical(16, 12)), + child: SizedBox( + width: 60, + height: 60, + child: CroppedFaceImageView( + enteFile: widget.file, + face: widget.face, + ), + ), + ), + ), + // TODO: the edges of the green line are still not properly rounded around ClipRRect + if (widget.editMode) + Positioned( + right: 0, + top: 0, + child: GestureDetector( + onTap: _cornerIconPressed, + child: isJustRemoved + ? const Icon( + CupertinoIcons.add_circled_solid, + color: Colors.green, + ) + : const Icon( + Icons.cancel, + color: Colors.red, + ), + ), + ), + ], ), const SizedBox(height: 8), if (widget.person != null) From 6fe8dc7c66efe5ab685d23482cf1821f366dca62 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Tue, 23 Apr 2024 12:18:19 +0530 Subject: [PATCH 13/69] [mob][photos] Check big and medium clusters first for suggestions --- .../face_ml/feedback/cluster_feedback.dart | 51 +++++++++++-------- 1 file changed, 30 insertions(+), 21 deletions(-) diff --git a/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart b/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart index e30dc375fe..e673b12ed6 100644 --- a/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart +++ b/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart @@ -1,5 +1,5 @@ import 'dart:developer' as dev; -import "dart:math" show Random; +import "dart:math" show Random, min; import "package:flutter/foundation.dart"; import "package:logging/logging.dart"; @@ -454,27 +454,36 @@ class ClusterFeedbackService { name: "getSuggestionsUsingMedian", ); - // First only do a simple check on the big clusters + // First only do a simple check on the big clusters, if the person does not have small clusters yet final EnteWatch watch = EnteWatch("ClusterFeedbackService")..start(); - final Map> clusterAvgBigClusters = - await _getUpdateClusterAvg( - allClusterIdsToCountMap, - ignoredClusters, - minClusterSize: kMinimumClusterSizeSearchResult, - ); - dev.log( - 'computed avg for ${clusterAvgBigClusters.length} clusters, in ${DateTime.now().difference(startTime).inMilliseconds} ms', - ); - final List<(int, double)> suggestionsMeanBigClusters = _calcSuggestionsMean( - clusterAvgBigClusters, - personClusters, - ignoredClusters, - goodMeanDistance, - ); - if (suggestionsMeanBigClusters.isNotEmpty) { - return suggestionsMeanBigClusters - .map((e) => (e.$1, e.$2, true)) - .toList(growable: false); + final smallestPersonClusterSize = personClusters + .map((clusterID) => allClusterIdsToCountMap[clusterID] ?? 0) + .reduce((value, element) => min(value, element)); + final checkSizes = [kMinimumClusterSizeSearchResult, 20, 10, 5]; + for (final minimumSize in checkSizes.toSet()) { + if (smallestPersonClusterSize >= minimumSize) { + final Map> clusterAvgBigClusters = + await _getUpdateClusterAvg( + allClusterIdsToCountMap, + ignoredClusters, + minClusterSize: minimumSize, + ); + dev.log( + 'computed avg for ${clusterAvgBigClusters.length} clusters, in ${DateTime.now().difference(startTime).inMilliseconds} ms', + ); + final List<(int, double)> suggestionsMeanBigClusters = + _calcSuggestionsMean( + clusterAvgBigClusters, + personClusters, + ignoredClusters, + goodMeanDistance, + ); + if (suggestionsMeanBigClusters.isNotEmpty) { + return suggestionsMeanBigClusters + .map((e) => (e.$1, e.$2, true)) + .toList(growable: false); + } + } } // Get and update the cluster summary to get the avg (centroid) and count From 3786c9def93cc439c663e4f3a774a8b35c53bd5e Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Tue, 23 Apr 2024 13:37:53 +0530 Subject: [PATCH 14/69] [mob][photos] Suggestions change parameters --- .../face_ml/feedback/cluster_feedback.dart | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart b/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart index e673b12ed6..cfd2301814 100644 --- a/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart +++ b/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart @@ -65,8 +65,13 @@ class ClusterFeedbackService { try { // Get the suggestions for the person using centroids and median + final startTime = DateTime.now(); final List<(int, double, bool)> suggestClusterIds = await _getSuggestions(person); + final findSuggestionsTime = DateTime.now(); + _logger.info( + '`_getSuggestions`: Found ${suggestClusterIds.length} suggestions in ${findSuggestionsTime.difference(startTime).inMilliseconds} ms', + ); // Get the files for the suggestions final Map> fileIdToClusterID = @@ -437,10 +442,10 @@ class ClusterFeedbackService { Future> _getSuggestions( PersonEntity p, { int sampleSize = 50, - double maxMedianDistance = 0.65, + double maxMedianDistance = 0.62, double goodMedianDistance = 0.55, double maxMeanDistance = 0.65, - double goodMeanDistance = 0.5, + double goodMeanDistance = 0.54, }) async { // Get all the cluster data final startTime = DateTime.now(); @@ -459,7 +464,7 @@ class ClusterFeedbackService { final smallestPersonClusterSize = personClusters .map((clusterID) => allClusterIdsToCountMap[clusterID] ?? 0) .reduce((value, element) => min(value, element)); - final checkSizes = [kMinimumClusterSizeSearchResult, 20, 10, 5]; + final checkSizes = [kMinimumClusterSizeSearchResult, 20, 10, 5, 1]; for (final minimumSize in checkSizes.toSet()) { if (smallestPersonClusterSize >= minimumSize) { final Map> clusterAvgBigClusters = From 7312633e02d0f3522127d53f59973d7ff7ae1df6 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Tue, 23 Apr 2024 14:26:30 +0530 Subject: [PATCH 15/69] [mob][photos] Only sort big suggestions --- .../face_ml/feedback/cluster_feedback.dart | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart b/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart index cfd2301814..10f7b90d66 100644 --- a/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart +++ b/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart @@ -855,18 +855,31 @@ class ClusterFeedbackService { Future _sortSuggestionsOnDistanceToPerson( PersonEntity person, - List suggestions, - ) async { + List suggestions, { + bool onlySortBigSuggestions = true, + }) async { if (suggestions.isEmpty) { debugPrint('No suggestions to sort'); return; } + if (onlySortBigSuggestions) { + final bigSuggestions = suggestions + .where( + (s) => s.filesInCluster.length > kMinimumClusterSizeSearchResult, + ) + .toList(); + if (bigSuggestions.isEmpty) { + debugPrint('No big suggestions to sort'); + return; + } + } final startTime = DateTime.now(); final faceMlDb = FaceMLDataDB.instance; // Get the cluster averages for the person's clusters and the suggestions' clusters final Map clusterToSummary = await faceMlDb.getAllClusterSummary(); + final clusterSummaryCallTime = DateTime.now(); // Calculate the avg embedding of the person final personClusters = await faceMlDb.getPersonClusterIDs(person.remoteID); @@ -913,7 +926,7 @@ class ClusterFeedbackService { final endTime = DateTime.now(); _logger.info( - "Sorting suggestions based on distance to person took ${endTime.difference(startTime).inMilliseconds} ms for ${suggestions.length} suggestions", + "Sorting suggestions based on distance to person took ${endTime.difference(startTime).inMilliseconds} ms for ${suggestions.length} suggestions, of which ${clusterSummaryCallTime.difference(startTime).inMilliseconds} ms was spent on the cluster summary call", ); } } From efb1170b44c9edebeeb280d155cb80fc68df6906 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Wed, 24 Apr 2024 08:39:45 +0530 Subject: [PATCH 16/69] [mob][photos] unawait network call when accepting suggestion --- mobile/lib/services/entity_service.dart | 2 +- .../face_ml/person/person_service.dart | 11 +++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/mobile/lib/services/entity_service.dart b/mobile/lib/services/entity_service.dart index ba4b4a6368..6ffe87358b 100644 --- a/mobile/lib/services/entity_service.dart +++ b/mobile/lib/services/entity_service.dart @@ -61,7 +61,7 @@ class EntityService { }) async { final key = await getOrCreateEntityKey(type); final encryptedKeyData = await CryptoUtil.encryptChaCha( - utf8.encode(plainText) as Uint8List, + utf8.encode(plainText), key, ); final String encryptedData = diff --git a/mobile/lib/services/machine_learning/face_ml/person/person_service.dart b/mobile/lib/services/machine_learning/face_ml/person/person_service.dart index 509862abc8..da62899538 100644 --- a/mobile/lib/services/machine_learning/face_ml/person/person_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/person/person_service.dart @@ -1,3 +1,4 @@ +import "dart:async" show unawaited; import "dart:convert"; import "package:flutter/foundation.dart"; @@ -102,10 +103,12 @@ class PersonService { faces: faceIds.toSet(), ); personData.assigned!.add(clusterInfo); - await entityService.addOrUpdate( - EntityType.person, - json.encode(personData.toJson()), - id: personID, + unawaited( + entityService.addOrUpdate( + EntityType.person, + json.encode(personData.toJson()), + id: personID, + ), ); await faceMLDataDB.assignClusterToPerson( personID: personID, From 7097ce3cf45069112b86c587075057b94fd05949 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Wed, 24 Apr 2024 09:06:20 +0530 Subject: [PATCH 17/69] [mob][photos] Faster DB call when sorting suggestions --- mobile/lib/face/db.dart | 15 +++++++++++++++ .../face_ml/feedback/cluster_feedback.dart | 12 ++++++------ 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/mobile/lib/face/db.dart b/mobile/lib/face/db.dart index 5945175470..905c2a3c49 100644 --- a/mobile/lib/face/db.dart +++ b/mobile/lib/face/db.dart @@ -846,6 +846,21 @@ class FaceMLDataDB { return result; } + Future> getClusterToClusterSummary(Iterable clusterIDs) async { + final db = await instance.sqliteAsyncDB; + final Map result = {}; + final rows = await db.getAll( + 'SELECT * FROM $clusterSummaryTable WHERE $clusterIDColumn IN (${clusterIDs.join(",")})', + ); + for (final r in rows) { + final id = r[clusterIDColumn] as int; + final avg = r[avgColumn] as Uint8List; + final count = r[countColumn] as int; + result[id] = (avg, count); + } + return result; + } + Future> getClusterIDToPersonID() async { final db = await instance.database; final List> maps = await db.rawQuery( diff --git a/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart b/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart index 10f7b90d66..c6c185741c 100644 --- a/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart +++ b/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart @@ -877,21 +877,21 @@ class ClusterFeedbackService { final faceMlDb = FaceMLDataDB.instance; // Get the cluster averages for the person's clusters and the suggestions' clusters - final Map clusterToSummary = - await faceMlDb.getAllClusterSummary(); + final personClusters = await faceMlDb.getPersonClusterIDs(person.remoteID); + final Map personClusterToSummary = + await faceMlDb.getClusterToClusterSummary(personClusters); final clusterSummaryCallTime = DateTime.now(); // Calculate the avg embedding of the person - final personClusters = await faceMlDb.getPersonClusterIDs(person.remoteID); final personEmbeddingsCount = personClusters - .map((e) => clusterToSummary[e]!.$2) + .map((e) => personClusterToSummary[e]!.$2) .reduce((a, b) => a + b); final List personAvg = List.filled(192, 0); for (final personClusterID in personClusters) { - final personClusterBlob = clusterToSummary[personClusterID]!.$1; + final personClusterBlob = personClusterToSummary[personClusterID]!.$1; final personClusterAvg = EVector.fromBuffer(personClusterBlob).values; final clusterWeight = - clusterToSummary[personClusterID]!.$2 / personEmbeddingsCount; + personClusterToSummary[personClusterID]!.$2 / personEmbeddingsCount; for (int i = 0; i < personClusterAvg.length; i++) { personAvg[i] += personClusterAvg[i] * clusterWeight; // Weighted sum of the cluster averages From c80208e754a8b16467392748b80adf1085a8a607 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Wed, 24 Apr 2024 09:53:42 +0530 Subject: [PATCH 18/69] [mob][photos] Fix in sorting suggestions --- mobile/lib/face/db.dart | 14 ++++++++ .../face_ml/feedback/cluster_feedback.dart | 33 +++++++++++++++---- 2 files changed, 40 insertions(+), 7 deletions(-) diff --git a/mobile/lib/face/db.dart b/mobile/lib/face/db.dart index 905c2a3c49..74b904ced4 100644 --- a/mobile/lib/face/db.dart +++ b/mobile/lib/face/db.dart @@ -332,6 +332,20 @@ class FaceMLDataDB { return mapRowToFace(result.first); } + Future>> getClusterToFaceIDs(Set clusterIDs) async { + final db = await instance.sqliteAsyncDB; + final Map> result = {}; + final List> maps = await db.getAll( + 'SELECT $fcClusterID, $fcFaceId FROM $faceClustersTable WHERE $fcClusterID IN (${clusterIDs.join(",")})', + ); + for (final map in maps) { + final clusterID = map[fcClusterID] as int; + final faceID = map[fcFaceId] as String; + result.putIfAbsent(clusterID, () => []).add(faceID); + } + return result; + } + Future> getFaceIDsForCluster(int clusterID) async { final db = await instance.sqliteAsyncDB; final List> maps = await db.getAll( diff --git a/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart b/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart index c6c185741c..86cd3b8384 100644 --- a/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart +++ b/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart @@ -25,12 +25,14 @@ class ClusterSuggestion { final double distancePersonToCluster; final bool usedOnlyMeanForSuggestion; final List filesInCluster; + final List faceIDsInCluster; ClusterSuggestion( this.clusterIDToMerge, this.distancePersonToCluster, this.usedOnlyMeanForSuggestion, this.filesInCluster, + this.faceIDsInCluster, ); } @@ -60,24 +62,27 @@ class ClusterFeedbackService { bool extremeFilesFirst = true, }) async { _logger.info( - 'getClusterFilesForPersonID ${kDebugMode ? person.data.name : person.remoteID}', + 'getSuggestionForPerson ${kDebugMode ? person.data.name : person.remoteID}', ); try { // Get the suggestions for the person using centroids and median final startTime = DateTime.now(); - final List<(int, double, bool)> suggestClusterIds = + final List<(int, double, bool)> foundSuggestions = await _getSuggestions(person); final findSuggestionsTime = DateTime.now(); _logger.info( - '`_getSuggestions`: Found ${suggestClusterIds.length} suggestions in ${findSuggestionsTime.difference(startTime).inMilliseconds} ms', + 'getSuggestionForPerson `_getSuggestions`: Found ${foundSuggestions.length} suggestions in ${findSuggestionsTime.difference(startTime).inMilliseconds} ms', ); // Get the files for the suggestions + final suggestionClusterIDs = foundSuggestions.map((e) => e.$1).toSet(); final Map> fileIdToClusterID = await FaceMLDataDB.instance.getFileIdToClusterIDSetForCluster( - suggestClusterIds.map((e) => e.$1).toSet(), + suggestionClusterIDs, ); + final clusterIdToFaceIDs = + await FaceMLDataDB.instance.getClusterToFaceIDs(suggestionClusterIDs); final Map> clusterIDToFiles = {}; final allFiles = await SearchService.instance.getAllFiles(); for (final f in allFiles) { @@ -95,7 +100,7 @@ class ClusterFeedbackService { } final List clusterIdAndFiles = []; - for (final clusterSuggestion in suggestClusterIds) { + for (final clusterSuggestion in foundSuggestions) { if (clusterIDToFiles.containsKey(clusterSuggestion.$1)) { clusterIdAndFiles.add( ClusterSuggestion( @@ -103,14 +108,20 @@ class ClusterFeedbackService { clusterSuggestion.$2, clusterSuggestion.$3, clusterIDToFiles[clusterSuggestion.$1]!, + clusterIdToFaceIDs[clusterSuggestion.$1]!.toList(), ), ); } } + final getFilesTime = DateTime.now(); + final sortingStartTime = DateTime.now(); if (extremeFilesFirst) { await _sortSuggestionsOnDistanceToPerson(person, clusterIdAndFiles); } + _logger.info( + 'getSuggestionForPerson post-processing suggestions took ${DateTime.now().difference(findSuggestionsTime).inMilliseconds} ms, of which sorting took ${DateTime.now().difference(sortingStartTime).inMilliseconds} ms and getting files took ${getFilesTime.difference(findSuggestionsTime).inMilliseconds} ms', + ); return clusterIdAndFiles; } catch (e, s) { @@ -883,6 +894,7 @@ class ClusterFeedbackService { final clusterSummaryCallTime = DateTime.now(); // Calculate the avg embedding of the person + final w = (kDebugMode ? EnteWatch('sortSuggestions') : null)?..start(); final personEmbeddingsCount = personClusters .map((e) => personClusterToSummary[e]!.$2) .reduce((a, b) => a + b); @@ -897,12 +909,17 @@ class ClusterFeedbackService { clusterWeight; // Weighted sum of the cluster averages } } + w?.log('calculated person avg'); // Sort the suggestions based on the distance to the person for (final suggestion in suggestions) { final clusterID = suggestion.clusterIDToMerge; - final faceIdToEmbeddingMap = await faceMlDb.getFaceEmbeddingMapForFile( - suggestion.filesInCluster.map((e) => e.uploadedFileID!).toList(), + final faceIDs = suggestion.faceIDsInCluster; + final faceIdToEmbeddingMap = await faceMlDb.getFaceEmbeddingMapForFaces( + faceIDs, + ); + w?.log( + 'got ${faceIdToEmbeddingMap.values.length} embeddings for ${suggestion.filesInCluster.length} files for cluster $clusterID', ); final fileIdToDistanceMap = {}; for (final entry in faceIdToEmbeddingMap.entries) { @@ -912,12 +929,14 @@ class ClusterFeedbackService { EVector.fromBuffer(entry.value).values, ); } + w?.log('calculated distances for cluster $clusterID'); suggestion.filesInCluster.sort((b, a) { //todo: review with @laurens, added this to avoid null safety issue final double distanceA = fileIdToDistanceMap[a.uploadedFileID!] ?? -1; final double distanceB = fileIdToDistanceMap[b.uploadedFileID!] ?? -1; return distanceA.compareTo(distanceB); }); + w?.log('sorted files for cluster $clusterID'); debugPrint( "[${_logger.name}] Sorted suggestions for cluster $clusterID based on distance to person: ${suggestion.filesInCluster.map((e) => fileIdToDistanceMap[e.uploadedFileID]).toList()}", From 759c8aa404ddc2102b5f4c226b838e608e4d0d4b Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Wed, 24 Apr 2024 10:01:07 +0530 Subject: [PATCH 19/69] [mob][photos] Extra check in sorting suggestions --- .../machine_learning/face_ml/feedback/cluster_feedback.dart | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart b/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart index 86cd3b8384..c01e3bc895 100644 --- a/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart +++ b/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart @@ -913,6 +913,11 @@ class ClusterFeedbackService { // Sort the suggestions based on the distance to the person for (final suggestion in suggestions) { + if (onlySortBigSuggestions) { + if (suggestion.filesInCluster.length <= 8) { + continue; + } + } final clusterID = suggestion.clusterIDToMerge; final faceIDs = suggestion.faceIDsInCluster; final faceIdToEmbeddingMap = await faceMlDb.getFaceEmbeddingMapForFaces( From 093f48fb63a6846531546283e60266d33c9a492c Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Wed, 24 Apr 2024 11:24:25 +0530 Subject: [PATCH 20/69] [mob][photos] Sort found suggestions based on distance --- .../face_ml/feedback/cluster_feedback.dart | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart b/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart index c01e3bc895..0d051a18a1 100644 --- a/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart +++ b/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart @@ -812,12 +812,12 @@ class ClusterFeedbackService { for (final List<(int, double)> suggestion in suggestions.values) { suggestClusterIds.addAll(suggestion); } + suggestClusterIds.sort( + (a, b) => a.$2.compareTo(b.$2), + ); // sort by distance + List? suggestClusterIdsSizes; if (allClusterIdsToCountMap != null) { - suggestClusterIds.sort( - (a, b) => allClusterIdsToCountMap[b.$1]! - .compareTo(allClusterIdsToCountMap[a.$1]!), - ); suggestClusterIdsSizes = suggestClusterIds .map((e) => allClusterIdsToCountMap[e.$1]!) .toList(growable: false); From 4b6641d7d8025b2b68a83164d81841811ad9e46c Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Wed, 24 Apr 2024 15:46:00 +0530 Subject: [PATCH 21/69] [mob][photos] Speed up suggestion calculation --- .../face_ml/feedback/cluster_feedback.dart | 45 +++++++++++++------ 1 file changed, 31 insertions(+), 14 deletions(-) diff --git a/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart b/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart index 0d051a18a1..b81a0c1abf 100644 --- a/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart +++ b/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart @@ -777,13 +777,25 @@ class ClusterFeedbackService { Map? allClusterIdsToCountMap, }) { final Map> suggestions = {}; - for (final otherClusterID in clusterAvg.keys) { + int suggestionCount = 0; + final w = (kDebugMode ? EnteWatch('getSuggestions') : null)?..start(); + final clusterAvgVectors = clusterAvg.map( + (key, value) => MapEntry( + key, + Vector.fromList( + value, + dtype: DType.float32, + ), + ), + ); + w?.log('converted avg to vectors for ${clusterAvg.length} averages'); + for (final otherClusterID in clusterAvgVectors.keys) { // ignore the cluster that belong to the person or is ignored if (personClusters.contains(otherClusterID) || ignoredClusters.contains(otherClusterID)) { continue; } - final otherAvg = clusterAvg[otherClusterID]!; + final otherAvg = clusterAvgVectors[otherClusterID]!; int? nearestPersonCluster; double? minDistance; for (final personCluster in personClusters) { @@ -791,8 +803,8 @@ class ClusterFeedbackService { _logger.info('no avg for cluster $personCluster'); continue; } - final avg = clusterAvg[personCluster]!; - final distance = cosineDistForNormVectors(avg, otherAvg); + final avg = clusterAvgVectors[personCluster]!; + final distance = 1 - avg.dot(otherAvg); if (distance < maxClusterDistance) { if (minDistance == null || distance < minDistance) { minDistance = distance; @@ -804,8 +816,13 @@ class ClusterFeedbackService { suggestions .putIfAbsent(nearestPersonCluster, () => []) .add((otherClusterID, minDistance)); + suggestionCount++; + } + if (suggestionCount >= 2000) { + break; } } + w?.log('calculation inside calcSuggestionsMean'); if (suggestions.isNotEmpty) { final List<(int, double)> suggestClusterIds = []; @@ -816,18 +833,18 @@ class ClusterFeedbackService { (a, b) => a.$2.compareTo(b.$2), ); // sort by distance - List? suggestClusterIdsSizes; - if (allClusterIdsToCountMap != null) { - suggestClusterIdsSizes = suggestClusterIds - .map((e) => allClusterIdsToCountMap[e.$1]!) - .toList(growable: false); - } - final suggestClusterIdsDistances = - suggestClusterIds.map((e) => e.$2).toList(growable: false); + // List? suggestClusterIdsSizes; + // if (allClusterIdsToCountMap != null) { + // suggestClusterIdsSizes = suggestClusterIds + // .map((e) => allClusterIdsToCountMap[e.$1]!) + // .toList(growable: false); + // } + // final suggestClusterIdsDistances = + // suggestClusterIds.map((e) => e.$2).toList(growable: false); _logger.info( - "Already found good suggestions using mean: $suggestClusterIds, ${suggestClusterIdsSizes != null ? 'with sizes $suggestClusterIdsSizes' : ''} and distances $suggestClusterIdsDistances", + "Already found ${suggestClusterIds.length} good suggestions using mean", ); - return suggestClusterIds; + return suggestClusterIds.sublist(0, min(suggestClusterIds.length, 20)); } else { _logger.info("No suggestions found using mean"); return <(int, double)>[]; From e829f7b62fe563a4ea31d633b76807a70c3625ca Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Wed, 24 Apr 2024 16:01:03 +0530 Subject: [PATCH 22/69] [mob][photos] Use vectors everywhere in cluster suggestion --- .../face_ml/feedback/cluster_feedback.dart | 65 +++++++++---------- 1 file changed, 29 insertions(+), 36 deletions(-) diff --git a/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart b/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart index b81a0c1abf..d11afa180d 100644 --- a/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart +++ b/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart @@ -3,6 +3,7 @@ import "dart:math" show Random, min; import "package:flutter/foundation.dart"; import "package:logging/logging.dart"; +import "package:ml_linalg/linalg.dart"; import "package:photos/core/event_bus.dart"; import "package:photos/db/files_db.dart"; // import "package:photos/events/files_updated_event.dart"; @@ -245,13 +246,13 @@ class ClusterFeedbackService { final ignoredClusters = await faceMlDb.getPersonIgnoredClusters(p.remoteID); final personClusters = await faceMlDb.getPersonClusterIDs(p.remoteID); dev.log( - 'existing clusters for ${p.data.name} are $personClusters', + '${p.data.name} has ${personClusters.length} existing clusters', name: "ClusterFeedbackService", ); // Get and update the cluster summary to get the avg (centroid) and count final EnteWatch watch = EnteWatch("ClusterFeedbackService")..start(); - final Map> clusterAvg = await _getUpdateClusterAvg( + final Map clusterAvg = await _getUpdateClusterAvg( allClusterIdsToCountMap, ignoredClusters, ); @@ -466,19 +467,19 @@ class ClusterFeedbackService { final ignoredClusters = await faceMlDb.getPersonIgnoredClusters(p.remoteID); final personClusters = await faceMlDb.getPersonClusterIDs(p.remoteID); dev.log( - 'existing clusters for ${p.data.name} are $personClusters, getting all database data took ${DateTime.now().difference(startTime).inMilliseconds} ms', + '${p.data.name} has ${personClusters.length} existing clusters, getting all database data took ${DateTime.now().difference(startTime).inMilliseconds} ms', name: "getSuggestionsUsingMedian", ); // First only do a simple check on the big clusters, if the person does not have small clusters yet - final EnteWatch watch = EnteWatch("ClusterFeedbackService")..start(); + final w = (kDebugMode ? EnteWatch('getSuggestions') : null)?..start(); final smallestPersonClusterSize = personClusters .map((clusterID) => allClusterIdsToCountMap[clusterID] ?? 0) .reduce((value, element) => min(value, element)); final checkSizes = [kMinimumClusterSizeSearchResult, 20, 10, 5, 1]; for (final minimumSize in checkSizes.toSet()) { if (smallestPersonClusterSize >= minimumSize) { - final Map> clusterAvgBigClusters = + final Map clusterAvgBigClusters = await _getUpdateClusterAvg( allClusterIdsToCountMap, ignoredClusters, @@ -487,6 +488,7 @@ class ClusterFeedbackService { dev.log( 'computed avg for ${clusterAvgBigClusters.length} clusters, in ${DateTime.now().difference(startTime).inMilliseconds} ms', ); + w?.log('Calculate avg for min size $minimumSize'); final List<(int, double)> suggestionsMeanBigClusters = _calcSuggestionsMean( clusterAvgBigClusters, @@ -494,6 +496,7 @@ class ClusterFeedbackService { ignoredClusters, goodMeanDistance, ); + w?.log('Calculate suggestions using mean for min size $minimumSize'); if (suggestionsMeanBigClusters.isNotEmpty) { return suggestionsMeanBigClusters .map((e) => (e.$1, e.$2, true)) @@ -501,9 +504,10 @@ class ClusterFeedbackService { } } } + w?.reset(); // Get and update the cluster summary to get the avg (centroid) and count - final Map> clusterAvg = await _getUpdateClusterAvg( + final Map clusterAvg = await _getUpdateClusterAvg( allClusterIdsToCountMap, ignoredClusters, ); @@ -547,7 +551,7 @@ class ClusterFeedbackService { "Found potential suggestions from loose mean for median test: $otherClusterIdsCandidates", ); - watch.logAndReset("Starting median test"); + w?.logAndReset("Starting median test"); // Take the embeddings from the person's clusters in one big list and sample from it final List personEmbeddingsProto = []; for (final clusterID in personClusters) { @@ -600,7 +604,7 @@ class ClusterFeedbackService { } } } - watch.log("Finished median test"); + w?.log("Finished median test"); if (suggestionsMedian.isEmpty) { _logger.info("No suggestions found using median"); return []; @@ -632,7 +636,7 @@ class ClusterFeedbackService { return finalSuggestionsMedian; } - Future>> _getUpdateClusterAvg( + Future> _getUpdateClusterAvg( Map allClusterIdsToCountMap, Set ignoredClusters, { int minClusterSize = 1, @@ -649,7 +653,7 @@ class ClusterFeedbackService { await faceMlDb.getAllClusterSummary(minClusterSize); final Map updatesForClusterSummary = {}; - final Map> clusterAvg = {}; + final Map clusterAvg = {}; dev.log( 'getUpdateClusterAvg database call for getAllClusterSummary took ${DateTime.now().difference(startTime).inMilliseconds} ms', @@ -666,7 +670,9 @@ class ClusterFeedbackService { } if (clusterToSummary[id]?.$2 == allClusterIdsToCountMap[id]) { allClusterIds.remove(id); - clusterAvg[id] = EVector.fromBuffer(clusterToSummary[id]!.$1).values; + clusterAvg[id] = Vector.fromList( + EVector.fromBuffer(clusterToSummary[id]!.$1).values, + dtype: DType.float32,); alreadyUpdatedClustersCnt++; } if (allClusterIdsToCountMap[id]! < minClusterSize) { @@ -731,19 +737,15 @@ class ClusterFeedbackService { ); for (final clusterID in clusterEmbeddings.keys) { - late List avg; - final Iterable embedings = clusterEmbeddings[clusterID]!; - final List sum = List.filled(192, 0); - for (final embedding in embedings) { - final data = EVector.fromBuffer(embedding).values; - for (int i = 0; i < sum.length; i++) { - sum[i] += data[i]; - } - } - avg = sum.map((e) => e / embedings.length).toList(); - final avgEmbeedingBuffer = EVector(values: avg).writeToBuffer(); + final Iterable embeddings = clusterEmbeddings[clusterID]!; + final Iterable vectors = embeddings.map((e) => Vector.fromList( + EVector.fromBuffer(e).values, + dtype: DType.float32, + ),); + final avg = vectors.reduce((a, b) => a + b) / vectors.length; + final avgEmbeddingBuffer = EVector(values: avg).writeToBuffer(); updatesForClusterSummary[clusterID] = - (avgEmbeedingBuffer, embedings.length); + (avgEmbeddingBuffer, embeddings.length); // store the intermediate updates indexedInCurrentRun++; if (updatesForClusterSummary.length > 100) { @@ -770,7 +772,7 @@ class ClusterFeedbackService { /// Returns a map of person's clusterID to map of closest clusterID to with disstance List<(int, double)> _calcSuggestionsMean( - Map> clusterAvg, + Map clusterAvg, Set personClusters, Set ignoredClusters, double maxClusterDistance, { @@ -779,23 +781,14 @@ class ClusterFeedbackService { final Map> suggestions = {}; int suggestionCount = 0; final w = (kDebugMode ? EnteWatch('getSuggestions') : null)?..start(); - final clusterAvgVectors = clusterAvg.map( - (key, value) => MapEntry( - key, - Vector.fromList( - value, - dtype: DType.float32, - ), - ), - ); w?.log('converted avg to vectors for ${clusterAvg.length} averages'); - for (final otherClusterID in clusterAvgVectors.keys) { + for (final otherClusterID in clusterAvg.keys) { // ignore the cluster that belong to the person or is ignored if (personClusters.contains(otherClusterID) || ignoredClusters.contains(otherClusterID)) { continue; } - final otherAvg = clusterAvgVectors[otherClusterID]!; + final Vector otherAvg = clusterAvg[otherClusterID]!; int? nearestPersonCluster; double? minDistance; for (final personCluster in personClusters) { @@ -803,7 +796,7 @@ class ClusterFeedbackService { _logger.info('no avg for cluster $personCluster'); continue; } - final avg = clusterAvgVectors[personCluster]!; + final Vector avg = clusterAvg[personCluster]!; final distance = 1 - avg.dot(otherAvg); if (distance < maxClusterDistance) { if (minDistance == null || distance < minDistance) { From 3806ee3232fd1ebdbd708ac50bb12d34134e643e Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Wed, 24 Apr 2024 16:19:10 +0530 Subject: [PATCH 23/69] [mob][photos] Use SIMD in sorting suggestions too --- .../face_ml/feedback/cluster_feedback.dart | 73 +++++++++++-------- 1 file changed, 44 insertions(+), 29 deletions(-) diff --git a/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart b/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart index d11afa180d..0e6d1bc95a 100644 --- a/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart +++ b/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart @@ -6,15 +6,12 @@ import "package:logging/logging.dart"; import "package:ml_linalg/linalg.dart"; import "package:photos/core/event_bus.dart"; import "package:photos/db/files_db.dart"; -// import "package:photos/events/files_updated_event.dart"; -// import "package:photos/events/local_photos_updated_event.dart"; import "package:photos/events/people_changed_event.dart"; import "package:photos/extensions/stop_watch.dart"; import "package:photos/face/db.dart"; import "package:photos/face/model/person.dart"; import "package:photos/generated/protos/ente/common/vector.pb.dart"; import "package:photos/models/file/file.dart"; -import 'package:photos/services/machine_learning/face_ml/face_clustering/cosine_distance.dart'; import "package:photos/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart"; import "package:photos/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart"; import "package:photos/services/machine_learning/face_ml/face_ml_result.dart"; @@ -555,17 +552,22 @@ class ClusterFeedbackService { // Take the embeddings from the person's clusters in one big list and sample from it final List personEmbeddingsProto = []; for (final clusterID in personClusters) { - final Iterable embedings = + final Iterable embeddings = await FaceMLDataDB.instance.getFaceEmbeddingsForCluster(clusterID); - personEmbeddingsProto.addAll(embedings); + personEmbeddingsProto.addAll(embeddings); } final List sampledEmbeddingsProto = _randomSampleWithoutReplacement( personEmbeddingsProto, sampleSize, ); - final List> sampledEmbeddings = sampledEmbeddingsProto - .map((embedding) => EVector.fromBuffer(embedding).values) + final List sampledEmbeddings = sampledEmbeddingsProto + .map( + (embedding) => Vector.fromList( + EVector.fromBuffer(embedding).values, + dtype: DType.float32, + ), + ) .toList(growable: false); // Find the actual closest clusters for the person using median @@ -581,16 +583,20 @@ class ClusterFeedbackService { otherEmbeddingsProto, sampleSize, ); - final List> sampledOtherEmbeddings = - sampledOtherEmbeddingsProto - .map((embedding) => EVector.fromBuffer(embedding).values) - .toList(growable: false); + final List sampledOtherEmbeddings = sampledOtherEmbeddingsProto + .map( + (embedding) => Vector.fromList( + EVector.fromBuffer(embedding).values, + dtype: DType.float32, + ), + ) + .toList(growable: false); // Calculate distances and find the median final List distances = []; for (final otherEmbedding in sampledOtherEmbeddings) { for (final embedding in sampledEmbeddings) { - distances.add(cosineDistForNormVectors(embedding, otherEmbedding)); + distances.add(1 - embedding.dot(otherEmbedding)); } } distances.sort(); @@ -671,8 +677,9 @@ class ClusterFeedbackService { if (clusterToSummary[id]?.$2 == allClusterIdsToCountMap[id]) { allClusterIds.remove(id); clusterAvg[id] = Vector.fromList( - EVector.fromBuffer(clusterToSummary[id]!.$1).values, - dtype: DType.float32,); + EVector.fromBuffer(clusterToSummary[id]!.$1).values, + dtype: DType.float32, + ); alreadyUpdatedClustersCnt++; } if (allClusterIdsToCountMap[id]! < minClusterSize) { @@ -738,10 +745,12 @@ class ClusterFeedbackService { for (final clusterID in clusterEmbeddings.keys) { final Iterable embeddings = clusterEmbeddings[clusterID]!; - final Iterable vectors = embeddings.map((e) => Vector.fromList( - EVector.fromBuffer(e).values, - dtype: DType.float32, - ),); + final Iterable vectors = embeddings.map( + (e) => Vector.fromList( + EVector.fromBuffer(e).values, + dtype: DType.float32, + ), + ); final avg = vectors.reduce((a, b) => a + b) / vectors.length; final avgEmbeddingBuffer = EVector(values: avg).writeToBuffer(); updatesForClusterSummary[clusterID] = @@ -908,16 +917,16 @@ class ClusterFeedbackService { final personEmbeddingsCount = personClusters .map((e) => personClusterToSummary[e]!.$2) .reduce((a, b) => a + b); - final List personAvg = List.filled(192, 0); + Vector personAvg = Vector.filled(192, 0); for (final personClusterID in personClusters) { final personClusterBlob = personClusterToSummary[personClusterID]!.$1; - final personClusterAvg = EVector.fromBuffer(personClusterBlob).values; + final personClusterAvg = Vector.fromList( + EVector.fromBuffer(personClusterBlob).values, + dtype: DType.float32, + ); final clusterWeight = personClusterToSummary[personClusterID]!.$2 / personEmbeddingsCount; - for (int i = 0; i < personClusterAvg.length; i++) { - personAvg[i] += personClusterAvg[i] * - clusterWeight; // Weighted sum of the cluster averages - } + personAvg += personClusterAvg * clusterWeight; } w?.log('calculated person avg'); @@ -933,16 +942,22 @@ class ClusterFeedbackService { final faceIdToEmbeddingMap = await faceMlDb.getFaceEmbeddingMapForFaces( faceIDs, ); + final faceIdToVectorMap = faceIdToEmbeddingMap.map( + (key, value) => MapEntry( + key, + Vector.fromList( + EVector.fromBuffer(value).values, + dtype: DType.float32, + ), + ), + ); w?.log( 'got ${faceIdToEmbeddingMap.values.length} embeddings for ${suggestion.filesInCluster.length} files for cluster $clusterID', ); final fileIdToDistanceMap = {}; - for (final entry in faceIdToEmbeddingMap.entries) { + for (final entry in faceIdToVectorMap.entries) { fileIdToDistanceMap[getFileIdFromFaceId(entry.key)] = - cosineDistForNormVectors( - personAvg, - EVector.fromBuffer(entry.value).values, - ); + 1 - personAvg.dot(entry.value); } w?.log('calculated distances for cluster $clusterID'); suggestion.filesInCluster.sort((b, a) { From 05a4e9f90b9a5057511f150bba6aa91a94481ef2 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Wed, 24 Apr 2024 16:31:36 +0530 Subject: [PATCH 24/69] [mob][photos] Remove redundant logging --- .../face_ml/feedback/cluster_feedback.dart | 39 ++++++++----------- 1 file changed, 16 insertions(+), 23 deletions(-) diff --git a/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart b/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart index 0e6d1bc95a..d3f3217b83 100644 --- a/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart +++ b/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart @@ -456,20 +456,17 @@ class ClusterFeedbackService { double maxMeanDistance = 0.65, double goodMeanDistance = 0.54, }) async { + final w = (kDebugMode ? EnteWatch('getSuggestions') : null)?..start(); // Get all the cluster data - final startTime = DateTime.now(); final faceMlDb = FaceMLDataDB.instance; - // final Map> suggestions = {}; final allClusterIdsToCountMap = await faceMlDb.clusterIdToFaceCount(); final ignoredClusters = await faceMlDb.getPersonIgnoredClusters(p.remoteID); final personClusters = await faceMlDb.getPersonClusterIDs(p.remoteID); - dev.log( - '${p.data.name} has ${personClusters.length} existing clusters, getting all database data took ${DateTime.now().difference(startTime).inMilliseconds} ms', - name: "getSuggestionsUsingMedian", + w?.log( + '${p.data.name} has ${personClusters.length} existing clusters, getting all database data done', ); // First only do a simple check on the big clusters, if the person does not have small clusters yet - final w = (kDebugMode ? EnteWatch('getSuggestions') : null)?..start(); final smallestPersonClusterSize = personClusters .map((clusterID) => allClusterIdsToCountMap[clusterID] ?? 0) .reduce((value, element) => min(value, element)); @@ -482,10 +479,9 @@ class ClusterFeedbackService { ignoredClusters, minClusterSize: minimumSize, ); - dev.log( - 'computed avg for ${clusterAvgBigClusters.length} clusters, in ${DateTime.now().difference(startTime).inMilliseconds} ms', + w?.log( + 'Calculate avg for ${clusterAvgBigClusters.length} clusters of min size $minimumSize', ); - w?.log('Calculate avg for min size $minimumSize'); final List<(int, double)> suggestionsMeanBigClusters = _calcSuggestionsMean( clusterAvgBigClusters, @@ -493,7 +489,9 @@ class ClusterFeedbackService { ignoredClusters, goodMeanDistance, ); - w?.log('Calculate suggestions using mean for min size $minimumSize'); + w?.log( + 'Calculate suggestions using mean for ${clusterAvgBigClusters.length} clusters of min size $minimumSize', + ); if (suggestionsMeanBigClusters.isNotEmpty) { return suggestionsMeanBigClusters .map((e) => (e.$1, e.$2, true)) @@ -508,8 +506,8 @@ class ClusterFeedbackService { allClusterIdsToCountMap, ignoredClusters, ); - dev.log( - 'computed avg for ${clusterAvg.length} clusters, in ${DateTime.now().difference(startTime).inMilliseconds} ms', + w?.log( + 'computed avg for ${clusterAvg.length} clusters,', ); // Find the other cluster candidates based on the mean @@ -649,6 +647,7 @@ class ClusterFeedbackService { int maxClusterInCurrentRun = 500, int maxEmbeddingToRead = 10000, }) async { + final w = (kDebugMode ? EnteWatch('_getUpdateClusterAvg') : null)?..start(); final startTime = DateTime.now(); final faceMlDb = FaceMLDataDB.instance; _logger.info( @@ -661,8 +660,8 @@ class ClusterFeedbackService { final Map clusterAvg = {}; - dev.log( - 'getUpdateClusterAvg database call for getAllClusterSummary took ${DateTime.now().difference(startTime).inMilliseconds} ms', + w?.log( + 'getUpdateClusterAvg database call for getAllClusterSummary', ); final allClusterIds = allClusterIdsToCountMap.keys.toSet(); @@ -687,8 +686,8 @@ class ClusterFeedbackService { smallerClustersCnt++; } } - dev.log( - 'serialization of embeddings took ${DateTime.now().difference(serializationTime).inMilliseconds} ms', + w?.log( + 'serialization of embeddings', ); _logger.info( 'Ignored $ignoredClustersCnt clusters, already updated $alreadyUpdatedClustersCnt clusters, $smallerClustersCnt clusters are smaller than $minClusterSize', @@ -708,12 +707,7 @@ class ClusterFeedbackService { allClusterIdsToCountMap[b]!.compareTo(allClusterIdsToCountMap[a]!), ); int indexedInCurrentRun = 0; - final EnteWatch? w = kDebugMode ? EnteWatch("computeAvg") : null; - w?.start(); - - w?.log( - 'reading embeddings for $maxClusterInCurrentRun or ${sortedClusterIDs.length} clusters', - ); + w?.reset(); int currentPendingRead = 0; final List clusterIdsToRead = []; @@ -790,7 +784,6 @@ class ClusterFeedbackService { final Map> suggestions = {}; int suggestionCount = 0; final w = (kDebugMode ? EnteWatch('getSuggestions') : null)?..start(); - w?.log('converted avg to vectors for ${clusterAvg.length} averages'); for (final otherClusterID in clusterAvg.keys) { // ignore the cluster that belong to the person or is ignored if (personClusters.contains(otherClusterID) || From 462d1d48547213edaaa348d14864e6b6a56648fb Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Wed, 24 Apr 2024 16:37:39 +0530 Subject: [PATCH 25/69] [mob][photos] Use cosineDistanceSIMD --- .../face_clustering/cosine_distance.dart | 13 +++++++++++ .../face_clustering_service.dart | 22 ++++++++++--------- .../face_ml/feedback/cluster_feedback.dart | 7 +++--- .../lib/ui/viewer/people/cluster_app_bar.dart | 16 +++++++------- 4 files changed, 37 insertions(+), 21 deletions(-) diff --git a/mobile/lib/services/machine_learning/face_ml/face_clustering/cosine_distance.dart b/mobile/lib/services/machine_learning/face_ml/face_clustering/cosine_distance.dart index f8f2e68a8f..7ba4c582b9 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_clustering/cosine_distance.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_clustering/cosine_distance.dart @@ -1,5 +1,18 @@ import 'dart:math' show sqrt; +import "package:ml_linalg/vector.dart"; + +/// Calculates the cosine distance between two embeddings/vectors using SIMD from ml_linalg +/// +/// WARNING: This assumes both vectors are already normalized! +double cosineDistanceSIMD(Vector vector1, Vector vector2) { + if (vector1.length != vector2.length) { + throw ArgumentError('Vectors must be the same length'); + } + + return 1 - vector1.dot(vector2); +} + /// Calculates the cosine distance between two embeddings/vectors. /// /// Throws an ArgumentError if the vectors are of different lengths or diff --git a/mobile/lib/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart b/mobile/lib/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart index de645c01d8..8a32578e99 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart @@ -560,10 +560,10 @@ class FaceClusteringService { for (int j = i - 1; j >= 0; j--) { late double distance; if (sortedFaceInfos[i].vEmbedding != null) { - distance = 1.0 - - sortedFaceInfos[i] - .vEmbedding! - .dot(sortedFaceInfos[j].vEmbedding!); + distance = cosineDistanceSIMD( + sortedFaceInfos[i].vEmbedding!, + sortedFaceInfos[j].vEmbedding!, + ); } else { distance = cosineDistForNormVectors( sortedFaceInfos[i].embedding!, @@ -804,8 +804,10 @@ class FaceClusteringService { double closestDistance = double.infinity; for (int j = 0; j < totalFaces; j++) { if (i == j) continue; - final double distance = - 1.0 - faceInfos[i].vEmbedding!.dot(faceInfos[j].vEmbedding!); + final double distance = cosineDistanceSIMD( + faceInfos[i].vEmbedding!, + faceInfos[j].vEmbedding!, + ); if (distance < closestDistance) { closestDistance = distance; closestIdx = j; @@ -855,10 +857,10 @@ class FaceClusteringService { for (int i = 0; i < clusterIds.length; i++) { for (int j = 0; j < clusterIds.length; j++) { if (i == j) continue; - final double newDistance = 1.0 - - clusterIdToMeanEmbeddingAndWeight[clusterIds[i]]!.$1.dot( - clusterIdToMeanEmbeddingAndWeight[clusterIds[j]]!.$1, - ); + final double newDistance = cosineDistanceSIMD( + clusterIdToMeanEmbeddingAndWeight[clusterIds[i]]!.$1, + clusterIdToMeanEmbeddingAndWeight[clusterIds[j]]!.$1, + ); if (newDistance < distance) { distance = newDistance; clusterIDsToMerge = (clusterIds[i], clusterIds[j]); diff --git a/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart b/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart index d3f3217b83..236f71c43d 100644 --- a/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart +++ b/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart @@ -12,6 +12,7 @@ import "package:photos/face/db.dart"; import "package:photos/face/model/person.dart"; import "package:photos/generated/protos/ente/common/vector.pb.dart"; import "package:photos/models/file/file.dart"; +import "package:photos/services/machine_learning/face_ml/face_clustering/cosine_distance.dart"; import "package:photos/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart"; import "package:photos/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart"; import "package:photos/services/machine_learning/face_ml/face_ml_result.dart"; @@ -594,7 +595,7 @@ class ClusterFeedbackService { final List distances = []; for (final otherEmbedding in sampledOtherEmbeddings) { for (final embedding in sampledEmbeddings) { - distances.add(1 - embedding.dot(otherEmbedding)); + distances.add(cosineDistanceSIMD(embedding,otherEmbedding)); } } distances.sort(); @@ -799,7 +800,7 @@ class ClusterFeedbackService { continue; } final Vector avg = clusterAvg[personCluster]!; - final distance = 1 - avg.dot(otherAvg); + final distance = cosineDistanceSIMD(avg,otherAvg); if (distance < maxClusterDistance) { if (minDistance == null || distance < minDistance) { minDistance = distance; @@ -950,7 +951,7 @@ class ClusterFeedbackService { final fileIdToDistanceMap = {}; for (final entry in faceIdToVectorMap.entries) { fileIdToDistanceMap[getFileIdFromFaceId(entry.key)] = - 1 - personAvg.dot(entry.value); + cosineDistanceSIMD(personAvg,entry.value); } w?.log('calculated distances for cluster $clusterID'); suggestion.filesInCluster.sort((b, a) { diff --git a/mobile/lib/ui/viewer/people/cluster_app_bar.dart b/mobile/lib/ui/viewer/people/cluster_app_bar.dart index 8990c1c2c9..61913a6b83 100644 --- a/mobile/lib/ui/viewer/people/cluster_app_bar.dart +++ b/mobile/lib/ui/viewer/people/cluster_app_bar.dart @@ -207,14 +207,14 @@ class _AppBarWidgetState extends State { if (embedding.key == otherEmbedding.key) { continue; } - final distance64 = 1.0 - - Vector.fromList(embedding.value, dtype: DType.float64).dot( - Vector.fromList(otherEmbedding.value, dtype: DType.float64), - ); - final distance32 = 1.0 - - Vector.fromList(embedding.value, dtype: DType.float32).dot( - Vector.fromList(otherEmbedding.value, dtype: DType.float32), - ); + final distance64 = cosineDistanceSIMD( + Vector.fromList(embedding.value, dtype: DType.float64), + Vector.fromList(otherEmbedding.value, dtype: DType.float64), + ); + final distance32 = cosineDistanceSIMD( + Vector.fromList(embedding.value, dtype: DType.float32), + Vector.fromList(otherEmbedding.value, dtype: DType.float32), + ); final distance = cosineDistForNormVectors( embedding.value, otherEmbedding.value, From e0fbb2620baf502ca4fa4097fce65982d2b5ed12 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Wed, 24 Apr 2024 17:06:35 +0530 Subject: [PATCH 26/69] [mob][photos] Correct suggestion logic again --- .../face_ml/feedback/cluster_feedback.dart | 35 ++++--------------- 1 file changed, 7 insertions(+), 28 deletions(-) diff --git a/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart b/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart index 236f71c43d..4caf79b316 100644 --- a/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart +++ b/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart @@ -455,7 +455,7 @@ class ClusterFeedbackService { double maxMedianDistance = 0.62, double goodMedianDistance = 0.55, double maxMeanDistance = 0.65, - double goodMeanDistance = 0.54, + double goodMeanDistance = 0.50, }) async { final w = (kDebugMode ? EnteWatch('getSuggestions') : null)?..start(); // Get all the cluster data @@ -472,10 +472,10 @@ class ClusterFeedbackService { .map((clusterID) => allClusterIdsToCountMap[clusterID] ?? 0) .reduce((value, element) => min(value, element)); final checkSizes = [kMinimumClusterSizeSearchResult, 20, 10, 5, 1]; + late final Map clusterAvgBigClusters; for (final minimumSize in checkSizes.toSet()) { if (smallestPersonClusterSize >= minimumSize) { - final Map clusterAvgBigClusters = - await _getUpdateClusterAvg( + clusterAvgBigClusters = await _getUpdateClusterAvg( allClusterIdsToCountMap, ignoredClusters, minClusterSize: minimumSize, @@ -502,29 +502,8 @@ class ClusterFeedbackService { } w?.reset(); - // Get and update the cluster summary to get the avg (centroid) and count - final Map clusterAvg = await _getUpdateClusterAvg( - allClusterIdsToCountMap, - ignoredClusters, - ); - w?.log( - 'computed avg for ${clusterAvg.length} clusters,', - ); - - // Find the other cluster candidates based on the mean - final List<(int, double)> suggestionsMean = _calcSuggestionsMean( - clusterAvg, - personClusters, - ignoredClusters, - goodMeanDistance, - ); - if (suggestionsMean.isNotEmpty) { - return suggestionsMean - .map((e) => (e.$1, e.$2, true)) - .toList(growable: false); - } - // Find the other cluster candidates based on the median + final clusterAvg = clusterAvgBigClusters; final List<(int, double)> moreSuggestionsMean = _calcSuggestionsMean( clusterAvg, personClusters, @@ -595,7 +574,7 @@ class ClusterFeedbackService { final List distances = []; for (final otherEmbedding in sampledOtherEmbeddings) { for (final embedding in sampledEmbeddings) { - distances.add(cosineDistanceSIMD(embedding,otherEmbedding)); + distances.add(cosineDistanceSIMD(embedding, otherEmbedding)); } } distances.sort(); @@ -800,7 +779,7 @@ class ClusterFeedbackService { continue; } final Vector avg = clusterAvg[personCluster]!; - final distance = cosineDistanceSIMD(avg,otherAvg); + final distance = cosineDistanceSIMD(avg, otherAvg); if (distance < maxClusterDistance) { if (minDistance == null || distance < minDistance) { minDistance = distance; @@ -951,7 +930,7 @@ class ClusterFeedbackService { final fileIdToDistanceMap = {}; for (final entry in faceIdToVectorMap.entries) { fileIdToDistanceMap[getFileIdFromFaceId(entry.key)] = - cosineDistanceSIMD(personAvg,entry.value); + cosineDistanceSIMD(personAvg, entry.value); } w?.log('calculated distances for cluster $clusterID'); suggestion.filesInCluster.sort((b, a) { From 72ff6e2cf3309195d61b5d95b8b350e94b4510b6 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Wed, 24 Apr 2024 17:07:28 +0530 Subject: [PATCH 27/69] [mob][photos] Tiny change --- .../machine_learning/face_ml/feedback/cluster_feedback.dart | 1 - 1 file changed, 1 deletion(-) diff --git a/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart b/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart index 4caf79b316..1be3e85909 100644 --- a/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart +++ b/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart @@ -647,7 +647,6 @@ class ClusterFeedbackService { final allClusterIds = allClusterIdsToCountMap.keys.toSet(); int ignoredClustersCnt = 0, alreadyUpdatedClustersCnt = 0; int smallerClustersCnt = 0; - final serializationTime = DateTime.now(); for (final id in allClusterIdsToCountMap.keys) { if (ignoredClusters.contains(id)) { allClusterIds.remove(id); From 07458fb247a40afc5b171deb05501f7edd16ab84 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Wed, 24 Apr 2024 17:15:11 +0530 Subject: [PATCH 28/69] [mob][photos] Recompute suggestions on rejected suggestion --- .../people/person_cluster_suggestion.dart | 23 +++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/mobile/lib/ui/viewer/people/person_cluster_suggestion.dart b/mobile/lib/ui/viewer/people/person_cluster_suggestion.dart index 583aaccf60..e310c5af88 100644 --- a/mobile/lib/ui/viewer/people/person_cluster_suggestion.dart +++ b/mobile/lib/ui/viewer/people/person_cluster_suggestion.dart @@ -116,20 +116,25 @@ class _PersonClustersState extends State { clusterID: clusterID, ); Bus.instance.fire(PeopleChangedEvent()); + // Increment the suggestion index + if (mounted) { + setState(() => currentSuggestionIndex++); + } + + // Check if we need to fetch new data + if (currentSuggestionIndex >= (numberOfSuggestions)) { + setState(() { + currentSuggestionIndex = 0; + futureBuilderKey = UniqueKey(); // Reset to trigger FutureBuilder + _fetchClusterSuggestions(); + }); + } } else { await FaceMLDataDB.instance.captureNotPersonFeedback( personID: widget.person.remoteID, clusterID: clusterID, ); - } - - // Increment the suggestion index - if (mounted) { - setState(() => currentSuggestionIndex++); - } - - // Check if we need to fetch new data - if (currentSuggestionIndex >= (numberOfSuggestions)) { + // Recalculate the suggestions when a suggestion is rejected setState(() { currentSuggestionIndex = 0; futureBuilderKey = UniqueKey(); // Reset to trigger FutureBuilder From 6f6f976dec336f17cd894ee354829e42f74f5fbf Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Wed, 24 Apr 2024 18:33:00 +0530 Subject: [PATCH 29/69] [mob][photos] Fix bug --- .../machine_learning/face_ml/feedback/cluster_feedback.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart b/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart index 1be3e85909..b017aa6919 100644 --- a/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart +++ b/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart @@ -472,7 +472,7 @@ class ClusterFeedbackService { .map((clusterID) => allClusterIdsToCountMap[clusterID] ?? 0) .reduce((value, element) => min(value, element)); final checkSizes = [kMinimumClusterSizeSearchResult, 20, 10, 5, 1]; - late final Map clusterAvgBigClusters; + late Map clusterAvgBigClusters; for (final minimumSize in checkSizes.toSet()) { if (smallestPersonClusterSize >= minimumSize) { clusterAvgBigClusters = await _getUpdateClusterAvg( From 3fbfa8c0e6cd07ff27791db5f32444885a5a1c0a Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Wed, 24 Apr 2024 18:59:08 +0530 Subject: [PATCH 30/69] [mob][photos] Precompute face thumbnails for suggestions --- .../people/person_cluster_suggestion.dart | 32 ++++++++++- .../search/result/person_face_widget.dart | 57 ++++++++++++++++++- 2 files changed, 85 insertions(+), 4 deletions(-) diff --git a/mobile/lib/ui/viewer/people/person_cluster_suggestion.dart b/mobile/lib/ui/viewer/people/person_cluster_suggestion.dart index e310c5af88..bf30f928c7 100644 --- a/mobile/lib/ui/viewer/people/person_cluster_suggestion.dart +++ b/mobile/lib/ui/viewer/people/person_cluster_suggestion.dart @@ -1,3 +1,4 @@ +import "dart:async" show unawaited; import "dart:math"; import "package:flutter/foundation.dart" show kDebugMode; @@ -61,8 +62,9 @@ class _PersonClustersState extends State { ), ); } - final numberOfDifferentSuggestions = snapshot.data!.length; - final currentSuggestion = snapshot.data![currentSuggestionIndex]; + final allSuggestions = snapshot.data!; + final numberOfDifferentSuggestions = allSuggestions.length; + final currentSuggestion = allSuggestions[currentSuggestionIndex]; final int clusterID = currentSuggestion.clusterIDToMerge; final double distance = currentSuggestion.distancePersonToCluster; final bool usingMean = currentSuggestion.usedOnlyMeanForSuggestion; @@ -90,6 +92,7 @@ class _PersonClustersState extends State { usingMean, files, numberOfDifferentSuggestions, + allSuggestions, ), ), ); @@ -155,8 +158,9 @@ class _PersonClustersState extends State { bool usingMean, List files, int numberOfSuggestions, + List allSuggestions, ) { - return Column( + final widgetToReturn = Column( key: ValueKey("cluster_id-$clusterID"), children: [ if (kDebugMode) @@ -233,6 +237,28 @@ class _PersonClustersState extends State { ), ], ); + // Precompute face thumbnails for next suggestions, in case there are + const precompute = 6; + const maxComputations = 10; + int compCount = 0; + + if (allSuggestions.length > currentSuggestionIndex + 1) { + for (final suggestion in allSuggestions.sublist( + currentSuggestionIndex + 1, + min(allSuggestions.length, currentSuggestionIndex + precompute), + )) { + final files = suggestion.filesInCluster; + final clusterID = suggestion.clusterIDToMerge; + for (final file in files.sublist(0, min(files.length, 8))) { + unawaited(PersonFaceWidget.precomputeFaceCrops(file, clusterID)); + compCount++; + if (compCount >= maxComputations) { + break; + } + } + } + } + return widgetToReturn; } List _buildThumbnailWidgets( diff --git a/mobile/lib/ui/viewer/search/result/person_face_widget.dart b/mobile/lib/ui/viewer/search/result/person_face_widget.dart index f2c96fc7a9..bdbfaab942 100644 --- a/mobile/lib/ui/viewer/search/result/person_face_widget.dart +++ b/mobile/lib/ui/viewer/search/result/person_face_widget.dart @@ -33,9 +33,64 @@ class PersonFaceWidget extends StatelessWidget { ), super(key: key); + static Future precomputeFaceCrops(file, clusterID) async { + try { + final Face? face = await FaceMLDataDB.instance.getCoverFaceForPerson( + recentFileID: file.uploadedFileID!, + clusterID: clusterID, + ); + if (face == null) { + debugPrint( + "No cover face for cluster $clusterID and recentFile ${file.uploadedFileID}", + ); + return; + } + final Uint8List? cachedFace = faceCropCache.get(face.faceID); + if (cachedFace != null) { + return; + } + final faceCropCacheFile = cachedFaceCropPath(face.faceID); + if ((await faceCropCacheFile.exists())) { + final data = await faceCropCacheFile.readAsBytes(); + faceCropCache.put(face.faceID, data); + return; + } + EnteFile? fileForFaceCrop = file; + if (face.fileID != file.uploadedFileID!) { + fileForFaceCrop = + await FilesDB.instance.getAnyUploadedFile(face.fileID); + } + if (fileForFaceCrop == null) { + return; + } + + final result = await pool.withResource( + () async => await getFaceCrops( + fileForFaceCrop!, + { + face.faceID: face.detection.box, + }, + ), + ); + final Uint8List? computedCrop = result?[face.faceID]; + if (computedCrop != null) { + faceCropCache.put(face.faceID, computedCrop); + faceCropCacheFile.writeAsBytes(computedCrop).ignore(); + } + return; + } catch (e, s) { + log( + "Error getting cover face for cluster $clusterID", + error: e, + stackTrace: s, + ); + return; + } + } + @override Widget build(BuildContext context) { - if (useGeneratedFaceCrops) { + if (!useGeneratedFaceCrops) { return FutureBuilder( future: getFaceCrop(), builder: (context, snapshot) { From 244d562207186e0918d81147fe44e9f51366dcd3 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Thu, 25 Apr 2024 08:50:44 +0530 Subject: [PATCH 31/69] [mob][photos] Increase the pool for face thumbnail generation --- mobile/lib/utils/face/face_box_crop.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobile/lib/utils/face/face_box_crop.dart b/mobile/lib/utils/face/face_box_crop.dart index 94228f8cd3..9bf36fbdd8 100644 --- a/mobile/lib/utils/face/face_box_crop.dart +++ b/mobile/lib/utils/face/face_box_crop.dart @@ -11,7 +11,7 @@ import "package:photos/utils/thumbnail_util.dart"; import "package:pool/pool.dart"; final LRUMap faceCropCache = LRUMap(1000); -final pool = Pool(5, timeout: const Duration(seconds: 15)); +final pool = Pool(10, timeout: const Duration(seconds: 15)); Future?> getFaceCrops( EnteFile file, Map faceBoxeMap, From b022ef6d1e6420205ae6a6baa28feffe70679d17 Mon Sep 17 00:00:00 2001 From: ashilkn Date: Thu, 25 Apr 2024 09:35:55 +0530 Subject: [PATCH 32/69] [mob] Crop image instead of using scale and translate transforms on OG image in CroppedFaceImageView widget --- .../people/cropped_face_image_view.dart | 144 +++++++++--------- mobile/lib/utils/image_util.dart | 35 +++++ 2 files changed, 103 insertions(+), 76 deletions(-) diff --git a/mobile/lib/ui/viewer/people/cropped_face_image_view.dart b/mobile/lib/ui/viewer/people/cropped_face_image_view.dart index 4ef3692595..cb7baff42e 100644 --- a/mobile/lib/ui/viewer/people/cropped_face_image_view.dart +++ b/mobile/lib/ui/viewer/people/cropped_face_image_view.dart @@ -1,11 +1,14 @@ -import 'dart:developer' show log; import "dart:io" show File; +import 'dart:ui' as ui; import 'package:flutter/material.dart'; +import "package:image/image.dart" as img; +import "package:logging/logging.dart"; import "package:photos/face/model/face.dart"; import "package:photos/models/file/file.dart"; import "package:photos/ui/viewer/file/thumbnail_widget.dart"; import "package:photos/utils/file_util.dart"; +import "package:photos/utils/image_util.dart"; class CroppedFaceInfo { final Image image; @@ -21,7 +24,7 @@ class CroppedFaceInfo { }); } -class CroppedFaceImageView extends StatelessWidget { +class CroppedFaceImageView extends StatefulWidget { final EnteFile enteFile; final Face face; @@ -32,85 +35,74 @@ class CroppedFaceImageView extends StatelessWidget { }) : super(key: key); @override - Widget build(BuildContext context) { - return FutureBuilder( - future: getImage(), - builder: (context, snapshot) { - if (snapshot.hasData) { - return LayoutBuilder( - builder: ((context, constraints) { - final double imageAspectRatio = enteFile.width / enteFile.height; - final Image image = snapshot.data!; + CroppedFaceImageViewState createState() => CroppedFaceImageViewState(); +} - final double viewWidth = constraints.maxWidth; - final double viewHeight = constraints.maxHeight; +class CroppedFaceImageViewState extends State { + ui.Image? _image; + final _logger = Logger("CroppedFaceImageView"); - final faceBox = face.detection.box; - - final double relativeFaceCenterX = - faceBox.xMin + faceBox.width / 2; - final double relativeFaceCenterY = - faceBox.yMin + faceBox.height / 2; - - const double desiredFaceHeightRelativeToWidget = 8 / 10; - final double scale = - (1 / faceBox.height) * desiredFaceHeightRelativeToWidget; - - final double widgetCenterX = viewWidth / 2; - final double widgetCenterY = viewHeight / 2; - - final double widgetAspectRatio = viewWidth / viewHeight; - final double imageToWidgetRatio = - imageAspectRatio / widgetAspectRatio; - - double offsetX = - (widgetCenterX - relativeFaceCenterX * viewWidth) * scale; - double offsetY = - (widgetCenterY - relativeFaceCenterY * viewHeight) * scale; - - if (imageAspectRatio < widgetAspectRatio) { - // Landscape Image: Adjust offsetX more conservatively - offsetX = offsetX * imageToWidgetRatio; - } else { - // Portrait Image: Adjust offsetY more conservatively - offsetY = offsetY / imageToWidgetRatio; - } - return ClipRRect( - borderRadius: const BorderRadius.all(Radius.elliptical(16, 12)), - child: Transform.translate( - offset: Offset( - offsetX, - offsetY, - ), - child: Transform.scale( - scale: scale, - child: image, - ), - ), - ); - }), - ); - } else { - if (snapshot.hasError) { - log('Error getting cover face for person: ${snapshot.error}'); - } - return ThumbnailWidget( - enteFile, - ); - } - }, - ); + @override + void initState() { + super.initState(); + _loadImage(); } - Future getImage() async { - final File? ioFile = await getFile(enteFile); - if (ioFile == null) { + @override + void dispose() { + super.dispose(); + _image?.dispose(); + } + + Future _loadImage() async { + final image = await getImage(); + if (mounted) { + setState(() { + _image = image; + }); + } + } + + @override + Widget build(BuildContext context) { + return _image != null + ? LayoutBuilder( + builder: (context, constraints) { + return RawImage( + image: _image!, + ); + }, + ) + : ThumbnailWidget(widget.enteFile); + } + + Future getImage() async { + try { + final faceBox = widget.face.detection.box; + final File? ioFile = await getFile(widget.enteFile); + if (ioFile == null) { + return null; + } + + final image = await img.decodeImageFile(ioFile.path); + + if (image == null) { + throw Exception("Failed decoding image file ${widget.enteFile.title}}"); + } + + final croppedImage = img.copyCrop( + image, + x: (image.width * faceBox.xMin).round(), + y: (image.height * faceBox.yMin).round(), + width: (image.width * faceBox.width).round(), + height: (image.height * faceBox.height).round(), + antialias: false, + ); + + return convertImageToFlutterUi(croppedImage); + } catch (e, s) { + _logger.severe("Error getting image", e, s); return null; } - - final imageData = await ioFile.readAsBytes(); - final image = Image.memory(imageData, fit: BoxFit.contain); - - return image; } } diff --git a/mobile/lib/utils/image_util.dart b/mobile/lib/utils/image_util.dart index a5bcb03a75..7eb1e39fc3 100644 --- a/mobile/lib/utils/image_util.dart +++ b/mobile/lib/utils/image_util.dart @@ -1,6 +1,8 @@ import 'dart:async'; +import 'dart:ui' as ui; import 'package:flutter/widgets.dart'; +import 'package:image/image.dart' as img; Future getImageInfo(ImageProvider imageProvider) { final completer = Completer(); @@ -14,3 +16,36 @@ Future getImageInfo(ImageProvider imageProvider) { completer.future.whenComplete(() => imageStream.removeListener(listener)); return completer.future; } + +///https://github.com/brendan-duncan/image/blob/main/doc/flutter.md +Future convertImageToFlutterUi(img.Image image) async { + if (image.format != img.Format.uint8 || image.numChannels != 4) { + final cmd = img.Command() + ..image(image) + ..convert(format: img.Format.uint8, numChannels: 4); + final rgba8 = await cmd.getImageThread(); + if (rgba8 != null) { + image = rgba8; + } + } + + final ui.ImmutableBuffer buffer = + await ui.ImmutableBuffer.fromUint8List(image.toUint8List()); + + final ui.ImageDescriptor id = ui.ImageDescriptor.raw( + buffer, + height: image.height, + width: image.width, + pixelFormat: ui.PixelFormat.rgba8888, + ); + + final ui.Codec codec = await id.instantiateCodec( + targetHeight: image.height, + targetWidth: image.width, + ); + + final ui.FrameInfo fi = await codec.getNextFrame(); + final ui.Image uiImage = fi.image; + + return uiImage; +} From d0420ce477d317c067285318bf189572bf50f53a Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Thu, 25 Apr 2024 10:03:05 +0530 Subject: [PATCH 33/69] [mob][photos] Better sorting of faces in file info --- .../file_details/faces_item_widget.dart | 26 ++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/mobile/lib/ui/viewer/file_details/faces_item_widget.dart b/mobile/lib/ui/viewer/file_details/faces_item_widget.dart index 783ed2262b..6ffdcbd37c 100644 --- a/mobile/lib/ui/viewer/file_details/faces_item_widget.dart +++ b/mobile/lib/ui/viewer/file_details/faces_item_widget.dart @@ -85,9 +85,6 @@ class _FacesItemWidgetState extends State { ]; } - // Sort the faces by score in descending order, so that the highest scoring face is first. - faces.sort((Face a, Face b) => b.score.compareTo(a.score)); - // TODO: add deduplication of faces of same person final faceIdsToClusterIds = await FaceMLDataDB.instance .getFaceIdsToClusterIds(faces.map((face) => face.faceID)); @@ -96,6 +93,29 @@ class _FacesItemWidgetState extends State { final clusterIDToPerson = await FaceMLDataDB.instance.getClusterIDToPersonID(); + // Sort faces by name and score + final faceIdToPersonID = {}; + for (final face in faces) { + final clusterID = faceIdsToClusterIds[face.faceID]; + if (clusterID != null) { + final personID = clusterIDToPerson[clusterID]; + if (personID != null) { + faceIdToPersonID[face.faceID] = personID; + } + } + } + faces.sort((Face a, Face b) { + final aPersonID = faceIdToPersonID[a.faceID]; + final bPersonID = faceIdToPersonID[b.faceID]; + if (aPersonID != null && bPersonID == null) { + return -1; + } else if (aPersonID == null && bPersonID != null) { + return 1; + } else { + return b.score.compareTo(a.score); + } + }); + final lastViewedClusterID = ClusterFeedbackService.lastViewedClusterID; final faceWidgets = []; From 1ae4482fe5a84ceaaa7b773c3ed72e849d8947df Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Thu, 25 Apr 2024 10:53:42 +0530 Subject: [PATCH 34/69] [mob][photos] Always check big clusters first for suggestions --- .../face_ml/feedback/cluster_feedback.dart | 48 +++++++++---------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart b/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart index b017aa6919..f56286e5d6 100644 --- a/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart +++ b/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart @@ -474,30 +474,30 @@ class ClusterFeedbackService { final checkSizes = [kMinimumClusterSizeSearchResult, 20, 10, 5, 1]; late Map clusterAvgBigClusters; for (final minimumSize in checkSizes.toSet()) { - if (smallestPersonClusterSize >= minimumSize) { - clusterAvgBigClusters = await _getUpdateClusterAvg( - allClusterIdsToCountMap, - ignoredClusters, - minClusterSize: minimumSize, - ); - w?.log( - 'Calculate avg for ${clusterAvgBigClusters.length} clusters of min size $minimumSize', - ); - final List<(int, double)> suggestionsMeanBigClusters = - _calcSuggestionsMean( - clusterAvgBigClusters, - personClusters, - ignoredClusters, - goodMeanDistance, - ); - w?.log( - 'Calculate suggestions using mean for ${clusterAvgBigClusters.length} clusters of min size $minimumSize', - ); - if (suggestionsMeanBigClusters.isNotEmpty) { - return suggestionsMeanBigClusters - .map((e) => (e.$1, e.$2, true)) - .toList(growable: false); - } + // if (smallestPersonClusterSize >= minimumSize) { + clusterAvgBigClusters = await _getUpdateClusterAvg( + allClusterIdsToCountMap, + ignoredClusters, + minClusterSize: minimumSize, + ); + w?.log( + 'Calculate avg for ${clusterAvgBigClusters.length} clusters of min size $minimumSize', + ); + final List<(int, double)> suggestionsMeanBigClusters = + _calcSuggestionsMean( + clusterAvgBigClusters, + personClusters, + ignoredClusters, + goodMeanDistance, + ); + w?.log( + 'Calculate suggestions using mean for ${clusterAvgBigClusters.length} clusters of min size $minimumSize', + ); + if (suggestionsMeanBigClusters.isNotEmpty) { + return suggestionsMeanBigClusters + .map((e) => (e.$1, e.$2, true)) + .toList(growable: false); + // } } } w?.reset(); From 7e00a470aad563d444ff5f3554a53b8b011cd856 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Thu, 25 Apr 2024 10:54:19 +0530 Subject: [PATCH 35/69] [mob][photos] Lower both hard and soft blur thresholds --- .../face_ml/face_filtering/face_filtering_constants.dart | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mobile/lib/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart b/mobile/lib/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart index 0feb275a7b..b0f954f8f9 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart @@ -1,8 +1,8 @@ import 'package:photos/services/machine_learning/face_ml/face_detection/face_detection_service.dart'; /// Blur detection threshold -const kLaplacianHardThreshold = 15; -const kLaplacianSoftThreshold = 100; +const kLaplacianHardThreshold = 10; +const kLaplacianSoftThreshold = 50; const kLaplacianVerySoftThreshold = 200; /// Default blur value From 7b8816a4bf9d08e510288e8ef2ccb635b1809f72 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Thu, 25 Apr 2024 10:54:49 +0530 Subject: [PATCH 36/69] [mob][photos] Higher conservative clustering threshold --- .../face_ml/face_clustering/face_clustering_service.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobile/lib/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart b/mobile/lib/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart index 8a32578e99..f504d55b18 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart @@ -69,7 +69,7 @@ class FaceClusteringService { bool isRunning = false; static const kRecommendedDistanceThreshold = 0.24; - static const kConservativeDistanceThreshold = 0.06; + static const kConservativeDistanceThreshold = 0.15; // singleton pattern FaceClusteringService._privateConstructor(); From d429efaf149623745f7edcee0a59ae1feb997766 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Thu, 25 Apr 2024 12:19:19 +0530 Subject: [PATCH 37/69] [mob][photos] Trailing commas --- mobile/lib/face/db.dart | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/mobile/lib/face/db.dart b/mobile/lib/face/db.dart index 74b904ced4..f054ebf65d 100644 --- a/mobile/lib/face/db.dart +++ b/mobile/lib/face/db.dart @@ -332,7 +332,9 @@ class FaceMLDataDB { return mapRowToFace(result.first); } - Future>> getClusterToFaceIDs(Set clusterIDs) async { + Future>> getClusterToFaceIDs( + Set clusterIDs, + ) async { final db = await instance.sqliteAsyncDB; final Map> result = {}; final List> maps = await db.getAll( @@ -860,7 +862,9 @@ class FaceMLDataDB { return result; } - Future> getClusterToClusterSummary(Iterable clusterIDs) async { + Future> getClusterToClusterSummary( + Iterable clusterIDs, + ) async { final db = await instance.sqliteAsyncDB; final Map result = {}; final rows = await db.getAll( From 2f7e0cd1ef6e5676e1d294b850835f1487e6c71c Mon Sep 17 00:00:00 2001 From: ashilkn Date: Thu, 25 Apr 2024 12:29:29 +0530 Subject: [PATCH 38/69] [mob] perf: Decode images from which face is to be cropped, in an isolate to avoid jank --- .../people/cropped_face_image_view.dart | 36 +++++++++++++++++-- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/mobile/lib/ui/viewer/people/cropped_face_image_view.dart b/mobile/lib/ui/viewer/people/cropped_face_image_view.dart index cb7baff42e..0da4129ebb 100644 --- a/mobile/lib/ui/viewer/people/cropped_face_image_view.dart +++ b/mobile/lib/ui/viewer/people/cropped_face_image_view.dart @@ -1,7 +1,10 @@ import "dart:io" show File; import 'dart:ui' as ui; +import "package:computer/computer.dart"; import 'package:flutter/material.dart'; +import "package:flutter/widgets.dart"; +import "package:flutter_image_compress/flutter_image_compress.dart"; import "package:image/image.dart" as img; import "package:logging/logging.dart"; import "package:photos/face/model/face.dart"; @@ -40,6 +43,7 @@ class CroppedFaceImageView extends StatefulWidget { class CroppedFaceImageViewState extends State { ui.Image? _image; + final _computer = Computer.shared(); final _logger = Logger("CroppedFaceImageView"); @override @@ -79,17 +83,32 @@ class CroppedFaceImageViewState extends State { Future getImage() async { try { final faceBox = widget.face.detection.box; + final File? ioFile = await getFile(widget.enteFile); if (ioFile == null) { return null; } - final image = await img.decodeImageFile(ioFile.path); + img.Image? image = await _computer + .compute(decodeImage, param: {"filePath": ioFile.path}); if (image == null) { - throw Exception("Failed decoding image file ${widget.enteFile.title}}"); + _logger.info( + "Failed to decode image ${widget.enteFile.title}. Compressing to jpg and decoding", + ); + final compressedJPGImage = + await FlutterImageCompress.compressWithFile(ioFile.path); + image = await _computer.compute( + decodeJPGImage, + param: {"image": compressedJPGImage}, + ); + + if (image == null) { + throw Exception("Failed to decode image"); + } } + final stopwatch = Stopwatch()..start(); final croppedImage = img.copyCrop( image, x: (image.width * faceBox.xMin).round(), @@ -98,7 +117,10 @@ class CroppedFaceImageViewState extends State { height: (image.height * faceBox.height).round(), antialias: false, ); - + _logger.info( + "Image crop took ${stopwatch.elapsedMilliseconds}ms ----------------", + ); + stopwatch.stop(); return convertImageToFlutterUi(croppedImage); } catch (e, s) { _logger.severe("Error getting image", e, s); @@ -106,3 +128,11 @@ class CroppedFaceImageViewState extends State { } } } + +Future decodeImage(Map args) async { + return await img.decodeImageFile(args["filePath"]); +} + +img.Image? decodeJPGImage(Map args) { + return img.decodeJpg(args["image"])!; +} From f0ebdb211cfbc510cb5894f5e2e05409e95fb2e4 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Thu, 25 Apr 2024 14:13:58 +0530 Subject: [PATCH 39/69] [mob][photos] Functionality to remove selected images from suggestion --- .../people/person_cluster_suggestion.dart | 29 +++++++++++++++++-- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/mobile/lib/ui/viewer/people/person_cluster_suggestion.dart b/mobile/lib/ui/viewer/people/person_cluster_suggestion.dart index bf30f928c7..ff9b1e7559 100644 --- a/mobile/lib/ui/viewer/people/person_cluster_suggestion.dart +++ b/mobile/lib/ui/viewer/people/person_cluster_suggestion.dart @@ -1,4 +1,4 @@ -import "dart:async" show unawaited; +import "dart:async" show StreamSubscription, unawaited; import "dart:math"; import "package:flutter/foundation.dart" show kDebugMode; @@ -30,16 +30,25 @@ class PersonReviewClusterSuggestion extends StatefulWidget { class _PersonClustersState extends State { int currentSuggestionIndex = 0; + bool fetch = true; Key futureBuilderKey = UniqueKey(); // Declare a variable for the future late Future> futureClusterSuggestions; + late StreamSubscription _peopleChangedEvent; @override void initState() { super.initState(); // Initialize the future in initState - _fetchClusterSuggestions(); + if (fetch) _fetchClusterSuggestions(); + fetch = true; + } + + @override + void dispose() { + _peopleChangedEvent.cancel(); + super.dispose(); } @override @@ -62,6 +71,7 @@ class _PersonClustersState extends State { ), ); } + final allSuggestions = snapshot.data!; final numberOfDifferentSuggestions = allSuggestions.length; final currentSuggestion = allSuggestions[currentSuggestionIndex]; @@ -69,6 +79,19 @@ class _PersonClustersState extends State { final double distance = currentSuggestion.distancePersonToCluster; final bool usingMean = currentSuggestion.usedOnlyMeanForSuggestion; final List files = currentSuggestion.filesInCluster; + + _peopleChangedEvent = + Bus.instance.on().listen((event) { + if (event.type == PeopleEventType.removedFilesFromCluster && + (event.source == clusterID.toString())) { + for (var updatedFile in event.relevantFiles!) { + files.remove(updatedFile); + } + fetch = false; + setState(() {}); + } + }); + return InkWell( onTap: () { Navigator.of(context).push( @@ -161,7 +184,7 @@ class _PersonClustersState extends State { List allSuggestions, ) { final widgetToReturn = Column( - key: ValueKey("cluster_id-$clusterID"), + key: ValueKey("cluster_id-$clusterID-files-${files.length}"), children: [ if (kDebugMode) Text( From 7370557b0866fc1343d4d12852c5728aa9afc73f Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Thu, 25 Apr 2024 15:41:26 +0530 Subject: [PATCH 40/69] [mob][photos] More use of sqlite async --- mobile/lib/face/db.dart | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/mobile/lib/face/db.dart b/mobile/lib/face/db.dart index f054ebf65d..53a3a22e8b 100644 --- a/mobile/lib/face/db.dart +++ b/mobile/lib/face/db.dart @@ -146,8 +146,8 @@ class FaceMLDataDB { } Future> clusterIdToFaceCount() async { - final db = await instance.database; - final List> maps = await db.rawQuery( + final db = await instance.sqliteAsyncDB; + final List> maps = await db.getAll( 'SELECT $fcClusterID, COUNT(*) as count FROM $faceClustersTable where $fcClusterID IS NOT NULL GROUP BY $fcClusterID ', ); final Map result = {}; @@ -158,15 +158,15 @@ class FaceMLDataDB { } Future> getPersonIgnoredClusters(String personID) async { - final db = await instance.database; + final db = await instance.sqliteAsyncDB; // find out clusterIds that are assigned to other persons using the clusters table - final List> maps = await db.rawQuery( + final List> maps = await db.getAll( 'SELECT $clusterIDColumn FROM $clusterPersonTable WHERE $personIdColumn != ? AND $personIdColumn IS NOT NULL', [personID], ); final Set ignoredClusterIDs = maps.map((e) => e[clusterIDColumn] as int).toSet(); - final List> rejectMaps = await db.rawQuery( + final List> rejectMaps = await db.getAll( 'SELECT $clusterIDColumn FROM $notPersonFeedback WHERE $personIdColumn = ?', [personID], ); @@ -176,8 +176,8 @@ class FaceMLDataDB { } Future> getPersonClusterIDs(String personID) async { - final db = await instance.database; - final List> maps = await db.rawQuery( + final db = await instance.sqliteAsyncDB; + final List> maps = await db.getAll( 'SELECT $clusterIDColumn FROM $clusterPersonTable WHERE $personIdColumn = ?', [personID], ); From 43cbfbfa332e64a239c647e65a273983210c5b76 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Thu, 25 Apr 2024 16:13:29 +0530 Subject: [PATCH 41/69] [mob][photos] Automatically reject overlapping suggestions --- .../face_ml/feedback/cluster_feedback.dart | 35 +++++++++++++++---- 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart b/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart index f56286e5d6..8734842723 100644 --- a/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart +++ b/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart @@ -98,10 +98,10 @@ class ClusterFeedbackService { } } - final List clusterIdAndFiles = []; + final List finalSuggestions = []; for (final clusterSuggestion in foundSuggestions) { if (clusterIDToFiles.containsKey(clusterSuggestion.$1)) { - clusterIdAndFiles.add( + finalSuggestions.add( ClusterSuggestion( clusterSuggestion.$1, clusterSuggestion.$2, @@ -116,13 +116,13 @@ class ClusterFeedbackService { final sortingStartTime = DateTime.now(); if (extremeFilesFirst) { - await _sortSuggestionsOnDistanceToPerson(person, clusterIdAndFiles); + await _sortSuggestionsOnDistanceToPerson(person, finalSuggestions); } _logger.info( 'getSuggestionForPerson post-processing suggestions took ${DateTime.now().difference(findSuggestionsTime).inMilliseconds} ms, of which sorting took ${DateTime.now().difference(sortingStartTime).inMilliseconds} ms and getting files took ${getFilesTime.difference(findSuggestionsTime).inMilliseconds} ms', ); - return clusterIdAndFiles; + return finalSuggestions; } catch (e, s) { _logger.severe("Error in getClusterFilesForPersonID", e, s); rethrow; @@ -463,9 +463,15 @@ class ClusterFeedbackService { final allClusterIdsToCountMap = await faceMlDb.clusterIdToFaceCount(); final ignoredClusters = await faceMlDb.getPersonIgnoredClusters(p.remoteID); final personClusters = await faceMlDb.getPersonClusterIDs(p.remoteID); + final personFaceIDs = + await FaceMLDataDB.instance.getFaceIDsForPerson(p.remoteID); + final personFileIDs = personFaceIDs.map(getFileIdFromFaceId).toSet(); w?.log( '${p.data.name} has ${personClusters.length} existing clusters, getting all database data done', ); + final allClusterIdToFaceIDs = + await FaceMLDataDB.instance.getAllClusterIdToFaceIDs(); + w?.log('getAllClusterIdToFaceIDs done'); // First only do a simple check on the big clusters, if the person does not have small clusters yet final smallestPersonClusterSize = personClusters @@ -473,6 +479,7 @@ class ClusterFeedbackService { .reduce((value, element) => min(value, element)); final checkSizes = [kMinimumClusterSizeSearchResult, 20, 10, 5, 1]; late Map clusterAvgBigClusters; + final List<(int, double)> suggestionsMean = []; for (final minimumSize in checkSizes.toSet()) { // if (smallestPersonClusterSize >= minimumSize) { clusterAvgBigClusters = await _getUpdateClusterAvg( @@ -493,8 +500,24 @@ class ClusterFeedbackService { w?.log( 'Calculate suggestions using mean for ${clusterAvgBigClusters.length} clusters of min size $minimumSize', ); - if (suggestionsMeanBigClusters.isNotEmpty) { - return suggestionsMeanBigClusters + for (final suggestion in suggestionsMeanBigClusters) { + // Skip suggestions that have a high overlap with the person's files + final suggestionSet = allClusterIdToFaceIDs[suggestion.$1]! + .map((faceID) => getFileIdFromFaceId(faceID)) + .toSet(); + final overlap = personFileIDs.intersection(suggestionSet); + if (overlap.isNotEmpty && + ((overlap.length / suggestionSet.length) > 0.5)) { + await FaceMLDataDB.instance.captureNotPersonFeedback( + personID: p.remoteID, + clusterID: suggestion.$1, + ); + continue; + } + suggestionsMean.add(suggestion); + } + if (suggestionsMean.isNotEmpty) { + return suggestionsMean .map((e) => (e.$1, e.$2, true)) .toList(growable: false); // } From 7fd5ffc0e6756c750b03ae628c797ee0ea0c9b51 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Thu, 25 Apr 2024 16:14:16 +0530 Subject: [PATCH 42/69] [mob][photos] Forgot method --- mobile/lib/face/db.dart | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/mobile/lib/face/db.dart b/mobile/lib/face/db.dart index 53a3a22e8b..a71fcc8864 100644 --- a/mobile/lib/face/db.dart +++ b/mobile/lib/face/db.dart @@ -348,6 +348,20 @@ class FaceMLDataDB { return result; } + Future>> getAllClusterIdToFaceIDs() async { + final db = await instance.sqliteAsyncDB; + final Map> result = {}; + final List> maps = await db.getAll( + 'SELECT $fcClusterID, $fcFaceId FROM $faceClustersTable', + ); + for (final map in maps) { + final clusterID = map[fcClusterID] as int; + final faceID = map[fcFaceId] as String; + result.putIfAbsent(clusterID, () => []).add(faceID); + } + return result; + } + Future> getFaceIDsForCluster(int clusterID) async { final db = await instance.sqliteAsyncDB; final List> maps = await db.getAll( From 52a7f2753ef8e6bd173d416f82236c25c74f3e1f Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Thu, 25 Apr 2024 16:15:04 +0530 Subject: [PATCH 43/69] [mob][photos] Tiny change --- .../machine_learning/face_ml/feedback/cluster_feedback.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart b/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart index 8734842723..008559d6d5 100644 --- a/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart +++ b/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart @@ -477,7 +477,7 @@ class ClusterFeedbackService { final smallestPersonClusterSize = personClusters .map((clusterID) => allClusterIdsToCountMap[clusterID] ?? 0) .reduce((value, element) => min(value, element)); - final checkSizes = [kMinimumClusterSizeSearchResult, 20, 10, 5, 1]; + final checkSizes = [20, kMinimumClusterSizeSearchResult, 10, 5, 1]; late Map clusterAvgBigClusters; final List<(int, double)> suggestionsMean = []; for (final minimumSize in checkSizes.toSet()) { From f101468a8dab53cf5a24e519578ddf6632e86336 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Thu, 25 Apr 2024 16:30:00 +0530 Subject: [PATCH 44/69] [mob][photos] Show faces in file info regardless of blur value --- mobile/lib/ui/viewer/file_details/faces_item_widget.dart | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mobile/lib/ui/viewer/file_details/faces_item_widget.dart b/mobile/lib/ui/viewer/file_details/faces_item_widget.dart index 6ffdcbd37c..3df244050d 100644 --- a/mobile/lib/ui/viewer/file_details/faces_item_widget.dart +++ b/mobile/lib/ui/viewer/file_details/faces_item_widget.dart @@ -71,9 +71,9 @@ class _FacesItemWidgetState extends State { ]; } - // Remove faces with low scores and blurry faces + // Remove faces with low scores if (!kDebugMode) { - faces.removeWhere((face) => (face.isBlurry || face.score < 0.75)); + faces.removeWhere((face) => (face.score < 0.75)); } if (faces.isEmpty) { From 3828fa328ee8ece4de856849dad3fdeebef0546a Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Thu, 25 Apr 2024 16:41:23 +0530 Subject: [PATCH 45/69] [mob][photos] Increase conservative clustering threshold slightly --- .../face_ml/face_clustering/face_clustering_service.dart | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mobile/lib/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart b/mobile/lib/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart index f504d55b18..f504eafa4b 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart @@ -69,7 +69,7 @@ class FaceClusteringService { bool isRunning = false; static const kRecommendedDistanceThreshold = 0.24; - static const kConservativeDistanceThreshold = 0.15; + static const kConservativeDistanceThreshold = 0.16; // singleton pattern FaceClusteringService._privateConstructor(); @@ -961,9 +961,9 @@ class FaceClusteringService { // Run the DBSCAN clustering final List> clusterOutput = dbscan.run(embeddings); - final List> clusteredFaceInfos = clusterOutput - .map((cluster) => cluster.map((idx) => faceInfos[idx]).toList()) - .toList(); + // final List> clusteredFaceInfos = clusterOutput + // .map((cluster) => cluster.map((idx) => faceInfos[idx]).toList()) + // .toList(); final List> clusteredFaceIDs = clusterOutput .map((cluster) => cluster.map((idx) => faceInfos[idx].faceID).toList()) .toList(); From f173bc4038d1325dbcc3290098a1f53187f228de Mon Sep 17 00:00:00 2001 From: ashilkn Date: Thu, 25 Apr 2024 16:56:43 +0530 Subject: [PATCH 46/69] [mob] Wrote util methods to generate face thumbnails from an image path Need to decide on which util method to use of the two after performance testing --- mobile/lib/utils/face/face_util.dart | 101 +++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 mobile/lib/utils/face/face_util.dart diff --git a/mobile/lib/utils/face/face_util.dart b/mobile/lib/utils/face/face_util.dart new file mode 100644 index 0000000000..99ec045121 --- /dev/null +++ b/mobile/lib/utils/face/face_util.dart @@ -0,0 +1,101 @@ +import "dart:typed_data"; + +import "package:computer/computer.dart"; +import "package:flutter_image_compress/flutter_image_compress.dart"; +import "package:image/image.dart" as img; +import "package:logging/logging.dart"; +import "package:photos/face/model/box.dart"; + +final _logger = Logger("FaceUtil"); +final _computer = Computer.shared(); + +///Convert img.Image to ui.Image and use RawImage to display. +Future> generateImgFaceThumbnails( + String imagePath, + List faceBoxes, +) async { + final faceThumbnails = []; + + img.Image? image = + await _computer.compute(_decodeImageFile, param: {"filePath": imagePath}); + + if (image == null) { + _logger.info( + "Failed to decode image. Compressing to jpg and decoding", + ); + final compressedJPGImage = + await FlutterImageCompress.compressWithFile(imagePath); + image = await _computer.compute( + _decodeJpg, + param: {"image": compressedJPGImage}, + ); + + if (image == null) { + throw Exception("Failed to decode image"); + } + } + + for (FaceBox faceBox in faceBoxes) { + final croppedImage = cropFaceBoxFromImage(image, faceBox); + faceThumbnails.add(croppedImage); + } + + return faceThumbnails; +} + +Future> generateJpgFaceThumbnails( + String imagePath, + List faceBoxes, +) async { + img.Image? image = + await _computer.compute(_decodeImageFile, param: {"filePath": imagePath}); + + if (image == null) { + _logger.info( + "Failed to decode image. Compressing to jpg and decoding", + ); + final compressedJPGImage = + await FlutterImageCompress.compressWithFile(imagePath); + image = await _computer.compute( + _decodeJpg, + param: {"image": compressedJPGImage}, + ); + + if (image == null) { + throw Exception("Failed to decode image"); + } + } + final croppedImages = []; + for (FaceBox faceBox in faceBoxes) { + final croppedImage = cropFaceBoxFromImage(image, faceBox); + croppedImages.add(croppedImage); + } + + return await _computer + .compute(_encodeImagesToJpg, param: {"images": croppedImages}); +} + +/// Returns an Image from 'package:image/image.dart' +img.Image cropFaceBoxFromImage(img.Image image, FaceBox faceBox) { + return img.copyCrop( + image, + x: (image.width * faceBox.xMin).round(), + y: (image.height * faceBox.yMin).round(), + width: (image.width * faceBox.width).round(), + height: (image.height * faceBox.height).round(), + antialias: false, + ); +} + +List _encodeImagesToJpg(Map args) { + final images = args["images"] as List; + return images.map((img.Image image) => img.encodeJpg(image)).toList(); +} + +Future _decodeImageFile(Map args) async { + return await img.decodeImageFile(args["filePath"]); +} + +img.Image? _decodeJpg(Map args) { + return img.decodeJpg(args["image"])!; +} From 7617817798d0f6888a9c0f9a620ba29069e4e7dd Mon Sep 17 00:00:00 2001 From: ashilkn Date: Thu, 25 Apr 2024 16:58:58 +0530 Subject: [PATCH 47/69] [mob] Two varients of CroppedFaceImageView for testing out which is more performant --- .../ui/viewer/file_details/face_widget.dart | 2 +- .../people/cropped_face_image_view.dart | 118 +++++++++++------- .../search/result/person_face_widget.dart | 2 +- 3 files changed, 76 insertions(+), 46 deletions(-) diff --git a/mobile/lib/ui/viewer/file_details/face_widget.dart b/mobile/lib/ui/viewer/file_details/face_widget.dart index da592a150b..9d1fa8dec1 100644 --- a/mobile/lib/ui/viewer/file_details/face_widget.dart +++ b/mobile/lib/ui/viewer/file_details/face_widget.dart @@ -267,7 +267,7 @@ class _FaceWidgetState extends State { SizedBox( width: 60, height: 60, - child: CroppedFaceImageView( + child: CroppedFaceImgImageView( enteFile: widget.file, face: widget.face, ), diff --git a/mobile/lib/ui/viewer/people/cropped_face_image_view.dart b/mobile/lib/ui/viewer/people/cropped_face_image_view.dart index 0da4129ebb..c7e840e713 100644 --- a/mobile/lib/ui/viewer/people/cropped_face_image_view.dart +++ b/mobile/lib/ui/viewer/people/cropped_face_image_view.dart @@ -1,15 +1,14 @@ import "dart:io" show File; +import "dart:typed_data"; import 'dart:ui' as ui; -import "package:computer/computer.dart"; import 'package:flutter/material.dart'; import "package:flutter/widgets.dart"; -import "package:flutter_image_compress/flutter_image_compress.dart"; -import "package:image/image.dart" as img; import "package:logging/logging.dart"; import "package:photos/face/model/face.dart"; import "package:photos/models/file/file.dart"; import "package:photos/ui/viewer/file/thumbnail_widget.dart"; +import "package:photos/utils/face/face_util.dart"; import "package:photos/utils/file_util.dart"; import "package:photos/utils/image_util.dart"; @@ -27,23 +26,22 @@ class CroppedFaceInfo { }); } -class CroppedFaceImageView extends StatefulWidget { +class CroppedFaceImgImageView extends StatefulWidget { final EnteFile enteFile; final Face face; - const CroppedFaceImageView({ + const CroppedFaceImgImageView({ Key? key, required this.enteFile, required this.face, }) : super(key: key); @override - CroppedFaceImageViewState createState() => CroppedFaceImageViewState(); + CroppedFaceImgImageViewState createState() => CroppedFaceImgImageViewState(); } -class CroppedFaceImageViewState extends State { +class CroppedFaceImgImageViewState extends State { ui.Image? _image; - final _computer = Computer.shared(); final _logger = Logger("CroppedFaceImageView"); @override @@ -89,39 +87,9 @@ class CroppedFaceImageViewState extends State { return null; } - img.Image? image = await _computer - .compute(decodeImage, param: {"filePath": ioFile.path}); + final image = await generateImgFaceThumbnails(ioFile.path, [faceBox]); - if (image == null) { - _logger.info( - "Failed to decode image ${widget.enteFile.title}. Compressing to jpg and decoding", - ); - final compressedJPGImage = - await FlutterImageCompress.compressWithFile(ioFile.path); - image = await _computer.compute( - decodeJPGImage, - param: {"image": compressedJPGImage}, - ); - - if (image == null) { - throw Exception("Failed to decode image"); - } - } - - final stopwatch = Stopwatch()..start(); - final croppedImage = img.copyCrop( - image, - x: (image.width * faceBox.xMin).round(), - y: (image.height * faceBox.yMin).round(), - width: (image.width * faceBox.width).round(), - height: (image.height * faceBox.height).round(), - antialias: false, - ); - _logger.info( - "Image crop took ${stopwatch.elapsedMilliseconds}ms ----------------", - ); - stopwatch.stop(); - return convertImageToFlutterUi(croppedImage); + return convertImageToFlutterUi(image.first); } catch (e, s) { _logger.severe("Error getting image", e, s); return null; @@ -129,10 +97,72 @@ class CroppedFaceImageViewState extends State { } } -Future decodeImage(Map args) async { - return await img.decodeImageFile(args["filePath"]); +class CroppedFaceJpgImageView extends StatefulWidget { + final EnteFile enteFile; + final Face face; + + const CroppedFaceJpgImageView({ + Key? key, + required this.enteFile, + required this.face, + }) : super(key: key); + + @override + CroppedFaceJpgImageViewState createState() => CroppedFaceJpgImageViewState(); } -img.Image? decodeJPGImage(Map args) { - return img.decodeJpg(args["image"])!; +class CroppedFaceJpgImageViewState extends State { + Uint8List? _image; + final _logger = Logger("CroppedFaceImageView"); + + @override + void initState() { + super.initState(); + _loadImage(); + } + + @override + void dispose() { + super.dispose(); + } + + Future _loadImage() async { + final image = await getImage(); + if (mounted) { + setState(() { + _image = image; + }); + } + } + + @override + Widget build(BuildContext context) { + return _image != null + ? LayoutBuilder( + builder: (context, constraints) { + return Image.memory( + _image!, + ); + }, + ) + : ThumbnailWidget(widget.enteFile); + } + + Future getImage() async { + try { + final faceBox = widget.face.detection.box; + + final File? ioFile = await getFile(widget.enteFile); + if (ioFile == null) { + return null; + } + + final image = await generateJpgFaceThumbnails(ioFile.path, [faceBox]); + + return image.first; + } catch (e, s) { + _logger.severe("Error getting image", e, s); + return null; + } + } } diff --git a/mobile/lib/ui/viewer/search/result/person_face_widget.dart b/mobile/lib/ui/viewer/search/result/person_face_widget.dart index f2c96fc7a9..0f872e9f3d 100644 --- a/mobile/lib/ui/viewer/search/result/person_face_widget.dart +++ b/mobile/lib/ui/viewer/search/result/person_face_widget.dart @@ -69,7 +69,7 @@ class PersonFaceWidget extends StatelessWidget { return Stack( fit: StackFit.expand, children: [ - CroppedFaceImageView(enteFile: file, face: face), + CroppedFaceImgImageView(enteFile: file, face: face), ], ); } else { From b2a20780458616821ba1a66e457a7b7cfe133f72 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Thu, 25 Apr 2024 17:06:40 +0530 Subject: [PATCH 48/69] [mob][photos] Moving more methods to sqlite async --- mobile/lib/face/db.dart | 34 +++++++++---------- .../face_ml/face_ml_service.dart | 4 +-- .../face_ml/feedback/cluster_feedback.dart | 2 +- .../face_ml/person/person_service.dart | 2 +- 4 files changed, 21 insertions(+), 21 deletions(-) diff --git a/mobile/lib/face/db.dart b/mobile/lib/face/db.dart index a71fcc8864..679a840a6e 100644 --- a/mobile/lib/face/db.dart +++ b/mobile/lib/face/db.dart @@ -98,7 +98,7 @@ class FaceMLDataDB { } } - Future updateClusterIdToFaceId( + Future updateFaceIdToClusterId( Map faceIDToClusterID, ) async { final db = await instance.database; @@ -197,8 +197,8 @@ class FaceMLDataDB { int clusterID, { int? limit, }) async { - final db = await instance.database; - final List> maps = await db.rawQuery( + final db = await instance.sqliteAsyncDB; + final List> maps = await db.getAll( 'SELECT $faceEmbeddingBlob FROM $facesTable WHERE $faceIDColumn in (SELECT $fcFaceId from $faceClustersTable where $fcClusterID = ?) ${limit != null ? 'LIMIT $limit' : ''}', [clusterID], ); @@ -209,7 +209,7 @@ class FaceMLDataDB { Iterable clusterIDs, { int? limit, }) async { - final db = await instance.database; + final db = await instance.sqliteAsyncDB; final Map> result = {}; final selectQuery = ''' @@ -220,7 +220,7 @@ class FaceMLDataDB { ${limit != null ? 'LIMIT $limit' : ''} '''; - final List> maps = await db.rawQuery(selectQuery); + final List> maps = await db.getAll(selectQuery); for (final map in maps) { final clusterID = map[fcClusterID] as int; @@ -321,8 +321,8 @@ class FaceMLDataDB { } Future getFaceForFaceID(String faceID) async { - final db = await instance.database; - final result = await db.rawQuery( + final db = await instance.sqliteAsyncDB; + final result = await db.getAll( 'SELECT * FROM $facesTable where $faceIDColumn = ?', [faceID], ); @@ -420,8 +420,8 @@ class FaceMLDataDB { Future> getFaceIdsToClusterIds( Iterable faceIds, ) async { - final db = await instance.database; - final List> maps = await db.rawQuery( + final db = await instance.sqliteAsyncDB; + final List> maps = await db.getAll( 'SELECT $fcFaceId, $fcClusterID FROM $faceClustersTable where $fcFaceId IN (${faceIds.map((id) => "'$id'").join(",")})', ); final Map result = {}; @@ -433,8 +433,8 @@ class FaceMLDataDB { Future>> getFileIdToClusterIds() async { final Map> result = {}; - final db = await instance.database; - final List> maps = await db.rawQuery( + final db = await instance.sqliteAsyncDB; + final List> maps = await db.getAll( 'SELECT $fcClusterID, $fcFaceId FROM $faceClustersTable', ); @@ -791,9 +791,9 @@ class FaceMLDataDB { // for a given personID, return a map of clusterID to fileIDs using join query Future>> getFileIdToClusterIDSet(String personID) { - final db = instance.database; + final db = instance.sqliteAsyncDB; return db.then((db) async { - final List> maps = await db.rawQuery( + final List> maps = await db.getAll( 'SELECT $faceClustersTable.$fcClusterID, $fcFaceId FROM $faceClustersTable ' 'INNER JOIN $clusterPersonTable ' 'ON $faceClustersTable.$fcClusterID = $clusterPersonTable.$clusterIDColumn ' @@ -814,9 +814,9 @@ class FaceMLDataDB { Future>> getFileIdToClusterIDSetForCluster( Set clusterIDs, ) { - final db = instance.database; + final db = instance.sqliteAsyncDB; return db.then((db) async { - final List> maps = await db.rawQuery( + final List> maps = await db.getAll( 'SELECT $fcClusterID, $fcFaceId FROM $faceClustersTable ' 'WHERE $fcClusterID IN (${clusterIDs.join(",")})', ); @@ -894,8 +894,8 @@ class FaceMLDataDB { } Future> getClusterIDToPersonID() async { - final db = await instance.database; - final List> maps = await db.rawQuery( + final db = await instance.sqliteAsyncDB; + final List> maps = await db.getAll( 'SELECT $personIdColumn, $clusterIDColumn FROM $clusterPersonTable', ); final Map result = {}; diff --git a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart index da57ad4e71..fbed736e8e 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart @@ -350,7 +350,7 @@ class FaceMlService { } await FaceMLDataDB.instance - .updateClusterIdToFaceId(clusteringResult.newFaceIdToCluster); + .updateFaceIdToClusterId(clusteringResult.newFaceIdToCluster); await FaceMLDataDB.instance .clusterSummaryUpdate(clusteringResult.newClusterSummaries!); _logger.info( @@ -403,7 +403,7 @@ class FaceMlService { 'Updating ${clusteringResult.newFaceIdToCluster.length} FaceIDs with clusterIDs in the DB', ); await FaceMLDataDB.instance - .updateClusterIdToFaceId(clusteringResult.newFaceIdToCluster); + .updateFaceIdToClusterId(clusteringResult.newFaceIdToCluster); await FaceMLDataDB.instance .clusterSummaryUpdate(clusteringResult.newClusterSummaries!); _logger.info('Done updating FaceIDs with clusterIDs in the DB, in ' diff --git a/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart b/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart index 008559d6d5..95b3f2b081 100644 --- a/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart +++ b/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart @@ -412,7 +412,7 @@ class ClusterFeedbackService { final newClusterID = startClusterID + blurValue ~/ 10; faceIdToCluster[faceID] = newClusterID; } - await FaceMLDataDB.instance.updateClusterIdToFaceId(faceIdToCluster); + await FaceMLDataDB.instance.updateFaceIdToClusterId(faceIdToCluster); Bus.instance.fire(PeopleChangedEvent()); } catch (e, s) { diff --git a/mobile/lib/services/machine_learning/face_ml/person/person_service.dart b/mobile/lib/services/machine_learning/face_ml/person/person_service.dart index da62899538..d860ca4b3f 100644 --- a/mobile/lib/services/machine_learning/face_ml/person/person_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/person/person_service.dart @@ -193,7 +193,7 @@ class PersonService { } logger.info("Storing feedback for ${faceIdToClusterID.length} faces"); - await faceMLDataDB.updateClusterIdToFaceId(faceIdToClusterID); + await faceMLDataDB.updateFaceIdToClusterId(faceIdToClusterID); await faceMLDataDB.bulkAssignClusterToPersonID(clusterToPersonID); } From 3eebfdd037ae491ff283b718bb8a449d660a4376 Mon Sep 17 00:00:00 2001 From: ashilkn Date: Fri, 26 Apr 2024 12:54:29 +0530 Subject: [PATCH 49/69] Revert "[mob] Two varients of CroppedFaceImageView for testing out which is more performant" This reverts commit 7617817798d0f6888a9c0f9a620ba29069e4e7dd. --- .../ui/viewer/file_details/face_widget.dart | 2 +- .../people/cropped_face_image_view.dart | 120 +++++++----------- .../search/result/person_face_widget.dart | 2 +- 3 files changed, 47 insertions(+), 77 deletions(-) diff --git a/mobile/lib/ui/viewer/file_details/face_widget.dart b/mobile/lib/ui/viewer/file_details/face_widget.dart index fa455eba4f..4caf1305f4 100644 --- a/mobile/lib/ui/viewer/file_details/face_widget.dart +++ b/mobile/lib/ui/viewer/file_details/face_widget.dart @@ -288,7 +288,7 @@ class _FaceWidgetState extends State { child: SizedBox( width: 60, height: 60, - child: CroppedFaceImgImageView( + child: CroppedFaceImageView( enteFile: widget.file, face: widget.face, ), diff --git a/mobile/lib/ui/viewer/people/cropped_face_image_view.dart b/mobile/lib/ui/viewer/people/cropped_face_image_view.dart index c7e840e713..0da4129ebb 100644 --- a/mobile/lib/ui/viewer/people/cropped_face_image_view.dart +++ b/mobile/lib/ui/viewer/people/cropped_face_image_view.dart @@ -1,14 +1,15 @@ import "dart:io" show File; -import "dart:typed_data"; import 'dart:ui' as ui; +import "package:computer/computer.dart"; import 'package:flutter/material.dart'; import "package:flutter/widgets.dart"; +import "package:flutter_image_compress/flutter_image_compress.dart"; +import "package:image/image.dart" as img; import "package:logging/logging.dart"; import "package:photos/face/model/face.dart"; import "package:photos/models/file/file.dart"; import "package:photos/ui/viewer/file/thumbnail_widget.dart"; -import "package:photos/utils/face/face_util.dart"; import "package:photos/utils/file_util.dart"; import "package:photos/utils/image_util.dart"; @@ -26,22 +27,23 @@ class CroppedFaceInfo { }); } -class CroppedFaceImgImageView extends StatefulWidget { +class CroppedFaceImageView extends StatefulWidget { final EnteFile enteFile; final Face face; - const CroppedFaceImgImageView({ + const CroppedFaceImageView({ Key? key, required this.enteFile, required this.face, }) : super(key: key); @override - CroppedFaceImgImageViewState createState() => CroppedFaceImgImageViewState(); + CroppedFaceImageViewState createState() => CroppedFaceImageViewState(); } -class CroppedFaceImgImageViewState extends State { +class CroppedFaceImageViewState extends State { ui.Image? _image; + final _computer = Computer.shared(); final _logger = Logger("CroppedFaceImageView"); @override @@ -87,82 +89,50 @@ class CroppedFaceImgImageViewState extends State { return null; } - final image = await generateImgFaceThumbnails(ioFile.path, [faceBox]); + img.Image? image = await _computer + .compute(decodeImage, param: {"filePath": ioFile.path}); - return convertImageToFlutterUi(image.first); - } catch (e, s) { - _logger.severe("Error getting image", e, s); - return null; - } - } -} + if (image == null) { + _logger.info( + "Failed to decode image ${widget.enteFile.title}. Compressing to jpg and decoding", + ); + final compressedJPGImage = + await FlutterImageCompress.compressWithFile(ioFile.path); + image = await _computer.compute( + decodeJPGImage, + param: {"image": compressedJPGImage}, + ); -class CroppedFaceJpgImageView extends StatefulWidget { - final EnteFile enteFile; - final Face face; - - const CroppedFaceJpgImageView({ - Key? key, - required this.enteFile, - required this.face, - }) : super(key: key); - - @override - CroppedFaceJpgImageViewState createState() => CroppedFaceJpgImageViewState(); -} - -class CroppedFaceJpgImageViewState extends State { - Uint8List? _image; - final _logger = Logger("CroppedFaceImageView"); - - @override - void initState() { - super.initState(); - _loadImage(); - } - - @override - void dispose() { - super.dispose(); - } - - Future _loadImage() async { - final image = await getImage(); - if (mounted) { - setState(() { - _image = image; - }); - } - } - - @override - Widget build(BuildContext context) { - return _image != null - ? LayoutBuilder( - builder: (context, constraints) { - return Image.memory( - _image!, - ); - }, - ) - : ThumbnailWidget(widget.enteFile); - } - - Future getImage() async { - try { - final faceBox = widget.face.detection.box; - - final File? ioFile = await getFile(widget.enteFile); - if (ioFile == null) { - return null; + if (image == null) { + throw Exception("Failed to decode image"); + } } - final image = await generateJpgFaceThumbnails(ioFile.path, [faceBox]); - - return image.first; + final stopwatch = Stopwatch()..start(); + final croppedImage = img.copyCrop( + image, + x: (image.width * faceBox.xMin).round(), + y: (image.height * faceBox.yMin).round(), + width: (image.width * faceBox.width).round(), + height: (image.height * faceBox.height).round(), + antialias: false, + ); + _logger.info( + "Image crop took ${stopwatch.elapsedMilliseconds}ms ----------------", + ); + stopwatch.stop(); + return convertImageToFlutterUi(croppedImage); } catch (e, s) { _logger.severe("Error getting image", e, s); return null; } } } + +Future decodeImage(Map args) async { + return await img.decodeImageFile(args["filePath"]); +} + +img.Image? decodeJPGImage(Map args) { + return img.decodeJpg(args["image"])!; +} diff --git a/mobile/lib/ui/viewer/search/result/person_face_widget.dart b/mobile/lib/ui/viewer/search/result/person_face_widget.dart index f30bb6f9f6..bdbfaab942 100644 --- a/mobile/lib/ui/viewer/search/result/person_face_widget.dart +++ b/mobile/lib/ui/viewer/search/result/person_face_widget.dart @@ -124,7 +124,7 @@ class PersonFaceWidget extends StatelessWidget { return Stack( fit: StackFit.expand, children: [ - CroppedFaceImgImageView(enteFile: file, face: face), + CroppedFaceImageView(enteFile: file, face: face), ], ); } else { From b256bb2757f07464fc4226fecda8faf489d6191b Mon Sep 17 00:00:00 2001 From: ashilkn Date: Fri, 26 Apr 2024 12:55:29 +0530 Subject: [PATCH 50/69] Revert "[mob] perf: Decode images from which face is to be cropped, in an isolate to avoid jank" This reverts commit 2f7e0cd1ef6e5676e1d294b850835f1487e6c71c. --- .../people/cropped_face_image_view.dart | 36 ++----------------- 1 file changed, 3 insertions(+), 33 deletions(-) diff --git a/mobile/lib/ui/viewer/people/cropped_face_image_view.dart b/mobile/lib/ui/viewer/people/cropped_face_image_view.dart index 0da4129ebb..cb7baff42e 100644 --- a/mobile/lib/ui/viewer/people/cropped_face_image_view.dart +++ b/mobile/lib/ui/viewer/people/cropped_face_image_view.dart @@ -1,10 +1,7 @@ import "dart:io" show File; import 'dart:ui' as ui; -import "package:computer/computer.dart"; import 'package:flutter/material.dart'; -import "package:flutter/widgets.dart"; -import "package:flutter_image_compress/flutter_image_compress.dart"; import "package:image/image.dart" as img; import "package:logging/logging.dart"; import "package:photos/face/model/face.dart"; @@ -43,7 +40,6 @@ class CroppedFaceImageView extends StatefulWidget { class CroppedFaceImageViewState extends State { ui.Image? _image; - final _computer = Computer.shared(); final _logger = Logger("CroppedFaceImageView"); @override @@ -83,32 +79,17 @@ class CroppedFaceImageViewState extends State { Future getImage() async { try { final faceBox = widget.face.detection.box; - final File? ioFile = await getFile(widget.enteFile); if (ioFile == null) { return null; } - img.Image? image = await _computer - .compute(decodeImage, param: {"filePath": ioFile.path}); + final image = await img.decodeImageFile(ioFile.path); if (image == null) { - _logger.info( - "Failed to decode image ${widget.enteFile.title}. Compressing to jpg and decoding", - ); - final compressedJPGImage = - await FlutterImageCompress.compressWithFile(ioFile.path); - image = await _computer.compute( - decodeJPGImage, - param: {"image": compressedJPGImage}, - ); - - if (image == null) { - throw Exception("Failed to decode image"); - } + throw Exception("Failed decoding image file ${widget.enteFile.title}}"); } - final stopwatch = Stopwatch()..start(); final croppedImage = img.copyCrop( image, x: (image.width * faceBox.xMin).round(), @@ -117,10 +98,7 @@ class CroppedFaceImageViewState extends State { height: (image.height * faceBox.height).round(), antialias: false, ); - _logger.info( - "Image crop took ${stopwatch.elapsedMilliseconds}ms ----------------", - ); - stopwatch.stop(); + return convertImageToFlutterUi(croppedImage); } catch (e, s) { _logger.severe("Error getting image", e, s); @@ -128,11 +106,3 @@ class CroppedFaceImageViewState extends State { } } } - -Future decodeImage(Map args) async { - return await img.decodeImageFile(args["filePath"]); -} - -img.Image? decodeJPGImage(Map args) { - return img.decodeJpg(args["image"])!; -} From a0e9913f43693886352ea7715b932308f09d0b95 Mon Sep 17 00:00:00 2001 From: ashilkn Date: Fri, 26 Apr 2024 12:56:12 +0530 Subject: [PATCH 51/69] Revert "[mob] Crop image instead of using scale and translate transforms on OG image in CroppedFaceImageView widget" This reverts commit b022ef6d1e6420205ae6a6baa28feffe70679d17. --- .../people/cropped_face_image_view.dart | 144 +++++++++--------- mobile/lib/utils/image_util.dart | 35 ----- 2 files changed, 76 insertions(+), 103 deletions(-) diff --git a/mobile/lib/ui/viewer/people/cropped_face_image_view.dart b/mobile/lib/ui/viewer/people/cropped_face_image_view.dart index cb7baff42e..4ef3692595 100644 --- a/mobile/lib/ui/viewer/people/cropped_face_image_view.dart +++ b/mobile/lib/ui/viewer/people/cropped_face_image_view.dart @@ -1,14 +1,11 @@ +import 'dart:developer' show log; import "dart:io" show File; -import 'dart:ui' as ui; import 'package:flutter/material.dart'; -import "package:image/image.dart" as img; -import "package:logging/logging.dart"; import "package:photos/face/model/face.dart"; import "package:photos/models/file/file.dart"; import "package:photos/ui/viewer/file/thumbnail_widget.dart"; import "package:photos/utils/file_util.dart"; -import "package:photos/utils/image_util.dart"; class CroppedFaceInfo { final Image image; @@ -24,7 +21,7 @@ class CroppedFaceInfo { }); } -class CroppedFaceImageView extends StatefulWidget { +class CroppedFaceImageView extends StatelessWidget { final EnteFile enteFile; final Face face; @@ -34,75 +31,86 @@ class CroppedFaceImageView extends StatefulWidget { required this.face, }) : super(key: key); - @override - CroppedFaceImageViewState createState() => CroppedFaceImageViewState(); -} - -class CroppedFaceImageViewState extends State { - ui.Image? _image; - final _logger = Logger("CroppedFaceImageView"); - - @override - void initState() { - super.initState(); - _loadImage(); - } - - @override - void dispose() { - super.dispose(); - _image?.dispose(); - } - - Future _loadImage() async { - final image = await getImage(); - if (mounted) { - setState(() { - _image = image; - }); - } - } - @override Widget build(BuildContext context) { - return _image != null - ? LayoutBuilder( - builder: (context, constraints) { - return RawImage( - image: _image!, + return FutureBuilder( + future: getImage(), + builder: (context, snapshot) { + if (snapshot.hasData) { + return LayoutBuilder( + builder: ((context, constraints) { + final double imageAspectRatio = enteFile.width / enteFile.height; + final Image image = snapshot.data!; + + final double viewWidth = constraints.maxWidth; + final double viewHeight = constraints.maxHeight; + + final faceBox = face.detection.box; + + final double relativeFaceCenterX = + faceBox.xMin + faceBox.width / 2; + final double relativeFaceCenterY = + faceBox.yMin + faceBox.height / 2; + + const double desiredFaceHeightRelativeToWidget = 8 / 10; + final double scale = + (1 / faceBox.height) * desiredFaceHeightRelativeToWidget; + + final double widgetCenterX = viewWidth / 2; + final double widgetCenterY = viewHeight / 2; + + final double widgetAspectRatio = viewWidth / viewHeight; + final double imageToWidgetRatio = + imageAspectRatio / widgetAspectRatio; + + double offsetX = + (widgetCenterX - relativeFaceCenterX * viewWidth) * scale; + double offsetY = + (widgetCenterY - relativeFaceCenterY * viewHeight) * scale; + + if (imageAspectRatio < widgetAspectRatio) { + // Landscape Image: Adjust offsetX more conservatively + offsetX = offsetX * imageToWidgetRatio; + } else { + // Portrait Image: Adjust offsetY more conservatively + offsetY = offsetY / imageToWidgetRatio; + } + return ClipRRect( + borderRadius: const BorderRadius.all(Radius.elliptical(16, 12)), + child: Transform.translate( + offset: Offset( + offsetX, + offsetY, + ), + child: Transform.scale( + scale: scale, + child: image, + ), + ), ); - }, - ) - : ThumbnailWidget(widget.enteFile); + }), + ); + } else { + if (snapshot.hasError) { + log('Error getting cover face for person: ${snapshot.error}'); + } + return ThumbnailWidget( + enteFile, + ); + } + }, + ); } - Future getImage() async { - try { - final faceBox = widget.face.detection.box; - final File? ioFile = await getFile(widget.enteFile); - if (ioFile == null) { - return null; - } - - final image = await img.decodeImageFile(ioFile.path); - - if (image == null) { - throw Exception("Failed decoding image file ${widget.enteFile.title}}"); - } - - final croppedImage = img.copyCrop( - image, - x: (image.width * faceBox.xMin).round(), - y: (image.height * faceBox.yMin).round(), - width: (image.width * faceBox.width).round(), - height: (image.height * faceBox.height).round(), - antialias: false, - ); - - return convertImageToFlutterUi(croppedImage); - } catch (e, s) { - _logger.severe("Error getting image", e, s); + Future getImage() async { + final File? ioFile = await getFile(enteFile); + if (ioFile == null) { return null; } + + final imageData = await ioFile.readAsBytes(); + final image = Image.memory(imageData, fit: BoxFit.contain); + + return image; } } diff --git a/mobile/lib/utils/image_util.dart b/mobile/lib/utils/image_util.dart index 7eb1e39fc3..a5bcb03a75 100644 --- a/mobile/lib/utils/image_util.dart +++ b/mobile/lib/utils/image_util.dart @@ -1,8 +1,6 @@ import 'dart:async'; -import 'dart:ui' as ui; import 'package:flutter/widgets.dart'; -import 'package:image/image.dart' as img; Future getImageInfo(ImageProvider imageProvider) { final completer = Completer(); @@ -16,36 +14,3 @@ Future getImageInfo(ImageProvider imageProvider) { completer.future.whenComplete(() => imageStream.removeListener(listener)); return completer.future; } - -///https://github.com/brendan-duncan/image/blob/main/doc/flutter.md -Future convertImageToFlutterUi(img.Image image) async { - if (image.format != img.Format.uint8 || image.numChannels != 4) { - final cmd = img.Command() - ..image(image) - ..convert(format: img.Format.uint8, numChannels: 4); - final rgba8 = await cmd.getImageThread(); - if (rgba8 != null) { - image = rgba8; - } - } - - final ui.ImmutableBuffer buffer = - await ui.ImmutableBuffer.fromUint8List(image.toUint8List()); - - final ui.ImageDescriptor id = ui.ImageDescriptor.raw( - buffer, - height: image.height, - width: image.width, - pixelFormat: ui.PixelFormat.rgba8888, - ); - - final ui.Codec codec = await id.instantiateCodec( - targetHeight: image.height, - targetWidth: image.width, - ); - - final ui.FrameInfo fi = await codec.getNextFrame(); - final ui.Image uiImage = fi.image; - - return uiImage; -} From 43f01c31da40e4d813a8644e975549f2c1adafda Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Fri, 26 Apr 2024 12:58:27 +0530 Subject: [PATCH 52/69] [mob][photos] Prevent sqlite disk corruption issue --- mobile/lib/face/db.dart | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mobile/lib/face/db.dart b/mobile/lib/face/db.dart index 679a840a6e..bdc4db0070 100644 --- a/mobile/lib/face/db.dart +++ b/mobile/lib/face/db.dart @@ -420,8 +420,8 @@ class FaceMLDataDB { Future> getFaceIdsToClusterIds( Iterable faceIds, ) async { - final db = await instance.sqliteAsyncDB; - final List> maps = await db.getAll( + final db = await instance.database; + final List> maps = await db.rawQuery( 'SELECT $fcFaceId, $fcClusterID FROM $faceClustersTable where $fcFaceId IN (${faceIds.map((id) => "'$id'").join(",")})', ); final Map result = {}; @@ -433,8 +433,8 @@ class FaceMLDataDB { Future>> getFileIdToClusterIds() async { final Map> result = {}; - final db = await instance.sqliteAsyncDB; - final List> maps = await db.getAll( + final db = await instance.database; + final List> maps = await db.rawQuery( 'SELECT $fcClusterID, $fcFaceId FROM $faceClustersTable', ); From 811ffe0117637e94c4211fc4fe2da8ba179c404e Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Fri, 26 Apr 2024 14:13:00 +0530 Subject: [PATCH 53/69] [mob][photos] Create new cluster when tapping unassigned face --- mobile/lib/face/db.dart | 12 +++ .../ui/viewer/file_details/face_widget.dart | 90 ++++++++++++++++++- 2 files changed, 100 insertions(+), 2 deletions(-) diff --git a/mobile/lib/face/db.dart b/mobile/lib/face/db.dart index bdc4db0070..f54b3c6ef4 100644 --- a/mobile/lib/face/db.dart +++ b/mobile/lib/face/db.dart @@ -348,6 +348,18 @@ class FaceMLDataDB { return result; } + Future getClusterIDForFaceID(String faceID) async { + final db = await instance.sqliteAsyncDB; + final List> maps = await db.getAll( + 'SELECT $fcClusterID FROM $faceClustersTable WHERE $fcFaceId = ?', + [faceID], + ); + if (maps.isEmpty) { + return null; + } + return maps.first[fcClusterID] as int; + } + Future>> getAllClusterIdToFaceIDs() async { final db = await instance.sqliteAsyncDB; final Map> result = {}; diff --git a/mobile/lib/ui/viewer/file_details/face_widget.dart b/mobile/lib/ui/viewer/file_details/face_widget.dart index 4caf1305f4..9be6193344 100644 --- a/mobile/lib/ui/viewer/file_details/face_widget.dart +++ b/mobile/lib/ui/viewer/file_details/face_widget.dart @@ -4,6 +4,7 @@ import "dart:typed_data"; import "package:flutter/cupertino.dart"; import "package:flutter/foundation.dart" show kDebugMode; import "package:flutter/material.dart"; +import "package:photos/extensions/stop_watch.dart"; import "package:photos/face/db.dart"; import "package:photos/face/model/face.dart"; import "package:photos/face/model/person.dart"; @@ -65,7 +66,50 @@ class _FaceWidgetState extends State { name: "FaceWidget", ); if (widget.person == null && widget.clusterID == null) { - return; + // Get faceID and double check that it doesn't belong to an existing clusterID. If it does, push that cluster page + final w = (kDebugMode ? EnteWatch('FaceWidget') : null) + ?..start(); + final existingClusterID = await FaceMLDataDB.instance + .getClusterIDForFaceID(widget.face.faceID); + w?.log('getting existing clusterID for faceID'); + if (existingClusterID != null) { + final fileIdsToClusterIds = + await FaceMLDataDB.instance.getFileIdToClusterIds(); + final files = await SearchService.instance.getAllFiles(); + final clusterFiles = files + .where( + (file) => + fileIdsToClusterIds[file.uploadedFileID] + ?.contains(existingClusterID) ?? + false, + ) + .toList(); + await Navigator.of(context).push( + MaterialPageRoute( + builder: (context) => ClusterPage( + clusterFiles, + clusterID: existingClusterID, + ), + ), + ); + } + + // Create new clusterID for the faceID and update DB to assign the faceID to the new clusterID + final int newClusterID = + DateTime.now().microsecondsSinceEpoch; + await FaceMLDataDB.instance.updateFaceIdToClusterId( + {widget.face.faceID: newClusterID}, + ); + + // Push page for the new cluster + await Navigator.of(context).push( + MaterialPageRoute( + builder: (context) => ClusterPage( + [widget.file], + clusterID: newClusterID, + ), + ), + ); } if (widget.person != null) { await Navigator.of(context).push( @@ -230,7 +274,49 @@ class _FaceWidgetState extends State { name: "FaceWidget", ); if (widget.person == null && widget.clusterID == null) { - return; + // Get faceID and double check that it doesn't belong to an existing clusterID. If it does, push that cluster page + final w = (kDebugMode ? EnteWatch('FaceWidget') : null) + ?..start(); + final existingClusterID = await FaceMLDataDB.instance + .getClusterIDForFaceID(widget.face.faceID); + w?.log('getting existing clusterID for faceID'); + if (existingClusterID != null) { + final fileIdsToClusterIds = + await FaceMLDataDB.instance.getFileIdToClusterIds(); + final files = await SearchService.instance.getAllFiles(); + final clusterFiles = files + .where( + (file) => + fileIdsToClusterIds[file.uploadedFileID] + ?.contains(existingClusterID) ?? + false, + ) + .toList(); + await Navigator.of(context).push( + MaterialPageRoute( + builder: (context) => ClusterPage( + clusterFiles, + clusterID: existingClusterID, + ), + ), + ); + } + + // Create new clusterID for the faceID and update DB to assign the faceID to the new clusterID + final int newClusterID = DateTime.now().microsecondsSinceEpoch; + await FaceMLDataDB.instance.updateFaceIdToClusterId( + {widget.face.faceID: newClusterID}, + ); + + // Push page for the new cluster + await Navigator.of(context).push( + MaterialPageRoute( + builder: (context) => ClusterPage( + [widget.file], + clusterID: newClusterID, + ), + ), + ); } if (widget.person != null) { await Navigator.of(context).push( From 44898415e7844373fc70f4ade990edd8ebff5934 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Fri, 26 Apr 2024 14:43:19 +0530 Subject: [PATCH 54/69] [mob][photos] Index videos using thumbnails --- .../machine_learning/face_ml/face_ml_service.dart | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart index fbed736e8e..cbc5aab3a4 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart @@ -966,7 +966,12 @@ class FaceMlService { switch (typeOfData) { case FileDataForML.fileData: final stopwatch = Stopwatch()..start(); - final File? file = await getFile(enteFile, isOrigin: true); + File? file; + if (enteFile.fileType == FileType.video) { + file = await getThumbnailForUploadedFile(enteFile); + } else { + file = await getFile(enteFile, isOrigin: true); + } if (file == null) { _logger.warning("Could not get file for $enteFile"); imagePath = null; @@ -1294,10 +1299,6 @@ class FaceMlService { if (!enteFile.isUploaded || enteFile.isOwner == false) { return true; } - // Skip if the file is a video - if (enteFile.fileType == FileType.video) { - return true; - } // I don't know how motionPhotos and livePhotos work, so I'm also just skipping them for now if (enteFile.fileType == FileType.other) { return true; From 2692d0a34fc4610773fc4ec893a68afd9ada7237 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Fri, 26 Apr 2024 14:50:14 +0530 Subject: [PATCH 55/69] [mob][photos] Fix issue in displaying face thumbnails for videos --- mobile/lib/ui/viewer/people/cropped_face_image_view.dart | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/mobile/lib/ui/viewer/people/cropped_face_image_view.dart b/mobile/lib/ui/viewer/people/cropped_face_image_view.dart index 4ef3692595..823a979fcd 100644 --- a/mobile/lib/ui/viewer/people/cropped_face_image_view.dart +++ b/mobile/lib/ui/viewer/people/cropped_face_image_view.dart @@ -4,8 +4,10 @@ import "dart:io" show File; import 'package:flutter/material.dart'; import "package:photos/face/model/face.dart"; import "package:photos/models/file/file.dart"; +import "package:photos/models/file/file_type.dart"; import "package:photos/ui/viewer/file/thumbnail_widget.dart"; import "package:photos/utils/file_util.dart"; +import "package:photos/utils/thumbnail_util.dart"; class CroppedFaceInfo { final Image image; @@ -103,7 +105,12 @@ class CroppedFaceImageView extends StatelessWidget { } Future getImage() async { - final File? ioFile = await getFile(enteFile); + final File? ioFile; + if (enteFile.fileType == FileType.video) { + ioFile = await getThumbnailForUploadedFile(enteFile); + } else { + ioFile = await getFile(enteFile); + } if (ioFile == null) { return null; } From caa72ba83022de5b48355ab9870497b2f598a42a Mon Sep 17 00:00:00 2001 From: ashilkn Date: Fri, 26 Apr 2024 15:44:07 +0530 Subject: [PATCH 56/69] [mob][photos] add option to pass decoded image to face thumbnail generation methods to avoid unnecessary decoding when possible --- mobile/lib/utils/face/face_util.dart | 57 +++++++++++++--------------- 1 file changed, 27 insertions(+), 30 deletions(-) diff --git a/mobile/lib/utils/face/face_util.dart b/mobile/lib/utils/face/face_util.dart index 99ec045121..3d7853f08d 100644 --- a/mobile/lib/utils/face/face_util.dart +++ b/mobile/lib/utils/face/face_util.dart @@ -12,28 +12,14 @@ final _computer = Computer.shared(); ///Convert img.Image to ui.Image and use RawImage to display. Future> generateImgFaceThumbnails( String imagePath, - List faceBoxes, -) async { + List faceBoxes, { + ///Pass decodedImage decoded by [decodeToImgImage] to avoid decoding image + ///multiple times if all faces are from the same image (eg: File info). + img.Image? decodedImage, +}) async { final faceThumbnails = []; - img.Image? image = - await _computer.compute(_decodeImageFile, param: {"filePath": imagePath}); - - if (image == null) { - _logger.info( - "Failed to decode image. Compressing to jpg and decoding", - ); - final compressedJPGImage = - await FlutterImageCompress.compressWithFile(imagePath); - image = await _computer.compute( - _decodeJpg, - param: {"image": compressedJPGImage}, - ); - - if (image == null) { - throw Exception("Failed to decode image"); - } - } + final image = decodedImage ?? await decodeToImgImage(imagePath); for (FaceBox faceBox in faceBoxes) { final croppedImage = cropFaceBoxFromImage(image, faceBox); @@ -45,8 +31,23 @@ Future> generateImgFaceThumbnails( Future> generateJpgFaceThumbnails( String imagePath, - List faceBoxes, -) async { + List faceBoxes, { + ///Pass decodedImage decoded by [decodeToImgImage] to avoid decoding image + ///multiple times if all faces are from the same image (eg: File info). + img.Image? decodedImage, +}) async { + final image = decodedImage ?? await decodeToImgImage(imagePath); + final croppedImages = []; + for (FaceBox faceBox in faceBoxes) { + final croppedImage = cropFaceBoxFromImage(image, faceBox); + croppedImages.add(croppedImage); + } + + return await _computer + .compute(_encodeImagesToJpg, param: {"images": croppedImages}); +} + +Future decodeToImgImage(String imagePath) async { img.Image? image = await _computer.compute(_decodeImageFile, param: {"filePath": imagePath}); @@ -63,16 +64,12 @@ Future> generateJpgFaceThumbnails( if (image == null) { throw Exception("Failed to decode image"); + } else { + return image; } + } else { + return image; } - final croppedImages = []; - for (FaceBox faceBox in faceBoxes) { - final croppedImage = cropFaceBoxFromImage(image, faceBox); - croppedImages.add(croppedImage); - } - - return await _computer - .compute(_encodeImagesToJpg, param: {"images": croppedImages}); } /// Returns an Image from 'package:image/image.dart' From 968eaaf5f6b8b47e203d1196163b9c352960b997 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Sat, 27 Apr 2024 09:39:12 +0530 Subject: [PATCH 57/69] [mob][photos] Better error logging --- .../ui/settings/debug/face_debug_section_widget.dart | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/mobile/lib/ui/settings/debug/face_debug_section_widget.dart b/mobile/lib/ui/settings/debug/face_debug_section_widget.dart index 6ce7e0609b..0112490a47 100644 --- a/mobile/lib/ui/settings/debug/face_debug_section_widget.dart +++ b/mobile/lib/ui/settings/debug/face_debug_section_widget.dart @@ -216,9 +216,14 @@ class _FaceDebugSectionWidgetState extends State { trailingIcon: Icons.chevron_right_outlined, trailingIconIsMuted: true, onTap: () async { - await FaceMLDataDB.instance.dropFeedbackTables(); - Bus.instance.fire(PeopleChangedEvent()); - showShortToast(context, "Done"); + try { + await FaceMLDataDB.instance.dropFeedbackTables(); + Bus.instance.fire(PeopleChangedEvent()); + showShortToast(context, "Done"); + } catch (e, s) { + _logger.warning('reset feedback failed ', e, s); + await showGenericErrorDialog(context: context, error: e); + } }, ), sectionOptionSpacing, From 8b236cde0946f014eabc86da9b81757d7d4a54d2 Mon Sep 17 00:00:00 2001 From: ashilkn Date: Sat, 27 Apr 2024 09:59:16 +0530 Subject: [PATCH 58/69] [mob][photos] When cropping a face from an image, make the image a square and add some buffer around it --- mobile/lib/face/model/box.dart | 20 ++++++++ mobile/lib/utils/face/face_util.dart | 71 ++++++++++++++++++++++++++-- 2 files changed, 87 insertions(+), 4 deletions(-) diff --git a/mobile/lib/face/model/box.dart b/mobile/lib/face/model/box.dart index 73d7dea388..dafe3d8384 100644 --- a/mobile/lib/face/model/box.dart +++ b/mobile/lib/face/model/box.dart @@ -41,3 +41,23 @@ class FaceBox { 'height': height, }; } + +/// Bounding box of a face. +/// +/// [xMin] and [yMin] are the coordinates of the top left corner of the box, and +/// [width] and [height] are the width and height of the box. +/// +/// One unit is equal to one pixel in the original image. +class FaceBoxImage { + final int xMin; + final int yMin; + final int width; + final int height; + + FaceBoxImage({ + required this.xMin, + required this.yMin, + required this.width, + required this.height, + }); +} diff --git a/mobile/lib/utils/face/face_util.dart b/mobile/lib/utils/face/face_util.dart index 3d7853f08d..b417657915 100644 --- a/mobile/lib/utils/face/face_util.dart +++ b/mobile/lib/utils/face/face_util.dart @@ -1,3 +1,4 @@ +import "dart:math"; import "dart:typed_data"; import "package:computer/computer.dart"; @@ -8,6 +9,7 @@ import "package:photos/face/model/box.dart"; final _logger = Logger("FaceUtil"); final _computer = Computer.shared(); +const _faceImageBufferFactor = 0.2; ///Convert img.Image to ui.Image and use RawImage to display. Future> generateImgFaceThumbnails( @@ -74,16 +76,77 @@ Future decodeToImgImage(String imagePath) async { /// Returns an Image from 'package:image/image.dart' img.Image cropFaceBoxFromImage(img.Image image, FaceBox faceBox) { + final squareFaceBox = _getSquareFaceBoxImage(image, faceBox); + final squareFaceBoxWithBuffer = + _addBufferAroundFaceBox(squareFaceBox, _faceImageBufferFactor); return img.copyCrop( image, - x: (image.width * faceBox.xMin).round(), - y: (image.height * faceBox.yMin).round(), - width: (image.width * faceBox.width).round(), - height: (image.height * faceBox.height).round(), + x: squareFaceBoxWithBuffer.xMin, + y: squareFaceBoxWithBuffer.yMin, + width: squareFaceBoxWithBuffer.width, + height: squareFaceBoxWithBuffer.height, antialias: false, ); } +/// Returns a square face box image from the original image with +/// side length equal to the maximum of the width and height of the face box in +/// the OG image. +FaceBoxImage _getSquareFaceBoxImage(img.Image image, FaceBox faceBox) { + final width = (image.width * faceBox.width).round(); + final height = (image.height * faceBox.height).round(); + final side = max(width, height); + final xImage = (image.width * faceBox.xMin).round(); + final yImage = (image.height * faceBox.yMin).round(); + + if (height >= width) { + final xImageAdj = (xImage - (height - width) / 2).round(); + return FaceBoxImage( + xMin: xImageAdj, + yMin: yImage, + width: side, + height: side, + ); + } else { + final yImageAdj = (yImage - (width - height) / 2).round(); + return FaceBoxImage( + xMin: xImage, + yMin: yImageAdj, + width: side, + height: side, + ); + } +} + +///To add some buffer around the face box so that the face isn't cropped +///too close to the face. +FaceBoxImage _addBufferAroundFaceBox( + FaceBoxImage faceBoxImage, + double bufferFactor, +) { + final heightBuffer = faceBoxImage.height * bufferFactor; + final widthBuffer = faceBoxImage.width * bufferFactor; + final xMinWithBuffer = faceBoxImage.xMin - widthBuffer; + final yMinWithBuffer = faceBoxImage.yMin - heightBuffer; + final widthWithBuffer = faceBoxImage.width + 2 * widthBuffer; + final heightWithBuffer = faceBoxImage.height + 2 * heightBuffer; + //Do not add buffer if the top left edge of the image is out of bounds + //after adding the buffer. + if (xMinWithBuffer < 0 || yMinWithBuffer < 0) { + return faceBoxImage; + } + //Another similar case that can be handled is when the bottom right edge + //of the image is out of bounds after adding the buffer. But the + //the visual difference is not as significant as when the top left edge + //is out of bounds, so we are not handling that case. + return FaceBoxImage( + xMin: xMinWithBuffer.round(), + yMin: yMinWithBuffer.round(), + width: widthWithBuffer.round(), + height: heightWithBuffer.round(), + ); +} + List _encodeImagesToJpg(Map args) { final images = args["images"] as List; return images.map((img.Image image) => img.encodeJpg(image)).toList(); From 707916f677c4b961e0cb80dd35ca38c6ee9ae72b Mon Sep 17 00:00:00 2001 From: ashilkn Date: Sat, 27 Apr 2024 10:57:49 +0530 Subject: [PATCH 59/69] [mob][photos] Add method to convert Image from Image package to UI image --- mobile/lib/utils/image_util.dart | 34 ++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/mobile/lib/utils/image_util.dart b/mobile/lib/utils/image_util.dart index a5bcb03a75..e5b0d72fac 100644 --- a/mobile/lib/utils/image_util.dart +++ b/mobile/lib/utils/image_util.dart @@ -1,6 +1,8 @@ import 'dart:async'; +import 'dart:ui' as ui; import 'package:flutter/widgets.dart'; +import 'package:image/image.dart' as img; Future getImageInfo(ImageProvider imageProvider) { final completer = Completer(); @@ -14,3 +16,35 @@ Future getImageInfo(ImageProvider imageProvider) { completer.future.whenComplete(() => imageStream.removeListener(listener)); return completer.future; } + +Future convertImageToFlutterUi(img.Image image) async { + if (image.format != img.Format.uint8 || image.numChannels != 4) { + final cmd = img.Command() + ..image(image) + ..convert(format: img.Format.uint8, numChannels: 4); + final rgba8 = await cmd.getImageThread(); + if (rgba8 != null) { + image = rgba8; + } + } + + final ui.ImmutableBuffer buffer = + await ui.ImmutableBuffer.fromUint8List(image.toUint8List()); + + final ui.ImageDescriptor id = ui.ImageDescriptor.raw( + buffer, + height: image.height, + width: image.width, + pixelFormat: ui.PixelFormat.rgba8888, + ); + + final ui.Codec codec = await id.instantiateCodec( + targetHeight: image.height, + targetWidth: image.width, + ); + + final ui.FrameInfo fi = await codec.getNextFrame(); + final ui.Image uiImage = fi.image; + + return uiImage; +} From 19f2c5f00a026478f7e22104586658c0e4ad0117 Mon Sep 17 00:00:00 2001 From: ashilkn Date: Sat, 27 Apr 2024 11:02:38 +0530 Subject: [PATCH 60/69] [mob][photos] remove negation --- mobile/lib/ui/viewer/search/result/person_face_widget.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobile/lib/ui/viewer/search/result/person_face_widget.dart b/mobile/lib/ui/viewer/search/result/person_face_widget.dart index bdbfaab942..de02938619 100644 --- a/mobile/lib/ui/viewer/search/result/person_face_widget.dart +++ b/mobile/lib/ui/viewer/search/result/person_face_widget.dart @@ -90,7 +90,7 @@ class PersonFaceWidget extends StatelessWidget { @override Widget build(BuildContext context) { - if (!useGeneratedFaceCrops) { + if (useGeneratedFaceCrops) { return FutureBuilder( future: getFaceCrop(), builder: (context, snapshot) { From ab5985a08b58f5f98e272ed8214532e34289c406 Mon Sep 17 00:00:00 2001 From: ashilkn Date: Sat, 27 Apr 2024 11:06:54 +0530 Subject: [PATCH 61/69] [mob][photos] Use generated face crops and crop it using the new method --- mobile/lib/ui/viewer/file_details/face_widget.dart | 2 +- mobile/lib/utils/face/face_box_crop.dart | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/mobile/lib/ui/viewer/file_details/face_widget.dart b/mobile/lib/ui/viewer/file_details/face_widget.dart index 4caf1305f4..52f173377f 100644 --- a/mobile/lib/ui/viewer/file_details/face_widget.dart +++ b/mobile/lib/ui/viewer/file_details/face_widget.dart @@ -20,7 +20,7 @@ import "package:photos/utils/face/face_box_crop.dart"; import "package:photos/utils/thumbnail_util.dart"; // import "package:photos/utils/toast_util.dart"; -const useGeneratedFaceCrops = false; +const useGeneratedFaceCrops = true; class FaceWidget extends StatefulWidget { final EnteFile file; diff --git a/mobile/lib/utils/face/face_box_crop.dart b/mobile/lib/utils/face/face_box_crop.dart index 9bf36fbdd8..7d032998a0 100644 --- a/mobile/lib/utils/face/face_box_crop.dart +++ b/mobile/lib/utils/face/face_box_crop.dart @@ -5,8 +5,8 @@ import "package:photos/core/cache/lru_map.dart"; import "package:photos/face/model/box.dart"; import "package:photos/models/file/file.dart"; import "package:photos/models/file/file_type.dart"; +import "package:photos/utils/face/face_util.dart"; import "package:photos/utils/file_util.dart"; -import "package:photos/utils/image_ml_isolate.dart"; import "package:photos/utils/thumbnail_util.dart"; import "package:pool/pool.dart"; @@ -37,7 +37,8 @@ Future?> getFaceCrops( faceBoxes.add(e.value); } final List faceCrop = - await ImageMlIsolate.instance.generateFaceThumbnailsForImage( + // await ImageMlIsolate.instance.generateFaceThumbnailsForImage( + await generateJpgFaceThumbnails( imagePath, faceBoxes, ); From f422e30a8ecd1c9788040e2eeb95bbba83c0d5dd Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Sat, 27 Apr 2024 11:13:52 +0530 Subject: [PATCH 62/69] [mob][photos] Migrate fully to sqlite async for faces, removing sqflite fully --- mobile/lib/face/db.dart | 567 ++++++++++++++++++++++------------------ 1 file changed, 313 insertions(+), 254 deletions(-) diff --git a/mobile/lib/face/db.dart b/mobile/lib/face/db.dart index f54b3c6ef4..05450b9687 100644 --- a/mobile/lib/face/db.dart +++ b/mobile/lib/face/db.dart @@ -1,5 +1,4 @@ import 'dart:async'; -import "dart:io" show Directory; import "dart:math"; import "package:collection/collection.dart"; @@ -14,7 +13,7 @@ import "package:photos/face/model/face.dart"; import "package:photos/models/file/file.dart"; import "package:photos/services/machine_learning/face_ml/face_clustering/face_info_for_clustering.dart"; import 'package:photos/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart'; -import 'package:sqflite/sqflite.dart'; +// import 'package:sqflite/sqflite.dart'; import 'package:sqlite_async/sqlite_async.dart' as sqlite_async; /// Stores all data for the ML-related features. The database can be accessed by `MlDataDB.instance.database`. @@ -33,75 +32,83 @@ class FaceMLDataDB { static final FaceMLDataDB instance = FaceMLDataDB._privateConstructor(); // only have a single app-wide reference to the database - static Future? _dbFuture; static Future? _sqliteAsyncDBFuture; - Future get database async { - _dbFuture ??= _initDatabase(); - return _dbFuture!; - } - - Future get sqliteAsyncDB async { + Future get asyncDB async { _sqliteAsyncDBFuture ??= _initSqliteAsyncDatabase(); return _sqliteAsyncDBFuture!; } - Future _initDatabase() async { + Future _initSqliteAsyncDatabase() async { final documentsDirectory = await getApplicationDocumentsDirectory(); - final String databaseDirectory = - join(documentsDirectory.path, _databaseName); - return await openDatabase( - databaseDirectory, - version: _databaseVersion, - onCreate: _onCreate, - ); - } - - Future _initSqliteAsyncDatabase() async { - final Directory documentsDirectory = - await getApplicationDocumentsDirectory(); final String databaseDirectory = join(documentsDirectory.path, _databaseName); _logger.info("Opening sqlite_async access: DB path " + databaseDirectory); - return sqlite_async.SqliteDatabase(path: databaseDirectory, maxReaders: 1); + final asyncDBConnection = + sqlite_async.SqliteDatabase(path: databaseDirectory, maxReaders: 2); + await _onCreate(asyncDBConnection); + return asyncDBConnection; } - Future _onCreate(Database db, int version) async { - await db.execute(createFacesTable); - await db.execute(createFaceClustersTable); - await db.execute(createClusterPersonTable); - await db.execute(createClusterSummaryTable); - await db.execute(createNotPersonFeedbackTable); - await db.execute(fcClusterIDIndex); + Future _onCreate(sqlite_async.SqliteDatabase asyncDBConnection) async { + final migrations = sqlite_async.SqliteMigrations() + ..add( + sqlite_async.SqliteMigration(_databaseVersion, (tx) async { + await tx.execute(createFacesTable); + await tx.execute(createFaceClustersTable); + await tx.execute(createClusterPersonTable); + await tx.execute(createClusterSummaryTable); + await tx.execute(createNotPersonFeedbackTable); + await tx.execute(fcClusterIDIndex); + }), + ); + await migrations.migrate(asyncDBConnection); } // bulkInsertFaces inserts the faces in the database in batches of 1000. // This is done to avoid the error "too many SQL variables" when inserting // a large number of faces. Future bulkInsertFaces(List faces) async { - final db = await instance.database; + final db = await instance.asyncDB; const batchSize = 500; final numBatches = (faces.length / batchSize).ceil(); for (int i = 0; i < numBatches; i++) { final start = i * batchSize; final end = min((i + 1) * batchSize, faces.length); final batch = faces.sublist(start, end); - final batchInsert = db.batch(); - for (final face in batch) { - batchInsert.insert( - facesTable, - mapRemoteToFaceDB(face), - conflictAlgorithm: ConflictAlgorithm.ignore, - ); - } - await batchInsert.commit(noResult: true); + + const String sql = ''' + INSERT INTO $facesTable ( + $fileIDColumn, $faceIDColumn, $faceDetectionColumn, $faceEmbeddingBlob, $faceScore, $faceBlur, $isSideways, $imageHeight, $imageWidth, $mlVersionColumn + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT($faceIDColumn, $mlVersionColumn) DO NOTHING + '''; + final parameterSets = batch.map((face) { + final map = mapRemoteToFaceDB(face); + return [ + map[fileIDColumn], + map[faceIDColumn], + map[faceDetectionColumn], + map[faceEmbeddingBlob], + map[faceScore], + map[faceBlur], + map[isSideways], + map[imageHeight], + map[imageWidth], + map[mlVersionColumn], + ]; + }).toList(); + + await db.writeTransaction((tx) async { + await tx.executeBatch(sql, parameterSets); + }); } } Future updateFaceIdToClusterId( Map faceIDToClusterID, ) async { - final db = await instance.database; + final db = await instance.asyncDB; const batchSize = 500; final numBatches = (faceIDToClusterID.length / batchSize).ceil(); for (int i = 0; i < numBatches; i++) { @@ -109,24 +116,21 @@ class FaceMLDataDB { final end = min((i + 1) * batchSize, faceIDToClusterID.length); final batch = faceIDToClusterID.entries.toList().sublist(start, end); - final batchUpdate = db.batch(); - - for (final entry in batch) { - final faceID = entry.key; - final clusterID = entry.value; - batchUpdate.insert( - faceClustersTable, - {fcClusterID: clusterID, fcFaceId: faceID}, - conflictAlgorithm: ConflictAlgorithm.replace, - ); - } - await batchUpdate.commit(noResult: true); + const String sql = ''' + INSERT INTO $faceClustersTable ($fcFaceId, $fcClusterID) + VALUES (?, ?) + ON CONFLICT($fcFaceId) DO UPDATE SET $fcClusterID = excluded.$fcClusterID + '''; + final parameterSets = batch.map((e) => [e.key, e.value]).toList(); + await db.writeTransaction((tx) async { + await tx.executeBatch(sql, parameterSets); + }); } } /// Returns a map of fileID to the indexed ML version Future> getIndexedFileIds() async { - final db = await instance.sqliteAsyncDB; + final db = await instance.asyncDB; final List> maps = await db.getAll( 'SELECT $fileIDColumn, $mlVersionColumn FROM $facesTable', ); @@ -138,7 +142,7 @@ class FaceMLDataDB { } Future getIndexedFileCount() async { - final db = await instance.sqliteAsyncDB; + final db = await instance.asyncDB; final List> maps = await db.getAll( 'SELECT COUNT(DISTINCT $fileIDColumn) as count FROM $facesTable', ); @@ -146,7 +150,7 @@ class FaceMLDataDB { } Future> clusterIdToFaceCount() async { - final db = await instance.sqliteAsyncDB; + final db = await instance.asyncDB; final List> maps = await db.getAll( 'SELECT $fcClusterID, COUNT(*) as count FROM $faceClustersTable where $fcClusterID IS NOT NULL GROUP BY $fcClusterID ', ); @@ -158,7 +162,7 @@ class FaceMLDataDB { } Future> getPersonIgnoredClusters(String personID) async { - final db = await instance.sqliteAsyncDB; + final db = await instance.asyncDB; // find out clusterIds that are assigned to other persons using the clusters table final List> maps = await db.getAll( 'SELECT $clusterIDColumn FROM $clusterPersonTable WHERE $personIdColumn != ? AND $personIdColumn IS NOT NULL', @@ -176,7 +180,7 @@ class FaceMLDataDB { } Future> getPersonClusterIDs(String personID) async { - final db = await instance.sqliteAsyncDB; + final db = await instance.asyncDB; final List> maps = await db.getAll( 'SELECT $clusterIDColumn FROM $clusterPersonTable WHERE $personIdColumn = ?', [personID], @@ -185,19 +189,22 @@ class FaceMLDataDB { } Future clearTable() async { - final db = await instance.database; - await db.delete(facesTable); - await db.delete(clusterPersonTable); - await db.delete(clusterSummaryTable); - await db.delete(personTable); - await db.delete(notPersonFeedback); + final db = await instance.asyncDB; + + await db.writeTransaction((tx) async { + await tx.execute(deleteFacesTable); + await tx.execute(dropClusterPersonTable); + await tx.execute(dropClusterSummaryTable); + await tx.execute(deletePersonTable); + await tx.execute(dropNotPersonFeedbackTable); + }); } Future> getFaceEmbeddingsForCluster( int clusterID, { int? limit, }) async { - final db = await instance.sqliteAsyncDB; + final db = await instance.asyncDB; final List> maps = await db.getAll( 'SELECT $faceEmbeddingBlob FROM $facesTable WHERE $faceIDColumn in (SELECT $fcFaceId from $faceClustersTable where $fcClusterID = ?) ${limit != null ? 'LIMIT $limit' : ''}', [clusterID], @@ -209,7 +216,7 @@ class FaceMLDataDB { Iterable clusterIDs, { int? limit, }) async { - final db = await instance.sqliteAsyncDB; + final db = await instance.asyncDB; final Map> result = {}; final selectQuery = ''' @@ -238,7 +245,7 @@ class FaceMLDataDB { int? clusterID, }) async { // read person from db - final db = await instance.database; + final db = await instance.asyncDB; if (personID != null) { final List fileId = [recentFileID]; int? avatarFileId; @@ -248,15 +255,18 @@ class FaceMLDataDB { fileId.add(avatarFileId); } } - final cluterRows = await db.query( - clusterPersonTable, - columns: [clusterIDColumn], - where: '$personIdColumn = ?', - whereArgs: [personID], + const String queryClusterID = ''' + SELECT $clusterIDColumn + FROM $clusterPersonTable + WHERE $personIdColumn = ? + '''; + final clusterRows = await db.getAll( + queryClusterID, + [personID], ); final clusterIDs = - cluterRows.map((e) => e[clusterIDColumn] as int).toList(); - final List> faceMaps = await db.rawQuery( + clusterRows.map((e) => e[clusterIDColumn] as int).toList(); + final List> faceMaps = await db.getAll( 'SELECT * FROM $facesTable where ' '$faceIDColumn in (SELECT $fcFaceId from $faceClustersTable where $fcClusterID IN (${clusterIDs.join(",")}))' 'AND $fileIDColumn in (${fileId.join(",")}) AND $faceScore > $kMinimumQualityFaceScore ORDER BY $faceScore DESC', @@ -274,11 +284,14 @@ class FaceMLDataDB { } } if (clusterID != null) { - final List> faceMaps = await db.query( - faceClustersTable, - columns: [fcFaceId], - where: '$fcClusterID = ?', - whereArgs: [clusterID], + const String queryFaceID = ''' + SELECT $fcFaceId + FROM $faceClustersTable + WHERE $fcClusterID = ? + '''; + final List> faceMaps = await db.getAll( + queryFaceID, + [clusterID], ); final List? faces = await getFacesForGivenFileID(recentFileID); if (faces != null) { @@ -297,22 +310,14 @@ class FaceMLDataDB { } Future?> getFacesForGivenFileID(int fileUploadID) async { - final db = await instance.database; - final List> maps = await db.query( - facesTable, - columns: [ - faceIDColumn, - fileIDColumn, - faceEmbeddingBlob, - faceScore, - faceDetectionColumn, - faceBlur, - imageHeight, - imageWidth, - mlVersionColumn, - ], - where: '$fileIDColumn = ?', - whereArgs: [fileUploadID], + final db = await instance.asyncDB; + const String query = ''' + SELECT * FROM $facesTable + WHERE $fileIDColumn = ? + '''; + final List> maps = await db.getAll( + query, + [fileUploadID], ); if (maps.isEmpty) { return null; @@ -321,7 +326,7 @@ class FaceMLDataDB { } Future getFaceForFaceID(String faceID) async { - final db = await instance.sqliteAsyncDB; + final db = await instance.asyncDB; final result = await db.getAll( 'SELECT * FROM $facesTable where $faceIDColumn = ?', [faceID], @@ -335,7 +340,7 @@ class FaceMLDataDB { Future>> getClusterToFaceIDs( Set clusterIDs, ) async { - final db = await instance.sqliteAsyncDB; + final db = await instance.asyncDB; final Map> result = {}; final List> maps = await db.getAll( 'SELECT $fcClusterID, $fcFaceId FROM $faceClustersTable WHERE $fcClusterID IN (${clusterIDs.join(",")})', @@ -349,7 +354,7 @@ class FaceMLDataDB { } Future getClusterIDForFaceID(String faceID) async { - final db = await instance.sqliteAsyncDB; + final db = await instance.asyncDB; final List> maps = await db.getAll( 'SELECT $fcClusterID FROM $faceClustersTable WHERE $fcFaceId = ?', [faceID], @@ -361,7 +366,7 @@ class FaceMLDataDB { } Future>> getAllClusterIdToFaceIDs() async { - final db = await instance.sqliteAsyncDB; + final db = await instance.asyncDB; final Map> result = {}; final List> maps = await db.getAll( 'SELECT $fcClusterID, $fcFaceId FROM $faceClustersTable', @@ -375,7 +380,7 @@ class FaceMLDataDB { } Future> getFaceIDsForCluster(int clusterID) async { - final db = await instance.sqliteAsyncDB; + final db = await instance.asyncDB; final List> maps = await db.getAll( 'SELECT $fcFaceId FROM $faceClustersTable ' 'WHERE $faceClustersTable.$fcClusterID = ?', @@ -385,7 +390,7 @@ class FaceMLDataDB { } Future> getFaceIDsForPerson(String personID) async { - final db = await instance.sqliteAsyncDB; + final db = await instance.asyncDB; final faceIdsResult = await db.getAll( 'SELECT $fcFaceId FROM $faceClustersTable LEFT JOIN $clusterPersonTable ' 'ON $faceClustersTable.$fcClusterID = $clusterPersonTable.$clusterIDColumn ' @@ -396,7 +401,7 @@ class FaceMLDataDB { } Future> getBlurValuesForCluster(int clusterID) async { - final db = await instance.sqliteAsyncDB; + final db = await instance.asyncDB; const String query = ''' SELECT $facesTable.$faceBlur FROM $facesTable @@ -418,7 +423,7 @@ class FaceMLDataDB { Future> getFaceIDsToBlurValues( int maxBlurValue, ) async { - final db = await instance.sqliteAsyncDB; + final db = await instance.asyncDB; final List> maps = await db.getAll( 'SELECT $faceIDColumn, $faceBlur FROM $facesTable WHERE $faceBlur < $maxBlurValue AND $faceBlur > 1 ORDER BY $faceBlur ASC', ); @@ -432,8 +437,8 @@ class FaceMLDataDB { Future> getFaceIdsToClusterIds( Iterable faceIds, ) async { - final db = await instance.database; - final List> maps = await db.rawQuery( + final db = await instance.asyncDB; + final List> maps = await db.getAll( 'SELECT $fcFaceId, $fcClusterID FROM $faceClustersTable where $fcFaceId IN (${faceIds.map((id) => "'$id'").join(",")})', ); final Map result = {}; @@ -445,8 +450,8 @@ class FaceMLDataDB { Future>> getFileIdToClusterIds() async { final Map> result = {}; - final db = await instance.database; - final List> maps = await db.rawQuery( + final db = await instance.asyncDB; + final List> maps = await db.getAll( 'SELECT $fcClusterID, $fcFaceId FROM $faceClustersTable', ); @@ -463,36 +468,33 @@ class FaceMLDataDB { Future forceUpdateClusterIds( Map faceIDToClusterID, ) async { - final db = await instance.database; + final db = await instance.asyncDB; - // Start a batch - final batch = db.batch(); - - for (final map in faceIDToClusterID.entries) { - final faceID = map.key; - final clusterID = map.value; - batch.insert( - faceClustersTable, - {fcFaceId: faceID, fcClusterID: clusterID}, - conflictAlgorithm: ConflictAlgorithm.replace, - ); - } - // Commit the batch - await batch.commit(noResult: true); + const String sql = ''' + INSERT INTO $faceClustersTable ($fcFaceId, $fcClusterID) + VALUES (?, ?) + ON CONFLICT($fcFaceId) DO UPDATE SET $fcClusterID = excluded.$fcClusterID + '''; + final parameterSets = + faceIDToClusterID.entries.map((e) => [e.key, e.value]).toList(); + await db.writeTransaction((tx) async { + await tx.executeBatch(sql, parameterSets); + }); } Future removePerson(String personID) async { - final db = await instance.database; - await db.delete( - clusterPersonTable, - where: '$personIdColumn = ?', - whereArgs: [personID], - ); - await db.delete( - notPersonFeedback, - where: '$personIdColumn = ?', - whereArgs: [personID], - ); + final db = await instance.asyncDB; + + await db.writeTransaction((tx) async { + await tx.execute( + 'DELETE FROM $clusterPersonTable WHERE $personIdColumn = ?', + [personID], + ); + await tx.execute( + 'DELETE FROM $notPersonFeedback WHERE $personIdColumn = ?', + [personID], + ); + }); } Future> getFaceInfoForClustering({ @@ -506,7 +508,7 @@ class FaceMLDataDB { w.logAndReset( 'reading as float offset: $offset, maxFaces: $maxFaces, batchSize: $batchSize', ); - final db = await instance.sqliteAsyncDB; + final db = await instance.asyncDB; final Set result = {}; while (true) { @@ -561,7 +563,7 @@ class FaceMLDataDB { w.logAndReset( 'reading as float offset: $offset, maxFaces: $maxFaces, batchSize: $batchSize', ); - final db = await instance.sqliteAsyncDB; + final db = await instance.asyncDB; final Map result = {}; while (true) { @@ -605,7 +607,7 @@ class FaceMLDataDB { List fileIDs, ) async { _logger.info('reading face embeddings for ${fileIDs.length} files'); - final db = await instance.database; + final db = await instance.asyncDB; // Define the batch size const batchSize = 10000; @@ -614,15 +616,23 @@ class FaceMLDataDB { final Map result = {}; while (true) { // Query a batch of rows - final List> maps = await db.query( - facesTable, - columns: [faceIDColumn, faceEmbeddingBlob], - where: - '$faceScore > $kMinimumQualityFaceScore AND $faceBlur > $kLaplacianHardThreshold AND $fileIDColumn IN (${fileIDs.join(",")})', - limit: batchSize, - offset: offset, - orderBy: '$faceIDColumn DESC', - ); + + final List> maps = await db.getAll(''' + SELECT $faceIDColumn, $faceEmbeddingBlob + FROM $facesTable + WHERE $faceScore > $kMinimumQualityFaceScore AND $faceBlur > $kLaplacianHardThreshold AND $fileIDColumn IN (${fileIDs.join(",")}) + ORDER BY $faceIDColumn DESC + LIMIT $batchSize OFFSET $offset + '''); + // final List> maps = await db.query( + // facesTable, + // columns: [faceIDColumn, faceEmbeddingBlob], + // where: + // '$faceScore > $kMinimumQualityFaceScore AND $faceBlur > $kLaplacianHardThreshold AND $fileIDColumn IN (${fileIDs.join(",")})', + // limit: batchSize, + // offset: offset, + // orderBy: '$faceIDColumn DESC', + // ); // Break the loop if no more rows if (maps.isEmpty) { break; @@ -644,7 +654,7 @@ class FaceMLDataDB { Iterable faceIDs, ) async { _logger.info('reading face embeddings for ${faceIDs.length} faces'); - final db = await instance.sqliteAsyncDB; + final db = await instance.asyncDB; // Define the batch size const batchSize = 10000; @@ -681,7 +691,7 @@ class FaceMLDataDB { Future getTotalFaceCount({ double minFaceScore = kMinimumQualityFaceScore, }) async { - final db = await instance.sqliteAsyncDB; + final db = await instance.asyncDB; final List> maps = await db.getAll( 'SELECT COUNT(*) as count FROM $facesTable WHERE $faceScore > $minFaceScore AND $faceBlur > $kLaplacianHardThreshold', ); @@ -689,7 +699,7 @@ class FaceMLDataDB { } Future getClusteredToTotalFacesRatio() async { - final db = await instance.sqliteAsyncDB; + final db = await instance.asyncDB; final List> totalFacesMaps = await db.getAll( 'SELECT COUNT(*) as count FROM $facesTable WHERE $faceScore > $kMinimumQualityFaceScore AND $faceBlur > $kLaplacianHardThreshold', @@ -707,103 +717,111 @@ class FaceMLDataDB { Future getBlurryFaceCount([ int blurThreshold = kLaplacianHardThreshold, ]) async { - final db = await instance.database; - final List> maps = await db.rawQuery( + final db = await instance.asyncDB; + final List> maps = await db.getAll( 'SELECT COUNT(*) as count FROM $facesTable WHERE $faceBlur <= $blurThreshold AND $faceScore > $kMinimumQualityFaceScore', ); return maps.first['count'] as int; } Future resetClusterIDs() async { - final db = await instance.database; - await db.execute(dropFaceClustersTable); - await db.execute(createFaceClustersTable); - await db.execute(fcClusterIDIndex); + try { + final db = await instance.asyncDB; + + await db.writeTransaction((tx) async { + await tx.execute(dropFaceClustersTable); + await tx.execute(createFaceClustersTable); + await tx.execute(fcClusterIDIndex); + }); + } catch (e, s) { + _logger.severe('Error resetting clusterIDs', e, s); + } } Future assignClusterToPerson({ required String personID, required int clusterID, }) async { - final db = await instance.database; - await db.insert( - clusterPersonTable, - { - personIdColumn: personID, - clusterIDColumn: clusterID, - }, - ); + final db = await instance.asyncDB; + + const String sql = ''' + INSERT INTO $clusterPersonTable ($personIdColumn, $clusterIDColumn) VALUES (?, ?) ON CONFLICT($personIdColumn, $clusterIDColumn) DO NOTHING + '''; + await db.execute(sql, [personID, clusterID]); } Future bulkAssignClusterToPersonID( Map clusterToPersonID, ) async { - final db = await instance.database; - final batch = db.batch(); - for (final entry in clusterToPersonID.entries) { - final clusterID = entry.key; - final personID = entry.value; - batch.insert( - clusterPersonTable, - { - personIdColumn: personID, - clusterIDColumn: clusterID, - }, - conflictAlgorithm: ConflictAlgorithm.replace, - ); - } - await batch.commit(noResult: true); + final db = await instance.asyncDB; + + const String sql = ''' + INSERT INTO $clusterPersonTable ($personIdColumn, $clusterIDColumn) VALUES (?, ?) ON CONFLICT($personIdColumn, $clusterIDColumn) DO NOTHING + '''; + final parameterSets = + clusterToPersonID.entries.map((e) => [e.value, e.key]).toList(); + await db.writeTransaction((tx) async { + await tx.executeBatch(sql, parameterSets); + }); + // final batch = db.batch(); + // for (final entry in clusterToPersonID.entries) { + // final clusterID = entry.key; + // final personID = entry.value; + // batch.insert( + // clusterPersonTable, + // { + // personIdColumn: personID, + // clusterIDColumn: clusterID, + // }, + // conflictAlgorithm: ConflictAlgorithm.replace, + // ); + // } + // await batch.commit(noResult: true); } Future captureNotPersonFeedback({ required String personID, required int clusterID, }) async { - final db = await instance.database; - await db.insert( - notPersonFeedback, - { - personIdColumn: personID, - clusterIDColumn: clusterID, - }, - ); + final db = await instance.asyncDB; + + const String sql = ''' + INSERT INTO $notPersonFeedback ($personIdColumn, $clusterIDColumn) VALUES (?, ?) ON CONFLICT($personIdColumn, $clusterIDColumn) DO NOTHING + '''; + await db.execute(sql, [personID, clusterID]); } Future bulkCaptureNotPersonFeedback( Map clusterToPersonID, ) async { - final db = await instance.database; - final batch = db.batch(); - for (final entry in clusterToPersonID.entries) { - final clusterID = entry.key; - final personID = entry.value; - batch.insert( - notPersonFeedback, - { - personIdColumn: personID, - clusterIDColumn: clusterID, - }, - conflictAlgorithm: ConflictAlgorithm.replace, - ); - } - await batch.commit(noResult: true); + final db = await instance.asyncDB; + + const String sql = ''' + INSERT INTO $notPersonFeedback ($personIdColumn, $clusterIDColumn) VALUES (?, ?) ON CONFLICT($personIdColumn, $clusterIDColumn) DO NOTHING + '''; + final parameterSets = + clusterToPersonID.entries.map((e) => [e.value, e.key]).toList(); + + await db.writeTransaction((tx) async { + await tx.executeBatch(sql, parameterSets); + }); } - Future removeClusterToPerson({ + Future removeClusterToPerson({ required String personID, required int clusterID, }) async { - final db = await instance.database; - return db.delete( - clusterPersonTable, - where: '$personIdColumn = ? AND $clusterIDColumn = ?', - whereArgs: [personID, clusterID], - ); + final db = await instance.asyncDB; + + const String sql = ''' + DELETE FROM $clusterPersonTable WHERE $personIdColumn = ? AND $clusterIDColumn = ? + '''; + await db.execute(sql, [personID, clusterID]); } // for a given personID, return a map of clusterID to fileIDs using join query Future>> getFileIdToClusterIDSet(String personID) { - final db = instance.sqliteAsyncDB; + final db = instance.asyncDB; return db.then((db) async { final List> maps = await db.getAll( 'SELECT $faceClustersTable.$fcClusterID, $fcFaceId FROM $faceClustersTable ' @@ -826,7 +844,7 @@ class FaceMLDataDB { Future>> getFileIdToClusterIDSetForCluster( Set clusterIDs, ) { - final db = instance.sqliteAsyncDB; + final db = instance.asyncDB; return db.then((db) async { final List> maps = await db.getAll( 'SELECT $fcClusterID, $fcFaceId FROM $faceClustersTable ' @@ -844,37 +862,61 @@ class FaceMLDataDB { } Future clusterSummaryUpdate(Map summary) async { - final db = await instance.database; - var batch = db.batch(); + final db = await instance.asyncDB; + + const String sql = ''' + INSERT INTO $clusterSummaryTable ($clusterIDColumn, $avgColumn, $countColumn) VALUES (?, ?, ?) ON CONFLICT($clusterIDColumn) DO UPDATE SET $avgColumn = excluded.$avgColumn, $countColumn = excluded.$countColumn + '''; + final List> parameterSets = []; int batchCounter = 0; for (final entry in summary.entries) { if (batchCounter == 400) { - await batch.commit(noResult: true); - batch = db.batch(); + await db.writeTransaction((tx) async { + await tx.executeBatch(sql, parameterSets); + }); batchCounter = 0; + parameterSets.clear(); } - final int cluserID = entry.key; + final int clusterID = entry.key; final int count = entry.value.$2; final Uint8List avg = entry.value.$1; - batch.insert( - clusterSummaryTable, - { - clusterIDColumn: cluserID, - avgColumn: avg, - countColumn: count, - }, - conflictAlgorithm: ConflictAlgorithm.replace, - ); + parameterSets.add([clusterID, avg, count]); batchCounter++; } - await batch.commit(noResult: true); + await db.writeTransaction((tx) async { + await tx.executeBatch(sql, parameterSets); + }); + + // var batch = db.batch(); + // int batchCounter = 0; + // for (final entry in summary.entries) { + // if (batchCounter == 400) { + // await batch.commit(noResult: true); + // batch = db.batch(); + // batchCounter = 0; + // } + // final int cluserID = entry.key; + // final int count = entry.value.$2; + // final Uint8List avg = entry.value.$1; + // batch.insert( + // clusterSummaryTable, + // { + // clusterIDColumn: cluserID, + // avgColumn: avg, + // countColumn: count, + // }, + // conflictAlgorithm: ConflictAlgorithm.replace, + // ); + // batchCounter++; + // } + // await batch.commit(noResult: true); } /// Returns a map of clusterID to (avg embedding, count) Future> getAllClusterSummary([ int? minClusterSize, ]) async { - final db = await instance.sqliteAsyncDB; + final db = await instance.asyncDB; final Map result = {}; final rows = await db.getAll( 'SELECT * FROM $clusterSummaryTable${minClusterSize != null ? ' WHERE $countColumn >= $minClusterSize' : ''}', @@ -891,7 +933,7 @@ class FaceMLDataDB { Future> getClusterToClusterSummary( Iterable clusterIDs, ) async { - final db = await instance.sqliteAsyncDB; + final db = await instance.asyncDB; final Map result = {}; final rows = await db.getAll( 'SELECT * FROM $clusterSummaryTable WHERE $clusterIDColumn IN (${clusterIDs.join(",")})', @@ -906,7 +948,7 @@ class FaceMLDataDB { } Future> getClusterIDToPersonID() async { - final db = await instance.sqliteAsyncDB; + final db = await instance.asyncDB; final List> maps = await db.getAll( 'SELECT $personIdColumn, $clusterIDColumn FROM $clusterPersonTable', ); @@ -919,43 +961,60 @@ class FaceMLDataDB { /// WARNING: This will delete ALL data in the database! Only use this for debug/testing purposes! Future dropClustersAndPersonTable({bool faces = false}) async { - final db = await instance.database; - if (faces) { - await db.execute(deleteFacesTable); - await db.execute(createFacesTable); - await db.execute(dropFaceClustersTable); - await db.execute(createFaceClustersTable); - await db.execute(fcClusterIDIndex); - } - await db.execute(deletePersonTable); - await db.execute(dropClusterPersonTable); - await db.execute(dropClusterSummaryTable); - await db.execute(dropNotPersonFeedbackTable); + try { + final db = await instance.asyncDB; + if (faces) { + await db.writeTransaction((tx) async { + await tx.execute(deleteFacesTable); + await tx.execute(createFacesTable); + await tx.execute(dropFaceClustersTable); + await tx.execute(createFaceClustersTable); + await tx.execute(fcClusterIDIndex); + }); + } - await db.execute(createClusterPersonTable); - await db.execute(createNotPersonFeedbackTable); - await db.execute(createClusterSummaryTable); + await db.writeTransaction((tx) async { + await tx.execute(deletePersonTable); + await tx.execute(dropClusterPersonTable); + await tx.execute(dropClusterSummaryTable); + await tx.execute(dropNotPersonFeedbackTable); + }); + + await db.writeTransaction((tx) async { + await tx.execute(createClusterPersonTable); + await tx.execute(createNotPersonFeedbackTable); + await tx.execute(createClusterSummaryTable); + }); + } catch (e, s) { + _logger.severe('Error dropping clusters and person table', e, s); + } } /// WARNING: This will delete ALL data in the database! Only use this for debug/testing purposes! Future dropFeedbackTables() async { - final db = await instance.database; + try { + final db = await instance.asyncDB; - await db.execute(deletePersonTable); - await db.execute(dropClusterPersonTable); - await db.execute(dropNotPersonFeedbackTable); - await db.execute(dropClusterSummaryTable); - await db.execute(createClusterPersonTable); - await db.execute(createNotPersonFeedbackTable); - await db.execute(createClusterSummaryTable); + await db.writeTransaction((tx) async { + await tx.execute(deletePersonTable); + await tx.execute(dropClusterPersonTable); + await tx.execute(dropNotPersonFeedbackTable); + await tx.execute(dropClusterSummaryTable); + await tx.execute(createClusterPersonTable); + await tx.execute(createNotPersonFeedbackTable); + await tx.execute(createClusterSummaryTable); + }); + } catch (e) { + _logger.severe('Error dropping feedback tables', e); + } } Future removeFilesFromPerson( List files, String personID, ) async { - final db = await instance.database; - final faceIdsResult = await db.rawQuery( + final db = await instance.asyncDB; + final faceIdsResult = await db.getAll( 'SELECT $fcFaceId FROM $faceClustersTable LEFT JOIN $clusterPersonTable ' 'ON $faceClustersTable.$fcClusterID = $clusterPersonTable.$clusterIDColumn ' 'WHERE $clusterPersonTable.$personIdColumn = ?', @@ -981,8 +1040,8 @@ class FaceMLDataDB { List files, int clusterID, ) async { - final db = await instance.database; - final faceIdsResult = await db.rawQuery( + final db = await instance.asyncDB; + final faceIdsResult = await db.getAll( 'SELECT $fcFaceId FROM $faceClustersTable ' 'WHERE $faceClustersTable.$fcClusterID = ?', [clusterID], From 6235f7ee78d19aa5452227da3cec6bfdcab70023 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Sat, 27 Apr 2024 12:19:29 +0530 Subject: [PATCH 63/69] [mob][photos] Move FaceBoxImage to face_util --- mobile/lib/face/model/box.dart | 20 -------------------- mobile/lib/utils/face/face_util.dart | 20 ++++++++++++++++++++ 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/mobile/lib/face/model/box.dart b/mobile/lib/face/model/box.dart index dafe3d8384..73d7dea388 100644 --- a/mobile/lib/face/model/box.dart +++ b/mobile/lib/face/model/box.dart @@ -41,23 +41,3 @@ class FaceBox { 'height': height, }; } - -/// Bounding box of a face. -/// -/// [xMin] and [yMin] are the coordinates of the top left corner of the box, and -/// [width] and [height] are the width and height of the box. -/// -/// One unit is equal to one pixel in the original image. -class FaceBoxImage { - final int xMin; - final int yMin; - final int width; - final int height; - - FaceBoxImage({ - required this.xMin, - required this.yMin, - required this.width, - required this.height, - }); -} diff --git a/mobile/lib/utils/face/face_util.dart b/mobile/lib/utils/face/face_util.dart index b417657915..17b1986679 100644 --- a/mobile/lib/utils/face/face_util.dart +++ b/mobile/lib/utils/face/face_util.dart @@ -7,6 +7,26 @@ import "package:image/image.dart" as img; import "package:logging/logging.dart"; import "package:photos/face/model/box.dart"; +/// Bounding box of a face. +/// +/// [xMin] and [yMin] are the coordinates of the top left corner of the box, and +/// [width] and [height] are the width and height of the box. +/// +/// One unit is equal to one pixel in the original image. +class FaceBoxImage { + final int xMin; + final int yMin; + final int width; + final int height; + + FaceBoxImage({ + required this.xMin, + required this.yMin, + required this.width, + required this.height, + }); +} + final _logger = Logger("FaceUtil"); final _computer = Computer.shared(); const _faceImageBufferFactor = 0.2; From 54d3ad974373622eb658361c51f5271db80498e2 Mon Sep 17 00:00:00 2001 From: ashilkn Date: Sat, 27 Apr 2024 12:37:42 +0530 Subject: [PATCH 64/69] [mob][photos] Remove unnecessary optional parameter --- mobile/lib/utils/face/face_util.dart | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/mobile/lib/utils/face/face_util.dart b/mobile/lib/utils/face/face_util.dart index b417657915..389336115f 100644 --- a/mobile/lib/utils/face/face_util.dart +++ b/mobile/lib/utils/face/face_util.dart @@ -14,14 +14,11 @@ const _faceImageBufferFactor = 0.2; ///Convert img.Image to ui.Image and use RawImage to display. Future> generateImgFaceThumbnails( String imagePath, - List faceBoxes, { - ///Pass decodedImage decoded by [decodeToImgImage] to avoid decoding image - ///multiple times if all faces are from the same image (eg: File info). - img.Image? decodedImage, -}) async { + List faceBoxes, +) async { final faceThumbnails = []; - final image = decodedImage ?? await decodeToImgImage(imagePath); + final image = await decodeToImgImage(imagePath); for (FaceBox faceBox in faceBoxes) { final croppedImage = cropFaceBoxFromImage(image, faceBox); @@ -33,12 +30,9 @@ Future> generateImgFaceThumbnails( Future> generateJpgFaceThumbnails( String imagePath, - List faceBoxes, { - ///Pass decodedImage decoded by [decodeToImgImage] to avoid decoding image - ///multiple times if all faces are from the same image (eg: File info). - img.Image? decodedImage, -}) async { - final image = decodedImage ?? await decodeToImgImage(imagePath); + List faceBoxes, +) async { + final image = await decodeToImgImage(imagePath); final croppedImages = []; for (FaceBox faceBox in faceBoxes) { final croppedImage = cropFaceBoxFromImage(image, faceBox); From a2023bd4579e29df9a5fa4c0b803ba9d221fbb8d Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Sat, 27 Apr 2024 13:38:34 +0530 Subject: [PATCH 65/69] [mob][photos] Fix database issue --- mobile/lib/face/db.dart | 2 +- mobile/lib/face/db_fields.dart | 2 +- .../lib/services/machine_learning/face_ml/face_ml_service.dart | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mobile/lib/face/db.dart b/mobile/lib/face/db.dart index 05450b9687..594f9aa4fe 100644 --- a/mobile/lib/face/db.dart +++ b/mobile/lib/face/db.dart @@ -81,7 +81,7 @@ class FaceMLDataDB { INSERT INTO $facesTable ( $fileIDColumn, $faceIDColumn, $faceDetectionColumn, $faceEmbeddingBlob, $faceScore, $faceBlur, $isSideways, $imageHeight, $imageWidth, $mlVersionColumn ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - ON CONFLICT($faceIDColumn, $mlVersionColumn) DO NOTHING + ON CONFLICT($fileIDColumn, $faceIDColumn) DO UPDATE SET $faceIDColumn = excluded.$faceIDColumn, $faceDetectionColumn = excluded.$faceDetectionColumn, $faceEmbeddingBlob = excluded.$faceEmbeddingBlob, $faceScore = excluded.$faceScore, $faceBlur = excluded.$faceBlur, $isSideways = excluded.$isSideways, $imageHeight = excluded.$imageHeight, $imageWidth = excluded.$imageWidth, $mlVersionColumn = excluded.$mlVersionColumn '''; final parameterSets = batch.map((face) { final map = mapRemoteToFaceDB(face); diff --git a/mobile/lib/face/db_fields.dart b/mobile/lib/face/db_fields.dart index c7d0c703c4..b9fc723125 100644 --- a/mobile/lib/face/db_fields.dart +++ b/mobile/lib/face/db_fields.dart @@ -16,7 +16,7 @@ const mlVersionColumn = 'ml_version'; const createFacesTable = '''CREATE TABLE IF NOT EXISTS $facesTable ( $fileIDColumn INTEGER NOT NULL, - $faceIDColumn TEXT NOT NULL, + $faceIDColumn TEXT NOT NULL UNIQUE, $faceDetectionColumn TEXT NOT NULL, $faceEmbeddingBlob BLOB NOT NULL, $faceScore REAL NOT NULL, diff --git a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart index cbc5aab3a4..b0aa9218f2 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart @@ -870,7 +870,7 @@ class FaceMlService { stopwatch.stop(); _logger.info( "Finished Analyze image (${result.faces.length} faces) with uploadedFileID ${enteFile.uploadedFileID}, in " - "${stopwatch.elapsedMilliseconds} ms", + "${stopwatch.elapsedMilliseconds} ms (including time waiting for inference engine availability)", ); return result; From 87571159ccbcbe7efbd08f207e7613055f122345 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Sat, 27 Apr 2024 14:30:34 +0530 Subject: [PATCH 66/69] [mob][photos] Fix faces db conflict --- mobile/lib/face/db_fields.dart | 1 + 1 file changed, 1 insertion(+) diff --git a/mobile/lib/face/db_fields.dart b/mobile/lib/face/db_fields.dart index b9fc723125..863b1cc782 100644 --- a/mobile/lib/face/db_fields.dart +++ b/mobile/lib/face/db_fields.dart @@ -96,6 +96,7 @@ const createNotPersonFeedbackTable = ''' CREATE TABLE IF NOT EXISTS $notPersonFeedback ( $personIdColumn TEXT NOT NULL, $clusterIDColumn INTEGER NOT NULL + PRIMARY KEY($personIdColumn, $clusterIDColumn) ); '''; const dropNotPersonFeedbackTable = 'DROP TABLE IF EXISTS $notPersonFeedback'; From d03d8d564de5cb177e2ffa06263dc872ead6cf0e Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Mon, 29 Apr 2024 15:13:51 +0530 Subject: [PATCH 67/69] [mob][photos] Fix DB issue --- mobile/lib/face/db.dart | 69 +++++++++++++++------------------- mobile/lib/face/db_fields.dart | 2 +- 2 files changed, 31 insertions(+), 40 deletions(-) diff --git a/mobile/lib/face/db.dart b/mobile/lib/face/db.dart index 594f9aa4fe..14f710017d 100644 --- a/mobile/lib/face/db.dart +++ b/mobile/lib/face/db.dart @@ -191,13 +191,11 @@ class FaceMLDataDB { Future clearTable() async { final db = await instance.asyncDB; - await db.writeTransaction((tx) async { - await tx.execute(deleteFacesTable); - await tx.execute(dropClusterPersonTable); - await tx.execute(dropClusterSummaryTable); - await tx.execute(deletePersonTable); - await tx.execute(dropNotPersonFeedbackTable); - }); + await db.execute(deleteFacesTable); + await db.execute(dropClusterPersonTable); + await db.execute(dropClusterSummaryTable); + await db.execute(deletePersonTable); + await db.execute(dropNotPersonFeedbackTable); } Future> getFaceEmbeddingsForCluster( @@ -728,11 +726,9 @@ class FaceMLDataDB { try { final db = await instance.asyncDB; - await db.writeTransaction((tx) async { - await tx.execute(dropFaceClustersTable); - await tx.execute(createFaceClustersTable); - await tx.execute(fcClusterIDIndex); - }); + await db.execute(dropFaceClustersTable); + await db.execute(createFaceClustersTable); + await db.execute(fcClusterIDIndex); } catch (e, s) { _logger.severe('Error resetting clusterIDs', e, s); } @@ -964,27 +960,21 @@ class FaceMLDataDB { try { final db = await instance.asyncDB; if (faces) { - await db.writeTransaction((tx) async { - await tx.execute(deleteFacesTable); - await tx.execute(createFacesTable); - await tx.execute(dropFaceClustersTable); - await tx.execute(createFaceClustersTable); - await tx.execute(fcClusterIDIndex); - }); + await db.execute(deleteFacesTable); + await db.execute(createFacesTable); + await db.execute(dropFaceClustersTable); + await db.execute(createFaceClustersTable); + await db.execute(fcClusterIDIndex); } - await db.writeTransaction((tx) async { - await tx.execute(deletePersonTable); - await tx.execute(dropClusterPersonTable); - await tx.execute(dropClusterSummaryTable); - await tx.execute(dropNotPersonFeedbackTable); - }); + await db.execute(deletePersonTable); + await db.execute(dropClusterPersonTable); + await db.execute(dropClusterSummaryTable); + await db.execute(dropNotPersonFeedbackTable); - await db.writeTransaction((tx) async { - await tx.execute(createClusterPersonTable); - await tx.execute(createNotPersonFeedbackTable); - await tx.execute(createClusterSummaryTable); - }); + await db.execute(createClusterPersonTable); + await db.execute(createNotPersonFeedbackTable); + await db.execute(createClusterSummaryTable); } catch (e, s) { _logger.severe('Error dropping clusters and person table', e, s); } @@ -995,15 +985,16 @@ class FaceMLDataDB { try { final db = await instance.asyncDB; - await db.writeTransaction((tx) async { - await tx.execute(deletePersonTable); - await tx.execute(dropClusterPersonTable); - await tx.execute(dropNotPersonFeedbackTable); - await tx.execute(dropClusterSummaryTable); - await tx.execute(createClusterPersonTable); - await tx.execute(createNotPersonFeedbackTable); - await tx.execute(createClusterSummaryTable); - }); + // Drop the tables + await db.execute(deletePersonTable); + await db.execute(dropClusterPersonTable); + await db.execute(dropNotPersonFeedbackTable); + await db.execute(dropClusterSummaryTable); + + // Recreate the tables + await db.execute(createClusterPersonTable); + await db.execute(createNotPersonFeedbackTable); + await db.execute(createClusterSummaryTable); } catch (e) { _logger.severe('Error dropping feedback tables', e); } diff --git a/mobile/lib/face/db_fields.dart b/mobile/lib/face/db_fields.dart index 863b1cc782..e6a70a7d4e 100644 --- a/mobile/lib/face/db_fields.dart +++ b/mobile/lib/face/db_fields.dart @@ -95,7 +95,7 @@ const notPersonFeedback = 'not_person_feedback'; const createNotPersonFeedbackTable = ''' CREATE TABLE IF NOT EXISTS $notPersonFeedback ( $personIdColumn TEXT NOT NULL, - $clusterIDColumn INTEGER NOT NULL + $clusterIDColumn INTEGER NOT NULL, PRIMARY KEY($personIdColumn, $clusterIDColumn) ); '''; From 8058c6b621bcbf065793764b5116ab4ebfbfdabe Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Mon, 29 Apr 2024 15:20:01 +0530 Subject: [PATCH 68/69] [mob][photos] Remove unnecessary write transactions --- mobile/lib/face/db.dart | 29 ++++++++--------------------- 1 file changed, 8 insertions(+), 21 deletions(-) diff --git a/mobile/lib/face/db.dart b/mobile/lib/face/db.dart index 14f710017d..5158a9cd4d 100644 --- a/mobile/lib/face/db.dart +++ b/mobile/lib/face/db.dart @@ -99,9 +99,7 @@ class FaceMLDataDB { ]; }).toList(); - await db.writeTransaction((tx) async { - await tx.executeBatch(sql, parameterSets); - }); + await db.executeBatch(sql, parameterSets); } } @@ -122,9 +120,8 @@ class FaceMLDataDB { ON CONFLICT($fcFaceId) DO UPDATE SET $fcClusterID = excluded.$fcClusterID '''; final parameterSets = batch.map((e) => [e.key, e.value]).toList(); - await db.writeTransaction((tx) async { - await tx.executeBatch(sql, parameterSets); - }); + + await db.executeBatch(sql, parameterSets); } } @@ -475,9 +472,7 @@ class FaceMLDataDB { '''; final parameterSets = faceIDToClusterID.entries.map((e) => [e.key, e.value]).toList(); - await db.writeTransaction((tx) async { - await tx.executeBatch(sql, parameterSets); - }); + await db.executeBatch(sql, parameterSets); } Future removePerson(String personID) async { @@ -756,9 +751,7 @@ class FaceMLDataDB { '''; final parameterSets = clusterToPersonID.entries.map((e) => [e.value, e.key]).toList(); - await db.writeTransaction((tx) async { - await tx.executeBatch(sql, parameterSets); - }); + await db.executeBatch(sql, parameterSets); // final batch = db.batch(); // for (final entry in clusterToPersonID.entries) { // final clusterID = entry.key; @@ -798,9 +791,7 @@ class FaceMLDataDB { final parameterSets = clusterToPersonID.entries.map((e) => [e.value, e.key]).toList(); - await db.writeTransaction((tx) async { - await tx.executeBatch(sql, parameterSets); - }); + await db.executeBatch(sql, parameterSets); } Future removeClusterToPerson({ @@ -867,9 +858,7 @@ class FaceMLDataDB { int batchCounter = 0; for (final entry in summary.entries) { if (batchCounter == 400) { - await db.writeTransaction((tx) async { - await tx.executeBatch(sql, parameterSets); - }); + await db.executeBatch(sql, parameterSets); batchCounter = 0; parameterSets.clear(); } @@ -879,9 +868,7 @@ class FaceMLDataDB { parameterSets.add([clusterID, avg, count]); batchCounter++; } - await db.writeTransaction((tx) async { - await tx.executeBatch(sql, parameterSets); - }); + await db.executeBatch(sql, parameterSets); // var batch = db.batch(); // int batchCounter = 0; From aad13277058db42b22d7d6fd2a5de065aebb6b60 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Mon, 29 Apr 2024 15:29:00 +0530 Subject: [PATCH 69/69] [mob][photos] Small cleanup --- mobile/lib/face/db.dart | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/mobile/lib/face/db.dart b/mobile/lib/face/db.dart index 5158a9cd4d..e77084ebee 100644 --- a/mobile/lib/face/db.dart +++ b/mobile/lib/face/db.dart @@ -13,14 +13,16 @@ import "package:photos/face/model/face.dart"; import "package:photos/models/file/file.dart"; import "package:photos/services/machine_learning/face_ml/face_clustering/face_info_for_clustering.dart"; import 'package:photos/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart'; -// import 'package:sqflite/sqflite.dart'; -import 'package:sqlite_async/sqlite_async.dart' as sqlite_async; +import 'package:sqlite_async/sqlite_async.dart'; -/// Stores all data for the ML-related features. The database can be accessed by `MlDataDB.instance.database`. +/// Stores all data for the FacesML-related features. The database can be accessed by `FaceMLDataDB.instance.database`. /// /// This includes: /// [facesTable] - Stores all the detected faces and its embeddings in the images. -/// [personTable] - Stores all the clusters of faces which are considered to be the same person. +/// [createFaceClustersTable] - Stores all the mappings from the faces (faceID) to the clusters (clusterID). +/// [clusterPersonTable] - Stores all the clusters that are mapped to a certain person. +/// [clusterSummaryTable] - Stores a summary of each cluster, containg the mean embedding and the number of faces in the cluster. +/// [notPersonFeedback] - Stores the clusters that are confirmed not to belong to a certain person by the user class FaceMLDataDB { static final Logger _logger = Logger("FaceMLDataDB"); @@ -32,28 +34,28 @@ class FaceMLDataDB { static final FaceMLDataDB instance = FaceMLDataDB._privateConstructor(); // only have a single app-wide reference to the database - static Future? _sqliteAsyncDBFuture; + static Future? _sqliteAsyncDBFuture; - Future get asyncDB async { + Future get asyncDB async { _sqliteAsyncDBFuture ??= _initSqliteAsyncDatabase(); return _sqliteAsyncDBFuture!; } - Future _initSqliteAsyncDatabase() async { + Future _initSqliteAsyncDatabase() async { final documentsDirectory = await getApplicationDocumentsDirectory(); final String databaseDirectory = join(documentsDirectory.path, _databaseName); _logger.info("Opening sqlite_async access: DB path " + databaseDirectory); final asyncDBConnection = - sqlite_async.SqliteDatabase(path: databaseDirectory, maxReaders: 2); + SqliteDatabase(path: databaseDirectory, maxReaders: 2); await _onCreate(asyncDBConnection); return asyncDBConnection; } - Future _onCreate(sqlite_async.SqliteDatabase asyncDBConnection) async { - final migrations = sqlite_async.SqliteMigrations() + Future _onCreate(SqliteDatabase asyncDBConnection) async { + final migrations = SqliteMigrations() ..add( - sqlite_async.SqliteMigration(_databaseVersion, (tx) async { + SqliteMigration(_databaseVersion, (tx) async { await tx.execute(createFacesTable); await tx.execute(createFaceClustersTable); await tx.execute(createClusterPersonTable); @@ -791,7 +793,7 @@ class FaceMLDataDB { final parameterSets = clusterToPersonID.entries.map((e) => [e.value, e.key]).toList(); - await db.executeBatch(sql, parameterSets); + await db.executeBatch(sql, parameterSets); } Future removeClusterToPerson({