From 5b94cbf912714e0c72cec01166ccf4d2165b6908 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Thu, 28 Mar 2024 13:22:32 +0530 Subject: [PATCH 1/5] [face][mob] Log db batch update progress --- mobile/lib/face/db.dart | 57 ++++++++++++++++++++++++----------------- 1 file changed, 33 insertions(+), 24 deletions(-) diff --git a/mobile/lib/face/db.dart b/mobile/lib/face/db.dart index 0b85e303c6..58a54782a4 100644 --- a/mobile/lib/face/db.dart +++ b/mobile/lib/face/db.dart @@ -78,6 +78,37 @@ class FaceMLDataDB { } } + Future updatePersonIDForFaceIDIFNotSet( + Map faceIDToPersonID, + ) async { + final db = await instance.database; + const batchSize = 500; + final numBatches = (faceIDToPersonID.length / batchSize).ceil(); + + for (int i = 0; i < numBatches; i++) { + _logger.info('updatePersonIDForFaceIDIFNotSet Batch $i of $numBatches'); + final start = i * batchSize; + final end = min((i + 1) * batchSize, faceIDToPersonID.length); + final batch = faceIDToPersonID.entries.toList().sublist(start, end); + + final batchUpdate = db.batch(); + + for (final entry in batch) { + final faceID = entry.key; + final personID = entry.value; + + batchUpdate.update( + facesTable, + {faceClusterId: personID}, + where: '$faceIDColumn = ? AND $faceClusterId IS NULL', + whereArgs: [faceID], + ); + } + + await batchUpdate.commit(noResult: true); + } + } + Future> getIndexedFileIds() async { final db = await instance.database; final List> maps = await db.rawQuery( @@ -290,28 +321,6 @@ class FaceMLDataDB { return result; } - Future updatePersonIDForFaceIDIFNotSet( - Map faceIDToPersonID, - ) async { - final db = await instance.database; - - // Start a batch - final batch = db.batch(); - - for (final map in faceIDToPersonID.entries) { - final faceID = map.key; - final personID = map.value; - batch.update( - facesTable, - {faceClusterId: personID}, - where: '$faceIDColumn = ? AND $faceClusterId IS NULL', - whereArgs: [faceID], - ); - } - // Commit the batch - await batch.commit(noResult: true); - } - Future forceUpdateClusterIds( Map faceIDToPersonID, ) async { @@ -340,7 +349,7 @@ class FaceMLDataDB { Future> getFaceEmbeddingMap({ double minScore = 0.78, int minClarity = kLaplacianThreshold, - int maxRows = 20000, + int maxRows = 10000, }) async { _logger.info('reading as float'); final db = await instance.database; @@ -370,7 +379,7 @@ class FaceMLDataDB { result[faceID] = (map[faceClusterId] as int?, map[faceEmbeddingBlob] as Uint8List); } - if (result.length >= 20000) { + if (result.length >= maxRows) { break; } offset += batchSize; From aa58989299301a7f485033850465d8fc48517023 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Thu, 28 Mar 2024 13:23:39 +0530 Subject: [PATCH 2/5] [mob] Remove commented out code --- .../linear_clustering_service.dart | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/mobile/lib/services/face_ml/face_clustering/linear_clustering_service.dart b/mobile/lib/services/face_ml/face_clustering/linear_clustering_service.dart index 8e77838592..2b36623154 100644 --- a/mobile/lib/services/face_ml/face_clustering/linear_clustering_service.dart +++ b/mobile/lib/services/face_ml/face_clustering/linear_clustering_service.dart @@ -349,21 +349,7 @@ class FaceLinearClustering { int moreThan50Count = 0; int moreThan100Count = 0; - // for (int i = 0; i < clusterSizes.length; i++) { - // if (clusterSizes[i] > 100) { - // moreThan100Count++; - // } else if (clusterSizes[i] > 50) { - // moreThan50Count++; - // } else if (clusterSizes[i] > 20) { - // moreThan20Count++; - // } else if (clusterSizes[i] > 10) { - // moreThan10Count++; - // } else if (clusterSizes[i] > 5) { - // moreThan5Count++; - // } else if (clusterSizes[i] == 1) { - // oneClusterCount++; - // } - // } + for (int i = 0; i < clusterSizes.length; i++) { if (clusterSizes[i] > 100) { moreThan100Count++; From 03e5f06bf295fa6edd9954ac3f630ec8e33028c9 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Thu, 28 Mar 2024 13:27:16 +0530 Subject: [PATCH 3/5] [mob][face] Add delay before starting faceIndexing --- mobile/lib/services/face_ml/face_ml_service.dart | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/mobile/lib/services/face_ml/face_ml_service.dart b/mobile/lib/services/face_ml/face_ml_service.dart index 1352823390..14ae63c9ff 100644 --- a/mobile/lib/services/face_ml/face_ml_service.dart +++ b/mobile/lib/services/face_ml/face_ml_service.dart @@ -118,7 +118,12 @@ class FaceMlService { if (LocalSettings.instance.isFaceIndexingEnabled == false) { return; } - unawaited(indexAllImages()); + // [neeraj] intentional delay in starting indexing on diff sync, this gives time for the user + // to disable face-indexing in case it's causing crash. In the future, we + // should have a better way to handle this. + Future.delayed(const Duration(seconds: 10), () { + unawaited(indexAllImages()); + }); }); } From d6db3de6b1b362e43cd35fcb8f6d597118bf0763 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Thu, 28 Mar 2024 13:40:15 +0530 Subject: [PATCH 4/5] [mob] Log time to fetch in debugMode --- mobile/lib/services/face_ml/face_ml_service.dart | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/mobile/lib/services/face_ml/face_ml_service.dart b/mobile/lib/services/face_ml/face_ml_service.dart index 14ae63c9ff..dd033da570 100644 --- a/mobile/lib/services/face_ml/face_ml_service.dart +++ b/mobile/lib/services/face_ml/face_ml_service.dart @@ -16,6 +16,7 @@ import "package:photos/core/event_bus.dart"; import "package:photos/db/ml_data_db.dart"; import "package:photos/events/diff_sync_complete_event.dart"; import "package:photos/extensions/list.dart"; +import "package:photos/extensions/stop_watch.dart"; import "package:photos/face/db.dart"; import "package:photos/face/model/box.dart"; import "package:photos/face/model/detection.dart" as face_detection; @@ -458,8 +459,11 @@ class FaceMlService { fileIds.add(f.uploadedFileID!); } try { + final EnteWatch? w = kDebugMode ? EnteWatch("face_em_fetch") : null; + w?.start(); final res = await RemoteFileMLService.instance.getFilessEmbedding(fileIds); + w?.logAndReset('fetched ${res.mlData.length} embeddings'); final List faces = []; final remoteFileIdToVersion = {}; for (FileMl fileMl in res.mlData.values) { @@ -481,6 +485,7 @@ class FaceMlService { remoteFileIdToVersion[fileMl.fileID] = fileMl.faceEmbedding.version; } await FaceMLDataDB.instance.bulkInsertFaces(faces); + w?.logAndReset('stored embeddings'); for (final entry in remoteFileIdToVersion.entries) { alreadyIndexedFiles[entry.key] = entry.value; } @@ -1167,7 +1172,6 @@ class FaceMlService { if (!enteFile.isUploaded || enteFile.isOwner == false) { return true; } - // Skip if the file is a video if (enteFile.fileType == FileType.video) { return true; From 6c287775d4766d25993004b49f8e6a6f22f09194 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Thu, 28 Mar 2024 16:25:09 +0530 Subject: [PATCH 5/5] [mob] Add logs for cluster feedback suggestions --- .../machine_learning/face_ml/feedback/cluster_feedback.dart | 6 +++++- mobile/lib/ui/settings/debug/face_debug_section_widget.dart | 2 +- mobile/lib/ui/viewer/people/person_cluster_suggestion.dart | 2 -- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart b/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart index 6a6fc28c0c..dfb6675b4b 100644 --- a/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart +++ b/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart @@ -79,7 +79,7 @@ class ClusterFeedbackService { final personClusters = await faceMlDb.getPersonClusterIDs(p.remoteID); dev.log( 'existing clusters for ${p.attr.name} are $personClusters', - name: "ClusterFeedbackService", + name: "getSuggestionsUsingMedian", ); // Get and update the cluster summary to get the avg (centroid) and count @@ -353,6 +353,9 @@ class ClusterFeedbackService { Set ignoredClusters, ) async { final faceMlDb = FaceMLDataDB.instance; + _logger.info( + 'start getUpdateClusterAvg for ${allClusterIdsToCountMap.length} clusters', + ); final Map clusterToSummary = await faceMlDb.clusterSummaryAll(); @@ -389,6 +392,7 @@ class ClusterFeedbackService { if (updatesForClusterSummary.isNotEmpty) { await faceMlDb.clusterSummaryUpdate(updatesForClusterSummary); } + _logger.info('end getUpdateClusterAvg for ${clusterAvg.length} clusters'); return clusterAvg; } diff --git a/mobile/lib/ui/settings/debug/face_debug_section_widget.dart b/mobile/lib/ui/settings/debug/face_debug_section_widget.dart index 4ee93b004c..f50c8d91d8 100644 --- a/mobile/lib/ui/settings/debug/face_debug_section_widget.dart +++ b/mobile/lib/ui/settings/debug/face_debug_section_widget.dart @@ -206,7 +206,7 @@ class _FaceDebugSectionWidgetState extends State { if (kDebugMode) MenuItemWidget( captionedTextWidget: const CaptionedTextWidget( - title: "Pull Embeddings From Local", + title: "Compute suggestions", ), pressedColor: getEnteColorScheme(context).fillFaint, trailingIcon: Icons.chevron_right_outlined, diff --git a/mobile/lib/ui/viewer/people/person_cluster_suggestion.dart b/mobile/lib/ui/viewer/people/person_cluster_suggestion.dart index 1bbcb4390b..26da744d6e 100644 --- a/mobile/lib/ui/viewer/people/person_cluster_suggestion.dart +++ b/mobile/lib/ui/viewer/people/person_cluster_suggestion.dart @@ -37,8 +37,6 @@ class _PersonClustersState extends State { super.initState(); // Initialize the future in initState _fetchClusterSuggestions(); - // futureClusterSuggestions = ClusterFeedbackService.instance - // .getClusterFilesForPersonID(widget.person); } @override