From a24d8f94d3f44a7d6b67ee1c0631d7f989ff001a Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Tue, 4 Jun 2024 17:02:45 +0530 Subject: [PATCH 1/4] [mob][photos] Wait on interaction in iOS --- .../machine_learning_controller.dart | 28 ++++++++----------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/mobile/lib/services/machine_learning/machine_learning_controller.dart b/mobile/lib/services/machine_learning/machine_learning_controller.dart index 7268032ca4..6d18d57199 100644 --- a/mobile/lib/services/machine_learning/machine_learning_controller.dart +++ b/mobile/lib/services/machine_learning/machine_learning_controller.dart @@ -18,7 +18,7 @@ class MachineLearningController { static const kMaximumTemperature = 42; // 42 degree celsius static const kMinimumBatteryLevel = 20; // 20% - static const kDefaultInteractionTimeout = Duration(seconds: 15); + final kDefaultInteractionTimeout = Duration(seconds: Platform.isIOS ? 5 : 15); static const kUnhealthyStates = ["over_heat", "over_voltage", "dead"]; bool _isDeviceHealthy = true; @@ -31,29 +31,26 @@ class MachineLearningController { void init() { _logger.info('init called'); - if (Platform.isAndroid) { - _startInteractionTimer(); - BatteryInfoPlugin() - .androidBatteryInfoStream - .listen((AndroidBatteryInfo? batteryInfo) { - _onAndroidBatteryStateUpdate(batteryInfo); - }); - } + _startInteractionTimer(kDefaultInteractionTimeout); if (Platform.isIOS) { BatteryInfoPlugin() .iosBatteryInfoStream .listen((IosBatteryInfo? batteryInfo) { _oniOSBatteryStateUpdate(batteryInfo); }); + } + if (Platform.isAndroid) { + BatteryInfoPlugin() + .androidBatteryInfoStream + .listen((AndroidBatteryInfo? batteryInfo) { + _onAndroidBatteryStateUpdate(batteryInfo); + }); } _fireControlEvent(); _logger.info('init done'); } void onUserInteraction() { - if (Platform.isIOS) { - return; - } if (!_isUserInteracting) { _logger.info("User is interacting with the app"); _isUserInteracting = true; @@ -63,8 +60,7 @@ class MachineLearningController { } bool _canRunGivenUserInteraction() { - return (Platform.isIOS ? true : !_isUserInteracting) || - mlInteractionOverride; + return !_isUserInteracting || mlInteractionOverride; } void forceOverrideML({required bool turnOn}) { @@ -84,7 +80,7 @@ class MachineLearningController { } } - void _startInteractionTimer({Duration timeout = kDefaultInteractionTimeout}) { + void _startInteractionTimer(Duration timeout) { _userInteractionTimer = Timer(timeout, () { _logger.info("User is not interacting with the app"); _isUserInteracting = false; @@ -94,7 +90,7 @@ class MachineLearningController { void _resetTimer() { _userInteractionTimer.cancel(); - _startInteractionTimer(); + _startInteractionTimer(kDefaultInteractionTimeout); } void _onAndroidBatteryStateUpdate(AndroidBatteryInfo? batteryInfo) { From 6743aa3db48dfbae01a9ef2d0db5363c48597969 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Tue, 4 Jun 2024 17:53:25 +0530 Subject: [PATCH 2/4] [mob][photos] Cooldown in indexing to prevent OS killing app --- .../face_ml/face_ml_service.dart | 136 ++++++++++++------ 1 file changed, 93 insertions(+), 43 deletions(-) diff --git a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart index 87a707995c..5a16b8d9e9 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart @@ -104,6 +104,8 @@ class FaceMlService { final int _fileDownloadLimit = 5; final int _embeddingFetchLimit = 200; final int _kForceClusteringFaceCount = 8000; + final int _kcooldownLimit = 300; + static const Duration _kCooldownDuration = Duration(minutes: 3); Future init({bool initializeImageMlIsolate = false}) async { if (LocalSettings.instance.isFaceIndexingEnabled == false) { @@ -167,7 +169,8 @@ class FaceMlService { pauseIndexingAndClustering(); } }); - if (Platform.isIOS && MachineLearningController.instance.isDeviceHealthy) { + if (Platform.isIOS && + MachineLearningController.instance.isDeviceHealthy) { _logger.info("Starting face indexing and clustering on iOS from init"); unawaited(indexAndClusterAll()); } @@ -272,7 +275,8 @@ class FaceMlService { switch (function) { case FaceMlOperation.analyzeImage: final time = DateTime.now(); - final FaceMlResult result = await FaceMlService.analyzeImageSync(args); + final FaceMlResult result = + await FaceMlService.analyzeImageSync(args); dev.log( "`analyzeImageSync` function executed in ${DateTime.now().difference(time).inMilliseconds} ms", ); @@ -285,7 +289,8 @@ class FaceMlService { error: e, stackTrace: stackTrace, ); - sendPort.send({'error': e.toString(), 'stackTrace': stackTrace.toString()}); + sendPort + .send({'error': e.toString(), 'stackTrace': stackTrace.toString()}); } }); } @@ -369,7 +374,8 @@ class FaceMlService { await sync(forceSync: _shouldSyncPeople); - final int unclusteredFacesCount = await FaceMLDataDB.instance.getUnclusteredFaceCount(); + final int unclusteredFacesCount = + await FaceMLDataDB.instance.getUnclusteredFaceCount(); if (unclusteredFacesCount > _kForceClusteringFaceCount) { _logger.info( "There are $unclusteredFacesCount unclustered faces, doing clustering first", @@ -396,10 +402,13 @@ class FaceMlService { _isIndexingOrClusteringRunning = true; _logger.info('starting image indexing'); - final w = (kDebugMode ? EnteWatch('prepare indexing files') : null)?..start(); - final Map alreadyIndexedFiles = await FaceMLDataDB.instance.getIndexedFileIds(); + final w = (kDebugMode ? EnteWatch('prepare indexing files') : null) + ?..start(); + final Map alreadyIndexedFiles = + await FaceMLDataDB.instance.getIndexedFileIds(); w?.log('getIndexedFileIds'); - final List enteFiles = await SearchService.instance.getAllFiles(); + final List enteFiles = + await SearchService.instance.getAllFiles(); w?.log('getAllFiles'); // Make sure the image conversion isolate is spawned @@ -408,6 +417,7 @@ class FaceMlService { int fileAnalyzedCount = 0; int fileSkippedCount = 0; + int cooldownCount = 0; final stopwatch = Stopwatch()..start(); final List filesWithLocalID = []; final List filesWithoutLocalID = []; @@ -426,7 +436,8 @@ class FaceMlService { } } w?.log('sifting through all normal files'); - final List hiddenFiles = await SearchService.instance.getHiddenFiles(); + final List hiddenFiles = + await SearchService.instance.getHiddenFiles(); w?.log('getHiddenFiles: ${hiddenFiles.length} hidden files'); for (final EnteFile enteFile in hiddenFiles) { if (_skipAnalysisEnteFile(enteFile, alreadyIndexedFiles)) { @@ -442,21 +453,22 @@ class FaceMlService { sortedBylocalID.addAll(filesWithoutLocalID); sortedBylocalID.addAll(hiddenFilesToIndex); w?.log('preparing all files to index'); - final List> chunks = sortedBylocalID.chunks(_embeddingFetchLimit); + final List> chunks = + sortedBylocalID.chunks(_embeddingFetchLimit); int fetchedCount = 0; outerLoop: for (final chunk in chunks) { - final futures = >[]; - if (LocalSettings.instance.remoteFetchEnabled) { try { - final Set fileIds = {}; // if there are duplicates here server returns 400 + final Set fileIds = + {}; // if there are duplicates here server returns 400 // Try to find embeddings on the remote server for (final f in chunk) { fileIds.add(f.uploadedFileID!); } _logger.info('starting remote fetch for ${fileIds.length} files'); - final res = await RemoteFileMLService.instance.getFilessEmbedding(fileIds); + final res = + await RemoteFileMLService.instance.getFilessEmbedding(fileIds); _logger.info('fetched ${res.mlData.length} embeddings'); fetchedCount += res.mlData.length; final List faces = []; @@ -478,7 +490,8 @@ class FaceMlService { faces.add(f); } } - remoteFileIdToVersion[fileMl.fileID] = fileMl.faceEmbedding.version; + remoteFileIdToVersion[fileMl.fileID] = + fileMl.faceEmbedding.version; } if (res.noEmbeddingFileIDs.isNotEmpty) { _logger.info( @@ -495,7 +508,8 @@ class FaceMlService { for (final entry in remoteFileIdToVersion.entries) { alreadyIndexedFiles[entry.key] = entry.value; } - _logger.info('already indexed files ${remoteFileIdToVersion.length}'); + _logger + .info('already indexed files ${remoteFileIdToVersion.length}'); } catch (e, s) { _logger.severe("err while getting files embeddings", e, s); if (retryFetchCount < 1000) { @@ -519,6 +533,7 @@ class FaceMlService { } final smallerChunks = chunk.chunks(_fileDownloadLimit); for (final smallestChunk in smallerChunks) { + final futures = >[]; if (!await canUseHighBandwidth()) { _logger.info( 'stopping indexing because user is not connected to wifi', @@ -545,12 +560,22 @@ class FaceMlService { (previousValue, element) => previousValue + (element ? 1 : 0), ); fileAnalyzedCount += sumFutures; + + if (fileAnalyzedCount > _kcooldownLimit) { + _logger.info( + 'Reached ${cooldownCount * _kcooldownLimit + fileAnalyzedCount} indexed files, cooling down to prevent OS from killing the app', + ); + cooldownCount++; + fileAnalyzedCount -= _kcooldownLimit; + await Future.delayed(_kCooldownDuration); + _logger.info('cooldown done, continuing indexing'); + } } } stopwatch.stop(); _logger.info( - "`indexAllImages()` finished. Fetched $fetchedCount and analyzed $fileAnalyzedCount images, in ${stopwatch.elapsed.inSeconds} seconds (avg of ${stopwatch.elapsed.inSeconds / fileAnalyzedCount} seconds per image, skipped $fileSkippedCount images)", + "`indexAllImages()` finished. Fetched $fetchedCount and analyzed ${cooldownCount * _kcooldownLimit + fileAnalyzedCount} images, in ${stopwatch.elapsed.inSeconds} seconds (avg of ${stopwatch.elapsed.inSeconds / fileAnalyzedCount} seconds per image, skipped $fileSkippedCount images, $cooldownCount cooldowns)", ); _logStatus(); } catch (e, s) { @@ -578,9 +603,10 @@ class FaceMlService { _showClusteringIsHappening = true; // Get a sense of the total number of faces in the database - final int totalFaces = - await FaceMLDataDB.instance.getTotalFaceCount(minFaceScore: minFaceScore); - final fileIDToCreationTime = await FilesDB.instance.getFileIDToCreationTime(); + final int totalFaces = await FaceMLDataDB.instance + .getTotalFaceCount(minFaceScore: minFaceScore); + final fileIDToCreationTime = + await FilesDB.instance.getFileIDToCreationTime(); final startEmbeddingFetch = DateTime.now(); // read all embeddings final result = await FaceMLDataDB.instance.getFaceInfoForClustering( @@ -598,7 +624,8 @@ class FaceMlService { } // sort the embeddings based on file creation time, newest first allFaceInfoForClustering.sort((b, a) { - return fileIDToCreationTime[a.fileID]!.compareTo(fileIDToCreationTime[b.fileID]!); + return fileIDToCreationTime[a.fileID]! + .compareTo(fileIDToCreationTime[b.fileID]!); }); _logger.info( 'Getting and sorting embeddings took ${DateTime.now().difference(startEmbeddingFetch).inMilliseconds} ms for ${allFaceInfoForClustering.length} embeddings' @@ -654,7 +681,8 @@ class FaceMlService { } } - final clusteringResult = await FaceClusteringService.instance.predictLinearIsolate( + final clusteringResult = + await FaceClusteringService.instance.predictLinearIsolate( faceInfoForClustering.toSet(), fileIDToCreationTime: fileIDToCreationTime, offset: offset, @@ -665,13 +693,17 @@ class FaceMlService { return; } - await FaceMLDataDB.instance.updateFaceIdToClusterId(clusteringResult.newFaceIdToCluster); - await FaceMLDataDB.instance.clusterSummaryUpdate(clusteringResult.newClusterSummaries); + await FaceMLDataDB.instance + .updateFaceIdToClusterId(clusteringResult.newFaceIdToCluster); + await FaceMLDataDB.instance + .clusterSummaryUpdate(clusteringResult.newClusterSummaries); Bus.instance.fire(PeopleChangedEvent()); for (final faceInfo in faceInfoForClustering) { - faceInfo.clusterId ??= clusteringResult.newFaceIdToCluster[faceInfo.faceID]; + faceInfo.clusterId ??= + clusteringResult.newFaceIdToCluster[faceInfo.faceID]; } - for (final clusterUpdate in clusteringResult.newClusterSummaries.entries) { + for (final clusterUpdate + in clusteringResult.newClusterSummaries.entries) { oldClusterSummaries[clusterUpdate.key] = clusterUpdate.value; } _logger.info( @@ -687,7 +719,8 @@ class FaceMlService { } else { final clusterStartTime = DateTime.now(); // Cluster the embeddings using the linear clustering algorithm, returning a map from faceID to clusterID - final clusteringResult = await FaceClusteringService.instance.predictLinearIsolate( + final clusteringResult = + await FaceClusteringService.instance.predictLinearIsolate( allFaceInfoForClustering.toSet(), fileIDToCreationTime: fileIDToCreationTime, oldClusterSummaries: oldClusterSummaries, @@ -705,8 +738,10 @@ class FaceMlService { _logger.info( 'Updating ${clusteringResult.newFaceIdToCluster.length} FaceIDs with clusterIDs in the DB', ); - await FaceMLDataDB.instance.updateFaceIdToClusterId(clusteringResult.newFaceIdToCluster); - await FaceMLDataDB.instance.clusterSummaryUpdate(clusteringResult.newClusterSummaries); + await FaceMLDataDB.instance + .updateFaceIdToClusterId(clusteringResult.newFaceIdToCluster); + await FaceMLDataDB.instance + .clusterSummaryUpdate(clusteringResult.newClusterSummaries); Bus.instance.fire(PeopleChangedEvent()); _logger.info('Done updating FaceIDs with clusterIDs in the DB, in ' '${DateTime.now().difference(clusterDoneTime).inSeconds} seconds'); @@ -739,7 +774,8 @@ class FaceMlService { allLandmarksEqual = false; break; } - if (face.detection.landmarks.any((landmark) => landmark.x != landmark.y)) { + if (face.detection.landmarks + .any((landmark) => landmark.x != landmark.y)) { allLandmarksEqual = false; break; } @@ -748,7 +784,10 @@ class FaceMlService { debugPrint("Discarding remote embedding for fileID ${fileMl.fileID} " "because landmarks are equal"); debugPrint( - fileMl.faceEmbedding.faces.map((e) => e.detection.landmarks.toString()).toList().toString(), + fileMl.faceEmbedding.faces + .map((e) => e.detection.landmarks.toString()) + .toList() + .toString(), ); return true; } @@ -786,9 +825,11 @@ class FaceMlService { Face.empty(result.fileId, error: result.errorOccured), ); } else { - if (result.decodedImageSize.width == -1 || result.decodedImageSize.height == -1) { - _logger.severe("decodedImageSize is not stored correctly for image with " - "ID: ${enteFile.uploadedFileID}"); + if (result.decodedImageSize.width == -1 || + result.decodedImageSize.height == -1) { + _logger + .severe("decodedImageSize is not stored correctly for image with " + "ID: ${enteFile.uploadedFileID}"); _logger.info( "Using aligned image size for image with ID: ${enteFile.uploadedFileID}. This size is ${result.decodedImageSize.width}x${result.decodedImageSize.height} compared to size of ${enteFile.width}x${enteFile.height} in the metadata", ); @@ -873,7 +914,8 @@ class FaceMlService { _checkEnteFileForID(enteFile); await ensureInitialized(); - final String? filePath = await _getImagePathForML(enteFile, typeOfData: FileDataForML.fileData); + final String? filePath = + await _getImagePathForML(enteFile, typeOfData: FileDataForML.fileData); if (filePath == null) { _logger.severe( @@ -892,8 +934,10 @@ class FaceMlService { { "enteFileID": enteFile.uploadedFileID ?? -1, "filePath": filePath, - "faceDetectionAddress": FaceDetectionService.instance.sessionAddress, - "faceEmbeddingAddress": FaceEmbeddingService.instance.sessionAddress, + "faceDetectionAddress": + FaceDetectionService.instance.sessionAddress, + "faceEmbeddingAddress": + FaceEmbeddingService.instance.sessionAddress, } ), ) as String?; @@ -947,7 +991,8 @@ class FaceMlService { stopwatch.reset(); // Get the faces - final List faceDetectionResult = await FaceMlService.detectFacesSync( + final List faceDetectionResult = + await FaceMlService.detectFacesSync( image, imgByteData, faceDetectionAddress, @@ -968,7 +1013,8 @@ class FaceMlService { stopwatch.reset(); // Align the faces - final Float32List faceAlignmentResult = await FaceMlService.alignFacesSync( + final Float32List faceAlignmentResult = + await FaceMlService.alignFacesSync( image, imgByteData, faceDetectionResult, @@ -1035,8 +1081,9 @@ class FaceMlService { } } if (file == null) { - _logger - .warning("Could not get file for $enteFile of type ${enteFile.fileType.toString()}"); + _logger.warning( + "Could not get file for $enteFile of type ${enteFile.fileType.toString()}", + ); imagePath = null; break; } @@ -1088,7 +1135,8 @@ class FaceMlService { }) async { try { // Get the bounding boxes of the faces - final (List faces, dataSize) = await FaceDetectionService.predictSync( + final (List faces, dataSize) = + await FaceDetectionService.predictSync( image, imageByteData, interpreterAddress, @@ -1198,7 +1246,8 @@ class FaceMlService { } bool _skipAnalysisEnteFile(EnteFile enteFile, Map indexedFileIds) { - if (_isIndexingOrClusteringRunning == false || _mlControllerStatus == false) { + if (_isIndexingOrClusteringRunning == false || + _mlControllerStatus == false) { return true; } // Skip if the file is not uploaded or not owned by the user @@ -1212,7 +1261,8 @@ class FaceMlService { // Skip if the file is already analyzed with the latest ml version final id = enteFile.uploadedFileID!; - return indexedFileIds.containsKey(id) && indexedFileIds[id]! >= faceMlVersion; + return indexedFileIds.containsKey(id) && + indexedFileIds[id]! >= faceMlVersion; } bool _cannotRunMLFunction({String function = ""}) { From 04048b20feb72682f710c33e6a68a1a78d0bbd5c Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Tue, 4 Jun 2024 18:11:15 +0530 Subject: [PATCH 3/4] [mob][photos] Make sure indexing is paused instantly --- .../face_ml/face_ml_service.dart | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart index 5a16b8d9e9..2fa0f9c871 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart @@ -303,6 +303,10 @@ class FaceMlService { return _functionLock.synchronized(() async { _resetInactivityTimer(); + if (_shouldPauseIndexingAndClustering) { + return null; + } + final completer = Completer(); final answerPort = ReceivePort(); @@ -811,9 +815,11 @@ class FaceMlService { // disposeImageIsolateAfterUse: false, ); if (result == null) { - _logger.severe( - "Failed to analyze image with uploadedFileID: ${enteFile.uploadedFileID}", - ); + if (!_shouldPauseIndexingAndClustering) { + _logger.severe( + "Failed to analyze image with uploadedFileID: ${enteFile.uploadedFileID}", + ); + } return false; } final List faces = []; @@ -942,7 +948,9 @@ class FaceMlService { ), ) as String?; if (resultJsonString == null) { - _logger.severe('Analyzing image in isolate is giving back null'); + if (!_shouldPauseIndexingAndClustering) { + _logger.severe('Analyzing image in isolate is giving back null'); + } return null; } result = FaceMlResult.fromJsonString(resultJsonString); From 465760e329f592617a41a5c3f763bcb91022573c Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Tue, 4 Jun 2024 18:11:54 +0530 Subject: [PATCH 4/4] [mob][photos] Increase file download limit to 10 --- .../lib/services/machine_learning/face_ml/face_ml_service.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart index 2fa0f9c871..e4620f6676 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart @@ -101,7 +101,7 @@ class FaceMlService { bool _shouldSyncPeople = false; bool _isSyncing = false; - final int _fileDownloadLimit = 5; + final int _fileDownloadLimit = 10; final int _embeddingFetchLimit = 200; final int _kForceClusteringFaceCount = 8000; final int _kcooldownLimit = 300;