From 0d12f1757691ded6de0b229ba7d1c536ea1258f3 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Wed, 26 Jun 2024 18:58:53 +0530 Subject: [PATCH 1/6] [mob][photos] Remove cooldown --- .../machine_learning/face_ml/face_ml_service.dart | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart index 277da70405..7123456805 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart @@ -99,8 +99,6 @@ class FaceMlService { final int _fileDownloadLimit = 10; final int _embeddingFetchLimit = 200; final int _kForceClusteringFaceCount = 8000; - final int _kcooldownLimit = 300; - static const Duration _kCooldownDuration = Duration(minutes: 3); Future init() async { if (LocalSettings.instance.isFaceIndexingEnabled == false) { @@ -405,7 +403,6 @@ class FaceMlService { int fileAnalyzedCount = 0; int fileSkippedCount = 0; - int cooldownCount = 0; final stopwatch = Stopwatch()..start(); final List filesWithLocalID = []; final List filesWithoutLocalID = []; @@ -548,22 +545,12 @@ class FaceMlService { (previousValue, element) => previousValue + (element ? 1 : 0), ); fileAnalyzedCount += sumFutures; - - if (fileAnalyzedCount > _kcooldownLimit) { - _logger.info( - 'Reached ${cooldownCount * _kcooldownLimit + fileAnalyzedCount} indexed files, cooling down to prevent OS from killing the app', - ); - cooldownCount++; - fileAnalyzedCount -= _kcooldownLimit; - await Future.delayed(_kCooldownDuration); - _logger.info('cooldown done, continuing indexing'); - } } } stopwatch.stop(); _logger.info( - "`indexAllImages()` finished. Fetched $fetchedCount and analyzed ${cooldownCount * _kcooldownLimit + fileAnalyzedCount} images, in ${stopwatch.elapsed.inSeconds} seconds (avg of ${stopwatch.elapsed.inSeconds / fileAnalyzedCount} seconds per image, skipped $fileSkippedCount images, $cooldownCount cooldowns)", + "`indexAllImages()` finished. Fetched $fetchedCount and analyzed $fileAnalyzedCount images, in ${stopwatch.elapsed.inSeconds} seconds (avg of ${stopwatch.elapsed.inSeconds / fileAnalyzedCount} seconds per image, skipped $fileSkippedCount images)", ); _logStatus(); } catch (e, s) { From 69bfaf2033ef5ce5201b17d23bb969f212f497fe Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Wed, 26 Jun 2024 22:00:31 +0530 Subject: [PATCH 2/6] [mob][photos] Only load models when actually indexing locally --- .../face_ml/face_ml_service.dart | 559 +++++++++--------- .../machine_learning_settings_page.dart | 2 +- 2 files changed, 269 insertions(+), 292 deletions(-) diff --git a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart index 7123456805..bffaa8469d 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart @@ -1,6 +1,6 @@ import "dart:async"; import "dart:developer" as dev show log; -import "dart:io" show File, Platform; +import "dart:io" show File; import "dart:isolate"; import "dart:math" show min; import "dart:typed_data" show Uint8List, Float32List, ByteData; @@ -40,7 +40,6 @@ import 'package:photos/services/machine_learning/face_ml/face_ml_result.dart'; import "package:photos/services/machine_learning/face_ml/person/person_service.dart"; import 'package:photos/services/machine_learning/file_ml/file_ml.dart'; import 'package:photos/services/machine_learning/file_ml/remote_fileml_service.dart'; -import "package:photos/services/machine_learning/machine_learning_controller.dart"; import 'package:photos/services/machine_learning/ml_exceptions.dart'; import "package:photos/services/search_service.dart"; import "package:photos/utils/file_util.dart"; @@ -50,7 +49,7 @@ import "package:photos/utils/network_util.dart"; import "package:photos/utils/thumbnail_util.dart"; import "package:synchronized/synchronized.dart"; -enum FileDataForML { thumbnailData, fileData, compressedFileData } +enum FileDataForML { thumbnailData, fileData } enum FaceMlOperation { analyzeImage } @@ -66,26 +65,29 @@ class FaceMlService { Timer? _inactivityTimer; final Duration _inactivityDuration = const Duration(seconds: 120); int _activeTasks = 0; - final _initLockIsolate = Lock(); late DartUiIsolate _isolate; late ReceivePort _receivePort = ReceivePort(); late SendPort _mainSendPort; - bool _isIsolateSpawned = false; - // Singleton pattern FaceMlService._privateConstructor(); static final instance = FaceMlService._privateConstructor(); factory FaceMlService() => instance; - final _initLock = Lock(); + final _initModelLock = Lock(); final _functionLock = Lock(); + final _initIsolateLock = Lock(); final _computer = Computer.shared(); - bool isInitialized = false; + bool _isInitialized = false; + bool _isModelsInitialized = false; + bool _isIsolateSpawned = false; + late String client; + bool get isInitialized => _isInitialized; + bool get showClusteringIsHappening => _showClusteringIsHappening; bool debugIndexingDisabled = false; @@ -96,250 +98,57 @@ class FaceMlService { bool _shouldSyncPeople = false; bool _isSyncing = false; - final int _fileDownloadLimit = 10; - final int _embeddingFetchLimit = 200; - final int _kForceClusteringFaceCount = 8000; + static const int _fileDownloadLimit = 10; + static const _embeddingFetchLimit = 200; + static const _kForceClusteringFaceCount = 8000; + /// Only call this function once at app startup, after that you can directly call [indexAndClusterAll] Future init() async { - if (LocalSettings.instance.isFaceIndexingEnabled == false) { + if (LocalSettings.instance.isFaceIndexingEnabled == false || + _isInitialized) { return; } - return _initLock.synchronized(() async { - if (isInitialized) { + _logger.info("init called"); + + // Listen on MachineLearningController + Bus.instance.on().listen((event) { + if (LocalSettings.instance.isFaceIndexingEnabled == false) { return; } - _logger.info("init called"); - _logStatus(); - await _computer.compute(initOrtEnv); - try { - await FaceDetectionService.instance.init(); - } catch (e, s) { - _logger.severe("Could not initialize yolo onnx", e, s); - } - try { - await FaceEmbeddingService.instance.init(); - } catch (e, s) { - _logger.severe("Could not initialize mobilefacenet", e, s); - } - - // Get client name - final packageInfo = await PackageInfo.fromPlatform(); - client = "${packageInfo.packageName}/${packageInfo.version}"; - _logger.info("client: $client"); - - isInitialized = true; - _mlControllerStatus = !Platform.isAndroid; - - /// hooking FaceML into [MachineLearningController] - Bus.instance.on().listen((event) { - if (LocalSettings.instance.isFaceIndexingEnabled == false) { - return; - } - _mlControllerStatus = event.shouldRun; - if (_mlControllerStatus) { - if (_shouldPauseIndexingAndClustering) { - _shouldPauseIndexingAndClustering = false; - _logger.info( - "MLController allowed running ML, faces indexing undoing previous pause", - ); - } else { - _logger.info( - "MLController allowed running ML, faces indexing starting", - ); - } - unawaited(indexAndClusterAll()); + _mlControllerStatus = event.shouldRun; + if (_mlControllerStatus) { + if (_shouldPauseIndexingAndClustering) { + _shouldPauseIndexingAndClustering = false; + _logger.info( + "MLController allowed running ML, faces indexing undoing previous pause", + ); } else { _logger.info( - "MLController stopped running ML, faces indexing will be paused (unless it's fetching embeddings)", + "MLController allowed running ML, faces indexing starting", ); - pauseIndexingAndClustering(); } - }); - if (Platform.isIOS && - MachineLearningController.instance.isDeviceHealthy) { - _logger.info("Starting face indexing and clustering on iOS from init"); unawaited(indexAndClusterAll()); + } else { + _logger.info( + "MLController stopped running ML, faces indexing will be paused (unless it's fetching embeddings)", + ); + pauseIndexingAndClustering(); } - - _listenIndexOnDiffSync(); - _listenOnPeopleChangedSync(); - - _logger.info('init done'); }); - } - static void initOrtEnv() async { - OrtEnv.instance.init(); - } - - void _listenIndexOnDiffSync() { + // Listen on DiffSync Bus.instance.on().listen((event) async { unawaited(sync()); }); - } - void _listenOnPeopleChangedSync() { + // Listne on PeopleChanged Bus.instance.on().listen((event) { if (event.type == PeopleEventType.syncDone) return; _shouldSyncPeople = true; }); - } - Future ensureInitialized() async { - if (!isInitialized) { - await init(); - } - } - - Future release() async { - return _initLock.synchronized(() async { - _logger.info("dispose called"); - if (!isInitialized) { - return; - } - try { - await FaceDetectionService.instance.release(); - } catch (e, s) { - _logger.severe("Could not dispose yolo onnx", e, s); - } - try { - await FaceEmbeddingService.instance.release(); - } catch (e, s) { - _logger.severe("Could not dispose mobilefacenet", e, s); - } - OrtEnv.instance.release(); - isInitialized = false; - }); - } - - Future _initIsolate() async { - return _initLockIsolate.synchronized(() async { - if (_isIsolateSpawned) return; - _logger.info("initIsolate called"); - - _receivePort = ReceivePort(); - - try { - _isolate = await DartUiIsolate.spawn( - _isolateMain, - _receivePort.sendPort, - ); - _mainSendPort = await _receivePort.first as SendPort; - _isIsolateSpawned = true; - - _resetInactivityTimer(); - } catch (e) { - _logger.severe('Could not spawn isolate', e); - _isIsolateSpawned = false; - } - }); - } - - Future _ensureSpawnedIsolate() async { - if (!_isIsolateSpawned) { - await _initIsolate(); - } - } - - /// The main execution function of the isolate. - @pragma('vm:entry-point') - static void _isolateMain(SendPort mainSendPort) async { - final receivePort = ReceivePort(); - mainSendPort.send(receivePort.sendPort); - - receivePort.listen((message) async { - final functionIndex = message[0] as int; - final function = FaceMlOperation.values[functionIndex]; - final args = message[1] as Map; - final sendPort = message[2] as SendPort; - - try { - switch (function) { - case FaceMlOperation.analyzeImage: - final time = DateTime.now(); - final FaceMlResult result = - await FaceMlService.analyzeImageSync(args); - dev.log( - "`analyzeImageSync` function executed in ${DateTime.now().difference(time).inMilliseconds} ms", - ); - sendPort.send(result.toJsonString()); - break; - } - } catch (e, stackTrace) { - dev.log( - "[SEVERE] Error in FaceML isolate: $e", - error: e, - stackTrace: stackTrace, - ); - sendPort - .send({'error': e.toString(), 'stackTrace': stackTrace.toString()}); - } - }); - } - - /// The common method to run any operation in the isolate. It sends the [message] to [_isolateMain] and waits for the result. - Future _runInIsolate( - (FaceMlOperation, Map) message, - ) async { - await _ensureSpawnedIsolate(); - return _functionLock.synchronized(() async { - _resetInactivityTimer(); - - if (_shouldPauseIndexingAndClustering) { - return null; - } - - final completer = Completer(); - final answerPort = ReceivePort(); - - _activeTasks++; - _mainSendPort.send([message.$1.index, message.$2, answerPort.sendPort]); - - answerPort.listen((receivedMessage) { - if (receivedMessage is Map && receivedMessage.containsKey('error')) { - // Handle the error - final errorMessage = receivedMessage['error']; - final errorStackTrace = receivedMessage['stackTrace']; - final exception = Exception(errorMessage); - final stackTrace = StackTrace.fromString(errorStackTrace); - completer.completeError(exception, stackTrace); - } else { - completer.complete(receivedMessage); - } - }); - _activeTasks--; - - return completer.future; - }); - } - - /// Resets a timer that kills the isolate after a certain amount of inactivity. - /// - /// Should be called after initialization (e.g. inside `init()`) and after every call to isolate (e.g. inside `_runInIsolate()`) - void _resetInactivityTimer() { - _inactivityTimer?.cancel(); - _inactivityTimer = Timer(_inactivityDuration, () { - if (_activeTasks > 0) { - _logger.info('Tasks are still running. Delaying isolate disposal.'); - // Optionally, reschedule the timer to check again later. - _resetInactivityTimer(); - } else { - _logger.info( - 'Clustering Isolate has been inactive for ${_inactivityDuration.inSeconds} seconds with no tasks running. Killing isolate.', - ); - _disposeIsolate(); - } - }); - } - - void _disposeIsolate() async { - if (!_isIsolateSpawned) return; - await release(); - - _isIsolateSpawned = false; - _isolate.kill(); - _receivePort.close(); - _inactivityTimer?.cancel(); + _isInitialized = true; + _logger.info('init done'); } Future sync({bool forceSync = true}) async { @@ -397,10 +206,6 @@ class FaceMlService { await SearchService.instance.getAllFiles(); w?.log('getAllFiles'); - // Make sure the image conversion isolate is spawned - // await ImageMlIsolate.instance.ensureSpawned(); - await ensureInitialized(); - int fileAnalyzedCount = 0; int fileSkippedCount = 0; final stopwatch = Stopwatch()..start(); @@ -537,6 +342,7 @@ class FaceMlService { fileSkippedCount++; continue; } + await _ensureReadyForInference(); futures.add(processImage(enteFile)); } final awaitedFutures = await Future.wait(futures); @@ -732,48 +538,6 @@ class FaceMlService { } } - bool _shouldDiscardRemoteEmbedding(FileMl fileMl) { - if (fileMl.faceEmbedding.version < faceMlVersion) { - debugPrint("Discarding remote embedding for fileID ${fileMl.fileID} " - "because version is ${fileMl.faceEmbedding.version} and we need $faceMlVersion"); - return true; - } - // are all landmarks equal? - bool allLandmarksEqual = true; - if (fileMl.faceEmbedding.faces.isEmpty) { - debugPrint("No face for ${fileMl.fileID}"); - allLandmarksEqual = false; - } - for (final face in fileMl.faceEmbedding.faces) { - if (face.detection.landmarks.isEmpty) { - allLandmarksEqual = false; - break; - } - if (face.detection.landmarks - .any((landmark) => landmark.x != landmark.y)) { - allLandmarksEqual = false; - break; - } - } - if (allLandmarksEqual) { - debugPrint("Discarding remote embedding for fileID ${fileMl.fileID} " - "because landmarks are equal"); - debugPrint( - fileMl.faceEmbedding.faces - .map((e) => e.detection.landmarks.toString()) - .toList() - .toString(), - ); - return true; - } - if (fileMl.width == null || fileMl.height == null) { - debugPrint("Discarding remote embedding for fileID ${fileMl.fileID} " - "because width is null"); - return true; - } - return false; - } - Future processImage(EnteFile enteFile) async { _logger.info( "`processImage` start processing image with uploadedFileID: ${enteFile.uploadedFileID}", @@ -886,10 +650,188 @@ class FaceMlService { } } - /// Analyzes the given image data by running the full pipeline for faces, using [analyzeImageSync] in the isolate. + Future _initModels() async { + if (LocalSettings.instance.isFaceIndexingEnabled) return; + return _initModelLock.synchronized(() async { + if (_isModelsInitialized) return; + _logger.info('initModels called'); + + // Get client name + final packageInfo = await PackageInfo.fromPlatform(); + client = "${packageInfo.packageName}/${packageInfo.version}"; + _logger.info("client: $client"); + + // Initialize models + await _computer.compute(() => OrtEnv.instance.init()); + try { + await FaceDetectionService.instance.init(); + } catch (e, s) { + _logger.severe("Could not initialize yolo onnx", e, s); + } + try { + await FaceEmbeddingService.instance.init(); + } catch (e, s) { + _logger.severe("Could not initialize mobilefacenet", e, s); + } + _isModelsInitialized = true; + _logger.info('initModels done'); + _logStatus(); + }); + } + + Future _releaseModels() async { + return _initModelLock.synchronized(() async { + _logger.info("dispose called"); + if (!_isInitialized) { + return; + } + try { + await FaceDetectionService.instance.release(); + } catch (e, s) { + _logger.severe("Could not dispose yolo onnx", e, s); + } + try { + await FaceEmbeddingService.instance.release(); + } catch (e, s) { + _logger.severe("Could not dispose mobilefacenet", e, s); + } + OrtEnv.instance.release(); + _isInitialized = false; + }); + } + + Future _initIsolate() async { + return _initIsolateLock.synchronized(() async { + if (_isIsolateSpawned) return; + _logger.info("initIsolate called"); + + _receivePort = ReceivePort(); + + try { + _isolate = await DartUiIsolate.spawn( + _isolateMain, + _receivePort.sendPort, + ); + _mainSendPort = await _receivePort.first as SendPort; + _isIsolateSpawned = true; + + _resetInactivityTimer(); + } catch (e) { + _logger.severe('Could not spawn isolate', e); + _isIsolateSpawned = false; + } + }); + } + + Future _ensureReadyForInference() async { + await _initModels(); + await _initIsolate(); + } + + /// The main execution function of the isolate. + @pragma('vm:entry-point') + static void _isolateMain(SendPort mainSendPort) async { + final receivePort = ReceivePort(); + mainSendPort.send(receivePort.sendPort); + + receivePort.listen((message) async { + final functionIndex = message[0] as int; + final function = FaceMlOperation.values[functionIndex]; + final args = message[1] as Map; + final sendPort = message[2] as SendPort; + + try { + switch (function) { + case FaceMlOperation.analyzeImage: + final time = DateTime.now(); + final FaceMlResult result = + await FaceMlService._analyzeImageSync(args); + dev.log( + "`analyzeImageSync` function executed in ${DateTime.now().difference(time).inMilliseconds} ms", + ); + sendPort.send(result.toJsonString()); + break; + } + } catch (e, stackTrace) { + dev.log( + "[SEVERE] Error in FaceML isolate: $e", + error: e, + stackTrace: stackTrace, + ); + sendPort + .send({'error': e.toString(), 'stackTrace': stackTrace.toString()}); + } + }); + } + + /// The common method to run any operation in the isolate. It sends the [message] to [_isolateMain] and waits for the result. + Future _runInIsolate( + (FaceMlOperation, Map) message, + ) async { + await _initIsolate(); + return _functionLock.synchronized(() async { + _resetInactivityTimer(); + + if (_shouldPauseIndexingAndClustering) { + return null; + } + + final completer = Completer(); + final answerPort = ReceivePort(); + + _activeTasks++; + _mainSendPort.send([message.$1.index, message.$2, answerPort.sendPort]); + + answerPort.listen((receivedMessage) { + if (receivedMessage is Map && receivedMessage.containsKey('error')) { + // Handle the error + final errorMessage = receivedMessage['error']; + final errorStackTrace = receivedMessage['stackTrace']; + final exception = Exception(errorMessage); + final stackTrace = StackTrace.fromString(errorStackTrace); + completer.completeError(exception, stackTrace); + } else { + completer.complete(receivedMessage); + } + }); + _activeTasks--; + + return completer.future; + }); + } + + /// Resets a timer that kills the isolate after a certain amount of inactivity. + /// + /// Should be called after initialization (e.g. inside `init()`) and after every call to isolate (e.g. inside `_runInIsolate()`) + void _resetInactivityTimer() { + _inactivityTimer?.cancel(); + _inactivityTimer = Timer(_inactivityDuration, () { + if (_activeTasks > 0) { + _logger.info('Tasks are still running. Delaying isolate disposal.'); + // Optionally, reschedule the timer to check again later. + _resetInactivityTimer(); + } else { + _logger.info( + 'Clustering Isolate has been inactive for ${_inactivityDuration.inSeconds} seconds with no tasks running. Killing isolate.', + ); + _dispose(); + } + }); + } + + void _dispose() async { + if (!_isIsolateSpawned) return; + _logger.info('Disposing isolate and models'); + await _releaseModels(); + _isIsolateSpawned = false; + _isolate.kill(); + _receivePort.close(); + _inactivityTimer?.cancel(); + } + + /// Analyzes the given image data by running the full pipeline for faces, using [_analyzeImageSync] in the isolate. Future _analyzeImageInSingleIsolate(EnteFile enteFile) async { _checkEnteFileForID(enteFile); - await ensureInitialized(); final String? filePath = await _getImagePathForML(enteFile, typeOfData: FileDataForML.fileData); @@ -948,7 +890,7 @@ class FaceMlService { return result; } - static Future analyzeImageSync(Map args) async { + static Future _analyzeImageSync(Map args) async { try { final int enteFileID = args["enteFileID"] as int; final String imagePath = args["filePath"] as String; @@ -973,7 +915,7 @@ class FaceMlService { // Get the faces final List faceDetectionResult = - await FaceMlService.detectFacesSync( + await FaceMlService._detectFacesSync( image, imgByteData, faceDetectionAddress, @@ -996,7 +938,7 @@ class FaceMlService { stopwatch.reset(); // Align the faces final Float32List faceAlignmentResult = - await FaceMlService.alignFacesSync( + await FaceMlService._alignFacesSync( image, imgByteData, faceDetectionResult, @@ -1008,7 +950,7 @@ class FaceMlService { stopwatch.reset(); // Get the embeddings of the faces - final embeddings = await FaceMlService.embedFacesSync( + final embeddings = await FaceMlService._embedFacesSync( faceAlignmentResult, faceEmbeddingAddress, resultBuilder: resultBuilder, @@ -1090,13 +1032,6 @@ class FaceMlService { "Getting thumbnail data for uploadedFileID ${enteFile.uploadedFileID} took ${stopwatch.elapsedMilliseconds} ms", ); break; - - case FileDataForML.compressedFileData: - _logger.warning( - "Getting compressed file data for uploadedFileID ${enteFile.uploadedFileID} is not implemented yet", - ); - imagePath = null; - break; } return imagePath; @@ -1107,7 +1042,7 @@ class FaceMlService { /// `imageData`: The image data to analyze. /// /// Returns a list of face detection results. - static Future> detectFacesSync( + static Future> _detectFacesSync( Image image, ByteData imageByteData, int interpreterAddress, { @@ -1142,7 +1077,7 @@ class FaceMlService { /// `faces`: The face detection results in a list of [FaceDetectionAbsolute] for the faces to align. /// /// Returns a list of the aligned faces as image data. - static Future alignFacesSync( + static Future _alignFacesSync( Image image, ByteData imageByteData, List faces, { @@ -1175,7 +1110,7 @@ class FaceMlService { } } - static Future>> embedFacesSync( + static Future>> _embedFacesSync( Float32List facesList, int interpreterAddress, { FaceMlResult? resultBuilder, @@ -1199,6 +1134,48 @@ class FaceMlService { } } + bool _shouldDiscardRemoteEmbedding(FileMl fileMl) { + if (fileMl.faceEmbedding.version < faceMlVersion) { + debugPrint("Discarding remote embedding for fileID ${fileMl.fileID} " + "because version is ${fileMl.faceEmbedding.version} and we need $faceMlVersion"); + return true; + } + // are all landmarks equal? + bool allLandmarksEqual = true; + if (fileMl.faceEmbedding.faces.isEmpty) { + debugPrint("No face for ${fileMl.fileID}"); + allLandmarksEqual = false; + } + for (final face in fileMl.faceEmbedding.faces) { + if (face.detection.landmarks.isEmpty) { + allLandmarksEqual = false; + break; + } + if (face.detection.landmarks + .any((landmark) => landmark.x != landmark.y)) { + allLandmarksEqual = false; + break; + } + } + if (allLandmarksEqual) { + debugPrint("Discarding remote embedding for fileID ${fileMl.fileID} " + "because landmarks are equal"); + debugPrint( + fileMl.faceEmbedding.faces + .map((e) => e.detection.landmarks.toString()) + .toList() + .toString(), + ); + return true; + } + if (fileMl.width == null || fileMl.height == null) { + debugPrint("Discarding remote embedding for fileID ${fileMl.fileID} " + "because width is null"); + return true; + } + return false; + } + /// Checks if the ente file to be analyzed actually can be analyzed: it must be uploaded and in the correct format. void _checkEnteFileForID(EnteFile enteFile) { if (_skipAnalysisEnteFile(enteFile, {})) { diff --git a/mobile/lib/ui/settings/machine_learning_settings_page.dart b/mobile/lib/ui/settings/machine_learning_settings_page.dart index 62ee7a1c0b..1ee333b591 100644 --- a/mobile/lib/ui/settings/machine_learning_settings_page.dart +++ b/mobile/lib/ui/settings/machine_learning_settings_page.dart @@ -223,7 +223,7 @@ class _MachineLearningSettingsPageState final isEnabled = await LocalSettings.instance.toggleFaceIndexing(); if (isEnabled) { - unawaited(FaceMlService.instance.ensureInitialized()); + unawaited(FaceMlService.instance.indexAndClusterAll()); } else { FaceMlService.instance.pauseIndexingAndClustering(); } From 3a8b2cf7b8f1a722025f75a29337dfe5e14aad1a Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Wed, 26 Jun 2024 23:06:10 +0530 Subject: [PATCH 3/6] [mob][photos] Small fix --- .../services/machine_learning/face_ml/face_ml_service.dart | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart index bffaa8469d..6737dac77a 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart @@ -651,7 +651,6 @@ class FaceMlService { } Future _initModels() async { - if (LocalSettings.instance.isFaceIndexingEnabled) return; return _initModelLock.synchronized(() async { if (_isModelsInitialized) return; _logger.info('initModels called'); @@ -682,7 +681,7 @@ class FaceMlService { Future _releaseModels() async { return _initModelLock.synchronized(() async { _logger.info("dispose called"); - if (!_isInitialized) { + if (!_isModelsInitialized) { return; } try { @@ -696,7 +695,7 @@ class FaceMlService { _logger.severe("Could not dispose mobilefacenet", e, s); } OrtEnv.instance.release(); - _isInitialized = false; + _isModelsInitialized = false; }); } From ca7096b87c4cba90d9ab71e5e3dc2ad1de2ebd39 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Wed, 26 Jun 2024 23:09:27 +0530 Subject: [PATCH 4/6] [mob][photos] Rename --- .../services/machine_learning/face_ml/face_ml_service.dart | 6 +++--- mobile/lib/ui/settings/debug/face_debug_section_widget.dart | 2 +- mobile/lib/ui/settings/machine_learning_settings_page.dart | 3 ++- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart index 6737dac77a..adad5afaf3 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart @@ -102,7 +102,7 @@ class FaceMlService { static const _embeddingFetchLimit = 200; static const _kForceClusteringFaceCount = 8000; - /// Only call this function once at app startup, after that you can directly call [indexAndClusterAll] + /// Only call this function once at app startup, after that you can directly call [runAllFaceML] Future init() async { if (LocalSettings.instance.isFaceIndexingEnabled == false || _isInitialized) { @@ -127,7 +127,7 @@ class FaceMlService { "MLController allowed running ML, faces indexing starting", ); } - unawaited(indexAndClusterAll()); + unawaited(runAllFaceML()); } else { _logger.info( "MLController stopped running ML, faces indexing will be paused (unless it's fetching embeddings)", @@ -164,7 +164,7 @@ class FaceMlService { _isSyncing = false; } - Future indexAndClusterAll() async { + Future runAllFaceML() async { if (_cannotRunMLFunction()) return; await sync(forceSync: _shouldSyncPeople); diff --git a/mobile/lib/ui/settings/debug/face_debug_section_widget.dart b/mobile/lib/ui/settings/debug/face_debug_section_widget.dart index 844f71c01b..1658d6a5f7 100644 --- a/mobile/lib/ui/settings/debug/face_debug_section_widget.dart +++ b/mobile/lib/ui/settings/debug/face_debug_section_widget.dart @@ -149,7 +149,7 @@ class _FaceDebugSectionWidgetState extends State { onTap: () async { try { FaceMlService.instance.debugIndexingDisabled = false; - unawaited(FaceMlService.instance.indexAndClusterAll()); + unawaited(FaceMlService.instance.runAllFaceML()); } catch (e, s) { _logger.warning('indexAndClusterAll failed ', e, s); await showGenericErrorDialog(context: context, error: e); diff --git a/mobile/lib/ui/settings/machine_learning_settings_page.dart b/mobile/lib/ui/settings/machine_learning_settings_page.dart index 1ee333b591..a284f6d98a 100644 --- a/mobile/lib/ui/settings/machine_learning_settings_page.dart +++ b/mobile/lib/ui/settings/machine_learning_settings_page.dart @@ -223,7 +223,8 @@ class _MachineLearningSettingsPageState final isEnabled = await LocalSettings.instance.toggleFaceIndexing(); if (isEnabled) { - unawaited(FaceMlService.instance.indexAndClusterAll()); + await FaceMlService.instance.init(); + unawaited(FaceMlService.instance.runAllFaceML()); } else { FaceMlService.instance.pauseIndexingAndClustering(); } From 2c9b0e77bacd64b7efe0db354fd457bc5ded977d Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Wed, 26 Jun 2024 23:11:23 +0530 Subject: [PATCH 5/6] [mob][photos] Force run on ML page --- .../services/machine_learning/face_ml/face_ml_service.dart | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart index adad5afaf3..3fdefb25bf 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart @@ -164,8 +164,11 @@ class FaceMlService { _isSyncing = false; } - Future runAllFaceML() async { - if (_cannotRunMLFunction()) return; + Future runAllFaceML({bool force = false}) async { + if (force) { + _mlControllerStatus = true; + } + if (_cannotRunMLFunction() && !force) return; await sync(forceSync: _shouldSyncPeople); From c5bcc1992c0bba363fdafc2aff829b0d4e2863df Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Wed, 26 Jun 2024 23:53:48 +0530 Subject: [PATCH 6/6] [mob][photos] Small fix --- .../lib/services/machine_learning/face_ml/face_ml_service.dart | 1 + mobile/lib/ui/settings/machine_learning_settings_page.dart | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart index 3fdefb25bf..99172153b5 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart @@ -718,6 +718,7 @@ class FaceMlService { _isIsolateSpawned = true; _resetInactivityTimer(); + _logger.info('initIsolate done'); } catch (e) { _logger.severe('Could not spawn isolate', e); _isIsolateSpawned = false; diff --git a/mobile/lib/ui/settings/machine_learning_settings_page.dart b/mobile/lib/ui/settings/machine_learning_settings_page.dart index a284f6d98a..c2f8ffaf54 100644 --- a/mobile/lib/ui/settings/machine_learning_settings_page.dart +++ b/mobile/lib/ui/settings/machine_learning_settings_page.dart @@ -224,7 +224,7 @@ class _MachineLearningSettingsPageState await LocalSettings.instance.toggleFaceIndexing(); if (isEnabled) { await FaceMlService.instance.init(); - unawaited(FaceMlService.instance.runAllFaceML()); + unawaited(FaceMlService.instance.runAllFaceML(force: true)); } else { FaceMlService.instance.pauseIndexingAndClustering(); }