From eb7f3501022bad3954c3ccaaea0a7817707312a8 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Thu, 27 Mar 2025 16:04:13 +0530 Subject: [PATCH 01/12] Separate face thumbnail generator from embeddings --- .../face_thumbnail_generator.dart | 47 +++++++++++++++++++ .../machine_learning/ml_computer.dart | 29 +++--------- mobile/lib/utils/face/face_box_crop.dart | 4 +- 3 files changed, 55 insertions(+), 25 deletions(-) create mode 100644 mobile/lib/services/machine_learning/face_thumbnail_generator.dart diff --git a/mobile/lib/services/machine_learning/face_thumbnail_generator.dart b/mobile/lib/services/machine_learning/face_thumbnail_generator.dart new file mode 100644 index 0000000000..8970256a08 --- /dev/null +++ b/mobile/lib/services/machine_learning/face_thumbnail_generator.dart @@ -0,0 +1,47 @@ +import 'dart:async'; +import 'dart:typed_data' show Uint8List; + +import "package:logging/logging.dart"; +import "package:photos/models/ml/face/box.dart"; +import "package:photos/services/isolate_functions.dart"; +import "package:photos/services/isolate_service.dart"; +import "package:photos/utils/image_ml_util.dart"; + +class FaceThumbnailGenerator extends SuperIsolate { + @override + Logger get logger => _logger; + final _logger = Logger('FaceThumbnailGenerator'); + + @override + bool get isDartUiIsolate => true; + + @override + String get isolateName => "FaceThumbnailGenerator"; + + @override + bool get shouldAutomaticDispose => false; + + // Singleton pattern + FaceThumbnailGenerator._privateConstructor(); + static final FaceThumbnailGenerator instance = + FaceThumbnailGenerator._privateConstructor(); + factory FaceThumbnailGenerator() => instance; + + /// Generates face thumbnails for all [faceBoxes] in [imageData]. + /// + /// Uses [generateFaceThumbnailsUsingCanvas] inside the isolate. + Future> generateFaceThumbnails( + String imagePath, + List faceBoxes, + ) async { + final List> faceBoxesJson = + faceBoxes.map((box) => box.toJson()).toList(); + return await runInIsolate( + IsolateOperation.generateFaceThumbnails, + { + 'imagePath': imagePath, + 'faceBoxesList': faceBoxesJson, + }, + ).then((value) => value.cast()); + } +} diff --git a/mobile/lib/services/machine_learning/ml_computer.dart b/mobile/lib/services/machine_learning/ml_computer.dart index 3301f6b7c6..2d34856a67 100644 --- a/mobile/lib/services/machine_learning/ml_computer.dart +++ b/mobile/lib/services/machine_learning/ml_computer.dart @@ -1,15 +1,12 @@ import 'dart:async'; -import 'dart:typed_data' show Uint8List; import "package:logging/logging.dart"; import "package:ml_linalg/linalg.dart"; -import "package:photos/models/ml/face/box.dart"; import "package:photos/models/ml/vector.dart"; import "package:photos/services/isolate_functions.dart"; import "package:photos/services/isolate_service.dart"; import "package:photos/services/machine_learning/semantic_search/clip/clip_text_encoder.dart"; import "package:photos/services/remote_assets_service.dart"; -import "package:photos/utils/image_ml_util.dart"; import "package:synchronized/synchronized.dart"; class MLComputer extends SuperIsolate { @@ -33,24 +30,6 @@ class MLComputer extends SuperIsolate { static final MLComputer instance = MLComputer._privateConstructor(); factory MLComputer() => instance; - /// Generates face thumbnails for all [faceBoxes] in [imageData]. - /// - /// Uses [generateFaceThumbnailsUsingCanvas] inside the isolate. - Future> generateFaceThumbnails( - String imagePath, - List faceBoxes, - ) async { - final List> faceBoxesJson = - faceBoxes.map((box) => box.toJson()).toList(); - return await runInIsolate( - IsolateOperation.generateFaceThumbnails, - { - 'imagePath': imagePath, - 'faceBoxesList': faceBoxesJson, - }, - ).then((value) => value.cast()); - } - Future> runClipText(String query) async { try { await _ensureLoadedClipTextModel(); @@ -66,9 +45,13 @@ class MLComputer extends SuperIsolate { } } - Future> compareEmbeddings(List embeddings, Vector otherEmbedding) async { + Future> compareEmbeddings( + List embeddings, + Vector otherEmbedding, + ) async { try { - final fileIdToSimilarity = await runInIsolate(IsolateOperation.compareEmbeddings, { + final fileIdToSimilarity = + await runInIsolate(IsolateOperation.compareEmbeddings, { "embeddings": embeddings.map((e) => e.toJsonString()).toList(), "otherEmbedding": otherEmbedding.toList(), }) as Map; diff --git a/mobile/lib/utils/face/face_box_crop.dart b/mobile/lib/utils/face/face_box_crop.dart index c75be4a533..dd31884ea1 100644 --- a/mobile/lib/utils/face/face_box_crop.dart +++ b/mobile/lib/utils/face/face_box_crop.dart @@ -10,7 +10,7 @@ import "package:photos/models/file/file.dart"; import "package:photos/models/file/file_type.dart"; import "package:photos/models/ml/face/box.dart"; import "package:photos/models/ml/face/face.dart"; -import "package:photos/services/machine_learning/ml_computer.dart"; +import "package:photos/services/machine_learning/face_thumbnail_generator.dart"; import "package:photos/utils/file_util.dart"; import "package:photos/utils/thumbnail_util.dart"; import "package:pool/pool.dart"; @@ -210,7 +210,7 @@ Future?> _getFaceCrops( faceBoxes.add(e.value); } final List faceCrop = - await MLComputer.instance.generateFaceThumbnails( + await FaceThumbnailGenerator.instance.generateFaceThumbnails( // await generateJpgFaceThumbnails( imagePath, faceBoxes, From a974a95fb24b00b685ef89aea2f6ab7d90230394 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Thu, 27 Mar 2025 16:06:36 +0530 Subject: [PATCH 02/12] Auto dispose face thumbnail generator isolate --- .../lib/services/machine_learning/face_thumbnail_generator.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobile/lib/services/machine_learning/face_thumbnail_generator.dart b/mobile/lib/services/machine_learning/face_thumbnail_generator.dart index 8970256a08..d98fe63793 100644 --- a/mobile/lib/services/machine_learning/face_thumbnail_generator.dart +++ b/mobile/lib/services/machine_learning/face_thumbnail_generator.dart @@ -19,7 +19,7 @@ class FaceThumbnailGenerator extends SuperIsolate { String get isolateName => "FaceThumbnailGenerator"; @override - bool get shouldAutomaticDispose => false; + bool get shouldAutomaticDispose => true; // Singleton pattern FaceThumbnailGenerator._privateConstructor(); From ac43ecf45b5f74b2ef78e3775ffd0d4ad29d0459 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Thu, 27 Mar 2025 16:07:28 +0530 Subject: [PATCH 03/12] Make MLComputer a regular isolate --- mobile/lib/services/machine_learning/ml_computer.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobile/lib/services/machine_learning/ml_computer.dart b/mobile/lib/services/machine_learning/ml_computer.dart index 2d34856a67..4749396db8 100644 --- a/mobile/lib/services/machine_learning/ml_computer.dart +++ b/mobile/lib/services/machine_learning/ml_computer.dart @@ -17,7 +17,7 @@ class MLComputer extends SuperIsolate { final _initModelLock = Lock(); @override - bool get isDartUiIsolate => true; + bool get isDartUiIsolate => false; @override String get isolateName => "MLComputerIsolate"; From 168a4936f86bbe892c88e18321bc8ede8e4f1c32 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Fri, 28 Mar 2025 11:42:06 +0530 Subject: [PATCH 04/12] cache clip embeddings inside MLComputer isolate --- mobile/lib/services/isolate_functions.dart | 63 +++++++++++---- mobile/lib/services/isolate_service.dart | 15 ++++ .../machine_learning/ml_computer.dart | 77 ++++++++++++++----- .../machine_learning/ml_constants.dart | 1 + .../semantic_search/query_result.dart | 6 ++ .../semantic_search_service.dart | 58 +++++++------- .../lib/services/smart_memories_service.dart | 4 +- 7 files changed, 158 insertions(+), 66 deletions(-) create mode 100644 mobile/lib/services/machine_learning/ml_constants.dart create mode 100644 mobile/lib/services/machine_learning/semantic_search/query_result.dart diff --git a/mobile/lib/services/isolate_functions.dart b/mobile/lib/services/isolate_functions.dart index c123facf79..ac0817f423 100644 --- a/mobile/lib/services/isolate_functions.dart +++ b/mobile/lib/services/isolate_functions.dart @@ -5,13 +5,17 @@ import "package:ml_linalg/linalg.dart"; import "package:photos/models/ml/face/box.dart"; import "package:photos/models/ml/vector.dart"; import "package:photos/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart"; +import "package:photos/services/machine_learning/ml_constants.dart"; import "package:photos/services/machine_learning/ml_model.dart"; import "package:photos/services/machine_learning/ml_result.dart"; import "package:photos/services/machine_learning/semantic_search/clip/clip_text_encoder.dart"; import "package:photos/services/machine_learning/semantic_search/clip/clip_text_tokenizer.dart"; +import "package:photos/services/machine_learning/semantic_search/query_result.dart"; import "package:photos/utils/image_ml_util.dart"; import "package:photos/utils/ml_util.dart"; +final Map _isolateCache = {}; + enum IsolateOperation { /// [MLIndexingIsolate] analyzeImage, @@ -35,10 +39,14 @@ enum IsolateOperation { runClipText, /// [MLComputer] - compareEmbeddings, + computeBulkSimilarities, /// [FaceClusteringService] linearIncrementalClustering, + + /// Cache operations + setIsolateCache, + clearIsolateCache, } /// WARNING: Only return primitives unless you know the method is only going @@ -121,19 +129,31 @@ Future isolateFunction( return List.from(textEmbedding, growable: false); /// MLComputer - case IsolateOperation.compareEmbeddings: - final List embeddings = - (args['embeddings'] as List) - .map((jsonString) => EmbeddingVector.fromJsonString(jsonString)) - .toList(); - final otherEmbedding = - Vector.fromList(args['otherEmbedding'] as List); - final Map result = {}; - for (final embedding in embeddings) { - final double similarity = embedding.vector.dot(otherEmbedding); - result[embedding.fileID] = similarity; + case IsolateOperation.computeBulkSimilarities: + final imageEmbeddings = + _isolateCache[imageEmbeddingsKey] as List; + final textEmbedding = + args["textQueryToEmbeddingMap"] as Map>; + final minimumSimilarityMap = + args["minimumSimilarityMap"] as Map; + final result = >{}; + for (final MapEntry> entry + in textEmbedding.entries) { + final query = entry.key; + final textVector = Vector.fromList(entry.value); + final minimumSimilarity = minimumSimilarityMap[query]!; + final queryResults = []; + for (final imageEmbedding in imageEmbeddings) { + final similarity = imageEmbedding.vector.dot(textVector); + if (similarity >= minimumSimilarity) { + queryResults.add(QueryResult(imageEmbedding.fileID, similarity)); + } + } + queryResults + .sort((first, second) => second.score.compareTo(first.score)); + result[query] = queryResults; } - return Map.from(result); + return result; /// Cases for MLComputer end here @@ -145,5 +165,22 @@ Future isolateFunction( return result; /// Cases for FaceClusteringService end here + + /// Cases for Caching start here + + /// Caching + case IsolateOperation.setIsolateCache: + final key = args['key'] as String; + final value = args['value']; + _isolateCache[key] = value; + return true; + + /// Caching + case IsolateOperation.clearIsolateCache: + final key = args['key'] as String; + _isolateCache.remove(key); + return true; + + /// Cases for Caching stop here } } diff --git a/mobile/lib/services/isolate_service.dart b/mobile/lib/services/isolate_service.dart index 66d41a13a5..799c821a8f 100644 --- a/mobile/lib/services/isolate_service.dart +++ b/mobile/lib/services/isolate_service.dart @@ -137,6 +137,20 @@ abstract class SuperIsolate { bool postFunctionlockStop(IsolateOperation operation) => false; + Future cacheData(String key, dynamic value) async { + await runInIsolate(IsolateOperation.setIsolateCache, { + 'key': key, + 'value': value, + }); + } + + /// Clears specific data from the isolate's cache + Future clearCachedData(String key) async { + await runInIsolate(IsolateOperation.clearIsolateCache, { + 'key': key, + }); + } + /// Resets a timer that kills the isolate after a certain amount of inactivity. /// /// Should be called after initialization (e.g. inside `init()`) and after every call to isolate (e.g. inside `_runInIsolate()`) @@ -161,6 +175,7 @@ abstract class SuperIsolate { void _disposeIsolate() async { if (!_isIsolateSpawned) return; logger.info('Disposing isolate'); + // await clearAllCachedData(); await onDispose(); _isIsolateSpawned = false; _isolate.kill(); diff --git a/mobile/lib/services/machine_learning/ml_computer.dart b/mobile/lib/services/machine_learning/ml_computer.dart index 4749396db8..edbc16b7ff 100644 --- a/mobile/lib/services/machine_learning/ml_computer.dart +++ b/mobile/lib/services/machine_learning/ml_computer.dart @@ -1,11 +1,12 @@ import 'dart:async'; import "package:logging/logging.dart"; -import "package:ml_linalg/linalg.dart"; import "package:photos/models/ml/vector.dart"; import "package:photos/services/isolate_functions.dart"; import "package:photos/services/isolate_service.dart"; +import "package:photos/services/machine_learning/ml_constants.dart"; import "package:photos/services/machine_learning/semantic_search/clip/clip_text_encoder.dart"; +import "package:photos/services/machine_learning/semantic_search/query_result.dart"; import "package:photos/services/remote_assets_service.dart"; import "package:synchronized/synchronized.dart"; @@ -45,23 +46,6 @@ class MLComputer extends SuperIsolate { } } - Future> compareEmbeddings( - List embeddings, - Vector otherEmbedding, - ) async { - try { - final fileIdToSimilarity = - await runInIsolate(IsolateOperation.compareEmbeddings, { - "embeddings": embeddings.map((e) => e.toJsonString()).toList(), - "otherEmbedding": otherEmbedding.toList(), - }) as Map; - return fileIdToSimilarity; - } catch (e, s) { - _logger.severe("Could not compare embeddings MLComputer isolate", e, s); - rethrow; - } - } - Future _ensureLoadedClipTextModel() async { return _initModelLock.synchronized(() async { if (ClipTextEncoder.instance.isInitialized) return; @@ -97,4 +81,61 @@ class MLComputer extends SuperIsolate { } }); } + + Future>> computeBulkSimilarities( + Map> textQueryToEmbeddingMap, + Map minimumSimilarityMap, + ) async { + try { + final queryToResults = + await runInIsolate(IsolateOperation.computeBulkSimilarities, { + "textQueryToEmbeddingMap": textQueryToEmbeddingMap, + "minimumSimilarityMap": minimumSimilarityMap, + }) as Map>; + return queryToResults; + } catch (e, s) { + _logger.severe( + "Could not bulk compare embeddings inside MLComputer isolate", + e, + s, + ); + rethrow; + } + } + + Future cacheImageEmbeddings(List embeddings) async { + try { + await runInIsolate( + IsolateOperation.setIsolateCache, + { + 'key': imageEmbeddingsKey, + 'value': embeddings, + }, + ) as bool; + _logger.info( + 'Cached ${embeddings.length} image embeddings inside MLComputer', + ); + return; + } catch (e, s) { + _logger.severe("Could not cache image embeddings in MLComputer", e, s); + rethrow; + } + } + + Future clearImageEmbeddingsCache() async { + try { + await runInIsolate( + IsolateOperation.clearIsolateCache, + {'key': imageEmbeddingsKey}, + ) as bool; + return; + } catch (e, s) { + _logger.severe( + "Could not clear image embeddings cache in MLComputer", + e, + s, + ); + rethrow; + } + } } diff --git a/mobile/lib/services/machine_learning/ml_constants.dart b/mobile/lib/services/machine_learning/ml_constants.dart new file mode 100644 index 0000000000..424cf584fc --- /dev/null +++ b/mobile/lib/services/machine_learning/ml_constants.dart @@ -0,0 +1 @@ +const imageEmbeddingsKey = "imageEmbeddings"; \ No newline at end of file diff --git a/mobile/lib/services/machine_learning/semantic_search/query_result.dart b/mobile/lib/services/machine_learning/semantic_search/query_result.dart new file mode 100644 index 0000000000..aa48ab9f85 --- /dev/null +++ b/mobile/lib/services/machine_learning/semantic_search/query_result.dart @@ -0,0 +1,6 @@ +class QueryResult { + final int id; + final double score; + + QueryResult(this.id, this.score); +} \ No newline at end of file diff --git a/mobile/lib/services/machine_learning/semantic_search/semantic_search_service.dart b/mobile/lib/services/machine_learning/semantic_search/semantic_search_service.dart index b5647b0b67..05f0a83fdb 100644 --- a/mobile/lib/services/machine_learning/semantic_search/semantic_search_service.dart +++ b/mobile/lib/services/machine_learning/semantic_search/semantic_search_service.dart @@ -3,7 +3,6 @@ import "dart:developer" as dev show log; import "dart:math" show min; import "dart:ui" show Image; -import "package:computer/computer.dart"; import "package:flutter/foundation.dart"; import "package:logging/logging.dart"; import "package:ml_linalg/vector.dart"; @@ -21,7 +20,9 @@ import "package:photos/services/collections_service.dart"; import "package:photos/services/machine_learning/ml_computer.dart"; import "package:photos/services/machine_learning/ml_result.dart"; import "package:photos/services/machine_learning/semantic_search/clip/clip_image_encoder.dart"; +import "package:photos/services/machine_learning/semantic_search/query_result.dart"; import "package:shared_preferences/shared_preferences.dart"; +import "package:synchronized/synchronized.dart"; class SemanticSearchService { static final _logger = Logger("SemanticSearchService"); @@ -30,7 +31,6 @@ class SemanticSearchService { static final SemanticSearchService instance = SemanticSearchService._privateConstructor(); - static final Computer _computer = Computer.shared(); final LRUMap> _queryEmbeddingCache = LRUMap(20); static const kMinimumSimilarityThreshold = 0.175; late final mlDataDB = MLDataDB.instance; @@ -38,7 +38,9 @@ class SemanticSearchService { bool _hasInitialized = false; bool _textModelIsLoaded = false; - Future>? _cachedImageEmbeddingVectors; + final _cacheLock = Lock(); + bool _imageEmbeddingsAreCached = false; + Future<(String, List)>? _searchScreenRequest; String? _latestPendingQuery; @@ -53,12 +55,15 @@ class SemanticSearchService { _logger.info("init called"); _hasInitialized = true; - // call getClipEmbeddings after 5 seconds + // cache clip embeddings after 5 seconds Future.delayed(const Duration(seconds: 5), () async { - await getClipVectors(); + await _cacheClipVectors(); }); Bus.instance.on().listen((event) { - _cachedImageEmbeddingVectors = null; + if (_imageEmbeddingsAreCached) { + MLComputer.instance.clearImageEmbeddingsCache(); + _imageEmbeddingsAreCached = false; + } }); unawaited(_loadTextModel(delay: true)); @@ -108,14 +113,17 @@ class SemanticSearchService { _logger.info("Indexes cleared"); } - Future> getClipVectors() async { - if (_cachedImageEmbeddingVectors != null) { - return _cachedImageEmbeddingVectors!; - } - _cachedImageEmbeddingVectors ??= mlDataDB.getAllClipVectors(); - _logger.info("read all embeddings from DB"); - - return _cachedImageEmbeddingVectors!; + Future _cacheClipVectors() async { + return _cacheLock.synchronized(() async { + if (_imageEmbeddingsAreCached) { + return; + } + final imageEmbeddings = await mlDataDB.getAllClipVectors(); + _logger.info("read all embeddings from DB"); + await MLComputer.instance.cacheImageEmbeddings(imageEmbeddings); + _imageEmbeddingsAreCached = true; + return; + }); } Future> getMatchingFiles( @@ -257,17 +265,10 @@ class SemanticSearchService { required Map minimumSimilarityMap, }) async { final startTime = DateTime.now(); - final imageEmbeddings = await getClipVectors(); - final Map> queryResults = await _computer - .compute, Map>>( - computeBulkSimilarities, - param: { - "imageEmbeddings": imageEmbeddings, - "textQueryToEmbeddingMap": textQueryToEmbeddingMap, - "minimumSimilarityMap": minimumSimilarityMap, - }, - taskName: "computeBulkSimilarities", - ); + await _cacheClipVectors(); + final Map> queryResults = await MLComputer + .instance + .computeBulkSimilarities(textQueryToEmbeddingMap, minimumSimilarityMap); final endTime = DateTime.now(); _logger.info( "computingSimilarities took for ${textQueryToEmbeddingMap.length} queries " + @@ -336,10 +337,3 @@ Map> computeBulkSimilarities(Map args) { } return result; } - -class QueryResult { - final int id; - final double score; - - QueryResult(this.id, this.score); -} diff --git a/mobile/lib/services/smart_memories_service.dart b/mobile/lib/services/smart_memories_service.dart index 493fbd1a31..5b66526b36 100644 --- a/mobile/lib/services/smart_memories_service.dart +++ b/mobile/lib/services/smart_memories_service.dart @@ -35,7 +35,6 @@ import "package:photos/services/location_service.dart"; import "package:photos/services/machine_learning/face_ml/person/person_service.dart"; import "package:photos/services/machine_learning/ml_computer.dart"; import "package:photos/services/machine_learning/ml_result.dart"; -import "package:photos/services/machine_learning/semantic_search/semantic_search_service.dart"; import "package:photos/services/search_service.dart"; class MemoriesResult { @@ -92,8 +91,7 @@ class SmartMemoriesService { await MLDataDB.instance.getFileIDsToFacesWithoutEmbedding(); _logger.finest('fileIdToFaces has ${fileIdToFaces.length} entries $t'); - final allImageEmbeddings = - await SemanticSearchService.instance.getClipVectors(); + final allImageEmbeddings = await MLDataDB.instance.getAllClipVectors(); _logger.finest( 'allImageEmbeddings has ${allImageEmbeddings.length} entries $t', ); From 1267587ae5ff98d6929d137dbc6b126a861661bc Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Fri, 28 Mar 2025 11:51:34 +0530 Subject: [PATCH 05/12] Fix using plugins in regular isolates --- mobile/lib/services/isolate_service.dart | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/mobile/lib/services/isolate_service.dart b/mobile/lib/services/isolate_service.dart index 799c821a8f..42fe4b8900 100644 --- a/mobile/lib/services/isolate_service.dart +++ b/mobile/lib/services/isolate_service.dart @@ -3,6 +3,7 @@ import 'dart:isolate'; import "package:dart_ui_isolate/dart_ui_isolate.dart"; import "package:flutter/foundation.dart" show kDebugMode; +import "package:flutter/services.dart"; import "package:logging/logging.dart"; import "package:photos/core/error-reporting/isolate_logging.dart"; import "package:photos/models/base/id.dart"; @@ -36,15 +37,22 @@ abstract class SuperIsolate { _receivePort = ReceivePort(); + // Get the root token before spawning the isolate + final rootToken = RootIsolateToken.instance; + if (rootToken == null && !isDartUiIsolate) { + logger.severe('Failed to get RootIsolateToken'); + return; + } + try { _isolate = isDartUiIsolate ? await DartUiIsolate.spawn( _isolateMain, - _receivePort.sendPort, + [_receivePort.sendPort, null], ) : await Isolate.spawn( _isolateMain, - _receivePort.sendPort, + [_receivePort.sendPort, rootToken], debugName: isolateName, ); _mainSendPort = await _receivePort.first as SendPort; @@ -60,12 +68,18 @@ abstract class SuperIsolate { } @pragma('vm:entry-point') - static void _isolateMain(SendPort mainSendPort) async { + static void _isolateMain(List args) async { + final SendPort mainSendPort = args[0] as SendPort; + final RootIsolateToken? rootToken = args[1] as RootIsolateToken?; + Logger.root.level = kDebugMode ? Level.ALL : Level.INFO; final IsolateLogger isolateLogger = IsolateLogger(); Logger.root.onRecord.listen(isolateLogger.onLogRecordInIsolate); final receivePort = ReceivePort(); mainSendPort.send(receivePort.sendPort); + if (rootToken != null) { + BackgroundIsolateBinaryMessenger.ensureInitialized(rootToken); + } receivePort.listen((message) async { final taskID = message[0] as String; From bd0818ec7db54a90e6a6b3d30579f0d46a106c49 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Fri, 28 Mar 2025 11:57:21 +0530 Subject: [PATCH 06/12] Reduce time to isolate disposal --- mobile/lib/services/isolate_service.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobile/lib/services/isolate_service.dart b/mobile/lib/services/isolate_service.dart index 42fe4b8900..24f9f63101 100644 --- a/mobile/lib/services/isolate_service.dart +++ b/mobile/lib/services/isolate_service.dart @@ -14,7 +14,7 @@ abstract class SuperIsolate { Logger get logger; Timer? _inactivityTimer; - final Duration _inactivityDuration = const Duration(seconds: 120); + final Duration _inactivityDuration = const Duration(seconds: 60); int _activeTasks = 0; final _initIsolateLock = Lock(); From 939d1a5d4097e0efeadc76593a57353c711ece79 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Fri, 28 Mar 2025 12:47:22 +0530 Subject: [PATCH 07/12] properly clear isolate cache --- mobile/lib/services/isolate_functions.dart | 6 ++++++ mobile/lib/services/isolate_service.dart | 6 +++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/mobile/lib/services/isolate_functions.dart b/mobile/lib/services/isolate_functions.dart index ac0817f423..929790c588 100644 --- a/mobile/lib/services/isolate_functions.dart +++ b/mobile/lib/services/isolate_functions.dart @@ -47,6 +47,7 @@ enum IsolateOperation { /// Cache operations setIsolateCache, clearIsolateCache, + clearAllIsolateCache, } /// WARNING: Only return primitives unless you know the method is only going @@ -181,6 +182,11 @@ Future isolateFunction( _isolateCache.remove(key); return true; + /// Caching + case IsolateOperation.clearAllIsolateCache: + _isolateCache.clear(); + return true; + /// Cases for Caching stop here } } diff --git a/mobile/lib/services/isolate_service.dart b/mobile/lib/services/isolate_service.dart index 24f9f63101..4a91c64473 100644 --- a/mobile/lib/services/isolate_service.dart +++ b/mobile/lib/services/isolate_service.dart @@ -165,6 +165,10 @@ abstract class SuperIsolate { }); } + Future clearAllCachedData() async { + await runInIsolate(IsolateOperation.clearAllIsolateCache, {}); + } + /// Resets a timer that kills the isolate after a certain amount of inactivity. /// /// Should be called after initialization (e.g. inside `init()`) and after every call to isolate (e.g. inside `_runInIsolate()`) @@ -189,7 +193,7 @@ abstract class SuperIsolate { void _disposeIsolate() async { if (!_isIsolateSpawned) return; logger.info('Disposing isolate'); - // await clearAllCachedData(); + await clearAllCachedData(); await onDispose(); _isIsolateSpawned = false; _isolate.kill(); From 5ffd51382675a7376c155a29f8e0786fdb3b0af1 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Fri, 28 Mar 2025 13:39:55 +0530 Subject: [PATCH 08/12] Speed up embedding db call --- mobile/lib/db/ml/db.dart | 34 +++++++++++++++------------------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/mobile/lib/db/ml/db.dart b/mobile/lib/db/ml/db.dart index 2ca30ab247..fde278a9f2 100644 --- a/mobile/lib/db/ml/db.dart +++ b/mobile/lib/db/ml/db.dart @@ -1166,8 +1166,21 @@ class MLDataDB with SqlDbBase implements IMLDataDB { Future> getAllClipVectors() async { Logger("ClipDB").info("reading all embeddings from DB"); final db = await instance.asyncDB; - final results = await db.getAll('SELECT * FROM $clipTable'); - return _convertToVectors(results); + final results = await db + .getAll('SELECT $fileIDColumn, $embeddingColumn FROM $clipTable'); + + // Convert rows to vectors + final List embeddings = []; + for (final result in results) { + // Convert to EmbeddingVector + final embedding = EmbeddingVector( + fileID: result[fileIDColumn], + embedding: Float32List.view(result[embeddingColumn].buffer), + ); + if (embedding.isEmpty) continue; + embeddings.add(embedding); + } + return embeddings; } // Get indexed FileIDs @@ -1229,23 +1242,6 @@ class MLDataDB with SqlDbBase implements IMLDataDB { Bus.instance.fire(EmbeddingUpdatedEvent()); } - List _convertToVectors(List> results) { - final List embeddings = []; - for (final result in results) { - final embedding = _getVectorFromRow(result); - if (embedding.isEmpty) continue; - embeddings.add(embedding); - } - return embeddings; - } - - EmbeddingVector _getVectorFromRow(Map row) { - final fileID = row[fileIDColumn] as int; - final bytes = row[embeddingColumn] as Uint8List; - final list = Float32List.view(bytes.buffer); - return EmbeddingVector(fileID: fileID, embedding: list); - } - List _getRowFromEmbedding(ClipEmbedding embedding) { return [ embedding.fileID, From 3457cc13698ddf6e61e7d4debc67c90589ef8f43 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Fri, 28 Mar 2025 13:40:58 +0530 Subject: [PATCH 09/12] log embeddings retrieval time --- .../semantic_search/semantic_search_service.dart | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mobile/lib/services/machine_learning/semantic_search/semantic_search_service.dart b/mobile/lib/services/machine_learning/semantic_search/semantic_search_service.dart index 05f0a83fdb..c4ecfa8362 100644 --- a/mobile/lib/services/machine_learning/semantic_search/semantic_search_service.dart +++ b/mobile/lib/services/machine_learning/semantic_search/semantic_search_service.dart @@ -118,8 +118,11 @@ class SemanticSearchService { if (_imageEmbeddingsAreCached) { return; } + final now = DateTime.now(); final imageEmbeddings = await mlDataDB.getAllClipVectors(); - _logger.info("read all embeddings from DB"); + _logger.info( + "read all ${imageEmbeddings.length} embeddings from DB in ${DateTime.now().difference(now).inMilliseconds} ms", + ); await MLComputer.instance.cacheImageEmbeddings(imageEmbeddings); _imageEmbeddingsAreCached = true; return; From bc65e2c256a16556904172017ce35e579f3697d2 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Fri, 28 Mar 2025 13:41:16 +0530 Subject: [PATCH 10/12] don't cache embeddings on startup --- .../semantic_search/semantic_search_service.dart | 4 ---- 1 file changed, 4 deletions(-) diff --git a/mobile/lib/services/machine_learning/semantic_search/semantic_search_service.dart b/mobile/lib/services/machine_learning/semantic_search/semantic_search_service.dart index c4ecfa8362..1952c9803c 100644 --- a/mobile/lib/services/machine_learning/semantic_search/semantic_search_service.dart +++ b/mobile/lib/services/machine_learning/semantic_search/semantic_search_service.dart @@ -55,10 +55,6 @@ class SemanticSearchService { _logger.info("init called"); _hasInitialized = true; - // cache clip embeddings after 5 seconds - Future.delayed(const Duration(seconds: 5), () async { - await _cacheClipVectors(); - }); Bus.instance.on().listen((event) { if (_imageEmbeddingsAreCached) { MLComputer.instance.clearImageEmbeddingsCache(); From a682fb4ecea213fb1b391cf7a8684fc427d91bee Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Fri, 28 Mar 2025 13:43:40 +0530 Subject: [PATCH 11/12] cleanup --- .../semantic_search_service.dart | 42 ------------------- 1 file changed, 42 deletions(-) diff --git a/mobile/lib/services/machine_learning/semantic_search/semantic_search_service.dart b/mobile/lib/services/machine_learning/semantic_search/semantic_search_service.dart index 1952c9803c..aa745e1af9 100644 --- a/mobile/lib/services/machine_learning/semantic_search/semantic_search_service.dart +++ b/mobile/lib/services/machine_learning/semantic_search/semantic_search_service.dart @@ -5,7 +5,6 @@ import "dart:ui" show Image; import "package:flutter/foundation.dart"; import "package:logging/logging.dart"; -import "package:ml_linalg/vector.dart"; import "package:photos/core/cache/lru_map.dart"; import "package:photos/core/event_bus.dart"; import "package:photos/db/files_db.dart"; @@ -14,7 +13,6 @@ import 'package:photos/events/embedding_updated_event.dart'; import "package:photos/models/file/file.dart"; import "package:photos/models/ml/clip.dart"; import "package:photos/models/ml/ml_versions.dart"; -import "package:photos/models/ml/vector.dart"; import "package:photos/service_locator.dart"; import "package:photos/services/collections_service.dart"; import "package:photos/services/machine_learning/ml_computer.dart"; @@ -296,43 +294,3 @@ class SemanticSearchService { return clipResult; } } - -Map> computeBulkSimilarities(Map args) { - final imageEmbeddings = args["imageEmbeddings"] as List; - final textEmbedding = - args["textQueryToEmbeddingMap"] as Map>; - final minimumSimilarityMap = - args["minimumSimilarityMap"] as Map; - final result = >{}; - for (final MapEntry> entry in textEmbedding.entries) { - final query = entry.key; - final textVector = Vector.fromList(entry.value); - final minimumSimilarity = minimumSimilarityMap[query]!; - final queryResults = []; - if (!kDebugMode) { - for (final imageEmbedding in imageEmbeddings) { - final similarity = imageEmbedding.vector.dot(textVector); - if (similarity >= minimumSimilarity) { - queryResults.add(QueryResult(imageEmbedding.fileID, similarity)); - } - } - } else { - double bestScore = 0.0; - for (final imageEmbedding in imageEmbeddings) { - final similarity = imageEmbedding.vector.dot(textVector); - if (similarity >= minimumSimilarity) { - queryResults.add(QueryResult(imageEmbedding.fileID, similarity)); - } - if (similarity > bestScore) { - bestScore = similarity; - } - } - if (kDebugMode && queryResults.isEmpty) { - dev.log("No results found for query with best score: $bestScore"); - } - } - queryResults.sort((first, second) => second.score.compareTo(first.score)); - result[query] = queryResults; - } - return result; -} From d11ff14ecd32fcfa98a6ad8c1924f4bca3c081ee Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Fri, 28 Mar 2025 13:55:23 +0530 Subject: [PATCH 12/12] Remove embeddings cache after inactivity --- .../semantic_search_service.dart | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/mobile/lib/services/machine_learning/semantic_search/semantic_search_service.dart b/mobile/lib/services/machine_learning/semantic_search/semantic_search_service.dart index aa745e1af9..cf3930a782 100644 --- a/mobile/lib/services/machine_learning/semantic_search/semantic_search_service.dart +++ b/mobile/lib/services/machine_learning/semantic_search/semantic_search_service.dart @@ -1,4 +1,4 @@ -import "dart:async" show unawaited; +import "dart:async" show Timer, unawaited; import "dart:developer" as dev show log; import "dart:math" show min; import "dart:ui" show Image; @@ -38,6 +38,8 @@ class SemanticSearchService { final _cacheLock = Lock(); bool _imageEmbeddingsAreCached = false; + Timer? _embeddingsCacheTimer; + final Duration _embeddingsCacheDuration = const Duration(seconds: 60); Future<(String, List)>? _searchScreenRequest; String? _latestPendingQuery; @@ -109,6 +111,7 @@ class SemanticSearchService { Future _cacheClipVectors() async { return _cacheLock.synchronized(() async { + _resetInactivityTimer(); if (_imageEmbeddingsAreCached) { return; } @@ -276,6 +279,19 @@ class SemanticSearchService { return queryResults; } + void _resetInactivityTimer() { + _embeddingsCacheTimer?.cancel(); + _embeddingsCacheTimer = Timer(_embeddingsCacheDuration, () { + _logger.info( + 'Embeddings cache is unused for ${_embeddingsCacheDuration.inSeconds} seconds. Removing cache.', + ); + if (_imageEmbeddingsAreCached) { + MLComputer.instance.clearImageEmbeddingsCache(); + _imageEmbeddingsAreCached = false; + } + }); + } + static Future runClipImage( int enteFileID, Image image,