diff --git a/mobile/lib/db/ml/base.dart b/mobile/lib/db/ml/base.dart new file mode 100644 index 0000000000..174ee00168 --- /dev/null +++ b/mobile/lib/db/ml/base.dart @@ -0,0 +1,106 @@ +import "dart:typed_data"; + +import "package:photos/models/ml/face/face.dart"; +import "package:photos/services/machine_learning/face_ml/face_clustering/face_db_info_for_clustering.dart"; + +abstract class IMLDataDB { + Future bulkInsertFaces(List faces); + Future updateFaceIdToClusterId(Map faceIDToClusterID); + Future> faceIndexedFileIds({int minimumMlVersion}); + Future getFaceIndexedFileCount({int minimumMlVersion}); + Future> clusterIdToFaceCount(); + Future> getPersonIgnoredClusters(String personID); + Future> getPersonClusterIDs(String personID); + Future> getPersonsClusterIDs(List personID); + Future clearTable(); + Future> getFaceEmbeddingsForCluster( + String clusterID, { + int? limit, + }); + Future>> getFaceEmbeddingsForClusters( + Iterable clusterIDs, { + int? limit, + }); + Future getCoverFaceForPerson({ + required int recentFileID, + String? personID, + String? avatarFaceId, + String? clusterID, + }); + Future?> getFacesForGivenFileID(int fileUploadID); + Future>> getClusterToFaceIDs( + Set clusterIDs, + ); + Future getClusterIDForFaceID(String faceID); + Future>> getAllClusterIdToFaceIDs(); + Future> getFaceIDsForCluster(String clusterID); + Future>>> getPersonToClusterIdToFaceIds(); + Future>> getClusterIdToFaceIdsForPerson( + String personID, + ); + Future> getFaceIDsForPerson(String personID); + Future> getBlurValuesForCluster(String clusterID); + Future> getFaceIdsToClusterIds(Iterable faceIds); + Future>> getFileIdToClusterIds(); + Future forceUpdateClusterIds(Map faceIDToClusterID); + Future removeFaceIdToClusterId(Map faceIDToClusterID); + Future removePerson(String personID); + Future> getFaceInfoForClustering({ + int maxFaces, + int offset, + int batchSize, + }); + Future> getFaceEmbeddingMapForFaces( + Iterable faceIDs, + ); + Future getTotalFaceCount(); + Future getErroredFaceCount(); + Future> getErroredFileIDs(); + Future deleteFaceIndexForFiles(List fileIDs); + Future getClusteredOrFacelessFileCount(); + Future getClusteredToIndexableFilesRatio(); + Future getUnclusteredFaceCount(); + Future assignClusterToPerson({ + required String personID, + required String clusterID, + }); + Future bulkAssignClusterToPersonID( + Map clusterToPersonID, + ); + Future captureNotPersonFeedback({ + required String personID, + required String clusterID, + }); + Future bulkCaptureNotPersonFeedback( + Map clusterToPersonID, + ); + Future removeNotPersonFeedback({ + required String personID, + required String clusterID, + }); + Future removeClusterToPerson({ + required String personID, + required String clusterID, + }); + Future>> getFileIdToClusterIDSet(String personID); + Future>> getFileIdToClusterIDSetForCluster( + Set clusterIDs, + ); + Future clusterSummaryUpdate(Map summary); + Future deleteClusterSummary(String clusterID); + Future> getAllClusterSummary([ + int? minClusterSize, + ]); + Future> getClusterToClusterSummary( + Iterable clusterIDs, + ); + Future> getClusterIDToPersonID(); + Future dropClustersAndPersonTable({bool faces}); + Future dropFacesFeedbackTables(); + Future> getFileIDsOfPersonID(String personID); + Future> getFileIDsOfClusterID(String clusterID); + Future> getAllFileIDsOfFaceIDsNotInAnyCluster(); + Future> getAllFilesAssociatedWithAllClusters({ + List? exceptClusters, + }); +} diff --git a/mobile/lib/db/ml/db.dart b/mobile/lib/db/ml/db.dart index 90f31d83ae..ddcdf09c5c 100644 --- a/mobile/lib/db/ml/db.dart +++ b/mobile/lib/db/ml/db.dart @@ -6,6 +6,7 @@ import "package:flutter/foundation.dart"; import 'package:logging/logging.dart'; import 'package:path/path.dart' show join; import 'package:path_provider/path_provider.dart'; +import "package:photos/db/ml/base.dart"; import 'package:photos/db/ml/db_fields.dart'; import "package:photos/db/ml/db_model_mappers.dart"; import "package:photos/extensions/stop_watch.dart"; @@ -28,7 +29,7 @@ import 'package:sqlite_async/sqlite_async.dart'; /// /// [clipTable] - Stores the embeddings of the CLIP model /// [fileDataTable] - Stores data about the files that are already processed by the ML models -class MLDataDB { +class MLDataDB extends IMLDataDB { static final Logger _logger = Logger("MLDataDB"); static const _databaseName = "ente.ml.db"; @@ -108,6 +109,7 @@ class MLDataDB { // bulkInsertFaces inserts the faces in the database in batches of 1000. // This is done to avoid the error "too many SQL variables" when inserting // a large number of faces. + @override Future bulkInsertFaces(List faces) async { final db = await instance.asyncDB; const batchSize = 500; @@ -143,6 +145,7 @@ class MLDataDB { } } + @override Future updateFaceIdToClusterId( Map faceIDToClusterID, ) async { @@ -166,6 +169,7 @@ class MLDataDB { } /// Returns a map of fileID to the indexed ML version + @override Future> faceIndexedFileIds({ int minimumMlVersion = faceMlVersion, }) async { @@ -183,6 +187,7 @@ class MLDataDB { return result; } + @override Future getFaceIndexedFileCount({ int minimumMlVersion = faceMlVersion, }) async { @@ -193,6 +198,7 @@ class MLDataDB { return maps.first['count'] as int; } + @override Future> clusterIdToFaceCount() async { final db = await instance.asyncDB; final List> maps = await db.getAll( @@ -205,6 +211,7 @@ class MLDataDB { return result; } + @override Future> getPersonIgnoredClusters(String personID) async { final db = await instance.asyncDB; // find out clusterIds that are assigned to other persons using the clusters table @@ -223,6 +230,7 @@ class MLDataDB { return ignoredClusterIDs.union(rejectClusterIDs); } + @override Future> getPersonClusterIDs(String personID) async { final db = await instance.asyncDB; final List> maps = await db.getAll( @@ -232,6 +240,7 @@ class MLDataDB { return maps.map((e) => e[clusterIDColumn] as String).toSet(); } + @override Future> getPersonsClusterIDs(List personID) async { final db = await instance.asyncDB; final inParam = personID.map((e) => "'$e'").join(','); @@ -241,6 +250,7 @@ class MLDataDB { return maps.map((e) => e[clusterIDColumn] as String).toSet(); } + @override Future clearTable() async { final db = await instance.asyncDB; @@ -253,6 +263,7 @@ class MLDataDB { await db.execute(deleteFileDataTable); } + @override Future> getFaceEmbeddingsForCluster( String clusterID, { int? limit, @@ -265,6 +276,7 @@ class MLDataDB { return maps.map((e) => e[embeddingColumn] as Uint8List); } + @override Future>> getFaceEmbeddingsForClusters( Iterable clusterIDs, { int? limit, @@ -297,6 +309,7 @@ class MLDataDB { return result; } + @override Future getCoverFaceForPerson({ required int recentFileID, String? personID, @@ -384,6 +397,7 @@ class MLDataDB { return null; } + @override Future?> getFacesForGivenFileID(int fileUploadID) async { final db = await instance.asyncDB; const String query = ''' @@ -400,6 +414,7 @@ class MLDataDB { return maps.map((e) => mapRowToFace(e)).toList(); } + @override Future>> getClusterToFaceIDs( Set clusterIDs, ) async { @@ -423,6 +438,7 @@ class MLDataDB { return result; } + @override Future getClusterIDForFaceID(String faceID) async { final db = await instance.asyncDB; final List> maps = await db.getAll( @@ -435,6 +451,7 @@ class MLDataDB { return maps.first[clusterIDColumn] as String; } + @override Future>> getAllClusterIdToFaceIDs() async { final db = await instance.asyncDB; final Map> result = {}; @@ -449,6 +466,7 @@ class MLDataDB { return result; } + @override Future> getFaceIDsForCluster(String clusterID) async { final db = await instance.asyncDB; final List> maps = await db.getAll( @@ -460,6 +478,7 @@ class MLDataDB { } // Get Map of personID to Map of clusterID to faceIDs + @override Future>>> getPersonToClusterIdToFaceIds() async { final db = await instance.asyncDB; @@ -480,6 +499,7 @@ class MLDataDB { return result; } + @override Future>> getClusterIdToFaceIdsForPerson( String personID, ) async { @@ -499,6 +519,7 @@ class MLDataDB { return result; } + @override Future> getFaceIDsForPerson(String personID) async { final db = await instance.asyncDB; final faceIdsResult = await db.getAll( @@ -510,6 +531,7 @@ class MLDataDB { return faceIdsResult.map((e) => e[faceIDColumn] as String).toSet(); } + @override Future> getBlurValuesForCluster(String clusterID) async { final db = await instance.asyncDB; const String query = ''' @@ -530,6 +552,7 @@ class MLDataDB { return maps.map((e) => e[faceBlur] as double).toSet(); } + @override Future> getFaceIdsToClusterIds( Iterable faceIds, ) async { @@ -544,6 +567,7 @@ class MLDataDB { return result; } + @override Future>> getFileIdToClusterIds() async { final Map> result = {}; final db = await instance.asyncDB; @@ -560,6 +584,7 @@ class MLDataDB { return result; } + @override Future forceUpdateClusterIds( Map faceIDToClusterID, ) async { @@ -575,6 +600,7 @@ class MLDataDB { await db.executeBatch(sql, parameterSets); } + @override Future removeFaceIdToClusterId( Map faceIDToClusterID, ) async { @@ -588,6 +614,7 @@ class MLDataDB { await db.executeBatch(sql, parameterSets); } + @override Future removePerson(String personID) async { final db = await instance.asyncDB; @@ -613,6 +640,7 @@ class MLDataDB { }); } + @override Future> getFaceInfoForClustering({ int maxFaces = 20000, int offset = 0, @@ -668,6 +696,7 @@ class MLDataDB { } } + @override Future> getFaceEmbeddingMapForFaces( Iterable faceIDs, ) async { @@ -706,6 +735,7 @@ class MLDataDB { return result; } + @override Future getTotalFaceCount() async { final db = await instance.asyncDB; final List> maps = await db.getAll( @@ -714,6 +744,7 @@ class MLDataDB { return maps.first['count'] as int; } + @override Future getErroredFaceCount() async { final db = await instance.asyncDB; final List> maps = await db.getAll( @@ -722,6 +753,7 @@ class MLDataDB { return maps.first['count'] as int; } + @override Future> getErroredFileIDs() async { final db = await instance.asyncDB; final List> maps = await db.getAll( @@ -730,6 +762,7 @@ class MLDataDB { return maps.map((e) => e[fileIDColumn] as int).toSet(); } + @override Future deleteFaceIndexForFiles(List fileIDs) async { final db = await instance.asyncDB; final String sql = ''' @@ -738,6 +771,7 @@ class MLDataDB { await db.execute(sql); } + @override Future getClusteredOrFacelessFileCount() async { final db = await instance.asyncDB; final List> clustered = await db.getAll( @@ -768,6 +802,7 @@ class MLDataDB { return clusteredFileIDs.length + trulyFacelessFiles.length; } + @override Future getClusteredToIndexableFilesRatio() async { final int indexableFiles = (await getIndexableFileIDs()).length; final int clusteredFiles = await getClusteredOrFacelessFileCount(); @@ -775,6 +810,7 @@ class MLDataDB { return clusteredFiles / indexableFiles; } + @override Future getUnclusteredFaceCount() async { final db = await instance.asyncDB; const String query = ''' @@ -791,6 +827,7 @@ class MLDataDB { /// WARNING: Only use this method if the person has just been created. /// Otherwise, use [ClusterFeedbackService.instance.addClusterToExistingPerson] instead. + @override Future assignClusterToPerson({ required String personID, required String clusterID, @@ -803,6 +840,7 @@ class MLDataDB { await db.execute(sql, [personID, clusterID]); } + @override Future bulkAssignClusterToPersonID( Map clusterToPersonID, ) async { @@ -816,6 +854,7 @@ class MLDataDB { await db.executeBatch(sql, parameterSets); } + @override Future captureNotPersonFeedback({ required String personID, required String clusterID, @@ -828,6 +867,7 @@ class MLDataDB { await db.execute(sql, [personID, clusterID]); } + @override Future bulkCaptureNotPersonFeedback( Map clusterToPersonID, ) async { @@ -842,6 +882,7 @@ class MLDataDB { await db.executeBatch(sql, parameterSets); } + @override Future removeNotPersonFeedback({ required String personID, required String clusterID, @@ -854,6 +895,7 @@ class MLDataDB { await db.execute(sql, [personID, clusterID]); } + @override Future removeClusterToPerson({ required String personID, required String clusterID, @@ -867,6 +909,7 @@ class MLDataDB { } // for a given personID, return a map of clusterID to fileIDs using join query + @override Future>> getFileIdToClusterIDSet(String personID) { final db = instance.asyncDB; return db.then((db) async { @@ -888,6 +931,7 @@ class MLDataDB { }); } + @override Future>> getFileIdToClusterIDSetForCluster( Set clusterIDs, ) { @@ -912,6 +956,7 @@ class MLDataDB { }); } + @override Future clusterSummaryUpdate( Map summary, ) async { @@ -937,6 +982,7 @@ class MLDataDB { await db.executeBatch(sql, parameterSets); } + @override Future deleteClusterSummary(String clusterID) async { final db = await instance.asyncDB; const String sqlDelete = @@ -945,6 +991,7 @@ class MLDataDB { } /// Returns a map of clusterID to (avg embedding, count) + @override Future> getAllClusterSummary([ int? minClusterSize, ]) async { @@ -962,6 +1009,7 @@ class MLDataDB { return result; } + @override Future> getClusterToClusterSummary( Iterable clusterIDs, ) async { @@ -982,6 +1030,7 @@ class MLDataDB { return result; } + @override Future> getClusterIDToPersonID() async { final db = await instance.asyncDB; final List> maps = await db.getAll( @@ -995,6 +1044,7 @@ class MLDataDB { } /// WARNING: This will delete ALL data in the database! Only use this for debug/testing purposes! + @override Future dropClustersAndPersonTable({bool faces = false}) async { try { final db = await instance.asyncDB; @@ -1022,6 +1072,7 @@ class MLDataDB { } /// WARNING: This will delete ALL data in the tables! Only use this for debug/testing purposes! + @override Future dropFacesFeedbackTables() async { try { final db = await instance.asyncDB; @@ -1038,6 +1089,7 @@ class MLDataDB { } } + @override Future> getFileIDsOfPersonID(String personID) async { final db = await instance.asyncDB; final result = await db.getAll( @@ -1054,6 +1106,7 @@ class MLDataDB { return [for (final row in result) row[fileIDColumn]]; } + @override Future> getFileIDsOfClusterID(String clusterID) async { final db = await instance.asyncDB; final result = await db.getAll( @@ -1069,6 +1122,7 @@ class MLDataDB { return [for (final row in result) row[fileIDColumn]]; } + @override Future> getAllFileIDsOfFaceIDsNotInAnyCluster() async { final db = await instance.asyncDB; final result = await db.getAll( @@ -1082,6 +1136,7 @@ class MLDataDB { return {for (final row in result) row[fileIDColumn]}; } + @override Future> getAllFilesAssociatedWithAllClusters({ List? exceptClusters, }) async { diff --git a/mobile/pubspec.lock b/mobile/pubspec.lock index 0cba96166c..40dcb0f65e 100644 --- a/mobile/pubspec.lock +++ b/mobile/pubspec.lock @@ -5,10 +5,10 @@ packages: dependency: transitive description: name: _fe_analyzer_shared - sha256: "16e298750b6d0af7ce8a3ba7c18c69c3785d11b15ec83f6dcd0ad2a0009b3cab" + sha256: f256b0c0ba6c7577c15e2e4e114755640a875e885099367bf6e012b19314c834 url: "https://pub.dev" source: hosted - version: "76.0.0" + version: "72.0.0" _flutterfire_internals: dependency: transitive description: @@ -21,7 +21,7 @@ packages: dependency: transitive description: dart source: sdk - version: "0.3.3" + version: "0.3.2" adaptive_theme: dependency: "direct main" description: @@ -34,10 +34,10 @@ packages: dependency: transitive description: name: analyzer - sha256: "1f14db053a8c23e260789e9b0980fa27f2680dd640932cae5e1137cce0e46e1e" + sha256: b652861553cd3990d8ed361f7979dc6d7053a9ac8843fa73820ab68ce5410139 url: "https://pub.dev" source: hosted - version: "6.11.0" + version: "6.7.0" android_intent_plus: dependency: "direct main" description: @@ -284,10 +284,10 @@ packages: dependency: "direct main" description: name: collection - sha256: a1ace0a119f20aabc852d165077c036cd864315bd99b7eaa10a60100341941bf + sha256: ee67cb0715911d28db6bf4af1026078bd6f0128b07a5f66fb2ed94ec6783c09a url: "https://pub.dev" source: hosted - version: "1.19.0" + version: "1.18.0" computer: dependency: "direct main" description: @@ -1359,18 +1359,18 @@ packages: dependency: transitive description: name: leak_tracker - sha256: "7bb2830ebd849694d1ec25bf1f44582d6ac531a57a365a803a6034ff751d2d06" + sha256: "3f87a60e8c63aecc975dda1ceedbc8f24de75f09e4856ea27daf8958f2f0ce05" url: "https://pub.dev" source: hosted - version: "10.0.7" + version: "10.0.5" leak_tracker_flutter_testing: dependency: transitive description: name: leak_tracker_flutter_testing - sha256: "9491a714cca3667b60b5c420da8217e6de0d1ba7a5ec322fab01758f6998f379" + sha256: "932549fb305594d82d7183ecd9fa93463e9914e1b67cacc34bc40906594a1806" url: "https://pub.dev" source: hosted - version: "3.0.8" + version: "3.0.5" leak_tracker_testing: dependency: transitive description: @@ -1487,10 +1487,10 @@ packages: dependency: transitive description: name: macros - sha256: "1d9e801cd66f7ea3663c45fc708450db1fa57f988142c64289142c9b7ee80656" + sha256: "0acaed5d6b7eab89f63350bccd82119e6c602df0f391260d0e32b5e23db79536" url: "https://pub.dev" source: hosted - version: "0.1.3-main.0" + version: "0.1.2-main.4" maps_launcher: dependency: "direct main" description: @@ -2293,7 +2293,7 @@ packages: dependency: transitive description: flutter source: sdk - version: "0.0.0" + version: "0.0.99" source_gen: dependency: transitive description: @@ -2418,10 +2418,10 @@ packages: dependency: transitive description: name: stack_trace - sha256: "9f47fd3630d76be3ab26f0ee06d213679aa425996925ff3feffdec504931c377" + sha256: "73713990125a6d93122541237550ee3352a2d84baad52d375a4cad2eb9b7ce0b" url: "https://pub.dev" source: hosted - version: "1.12.0" + version: "1.11.1" step_progress_indicator: dependency: "direct main" description: @@ -2450,10 +2450,10 @@ packages: dependency: transitive description: name: string_scanner - sha256: "688af5ed3402a4bde5b3a6c15fd768dbf2621a614950b17f04626c431ab3c4c3" + sha256: "556692adab6cfa87322a115640c11f13cb77b3f076ddcc5d6ae3c20242bedcde" url: "https://pub.dev" source: hosted - version: "1.3.0" + version: "1.2.0" styled_text: dependency: "direct main" description: @@ -2514,26 +2514,26 @@ packages: dependency: "direct dev" description: name: test - sha256: "713a8789d62f3233c46b4a90b174737b2c04cb6ae4500f2aa8b1be8f03f5e67f" + sha256: "7ee44229615f8f642b68120165ae4c2a75fe77ae2065b1e55ae4711f6cf0899e" url: "https://pub.dev" source: hosted - version: "1.25.8" + version: "1.25.7" test_api: dependency: transitive description: name: test_api - sha256: "664d3a9a64782fcdeb83ce9c6b39e78fd2971d4e37827b9b06c3aa1edc5e760c" + sha256: "5b8a98dafc4d5c4c9c72d8b31ab2b23fc13422348d2997120294d3bac86b4ddb" url: "https://pub.dev" source: hosted - version: "0.7.3" + version: "0.7.2" test_core: dependency: transitive description: name: test_core - sha256: "12391302411737c176b0b5d6491f466b0dd56d4763e347b6714efbaa74d7953d" + sha256: "55ea5a652e38a1dfb32943a7973f3681a60f872f8c3a05a14664ad54ef9c6696" url: "https://pub.dev" source: hosted - version: "0.6.5" + version: "0.6.4" timezone: dependency: transitive description: @@ -2820,10 +2820,10 @@ packages: dependency: transitive description: name: vm_service - sha256: f6be3ed8bd01289b34d679c2b62226f63c0e69f9fd2e50a6b3c1c729a961041b + sha256: "5c5f338a667b4c644744b661f309fb8080bb94b18a7e91ef1dbd343bed00ed6d" url: "https://pub.dev" source: hosted - version: "14.3.0" + version: "14.2.5" volume_controller: dependency: transitive description: @@ -2892,10 +2892,10 @@ packages: dependency: transitive description: name: webdriver - sha256: "3d773670966f02a646319410766d3b5e1037efb7f07cc68f844d5e06cd4d61c8" + sha256: "003d7da9519e1e5f329422b36c4dcdf18d7d2978d1ba099ea4e45ba490ed845e" url: "https://pub.dev" source: hosted - version: "3.0.4" + version: "3.0.3" webkit_inspection_protocol: dependency: transitive description: