diff --git a/mobile/lib/db/ml_data_db.dart b/mobile/lib/db/ml_data_db.dart deleted file mode 100644 index 46ca06466b..0000000000 --- a/mobile/lib/db/ml_data_db.dart +++ /dev/null @@ -1,714 +0,0 @@ -import 'dart:async'; - -import 'package:logging/logging.dart'; -import 'package:path/path.dart' show join; -import 'package:path_provider/path_provider.dart'; -import 'package:photos/models/ml/ml_typedefs.dart'; -import 'package:photos/services/machine_learning/face_ml/face_feedback.dart/cluster_feedback.dart'; -import 'package:photos/services/machine_learning/face_ml/face_feedback.dart/feedback_types.dart'; -import 'package:photos/services/machine_learning/face_ml/face_ml_result.dart'; -import 'package:sqflite/sqflite.dart'; - -/// Stores all data for the ML-related features. The database can be accessed by `MlDataDB.instance.database`. -/// -/// This includes: -/// [facesTable] - Stores all the detected faces and its embeddings in the images. -/// [peopleTable] - Stores all the clusters of faces which are considered to be the same person. -class MlDataDB { - static final Logger _logger = Logger("MlDataDB"); - - // TODO: [BOB] put the db in files - static const _databaseName = "ente.ml_data.db"; - static const _databaseVersion = 1; - - static const facesTable = 'faces'; - static const fileIDColumn = 'file_id'; - static const faceMlResultColumn = 'face_ml_result'; - static const mlVersionColumn = 'ml_version'; - - static const peopleTable = 'people'; - static const personIDColumn = 'person_id'; - static const clusterResultColumn = 'cluster_result'; - static const centroidColumn = 'cluster_centroid'; - static const centroidDistanceThresholdColumn = 'centroid_distance_threshold'; - - static const feedbackTable = 'feedback'; - static const feedbackIDColumn = 'feedback_id'; - static const feedbackTypeColumn = 'feedback_type'; - static const feedbackDataColumn = 'feedback_data'; - static const feedbackTimestampColumn = 'feedback_timestamp'; - static const feedbackFaceMlVersionColumn = 'feedback_face_ml_version'; - static const feedbackClusterMlVersionColumn = 'feedback_cluster_ml_version'; - - static const createFacesTable = '''CREATE TABLE IF NOT EXISTS $facesTable ( - $fileIDColumn INTEGER NOT NULL UNIQUE, - $faceMlResultColumn TEXT NOT NULL, - $mlVersionColumn INTEGER NOT NULL, - PRIMARY KEY($fileIDColumn) - ); - '''; - static const createPeopleTable = '''CREATE TABLE IF NOT EXISTS $peopleTable ( - $personIDColumn INTEGER NOT NULL UNIQUE, - $clusterResultColumn TEXT NOT NULL, - $centroidColumn TEXT NOT NULL, - $centroidDistanceThresholdColumn REAL NOT NULL, - PRIMARY KEY($personIDColumn) - ); - '''; - static const createFeedbackTable = - '''CREATE TABLE IF NOT EXISTS $feedbackTable ( - $feedbackIDColumn TEXT NOT NULL UNIQUE, - $feedbackTypeColumn TEXT NOT NULL, - $feedbackDataColumn TEXT NOT NULL, - $feedbackTimestampColumn TEXT NOT NULL, - $feedbackFaceMlVersionColumn INTEGER NOT NULL, - $feedbackClusterMlVersionColumn INTEGER NOT NULL, - PRIMARY KEY($feedbackIDColumn) - ); - '''; - static const _deleteFacesTable = 'DROP TABLE IF EXISTS $facesTable'; - static const _deletePeopleTable = 'DROP TABLE IF EXISTS $peopleTable'; - static const _deleteFeedbackTable = 'DROP TABLE IF EXISTS $feedbackTable'; - - MlDataDB._privateConstructor(); - static final MlDataDB instance = MlDataDB._privateConstructor(); - - static Future? _dbFuture; - Future get database async { - _dbFuture ??= _initDatabase(); - return _dbFuture!; - } - - Future _initDatabase() async { - final documentsDirectory = await getApplicationDocumentsDirectory(); - final String databaseDirectory = - join(documentsDirectory.path, _databaseName); - return await openDatabase( - databaseDirectory, - version: _databaseVersion, - onCreate: _onCreate, - ); - } - - Future _onCreate(Database db, int version) async { - await db.execute(createFacesTable); - await db.execute(createPeopleTable); - await db.execute(createFeedbackTable); - } - - /// WARNING: This will delete ALL data in the database! Only use this for debug/testing purposes! - Future cleanTables({ - bool cleanFaces = false, - bool cleanPeople = false, - bool cleanFeedback = false, - }) async { - _logger.fine('`cleanTables()` called'); - final db = await instance.database; - - if (cleanFaces) { - _logger.fine('`cleanTables()`: Cleaning faces table'); - await db.execute(_deleteFacesTable); - } - - if (cleanPeople) { - _logger.fine('`cleanTables()`: Cleaning people table'); - await db.execute(_deletePeopleTable); - } - - if (cleanFeedback) { - _logger.fine('`cleanTables()`: Cleaning feedback table'); - await db.execute(_deleteFeedbackTable); - } - - if (!cleanFaces && !cleanPeople && !cleanFeedback) { - _logger.fine( - '`cleanTables()`: No tables cleaned, since no table was specified. Please be careful with this function!', - ); - } - - await db.execute(createFacesTable); - await db.execute(createPeopleTable); - await db.execute(createFeedbackTable); - } - - Future createFaceMlResult(FaceMlResult faceMlResult) async { - _logger.fine('createFaceMlResult called'); - - final existingResult = await getFaceMlResult(faceMlResult.fileId); - if (existingResult != null) { - if (faceMlResult.mlVersion <= existingResult.mlVersion) { - _logger.fine( - 'FaceMlResult with file ID ${faceMlResult.fileId} already exists with equal or higher version. Skipping insert.', - ); - return; - } - } - - final db = await instance.database; - await db.insert( - facesTable, - { - fileIDColumn: faceMlResult.fileId, - faceMlResultColumn: faceMlResult.toJsonString(), - mlVersionColumn: faceMlResult.mlVersion, - }, - conflictAlgorithm: ConflictAlgorithm.replace, - ); - } - - Future doesFaceMlResultExist(int fileId, {int? mlVersion}) async { - _logger.fine('doesFaceMlResultExist called'); - final db = await instance.database; - - String whereString = '$fileIDColumn = ?'; - final List whereArgs = [fileId]; - - if (mlVersion != null) { - whereString += ' AND $mlVersionColumn = ?'; - whereArgs.add(mlVersion); - } - - final result = await db.query( - facesTable, - where: whereString, - whereArgs: whereArgs, - limit: 1, - ); - return result.isNotEmpty; - } - - Future getFaceMlResult(int fileId, {int? mlVersion}) async { - _logger.fine('getFaceMlResult called'); - final db = await instance.database; - - String whereString = '$fileIDColumn = ?'; - final List whereArgs = [fileId]; - - if (mlVersion != null) { - whereString += ' AND $mlVersionColumn = ?'; - whereArgs.add(mlVersion); - } - - final result = await db.query( - facesTable, - where: whereString, - whereArgs: whereArgs, - limit: 1, - ); - if (result.isNotEmpty) { - return FaceMlResult.fromJsonString( - result.first[faceMlResultColumn] as String, - ); - } - _logger.fine( - 'No faceMlResult found for fileID $fileId and mlVersion $mlVersion (null if not specified)', - ); - return null; - } - - /// Returns the faceMlResults for the given [fileIds]. - Future> getSelectedFaceMlResults( - List fileIds, - ) async { - _logger.fine('getSelectedFaceMlResults called'); - final db = await instance.database; - - if (fileIds.isEmpty) { - _logger.warning('getSelectedFaceMlResults called with empty fileIds'); - return []; - } - - final List> results = await db.query( - facesTable, - columns: [faceMlResultColumn], - where: '$fileIDColumn IN (${fileIds.join(',')})', - orderBy: fileIDColumn, - ); - - return results - .map( - (result) => - FaceMlResult.fromJsonString(result[faceMlResultColumn] as String), - ) - .toList(); - } - - Future> getAllFaceMlResults({int? mlVersion}) async { - _logger.fine('getAllFaceMlResults called'); - final db = await instance.database; - - String? whereString; - List? whereArgs; - - if (mlVersion != null) { - whereString = '$mlVersionColumn = ?'; - whereArgs = [mlVersion]; - } - - final results = await db.query( - facesTable, - where: whereString, - whereArgs: whereArgs, - orderBy: fileIDColumn, - ); - - return results - .map( - (result) => - FaceMlResult.fromJsonString(result[faceMlResultColumn] as String), - ) - .toList(); - } - - /// getAllFileIDs returns a set of all fileIDs from the facesTable, meaning all the fileIDs for which a FaceMlResult exists, optionally filtered by mlVersion. - Future> getAllFaceMlResultFileIDs({int? mlVersion}) async { - _logger.fine('getAllFaceMlResultFileIDs called'); - final db = await instance.database; - - String? whereString; - List? whereArgs; - - if (mlVersion != null) { - whereString = '$mlVersionColumn = ?'; - whereArgs = [mlVersion]; - } - - final List> results = await db.query( - facesTable, - where: whereString, - whereArgs: whereArgs, - orderBy: fileIDColumn, - ); - - return results.map((result) => result[fileIDColumn] as int).toSet(); - } - - Future> getAllFaceMlResultFileIDsProcessedWithThumbnailOnly({ - int? mlVersion, - }) async { - _logger.fine('getAllFaceMlResultFileIDsProcessedWithThumbnailOnly called'); - final db = await instance.database; - - String? whereString; - List? whereArgs; - - if (mlVersion != null) { - whereString = '$mlVersionColumn = ?'; - whereArgs = [mlVersion]; - } - - final List> results = await db.query( - facesTable, - where: whereString, - whereArgs: whereArgs, - orderBy: fileIDColumn, - ); - - return results - .map( - (result) => - FaceMlResult.fromJsonString(result[faceMlResultColumn] as String), - ) - .where((element) => element.onlyThumbnailUsed) - .map((result) => result.fileId) - .toSet(); - } - - /// Updates the faceMlResult for the given [faceMlResult.fileId]. Update is done regardless of the [faceMlResult.mlVersion]. - /// However, if [updateHigherVersionOnly] is set to true, the update is only done if the [faceMlResult.mlVersion] is higher than the existing one. - Future updateFaceMlResult( - FaceMlResult faceMlResult, { - bool updateHigherVersionOnly = false, - }) async { - _logger.fine('updateFaceMlResult called'); - - if (updateHigherVersionOnly) { - final existingResult = await getFaceMlResult(faceMlResult.fileId); - if (existingResult != null) { - if (faceMlResult.mlVersion <= existingResult.mlVersion) { - _logger.fine( - 'FaceMlResult with file ID ${faceMlResult.fileId} already exists with equal or higher version. Skipping update.', - ); - return 0; - } - } - } - - final db = await instance.database; - return await db.update( - facesTable, - { - fileIDColumn: faceMlResult.fileId, - faceMlResultColumn: faceMlResult.toJsonString(), - mlVersionColumn: faceMlResult.mlVersion, - }, - where: '$fileIDColumn = ?', - whereArgs: [faceMlResult.fileId], - ); - } - - Future deleteFaceMlResult(int fileId) async { - _logger.fine('deleteFaceMlResult called'); - final db = await instance.database; - final deleteCount = await db.delete( - facesTable, - where: '$fileIDColumn = ?', - whereArgs: [fileId], - ); - _logger.fine('Deleted $deleteCount faceMlResults'); - return deleteCount; - } - - Future createAllClusterResults( - List clusterResults, { - bool cleanExistingClusters = true, - }) async { - _logger.fine('createClusterResults called'); - final db = await instance.database; - - if (clusterResults.isEmpty) { - _logger.fine('No clusterResults given, skipping insert.'); - return; - } - - // Completely clean the table and start fresh - if (cleanExistingClusters) { - await deleteAllClusterResults(); - } - - // Insert all the cluster results - for (final clusterResult in clusterResults) { - await db.insert( - peopleTable, - { - personIDColumn: clusterResult.personId, - clusterResultColumn: clusterResult.toJsonString(), - centroidColumn: clusterResult.medoid.toString(), - centroidDistanceThresholdColumn: - clusterResult.medoidDistanceThreshold, - }, - conflictAlgorithm: ConflictAlgorithm.replace, - ); - } - } - - Future getClusterResult(int personId) async { - _logger.fine('getClusterResult called'); - final db = await instance.database; - - final result = await db.query( - peopleTable, - where: '$personIDColumn = ?', - whereArgs: [personId], - limit: 1, - ); - if (result.isNotEmpty) { - return ClusterResult.fromJsonString( - result.first[clusterResultColumn] as String, - ); - } - _logger.fine('No clusterResult found for personID $personId'); - return null; - } - - /// Returns the ClusterResult objects for the given [personIDs]. - Future> getSelectedClusterResults( - List personIDs, - ) async { - _logger.fine('getSelectedClusterResults called'); - final db = await instance.database; - - if (personIDs.isEmpty) { - _logger.warning('getSelectedClusterResults called with empty personIDs'); - return []; - } - - final results = await db.query( - peopleTable, - where: '$personIDColumn IN (${personIDs.join(',')})', - orderBy: personIDColumn, - ); - - return results - .map( - (result) => ClusterResult.fromJsonString( - result[clusterResultColumn] as String, - ), - ) - .toList(); - } - - Future> getAllClusterResults() async { - _logger.fine('getAllClusterResults called'); - final db = await instance.database; - - final results = await db.query( - peopleTable, - ); - - return results - .map( - (result) => ClusterResult.fromJsonString( - result[clusterResultColumn] as String, - ), - ) - .toList(); - } - - /// Returns the personIDs of all clustered people in the database. - Future> getAllClusterIds() async { - _logger.fine('getAllClusterIds called'); - final db = await instance.database; - - final results = await db.query( - peopleTable, - columns: [personIDColumn], - ); - - return results.map((result) => result[personIDColumn] as int).toList(); - } - - /// Returns the fileIDs of all files associated with a given [personId]. - Future> getClusterFileIds(int personId) async { - _logger.fine('getClusterFileIds called'); - - final ClusterResult? clusterResult = await getClusterResult(personId); - if (clusterResult == null) { - return []; - } - return clusterResult.uniqueFileIds; - } - - Future> getClusterFaceIds(int personId) async { - _logger.fine('getClusterFaceIds called'); - - final ClusterResult? clusterResult = await getClusterResult(personId); - if (clusterResult == null) { - return []; - } - return clusterResult.faceIDs; - } - - Future> getClusterEmbeddings( - int personId, - ) async { - _logger.fine('getClusterEmbeddings called'); - - final ClusterResult? clusterResult = await getClusterResult(personId); - if (clusterResult == null) return []; - - final fileIds = clusterResult.uniqueFileIds; - final faceIds = clusterResult.faceIDs; - if (fileIds.length != faceIds.length) { - _logger.severe( - 'fileIds and faceIds have different lengths: ${fileIds.length} vs ${faceIds.length}. This should not happen!', - ); - return []; - } - - final faceMlResults = await getSelectedFaceMlResults(fileIds); - if (faceMlResults.isEmpty) return []; - - final embeddings = []; - for (var i = 0; i < faceMlResults.length; i++) { - final faceMlResult = faceMlResults[i]; - final int faceIndex = faceMlResult.allFaceIds.indexOf(faceIds[i]); - if (faceIndex == -1) { - _logger.severe( - 'Could not find faceIndex for faceId ${faceIds[i]} in faceMlResult ${faceMlResult.fileId}', - ); - return []; - } - embeddings.add(faceMlResult.faces[faceIndex].embedding); - } - - return embeddings; - } - - Future updateClusterResult(ClusterResult clusterResult) async { - _logger.fine('updateClusterResult called'); - final db = await instance.database; - await db.update( - peopleTable, - { - personIDColumn: clusterResult.personId, - clusterResultColumn: clusterResult.toJsonString(), - centroidColumn: clusterResult.medoid.toString(), - centroidDistanceThresholdColumn: clusterResult.medoidDistanceThreshold, - }, - where: '$personIDColumn = ?', - whereArgs: [clusterResult.personId], - ); - } - - Future deleteClusterResult(int personId) async { - _logger.fine('deleteClusterResult called'); - final db = await instance.database; - final deleteCount = await db.delete( - peopleTable, - where: '$personIDColumn = ?', - whereArgs: [personId], - ); - _logger.fine('Deleted $deleteCount clusterResults'); - return deleteCount; - } - - Future deleteAllClusterResults() async { - _logger.fine('deleteAllClusterResults called'); - final db = await instance.database; - await db.execute(_deletePeopleTable); - await db.execute(createPeopleTable); - } - - // TODO: current function implementation will skip inserting for a similar feedback, which means I can't remove two photos from the same person in a row - Future createClusterFeedback( - T feedback, { - bool skipIfSimilarFeedbackExists = false, - }) async { - _logger.fine('createClusterFeedback called'); - - // TODO: this skipping might cause issues for adding photos to the same person in a row!! - if (skipIfSimilarFeedbackExists && - await doesSimilarClusterFeedbackExist(feedback)) { - _logger.fine( - 'ClusterFeedback with ID ${feedback.feedbackID} already has a similar feedback installed. Skipping insert.', - ); - return; - } - - final db = await instance.database; - await db.insert( - feedbackTable, - { - feedbackIDColumn: feedback.feedbackID, - feedbackTypeColumn: feedback.typeString, - feedbackDataColumn: feedback.toJsonString(), - feedbackTimestampColumn: feedback.timestampString, - feedbackFaceMlVersionColumn: feedback.madeOnFaceMlVersion, - feedbackClusterMlVersionColumn: feedback.madeOnClusterMlVersion, - }, - conflictAlgorithm: ConflictAlgorithm.replace, - ); - return; - } - - Future doesSimilarClusterFeedbackExist( - T feedback, - ) async { - _logger.fine('doesClusterFeedbackExist called'); - - final List existingFeedback = - await getAllClusterFeedback(type: feedback.type); - - if (existingFeedback.isNotEmpty) { - for (final existingFeedbackItem in existingFeedback) { - assert( - existingFeedbackItem.type == feedback.type, - 'Feedback types should be the same!', - ); - if (feedback.looselyMatchesMedoid(existingFeedbackItem)) { - _logger.fine( - 'ClusterFeedback of type ${feedback.typeString} with ID ${feedback.feedbackID} already has a similar feedback installed!', - ); - return true; - } - } - } - return false; - } - - /// Returns all the clusterFeedbacks of type [T] which match the given [feedback], sorted by timestamp (latest first). - Future> getAllMatchingClusterFeedback( - T feedback, { - bool sortNewestFirst = true, - }) async { - _logger.fine('getAllMatchingClusterFeedback called'); - - final List existingFeedback = - await getAllClusterFeedback(type: feedback.type); - final List matchingFeedback = []; - if (existingFeedback.isNotEmpty) { - for (final existingFeedbackItem in existingFeedback) { - assert( - existingFeedbackItem.type == feedback.type, - 'Feedback types should be the same!', - ); - if (feedback.looselyMatchesMedoid(existingFeedbackItem)) { - _logger.fine( - 'ClusterFeedback of type ${feedback.typeString} with ID ${feedback.feedbackID} already has a similar feedback installed!', - ); - matchingFeedback.add(existingFeedbackItem); - } - } - } - if (sortNewestFirst) { - matchingFeedback.sort((a, b) => b.timestamp.compareTo(a.timestamp)); - } - return matchingFeedback; - } - - Future> getAllClusterFeedback({ - required FeedbackType type, - int? mlVersion, - int? clusterMlVersion, - }) async { - _logger.fine('getAllClusterFeedback called'); - final db = await instance.database; - - // TODO: implement the versions for FeedbackType.imageFeedback and FeedbackType.faceFeedback and rename this function to getAllFeedback? - - String whereString = '$feedbackTypeColumn = ?'; - final List whereArgs = [type.toValueString()]; - - if (mlVersion != null) { - whereString += ' AND $feedbackFaceMlVersionColumn = ?'; - whereArgs.add(mlVersion); - } - if (clusterMlVersion != null) { - whereString += ' AND $feedbackClusterMlVersionColumn = ?'; - whereArgs.add(clusterMlVersion); - } - - final results = await db.query( - feedbackTable, - where: whereString, - whereArgs: whereArgs, - ); - - if (results.isNotEmpty) { - if (ClusterFeedback.fromJsonStringRegistry.containsKey(type)) { - final Function(String) fromJsonString = - ClusterFeedback.fromJsonStringRegistry[type]!; - return results - .map((e) => fromJsonString(e[feedbackDataColumn] as String) as T) - .toList(); - } else { - _logger.severe( - 'No fromJsonString function found for type ${type.name}. This should not happen!', - ); - } - } - _logger.fine( - 'No clusterFeedback results found of type $type' + - (mlVersion != null ? ' and mlVersion $mlVersion' : '') + - (clusterMlVersion != null - ? ' and clusterMlVersion $clusterMlVersion' - : ''), - ); - return []; - } - - Future deleteClusterFeedback( - T feedback, - ) async { - _logger.fine('deleteClusterFeedback called'); - final db = await instance.database; - final deleteCount = await db.delete( - feedbackTable, - where: '$feedbackIDColumn = ?', - whereArgs: [feedback.feedbackID], - ); - _logger.fine('Deleted $deleteCount clusterFeedbacks'); - return deleteCount; - } -} diff --git a/mobile/lib/services/machine_learning/face_ml/face_feedback.dart/cluster_feedback.dart b/mobile/lib/services/machine_learning/face_ml/face_feedback.dart/cluster_feedback.dart deleted file mode 100644 index a7da4fa556..0000000000 --- a/mobile/lib/services/machine_learning/face_ml/face_feedback.dart/cluster_feedback.dart +++ /dev/null @@ -1,379 +0,0 @@ -import "dart:convert"; - -import 'package:photos/services/machine_learning/face_ml/face_clustering/cosine_distance.dart'; -import 'package:photos/services/machine_learning/face_ml/face_feedback.dart/feedback.dart'; -import 'package:photos/services/machine_learning/face_ml/face_feedback.dart/feedback_types.dart'; - -abstract class ClusterFeedback extends Feedback { - static final Map fromJsonStringRegistry = { - FeedbackType.deleteClusterFeedback: DeleteClusterFeedback.fromJsonString, - FeedbackType.mergeClusterFeedback: MergeClusterFeedback.fromJsonString, - FeedbackType.renameOrCustomThumbnailClusterFeedback: - RenameOrCustomThumbnailClusterFeedback.fromJsonString, - FeedbackType.removePhotosClusterFeedback: - RemovePhotosClusterFeedback.fromJsonString, - FeedbackType.addPhotosClusterFeedback: - AddPhotosClusterFeedback.fromJsonString, - }; - - final List medoid; - final double medoidDistanceThreshold; - // TODO: work out the optimal distance threshold so there's never an overlap between clusters - - ClusterFeedback( - FeedbackType type, - this.medoid, - this.medoidDistanceThreshold, { - String? feedbackID, - DateTime? timestamp, - int? madeOnFaceMlVersion, - int? madeOnClusterMlVersion, - }) : super( - type, - feedbackID: feedbackID, - timestamp: timestamp, - madeOnFaceMlVersion: madeOnFaceMlVersion, - madeOnClusterMlVersion: madeOnClusterMlVersion, - ); - - /// Compares this feedback with another [ClusterFeedback] to see if they are similar enough that only one should be kept. - /// - /// It checks this by comparing the distance between the two medoids with the medoidDistanceThreshold of each feedback. - /// - /// Returns true if they are similar enough, false otherwise. - /// // TODO: Should it maybe return a merged feedback instead, when you are similar enough? - bool looselyMatchesMedoid(ClusterFeedback other) { - // Using the cosineDistance function you mentioned - final double distance = cosineDistance(medoid, other.medoid); - - // Check if the distance is less than either of the threshold values - return distance < medoidDistanceThreshold || - distance < other.medoidDistanceThreshold; - } - - bool exactlyMatchesMedoid(ClusterFeedback other) { - if (medoid.length != other.medoid.length) { - return false; - } - for (int i = 0; i < medoid.length; i++) { - if (medoid[i] != other.medoid[i]) { - return false; - } - } - return true; - } -} - -class DeleteClusterFeedback extends ClusterFeedback { - DeleteClusterFeedback({ - required List medoid, - required double medoidDistanceThreshold, - String? feedbackID, - DateTime? timestamp, - int? madeOnFaceMlVersion, - int? madeOnClusterMlVersion, - }) : super( - FeedbackType.deleteClusterFeedback, - medoid, - medoidDistanceThreshold, - feedbackID: feedbackID, - timestamp: timestamp, - madeOnFaceMlVersion: madeOnFaceMlVersion, - madeOnClusterMlVersion: madeOnClusterMlVersion, - ); - - @override - Map toJson() { - return { - 'type': type.toValueString(), - 'medoid': medoid, - 'medoidDistanceThreshold': medoidDistanceThreshold, - 'feedbackID': feedbackID, - 'timestamp': timestamp.toIso8601String(), - 'madeOnFaceMlVersion': madeOnFaceMlVersion, - 'madeOnClusterMlVersion': madeOnClusterMlVersion, - }; - } - - @override - String toJsonString() => jsonEncode(toJson()); - - static DeleteClusterFeedback fromJson(Map json) { - assert(json['type'] == FeedbackType.deleteClusterFeedback.toValueString()); - return DeleteClusterFeedback( - medoid: - (json['medoid'] as List?)?.map((item) => item as double).toList() ?? - [], - medoidDistanceThreshold: json['medoidDistanceThreshold'], - feedbackID: json['feedbackID'], - timestamp: DateTime.parse(json['timestamp']), - madeOnFaceMlVersion: json['madeOnFaceMlVersion'], - madeOnClusterMlVersion: json['madeOnClusterMlVersion'], - ); - } - - static fromJsonString(String jsonString) { - return fromJson(jsonDecode(jsonString)); - } -} - -class MergeClusterFeedback extends ClusterFeedback { - final List medoidToMoveTo; - - MergeClusterFeedback({ - required List medoid, - required double medoidDistanceThreshold, - required this.medoidToMoveTo, - String? feedbackID, - DateTime? timestamp, - int? madeOnFaceMlVersion, - int? madeOnClusterMlVersion, - }) : super( - FeedbackType.mergeClusterFeedback, - medoid, - medoidDistanceThreshold, - feedbackID: feedbackID, - timestamp: timestamp, - madeOnFaceMlVersion: madeOnFaceMlVersion, - madeOnClusterMlVersion: madeOnClusterMlVersion, - ); - - @override - Map toJson() { - return { - 'type': type.toValueString(), - 'medoid': medoid, - 'medoidDistanceThreshold': medoidDistanceThreshold, - 'medoidToMoveTo': medoidToMoveTo, - 'feedbackID': feedbackID, - 'timestamp': timestamp.toIso8601String(), - 'madeOnFaceMlVersion': madeOnFaceMlVersion, - 'madeOnClusterMlVersion': madeOnClusterMlVersion, - }; - } - - @override - String toJsonString() => jsonEncode(toJson()); - - static MergeClusterFeedback fromJson(Map json) { - assert(json['type'] == FeedbackType.mergeClusterFeedback.toValueString()); - return MergeClusterFeedback( - medoid: - (json['medoid'] as List?)?.map((item) => item as double).toList() ?? - [], - medoidDistanceThreshold: json['medoidDistanceThreshold'], - medoidToMoveTo: (json['medoidToMoveTo'] as List?) - ?.map((item) => item as double) - .toList() ?? - [], - feedbackID: json['feedbackID'], - timestamp: DateTime.parse(json['timestamp']), - madeOnFaceMlVersion: json['madeOnFaceMlVersion'], - madeOnClusterMlVersion: json['madeOnClusterMlVersion'], - ); - } - - static MergeClusterFeedback fromJsonString(String jsonString) { - return fromJson(jsonDecode(jsonString)); - } -} - -class RenameOrCustomThumbnailClusterFeedback extends ClusterFeedback { - String? customName; - String? customThumbnailFaceId; - - RenameOrCustomThumbnailClusterFeedback({ - required List medoid, - required double medoidDistanceThreshold, - this.customName, - this.customThumbnailFaceId, - String? feedbackID, - DateTime? timestamp, - int? madeOnFaceMlVersion, - int? madeOnClusterMlVersion, - }) : assert( - customName != null || customThumbnailFaceId != null, - "Either customName or customThumbnailFaceId must be non-null!", - ), - super( - FeedbackType.renameOrCustomThumbnailClusterFeedback, - medoid, - medoidDistanceThreshold, - feedbackID: feedbackID, - timestamp: timestamp, - madeOnFaceMlVersion: madeOnFaceMlVersion, - madeOnClusterMlVersion: madeOnClusterMlVersion, - ); - - @override - Map toJson() { - return { - 'type': type.toValueString(), - 'medoid': medoid, - 'medoidDistanceThreshold': medoidDistanceThreshold, - if (customName != null) 'customName': customName, - if (customThumbnailFaceId != null) - 'customThumbnailFaceId': customThumbnailFaceId, - 'feedbackID': feedbackID, - 'timestamp': timestamp.toIso8601String(), - 'madeOnFaceMlVersion': madeOnFaceMlVersion, - 'madeOnClusterMlVersion': madeOnClusterMlVersion, - }; - } - - @override - String toJsonString() => jsonEncode(toJson()); - - static RenameOrCustomThumbnailClusterFeedback fromJson( - Map json, - ) { - assert( - json['type'] == - FeedbackType.renameOrCustomThumbnailClusterFeedback.toValueString(), - ); - return RenameOrCustomThumbnailClusterFeedback( - medoid: - (json['medoid'] as List?)?.map((item) => item as double).toList() ?? - [], - medoidDistanceThreshold: json['medoidDistanceThreshold'], - customName: json['customName'], - customThumbnailFaceId: json['customThumbnailFaceId'], - feedbackID: json['feedbackID'], - timestamp: DateTime.parse(json['timestamp']), - madeOnFaceMlVersion: json['madeOnFaceMlVersion'], - madeOnClusterMlVersion: json['madeOnClusterMlVersion'], - ); - } - - static RenameOrCustomThumbnailClusterFeedback fromJsonString( - String jsonString, - ) { - return fromJson(jsonDecode(jsonString)); - } -} - -class RemovePhotosClusterFeedback extends ClusterFeedback { - final List removedPhotosFileID; - - RemovePhotosClusterFeedback({ - required List medoid, - required double medoidDistanceThreshold, - required this.removedPhotosFileID, - String? feedbackID, - DateTime? timestamp, - int? madeOnFaceMlVersion, - int? madeOnClusterMlVersion, - }) : super( - FeedbackType.removePhotosClusterFeedback, - medoid, - medoidDistanceThreshold, - feedbackID: feedbackID, - timestamp: timestamp, - madeOnFaceMlVersion: madeOnFaceMlVersion, - madeOnClusterMlVersion: madeOnClusterMlVersion, - ); - - @override - Map toJson() { - return { - 'type': type.toValueString(), - 'medoid': medoid, - 'medoidDistanceThreshold': medoidDistanceThreshold, - 'removedPhotosFileID': removedPhotosFileID, - 'feedbackID': feedbackID, - 'timestamp': timestamp.toIso8601String(), - 'madeOnFaceMlVersion': madeOnFaceMlVersion, - 'madeOnClusterMlVersion': madeOnClusterMlVersion, - }; - } - - @override - String toJsonString() => jsonEncode(toJson()); - - static RemovePhotosClusterFeedback fromJson(Map json) { - assert( - json['type'] == FeedbackType.removePhotosClusterFeedback.toValueString(), - ); - return RemovePhotosClusterFeedback( - medoid: - (json['medoid'] as List?)?.map((item) => item as double).toList() ?? - [], - medoidDistanceThreshold: json['medoidDistanceThreshold'], - removedPhotosFileID: (json['removedPhotosFileID'] as List?) - ?.map((item) => item as int) - .toList() ?? - [], - feedbackID: json['feedbackID'], - timestamp: DateTime.parse(json['timestamp']), - madeOnFaceMlVersion: json['madeOnFaceMlVersion'], - madeOnClusterMlVersion: json['madeOnClusterMlVersion'], - ); - } - - static RemovePhotosClusterFeedback fromJsonString(String jsonString) { - return fromJson(jsonDecode(jsonString)); - } -} - -class AddPhotosClusterFeedback extends ClusterFeedback { - final List addedPhotoFileIDs; - - AddPhotosClusterFeedback({ - required List medoid, - required double medoidDistanceThreshold, - required this.addedPhotoFileIDs, - String? feedbackID, - DateTime? timestamp, - int? madeOnFaceMlVersion, - int? madeOnClusterMlVersion, - }) : super( - FeedbackType.addPhotosClusterFeedback, - medoid, - medoidDistanceThreshold, - feedbackID: feedbackID, - timestamp: timestamp, - madeOnFaceMlVersion: madeOnFaceMlVersion, - madeOnClusterMlVersion: madeOnClusterMlVersion, - ); - - @override - Map toJson() { - return { - 'type': type.toValueString(), - 'medoid': medoid, - 'medoidDistanceThreshold': medoidDistanceThreshold, - 'addedPhotoFileIDs': addedPhotoFileIDs, - 'feedbackID': feedbackID, - 'timestamp': timestamp.toIso8601String(), - 'madeOnFaceMlVersion': madeOnFaceMlVersion, - 'madeOnClusterMlVersion': madeOnClusterMlVersion, - }; - } - - @override - String toJsonString() => jsonEncode(toJson()); - - static AddPhotosClusterFeedback fromJson(Map json) { - assert( - json['type'] == FeedbackType.addPhotosClusterFeedback.toValueString(), - ); - return AddPhotosClusterFeedback( - medoid: - (json['medoid'] as List?)?.map((item) => item as double).toList() ?? - [], - medoidDistanceThreshold: json['medoidDistanceThreshold'], - addedPhotoFileIDs: (json['addedPhotoFileIDs'] as List?) - ?.map((item) => item as int) - .toList() ?? - [], - feedbackID: json['feedbackID'], - timestamp: DateTime.parse(json['timestamp']), - madeOnFaceMlVersion: json['madeOnFaceMlVersion'], - madeOnClusterMlVersion: json['madeOnClusterMlVersion'], - ); - } - - static AddPhotosClusterFeedback fromJsonString(String jsonString) { - return fromJson(jsonDecode(jsonString)); - } -} diff --git a/mobile/lib/services/machine_learning/face_ml/face_feedback.dart/face_feedback_service.dart b/mobile/lib/services/machine_learning/face_ml/face_feedback.dart/face_feedback_service.dart deleted file mode 100644 index c94c8c8d85..0000000000 --- a/mobile/lib/services/machine_learning/face_ml/face_feedback.dart/face_feedback_service.dart +++ /dev/null @@ -1,416 +0,0 @@ -import "package:logging/logging.dart"; -import "package:photos/db/ml_data_db.dart"; -import 'package:photos/services/machine_learning/face_ml/face_detection/detection.dart'; -import 'package:photos/services/machine_learning/face_ml/face_feedback.dart/cluster_feedback.dart'; -import 'package:photos/services/machine_learning/face_ml/face_ml_result.dart'; - -class FaceFeedbackService { - final _logger = Logger("FaceFeedbackService"); - - final _mlDatabase = MlDataDB.instance; - - int executedFeedbackCount = 0; - final int _reclusterFeedbackThreshold = 10; - - // singleton pattern - FaceFeedbackService._privateConstructor(); - static final instance = FaceFeedbackService._privateConstructor(); - factory FaceFeedbackService() => instance; - - /// Returns the updated cluster after removing the given file from the given person's cluster. - /// - /// If the file is not in the cluster, returns null. - /// - /// The updated cluster is also updated in [MlDataDB]. - Future removePhotosFromCluster( - List fileIDs, - int personID, - ) async { - // TODO: check if photo was originally added to cluster by user. If so, we should remove that addition instead of changing the embedding, because there is no embedding... - _logger.info( - 'removePhotoFromCluster called with fileIDs $fileIDs and personID $personID', - ); - - if (fileIDs.isEmpty) { - _logger.severe( - "No fileIDs given, unable to add photos to cluster!", - ); - throw ArgumentError( - "No fileIDs given, unable to add photos to cluster!", - ); - } - - // Get the relevant cluster - final ClusterResult? cluster = await _mlDatabase.getClusterResult(personID); - if (cluster == null) { - _logger.severe( - "No cluster found for personID $personID, unable to remove photo from non-existent cluster!", - ); - throw ArgumentError( - "No cluster found for personID $personID, unable to remove photo from non-existent cluster!", - ); - } - // Get the relevant faceMlResults - final List faceMlResults = - await _mlDatabase.getSelectedFaceMlResults(fileIDs); - if (faceMlResults.length != fileIDs.length) { - final List foundFileIDs = - faceMlResults.map((faceMlResult) => faceMlResult.fileId).toList(); - _logger.severe( - "Couldn't find all facemlresults for fileIDs $fileIDs, only found for $foundFileIDs. Unable to remove unindexed photos from cluster!", - ); - throw ArgumentError( - "Couldn't find all facemlresults for fileIDs $fileIDs, only found for $foundFileIDs. Unable to remove unindexed photos from cluster!", - ); - } - - // Check if at least one of the files is in the cluster. If all files are already not in the cluster, return the cluster. - final List fileIDsInCluster = fileIDs - .where((fileID) => cluster.uniqueFileIds.contains(fileID)) - .toList(); - if (fileIDsInCluster.isEmpty) { - _logger.warning( - "All fileIDs are already not in the cluster, unable to remove photos from cluster!", - ); - return cluster; - } - final List faceMlResultsInCluster = faceMlResults - .where((faceMlResult) => fileIDsInCluster.contains(faceMlResult.fileId)) - .toList(); - assert(faceMlResultsInCluster.length == fileIDsInCluster.length); - - for (var i = 0; i < fileIDsInCluster.length; i++) { - // Find the faces/embeddings associated with both the fileID and personID - final List faceIDs = faceMlResultsInCluster[i].allFaceIds; - final List faceIDsInCluster = cluster.faceIDs; - final List relevantFaceIDs = - faceIDsInCluster.where((faceID) => faceIDs.contains(faceID)).toList(); - if (relevantFaceIDs.isEmpty) { - _logger.severe( - "No faces found in both cluster and file, unable to remove photo from cluster!", - ); - throw ArgumentError( - "No faces found in both cluster and file, unable to remove photo from cluster!", - ); - } - - // Set the embeddings to [10, 10,..., 10] and save the updated faceMlResult - faceMlResultsInCluster[i].setEmbeddingsToTen(relevantFaceIDs); - await _mlDatabase.updateFaceMlResult(faceMlResultsInCluster[i]); - - // Make sure there is a manual override for [10, 10,..., 10] embeddings (not actually here, but in building the clusters, see _checkIfClusterIsDeleted function) - - // Manually remove the fileID from the cluster - cluster.removeFileId(fileIDsInCluster[i]); - } - - // TODO: see below - // Re-cluster and check if this leads to more deletions. If so, save them and ask the user if they want to delete them too. - executedFeedbackCount++; - if (executedFeedbackCount % _reclusterFeedbackThreshold == 0) { - // await recluster(); - } - - // Update the cluster in the database - await _mlDatabase.updateClusterResult(cluster); - - // TODO: see below - // Safe the given feedback to the database - final removePhotoFeedback = RemovePhotosClusterFeedback( - medoid: cluster.medoid, - medoidDistanceThreshold: cluster.medoidDistanceThreshold, - removedPhotosFileID: fileIDsInCluster, - ); - await _mlDatabase.createClusterFeedback( - removePhotoFeedback, - skipIfSimilarFeedbackExists: false, - ); - - // Return the updated cluster - return cluster; - } - - Future addPhotosToCluster(List fileIDs, personID) async { - _logger.info( - 'addPhotosToCluster called with fileIDs $fileIDs and personID $personID', - ); - - if (fileIDs.isEmpty) { - _logger.severe( - "No fileIDs given, unable to add photos to cluster!", - ); - throw ArgumentError( - "No fileIDs given, unable to add photos to cluster!", - ); - } - - // Get the relevant cluster - final ClusterResult? cluster = await _mlDatabase.getClusterResult(personID); - if (cluster == null) { - _logger.severe( - "No cluster found for personID $personID, unable to add photos to non-existent cluster!", - ); - throw ArgumentError( - "No cluster found for personID $personID, unable to add photos to non-existent cluster!", - ); - } - - // Check if at least one of the files is not in the cluster. If all files are already in the cluster, return the cluster. - final List fileIDsNotInCluster = fileIDs - .where((fileID) => !cluster.uniqueFileIds.contains(fileID)) - .toList(); - if (fileIDsNotInCluster.isEmpty) { - _logger.warning( - "All fileIDs are already in the cluster, unable to add new photos to cluster!", - ); - return cluster; - } - final List faceIDsNotInCluster = fileIDsNotInCluster - .map((fileID) => FaceDetectionRelative.toFaceIDEmpty(fileID: fileID)) - .toList(); - - // Add the new files to the cluster - cluster.addFileIDsAndFaceIDs(fileIDsNotInCluster, faceIDsNotInCluster); - - // Update the cluster in the database - await _mlDatabase.updateClusterResult(cluster); - - // Build the addPhotoFeedback - final AddPhotosClusterFeedback addPhotosFeedback = AddPhotosClusterFeedback( - medoid: cluster.medoid, - medoidDistanceThreshold: cluster.medoidDistanceThreshold, - addedPhotoFileIDs: fileIDsNotInCluster, - ); - - // TODO: check for exact match and update feedback if necessary - - // Save the addPhotoFeedback to the database - await _mlDatabase.createClusterFeedback( - addPhotosFeedback, - skipIfSimilarFeedbackExists: false, - ); - - // Return the updated cluster - return cluster; - } - - /// Deletes the given cluster completely. - Future deleteCluster(int personID) async { - _logger.info( - 'deleteCluster called with personID $personID', - ); - - // Get the relevant cluster - final cluster = await _mlDatabase.getClusterResult(personID); - if (cluster == null) { - _logger.severe( - "No cluster found for personID $personID, unable to delete non-existent cluster!", - ); - throw ArgumentError( - "No cluster found for personID $personID, unable to delete non-existent cluster!", - ); - } - - // Delete the cluster from the database - await _mlDatabase.deleteClusterResult(cluster.personId); - - // TODO: look into the right threshold distance. - // Build the deleteClusterFeedback - final DeleteClusterFeedback deleteClusterFeedback = DeleteClusterFeedback( - medoid: cluster.medoid, - medoidDistanceThreshold: cluster.medoidDistanceThreshold, - ); - - // TODO: maybe I should merge the two feedbacks if they are similar enough? Or alternatively, I keep them both? - // Check if feedback doesn't already exist - if (await _mlDatabase - .doesSimilarClusterFeedbackExist(deleteClusterFeedback)) { - _logger.warning( - "Feedback already exists for deleting cluster $personID, unable to delete cluster!", - ); - return; - } - - // Save the deleteClusterFeedback to the database - await _mlDatabase.createClusterFeedback(deleteClusterFeedback); - } - - /// Renames the given cluster and/or sets the thumbnail of the given cluster. - /// - /// Requires either a [customName] or a [customFaceID]. If both are given, both are used. If neither are given, an error is thrown. - Future renameOrSetThumbnailCluster( - int personID, { - String? customName, - String? customFaceID, - }) async { - _logger.info( - 'renameOrSetThumbnailCluster called with personID $personID, customName $customName, and customFaceID $customFaceID', - ); - - if (customFaceID != null && - FaceDetectionRelative.isFaceIDEmpty(customFaceID)) { - _logger.severe( - "customFaceID $customFaceID is belongs to empty detection, unable to set as thumbnail of cluster!", - ); - customFaceID = null; - } - if (customName == null && customFaceID == null) { - _logger.severe( - "No name or faceID given, unable to rename or set thumbnail of cluster!", - ); - throw ArgumentError( - "No name or faceID given, unable to rename or set thumbnail of cluster!", - ); - } - - // Get the relevant cluster - final cluster = await _mlDatabase.getClusterResult(personID); - if (cluster == null) { - _logger.severe( - "No cluster found for personID $personID, unable to delete non-existent cluster!", - ); - throw ArgumentError( - "No cluster found for personID $personID, unable to delete non-existent cluster!", - ); - } - - // Update the cluster - if (customName != null) cluster.setUserDefinedName = customName; - if (customFaceID != null) cluster.setThumbnailFaceId = customFaceID; - - // Update the cluster in the database - await _mlDatabase.updateClusterResult(cluster); - - // Build the RenameOrCustomThumbnailClusterFeedback - final RenameOrCustomThumbnailClusterFeedback renameClusterFeedback = - RenameOrCustomThumbnailClusterFeedback( - medoid: cluster.medoid, - medoidDistanceThreshold: cluster.medoidDistanceThreshold, - customName: customName, - customThumbnailFaceId: customFaceID, - ); - - // TODO: maybe I should merge the two feedbacks if they are similar enough? - // Check if feedback doesn't already exist - final matchingFeedbacks = - await _mlDatabase.getAllMatchingClusterFeedback(renameClusterFeedback); - for (final matchingFeedback in matchingFeedbacks) { - // Update the current feedback wherever possible - renameClusterFeedback.customName ??= matchingFeedback.customName; - renameClusterFeedback.customThumbnailFaceId ??= - matchingFeedback.customThumbnailFaceId; - - // Delete the old feedback (since we want the user to be able to overwrite their earlier feedback) - await _mlDatabase.deleteClusterFeedback(matchingFeedback); - } - - // Save the RenameOrCustomThumbnailClusterFeedback to the database - await _mlDatabase.createClusterFeedback(renameClusterFeedback); - - // Return the updated cluster - return cluster; - } - - /// Merges the given clusters. The largest cluster is kept and the other clusters are deleted. - /// - /// Requires either a [clusters] or [personIDs]. If both are given, the [clusters] are used. - Future mergeClusters(List personIDs) async { - _logger.info( - 'mergeClusters called with personIDs $personIDs', - ); - - // Get the relevant clusters - final List clusters = - await _mlDatabase.getSelectedClusterResults(personIDs); - if (clusters.length <= 1) { - _logger.severe( - "${clusters.length} clusters found for personIDs $personIDs, unable to merge non-existent clusters!", - ); - throw ArgumentError( - "${clusters.length} clusters found for personIDs $personIDs, unable to merge non-existent clusters!", - ); - } - - // Find the largest cluster - clusters.sort((a, b) => b.clusterSize.compareTo(a.clusterSize)); - final ClusterResult largestCluster = clusters.first; - - // Now iterate through the clusters to be merged and deleted - for (var i = 1; i < clusters.length; i++) { - final ClusterResult clusterToBeMerged = clusters[i]; - - // Add the files and faces of the cluster to be merged to the largest cluster - largestCluster.addFileIDsAndFaceIDs( - clusterToBeMerged.fileIDsIncludingPotentialDuplicates, - clusterToBeMerged.faceIDs, - ); - - // TODO: maybe I should wrap the logic below in a separate function, since it's also used in renameOrSetThumbnailCluster - // Merge any names and thumbnails if the largest cluster doesn't have them - bool shouldCreateNamingFeedback = false; - String? nameToBeMerged; - String? thumbnailToBeMerged; - if (!largestCluster.hasUserDefinedName && - clusterToBeMerged.hasUserDefinedName) { - largestCluster.setUserDefinedName = clusterToBeMerged.userDefinedName!; - nameToBeMerged = clusterToBeMerged.userDefinedName!; - shouldCreateNamingFeedback = true; - } - if (!largestCluster.thumbnailFaceIdIsUserDefined && - clusterToBeMerged.thumbnailFaceIdIsUserDefined) { - largestCluster.setThumbnailFaceId = clusterToBeMerged.thumbnailFaceId; - thumbnailToBeMerged = clusterToBeMerged.thumbnailFaceId; - shouldCreateNamingFeedback = true; - } - if (shouldCreateNamingFeedback) { - final RenameOrCustomThumbnailClusterFeedback renameClusterFeedback = - RenameOrCustomThumbnailClusterFeedback( - medoid: largestCluster.medoid, - medoidDistanceThreshold: largestCluster.medoidDistanceThreshold, - customName: nameToBeMerged, - customThumbnailFaceId: thumbnailToBeMerged, - ); - // Check if feedback doesn't already exist - final matchingFeedbacks = await _mlDatabase - .getAllMatchingClusterFeedback(renameClusterFeedback); - for (final matchingFeedback in matchingFeedbacks) { - // Update the current feedback wherever possible - renameClusterFeedback.customName ??= matchingFeedback.customName; - renameClusterFeedback.customThumbnailFaceId ??= - matchingFeedback.customThumbnailFaceId; - - // Delete the old feedback (since we want the user to be able to overwrite their earlier feedback) - await _mlDatabase.deleteClusterFeedback(matchingFeedback); - } - - // Save the RenameOrCustomThumbnailClusterFeedback to the database - await _mlDatabase.createClusterFeedback(renameClusterFeedback); - } - - // Build the mergeClusterFeedback - final MergeClusterFeedback mergeClusterFeedback = MergeClusterFeedback( - medoid: clusterToBeMerged.medoid, - medoidDistanceThreshold: clusterToBeMerged.medoidDistanceThreshold, - medoidToMoveTo: largestCluster.medoid, - ); - - // Save the mergeClusterFeedback to the database and delete any old matching feedbacks - final matchingFeedbacks = - await _mlDatabase.getAllMatchingClusterFeedback(mergeClusterFeedback); - for (final matchingFeedback in matchingFeedbacks) { - await _mlDatabase.deleteClusterFeedback(matchingFeedback); - } - await _mlDatabase.createClusterFeedback(mergeClusterFeedback); - - // Delete the cluster from the database - await _mlDatabase.deleteClusterResult(clusterToBeMerged.personId); - } - - // TODO: should I update the medoid of this new cluster? My intuition says no, but I'm not sure. - // Update the largest cluster in the database - await _mlDatabase.updateClusterResult(largestCluster); - - // Return the merged cluster - return largestCluster; - } -} diff --git a/mobile/lib/services/machine_learning/face_ml/face_feedback.dart/feedback.dart b/mobile/lib/services/machine_learning/face_ml/face_feedback.dart/feedback.dart deleted file mode 100644 index 8b3eb3c6ad..0000000000 --- a/mobile/lib/services/machine_learning/face_ml/face_feedback.dart/feedback.dart +++ /dev/null @@ -1,34 +0,0 @@ -import "package:photos/models/ml/ml_versions.dart"; -import 'package:photos/services/machine_learning/face_ml/face_feedback.dart/feedback_types.dart'; -import "package:uuid/uuid.dart"; - -abstract class Feedback { - final FeedbackType type; - final String feedbackID; - final DateTime timestamp; - final int madeOnFaceMlVersion; - final int madeOnClusterMlVersion; - - get typeString => type.toValueString(); - - get timestampString => timestamp.toIso8601String(); - - Feedback( - this.type, { - String? feedbackID, - DateTime? timestamp, - int? madeOnFaceMlVersion, - int? madeOnClusterMlVersion, - }) : feedbackID = feedbackID ?? const Uuid().v4(), - timestamp = timestamp ?? DateTime.now(), - madeOnFaceMlVersion = madeOnFaceMlVersion ?? faceMlVersion, - madeOnClusterMlVersion = madeOnClusterMlVersion ?? clusterMlVersion; - - Map toJson(); - - String toJsonString(); - - // Feedback fromJson(Map json); - - // Feedback fromJsonString(String jsonString); -} diff --git a/mobile/lib/services/machine_learning/face_ml/face_feedback.dart/feedback_types.dart b/mobile/lib/services/machine_learning/face_ml/face_feedback.dart/feedback_types.dart deleted file mode 100644 index 8451d4790a..0000000000 --- a/mobile/lib/services/machine_learning/face_ml/face_feedback.dart/feedback_types.dart +++ /dev/null @@ -1,26 +0,0 @@ -enum FeedbackType { - removePhotosClusterFeedback, - addPhotosClusterFeedback, - deleteClusterFeedback, - mergeClusterFeedback, - renameOrCustomThumbnailClusterFeedback; // I have merged renameClusterFeedback and customThumbnailClusterFeedback, since I suspect they will be used together often - - factory FeedbackType.fromValueString(String value) { - switch (value) { - case 'deleteClusterFeedback': - return FeedbackType.deleteClusterFeedback; - case 'mergeClusterFeedback': - return FeedbackType.mergeClusterFeedback; - case 'renameOrCustomThumbnailClusterFeedback': - return FeedbackType.renameOrCustomThumbnailClusterFeedback; - case 'removePhotoClusterFeedback': - return FeedbackType.removePhotosClusterFeedback; - case 'addPhotoClusterFeedback': - return FeedbackType.addPhotosClusterFeedback; - default: - throw Exception('Invalid FeedbackType: $value'); - } - } - - String toValueString() => name; -} diff --git a/mobile/lib/services/machine_learning/face_ml/face_ml_result.dart b/mobile/lib/services/machine_learning/face_ml/face_ml_result.dart index 58fc72ac5a..892ce84834 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_ml_result.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_ml_result.dart @@ -2,20 +2,19 @@ import "dart:convert" show jsonEncode, jsonDecode; import "package:flutter/material.dart" show Size, debugPrint, immutable; import "package:logging/logging.dart"; -import "package:photos/db/ml_data_db.dart"; import "package:photos/models/file/file.dart"; import 'package:photos/models/ml/ml_typedefs.dart'; import "package:photos/models/ml/ml_versions.dart"; import 'package:photos/services/machine_learning/face_ml/face_alignment/alignment_result.dart'; import 'package:photos/services/machine_learning/face_ml/face_clustering/cosine_distance.dart'; import 'package:photos/services/machine_learning/face_ml/face_detection/detection.dart'; -import 'package:photos/services/machine_learning/face_ml/face_feedback.dart/cluster_feedback.dart'; import 'package:photos/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart'; import 'package:photos/services/machine_learning/face_ml/face_ml_methods.dart'; final _logger = Logger('ClusterResult_FaceMlResult'); // TODO: should I add [faceMlVersion] and [clusterMlVersion] to the [ClusterResult] class? +@Deprecated('We are now just storing the cluster results directly in DB') class ClusterResult { final int personId; String? userDefinedName; @@ -263,64 +262,6 @@ class ClusterResultBuilder { return (medoid!, kthDistance); } - Future _checkIfClusterIsDeleted() async { - assert(medoidAndThresholdCalculated); - - // Check if the medoid is the default medoid for deleted faces - if (cosineDistance(medoid, List.filled(medoid.length, 10.0)) < 0.001) { - return true; - } - - final tempFeedback = DeleteClusterFeedback( - medoid: medoid, - medoidDistanceThreshold: medoidDistanceThreshold, - ); - return await MlDataDB.instance - .doesSimilarClusterFeedbackExist(tempFeedback); - } - - Future _checkAndAddPhotos() async { - assert(medoidAndThresholdCalculated); - - final tempFeedback = AddPhotosClusterFeedback( - medoid: medoid, - medoidDistanceThreshold: medoidDistanceThreshold, - addedPhotoFileIDs: [], - ); - final allAddPhotosFeedbacks = - await MlDataDB.instance.getAllMatchingClusterFeedback(tempFeedback); - - for (final addPhotosFeedback in allAddPhotosFeedbacks) { - final fileIDsToAdd = addPhotosFeedback.addedPhotoFileIDs; - final faceIDsToAdd = fileIDsToAdd - .map((fileID) => FaceDetectionRelative.toFaceIDEmpty(fileID: fileID)) - .toList(); - addFileIDsAndFaceIDs(fileIDsToAdd, faceIDsToAdd); - } - } - - Future _checkAndAddCustomName() async { - assert(medoidAndThresholdCalculated); - - final tempFeedback = RenameOrCustomThumbnailClusterFeedback( - medoid: medoid, - medoidDistanceThreshold: medoidDistanceThreshold, - customName: 'test', - ); - final allRenameFeedbacks = - await MlDataDB.instance.getAllMatchingClusterFeedback(tempFeedback); - - for (final nameFeedback in allRenameFeedbacks) { - userDefinedName ??= nameFeedback.customName; - if (!thumbnailFaceIdIsUserDefined) { - thumbnailFaceId = nameFeedback.customThumbnailFaceId ?? thumbnailFaceId; - thumbnailFaceIdIsUserDefined = - nameFeedback.customThumbnailFaceId != null; - } - } - return; - } - void changeThumbnailFaceId(String faceId) { if (!faceIds.contains(faceId)) { throw Exception( @@ -335,113 +276,6 @@ class ClusterResultBuilder { fileIds.addAll(addedFileIDs); faceIds.addAll(addedFaceIDs); } - - static Future> buildClusters( - List clusterBuilders, - ) async { - final List deletedClusterIndices = []; - for (var i = 0; i < clusterBuilders.length; i++) { - final clusterBuilder = clusterBuilders[i]; - clusterBuilder.calculateAndSetMedoidAndThreshold(); - - // Check if the cluster has been deleted - if (await clusterBuilder._checkIfClusterIsDeleted()) { - deletedClusterIndices.add(i); - } - - await clusterBuilder._checkAndAddPhotos(); - } - - // Check if a cluster should be merged with another cluster - for (var i = 0; i < clusterBuilders.length; i++) { - // Don't check for clusters that have been deleted - if (deletedClusterIndices.contains(i)) { - continue; - } - final clusterBuilder = clusterBuilders[i]; - final List allMatchingMergeFeedback = - await MlDataDB.instance.getAllMatchingClusterFeedback( - MergeClusterFeedback( - medoid: clusterBuilder.medoid, - medoidDistanceThreshold: clusterBuilder.medoidDistanceThreshold, - medoidToMoveTo: clusterBuilder.medoid, - ), - ); - if (allMatchingMergeFeedback.isEmpty) { - continue; - } - // Merge the cluster with the first merge feedback - final mainFeedback = allMatchingMergeFeedback.first; - if (allMatchingMergeFeedback.length > 1) { - // This is the BUG!!!! - _logger.warning( - "There are ${allMatchingMergeFeedback.length} merge feedbacks for cluster ${clusterBuilder.personId}. Using the first one.", - ); - } - for (var j = 0; j < clusterBuilders.length; j++) { - if (i == j) continue; - final clusterBuilderToMergeTo = clusterBuilders[j]; - final distance = cosineDistance( - // BUG: it hasn't calculated the medoid for every clusterBuilder yet!!! - mainFeedback.medoidToMoveTo, - clusterBuilderToMergeTo.medoid, - ); - if (distance < mainFeedback.medoidDistanceThreshold || - distance < clusterBuilderToMergeTo.medoidDistanceThreshold) { - clusterBuilderToMergeTo.addFileIDsAndFaceIDs( - clusterBuilder.fileIds, - clusterBuilder.faceIds, - ); - deletedClusterIndices.add(i); - } - } - } - - final clusterResults = []; - for (var i = 0; i < clusterBuilders.length; i++) { - // Don't build the cluster if it has been deleted or merged - if (deletedClusterIndices.contains(i)) { - continue; - } - final clusterBuilder = clusterBuilders[i]; - // Check if the cluster has a custom name or thumbnail - await clusterBuilder._checkAndAddCustomName(); - - // Build the clusterResult - clusterResults.add( - ClusterResult( - personId: clusterBuilder.personId, - thumbnailFaceId: clusterBuilder.thumbnailFaceId, - fileIds: clusterBuilder.fileIds, - faceIds: clusterBuilder.faceIds, - medoid: clusterBuilder.medoid, - medoidDistanceThreshold: clusterBuilder.medoidDistanceThreshold, - userDefinedName: clusterBuilder.userDefinedName, - thumbnailFaceIdIsUserDefined: - clusterBuilder.thumbnailFaceIdIsUserDefined, - ), - ); - } - - return clusterResults; - } - - // TODO: This function should include the feedback from the user. Should also be nullable, since user might want to delete the cluster. - Future _buildSingleCluster() async { - calculateAndSetMedoidAndThreshold(); - if (await _checkIfClusterIsDeleted()) { - return null; - } - await _checkAndAddCustomName(); - return ClusterResult( - personId: personId, - thumbnailFaceId: thumbnailFaceId, - fileIds: fileIds, - faceIds: faceIds, - medoid: medoid, - medoidDistanceThreshold: medoidDistanceThreshold, - ); - } } @immutable diff --git a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart index 2d7608c959..d8c003afb4 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart @@ -14,7 +14,6 @@ import "package:onnxruntime/onnxruntime.dart"; import "package:photos/core/configuration.dart"; import "package:photos/core/event_bus.dart"; import "package:photos/db/files_db.dart"; -import "package:photos/db/ml_data_db.dart"; import "package:photos/events/diff_sync_complete_event.dart"; import "package:photos/extensions/list.dart"; import "package:photos/extensions/stop_watch.dart"; @@ -699,48 +698,6 @@ class FaceMlService { isImageIndexRunning = false; } - /// Analyzes the given image data by running the full pipeline using [analyzeImageInComputerAndImageIsolate] and stores the result in the database [MlDataDB]. - /// This function first checks if the image has already been analyzed (with latest ml version) and stored in the database. If so, it returns the stored result. - /// - /// 'enteFile': The ente file to analyze. - /// - /// Returns an immutable [FaceMlResult] instance containing the results of the analysis. The result is also stored in the database. - Future indexImage(EnteFile enteFile) async { - _logger.info( - "`indexImage` called on image with uploadedFileID ${enteFile.uploadedFileID}", - ); - _checkEnteFileForID(enteFile); - - // Check if the image has already been analyzed and stored in the database with the latest ml version - final existingResult = await _checkForExistingUpToDateResult(enteFile); - if (existingResult != null) { - return existingResult; - } - - // If the image has not been analyzed and stored in the database, analyze it and store the result in the database - _logger.info( - "Image with uploadedFileID ${enteFile.uploadedFileID} has not been analyzed and stored in the database. Analyzing it now.", - ); - FaceMlResult result; - try { - result = await analyzeImageInComputerAndImageIsolate(enteFile); - } catch (e, s) { - _logger.severe( - "`indexImage` failed on image with uploadedFileID ${enteFile.uploadedFileID}", - e, - s, - ); - throw GeneralFaceMlException( - "`indexImage` failed on image with uploadedFileID ${enteFile.uploadedFileID}", - ); - } - - // Store the result in the database - await MlDataDB.instance.createFaceMlResult(result); - - return result; - } - /// Analyzes the given image data by running the full pipeline (face detection, face alignment, face embedding). /// /// [enteFile] The ente file to analyze. @@ -1266,22 +1223,4 @@ class FaceMlService { indexedFileIds[id]! >= faceMlVersion; } - Future _checkForExistingUpToDateResult( - EnteFile enteFile, - ) async { - // Check if the image has already been analyzed and stored in the database - final existingResult = - await MlDataDB.instance.getFaceMlResult(enteFile.uploadedFileID!); - - // If the image has already been analyzed and stored in the database, return the stored result - if (existingResult != null) { - if (existingResult.mlVersion >= faceMlVersion) { - _logger.info( - "Image with uploadedFileID ${enteFile.uploadedFileID} has already been analyzed and stored in the database with the latest ml version. Returning the stored result.", - ); - return existingResult; - } - } - return null; - } } diff --git a/mobile/lib/services/machine_learning/face_ml/face_search_service.dart b/mobile/lib/services/machine_learning/face_ml/face_search_service.dart deleted file mode 100644 index 29bd467439..0000000000 --- a/mobile/lib/services/machine_learning/face_ml/face_search_service.dart +++ /dev/null @@ -1,123 +0,0 @@ -// import "dart:io" show File; -// import "dart:typed_data" show Uint8List; - -import "package:logging/logging.dart"; -import "package:photos/db/files_db.dart"; -import "package:photos/db/ml_data_db.dart"; -import "package:photos/models/file/file.dart"; -// import 'package:photos/utils/image_ml_isolate.dart'; -// import "package:photos/utils/thumbnail_util.dart"; - -class FaceSearchService { - final _logger = Logger("FaceSearchService"); - - final _mlDatabase = MlDataDB.instance; - final _filesDatabase = FilesDB.instance; - - // singleton pattern - FaceSearchService._privateConstructor(); - static final instance = FaceSearchService._privateConstructor(); - factory FaceSearchService() => instance; - - /// Returns the personIDs of all clustered people in the database. - Future> getAllPeople() async { - final peopleIds = await _mlDatabase.getAllClusterIds(); - return peopleIds; - } - - // /// Returns the thumbnail associated with a given personId. - // Future getPersonThumbnail(int personID) async { - // // get the cluster associated with the personID - // final cluster = await _mlDatabase.getClusterResult(personID); - // if (cluster == null) { - // _logger.warning( - // "No cluster found for personID $personID, unable to get thumbnail.", - // ); - // return null; - // } - - // // get the faceID and fileID you want to use to generate the thumbnail - // final String thumbnailFaceID = cluster.thumbnailFaceId; - // final int thumbnailFileID = cluster.thumbnailFileId; - - // // get the full file thumbnail - // final EnteFile enteFile = await _filesDatabase - // .getFilesFromIDs([thumbnailFileID]).then((value) => value.values.first); - // final File? fileThumbnail = await getThumbnailForUploadedFile(enteFile); - // if (fileThumbnail == null) { - // _logger.warning( - // "No full file thumbnail found for thumbnail faceID $thumbnailFaceID, unable to get thumbnail.", - // ); - // return null; - // } - - // // get the face detection for the thumbnail - // final thumbnailMlResult = - // await _mlDatabase.getFaceMlResult(thumbnailFileID); - // if (thumbnailMlResult == null) { - // _logger.warning( - // "No face ml result found for thumbnail faceID $thumbnailFaceID, unable to get thumbnail.", - // ); - // return null; - // } - // final detection = thumbnailMlResult.getDetectionForFaceId(thumbnailFaceID); - - // // create the thumbnail from the full file thumbnail and the face detection - // Uint8List faceThumbnail; - // try { - // faceThumbnail = await ImageMlIsolate.instance - // .generateFaceThumbnailsForImage( - // fileThumbnail.path, - // detection, - // ) - // .then((value) => value[0]); - // } catch (e, s) { - // _logger.warning( - // "Unable to generate face thumbnail for thumbnail faceID $thumbnailFaceID, unable to get thumbnail.", - // e, - // s, - // ); - // return null; - // } - - // return faceThumbnail; - // } - - /// Returns all files associated with a given personId. - Future> getFilesForPerson(int personID) async { - final fileIDs = await _mlDatabase.getClusterFileIds(personID); - - final Map files = - await _filesDatabase.getFilesFromIDs(fileIDs); - return files.values.toList(); - } - - Future> getFilesForIntersectOfPeople( - List personIDs, - ) async { - if (personIDs.length <= 1) { - _logger - .warning('Cannot get intersection of files for less than 2 people'); - return []; - } - - final Set fileIDsFirstCluster = await _mlDatabase - .getClusterFileIds(personIDs.first) - .then((value) => value.toSet()); - for (final personID in personIDs.sublist(1)) { - final fileIDsSingleCluster = - await _mlDatabase.getClusterFileIds(personID); - fileIDsFirstCluster.retainAll(fileIDsSingleCluster); - - // Early termination if intersection is empty - if (fileIDsFirstCluster.isEmpty) { - return []; - } - } - - final Map files = - await _filesDatabase.getFilesFromIDs(fileIDsFirstCluster.toList()); - - return files.values.toList(); - } -} diff --git a/mobile/lib/ui/viewer/actions/file_selection_actions_widget.dart b/mobile/lib/ui/viewer/actions/file_selection_actions_widget.dart index 869c8c317f..3da03fa055 100644 --- a/mobile/lib/ui/viewer/actions/file_selection_actions_widget.dart +++ b/mobile/lib/ui/viewer/actions/file_selection_actions_widget.dart @@ -402,7 +402,7 @@ class _FileSelectionActionsWidgetState ); // if (widget.type == GalleryType.cluster && widget.clusterID != null) { - if (widget.type == GalleryType.cluster) { + if (widget.type == GalleryType.cluster && widget.clusterID != null) { items.add( SelectionActionButton( labelText: 'Remove', diff --git a/mobile/lib/ui/viewer/actions/file_selection_overlay_bar.dart b/mobile/lib/ui/viewer/actions/file_selection_overlay_bar.dart index f0d258956d..c236f56a4c 100644 --- a/mobile/lib/ui/viewer/actions/file_selection_overlay_bar.dart +++ b/mobile/lib/ui/viewer/actions/file_selection_overlay_bar.dart @@ -12,6 +12,7 @@ class FileSelectionOverlayBar extends StatefulWidget { final Collection? collection; final Color? backgroundColor; final Person? person; + final int? clusterID; const FileSelectionOverlayBar( this.galleryType, @@ -19,6 +20,7 @@ class FileSelectionOverlayBar extends StatefulWidget { this.collection, this.backgroundColor, this.person, + this.clusterID, Key? key, }) : super(key: key); @@ -69,6 +71,7 @@ class _FileSelectionOverlayBarState extends State { galleryType: widget.galleryType, collection: widget.collection, person: widget.person, + clusterID: widget.clusterID, onCancel: () { if (widget.selectedFiles.files.isNotEmpty) { widget.selectedFiles.clearAll(); diff --git a/mobile/lib/ui/viewer/people/cluster_page.dart b/mobile/lib/ui/viewer/people/cluster_page.dart index 04a14d0120..a360e111f5 100644 --- a/mobile/lib/ui/viewer/people/cluster_page.dart +++ b/mobile/lib/ui/viewer/people/cluster_page.dart @@ -159,6 +159,7 @@ class _ClusterPageState extends State { FileSelectionOverlayBar( ClusterPage.overlayType, _selectedFiles, + clusterID: widget.cluserID, ), ], ),