From 5bd845d32b5d4f168a67f9495ba2d97051b33cc1 Mon Sep 17 00:00:00 2001 From: ashilkn Date: Mon, 13 May 2024 15:39:12 +0530 Subject: [PATCH 001/354] [mob][photos] Migrate to sqlite_async (1) --- mobile/lib/db/files_db.dart | 201 +++++++++++++++++------------------- 1 file changed, 93 insertions(+), 108 deletions(-) diff --git a/mobile/lib/db/files_db.dart b/mobile/lib/db/files_db.dart index 7022100b73..d61bbd2bdf 100644 --- a/mobile/lib/db/files_db.dart +++ b/mobile/lib/db/files_db.dart @@ -778,15 +778,13 @@ class FilesDB { // Files which user added to a collection manually but they are not // uploaded yet or files belonging to a collection which is marked for backup Future> getFilesPendingForUpload() async { - final db = await instance.database; - final results = await db.query( - filesTable, - where: - '($columnUploadedFileID IS NULL OR $columnUploadedFileID IS -1) AND ' - '$columnCollectionID IS NOT NULL AND $columnCollectionID IS NOT -1 AND ' - '$columnLocalID IS NOT NULL AND $columnLocalID IS NOT -1', - orderBy: '$columnCreationTime DESC', - groupBy: columnLocalID, + final db = await instance.sqliteAsyncDB; + final results = await db.getAll( + 'SELECT * FROM $filesTable WHERE ($columnUploadedFileID IS NULL OR ' + '$columnUploadedFileID IS -1) AND $columnCollectionID IS NOT NULL AND ' + '$columnCollectionID IS NOT -1 AND $columnLocalID IS NOT NULL AND ' + '$columnLocalID IS NOT -1 GROUP BY $columnLocalID ' + 'ORDER BY $columnCreationTime DESC', ); final files = convertToFiles(results); // future-safe filter just to ensure that the query doesn't end up returning files @@ -801,30 +799,22 @@ class FilesDB { } Future> getUnUploadedLocalFiles() async { - final db = await instance.database; - final results = await db.query( - filesTable, - where: - '($columnUploadedFileID IS NULL OR $columnUploadedFileID IS -1) AND $columnLocalID IS NOT NULL', - orderBy: '$columnCreationTime DESC', - groupBy: columnLocalID, + final db = await instance.sqliteAsyncDB; + final results = await db.getAll( + 'SELECT * FROM $filesTable WHERE ($columnUploadedFileID IS NULL OR ' + '$columnUploadedFileID IS -1) AND $columnLocalID IS NOT NULL ' + 'GROUP BY $columnLocalID ORDER BY $columnCreationTime DESC', ); return convertToFiles(results); } Future> getUploadedFileIDsToBeUpdated(int ownerID) async { - final db = await instance.database; - final rows = await db.query( - filesTable, - columns: [columnUploadedFileID], - where: '($columnLocalID IS NOT NULL AND $columnOwnerID = ? AND ' - '($columnUploadedFileID ' - 'IS NOT ' - 'NULL AND $columnUploadedFileID IS NOT -1) AND $columnUpdationTime IS NULL)', - whereArgs: [ownerID], - orderBy: '$columnCreationTime DESC', - distinct: true, - ); + final db = await instance.sqliteAsyncDB; + final rows = await db.getAll( + 'SELECT DISTINCT $columnUploadedFileID FROM $filesTable WHERE ' + '($columnLocalID IS NOT NULL AND $columnOwnerID = ? AND ' + '($columnUploadedFileID IS NOT NULL AND $columnUploadedFileID IS NOT -1) ' + 'AND $columnUpdationTime IS NULL) ORDER BY $columnCreationTime DESC '); final uploadedFileIDs = []; for (final row in rows) { uploadedFileIDs.add(row[columnUploadedFileID] as int); @@ -836,15 +826,11 @@ class FilesDB { int uploadedFileID, int userID, ) async { - final db = await instance.database; - final results = await db.query( - filesTable, - where: '$columnLocalID IS NOT NULL AND $columnOwnerID = ? AND ' - '$columnUploadedFileID = ?', - whereArgs: [ - userID, - uploadedFileID, - ], + final db = await instance.sqliteAsyncDB; + final results = await db.getAll( + 'SELECT * FROM $filesTable WHERE $columnLocalID IS NOT NULL AND ' + '$columnOwnerID = ? AND $columnUploadedFileID = ?', + [userID, uploadedFileID], ); if (results.isEmpty) { return []; @@ -853,14 +839,12 @@ class FilesDB { } Future> getExistingLocalFileIDs(int ownerID) async { - final db = await instance.database; - final rows = await db.query( - filesTable, - columns: [columnLocalID], - distinct: true, - where: '$columnLocalID IS NOT NULL AND ($columnOwnerID IS NULL OR ' - '$columnOwnerID = ?)', - whereArgs: [ownerID], + final db = await instance.sqliteAsyncDB; + final rows = await db.getAll( + 'SELECT DISTINCT $columnLocalID FROM $filesTable ' + 'WHERE $columnLocalID IS NOT NULL AND ($columnOwnerID IS NULL OR ' + '$columnOwnerID = ?)', + [ownerID], ); final result = {}; for (final row in rows) { @@ -870,16 +854,13 @@ class FilesDB { } Future> getLocalIDsMarkedForOrAlreadyUploaded(int ownerID) async { - final db = await instance.database; - final rows = await db.query( - filesTable, - columns: [columnLocalID], - distinct: true, - where: '$columnLocalID IS NOT NULL AND ($columnCollectionID IS NOT NULL ' - 'AND ' - '$columnCollectionID != -1) AND ($columnOwnerID = ? OR ' - '$columnOwnerID IS NULL)', - whereArgs: [ownerID], + final db = await instance.sqliteAsyncDB; + final rows = await db.getAll( + 'SELECT DISTINCT $columnLocalID FROM $filesTable ' + 'WHERE $columnLocalID IS NOT NULL AND ($columnCollectionID IS NOT NULL ' + 'AND $columnCollectionID != -1) AND ($columnOwnerID = ? OR ' + '$columnOwnerID IS NULL)', + [ownerID], ); final result = {}; for (final row in rows) { @@ -889,12 +870,11 @@ class FilesDB { } Future> getLocalFileIDsForCollection(int collectionID) async { - final db = await instance.database; - final rows = await db.query( - filesTable, - columns: [columnLocalID], - where: '$columnLocalID IS NOT NULL AND $columnCollectionID = ?', - whereArgs: [collectionID], + final db = await instance.sqliteAsyncDB; + final rows = await db.getAll( + 'SELECT $columnLocalID FROM $filesTable ' + 'WHERE $columnLocalID IS NOT NULL AND $columnCollectionID = ?', + [collectionID], ); final result = {}; for (final row in rows) { @@ -905,17 +885,17 @@ class FilesDB { // Sets the collectionID for the files with given LocalIDs if the // corresponding file entries are not already mapped to some other collection - Future setCollectionIDForUnMappedLocalFiles( + Future setCollectionIDForUnMappedLocalFiles( int collectionID, Set localIDs, ) async { - final db = await instance.database; + final db = await instance.sqliteAsyncDB; String inParam = ""; for (final localID in localIDs) { inParam += "'" + localID + "',"; } inParam = inParam.substring(0, inParam.length - 1); - return await db.rawUpdate( + await db.execute( ''' UPDATE $filesTable SET $columnCollectionID = $collectionID @@ -925,7 +905,7 @@ class FilesDB { ); } - Future markFilesForReUpload( + Future markFilesForReUpload( int ownerID, String localID, String? title, @@ -934,22 +914,32 @@ class FilesDB { int modificationTime, FileType fileType, ) async { - final db = await instance.database; - return await db.update( - filesTable, - { - columnTitle: title, - columnLatitude: location?.latitude, - columnLongitude: location?.longitude, - columnCreationTime: creationTime, - columnModificationTime: modificationTime, - // #hack reset updation time to null for re-upload - columnUpdationTime: null, - columnFileType: getInt(fileType), - }, - where: - '$columnLocalID = ? AND ($columnOwnerID = ? OR $columnOwnerID IS NULL)', - whereArgs: [localID, ownerID], + final db = await instance.sqliteAsyncDB; + + await db.execute( + ''' + UPDATE $filesTable + SET $columnTitle = ?, + $columnLatitude = ?, + $columnLongitude = ?, + $columnCreationTime = ?, + $columnModificationTime = ?, + $columnUpdationTime = NULL, + $columnFileType = ? + WHERE $columnLocalID = ? AND ($columnOwnerID = ? OR $columnOwnerID IS NULL); + ''', + [ + title, + location?.latitude, + location?.longitude, + localID, + creationTime, + modificationTime, + null, + getInt(fileType), + localID, + ownerID, + ], ); } @@ -964,12 +954,12 @@ class FilesDB { required String title, required String deviceFolder, }) async { - final db = await instance.database; + final db = await instance.sqliteAsyncDB; // on iOS, match using localID and fileType. title can either match or // might be null based on how the file was imported - String whereClause = ''' ($columnOwnerID = ? OR $columnOwnerID IS NULL) AND - $columnLocalID = ? AND $columnFileType = ? AND - ($columnTitle=? OR $columnTitle IS NULL) '''; + String query = '''SELECT * FROM $filesTable WHERE ($columnOwnerID = ? + OR $columnOwnerID IS NULL) AND $columnLocalID = ? + AND $columnFileType = ? AND ($columnTitle=? OR $columnTitle IS NULL) '''; List whereArgs = [ ownerID, localID, @@ -977,9 +967,9 @@ class FilesDB { title, ]; if (Platform.isAndroid) { - whereClause = ''' ($columnOwnerID = ? OR $columnOwnerID IS NULL) AND - $columnLocalID = ? AND $columnFileType = ? AND $columnTitle=? AND $columnDeviceFolder= ? - '''; + query = '''SELECT * FROM $filesTable WHERE ($columnOwnerID = ? OR + $columnOwnerID IS NULL) AND $columnLocalID = ? AND $columnFileType = ? + AND $columnTitle=? AND $columnDeviceFolder= ? '''; whereArgs = [ ownerID, localID, @@ -989,10 +979,9 @@ class FilesDB { ]; } - final rows = await db.query( - filesTable, - where: whereClause, - whereArgs: whereArgs, + final rows = await db.getAll( + query, + whereArgs, ); return convertToFiles(rows); @@ -1030,14 +1019,12 @@ class FilesDB { if (fileType == FileType.livePhoto && hashData.zipHash != null) { inParam += ",'${hashData.zipHash}'"; } - final db = await instance.database; - final rows = await db.query( - filesTable, - where: '($columnUploadedFileID != NULL OR $columnUploadedFileID != -1) ' - 'AND $columnOwnerID = ? AND $columnFileType =' - ' ? ' - 'AND $columnHash IN ($inParam)', - whereArgs: [ + final db = await instance.sqliteAsyncDB; + final rows = await db.getAll( + 'SELECT * FROM $filesTable WHERE $columnUploadedFileID != NULL OR ' + '$columnUploadedFileID != -1) AND $columnOwnerID = ? AND ' + '$columnFileType = ? AND $columnHash IN ($inParam)', + [ ownerID, getInt(fileType), ], @@ -1371,10 +1358,9 @@ class FilesDB { inParam += "'" + id.toString() + "',"; } inParam = inParam.substring(0, inParam.length - 1); - final db = await instance.database; - final results = await db.query( - filesTable, - where: '$columnUploadedFileID IN ($inParam)', + final db = await instance.sqliteAsyncDB; + final results = await db.getAll( + 'SELECT * FROM $filesTable WHERE $columnUploadedFileID IN ($inParam)', ); final files = convertToFiles(results); for (final file in files) { @@ -1393,10 +1379,9 @@ class FilesDB { inParam += "'" + id.toString() + "',"; } inParam = inParam.substring(0, inParam.length - 1); - final db = await instance.database; - final results = await db.query( - filesTable, - where: '$columnGeneratedID IN ($inParam)', + final db = await instance.sqliteAsyncDB; + final results = await db.getAll( + 'SELECT * FROM $filesTable WHERE $columnGeneratedID IN ($inParam)', ); final files = convertToFiles(results); for (final file in files) { From 3a0882a1a9fcef9e09b985e31819edf52b38a4fb Mon Sep 17 00:00:00 2001 From: ashilkn Date: Mon, 13 May 2024 17:57:22 +0530 Subject: [PATCH 002/354] [mob][photos] Migrate to sqlite_async (2): Migrate all update queries in filesDB --- mobile/lib/db/files_db.dart | 94 ++++++++++++------- .../services/local_file_update_service.dart | 5 +- 2 files changed, 62 insertions(+), 37 deletions(-) diff --git a/mobile/lib/db/files_db.dart b/mobile/lib/db/files_db.dart index d61bbd2bdf..88c58a7ca1 100644 --- a/mobile/lib/db/files_db.dart +++ b/mobile/lib/db/files_db.dart @@ -1032,33 +1032,32 @@ class FilesDB { return convertToFiles(rows); } - Future update(EnteFile file) async { - final db = await instance.database; - return await db.update( - filesTable, - _getRowForFile(file), - where: '$columnGeneratedID = ?', - whereArgs: [file.generatedID], + Future update(EnteFile file) async { + final db = await instance.sqliteAsyncDB; + final setClause = _getSetClauseForFile(file); + await db.execute( + 'UPDATE $filesTable SET ' + '$setClause WHERE $columnGeneratedID = ?', + [file.generatedID], ); } - Future updateUploadedFileAcrossCollections(EnteFile file) async { - final db = await instance.database; - return await db.update( - filesTable, - _getRowForFileWithoutCollection(file), - where: '$columnUploadedFileID = ?', - whereArgs: [file.uploadedFileID], + Future updateUploadedFileAcrossCollections(EnteFile file) async { + final db = await instance.sqliteAsyncDB; + final setClause = _getSetClauseForFileWithoutCollection(file); + await db.execute( + 'UPDATE $filesTable SET ' + '$setClause WHERE $columnUploadedFileID = ?', + [file.uploadedFileID], ); } - Future updateLocalIDForUploaded(int uploadedID, String localID) async { - final db = await instance.database; - return await db.update( - filesTable, - {columnLocalID: localID}, - where: '$columnUploadedFileID = ? AND $columnLocalID IS NULL', - whereArgs: [uploadedID], + Future updateLocalIDForUploaded(int uploadedID, String localID) async { + final db = await instance.sqliteAsyncDB; + await db.execute( + 'UPDATE $filesTable SET $columnLocalID = ? WHERE $columnUploadedFileID = ?' + ' AND $columnLocalID IS NULL', + [localID, uploadedID], ); } @@ -1123,8 +1122,8 @@ class FilesDB { inParam += "'" + localID + "',"; } inParam = inParam.substring(0, inParam.length - 1); - final db = await instance.database; - await db.rawQuery( + final db = await instance.sqliteAsyncDB; + await db.execute( ''' UPDATE $filesTable SET $columnLocalID = NULL @@ -1323,8 +1322,8 @@ class FilesDB { inParam += "'" + localID + "',"; } inParam = inParam.substring(0, inParam.length - 1); - final db = await instance.database; - await db.rawUpdate( + final db = await instance.sqliteAsyncDB; + await db.execute( ''' UPDATE $filesTable SET $columnUpdationTime = NULL @@ -1552,17 +1551,24 @@ class FilesDB { if (uploadedFileIDToSize.isEmpty) { return; } - final db = await instance.database; - final batch = db.batch(); + final db = await instance.sqliteAsyncDB; + final parameterSets = >[]; + for (final uploadedFileID in uploadedFileIDToSize.keys) { - batch.update( - filesTable, - {columnFileSize: uploadedFileIDToSize[uploadedFileID]}, - where: '$columnUploadedFileID = ?', - whereArgs: [uploadedFileID], - ); + parameterSets.add([ + uploadedFileIDToSize[uploadedFileID], + uploadedFileID, + ]); } - await batch.commit(noResult: true); + + await db.executeBatch( + ''' + UPDATE $filesTable + SET $columnFileSize = ? + WHERE $columnUploadedFileID = ?; + ''', + parameterSets, + ); } Future> getAllFilesFromDB( @@ -1661,6 +1667,26 @@ class FilesDB { return convertToFiles(results); } + String _getSetClauseForFile(EnteFile file) { + final row = _getRowForFile(file); + final setClause = []; + for (int i = 0; i < row.entries.length; i++) { + final entry = row.entries.elementAt(i); + setClause.add('${entry.key} = ${entry.value}'); + } + return setClause.join(', '); + } + + String _getSetClauseForFileWithoutCollection(EnteFile file) { + final row = _getRowForFileWithoutCollection(file); + final setClause = []; + for (int i = 0; i < row.entries.length; i++) { + final entry = row.entries.elementAt(i); + setClause.add('${entry.key} = ${entry.value}'); + } + return setClause.join(', '); + } + Map _getRowForFile(EnteFile file) { final row = {}; if (file.generatedID != null) { diff --git a/mobile/lib/services/local_file_update_service.dart b/mobile/lib/services/local_file_update_service.dart index e00ac6c459..ce5a9080af 100644 --- a/mobile/lib/services/local_file_update_service.dart +++ b/mobile/lib/services/local_file_update_service.dart @@ -193,7 +193,7 @@ class LocalFileUpdateService { } else if (e.reason == InvalidReason.imageToLivePhotoTypeChanged) { fileType = FileType.livePhoto; } - final int count = await FilesDB.instance.markFilesForReUpload( + await FilesDB.instance.markFilesForReUpload( userID, file.localID!, file.title, @@ -202,8 +202,7 @@ class LocalFileUpdateService { file.modificationTime!, fileType, ); - _logger.fine('fileType changed for ${file.tag} to ${e.reason} for ' - '$count files'); + _logger.fine('fileType changed for ${file.tag} to ${e.reason} for '); } else { _logger.severe("failed to check hash: invalid file ${file.tag}", e); } From 8fcd05b95f5e3a6417eadb3fd372a729b1c96b0c Mon Sep 17 00:00:00 2001 From: ashilkn Date: Mon, 13 May 2024 18:29:01 +0530 Subject: [PATCH 003/354] [mob][photos] Migrate to sqlite_async (3) --- mobile/lib/db/files_db.dart | 122 +++++++++++++++++------------------- 1 file changed, 57 insertions(+), 65 deletions(-) diff --git a/mobile/lib/db/files_db.dart b/mobile/lib/db/files_db.dart index 88c58a7ca1..1468f961d2 100644 --- a/mobile/lib/db/files_db.dart +++ b/mobile/lib/db/files_db.dart @@ -1,3 +1,4 @@ +import "dart:async"; import "dart:io"; import "package:computer/computer.dart"; @@ -1061,57 +1062,53 @@ class FilesDB { ); } - Future delete(int uploadedFileID) async { - final db = await instance.database; - return db.delete( - filesTable, - where: '$columnUploadedFileID =?', - whereArgs: [uploadedFileID], + Future deleteByGeneratedID(int genID) async { + final db = await instance.sqliteAsyncDB; + + await db.execute( + 'DELETE FROM $filesTable WHERE $columnGeneratedID = ?', + [genID], ); } - Future deleteByGeneratedID(int genID) async { - final db = await instance.database; - return db.delete( - filesTable, - where: '$columnGeneratedID =?', - whereArgs: [genID], + Future deleteMultipleUploadedFiles(List uploadedFileIDs) async { + final db = await instance.sqliteAsyncDB; + final inParam = uploadedFileIDs.join(','); + + await db.execute( + 'DELETE FROM $filesTable WHERE $columnUploadedFileID IN ($inParam)', ); } - Future deleteMultipleUploadedFiles(List uploadedFileIDs) async { - final db = await instance.database; - return await db.delete( - filesTable, - where: '$columnUploadedFileID IN (${uploadedFileIDs.join(', ')})', - ); - } - - Future deleteMultipleByGeneratedIDs(List generatedIDs) async { + Future deleteMultipleByGeneratedIDs(List generatedIDs) async { if (generatedIDs.isEmpty) { - return 0; + return; } - final db = await instance.database; - return await db.delete( - filesTable, - where: '$columnGeneratedID IN (${generatedIDs.join(', ')})', + + final db = await instance.sqliteAsyncDB; + final inParam = generatedIDs.join(','); + + await db.execute( + 'DELETE FROM $filesTable WHERE $columnGeneratedID IN ($inParam)', ); } - Future deleteLocalFile(EnteFile file) async { - final db = await instance.database; + Future deleteLocalFile(EnteFile file) async { + final db = await instance.sqliteAsyncDB; if (file.localID != null) { // delete all files with same local ID - return db.delete( - filesTable, - where: '$columnLocalID =?', - whereArgs: [file.localID], + unawaited( + db.execute( + 'DELETE FROM $filesTable WHERE $columnLocalID = ?', + [file.localID], + ), ); } else { - return db.delete( - filesTable, - where: '$columnGeneratedID =?', - whereArgs: [file.generatedID], + unawaited( + db.execute( + 'DELETE FROM $filesTable WHERE $columnGeneratedID = ?', + [file.generatedID], + ), ); } } @@ -1138,37 +1135,34 @@ class FilesDB { inParam += "'" + localID + "',"; } inParam = inParam.substring(0, inParam.length - 1); - final db = await instance.database; - final results = await db.query( - filesTable, - where: '$columnLocalID IN ($inParam)', + final db = await instance.sqliteAsyncDB; + final results = await db.getAll( + ''' + SELECT * FROM $filesTable + WHERE $columnLocalID IN ($inParam); + ''', ); return convertToFiles(results); } - Future deleteUnSyncedLocalFiles(List localIDs) async { + Future deleteUnSyncedLocalFiles(List localIDs) async { String inParam = ""; for (final localID in localIDs) { inParam += "'" + localID + "',"; } inParam = inParam.substring(0, inParam.length - 1); - final db = await instance.database; - return db.delete( - filesTable, - where: - '($columnUploadedFileID is NULL OR $columnUploadedFileID = -1 ) AND $columnLocalID IN ($inParam)', - ); - } - - Future deleteFromCollection(int uploadedFileID, int collectionID) async { - final db = await instance.database; - return db.delete( - filesTable, - where: '$columnUploadedFileID = ? AND $columnCollectionID = ?', - whereArgs: [uploadedFileID, collectionID], + final db = await instance.sqliteAsyncDB; + unawaited( + db.execute( + ''' + DELETE FROM $filesTable + WHERE ($columnUploadedFileID is NULL OR $columnUploadedFileID = -1 ) AND $columnLocalID IN ($inParam) + ''', + ), ); } + /// Uses int in return value. Future deleteFilesFromCollection( int collectionID, List uploadedFileIDs, @@ -1183,9 +1177,9 @@ class FilesDB { } Future collectionFileCount(int collectionID) async { - final db = await instance.database; + final db = await instance.sqliteAsyncDB; final count = Sqflite.firstIntValue( - await db.rawQuery( + await db.execute( 'SELECT COUNT(*) FROM $filesTable where $columnCollectionID = ' '$collectionID AND $columnUploadedFileID IS NOT -1', ), @@ -1198,15 +1192,13 @@ class FilesDB { int ownerID, Set hiddenCollections, ) async { - final db = await instance.database; - final count = Sqflite.firstIntValue( - await db.rawQuery( - 'SELECT COUNT(distinct($columnUploadedFileID)) FROM $filesTable where ' - '$columnMMdVisibility' - ' = $visibility AND $columnOwnerID = $ownerID AND $columnCollectionID NOT IN (${hiddenCollections.join(', ')})', - ), + final db = await instance.sqliteAsyncDB; + final count = await db.execute( + 'SELECT COUNT(distinct($columnUploadedFileID)) as COUNT FROM $filesTable where ' + '$columnMMdVisibility' + ' = $visibility AND $columnOwnerID = $ownerID AND $columnCollectionID NOT IN (${hiddenCollections.join(', ')})', ); - return count ?? 0; + return count.first['COUNT'] as int; } Future deleteCollection(int collectionID) async { From ff14eb1d5a34ab5392869a631880dc721bf8b1b4 Mon Sep 17 00:00:00 2001 From: ashilkn Date: Tue, 14 May 2024 14:59:03 +0530 Subject: [PATCH 004/354] [mob][photos] Migrate to sqlite_async (4) --- mobile/lib/db/files_db.dart | 264 ++++++++++++++++++++---------------- 1 file changed, 148 insertions(+), 116 deletions(-) diff --git a/mobile/lib/db/files_db.dart b/mobile/lib/db/files_db.dart index 1468f961d2..887932250b 100644 --- a/mobile/lib/db/files_db.dart +++ b/mobile/lib/db/files_db.dart @@ -1201,32 +1201,36 @@ class FilesDB { return count.first['COUNT'] as int; } - Future deleteCollection(int collectionID) async { - final db = await instance.database; - return db.delete( - filesTable, - where: '$columnCollectionID = ?', - whereArgs: [collectionID], + Future deleteCollection(int collectionID) async { + final db = await instance.sqliteAsyncDB; + unawaited( + db.execute( + 'DELETE FROM $filesTable WHERE $columnCollectionID = ?', + [collectionID], + ), ); } - Future removeFromCollection(int collectionID, List fileIDs) async { - final db = await instance.database; - return db.delete( - filesTable, - where: - '$columnCollectionID =? AND $columnUploadedFileID IN (${fileIDs.join(', ')})', - whereArgs: [collectionID], + Future removeFromCollection(int collectionID, List fileIDs) async { + final db = await instance.sqliteAsyncDB; + final inParam = fileIDs.join(','); + unawaited( + db.execute( + ''' + DELETE FROM $filesTable + WHERE $columnCollectionID = ? AND $columnUploadedFileID IN ($inParam); + ''', + [collectionID], + ), ); } Future> getPendingUploadForCollection(int collectionID) async { - final db = await instance.database; - final results = await db.query( - filesTable, - where: '$columnCollectionID = ? AND ($columnUploadedFileID IS NULL OR ' - '$columnUploadedFileID = -1)', - whereArgs: [collectionID], + final db = await instance.sqliteAsyncDB; + final results = await db.getAll( + 'SELECT * FROM $filesTable WHERE $columnCollectionID = ? AND ' + '($columnUploadedFileID IS NULL OR $columnUploadedFileID = -1)', + [collectionID], ); return convertToFiles(results); } @@ -1242,8 +1246,8 @@ class FilesDB { } } inParam = inParam.substring(0, inParam.length - 1); - final db = await instance.database; - final rows = await db.rawQuery( + final db = await instance.sqliteAsyncDB; + final rows = await db.execute( ''' SELECT $columnLocalID FROM $filesTable @@ -1262,8 +1266,8 @@ class FilesDB { // creationTime of the files in the collection. Future> getCollectionIDToMaxCreationTime() async { final enteWatch = EnteWatch("getCollectionIDToMaxCreationTime")..start(); - final db = await instance.database; - final rows = await db.rawQuery( + final db = await instance.sqliteAsyncDB; + final rows = await db.execute( ''' SELECT $columnCollectionID, MAX($columnCreationTime) AS max_creation_time FROM $filesTable @@ -1288,16 +1292,17 @@ class FilesDB { int collectionID, bool sortAsc, ) async { - final db = await instance.database; + final db = await instance.sqliteAsyncDB; final order = sortAsc ? 'ASC' : 'DESC'; - final rows = await db.query( - filesTable, - where: '$columnCollectionID = ? AND ($columnUploadedFileID IS NOT NULL ' - 'AND $columnUploadedFileID IS NOT -1)', - whereArgs: [collectionID], - orderBy: - '$columnCreationTime ' + order + ', $columnModificationTime ' + order, - limit: 1, + final rows = await db.getAll( + ''' + SELECT * FROM $filesTable + WHERE $columnCollectionID = ? AND ($columnUploadedFileID IS NOT NULL + AND $columnUploadedFileID IS NOT -1) + ORDER BY $columnCreationTime $order, $columnModificationTime $order + LIMIT 1; + ''', + [collectionID], ); if (rows.isEmpty) { return null; @@ -1329,12 +1334,11 @@ class FilesDB { int uploadedFileID, int collectionID, ) async { - final db = await instance.database; - final rows = await db.query( - filesTable, - where: '$columnUploadedFileID = ? AND $columnCollectionID = ?', - whereArgs: [uploadedFileID, collectionID], - limit: 1, + final db = await instance.sqliteAsyncDB; + final rows = await db.getAll( + 'SELECT * FROM $filesTable WHERE $columnUploadedFileID = ? AND ' + '$columnCollectionID = ? LIMIT 1', + [uploadedFileID, collectionID], ); return rows.isNotEmpty; } @@ -1393,10 +1397,9 @@ class FilesDB { inParam += "'" + id.toString() + "',"; } inParam = inParam.substring(0, inParam.length - 1); - final db = await instance.database; - final results = await db.query( - filesTable, - where: '$columnUploadedFileID IN ($inParam)', + final db = await instance.sqliteAsyncDB; + final results = await db.getAll( + 'SELECT * FROM $filesTable WHERE $columnUploadedFileID IN ($inParam)', ); final files = convertToFiles(results); for (EnteFile eachFile in files) { @@ -1411,13 +1414,13 @@ class FilesDB { Future> getAllCollectionIDsOfFile( int uploadedFileID, ) async { - final db = await instance.database; - final results = await db.query( - filesTable, - where: '$columnUploadedFileID = ? AND $columnCollectionID != -1', - columns: [columnCollectionID], - whereArgs: [uploadedFileID], - distinct: true, + final db = await instance.sqliteAsyncDB; + final results = await db.getAll( + ''' + SELECT DISTINCT $columnCollectionID FROM $filesTable + WHERE $columnUploadedFileID = ? AND $columnCollectionID != -1 + ''', + [uploadedFileID], ); final collectionIDsOfFile = {}; for (var result in results) { @@ -1446,14 +1449,13 @@ class FilesDB { int cutOffTime, int ownerID, ) async { - final db = await instance.database; - final rows = await db.query( - filesTable, - columns: [columnGeneratedID], - distinct: true, - where: - '$columnCreationTime <= ? AND ($columnOwnerID IS NULL OR $columnOwnerID = ?)', - whereArgs: [cutOffTime, ownerID], + final db = await instance.sqliteAsyncDB; + final rows = await db.getAll( + ''' + SELECT DISTINCT $columnGeneratedID FROM $filesTable + WHERE $columnCreationTime <= ? AND ($columnOwnerID IS NULL OR $columnOwnerID = ?) + ''', + [cutOffTime, ownerID], ); final result = []; for (final row in rows) { @@ -1465,15 +1467,14 @@ class FilesDB { // For givenUserID, get List of unique LocalIDs for files which are // uploaded by the given user and location is missing Future> getLocalIDsForFilesWithoutLocation(int ownerID) async { - final db = await instance.database; - final rows = await db.query( - filesTable, - columns: [columnLocalID], - distinct: true, - where: '$columnOwnerID = ? AND $columnLocalID IS NOT NULL AND ' - '($columnLatitude IS NULL OR ' - '$columnLongitude IS NULL OR $columnLongitude = 0.0 or $columnLongitude = 0.0)', - whereArgs: [ownerID], + final db = await instance.sqliteAsyncDB; + final rows = await db.getAll( + ''' + SELECT DISTINCT $columnLocalID FROM $filesTable + WHERE $columnOwnerID = ? AND $columnLocalID IS NOT NULL AND + ($columnLatitude IS NULL OR $columnLongitude IS NULL OR $columnLatitude = 0.0 or $columnLongitude = 0.0) + ''', + [ownerID], ); final result = []; for (final row in rows) { @@ -1484,13 +1485,13 @@ class FilesDB { // For a given userID, return unique uploadedFileId for the given userID Future> getUploadIDsWithMissingSize(int userId) async { - final db = await instance.database; - final rows = await db.query( - filesTable, - columns: [columnUploadedFileID], - distinct: true, - where: '$columnOwnerID = ? AND $columnFileSize IS NULL', - whereArgs: [userId], + final db = await instance.sqliteAsyncDB; + final rows = await db.getAll( + ''' + SELECT DISTINCT $columnUploadedFileID FROM $filesTable + WHERE $columnOwnerID = ? AND $columnFileSize IS NULL + ''', + [userId], ); final result = []; for (final row in rows) { @@ -1501,14 +1502,13 @@ class FilesDB { // For a given userID, return unique localID for all uploaded live photos Future> getLivePhotosForUser(int userId) async { - final db = await instance.database; - final rows = await db.query( - filesTable, - columns: [columnLocalID], - distinct: true, - where: '$columnOwnerID = ? AND ' - '$columnFileType = ? AND $columnLocalID IS NOT NULL', - whereArgs: [userId, getInt(FileType.livePhoto)], + final db = await instance.sqliteAsyncDB; + final rows = await db.getAll( + ''' + SELECT DISTINCT $columnLocalID FROM $filesTable + WHERE $columnOwnerID = ? AND $columnFileType = ? AND $columnLocalID IS NOT NULL + ''', + [userId, getInt(FileType.livePhoto)], ); final result = []; for (final row in rows) { @@ -1518,15 +1518,16 @@ class FilesDB { } Future> getLocalFilesBackedUpWithoutLocation(int userId) async { - final db = await instance.database; - final rows = await db.query( - filesTable, - columns: [columnLocalID], - distinct: true, - where: - '$columnOwnerID = ? AND $columnLocalID IS NOT NULL AND ($columnUploadedFileID IS NOT NULL AND $columnUploadedFileID IS NOT -1) ' - 'AND ($columnLatitude IS NULL OR $columnLongitude IS NULL OR $columnLongitude = 0.0 or $columnLongitude = 0.0)', - whereArgs: [userId], + final db = await instance.sqliteAsyncDB; + final rows = await db.getAll( + ''' + SELECT DISTINCT $columnLocalID FROM $filesTable + WHERE $columnOwnerID = ? AND $columnLocalID IS NOT NULL AND + ($columnUploadedFileID IS NOT NULL AND $columnUploadedFileID IS NOT -1) + AND ($columnLatitude IS NULL OR $columnLongitude IS NULL OR + $columnLatitude = 0.0 or $columnLongitude = 0.0) + ''', + [userId], ); final result = []; for (final row in rows) { @@ -1585,9 +1586,13 @@ class FilesDB { } Future> fetchFilesCountbyType(int userID) async { - final db = await instance.database; - final result = await db.rawQuery( - "SELECT $columnFileType, COUNT(DISTINCT $columnUploadedFileID) FROM $filesTable WHERE $columnUploadedFileID != -1 AND $columnOwnerID == $userID GROUP BY $columnFileType", + final db = await instance.sqliteAsyncDB; + final result = await db.execute( + ''' + SELECT $columnFileType, COUNT(DISTINCT $columnUploadedFileID) + FROM $filesTable WHERE $columnUploadedFileID != -1 AND + $columnOwnerID == $userID GROUP BY $columnFileType + ''', ); final filesCount = {}; @@ -1606,18 +1611,20 @@ class FilesDB { bool? asc, required DBFilterOptions? filterOptions, }) async { - final db = await instance.database; + final db = await instance.sqliteAsyncDB; final order = (asc ?? false ? 'ASC' : 'DESC'); - final results = await db.query( - filesTable, - where: - '$columnLatitude IS NOT NULL AND $columnLongitude IS NOT NULL AND ($columnLatitude IS NOT 0 OR $columnLongitude IS NOT 0)' - ' AND $columnCreationTime >= ? AND $columnCreationTime <= ?' - ' AND ($columnLocalID IS NOT NULL OR ($columnCollectionID IS NOT NULL AND $columnCollectionID IS NOT -1))', - whereArgs: [startTime, endTime], - orderBy: - '$columnCreationTime ' + order + ', $columnModificationTime ' + order, - limit: limit, + final results = await db.getAll( + ''' + SELECT * FROM $filesTable + WHERE $columnLatitude IS NOT NULL AND $columnLongitude IS NOT NULL AND + ($columnLatitude IS NOT 0 OR $columnLongitude IS NOT 0) AND + $columnCreationTime >= ? AND $columnCreationTime <= ? AND + ($columnLocalID IS NOT NULL OR ($columnCollectionID IS NOT NULL AND + $columnCollectionID IS NOT -1)) + ORDER BY $columnCreationTime $order, $columnModificationTime $order + LIMIT $limit + ''', + [startTime, endTime], ); final files = convertToFiles(results); final List filteredFiles = @@ -1626,13 +1633,14 @@ class FilesDB { } Future> getOwnedFileIDs(int ownerID) async { - final db = await instance.database; - final results = await db.query( - filesTable, - columns: [columnUploadedFileID], - where: - '($columnOwnerID = $ownerID AND $columnUploadedFileID IS NOT NULL AND $columnUploadedFileID IS NOT -1)', - distinct: true, + final db = await instance.sqliteAsyncDB; + final results = await db.getAll( + ''' + SELECT DISTINCT $columnUploadedFileID FROM $filesTable + WHERE $columnOwnerID = ? AND $columnUploadedFileID IS NOT NULL AND + $columnUploadedFileID IS NOT -1) + ''', + [ownerID], ); final ids = []; for (final result in results) { @@ -1642,16 +1650,17 @@ class FilesDB { } Future> getUploadedFiles(List uploadedIDs) async { - final db = await instance.database; + final db = await instance.sqliteAsyncDB; String inParam = ""; for (final id in uploadedIDs) { inParam += "'" + id.toString() + "',"; } inParam = inParam.substring(0, inParam.length - 1); - final results = await db.query( - filesTable, - where: '$columnUploadedFileID IN ($inParam)', - groupBy: columnUploadedFileID, + final results = await db.getAll( + ''' + SELECT * FROM $filesTable WHERE $columnUploadedFileID IN ($inParam) + GROUP BY $columnUploadedFileID +''', ); if (results.isEmpty) { return []; @@ -1659,6 +1668,29 @@ class FilesDB { return convertToFiles(results); } + ///For insertion, the syntax is as follows: + ///INSERT INTO table (column1,column2 ,..) + ///VALUES( value1, value2 ,...); + ///This method returns: + ///{ + /// 'columns': 'column1,column2 ,..', + /// 'values': 'value1, value2 ,...' + /// } + Map _getColumnsAndValuesForInsertion(EnteFile file) { + final row = _getRowForFile(file); + final columns = []; + final values = []; + for (int i = 0; i < row.entries.length; i++) { + final entry = row.entries.elementAt(i); + columns.add(entry.key); + values.add(entry.value.toString()); + } + return { + 'columns': columns.join(', '), + 'values': values.join(', '), + }; + } + String _getSetClauseForFile(EnteFile file) { final row = _getRowForFile(file); final setClause = []; From d1a5921c271303c2f568d62a04d88668a44f0ba3 Mon Sep 17 00:00:00 2001 From: ashilkn Date: Wed, 15 May 2024 15:28:24 +0530 Subject: [PATCH 005/354] [mob][photos] Migrate to sqlite_async(5): Create a method to get parameter set from file without calling getRowForFile() --- mobile/lib/db/files_db.dart | 98 ++++++++++++++++++++++++++----------- 1 file changed, 70 insertions(+), 28 deletions(-) diff --git a/mobile/lib/db/files_db.dart b/mobile/lib/db/files_db.dart index 887932250b..7fde7ee2e7 100644 --- a/mobile/lib/db/files_db.dart +++ b/mobile/lib/db/files_db.dart @@ -401,11 +401,11 @@ class FilesDB { } Future clearTable() async { - final db = await instance.database; - await db.delete(filesTable); - await db.delete("device_files"); - await db.delete("device_collections"); - await db.delete("entities"); + final db = await instance.sqliteAsyncDB; + await db.execute('DELETE FROM $filesTable'); + await db.execute('DELETE FROM device_files'); + await db.execute('DELETE FROM device_collections'); + await db.execute('DELETE FROM entities'); } Future deleteDB() async { @@ -1668,29 +1668,6 @@ class FilesDB { return convertToFiles(results); } - ///For insertion, the syntax is as follows: - ///INSERT INTO table (column1,column2 ,..) - ///VALUES( value1, value2 ,...); - ///This method returns: - ///{ - /// 'columns': 'column1,column2 ,..', - /// 'values': 'value1, value2 ,...' - /// } - Map _getColumnsAndValuesForInsertion(EnteFile file) { - final row = _getRowForFile(file); - final columns = []; - final values = []; - for (int i = 0; i < row.entries.length; i++) { - final entry = row.entries.elementAt(i); - columns.add(entry.key); - values.add(entry.value.toString()); - } - return { - 'columns': columns.join(', '), - 'values': values.join(', '), - }; - } - String _getSetClauseForFile(EnteFile file) { final row = _getRowForFile(file); final setClause = []; @@ -1701,6 +1678,71 @@ class FilesDB { return setClause.join(', '); } + List _getParameterSetForFile(EnteFile file) { + final row = _getRowForFile(file); + final values = []; + for (int i = 0; i < row.entries.length; i++) { + values.add(row.entries.elementAt(i).value); + } + return values; + } + + List _getParameterSetForFileNew( + EnteFile file, { + bool omitNullGenId = true, + }) { + final values = []; + double? latitude; + double? longitude; + int? creationTime = file.creationTime; + if (file.pubMagicMetadata != null) { + if (file.pubMagicMetadata!.editedTime != null) { + creationTime = file.pubMagicMetadata!.editedTime; + } + if (file.pubMagicMetadata!.lat != null && + file.pubMagicMetadata!.long != null) { + latitude = file.pubMagicMetadata!.lat; + longitude = file.pubMagicMetadata!.long; + } + } + if (file.generatedID != null || !omitNullGenId) { + values.add(file.generatedID); + } + values.addAll([ + file.localID, + file.uploadedFileID ?? -1, + file.ownerID, + file.collectionID ?? -1, + file.title, + file.deviceFolder, + latitude, + longitude, + getInt(file.fileType), + file.modificationTime, + file.encryptedKey, + file.keyDecryptionNonce, + file.fileDecryptionHeader, + file.thumbnailDecryptionHeader, + file.metadataDecryptionHeader, + creationTime, + file.updationTime, + file.fileSubType ?? -1, + file.duration ?? 0, + file.exif, + file.hash, + file.metadataVersion, + file.mMdEncodedJson ?? {}, + file.mMdVersion, + file.magicMetadata.visibility, + file.pubMmdEncodedJson ?? {}, + file.pubMmdVersion, + file.fileSize, + file.addedTime ?? DateTime.now().microsecondsSinceEpoch, + ]); + + return values; + } + String _getSetClauseForFileWithoutCollection(EnteFile file) { final row = _getRowForFileWithoutCollection(file); final setClause = []; From 25554209ec7375b3d4e62e72608a0c242715d921 Mon Sep 17 00:00:00 2001 From: ashilkn Date: Wed, 15 May 2024 19:52:55 +0530 Subject: [PATCH 006/354] [mob][photos] Migrate to sqlite_async)(6): Migrate insertMultipleNew to use sqlite_async --- mobile/lib/db/files_db.dart | 158 +++++++++++++++++++++++- mobile/lib/utils/primitive_wrapper.dart | 6 + 2 files changed, 163 insertions(+), 1 deletion(-) create mode 100644 mobile/lib/utils/primitive_wrapper.dart diff --git a/mobile/lib/db/files_db.dart b/mobile/lib/db/files_db.dart index 7fde7ee2e7..4b014b9e84 100644 --- a/mobile/lib/db/files_db.dart +++ b/mobile/lib/db/files_db.dart @@ -15,6 +15,7 @@ import 'package:photos/models/location/location.dart'; import "package:photos/models/metadata/common_keys.dart"; import "package:photos/services/filter/db_filters.dart"; import 'package:photos/utils/file_uploader_util.dart'; +import "package:photos/utils/primitive_wrapper.dart"; import 'package:sqflite/sqflite.dart'; import 'package:sqflite_migration/sqflite_migration.dart'; import 'package:sqlite_async/sqlite_async.dart' as sqlite_async; @@ -91,6 +92,39 @@ class FilesDB { ...addAddedTime(), ]; + static const List _columnNames = [ + columnGeneratedID, + columnLocalID, + columnUploadedFileID, + columnOwnerID, + columnCollectionID, + columnTitle, + columnDeviceFolder, + columnLatitude, + columnLongitude, + columnFileType, + columnModificationTime, + columnEncryptedKey, + columnKeyDecryptionNonce, + columnFileDecryptionHeader, + columnThumbnailDecryptionHeader, + columnMetadataDecryptionHeader, + columnCreationTime, + columnUpdationTime, + columnFileSubType, + columnDuration, + columnExif, + columnHash, + columnMetadataVersion, + columnMMdEncodedJson, + columnMMdVersion, + columnMMdVisibility, + columnPubMMdEncodedJson, + columnPubMMdVersion, + columnFileSize, + columnAddedTime, + ]; + final dbConfig = MigrationConfig( initializationScript: initializationScript, migrationScripts: migrationScripts, @@ -455,6 +489,128 @@ class FilesDB { ); } + Future insertMultipleNew( + List files, { + ConflictAlgorithm conflictAlgorithm = ConflictAlgorithm.replace, + }) async { + final startTime = DateTime.now(); + final db = await sqliteAsyncDB; + + ///Strong batch counter in an object so that it gets passed by reference + ///Primitives are passed by value + final genIdNotNullbatchCounter = PrimitiveWrapper(0); + final genIdNullbatchCounter = PrimitiveWrapper(0); + final genIdNullParameterSets = >[]; + final genIdNotNullParameterSets = >[]; + + final genIdNullcolumnNames = + _columnNames.where((element) => element != columnGeneratedID); + + for (EnteFile file in files) { + final fileGenIdIsNull = file.generatedID == null; + + if (!fileGenIdIsNull) { + await _batchAndInsertFile( + file, + conflictAlgorithm, + db, + genIdNotNullParameterSets, + genIdNotNullbatchCounter, + isGenIdNull: fileGenIdIsNull, + ); + } else { + await _batchAndInsertFile( + file, + conflictAlgorithm, + db, + genIdNullParameterSets, + genIdNullbatchCounter, + isGenIdNull: fileGenIdIsNull, + ); + } + } + + if (genIdNotNullbatchCounter.value > 0) { + await _insertBatch( + conflictAlgorithm, + _columnNames, + db, + genIdNotNullParameterSets, + ); + genIdNotNullbatchCounter.value = 0; + genIdNotNullParameterSets.clear(); + } + if (genIdNullbatchCounter.value > 0) { + await _insertBatch( + conflictAlgorithm, + genIdNullcolumnNames, + db, + genIdNullParameterSets, + ); + genIdNullbatchCounter.value = 0; + genIdNullParameterSets.clear(); + } + + final endTime = DateTime.now(); + final duration = Duration( + microseconds: + endTime.microsecondsSinceEpoch - startTime.microsecondsSinceEpoch, + ); + _logger.info( + "Batch insert of " + + files.length.toString() + + " took " + + duration.inMilliseconds.toString() + + "ms.", + ); + } + + @pragma('vm:prefer-inline') + Future _batchAndInsertFile( + EnteFile file, + ConflictAlgorithm conflictAlgorithm, + sqlite_async.SqliteDatabase db, + List> parameterSets, + PrimitiveWrapper batchCounter, { + required bool isGenIdNull, + }) async { + parameterSets.add(_getParameterSetForFileV2(file)); + batchCounter.value++; + + final columnNames = isGenIdNull + ? _columnNames.where((column) => column != columnGeneratedID) + : _columnNames; + if (batchCounter.value == 400) { + _logger.info("Inserting batch with genIdNull: $isGenIdNull"); + await _insertBatch(conflictAlgorithm, columnNames, db, parameterSets); + // await db.executeBatch( + // ''' + // INSERT OR ${conflictAlgorithm.name.toUpperCase()} INTO $filesTable($columnNames) VALUES($valuesPlaceholders) + // ''', + // parameterSets, + // ); + batchCounter.value = 0; + parameterSets.clear(); + } + } + + Future _insertBatch( + ConflictAlgorithm conflictAlgorithm, + Iterable columnNames, + sqlite_async.SqliteDatabase db, + List> parameterSets, + ) async { + final valuesPlaceholders = List.filled(columnNames.length, "?").join(","); + final columnNamesJoined = columnNames.join(","); + await db.executeBatch( + ''' + INSERT OR ${conflictAlgorithm.name.toUpperCase()} INTO $filesTable($columnNamesJoined) VALUES($valuesPlaceholders) + ''', + parameterSets, + ); + } + + @pragma('vm:prefer-inline') Future insert(EnteFile file) async { _logger.info("Inserting $file"); final db = await instance.database; @@ -1687,7 +1843,7 @@ class FilesDB { return values; } - List _getParameterSetForFileNew( + List _getParameterSetForFileV2( EnteFile file, { bool omitNullGenId = true, }) { diff --git a/mobile/lib/utils/primitive_wrapper.dart b/mobile/lib/utils/primitive_wrapper.dart new file mode 100644 index 0000000000..20ea9bbb6e --- /dev/null +++ b/mobile/lib/utils/primitive_wrapper.dart @@ -0,0 +1,6 @@ +///This is useful when you want to pass a primitive by reference. + +class PrimitiveWrapper { + var value; + PrimitiveWrapper(this.value); +} From e179d351d990d82230ae42a8c650d17efbf580ed Mon Sep 17 00:00:00 2001 From: ashilkn Date: Wed, 15 May 2024 21:04:32 +0530 Subject: [PATCH 007/354] [mob][photos] Migrate to sqlite_async(7): Assign String '{}' instead of map object {} to fix unexpected behaviour --- mobile/lib/db/files_db.dart | 40 ++----------------------------------- 1 file changed, 2 insertions(+), 38 deletions(-) diff --git a/mobile/lib/db/files_db.dart b/mobile/lib/db/files_db.dart index 4b014b9e84..b8779c5fd5 100644 --- a/mobile/lib/db/files_db.dart +++ b/mobile/lib/db/files_db.dart @@ -456,42 +456,6 @@ class FilesDB { Future insertMultiple( List files, { ConflictAlgorithm conflictAlgorithm = ConflictAlgorithm.replace, - }) async { - final startTime = DateTime.now(); - final db = await database; - var batch = db.batch(); - int batchCounter = 0; - for (EnteFile file in files) { - if (batchCounter == 400) { - await batch.commit(noResult: true); - batch = db.batch(); - batchCounter = 0; - } - batch.insert( - filesTable, - _getRowForFile(file), - conflictAlgorithm: conflictAlgorithm, - ); - batchCounter++; - } - await batch.commit(noResult: true); - final endTime = DateTime.now(); - final duration = Duration( - microseconds: - endTime.microsecondsSinceEpoch - startTime.microsecondsSinceEpoch, - ); - _logger.info( - "Batch insert of " + - files.length.toString() + - " took " + - duration.inMilliseconds.toString() + - "ms.", - ); - } - - Future insertMultipleNew( - List files, { - ConflictAlgorithm conflictAlgorithm = ConflictAlgorithm.replace, }) async { final startTime = DateTime.now(); final db = await sqliteAsyncDB; @@ -1887,10 +1851,10 @@ class FilesDB { file.exif, file.hash, file.metadataVersion, - file.mMdEncodedJson ?? {}, + file.mMdEncodedJson ?? '{}', file.mMdVersion, file.magicMetadata.visibility, - file.pubMmdEncodedJson ?? {}, + file.pubMmdEncodedJson ?? '{}', file.pubMmdVersion, file.fileSize, file.addedTime ?? DateTime.now().microsecondsSinceEpoch, From 56478fcb8a620af7891eef9230405be3bc2d373b Mon Sep 17 00:00:00 2001 From: ashilkn Date: Wed, 15 May 2024 21:10:37 +0530 Subject: [PATCH 008/354] [mob][photos] avoid unnecessary compute --- mobile/lib/db/files_db.dart | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mobile/lib/db/files_db.dart b/mobile/lib/db/files_db.dart index b8779c5fd5..97f76ed8c8 100644 --- a/mobile/lib/db/files_db.dart +++ b/mobile/lib/db/files_db.dart @@ -457,6 +457,8 @@ class FilesDB { List files, { ConflictAlgorithm conflictAlgorithm = ConflictAlgorithm.replace, }) async { + if (files.isEmpty) return; + final startTime = DateTime.now(); final db = await sqliteAsyncDB; From 1e7779a81996eccab1742275b8c012c9f32470fc Mon Sep 17 00:00:00 2001 From: ashilkn Date: Wed, 15 May 2024 21:18:14 +0530 Subject: [PATCH 009/354] [mob][photos] Remove method inline annotation which doesn 't have noticeable perf improvement + remove commented out code --- mobile/lib/db/files_db.dart | 8 -------- 1 file changed, 8 deletions(-) diff --git a/mobile/lib/db/files_db.dart b/mobile/lib/db/files_db.dart index 97f76ed8c8..4ff7f20e2b 100644 --- a/mobile/lib/db/files_db.dart +++ b/mobile/lib/db/files_db.dart @@ -531,7 +531,6 @@ class FilesDB { ); } - @pragma('vm:prefer-inline') Future _batchAndInsertFile( EnteFile file, ConflictAlgorithm conflictAlgorithm, @@ -549,12 +548,6 @@ class FilesDB { if (batchCounter.value == 400) { _logger.info("Inserting batch with genIdNull: $isGenIdNull"); await _insertBatch(conflictAlgorithm, columnNames, db, parameterSets); - // await db.executeBatch( - // ''' - // INSERT OR ${conflictAlgorithm.name.toUpperCase()} INTO $filesTable($columnNames) VALUES($valuesPlaceholders) - // ''', - // parameterSets, - // ); batchCounter.value = 0; parameterSets.clear(); } @@ -576,7 +569,6 @@ class FilesDB { ); } - @pragma('vm:prefer-inline') Future insert(EnteFile file) async { _logger.info("Inserting $file"); final db = await instance.database; From 7fdc2b5e669feab6ceaaa6c8c6218e5508475ace Mon Sep 17 00:00:00 2001 From: ashilkn Date: Thu, 16 May 2024 12:48:21 +0530 Subject: [PATCH 010/354] [mob][photos] Migrate to sqlite_async(8): Fix faulty update statements due to incorrect query generation --- mobile/lib/db/files_db.dart | 72 ++++++++++++++++++++++++------------- 1 file changed, 48 insertions(+), 24 deletions(-) diff --git a/mobile/lib/db/files_db.dart b/mobile/lib/db/files_db.dart index 4ff7f20e2b..efd24b28db 100644 --- a/mobile/lib/db/files_db.dart +++ b/mobile/lib/db/files_db.dart @@ -539,7 +539,7 @@ class FilesDB { PrimitiveWrapper batchCounter, { required bool isGenIdNull, }) async { - parameterSets.add(_getParameterSetForFileV2(file)); + parameterSets.add(_getParameterSetForFile(file)); batchCounter.value++; final columnNames = isGenIdNull @@ -1149,11 +1149,14 @@ class FilesDB { Future update(EnteFile file) async { final db = await instance.sqliteAsyncDB; - final setClause = _getSetClauseForFile(file); + final parameterSet = _getParameterSetForFile(file)..add(file.generatedID); + final updateAssignments = _generateUpdateAssignmentsWithPlaceholders( + fileGenId: file.generatedID, + ); await db.execute( - 'UPDATE $filesTable SET ' - '$setClause WHERE $columnGeneratedID = ?', - [file.generatedID], + 'UPDATE $filesTable ' + 'SET $updateAssignments WHERE $columnGeneratedID = ?', + parameterSet, ); } @@ -1167,6 +1170,21 @@ class FilesDB { ); } + Future updateUploadedFileAcrossCollectionsNew(EnteFile file) async { + final db = await instance.sqliteAsyncDB; + final parameterSet = _getParameterSetForFile(file, omitCollectionId: true) + ..add(file.uploadedFileID); + final updateAssignments = _generateUpdateAssignmentsWithPlaceholders( + fileGenId: file.generatedID, + omitCollectionId: true, + ); + await db.execute( + 'UPDATE $filesTable' + 'SET $updateAssignments WHERE $columnUploadedFileID = ?', + parameterSet, + ); + } + Future updateLocalIDForUploaded(int uploadedID, String localID) async { final db = await instance.sqliteAsyncDB; await db.execute( @@ -1782,8 +1800,8 @@ class FilesDB { return convertToFiles(results); } - String _getSetClauseForFile(EnteFile file) { - final row = _getRowForFile(file); + String _getSetClauseForFileWithoutCollection(EnteFile file) { + final row = _getRowForFileWithoutCollection(file); final setClause = []; for (int i = 0; i < row.entries.length; i++) { final entry = row.entries.elementAt(i); @@ -1792,18 +1810,30 @@ class FilesDB { return setClause.join(', '); } - List _getParameterSetForFile(EnteFile file) { - final row = _getRowForFile(file); - final values = []; - for (int i = 0; i < row.entries.length; i++) { - values.add(row.entries.elementAt(i).value); + ///Returns "columnName1 = ?, columnName2 = ?, ..." + String _generateUpdateAssignmentsWithPlaceholders({ + required int? fileGenId, + bool omitCollectionId = false, + }) { + final setClauses = []; + + for (String columnName in _columnNames) { + if (columnName == columnGeneratedID && fileGenId == null) { + continue; + } + if (columnName == columnCollectionID && omitCollectionId) { + continue; + } + setClauses.add("$columnName = ?"); } - return values; + + return setClauses.join(","); } - List _getParameterSetForFileV2( + List _getParameterSetForFile( EnteFile file, { bool omitNullGenId = true, + bool omitCollectionId = false, }) { final values = []; double? latitude; @@ -1854,17 +1884,11 @@ class FilesDB { file.addedTime ?? DateTime.now().microsecondsSinceEpoch, ]); - return values; - } - - String _getSetClauseForFileWithoutCollection(EnteFile file) { - final row = _getRowForFileWithoutCollection(file); - final setClause = []; - for (int i = 0; i < row.entries.length; i++) { - final entry = row.entries.elementAt(i); - setClause.add('${entry.key} = ${entry.value}'); + if (omitCollectionId) { + values.removeAt(3); } - return setClause.join(', '); + + return values; } Map _getRowForFile(EnteFile file) { From cd023b621a698485d1d1af8a31d364fb47884680 Mon Sep 17 00:00:00 2001 From: ashilkn Date: Thu, 16 May 2024 12:59:19 +0530 Subject: [PATCH 011/354] [mob][photos] Remove optional parameter which should never be used Since generatedID (_id) has NOT NULL constrain, it shouldn't be in a parameter set of a query --- mobile/lib/db/files_db.dart | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/mobile/lib/db/files_db.dart b/mobile/lib/db/files_db.dart index efd24b28db..dd71e4b68c 100644 --- a/mobile/lib/db/files_db.dart +++ b/mobile/lib/db/files_db.dart @@ -1161,16 +1161,6 @@ class FilesDB { } Future updateUploadedFileAcrossCollections(EnteFile file) async { - final db = await instance.sqliteAsyncDB; - final setClause = _getSetClauseForFileWithoutCollection(file); - await db.execute( - 'UPDATE $filesTable SET ' - '$setClause WHERE $columnUploadedFileID = ?', - [file.uploadedFileID], - ); - } - - Future updateUploadedFileAcrossCollectionsNew(EnteFile file) async { final db = await instance.sqliteAsyncDB; final parameterSet = _getParameterSetForFile(file, omitCollectionId: true) ..add(file.uploadedFileID); @@ -1832,7 +1822,6 @@ class FilesDB { List _getParameterSetForFile( EnteFile file, { - bool omitNullGenId = true, bool omitCollectionId = false, }) { final values = []; @@ -1849,7 +1838,7 @@ class FilesDB { longitude = file.pubMagicMetadata!.long; } } - if (file.generatedID != null || !omitNullGenId) { + if (file.generatedID != null) { values.add(file.generatedID); } values.addAll([ From 584a37d2a20f66ac1b79f22fb1bd1350ff16fb70 Mon Sep 17 00:00:00 2001 From: ashilkn Date: Thu, 16 May 2024 14:20:03 +0530 Subject: [PATCH 012/354] [mob][photos] Remove obsolete code This code is from when we used to support favoriting un-uploaded files --- mobile/lib/services/favorites_service.dart | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/mobile/lib/services/favorites_service.dart b/mobile/lib/services/favorites_service.dart index fef4a323a8..2f64e63d41 100644 --- a/mobile/lib/services/favorites_service.dart +++ b/mobile/lib/services/favorites_service.dart @@ -151,9 +151,7 @@ class FavoritesService { final collectionID = await _getOrCreateFavoriteCollectionID(); final List files = [file]; if (file.uploadedFileID == null) { - file.collectionID = collectionID; - await _filesDB.insert(file); - Bus.instance.fire(CollectionUpdatedEvent(collectionID, files, "addTFav")); + throw AssertionError("Can only favorite uploaded items"); } else { await _collectionsService.addOrCopyToCollection(collectionID, files); } From 1a360d3ee7381c047c46192d26b61729fcc9959e Mon Sep 17 00:00:00 2001 From: ashilkn Date: Thu, 16 May 2024 15:37:00 +0530 Subject: [PATCH 013/354] [mob][photos] Migrate to sqlite_async(8): Migrate insert() + rearrange + clean up --- mobile/lib/db/files_db.dart | 186 +++++++++--------- .../ui/tools/editor/image_editor_page.dart | 2 +- 2 files changed, 89 insertions(+), 99 deletions(-) diff --git a/mobile/lib/db/files_db.dart b/mobile/lib/db/files_db.dart index dd71e4b68c..68b80d9d55 100644 --- a/mobile/lib/db/files_db.dart +++ b/mobile/lib/db/files_db.dart @@ -531,54 +531,35 @@ class FilesDB { ); } - Future _batchAndInsertFile( - EnteFile file, - ConflictAlgorithm conflictAlgorithm, - sqlite_async.SqliteDatabase db, - List> parameterSets, - PrimitiveWrapper batchCounter, { - required bool isGenIdNull, - }) async { - parameterSets.add(_getParameterSetForFile(file)); - batchCounter.value++; - - final columnNames = isGenIdNull - ? _columnNames.where((column) => column != columnGeneratedID) - : _columnNames; - if (batchCounter.value == 400) { - _logger.info("Inserting batch with genIdNull: $isGenIdNull"); - await _insertBatch(conflictAlgorithm, columnNames, db, parameterSets); - batchCounter.value = 0; - parameterSets.clear(); - } - } - - Future _insertBatch( - ConflictAlgorithm conflictAlgorithm, - Iterable columnNames, - sqlite_async.SqliteDatabase db, - List> parameterSets, - ) async { - final valuesPlaceholders = List.filled(columnNames.length, "?").join(","); - final columnNamesJoined = columnNames.join(","); - await db.executeBatch( - ''' - INSERT OR ${conflictAlgorithm.name.toUpperCase()} INTO $filesTable($columnNamesJoined) VALUES($valuesPlaceholders) - ''', - parameterSets, - ); - } - - Future insert(EnteFile file) async { + Future insert(EnteFile file) async { _logger.info("Inserting $file"); - final db = await instance.database; - return db.insert( - filesTable, - _getRowForFile(file), - conflictAlgorithm: ConflictAlgorithm.replace, + final db = await instance.sqliteAsyncDB; + final columnsAndPlaceholders = + _generateColumnsAndPlaceholdersForInsert(fileGenId: file.generatedID); + final values = _getParameterSetForFile(file); + + await db.execute( + 'INSERT OR REPLACE INTO $filesTable (${columnsAndPlaceholders["columns"]}) VALUES (${columnsAndPlaceholders["placeholders"]})', + values, ); } + Future insertAndGetId(EnteFile file) async { + _logger.info("Inserting $file"); + final db = await instance.sqliteAsyncDB; + final columnsAndPlaceholders = + _generateColumnsAndPlaceholdersForInsert(fileGenId: file.generatedID); + final values = _getParameterSetForFile(file); + return await db.writeTransaction((tx) async { + await tx.execute( + 'INSERT OR REPLACE INTO $filesTable (${columnsAndPlaceholders["columns"]}) VALUES (${columnsAndPlaceholders["placeholders"]})', + values, + ); + final result = await tx.get('SELECT last_insert_rowid()'); + return result["last_insert_rowid()"] as int; + }); + } + Future getFile(int generatedID) async { final db = await instance.sqliteAsyncDB; final results = await db.getAll( @@ -1790,22 +1771,12 @@ class FilesDB { return convertToFiles(results); } - String _getSetClauseForFileWithoutCollection(EnteFile file) { - final row = _getRowForFileWithoutCollection(file); - final setClause = []; - for (int i = 0; i < row.entries.length; i++) { - final entry = row.entries.elementAt(i); - setClause.add('${entry.key} = ${entry.value}'); - } - return setClause.join(', '); - } - ///Returns "columnName1 = ?, columnName2 = ?, ..." String _generateUpdateAssignmentsWithPlaceholders({ required int? fileGenId, bool omitCollectionId = false, }) { - final setClauses = []; + final assignments = []; for (String columnName in _columnNames) { if (columnName == columnGeneratedID && fileGenId == null) { @@ -1814,10 +1785,29 @@ class FilesDB { if (columnName == columnCollectionID && omitCollectionId) { continue; } - setClauses.add("$columnName = ?"); + assignments.add("$columnName = ?"); } - return setClauses.join(","); + return assignments.join(","); + } + + Map _generateColumnsAndPlaceholdersForInsert({ + required int? fileGenId, + }) { + final columnNames = []; + + for (String columnName in _columnNames) { + if (columnName == columnGeneratedID && fileGenId == null) { + continue; + } + + columnNames.add(columnName); + } + + return { + "columns": columnNames.join(","), + "placeholders": List.filled(columnNames.length, "?").join(","), + }; } List _getParameterSetForFile( @@ -1825,8 +1815,10 @@ class FilesDB { bool omitCollectionId = false, }) { final values = []; + double? latitude; double? longitude; + int? creationTime = file.creationTime; if (file.pubMagicMetadata != null) { if (file.pubMagicMetadata!.editedTime != null) { @@ -1838,6 +1830,7 @@ class FilesDB { longitude = file.pubMagicMetadata!.long; } } + if (file.generatedID != null) { values.add(file.generatedID); } @@ -1880,6 +1873,44 @@ class FilesDB { return values; } + Future _batchAndInsertFile( + EnteFile file, + ConflictAlgorithm conflictAlgorithm, + sqlite_async.SqliteDatabase db, + List> parameterSets, + PrimitiveWrapper batchCounter, { + required bool isGenIdNull, + }) async { + parameterSets.add(_getParameterSetForFile(file)); + batchCounter.value++; + + final columnNames = isGenIdNull + ? _columnNames.where((column) => column != columnGeneratedID) + : _columnNames; + if (batchCounter.value == 400) { + _logger.info("Inserting batch with genIdNull: $isGenIdNull"); + await _insertBatch(conflictAlgorithm, columnNames, db, parameterSets); + batchCounter.value = 0; + parameterSets.clear(); + } + } + + Future _insertBatch( + ConflictAlgorithm conflictAlgorithm, + Iterable columnNames, + sqlite_async.SqliteDatabase db, + List> parameterSets, + ) async { + final valuesPlaceholders = List.filled(columnNames.length, "?").join(","); + final columnNamesJoined = columnNames.join(","); + await db.executeBatch( + ''' + INSERT OR ${conflictAlgorithm.name.toUpperCase()} INTO $filesTable($columnNamesJoined) VALUES($valuesPlaceholders) + ''', + parameterSets, + ); + } + Map _getRowForFile(EnteFile file) { final row = {}; if (file.generatedID != null) { @@ -1935,47 +1966,6 @@ class FilesDB { return row; } - Map _getRowForFileWithoutCollection(EnteFile file) { - final row = {}; - row[columnLocalID] = file.localID; - row[columnUploadedFileID] = file.uploadedFileID ?? -1; - row[columnOwnerID] = file.ownerID; - row[columnTitle] = file.title; - row[columnDeviceFolder] = file.deviceFolder; - if (file.location != null) { - row[columnLatitude] = file.location!.latitude; - row[columnLongitude] = file.location!.longitude; - } - row[columnFileType] = getInt(file.fileType); - row[columnCreationTime] = file.creationTime; - row[columnModificationTime] = file.modificationTime; - row[columnUpdationTime] = file.updationTime; - row[columnAddedTime] = - file.addedTime ?? DateTime.now().microsecondsSinceEpoch; - row[columnFileDecryptionHeader] = file.fileDecryptionHeader; - row[columnThumbnailDecryptionHeader] = file.thumbnailDecryptionHeader; - row[columnMetadataDecryptionHeader] = file.metadataDecryptionHeader; - row[columnFileSubType] = file.fileSubType ?? -1; - row[columnDuration] = file.duration ?? 0; - row[columnExif] = file.exif; - row[columnHash] = file.hash; - row[columnMetadataVersion] = file.metadataVersion; - - row[columnMMdVersion] = file.mMdVersion; - row[columnMMdEncodedJson] = file.mMdEncodedJson ?? '{}'; - row[columnMMdVisibility] = file.magicMetadata.visibility; - - row[columnPubMMdVersion] = file.pubMmdVersion; - row[columnPubMMdEncodedJson] = file.pubMmdEncodedJson ?? '{}'; - if (file.pubMagicMetadata != null && - file.pubMagicMetadata!.editedTime != null) { - // override existing creationTime to avoid re-writing all queries related - // to loading the gallery - row[columnCreationTime] = file.pubMagicMetadata!.editedTime!; - } - return row; - } - EnteFile _getFileFromRow(Map row) { final file = EnteFile(); file.generatedID = row[columnGeneratedID]; diff --git a/mobile/lib/ui/tools/editor/image_editor_page.dart b/mobile/lib/ui/tools/editor/image_editor_page.dart index 4830df9523..5314d9ca86 100644 --- a/mobile/lib/ui/tools/editor/image_editor_page.dart +++ b/mobile/lib/ui/tools/editor/image_editor_page.dart @@ -371,7 +371,7 @@ class _ImageEditorPageState extends State { ); } } - newFile.generatedID = await FilesDB.instance.insert(newFile); + newFile.generatedID = await FilesDB.instance.insertAndGetId(newFile); Bus.instance.fire(LocalPhotosUpdatedEvent([newFile], source: "editSave")); unawaited(SyncService.instance.sync()); showShortToast(context, S.of(context).editsSaved); From dec7c45310da3e668e7fd5e3f77e20858fbb48a4 Mon Sep 17 00:00:00 2001 From: ashilkn Date: Thu, 16 May 2024 16:41:57 +0530 Subject: [PATCH 014/354] [mob][photos] Migrate to sqlite_async(9) --- mobile/lib/db/files_db.dart | 87 ++++++++----------------------------- 1 file changed, 17 insertions(+), 70 deletions(-) diff --git a/mobile/lib/db/files_db.dart b/mobile/lib/db/files_db.dart index 68b80d9d55..a5e00f352c 100644 --- a/mobile/lib/db/files_db.dart +++ b/mobile/lib/db/files_db.dart @@ -606,13 +606,11 @@ class FilesDB { Future<(Set, Map)> getUploadAndHash( int collectionID, ) async { - final db = await instance.database; - final results = await db.query( - filesTable, - columns: [columnUploadedFileID, columnHash], - where: - '$columnCollectionID = ? AND ($columnUploadedFileID IS NOT NULL AND $columnUploadedFileID IS NOT -1)', - whereArgs: [ + final db = await instance.sqliteAsyncDB; + final results = await db.getAll( + 'SELECT $columnUploadedFileID, $columnHash FROM $filesTable' + ' WHERE $columnCollectionID = ? AND ($columnUploadedFileID IS NOT NULL AND $columnUploadedFileID IS NOT -1)', + [ collectionID, ], ); @@ -1265,18 +1263,22 @@ class FilesDB { ); } - /// Uses int in return value. Future deleteFilesFromCollection( int collectionID, List uploadedFileIDs, ) async { - final db = await instance.database; - return db.delete( - filesTable, - where: - '$columnCollectionID = ? AND $columnUploadedFileID IN (${uploadedFileIDs.join(', ')})', - whereArgs: [collectionID], - ); + final db = await instance.sqliteAsyncDB; + return db.writeTransaction((tx) async { + await tx.execute( + ''' + DELETE FROM $filesTable + WHERE $columnCollectionID = ? AND $columnUploadedFileID IN (${uploadedFileIDs.join(', ')}); + ''', + [collectionID], + ); + final res = await tx.get('SELECT changes()'); + return res['changes()'] as int; + }); } Future collectionFileCount(int collectionID) async { @@ -1911,61 +1913,6 @@ class FilesDB { ); } - Map _getRowForFile(EnteFile file) { - final row = {}; - if (file.generatedID != null) { - row[columnGeneratedID] = file.generatedID; - } - row[columnLocalID] = file.localID; - row[columnUploadedFileID] = file.uploadedFileID ?? -1; - row[columnOwnerID] = file.ownerID; - row[columnCollectionID] = file.collectionID ?? -1; - row[columnTitle] = file.title; - row[columnDeviceFolder] = file.deviceFolder; - // if (file.location == null || - // (file.location!.latitude == null && file.location!.longitude == null)) { - // file.location = Location.randomLocation(); - // } - if (file.location != null) { - row[columnLatitude] = file.location!.latitude; - row[columnLongitude] = file.location!.longitude; - } - row[columnFileType] = getInt(file.fileType); - row[columnCreationTime] = file.creationTime; - row[columnModificationTime] = file.modificationTime; - row[columnUpdationTime] = file.updationTime; - row[columnAddedTime] = - file.addedTime ?? DateTime.now().microsecondsSinceEpoch; - row[columnEncryptedKey] = file.encryptedKey; - row[columnKeyDecryptionNonce] = file.keyDecryptionNonce; - row[columnFileDecryptionHeader] = file.fileDecryptionHeader; - row[columnThumbnailDecryptionHeader] = file.thumbnailDecryptionHeader; - row[columnMetadataDecryptionHeader] = file.metadataDecryptionHeader; - row[columnFileSubType] = file.fileSubType ?? -1; - row[columnDuration] = file.duration ?? 0; - row[columnExif] = file.exif; - row[columnHash] = file.hash; - row[columnMetadataVersion] = file.metadataVersion; - row[columnFileSize] = file.fileSize; - row[columnMMdVersion] = file.mMdVersion; - row[columnMMdEncodedJson] = file.mMdEncodedJson ?? '{}'; - row[columnMMdVisibility] = file.magicMetadata.visibility; - row[columnPubMMdVersion] = file.pubMmdVersion; - row[columnPubMMdEncodedJson] = file.pubMmdEncodedJson ?? '{}'; - // override existing fields to avoid re-writing all queries and logic - if (file.pubMagicMetadata != null) { - if (file.pubMagicMetadata!.editedTime != null) { - row[columnCreationTime] = file.pubMagicMetadata!.editedTime; - } - if (file.pubMagicMetadata!.lat != null && - file.pubMagicMetadata!.long != null) { - row[columnLatitude] = file.pubMagicMetadata!.lat; - row[columnLongitude] = file.pubMagicMetadata!.long; - } - } - return row; - } - EnteFile _getFileFromRow(Map row) { final file = EnteFile(); file.generatedID = row[columnGeneratedID]; From 16d54645bc76aaf4be1af6392fdecef51898169c Mon Sep 17 00:00:00 2001 From: ashilkn Date: Thu, 16 May 2024 18:02:39 +0530 Subject: [PATCH 015/354] [mob][photos] Migrate to sqlite_async(10) --- mobile/lib/db/device_files_db.dart | 199 +++++++++++++++++------------ 1 file changed, 117 insertions(+), 82 deletions(-) diff --git a/mobile/lib/db/device_files_db.dart b/mobile/lib/db/device_files_db.dart index 25c88daca1..73fc3a62b3 100644 --- a/mobile/lib/db/device_files_db.dart +++ b/mobile/lib/db/device_files_db.dart @@ -22,61 +22,55 @@ extension DeviceFiles on FilesDB { ConflictAlgorithm conflictAlgorithm = ConflictAlgorithm.ignore, }) async { debugPrint("Inserting missing PathIDToLocalIDMapping"); - final db = await database; - var batch = db.batch(); + final parameterSets = >[]; int batchCounter = 0; for (MapEntry e in mappingToAdd.entries) { final String pathID = e.key; for (String localID in e.value) { + parameterSets.add([localID, pathID]); + batchCounter++; + if (batchCounter == 400) { - await batch.commit(noResult: true); - batch = db.batch(); + await _insertBatch(parameterSets, conflictAlgorithm); + parameterSets.clear(); batchCounter = 0; } - batch.insert( - "device_files", - { - "id": localID, - "path_id": pathID, - }, - conflictAlgorithm: conflictAlgorithm, - ); - batchCounter++; } } - await batch.commit(noResult: true); + await _insertBatch(parameterSets, conflictAlgorithm); + parameterSets.clear(); + batchCounter = 0; } Future deletePathIDToLocalIDMapping( Map> mappingsToRemove, ) async { debugPrint("removing PathIDToLocalIDMapping"); - final db = await database; - var batch = db.batch(); + final parameterSets = >[]; int batchCounter = 0; for (MapEntry e in mappingsToRemove.entries) { final String pathID = e.key; + for (String localID in e.value) { + parameterSets.add([localID, pathID]); + batchCounter++; + if (batchCounter == 400) { - await batch.commit(noResult: true); - batch = db.batch(); + await _deleteBatch(parameterSets); + parameterSets.clear(); batchCounter = 0; } - batch.delete( - "device_files", - where: 'id = ? AND path_id = ?', - whereArgs: [localID, pathID], - ); - batchCounter++; } } - await batch.commit(noResult: true); + await _deleteBatch(parameterSets); + parameterSets.clear(); + batchCounter = 0; } Future> getDevicePathIDToImportedFileCount() async { try { - final db = await database; - final rows = await db.rawQuery( + final db = await sqliteAsyncDB; + final rows = await db.getAll( ''' SELECT count(*) as count, path_id FROM device_files @@ -96,8 +90,8 @@ extension DeviceFiles on FilesDB { Future>> getDevicePathIDToLocalIDMap() async { try { - final db = await database; - final rows = await db.rawQuery( + final db = await sqliteAsyncDB; + final rows = await db.getAll( ''' SELECT id, path_id FROM device_files; ''', ); final result = >{}; @@ -116,8 +110,8 @@ extension DeviceFiles on FilesDB { } Future> getDevicePathIDs() async { - final Database db = await database; - final rows = await db.rawQuery( + final db = await sqliteAsyncDB; + final rows = await db.getAll( ''' SELECT id FROM device_collections ''', @@ -133,34 +127,42 @@ extension DeviceFiles on FilesDB { List localPathAssets, { bool shouldAutoBackup = false, }) async { - final Database db = await database; + final db = await sqliteAsyncDB; final Map> pathIDToLocalIDsMap = {}; try { - final batch = db.batch(); final Set existingPathIds = await getDevicePathIDs(); + final parameterSetsForUpdate = >[]; + final parameterSetsForInsert = >[]; for (LocalPathAsset localPathAsset in localPathAssets) { if (localPathAsset.localIDs.isNotEmpty) { pathIDToLocalIDsMap[localPathAsset.pathID] = localPathAsset.localIDs; } if (existingPathIds.contains(localPathAsset.pathID)) { - batch.rawUpdate( - "UPDATE device_collections SET name = ? where id = " - "?", - [localPathAsset.pathName, localPathAsset.pathID], - ); + parameterSetsForUpdate + .add([localPathAsset.pathName, localPathAsset.pathID]); } else if (localPathAsset.localIDs.isNotEmpty) { - batch.insert( - "device_collections", - { - "id": localPathAsset.pathID, - "name": localPathAsset.pathName, - "should_backup": shouldAutoBackup ? _sqlBoolTrue : _sqlBoolFalse, - }, - conflictAlgorithm: ConflictAlgorithm.ignore, - ); + parameterSetsForInsert.add([ + localPathAsset.pathID, + localPathAsset.pathName, + shouldAutoBackup ? _sqlBoolTrue : _sqlBoolFalse, + ]); } } - await batch.commit(noResult: true); + + await db.executeBatch( + ''' + INSERT OR IGNORE INTO device_collections (id, name, should_backup) VALUES (?, ?, ?); + ''', + parameterSetsForInsert, + ); + + await db.executeBatch( + ''' + UPDATE device_collections SET name = ? WHERE id = ?; + ''', + parameterSetsForUpdate, + ); + // add the mappings for localIDs if (pathIDToLocalIDsMap.isNotEmpty) { await insertPathIDToLocalIDMapping(pathIDToLocalIDsMap); @@ -177,7 +179,7 @@ extension DeviceFiles on FilesDB { }) async { bool hasUpdated = false; try { - final Database db = await database; + final db = await sqliteAsyncDB; final Set existingPathIds = await getDevicePathIDs(); for (Tuple2 tup in devicePathInfo) { final AssetPathEntity pathEntity = tup.item1; @@ -185,35 +187,42 @@ extension DeviceFiles on FilesDB { final String localID = tup.item2; final bool shouldUpdate = existingPathIds.contains(pathEntity.id); if (shouldUpdate) { - final rowUpdated = await db.rawUpdate( - "UPDATE device_collections SET name = ?, cover_id = ?, count" - " = ? where id = ? AND (name != ? OR cover_id != ? OR count != ?)", - [ - pathEntity.name, - localID, - assetCount, - pathEntity.id, - pathEntity.name, - localID, - assetCount, - ], - ); + final rowUpdated = await db.writeTransaction((tx) async { + await tx.execute( + "UPDATE device_collections SET name = ?, cover_id = ?, count" + " = ? where id = ? AND (name != ? OR cover_id != ? OR count != ?)", + [ + pathEntity.name, + localID, + assetCount, + pathEntity.id, + pathEntity.name, + localID, + assetCount, + ], + ); + final result = await tx.get("SELECT changes();"); + return result["changes()"] as int; + }); + if (rowUpdated > 0) { _logger.fine("Updated $rowUpdated rows for ${pathEntity.name}"); hasUpdated = true; } } else { hasUpdated = true; - await db.insert( - "device_collections", - { - "id": pathEntity.id, - "name": pathEntity.name, - "count": assetCount, - "cover_id": localID, - "should_backup": shouldBackup ? _sqlBoolTrue : _sqlBoolFalse, - }, - conflictAlgorithm: ConflictAlgorithm.ignore, + await db.execute( + ''' + INSERT INTO device_collections (id, name, count, cover_id, should_backup) + VALUES (?, ?, ?, ?, ?); + ''', + [ + pathEntity.id, + pathEntity.name, + assetCount, + localID, + shouldBackup ? _sqlBoolTrue : _sqlBoolFalse, + ], ); } } @@ -231,15 +240,17 @@ extension DeviceFiles on FilesDB { // feature, where we delete files which are backed up. Deleting such // entries here result in us losing out on the information that // those folders were marked for automatic backup. - await db.delete( - "device_collections", - where: 'id = ? and should_backup = $_sqlBoolFalse ', - whereArgs: [pathID], + await db.execute( + ''' + DELETE FROM device_collections WHERE id = ? AND should_backup = $_sqlBoolFalse; + ''', + [pathID], ); - await db.delete( - "device_files", - where: 'path_id = ?', - whereArgs: [pathID], + await db.execute( + ''' + DELETE FROM device_files WHERE path_id = ?; + ''', + [pathID], ); } } @@ -253,8 +264,8 @@ extension DeviceFiles on FilesDB { // getDeviceSyncCollectionIDs returns the collectionIDs for the // deviceCollections which are marked for auto-backup Future> getDeviceSyncCollectionIDs() async { - final Database db = await database; - final rows = await db.rawQuery( + final db = await sqliteAsyncDB; + final rows = await db.getAll( ''' SELECT collection_id FROM device_collections where should_backup = $_sqlBoolTrue @@ -447,4 +458,28 @@ extension DeviceFiles on FilesDB { return null; } } + + Future _insertBatch( + List> parameterSets, + ConflictAlgorithm conflictAlgorithm, + ) async { + final db = await sqliteAsyncDB; + await db.executeBatch( + ''' + INSERT OR ${conflictAlgorithm.name.toUpperCase()} + INTO device_files (id, path_id) VALUES (?, ?); + ''', + parameterSets, + ); + } + + Future _deleteBatch(List> parameterSets) async { + final db = await sqliteAsyncDB; + await db.executeBatch( + ''' + DELETE FROM device_files WHERE id = ? AND path_id = ?; + ''', + parameterSets, + ); + } } From 2b0fa9bae62d7db88bf2f83d015354b90e36b6e4 Mon Sep 17 00:00:00 2001 From: ashilkn Date: Thu, 16 May 2024 19:34:59 +0530 Subject: [PATCH 016/354] [mob][photos] Migrate to sqlite_async(11) --- mobile/lib/db/device_files_db.dart | 67 +++++++++++++++++------------- 1 file changed, 37 insertions(+), 30 deletions(-) diff --git a/mobile/lib/db/device_files_db.dart b/mobile/lib/db/device_files_db.dart index 73fc3a62b3..fbd1649e26 100644 --- a/mobile/lib/db/device_files_db.dart +++ b/mobile/lib/db/device_files_db.dart @@ -279,40 +279,47 @@ extension DeviceFiles on FilesDB { return result; } - Future updateDevicePathSyncStatus(Map syncStatus) async { - final db = await database; - var batch = db.batch(); + Future updateDevicePathSyncStatus( + Map syncStatus, + ) async { + final db = await sqliteAsyncDB; int batchCounter = 0; + final parameterSets = >[]; for (MapEntry e in syncStatus.entries) { final String pathID = e.key; + parameterSets.add([e.value ? _sqlBoolTrue : _sqlBoolFalse, pathID]); + batchCounter++; + if (batchCounter == 400) { - await batch.commit(noResult: true); - batch = db.batch(); + await db.executeBatch( + ''' + UPDATE device_collections SET should_backup = ? WHERE id = ?; + ''', + parameterSets, + ); + parameterSets.clear(); batchCounter = 0; } - batch.update( - "device_collections", - { - "should_backup": e.value ? _sqlBoolTrue : _sqlBoolFalse, - }, - where: 'id = ?', - whereArgs: [pathID], - ); - batchCounter++; } - await batch.commit(noResult: true); + + await db.executeBatch( + ''' + UPDATE device_collections SET should_backup = ? WHERE id = ?; + ''', + parameterSets, + ); } Future updateDeviceCollection( String pathID, int collectionID, ) async { - final db = await database; - await db.update( - "device_collections", - {"collection_id": collectionID}, - where: 'id = ?', - whereArgs: [pathID], + final db = await sqliteAsyncDB; + await db.execute( + ''' + UPDATE device_collections SET collection_id = ? WHERE id = ?; + ''', + [collectionID, pathID], ); return; } @@ -325,7 +332,7 @@ extension DeviceFiles on FilesDB { int? limit, bool? asc, }) async { - final db = await database; + final db = await sqliteAsyncDB; final order = (asc ?? false ? 'ASC' : 'DESC'); final String rawQuery = ''' SELECT * @@ -340,7 +347,7 @@ extension DeviceFiles on FilesDB { ORDER BY ${FilesDB.columnCreationTime} $order , ${FilesDB.columnModificationTime} $order ''' + (limit != null ? ' limit $limit;' : ';'); - final results = await db.rawQuery(rawQuery); + final results = await db.getAll(rawQuery); final files = convertToFiles(results); final dedupe = deduplicateByLocalID(files); return FileLoadResult(dedupe, files.length == limit); @@ -350,7 +357,7 @@ extension DeviceFiles on FilesDB { String pathID, int ownerID, ) async { - final db = await database; + final db = await sqliteAsyncDB; const String rawQuery = ''' SELECT ${FilesDB.columnLocalID}, ${FilesDB.columnUploadedFileID}, ${FilesDB.columnFileSize} @@ -362,7 +369,7 @@ extension DeviceFiles on FilesDB { ${FilesDB.columnLocalID} IN (SELECT id FROM device_files where path_id = ?) '''; - final results = await db.rawQuery(rawQuery, [ownerID, pathID]); + final results = await db.getAll(rawQuery, [ownerID, pathID]); final localIDs = {}; final uploadedIDs = {}; int localSize = 0; @@ -386,17 +393,17 @@ extension DeviceFiles on FilesDB { "$includeCoverThumbnail", ); try { - final db = await database; + final db = await sqliteAsyncDB; final coverFiles = []; if (includeCoverThumbnail) { - final fileRows = await db.rawQuery( + final fileRows = await db.getAll( '''SELECT * FROM FILES where local_id in (select cover_id from device_collections) group by local_id; ''', ); final files = convertToFiles(fileRows); coverFiles.addAll(files); } - final deviceCollectionRows = await db.rawQuery( + final deviceCollectionRows = await db.getAll( '''SELECT * from device_collections''', ); final List deviceCollections = []; @@ -444,8 +451,8 @@ extension DeviceFiles on FilesDB { Future getDeviceCollectionThumbnail(String pathID) async { debugPrint("Call fallback method to get potential thumbnail"); - final db = await database; - final fileRows = await db.rawQuery( + final db = await sqliteAsyncDB; + final fileRows = await db.getAll( '''SELECT * FROM FILES f JOIN device_files df on f.local_id = df.id and df.path_id= ? order by f.creation_time DESC limit 1; ''', From 28ddb93747007b64ed6a17b4fe44a72f45a9886d Mon Sep 17 00:00:00 2001 From: ashilkn Date: Thu, 16 May 2024 20:17:58 +0530 Subject: [PATCH 017/354] [mob][photos] Add missing parameters for query --- mobile/lib/db/files_db.dart | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/mobile/lib/db/files_db.dart b/mobile/lib/db/files_db.dart index a5e00f352c..2b6219cbe7 100644 --- a/mobile/lib/db/files_db.dart +++ b/mobile/lib/db/files_db.dart @@ -905,10 +905,12 @@ class FilesDB { Future> getUploadedFileIDsToBeUpdated(int ownerID) async { final db = await instance.sqliteAsyncDB; final rows = await db.getAll( - 'SELECT DISTINCT $columnUploadedFileID FROM $filesTable WHERE ' - '($columnLocalID IS NOT NULL AND $columnOwnerID = ? AND ' - '($columnUploadedFileID IS NOT NULL AND $columnUploadedFileID IS NOT -1) ' - 'AND $columnUpdationTime IS NULL) ORDER BY $columnCreationTime DESC '); + 'SELECT DISTINCT $columnUploadedFileID FROM $filesTable WHERE ' + '($columnLocalID IS NOT NULL AND $columnOwnerID = ? AND ' + '($columnUploadedFileID IS NOT NULL AND $columnUploadedFileID IS NOT -1) ' + 'AND $columnUpdationTime IS NULL) ORDER BY $columnCreationTime DESC ', + [ownerID], + ); final uploadedFileIDs = []; for (final row in rows) { uploadedFileIDs.add(row[columnUploadedFileID] as int); From a44e5f950528cc3d1f14286b4a21b3765bafed09 Mon Sep 17 00:00:00 2001 From: ashilkn Date: Fri, 17 May 2024 11:47:32 +0530 Subject: [PATCH 018/354] [mob][photos] Migrate to sqlite_async(12): Migrate entities --- mobile/lib/db/entities_db.dart | 79 ++++++++++++++++++++++------------ 1 file changed, 51 insertions(+), 28 deletions(-) diff --git a/mobile/lib/db/entities_db.dart b/mobile/lib/db/entities_db.dart index b8b48fbe4a..d3759389d6 100644 --- a/mobile/lib/db/entities_db.dart +++ b/mobile/lib/db/entities_db.dart @@ -10,53 +10,76 @@ extension EntitiesDB on FilesDB { ConflictAlgorithm conflictAlgorithm = ConflictAlgorithm.replace, }) async { debugPrint("Inserting missing PathIDToLocalIDMapping"); - final db = await database; - var batch = db.batch(); + final db = await sqliteAsyncDB; + final parameterSets = >[]; int batchCounter = 0; for (LocalEntityData e in data) { + parameterSets.add([ + e.id, + e.type.name, + e.ownerID, + e.data, + e.updatedAt, + ]); + batchCounter++; + if (batchCounter == 400) { - await batch.commit(noResult: true); - batch = db.batch(); + await db.executeBatch( + ''' + INSERT OR ${conflictAlgorithm.name.toUpperCase()} + INTO entities (id, type, ownerID, data, updatedAt) +''', + parameterSets, + ); + parameterSets.clear(); batchCounter = 0; } - batch.insert( - "entities", - e.toJson(), - conflictAlgorithm: conflictAlgorithm, - ); - batchCounter++; } - await batch.commit(noResult: true); + await db.executeBatch( + ''' + INSERT OR ${conflictAlgorithm.name.toUpperCase()} + INTO entities (id, type, ownerID, data, updatedAt) +''', + parameterSets, + ); } Future deleteEntities( List ids, ) async { - final db = await database; - var batch = db.batch(); + final db = await sqliteAsyncDB; + final parameterSets = >[]; int batchCounter = 0; for (String id in ids) { - if (batchCounter == 400) { - await batch.commit(noResult: true); - batch = db.batch(); - batchCounter = 0; - } - batch.delete( - "entities", - where: "id = ?", - whereArgs: [id], + parameterSets.add( + [id], ); batchCounter++; + + if (batchCounter == 400) { + await db.executeBatch( + ''' + DELETE FROM entities WHERE id = ? + ''', + parameterSets, + ); + parameterSets.clear(); + batchCounter = 0; + } } - await batch.commit(noResult: true); + await db.executeBatch( + ''' + DELETE FROM entities WHERE id = ? + ''', + parameterSets, + ); } Future> getEntities(EntityType type) async { - final db = await database; - final List> maps = await db.query( - "entities", - where: "type = ?", - whereArgs: [type.typeToString()], + final db = await sqliteAsyncDB; + final List> maps = await db.getAll( + 'SELECT * FROM entities WHERE type = ?', + [type.name], ); return List.generate(maps.length, (i) { return LocalEntityData.fromJson(maps[i]); From c2b6032b6f6214ac074efa208a753f4ce52cde0c Mon Sep 17 00:00:00 2001 From: ashilkn Date: Fri, 17 May 2024 13:40:38 +0530 Subject: [PATCH 019/354] [mob][photos] Fix broken query --- mobile/lib/db/files_db.dart | 26 ++++++++++--------- .../ui/viewer/location/location_screen.dart | 4 +-- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/mobile/lib/db/files_db.dart b/mobile/lib/db/files_db.dart index 2b6219cbe7..16db97337e 100644 --- a/mobile/lib/db/files_db.dart +++ b/mobile/lib/db/files_db.dart @@ -139,14 +139,8 @@ class FilesDB { static Future? _dbFuture; static Future? _sqliteAsyncDBFuture; - @Deprecated("Use sqliteAsyncDB instead (sqlite_async)") - Future get database async { - // lazily instantiate the db the first time it is accessed - _dbFuture ??= _initDatabase(); - return _dbFuture!; - } - Future get sqliteAsyncDB async { + // lazily instantiate the db the first time it is accessed _sqliteAsyncDBFuture ??= _initSqliteAsyncDatabase(); return _sqliteAsyncDBFuture!; } @@ -1720,8 +1714,7 @@ class FilesDB { }) async { final db = await instance.sqliteAsyncDB; final order = (asc ?? false ? 'ASC' : 'DESC'); - final results = await db.getAll( - ''' + String query = ''' SELECT * FROM $filesTable WHERE $columnLatitude IS NOT NULL AND $columnLongitude IS NOT NULL AND ($columnLatitude IS NOT 0 OR $columnLongitude IS NOT 0) AND @@ -1729,9 +1722,18 @@ class FilesDB { ($columnLocalID IS NOT NULL OR ($columnCollectionID IS NOT NULL AND $columnCollectionID IS NOT -1)) ORDER BY $columnCreationTime $order, $columnModificationTime $order - LIMIT $limit - ''', - [startTime, endTime], + '''; + + final args = [startTime, endTime]; + + if (limit != null) { + query += ' LIMIT ?'; + args.add(limit); + } + + final results = await db.getAll( + query, + args, ); final files = convertToFiles(results); final List filteredFiles = diff --git a/mobile/lib/ui/viewer/location/location_screen.dart b/mobile/lib/ui/viewer/location/location_screen.dart index 374d70cb9f..55975dd3fa 100644 --- a/mobile/lib/ui/viewer/location/location_screen.dart +++ b/mobile/lib/ui/viewer/location/location_screen.dart @@ -146,6 +146,8 @@ class _LocationGalleryWidgetState extends State { late final StreamSubscription _filesUpdateEvent; @override void initState() { + super.initState(); + final collectionsToHide = CollectionsService.instance.archivedOrHiddenCollectionIds(); fileLoadResult = FilesDB.instance @@ -179,8 +181,6 @@ class _LocationGalleryWidgetState extends State { }); galleryHeaderWidget = const GalleryHeaderWidget(); - - super.initState(); } @override From 16178b6f098f00914bf872cc90040fb98541b87f Mon Sep 17 00:00:00 2001 From: ashilkn Date: Fri, 17 May 2024 15:09:10 +0530 Subject: [PATCH 020/354] [mob][photos] Add missing paranthesis --- mobile/lib/db/files_db.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobile/lib/db/files_db.dart b/mobile/lib/db/files_db.dart index 16db97337e..444395f7de 100644 --- a/mobile/lib/db/files_db.dart +++ b/mobile/lib/db/files_db.dart @@ -1746,7 +1746,7 @@ class FilesDB { final results = await db.getAll( ''' SELECT DISTINCT $columnUploadedFileID FROM $filesTable - WHERE $columnOwnerID = ? AND $columnUploadedFileID IS NOT NULL AND + WHERE ($columnOwnerID = ? AND $columnUploadedFileID IS NOT NULL AND $columnUploadedFileID IS NOT -1) ''', [ownerID], From 48436694ebbfb087bb92bb021389c85e2da00ad9 Mon Sep 17 00:00:00 2001 From: ashilkn Date: Fri, 17 May 2024 16:28:13 +0530 Subject: [PATCH 021/354] [mob][photos] Fix incorrent sqlite operation --- mobile/lib/db/entities_db.dart | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mobile/lib/db/entities_db.dart b/mobile/lib/db/entities_db.dart index d3759389d6..228e5e2b01 100644 --- a/mobile/lib/db/entities_db.dart +++ b/mobile/lib/db/entities_db.dart @@ -28,6 +28,7 @@ extension EntitiesDB on FilesDB { ''' INSERT OR ${conflictAlgorithm.name.toUpperCase()} INTO entities (id, type, ownerID, data, updatedAt) + VALUES (?, ?, ?, ?, ?) ''', parameterSets, ); @@ -39,6 +40,7 @@ extension EntitiesDB on FilesDB { ''' INSERT OR ${conflictAlgorithm.name.toUpperCase()} INTO entities (id, type, ownerID, data, updatedAt) + VALUES (?, ?, ?, ?, ?) ''', parameterSets, ); From 18d68bbdf34051a77f03322cc6a257bad5baf5c9 Mon Sep 17 00:00:00 2001 From: ashilkn Date: Fri, 17 May 2024 16:34:04 +0530 Subject: [PATCH 022/354] Migrate to sqlite_async(13): Migrate db migration to use sqlite_async --- mobile/lib/db/files_db.dart | 42 +++++++++++++++++++------------------ 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/mobile/lib/db/files_db.dart b/mobile/lib/db/files_db.dart index 444395f7de..9ac322937c 100644 --- a/mobile/lib/db/files_db.dart +++ b/mobile/lib/db/files_db.dart @@ -17,7 +17,6 @@ import "package:photos/services/filter/db_filters.dart"; import 'package:photos/utils/file_uploader_util.dart'; import "package:photos/utils/primitive_wrapper.dart"; import 'package:sqflite/sqflite.dart'; -import 'package:sqflite_migration/sqflite_migration.dart'; import 'package:sqlite_async/sqlite_async.dart' as sqlite_async; class FilesDB { @@ -74,10 +73,8 @@ class FilesDB { // we need to write query based on that field static const columnMMdVisibility = 'mmd_visibility'; - static final initializationScript = [ - ...createTable(filesTable), - ]; static final migrationScripts = [ + ...createTable(filesTable), ...alterDeviceFolderToAllowNULL(), ...alterTimestampColumnTypes(), ...addIndices(), @@ -125,18 +122,12 @@ class FilesDB { columnAddedTime, ]; - final dbConfig = MigrationConfig( - initializationScript: initializationScript, - migrationScripts: migrationScripts, - ); - // make this a singleton class FilesDB._privateConstructor(); static final FilesDB instance = FilesDB._privateConstructor(); // only have a single app-wide reference to the database - static Future? _dbFuture; static Future? _sqliteAsyncDBFuture; Future get sqliteAsyncDB async { @@ -146,20 +137,31 @@ class FilesDB { } // this opens the database (and creates it if it doesn't exist) - Future _initDatabase() async { - final Directory documentsDirectory = - await getApplicationDocumentsDirectory(); - final String path = join(documentsDirectory.path, _databaseName); - _logger.info("DB path " + path); - return await openDatabaseWithMigration(path, dbConfig); - } - Future _initSqliteAsyncDatabase() async { final Directory documentsDirectory = await getApplicationDocumentsDirectory(); final String path = join(documentsDirectory.path, _databaseName); _logger.info("DB path " + path); - return sqlite_async.SqliteDatabase(path: path); + final migrations = getMigrations(); + final database = sqlite_async.SqliteDatabase(path: path); + await migrations.migrate(database); + return database; + } + + sqlite_async.SqliteMigrations getMigrations() { + final numberOfMigrationScripts = migrationScripts.length; + final migrations = sqlite_async.SqliteMigrations(); + for (int i = 0; i < numberOfMigrationScripts; i++) { + migrations.add( + sqlite_async.SqliteMigration( + i + 1, + (tx) async { + await tx.execute(migrationScripts[i]); + }, + ), + ); + } + return migrations; } // SQL code to create the database table @@ -443,7 +445,7 @@ class FilesDB { await getApplicationDocumentsDirectory(); final String path = join(documentsDirectory.path, _databaseName); File(path).deleteSync(recursive: true); - _dbFuture = null; + _sqliteAsyncDBFuture = null; } } From ab9cef689de29a6c2e9bda85acb12e428457f5b2 Mon Sep 17 00:00:00 2001 From: ashilkn Date: Fri, 17 May 2024 16:40:59 +0530 Subject: [PATCH 023/354] [mob][photos] Create ConflictAlgorithm enum and stop using it from sqflite --- mobile/lib/db/files_db.dart | 3 ++- mobile/lib/utils/sqlite_util.dart | 39 +++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) create mode 100644 mobile/lib/utils/sqlite_util.dart diff --git a/mobile/lib/db/files_db.dart b/mobile/lib/db/files_db.dart index 9ac322937c..fdd361a677 100644 --- a/mobile/lib/db/files_db.dart +++ b/mobile/lib/db/files_db.dart @@ -16,7 +16,8 @@ import "package:photos/models/metadata/common_keys.dart"; import "package:photos/services/filter/db_filters.dart"; import 'package:photos/utils/file_uploader_util.dart'; import "package:photos/utils/primitive_wrapper.dart"; -import 'package:sqflite/sqflite.dart'; +import "package:photos/utils/sqlite_util.dart"; +// import 'package:sqflite/sqflite.dart'; import 'package:sqlite_async/sqlite_async.dart' as sqlite_async; class FilesDB { diff --git a/mobile/lib/utils/sqlite_util.dart b/mobile/lib/utils/sqlite_util.dart new file mode 100644 index 0000000000..d236679bcc --- /dev/null +++ b/mobile/lib/utils/sqlite_util.dart @@ -0,0 +1,39 @@ +enum ConflictAlgorithm { + /// When a constraint violation occurs, an immediate ROLLBACK occurs, + /// thus ending the current transaction, and the command aborts with a + /// return code of SQLITE_CONSTRAINT. If no transaction is active + /// (other than the implied transaction that is created on every command) + /// then this algorithm works the same as ABORT. + rollback, + + /// When a constraint violation occurs,no ROLLBACK is executed + /// so changes from prior commands within the same transaction + /// are preserved. This is the default behavior. + abort, + + /// When a constraint violation occurs, the command aborts with a return + /// code SQLITE_CONSTRAINT. But any changes to the database that + /// the command made prior to encountering the constraint violation + /// are preserved and are not backed out. + fail, + + /// When a constraint violation occurs, the one row that contains + /// the constraint violation is not inserted or changed. + /// But the command continues executing normally. Other rows before and + /// after the row that contained the constraint violation continue to be + /// inserted or updated normally. No error is returned. + ignore, + + /// When a UNIQUE constraint violation occurs, the pre-existing rows that + /// are causing the constraint violation are removed prior to inserting + /// or updating the current row. Thus the insert or update always occurs. + /// The command continues executing normally. No error is returned. + /// If a NOT NULL constraint violation occurs, the NULL value is replaced + /// by the default value for that column. If the column has no default + /// value, then the ABORT algorithm is used. If a CHECK constraint + /// violation occurs then the IGNORE algorithm is used. When this conflict + /// resolution strategy deletes rows in order to satisfy a constraint, + /// it does not invoke delete triggers on those rows. + /// This behavior might change in a future release. + replace, +} From a7e0f3df7bc252b0f422021c410725c15c029429 Mon Sep 17 00:00:00 2001 From: ashilkn Date: Fri, 17 May 2024 17:05:58 +0530 Subject: [PATCH 024/354] [mob][photos] Remove sqflite import in filesDB --- mobile/lib/db/files_db.dart | 17 ++++++++--------- mobile/lib/services/local_sync_service.dart | 6 +++--- mobile/lib/utils/sqlite_util.dart | 2 +- 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/mobile/lib/db/files_db.dart b/mobile/lib/db/files_db.dart index fdd361a677..7f82759e52 100644 --- a/mobile/lib/db/files_db.dart +++ b/mobile/lib/db/files_db.dart @@ -452,7 +452,8 @@ class FilesDB { Future insertMultiple( List files, { - ConflictAlgorithm conflictAlgorithm = ConflictAlgorithm.replace, + SqliteAsyncConflictAlgorithm conflictAlgorithm = + SqliteAsyncConflictAlgorithm.replace, }) async { if (files.isEmpty) return; @@ -1282,13 +1283,11 @@ class FilesDB { Future collectionFileCount(int collectionID) async { final db = await instance.sqliteAsyncDB; - final count = Sqflite.firstIntValue( - await db.execute( - 'SELECT COUNT(*) FROM $filesTable where $columnCollectionID = ' - '$collectionID AND $columnUploadedFileID IS NOT -1', - ), + final row = await db.get( + 'SELECT COUNT(*) FROM $filesTable where $columnCollectionID = ' + '$collectionID AND $columnUploadedFileID IS NOT -1', ); - return count ?? 0; + return row['COUNT(*)'] as int; } Future archivedFilesCount( @@ -1884,7 +1883,7 @@ class FilesDB { Future _batchAndInsertFile( EnteFile file, - ConflictAlgorithm conflictAlgorithm, + SqliteAsyncConflictAlgorithm conflictAlgorithm, sqlite_async.SqliteDatabase db, List> parameterSets, PrimitiveWrapper batchCounter, { @@ -1905,7 +1904,7 @@ class FilesDB { } Future _insertBatch( - ConflictAlgorithm conflictAlgorithm, + SqliteAsyncConflictAlgorithm conflictAlgorithm, Iterable columnNames, sqlite_async.SqliteDatabase db, List> parameterSets, diff --git a/mobile/lib/services/local_sync_service.dart b/mobile/lib/services/local_sync_service.dart index 93b3c94373..1915ac30c2 100644 --- a/mobile/lib/services/local_sync_service.dart +++ b/mobile/lib/services/local_sync_service.dart @@ -21,8 +21,8 @@ import "package:photos/services/ignored_files_service.dart"; import 'package:photos/services/local/local_sync_util.dart'; import "package:photos/utils/debouncer.dart"; import "package:photos/utils/photo_manager_util.dart"; +import "package:photos/utils/sqlite_util.dart"; import 'package:shared_preferences/shared_preferences.dart'; -import 'package:sqflite/sqflite.dart'; import 'package:tuple/tuple.dart'; class LocalSyncService { @@ -184,7 +184,7 @@ class LocalSyncService { if (hasUnsyncedFiles) { await _db.insertMultiple( localDiffResult.uniqueLocalFiles!, - conflictAlgorithm: ConflictAlgorithm.ignore, + conflictAlgorithm: SqliteAsyncConflictAlgorithm.ignore, ); _logger.info( "Inserted ${localDiffResult.uniqueLocalFiles?.length} " @@ -321,7 +321,7 @@ class LocalSyncService { files.removeWhere((file) => existingLocalDs.contains(file.localID)); await _db.insertMultiple( files, - conflictAlgorithm: ConflictAlgorithm.ignore, + conflictAlgorithm: SqliteAsyncConflictAlgorithm.ignore, ); _logger.info('Inserted ${files.length} files'); Bus.instance.fire( diff --git a/mobile/lib/utils/sqlite_util.dart b/mobile/lib/utils/sqlite_util.dart index d236679bcc..b83bd58e15 100644 --- a/mobile/lib/utils/sqlite_util.dart +++ b/mobile/lib/utils/sqlite_util.dart @@ -1,4 +1,4 @@ -enum ConflictAlgorithm { +enum SqliteAsyncConflictAlgorithm { /// When a constraint violation occurs, an immediate ROLLBACK occurs, /// thus ending the current transaction, and the command aborts with a /// return code of SQLITE_CONSTRAINT. If no transaction is active From 49e64b3d4ce3a7c4523e6562f24137c04602c4d5 Mon Sep 17 00:00:00 2001 From: ashilkn Date: Tue, 21 May 2024 16:12:44 +0530 Subject: [PATCH 025/354] [mob][photos] Fix issue with EnteFile not having location data --- mobile/lib/db/files_db.dart | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mobile/lib/db/files_db.dart b/mobile/lib/db/files_db.dart index 7f82759e52..5bad957583 100644 --- a/mobile/lib/db/files_db.dart +++ b/mobile/lib/db/files_db.dart @@ -1824,8 +1824,8 @@ class FilesDB { }) { final values = []; - double? latitude; - double? longitude; + double? latitude = file.location?.latitude; + double? longitude = file.location?.longitude; int? creationTime = file.creationTime; if (file.pubMagicMetadata != null) { From b2a359ca598cc3b0972dcf15dba5106eee8c5113 Mon Sep 17 00:00:00 2001 From: ashilkn Date: Tue, 21 May 2024 16:53:49 +0530 Subject: [PATCH 026/354] [mob][photos] Migrate to sqlite_async(13): Use getAll() instead of execute() for SELECT commands --- mobile/lib/db/files_db.dart | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/mobile/lib/db/files_db.dart b/mobile/lib/db/files_db.dart index 323c1aed22..f9435b8329 100644 --- a/mobile/lib/db/files_db.dart +++ b/mobile/lib/db/files_db.dart @@ -1120,7 +1120,7 @@ class FilesDB { return {}; } final inParam = hashes.map((e) => "'$e'").join(','); - final rows = await db.execute(''' + final rows = await db.getAll(''' SELECT * FROM $filesTable WHERE $columnHash IN ($inParam) AND $columnOwnerID = $userID; '''); final matchedFiles = convertToFiles(rows); @@ -1319,7 +1319,7 @@ class FilesDB { Set hiddenCollections, ) async { final db = await instance.sqliteAsyncDB; - final count = await db.execute( + final count = await db.getAll( 'SELECT COUNT(distinct($columnUploadedFileID)) as COUNT FROM $filesTable where ' '$columnMMdVisibility' ' = $visibility AND $columnOwnerID = $ownerID AND $columnCollectionID NOT IN (${hiddenCollections.join(', ')})', @@ -1373,7 +1373,7 @@ class FilesDB { } inParam = inParam.substring(0, inParam.length - 1); final db = await instance.sqliteAsyncDB; - final rows = await db.execute( + final rows = await db.getAll( ''' SELECT $columnLocalID FROM $filesTable @@ -1393,7 +1393,7 @@ class FilesDB { Future> getCollectionIDToMaxCreationTime() async { final enteWatch = EnteWatch("getCollectionIDToMaxCreationTime")..start(); final db = await instance.sqliteAsyncDB; - final rows = await db.execute( + final rows = await db.getAll( ''' SELECT $columnCollectionID, MAX($columnCreationTime) AS max_creation_time FROM $filesTable @@ -1730,7 +1730,7 @@ class FilesDB { Future> fetchFilesCountbyType(int userID) async { final db = await instance.sqliteAsyncDB; - final result = await db.execute( + final result = await db.getAll( ''' SELECT $columnFileType, COUNT(DISTINCT $columnUploadedFileID) FROM $filesTable WHERE $columnUploadedFileID != -1 AND From 159fdf83ad34cea421eb8882869eaa1919bb086d Mon Sep 17 00:00:00 2001 From: ashilkn Date: Tue, 21 May 2024 16:54:09 +0530 Subject: [PATCH 027/354] [mob][photos] Migrate to sqlite_async(14) --- mobile/lib/db/entities_db.dart | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/mobile/lib/db/entities_db.dart b/mobile/lib/db/entities_db.dart index 004fdf0aee..3cd9d47639 100644 --- a/mobile/lib/db/entities_db.dart +++ b/mobile/lib/db/entities_db.dart @@ -89,11 +89,10 @@ extension EntitiesDB on FilesDB { } Future getEntity(EntityType type, String id) async { - final db = await database; - final List> maps = await db.query( - "entities", - where: "type = ? AND id = ?", - whereArgs: [type.typeToString(), id], + final db = await sqliteAsyncDB; + final List> maps = await db.getAll( + 'SELECT * FROM entities WHERE type = ? AND id = ?', + [type.name, id], ); if (maps.isEmpty) { return null; From 5a017616f5922ae5d49d2527eedcf802e09304ca Mon Sep 17 00:00:00 2001 From: ashilkn Date: Tue, 21 May 2024 17:10:42 +0530 Subject: [PATCH 028/354] [mob][photos] Fix sqlite command syntax errors --- mobile/lib/db/files_db.dart | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mobile/lib/db/files_db.dart b/mobile/lib/db/files_db.dart index f9435b8329..e9fc70d458 100644 --- a/mobile/lib/db/files_db.dart +++ b/mobile/lib/db/files_db.dart @@ -1138,7 +1138,7 @@ class FilesDB { } final db = await instance.sqliteAsyncDB; final rows = await db.getAll( - 'SELECT * FROM $filesTable WHERE $columnUploadedFileID != NULL OR ' + 'SELECT * FROM $filesTable WHERE ($columnUploadedFileID != NULL OR ' '$columnUploadedFileID != -1) AND $columnOwnerID = ? AND ' '$columnFileType = ? AND $columnHash IN ($inParam)', [ @@ -1734,7 +1734,7 @@ class FilesDB { ''' SELECT $columnFileType, COUNT(DISTINCT $columnUploadedFileID) FROM $filesTable WHERE $columnUploadedFileID != -1 AND - $columnOwnerID == $userID GROUP BY $columnFileType + $columnOwnerID IS $userID GROUP BY $columnFileType ''', ); From e3ea22f479b2db6433117cdd9491dcd2026525dc Mon Sep 17 00:00:00 2001 From: ashilkn Date: Tue, 21 May 2024 17:44:38 +0530 Subject: [PATCH 029/354] [mob][photos] add comment --- mobile/lib/db/files_db.dart | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mobile/lib/db/files_db.dart b/mobile/lib/db/files_db.dart index e9fc70d458..03f5d832b8 100644 --- a/mobile/lib/db/files_db.dart +++ b/mobile/lib/db/files_db.dart @@ -74,6 +74,9 @@ class FilesDB { // we need to write query based on that field static const columnMMdVisibility = 'mmd_visibility'; +//If adding or removing a new column, make sure to update the `_columnNames` list +//and update `_generateColumnsAndPlaceholdersForInsert` and +//`_generateUpdateAssignmentsWithPlaceholders` static final migrationScripts = [ ...createTable(filesTable), ...alterDeviceFolderToAllowNULL(), From eaca151a9fc503ff48c9a9c7f73b1de73eeb3954 Mon Sep 17 00:00:00 2001 From: ashilkn Date: Tue, 21 May 2024 18:34:11 +0530 Subject: [PATCH 030/354] [mob][photos] Minor change --- mobile/lib/db/files_db.dart | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/mobile/lib/db/files_db.dart b/mobile/lib/db/files_db.dart index 03f5d832b8..0de84df68c 100644 --- a/mobile/lib/db/files_db.dart +++ b/mobile/lib/db/files_db.dart @@ -17,8 +17,7 @@ import "package:photos/services/filter/db_filters.dart"; import 'package:photos/utils/file_uploader_util.dart'; import "package:photos/utils/primitive_wrapper.dart"; import "package:photos/utils/sqlite_util.dart"; -// import 'package:sqflite/sqflite.dart'; -import 'package:sqlite_async/sqlite_async.dart' as sqlite_async; +import 'package:sqlite_async/sqlite_async.dart'; class FilesDB { /* @@ -132,32 +131,32 @@ class FilesDB { static final FilesDB instance = FilesDB._privateConstructor(); // only have a single app-wide reference to the database - static Future? _sqliteAsyncDBFuture; + static Future? _sqliteAsyncDBFuture; - Future get sqliteAsyncDB async { + Future get sqliteAsyncDB async { // lazily instantiate the db the first time it is accessed _sqliteAsyncDBFuture ??= _initSqliteAsyncDatabase(); return _sqliteAsyncDBFuture!; } // this opens the database (and creates it if it doesn't exist) - Future _initSqliteAsyncDatabase() async { + Future _initSqliteAsyncDatabase() async { final Directory documentsDirectory = await getApplicationDocumentsDirectory(); final String path = join(documentsDirectory.path, _databaseName); _logger.info("DB path " + path); final migrations = getMigrations(); - final database = sqlite_async.SqliteDatabase(path: path); + final database = SqliteDatabase(path: path); await migrations.migrate(database); return database; } - sqlite_async.SqliteMigrations getMigrations() { + SqliteMigrations getMigrations() { final numberOfMigrationScripts = migrationScripts.length; - final migrations = sqlite_async.SqliteMigrations(); + final migrations = SqliteMigrations(); for (int i = 0; i < numberOfMigrationScripts; i++) { migrations.add( - sqlite_async.SqliteMigration( + SqliteMigration( i + 1, (tx) async { await tx.execute(migrationScripts[i]); @@ -1927,7 +1926,7 @@ class FilesDB { Future _batchAndInsertFile( EnteFile file, SqliteAsyncConflictAlgorithm conflictAlgorithm, - sqlite_async.SqliteDatabase db, + SqliteDatabase db, List> parameterSets, PrimitiveWrapper batchCounter, { required bool isGenIdNull, @@ -1949,7 +1948,7 @@ class FilesDB { Future _insertBatch( SqliteAsyncConflictAlgorithm conflictAlgorithm, Iterable columnNames, - sqlite_async.SqliteDatabase db, + SqliteDatabase db, List> parameterSets, ) async { final valuesPlaceholders = List.filled(columnNames.length, "?").join(","); From 4fb9e75394a0fdddee2e9b7772c9dc9461900353 Mon Sep 17 00:00:00 2001 From: ashilkn Date: Tue, 21 May 2024 18:36:01 +0530 Subject: [PATCH 031/354] [mob][photos] Bump up version to 0.8.99 --- mobile/pubspec.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobile/pubspec.yaml b/mobile/pubspec.yaml index b9d5345c39..31c605ab7f 100644 --- a/mobile/pubspec.yaml +++ b/mobile/pubspec.yaml @@ -12,7 +12,7 @@ description: ente photos application # Read more about iOS versioning at # https://developer.apple.com/library/archive/documentation/General/Reference/InfoPlistKeyReference/Articles/CoreFoundationKeys.html -version: 0.8.98+618 +version: 0.8.99+619 publish_to: none environment: From f513473362568278e8c2c5ba5c78245af4d8b127 Mon Sep 17 00:00:00 2001 From: ashilkn Date: Wed, 22 May 2024 13:44:19 +0530 Subject: [PATCH 032/354] [mob][photos] Check db version when sqflite was used and run only migrations that are necessary using sqlite_async Tested adding a new migration and it works. Tested two cases (a)Fresh install (b)Opening app with new migration added and the last db migration was done when sqflite was used --- mobile/lib/db/files_db.dart | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/mobile/lib/db/files_db.dart b/mobile/lib/db/files_db.dart index 0de84df68c..fd258b3690 100644 --- a/mobile/lib/db/files_db.dart +++ b/mobile/lib/db/files_db.dart @@ -145,16 +145,23 @@ class FilesDB { await getApplicationDocumentsDirectory(); final String path = join(documentsDirectory.path, _databaseName); _logger.info("DB path " + path); - final migrations = getMigrations(); final database = SqliteDatabase(path: path); + final versionRow = await database.execute('PRAGMA user_version'); + + //db version used to be stored in `user_version` when using sqflite. + //sqlite_async doesn't use `user_version` to store db version. + // `oldVersionNumber` = 0 for fresh install + final oldVersionNumber = versionRow[0]['user_version'] as int; + final migrations = getMigrations(oldVersionNumber); + await migrations.migrate(database); return database; } - SqliteMigrations getMigrations() { + SqliteMigrations getMigrations(int oldSqfliteDBVersion) { final numberOfMigrationScripts = migrationScripts.length; final migrations = SqliteMigrations(); - for (int i = 0; i < numberOfMigrationScripts; i++) { + for (int i = oldSqfliteDBVersion; i < numberOfMigrationScripts; i++) { migrations.add( SqliteMigration( i + 1, From cb9ac0d939214e79231c7489e2ff9f926479c5e0 Mon Sep 17 00:00:00 2001 From: ashilkn Date: Wed, 22 May 2024 14:21:31 +0530 Subject: [PATCH 033/354] [mob][photos] bump up version to 0.8.100 --- mobile/pubspec.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobile/pubspec.yaml b/mobile/pubspec.yaml index 31c605ab7f..7b429b4578 100644 --- a/mobile/pubspec.yaml +++ b/mobile/pubspec.yaml @@ -12,7 +12,7 @@ description: ente photos application # Read more about iOS versioning at # https://developer.apple.com/library/archive/documentation/General/Reference/InfoPlistKeyReference/Articles/CoreFoundationKeys.html -version: 0.8.99+619 +version: 0.8.100+620 publish_to: none environment: From 22fc67c8c3872adc4d26c623a22fe6e7400a8c1e Mon Sep 17 00:00:00 2001 From: ashilkn Date: Wed, 22 May 2024 16:17:05 +0530 Subject: [PATCH 034/354] [mob][photos] Remove unnecessary parameters --- mobile/lib/db/files_db.dart | 2 -- 1 file changed, 2 deletions(-) diff --git a/mobile/lib/db/files_db.dart b/mobile/lib/db/files_db.dart index fd258b3690..d690e945e7 100644 --- a/mobile/lib/db/files_db.dart +++ b/mobile/lib/db/files_db.dart @@ -1058,10 +1058,8 @@ class FilesDB { title, location?.latitude, location?.longitude, - localID, creationTime, modificationTime, - null, getInt(fileType), localID, ownerID, From 7aa26a950d35c0f4c64eccfe4c9c273e0dc79d90 Mon Sep 17 00:00:00 2001 From: ashilkn Date: Wed, 22 May 2024 20:44:10 +0530 Subject: [PATCH 035/354] [mob][photos] Bump up to version 0.8.103 --- mobile/pubspec.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobile/pubspec.yaml b/mobile/pubspec.yaml index 2e80192929..b1d4fff03f 100644 --- a/mobile/pubspec.yaml +++ b/mobile/pubspec.yaml @@ -12,7 +12,7 @@ description: ente photos application # Read more about iOS versioning at # https://developer.apple.com/library/archive/documentation/General/Reference/InfoPlistKeyReference/Articles/CoreFoundationKeys.html -version: 0.8.102+626 +version: 0.8.103+627 publish_to: none environment: From 637adb46170bbb9cca2a5c7f9ab3e144ef1f171a Mon Sep 17 00:00:00 2001 From: ashilkn Date: Fri, 24 May 2024 14:21:02 +0530 Subject: [PATCH 036/354] [mob][photos] Simplify how FilesDB migrates --- mobile/lib/db/files_db.dart | 44 +++++++++++++++++++------------------ 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/mobile/lib/db/files_db.dart b/mobile/lib/db/files_db.dart index d690e945e7..a0b400cbc1 100644 --- a/mobile/lib/db/files_db.dart +++ b/mobile/lib/db/files_db.dart @@ -146,32 +146,34 @@ class FilesDB { final String path = join(documentsDirectory.path, _databaseName); _logger.info("DB path " + path); final database = SqliteDatabase(path: path); - final versionRow = await database.execute('PRAGMA user_version'); + await _migrate(database); - //db version used to be stored in `user_version` when using sqflite. - //sqlite_async doesn't use `user_version` to store db version. - // `oldVersionNumber` = 0 for fresh install - final oldVersionNumber = versionRow[0]['user_version'] as int; - final migrations = getMigrations(oldVersionNumber); - - await migrations.migrate(database); return database; } - SqliteMigrations getMigrations(int oldSqfliteDBVersion) { - final numberOfMigrationScripts = migrationScripts.length; - final migrations = SqliteMigrations(); - for (int i = oldSqfliteDBVersion; i < numberOfMigrationScripts; i++) { - migrations.add( - SqliteMigration( - i + 1, - (tx) async { - await tx.execute(migrationScripts[i]); - }, - ), - ); + Future _migrate( + SqliteDatabase database, + ) async { + final result = await database.execute('PRAGMA user_version'); + final currentVersion = result[0]['user_version'] as int; + final toVersion = migrationScripts.length; + + _logger.info("currentVersion: $currentVersion"); + _logger.info("toVersion: $toVersion"); + + if (currentVersion < toVersion) { + _logger.info("Migrating database from $currentVersion to $toVersion"); + await database.writeTransaction((tx) async { + for (int i = currentVersion + 1; i <= toVersion; i++) { + await tx.execute(migrationScripts[i - 1]); + } + await tx.execute('PRAGMA user_version = $toVersion'); + }); + } else if (currentVersion > toVersion) { + throw AssertionError("currentVersion cannot be greater than toVersion"); + } else { + _logger.info("Database is already at version $toVersion"); } - return migrations; } // SQL code to create the database table From 500d7da30671e6df35cb8b8d661a1ef02452beee Mon Sep 17 00:00:00 2001 From: ashilkn Date: Fri, 24 May 2024 14:39:16 +0530 Subject: [PATCH 037/354] [mob][photos] Remove log lines used for testing --- mobile/lib/db/files_db.dart | 5 ----- 1 file changed, 5 deletions(-) diff --git a/mobile/lib/db/files_db.dart b/mobile/lib/db/files_db.dart index a0b400cbc1..b7c67ff36c 100644 --- a/mobile/lib/db/files_db.dart +++ b/mobile/lib/db/files_db.dart @@ -158,9 +158,6 @@ class FilesDB { final currentVersion = result[0]['user_version'] as int; final toVersion = migrationScripts.length; - _logger.info("currentVersion: $currentVersion"); - _logger.info("toVersion: $toVersion"); - if (currentVersion < toVersion) { _logger.info("Migrating database from $currentVersion to $toVersion"); await database.writeTransaction((tx) async { @@ -171,8 +168,6 @@ class FilesDB { }); } else if (currentVersion > toVersion) { throw AssertionError("currentVersion cannot be greater than toVersion"); - } else { - _logger.info("Database is already at version $toVersion"); } } From a79d11c2636b2cbe09d166e3b80491b962695f61 Mon Sep 17 00:00:00 2001 From: ashilkn Date: Fri, 24 May 2024 14:43:39 +0530 Subject: [PATCH 038/354] [mob][photos] Add more info in error message --- mobile/lib/db/files_db.dart | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mobile/lib/db/files_db.dart b/mobile/lib/db/files_db.dart index b7c67ff36c..c6ba617e0e 100644 --- a/mobile/lib/db/files_db.dart +++ b/mobile/lib/db/files_db.dart @@ -167,7 +167,9 @@ class FilesDB { await tx.execute('PRAGMA user_version = $toVersion'); }); } else if (currentVersion > toVersion) { - throw AssertionError("currentVersion cannot be greater than toVersion"); + throw AssertionError( + "currentVersion($currentVersion) cannot be greater than toVersion($toVersion)", + ); } } From 022448155dc01478527145d8a4323ad9d24178be Mon Sep 17 00:00:00 2001 From: ashilkn Date: Fri, 24 May 2024 15:48:39 +0530 Subject: [PATCH 039/354] [mob][photos] Bump up version to v0.8.111 --- mobile/pubspec.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobile/pubspec.yaml b/mobile/pubspec.yaml index 1417d17f3e..c9df14e163 100644 --- a/mobile/pubspec.yaml +++ b/mobile/pubspec.yaml @@ -12,7 +12,7 @@ description: ente photos application # Read more about iOS versioning at # https://developer.apple.com/library/archive/documentation/General/Reference/InfoPlistKeyReference/Articles/CoreFoundationKeys.html -version: 0.8.110+634 +version: 0.8.111+635 publish_to: none environment: From 1ec7e026950ea1be4d7d50879009d0b461cda739 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Sat, 25 May 2024 12:03:34 +0530 Subject: [PATCH 040/354] [mob][photos] Copy change --- mobile/lib/generated/intl/messages_en.dart | 2 +- mobile/lib/generated/l10n.dart | 4 ++-- mobile/lib/l10n/intl_en.arb | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/mobile/lib/generated/intl/messages_en.dart b/mobile/lib/generated/intl/messages_en.dart index 320df2c1d0..baf50af582 100644 --- a/mobile/lib/generated/intl/messages_en.dart +++ b/mobile/lib/generated/intl/messages_en.dart @@ -814,7 +814,7 @@ class MessageLookup extends MessageLookupByLibrary { MessageLookupByLibrary.simpleMessage("Incorrect recovery key"), "indexedItems": MessageLookupByLibrary.simpleMessage("Indexed items"), "indexingIsPaused": MessageLookupByLibrary.simpleMessage( - "Indexing is paused, will automatically resume when device is ready"), + "Indexing is paused. It will automatically resume when device is ready"), "insecureDevice": MessageLookupByLibrary.simpleMessage("Insecure device"), "installManually": diff --git a/mobile/lib/generated/l10n.dart b/mobile/lib/generated/l10n.dart index de8922161d..a6cd47cbdc 100644 --- a/mobile/lib/generated/l10n.dart +++ b/mobile/lib/generated/l10n.dart @@ -8794,10 +8794,10 @@ class S { ); } - /// `Indexing is paused, will automatically resume when device is ready` + /// `Indexing is paused. It will automatically resume when device is ready` String get indexingIsPaused { return Intl.message( - 'Indexing is paused, will automatically resume when device is ready', + 'Indexing is paused. It will automatically resume when device is ready', name: 'indexingIsPaused', desc: '', args: [], diff --git a/mobile/lib/l10n/intl_en.arb b/mobile/lib/l10n/intl_en.arb index 08e794074b..a63d8674e5 100644 --- a/mobile/lib/l10n/intl_en.arb +++ b/mobile/lib/l10n/intl_en.arb @@ -1236,5 +1236,5 @@ "faceRecognitionIndexingDescription": "Please note that this will result in a higher bandwidth and battery usage until all items are indexed.", "foundFaces": "Found faces", "clusteringProgress": "Clustering progress", - "indexingIsPaused": "Indexing is paused, will automatically resume when device is ready" -} \ No newline at end of file + "indexingIsPaused": "Indexing is paused. It will automatically resume when device is ready" +} From 7b4559f3cafaedb9b064e2a0ce38ed10675f416d Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Mon, 27 May 2024 10:49:42 +0530 Subject: [PATCH 041/354] [mob][photos] Reduce clustering frequency --- .../lib/services/machine_learning/face_ml/face_ml_service.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart index bbe719dbe1..de02c5672a 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart @@ -99,7 +99,7 @@ class FaceMlService { final int _fileDownloadLimit = 5; final int _embeddingFetchLimit = 200; - final int _kForceClusteringFaceCount = 4000; + final int _kForceClusteringFaceCount = 8000; Future init({bool initializeImageMlIsolate = false}) async { if (LocalSettings.instance.isFaceIndexingEnabled == false) { From b100f1d4bfe27c646bca14c761e48b320bd052c8 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Mon, 27 May 2024 11:28:05 +0530 Subject: [PATCH 042/354] [mob][photos] Catch and stopwatch on faces db creation --- mobile/lib/face/db.dart | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/mobile/lib/face/db.dart b/mobile/lib/face/db.dart index 3ad90915d4..388b30b1e3 100644 --- a/mobile/lib/face/db.dart +++ b/mobile/lib/face/db.dart @@ -55,12 +55,21 @@ class FaceMLDataDB { } Future _onCreate(SqliteDatabase asyncDBConnection) async { - await asyncDBConnection.execute(createFacesTable); - await asyncDBConnection.execute(createFaceClustersTable); - await asyncDBConnection.execute(createClusterPersonTable); - await asyncDBConnection.execute(createClusterSummaryTable); - await asyncDBConnection.execute(createNotPersonFeedbackTable); - await asyncDBConnection.execute(fcClusterIDIndex); + try { + final startTime = DateTime.now(); + await asyncDBConnection.execute(createFacesTable); + await asyncDBConnection.execute(createFaceClustersTable); + await asyncDBConnection.execute(createClusterPersonTable); + await asyncDBConnection.execute(createClusterSummaryTable); + await asyncDBConnection.execute(createNotPersonFeedbackTable); + await asyncDBConnection.execute(fcClusterIDIndex); + _logger.info( + 'FaceMLDataDB tables created in ${DateTime.now().difference(startTime).inMilliseconds}ms', + ); + } catch (e, s) { + _logger.severe("Error creating FaceMLDataDB tables", e, s); + rethrow; + } } // bulkInsertFaces inserts the faces in the database in batches of 1000. From b2e8c3c0eb86c2a1b1ff07420b4031a764e347ca Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Mon, 27 May 2024 11:51:20 +0530 Subject: [PATCH 043/354] [mob][photos] Remove restriction for ML for F-Droid --- mobile/lib/main.dart | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/mobile/lib/main.dart b/mobile/lib/main.dart index 50de0b9a11..3096f8fd3e 100644 --- a/mobile/lib/main.dart +++ b/mobile/lib/main.dart @@ -238,14 +238,12 @@ Future _init(bool isBackground, {String via = ''}) async { // Can not including existing tf/ml binaries as they are not being built // from source. // See https://gitlab.com/fdroid/fdroiddata/-/merge_requests/12671#note_1294346819 - if (!UpdateService.instance.isFdroidFlavor()) { - // unawaited(ObjectDetectionService.instance.init()); - if (flagService.faceSearchEnabled) { - unawaited(FaceMlService.instance.init()); - } else { - if (LocalSettings.instance.isFaceIndexingEnabled) { - unawaited(LocalSettings.instance.toggleFaceIndexing()); - } + // unawaited(ObjectDetectionService.instance.init()); + if (flagService.faceSearchEnabled) { + unawaited(FaceMlService.instance.init()); + } else { + if (LocalSettings.instance.isFaceIndexingEnabled) { + unawaited(LocalSettings.instance.toggleFaceIndexing()); } } PersonService.init( From 86fb8ebfafa83ba1a7da2ebfee4d50cdd7748fc3 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Mon, 27 May 2024 11:57:40 +0530 Subject: [PATCH 044/354] [mob][photos] Fix indexing issue on iOS --- .../face_ml/face_ml_service.dart | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart index de02c5672a..9f153ffa89 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart @@ -43,6 +43,7 @@ import 'package:photos/services/machine_learning/face_ml/face_ml_result.dart'; import "package:photos/services/machine_learning/face_ml/person/person_service.dart"; import 'package:photos/services/machine_learning/file_ml/file_ml.dart'; import 'package:photos/services/machine_learning/file_ml/remote_fileml_service.dart'; +import "package:photos/services/machine_learning/machine_learning_controller.dart"; import "package:photos/services/search_service.dart"; import "package:photos/utils/file_util.dart"; import 'package:photos/utils/image_ml_isolate.dart'; @@ -163,9 +164,16 @@ class FaceMlService { pauseIndexingAndClustering(); } }); + if (Platform.isIOS && + MachineLearningController.instance.isDeviceHealthy) { + _logger.info("Starting face indexing and clustering on iOS from init"); + unawaited(indexAndClusterAll()); + } _listenIndexOnDiffSync(); _listenOnPeopleChangedSync(); + + _logger.info('init done'); }); } @@ -1016,9 +1024,13 @@ class FaceMlService { File? file; if (enteFile.fileType == FileType.video) { try { - file = await getThumbnailForUploadedFile(enteFile); + file = await getThumbnailForUploadedFile(enteFile); } on PlatformException catch (e, s) { - _logger.severe("Could not get thumbnail for $enteFile due to PlatformException", e, s); + _logger.severe( + "Could not get thumbnail for $enteFile due to PlatformException", + e, + s, + ); throw ThumbnailRetrievalException(e.toString(), s); } } else { From 30ade541dfe1525197f0b837f22ced8d963eac37 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Mon, 27 May 2024 11:57:46 +0530 Subject: [PATCH 045/354] [mob][photos] Logging --- .../services/machine_learning/machine_learning_controller.dart | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mobile/lib/services/machine_learning/machine_learning_controller.dart b/mobile/lib/services/machine_learning/machine_learning_controller.dart index 1b70ea48d8..40a52a419d 100644 --- a/mobile/lib/services/machine_learning/machine_learning_controller.dart +++ b/mobile/lib/services/machine_learning/machine_learning_controller.dart @@ -31,6 +31,7 @@ class MachineLearningController { bool get isDeviceHealthy => _isDeviceHealthy; void init() { + _logger.info('init called'); if (Platform.isAndroid) { _startInteractionTimer(); BatteryInfoPlugin() @@ -47,6 +48,7 @@ class MachineLearningController { }); } _fireControlEvent(); + _logger.info('init done'); } void onUserInteraction() { From baa90c42ad188ef90598f3cd8f099173d751ed23 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Mon, 27 May 2024 11:59:36 +0530 Subject: [PATCH 046/354] [mob][photos] Remove stale comments --- mobile/lib/main.dart | 4 ---- 1 file changed, 4 deletions(-) diff --git a/mobile/lib/main.dart b/mobile/lib/main.dart index 3096f8fd3e..ac1e8441d0 100644 --- a/mobile/lib/main.dart +++ b/mobile/lib/main.dart @@ -235,10 +235,6 @@ Future _init(bool isBackground, {String via = ''}) async { unawaited(SemanticSearchService.instance.init()); MachineLearningController.instance.init(); - // Can not including existing tf/ml binaries as they are not being built - // from source. - // See https://gitlab.com/fdroid/fdroiddata/-/merge_requests/12671#note_1294346819 - // unawaited(ObjectDetectionService.instance.init()); if (flagService.faceSearchEnabled) { unawaited(FaceMlService.instance.init()); } else { From ee8976e92bb8e0b0b687ad7764cb62c3c9faa919 Mon Sep 17 00:00:00 2001 From: ashilkn Date: Mon, 27 May 2024 12:56:20 +0530 Subject: [PATCH 047/354] [mob][photos] Add schema migration easier on FaceMLDataDB --- mobile/lib/face/db.dart | 44 +++++++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 15 deletions(-) diff --git a/mobile/lib/face/db.dart b/mobile/lib/face/db.dart index 388b30b1e3..c1b96767e9 100644 --- a/mobile/lib/face/db.dart +++ b/mobile/lib/face/db.dart @@ -35,6 +35,15 @@ class FaceMLDataDB { static final FaceMLDataDB instance = FaceMLDataDB._privateConstructor(); + static final _migrationScripts = [ + createFacesTable, + createFaceClustersTable, + createClusterPersonTable, + createClusterSummaryTable, + createNotPersonFeedbackTable, + fcClusterIDIndex, + ]; + // only have a single app-wide reference to the database static Future? _sqliteAsyncDBFuture; @@ -50,25 +59,30 @@ class FaceMLDataDB { _logger.info("Opening sqlite_async access: DB path " + databaseDirectory); final asyncDBConnection = SqliteDatabase(path: databaseDirectory, maxReaders: 2); - await _onCreate(asyncDBConnection); + await _migrate(asyncDBConnection); + return asyncDBConnection; } - Future _onCreate(SqliteDatabase asyncDBConnection) async { - try { - final startTime = DateTime.now(); - await asyncDBConnection.execute(createFacesTable); - await asyncDBConnection.execute(createFaceClustersTable); - await asyncDBConnection.execute(createClusterPersonTable); - await asyncDBConnection.execute(createClusterSummaryTable); - await asyncDBConnection.execute(createNotPersonFeedbackTable); - await asyncDBConnection.execute(fcClusterIDIndex); - _logger.info( - 'FaceMLDataDB tables created in ${DateTime.now().difference(startTime).inMilliseconds}ms', + Future _migrate( + SqliteDatabase database, + ) async { + final result = await database.execute('PRAGMA user_version'); + final currentVersion = result[0]['user_version'] as int; + final toVersion = _migrationScripts.length; + + if (currentVersion < toVersion) { + _logger.info("Migrating database from $currentVersion to $toVersion"); + await database.writeTransaction((tx) async { + for (int i = currentVersion + 1; i <= toVersion; i++) { + await tx.execute(_migrationScripts[i - 1]); + } + await tx.execute('PRAGMA user_version = $toVersion'); + }); + } else if (currentVersion > toVersion) { + throw AssertionError( + "currentVersion($currentVersion) cannot be greater than toVersion($toVersion)", ); - } catch (e, s) { - _logger.severe("Error creating FaceMLDataDB tables", e, s); - rethrow; } } From d413c4f4c117078d8b5bfaa43ab8a1c980f1f72a Mon Sep 17 00:00:00 2001 From: ashilkn Date: Mon, 27 May 2024 12:57:25 +0530 Subject: [PATCH 048/354] [mob][photos] Add try catch + logs for debugging in FaceMLDataDB --- mobile/lib/face/db.dart | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/mobile/lib/face/db.dart b/mobile/lib/face/db.dart index c1b96767e9..b00d56650c 100644 --- a/mobile/lib/face/db.dart +++ b/mobile/lib/face/db.dart @@ -59,7 +59,13 @@ class FaceMLDataDB { _logger.info("Opening sqlite_async access: DB path " + databaseDirectory); final asyncDBConnection = SqliteDatabase(path: databaseDirectory, maxReaders: 2); + final stopwatch = Stopwatch()..start(); + _logger.info("FaceMLDataDB: Starting migration"); await _migrate(asyncDBConnection); + _logger.info( + "FaceMLDataDB Migration took ${stopwatch.elapsedMilliseconds} ms", + ); + stopwatch.stop(); return asyncDBConnection; } @@ -75,7 +81,12 @@ class FaceMLDataDB { _logger.info("Migrating database from $currentVersion to $toVersion"); await database.writeTransaction((tx) async { for (int i = currentVersion + 1; i <= toVersion; i++) { - await tx.execute(_migrationScripts[i - 1]); + try { + await tx.execute(_migrationScripts[i - 1]); + } catch (e) { + _logger.severe("Error running migration script index ${i - 1}", e); + rethrow; + } } await tx.execute('PRAGMA user_version = $toVersion'); }); From a2a209a8499f6653d4a5c00fa6656e5a381a23d7 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Mon, 27 May 2024 12:59:32 +0530 Subject: [PATCH 049/354] [web] Fix display of codes on Safari --- web/apps/auth/src/services/code.ts | 50 +++++++++++++++++++++--------- 1 file changed, 36 insertions(+), 14 deletions(-) diff --git a/web/apps/auth/src/services/code.ts b/web/apps/auth/src/services/code.ts index 1df085b4e3..c604bae0ce 100644 --- a/web/apps/auth/src/services/code.ts +++ b/web/apps/auth/src/services/code.ts @@ -84,20 +84,6 @@ export const codeFromURIString = (id: string, uriString: string): Code => { const _codeFromURIString = (id: string, uriString: string): Code => { const url = new URL(uriString); - // A URL like - // - // new URL("otpauth://hotp/Test?secret=AAABBBCCCDDDEEEFFF&issuer=Test&counter=0") - // - // is parsed differently by the browser and Node depending on the scheme. - // When the scheme is http(s), then both of them consider "hotp" as the - // `host`. However, when the scheme is "otpauth", as is our case, the - // browser considers the entire thing as part of the pathname. so we get. - // - // host: "" - // pathname: "//hotp/Test" - // - // Since this code run on browsers only, we parse as per that behaviour. - const [type, path] = parsePathname(url); return { @@ -115,10 +101,46 @@ const _codeFromURIString = (id: string, uriString: string): Code => { }; const parsePathname = (url: URL): [type: Code["type"], path: string] => { + // A URL like + // + // new + // URL("otpauth://hotp/Test?secret=AAABBBCCCDDDEEEFFF&issuer=Test&counter=0") + // + // is parsed differently by different browsers, and there are differences + // even depending on the scheme. + // + // When the scheme is http(s), then all of them consider "hotp" as the + // `host`. However, when the scheme is "otpauth", as is our case, Safari + // splits it into + // + // host: "hotp" + // pathname: "/Test" + // + // while Chrome and Firefox consider the entire thing as part of the + // pathname + // + // host: "" + // pathname: "//hotp/Test" + // + // So we try to handle both scenarios by first checking for the host match, + // and if not fall back to deducing the "host" from the pathname. + + switch (url.host.toLowerCase()) { + case "totp": + return ["totp", url.pathname.toLowerCase()]; + case "hotp": + return ["hotp", url.pathname.toLowerCase()]; + case "steam": + return ["steam", url.pathname.toLowerCase()]; + default: + break; + } + const p = url.pathname.toLowerCase(); if (p.startsWith("//totp")) return ["totp", url.pathname.slice(6)]; if (p.startsWith("//hotp")) return ["hotp", url.pathname.slice(6)]; if (p.startsWith("//steam")) return ["steam", url.pathname.slice(7)]; + throw new Error(`Unsupported code or unparseable path "${url.pathname}"`); }; From 9f361237b1c25e4a1bfbfc6c20050ce927357743 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Mon, 27 May 2024 13:04:20 +0530 Subject: [PATCH 050/354] [mob][photos] Fix cluster appbar not showing --- mobile/lib/ui/viewer/people/cluster_app_bar.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobile/lib/ui/viewer/people/cluster_app_bar.dart b/mobile/lib/ui/viewer/people/cluster_app_bar.dart index 0896d06896..83ebe54284 100644 --- a/mobile/lib/ui/viewer/people/cluster_app_bar.dart +++ b/mobile/lib/ui/viewer/people/cluster_app_bar.dart @@ -97,7 +97,7 @@ class _AppBarWidgetState extends State { maxLines: 2, overflow: TextOverflow.ellipsis, ), - actions: kDebugMode ? _getDefaultActions(context) : null, + actions: _getDefaultActions(context), ); } From 04be2b6a2cbc3f3606b136cc35ba8872defca047 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Mon, 27 May 2024 14:00:24 +0530 Subject: [PATCH 051/354] Update electron updater Trying to rule out https://github.com/electron-userland/electron-builder/issues/7127 --- desktop/package.json | 2 +- desktop/yarn.lock | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/desktop/package.json b/desktop/package.json index 236dd55927..085d966817 100644 --- a/desktop/package.json +++ b/desktop/package.json @@ -30,7 +30,7 @@ "compare-versions": "^6.1", "electron-log": "^5.1", "electron-store": "^8.2", - "electron-updater": "^6.1", + "electron-updater": "^6.2", "ffmpeg-static": "^5.2", "html-entities": "^2.5", "jpeg-js": "^0.4", diff --git a/desktop/yarn.lock b/desktop/yarn.lock index 2aa060efc0..0ad96e95d8 100644 --- a/desktop/yarn.lock +++ b/desktop/yarn.lock @@ -743,10 +743,10 @@ buffer@^5.1.0, buffer@^5.5.0: base64-js "^1.3.1" ieee754 "^1.1.13" -builder-util-runtime@9.2.3: - version "9.2.3" - resolved "https://registry.yarnpkg.com/builder-util-runtime/-/builder-util-runtime-9.2.3.tgz#0a82c7aca8eadef46d67b353c638f052c206b83c" - integrity sha512-FGhkqXdFFZ5dNC4C+yuQB9ak311rpGAw+/ASz8ZdxwODCv1GGMWgLDeofRkdi0F3VCHQEWy/aXcJQozx2nOPiw== +builder-util-runtime@9.2.4: + version "9.2.4" + resolved "https://registry.yarnpkg.com/builder-util-runtime/-/builder-util-runtime-9.2.4.tgz#13cd1763da621e53458739a1e63f7fcba673c42a" + integrity sha512-upp+biKpN/XZMLim7aguUyW8s0FUpDvOtK6sbanMFDAMBzpHDqdhgVYm6zc9HJ6nWo7u2Lxk60i2M6Jd3aiNrA== dependencies: debug "^4.3.4" sax "^1.2.4" @@ -1251,12 +1251,12 @@ electron-store@^8.2: conf "^10.2.0" type-fest "^2.17.0" -electron-updater@^6.1: - version "6.1.8" - resolved "https://registry.yarnpkg.com/electron-updater/-/electron-updater-6.1.8.tgz#17637bca165322f4e526b13c99165f43e6f697d8" - integrity sha512-hhOTfaFAd6wRHAfUaBhnAOYc+ymSGCWJLtFkw4xJqOvtpHmIdNHnXDV9m1MHC+A6q08Abx4Ykgyz/R5DGKNAMQ== +electron-updater@^6.2: + version "6.2.1" + resolved "https://registry.yarnpkg.com/electron-updater/-/electron-updater-6.2.1.tgz#1c9adb9ba2a21a5dc50a8c434c45360d5e9fe6c9" + integrity sha512-83eKIPW14qwZqUUM6wdsIRwVKZyjmHxQ4/8G+1C6iS5PdDt7b1umYQyj1/qPpH510GmHEQe4q0kCPe3qmb3a0Q== dependencies: - builder-util-runtime "9.2.3" + builder-util-runtime "9.2.4" fs-extra "^10.1.0" js-yaml "^4.1.0" lazy-val "^1.0.5" From 9f58f1eeb34f9863a57da6870270ee9cef693800 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Mon, 27 May 2024 14:41:32 +0530 Subject: [PATCH 052/354] Fix error on refresh while a folder watch is being set up Notes: From QA > This error mostly happens if i add a watch folder and before watch folders start to upload and i refresh the app. e is undefined in let {watches: e, removeWatch: n} = t; return 0 === e.length ? (0,... Results in Next throwing Application error: a client-side exception has occurred (see the browser console for more information). --- web/apps/photos/src/components/WatchFolder.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/web/apps/photos/src/components/WatchFolder.tsx b/web/apps/photos/src/components/WatchFolder.tsx index 812f54c6b7..e99fbb0073 100644 --- a/web/apps/photos/src/components/WatchFolder.tsx +++ b/web/apps/photos/src/components/WatchFolder.tsx @@ -153,12 +153,12 @@ const Title_ = styled("div")` `; interface WatchList { - watches: FolderWatch[]; + watches: FolderWatch[] | undefined; removeWatch: (watch: FolderWatch) => void; } const WatchList: React.FC = ({ watches, removeWatch }) => { - return watches.length === 0 ? ( + return (watches ?? []).length === 0 ? ( ) : ( From ced1f0bd7948dce1ac4fce0251f6eef42fc51932 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Mon, 27 May 2024 14:55:52 +0530 Subject: [PATCH 053/354] [mob][photos] Don't remove last cluster of person --- .../viewer/people/person_clusters_page.dart | 64 +++++++++++-------- 1 file changed, 36 insertions(+), 28 deletions(-) diff --git a/mobile/lib/ui/viewer/people/person_clusters_page.dart b/mobile/lib/ui/viewer/people/person_clusters_page.dart index 2c493fc21f..4f7454f310 100644 --- a/mobile/lib/ui/viewer/people/person_clusters_page.dart +++ b/mobile/lib/ui/viewer/people/person_clusters_page.dart @@ -38,12 +38,17 @@ class _PersonClustersPageState extends State { .getClusterFilesForPersonID(widget.person.remoteID), builder: (context, snapshot) { if (snapshot.hasData) { - final List keys = snapshot.data!.keys.toList(); + final clusters = snapshot.data!; + final List keys = clusters.keys.toList(); + // Sort the clusters by the number of files in each cluster, largest first + keys.sort( + (b, a) => clusters[a]!.length.compareTo(clusters[b]!.length), + ); return ListView.builder( itemCount: keys.length, itemBuilder: (context, index) { final int clusterID = keys[index]; - final List files = snapshot.data![keys[index]]!; + final List files = clusters[clusterID]!; return InkWell( onTap: () { Navigator.of(context).push( @@ -93,34 +98,37 @@ class _PersonClustersPageState extends State { mainAxisAlignment: MainAxisAlignment.spaceBetween, children: [ Text( - "${snapshot.data![keys[index]]!.length} photos", + "${files.length} photos", style: getEnteTextTheme(context).body, ), - GestureDetector( - onTap: () async { - try { - await PersonService.instance - .removeClusterToPerson( - personID: widget.person.remoteID, - clusterID: clusterID, - ); - _logger.info( - "Removed cluster $clusterID from person ${widget.person.remoteID}", - ); - Bus.instance.fire(PeopleChangedEvent()); - setState(() {}); - } catch (e) { - _logger.severe( - "removing cluster from person,", - e, - ); - } - }, - child: const Icon( - CupertinoIcons.minus_circled, - color: Colors.red, - ), - ), + (index != 0) + ? GestureDetector( + onTap: () async { + try { + await PersonService.instance + .removeClusterToPerson( + personID: widget.person.remoteID, + clusterID: clusterID, + ); + _logger.info( + "Removed cluster $clusterID from person ${widget.person.remoteID}", + ); + Bus.instance + .fire(PeopleChangedEvent()); + setState(() {}); + } catch (e) { + _logger.severe( + "removing cluster from person,", + e, + ); + } + }, + child: const Icon( + CupertinoIcons.minus_circled, + color: Colors.red, + ), + ) + : const SizedBox.shrink(), ], ), ), From 8f474a4500c9a48c12700484c9ab1bf43c82f9e7 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Mon, 27 May 2024 15:54:10 +0530 Subject: [PATCH 054/354] [mob][photos] Set MLController timer to 10 seconds --- .../machine_learning/machine_learning_controller.dart | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/mobile/lib/services/machine_learning/machine_learning_controller.dart b/mobile/lib/services/machine_learning/machine_learning_controller.dart index 40a52a419d..3b78fd8c9e 100644 --- a/mobile/lib/services/machine_learning/machine_learning_controller.dart +++ b/mobile/lib/services/machine_learning/machine_learning_controller.dart @@ -4,7 +4,6 @@ import "dart:io"; import "package:battery_info/battery_info_plugin.dart"; import "package:battery_info/model/android_battery_info.dart"; import "package:battery_info/model/iso_battery_info.dart"; -import "package:flutter/foundation.dart" show kDebugMode; import "package:logging/logging.dart"; import "package:photos/core/event_bus.dart"; import "package:photos/events/machine_learning_control_event.dart"; @@ -19,8 +18,7 @@ class MachineLearningController { static const kMaximumTemperature = 42; // 42 degree celsius static const kMinimumBatteryLevel = 20; // 20% - static const kDefaultInteractionTimeout = - kDebugMode ? Duration(seconds: 3) : Duration(seconds: 5); + static const kDefaultInteractionTimeout = Duration(seconds: 10); static const kUnhealthyStates = ["over_heat", "over_voltage", "dead"]; bool _isDeviceHealthy = true; From 9cf5691e426f939253eba61f73f86d6a7c23b5df Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Mon, 27 May 2024 17:09:33 +0530 Subject: [PATCH 055/354] [mob][photos] Delete instead of drop table --- mobile/lib/face/db.dart | 26 ++++++++++++-------------- mobile/lib/face/db_fields.dart | 16 +++++----------- 2 files changed, 17 insertions(+), 25 deletions(-) diff --git a/mobile/lib/face/db.dart b/mobile/lib/face/db.dart index b00d56650c..abe4e19227 100644 --- a/mobile/lib/face/db.dart +++ b/mobile/lib/face/db.dart @@ -229,10 +229,10 @@ class FaceMLDataDB { final db = await instance.asyncDB; await db.execute(deleteFacesTable); - await db.execute(dropClusterPersonTable); - await db.execute(dropClusterSummaryTable); - await db.execute(deletePersonTable); - await db.execute(dropNotPersonFeedbackTable); + await db.execute(deleteFaceClustersTable); + await db.execute(deleteClusterPersonTable); + await db.execute(deleteClusterSummaryTable); + await db.execute(deleteNotPersonFeedbackTable); } Future> getFaceEmbeddingsForCluster( @@ -768,7 +768,7 @@ class FaceMLDataDB { try { final db = await instance.asyncDB; - await db.execute(dropFaceClustersTable); + await db.execute(deleteFaceClustersTable); await db.execute(createFaceClustersTable); await db.execute(fcClusterIDIndex); } catch (e, s) { @@ -979,16 +979,15 @@ class FaceMLDataDB { if (faces) { await db.execute(deleteFacesTable); await db.execute(createFacesTable); - await db.execute(dropFaceClustersTable); + await db.execute(deleteFaceClustersTable); await db.execute(createFaceClustersTable); await db.execute(fcClusterIDIndex); } - await db.execute(deletePersonTable); - await db.execute(dropClusterPersonTable); - await db.execute(dropNotPersonFeedbackTable); - await db.execute(dropClusterSummaryTable); - await db.execute(dropFaceClustersTable); + await db.execute(deleteClusterPersonTable); + await db.execute(deleteNotPersonFeedbackTable); + await db.execute(deleteClusterSummaryTable); + await db.execute(deleteFaceClustersTable); await db.execute(createClusterPersonTable); await db.execute(createNotPersonFeedbackTable); @@ -1006,9 +1005,8 @@ class FaceMLDataDB { final db = await instance.asyncDB; // Drop the tables - await db.execute(deletePersonTable); - await db.execute(dropClusterPersonTable); - await db.execute(dropNotPersonFeedbackTable); + await db.execute(deleteClusterPersonTable); + await db.execute(deleteNotPersonFeedbackTable); // Recreate the tables await db.execute(createClusterPersonTable); diff --git a/mobile/lib/face/db_fields.dart b/mobile/lib/face/db_fields.dart index e6a70a7d4e..8ad14ae282 100644 --- a/mobile/lib/face/db_fields.dart +++ b/mobile/lib/face/db_fields.dart @@ -29,7 +29,7 @@ const createFacesTable = '''CREATE TABLE IF NOT EXISTS $facesTable ( ); '''; -const deleteFacesTable = 'DROP TABLE IF EXISTS $facesTable'; +const deleteFacesTable = 'DELETE FROM $facesTable'; // End of Faces Table Fields & Schema Queries //##region Face Clusters Table Fields & Schema Queries @@ -48,15 +48,9 @@ CREATE TABLE IF NOT EXISTS $faceClustersTable ( // -- Creating a non-unique index on clusterID for query optimization const fcClusterIDIndex = '''CREATE INDEX IF NOT EXISTS idx_fcClusterID ON $faceClustersTable($fcClusterID);'''; -const dropFaceClustersTable = 'DROP TABLE IF EXISTS $faceClustersTable'; +const deleteFaceClustersTable = 'DELETE FROM $faceClustersTable'; //##endregion -// People Table Fields & Schema Queries -const personTable = 'person'; - -const deletePersonTable = 'DROP TABLE IF EXISTS $personTable'; -//End People Table Fields & Schema Queries - // Clusters Table Fields & Schema Queries const clusterPersonTable = 'cluster_person'; const personIdColumn = 'person_id'; @@ -69,7 +63,7 @@ CREATE TABLE IF NOT EXISTS $clusterPersonTable ( PRIMARY KEY($personIdColumn, $clusterIDColumn) ); '''; -const dropClusterPersonTable = 'DROP TABLE IF EXISTS $clusterPersonTable'; +const deleteClusterPersonTable = 'DELETE FROM $clusterPersonTable'; // End Clusters Table Fields & Schema Queries /// Cluster Summary Table Fields & Schema Queries @@ -85,7 +79,7 @@ CREATE TABLE IF NOT EXISTS $clusterSummaryTable ( ); '''; -const dropClusterSummaryTable = 'DROP TABLE IF EXISTS $clusterSummaryTable'; +const deleteClusterSummaryTable = 'DELETE FROM $clusterSummaryTable'; /// End Cluster Summary Table Fields & Schema Queries @@ -99,5 +93,5 @@ CREATE TABLE IF NOT EXISTS $notPersonFeedback ( PRIMARY KEY($personIdColumn, $clusterIDColumn) ); '''; -const dropNotPersonFeedbackTable = 'DROP TABLE IF EXISTS $notPersonFeedback'; +const deleteNotPersonFeedbackTable = 'DELETE FROM $notPersonFeedback'; // End Clusters Table Fields & Schema Queries From 380d37267b99e994dbb2be636959cbc7e5467159 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Mon, 27 May 2024 17:19:06 +0530 Subject: [PATCH 056/354] [mob][photos] Don't pop too often --- mobile/lib/ui/settings/machine_learning_settings_page.dart | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mobile/lib/ui/settings/machine_learning_settings_page.dart b/mobile/lib/ui/settings/machine_learning_settings_page.dart index 0ea1588a0e..257d5dd0fa 100644 --- a/mobile/lib/ui/settings/machine_learning_settings_page.dart +++ b/mobile/lib/ui/settings/machine_learning_settings_page.dart @@ -89,8 +89,8 @@ class _MachineLearningSettingsPageState iconButtonType: IconButtonType.secondary, onTap: () { Navigator.pop(context); - Navigator.pop(context); - Navigator.pop(context); + if (Navigator.canPop(context)) Navigator.pop(context); + if (Navigator.canPop(context)) Navigator.pop(context); }, ), ], From 89a61b3bf7ea6fd5df0fd6e9f9bec9aa9dc0723c Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Mon, 27 May 2024 17:21:29 +0530 Subject: [PATCH 057/354] [mob][photos] Bump --- mobile/pubspec.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobile/pubspec.yaml b/mobile/pubspec.yaml index 1417d17f3e..d3f49380f9 100644 --- a/mobile/pubspec.yaml +++ b/mobile/pubspec.yaml @@ -12,7 +12,7 @@ description: ente photos application # Read more about iOS versioning at # https://developer.apple.com/library/archive/documentation/General/Reference/InfoPlistKeyReference/Articles/CoreFoundationKeys.html -version: 0.8.110+634 +version: 0.8.112+636 publish_to: none environment: From f25f119ca1b6f6deab2a797f8820c775c4bd8714 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Mon, 27 May 2024 17:26:14 +0530 Subject: [PATCH 058/354] [mob][photos] Copy --- mobile/lib/generated/intl/messages_en.dart | 2 +- mobile/lib/generated/l10n.dart | 4 ++-- mobile/lib/l10n/intl_en.arb | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mobile/lib/generated/intl/messages_en.dart b/mobile/lib/generated/intl/messages_en.dart index baf50af582..b715eb4851 100644 --- a/mobile/lib/generated/intl/messages_en.dart +++ b/mobile/lib/generated/intl/messages_en.dart @@ -814,7 +814,7 @@ class MessageLookup extends MessageLookupByLibrary { MessageLookupByLibrary.simpleMessage("Incorrect recovery key"), "indexedItems": MessageLookupByLibrary.simpleMessage("Indexed items"), "indexingIsPaused": MessageLookupByLibrary.simpleMessage( - "Indexing is paused. It will automatically resume when device is ready"), + "Indexing is paused. It will automatically resume when device is ready."), "insecureDevice": MessageLookupByLibrary.simpleMessage("Insecure device"), "installManually": diff --git a/mobile/lib/generated/l10n.dart b/mobile/lib/generated/l10n.dart index a6cd47cbdc..23b67ff0bf 100644 --- a/mobile/lib/generated/l10n.dart +++ b/mobile/lib/generated/l10n.dart @@ -8794,10 +8794,10 @@ class S { ); } - /// `Indexing is paused. It will automatically resume when device is ready` + /// `Indexing is paused. It will automatically resume when device is ready.` String get indexingIsPaused { return Intl.message( - 'Indexing is paused. It will automatically resume when device is ready', + 'Indexing is paused. It will automatically resume when device is ready.', name: 'indexingIsPaused', desc: '', args: [], diff --git a/mobile/lib/l10n/intl_en.arb b/mobile/lib/l10n/intl_en.arb index a63d8674e5..df2894e4ce 100644 --- a/mobile/lib/l10n/intl_en.arb +++ b/mobile/lib/l10n/intl_en.arb @@ -1236,5 +1236,5 @@ "faceRecognitionIndexingDescription": "Please note that this will result in a higher bandwidth and battery usage until all items are indexed.", "foundFaces": "Found faces", "clusteringProgress": "Clustering progress", - "indexingIsPaused": "Indexing is paused. It will automatically resume when device is ready" + "indexingIsPaused": "Indexing is paused. It will automatically resume when device is ready." } From d854d5820eb0f2282220ead177559405c98b4a25 Mon Sep 17 00:00:00 2001 From: Crowdin Bot Date: Mon, 27 May 2024 12:17:06 +0000 Subject: [PATCH 059/354] New Crowdin translations by GitHub Action --- .../metadata/playstore/fr/full_description.txt | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/mobile/fastlane/metadata/playstore/fr/full_description.txt b/mobile/fastlane/metadata/playstore/fr/full_description.txt index 9a7f5975eb..07ff21f85f 100644 --- a/mobile/fastlane/metadata/playstore/fr/full_description.txt +++ b/mobile/fastlane/metadata/playstore/fr/full_description.txt @@ -1,30 +1,30 @@ -Entre est une application simple qui sauvegarde et organisé vos photos et vidéos. +Entre est une application simple qui sauvegarde et organise vos photos et vidéos. -Si vous recherchez une alternative respectueuse de la vie privée pour préserver vos souvenirs, vous êtes au bon endroit. Avec Ente, ils sont stockés chiffrés de bout-en-bout (e2ee). Cela signifie que vous-seul pouvez les voir. +Si vous recherchez une alternative respectueuse de votre vie privée pour préserver vos souvenirs, vous êtes au bon endroit. Avec Ente, ils sont stockés chiffrés de bout-en-bout (e2ee). Cela signifie que vous-seul pouvez les voir. -Nous avons des applications sur Android, iOS, Web et Ordinateur, et vos photos seront synchronisées de manière transparente entre tous vos appareils chiffrée de bout en bout (e2ee). +Nous avons des applications pour Android, iOS, Web et Ordinateur, et vos photos seront synchronisées de manière transparente entre tous vos appareils avec une méthode de chiffrement de bout en bout (e2ee). Ente vous permet également de partager vos albums avec vos proches. Vous pouvez soit les partager directement avec d'autres utilisateurs Ente, chiffrés de bout en bout ou avec des liens visibles publiquement. -Vos données chiffrées sont stockées à travers de multiples endroits, dont un abri antiatomique à Paris. Nous prenons la postérité au sérieux et facilitons la conservation de vos souvenirs. +Vos données chiffrées sont stockées dans de multiples endroits, dont un abri antiatomique à Paris. Nous prenons la postérité au sérieux et facilitons la conservation de vos souvenirs. Nous sommes là pour faire l'application photo la plus sûre de tous les temps, rejoignez-nous ! ✨ CARACTÉRISTIQUES -- Sauvegardes de qualité originales, car chaque pixel est important +- Sauvegardes en qualité originale, car chaque pixel est important - Abonnement familiaux, pour que vous puissiez partager l'espace de stockage avec votre famille - Dossiers partagés, si vous voulez que votre partenaire profite de vos clichés -- Liens ves les albums qui peuvent être protégés par un mot de passe et être configurés pour expirer +- Liens vers les albums, qui peuvent être protégés par un mot de passe et être configurés pour expirer - Possibilité de libérer de l'espace en supprimant les fichiers qui ont été sauvegardés en toute sécurité - Éditeur d'images, pour ajouter des touches de finition -- Favoriser, cacher et revivre vos souvenirs, car ils sont précieux +- Favoris, cacher et revivre vos souvenirs, car ils sont précieux - Importation en un clic depuis Google, Apple, votre disque dur et plus encore - Thème sombre, parce que vos photos y sont jolies - 2FA, 3FA, authentification biométrique - et beaucoup de choses encore ! 💲 PRIX -Nous ne proposons pas d'abonnement gratuits pour toujours, car il est important pour nous de rester durables et de résister à l'épreuve du temps. Au lieu de cela, nous vous proposons des abonnements abordables que vous pouvez partager librement avec votre famille. Vous pouvez trouver plus d'informations sur ente.io. +Nous ne proposons pas d'abonnements gratuits à vie, car il est important pour nous de rester pérenne et de résister à l'épreuve du temps. Au lieu de cela, nous vous proposons des abonnements abordables que vous pouvez partager librement avec votre famille. Vous pouvez trouver plus d'informations sur ente.io. 🙋 ASSISTANCE Nous sommes fiers d'offrir un support humain. Si vous êtes un abonné, vous pouvez contacter team@ente.io et vous recevrez une réponse de notre équipe dans les 24 heures. \ No newline at end of file From c291fa70d32b4ab8e0405af967f12fc99c16ecd3 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Mon, 27 May 2024 18:12:21 +0530 Subject: [PATCH 060/354] Wrap add person name banner inside safeArea --- mobile/lib/ui/viewer/people/cluster_page.dart | 74 ++++++++++--------- 1 file changed, 38 insertions(+), 36 deletions(-) diff --git a/mobile/lib/ui/viewer/people/cluster_page.dart b/mobile/lib/ui/viewer/people/cluster_page.dart index f6b720f023..ef069887f9 100644 --- a/mobile/lib/ui/viewer/people/cluster_page.dart +++ b/mobile/lib/ui/viewer/people/cluster_page.dart @@ -161,43 +161,45 @@ class _ClusterPageState extends State { ), ), showNamingBanner - ? Dismissible( - key: const Key("namingBanner"), - direction: DismissDirection.horizontal, - onDismissed: (direction) { - setState(() { - userDismissedNamingBanner = true; - }); - }, - child: PeopleBanner( - type: PeopleBannerType.addName, - faceWidget: PersonFaceWidget( - files.first, - clusterID: widget.clusterID, - ), - actionIcon: Icons.add_outlined, - text: S.of(context).addAName, - subText: S.of(context).findPeopleByName, - onTap: () async { - if (widget.personID == null) { - final result = await showAssignPersonAction( - context, - clusterID: widget.clusterID, - ); - if (result != null && - result is (PersonEntity, EnteFile)) { - Navigator.pop(context); - // ignore: unawaited_futures - routeToPage(context, PeoplePage(person: result.$1)); - } else if (result != null && result is PersonEntity) { - Navigator.pop(context); - // ignore: unawaited_futures - routeToPage(context, PeoplePage(person: result)); - } - } else { - showShortToast(context, "No personID or clusterID"); - } + ? SafeArea( + child: Dismissible( + key: const Key("namingBanner"), + direction: DismissDirection.horizontal, + onDismissed: (direction) { + setState(() { + userDismissedNamingBanner = true; + }); }, + child: PeopleBanner( + type: PeopleBannerType.addName, + faceWidget: PersonFaceWidget( + files.first, + clusterID: widget.clusterID, + ), + actionIcon: Icons.add_outlined, + text: S.of(context).addAName, + subText: S.of(context).findPeopleByName, + onTap: () async { + if (widget.personID == null) { + final result = await showAssignPersonAction( + context, + clusterID: widget.clusterID, + ); + if (result != null && + result is (PersonEntity, EnteFile)) { + Navigator.pop(context); + // ignore: unawaited_futures + routeToPage(context, PeoplePage(person: result.$1)); + } else if (result != null && result is PersonEntity) { + Navigator.pop(context); + // ignore: unawaited_futures + routeToPage(context, PeoplePage(person: result)); + } + } else { + showShortToast(context, "No personID or clusterID"); + } + }, + ), ), ) : const SizedBox.shrink(), From 90e467c7c02fd5d9ecd2121f3494752d01915c76 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Mon, 27 May 2024 18:31:17 +0530 Subject: [PATCH 061/354] [mob][photos] Fetch remote feedback before clustering --- .../services/machine_learning/face_ml/face_ml_service.dart | 3 +++ .../machine_learning/face_ml/person/person_service.dart | 4 ++-- mobile/lib/ui/settings/debug/face_debug_section_widget.dart | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart index 9f153ffa89..222fd50b8f 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart @@ -580,6 +580,9 @@ class FaceMlService { _isIndexingOrClusteringRunning = true; final clusterAllImagesTime = DateTime.now(); + _logger.info('Pulling remote feedback before actually clustering'); + await PersonService.instance.fetchRemoteClusterFeedback(); + try { // Get a sense of the total number of faces in the database final int totalFaces = await FaceMLDataDB.instance diff --git a/mobile/lib/services/machine_learning/face_ml/person/person_service.dart b/mobile/lib/services/machine_learning/face_ml/person/person_service.dart index 7517d057d5..682deaff0c 100644 --- a/mobile/lib/services/machine_learning/face_ml/person/person_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/person/person_service.dart @@ -73,7 +73,7 @@ class PersonService { Future reconcileClusters() async { final EnteWatch? w = kDebugMode ? EnteWatch("reconcileClusters") : null; w?.start(); - await storeRemoteFeedback(); + await fetchRemoteClusterFeedback(); w?.log("Stored remote feedback"); final dbPersonClusterInfo = await faceMLDataDB.getPersonToClusterIdToFaceIds(); @@ -225,7 +225,7 @@ class PersonService { Bus.instance.fire(PeopleChangedEvent()); } - Future storeRemoteFeedback() async { + Future fetchRemoteClusterFeedback() async { await entityService.syncEntities(); final entities = await entityService.getEntities(EntityType.person); entities.sort((a, b) => a.updatedAt.compareTo(b.updatedAt)); diff --git a/mobile/lib/ui/settings/debug/face_debug_section_widget.dart b/mobile/lib/ui/settings/debug/face_debug_section_widget.dart index 376793769f..844f71c01b 100644 --- a/mobile/lib/ui/settings/debug/face_debug_section_widget.dart +++ b/mobile/lib/ui/settings/debug/face_debug_section_widget.dart @@ -193,7 +193,7 @@ class _FaceDebugSectionWidgetState extends State { trailingIconIsMuted: true, onTap: () async { try { - await PersonService.instance.storeRemoteFeedback(); + await PersonService.instance.fetchRemoteClusterFeedback(); FaceMlService.instance.debugIndexingDisabled = false; await FaceMlService.instance .clusterAllImages(clusterInBuckets: true); From 37d3776e285e21378712fe59bb11f078220eef84 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Mon, 27 May 2024 18:34:26 +0530 Subject: [PATCH 062/354] [mob][photos] Bump --- mobile/pubspec.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobile/pubspec.yaml b/mobile/pubspec.yaml index d3f49380f9..1311b4e793 100644 --- a/mobile/pubspec.yaml +++ b/mobile/pubspec.yaml @@ -12,7 +12,7 @@ description: ente photos application # Read more about iOS versioning at # https://developer.apple.com/library/archive/documentation/General/Reference/InfoPlistKeyReference/Articles/CoreFoundationKeys.html -version: 0.8.112+636 +version: 0.8.113+637 publish_to: none environment: From e90eb50a5005e07b100f0541bff70514ff8beb88 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Mon, 27 May 2024 11:15:58 +0530 Subject: [PATCH 063/354] [desktop] Code sign on Windows --- desktop/.github/workflows/desktop-release.yml | 38 ++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/desktop/.github/workflows/desktop-release.yml b/desktop/.github/workflows/desktop-release.yml index 70eedf3ea6..5c54c2c33f 100644 --- a/desktop/.github/workflows/desktop-release.yml +++ b/desktop/.github/workflows/desktop-release.yml @@ -54,8 +54,16 @@ jobs: # https://github.com/electron-userland/electron-builder/issues/4181 run: sudo apt-get install libarchive-tools - - name: Build + - name: Export Windows code signing certificate + if: startsWith(matrix.os, 'windows') + run: echo "$CSC_LINK_PFX_B64" | base64 -d > windows-csc.pfx + shell: bash + env: + CSC_LINK_PFX_B64: ${{ secrets.WINDOWS_CSC_LINK_PFX_B64 }} + + - name: Build Linux uses: ente-io/action-electron-builder@v1.0.0 + if: startsWith(matrix.os, 'ubuntu') with: package_root: desktop build_script_name: build:ci @@ -68,6 +76,19 @@ jobs: # create a (draft) release after building. Otherwise upload # assets to the existing draft named after the version. release: ${{ startsWith(github.ref, 'refs/tags/v') }} + env: + # Workaround recommended in + # https://github.com/electron-userland/electron-builder/issues/3179 + USE_HARD_LINKS: false + + - name: Build macOS + uses: ente-io/action-electron-builder@v1.0.0 + if: startsWith(matrix.os, 'macos') + with: + package_root: desktop + build_script_name: build:ci + github_token: ${{ secrets.GITHUB_TOKEN }} + release: ${{ startsWith(github.ref, 'refs/tags/v') }} mac_certs: ${{ secrets.MAC_CERTS }} mac_certs_password: ${{ secrets.MAC_CERTS_PASSWORD }} @@ -78,3 +99,18 @@ jobs: ${{ secrets.APPLE_APP_SPECIFIC_PASSWORD }} APPLE_TEAM_ID: ${{ secrets.APPLE_TEAM_ID }} USE_HARD_LINKS: false + + - name: Build Windows + uses: ente-io/action-electron-builder@v1.0.0 + if: startsWith(matrix.os, 'windows') + with: + package_root: desktop + build_script_name: build:ci + github_token: ${{ secrets.GITHUB_TOKEN }} + release: ${{ startsWith(github.ref, 'refs/tags/v') }} + env: + # Windows signing credentials + # https://www.electron.build/code-signing + CSC_LINK: "windows-csc.pfx" + CSC_KEY_PASSWORD: ${{ secrets.WINDOWS_CSC_KEY_PASSWORD }} + USE_HARD_LINKS: false From 3d122b9f9de40d165be3269ec3113e3f108e97ab Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Mon, 27 May 2024 15:49:55 +0530 Subject: [PATCH 064/354] Add publisher > no certificates with ExtKeyUsageCodeSigning Cannot extract publisher name from code signing certificate. As workaround, set win.publisherName. --- desktop/electron-builder.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/desktop/electron-builder.yml b/desktop/electron-builder.yml index c2c000ce9f..0412cdecff 100644 --- a/desktop/electron-builder.yml +++ b/desktop/electron-builder.yml @@ -7,6 +7,8 @@ extraFiles: - from: build to: resources win: + # prettier-ignore + publisherName: "C=US, ST=Delaware, L=Dover, O=Ente Technologies, Inc., CN=Ente, emailAddress=code@ente.io" target: - target: nsis arch: [x64, arm64] From 03bc8f0493da3bf2388145abbf9e44424be4a4f1 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Mon, 27 May 2024 15:54:27 +0530 Subject: [PATCH 065/354] Let prettier have a go at it > Example 7.5 Double Quoted Line Breaks > All leading and trailing white space characters on each line are excluded from the content. > > https://yaml.org/spec/1.2.2/ --- desktop/electron-builder.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/desktop/electron-builder.yml b/desktop/electron-builder.yml index 0412cdecff..aab8d0effb 100644 --- a/desktop/electron-builder.yml +++ b/desktop/electron-builder.yml @@ -7,8 +7,9 @@ extraFiles: - from: build to: resources win: - # prettier-ignore - publisherName: "C=US, ST=Delaware, L=Dover, O=Ente Technologies, Inc., CN=Ente, emailAddress=code@ente.io" + publisherName: + "C=US, ST=Delaware, L=Dover, O=Ente Technologies, Inc., CN=Ente, + emailAddress=code@ente.io" target: - target: nsis arch: [x64, arm64] From bed57eb03e03e403d06d182b110a7bba10eba25b Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Mon, 27 May 2024 20:15:50 +0530 Subject: [PATCH 066/354] Fix the actual issue (the signing thing was a red herring) --- desktop/src/main.ts | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/desktop/src/main.ts b/desktop/src/main.ts index 463774dc2b..a5abc26271 100644 --- a/desktop/src/main.ts +++ b/desktop/src/main.ts @@ -322,6 +322,13 @@ const setupTrayItem = (mainWindow: BrowserWindow) => { * once most people have upgraded to newer versions. */ const deleteLegacyDiskCacheDirIfExists = async () => { + const removeIfExists = async (dirPath: string) => { + if (existsSync(dirPath)) { + log.info(`Removing legacy disk cache from ${dirPath}`); + await fs.rm(dirPath, { recursive: true }); + } + }; + // [Note: Getting the cache path] // // The existing code was passing "cache" as a parameter to getPath. @@ -338,9 +345,18 @@ const deleteLegacyDiskCacheDirIfExists = async () => { // // @ts-expect-error "cache" works but is not part of the public API. const cacheDir = path.join(app.getPath("cache"), "ente"); - if (existsSync(cacheDir)) { - log.info(`Removing legacy disk cache from ${cacheDir}`); - await fs.rm(cacheDir, { recursive: true }); + if (process.platform == "win32") { + // On Windows the cache dir is the same as the app data (!). So deleting + // the ente subfolder of the cache dir is equivalent to deleting the + // user data dir. + // + // Obviously, that's not good. So instead of Windows we explicitly + // delete the named cache directories. + await removeIfExists(path.join(cacheDir, "thumbs")); + await removeIfExists(path.join(cacheDir, "files")); + await removeIfExists(path.join(cacheDir, "face-crops")); + } else { + await removeIfExists(cacheDir); } }; From 69e2a36933f8e2c9e4a482f368f60c23d4341c00 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Mon, 27 May 2024 20:16:57 +0530 Subject: [PATCH 067/354] Revert "[desktop] Code sign on Windows" This reverts commit 7e6b75004026f24cc340bc5da806fbe8fc20e6c8 and its never siblings. Retaining them in git history though. --- desktop/.github/workflows/desktop-release.yml | 38 +------------------ desktop/electron-builder.yml | 3 -- 2 files changed, 1 insertion(+), 40 deletions(-) diff --git a/desktop/.github/workflows/desktop-release.yml b/desktop/.github/workflows/desktop-release.yml index 5c54c2c33f..70eedf3ea6 100644 --- a/desktop/.github/workflows/desktop-release.yml +++ b/desktop/.github/workflows/desktop-release.yml @@ -54,16 +54,8 @@ jobs: # https://github.com/electron-userland/electron-builder/issues/4181 run: sudo apt-get install libarchive-tools - - name: Export Windows code signing certificate - if: startsWith(matrix.os, 'windows') - run: echo "$CSC_LINK_PFX_B64" | base64 -d > windows-csc.pfx - shell: bash - env: - CSC_LINK_PFX_B64: ${{ secrets.WINDOWS_CSC_LINK_PFX_B64 }} - - - name: Build Linux + - name: Build uses: ente-io/action-electron-builder@v1.0.0 - if: startsWith(matrix.os, 'ubuntu') with: package_root: desktop build_script_name: build:ci @@ -76,19 +68,6 @@ jobs: # create a (draft) release after building. Otherwise upload # assets to the existing draft named after the version. release: ${{ startsWith(github.ref, 'refs/tags/v') }} - env: - # Workaround recommended in - # https://github.com/electron-userland/electron-builder/issues/3179 - USE_HARD_LINKS: false - - - name: Build macOS - uses: ente-io/action-electron-builder@v1.0.0 - if: startsWith(matrix.os, 'macos') - with: - package_root: desktop - build_script_name: build:ci - github_token: ${{ secrets.GITHUB_TOKEN }} - release: ${{ startsWith(github.ref, 'refs/tags/v') }} mac_certs: ${{ secrets.MAC_CERTS }} mac_certs_password: ${{ secrets.MAC_CERTS_PASSWORD }} @@ -99,18 +78,3 @@ jobs: ${{ secrets.APPLE_APP_SPECIFIC_PASSWORD }} APPLE_TEAM_ID: ${{ secrets.APPLE_TEAM_ID }} USE_HARD_LINKS: false - - - name: Build Windows - uses: ente-io/action-electron-builder@v1.0.0 - if: startsWith(matrix.os, 'windows') - with: - package_root: desktop - build_script_name: build:ci - github_token: ${{ secrets.GITHUB_TOKEN }} - release: ${{ startsWith(github.ref, 'refs/tags/v') }} - env: - # Windows signing credentials - # https://www.electron.build/code-signing - CSC_LINK: "windows-csc.pfx" - CSC_KEY_PASSWORD: ${{ secrets.WINDOWS_CSC_KEY_PASSWORD }} - USE_HARD_LINKS: false diff --git a/desktop/electron-builder.yml b/desktop/electron-builder.yml index aab8d0effb..c2c000ce9f 100644 --- a/desktop/electron-builder.yml +++ b/desktop/electron-builder.yml @@ -7,9 +7,6 @@ extraFiles: - from: build to: resources win: - publisherName: - "C=US, ST=Delaware, L=Dover, O=Ente Technologies, Inc., CN=Ente, - emailAddress=code@ente.io" target: - target: nsis arch: [x64, arm64] From 0f502eb9c28d1bdde6ffe252db6ab00da15ba6c8 Mon Sep 17 00:00:00 2001 From: Griffin Wiebel Date: Mon, 27 May 2024 11:59:57 -0700 Subject: [PATCH 068/354] Create a custom icon for YNAB --- .../custom-icons/_data/custom-icons.json | 8 ++++++++ auth/assets/custom-icons/icons/ynab.svg | 20 +++++++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 auth/assets/custom-icons/icons/ynab.svg diff --git a/auth/assets/custom-icons/_data/custom-icons.json b/auth/assets/custom-icons/_data/custom-icons.json index 3e8f8ce679..103d59416b 100644 --- a/auth/assets/custom-icons/_data/custom-icons.json +++ b/auth/assets/custom-icons/_data/custom-icons.json @@ -412,6 +412,14 @@ "Яндекс" ], "slug": "Yandex" + }, + { + "title": "YNAB", + "altNames": [ + "You Need A Budget" + ], + "slug": "ynab", + "hex": "3B5EDA" } ] } \ No newline at end of file diff --git a/auth/assets/custom-icons/icons/ynab.svg b/auth/assets/custom-icons/icons/ynab.svg new file mode 100644 index 0000000000..4ddfc9fa07 --- /dev/null +++ b/auth/assets/custom-icons/icons/ynab.svg @@ -0,0 +1,20 @@ + + + + + + + + + + + + + + + + + + + + From eaaa26c2e32bf6f4a2114db8aed2103ee3b541c6 Mon Sep 17 00:00:00 2001 From: Xeiv Date: Tue, 28 May 2024 01:14:34 +0530 Subject: [PATCH 069/354] [auth] Create a custom icon for RuneMate --- auth/assets/custom-icons/_data/custom-icons.json | 4 ++++ auth/assets/custom-icons/icons/runemate.svg | 8 ++++++++ 2 files changed, 12 insertions(+) create mode 100644 auth/assets/custom-icons/icons/runemate.svg diff --git a/auth/assets/custom-icons/_data/custom-icons.json b/auth/assets/custom-icons/_data/custom-icons.json index 3e8f8ce679..88acd244bf 100644 --- a/auth/assets/custom-icons/_data/custom-icons.json +++ b/auth/assets/custom-icons/_data/custom-icons.json @@ -302,6 +302,10 @@ "title": "Rockstar Games", "slug": "rockstar_games" }, + { + "title": "RuneMate", + "hex": "2ECC71" + }, { "title": "Rust Language Forum", "slug": "rust_language_forum", diff --git a/auth/assets/custom-icons/icons/runemate.svg b/auth/assets/custom-icons/icons/runemate.svg new file mode 100644 index 0000000000..1855afb8d2 --- /dev/null +++ b/auth/assets/custom-icons/icons/runemate.svg @@ -0,0 +1,8 @@ + + + + + + + + From 321ae0b7fc5d985ca8fceb84604960913e84f2c0 Mon Sep 17 00:00:00 2001 From: Crowdin Bot Date: Tue, 28 May 2024 01:42:06 +0000 Subject: [PATCH 070/354] New Crowdin translations by GitHub Action --- .../next/locales/pt-BR/translation.json | 2 +- .../next/locales/sv-SE/translation.json | 160 +++++++++--------- 2 files changed, 81 insertions(+), 81 deletions(-) diff --git a/web/packages/next/locales/pt-BR/translation.json b/web/packages/next/locales/pt-BR/translation.json index ac353e9f9f..006aad0183 100644 --- a/web/packages/next/locales/pt-BR/translation.json +++ b/web/packages/next/locales/pt-BR/translation.json @@ -315,7 +315,7 @@ "TRASH": "Lixeira", "MOVE_TO_TRASH": "Mover para a lixeira", "TRASH_FILES_MESSAGE": "Os itens selecionados serão excluídos de todos os álbuns e movidos para a lixeira.", - "TRASH_FILE_MESSAGE": "Os itens selecionados serão excluídos de todos os álbuns e movidos para o lixo.", + "TRASH_FILE_MESSAGE": "O item será removido de todos os álbuns e movido para a lixeira.", "DELETE_PERMANENTLY": "Excluir permanentemente", "RESTORE": "Restaurar", "RESTORE_TO_COLLECTION": "Restaurar para álbum", diff --git a/web/packages/next/locales/sv-SE/translation.json b/web/packages/next/locales/sv-SE/translation.json index 3c6adfb1ba..758d0ee314 100644 --- a/web/packages/next/locales/sv-SE/translation.json +++ b/web/packages/next/locales/sv-SE/translation.json @@ -1,65 +1,65 @@ { - "HERO_SLIDE_1_TITLE": "", - "HERO_SLIDE_1": "", + "HERO_SLIDE_1_TITLE": "
Privata säkerhetskopior
för dina minnen
", + "HERO_SLIDE_1": "Totalsträckskryptering som standard", "HERO_SLIDE_2_TITLE": "", - "HERO_SLIDE_2": "", - "HERO_SLIDE_3_TITLE": "", - "HERO_SLIDE_3": "", - "LOGIN": "", - "SIGN_UP": "", + "HERO_SLIDE_2": "Utformad för att överleva", + "HERO_SLIDE_3_TITLE": "
Tillgänglig
överallt
", + "HERO_SLIDE_3": "Android, iOS, webb, skrivbord", + "LOGIN": "Logga in", + "SIGN_UP": "Registrera", "NEW_USER": "", - "EXISTING_USER": "", + "EXISTING_USER": "Befintlig användare", "ENTER_NAME": "Ange namn", - "PUBLIC_UPLOADER_NAME_MESSAGE": "", + "PUBLIC_UPLOADER_NAME_MESSAGE": "Lägg till ett namn så att dina vänner vet vem de ska tacka för dessa fantastiska bilder!", "ENTER_EMAIL": "Ange e-postadress", "EMAIL_ERROR": "Ange en giltig e-postadress", - "REQUIRED": "", - "EMAIL_SENT": "", - "CHECK_INBOX": "", - "ENTER_OTT": "", - "RESEND_MAIL": "", - "VERIFY": "", - "UNKNOWN_ERROR": "", - "INVALID_CODE": "", - "EXPIRED_CODE": "", - "SENDING": "", - "SENT": "", + "REQUIRED": "Obligatoriskt", + "EMAIL_SENT": "Verifikationskoden skickad till {{email}}", + "CHECK_INBOX": "Kontrollera din inkorg (och skräppost) för att slutföra verifieringen", + "ENTER_OTT": "Verifieringskod", + "RESEND_MAIL": "Skicka kod igen", + "VERIFY": "Bekräfta", + "UNKNOWN_ERROR": "Något gick fel, vänligen försök igen", + "INVALID_CODE": "Ogiltig verifieringskod", + "EXPIRED_CODE": "Din verifieringskod har löpt ut", + "SENDING": "Skickar...", + "SENT": "Skickat!", "PASSWORD": "Lösenord", - "LINK_PASSWORD": "", + "LINK_PASSWORD": "Ange lösenord för att låsa upp albumet", "RETURN_PASSPHRASE_HINT": "Lösenord", - "SET_PASSPHRASE": "", + "SET_PASSPHRASE": "Välj lösenord", "VERIFY_PASSPHRASE": "Logga in", - "INCORRECT_PASSPHRASE": "", - "ENTER_ENC_PASSPHRASE": "", - "PASSPHRASE_DISCLAIMER": "", + "INCORRECT_PASSPHRASE": "Felaktigt lösenord", + "ENTER_ENC_PASSPHRASE": "Ange ett lösenord som vi kan använda för att kryptera din data", + "PASSPHRASE_DISCLAIMER": "Vi lagrar inte ditt lösenord, så om du glömmer det, vi kommer inte att kunna hjälpa dig återställa dina data utan en återställningsnyckel.", "WELCOME_TO_ENTE_HEADING": "Välkommen till ", - "WELCOME_TO_ENTE_SUBHEADING": "", - "WHERE_YOUR_BEST_PHOTOS_LIVE": "", - "KEY_GENERATION_IN_PROGRESS_MESSAGE": "", + "WELCOME_TO_ENTE_SUBHEADING": "Totalsträckskrypterad lagring och delning av foton", + "WHERE_YOUR_BEST_PHOTOS_LIVE": "Där dina bästa bilder bor", + "KEY_GENERATION_IN_PROGRESS_MESSAGE": "Skapar krypteringsnycklar...", "PASSPHRASE_HINT": "Lösenord", "CONFIRM_PASSPHRASE": "Bekräfta lösenord", - "REFERRAL_CODE_HINT": "", - "REFERRAL_INFO": "", + "REFERRAL_CODE_HINT": "Hur hörde du talas om Ente? (valfritt)", + "REFERRAL_INFO": "Vi spårar inte appinstallationer, Det skulle hjälpa oss om du berättade var du hittade oss!", "PASSPHRASE_MATCH_ERROR": "Lösenorden matchar inte", - "CREATE_COLLECTION": "", - "ENTER_ALBUM_NAME": "", - "CLOSE_OPTION": "", + "CREATE_COLLECTION": "Nytt album", + "ENTER_ALBUM_NAME": "Albumnamn", + "CLOSE_OPTION": "Stäng (Esc)", "ENTER_FILE_NAME": "Filnamn", "CLOSE": "Stäng", "NO": "Nej", - "NOTHING_HERE": "", - "UPLOAD": "", - "IMPORT": "", - "ADD_PHOTOS": "", - "ADD_MORE_PHOTOS": "", - "add_photos_one": "", - "add_photos_other": "", - "SELECT_PHOTOS": "", - "FILE_UPLOAD": "", + "NOTHING_HERE": "Inget att se här ännu 👀", + "UPLOAD": "Ladda upp", + "IMPORT": "Importera", + "ADD_PHOTOS": "Lägg till bilder", + "ADD_MORE_PHOTOS": "Lägg till fler bilder", + "add_photos_one": "Lägg till ett (1) objekt", + "add_photos_other": "Lägg till {{count, number}} objekt", + "SELECT_PHOTOS": "Välj bilder", + "FILE_UPLOAD": "Ladda upp", "UPLOAD_STAGE_MESSAGE": { - "0": "", - "1": "", - "2": "", + "0": "Förbereder att ladda upp", + "1": "Läser Google metadatafiler", + "2": "Metadata för {{uploadCounter.finished, number}} / {{uploadCounter.total, number}} filer extraherat", "3": "", "4": "", "5": "" @@ -114,11 +114,11 @@ "RECOVER_KEY_GENERATION_FAILED": "", "KEY_NOT_STORED_DISCLAIMER": "", "FORGOT_PASSWORD": "Glömt lösenord", - "RECOVER_ACCOUNT": "", - "RECOVERY_KEY_HINT": "", - "RECOVER": "", + "RECOVER_ACCOUNT": "Återställ konto", + "RECOVERY_KEY_HINT": "Återställningsnyckel", + "RECOVER": "Återställ", "NO_RECOVERY_KEY": "Ingen återställningsnyckel?", - "INCORRECT_RECOVERY_KEY": "", + "INCORRECT_RECOVERY_KEY": "Felaktig återställningsnyckel", "SORRY": "", "NO_RECOVERY_KEY_MESSAGE": "", "NO_TWO_FACTOR_RECOVERY_KEY_MESSAGE": "", @@ -140,12 +140,12 @@ "DOWNLOAD_APP_MESSAGE": "", "DOWNLOAD_APP": "", "EXPORT": "", - "SUBSCRIPTION": "", + "SUBSCRIPTION": "Prenumeration", "SUBSCRIBE": "Prenumerera", "MANAGEMENT_PORTAL": "Hantera betalningsmetod", "MANAGE_FAMILY_PORTAL": "", "LEAVE_FAMILY_PLAN": "", - "LEAVE": "", + "LEAVE": "Lämna", "LEAVE_FAMILY_CONFIRM": "", "CHOOSE_PLAN": "", "MANAGE_PLAN": "Hantera din prenumeration", @@ -179,7 +179,7 @@ "REACTIVATE_SUBSCRIPTION_MESSAGE": "", "SUBSCRIPTION_ACTIVATE_SUCCESS": "", "SUBSCRIPTION_ACTIVATE_FAILED": "", - "SUBSCRIPTION_PURCHASE_SUCCESS_TITLE": "", + "SUBSCRIPTION_PURCHASE_SUCCESS_TITLE": "Tack", "CANCEL_SUBSCRIPTION_ON_MOBILE": "", "CANCEL_SUBSCRIPTION_ON_MOBILE_MESSAGE": "", "MAIL_TO_MANAGE_SUBSCRIPTION": "", @@ -191,7 +191,7 @@ "DELETE_COLLECTION_MESSAGE": "", "DELETE_PHOTOS": "", "KEEP_PHOTOS": "", - "SHARE_COLLECTION": "", + "SHARE_COLLECTION": "Dela album", "SHARE_WITH_SELF": "", "ALREADY_SHARED": "", "SHARING_BAD_REQUEST_ERROR": "", @@ -230,10 +230,10 @@ "INFO": "", "INFO_OPTION": "", "FILE_NAME": "", - "CAPTION_PLACEHOLDER": "", + "CAPTION_PLACEHOLDER": "Lägg till en beskrivning", "LOCATION": "", "SHOW_ON_MAP": "", - "MAP": "", + "MAP": "Karta", "MAP_SETTINGS": "", "ENABLE_MAPS": "", "ENABLE_MAP": "", @@ -247,23 +247,23 @@ "EXIF": "", "ISO": "", "TWO_FACTOR": "", - "TWO_FACTOR_AUTHENTICATION": "", + "TWO_FACTOR_AUTHENTICATION": "Tvåfaktorsautentisering", "TWO_FACTOR_QR_INSTRUCTION": "", "ENTER_CODE_MANUALLY": "", "TWO_FACTOR_MANUAL_CODE_INSTRUCTION": "", "SCAN_QR_CODE": "", "ENABLE_TWO_FACTOR": "", - "ENABLE": "", + "ENABLE": "Aktivera", "LOST_DEVICE": "", - "INCORRECT_CODE": "", + "INCORRECT_CODE": "Felaktig kod", "TWO_FACTOR_INFO": "", - "DISABLE_TWO_FACTOR_LABEL": "", + "DISABLE_TWO_FACTOR_LABEL": "Inaktivera tvåfaktorsautentisering", "UPDATE_TWO_FACTOR_LABEL": "", - "DISABLE": "", + "DISABLE": "Inaktivera", "RECONFIGURE": "", "UPDATE_TWO_FACTOR": "", "UPDATE_TWO_FACTOR_MESSAGE": "", - "UPDATE": "", + "UPDATE": "Uppdatera", "DISABLE_TWO_FACTOR": "", "DISABLE_TWO_FACTOR_MESSAGE": "", "TWO_FACTOR_DISABLE_FAILED": "", @@ -298,7 +298,7 @@ "UPLOAD_TO_COLLECTION": "", "UNCATEGORIZED": "", "ARCHIVE": "", - "FAVORITES": "", + "FAVORITES": "Favoriter", "ARCHIVE_COLLECTION": "", "ARCHIVE_SECTION_NAME": "", "ALL_SECTION_NAME": "", @@ -308,16 +308,16 @@ "HIDE_COLLECTION": "", "UNHIDE_COLLECTION": "", "MOVE": "", - "ADD": "", + "ADD": "Lägg till", "REMOVE": "", "YES_REMOVE": "", "REMOVE_FROM_COLLECTION": "", - "TRASH": "", - "MOVE_TO_TRASH": "", + "TRASH": "Papperskorg", + "MOVE_TO_TRASH": "Flytta till papperskorg", "TRASH_FILES_MESSAGE": "", "TRASH_FILE_MESSAGE": "", "DELETE_PERMANENTLY": "", - "RESTORE": "", + "RESTORE": "Återställ", "RESTORE_TO_COLLECTION": "", "EMPTY_TRASH": "", "EMPTY_TRASH_TITLE": "", @@ -331,7 +331,7 @@ "CONFIRM_SELF_AND_OTHER_REMOVE_MESSAGE": "", "SORT_BY_CREATION_TIME_ASCENDING": "", "SORT_BY_UPDATION_TIME_DESCENDING": "", - "SORT_BY_NAME": "", + "SORT_BY_NAME": "Namn", "FIX_CREATION_TIME": "", "FIX_CREATION_TIME_IN_PROGRESS": "", "CREATION_TIME_UPDATED": "", @@ -355,7 +355,7 @@ "shared_with_people_other": "", "participants_zero": "Inga deltagare", "participants_one": "1 deltagare", - "participants_other": "", + "participants_other": "{{count, number}} deltagare", "ADD_VIEWERS": "", "CHANGE_PERMISSIONS_TO_VIEWER": "", "CHANGE_PERMISSIONS_TO_COLLABORATOR": "", @@ -377,7 +377,7 @@ "NOT_FOUND": "", "LINK_EXPIRED": "", "LINK_EXPIRED_MESSAGE": "", - "MANAGE_LINK": "", + "MANAGE_LINK": "Hantera länk", "LINK_TOO_MANY_REQUESTS": "", "FILE_DOWNLOAD": "", "LINK_PASSWORD_LOCK": "", @@ -385,7 +385,7 @@ "LINK_DEVICE_LIMIT": "", "NO_DEVICE_LIMIT": "", "LINK_EXPIRY": "", - "NEVER": "", + "NEVER": "Aldrig", "DISABLE_FILE_DOWNLOAD": "", "DISABLE_FILE_DOWNLOAD_MESSAGE": "", "SHARED_USING": "", @@ -418,17 +418,17 @@ "HIDDEN_ALBUMS": "", "HIDDEN_ITEMS": "", "ENTER_TWO_FACTOR_OTP": "", - "CREATE_ACCOUNT": "", + "CREATE_ACCOUNT": "Skapa konto", "COPIED": "", "WATCH_FOLDERS": "", "UPGRADE_NOW": "", "RENEW_NOW": "", "STORAGE": "", "USED": "", - "YOU": "", + "YOU": "Du", "FAMILY": "", - "FREE": "", - "OF": "", + "FREE": "gratis", + "OF": "av", "WATCHED_FOLDERS": "", "NO_FOLDERS_ADDED": "", "FOLDERS_AUTOMATICALLY_MONITORED": "", @@ -495,9 +495,9 @@ "storage_unit": { "b": "", "kb": "", - "mb": "", - "gb": "", - "tb": "" + "mb": "MB", + "gb": "GB", + "tb": "TB" }, "AFTER_TIME": { "HOUR": "", @@ -570,8 +570,8 @@ "CONVERT": "", "CONFIRM_EDITOR_CLOSE_MESSAGE": "", "CONFIRM_EDITOR_CLOSE_DESCRIPTION": "", - "BRIGHTNESS": "", - "CONTRAST": "", + "BRIGHTNESS": "Ljusstyrka", + "CONTRAST": "Kontrast", "SATURATION": "", "BLUR": "", "INVERT_COLORS": "", @@ -619,7 +619,7 @@ "PASSKEY_LOGIN_FAILED": "", "PASSKEY_LOGIN_URL_INVALID": "", "PASSKEY_LOGIN_ERRORED": "", - "TRY_AGAIN": "", + "TRY_AGAIN": "Försök igen", "PASSKEY_FOLLOW_THE_STEPS_FROM_YOUR_BROWSER": "", "LOGIN_WITH_PASSKEY": "", "autogenerated_first_album_name": "", From 50556b993097c1ffe61588960ba96c53de1cfaed Mon Sep 17 00:00:00 2001 From: Crowdin Bot Date: Tue, 28 May 2024 02:07:43 +0000 Subject: [PATCH 071/354] New Crowdin translations by GitHub Action --- auth/lib/l10n/arb/app_fr.arb | 25 +++++++++++++++++++- auth/lib/l10n/arb/app_pt.arb | 16 +++++++------ auth/lib/l10n/arb/app_ru.arb | 34 +++++++++++++++++++++++++++- auth/lib/l10n/arb/app_sv.arb | 4 ++++ auth/lib/l10n/arb/app_tr.arb | 44 ++++++++++++++++++++++++++++++++++-- 5 files changed, 112 insertions(+), 11 deletions(-) diff --git a/auth/lib/l10n/arb/app_fr.arb b/auth/lib/l10n/arb/app_fr.arb index 71ddc0b31c..d789fc48c2 100644 --- a/auth/lib/l10n/arb/app_fr.arb +++ b/auth/lib/l10n/arb/app_fr.arb @@ -20,6 +20,8 @@ "codeIssuerHint": "Émetteur", "codeSecretKeyHint": "Clé secrète", "codeAccountHint": "Compte (vous@exemple.com)", + "codeTagHint": "Tag", + "accountKeyType": "Type de clé", "sessionExpired": "Session expirée", "@sessionExpired": { "description": "Title of the dialog when the users current session is invalid/expired" @@ -77,12 +79,14 @@ "data": "Données", "importCodes": "Importer les codes", "importTypePlainText": "Texte brut", + "importTypeEnteEncrypted": "Export chiffré Ente", "passwordForDecryptingExport": "Mot de passe pour déchiffrer l'exportation", "passwordEmptyError": "Le mot de passe ne peut pas être vide", "importFromApp": "Importer des codes depuis {appName}", "importGoogleAuthGuide": "Exportez vos comptes depuis Google Authenticator vers un code QR en utilisant l'option \"Transférer des comptes\". Ensuite, en utilisant un autre appareil, scannez le code QR.\n\nAstuce : Vous pouvez utiliser la webcam de votre ordinateur portable pour prendre une photo du code QR.", "importSelectJsonFile": "Sélectionnez un fichier JSON", "importSelectAppExport": "Sélectionnez le fichier d'exportation {appName}", + "importEnteEncGuide": "Sélectionnez le fichier chiffré JSON exporté depuis Ente", "importRaivoGuide": "Utilisez l'option \"Exporter les OTPs vers l'archive Zip\" dans les paramètres de Raivo.\n\nExtrayez le fichier zip et importez le fichier JSON.", "importBitwardenGuide": "Utilisez l'option « Exporter le coffre » dans les outils Bitwarden et importez le fichier JSON non chiffré.", "importAegisGuide": "Utilisez l'option \"Exporter le coffre-fort\" dans les paramètres d'Aegis.\n\nSi votre coffre-fort est crypté, vous devrez saisir le mot de passe du coffre-fort pour déchiffrer le coffre-fort.", @@ -112,18 +116,22 @@ "copied": "Copié", "pleaseTryAgain": "Veuillez réessayer", "existingUser": "Utilisateur existant", + "newUser": "Nouveau dans Ente", "delete": "Supprimer", "enterYourPasswordHint": "Saisir votre mot de passe", "forgotPassword": "Mot de passe oublié", "oops": "Oups", "suggestFeatures": "Suggérer des fonctionnalités", "faq": "FAQ", + "faq_q_1": "Quelle est la sécurité de Auth?", + "faq_a_1": "Tous les codes que vous sauvegardez via ente sont chiffrés de bout en bout. Cela signifie que vous seul pouvez accéder à vos codes. Nos applications sont open source et notre cryptographie ont fait l'objet d'un audit externe.", "faq_q_2": "Puis-je accéder à mes codes sur mon ordinateur ?", "faq_a_2": "Vous pouvez accéder à vos codes sur le web via auth.ente.io.", "faq_q_3": "Comment puis-je supprimer des codes ?", "faq_a_3": "Vous pouvez supprimer un code en glissant vers la gauche.", "faq_q_4": "Comment puis-je soutenir le projet ?", "faq_a_4": "Vous pouvez soutenir le développement de ce projet en vous abonnant à notre application Photos, ente.io.", + "faq_q_5": "Comment puis-je activer le verrouillage FaceID dans Auth", "faq_a_5": "Vous pouvez activer le verrouillage FaceID dans Paramètres → Sécurité → Écran de verrouillage.", "somethingWentWrongMessage": "Quelque chose s'est mal passé, veuillez recommencer", "leaveFamily": "Quitter le plan familial", @@ -150,6 +158,7 @@ } } }, + "invalidQRCode": "QR code non valide", "noRecoveryKeyTitle": "Pas de clé de récupération ?", "enterEmailHint": "Entrez votre adresse e-mail", "invalidEmailTitle": "Adresse e-mail invalide", @@ -343,6 +352,7 @@ "deleteCodeAuthMessage": "Authentification requise pour supprimer le code", "showQRAuthMessage": "Authentification requise pour afficher le code QR", "confirmAccountDeleteTitle": "Confirmer la suppression du compte", + "confirmAccountDeleteMessage": "Ce compte est lié à d'autres applications ente, si vous en utilisez une.\n\nVos données téléchargées, dans toutes les applications ente, seront planifiées pour suppression, et votre compte sera définitivement supprimé.", "androidBiometricHint": "Vérifier l’identité", "@androidBiometricHint": { "description": "Hint message advising the user how to authenticate with biometrics. It is used on Android side. Maximum 60 characters." @@ -413,5 +423,18 @@ "invalidEndpoint": "Point de terminaison non valide", "invalidEndpointMessage": "Désolé, le point de terminaison que vous avez entré n'est pas valide. Veuillez en entrer un valide puis réessayez.", "endpointUpdatedMessage": "Point de terminaison mis à jour avec succès", - "customEndpoint": "Connecté à {endpoint}" + "customEndpoint": "Connecté à {endpoint}", + "pinText": "Épingler", + "unpinText": "Désépingler", + "pinnedCodeMessage": "{code} a été épinglé", + "unpinnedCodeMessage": "{code} a été désépinglé", + "tags": "Tags", + "createNewTag": "Créer un nouveau tag", + "tag": "Tag", + "create": "Créer", + "editTag": "Modifier le tag", + "deleteTagTitle": "Supprimer le tag ?", + "deleteTagMessage": "Êtes-vous sûr de vouloir supprimer ce tag ? Cette action est irréversible.", + "somethingWentWrongParsingCode": "Impossible d'analyser les codes {x}.", + "updateNotAvailable": "Mise à jour non disponible" } \ No newline at end of file diff --git a/auth/lib/l10n/arb/app_pt.arb b/auth/lib/l10n/arb/app_pt.arb index 3f92822d9a..232c1becfe 100644 --- a/auth/lib/l10n/arb/app_pt.arb +++ b/auth/lib/l10n/arb/app_pt.arb @@ -31,7 +31,7 @@ "timeBasedKeyType": "Baseado no horário (TOTP)", "counterBasedKeyType": "Baseado em um contador (HOTP)", "saveAction": "Salvar", - "nextTotpTitle": "próximo", + "nextTotpTitle": "avançar", "deleteCodeTitle": "Excluir código?", "deleteCodeMessage": "Tem certeza de que deseja excluir este código? Esta ação é irreversível.", "viewLogsAction": "Ver logs", @@ -105,7 +105,7 @@ "authToChangeYourPassword": "Por favor, autentique-se para alterar sua senha", "authToViewSecrets": "Por favor, autentique-se para ver as suas chaves secretas", "authToInitiateSignIn": "Por favor, autentique-se para iniciar o login para um backup.", - "ok": "Ok", + "ok": "OK", "cancel": "Cancelar", "yes": "Sim", "no": "Não", @@ -120,7 +120,7 @@ "delete": "Excluir", "enterYourPasswordHint": "Insira sua senha", "forgotPassword": "Esqueci a senha", - "oops": "Oops", + "oops": "Opa", "suggestFeatures": "Sugerir funcionalidades", "faq": "Perguntas frequentes", "faq_q_1": "Quão seguro é o Auth?", @@ -239,8 +239,8 @@ "howItWorks": "Como funciona", "ackPasswordLostWarning": "Eu entendo que se eu perder minha senha, posso perder meus dados, já que meus dados são criptografados de ponta a ponta.", "loginTerms": "Ao clicar em login, eu concordo com os termos de serviço e a política de privacidade", - "logInLabel": "Login", - "logout": "Encerrar sessão", + "logInLabel": "Entrar", + "logout": "Sair", "areYouSureYouWantToLogout": "Você tem certeza que deseja encerrar a sessão?", "yesLogout": "Sim, encerrar sessão", "exit": "Sair", @@ -282,7 +282,7 @@ "description": "Text for the button to confirm the user understands the warning" }, "authToExportCodes": "Por favor, autentique-se para exportar seus códigos", - "importSuccessTitle": "Yay!", + "importSuccessTitle": "Oba!", "importSuccessDesc": "Você importou {count} códigos!", "@importSuccessDesc": { "placeholders": { @@ -401,7 +401,7 @@ "@iOSGoToSettingsDescription": { "description": "Message advising the user to go to the settings and configure Biometrics for their device. It shows in a dialog on iOS side." }, - "iOSOkButton": "Ok", + "iOSOkButton": "OK", "@iOSOkButton": { "description": "Message showed on a button that the user can click to leave the current dialog. It is used on iOS side. Maximum 30 characters." }, @@ -425,7 +425,9 @@ "endpointUpdatedMessage": "Endpoint atualizado com sucesso", "customEndpoint": "Conectado a {endpoint}", "pinText": "Fixar", + "unpinText": "Desafixar", "pinnedCodeMessage": "{code} foi fixado", + "unpinnedCodeMessage": "{code} foi desafixado", "tags": "Etiquetas", "createNewTag": "Criar etiqueta", "tag": "Etiqueta", diff --git a/auth/lib/l10n/arb/app_ru.arb b/auth/lib/l10n/arb/app_ru.arb index 42571a166b..9c5dceaaba 100644 --- a/auth/lib/l10n/arb/app_ru.arb +++ b/auth/lib/l10n/arb/app_ru.arb @@ -20,6 +20,8 @@ "codeIssuerHint": "Эмитент", "codeSecretKeyHint": "Секретный ключ", "codeAccountHint": "Аккаунт (you@domain.com)", + "codeTagHint": "Метка", + "accountKeyType": "Тип ключа", "sessionExpired": "Сеанс истек", "@sessionExpired": { "description": "Title of the dialog when the users current session is invalid/expired" @@ -77,16 +79,19 @@ "data": "Данные", "importCodes": "Импортировать коды", "importTypePlainText": "Обычный текст", + "importTypeEnteEncrypted": "Ente Зашифрованный экспорт", "passwordForDecryptingExport": "Пароль для расшифровки экспорта", "passwordEmptyError": "Пароль не может быть пустым", "importFromApp": "Импорт кодов из {appName}", "importGoogleAuthGuide": "Экспортируйте учетные записи из Google Authenticator в QR-код, используя опцию «Перенести учетные записи». Затем с помощью другого устройства отсканируйте QR-код.\n\nСовет: Чтобы сфотографировать QR-код, можно воспользоваться веб-камерой ноутбука.", "importSelectJsonFile": "Выбрать JSON-файл", "importSelectAppExport": "Выбрать файл экспорта {appName}", + "importEnteEncGuide": "Выберите зашифрованный JSON файл, экспортированный из Ente", "importRaivoGuide": "Используйте опцию «Export OTPs to Zip archive» в настройках Raivo.\n\nРаспакуйте zip-архив и импортируйте JSON-файл.", "importBitwardenGuide": "Используйте опцию \"Экспортировать хранилище\" в Bitwarden Tools и импортируйте незашифрованный JSON файл.", "importAegisGuide": "Используйте опцию «Экспортировать хранилище» в настройках Aegis.\n\nЕсли ваше хранилище зашифровано, то для его расшифровки потребуется ввести пароль хранилища.", "import2FasGuide": "Используйте опцию \"Settings->Backup -Export\" в 2FAS.\n\nЕсли ваша резервная копия зашифрована, то для расшифровки резервной копии необходимо ввести пароль", + "importLastpassGuide": "Используйте опцию \"Перенести аккаунты\" в настройках Lastpass Authenticator и нажмите на \"Экспортировать учетные записи в файл\". Импортируйте загружённый JSON файл.", "exportCodes": "Экспортировать коды", "importLabel": "Импорт", "importInstruction": "Пожалуйста, выберите файл, содержащий список ваших кодов в следующем формате", @@ -99,6 +104,7 @@ "authToChangeYourEmail": "Пожалуйста, авторизуйтесь, чтобы изменить адрес электронной почты", "authToChangeYourPassword": "Пожалуйста, авторизуйтесь, чтобы изменить пароль", "authToViewSecrets": "Пожалуйста, авторизуйтесь для просмотра ваших секретов", + "authToInitiateSignIn": "Пожалуйста, авторизуйтесь, чтобы начать вход для резервного копирования.", "ok": "Ок", "cancel": "Отменить", "yes": "Да", @@ -110,18 +116,22 @@ "copied": "Скопировано", "pleaseTryAgain": "Пожалуйста, попробуйте ещё раз", "existingUser": "Существующий пользователь", + "newUser": "Впервые здесь, в Ente", "delete": "Удалить", "enterYourPasswordHint": "Введите пароль", "forgotPassword": "Забыл пароль", "oops": "Ой", "suggestFeatures": "Предложить идеи", "faq": "FAQ", + "faq_q_1": "Насколько безопасен Auth?", + "faq_a_1": "Все коды, которые вы резервируете с помощью Auth, хранятся в зашифрованном виде. Это означает, что только вы можете получить доступ к своим кодам. Наши приложения имеют открытый исходный код, а наша криптография прошла внешний аудит.", "faq_q_2": "Могу ли я получить доступ к моим кодам на компьютере?", "faq_a_2": "Вы можете получить доступ к своим кодам на сайте @ auth.ente.io.", "faq_q_3": "Как я могу удалить коды?", "faq_a_3": "Вы можете удалить код, проведя пальцем влево по этому элементу.", "faq_q_4": "Как я могу поддержать этот проект?", "faq_a_4": "Вы можете поддержать развитие этого проекта, подписавшись на наше приложение Photos @ ente.io.", + "faq_q_5": "Как мне включить FaceID в Auth", "faq_a_5": "Вы можете включить блокировку FaceID в Настройки → Безопасность → Экран блокировки.", "somethingWentWrongMessage": "Что-то пошло не так. Попробуйте еще раз", "leaveFamily": "Покинуть семью", @@ -135,6 +145,8 @@ "enterCodeHint": "Введите 6-значный код из\nвашего приложения-аутентификатора", "lostDeviceTitle": "Потеряно устройство?", "twoFactorAuthTitle": "Двухфакторная аутентификация", + "passkeyAuthTitle": "Проверка с помощью пароля", + "verifyPasskey": "Подтвердить пароль", "recoverAccount": "Восстановить аккаунт", "enterRecoveryKeyHint": "Введите свой ключ восстановления", "recover": "Восстановить", @@ -146,6 +158,7 @@ } } }, + "invalidQRCode": "Неверный QR-код", "noRecoveryKeyTitle": "Нет ключа восстановления?", "enterEmailHint": "Введите свою почту", "invalidEmailTitle": "Неверный адрес электронной почты", @@ -190,6 +203,8 @@ "saveKey": "Сохранить ключ", "save": "Сохранить", "send": "Отправить", + "saveOrSendDescription": "Вы хотите сохранить это в хранилище (папку загрузок по умолчанию) или отправить в другие приложения?", + "saveOnlyDescription": "Вы хотите сохранить это в хранилище (по умолчанию папка загрузок)?", "back": "Вернуться", "createAccount": "Создать аккаунт", "passwordStrength": "Мощность пароля: {passwordStrengthValue}", @@ -337,6 +352,7 @@ "deleteCodeAuthMessage": "Аутентификация для удаления кода", "showQRAuthMessage": "Аутентификация для отображения QR-кода", "confirmAccountDeleteTitle": "Подтвердить удаление аккаунта", + "confirmAccountDeleteMessage": "Эта учетная запись связана с другими приложениями Ente, если вы ими пользуетесь.\n\nЗагруженные вами данные во всех приложениях ente будут запланированы к удалению, а ваша учетная запись будет удалена без возможности восстановления.", "androidBiometricHint": "Подтвердите личность", "@androidBiometricHint": { "description": "Hint message advising the user how to authenticate with biometrics. It is used on Android side. Maximum 60 characters." @@ -397,12 +413,28 @@ "doNotSignOut": "Не выходить", "hearUsWhereTitle": "Как вы узнали о Ente? (необязательно)", "hearUsExplanation": "Будет полезно, если вы укажете, где нашли нас, так как мы не отслеживаем установки приложения", + "recoveryKeySaved": "Ключ восстановления сохранён в папке Загрузки!", + "waitingForBrowserRequest": "Ожидание запроса браузера...", "waitingForVerification": "Ожидание подтверждения...", + "passkey": "Ключ", "developerSettingsWarning": "Вы уверены, что хотите изменить настройки разработчика?", "developerSettings": "Настройки разработчика", "serverEndpoint": "Конечная точка сервера", "invalidEndpoint": "Неверная конечная точка", "invalidEndpointMessage": "Извините, введенная вами конечная точка неверна. Пожалуйста, введите корректную конечную точку и повторите попытку.", "endpointUpdatedMessage": "Конечная точка успешно обновлена", - "customEndpoint": "Подключено к {endpoint}" + "customEndpoint": "Подключено к {endpoint}", + "pinText": "Прикрепить", + "unpinText": "Открепить", + "pinnedCodeMessage": "{code} прикреплен", + "unpinnedCodeMessage": "{code} откреплен", + "tags": "Метки", + "createNewTag": "Создать новую метку", + "tag": "Метка", + "create": "Создать", + "editTag": "Изменить метку", + "deleteTagTitle": "Удалить метку?", + "deleteTagMessage": "Вы уверены, что хотите удалить эту метку? Это действие необратимо.", + "somethingWentWrongParsingCode": "Мы не смогли разобрать коды {x}.", + "updateNotAvailable": "Обновление недоступно" } \ No newline at end of file diff --git a/auth/lib/l10n/arb/app_sv.arb b/auth/lib/l10n/arb/app_sv.arb index 9761325ce1..41aa2f8a86 100644 --- a/auth/lib/l10n/arb/app_sv.arb +++ b/auth/lib/l10n/arb/app_sv.arb @@ -61,6 +61,7 @@ "welcomeBack": "Välkommen tillbaka!", "changePassword": "Ändra lösenord", "importCodes": "Importera koder", + "exportCodes": "Exportera koder", "cancel": "Avbryt", "yes": "Ja", "no": "Nej", @@ -76,6 +77,7 @@ "scan": "Skanna", "twoFactorAuthTitle": "Tvåfaktorsautentisering", "enterRecoveryKeyHint": "Ange din återställningsnyckel", + "invalidQRCode": "Ogiltig QR-kod", "noRecoveryKeyTitle": "Ingen återställningsnyckel?", "enterEmailHint": "Ange din e-postadress", "invalidEmailTitle": "Ogiltig e-postadress", @@ -143,6 +145,8 @@ }, "pendingSyncs": "Varning", "activeSessions": "Aktiva sessioner", + "incorrectCode": "Felaktig kod", + "incorrectRecoveryKey": "Felaktig återställningsnyckel", "enterPassword": "Ange lösenord", "export": "Exportera", "singIn": "Logga in", diff --git a/auth/lib/l10n/arb/app_tr.arb b/auth/lib/l10n/arb/app_tr.arb index 322af5f48c..2473067b74 100644 --- a/auth/lib/l10n/arb/app_tr.arb +++ b/auth/lib/l10n/arb/app_tr.arb @@ -20,6 +20,8 @@ "codeIssuerHint": "Yayınlayan", "codeSecretKeyHint": "Gizli Anahtar", "codeAccountHint": "Hesap (ornek@domain.com)", + "codeTagHint": "Etiket", + "accountKeyType": "Anahtar türü", "sessionExpired": "Oturum süresi doldu", "@sessionExpired": { "description": "Title of the dialog when the users current session is invalid/expired" @@ -46,7 +48,7 @@ }, "copyEmailAction": "E-postayı Kopyala", "exportLogsAction": "Günlüğü dışa aktar", - "reportABug": "Bir hata bildir", + "reportABug": "Hata bildirin", "crashAndErrorReporting": "Çökme ve hata bildirimi", "reportBug": "Hata bildir", "emailUsMessage": "Lütfen bize {email} adresinden ulaşın", @@ -77,12 +79,14 @@ "data": "Veri", "importCodes": "Kodu içe aktar", "importTypePlainText": "Salt metin", + "importTypeEnteEncrypted": "Ente Şifreli dışa aktarma", "passwordForDecryptingExport": "Dışa aktarımın şifresini çözmek için parola", "passwordEmptyError": "Şifre boş olamaz", "importFromApp": "Kodları {appName} uygulamasından içe aktarın", "importGoogleAuthGuide": "\"Hesapları Aktar\" seçeneğini kullanarak hesaplarınızı Google Authenticator'dan bir QR koduna aktarın. Ardından başka bir cihaz kullanarak QR kodunu tarayın.\n\nİpucu: QR kodunun fotoğrafını çekmek için dizüstü bilgisayarınızın kamerasını kullanabilirsiniz.", "importSelectJsonFile": "JSON dosyasını seçin", "importSelectAppExport": "{appName} dışarı aktarma dosyasını seçin", + "importEnteEncGuide": "Ente'den dışa aktarılan şifrelenmiş JSON dosyasını seçin", "importRaivoGuide": "Raivo'nun ayarlarında \"OTP'leri Zip arşivine aktar\" seçeneğini kullanın.\n\nZip dosyasını çıkarın ve JSON dosyasını içe aktarın.", "importBitwardenGuide": "Bitwarden Tools içindeki \"Kasayı dışa aktar\" seçeneğini kullanın ve şifrelenmemiş JSON dosyasını içe aktarın.", "importAegisGuide": "Aegis'in Ayarlarında \"Kasayı dışa aktar\" seçeneğini kullanın.\n\nKasanız şifrelenmişse, kasanın şifresini çözmek için kasa parolasını girmeniz gerekecektir.", @@ -112,18 +116,22 @@ "copied": "Kopyalandı", "pleaseTryAgain": "Lütfen tekrar deneyin", "existingUser": "Mevcut kullanıcı", + "newUser": "Ente'de Yeni", "delete": "Sil", "enterYourPasswordHint": "Parolanızı girin", "forgotPassword": "Şifremi unuttum", "oops": "Hay aksi", "suggestFeatures": "Özellik önerin", "faq": "SSS", + "faq_q_1": "Kimlik doğrulayıcı ne kadar güvenli?", + "faq_a_1": "Auth aracılığıyla yedeklediğiniz tüm kodlar uçtan uca şifrelenmiş olarak saklanır. Böylece kodlarınıza yalnızca siz erişebilirsiniz. Uygulamalarımız açık kaynaklıdır ve şifrelememiz dış denetimden geçmiştir.", "faq_q_2": "Kodlarıma masaüstünden erişebilir miyim?", "faq_a_2": "Kodlarınıza internet üzerinden @ auth.ente.io adresinden erişebilirsiniz.", "faq_q_3": "Kodları nasıl silebilirim?", "faq_a_3": "Bir kodu, o öğenin üzerinde sola kaydırarak silebilirsiniz.", "faq_q_4": "Bu projeye nasıl destek olabilirim?", "faq_a_4": "Fotoğraflar uygulamamıza @ ente.io abone olarak bu projenin geliştirilmesine destek olabilirsiniz.", + "faq_q_5": "Auth'ta FaceID kilidini nasıl etkinleştirebilirim", "faq_a_5": "FaceID kilidini Ayarlar → Güvenlik → Kilit Ekranı altında etkinleştirebilirsiniz.", "somethingWentWrongMessage": "Bir şeyler ters gitti, lütfen tekrar deneyin", "leaveFamily": "Aile planından ayrıl", @@ -137,6 +145,8 @@ "enterCodeHint": "Kimlik doğrulayıcı uygulamanızdaki 6 haneli doğrulama kodunu girin", "lostDeviceTitle": "Cihazınızı mı kaybettiniz?", "twoFactorAuthTitle": "İki faktörlü kimlik doğrulama", + "passkeyAuthTitle": "Geçiş anahtarı doğrulaması", + "verifyPasskey": "Geçiş anahtarını doğrula", "recoverAccount": "Hesap kurtarma", "enterRecoveryKeyHint": "Kurtarma anahtarınızı girin", "recover": "Kurtar", @@ -148,6 +158,7 @@ } } }, + "invalidQRCode": "Geçersiz QR kodu", "noRecoveryKeyTitle": "Kurtarma anahtarınız yok mu?", "enterEmailHint": "E-posta adresinizi girin", "invalidEmailTitle": "Geçersiz e-posta adresi", @@ -190,6 +201,10 @@ "recoveryKeySaveDescription": "Biz bu anahtarı saklamıyoruz, lütfen. bu 24 kelimelik anahtarı güvenli bir yerde saklayın.", "doThisLater": "Bunu daha sonra yap", "saveKey": "Anahtarı kaydet", + "save": "Kaydet", + "send": "Gönder", + "saveOrSendDescription": "Bunu belleğinize mi kaydedeceksiniz (İndirilenler klasörü varsayılandır) yoksa diğer uygulamalara mı göndereceksiniz?", + "saveOnlyDescription": "Bunu belleğinize kaydetmek ister misiniz? (İndirilenler klasörü varsayılandır)", "back": "Geri", "createAccount": "Hesap oluştur", "passwordStrength": "Şifre gücü: {passwordStrengthValue}", @@ -337,6 +352,7 @@ "deleteCodeAuthMessage": "Kodu silmek için doğrulama yapın", "showQRAuthMessage": "QR kodunu göstermek için doğrulama yapın", "confirmAccountDeleteTitle": "Hesap silme işlemini onayla", + "confirmAccountDeleteMessage": "Kullandığınız Ente uygulamaları varsa bu hesap diğer Ente uygulamalarıyla bağlantılıdır.\n\nTüm Ente uygulamalarına yüklediğiniz veriler ve hesabınız kalıcı olarak silinecektir.", "androidBiometricHint": "Kimliği doğrula", "@androidBiometricHint": { "description": "Hint message advising the user how to authenticate with biometrics. It is used on Android side. Maximum 60 characters." @@ -396,5 +412,29 @@ "signOutOtherDevices": "Diğer cihazlardan çıkış yap", "doNotSignOut": "Çıkış yapma", "hearUsWhereTitle": "Ente'yi nereden duydunuz? (opsiyonel)", - "hearUsExplanation": "Biz uygulama kurulumlarını takip etmiyoruz. Bizi nereden duyduğunuzdan bahsetmeniz bize çok yardımcı olacak!" + "hearUsExplanation": "Biz uygulama kurulumlarını takip etmiyoruz. Bizi nereden duyduğunuzdan bahsetmeniz bize çok yardımcı olacak!", + "recoveryKeySaved": "Kurtarma anahtarı İndirilenler klasörüne kaydedildi!", + "waitingForBrowserRequest": "Tarayıcı isteği bekleniyor...", + "waitingForVerification": "Doğrulama bekleniyor...", + "passkey": "Geçiş anahtarı", + "developerSettingsWarning": "Geliştirici ayarlarını değiştirmekten emin misiniz?", + "developerSettings": "Geliştirici ayarları", + "serverEndpoint": "Sunucu uç noktası", + "invalidEndpoint": "Geçersiz uç nokta", + "invalidEndpointMessage": "Üzgünüz, girdiğiniz uç nokta geçersiz. Lütfen geçerli bir uç nokta girin ve tekrar deneyin.", + "endpointUpdatedMessage": "Uç nokta başarıyla güncellendi", + "customEndpoint": "Bağlandı: {endpoint}", + "pinText": "Sabitle", + "unpinText": "Sabitlemeyi kaldır", + "pinnedCodeMessage": "{code} sabitlendi", + "unpinnedCodeMessage": "{code} sabitlemesi kaldırıldı", + "tags": "Etiketler", + "createNewTag": "Yeni etiket oluştur", + "tag": "Etiket", + "create": "Oluştur", + "editTag": "Etiketi düzenle", + "deleteTagTitle": "Etiket silinsin mi?", + "deleteTagMessage": "Bu etiketi silmek istediğinizden emin misiniz? Bu işlem geri alınamaz.", + "somethingWentWrongParsingCode": "{x} kodu ayrıştıramadık.", + "updateNotAvailable": "Güncelleme mevcut değil" } \ No newline at end of file From 07dc0231ee1795c3350067e68bce6d460078dc79 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 28 May 2024 09:36:44 +0530 Subject: [PATCH 072/354] photosd-v1.7.0 --- desktop/CHANGELOG.md | 2 +- desktop/docs/release.md | 6 +++--- desktop/package.json | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/desktop/CHANGELOG.md b/desktop/CHANGELOG.md index 5fbbefaaa8..094145d261 100644 --- a/desktop/CHANGELOG.md +++ b/desktop/CHANGELOG.md @@ -1,6 +1,6 @@ # CHANGELOG -## v1.7.0 (Unreleased) +## v1.7.0 v1.7 is a major rewrite to improve the security of our app. In particular, the UI and the native parts of the app now run isolated from each other and diff --git a/desktop/docs/release.md b/desktop/docs/release.md index 1cda1c11b1..53d0355c3e 100644 --- a/desktop/docs/release.md +++ b/desktop/docs/release.md @@ -39,8 +39,8 @@ artifacts attached to the same draft. ## Workflow - Release -1. Update source repo to set version `1.x.x` in `package.json` and finialize - the CHANGELOG. +1. Update source repo to set version `1.x.x` in `package.json` and finalize the + CHANGELOG. 2. Push code to the `desktop/rc` branch in the source repo. @@ -53,7 +53,7 @@ artifacts attached to the same draft. 4. If the build is successful, tag `desktop/rc` in the source repo. ```sh - # Assuming we're on desktop/rc that just got build + # Assuming we're on desktop/rc that just got built git tag photosd-v1.x.x git push origin photosd-v1.x.x diff --git a/desktop/package.json b/desktop/package.json index 085d966817..a38ce92f24 100644 --- a/desktop/package.json +++ b/desktop/package.json @@ -1,6 +1,6 @@ { "name": "ente", - "version": "1.7.0-rc", + "version": "1.7.0", "private": true, "description": "Desktop client for Ente Photos", "repository": "github:ente-io/photos-desktop", From cad07cd96f1205de2b44e19a611fcc8dbc7e1d49 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Tue, 28 May 2024 10:53:50 +0530 Subject: [PATCH 073/354] [mob][photos] Fire PeopleChangedEvent after each cluster bucket --- .../lib/services/machine_learning/face_ml/face_ml_service.dart | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart index 222fd50b8f..8ea0a30d93 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart @@ -666,6 +666,7 @@ class FaceMlService { .updateFaceIdToClusterId(clusteringResult.newFaceIdToCluster); await FaceMLDataDB.instance .clusterSummaryUpdate(clusteringResult.newClusterSummaries!); + Bus.instance.fire(PeopleChangedEvent()); for (final faceInfo in faceInfoForClustering) { faceInfo.clusterId ??= clusteringResult.newFaceIdToCluster[faceInfo.faceID]; @@ -710,10 +711,10 @@ class FaceMlService { .updateFaceIdToClusterId(clusteringResult.newFaceIdToCluster); await FaceMLDataDB.instance .clusterSummaryUpdate(clusteringResult.newClusterSummaries!); + Bus.instance.fire(PeopleChangedEvent()); _logger.info('Done updating FaceIDs with clusterIDs in the DB, in ' '${DateTime.now().difference(clusterDoneTime).inSeconds} seconds'); } - Bus.instance.fire(PeopleChangedEvent()); _logger.info('clusterAllImages() finished, in ' '${DateTime.now().difference(clusterAllImagesTime).inSeconds} seconds'); } catch (e, s) { From 705fae35e68f7ebbbe19fafd5779e730b072e7cf Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Tue, 28 May 2024 10:55:04 +0530 Subject: [PATCH 074/354] [mob][photos] Fire PeopleChangedEvent after syncing --- .../lib/services/machine_learning/face_ml/face_ml_service.dart | 1 + 1 file changed, 1 insertion(+) diff --git a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart index 8ea0a30d93..a8d13297d8 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart @@ -358,6 +358,7 @@ class FaceMlService { _isSyncing = true; if (forceSync) { await PersonService.instance.reconcileClusters(); + Bus.instance.fire(PeopleChangedEvent()); _shouldSyncPeople = false; } _isSyncing = false; From d1b2d5696a11793aab5319677d87cf7c4221c2d6 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Tue, 28 May 2024 11:05:40 +0530 Subject: [PATCH 075/354] [mob][photos] Wrap people banner in a SafeArea --- .../lib/ui/viewer/people/people_banner.dart | 112 +++++++++--------- 1 file changed, 57 insertions(+), 55 deletions(-) diff --git a/mobile/lib/ui/viewer/people/people_banner.dart b/mobile/lib/ui/viewer/people/people_banner.dart index db242a5230..e602d9aceb 100644 --- a/mobile/lib/ui/viewer/people/people_banner.dart +++ b/mobile/lib/ui/viewer/people/people_banner.dart @@ -68,65 +68,67 @@ class PeopleBanner extends StatelessWidget { roundedActionIcon = false; } - return RepaintBoundary( - child: Center( - child: GestureDetector( - onTap: onTap, - child: Container( - decoration: BoxDecoration( - boxShadow: Theme.of(context).colorScheme.enteTheme.shadowMenu, - color: backgroundColor, - ), - child: Padding( - padding: const EdgeInsets.symmetric(horizontal: 2, vertical: 2), - child: Row( - mainAxisAlignment: MainAxisAlignment.spaceBetween, - children: [ - startWidget, - const SizedBox(width: 12), - Expanded( - child: Column( - mainAxisAlignment: MainAxisAlignment.spaceEvenly, - crossAxisAlignment: CrossAxisAlignment.start, - children: [ - Text( - text, - style: mainTextStyle, - textAlign: TextAlign.left, - ), - subText != null - ? const SizedBox(height: 6) - : const SizedBox.shrink(), - subText != null - ? Text( - subText!, - style: subTextStyle, - ) - : const SizedBox.shrink(), - ], + return SafeArea( + child: RepaintBoundary( + child: Center( + child: GestureDetector( + onTap: onTap, + child: Container( + decoration: BoxDecoration( + boxShadow: Theme.of(context).colorScheme.enteTheme.shadowMenu, + color: backgroundColor, + ), + child: Padding( + padding: const EdgeInsets.symmetric(horizontal: 2, vertical: 2), + child: Row( + mainAxisAlignment: MainAxisAlignment.spaceBetween, + children: [ + startWidget, + const SizedBox(width: 12), + Expanded( + child: Column( + mainAxisAlignment: MainAxisAlignment.spaceEvenly, + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + Text( + text, + style: mainTextStyle, + textAlign: TextAlign.left, + ), + subText != null + ? const SizedBox(height: 6) + : const SizedBox.shrink(), + subText != null + ? Text( + subText!, + style: subTextStyle, + ) + : const SizedBox.shrink(), + ], + ), ), - ), - const SizedBox(width: 12), - IconButtonWidget( - icon: actionIcon, - iconButtonType: IconButtonType.primary, - iconColor: colorScheme.strokeBase, - defaultColor: colorScheme.fillFaint, - pressedColor: colorScheme.fillMuted, - roundedIcon: roundedActionIcon, - onTap: onTap, - ), - const SizedBox(width: 6), - ], + const SizedBox(width: 12), + IconButtonWidget( + icon: actionIcon, + iconButtonType: IconButtonType.primary, + iconColor: colorScheme.strokeBase, + defaultColor: colorScheme.fillFaint, + pressedColor: colorScheme.fillMuted, + roundedIcon: roundedActionIcon, + onTap: onTap, + ), + const SizedBox(width: 6), + ], + ), ), ), ), - ), - ).animate(onPlay: (controller) => controller.repeat()).shimmer( - duration: 1000.ms, - delay: 3200.ms, - size: 0.6, - ), + ).animate(onPlay: (controller) => controller.repeat()).shimmer( + duration: 1000.ms, + delay: 3200.ms, + size: 0.6, + ), + ), ); } } From b2df698e42425dda6dde2e898a3f2d879e664e93 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 28 May 2024 12:13:36 +0530 Subject: [PATCH 076/354] [desktop] Start next release sequence --- desktop/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/desktop/package.json b/desktop/package.json index a38ce92f24..4e493e30c8 100644 --- a/desktop/package.json +++ b/desktop/package.json @@ -1,6 +1,6 @@ { "name": "ente", - "version": "1.7.0", + "version": "1.7.1-rc", "private": true, "description": "Desktop client for Ente Photos", "repository": "github:ente-io/photos-desktop", From 07552f7a8950543b339f3b557abf18f2d4ad5d23 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Tue, 28 May 2024 12:19:28 +0530 Subject: [PATCH 077/354] Handle case when person has no file mapping --- .../lib/ui/viewer/people/add_person_action_sheet.dart | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/mobile/lib/ui/viewer/people/add_person_action_sheet.dart b/mobile/lib/ui/viewer/people/add_person_action_sheet.dart index 7a0c3a4713..a0f9f76a70 100644 --- a/mobile/lib/ui/viewer/people/add_person_action_sheet.dart +++ b/mobile/lib/ui/viewer/people/add_person_action_sheet.dart @@ -2,6 +2,7 @@ import "dart:async"; import "dart:developer"; import "dart:math" as math; +import "package:flutter/foundation.dart"; import 'package:flutter/material.dart'; import "package:logging/logging.dart"; import 'package:modal_bottom_sheet/modal_bottom_sheet.dart'; @@ -182,7 +183,7 @@ class _PersonActionSheetState extends State { child: Padding( padding: const EdgeInsets.fromLTRB(16, 24, 4, 0), child: FutureBuilder>( - future: _getPersons(), + future: _getPersonsWithRecentFile(), builder: (context, snapshot) { if (snapshot.hasError) { log("Error: ${snapshot.error} ${snapshot.stackTrace}}"); @@ -303,7 +304,7 @@ class _PersonActionSheetState extends State { } } - Future> _getPersons({ + Future> _getPersonsWithRecentFile({ bool excludeHidden = true, }) async { final persons = await PersonService.instance.getPersons(); @@ -317,6 +318,11 @@ class _PersonActionSheetState extends State { person.remoteID, ); final files = clustersToFiles.values.expand((e) => e).toList(); + if (files.isEmpty) { + debugPrint( + "Person ${kDebugMode ? person.data.name : person.remoteID} has no files"); + continue; + } personAndFileID.add((person, files.first)); } return personAndFileID; From cb8d57295147cab8f890273e9d8b815cbc83d137 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Tue, 28 May 2024 12:20:39 +0530 Subject: [PATCH 078/354] Bump version --- mobile/pubspec.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobile/pubspec.yaml b/mobile/pubspec.yaml index 1311b4e793..ed3bf47193 100644 --- a/mobile/pubspec.yaml +++ b/mobile/pubspec.yaml @@ -12,7 +12,7 @@ description: ente photos application # Read more about iOS versioning at # https://developer.apple.com/library/archive/documentation/General/Reference/InfoPlistKeyReference/Articles/CoreFoundationKeys.html -version: 0.8.113+637 +version: 0.8.120+640 publish_to: none environment: From 54aecfd7214c4f1de74740f563109f7de5e8fdb7 Mon Sep 17 00:00:00 2001 From: Ashil <77285023+ashilkn@users.noreply.github.com> Date: Tue, 28 May 2024 12:24:37 +0530 Subject: [PATCH 079/354] Revert "Upgrade to flutter 3.22.0 (#1804)" (#1901) ## Description This reverts commit a41f705dada762f4c6254734823aa564f691d677. Need to update `flutter_map` dependency to work with flutter 3.22.0. --- .github/workflows/mobile-lint.yml | 2 +- mobile/README.md | 2 +- mobile/ios/Podfile.lock | 2 +- mobile/pubspec.lock | 54 +++++++++++++++---------------- mobile/pubspec.yaml | 13 ++++---- 5 files changed, 36 insertions(+), 37 deletions(-) diff --git a/.github/workflows/mobile-lint.yml b/.github/workflows/mobile-lint.yml index 3a43924a35..493185b6bd 100644 --- a/.github/workflows/mobile-lint.yml +++ b/.github/workflows/mobile-lint.yml @@ -9,7 +9,7 @@ on: - ".github/workflows/mobile-lint.yml" env: - FLUTTER_VERSION: "3.22.0" + FLUTTER_VERSION: "3.19.5" jobs: lint: diff --git a/mobile/README.md b/mobile/README.md index 6d86ad5344..fc17f6b26e 100644 --- a/mobile/README.md +++ b/mobile/README.md @@ -46,7 +46,7 @@ You can alternatively install the build from PlayStore or F-Droid. ## 🧑‍💻 Building from source -1. [Install Flutter v3.22.0](https://flutter.dev/docs/get-started/install). +1. [Install Flutter v3.19.3](https://flutter.dev/docs/get-started/install). 2. Pull in all submodules with `git submodule update --init --recursive` diff --git a/mobile/ios/Podfile.lock b/mobile/ios/Podfile.lock index 9f74d552a7..558a279108 100644 --- a/mobile/ios/Podfile.lock +++ b/mobile/ios/Podfile.lock @@ -427,7 +427,7 @@ SPEC CHECKSUMS: home_widget: 0434835a4c9a75704264feff6be17ea40e0f0d57 image_editor_common: d6f6644ae4a6de80481e89fe6d0a8c49e30b4b43 in_app_purchase_storekit: 0e4b3c2e43ba1e1281f4f46dd71b0593ce529892 - integration_test: ce0a3ffa1de96d1a89ca0ac26fca7ea18a749ef4 + integration_test: 13825b8a9334a850581300559b8839134b124670 libwebp: 1786c9f4ff8a279e4dac1e8f385004d5fc253009 local_auth_darwin: c7e464000a6a89e952235699e32b329457608d98 local_auth_ios: 5046a18c018dd973247a0564496c8898dbb5adf9 diff --git a/mobile/pubspec.lock b/mobile/pubspec.lock index 8b71025e9f..1d1082bfd3 100644 --- a/mobile/pubspec.lock +++ b/mobile/pubspec.lock @@ -45,10 +45,10 @@ packages: dependency: "direct main" description: name: animated_list_plus - sha256: fb3d7f1fbaf5af84907f3c739236bacda8bf32cbe1f118dd51510752883ff50c + sha256: fe66f9c300d715254727fbdf050487844d17b013fec344fa28081d29bddbdf1a url: "https://pub.dev" source: hosted - version: "0.5.2" + version: "0.4.5" animated_stack_widget: dependency: transitive description: @@ -971,10 +971,10 @@ packages: dependency: "direct main" description: name: home_widget - sha256: "2a0fdd6267ff975bd07bedf74686bd5577200f504f5de36527ac1b56bdbe68e3" + sha256: "29565bfee4b32eaf9e7e8b998d504618b779a74b2b1ac62dd4dac7468e66f1a3" url: "https://pub.dev" source: hosted - version: "0.6.0" + version: "0.5.0" html: dependency: transitive description: @@ -1152,26 +1152,26 @@ packages: dependency: transitive description: name: leak_tracker - sha256: "7f0df31977cb2c0b88585095d168e689669a2cc9b97c309665e3386f3e9d341a" + sha256: "78eb209deea09858f5269f5a5b02be4049535f568c07b275096836f01ea323fa" url: "https://pub.dev" source: hosted - version: "10.0.4" + version: "10.0.0" leak_tracker_flutter_testing: dependency: transitive description: name: leak_tracker_flutter_testing - sha256: "06e98f569d004c1315b991ded39924b21af84cf14cc94791b8aea337d25b57f8" + sha256: b46c5e37c19120a8a01918cfaf293547f47269f7cb4b0058f21531c2465d6ef0 url: "https://pub.dev" source: hosted - version: "3.0.3" + version: "2.0.1" leak_tracker_testing: dependency: transitive description: name: leak_tracker_testing - sha256: "6ba465d5d76e67ddf503e1161d1f4a6bc42306f9d66ca1e8f079a47290fb06d3" + sha256: a597f72a664dbd293f3bfc51f9ba69816f84dcd403cdac7066cb3f6003f3ab47 url: "https://pub.dev" source: hosted - version: "3.0.1" + version: "2.0.1" like_button: dependency: "direct main" description: @@ -1368,10 +1368,10 @@ packages: dependency: transitive description: name: meta - sha256: "7687075e408b093f36e6bbf6c91878cc0d4cd10f409506f7bc996f68220b9136" + sha256: d584fa6707a52763a52446f02cc621b077888fb63b93bbcb1143a7be5a0c0c04 url: "https://pub.dev" source: hosted - version: "1.12.0" + version: "1.11.0" mgrs_dart: dependency: transitive description: @@ -2144,10 +2144,10 @@ packages: dependency: "direct main" description: name: styled_text - sha256: fd624172cf629751b4f171dd0ecf9acf02a06df3f8a81bb56c0caa4f1df706c3 + sha256: f72928d1ebe8cb149e3b34a689cb1ddca696b808187cf40ac3a0bd183dff379c url: "https://pub.dev" source: hosted - version: "8.1.0" + version: "7.0.0" sync_http: dependency: transitive description: @@ -2160,18 +2160,18 @@ packages: dependency: "direct main" description: name: syncfusion_flutter_core - sha256: "63108a33f9b0d89f7b6b56cce908b8e519fe433dbbe0efcf41ad3e8bb2081bd9" + sha256: "9be1bb9bbdb42823439a18da71484f1964c14dbe1c255ab1b931932b12fa96e8" url: "https://pub.dev" source: hosted - version: "25.2.5" + version: "19.4.56" syncfusion_flutter_sliders: dependency: "direct main" description: name: syncfusion_flutter_sliders - sha256: f27310bedc0e96e84054f0a70ac593d1a3c38397c158c5226ba86027ad77b2c1 + sha256: "1f6a63ccab4180b544074b9264a20f01ee80b553de154192fe1d7b434089d3c2" url: "https://pub.dev" source: hosted - version: "25.2.5" + version: "19.4.56" synchronized: dependency: "direct main" description: @@ -2192,26 +2192,26 @@ packages: dependency: "direct dev" description: name: test - sha256: "7ee446762c2c50b3bd4ea96fe13ffac69919352bd3b4b17bac3f3465edc58073" + sha256: a1f7595805820fcc05e5c52e3a231aedd0b72972cb333e8c738a8b1239448b6f url: "https://pub.dev" source: hosted - version: "1.25.2" + version: "1.24.9" test_api: dependency: transitive description: name: test_api - sha256: "9955ae474176f7ac8ee4e989dadfb411a58c30415bcfb648fa04b2b8a03afa7f" + sha256: "5c2f730018264d276c20e4f1503fd1308dfbbae39ec8ee63c5236311ac06954b" url: "https://pub.dev" source: hosted - version: "0.7.0" + version: "0.6.1" test_core: dependency: transitive description: name: test_core - sha256: "2bc4b4ecddd75309300d8096f781c0e3280ca1ef85beda558d33fcbedc2eead4" + sha256: a757b14fc47507060a162cc2530d9a4a2f92f5100a952c7443b5cad5ef5b106a url: "https://pub.dev" source: hosted - version: "0.6.0" + version: "0.5.9" timezone: dependency: transitive description: @@ -2441,10 +2441,10 @@ packages: dependency: transitive description: name: vm_service - sha256: "3923c89304b715fb1eb6423f017651664a03bf5f4b29983627c4da791f74a4ec" + sha256: b3d56ff4341b8f182b96aceb2fa20e3dcb336b9f867bc0eafc0de10f1048e957 url: "https://pub.dev" source: hosted - version: "14.2.1" + version: "13.0.0" volume_controller: dependency: transitive description: @@ -2591,4 +2591,4 @@ packages: version: "3.1.2" sdks: dart: ">=3.3.0 <4.0.0" - flutter: ">=3.20.0-1.2.pre" + flutter: ">=3.19.0" diff --git a/mobile/pubspec.yaml b/mobile/pubspec.yaml index ed3bf47193..934a99a8f0 100644 --- a/mobile/pubspec.yaml +++ b/mobile/pubspec.yaml @@ -21,7 +21,7 @@ environment: dependencies: adaptive_theme: ^3.1.0 animate_do: ^2.0.0 - animated_list_plus: ^0.5.2 + animated_list_plus: ^0.4.5 archive: ^3.1.2 background_fetch: ^1.2.1 battery_info: ^1.1.1 @@ -93,13 +93,13 @@ dependencies: fluttertoast: ^8.0.6 freezed_annotation: ^2.4.1 google_nav_bar: ^5.0.5 - home_widget: ^0.6.0 + home_widget: ^0.5.0 html_unescape: ^2.0.0 http: ^1.1.0 image: ^4.0.17 image_editor: ^1.3.0 in_app_purchase: ^3.0.7 - intl: ^0.19.0 + intl: ^0.18.0 json_annotation: ^4.8.0 latlong2: ^0.9.0 like_button: ^2.0.5 @@ -152,9 +152,9 @@ dependencies: sqlite3_flutter_libs: ^0.5.20 sqlite_async: ^0.6.1 step_progress_indicator: ^1.0.2 - styled_text: ^8.1.0 - syncfusion_flutter_core: ^25.2.5 - syncfusion_flutter_sliders: ^25.2.5 + styled_text: ^7.0.0 + syncfusion_flutter_core: ^19.2.49 + syncfusion_flutter_sliders: ^19.2.49 synchronized: ^3.1.0 tuple: ^2.0.0 uni_links: ^0.5.1 @@ -177,7 +177,6 @@ dependency_overrides: # Remove this after removing dependency from flutter_sodium. # Newer flutter packages depends on ffi > 2.0.0 while flutter_sodium depends on ffi < 2.0.0 ffi: 2.1.0 - intl: 0.18.1 video_player: git: url: https://github.com/ente-io/packages.git From d33c92a51c9acef6d51a97b894adb403facccddf Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Tue, 28 May 2024 12:26:54 +0530 Subject: [PATCH 080/354] Lint fix --- mobile/lib/ui/viewer/people/add_person_action_sheet.dart | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mobile/lib/ui/viewer/people/add_person_action_sheet.dart b/mobile/lib/ui/viewer/people/add_person_action_sheet.dart index a0f9f76a70..8cc483a834 100644 --- a/mobile/lib/ui/viewer/people/add_person_action_sheet.dart +++ b/mobile/lib/ui/viewer/people/add_person_action_sheet.dart @@ -320,7 +320,8 @@ class _PersonActionSheetState extends State { final files = clustersToFiles.values.expand((e) => e).toList(); if (files.isEmpty) { debugPrint( - "Person ${kDebugMode ? person.data.name : person.remoteID} has no files"); + "Person ${kDebugMode ? person.data.name : person.remoteID} has no files", + ); continue; } personAndFileID.add((person, files.first)); From dd83edf0e3e7bd13f09381a88cf62201cb4e6a39 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Tue, 28 May 2024 12:33:33 +0530 Subject: [PATCH 081/354] [mob] Use same flutter version in all workflows --- .github/workflows/mobile-internal-release.yml | 2 +- .github/workflows/mobile-lint.yml | 2 +- .github/workflows/mobile-release.yml | 2 +- mobile/README.md | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/mobile-internal-release.yml b/.github/workflows/mobile-internal-release.yml index fac4eb1d2f..7b0696c1bd 100644 --- a/.github/workflows/mobile-internal-release.yml +++ b/.github/workflows/mobile-internal-release.yml @@ -4,7 +4,7 @@ on: workflow_dispatch: # Allow manually running the action env: - FLUTTER_VERSION: "3.22.0" + FLUTTER_VERSION: "3.19.4" jobs: build: diff --git a/.github/workflows/mobile-lint.yml b/.github/workflows/mobile-lint.yml index 493185b6bd..3efebf4f2d 100644 --- a/.github/workflows/mobile-lint.yml +++ b/.github/workflows/mobile-lint.yml @@ -9,7 +9,7 @@ on: - ".github/workflows/mobile-lint.yml" env: - FLUTTER_VERSION: "3.19.5" + FLUTTER_VERSION: "3.19.4" jobs: lint: diff --git a/.github/workflows/mobile-release.yml b/.github/workflows/mobile-release.yml index 6211f2c262..146370010b 100644 --- a/.github/workflows/mobile-release.yml +++ b/.github/workflows/mobile-release.yml @@ -9,7 +9,7 @@ on: - "photos-v*" env: - FLUTTER_VERSION: "3.19.3" + FLUTTER_VERSION: "3.19.4" jobs: build: diff --git a/mobile/README.md b/mobile/README.md index fc17f6b26e..4a4579adfc 100644 --- a/mobile/README.md +++ b/mobile/README.md @@ -46,7 +46,7 @@ You can alternatively install the build from PlayStore or F-Droid. ## 🧑‍💻 Building from source -1. [Install Flutter v3.19.3](https://flutter.dev/docs/get-started/install). +1. [Install Flutter v3.19.4](https://flutter.dev/docs/get-started/install). 2. Pull in all submodules with `git submodule update --init --recursive` From b45dfa9cfc58474e5fb0c96de6fff4bafc701f9e Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Tue, 28 May 2024 12:58:10 +0530 Subject: [PATCH 082/354] [mob][photos] Show error on UI in debugMode --- .../ui/viewer/people/add_person_action_sheet.dart | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/mobile/lib/ui/viewer/people/add_person_action_sheet.dart b/mobile/lib/ui/viewer/people/add_person_action_sheet.dart index a0f9f76a70..83a19ac316 100644 --- a/mobile/lib/ui/viewer/people/add_person_action_sheet.dart +++ b/mobile/lib/ui/viewer/people/add_person_action_sheet.dart @@ -188,7 +188,16 @@ class _PersonActionSheetState extends State { if (snapshot.hasError) { log("Error: ${snapshot.error} ${snapshot.stackTrace}}"); //Need to show an error on the UI here - return const SizedBox.shrink(); + if (kDebugMode) { + return Column( + children: [ + Text('${snapshot.error}'), + Text('${snapshot.stackTrace}'), + ], + ); + } else { + return const SizedBox.shrink(); + } } else if (snapshot.hasData) { final persons = snapshot.data!; final searchResults = _searchQuery.isNotEmpty @@ -320,7 +329,8 @@ class _PersonActionSheetState extends State { final files = clustersToFiles.values.expand((e) => e).toList(); if (files.isEmpty) { debugPrint( - "Person ${kDebugMode ? person.data.name : person.remoteID} has no files"); + "Person ${kDebugMode ? person.data.name : person.remoteID} has no files", + ); continue; } personAndFileID.add((person, files.first)); From 1535f6165327ad881b4b345a74576d45b6d110be Mon Sep 17 00:00:00 2001 From: ashilkn Date: Tue, 28 May 2024 14:01:12 +0530 Subject: [PATCH 083/354] [mob][photos] upgrade to flutter 3.22.0 --- .github/workflows/mobile-lint.yml | 3 +- mobile/README.md | 2 +- mobile/ios/Podfile.lock | 2 +- mobile/pubspec.lock | 54 +++++++++++++++---------------- mobile/pubspec.yaml | 13 ++++---- 5 files changed, 38 insertions(+), 36 deletions(-) diff --git a/.github/workflows/mobile-lint.yml b/.github/workflows/mobile-lint.yml index 3efebf4f2d..8f231079f3 100644 --- a/.github/workflows/mobile-lint.yml +++ b/.github/workflows/mobile-lint.yml @@ -9,7 +9,8 @@ on: - ".github/workflows/mobile-lint.yml" env: - FLUTTER_VERSION: "3.19.4" + + FLUTTER_VERSION: "3.22.0" jobs: lint: diff --git a/mobile/README.md b/mobile/README.md index 4a4579adfc..6d86ad5344 100644 --- a/mobile/README.md +++ b/mobile/README.md @@ -46,7 +46,7 @@ You can alternatively install the build from PlayStore or F-Droid. ## 🧑‍💻 Building from source -1. [Install Flutter v3.19.4](https://flutter.dev/docs/get-started/install). +1. [Install Flutter v3.22.0](https://flutter.dev/docs/get-started/install). 2. Pull in all submodules with `git submodule update --init --recursive` diff --git a/mobile/ios/Podfile.lock b/mobile/ios/Podfile.lock index 558a279108..9f74d552a7 100644 --- a/mobile/ios/Podfile.lock +++ b/mobile/ios/Podfile.lock @@ -427,7 +427,7 @@ SPEC CHECKSUMS: home_widget: 0434835a4c9a75704264feff6be17ea40e0f0d57 image_editor_common: d6f6644ae4a6de80481e89fe6d0a8c49e30b4b43 in_app_purchase_storekit: 0e4b3c2e43ba1e1281f4f46dd71b0593ce529892 - integration_test: 13825b8a9334a850581300559b8839134b124670 + integration_test: ce0a3ffa1de96d1a89ca0ac26fca7ea18a749ef4 libwebp: 1786c9f4ff8a279e4dac1e8f385004d5fc253009 local_auth_darwin: c7e464000a6a89e952235699e32b329457608d98 local_auth_ios: 5046a18c018dd973247a0564496c8898dbb5adf9 diff --git a/mobile/pubspec.lock b/mobile/pubspec.lock index 1d1082bfd3..8b71025e9f 100644 --- a/mobile/pubspec.lock +++ b/mobile/pubspec.lock @@ -45,10 +45,10 @@ packages: dependency: "direct main" description: name: animated_list_plus - sha256: fe66f9c300d715254727fbdf050487844d17b013fec344fa28081d29bddbdf1a + sha256: fb3d7f1fbaf5af84907f3c739236bacda8bf32cbe1f118dd51510752883ff50c url: "https://pub.dev" source: hosted - version: "0.4.5" + version: "0.5.2" animated_stack_widget: dependency: transitive description: @@ -971,10 +971,10 @@ packages: dependency: "direct main" description: name: home_widget - sha256: "29565bfee4b32eaf9e7e8b998d504618b779a74b2b1ac62dd4dac7468e66f1a3" + sha256: "2a0fdd6267ff975bd07bedf74686bd5577200f504f5de36527ac1b56bdbe68e3" url: "https://pub.dev" source: hosted - version: "0.5.0" + version: "0.6.0" html: dependency: transitive description: @@ -1152,26 +1152,26 @@ packages: dependency: transitive description: name: leak_tracker - sha256: "78eb209deea09858f5269f5a5b02be4049535f568c07b275096836f01ea323fa" + sha256: "7f0df31977cb2c0b88585095d168e689669a2cc9b97c309665e3386f3e9d341a" url: "https://pub.dev" source: hosted - version: "10.0.0" + version: "10.0.4" leak_tracker_flutter_testing: dependency: transitive description: name: leak_tracker_flutter_testing - sha256: b46c5e37c19120a8a01918cfaf293547f47269f7cb4b0058f21531c2465d6ef0 + sha256: "06e98f569d004c1315b991ded39924b21af84cf14cc94791b8aea337d25b57f8" url: "https://pub.dev" source: hosted - version: "2.0.1" + version: "3.0.3" leak_tracker_testing: dependency: transitive description: name: leak_tracker_testing - sha256: a597f72a664dbd293f3bfc51f9ba69816f84dcd403cdac7066cb3f6003f3ab47 + sha256: "6ba465d5d76e67ddf503e1161d1f4a6bc42306f9d66ca1e8f079a47290fb06d3" url: "https://pub.dev" source: hosted - version: "2.0.1" + version: "3.0.1" like_button: dependency: "direct main" description: @@ -1368,10 +1368,10 @@ packages: dependency: transitive description: name: meta - sha256: d584fa6707a52763a52446f02cc621b077888fb63b93bbcb1143a7be5a0c0c04 + sha256: "7687075e408b093f36e6bbf6c91878cc0d4cd10f409506f7bc996f68220b9136" url: "https://pub.dev" source: hosted - version: "1.11.0" + version: "1.12.0" mgrs_dart: dependency: transitive description: @@ -2144,10 +2144,10 @@ packages: dependency: "direct main" description: name: styled_text - sha256: f72928d1ebe8cb149e3b34a689cb1ddca696b808187cf40ac3a0bd183dff379c + sha256: fd624172cf629751b4f171dd0ecf9acf02a06df3f8a81bb56c0caa4f1df706c3 url: "https://pub.dev" source: hosted - version: "7.0.0" + version: "8.1.0" sync_http: dependency: transitive description: @@ -2160,18 +2160,18 @@ packages: dependency: "direct main" description: name: syncfusion_flutter_core - sha256: "9be1bb9bbdb42823439a18da71484f1964c14dbe1c255ab1b931932b12fa96e8" + sha256: "63108a33f9b0d89f7b6b56cce908b8e519fe433dbbe0efcf41ad3e8bb2081bd9" url: "https://pub.dev" source: hosted - version: "19.4.56" + version: "25.2.5" syncfusion_flutter_sliders: dependency: "direct main" description: name: syncfusion_flutter_sliders - sha256: "1f6a63ccab4180b544074b9264a20f01ee80b553de154192fe1d7b434089d3c2" + sha256: f27310bedc0e96e84054f0a70ac593d1a3c38397c158c5226ba86027ad77b2c1 url: "https://pub.dev" source: hosted - version: "19.4.56" + version: "25.2.5" synchronized: dependency: "direct main" description: @@ -2192,26 +2192,26 @@ packages: dependency: "direct dev" description: name: test - sha256: a1f7595805820fcc05e5c52e3a231aedd0b72972cb333e8c738a8b1239448b6f + sha256: "7ee446762c2c50b3bd4ea96fe13ffac69919352bd3b4b17bac3f3465edc58073" url: "https://pub.dev" source: hosted - version: "1.24.9" + version: "1.25.2" test_api: dependency: transitive description: name: test_api - sha256: "5c2f730018264d276c20e4f1503fd1308dfbbae39ec8ee63c5236311ac06954b" + sha256: "9955ae474176f7ac8ee4e989dadfb411a58c30415bcfb648fa04b2b8a03afa7f" url: "https://pub.dev" source: hosted - version: "0.6.1" + version: "0.7.0" test_core: dependency: transitive description: name: test_core - sha256: a757b14fc47507060a162cc2530d9a4a2f92f5100a952c7443b5cad5ef5b106a + sha256: "2bc4b4ecddd75309300d8096f781c0e3280ca1ef85beda558d33fcbedc2eead4" url: "https://pub.dev" source: hosted - version: "0.5.9" + version: "0.6.0" timezone: dependency: transitive description: @@ -2441,10 +2441,10 @@ packages: dependency: transitive description: name: vm_service - sha256: b3d56ff4341b8f182b96aceb2fa20e3dcb336b9f867bc0eafc0de10f1048e957 + sha256: "3923c89304b715fb1eb6423f017651664a03bf5f4b29983627c4da791f74a4ec" url: "https://pub.dev" source: hosted - version: "13.0.0" + version: "14.2.1" volume_controller: dependency: transitive description: @@ -2591,4 +2591,4 @@ packages: version: "3.1.2" sdks: dart: ">=3.3.0 <4.0.0" - flutter: ">=3.19.0" + flutter: ">=3.20.0-1.2.pre" diff --git a/mobile/pubspec.yaml b/mobile/pubspec.yaml index 934a99a8f0..ed3bf47193 100644 --- a/mobile/pubspec.yaml +++ b/mobile/pubspec.yaml @@ -21,7 +21,7 @@ environment: dependencies: adaptive_theme: ^3.1.0 animate_do: ^2.0.0 - animated_list_plus: ^0.4.5 + animated_list_plus: ^0.5.2 archive: ^3.1.2 background_fetch: ^1.2.1 battery_info: ^1.1.1 @@ -93,13 +93,13 @@ dependencies: fluttertoast: ^8.0.6 freezed_annotation: ^2.4.1 google_nav_bar: ^5.0.5 - home_widget: ^0.5.0 + home_widget: ^0.6.0 html_unescape: ^2.0.0 http: ^1.1.0 image: ^4.0.17 image_editor: ^1.3.0 in_app_purchase: ^3.0.7 - intl: ^0.18.0 + intl: ^0.19.0 json_annotation: ^4.8.0 latlong2: ^0.9.0 like_button: ^2.0.5 @@ -152,9 +152,9 @@ dependencies: sqlite3_flutter_libs: ^0.5.20 sqlite_async: ^0.6.1 step_progress_indicator: ^1.0.2 - styled_text: ^7.0.0 - syncfusion_flutter_core: ^19.2.49 - syncfusion_flutter_sliders: ^19.2.49 + styled_text: ^8.1.0 + syncfusion_flutter_core: ^25.2.5 + syncfusion_flutter_sliders: ^25.2.5 synchronized: ^3.1.0 tuple: ^2.0.0 uni_links: ^0.5.1 @@ -177,6 +177,7 @@ dependency_overrides: # Remove this after removing dependency from flutter_sodium. # Newer flutter packages depends on ffi > 2.0.0 while flutter_sodium depends on ffi < 2.0.0 ffi: 2.1.0 + intl: 0.18.1 video_player: git: url: https://github.com/ente-io/packages.git From 284bca782e737c9dc39ef419243e8a5e9b0e159a Mon Sep 17 00:00:00 2001 From: ashilkn Date: Tue, 28 May 2024 14:01:58 +0530 Subject: [PATCH 084/354] [mob][photos] Update flutter version in internal release workflow --- .github/workflows/mobile-internal-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/mobile-internal-release.yml b/.github/workflows/mobile-internal-release.yml index 7b0696c1bd..fac4eb1d2f 100644 --- a/.github/workflows/mobile-internal-release.yml +++ b/.github/workflows/mobile-internal-release.yml @@ -4,7 +4,7 @@ on: workflow_dispatch: # Allow manually running the action env: - FLUTTER_VERSION: "3.19.4" + FLUTTER_VERSION: "3.22.0" jobs: build: From 9a8c4d9cfd99ac128d007599258ccb59e847a1b3 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Tue, 28 May 2024 15:06:22 +0530 Subject: [PATCH 085/354] [mob][photos] Calculate cosine distance inline --- .../face_clustering/cosine_distance.dart | 8 +-- .../face_clustering_service.dart | 22 ++++---- .../face_ml/feedback/cluster_feedback.dart | 17 +++--- .../lib/ui/viewer/people/cluster_app_bar.dart | 55 ------------------- 4 files changed, 21 insertions(+), 81 deletions(-) diff --git a/mobile/lib/services/machine_learning/face_ml/face_clustering/cosine_distance.dart b/mobile/lib/services/machine_learning/face_ml/face_clustering/cosine_distance.dart index 0611a1d838..c081ef4520 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_clustering/cosine_distance.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_clustering/cosine_distance.dart @@ -5,17 +5,15 @@ import "package:ml_linalg/linalg.dart"; /// Calculates the cosine distance between two embeddings/vectors using SIMD from ml_linalg /// /// WARNING: This assumes both vectors are already normalized! +/// WARNING: For even more performance, consider calculating the logic below inline! +@pragma("vm:prefer-inline") double cosineDistanceSIMD(Vector vector1, Vector vector2) { - if (vector1.length != vector2.length) { - throw ArgumentError('Vectors must be the same length'); - } - return 1 - vector1.dot(vector2); } /// Calculates the cosine distance between two embeddings/vectors using SIMD from ml_linalg /// -/// WARNING: Only use when you're not sure if vectors are normalized. If you're sure they are, use [cosineDistanceSIMD] instead for better performance. +/// WARNING: Only use when you're not sure if vectors are normalized. If you're sure they are, use [cosineDistanceSIMD] instead for better performance, or inline for best performance. double cosineDistanceSIMDSafe(Vector vector1, Vector vector2) { if (vector1.length != vector2.length) { throw ArgumentError('Vectors must be the same length'); diff --git a/mobile/lib/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart b/mobile/lib/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart index 1a635b0f07..8407698640 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart @@ -565,10 +565,10 @@ class FaceClusteringService { for (int j = i - 1; j >= 0; j--) { late double distance; if (sortedFaceInfos[i].vEmbedding != null) { - distance = cosineDistanceSIMD( - sortedFaceInfos[i].vEmbedding!, - sortedFaceInfos[j].vEmbedding!, - ); + distance = 1 - + sortedFaceInfos[i] + .vEmbedding! + .dot(sortedFaceInfos[j].vEmbedding!); } else { distance = cosineDistForNormVectors( sortedFaceInfos[i].embedding!, @@ -814,10 +814,8 @@ class FaceClusteringService { double closestDistance = double.infinity; for (int j = 0; j < totalFaces; j++) { if (i == j) continue; - final double distance = cosineDistanceSIMD( - faceInfos[i].vEmbedding!, - faceInfos[j].vEmbedding!, - ); + final double distance = + 1 - faceInfos[i].vEmbedding!.dot(faceInfos[j].vEmbedding!); if (distance < closestDistance) { closestDistance = distance; closestIdx = j; @@ -870,10 +868,10 @@ class FaceClusteringService { for (int i = 0; i < clusterIds.length; i++) { for (int j = 0; j < clusterIds.length; j++) { if (i == j) continue; - final double newDistance = cosineDistanceSIMD( - clusterIdToMeanEmbeddingAndWeight[clusterIds[i]]!.$1, - clusterIdToMeanEmbeddingAndWeight[clusterIds[j]]!.$1, - ); + final double newDistance = 1 - + clusterIdToMeanEmbeddingAndWeight[clusterIds[i]]! + .$1 + .dot(clusterIdToMeanEmbeddingAndWeight[clusterIds[j]]!.$1); if (newDistance < distance) { distance = newDistance; clusterIDsToMerge = (clusterIds[i], clusterIds[j]); diff --git a/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart b/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart index 8567e88685..b354719f9a 100644 --- a/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart +++ b/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart @@ -13,7 +13,6 @@ import "package:photos/face/db.dart"; import "package:photos/face/model/person.dart"; import "package:photos/generated/protos/ente/common/vector.pb.dart"; import "package:photos/models/file/file.dart"; -import "package:photos/services/machine_learning/face_ml/face_clustering/cosine_distance.dart"; import "package:photos/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart"; import "package:photos/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart"; import "package:photos/services/machine_learning/face_ml/face_ml_result.dart"; @@ -434,7 +433,9 @@ class ClusterFeedbackService { distanceThreshold: 0.22, ); - if (clusterResult == null || clusterResult.newClusterIdToFaceIds == null || clusterResult.isEmpty) { + if (clusterResult == null || + clusterResult.newClusterIdToFaceIds == null || + clusterResult.isEmpty) { _logger.warning('No clusters found or something went wrong'); return ClusteringResult(newFaceIdToCluster: {}); } @@ -537,8 +538,7 @@ class ClusterFeedbackService { EVector.fromBuffer(clusterSummary.$1).values, dtype: DType.float32, ); - final bigClustersMeanDistance = - cosineDistanceSIMD(biggestMean, currentMean); + final bigClustersMeanDistance = 1 - biggestMean.dot(currentMean); _logger.info( "Mean distance between biggest cluster and current cluster: $bigClustersMeanDistance", ); @@ -595,8 +595,7 @@ class ClusterFeedbackService { final List trueDistances = []; for (final biggestEmbedding in biggestSampledEmbeddings) { for (final currentEmbedding in currentSampledEmbeddings) { - distances - .add(cosineDistanceSIMD(biggestEmbedding, currentEmbedding)); + distances.add(1 - biggestEmbedding.dot(currentEmbedding)); trueDistances.add( biggestEmbedding.distanceTo( currentEmbedding, @@ -789,7 +788,7 @@ class ClusterFeedbackService { final List distances = []; for (final otherEmbedding in sampledOtherEmbeddings) { for (final embedding in sampledEmbeddings) { - distances.add(cosineDistanceSIMD(embedding, otherEmbedding)); + distances.add(1 - embedding.dot(otherEmbedding)); } } distances.sort(); @@ -1086,7 +1085,7 @@ class ClusterFeedbackService { final fileIdToDistanceMap = {}; for (final entry in faceIdToVectorMap.entries) { fileIdToDistanceMap[getFileIdFromFaceId(entry.key)] = - cosineDistanceSIMD(personAvg, entry.value); + 1 - personAvg.dot(entry.value); } w?.log('calculated distances for cluster $clusterID'); suggestion.filesInCluster.sort((b, a) { @@ -1141,7 +1140,7 @@ List<(int, double)> _calcSuggestionsMean(Map args) { continue; } final Vector avg = clusterAvg[personCluster]!; - final distance = cosineDistanceSIMD(avg, otherAvg); + final distance = 1 - avg.dot(otherAvg); comparisons++; if (distance < maxClusterDistance) { if (minDistance == null || distance < minDistance) { diff --git a/mobile/lib/ui/viewer/people/cluster_app_bar.dart b/mobile/lib/ui/viewer/people/cluster_app_bar.dart index 83ebe54284..8b95d4247e 100644 --- a/mobile/lib/ui/viewer/people/cluster_app_bar.dart +++ b/mobile/lib/ui/viewer/people/cluster_app_bar.dart @@ -3,7 +3,6 @@ import 'dart:async'; import "package:flutter/foundation.dart"; import 'package:flutter/material.dart'; import 'package:logging/logging.dart'; -import "package:ml_linalg/linalg.dart"; import 'package:photos/core/configuration.dart'; import 'package:photos/core/event_bus.dart'; import "package:photos/db/files_db.dart"; @@ -11,12 +10,10 @@ import "package:photos/events/people_changed_event.dart"; import 'package:photos/events/subscription_purchased_event.dart'; import "package:photos/face/db.dart"; import "package:photos/face/model/person.dart"; -import "package:photos/generated/protos/ente/common/vector.pb.dart"; import "package:photos/models/file/file.dart"; import 'package:photos/models/gallery_type.dart'; import 'package:photos/models/selected_files.dart'; import 'package:photos/services/collections_service.dart'; -import "package:photos/services/machine_learning/face_ml/face_clustering/cosine_distance.dart"; import "package:photos/services/machine_learning/face_ml/face_ml_result.dart"; import "package:photos/services/machine_learning/face_ml/feedback/cluster_feedback.dart"; import 'package:photos/ui/actions/collection/collection_sharing_actions.dart'; @@ -160,58 +157,6 @@ class _AppBarWidgetState extends State { return actions; } - @Deprecated( - 'Used for debugging an issue with conflicts on cluster IDs, resolved now', - ) - Future _validateCluster(BuildContext context) async { - _logger.info('_validateCluster called'); - final faceMlDb = FaceMLDataDB.instance; - - final faceIDs = await faceMlDb.getFaceIDsForCluster(widget.clusterID); - final fileIDs = faceIDs.map((e) => getFileIdFromFaceId(e)).toList(); - - final embeddingsBlobs = await faceMlDb.getFaceEmbeddingMapForFile(fileIDs); - embeddingsBlobs.removeWhere((key, value) => !faceIDs.contains(key)); - final embeddings = embeddingsBlobs - .map((key, value) => MapEntry(key, EVector.fromBuffer(value).values)); - - for (final MapEntry> embedding in embeddings.entries) { - double closestDistance = double.infinity; - double closestDistance32 = double.infinity; - double closestDistance64 = double.infinity; - String? closestFaceID; - for (final MapEntry> otherEmbedding - in embeddings.entries) { - if (embedding.key == otherEmbedding.key) { - continue; - } - final distance64 = cosineDistanceSIMD( - Vector.fromList(embedding.value, dtype: DType.float64), - Vector.fromList(otherEmbedding.value, dtype: DType.float64), - ); - final distance32 = cosineDistanceSIMD( - Vector.fromList(embedding.value, dtype: DType.float32), - Vector.fromList(otherEmbedding.value, dtype: DType.float32), - ); - final distance = cosineDistForNormVectors( - embedding.value, - otherEmbedding.value, - ); - if (distance < closestDistance) { - closestDistance = distance; - closestDistance32 = distance32; - closestDistance64 = distance64; - closestFaceID = otherEmbedding.key; - } - } - if (closestDistance > 0.3) { - _logger.severe( - "Face ${embedding.key} is similar to $closestFaceID with distance $closestDistance, and float32 distance $closestDistance32, and float64 distance $closestDistance64", - ); - } - } - } - Future _onIgnoredClusterClicked(BuildContext context) async { await showChoiceDialog( context, From 89a47026d95eb8bc556cd720fe3ec76f38ce1689 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Tue, 28 May 2024 15:18:44 +0530 Subject: [PATCH 086/354] [mob][photos] Clustering cleanup --- .../face_clustering_service.dart | 120 +----------------- 1 file changed, 2 insertions(+), 118 deletions(-) diff --git a/mobile/lib/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart b/mobile/lib/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart index 8407698640..2a35cd0dac 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart @@ -10,11 +10,9 @@ import "package:logging/logging.dart"; import "package:ml_linalg/dtype.dart"; import "package:ml_linalg/vector.dart"; import "package:photos/generated/protos/ente/common/vector.pb.dart"; -import 'package:photos/services/machine_learning/face_ml/face_clustering/cosine_distance.dart'; import "package:photos/services/machine_learning/face_ml/face_clustering/face_info_for_clustering.dart"; import "package:photos/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart"; import "package:photos/services/machine_learning/face_ml/face_ml_result.dart"; -import "package:simple_cluster/simple_cluster.dart"; import "package:synchronized/synchronized.dart"; class FaceInfo { @@ -22,7 +20,6 @@ class FaceInfo { final double? faceScore; final double? blurValue; final bool? badFace; - final List? embedding; final Vector? vEmbedding; int? clusterId; String? closestFaceId; @@ -33,14 +30,13 @@ class FaceInfo { this.faceScore, this.blurValue, this.badFace, - this.embedding, this.vEmbedding, this.clusterId, this.fileCreationTime, }); } -enum ClusterOperation { linearIncrementalClustering, dbscanClustering } +enum ClusterOperation { linearIncrementalClustering } class ClusteringResult { final Map newFaceIdToCluster; @@ -129,10 +125,6 @@ class FaceClusteringService { final result = FaceClusteringService.runLinearClustering(args); sendPort.send(result); break; - case ClusterOperation.dbscanClustering: - final result = FaceClusteringService._runDbscanClustering(args); - sendPort.send(result); - break; } } catch (e, stackTrace) { sendPort @@ -203,8 +195,6 @@ class FaceClusteringService { /// Runs the clustering algorithm [runLinearClustering] on the given [input], in an isolate. /// /// Returns the clustering result, which is a list of clusters, where each cluster is a list of indices of the dataset. - /// - /// WARNING: Make sure to always input data in the same ordering, otherwise the clustering can less less deterministic. Future predictLinear( Set input, { Map? fileIDToCreationTime, @@ -401,55 +391,6 @@ class FaceClusteringService { } } - Future>> predictDbscan( - Map input, { - Map? fileIDToCreationTime, - double eps = 0.3, - int minPts = 5, - }) async { - if (input.isEmpty) { - _logger.warning( - "DBSCAN Clustering dataset of embeddings is empty, returning empty list.", - ); - return []; - } - if (isRunning) { - _logger.warning( - "DBSCAN Clustering is already running, returning empty list.", - ); - return []; - } - - isRunning = true; - - // Clustering inside the isolate - _logger.info( - "Start DBSCAN clustering on ${input.length} embeddings inside computer isolate", - ); - final stopwatchClustering = Stopwatch()..start(); - // final Map faceIdToCluster = - // await _runLinearClusteringInComputer(input); - final List> clusterFaceIDs = await _runInIsolate( - ( - ClusterOperation.dbscanClustering, - { - 'input': input, - 'fileIDToCreationTime': fileIDToCreationTime, - 'eps': eps, - 'minPts': minPts, - } - ), - ); - // return _runLinearClusteringInComputer(input); - _logger.info( - 'DBSCAN Clustering executed in ${stopwatchClustering.elapsed.inSeconds} seconds', - ); - - isRunning = false; - - return clusterFaceIDs; - } - static ClusteringResult? runLinearClustering(Map args) { // final input = args['input'] as Map; final input = args['input'] as Set; @@ -563,18 +504,10 @@ class FaceClusteringService { log("[ClusterIsolate] ${DateTime.now()} Processed ${offset != null ? i + offset : i} faces"); } for (int j = i - 1; j >= 0; j--) { - late double distance; - if (sortedFaceInfos[i].vEmbedding != null) { - distance = 1 - + final double distance = 1 - sortedFaceInfos[i] .vEmbedding! .dot(sortedFaceInfos[j].vEmbedding!); - } else { - distance = cosineDistForNormVectors( - sortedFaceInfos[i].embedding!, - sortedFaceInfos[j].embedding!, - ); - } if (distance < closestDistance) { if (sortedFaceInfos[j].badFace! && distance > conservativeDistanceThreshold) { @@ -942,55 +875,6 @@ class FaceClusteringService { newClusterIdToFaceIds: clusterIdToFaceIds, ); } - - static List> _runDbscanClustering(Map args) { - final input = args['input'] as Map; - final fileIDToCreationTime = args['fileIDToCreationTime'] as Map?; - final eps = args['eps'] as double; - final minPts = args['minPts'] as int; - - log( - "[ClusterIsolate] ${DateTime.now()} Copied to isolate ${input.length} faces", - ); - - final DBSCAN dbscan = DBSCAN( - epsilon: eps, - minPoints: minPts, - distanceMeasure: cosineDistForNormVectors, - ); - - // Organize everything into a list of FaceInfo objects - final List faceInfos = []; - for (final entry in input.entries) { - faceInfos.add( - FaceInfo( - faceID: entry.key, - embedding: EVector.fromBuffer(entry.value).values, - fileCreationTime: - fileIDToCreationTime?[getFileIdFromFaceId(entry.key)], - ), - ); - } - - if (fileIDToCreationTime != null) { - _sortFaceInfosOnCreationTime(faceInfos); - } - - // Get the embeddings - final List> embeddings = - faceInfos.map((faceInfo) => faceInfo.embedding!).toList(); - - // Run the DBSCAN clustering - final List> clusterOutput = dbscan.run(embeddings); - // final List> clusteredFaceInfos = clusterOutput - // .map((cluster) => cluster.map((idx) => faceInfos[idx]).toList()) - // .toList(); - final List> clusteredFaceIDs = clusterOutput - .map((cluster) => cluster.map((idx) => faceInfos[idx].faceID).toList()) - .toList(); - - return clusteredFaceIDs; - } } /// Sort the faceInfos based on fileCreationTime, in descending order, so newest faces are first From 50968fd6a112be0d9942a3241f15bfab1c0e7a85 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Tue, 28 May 2024 15:20:44 +0530 Subject: [PATCH 087/354] [mob][photos] Comment --- .../face_ml/face_clustering/face_clustering_service.dart | 1 + 1 file changed, 1 insertion(+) diff --git a/mobile/lib/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart b/mobile/lib/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart index 2a35cd0dac..f826cc7855 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart @@ -503,6 +503,7 @@ class FaceClusteringService { if (i % 250 == 0) { log("[ClusterIsolate] ${DateTime.now()} Processed ${offset != null ? i + offset : i} faces"); } + // WARNING: The loop below is now O(n^2) so be very careful with anything you put in there! for (int j = i - 1; j >= 0; j--) { final double distance = 1 - sortedFaceInfos[i] From b64077d5e7a651a23fa817eaa7e42ba06c40ab18 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Tue, 28 May 2024 15:41:20 +0530 Subject: [PATCH 088/354] [mob][photos] Skip cluster bucket if everything already has a clusterID --- .../face_clustering/face_clustering_service.dart | 4 +--- .../machine_learning/face_ml/face_ml_service.dart | 12 ++++++++++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/mobile/lib/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart b/mobile/lib/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart index f826cc7855..310deb964a 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart @@ -506,9 +506,7 @@ class FaceClusteringService { // WARNING: The loop below is now O(n^2) so be very careful with anything you put in there! for (int j = i - 1; j >= 0; j--) { final double distance = 1 - - sortedFaceInfos[i] - .vEmbedding! - .dot(sortedFaceInfos[j].vEmbedding!); + sortedFaceInfos[i].vEmbedding!.dot(sortedFaceInfos[j].vEmbedding!); if (distance < closestDistance) { if (sortedFaceInfos[j].badFace! && distance > conservativeDistanceThreshold) { diff --git a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart index a8d13297d8..b9ec603e2a 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart @@ -651,6 +651,18 @@ class FaceMlService { min(offset + bucketSize, allFaceInfoForClustering.length), ); + if (faceInfoForClustering.every((face) => face.clusterId != null)) { + _logger.info('Everything in bucket $bucket is already clustered'); + if (offset + bucketSize >= totalFaces) { + _logger.info('All faces clustered'); + break; + } else { + _logger.info('Skipping to next bucket'); + offset += offsetIncrement; + bucket++; + } + } + final clusteringResult = await FaceClusteringService.instance.predictLinear( faceInfoForClustering.toSet(), From 8975af7a716429663b8f78ce911f124065934462 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Tue, 28 May 2024 15:47:46 +0530 Subject: [PATCH 089/354] [mob][photos] Dont forget to continue --- .../lib/services/machine_learning/face_ml/face_ml_service.dart | 1 + 1 file changed, 1 insertion(+) diff --git a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart index b9ec603e2a..1297f4cac0 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart @@ -660,6 +660,7 @@ class FaceMlService { _logger.info('Skipping to next bucket'); offset += offsetIncrement; bucket++; + continue; } } From 77f3503a0bcee2bccc1467f8678fdd83798a241c Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 28 May 2024 12:31:03 +0530 Subject: [PATCH 090/354] Make space --- .../components/Search/SearchBar/searchInput/MenuWithPeople.tsx | 2 +- web/apps/photos/src/components/ml/PeopleList.tsx | 2 +- web/apps/photos/src/services/face/{db.ts => db-old.ts} | 0 .../src/services/machineLearning/machineLearningService.ts | 2 +- web/apps/photos/src/services/machineLearning/mlWorkManager.ts | 2 +- web/apps/photos/src/services/searchService.ts | 2 +- web/apps/photos/src/types/search/index.ts | 2 +- 7 files changed, 6 insertions(+), 6 deletions(-) rename web/apps/photos/src/services/face/{db.ts => db-old.ts} (100%) diff --git a/web/apps/photos/src/components/Search/SearchBar/searchInput/MenuWithPeople.tsx b/web/apps/photos/src/components/Search/SearchBar/searchInput/MenuWithPeople.tsx index 3b739520e2..aaca1c3906 100644 --- a/web/apps/photos/src/components/Search/SearchBar/searchInput/MenuWithPeople.tsx +++ b/web/apps/photos/src/components/Search/SearchBar/searchInput/MenuWithPeople.tsx @@ -5,7 +5,7 @@ import { t } from "i18next"; import { AppContext } from "pages/_app"; import { useContext } from "react"; import { components } from "react-select"; -import { IndexStatus } from "services/face/db"; +import { IndexStatus } from "services/face/db-old"; import { Suggestion, SuggestionType } from "types/search"; const { Menu } = components; diff --git a/web/apps/photos/src/components/ml/PeopleList.tsx b/web/apps/photos/src/components/ml/PeopleList.tsx index da003d97d5..c77bb38a67 100644 --- a/web/apps/photos/src/components/ml/PeopleList.tsx +++ b/web/apps/photos/src/components/ml/PeopleList.tsx @@ -4,7 +4,7 @@ import { Skeleton, styled } from "@mui/material"; import { Legend } from "components/PhotoViewer/styledComponents/Legend"; import { t } from "i18next"; import React, { useEffect, useState } from "react"; -import mlIDbStorage from "services/face/db"; +import mlIDbStorage from "services/face/db-old"; import type { Person } from "services/face/people"; import { EnteFile } from "types/file"; diff --git a/web/apps/photos/src/services/face/db.ts b/web/apps/photos/src/services/face/db-old.ts similarity index 100% rename from web/apps/photos/src/services/face/db.ts rename to web/apps/photos/src/services/face/db-old.ts diff --git a/web/apps/photos/src/services/machineLearning/machineLearningService.ts b/web/apps/photos/src/services/machineLearning/machineLearningService.ts index 954a88c66d..f871584743 100644 --- a/web/apps/photos/src/services/machineLearning/machineLearningService.ts +++ b/web/apps/photos/src/services/machineLearning/machineLearningService.ts @@ -4,7 +4,7 @@ import PQueue from "p-queue"; import mlIDbStorage, { ML_SEARCH_CONFIG_NAME, type MinimalPersistedFileData, -} from "services/face/db"; +} from "services/face/db-old"; import { putFaceEmbedding } from "services/face/remote"; import { getLocalFiles } from "services/fileService"; import { EnteFile } from "types/file"; diff --git a/web/apps/photos/src/services/machineLearning/mlWorkManager.ts b/web/apps/photos/src/services/machineLearning/mlWorkManager.ts index c1b2ef6a70..0ec5f29541 100644 --- a/web/apps/photos/src/services/machineLearning/mlWorkManager.ts +++ b/web/apps/photos/src/services/machineLearning/mlWorkManager.ts @@ -8,7 +8,7 @@ import { getToken, getUserID } from "@ente/shared/storage/localStorage/helpers"; import debounce from "debounce"; import PQueue from "p-queue"; import { createFaceComlinkWorker } from "services/face"; -import mlIDbStorage from "services/face/db"; +import mlIDbStorage from "services/face/db-old"; import type { DedicatedMLWorker } from "services/face/face.worker"; import { EnteFile } from "types/file"; diff --git a/web/apps/photos/src/services/searchService.ts b/web/apps/photos/src/services/searchService.ts index 4bbab115c3..b48778f690 100644 --- a/web/apps/photos/src/services/searchService.ts +++ b/web/apps/photos/src/services/searchService.ts @@ -2,7 +2,7 @@ import { FILE_TYPE } from "@/media/file-type"; import log from "@/next/log"; import * as chrono from "chrono-node"; import { t } from "i18next"; -import mlIDbStorage from "services/face/db"; +import mlIDbStorage from "services/face/db-old"; import type { Person } from "services/face/people"; import { defaultMLVersion } from "services/machineLearning/machineLearningService"; import { Collection } from "types/collection"; diff --git a/web/apps/photos/src/types/search/index.ts b/web/apps/photos/src/types/search/index.ts index 33f5eba9a0..3f3de9b460 100644 --- a/web/apps/photos/src/types/search/index.ts +++ b/web/apps/photos/src/types/search/index.ts @@ -1,5 +1,5 @@ import { FILE_TYPE } from "@/media/file-type"; -import { IndexStatus } from "services/face/db"; +import { IndexStatus } from "services/face/db-old"; import type { Person } from "services/face/people"; import { City } from "services/locationSearchService"; import { LocationTagData } from "types/entity"; From 8ea7a742b1ea50b227cdc88c9f5daabe53291547 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 28 May 2024 12:57:40 +0530 Subject: [PATCH 091/354] Outline --- web/apps/photos/src/services/face/db.ts | 27 +++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 web/apps/photos/src/services/face/db.ts diff --git a/web/apps/photos/src/services/face/db.ts b/web/apps/photos/src/services/face/db.ts new file mode 100644 index 0000000000..8edaaca6b8 --- /dev/null +++ b/web/apps/photos/src/services/face/db.ts @@ -0,0 +1,27 @@ +/** + * The faces in a file (and an embedding for each of them). + * + * This interface describes the format of both local and remote face data. + * + * - Local face detections and embeddings (collectively called as the face + * index) are generated by the current client when uploading a file (or when + * noticing a file which doesn't yet have a face index), stored in the local + * IndexedDB ("face/db") and also uploaded (E2EE) to remote. + * + * - Remote embeddings are fetched by subsequent clients to avoid them having to + * reindex (indexing faces is a costly operation, esp for mobile clients). + * + * In both these scenarios (whether generated locally or fetched from remote), + * we end up with an face index described by this {@link FaceIndex} interface. + * + * It has a top level envelope with information about the client (in particular + * the primary key {@link fileID}), an inner envelope {@link faceEmbedding} with + * metadata about the indexing, and an array of {@link faces} each containing + * the result of a face detection and an embedding for that detected face. + * + * This last one (faceEmbedding > faces > embedding) is the "actual" embedding, + * but sometimes we colloquially refer to the inner envelope (the + * "faceEmbedding") also an embedding since a file can have other types of + * embedding (envelopes) like a "clipEmbedding". + */ +export interface FaceIndex {} From 3664532f9164bb3dfb6af83c19c8a9df1a47c28d Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 28 May 2024 13:49:03 +0530 Subject: [PATCH 092/354] Document --- web/apps/photos/src/services/face/db.ts | 122 ++++++++++++++++++++++-- 1 file changed, 116 insertions(+), 6 deletions(-) diff --git a/web/apps/photos/src/services/face/db.ts b/web/apps/photos/src/services/face/db.ts index 8edaaca6b8..ff0a1061b9 100644 --- a/web/apps/photos/src/services/face/db.ts +++ b/web/apps/photos/src/services/face/db.ts @@ -1,3 +1,5 @@ +import type { Box, Point } from "./types"; + /** * The faces in a file (and an embedding for each of them). * @@ -14,14 +16,122 @@ * In both these scenarios (whether generated locally or fetched from remote), * we end up with an face index described by this {@link FaceIndex} interface. * - * It has a top level envelope with information about the client (in particular + * It has a top level envelope with information about the file (in particular * the primary key {@link fileID}), an inner envelope {@link faceEmbedding} with * metadata about the indexing, and an array of {@link faces} each containing * the result of a face detection and an embedding for that detected face. * - * This last one (faceEmbedding > faces > embedding) is the "actual" embedding, - * but sometimes we colloquially refer to the inner envelope (the - * "faceEmbedding") also an embedding since a file can have other types of - * embedding (envelopes) like a "clipEmbedding". + * The word embedding is used to refer two things: The last one (faceEmbedding > + * faces > embedding) is the "actual" embedding, but sometimes we colloquially + * refer to the inner envelope (the "faceEmbedding") also an embedding since a + * file can have other types of embedding (envelopes), e.g. a "clipEmbedding". */ -export interface FaceIndex {} +export interface FaceIndex { + /** + * The ID of the {@link EnteFile} whose index this is. + * + * This is used as the primary key when storing the index locally (An + * {@link EnteFile} is guaranteed to have its fileID be unique in the + * namespace of the user. Even if someone shares a file with the user the + * user will get a file entry with a fileID unique to them). + */ + fileID: number; + /** + * The width (in px) of the image (file). + */ + width: number; + /** + * The height (in px) of the image (file). + */ + height: number; + /** + * The "face embedding" for the file. + * + * This is an envelope that contains a list of indexed faces and metadata + * about the indexing. + */ + faceEmbedding: { + /** + * An integral version number of the indexing algorithm / pipeline. + * + * Clients agree out of band what a particular version means. The + * guarantee is that an embedding with a particular version will be the + * same (to negligible floating point epsilons) irrespective of the + * client that indexed the file. + */ + version: number; + /** The UA for the client which generated this embedding. */ + client: string; + /** The list of faces (and their embeddings) detected in the file. */ + faces: Face[]; + }; +} + +/** + * A face detected in a file, and an embedding for this detected face. + * + * During face indexing, we first detect all the faces in a particular file. + * Then for each such detected region, we compute an embedding of that part of + * the file. Together, this detection region and the emedding travel together in + * this {@link Face} interface. + */ +export interface Face { + /** + * A unique identifier for the face. + * + * This ID is guaranteed to be unique for all the faces detected in all the + * files for the user. In particular, each file can have multiple faces but + * they all will get their own unique {@link faceID}. + */ + faceID: string; + /** + * The face detection. Describes the region within the image that was + * detected to be a face, and a set of landmarks (e.g. "eyes") of the + * detection. + * + * All coordinates are relative within the image's dimension, i.e. they have + * been normalized to lie between 0 and 1, with 0 being the left (or top) + * and 1 being the width (or height) of the image. + */ + detection: { + /** + * The region within the image that contains the face. + * + * All coordinates and sizes are between 0 and 1, normalized by the + * dimensions of the image. + * */ + box: Box; + /** + * Face "landmarks", e.g. eyes. + * + * The exact landmarks and their order depends on the face detection + * algorithm being used. + * + * The coordinatesare between 0 and 1, normalized by the dimensions of + * the image. + */ + landmarks: Point[]; + }; + /** + * An correctness probability (0 to 1) that the face detection algorithm + * gave to the detection. Higher values are better. + */ + score: number; + /** + * The computed blur for the detected face. + * + * The exact semantics and range for these (floating point) values depend on + * the face indexing algorithm / pipeline version being used. + * */ + blur: number; + /** + * An embedding for the face. + * + * This is an opaque numeric (signed floating point) vector whose semantics + * and length depend on the version of the face indexing algorithm / + * pipeline that we are using. However, within a set of embeddings with the + * same version, the property is that two such embedding vectors will be + * "cosine similar" to each other if they are both faces of the same person. + */ + embedding: number[]; +} From 5e49b8a528c362d7faeb59c8a84246b1a4049fcf Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 28 May 2024 13:52:40 +0530 Subject: [PATCH 093/354] Move --- web/apps/photos/src/services/face/crop.ts | 2 +- web/apps/photos/src/services/face/db-old.ts | 2 +- web/apps/photos/src/services/face/db.ts | 137 ------------- web/apps/photos/src/services/face/f-index.ts | 15 +- web/apps/photos/src/services/face/remote.ts | 2 +- .../photos/src/services/face/types-old.ts | 46 +++++ web/apps/photos/src/services/face/types.ts | 181 +++++++++++++----- 7 files changed, 192 insertions(+), 193 deletions(-) delete mode 100644 web/apps/photos/src/services/face/db.ts create mode 100644 web/apps/photos/src/services/face/types-old.ts diff --git a/web/apps/photos/src/services/face/crop.ts b/web/apps/photos/src/services/face/crop.ts index 369dfc654a..d4d3753825 100644 --- a/web/apps/photos/src/services/face/crop.ts +++ b/web/apps/photos/src/services/face/crop.ts @@ -1,5 +1,5 @@ import { blobCache } from "@/next/blob-cache"; -import type { Box, Face, FaceAlignment } from "./types"; +import type { Box, Face, FaceAlignment } from "./types-old"; export const saveFaceCrop = async (imageBitmap: ImageBitmap, face: Face) => { const faceCrop = extractFaceCrop(imageBitmap, face.alignment); diff --git a/web/apps/photos/src/services/face/db-old.ts b/web/apps/photos/src/services/face/db-old.ts index 4742dd9d73..a70e94bee7 100644 --- a/web/apps/photos/src/services/face/db-old.ts +++ b/web/apps/photos/src/services/face/db-old.ts @@ -10,7 +10,7 @@ import { } from "idb"; import isElectron from "is-electron"; import type { Person } from "services/face/people"; -import type { MlFileData } from "services/face/types"; +import type { MlFileData } from "services/face/types-old"; import { DEFAULT_ML_SEARCH_CONFIG, MAX_ML_SYNC_ERROR_COUNT, diff --git a/web/apps/photos/src/services/face/db.ts b/web/apps/photos/src/services/face/db.ts deleted file mode 100644 index ff0a1061b9..0000000000 --- a/web/apps/photos/src/services/face/db.ts +++ /dev/null @@ -1,137 +0,0 @@ -import type { Box, Point } from "./types"; - -/** - * The faces in a file (and an embedding for each of them). - * - * This interface describes the format of both local and remote face data. - * - * - Local face detections and embeddings (collectively called as the face - * index) are generated by the current client when uploading a file (or when - * noticing a file which doesn't yet have a face index), stored in the local - * IndexedDB ("face/db") and also uploaded (E2EE) to remote. - * - * - Remote embeddings are fetched by subsequent clients to avoid them having to - * reindex (indexing faces is a costly operation, esp for mobile clients). - * - * In both these scenarios (whether generated locally or fetched from remote), - * we end up with an face index described by this {@link FaceIndex} interface. - * - * It has a top level envelope with information about the file (in particular - * the primary key {@link fileID}), an inner envelope {@link faceEmbedding} with - * metadata about the indexing, and an array of {@link faces} each containing - * the result of a face detection and an embedding for that detected face. - * - * The word embedding is used to refer two things: The last one (faceEmbedding > - * faces > embedding) is the "actual" embedding, but sometimes we colloquially - * refer to the inner envelope (the "faceEmbedding") also an embedding since a - * file can have other types of embedding (envelopes), e.g. a "clipEmbedding". - */ -export interface FaceIndex { - /** - * The ID of the {@link EnteFile} whose index this is. - * - * This is used as the primary key when storing the index locally (An - * {@link EnteFile} is guaranteed to have its fileID be unique in the - * namespace of the user. Even if someone shares a file with the user the - * user will get a file entry with a fileID unique to them). - */ - fileID: number; - /** - * The width (in px) of the image (file). - */ - width: number; - /** - * The height (in px) of the image (file). - */ - height: number; - /** - * The "face embedding" for the file. - * - * This is an envelope that contains a list of indexed faces and metadata - * about the indexing. - */ - faceEmbedding: { - /** - * An integral version number of the indexing algorithm / pipeline. - * - * Clients agree out of band what a particular version means. The - * guarantee is that an embedding with a particular version will be the - * same (to negligible floating point epsilons) irrespective of the - * client that indexed the file. - */ - version: number; - /** The UA for the client which generated this embedding. */ - client: string; - /** The list of faces (and their embeddings) detected in the file. */ - faces: Face[]; - }; -} - -/** - * A face detected in a file, and an embedding for this detected face. - * - * During face indexing, we first detect all the faces in a particular file. - * Then for each such detected region, we compute an embedding of that part of - * the file. Together, this detection region and the emedding travel together in - * this {@link Face} interface. - */ -export interface Face { - /** - * A unique identifier for the face. - * - * This ID is guaranteed to be unique for all the faces detected in all the - * files for the user. In particular, each file can have multiple faces but - * they all will get their own unique {@link faceID}. - */ - faceID: string; - /** - * The face detection. Describes the region within the image that was - * detected to be a face, and a set of landmarks (e.g. "eyes") of the - * detection. - * - * All coordinates are relative within the image's dimension, i.e. they have - * been normalized to lie between 0 and 1, with 0 being the left (or top) - * and 1 being the width (or height) of the image. - */ - detection: { - /** - * The region within the image that contains the face. - * - * All coordinates and sizes are between 0 and 1, normalized by the - * dimensions of the image. - * */ - box: Box; - /** - * Face "landmarks", e.g. eyes. - * - * The exact landmarks and their order depends on the face detection - * algorithm being used. - * - * The coordinatesare between 0 and 1, normalized by the dimensions of - * the image. - */ - landmarks: Point[]; - }; - /** - * An correctness probability (0 to 1) that the face detection algorithm - * gave to the detection. Higher values are better. - */ - score: number; - /** - * The computed blur for the detected face. - * - * The exact semantics and range for these (floating point) values depend on - * the face indexing algorithm / pipeline version being used. - * */ - blur: number; - /** - * An embedding for the face. - * - * This is an opaque numeric (signed floating point) vector whose semantics - * and length depend on the version of the face indexing algorithm / - * pipeline that we are using. However, within a set of embeddings with the - * same version, the property is that two such embedding vectors will be - * "cosine similar" to each other if they are both faces of the same person. - */ - embedding: number[]; -} diff --git a/web/apps/photos/src/services/face/f-index.ts b/web/apps/photos/src/services/face/f-index.ts index 5197214b24..5e93f60bd6 100644 --- a/web/apps/photos/src/services/face/f-index.ts +++ b/web/apps/photos/src/services/face/f-index.ts @@ -2,14 +2,6 @@ import { FILE_TYPE } from "@/media/file-type"; import log from "@/next/log"; import { workerBridge } from "@/next/worker/worker-bridge"; import { Matrix } from "ml-matrix"; -import type { - Box, - Dimensions, - Face, - FaceAlignment, - FaceDetection, - MlFileData, -} from "services/face/types"; import { defaultMLVersion } from "services/machineLearning/machineLearningService"; import { getSimilarityTransformation } from "similarity-transformation"; import { @@ -28,6 +20,13 @@ import { pixelRGBBilinear, warpAffineFloat32List, } from "./image"; +import type { Box, Dimensions } from "./types"; +import type { + Face, + FaceAlignment, + FaceDetection, + MlFileData, +} from "./types-old"; /** * Index faces in the given file. diff --git a/web/apps/photos/src/services/face/remote.ts b/web/apps/photos/src/services/face/remote.ts index 3c64ca30cc..32d0fddad8 100644 --- a/web/apps/photos/src/services/face/remote.ts +++ b/web/apps/photos/src/services/face/remote.ts @@ -2,7 +2,7 @@ import log from "@/next/log"; import ComlinkCryptoWorker from "@ente/shared/crypto"; import { putEmbedding } from "services/embeddingService"; import type { EnteFile } from "types/file"; -import type { Face, FaceDetection, MlFileData, Point } from "./types"; +import type { Face, FaceDetection, MlFileData, Point } from "./types-old"; export const putFaceEmbedding = async ( enteFile: EnteFile, diff --git a/web/apps/photos/src/services/face/types-old.ts b/web/apps/photos/src/services/face/types-old.ts new file mode 100644 index 0000000000..66eec9cf55 --- /dev/null +++ b/web/apps/photos/src/services/face/types-old.ts @@ -0,0 +1,46 @@ +import type { Box, Dimensions, Point } from "./types"; + +export interface FaceDetection { + // box and landmarks is relative to image dimentions stored at mlFileData + box: Box; + landmarks?: Point[]; + probability?: number; +} + +export interface FaceAlignment { + /** + * An affine transformation matrix (rotation, translation, scaling) to align + * the face extracted from the image. + */ + affineMatrix: number[][]; + /** + * The bounding box of the transformed box. + * + * The affine transformation shifts the original detection box a new, + * transformed, box (possibily rotated). This property is the bounding box + * of that transformed box. It is in the coordinate system of the original, + * full, image on which the detection occurred. + */ + boundingBox: Box; +} + +export interface Face { + fileId: number; + detection: FaceDetection; + id: string; + + alignment?: FaceAlignment; + blurValue?: number; + + embedding?: Float32Array; + + personId?: number; +} + +export interface MlFileData { + fileId: number; + faces?: Face[]; + imageDimensions?: Dimensions; + mlVersion: number; + errorCount: number; +} diff --git a/web/apps/photos/src/services/face/types.ts b/web/apps/photos/src/services/face/types.ts index 0b1b2f9757..a1db97a9af 100644 --- a/web/apps/photos/src/services/face/types.ts +++ b/web/apps/photos/src/services/face/types.ts @@ -1,3 +1,139 @@ +/** + * The faces in a file (and an embedding for each of them). + * + * This interface describes the format of both local and remote face data. + * + * - Local face detections and embeddings (collectively called as the face + * index) are generated by the current client when uploading a file (or when + * noticing a file which doesn't yet have a face index), stored in the local + * IndexedDB ("face/db") and also uploaded (E2EE) to remote. + * + * - Remote embeddings are fetched by subsequent clients to avoid them having to + * reindex (indexing faces is a costly operation, esp for mobile clients). + * + * In both these scenarios (whether generated locally or fetched from remote), + * we end up with an face index described by this {@link FaceIndex} interface. + * + * It has a top level envelope with information about the file (in particular + * the primary key {@link fileID}), an inner envelope {@link faceEmbedding} with + * metadata about the indexing, and an array of {@link faces} each containing + * the result of a face detection and an embedding for that detected face. + * + * The word embedding is used to refer two things: The last one (faceEmbedding > + * faces > embedding) is the "actual" embedding, but sometimes we colloquially + * refer to the inner envelope (the "faceEmbedding") also an embedding since a + * file can have other types of embedding (envelopes), e.g. a "clipEmbedding". + */ +export interface FaceIndex { + /** + * The ID of the {@link EnteFile} whose index this is. + * + * This is used as the primary key when storing the index locally (An + * {@link EnteFile} is guaranteed to have its fileID be unique in the + * namespace of the user. Even if someone shares a file with the user the + * user will get a file entry with a fileID unique to them). + */ + fileID: number; + /** + * The width (in px) of the image (file). + */ + width: number; + /** + * The height (in px) of the image (file). + */ + height: number; + /** + * The "face embedding" for the file. + * + * This is an envelope that contains a list of indexed faces and metadata + * about the indexing. + */ + faceEmbedding: { + /** + * An integral version number of the indexing algorithm / pipeline. + * + * Clients agree out of band what a particular version means. The + * guarantee is that an embedding with a particular version will be the + * same (to negligible floating point epsilons) irrespective of the + * client that indexed the file. + */ + version: number; + /** The UA for the client which generated this embedding. */ + client: string; + /** The list of faces (and their embeddings) detected in the file. */ + faces: Face[]; + }; +} + +/** + * A face detected in a file, and an embedding for this detected face. + * + * During face indexing, we first detect all the faces in a particular file. + * Then for each such detected region, we compute an embedding of that part of + * the file. Together, this detection region and the emedding travel together in + * this {@link Face} interface. + */ +export interface Face { + /** + * A unique identifier for the face. + * + * This ID is guaranteed to be unique for all the faces detected in all the + * files for the user. In particular, each file can have multiple faces but + * they all will get their own unique {@link faceID}. + */ + faceID: string; + /** + * The face detection. Describes the region within the image that was + * detected to be a face, and a set of landmarks (e.g. "eyes") of the + * detection. + * + * All coordinates are relative to and normalized by the image's dimension, + * i.e. they have been normalized to lie between 0 and 1, with 0 being the + * left (or top) and 1 being the width (or height) of the image. + */ + detection: { + /** + * The region within the image that contains the face. + * + * All coordinates and sizes are between 0 and 1, normalized by the + * dimensions of the image. + * */ + box: Box; + /** + * Face "landmarks", e.g. eyes. + * + * The exact landmarks and their order depends on the face detection + * algorithm being used. + * + * The coordinatesare between 0 and 1, normalized by the dimensions of + * the image. + */ + landmarks: Point[]; + }; + /** + * An correctness probability (0 to 1) that the face detection algorithm + * gave to the detection. Higher values are better. + */ + score: number; + /** + * The computed blur for the detected face. + * + * The exact semantics and range for these (floating point) values depend on + * the face indexing algorithm / pipeline version being used. + * */ + blur: number; + /** + * An embedding for the face. + * + * This is an opaque numeric (signed floating point) vector whose semantics + * and length depend on the version of the face indexing algorithm / + * pipeline that we are using. However, within a set of embeddings with the + * same version, the property is that two such embedding vectors will be + * "cosine similar" to each other if they are both faces of the same person. + */ + embedding: number[]; +} + /** The x and y coordinates of a point. */ export interface Point { x: number; @@ -21,48 +157,3 @@ export interface Box { /** The height of the box. */ height: number; } - -export interface FaceDetection { - // box and landmarks is relative to image dimentions stored at mlFileData - box: Box; - landmarks?: Point[]; - probability?: number; -} - -export interface FaceAlignment { - /** - * An affine transformation matrix (rotation, translation, scaling) to align - * the face extracted from the image. - */ - affineMatrix: number[][]; - /** - * The bounding box of the transformed box. - * - * The affine transformation shifts the original detection box a new, - * transformed, box (possibily rotated). This property is the bounding box - * of that transformed box. It is in the coordinate system of the original, - * full, image on which the detection occurred. - */ - boundingBox: Box; -} - -export interface Face { - fileId: number; - detection: FaceDetection; - id: string; - - alignment?: FaceAlignment; - blurValue?: number; - - embedding?: Float32Array; - - personId?: number; -} - -export interface MlFileData { - fileId: number; - faces?: Face[]; - imageDimensions?: Dimensions; - mlVersion: number; - errorCount: number; -} From 126727a9cc64f909df1d96ca54a31d0a3bd82119 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 28 May 2024 14:13:46 +0530 Subject: [PATCH 094/354] Document --- web/apps/photos/src/services/face/db.ts | 60 +++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 web/apps/photos/src/services/face/db.ts diff --git a/web/apps/photos/src/services/face/db.ts b/web/apps/photos/src/services/face/db.ts new file mode 100644 index 0000000000..1710f4382e --- /dev/null +++ b/web/apps/photos/src/services/face/db.ts @@ -0,0 +1,60 @@ +import type { FaceIndex } from "./types"; + +/** + * [Note: Face DB schema] + * + * There "face" database is made of two object stores: + * + * - "face-index": Contains {@link FaceIndex} objects, either indexed locally or + * fetched from remote storage. + * + * - "file-status": Contains {@link FileStatus} objects, one for each + * {@link EnteFile} that the current client knows about. + * + * Both the stores are keyed by {@link fileID}, and are expected to contain the + * exact same set of {@link fileID}s. The face-index can be thought of as the + * "original" indexing result, whilst file-status bookkeeps information about + * the indexing process (whether or not a file needs indexing, or if there were + * errors doing so). + * + * In tandem, these serve as the underlying storage for the functions exposed by + * this file. + */ + +/** + * Save the given {@link faceIndex} locally. + * + * @param faceIndex A {@link FaceIndex} representing the faces that we detected + * (and their corresponding embeddings) in some file. + * + * This function adds a new entry, overwriting any existing ones (No merging is + * performed, the existing entry is unconditionally overwritten). + */ +// eslint-disable-next-line @typescript-eslint/no-unused-vars +export const saveFaceIndex = (faceIndex: FaceIndex) => {}; + +/** + * Record the existence of a fileID so that entities in the face indexing + * universe know about it. + * + * @param fileID The ID of an {@link EnteFile}. + * + * This function does not overwrite existing entries. If an entry already exists + * for the given {@link fileID} (e.g. if it was indexed and + * {@link saveFaceIndex} called with the result), its existing status remains + * unperturbed. + */ +// eslint-disable-next-line @typescript-eslint/no-unused-vars +export const addFileEntry = (fileID: string) => {}; + +/** + * Increment the failure count associated with the given {@link fileID}. + * + * @param fileID The ID of an {@link EnteFile}. + * + * If an entry does not exist yet for the given file, then a new one is created + * and its failure count is set to 1. Otherwise the failure count of the + * existing entry is incremented. + */ +// eslint-disable-next-line @typescript-eslint/no-unused-vars +export const markIndexingFailed = (fileID: string) => {}; From f5947a0c4a72b604852780b6cc7dbb62ab3b3577 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 28 May 2024 15:04:33 +0530 Subject: [PATCH 095/354] Introduce idb --- web/apps/photos/src/services/face/db.ts | 9 ++++++++ web/docs/dependencies.md | 7 +++++- web/docs/storage.md | 30 ++++++++++++++++--------- 3 files changed, 35 insertions(+), 11 deletions(-) diff --git a/web/apps/photos/src/services/face/db.ts b/web/apps/photos/src/services/face/db.ts index 1710f4382e..e099e823e3 100644 --- a/web/apps/photos/src/services/face/db.ts +++ b/web/apps/photos/src/services/face/db.ts @@ -20,6 +20,8 @@ import type { FaceIndex } from "./types"; * In tandem, these serve as the underlying storage for the functions exposed by * this file. */ +// eslint-disable-next-line @typescript-eslint/no-unused-vars +const openFaceDB = () => {}; /** * Save the given {@link faceIndex} locally. @@ -58,3 +60,10 @@ export const addFileEntry = (fileID: string) => {}; */ // eslint-disable-next-line @typescript-eslint/no-unused-vars export const markIndexingFailed = (fileID: string) => {}; + +/** + * Clear any data stored by the face module. + * + * Meant to be called during logout. + */ +export const clearFaceData = () => {}; diff --git a/web/docs/dependencies.md b/web/docs/dependencies.md index 3ea8fb2409..6f959e61f5 100644 --- a/web/docs/dependencies.md +++ b/web/docs/dependencies.md @@ -161,11 +161,16 @@ some cases. - [heic-convert](https://github.com/catdad-experiments/heic-convert) is used for converting HEIC files (which browsers don't natively support) into JPEG. -## Processing +## General - [comlink](https://github.com/GoogleChromeLabs/comlink) provides a minimal layer on top of Web Workers to make them more easier to use. +- [idb](https://github.com/jakearchibald/idb) provides a promise API over the + browser-native IndexedDB APIs. + + > For more details about IDB and its role, see [storage.md](storage.md). + ## Photos app specific - [react-dropzone](https://github.com/react-dropzone/react-dropzone/) is a diff --git a/web/docs/storage.md b/web/docs/storage.md index 9f19a6a46d..ae6d01eec5 100644 --- a/web/docs/storage.md +++ b/web/docs/storage.md @@ -1,9 +1,15 @@ # Storage +## Session Storage + +Data tied to the browser tab's lifetime. + +We store the user's encryption key here. + ## Local Storage -Data in the local storage is persisted even after the user closes the tab (or -the browser itself). This is in contrast with session storage, where the data is +Data in the local storage is persisted even after the user closes the tab, or +the browser itself. This is in contrast with session storage, where the data is cleared when the browser tab is closed. The data in local storage is tied to the Document's origin (scheme + host). @@ -15,19 +21,23 @@ Some things that get stored here are: - Various user preferences -## Session Storage +## IndexedDB -Data tied to the browser tab's lifetime. +IndexedDB is a transactional NoSQL store provided by browsers. It has quite +large storage limits, and data is stored per origin (and remains persistent +across tab restarts). -We store the user's encryption key here. +Older code used the LocalForage library for storing things in Indexed DB. This +library falls back to localStorage in case Indexed DB storage is not available. -## Indexed DB +Newer code uses the idb library which provides a promise API over the IndexedDB, +but otherwise does not introduce any new abstractions. -We use the LocalForage library for storing things in Indexed DB. This library -falls back to localStorage in case Indexed DB storage is not available. +For more details, see: + +- https://web.dev/articles/indexeddb +- https://github.com/jakearchibald/idb -Indexed DB allows for larger sizes than local/session storage, and is generally -meant for larger, tabular data. ## OPFS From f1b2e2bec2b6790bfc4097379d184894c37af1df Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 28 May 2024 15:16:19 +0530 Subject: [PATCH 096/354] Update to idb 8 No breaking changes that impact us https://github.com/jakearchibald/idb/blob/main/CHANGELOG.md --- web/apps/photos/package.json | 2 +- web/apps/photos/src/services/face/db.ts | 1 + web/yarn.lock | 8 ++++---- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/web/apps/photos/package.json b/web/apps/photos/package.json index 0ec924b29b..001fc1036a 100644 --- a/web/apps/photos/package.json +++ b/web/apps/photos/package.json @@ -22,7 +22,7 @@ "ffmpeg-wasm": "file:./thirdparty/ffmpeg-wasm", "formik": "^2.1.5", "hdbscan": "0.0.1-alpha.5", - "idb": "^7.1.1", + "idb": "^8", "leaflet": "^1.9.4", "leaflet-defaulticon-compatibility": "^0.1.1", "localforage": "^1.9.0", diff --git a/web/apps/photos/src/services/face/db.ts b/web/apps/photos/src/services/face/db.ts index e099e823e3..a5fa854bcd 100644 --- a/web/apps/photos/src/services/face/db.ts +++ b/web/apps/photos/src/services/face/db.ts @@ -1,4 +1,5 @@ import type { FaceIndex } from "./types"; +// import { openDB } from "idb"; /** * [Note: Face DB schema] diff --git a/web/yarn.lock b/web/yarn.lock index aaa0d517a8..226b3add6b 100644 --- a/web/yarn.lock +++ b/web/yarn.lock @@ -2922,10 +2922,10 @@ i18next@^23.10: dependencies: "@babel/runtime" "^7.23.2" -idb@^7.1.1: - version "7.1.1" - resolved "https://registry.yarnpkg.com/idb/-/idb-7.1.1.tgz#d910ded866d32c7ced9befc5bfdf36f572ced72b" - integrity sha512-gchesWBzyvGHRO9W8tzUWFDycow5gwjvFKfyV9FF32Y7F50yZMp7mP+T2mJIWFx49zicqyC4uefHM17o6xKIVQ== +idb@^8: + version "8.0.0" + resolved "https://registry.yarnpkg.com/idb/-/idb-8.0.0.tgz#33d7ed894ed36e23bcb542fb701ad579bfaad41f" + integrity sha512-l//qvlAKGmQO31Qn7xdzagVPPaHTxXx199MhrAFuVBTPqydcPYBWjkrbv4Y0ktB+GmWOiwHl237UUOrLmQxLvw== ieee754@^1.2.1: version "1.2.1" From ca7b60921786bba64c493b96fd0190b8f8a851e4 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 28 May 2024 15:40:08 +0530 Subject: [PATCH 097/354] Schema --- web/apps/photos/src/services/face/db.ts | 53 ++++++++++++++++++++++++- 1 file changed, 51 insertions(+), 2 deletions(-) diff --git a/web/apps/photos/src/services/face/db.ts b/web/apps/photos/src/services/face/db.ts index a5fa854bcd..4cbf6fda24 100644 --- a/web/apps/photos/src/services/face/db.ts +++ b/web/apps/photos/src/services/face/db.ts @@ -1,5 +1,6 @@ +import log from "@/next/log"; +import { openDB, type DBSchema } from "idb"; import type { FaceIndex } from "./types"; -// import { openDB } from "idb"; /** * [Note: Face DB schema] @@ -21,8 +22,56 @@ import type { FaceIndex } from "./types"; * In tandem, these serve as the underlying storage for the functions exposed by * this file. */ +interface FaceDBSchema extends DBSchema { + "face-index": { + key: string; + value: FaceIndex; + }; + "file-status": { + key: string; + value: FileStatus; + }; +} + +interface FileStatus { + /** The ID of the {@link EnteFile} whose indexing status we represent. */ + fileID: string; + /** + * `1` if we have indexed a file with this {@link fileID}, `0` otherwise. + * + * It is guaranteed that "face-index" will have an entry for the same + * {@link fileID} if and only if {@link isIndexed} is `1`. + * + * [Note: Boolean IndexedDB indexes]. + * + * IndexedDB does not (currently) supported indexes on boolean fields. + * https://github.com/w3c/IndexedDB/issues/76 + * + * As a workaround, we use numeric fields where `0` denotes `false` and `1` + * denotes `true`. + */ + isIndexed: number; + /** + * The number of times attempts to index this file failed. + * + * This is guaranteed to be `0` for files which have already been + * sucessfully indexed (i.e. files for which `isIndexed` is true). + */ + failureCount: number; +} + // eslint-disable-next-line @typescript-eslint/no-unused-vars -const openFaceDB = () => {}; +const openFaceDB = () => + openDB("face", 1, { + // eslint-disable-next-line @typescript-eslint/no-unused-vars + upgrade(db, oldVersion, newVersion, tx) { + log.info(`Upgrading face DB ${oldVersion} => ${newVersion}`); + if (oldVersion < 1) { + // db.createObjectStore(); + } + }, + // TODO: FDB + }); /** * Save the given {@link faceIndex} locally. From 9887d44416b96dee33ad8b3f8e7fb9d4a302c90b Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 28 May 2024 15:52:02 +0530 Subject: [PATCH 098/354] index --- web/apps/photos/src/services/face/db.ts | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/web/apps/photos/src/services/face/db.ts b/web/apps/photos/src/services/face/db.ts index 4cbf6fda24..8f2ad681e2 100644 --- a/web/apps/photos/src/services/face/db.ts +++ b/web/apps/photos/src/services/face/db.ts @@ -30,6 +30,7 @@ interface FaceDBSchema extends DBSchema { "file-status": { key: string; value: FileStatus; + indexes: { isIndexed: number }; }; } @@ -42,6 +43,11 @@ interface FileStatus { * It is guaranteed that "face-index" will have an entry for the same * {@link fileID} if and only if {@link isIndexed} is `1`. * + * > Somewhat confusingly, we also have a (IndexedDB) "index" on this field. + * That (IDB) index allows us to effectively select {@link fileIDs} that + * still need indexing (where {@link isIndexed} is not `1`), so there is + * utility, it is just that if I say the word "index" one more time... + * * [Note: Boolean IndexedDB indexes]. * * IndexedDB does not (currently) supported indexes on boolean fields. @@ -64,10 +70,15 @@ interface FileStatus { const openFaceDB = () => openDB("face", 1, { // eslint-disable-next-line @typescript-eslint/no-unused-vars - upgrade(db, oldVersion, newVersion, tx) { + upgrade(db, oldVersion, newVersion) { log.info(`Upgrading face DB ${oldVersion} => ${newVersion}`); if (oldVersion < 1) { - // db.createObjectStore(); + db.createObjectStore("face-index", { keyPath: "fileID" }); + + const statusStore = db.createObjectStore("file-status", { + keyPath: "fileID", + }); + statusStore.createIndex("isIndexed", "isIndexed"); } }, // TODO: FDB @@ -86,8 +97,8 @@ const openFaceDB = () => export const saveFaceIndex = (faceIndex: FaceIndex) => {}; /** - * Record the existence of a fileID so that entities in the face indexing - * universe know about it. + * Record the existence of a file so that entities in the face indexing universe + * know about it (e.g. can index it if it is new and it needs indexing). * * @param fileID The ID of an {@link EnteFile}. * From 853f291de3268f926051f3445b2d4c192733e5b6 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Tue, 28 May 2024 16:20:33 +0530 Subject: [PATCH 099/354] [mob][photos] Fix face thumbnail generation pool issue --- mobile/lib/ui/viewer/file_details/face_widget.dart | 6 +++++- .../ui/viewer/file_details/faces_item_widget.dart | 8 +++++++- .../ui/viewer/search/result/person_face_widget.dart | 6 +++++- mobile/lib/utils/face/face_box_crop.dart | 13 +++++++++++-- 4 files changed, 28 insertions(+), 5 deletions(-) diff --git a/mobile/lib/ui/viewer/file_details/face_widget.dart b/mobile/lib/ui/viewer/file_details/face_widget.dart index 1ec7a2eb2d..67d2368d71 100644 --- a/mobile/lib/ui/viewer/file_details/face_widget.dart +++ b/mobile/lib/ui/viewer/file_details/face_widget.dart @@ -293,7 +293,7 @@ class _FaceWidgetState extends State { } } - Future?> getFaceCrop() async { + Future?> getFaceCrop({int fetchAttempt = 1}) async { try { final Uint8List? cachedFace = faceCropCache.get(widget.face.faceID); if (cachedFace != null) { @@ -326,6 +326,10 @@ class _FaceWidgetState extends State { error: e, stackTrace: s, ); + resetPool(fullFile: true); + if (fetchAttempt <= retryLimit) { + return getFaceCrop(fetchAttempt: fetchAttempt + 1); + } return null; } } diff --git a/mobile/lib/ui/viewer/file_details/faces_item_widget.dart b/mobile/lib/ui/viewer/file_details/faces_item_widget.dart index ed2fb0f12e..cb22e53b82 100644 --- a/mobile/lib/ui/viewer/file_details/faces_item_widget.dart +++ b/mobile/lib/ui/viewer/file_details/faces_item_widget.dart @@ -173,7 +173,9 @@ class _FacesItemWidgetState extends State { } Future?> getRelevantFaceCrops( - Iterable faces, + Iterable faces, { + int fetchAttempt = 1, + } ) async { try { final faceIdToCrop = {}; @@ -223,6 +225,10 @@ class _FacesItemWidgetState extends State { error: e, stackTrace: s, ); + resetPool(fullFile: true); + if(fetchAttempt <= retryLimit) { + return getRelevantFaceCrops(faces, fetchAttempt: fetchAttempt + 1); + } return null; } } diff --git a/mobile/lib/ui/viewer/search/result/person_face_widget.dart b/mobile/lib/ui/viewer/search/result/person_face_widget.dart index 8be99e5f6e..57fe5af654 100644 --- a/mobile/lib/ui/viewer/search/result/person_face_widget.dart +++ b/mobile/lib/ui/viewer/search/result/person_face_widget.dart @@ -121,7 +121,7 @@ class PersonFaceWidget extends StatelessWidget { ); } - Future getFaceCrop() async { + Future getFaceCrop({int fetchAttempt = 1}) async { try { final Face? face = await _getFace(); if (face == null) { @@ -187,6 +187,10 @@ class PersonFaceWidget extends StatelessWidget { error: e, stackTrace: s, ); + resetPool(fullFile: useFullFile); + if(fetchAttempt <= retryLimit) { + return getFaceCrop(fetchAttempt: fetchAttempt + 1); + } return null; } } diff --git a/mobile/lib/utils/face/face_box_crop.dart b/mobile/lib/utils/face/face_box_crop.dart index 281c0ef495..4dd1778f72 100644 --- a/mobile/lib/utils/face/face_box_crop.dart +++ b/mobile/lib/utils/face/face_box_crop.dart @@ -11,11 +11,20 @@ import "package:photos/utils/image_ml_isolate.dart"; import "package:photos/utils/thumbnail_util.dart"; import "package:pool/pool.dart"; +void resetPool({required bool fullFile}) { + if (fullFile) { + poolFullFileFaceGenerations = Pool(20, timeout: const Duration(seconds: 15)); + } else { + poolThumbnailFaceGenerations = Pool(100, timeout: const Duration(seconds: 15)); + } +} + +const int retryLimit = 3; final LRUMap faceCropCache = LRUMap(1000); final LRUMap faceCropThumbnailCache = LRUMap(1000); -final poolFullFileFaceGenerations = +Pool poolFullFileFaceGenerations = Pool(20, timeout: const Duration(seconds: 15)); -final poolThumbnailFaceGenerations = +Pool poolThumbnailFaceGenerations = Pool(100, timeout: const Duration(seconds: 15)); Future?> getFaceCrops( EnteFile file, From cb0cffce3d8411c7f2d3d7ac7c193d87a57080b4 Mon Sep 17 00:00:00 2001 From: ashilkn Date: Tue, 28 May 2024 16:28:39 +0530 Subject: [PATCH 100/354] [mob][photos] Migrating to flutter_map v6 (1) --- mobile/lib/ui/map/map_view.dart | 35 +- .../map/tile/attribution/map_attribution.dart | 544 +++++++++--------- mobile/lib/ui/map/tile/layers.dart | 58 +- 3 files changed, 311 insertions(+), 326 deletions(-) diff --git a/mobile/lib/ui/map/map_view.dart b/mobile/lib/ui/map/map_view.dart index 15b5c1d8b9..2a8c0ed4c7 100644 --- a/mobile/lib/ui/map/map_view.dart +++ b/mobile/lib/ui/map/map_view.dart @@ -60,11 +60,6 @@ class _MapViewState extends State { _markers = _buildMakers(); } - @override - void dispose() { - super.dispose(); - } - void onChange(LatLngBounds bounds) { _debouncer.run( () async { @@ -85,37 +80,31 @@ class _MapViewState extends State { widget.onTap!.call(); } : null, - center: widget.center, + initialCenter: widget.center, minZoom: widget.minZoom, maxZoom: widget.maxZoom, - enableMultiFingerGestureRace: true, - zoom: widget.initialZoom, - maxBounds: LatLngBounds( - const LatLng(-90, -180), - const LatLng(90, 180), + interactionOptions: InteractionOptions( + flags: widget.interactiveFlags, + enableMultiFingerGestureRace: true, + ), + initialZoom: widget.initialZoom, + cameraConstraint: CameraConstraint.contain( + bounds: LatLngBounds( + const LatLng(-90, -180), + const LatLng(90, 180), + ), ), onPositionChanged: (position, hasGesture) { if (position.bounds != null) { onChange(position.bounds!); } }, - interactiveFlags: widget.interactiveFlags, ), - nonRotatedChildren: [ - Padding( - padding: EdgeInsets.only( - bottom: widget.bottomSheetDraggableAreaHeight, - ), - child: OSMFranceTileAttributes( - options: widget.mapAttributionOptions, - ), - ), - ], children: [ const OSMFranceTileLayer(), MarkerClusterLayerWidget( options: MarkerClusterLayerOptions( - anchorPos: AnchorPos.align(AnchorAlign.top), + alignment: Alignment.topCenter, maxClusterRadius: 100, showPolygon: false, size: widget.markerSize, diff --git a/mobile/lib/ui/map/tile/attribution/map_attribution.dart b/mobile/lib/ui/map/tile/attribution/map_attribution.dart index e00e1a3e64..370883bf08 100644 --- a/mobile/lib/ui/map/tile/attribution/map_attribution.dart +++ b/mobile/lib/ui/map/tile/attribution/map_attribution.dart @@ -1,296 +1,296 @@ -// ignore_for_file: invalid_use_of_internal_member +// // ignore_for_file: invalid_use_of_internal_member -import "dart:async"; +// import "dart:async"; -import "package:flutter/material.dart"; -import "package:flutter_map/plugin_api.dart"; -import "package:photos/extensions/list.dart"; -import "package:photos/theme/colors.dart"; -import "package:photos/theme/ente_theme.dart"; -import "package:photos/ui/components/buttons/icon_button_widget.dart"; +// import "package:flutter/material.dart"; +// import "package:flutter_map/plugin_api.dart"; +// import "package:photos/extensions/list.dart"; +// import "package:photos/theme/colors.dart"; +// import "package:photos/theme/ente_theme.dart"; +// import "package:photos/ui/components/buttons/icon_button_widget.dart"; -// Credit: This code is based on the Rich Attribution widget from the flutter_map -class MapAttributionWidget extends StatefulWidget { - /// List of attributions to display - /// - /// [TextSourceAttribution]s are shown in a popup box (toggled by a tap/click - /// on the [openButton]/[closeButton]), unlike [LogoSourceAttribution], which - /// are visible permanently adjacent to the open/close button. - final List attributions; +// // Credit: This code is based on the Rich Attribution widget from the flutter_map +// class MapAttributionWidget extends StatefulWidget { +// /// List of attributions to display +// /// +// /// [TextSourceAttribution]s are shown in a popup box (toggled by a tap/click +// /// on the [openButton]/[closeButton]), unlike [LogoSourceAttribution], which +// /// are visible permanently adjacent to the open/close button. +// final List attributions; - /// The position in which to anchor this widget - final AttributionAlignment alignment; +// /// The position in which to anchor this widget +// final AttributionAlignment alignment; - /// The widget (usually an [IconButton]) to display when the popup box is - /// closed, that opens the popup box via the `open` callback - final Widget Function(BuildContext context, VoidCallback open)? openButton; +// /// The widget (usually an [IconButton]) to display when the popup box is +// /// closed, that opens the popup box via the `open` callback +// final Widget Function(BuildContext context, VoidCallback open)? openButton; - /// The widget (usually an [IconButton]) to display when the popup box is open, - /// that closes the popup box via the `close` callback - final Widget Function(BuildContext context, VoidCallback close)? closeButton; +// /// The widget (usually an [IconButton]) to display when the popup box is open, +// /// that closes the popup box via the `close` callback +// final Widget Function(BuildContext context, VoidCallback close)? closeButton; - /// The color to use as the popup box's background color, defaulting to the - /// [Theme]s background color - final Color? popupBackgroundColor; +// /// The color to use as the popup box's background color, defaulting to the +// /// [Theme]s background color +// final Color? popupBackgroundColor; - /// The radius of the edges of the popup box - final BorderRadius? popupBorderRadius; +// /// The radius of the edges of the popup box +// final BorderRadius? popupBorderRadius; - /// The height of the permanent row in which is found the popup menu toggle - /// button - /// - /// Also determines spacing between the items within the row. - /// - /// Also set [LogoSourceAttribution.height] to the same value, if adjusted. - final double permanentHeight; +// /// The height of the permanent row in which is found the popup menu toggle +// /// button +// /// +// /// Also determines spacing between the items within the row. +// /// +// /// Also set [LogoSourceAttribution.height] to the same value, if adjusted. +// final double permanentHeight; - /// Whether to add an additional attribution logo and text for 'flutter_map' - final bool showFlutterMapAttribution; +// /// Whether to add an additional attribution logo and text for 'flutter_map' +// final bool showFlutterMapAttribution; - /// Animation configuration, through the properties and handler/builder - /// defined by a [RichAttributionWidgetAnimation] implementation - /// - /// Can be extensivley customized by implementing a custom - /// [RichAttributionWidgetAnimation], or the prebuilt [FadeRAWA] and - /// [ScaleRAWA] animations can be used with limited customization. - final RichAttributionWidgetAnimation animationConfig; +// /// Animation configuration, through the properties and handler/builder +// /// defined by a [RichAttributionWidgetAnimation] implementation +// /// +// /// Can be extensivley customized by implementing a custom +// /// [RichAttributionWidgetAnimation], or the prebuilt [FadeRAWA] and +// /// [ScaleRAWA] animations can be used with limited customization. +// final RichAttributionWidgetAnimation animationConfig; - /// If not [Duration.zero] (default), the popup box will be open by default and - /// hidden this long after the map is initialised - /// - /// This is useful with certain sources/tile servers that make immediate - /// attribution mandatory and are not attributed with a permanently visible - /// [LogoSourceAttribution]. - final Duration popupInitialDisplayDuration; +// /// If not [Duration.zero] (default), the popup box will be open by default and +// /// hidden this long after the map is initialised +// /// +// /// This is useful with certain sources/tile servers that make immediate +// /// attribution mandatory and are not attributed with a permanently visible +// /// [LogoSourceAttribution]. +// final Duration popupInitialDisplayDuration; - /// A prebuilt dynamic attribution layer that supports both logos and text - /// through [SourceAttribution]s - /// - /// [TextSourceAttribution]s are shown in a popup box that can be visible or - /// invisible. Its state is toggled by a tri-state [openButton]/[closeButton] : - /// 1. Not hovered, not opened: faded button, invisible box - /// 2. Hovered, not opened: full opacity button, invisible box - /// 3. Opened: full opacity button, visible box - /// - /// The hover state on mobile devices is unspecified, but the behaviour is - /// usually inconsequential on mobile devices anyway, due to the fingertip - /// covering the entire button. - /// - /// [LogoSourceAttribution]s are shown adjacent to the open/close button, to - /// comply with some stricter tile server requirements (such as Mapbox). These - /// are usually supplemented with a [TextSourceAttribution]. - /// - /// The popup box also closes automatically on any interaction with the map. - /// - /// Animations are built in by default, and configured/handled through - /// [RichAttributionWidgetAnimation] - see that class and the [animationConfig] - /// property for more information. By default, a simple fade/opacity animation - /// is provided by [FadeRAWA]. [ScaleRAWA] is also available. - /// - /// Read the documentation on the individual properties for more information - /// and customizability. +// /// A prebuilt dynamic attribution layer that supports both logos and text +// /// through [SourceAttribution]s +// /// +// /// [TextSourceAttribution]s are shown in a popup box that can be visible or +// /// invisible. Its state is toggled by a tri-state [openButton]/[closeButton] : +// /// 1. Not hovered, not opened: faded button, invisible box +// /// 2. Hovered, not opened: full opacity button, invisible box +// /// 3. Opened: full opacity button, visible box +// /// +// /// The hover state on mobile devices is unspecified, but the behaviour is +// /// usually inconsequential on mobile devices anyway, due to the fingertip +// /// covering the entire button. +// /// +// /// [LogoSourceAttribution]s are shown adjacent to the open/close button, to +// /// comply with some stricter tile server requirements (such as Mapbox). These +// /// are usually supplemented with a [TextSourceAttribution]. +// /// +// /// The popup box also closes automatically on any interaction with the map. +// /// +// /// Animations are built in by default, and configured/handled through +// /// [RichAttributionWidgetAnimation] - see that class and the [animationConfig] +// /// property for more information. By default, a simple fade/opacity animation +// /// is provided by [FadeRAWA]. [ScaleRAWA] is also available. +// /// +// /// Read the documentation on the individual properties for more information +// /// and customizability. - final double iconSize; - const MapAttributionWidget({ - super.key, - required this.attributions, - this.alignment = AttributionAlignment.bottomRight, - this.openButton, - this.closeButton, - this.popupBackgroundColor, - this.popupBorderRadius, - this.permanentHeight = 24, - this.showFlutterMapAttribution = true, - this.animationConfig = const FadeRAWA(), - this.popupInitialDisplayDuration = Duration.zero, - this.iconSize = 20, - }); +// final double iconSize; +// const MapAttributionWidget({ +// super.key, +// required this.attributions, +// this.alignment = AttributionAlignment.bottomRight, +// this.openButton, +// this.closeButton, +// this.popupBackgroundColor, +// this.popupBorderRadius, +// this.permanentHeight = 24, +// this.showFlutterMapAttribution = true, +// this.animationConfig = const FadeRAWA(), +// this.popupInitialDisplayDuration = Duration.zero, +// this.iconSize = 20, +// }); - @override - State createState() => MapAttributionWidgetState(); -} +// @override +// State createState() => MapAttributionWidgetState(); +// } -class MapAttributionWidgetState extends State { - StreamSubscription? mapEventSubscription; +// class MapAttributionWidgetState extends State { +// StreamSubscription? mapEventSubscription; - final persistentAttributionKey = GlobalKey(); - Size? persistentAttributionSize; +// final persistentAttributionKey = GlobalKey(); +// Size? persistentAttributionSize; - late bool popupExpanded = widget.popupInitialDisplayDuration != Duration.zero; - bool persistentHovered = false; +// late bool popupExpanded = widget.popupInitialDisplayDuration != Duration.zero; +// bool persistentHovered = false; - @override - void initState() { - super.initState(); +// @override +// void initState() { +// super.initState(); - if (popupExpanded) { - Future.delayed( - widget.popupInitialDisplayDuration, - () => setState(() => popupExpanded = false), - ); - } +// if (popupExpanded) { +// Future.delayed( +// widget.popupInitialDisplayDuration, +// () => setState(() => popupExpanded = false), +// ); +// } - WidgetsBinding.instance.addPostFrameCallback( - (_) => WidgetsBinding.instance.addPostFrameCallback((_) { - if (mounted) { - setState( - () => persistentAttributionSize = - (persistentAttributionKey.currentContext!.findRenderObject() - as RenderBox) - .size, - ); - } - }), - ); - } +// WidgetsBinding.instance.addPostFrameCallback( +// (_) => WidgetsBinding.instance.addPostFrameCallback((_) { +// if (mounted) { +// setState( +// () => persistentAttributionSize = +// (persistentAttributionKey.currentContext!.findRenderObject() +// as RenderBox) +// .size, +// ); +// } +// }), +// ); +// } - @override - void dispose() { - mapEventSubscription?.cancel(); - super.dispose(); - } +// @override +// void dispose() { +// mapEventSubscription?.cancel(); +// super.dispose(); +// } - @override - Widget build(BuildContext context) { - final persistentAttributionItems = [ - ...List.from( - widget.attributions.whereType(), - growable: false, - ).interleave(SizedBox(width: widget.permanentHeight / 1.5)), - if (widget.showFlutterMapAttribution) - LogoSourceAttribution( - Image.asset( - 'lib/assets/flutter_map_logo.png', - package: 'flutter_map', - ), - tooltip: 'flutter_map', - height: widget.permanentHeight, - ), - SizedBox(width: widget.permanentHeight * 0.1), - AnimatedSwitcher( - switchInCurve: widget.animationConfig.buttonCurve, - switchOutCurve: widget.animationConfig.buttonCurve, - duration: widget.animationConfig.buttonDuration, - child: popupExpanded - ? (widget.closeButton ?? - (context, close) => IconButtonWidget( - size: widget.iconSize, - onTap: close, - icon: Icons.cancel_outlined, - iconButtonType: IconButtonType.primary, - iconColor: getEnteColorScheme(context).strokeBase, - ))( - context, - () => setState(() => popupExpanded = false), - ) - : (widget.openButton ?? - (context, open) => IconButtonWidget( - size: widget.iconSize, - onTap: open, - icon: Icons.info_outlined, - iconButtonType: IconButtonType.primary, - iconColor: strokeBaseLight, - ))( - context, - () { - setState(() => popupExpanded = true); - mapEventSubscription = FlutterMapState.of(context) - .mapController - .mapEventStream - .listen((e) { - setState(() => popupExpanded = false); - mapEventSubscription?.cancel(); - }); - }, - ), - ), - ]; +// @override +// Widget build(BuildContext context) { +// final persistentAttributionItems = [ +// ...List.from( +// widget.attributions.whereType(), +// growable: false, +// ).interleave(SizedBox(width: widget.permanentHeight / 1.5)), +// if (widget.showFlutterMapAttribution) +// LogoSourceAttribution( +// Image.asset( +// 'lib/assets/flutter_map_logo.png', +// package: 'flutter_map', +// ), +// tooltip: 'flutter_map', +// height: widget.permanentHeight, +// ), +// SizedBox(width: widget.permanentHeight * 0.1), +// AnimatedSwitcher( +// switchInCurve: widget.animationConfig.buttonCurve, +// switchOutCurve: widget.animationConfig.buttonCurve, +// duration: widget.animationConfig.buttonDuration, +// child: popupExpanded +// ? (widget.closeButton ?? +// (context, close) => IconButtonWidget( +// size: widget.iconSize, +// onTap: close, +// icon: Icons.cancel_outlined, +// iconButtonType: IconButtonType.primary, +// iconColor: getEnteColorScheme(context).strokeBase, +// ))( +// context, +// () => setState(() => popupExpanded = false), +// ) +// : (widget.openButton ?? +// (context, open) => IconButtonWidget( +// size: widget.iconSize, +// onTap: open, +// icon: Icons.info_outlined, +// iconButtonType: IconButtonType.primary, +// iconColor: strokeBaseLight, +// ))( +// context, +// () { +// setState(() => popupExpanded = true); +// mapEventSubscription = FlutterMapState.of(context) +// .mapController +// .mapEventStream +// .listen((e) { +// setState(() => popupExpanded = false); +// mapEventSubscription?.cancel(); +// }); +// }, +// ), +// ), +// ]; - return LayoutBuilder( - builder: (context, constraints) => Align( - alignment: widget.alignment.real, - child: Stack( - alignment: widget.alignment.real, - children: [ - if (persistentAttributionSize != null) - Padding( - padding: const EdgeInsets.all(6), - child: AnimatedScale( - scale: popupExpanded ? 1 : 0, - duration: const Duration(milliseconds: 200), - curve: popupExpanded ? Curves.easeOut : Curves.easeIn, - alignment: widget.alignment.real, - child: Container( - decoration: BoxDecoration( - color: widget.popupBackgroundColor ?? - Theme.of(context).colorScheme.background, - border: Border.all(width: 0, style: BorderStyle.none), - borderRadius: widget.popupBorderRadius ?? - BorderRadius.only( - topLeft: const Radius.circular(10), - topRight: const Radius.circular(10), - bottomLeft: widget.alignment == - AttributionAlignment.bottomLeft - ? Radius.zero - : const Radius.circular(10), - bottomRight: widget.alignment == - AttributionAlignment.bottomRight - ? Radius.zero - : const Radius.circular(10), - ), - ), - constraints: BoxConstraints( - minWidth: constraints.maxWidth < 420 - ? constraints.maxWidth - : persistentAttributionSize!.width, - ), - child: Padding( - padding: const EdgeInsets.all(8), - child: SizedBox( - height: widget.attributions.length * 32, - child: Column( - mainAxisSize: MainAxisSize.max, - mainAxisAlignment: MainAxisAlignment.spaceBetween, - crossAxisAlignment: CrossAxisAlignment.start, - children: [ - ...widget.attributions - .whereType(), - SizedBox( - height: (widget.permanentHeight - 24) + 32, - ), - ], - ), - ), - ), - ), - ), - ), - MouseRegion( - key: persistentAttributionKey, - onEnter: (_) => setState(() => persistentHovered = true), - onExit: (_) => setState(() => persistentHovered = false), - cursor: SystemMouseCursors.click, - child: AnimatedOpacity( - opacity: persistentHovered || popupExpanded ? 1 : 0.5, - curve: widget.animationConfig.buttonCurve, - duration: widget.animationConfig.buttonDuration, - child: Padding( - padding: const EdgeInsets.all(4), - child: FittedBox( - child: Row( - mainAxisSize: MainAxisSize.min, - children: - widget.alignment == AttributionAlignment.bottomLeft - ? persistentAttributionItems.reversed.toList() - : persistentAttributionItems, - ), - ), - ), - ), - ), - ], - ), - ), - ); - } -} +// return LayoutBuilder( +// builder: (context, constraints) => Align( +// alignment: widget.alignment.real, +// child: Stack( +// alignment: widget.alignment.real, +// children: [ +// if (persistentAttributionSize != null) +// Padding( +// padding: const EdgeInsets.all(6), +// child: AnimatedScale( +// scale: popupExpanded ? 1 : 0, +// duration: const Duration(milliseconds: 200), +// curve: popupExpanded ? Curves.easeOut : Curves.easeIn, +// alignment: widget.alignment.real, +// child: Container( +// decoration: BoxDecoration( +// color: widget.popupBackgroundColor ?? +// Theme.of(context).colorScheme.background, +// border: Border.all(width: 0, style: BorderStyle.none), +// borderRadius: widget.popupBorderRadius ?? +// BorderRadius.only( +// topLeft: const Radius.circular(10), +// topRight: const Radius.circular(10), +// bottomLeft: widget.alignment == +// AttributionAlignment.bottomLeft +// ? Radius.zero +// : const Radius.circular(10), +// bottomRight: widget.alignment == +// AttributionAlignment.bottomRight +// ? Radius.zero +// : const Radius.circular(10), +// ), +// ), +// constraints: BoxConstraints( +// minWidth: constraints.maxWidth < 420 +// ? constraints.maxWidth +// : persistentAttributionSize!.width, +// ), +// child: Padding( +// padding: const EdgeInsets.all(8), +// child: SizedBox( +// height: widget.attributions.length * 32, +// child: Column( +// mainAxisSize: MainAxisSize.max, +// mainAxisAlignment: MainAxisAlignment.spaceBetween, +// crossAxisAlignment: CrossAxisAlignment.start, +// children: [ +// ...widget.attributions +// .whereType(), +// SizedBox( +// height: (widget.permanentHeight - 24) + 32, +// ), +// ], +// ), +// ), +// ), +// ), +// ), +// ), +// MouseRegion( +// key: persistentAttributionKey, +// onEnter: (_) => setState(() => persistentHovered = true), +// onExit: (_) => setState(() => persistentHovered = false), +// cursor: SystemMouseCursors.click, +// child: AnimatedOpacity( +// opacity: persistentHovered || popupExpanded ? 1 : 0.5, +// curve: widget.animationConfig.buttonCurve, +// duration: widget.animationConfig.buttonDuration, +// child: Padding( +// padding: const EdgeInsets.all(4), +// child: FittedBox( +// child: Row( +// mainAxisSize: MainAxisSize.min, +// children: +// widget.alignment == AttributionAlignment.bottomLeft +// ? persistentAttributionItems.reversed.toList() +// : persistentAttributionItems, +// ), +// ), +// ), +// ), +// ), +// ], +// ), +// ), +// ); +// } +// } diff --git a/mobile/lib/ui/map/tile/layers.dart b/mobile/lib/ui/map/tile/layers.dart index 2597dddea2..c57ba2c639 100644 --- a/mobile/lib/ui/map/tile/layers.dart +++ b/mobile/lib/ui/map/tile/layers.dart @@ -1,11 +1,6 @@ import "package:flutter/material.dart"; import "package:flutter_map/flutter_map.dart"; -import "package:photos/generated/l10n.dart"; -import "package:photos/theme/ente_theme.dart"; -import "package:photos/ui/map/tile/attribution/map_attribution.dart"; import "package:photos/ui/map/tile/cache.dart"; -import "package:url_launcher/url_launcher.dart"; -import "package:url_launcher/url_launcher_string.dart"; const String _userAgent = "io.ente.photos"; @@ -62,32 +57,33 @@ class OSMFranceTileAttributes extends StatelessWidget { @override Widget build(BuildContext context) { - final textTheme = getEnteTextTheme(context).tinyBold; - return MapAttributionWidget( - alignment: AttributionAlignment.bottomLeft, - showFlutterMapAttribution: false, - permanentHeight: options.permanentHeight, - popupBackgroundColor: getEnteColorScheme(context).backgroundElevated, - popupBorderRadius: options.popupBorderRadius, - iconSize: options.iconSize, - attributions: [ - TextSourceAttribution( - S.of(context).openstreetmapContributors, - textStyle: textTheme, - onTap: () => launchUrlString('https://openstreetmap.org/copyright'), - ), - TextSourceAttribution( - 'HOT Tiles', - textStyle: textTheme, - onTap: () => launchUrl(Uri.parse('https://www.hotosm.org/')), - ), - TextSourceAttribution( - S.of(context).hostedAtOsmFrance, - textStyle: textTheme, - onTap: () => launchUrl(Uri.parse('https://www.openstreetmap.fr/')), - ), - ], - ); + // final textTheme = getEnteTextTheme(context).tinyBold; + // return MapAttributionWidget( + // alignment: AttributionAlignment.bottomLeft, + // showFlutterMapAttribution: false, + // permanentHeight: options.permanentHeight, + // popupBackgroundColor: getEnteColorScheme(context).backgroundElevated, + // popupBorderRadius: options.popupBorderRadius, + // iconSize: options.iconSize, + // attributions: [ + // TextSourceAttribution( + // S.of(context).openstreetmapContributors, + // textStyle: textTheme, + // onTap: () => launchUrlString('https://openstreetmap.org/copyright'), + // ), + // TextSourceAttribution( + // 'HOT Tiles', + // textStyle: textTheme, + // onTap: () => launchUrl(Uri.parse('https://www.hotosm.org/')), + // ), + // TextSourceAttribution( + // S.of(context).hostedAtOsmFrance, + // textStyle: textTheme, + // onTap: () => launchUrl(Uri.parse('https://www.openstreetmap.fr/')), + // ), + // ], + // ); + return const SizedBox.shrink(); } } From 433c23ca0782a2dfb375fbcb87d890a9ac49ebc4 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Tue, 28 May 2024 16:35:56 +0530 Subject: [PATCH 101/354] [mob][photos] Put MLController timeout back to 15 seconds --- .../services/machine_learning/machine_learning_controller.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobile/lib/services/machine_learning/machine_learning_controller.dart b/mobile/lib/services/machine_learning/machine_learning_controller.dart index 3b78fd8c9e..1ecb053f01 100644 --- a/mobile/lib/services/machine_learning/machine_learning_controller.dart +++ b/mobile/lib/services/machine_learning/machine_learning_controller.dart @@ -18,7 +18,7 @@ class MachineLearningController { static const kMaximumTemperature = 42; // 42 degree celsius static const kMinimumBatteryLevel = 20; // 20% - static const kDefaultInteractionTimeout = Duration(seconds: 10); + static const kDefaultInteractionTimeout = Duration(seconds: 15); static const kUnhealthyStates = ["over_heat", "over_voltage", "dead"]; bool _isDeviceHealthy = true; From b10f4ee18a28ebde48ed177cfc82f6d4e64afb99 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Tue, 28 May 2024 16:41:40 +0530 Subject: [PATCH 102/354] [mob][photos] Bump --- mobile/pubspec.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobile/pubspec.yaml b/mobile/pubspec.yaml index ed3bf47193..7b06f7f420 100644 --- a/mobile/pubspec.yaml +++ b/mobile/pubspec.yaml @@ -12,7 +12,7 @@ description: ente photos application # Read more about iOS versioning at # https://developer.apple.com/library/archive/documentation/General/Reference/InfoPlistKeyReference/Articles/CoreFoundationKeys.html -version: 0.8.120+640 +version: 0.8.122+642 publish_to: none environment: From f34a4d4a219f83c2ef17bad256b077339edff682 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 28 May 2024 18:18:41 +0530 Subject: [PATCH 103/354] lf --- web/apps/photos/src/services/face/db.ts | 1 - web/docs/storage.md | 1 - 2 files changed, 2 deletions(-) diff --git a/web/apps/photos/src/services/face/db.ts b/web/apps/photos/src/services/face/db.ts index 8f2ad681e2..886309ffbd 100644 --- a/web/apps/photos/src/services/face/db.ts +++ b/web/apps/photos/src/services/face/db.ts @@ -69,7 +69,6 @@ interface FileStatus { // eslint-disable-next-line @typescript-eslint/no-unused-vars const openFaceDB = () => openDB("face", 1, { - // eslint-disable-next-line @typescript-eslint/no-unused-vars upgrade(db, oldVersion, newVersion) { log.info(`Upgrading face DB ${oldVersion} => ${newVersion}`); if (oldVersion < 1) { diff --git a/web/docs/storage.md b/web/docs/storage.md index ae6d01eec5..f4b28bda16 100644 --- a/web/docs/storage.md +++ b/web/docs/storage.md @@ -38,7 +38,6 @@ For more details, see: - https://web.dev/articles/indexeddb - https://github.com/jakearchibald/idb - ## OPFS OPFS is used for caching entire files when we're running under Electron (the Web From 4aaafd3b084fb13d7bbe5ad015bc3b36a99826c9 Mon Sep 17 00:00:00 2001 From: Raphael Le Goaller Date: Tue, 28 May 2024 15:49:08 +0200 Subject: [PATCH 104/354] [docs] Fix typo in custom server documentation --- docs/docs/self-hosting/guides/custom-server/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docs/self-hosting/guides/custom-server/index.md b/docs/docs/self-hosting/guides/custom-server/index.md index 110e3dbb88..81ba95a78a 100644 --- a/docs/docs/self-hosting/guides/custom-server/index.md +++ b/docs/docs/self-hosting/guides/custom-server/index.md @@ -37,7 +37,7 @@ endpoint: (Another [example](https://github.com/ente-io/ente/blob/main/cli/config.yaml.example)) -## Web appps and Photos desktop app +## Web apps and Photos desktop app You will need to build the app from source and use the `NEXT_PUBLIC_ENTE_ENDPOINT` environment variable to tell it which server to From b1e64cadf68975dca8a3839253e4a10b90b8bc48 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 29 May 2024 11:07:11 +0530 Subject: [PATCH 105/354] Lifecycle --- web/apps/photos/src/services/face/db.ts | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/web/apps/photos/src/services/face/db.ts b/web/apps/photos/src/services/face/db.ts index 886309ffbd..9f11425b7d 100644 --- a/web/apps/photos/src/services/face/db.ts +++ b/web/apps/photos/src/services/face/db.ts @@ -45,8 +45,8 @@ interface FileStatus { * * > Somewhat confusingly, we also have a (IndexedDB) "index" on this field. * That (IDB) index allows us to effectively select {@link fileIDs} that - * still need indexing (where {@link isIndexed} is not `1`), so there is - * utility, it is just that if I say the word "index" one more time... + * still need indexing (where {@link isIndexed} is not `1`), so it is all + * sensible, just that if I say the word "index" one more time... * * [Note: Boolean IndexedDB indexes]. * @@ -80,7 +80,22 @@ const openFaceDB = () => statusStore.createIndex("isIndexed", "isIndexed"); } }, - // TODO: FDB + blocking() { + log.info( + "Another client is attempting to open a new version of face DB", + ); + // TODO: FBD: close via our promise + // TODO: FBD: clear our promise + }, + blocked() { + log.warn( + "Waiting for an existing client to close their connection so that we can update the face DB version", + ); + }, + terminated() { + log.warn("Our connection to face DB was unexpectedly terminated"); + // TODO: FBD: clear our promise + }, }); /** From cfb4ded9913d87f638753aef3ed08a2ce932b8c5 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Wed, 29 May 2024 11:13:21 +0530 Subject: [PATCH 106/354] [mob][photos] Fix breakupCluster not returning cluster summaries --- .../face_clustering_service.dart | 83 +++++++++---------- .../face_ml/feedback/cluster_feedback.dart | 2 +- 2 files changed, 41 insertions(+), 44 deletions(-) diff --git a/mobile/lib/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart b/mobile/lib/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart index 310deb964a..35a25748ac 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart @@ -45,6 +45,8 @@ class ClusteringResult { bool get isEmpty => newFaceIdToCluster.isEmpty; + bool get hasAllResults => newClusterSummaries != null && newClusterIdToFaceIds != null; + ClusteringResult({ required this.newFaceIdToCluster, this.newClusterSummaries, @@ -127,8 +129,7 @@ class FaceClusteringService { break; } } catch (e, stackTrace) { - sendPort - .send({'error': e.toString(), 'stackTrace': stackTrace.toString()}); + sendPort.send({'error': e.toString(), 'stackTrace': stackTrace.toString()}); } }); } @@ -256,6 +257,7 @@ class FaceClusteringService { Future predictLinearComputer( Map input, { Map? fileIDToCreationTime, + required Map oldClusterSummaries, double distanceThreshold = kRecommendedDistanceThreshold, }) async { if (input.isEmpty) { @@ -291,6 +293,7 @@ class FaceClusteringService { param: { "input": clusteringInput, "fileIDToCreationTime": fileIDToCreationTime, + "oldClusterSummaries": oldClusterSummaries, "distanceThreshold": distanceThreshold, "conservativeDistanceThreshold": distanceThreshold - 0.08, "useDynamicThreshold": false, @@ -314,6 +317,7 @@ class FaceClusteringService { Future predictCompleteComputer( Map input, { Map? fileIDToCreationTime, + required Map oldClusterSummaries, double distanceThreshold = kRecommendedDistanceThreshold, double mergeThreshold = 0.30, }) async { @@ -336,6 +340,7 @@ class FaceClusteringService { param: { "input": input, "fileIDToCreationTime": fileIDToCreationTime, + "oldClusterSummaries": oldClusterSummaries, "distanceThreshold": distanceThreshold, "mergeThreshold": mergeThreshold, }, @@ -355,6 +360,7 @@ class FaceClusteringService { Future predictWithinClusterComputer( Map input, { Map? fileIDToCreationTime, + Map oldClusterSummaries = const {}, double distanceThreshold = kRecommendedDistanceThreshold, }) async { _logger.info( @@ -369,6 +375,7 @@ class FaceClusteringService { final result = await predictCompleteComputer( input, fileIDToCreationTime: fileIDToCreationTime, + oldClusterSummaries: oldClusterSummaries, distanceThreshold: distanceThreshold - 0.08, mergeThreshold: mergeThreshold, ); @@ -381,6 +388,7 @@ class FaceClusteringService { final clusterResult = await predictLinearComputer( input, fileIDToCreationTime: fileIDToCreationTime, + oldClusterSummaries: oldClusterSummaries, distanceThreshold: distanceThreshold, ); return clusterResult; @@ -396,12 +404,10 @@ class FaceClusteringService { final input = args['input'] as Set; final fileIDToCreationTime = args['fileIDToCreationTime'] as Map?; final distanceThreshold = args['distanceThreshold'] as double; - final conservativeDistanceThreshold = - args['conservativeDistanceThreshold'] as double; + final conservativeDistanceThreshold = args['conservativeDistanceThreshold'] as double; final useDynamicThreshold = args['useDynamicThreshold'] as bool; final offset = args['offset'] as int?; - final oldClusterSummaries = - args['oldClusterSummaries'] as Map?; + final oldClusterSummaries = args['oldClusterSummaries'] as Map?; log( "[ClusterIsolate] ${DateTime.now()} Copied to isolate ${input.length} faces", @@ -425,8 +431,7 @@ class FaceClusteringService { dtype: DType.float32, ), clusterId: face.clusterId, - fileCreationTime: - fileIDToCreationTime?[getFileIdFromFaceId(face.faceID)], + fileCreationTime: fileIDToCreationTime?[getFileIdFromFaceId(face.faceID)], ), ); } @@ -493,9 +498,8 @@ class FaceClusteringService { double closestDistance = double.infinity; late double thresholdValue; if (useDynamicThreshold) { - thresholdValue = sortedFaceInfos[i].badFace! - ? conservativeDistanceThreshold - : distanceThreshold; + thresholdValue = + sortedFaceInfos[i].badFace! ? conservativeDistanceThreshold : distanceThreshold; if (sortedFaceInfos[i].badFace!) dynamicThresholdCount++; } else { thresholdValue = distanceThreshold; @@ -505,11 +509,10 @@ class FaceClusteringService { } // WARNING: The loop below is now O(n^2) so be very careful with anything you put in there! for (int j = i - 1; j >= 0; j--) { - final double distance = 1 - - sortedFaceInfos[i].vEmbedding!.dot(sortedFaceInfos[j].vEmbedding!); + final double distance = + 1 - sortedFaceInfos[i].vEmbedding!.dot(sortedFaceInfos[j].vEmbedding!); if (distance < closestDistance) { - if (sortedFaceInfos[j].badFace! && - distance > conservativeDistanceThreshold) { + if (sortedFaceInfos[j].badFace! && distance > conservativeDistanceThreshold) { continue; } closestDistance = distance; @@ -535,8 +538,7 @@ class FaceClusteringService { // Finally, assign the new clusterId to the faces final Map newFaceIdToCluster = {}; - final newClusteredFaceInfos = - sortedFaceInfos.sublist(alreadyClusteredCount); + final newClusteredFaceInfos = sortedFaceInfos.sublist(alreadyClusteredCount); for (final faceInfo in newClusteredFaceInfos) { newFaceIdToCluster[faceInfo.faceID] = faceInfo.clusterId!; } @@ -597,9 +599,8 @@ class FaceClusteringService { final Map newClusterSummaries = {}; for (final clusterId in newClusterIdToFaceInfos.keys) { - final List newEmbeddings = newClusterIdToFaceInfos[clusterId]! - .map((faceInfo) => faceInfo.vEmbedding!) - .toList(); + final List newEmbeddings = + newClusterIdToFaceInfos[clusterId]!.map((faceInfo) => faceInfo.vEmbedding!).toList(); final newCount = newEmbeddings.length; if (oldSummary.containsKey(clusterId)) { final oldMean = Vector.fromList( @@ -609,8 +610,7 @@ class FaceClusteringService { final oldCount = oldSummary[clusterId]!.$2; final oldEmbeddings = oldMean * oldCount; newEmbeddings.add(oldEmbeddings); - final newMeanVector = - newEmbeddings.reduce((a, b) => a + b) / (oldCount + newCount); + final newMeanVector = newEmbeddings.reduce((a, b) => a + b) / (oldCount + newCount); final newMeanVectorNormalized = newMeanVector / newMeanVector.norm(); newClusterSummaries[clusterId] = ( EVector(values: newMeanVectorNormalized.toList()).writeToBuffer(), @@ -619,10 +619,8 @@ class FaceClusteringService { } else { final newMeanVector = newEmbeddings.reduce((a, b) => a + b); final newMeanVectorNormalized = newMeanVector / newMeanVector.norm(); - newClusterSummaries[clusterId] = ( - EVector(values: newMeanVectorNormalized.toList()).writeToBuffer(), - newCount - ); + newClusterSummaries[clusterId] = + (EVector(values: newMeanVectorNormalized.toList()).writeToBuffer(), newCount); } } log( @@ -696,6 +694,7 @@ class FaceClusteringService { final fileIDToCreationTime = args['fileIDToCreationTime'] as Map?; final distanceThreshold = args['distanceThreshold'] as double; final mergeThreshold = args['mergeThreshold'] as double; + final oldClusterSummaries = args['oldClusterSummaries'] as Map?; log( "[CompleteClustering] ${DateTime.now()} Copied to isolate ${input.length} faces for clustering", @@ -711,8 +710,7 @@ class FaceClusteringService { EVector.fromBuffer(entry.value).values, dtype: DType.float32, ), - fileCreationTime: - fileIDToCreationTime?[getFileIdFromFaceId(entry.key)], + fileCreationTime: fileIDToCreationTime?[getFileIdFromFaceId(entry.key)], ), ); } @@ -746,8 +744,7 @@ class FaceClusteringService { double closestDistance = double.infinity; for (int j = 0; j < totalFaces; j++) { if (i == j) continue; - final double distance = - 1 - faceInfos[i].vEmbedding!.dot(faceInfos[j].vEmbedding!); + final double distance = 1 - faceInfos[i].vEmbedding!.dot(faceInfos[j].vEmbedding!); if (distance < closestDistance) { closestDistance = distance; closestIdx = j; @@ -777,21 +774,17 @@ class FaceClusteringService { } final Map clusterIdToMeanEmbeddingAndWeight = {}; for (final clusterId in clusterIdToFaceInfos.keys) { - final List embeddings = clusterIdToFaceInfos[clusterId]! - .map((faceInfo) => faceInfo.vEmbedding!) - .toList(); + final List embeddings = + clusterIdToFaceInfos[clusterId]!.map((faceInfo) => faceInfo.vEmbedding!).toList(); final count = clusterIdToFaceInfos[clusterId]!.length; final Vector meanEmbedding = embeddings.reduce((a, b) => a + b) / count; - final Vector meanEmbeddingNormalized = - meanEmbedding / meanEmbedding.norm(); - clusterIdToMeanEmbeddingAndWeight[clusterId] = - (meanEmbeddingNormalized, count); + final Vector meanEmbeddingNormalized = meanEmbedding / meanEmbedding.norm(); + clusterIdToMeanEmbeddingAndWeight[clusterId] = (meanEmbeddingNormalized, count); } // Now merge the clusters that are close to each other, based on mean embedding final List<(int, int)> mergedClustersList = []; - final List clusterIds = - clusterIdToMeanEmbeddingAndWeight.keys.toList(); + final List clusterIds = clusterIdToMeanEmbeddingAndWeight.keys.toList(); log(' [CompleteClustering] ${DateTime.now()} ${clusterIds.length} clusters found, now checking for merges'); while (true) { if (clusterIds.length < 2) break; @@ -858,10 +851,14 @@ class FaceClusteringService { } } - final newClusterSummaries = FaceClusteringService.updateClusterSummaries( - oldSummary: {}, - newFaceInfos: faceInfos, - ); + // Now calculate the mean of the embeddings for each cluster and update the cluster summaries + Map? newClusterSummaries; + if (oldClusterSummaries != null) { + newClusterSummaries = FaceClusteringService.updateClusterSummaries( + oldSummary: oldClusterSummaries, + newFaceInfos: faceInfos, + ); + } stopwatchClustering.stop(); log( diff --git a/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart b/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart index b354719f9a..5a41a60df7 100644 --- a/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart +++ b/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart @@ -434,7 +434,7 @@ class ClusterFeedbackService { ); if (clusterResult == null || - clusterResult.newClusterIdToFaceIds == null || + !clusterResult.hasAllResults || clusterResult.isEmpty) { _logger.warning('No clusters found or something went wrong'); return ClusteringResult(newFaceIdToCluster: {}); From 2f7d1401cd81b5a1840db9056668365b4d52fcf1 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 29 May 2024 11:26:38 +0530 Subject: [PATCH 107/354] Promise --- web/apps/photos/src/services/face/db.ts | 37 +++++++++++++++++++++---- 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/web/apps/photos/src/services/face/db.ts b/web/apps/photos/src/services/face/db.ts index 9f11425b7d..14900638cd 100644 --- a/web/apps/photos/src/services/face/db.ts +++ b/web/apps/photos/src/services/face/db.ts @@ -66,9 +66,26 @@ interface FileStatus { failureCount: number; } -// eslint-disable-next-line @typescript-eslint/no-unused-vars -const openFaceDB = () => - openDB("face", 1, { +/** + * A promise to the face DB. + * + * We open the database once (lazily), and thereafter save and reuse the promise + * each time something wants to connect to it. + * + * This promise can subsequently get cleared if we need to relinquish our + * connection (e.g. if another client wants to open the face DB with a newer + * version of the schema). + * + * Note that this is module specific state, so the main thread and each worker + * thread that calls the functions in this module will get their own independent + * connection. When we logout, the workers will presumably get resurrected and + * close their connections, while the connection kept by the main thread will be + * used to delete the database in {@link clearFaceData}. + */ +let _faceDB: ReturnType | undefined; + +const openFaceDB = async () => { + const db = await openDB("face", 1, { upgrade(db, oldVersion, newVersion) { log.info(`Upgrading face DB ${oldVersion} => ${newVersion}`); if (oldVersion < 1) { @@ -84,8 +101,8 @@ const openFaceDB = () => log.info( "Another client is attempting to open a new version of face DB", ); - // TODO: FBD: close via our promise - // TODO: FBD: clear our promise + db.close(); + _faceDB = undefined; }, blocked() { log.warn( @@ -94,9 +111,17 @@ const openFaceDB = () => }, terminated() { log.warn("Our connection to face DB was unexpectedly terminated"); - // TODO: FBD: clear our promise + _faceDB = undefined; }, }); + return db; +}; + +/** + * @returns a lazily created, cached connection to the face DB. + */ +// eslint-disable-next-line @typescript-eslint/no-unused-vars +const faceDB = () => (_faceDB ??= openFaceDB()); /** * Save the given {@link faceIndex} locally. From 0cae667b44f0071cac8a4c7ccdaf06e478e9736c Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 29 May 2024 12:14:18 +0530 Subject: [PATCH 108/354] Add a close Ref: https://www.w3.org/TR/IndexedDB-2/ --- web/apps/photos/src/services/face/db.ts | 44 ++++++++++++++++++------- 1 file changed, 33 insertions(+), 11 deletions(-) diff --git a/web/apps/photos/src/services/face/db.ts b/web/apps/photos/src/services/face/db.ts index 14900638cd..adaf097cee 100644 --- a/web/apps/photos/src/services/face/db.ts +++ b/web/apps/photos/src/services/face/db.ts @@ -1,5 +1,5 @@ import log from "@/next/log"; -import { openDB, type DBSchema } from "idb"; +import { deleteDB, openDB, type DBSchema } from "idb"; import type { FaceIndex } from "./types"; /** @@ -78,8 +78,8 @@ interface FileStatus { * * Note that this is module specific state, so the main thread and each worker * thread that calls the functions in this module will get their own independent - * connection. When we logout, the workers will presumably get resurrected and - * close their connections, while the connection kept by the main thread will be + * connection. To ensure that all connections get torn down correctly, we need + * to call closeFaceDBConnection * used to delete the database in {@link clearFaceData}. */ let _faceDB: ReturnType | undefined; @@ -123,6 +123,35 @@ const openFaceDB = async () => { // eslint-disable-next-line @typescript-eslint/no-unused-vars const faceDB = () => (_faceDB ??= openFaceDB()); +/** + * Close the face DB connection (if any) opened by this module. + * + * To ensure proper teardown of the DB connections, this function must be called + * at least once by any execution context that has called any of the other + * functions in this module. + */ +export const closeFaceDBConnectionsIfNeeded = async () => { + try { + if (_faceDB) (await _faceDB).close(); + } finally { + _faceDB = undefined; + } +}; + +/** + * Clear any data stored by the face module. + * + * Meant to be called during logout. + */ +export const clearFaceData = () => + deleteDB("face", { + blocked() { + log.warn( + "Waiting for an existing client to close their connection so that we can delete the face DB", + ); + }, + }); + /** * Save the given {@link faceIndex} locally. * @@ -133,7 +162,7 @@ const faceDB = () => (_faceDB ??= openFaceDB()); * performed, the existing entry is unconditionally overwritten). */ // eslint-disable-next-line @typescript-eslint/no-unused-vars -export const saveFaceIndex = (faceIndex: FaceIndex) => {}; +export const saveFaceIndex = async (faceIndex: FaceIndex) => {}; /** * Record the existence of a file so that entities in the face indexing universe @@ -160,10 +189,3 @@ export const addFileEntry = (fileID: string) => {}; */ // eslint-disable-next-line @typescript-eslint/no-unused-vars export const markIndexingFailed = (fileID: string) => {}; - -/** - * Clear any data stored by the face module. - * - * Meant to be called during logout. - */ -export const clearFaceData = () => {}; From 9c60fe6f3fd258d385cdc0f3abee4e29086fed10 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 29 May 2024 12:22:58 +0530 Subject: [PATCH 109/354] logout --- web/apps/photos/src/services/face/db.ts | 10 ++++++---- web/apps/photos/src/services/logout.ts | 7 +++++++ 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/web/apps/photos/src/services/face/db.ts b/web/apps/photos/src/services/face/db.ts index adaf097cee..5c23bb19f6 100644 --- a/web/apps/photos/src/services/face/db.ts +++ b/web/apps/photos/src/services/face/db.ts @@ -79,8 +79,8 @@ interface FileStatus { * Note that this is module specific state, so the main thread and each worker * thread that calls the functions in this module will get their own independent * connection. To ensure that all connections get torn down correctly, we need - * to call closeFaceDBConnection - * used to delete the database in {@link clearFaceData}. + * to call {@link closeFaceDBConnectionsIfNeeded} from both the main thread and + * all the worker threads that use this module. */ let _faceDB: ReturnType | undefined; @@ -143,14 +143,16 @@ export const closeFaceDBConnectionsIfNeeded = async () => { * * Meant to be called during logout. */ -export const clearFaceData = () => - deleteDB("face", { +export const clearFaceData = async () => { + await closeFaceDBConnectionsIfNeeded(); + return deleteDB("face", { blocked() { log.warn( "Waiting for an existing client to close their connection so that we can delete the face DB", ); }, }); +}; /** * Save the given {@link faceIndex} locally. diff --git a/web/apps/photos/src/services/logout.ts b/web/apps/photos/src/services/logout.ts index a6b155c8c2..9b664bf579 100644 --- a/web/apps/photos/src/services/logout.ts +++ b/web/apps/photos/src/services/logout.ts @@ -3,6 +3,7 @@ import { accountLogout } from "@ente/accounts/services/logout"; import { clipService } from "services/clip-service"; import DownloadManager from "./download"; import exportService from "./export"; +import { clearFaceData } from "./face/db"; import mlWorkManager from "./machineLearning/mlWorkManager"; /** @@ -35,6 +36,12 @@ export const photosLogout = async () => { log.error("Ignoring error during logout (ML)", e); } + try { + await clearFaceData(); + } catch (e) { + log.error("Ignoring error during logout (face)", e); + } + try { exportService.disableContinuousExport(); } catch (e) { From f0f3af96d1bcf227cc55221ceff95d31da063d78 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 29 May 2024 12:25:28 +0530 Subject: [PATCH 110/354] dedup --- web/apps/photos/src/services/logout.ts | 15 +++++++++------ web/packages/accounts/services/logout.ts | 15 +++++++++------ 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/web/apps/photos/src/services/logout.ts b/web/apps/photos/src/services/logout.ts index 9b664bf579..33854f5578 100644 --- a/web/apps/photos/src/services/logout.ts +++ b/web/apps/photos/src/services/logout.ts @@ -14,18 +14,21 @@ import mlWorkManager from "./machineLearning/mlWorkManager"; * See: [Note: Do not throw during logout]. */ export const photosLogout = async () => { + const ignoreError = (e: unknown, label: string) => + log.error(`Ignoring error during logout (${label})`, e); + await accountLogout(); try { await DownloadManager.logout(); } catch (e) { - log.error("Ignoring error during logout (download)", e); + ignoreError("download", e); } try { await clipService.logout(); } catch (e) { - log.error("Ignoring error during logout (CLIP)", e); + ignoreError("CLIP", e); } const electron = globalThis.electron; @@ -33,25 +36,25 @@ export const photosLogout = async () => { try { await mlWorkManager.logout(); } catch (e) { - log.error("Ignoring error during logout (ML)", e); + ignoreError("ML", e); } try { await clearFaceData(); } catch (e) { - log.error("Ignoring error during logout (face)", e); + ignoreError("face", e); } try { exportService.disableContinuousExport(); } catch (e) { - log.error("Ignoring error during logout (export)", e); + ignoreError("export", e); } try { await electron?.logout(); } catch (e) { - log.error("Ignoring error during logout (electron)", e); + ignoreError("electron", e); } } }; diff --git a/web/packages/accounts/services/logout.ts b/web/packages/accounts/services/logout.ts index 1858ec7cc3..d70b1c36b3 100644 --- a/web/packages/accounts/services/logout.ts +++ b/web/packages/accounts/services/logout.ts @@ -17,34 +17,37 @@ import { logout as remoteLogout } from "../api/user"; * gets in an unexpected state. */ export const accountLogout = async () => { + const ignoreError = (e: unknown, label: string) => + log.error(`Ignoring error during logout (${label})`, e); + try { await remoteLogout(); } catch (e) { - log.error("Ignoring error during logout (remote)", e); + ignoreError("remote", e); } try { InMemoryStore.clear(); } catch (e) { - log.error("Ignoring error during logout (in-memory store)", e); + ignoreError("in-memory store", e); } try { clearKeys(); } catch (e) { - log.error("Ignoring error during logout (session store)", e); + ignoreError("session store", e); } try { clearData(); } catch (e) { - log.error("Ignoring error during logout (local storage)", e); + ignoreError("local storage", e); } try { await localForage.clear(); } catch (e) { - log.error("Ignoring error during logout (local forage)", e); + ignoreError("local forage", e); } try { await clearBlobCaches(); } catch (e) { - log.error("Ignoring error during logout (cache)", e); + ignoreError("cache", e); } }; From bb46e98e857e554e660aba379a0285740367be8f Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 29 May 2024 12:28:56 +0530 Subject: [PATCH 111/354] Desktop --- desktop/src/main/services/logout.ts | 9 ++++++--- web/apps/photos/src/services/logout.ts | 2 +- web/packages/accounts/services/logout.ts | 2 +- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/desktop/src/main/services/logout.ts b/desktop/src/main/services/logout.ts index e6cb7666ca..37e73309a4 100644 --- a/desktop/src/main/services/logout.ts +++ b/desktop/src/main/services/logout.ts @@ -12,19 +12,22 @@ import { watchReset } from "./watch"; * See: [Note: Do not throw during logout]. */ export const logout = (watcher: FSWatcher) => { + const ignoreError = (label: string, e: unknown) => + log.error(`Ignoring error during logout (${label})`, e); + try { watchReset(watcher); } catch (e) { - log.error("Ignoring error during logout (FS watch)", e); + ignoreError("FS watch", e); } try { clearConvertToMP4Results(); } catch (e) { - log.error("Ignoring error during logout (convert-to-mp4)", e); + ignoreError("convert-to-mp4", e); } try { clearStores(); } catch (e) { - log.error("Ignoring error during logout (native stores)", e); + ignoreError("native stores", e); } }; diff --git a/web/apps/photos/src/services/logout.ts b/web/apps/photos/src/services/logout.ts index 33854f5578..4e09516dee 100644 --- a/web/apps/photos/src/services/logout.ts +++ b/web/apps/photos/src/services/logout.ts @@ -14,7 +14,7 @@ import mlWorkManager from "./machineLearning/mlWorkManager"; * See: [Note: Do not throw during logout]. */ export const photosLogout = async () => { - const ignoreError = (e: unknown, label: string) => + const ignoreError = (label: string, e: unknown) => log.error(`Ignoring error during logout (${label})`, e); await accountLogout(); diff --git a/web/packages/accounts/services/logout.ts b/web/packages/accounts/services/logout.ts index d70b1c36b3..7a150384db 100644 --- a/web/packages/accounts/services/logout.ts +++ b/web/packages/accounts/services/logout.ts @@ -17,7 +17,7 @@ import { logout as remoteLogout } from "../api/user"; * gets in an unexpected state. */ export const accountLogout = async () => { - const ignoreError = (e: unknown, label: string) => + const ignoreError = (label: string, e: unknown) => log.error(`Ignoring error during logout (${label})`, e); try { From 431cd3935886b05b0a841be080890e8f5adfe743 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 29 May 2024 13:00:02 +0530 Subject: [PATCH 112/354] Save --- web/apps/photos/src/services/face/db.ts | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/web/apps/photos/src/services/face/db.ts b/web/apps/photos/src/services/face/db.ts index 5c23bb19f6..0dda336481 100644 --- a/web/apps/photos/src/services/face/db.ts +++ b/web/apps/photos/src/services/face/db.ts @@ -36,7 +36,7 @@ interface FaceDBSchema extends DBSchema { interface FileStatus { /** The ID of the {@link EnteFile} whose indexing status we represent. */ - fileID: string; + fileID: number; /** * `1` if we have indexed a file with this {@link fileID}, `0` otherwise. * @@ -163,8 +163,21 @@ export const clearFaceData = async () => { * This function adds a new entry, overwriting any existing ones (No merging is * performed, the existing entry is unconditionally overwritten). */ -// eslint-disable-next-line @typescript-eslint/no-unused-vars -export const saveFaceIndex = async (faceIndex: FaceIndex) => {}; +export const saveFaceIndex = async (faceIndex: FaceIndex) => { + const db = await faceDB(); + const tx = db.transaction(["face-index", "file-status"], "readwrite"); + const indexStore = tx.objectStore("face-index"); + const statusStore = tx.objectStore("file-status"); + return Promise.all([ + indexStore.put(faceIndex), + statusStore.put({ + fileID: faceIndex.fileID, + isIndexed: 1, + failureCount: 0 + }), + tx.done + ]) +}; /** * Record the existence of a file so that entities in the face indexing universe From 34d4aeaf56aa1ef60b670060dc44279eed3bc777 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 29 May 2024 13:04:09 +0530 Subject: [PATCH 113/354] file entry --- web/apps/photos/src/services/face/db.ts | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/web/apps/photos/src/services/face/db.ts b/web/apps/photos/src/services/face/db.ts index 0dda336481..d3d83902cf 100644 --- a/web/apps/photos/src/services/face/db.ts +++ b/web/apps/photos/src/services/face/db.ts @@ -24,11 +24,11 @@ import type { FaceIndex } from "./types"; */ interface FaceDBSchema extends DBSchema { "face-index": { - key: string; + key: number; value: FaceIndex; }; "file-status": { - key: string; + key: number; value: FileStatus; indexes: { isIndexed: number }; }; @@ -173,10 +173,10 @@ export const saveFaceIndex = async (faceIndex: FaceIndex) => { statusStore.put({ fileID: faceIndex.fileID, isIndexed: 1, - failureCount: 0 + failureCount: 0, }), - tx.done - ]) + tx.done, + ]); }; /** @@ -190,8 +190,18 @@ export const saveFaceIndex = async (faceIndex: FaceIndex) => { * {@link saveFaceIndex} called with the result), its existing status remains * unperturbed. */ -// eslint-disable-next-line @typescript-eslint/no-unused-vars -export const addFileEntry = (fileID: string) => {}; +export const addFileEntry = async (fileID: number) => { + const db = await faceDB(); + const tx = db.transaction("file-status", "readwrite"); + if ((await tx.store.getKey(fileID)) === undefined) { + await tx.store.put({ + fileID, + isIndexed: 0, + failureCount: 0, + }); + } + return tx.done; +}; /** * Increment the failure count associated with the given {@link fileID}. From 8dd0d58319043173b14038eae4e2242d0b03d05f Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 29 May 2024 13:06:52 +0530 Subject: [PATCH 114/354] tick --- web/apps/photos/src/services/face/db.ts | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/web/apps/photos/src/services/face/db.ts b/web/apps/photos/src/services/face/db.ts index d3d83902cf..56ab5d8368 100644 --- a/web/apps/photos/src/services/face/db.ts +++ b/web/apps/photos/src/services/face/db.ts @@ -212,5 +212,15 @@ export const addFileEntry = async (fileID: number) => { * and its failure count is set to 1. Otherwise the failure count of the * existing entry is incremented. */ -// eslint-disable-next-line @typescript-eslint/no-unused-vars -export const markIndexingFailed = (fileID: string) => {}; +export const markIndexingFailed = async (fileID: number) => { + const db = await faceDB(); + const tx = db.transaction("file-status", "readwrite"); + const status = (await tx.store.get(fileID)) ?? { + fileID, + isIndexed: 0, + failureCount: 0, + }; + status.failureCount = status.failureCount + 1; + await tx.store.put(status); + return tx.done; +}; From cee093c214dc2eaa36c2e7991da8fd2ee01e5cb7 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 29 May 2024 13:22:40 +0530 Subject: [PATCH 115/354] query --- web/apps/photos/src/services/face/db.ts | 60 ++++++++++++++----------- 1 file changed, 34 insertions(+), 26 deletions(-) diff --git a/web/apps/photos/src/services/face/db.ts b/web/apps/photos/src/services/face/db.ts index 56ab5d8368..fa4f8dd63f 100644 --- a/web/apps/photos/src/services/face/db.ts +++ b/web/apps/photos/src/services/face/db.ts @@ -30,7 +30,7 @@ interface FaceDBSchema extends DBSchema { "file-status": { key: number; value: FileStatus; - indexes: { isIndexed: number }; + indexes: { isIndexable: number }; }; } @@ -38,15 +38,11 @@ interface FileStatus { /** The ID of the {@link EnteFile} whose indexing status we represent. */ fileID: number; /** - * `1` if we have indexed a file with this {@link fileID}, `0` otherwise. - * - * It is guaranteed that "face-index" will have an entry for the same - * {@link fileID} if and only if {@link isIndexed} is `1`. + * `1` if this file needs to be indexed, `0` otherwise. * * > Somewhat confusingly, we also have a (IndexedDB) "index" on this field. - * That (IDB) index allows us to effectively select {@link fileIDs} that - * still need indexing (where {@link isIndexed} is not `1`), so it is all - * sensible, just that if I say the word "index" one more time... + * That (IDB) index allows us to efficiently select {@link fileIDs} that + * still need indexing (i.e. entries where {@link isIndexed} is `1`). * * [Note: Boolean IndexedDB indexes]. * @@ -56,12 +52,13 @@ interface FileStatus { * As a workaround, we use numeric fields where `0` denotes `false` and `1` * denotes `true`. */ - isIndexed: number; + isIndexable: number; /** * The number of times attempts to index this file failed. * * This is guaranteed to be `0` for files which have already been - * sucessfully indexed (i.e. files for which `isIndexed` is true). + * sucessfully indexed (i.e. files for which `isIndexable` is 0 and which + * have a corresponding entry in the "face-index" object store). */ failureCount: number; } @@ -77,10 +74,10 @@ interface FileStatus { * version of the schema). * * Note that this is module specific state, so the main thread and each worker - * thread that calls the functions in this module will get their own independent - * connection. To ensure that all connections get torn down correctly, we need - * to call {@link closeFaceDBConnectionsIfNeeded} from both the main thread and - * all the worker threads that use this module. + * thread that calls the functions in this module will have their own promises. + * To ensure that all connections get torn down correctly, we need to call + * {@link closeFaceDBConnectionsIfNeeded} from both the main thread and all the + * worker threads that use this module. */ let _faceDB: ReturnType | undefined; @@ -90,11 +87,9 @@ const openFaceDB = async () => { log.info(`Upgrading face DB ${oldVersion} => ${newVersion}`); if (oldVersion < 1) { db.createObjectStore("face-index", { keyPath: "fileID" }); - - const statusStore = db.createObjectStore("file-status", { + db.createObjectStore("file-status", { keyPath: "fileID", - }); - statusStore.createIndex("isIndexed", "isIndexed"); + }).createIndex("isIndexable", "isIndexable"); } }, blocking() { @@ -172,7 +167,7 @@ export const saveFaceIndex = async (faceIndex: FaceIndex) => { indexStore.put(faceIndex), statusStore.put({ fileID: faceIndex.fileID, - isIndexed: 1, + isIndexable: 0, failureCount: 0, }), tx.done, @@ -196,13 +191,27 @@ export const addFileEntry = async (fileID: number) => { if ((await tx.store.getKey(fileID)) === undefined) { await tx.store.put({ fileID, - isIndexed: 0, + isIndexable: 1, failureCount: 0, }); } return tx.done; }; +/** + * Return a list of fileIDs that need to be indexed. + * + * This list is from the universe of the file IDs that the face DB knows about + * (can use {@link addFileEntry} to inform it about new files). From this + * universe, we filter out fileIDs the files corresponding to which have already + * been indexed, or for which we attempted indexing but failed. + */ +export const unindexedFileIDs = async () => { + const db = await faceDB(); + const tx = db.transaction("file-status", "readonly"); + return tx.store.index("isIndexable").getAllKeys(IDBKeyRange.only(1)); +}; + /** * Increment the failure count associated with the given {@link fileID}. * @@ -215,12 +224,11 @@ export const addFileEntry = async (fileID: number) => { export const markIndexingFailed = async (fileID: number) => { const db = await faceDB(); const tx = db.transaction("file-status", "readwrite"); - const status = (await tx.store.get(fileID)) ?? { + const failureCount = ((await tx.store.get(fileID)).failureCount ?? 0) + 1; + await tx.store.put({ fileID, - isIndexed: 0, - failureCount: 0, - }; - status.failureCount = status.failureCount + 1; - await tx.store.put(status); + isIndexable: 0, + failureCount, + }); return tx.done; }; From b3a0bc624bf8b6cfb4457f5df58957060c533258 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 29 May 2024 13:25:41 +0530 Subject: [PATCH 116/354] lf --- web/apps/photos/src/services/face/crop.ts | 3 ++- web/apps/photos/src/services/face/db.ts | 1 - web/apps/photos/src/services/face/remote.ts | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/web/apps/photos/src/services/face/crop.ts b/web/apps/photos/src/services/face/crop.ts index d4d3753825..faf7f0ac9b 100644 --- a/web/apps/photos/src/services/face/crop.ts +++ b/web/apps/photos/src/services/face/crop.ts @@ -1,5 +1,6 @@ import { blobCache } from "@/next/blob-cache"; -import type { Box, Face, FaceAlignment } from "./types-old"; +import type { Box } from "./types"; +import type { Face, FaceAlignment } from "./types-old"; export const saveFaceCrop = async (imageBitmap: ImageBitmap, face: Face) => { const faceCrop = extractFaceCrop(imageBitmap, face.alignment); diff --git a/web/apps/photos/src/services/face/db.ts b/web/apps/photos/src/services/face/db.ts index fa4f8dd63f..1128395237 100644 --- a/web/apps/photos/src/services/face/db.ts +++ b/web/apps/photos/src/services/face/db.ts @@ -115,7 +115,6 @@ const openFaceDB = async () => { /** * @returns a lazily created, cached connection to the face DB. */ -// eslint-disable-next-line @typescript-eslint/no-unused-vars const faceDB = () => (_faceDB ??= openFaceDB()); /** diff --git a/web/apps/photos/src/services/face/remote.ts b/web/apps/photos/src/services/face/remote.ts index 32d0fddad8..36ef724bf3 100644 --- a/web/apps/photos/src/services/face/remote.ts +++ b/web/apps/photos/src/services/face/remote.ts @@ -2,7 +2,8 @@ import log from "@/next/log"; import ComlinkCryptoWorker from "@ente/shared/crypto"; import { putEmbedding } from "services/embeddingService"; import type { EnteFile } from "types/file"; -import type { Face, FaceDetection, MlFileData, Point } from "./types-old"; +import type { Point } from "./types"; +import type { Face, FaceDetection, MlFileData } from "./types-old"; export const putFaceEmbedding = async ( enteFile: EnteFile, From f8f2bae1730f2f87a7587ca91d1791d153257e32 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Wed, 29 May 2024 13:46:06 +0530 Subject: [PATCH 117/354] Log error during init --- mobile/lib/main.dart | 157 ++++++++++++++++++++++--------------------- 1 file changed, 81 insertions(+), 76 deletions(-) diff --git a/mobile/lib/main.dart b/mobile/lib/main.dart index a3a1b36f6a..5418afed89 100644 --- a/mobile/lib/main.dart +++ b/mobile/lib/main.dart @@ -181,86 +181,91 @@ void _headlessTaskHandler(HeadlessTask task) { } Future _init(bool isBackground, {String via = ''}) async { - bool initComplete = false; - Future.delayed(const Duration(seconds: 15), () { - if (!initComplete && !isBackground) { - sendLogsForInit( - "support@ente.io", - "Stuck on splash screen for >= 15 seconds", - null, - ); - } - }); - _isProcessRunning = true; - _logger.info("Initializing... inBG =$isBackground via: $via"); - final SharedPreferences preferences = await SharedPreferences.getInstance(); - - await _logFGHeartBeatInfo(); - unawaited(_scheduleHeartBeat(preferences, isBackground)); - AppLifecycleService.instance.init(preferences); - if (isBackground) { - AppLifecycleService.instance.onAppInBackground('init via: $via'); - } else { - AppLifecycleService.instance.onAppInForeground('init via: $via'); - } - // Start workers asynchronously. No need to wait for them to start - Computer.shared().turnOn(workersCount: 4).ignore(); - CryptoUtil.init(); - await Configuration.instance.init(); - await NetworkClient.instance.init(); - ServiceLocator.instance.init(preferences, NetworkClient.instance.enteDio); - await UserService.instance.init(); - await EntityService.instance.init(); - LocationService.instance.init(preferences); - - await UserRemoteFlagService.instance.init(); - await UpdateService.instance.init(); - BillingService.instance.init(); - await CollectionsService.instance.init(preferences); - FavoritesService.instance.initFav().ignore(); - await FileUploader.instance.init(preferences, isBackground); - await LocalSyncService.instance.init(preferences); - TrashSyncService.instance.init(preferences); - RemoteSyncService.instance.init(preferences); - await SyncService.instance.init(preferences); - MemoriesService.instance.init(preferences); - LocalSettings.instance.init(preferences); - LocalFileUpdateService.instance.init(preferences); - SearchService.instance.init(); - StorageBonusService.instance.init(preferences); - RemoteFileMLService.instance.init(preferences); - if (!isBackground && - Platform.isAndroid && - await HomeWidgetService.instance.countHomeWidgets() == 0) { - unawaited(HomeWidgetService.instance.initHomeWidget()); - } - - if (Platform.isIOS) { - // ignore: unawaited_futures - PushService.instance.init().then((_) { - FirebaseMessaging.onBackgroundMessage( - _firebaseMessagingBackgroundHandler, - ); + try { + bool initComplete = false; + Future.delayed(const Duration(seconds: 15), () { + if (!initComplete && !isBackground) { + sendLogsForInit( + "support@ente.io", + "Stuck on splash screen for >= 15 seconds", + null, + ); + } }); - } + _isProcessRunning = true; + _logger.info("Initializing... inBG =$isBackground via: $via"); + final SharedPreferences preferences = await SharedPreferences.getInstance(); - unawaited(SemanticSearchService.instance.init()); - MachineLearningController.instance.init(); - if (flagService.faceSearchEnabled) { - unawaited(FaceMlService.instance.init()); - } else { - if (LocalSettings.instance.isFaceIndexingEnabled) { - unawaited(LocalSettings.instance.toggleFaceIndexing()); + await _logFGHeartBeatInfo(); + unawaited(_scheduleHeartBeat(preferences, isBackground)); + AppLifecycleService.instance.init(preferences); + if (isBackground) { + AppLifecycleService.instance.onAppInBackground('init via: $via'); + } else { + AppLifecycleService.instance.onAppInForeground('init via: $via'); } - } - PersonService.init( - EntityService.instance, - FaceMLDataDB.instance, - preferences, - ); + // Start workers asynchronously. No need to wait for them to start + Computer.shared().turnOn(workersCount: 4).ignore(); + CryptoUtil.init(); + await Configuration.instance.init(); + await NetworkClient.instance.init(); + ServiceLocator.instance.init(preferences, NetworkClient.instance.enteDio); + await UserService.instance.init(); + await EntityService.instance.init(); + LocationService.instance.init(preferences); - initComplete = true; - _logger.info("Initialization done"); + await UserRemoteFlagService.instance.init(); + await UpdateService.instance.init(); + BillingService.instance.init(); + await CollectionsService.instance.init(preferences); + FavoritesService.instance.initFav().ignore(); + await FileUploader.instance.init(preferences, isBackground); + await LocalSyncService.instance.init(preferences); + TrashSyncService.instance.init(preferences); + RemoteSyncService.instance.init(preferences); + await SyncService.instance.init(preferences); + MemoriesService.instance.init(preferences); + LocalSettings.instance.init(preferences); + LocalFileUpdateService.instance.init(preferences); + SearchService.instance.init(); + StorageBonusService.instance.init(preferences); + RemoteFileMLService.instance.init(preferences); + if (!isBackground && + Platform.isAndroid && + await HomeWidgetService.instance.countHomeWidgets() == 0) { + unawaited(HomeWidgetService.instance.initHomeWidget()); + } + + if (Platform.isIOS) { + // ignore: unawaited_futures + PushService.instance.init().then((_) { + FirebaseMessaging.onBackgroundMessage( + _firebaseMessagingBackgroundHandler, + ); + }); + } + + unawaited(SemanticSearchService.instance.init()); + MachineLearningController.instance.init(); + if (flagService.faceSearchEnabled) { + unawaited(FaceMlService.instance.init()); + } else { + if (LocalSettings.instance.isFaceIndexingEnabled) { + unawaited(LocalSettings.instance.toggleFaceIndexing()); + } + } + PersonService.init( + EntityService.instance, + FaceMLDataDB.instance, + preferences, + ); + + initComplete = true; + _logger.info("Initialization done"); + } catch (e, s) { + _logger.severe("Error in init", e, s); + rethrow; + } } Future _sync(String caller) async { From 979730d7404987721e76d7ce1286a1b4a81abd66 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Wed, 29 May 2024 13:46:39 +0530 Subject: [PATCH 118/354] [mob][photos] Small refactor FaceClusteringService --- .../face_clustering_service.dart | 1063 ++++++++--------- .../face_ml/face_ml_service.dart | 113 +- .../face_ml/feedback/cluster_feedback.dart | 206 ++-- .../lib/ui/viewer/people/cluster_app_bar.dart | 6 +- 4 files changed, 649 insertions(+), 739 deletions(-) diff --git a/mobile/lib/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart b/mobile/lib/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart index 35a25748ac..84b180eb8b 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart @@ -40,18 +40,24 @@ enum ClusterOperation { linearIncrementalClustering } class ClusteringResult { final Map newFaceIdToCluster; - final Map>? newClusterIdToFaceIds; - final Map? newClusterSummaries; + final Map> newClusterIdToFaceIds; + final Map newClusterSummaries; bool get isEmpty => newFaceIdToCluster.isEmpty; - - bool get hasAllResults => newClusterSummaries != null && newClusterIdToFaceIds != null; - + ClusteringResult({ required this.newFaceIdToCluster, - this.newClusterSummaries, - this.newClusterIdToFaceIds, + required this.newClusterSummaries, + required this.newClusterIdToFaceIds, }); + + factory ClusteringResult.empty() { + return ClusteringResult( + newFaceIdToCluster: {}, + newClusterIdToFaceIds: {}, + newClusterSummaries: {}, + ); + } } class FaceClusteringService { @@ -59,7 +65,7 @@ class FaceClusteringService { final _computer = Computer.shared(); Timer? _inactivityTimer; - final Duration _inactivityDuration = const Duration(minutes: 3); + final Duration _inactivityDuration = const Duration(minutes: 2); int _activeTasks = 0; final _initLock = Lock(); @@ -82,7 +88,7 @@ class FaceClusteringService { static final instance = FaceClusteringService._privateConstructor(); factory FaceClusteringService() => instance; - Future init() async { + Future _initIsolate() async { return _initLock.synchronized(() async { if (isSpawned) return; @@ -104,9 +110,9 @@ class FaceClusteringService { }); } - Future ensureSpawned() async { + Future _ensureSpawnedIsolate() async { if (!isSpawned) { - await init(); + await _initIsolate(); } } @@ -124,7 +130,7 @@ class FaceClusteringService { try { switch (function) { case ClusterOperation.linearIncrementalClustering: - final result = FaceClusteringService.runLinearClustering(args); + final ClusteringResult result = _runLinearClustering(args); sendPort.send(result); break; } @@ -138,7 +144,7 @@ class FaceClusteringService { Future _runInIsolate( (ClusterOperation, Map) message, ) async { - await ensureSpawned(); + await _ensureSpawnedIsolate(); _resetInactivityTimer(); final completer = Completer(); final answerPort = ReceivePort(); @@ -178,13 +184,13 @@ class FaceClusteringService { _logger.info( 'Clustering Isolate has been inactive for ${_inactivityDuration.inSeconds} seconds with no tasks running. Killing isolate.', ); - dispose(); + _dispose(); } }); } /// Disposes the isolate worker. - void dispose() { + void _dispose() { if (!isSpawned) return; isSpawned = false; @@ -193,10 +199,10 @@ class FaceClusteringService { _inactivityTimer?.cancel(); } - /// Runs the clustering algorithm [runLinearClustering] on the given [input], in an isolate. + /// Runs the clustering algorithm [_runLinearClustering] on the given [input], in an isolate. /// /// Returns the clustering result, which is a list of clusters, where each cluster is a list of indices of the dataset. - Future predictLinear( + Future predictLinearIsolate( Set input, { Map? fileIDToCreationTime, double distanceThreshold = kRecommendedDistanceThreshold, @@ -225,7 +231,7 @@ class FaceClusteringService { final stopwatchClustering = Stopwatch()..start(); // final Map faceIdToCluster = // await _runLinearClusteringInComputer(input); - final ClusteringResult? faceIdToCluster = await _runInIsolate( + final ClusteringResult faceIdToCluster = await _runInIsolate( ( ClusterOperation.linearIncrementalClustering, { @@ -253,8 +259,49 @@ class FaceClusteringService { } } - /// Runs the clustering algorithm [runLinearClustering] on the given [input], in computer, without any dynamic thresholding - Future predictLinearComputer( + Future predictWithinClusterComputer( + Map input, { + Map? fileIDToCreationTime, + Map oldClusterSummaries = const {}, + double distanceThreshold = kRecommendedDistanceThreshold, + }) async { + _logger.info( + '`predictWithinClusterComputer` called with ${input.length} faces and distance threshold $distanceThreshold', + ); + try { + if (input.length < 500) { + final mergeThreshold = distanceThreshold; + _logger.info( + 'Running complete clustering on ${input.length} faces with distance threshold $mergeThreshold', + ); + final ClusteringResult clusterResult = await predictCompleteComputer( + input, + fileIDToCreationTime: fileIDToCreationTime, + oldClusterSummaries: oldClusterSummaries, + distanceThreshold: distanceThreshold - 0.08, + mergeThreshold: mergeThreshold, + ); + return clusterResult; + } else { + _logger.info( + 'Running linear clustering on ${input.length} faces with distance threshold $distanceThreshold', + ); + final ClusteringResult clusterResult = await predictLinearComputer( + input, + fileIDToCreationTime: fileIDToCreationTime, + oldClusterSummaries: oldClusterSummaries, + distanceThreshold: distanceThreshold, + ); + return clusterResult; + } + } catch (e, s) { + _logger.severe(e, s); + rethrow; + } + } + + /// Runs the clustering algorithm [_runLinearClustering] on the given [input], in computer, without any dynamic thresholding + Future predictLinearComputer( Map input, { Map? fileIDToCreationTime, required Map oldClusterSummaries, @@ -264,7 +311,7 @@ class FaceClusteringService { _logger.warning( "Linear Clustering dataset of embeddings is empty, returning empty list.", ); - return null; + return ClusteringResult.empty(); } // Clustering inside the isolate @@ -289,7 +336,7 @@ class FaceClusteringService { .toSet(); final startTime = DateTime.now(); final faceIdToCluster = await _computer.compute( - runLinearClustering, + _runLinearClustering, param: { "input": clusteringInput, "fileIDToCreationTime": fileIDToCreationTime, @@ -311,7 +358,7 @@ class FaceClusteringService { } } - /// Runs the clustering algorithm [runCompleteClustering] on the given [input], in computer. + /// Runs the clustering algorithm [_runCompleteClustering] on the given [input], in computer. /// /// WARNING: Only use on small datasets, as it is not optimized for large datasets. Future predictCompleteComputer( @@ -325,7 +372,7 @@ class FaceClusteringService { _logger.warning( "Complete Clustering dataset of embeddings is empty, returning empty list.", ); - return ClusteringResult(newFaceIdToCluster: {}); + return ClusteringResult.empty(); } // Clustering inside the isolate @@ -336,7 +383,7 @@ class FaceClusteringService { try { final startTime = DateTime.now(); final clusteringResult = await _computer.compute( - runCompleteClustering, + _runCompleteClustering, param: { "input": input, "fileIDToCreationTime": fileIDToCreationTime, @@ -356,521 +403,367 @@ class FaceClusteringService { rethrow; } } +} - Future predictWithinClusterComputer( - Map input, { - Map? fileIDToCreationTime, - Map oldClusterSummaries = const {}, - double distanceThreshold = kRecommendedDistanceThreshold, - }) async { - _logger.info( - '`predictWithinClusterComputer` called with ${input.length} faces and distance threshold $distanceThreshold', - ); - try { - if (input.length < 500) { - final mergeThreshold = distanceThreshold; - _logger.info( - 'Running complete clustering on ${input.length} faces with distance threshold $mergeThreshold', - ); - final result = await predictCompleteComputer( - input, - fileIDToCreationTime: fileIDToCreationTime, - oldClusterSummaries: oldClusterSummaries, - distanceThreshold: distanceThreshold - 0.08, - mergeThreshold: mergeThreshold, - ); - if (result.newFaceIdToCluster.isEmpty) return null; - return result; - } else { - _logger.info( - 'Running linear clustering on ${input.length} faces with distance threshold $distanceThreshold', - ); - final clusterResult = await predictLinearComputer( - input, - fileIDToCreationTime: fileIDToCreationTime, - oldClusterSummaries: oldClusterSummaries, - distanceThreshold: distanceThreshold, - ); - return clusterResult; - } - } catch (e, s) { - _logger.severe(e, s); - rethrow; - } - } +ClusteringResult _runLinearClustering(Map args) { + // final input = args['input'] as Map; + final input = args['input'] as Set; + final fileIDToCreationTime = args['fileIDToCreationTime'] as Map?; + final distanceThreshold = args['distanceThreshold'] as double; + final conservativeDistanceThreshold = args['conservativeDistanceThreshold'] as double; + final useDynamicThreshold = args['useDynamicThreshold'] as bool; + final offset = args['offset'] as int?; + final oldClusterSummaries = args['oldClusterSummaries'] as Map?; - static ClusteringResult? runLinearClustering(Map args) { - // final input = args['input'] as Map; - final input = args['input'] as Set; - final fileIDToCreationTime = args['fileIDToCreationTime'] as Map?; - final distanceThreshold = args['distanceThreshold'] as double; - final conservativeDistanceThreshold = args['conservativeDistanceThreshold'] as double; - final useDynamicThreshold = args['useDynamicThreshold'] as bool; - final offset = args['offset'] as int?; - final oldClusterSummaries = args['oldClusterSummaries'] as Map?; + log( + "[ClusterIsolate] ${DateTime.now()} Copied to isolate ${input.length} faces", + ); - log( - "[ClusterIsolate] ${DateTime.now()} Copied to isolate ${input.length} faces", - ); - - // Organize everything into a list of FaceInfo objects - final List faceInfos = []; - for (final face in input) { - faceInfos.add( - FaceInfo( - faceID: face.faceID, - faceScore: face.faceScore, - blurValue: face.blurValue, - badFace: face.faceScore < kMinimumQualityFaceScore || - face.blurValue < kLaplacianSoftThreshold || - (face.blurValue < kLaplacianVerySoftThreshold && - face.faceScore < kMediumQualityFaceScore) || - face.isSideways, - vEmbedding: Vector.fromList( - EVector.fromBuffer(face.embeddingBytes).values, - dtype: DType.float32, - ), - clusterId: face.clusterId, - fileCreationTime: fileIDToCreationTime?[getFileIdFromFaceId(face.faceID)], - ), - ); - } - - // Assert that the embeddings are normalized - for (final faceInfo in faceInfos) { - if (faceInfo.vEmbedding != null) { - final norm = faceInfo.vEmbedding!.norm(); - assert((norm - 1.0).abs() < 1e-5); - } - } - - if (fileIDToCreationTime != null) { - _sortFaceInfosOnCreationTime(faceInfos); - } - - // Sort the faceInfos such that the ones with null clusterId are at the end - final List facesWithClusterID = []; - final List facesWithoutClusterID = []; - for (final FaceInfo faceInfo in faceInfos) { - if (faceInfo.clusterId == null) { - facesWithoutClusterID.add(faceInfo); - } else { - facesWithClusterID.add(faceInfo); - } - } - final alreadyClusteredCount = facesWithClusterID.length; - final sortedFaceInfos = []; - sortedFaceInfos.addAll(facesWithClusterID); - sortedFaceInfos.addAll(facesWithoutClusterID); - - log( - "[ClusterIsolate] ${DateTime.now()} Clustering ${facesWithoutClusterID.length} new faces without clusterId, and $alreadyClusteredCount faces with clusterId", - ); - - // Make sure the first face has a clusterId - final int totalFaces = sortedFaceInfos.length; - int dynamicThresholdCount = 0; - - if (sortedFaceInfos.isEmpty) { - return null; - } - - // Start actual clustering - log( - "[ClusterIsolate] ${DateTime.now()} Processing $totalFaces faces in total in this round ${offset != null ? "on top of ${offset + facesWithClusterID.length} earlier processed faces" : ""}", - ); - // set current epoch time as clusterID - int clusterID = DateTime.now().microsecondsSinceEpoch; - if (facesWithClusterID.isEmpty) { - // assign a clusterID to the first face - sortedFaceInfos[0].clusterId = clusterID; - clusterID++; - } - final stopwatchClustering = Stopwatch()..start(); - for (int i = 1; i < totalFaces; i++) { - // Incremental clustering, so we can skip faces that already have a clusterId - if (sortedFaceInfos[i].clusterId != null) { - clusterID = max(clusterID, sortedFaceInfos[i].clusterId!); - continue; - } - - int closestIdx = -1; - double closestDistance = double.infinity; - late double thresholdValue; - if (useDynamicThreshold) { - thresholdValue = - sortedFaceInfos[i].badFace! ? conservativeDistanceThreshold : distanceThreshold; - if (sortedFaceInfos[i].badFace!) dynamicThresholdCount++; - } else { - thresholdValue = distanceThreshold; - } - if (i % 250 == 0) { - log("[ClusterIsolate] ${DateTime.now()} Processed ${offset != null ? i + offset : i} faces"); - } - // WARNING: The loop below is now O(n^2) so be very careful with anything you put in there! - for (int j = i - 1; j >= 0; j--) { - final double distance = - 1 - sortedFaceInfos[i].vEmbedding!.dot(sortedFaceInfos[j].vEmbedding!); - if (distance < closestDistance) { - if (sortedFaceInfos[j].badFace! && distance > conservativeDistanceThreshold) { - continue; - } - closestDistance = distance; - closestIdx = j; - } - } - - if (closestDistance < thresholdValue) { - if (sortedFaceInfos[closestIdx].clusterId == null) { - // Ideally this should never happen, but just in case log it - log( - " [ClusterIsolate] [WARNING] ${DateTime.now()} Found new cluster $clusterID", - ); - clusterID++; - sortedFaceInfos[closestIdx].clusterId = clusterID; - } - sortedFaceInfos[i].clusterId = sortedFaceInfos[closestIdx].clusterId; - } else { - clusterID++; - sortedFaceInfos[i].clusterId = clusterID; - } - } - - // Finally, assign the new clusterId to the faces - final Map newFaceIdToCluster = {}; - final newClusteredFaceInfos = sortedFaceInfos.sublist(alreadyClusteredCount); - for (final faceInfo in newClusteredFaceInfos) { - newFaceIdToCluster[faceInfo.faceID] = faceInfo.clusterId!; - } - - // Create a map of clusterId to faceIds - final Map> clusterIdToFaceIds = {}; - for (final entry in newFaceIdToCluster.entries) { - final clusterID = entry.value; - if (clusterIdToFaceIds.containsKey(clusterID)) { - clusterIdToFaceIds[clusterID]!.add(entry.key); - } else { - clusterIdToFaceIds[clusterID] = [entry.key]; - } - } - - stopwatchClustering.stop(); - log( - ' [ClusterIsolate] ${DateTime.now()} Clustering for ${sortedFaceInfos.length} embeddings executed in ${stopwatchClustering.elapsedMilliseconds}ms', - ); - if (useDynamicThreshold) { - log( - "[ClusterIsolate] ${DateTime.now()} Dynamic thresholding: $dynamicThresholdCount faces had a low face score or low blur clarity", - ); - } - - // Now calculate the mean of the embeddings for each cluster and update the cluster summaries - Map? newClusterSummaries; - if (oldClusterSummaries != null) { - newClusterSummaries = FaceClusteringService.updateClusterSummaries( - oldSummary: oldClusterSummaries, - newFaceInfos: newClusteredFaceInfos, - ); - } - - // analyze the results - // FaceClusteringService._analyzeClusterResults(sortedFaceInfos); - - return ClusteringResult( - newFaceIdToCluster: newFaceIdToCluster, - newClusterSummaries: newClusterSummaries, - newClusterIdToFaceIds: clusterIdToFaceIds, - ); - } - - static Map updateClusterSummaries({ - required Map oldSummary, - required List newFaceInfos, - }) { - final calcSummariesStart = DateTime.now(); - final Map> newClusterIdToFaceInfos = {}; - for (final faceInfo in newFaceInfos) { - if (newClusterIdToFaceInfos.containsKey(faceInfo.clusterId!)) { - newClusterIdToFaceInfos[faceInfo.clusterId!]!.add(faceInfo); - } else { - newClusterIdToFaceInfos[faceInfo.clusterId!] = [faceInfo]; - } - } - - final Map newClusterSummaries = {}; - for (final clusterId in newClusterIdToFaceInfos.keys) { - final List newEmbeddings = - newClusterIdToFaceInfos[clusterId]!.map((faceInfo) => faceInfo.vEmbedding!).toList(); - final newCount = newEmbeddings.length; - if (oldSummary.containsKey(clusterId)) { - final oldMean = Vector.fromList( - EVector.fromBuffer(oldSummary[clusterId]!.$1).values, + // Organize everything into a list of FaceInfo objects + final List faceInfos = []; + for (final face in input) { + faceInfos.add( + FaceInfo( + faceID: face.faceID, + faceScore: face.faceScore, + blurValue: face.blurValue, + badFace: face.faceScore < kMinimumQualityFaceScore || + face.blurValue < kLaplacianSoftThreshold || + (face.blurValue < kLaplacianVerySoftThreshold && + face.faceScore < kMediumQualityFaceScore) || + face.isSideways, + vEmbedding: Vector.fromList( + EVector.fromBuffer(face.embeddingBytes).values, dtype: DType.float32, - ); - final oldCount = oldSummary[clusterId]!.$2; - final oldEmbeddings = oldMean * oldCount; - newEmbeddings.add(oldEmbeddings); - final newMeanVector = newEmbeddings.reduce((a, b) => a + b) / (oldCount + newCount); - final newMeanVectorNormalized = newMeanVector / newMeanVector.norm(); - newClusterSummaries[clusterId] = ( - EVector(values: newMeanVectorNormalized.toList()).writeToBuffer(), - oldCount + newCount - ); - } else { - final newMeanVector = newEmbeddings.reduce((a, b) => a + b); - final newMeanVectorNormalized = newMeanVector / newMeanVector.norm(); - newClusterSummaries[clusterId] = - (EVector(values: newMeanVectorNormalized.toList()).writeToBuffer(), newCount); - } - } - log( - "[ClusterIsolate] ${DateTime.now()} Calculated cluster summaries in ${DateTime.now().difference(calcSummariesStart).inMilliseconds}ms", - ); - - return newClusterSummaries; - } - - static void _analyzeClusterResults(List sortedFaceInfos) { - if (!kDebugMode) return; - final stopwatch = Stopwatch()..start(); - - final Map faceIdToCluster = {}; - for (final faceInfo in sortedFaceInfos) { - faceIdToCluster[faceInfo.faceID] = faceInfo.clusterId!; - } - - // Find faceIDs that are part of a cluster which is larger than 5 and are new faceIDs - final Map clusterIdToSize = {}; - faceIdToCluster.forEach((key, value) { - if (clusterIdToSize.containsKey(value)) { - clusterIdToSize[value] = clusterIdToSize[value]! + 1; - } else { - clusterIdToSize[value] = 1; - } - }); - - // print top 10 cluster ids and their sizes based on the internal cluster id - final clusterIds = faceIdToCluster.values.toSet(); - final clusterSizes = clusterIds.map((clusterId) { - return faceIdToCluster.values.where((id) => id == clusterId).length; - }).toList(); - clusterSizes.sort(); - // find clusters whose size is greater than 1 - int oneClusterCount = 0; - int moreThan5Count = 0; - int moreThan10Count = 0; - int moreThan20Count = 0; - int moreThan50Count = 0; - int moreThan100Count = 0; - - for (int i = 0; i < clusterSizes.length; i++) { - if (clusterSizes[i] > 100) { - moreThan100Count++; - } else if (clusterSizes[i] > 50) { - moreThan50Count++; - } else if (clusterSizes[i] > 20) { - moreThan20Count++; - } else if (clusterSizes[i] > 10) { - moreThan10Count++; - } else if (clusterSizes[i] > 5) { - moreThan5Count++; - } else if (clusterSizes[i] == 1) { - oneClusterCount++; - } - } - - // print the metrics - log( - "[ClusterIsolate] Total clusters ${clusterIds.length}: \n oneClusterCount $oneClusterCount \n moreThan5Count $moreThan5Count \n moreThan10Count $moreThan10Count \n moreThan20Count $moreThan20Count \n moreThan50Count $moreThan50Count \n moreThan100Count $moreThan100Count", - ); - stopwatch.stop(); - log( - "[ClusterIsolate] Clustering additional analysis took ${stopwatch.elapsedMilliseconds} ms", - ); - } - - static ClusteringResult runCompleteClustering(Map args) { - final input = args['input'] as Map; - final fileIDToCreationTime = args['fileIDToCreationTime'] as Map?; - final distanceThreshold = args['distanceThreshold'] as double; - final mergeThreshold = args['mergeThreshold'] as double; - final oldClusterSummaries = args['oldClusterSummaries'] as Map?; - - log( - "[CompleteClustering] ${DateTime.now()} Copied to isolate ${input.length} faces for clustering", - ); - - // Organize everything into a list of FaceInfo objects - final List faceInfos = []; - for (final entry in input.entries) { - faceInfos.add( - FaceInfo( - faceID: entry.key, - vEmbedding: Vector.fromList( - EVector.fromBuffer(entry.value).values, - dtype: DType.float32, - ), - fileCreationTime: fileIDToCreationTime?[getFileIdFromFaceId(entry.key)], ), - ); - } - - if (fileIDToCreationTime != null) { - _sortFaceInfosOnCreationTime(faceInfos); - } - - if (faceInfos.isEmpty) { - ClusteringResult(newFaceIdToCluster: {}); - } - final int totalFaces = faceInfos.length; - - // Start actual clustering - log( - "[CompleteClustering] ${DateTime.now()} Processing $totalFaces faces in one single round of complete clustering", - ); - - // set current epoch time as clusterID - int clusterID = DateTime.now().microsecondsSinceEpoch; - - // Start actual clustering - final Map newFaceIdToCluster = {}; - final stopwatchClustering = Stopwatch()..start(); - for (int i = 0; i < totalFaces; i++) { - if ((i + 1) % 250 == 0) { - log("[CompleteClustering] ${DateTime.now()} Processed ${i + 1} faces"); - } - if (faceInfos[i].clusterId != null) continue; - int closestIdx = -1; - double closestDistance = double.infinity; - for (int j = 0; j < totalFaces; j++) { - if (i == j) continue; - final double distance = 1 - faceInfos[i].vEmbedding!.dot(faceInfos[j].vEmbedding!); - if (distance < closestDistance) { - closestDistance = distance; - closestIdx = j; - } - } - - if (closestDistance < distanceThreshold) { - if (faceInfos[closestIdx].clusterId == null) { - clusterID++; - faceInfos[closestIdx].clusterId = clusterID; - } - faceInfos[i].clusterId = faceInfos[closestIdx].clusterId!; - } else { - clusterID++; - faceInfos[i].clusterId = clusterID; - } - } - - // Now calculate the mean of the embeddings for each cluster - final Map> clusterIdToFaceInfos = {}; - for (final faceInfo in faceInfos) { - if (clusterIdToFaceInfos.containsKey(faceInfo.clusterId)) { - clusterIdToFaceInfos[faceInfo.clusterId]!.add(faceInfo); - } else { - clusterIdToFaceInfos[faceInfo.clusterId!] = [faceInfo]; - } - } - final Map clusterIdToMeanEmbeddingAndWeight = {}; - for (final clusterId in clusterIdToFaceInfos.keys) { - final List embeddings = - clusterIdToFaceInfos[clusterId]!.map((faceInfo) => faceInfo.vEmbedding!).toList(); - final count = clusterIdToFaceInfos[clusterId]!.length; - final Vector meanEmbedding = embeddings.reduce((a, b) => a + b) / count; - final Vector meanEmbeddingNormalized = meanEmbedding / meanEmbedding.norm(); - clusterIdToMeanEmbeddingAndWeight[clusterId] = (meanEmbeddingNormalized, count); - } - - // Now merge the clusters that are close to each other, based on mean embedding - final List<(int, int)> mergedClustersList = []; - final List clusterIds = clusterIdToMeanEmbeddingAndWeight.keys.toList(); - log(' [CompleteClustering] ${DateTime.now()} ${clusterIds.length} clusters found, now checking for merges'); - while (true) { - if (clusterIds.length < 2) break; - double distance = double.infinity; - (int, int) clusterIDsToMerge = (-1, -1); - for (int i = 0; i < clusterIds.length; i++) { - for (int j = 0; j < clusterIds.length; j++) { - if (i == j) continue; - final double newDistance = 1 - - clusterIdToMeanEmbeddingAndWeight[clusterIds[i]]! - .$1 - .dot(clusterIdToMeanEmbeddingAndWeight[clusterIds[j]]!.$1); - if (newDistance < distance) { - distance = newDistance; - clusterIDsToMerge = (clusterIds[i], clusterIds[j]); - } - } - } - if (distance < mergeThreshold) { - mergedClustersList.add(clusterIDsToMerge); - final clusterID1 = clusterIDsToMerge.$1; - final clusterID2 = clusterIDsToMerge.$2; - final mean1 = clusterIdToMeanEmbeddingAndWeight[clusterID1]!.$1; - final mean2 = clusterIdToMeanEmbeddingAndWeight[clusterID2]!.$1; - final count1 = clusterIdToMeanEmbeddingAndWeight[clusterID1]!.$2; - final count2 = clusterIdToMeanEmbeddingAndWeight[clusterID2]!.$2; - final weight1 = count1 / (count1 + count2); - final weight2 = count2 / (count1 + count2); - final weightedMean = mean1 * weight1 + mean2 * weight2; - final weightedMeanNormalized = weightedMean / weightedMean.norm(); - clusterIdToMeanEmbeddingAndWeight[clusterID1] = ( - weightedMeanNormalized, - count1 + count2, - ); - clusterIdToMeanEmbeddingAndWeight.remove(clusterID2); - clusterIds.remove(clusterID2); - } else { - break; - } - } - log(' [CompleteClustering] ${DateTime.now()} ${mergedClustersList.length} clusters merged'); - - // Now assign the new clusterId to the faces - for (final faceInfo in faceInfos) { - for (final mergedClusters in mergedClustersList) { - if (faceInfo.clusterId == mergedClusters.$2) { - faceInfo.clusterId = mergedClusters.$1; - } - } - } - - // Finally, assign the new clusterId to the faces - for (final faceInfo in faceInfos) { - newFaceIdToCluster[faceInfo.faceID] = faceInfo.clusterId!; - } - - final Map> clusterIdToFaceIds = {}; - for (final entry in newFaceIdToCluster.entries) { - final clusterID = entry.value; - if (clusterIdToFaceIds.containsKey(clusterID)) { - clusterIdToFaceIds[clusterID]!.add(entry.key); - } else { - clusterIdToFaceIds[clusterID] = [entry.key]; - } - } - - // Now calculate the mean of the embeddings for each cluster and update the cluster summaries - Map? newClusterSummaries; - if (oldClusterSummaries != null) { - newClusterSummaries = FaceClusteringService.updateClusterSummaries( - oldSummary: oldClusterSummaries, - newFaceInfos: faceInfos, - ); - } - - stopwatchClustering.stop(); - log( - ' [CompleteClustering] ${DateTime.now()} Clustering for ${faceInfos.length} embeddings executed in ${stopwatchClustering.elapsedMilliseconds}ms', - ); - - return ClusteringResult( - newFaceIdToCluster: newFaceIdToCluster, - newClusterSummaries: newClusterSummaries, - newClusterIdToFaceIds: clusterIdToFaceIds, + clusterId: face.clusterId, + fileCreationTime: fileIDToCreationTime?[getFileIdFromFaceId(face.faceID)], + ), ); } + + // Assert that the embeddings are normalized + for (final faceInfo in faceInfos) { + if (faceInfo.vEmbedding != null) { + final norm = faceInfo.vEmbedding!.norm(); + assert((norm - 1.0).abs() < 1e-5); + } + } + + if (fileIDToCreationTime != null) { + _sortFaceInfosOnCreationTime(faceInfos); + } + + // Sort the faceInfos such that the ones with null clusterId are at the end + final List facesWithClusterID = []; + final List facesWithoutClusterID = []; + for (final FaceInfo faceInfo in faceInfos) { + if (faceInfo.clusterId == null) { + facesWithoutClusterID.add(faceInfo); + } else { + facesWithClusterID.add(faceInfo); + } + } + final alreadyClusteredCount = facesWithClusterID.length; + final sortedFaceInfos = []; + sortedFaceInfos.addAll(facesWithClusterID); + sortedFaceInfos.addAll(facesWithoutClusterID); + + log( + "[ClusterIsolate] ${DateTime.now()} Clustering ${facesWithoutClusterID.length} new faces without clusterId, and $alreadyClusteredCount faces with clusterId", + ); + + // Make sure the first face has a clusterId + final int totalFaces = sortedFaceInfos.length; + int dynamicThresholdCount = 0; + + if (sortedFaceInfos.isEmpty) { + return ClusteringResult.empty(); + } + + // Start actual clustering + log( + "[ClusterIsolate] ${DateTime.now()} Processing $totalFaces faces in total in this round ${offset != null ? "on top of ${offset + facesWithClusterID.length} earlier processed faces" : ""}", + ); + // set current epoch time as clusterID + int clusterID = DateTime.now().microsecondsSinceEpoch; + if (facesWithClusterID.isEmpty) { + // assign a clusterID to the first face + sortedFaceInfos[0].clusterId = clusterID; + clusterID++; + } + final stopwatchClustering = Stopwatch()..start(); + for (int i = 1; i < totalFaces; i++) { + // Incremental clustering, so we can skip faces that already have a clusterId + if (sortedFaceInfos[i].clusterId != null) { + clusterID = max(clusterID, sortedFaceInfos[i].clusterId!); + continue; + } + + int closestIdx = -1; + double closestDistance = double.infinity; + late double thresholdValue; + if (useDynamicThreshold) { + thresholdValue = + sortedFaceInfos[i].badFace! ? conservativeDistanceThreshold : distanceThreshold; + if (sortedFaceInfos[i].badFace!) dynamicThresholdCount++; + } else { + thresholdValue = distanceThreshold; + } + if (i % 250 == 0) { + log("[ClusterIsolate] ${DateTime.now()} Processed ${offset != null ? i + offset : i} faces"); + } + // WARNING: The loop below is now O(n^2) so be very careful with anything you put in there! + for (int j = i - 1; j >= 0; j--) { + final double distance = + 1 - sortedFaceInfos[i].vEmbedding!.dot(sortedFaceInfos[j].vEmbedding!); + if (distance < closestDistance) { + if (sortedFaceInfos[j].badFace! && distance > conservativeDistanceThreshold) { + continue; + } + closestDistance = distance; + closestIdx = j; + } + } + + if (closestDistance < thresholdValue) { + if (sortedFaceInfos[closestIdx].clusterId == null) { + // Ideally this should never happen, but just in case log it + log( + " [ClusterIsolate] [WARNING] ${DateTime.now()} Found new cluster $clusterID", + ); + clusterID++; + sortedFaceInfos[closestIdx].clusterId = clusterID; + } + sortedFaceInfos[i].clusterId = sortedFaceInfos[closestIdx].clusterId; + } else { + clusterID++; + sortedFaceInfos[i].clusterId = clusterID; + } + } + + // Finally, assign the new clusterId to the faces + final Map newFaceIdToCluster = {}; + final newClusteredFaceInfos = sortedFaceInfos.sublist(alreadyClusteredCount); + for (final faceInfo in newClusteredFaceInfos) { + newFaceIdToCluster[faceInfo.faceID] = faceInfo.clusterId!; + } + + // Create a map of clusterId to faceIds + final Map> clusterIdToFaceIds = {}; + for (final entry in newFaceIdToCluster.entries) { + final clusterID = entry.value; + if (clusterIdToFaceIds.containsKey(clusterID)) { + clusterIdToFaceIds[clusterID]!.add(entry.key); + } else { + clusterIdToFaceIds[clusterID] = [entry.key]; + } + } + + stopwatchClustering.stop(); + log( + ' [ClusterIsolate] ${DateTime.now()} Clustering for ${sortedFaceInfos.length} embeddings executed in ${stopwatchClustering.elapsedMilliseconds}ms', + ); + if (useDynamicThreshold) { + log( + "[ClusterIsolate] ${DateTime.now()} Dynamic thresholding: $dynamicThresholdCount faces had a low face score or low blur clarity", + ); + } + + // Now calculate the mean of the embeddings for each cluster and update the cluster summaries + final newClusterSummaries = _updateClusterSummaries( + newFaceInfos: newClusteredFaceInfos, + oldSummary: oldClusterSummaries, + ); + + // analyze the results + // FaceClusteringService._analyzeClusterResults(sortedFaceInfos); + + return ClusteringResult( + newFaceIdToCluster: newFaceIdToCluster, + newClusterSummaries: newClusterSummaries, + newClusterIdToFaceIds: clusterIdToFaceIds, + ); +} + +ClusteringResult _runCompleteClustering(Map args) { + final input = args['input'] as Map; + final fileIDToCreationTime = args['fileIDToCreationTime'] as Map?; + final distanceThreshold = args['distanceThreshold'] as double; + final mergeThreshold = args['mergeThreshold'] as double; + final oldClusterSummaries = args['oldClusterSummaries'] as Map?; + + log( + "[CompleteClustering] ${DateTime.now()} Copied to isolate ${input.length} faces for clustering", + ); + + // Organize everything into a list of FaceInfo objects + final List faceInfos = []; + for (final entry in input.entries) { + faceInfos.add( + FaceInfo( + faceID: entry.key, + vEmbedding: Vector.fromList( + EVector.fromBuffer(entry.value).values, + dtype: DType.float32, + ), + fileCreationTime: fileIDToCreationTime?[getFileIdFromFaceId(entry.key)], + ), + ); + } + + if (fileIDToCreationTime != null) { + _sortFaceInfosOnCreationTime(faceInfos); + } + + if (faceInfos.isEmpty) { + ClusteringResult.empty(); + } + final int totalFaces = faceInfos.length; + + // Start actual clustering + log( + "[CompleteClustering] ${DateTime.now()} Processing $totalFaces faces in one single round of complete clustering", + ); + + // set current epoch time as clusterID + int clusterID = DateTime.now().microsecondsSinceEpoch; + + // Start actual clustering + final Map newFaceIdToCluster = {}; + final stopwatchClustering = Stopwatch()..start(); + for (int i = 0; i < totalFaces; i++) { + if ((i + 1) % 250 == 0) { + log("[CompleteClustering] ${DateTime.now()} Processed ${i + 1} faces"); + } + if (faceInfos[i].clusterId != null) continue; + int closestIdx = -1; + double closestDistance = double.infinity; + for (int j = 0; j < totalFaces; j++) { + if (i == j) continue; + final double distance = 1 - faceInfos[i].vEmbedding!.dot(faceInfos[j].vEmbedding!); + if (distance < closestDistance) { + closestDistance = distance; + closestIdx = j; + } + } + + if (closestDistance < distanceThreshold) { + if (faceInfos[closestIdx].clusterId == null) { + clusterID++; + faceInfos[closestIdx].clusterId = clusterID; + } + faceInfos[i].clusterId = faceInfos[closestIdx].clusterId!; + } else { + clusterID++; + faceInfos[i].clusterId = clusterID; + } + } + + // Now calculate the mean of the embeddings for each cluster + final Map> clusterIdToFaceInfos = {}; + for (final faceInfo in faceInfos) { + if (clusterIdToFaceInfos.containsKey(faceInfo.clusterId)) { + clusterIdToFaceInfos[faceInfo.clusterId]!.add(faceInfo); + } else { + clusterIdToFaceInfos[faceInfo.clusterId!] = [faceInfo]; + } + } + final Map clusterIdToMeanEmbeddingAndWeight = {}; + for (final clusterId in clusterIdToFaceInfos.keys) { + final List embeddings = + clusterIdToFaceInfos[clusterId]!.map((faceInfo) => faceInfo.vEmbedding!).toList(); + final count = clusterIdToFaceInfos[clusterId]!.length; + final Vector meanEmbedding = embeddings.reduce((a, b) => a + b) / count; + final Vector meanEmbeddingNormalized = meanEmbedding / meanEmbedding.norm(); + clusterIdToMeanEmbeddingAndWeight[clusterId] = (meanEmbeddingNormalized, count); + } + + // Now merge the clusters that are close to each other, based on mean embedding + final List<(int, int)> mergedClustersList = []; + final List clusterIds = clusterIdToMeanEmbeddingAndWeight.keys.toList(); + log(' [CompleteClustering] ${DateTime.now()} ${clusterIds.length} clusters found, now checking for merges'); + while (true) { + if (clusterIds.length < 2) break; + double distance = double.infinity; + (int, int) clusterIDsToMerge = (-1, -1); + for (int i = 0; i < clusterIds.length; i++) { + for (int j = 0; j < clusterIds.length; j++) { + if (i == j) continue; + final double newDistance = 1 - + clusterIdToMeanEmbeddingAndWeight[clusterIds[i]]! + .$1 + .dot(clusterIdToMeanEmbeddingAndWeight[clusterIds[j]]!.$1); + if (newDistance < distance) { + distance = newDistance; + clusterIDsToMerge = (clusterIds[i], clusterIds[j]); + } + } + } + if (distance < mergeThreshold) { + mergedClustersList.add(clusterIDsToMerge); + final clusterID1 = clusterIDsToMerge.$1; + final clusterID2 = clusterIDsToMerge.$2; + final mean1 = clusterIdToMeanEmbeddingAndWeight[clusterID1]!.$1; + final mean2 = clusterIdToMeanEmbeddingAndWeight[clusterID2]!.$1; + final count1 = clusterIdToMeanEmbeddingAndWeight[clusterID1]!.$2; + final count2 = clusterIdToMeanEmbeddingAndWeight[clusterID2]!.$2; + final weight1 = count1 / (count1 + count2); + final weight2 = count2 / (count1 + count2); + final weightedMean = mean1 * weight1 + mean2 * weight2; + final weightedMeanNormalized = weightedMean / weightedMean.norm(); + clusterIdToMeanEmbeddingAndWeight[clusterID1] = ( + weightedMeanNormalized, + count1 + count2, + ); + clusterIdToMeanEmbeddingAndWeight.remove(clusterID2); + clusterIds.remove(clusterID2); + } else { + break; + } + } + log(' [CompleteClustering] ${DateTime.now()} ${mergedClustersList.length} clusters merged'); + + // Now assign the new clusterId to the faces + for (final faceInfo in faceInfos) { + for (final mergedClusters in mergedClustersList) { + if (faceInfo.clusterId == mergedClusters.$2) { + faceInfo.clusterId = mergedClusters.$1; + } + } + } + + // Finally, assign the new clusterId to the faces + for (final faceInfo in faceInfos) { + newFaceIdToCluster[faceInfo.faceID] = faceInfo.clusterId!; + } + + final Map> clusterIdToFaceIds = {}; + for (final entry in newFaceIdToCluster.entries) { + final clusterID = entry.value; + if (clusterIdToFaceIds.containsKey(clusterID)) { + clusterIdToFaceIds[clusterID]!.add(entry.key); + } else { + clusterIdToFaceIds[clusterID] = [entry.key]; + } + } + + // Now calculate the mean of the embeddings for each cluster and update the cluster summaries + final newClusterSummaries = _updateClusterSummaries( + newFaceInfos: faceInfos, + oldSummary: oldClusterSummaries, + ); + + stopwatchClustering.stop(); + log( + ' [CompleteClustering] ${DateTime.now()} Clustering for ${faceInfos.length} embeddings executed in ${stopwatchClustering.elapsedMilliseconds}ms', + ); + + return ClusteringResult( + newFaceIdToCluster: newFaceIdToCluster, + newClusterSummaries: newClusterSummaries, + newClusterIdToFaceIds: clusterIdToFaceIds, + ); } /// Sort the faceInfos based on fileCreationTime, in descending order, so newest faces are first @@ -889,3 +782,107 @@ void _sortFaceInfosOnCreationTime( } }); } + +Map _updateClusterSummaries({ + required List newFaceInfos, + Map? oldSummary, +}) { + final calcSummariesStart = DateTime.now(); + final Map> newClusterIdToFaceInfos = {}; + for (final faceInfo in newFaceInfos) { + if (newClusterIdToFaceInfos.containsKey(faceInfo.clusterId!)) { + newClusterIdToFaceInfos[faceInfo.clusterId!]!.add(faceInfo); + } else { + newClusterIdToFaceInfos[faceInfo.clusterId!] = [faceInfo]; + } + } + + final Map newClusterSummaries = {}; + for (final clusterId in newClusterIdToFaceInfos.keys) { + final List newEmbeddings = + newClusterIdToFaceInfos[clusterId]!.map((faceInfo) => faceInfo.vEmbedding!).toList(); + final newCount = newEmbeddings.length; + if (oldSummary != null && oldSummary.containsKey(clusterId)) { + final oldMean = Vector.fromList( + EVector.fromBuffer(oldSummary[clusterId]!.$1).values, + dtype: DType.float32, + ); + final oldCount = oldSummary[clusterId]!.$2; + final oldEmbeddings = oldMean * oldCount; + newEmbeddings.add(oldEmbeddings); + final newMeanVector = newEmbeddings.reduce((a, b) => a + b) / (oldCount + newCount); + final newMeanVectorNormalized = newMeanVector / newMeanVector.norm(); + newClusterSummaries[clusterId] = + (EVector(values: newMeanVectorNormalized.toList()).writeToBuffer(), oldCount + newCount); + } else { + final newMeanVector = newEmbeddings.reduce((a, b) => a + b); + final newMeanVectorNormalized = newMeanVector / newMeanVector.norm(); + newClusterSummaries[clusterId] = + (EVector(values: newMeanVectorNormalized.toList()).writeToBuffer(), newCount); + } + } + log( + "[ClusterIsolate] ${DateTime.now()} Calculated cluster summaries in ${DateTime.now().difference(calcSummariesStart).inMilliseconds}ms", + ); + + return newClusterSummaries; +} + +void _analyzeClusterResults(List sortedFaceInfos) { + if (!kDebugMode) return; + final stopwatch = Stopwatch()..start(); + + final Map faceIdToCluster = {}; + for (final faceInfo in sortedFaceInfos) { + faceIdToCluster[faceInfo.faceID] = faceInfo.clusterId!; + } + + // Find faceIDs that are part of a cluster which is larger than 5 and are new faceIDs + final Map clusterIdToSize = {}; + faceIdToCluster.forEach((key, value) { + if (clusterIdToSize.containsKey(value)) { + clusterIdToSize[value] = clusterIdToSize[value]! + 1; + } else { + clusterIdToSize[value] = 1; + } + }); + + // print top 10 cluster ids and their sizes based on the internal cluster id + final clusterIds = faceIdToCluster.values.toSet(); + final clusterSizes = clusterIds.map((clusterId) { + return faceIdToCluster.values.where((id) => id == clusterId).length; + }).toList(); + clusterSizes.sort(); + // find clusters whose size is greater than 1 + int oneClusterCount = 0; + int moreThan5Count = 0; + int moreThan10Count = 0; + int moreThan20Count = 0; + int moreThan50Count = 0; + int moreThan100Count = 0; + + for (int i = 0; i < clusterSizes.length; i++) { + if (clusterSizes[i] > 100) { + moreThan100Count++; + } else if (clusterSizes[i] > 50) { + moreThan50Count++; + } else if (clusterSizes[i] > 20) { + moreThan20Count++; + } else if (clusterSizes[i] > 10) { + moreThan10Count++; + } else if (clusterSizes[i] > 5) { + moreThan5Count++; + } else if (clusterSizes[i] == 1) { + oneClusterCount++; + } + } + + // print the metrics + log( + "[ClusterIsolate] Total clusters ${clusterIds.length}: \n oneClusterCount $oneClusterCount \n moreThan5Count $moreThan5Count \n moreThan10Count $moreThan10Count \n moreThan20Count $moreThan20Count \n moreThan50Count $moreThan50Count \n moreThan100Count $moreThan100Count", + ); + stopwatch.stop(); + log( + "[ClusterIsolate] Clustering additional analysis took ${stopwatch.elapsedMilliseconds} ms", + ); +} diff --git a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart index 1297f4cac0..de4287465e 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart @@ -164,8 +164,7 @@ class FaceMlService { pauseIndexingAndClustering(); } }); - if (Platform.isIOS && - MachineLearningController.instance.isDeviceHealthy) { + if (Platform.isIOS && MachineLearningController.instance.isDeviceHealthy) { _logger.info("Starting face indexing and clustering on iOS from init"); unawaited(indexAndClusterAll()); } @@ -270,8 +269,7 @@ class FaceMlService { switch (function) { case FaceMlOperation.analyzeImage: final time = DateTime.now(); - final FaceMlResult result = - await FaceMlService.analyzeImageSync(args); + final FaceMlResult result = await FaceMlService.analyzeImageSync(args); dev.log( "`analyzeImageSync` function executed in ${DateTime.now().difference(time).inMilliseconds} ms", ); @@ -284,8 +282,7 @@ class FaceMlService { error: e, stackTrace: stackTrace, ); - sendPort - .send({'error': e.toString(), 'stackTrace': stackTrace.toString()}); + sendPort.send({'error': e.toString(), 'stackTrace': stackTrace.toString()}); } }); } @@ -369,8 +366,7 @@ class FaceMlService { await sync(forceSync: _shouldSyncPeople); - final int unclusteredFacesCount = - await FaceMLDataDB.instance.getUnclusteredFaceCount(); + final int unclusteredFacesCount = await FaceMLDataDB.instance.getUnclusteredFaceCount(); if (unclusteredFacesCount > _kForceClusteringFaceCount) { _logger.info( "There are $unclusteredFacesCount unclustered faces, doing clustering first", @@ -397,13 +393,10 @@ class FaceMlService { _isIndexingOrClusteringRunning = true; _logger.info('starting image indexing'); - final w = (kDebugMode ? EnteWatch('prepare indexing files') : null) - ?..start(); - final Map alreadyIndexedFiles = - await FaceMLDataDB.instance.getIndexedFileIds(); + final w = (kDebugMode ? EnteWatch('prepare indexing files') : null)?..start(); + final Map alreadyIndexedFiles = await FaceMLDataDB.instance.getIndexedFileIds(); w?.log('getIndexedFileIds'); - final List enteFiles = - await SearchService.instance.getAllFiles(); + final List enteFiles = await SearchService.instance.getAllFiles(); w?.log('getAllFiles'); // Make sure the image conversion isolate is spawned @@ -430,8 +423,7 @@ class FaceMlService { } } w?.log('sifting through all normal files'); - final List hiddenFiles = - await SearchService.instance.getHiddenFiles(); + final List hiddenFiles = await SearchService.instance.getHiddenFiles(); w?.log('getHiddenFiles: ${hiddenFiles.length} hidden files'); for (final EnteFile enteFile in hiddenFiles) { if (_skipAnalysisEnteFile(enteFile, alreadyIndexedFiles)) { @@ -447,8 +439,7 @@ class FaceMlService { sortedBylocalID.addAll(filesWithoutLocalID); sortedBylocalID.addAll(hiddenFilesToIndex); w?.log('preparing all files to index'); - final List> chunks = - sortedBylocalID.chunks(_embeddingFetchLimit); + final List> chunks = sortedBylocalID.chunks(_embeddingFetchLimit); int fetchedCount = 0; outerLoop: for (final chunk in chunks) { @@ -456,15 +447,13 @@ class FaceMlService { if (LocalSettings.instance.remoteFetchEnabled) { try { - final Set fileIds = - {}; // if there are duplicates here server returns 400 + final Set fileIds = {}; // if there are duplicates here server returns 400 // Try to find embeddings on the remote server for (final f in chunk) { fileIds.add(f.uploadedFileID!); } _logger.info('starting remote fetch for ${fileIds.length} files'); - final res = - await RemoteFileMLService.instance.getFilessEmbedding(fileIds); + final res = await RemoteFileMLService.instance.getFilessEmbedding(fileIds); _logger.info('fetched ${res.mlData.length} embeddings'); fetchedCount += res.mlData.length; final List faces = []; @@ -486,8 +475,7 @@ class FaceMlService { faces.add(f); } } - remoteFileIdToVersion[fileMl.fileID] = - fileMl.faceEmbedding.version; + remoteFileIdToVersion[fileMl.fileID] = fileMl.faceEmbedding.version; } if (res.noEmbeddingFileIDs.isNotEmpty) { _logger.info( @@ -504,8 +492,7 @@ class FaceMlService { for (final entry in remoteFileIdToVersion.entries) { alreadyIndexedFiles[entry.key] = entry.value; } - _logger - .info('already indexed files ${remoteFileIdToVersion.length}'); + _logger.info('already indexed files ${remoteFileIdToVersion.length}'); } catch (e, s) { _logger.severe("err while getting files embeddings", e, s); if (retryFetchCount < 1000) { @@ -586,10 +573,9 @@ class FaceMlService { try { // Get a sense of the total number of faces in the database - final int totalFaces = await FaceMLDataDB.instance - .getTotalFaceCount(minFaceScore: minFaceScore); - final fileIDToCreationTime = - await FilesDB.instance.getFileIDToCreationTime(); + final int totalFaces = + await FaceMLDataDB.instance.getTotalFaceCount(minFaceScore: minFaceScore); + final fileIDToCreationTime = await FilesDB.instance.getFileIDToCreationTime(); final startEmbeddingFetch = DateTime.now(); // read all embeddings final result = await FaceMLDataDB.instance.getFaceInfoForClustering( @@ -607,8 +593,7 @@ class FaceMlService { } // sort the embeddings based on file creation time, newest first allFaceInfoForClustering.sort((b, a) { - return fileIDToCreationTime[a.fileID]! - .compareTo(fileIDToCreationTime[b.fileID]!); + return fileIDToCreationTime[a.fileID]!.compareTo(fileIDToCreationTime[b.fileID]!); }); _logger.info( 'Getting and sorting embeddings took ${DateTime.now().difference(startEmbeddingFetch).inMilliseconds} ms for ${allFaceInfoForClustering.length} embeddings' @@ -664,8 +649,7 @@ class FaceMlService { } } - final clusteringResult = - await FaceClusteringService.instance.predictLinear( + final clusteringResult = await FaceClusteringService.instance.predictLinearIsolate( faceInfoForClustering.toSet(), fileIDToCreationTime: fileIDToCreationTime, offset: offset, @@ -676,17 +660,13 @@ class FaceMlService { return; } - await FaceMLDataDB.instance - .updateFaceIdToClusterId(clusteringResult.newFaceIdToCluster); - await FaceMLDataDB.instance - .clusterSummaryUpdate(clusteringResult.newClusterSummaries!); + await FaceMLDataDB.instance.updateFaceIdToClusterId(clusteringResult.newFaceIdToCluster); + await FaceMLDataDB.instance.clusterSummaryUpdate(clusteringResult.newClusterSummaries!); Bus.instance.fire(PeopleChangedEvent()); for (final faceInfo in faceInfoForClustering) { - faceInfo.clusterId ??= - clusteringResult.newFaceIdToCluster[faceInfo.faceID]; + faceInfo.clusterId ??= clusteringResult.newFaceIdToCluster[faceInfo.faceID]; } - for (final clusterUpdate - in clusteringResult.newClusterSummaries!.entries) { + for (final clusterUpdate in clusteringResult.newClusterSummaries!.entries) { oldClusterSummaries[clusterUpdate.key] = clusterUpdate.value; } _logger.info( @@ -702,8 +682,7 @@ class FaceMlService { } else { final clusterStartTime = DateTime.now(); // Cluster the embeddings using the linear clustering algorithm, returning a map from faceID to clusterID - final clusteringResult = - await FaceClusteringService.instance.predictLinear( + final clusteringResult = await FaceClusteringService.instance.predictLinearIsolate( allFaceInfoForClustering.toSet(), fileIDToCreationTime: fileIDToCreationTime, oldClusterSummaries: oldClusterSummaries, @@ -721,10 +700,8 @@ class FaceMlService { _logger.info( 'Updating ${clusteringResult.newFaceIdToCluster.length} FaceIDs with clusterIDs in the DB', ); - await FaceMLDataDB.instance - .updateFaceIdToClusterId(clusteringResult.newFaceIdToCluster); - await FaceMLDataDB.instance - .clusterSummaryUpdate(clusteringResult.newClusterSummaries!); + await FaceMLDataDB.instance.updateFaceIdToClusterId(clusteringResult.newFaceIdToCluster); + await FaceMLDataDB.instance.clusterSummaryUpdate(clusteringResult.newClusterSummaries!); Bus.instance.fire(PeopleChangedEvent()); _logger.info('Done updating FaceIDs with clusterIDs in the DB, in ' '${DateTime.now().difference(clusterDoneTime).inSeconds} seconds'); @@ -756,8 +733,7 @@ class FaceMlService { allLandmarksEqual = false; break; } - if (face.detection.landmarks - .any((landmark) => landmark.x != landmark.y)) { + if (face.detection.landmarks.any((landmark) => landmark.x != landmark.y)) { allLandmarksEqual = false; break; } @@ -766,10 +742,7 @@ class FaceMlService { debugPrint("Discarding remote embedding for fileID ${fileMl.fileID} " "because landmarks are equal"); debugPrint( - fileMl.faceEmbedding.faces - .map((e) => e.detection.landmarks.toString()) - .toList() - .toString(), + fileMl.faceEmbedding.faces.map((e) => e.detection.landmarks.toString()).toList().toString(), ); return true; } @@ -807,11 +780,9 @@ class FaceMlService { Face.empty(result.fileId, error: result.errorOccured), ); } else { - if (result.decodedImageSize.width == -1 || - result.decodedImageSize.height == -1) { - _logger - .severe("decodedImageSize is not stored correctly for image with " - "ID: ${enteFile.uploadedFileID}"); + if (result.decodedImageSize.width == -1 || result.decodedImageSize.height == -1) { + _logger.severe("decodedImageSize is not stored correctly for image with " + "ID: ${enteFile.uploadedFileID}"); _logger.info( "Using aligned image size for image with ID: ${enteFile.uploadedFileID}. This size is ${result.decodedImageSize.width}x${result.decodedImageSize.height} compared to size of ${enteFile.width}x${enteFile.height} in the metadata", ); @@ -896,8 +867,7 @@ class FaceMlService { _checkEnteFileForID(enteFile); await ensureInitialized(); - final String? filePath = - await _getImagePathForML(enteFile, typeOfData: FileDataForML.fileData); + final String? filePath = await _getImagePathForML(enteFile, typeOfData: FileDataForML.fileData); if (filePath == null) { _logger.severe( @@ -916,10 +886,8 @@ class FaceMlService { { "enteFileID": enteFile.uploadedFileID ?? -1, "filePath": filePath, - "faceDetectionAddress": - FaceDetectionService.instance.sessionAddress, - "faceEmbeddingAddress": - FaceEmbeddingService.instance.sessionAddress, + "faceDetectionAddress": FaceDetectionService.instance.sessionAddress, + "faceEmbeddingAddress": FaceEmbeddingService.instance.sessionAddress, } ), ) as String?; @@ -973,8 +941,7 @@ class FaceMlService { stopwatch.reset(); // Get the faces - final List faceDetectionResult = - await FaceMlService.detectFacesSync( + final List faceDetectionResult = await FaceMlService.detectFacesSync( image, imgByteData, faceDetectionAddress, @@ -995,8 +962,7 @@ class FaceMlService { stopwatch.reset(); // Align the faces - final Float32List faceAlignmentResult = - await FaceMlService.alignFacesSync( + final Float32List faceAlignmentResult = await FaceMlService.alignFacesSync( image, imgByteData, faceDetectionResult, @@ -1108,8 +1074,7 @@ class FaceMlService { }) async { try { // Get the bounding boxes of the faces - final (List faces, dataSize) = - await FaceDetectionService.predictSync( + final (List faces, dataSize) = await FaceDetectionService.predictSync( image, imageByteData, interpreterAddress, @@ -1219,8 +1184,7 @@ class FaceMlService { } bool _skipAnalysisEnteFile(EnteFile enteFile, Map indexedFileIds) { - if (_isIndexingOrClusteringRunning == false || - _mlControllerStatus == false) { + if (_isIndexingOrClusteringRunning == false || _mlControllerStatus == false) { return true; } // Skip if the file is not uploaded or not owned by the user @@ -1234,8 +1198,7 @@ class FaceMlService { // Skip if the file is already analyzed with the latest ml version final id = enteFile.uploadedFileID!; - return indexedFileIds.containsKey(id) && - indexedFileIds[id]! >= faceMlVersion; + return indexedFileIds.containsKey(id) && indexedFileIds[id]! >= faceMlVersion; } bool _cannotRunMLFunction({String function = ""}) { diff --git a/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart b/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart index 5a41a60df7..af6f07a1ab 100644 --- a/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart +++ b/mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart @@ -40,8 +40,7 @@ class ClusterFeedbackService { final _computer = Computer.shared(); ClusterFeedbackService._privateConstructor(); - static final ClusterFeedbackService instance = - ClusterFeedbackService._privateConstructor(); + static final ClusterFeedbackService instance = ClusterFeedbackService._privateConstructor(); static int lastViewedClusterID = -1; static setLastViewedClusterID(int clusterID) { @@ -68,8 +67,7 @@ class ClusterFeedbackService { try { // Get the suggestions for the person using centroids and median final startTime = DateTime.now(); - final List<(int, double, bool)> foundSuggestions = - await _getSuggestions(person); + final List<(int, double, bool)> foundSuggestions = await _getSuggestions(person); final findSuggestionsTime = DateTime.now(); _logger.info( 'getSuggestionForPerson `_getSuggestions`: Found ${foundSuggestions.length} suggestions in ${findSuggestionsTime.difference(startTime).inMilliseconds} ms', @@ -143,36 +141,32 @@ class ClusterFeedbackService { final fileID = getFileIdFromFaceId(faceID); return files.any((file) => file.uploadedFileID == fileID); }); - final embeddings = - await FaceMLDataDB.instance.getFaceEmbeddingMapForFaces(faceIDs); + final embeddings = await FaceMLDataDB.instance.getFaceEmbeddingMapForFaces(faceIDs); - final fileIDToCreationTime = - await FilesDB.instance.getFileIDToCreationTime(); + final fileIDToCreationTime = await FilesDB.instance.getFileIDToCreationTime(); // Re-cluster within the deleted faces - final clusterResult = - await FaceClusteringService.instance.predictWithinClusterComputer( + final clusterResult = await FaceClusteringService.instance.predictWithinClusterComputer( embeddings, fileIDToCreationTime: fileIDToCreationTime, distanceThreshold: 0.20, ); - if (clusterResult == null || clusterResult.isEmpty) { + if (clusterResult.isEmpty) { + _logger.warning('No clusters found or something went wrong'); return; } final newFaceIdToClusterID = clusterResult.newFaceIdToCluster; // Update the deleted faces await FaceMLDataDB.instance.forceUpdateClusterIds(newFaceIdToClusterID); - await FaceMLDataDB.instance - .clusterSummaryUpdate(clusterResult.newClusterSummaries!); + await FaceMLDataDB.instance.clusterSummaryUpdate(clusterResult.newClusterSummaries); // Make sure the deleted faces don't get suggested in the future final notClusterIdToPersonId = {}; for (final clusterId in newFaceIdToClusterID.values.toSet()) { notClusterIdToPersonId[clusterId] = p.remoteID; } - await FaceMLDataDB.instance - .bulkCaptureNotPersonFeedback(notClusterIdToPersonId); + await FaceMLDataDB.instance.bulkCaptureNotPersonFeedback(notClusterIdToPersonId); Bus.instance.fire(PeopleChangedEvent()); return; @@ -195,28 +189,24 @@ class ClusterFeedbackService { final fileID = getFileIdFromFaceId(faceID); return files.any((file) => file.uploadedFileID == fileID); }); - final embeddings = - await FaceMLDataDB.instance.getFaceEmbeddingMapForFaces(faceIDs); + final embeddings = await FaceMLDataDB.instance.getFaceEmbeddingMapForFaces(faceIDs); - final fileIDToCreationTime = - await FilesDB.instance.getFileIDToCreationTime(); + final fileIDToCreationTime = await FilesDB.instance.getFileIDToCreationTime(); // Re-cluster within the deleted faces - final clusterResult = - await FaceClusteringService.instance.predictWithinClusterComputer( + final clusterResult = await FaceClusteringService.instance.predictWithinClusterComputer( embeddings, fileIDToCreationTime: fileIDToCreationTime, distanceThreshold: 0.20, ); - if (clusterResult == null || clusterResult.isEmpty) { + if (clusterResult.isEmpty) { return; } final newFaceIdToClusterID = clusterResult.newFaceIdToCluster; // Update the deleted faces await FaceMLDataDB.instance.forceUpdateClusterIds(newFaceIdToClusterID); - await FaceMLDataDB.instance - .clusterSummaryUpdate(clusterResult.newClusterSummaries!); + await FaceMLDataDB.instance.clusterSummaryUpdate(clusterResult.newClusterSummaries); Bus.instance.fire( PeopleChangedEvent( @@ -319,14 +309,12 @@ class ClusterFeedbackService { final allClusterToFaceCount = await faceMlDb.clusterIdToFaceCount(); final clustersToInspect = []; for (final clusterID in allClusterToFaceCount.keys) { - if (allClusterToFaceCount[clusterID]! > 20 && - allClusterToFaceCount[clusterID]! < 500) { + if (allClusterToFaceCount[clusterID]! > 20 && allClusterToFaceCount[clusterID]! < 500) { clustersToInspect.add(clusterID); } } - final fileIDToCreationTime = - await FilesDB.instance.getFileIDToCreationTime(); + final fileIDToCreationTime = await FilesDB.instance.getFileIDToCreationTime(); final susClusters = <(int, int)>[]; @@ -337,24 +325,20 @@ class ClusterFeedbackService { final embeddings = await faceMlDb.getFaceEmbeddingMapForFaces(faceIDs); - final clusterResult = - await FaceClusteringService.instance.predictWithinClusterComputer( + final clusterResult = await FaceClusteringService.instance.predictWithinClusterComputer( embeddings, fileIDToCreationTime: fileIDToCreationTime, distanceThreshold: 0.22, ); - if (clusterResult == null || - clusterResult.newClusterIdToFaceIds == null || - clusterResult.isEmpty) { + if (clusterResult.isEmpty) { _logger.warning( '[CheckMixedClusters] Clustering did not seem to work for cluster $clusterID of size ${allClusterToFaceCount[clusterID]}', ); continue; } - final newClusterIdToCount = - clusterResult.newClusterIdToFaceIds!.map((key, value) { + final newClusterIdToCount = clusterResult.newClusterIdToFaceIds.map((key, value) { return MapEntry(key, value.length); }); final amountOfNewClusters = newClusterIdToCount.length; @@ -375,15 +359,12 @@ class ClusterFeedbackService { final int secondBiggestClusterID = clusterIDs.reduce((a, b) { return newClusterIdToCount[a]! > newClusterIdToCount[b]! ? a : b; }); - final int secondBiggestSize = - newClusterIdToCount[secondBiggestClusterID]!; + final int secondBiggestSize = newClusterIdToCount[secondBiggestClusterID]!; final secondBiggestRatio = secondBiggestSize / originalClusterSize; if (biggestRatio < 0.5 || secondBiggestRatio > 0.2) { - final faceIdsOfCluster = - await faceMlDb.getFaceIDsForCluster(clusterID); - final uniqueFileIDs = - faceIdsOfCluster.map(getFileIdFromFaceId).toSet(); + final faceIdsOfCluster = await faceMlDb.getFaceIDsForCluster(clusterID); + final uniqueFileIDs = faceIdsOfCluster.map(getFileIdFromFaceId).toSet(); susClusters.add((clusterID, uniqueFileIDs.length)); _logger.info( '[CheckMixedClusters] Detected that cluster $clusterID with size ${uniqueFileIDs.length} might be mixed', @@ -423,25 +404,25 @@ class ClusterFeedbackService { final embeddings = await faceMlDb.getFaceEmbeddingMapForFaces(faceIDs); - final fileIDToCreationTime = - await FilesDB.instance.getFileIDToCreationTime(); + if (embeddings.isEmpty) { + _logger.warning('No embeddings found for cluster $clusterID'); + return ClusteringResult.empty(); + } - final clusterResult = - await FaceClusteringService.instance.predictWithinClusterComputer( + final fileIDToCreationTime = await FilesDB.instance.getFileIDToCreationTime(); + + final clusterResult = await FaceClusteringService.instance.predictWithinClusterComputer( embeddings, fileIDToCreationTime: fileIDToCreationTime, distanceThreshold: 0.22, ); - if (clusterResult == null || - !clusterResult.hasAllResults || - clusterResult.isEmpty) { + if (clusterResult.isEmpty) { _logger.warning('No clusters found or something went wrong'); - return ClusteringResult(newFaceIdToCluster: {}); + return ClusteringResult.empty(); } - final clusterIdToCount = - clusterResult.newClusterIdToFaceIds!.map((key, value) { + final clusterIdToCount = clusterResult.newClusterIdToFaceIds.map((key, value) { return MapEntry(key, value.length); }); final amountOfNewClusters = clusterIdToCount.length; @@ -452,8 +433,7 @@ class ClusterFeedbackService { if (kDebugMode) { final Set allClusteredFaceIDsSet = {}; - for (final List value - in clusterResult.newClusterIdToFaceIds!.values) { + for (final List value in clusterResult.newClusterIdToFaceIds.values) { allClusteredFaceIDsSet.addAll(value); } assert((originalFaceIDsSet.difference(allClusteredFaceIDsSet)).isEmpty); @@ -467,16 +447,14 @@ class ClusterFeedbackService { try { // Delete old clusters await FaceMLDataDB.instance.dropClustersAndPersonTable(); - final List persons = - await PersonService.instance.getPersons(); + final List persons = await PersonService.instance.getPersons(); for (final PersonEntity p in persons) { await PersonService.instance.deletePerson(p.remoteID); } // Create new fake clusters based on blur value. One for values between 0 and 10, one for 10-20, etc till 200 final int startClusterID = DateTime.now().microsecondsSinceEpoch; - final faceIDsToBlurValues = - await FaceMLDataDB.instance.getFaceIDsToBlurValues(200); + final faceIDsToBlurValues = await FaceMLDataDB.instance.getFaceIDsToBlurValues(200); final faceIdToCluster = {}; for (final entry in faceIDsToBlurValues.entries) { final faceID = entry.key; @@ -549,46 +527,40 @@ class ClusterFeedbackService { _logger.info( 'L2 norm of current mean: $currentL2Norm', ); - final trueDistance = - biggestMean.distanceTo(currentMean, distance: Distance.cosine); + final trueDistance = biggestMean.distanceTo(currentMean, distance: Distance.cosine); _logger.info('True distance between the two means: $trueDistance'); // Median distance const sampleSize = 100; - final Iterable biggestEmbeddings = await FaceMLDataDB - .instance - .getFaceEmbeddingsForCluster(biggestClusterID); - final List biggestSampledEmbeddingsProto = - _randomSampleWithoutReplacement( + final Iterable biggestEmbeddings = + await FaceMLDataDB.instance.getFaceEmbeddingsForCluster(biggestClusterID); + final List biggestSampledEmbeddingsProto = _randomSampleWithoutReplacement( biggestEmbeddings, sampleSize, ); - final List biggestSampledEmbeddings = - biggestSampledEmbeddingsProto - .map( - (embedding) => Vector.fromList( - EVector.fromBuffer(embedding).values, - dtype: DType.float32, - ), - ) - .toList(growable: false); + final List biggestSampledEmbeddings = biggestSampledEmbeddingsProto + .map( + (embedding) => Vector.fromList( + EVector.fromBuffer(embedding).values, + dtype: DType.float32, + ), + ) + .toList(growable: false); final Iterable currentEmbeddings = await FaceMLDataDB.instance.getFaceEmbeddingsForCluster(clusterID); - final List currentSampledEmbeddingsProto = - _randomSampleWithoutReplacement( + final List currentSampledEmbeddingsProto = _randomSampleWithoutReplacement( currentEmbeddings, sampleSize, ); - final List currentSampledEmbeddings = - currentSampledEmbeddingsProto - .map( - (embedding) => Vector.fromList( - EVector.fromBuffer(embedding).values, - dtype: DType.float32, - ), - ) - .toList(growable: false); + final List currentSampledEmbeddings = currentSampledEmbeddingsProto + .map( + (embedding) => Vector.fromList( + EVector.fromBuffer(embedding).values, + dtype: DType.float32, + ), + ) + .toList(growable: false); // Calculate distances and find the median final List distances = []; @@ -607,8 +579,7 @@ class ClusterFeedbackService { distances.sort(); trueDistances.sort(); final double medianDistance = distances[distances.length ~/ 2]; - final double trueMedianDistance = - trueDistances[trueDistances.length ~/ 2]; + final double trueMedianDistance = trueDistances[trueDistances.length ~/ 2]; _logger.info( "Median distance between biggest cluster and current cluster: $medianDistance (using sample of $sampleSize)", ); @@ -623,8 +594,7 @@ class ClusterFeedbackService { final List blurValues = await FaceMLDataDB.instance .getBlurValuesForCluster(clusterID) .then((value) => value.toList()); - final blurValuesIntegers = - blurValues.map((value) => value.round()).toList(); + final blurValuesIntegers = blurValues.map((value) => value.round()).toList(); blurValuesIntegers.sort(); _logger.info( "Blur values for cluster $clusterID${clusterSize != null ? ' with $clusterSize photos' : ''}: $blurValuesIntegers", @@ -652,14 +622,12 @@ class ClusterFeedbackService { final allClusterIdsToCountMap = await faceMlDb.clusterIdToFaceCount(); final ignoredClusters = await faceMlDb.getPersonIgnoredClusters(p.remoteID); final personClusters = await faceMlDb.getPersonClusterIDs(p.remoteID); - final personFaceIDs = - await FaceMLDataDB.instance.getFaceIDsForPerson(p.remoteID); + final personFaceIDs = await FaceMLDataDB.instance.getFaceIDsForPerson(p.remoteID); final personFileIDs = personFaceIDs.map(getFileIdFromFaceId).toSet(); w?.log( '${p.data.name} has ${personClusters.length} existing clusters, getting all database data done', ); - final allClusterIdToFaceIDs = - await FaceMLDataDB.instance.getAllClusterIdToFaceIDs(); + final allClusterIdToFaceIDs = await FaceMLDataDB.instance.getAllClusterIdToFaceIDs(); w?.log('getAllClusterIdToFaceIDs done'); // First only do a simple check on the big clusters, if the person does not have small clusters yet @@ -670,8 +638,7 @@ class ClusterFeedbackService { late Map clusterAvgBigClusters; final List<(int, double)> suggestionsMean = []; for (final minimumSize in checkSizes.toSet()) { - if (smallestPersonClusterSize >= - min(minimumSize, kMinimumClusterSizeSearchResult)) { + if (smallestPersonClusterSize >= min(minimumSize, kMinimumClusterSizeSearchResult)) { clusterAvgBigClusters = await _getUpdateClusterAvg( allClusterIdsToCountMap, ignoredClusters, @@ -680,8 +647,7 @@ class ClusterFeedbackService { w?.log( 'Calculate avg for ${clusterAvgBigClusters.length} clusters of min size $minimumSize', ); - final List<(int, double)> suggestionsMeanBigClusters = - await calcSuggestionsMeanInComputer( + final List<(int, double)> suggestionsMeanBigClusters = await calcSuggestionsMeanInComputer( clusterAvgBigClusters, personClusters, ignoredClusters, @@ -696,8 +662,7 @@ class ClusterFeedbackService { .map((faceID) => getFileIdFromFaceId(faceID)) .toSet(); final overlap = personFileIDs.intersection(suggestionSet); - if (overlap.isNotEmpty && - ((overlap.length / suggestionSet.length) > 0.5)) { + if (overlap.isNotEmpty && ((overlap.length / suggestionSet.length) > 0.5)) { await FaceMLDataDB.instance.captureNotPersonFeedback( personID: p.remoteID, clusterID: suggestion.$1, @@ -707,9 +672,7 @@ class ClusterFeedbackService { suggestionsMean.add(suggestion); } if (suggestionsMean.isNotEmpty) { - return suggestionsMean - .map((e) => (e.$1, e.$2, true)) - .toList(growable: false); + return suggestionsMean.map((e) => (e.$1, e.$2, true)).toList(growable: false); } } } @@ -717,16 +680,14 @@ class ClusterFeedbackService { // Find the other cluster candidates based on the median final clusterAvg = clusterAvgBigClusters; - final List<(int, double)> moreSuggestionsMean = - await calcSuggestionsMeanInComputer( + final List<(int, double)> moreSuggestionsMean = await calcSuggestionsMeanInComputer( clusterAvg, personClusters, ignoredClusters, maxMeanDistance, ); if (moreSuggestionsMean.isEmpty) { - _logger - .info("No suggestions found using mean, even with higher threshold"); + _logger.info("No suggestions found using mean, even with higher threshold"); return []; } @@ -748,8 +709,7 @@ class ClusterFeedbackService { await FaceMLDataDB.instance.getFaceEmbeddingsForCluster(clusterID); personEmbeddingsProto.addAll(embeddings); } - final List sampledEmbeddingsProto = - _randomSampleWithoutReplacement( + final List sampledEmbeddingsProto = _randomSampleWithoutReplacement( personEmbeddingsProto, sampleSize, ); @@ -893,8 +853,7 @@ class ClusterFeedbackService { // get clusterIDs sorted by count in descending order final sortedClusterIDs = allClusterIds.toList(); sortedClusterIDs.sort( - (a, b) => - allClusterIdsToCountMap[b]!.compareTo(allClusterIdsToCountMap[a]!), + (a, b) => allClusterIdsToCountMap[b]!.compareTo(allClusterIdsToCountMap[a]!), ); int indexedInCurrentRun = 0; w?.reset(); @@ -909,8 +868,7 @@ class ClusterFeedbackService { currentPendingRead = allClusterIdsToCountMap[clusterID] ?? 0; clusterIdsToRead.add(clusterID); } else { - if ((currentPendingRead + allClusterIdsToCountMap[clusterID]!) < - maxEmbeddingToRead) { + if ((currentPendingRead + allClusterIdsToCountMap[clusterID]!) < maxEmbeddingToRead) { clusterIdsToRead.add(clusterID); currentPendingRead += allClusterIdsToCountMap[clusterID]!; } else { @@ -919,9 +877,8 @@ class ClusterFeedbackService { } } - final Map> clusterEmbeddings = await FaceMLDataDB - .instance - .getFaceEmbeddingsForClusters(clusterIdsToRead); + final Map> clusterEmbeddings = + await FaceMLDataDB.instance.getFaceEmbeddingsForClusters(clusterIdsToRead); w?.logAndReset( 'read $currentPendingRead embeddings for ${clusterEmbeddings.length} clusters', @@ -938,8 +895,7 @@ class ClusterFeedbackService { final avg = vectors.reduce((a, b) => a + b) / vectors.length; final avgNormalized = avg / avg.norm(); final avgEmbeddingBuffer = EVector(values: avgNormalized).writeToBuffer(); - updatesForClusterSummary[clusterID] = - (avgEmbeddingBuffer, embeddings.length); + updatesForClusterSummary[clusterID] = (avgEmbeddingBuffer, embeddings.length); // store the intermediate updates indexedInCurrentRun++; if (updatesForClusterSummary.length > 100) { @@ -1042,9 +998,8 @@ class ClusterFeedbackService { // Calculate the avg embedding of the person final w = (kDebugMode ? EnteWatch('sortSuggestions') : null)?..start(); - final personEmbeddingsCount = personClusters - .map((e) => personClusterToSummary[e]!.$2) - .reduce((a, b) => a + b); + final personEmbeddingsCount = + personClusters.map((e) => personClusterToSummary[e]!.$2).reduce((a, b) => a + b); Vector personAvg = Vector.filled(192, 0); for (final personClusterID in personClusters) { final personClusterBlob = personClusterToSummary[personClusterID]!.$1; @@ -1052,8 +1007,7 @@ class ClusterFeedbackService { EVector.fromBuffer(personClusterBlob).values, dtype: DType.float32, ); - final clusterWeight = - personClusterToSummary[personClusterID]!.$2 / personEmbeddingsCount; + final clusterWeight = personClusterToSummary[personClusterID]!.$2 / personEmbeddingsCount; personAvg += personClusterAvg * clusterWeight; } w?.log('calculated person avg'); @@ -1084,8 +1038,7 @@ class ClusterFeedbackService { ); final fileIdToDistanceMap = {}; for (final entry in faceIdToVectorMap.entries) { - fileIdToDistanceMap[getFileIdFromFaceId(entry.key)] = - 1 - personAvg.dot(entry.value); + fileIdToDistanceMap[getFileIdFromFaceId(entry.key)] = 1 - personAvg.dot(entry.value); } w?.log('calculated distances for cluster $clusterID'); suggestion.filesInCluster.sort((b, a) { @@ -1150,9 +1103,7 @@ List<(int, double)> _calcSuggestionsMean(Map args) { } } if (nearestPersonCluster != null && minDistance != null) { - suggestions - .putIfAbsent(nearestPersonCluster, () => []) - .add((otherClusterID, minDistance)); + suggestions.putIfAbsent(nearestPersonCluster, () => []).add((otherClusterID, minDistance)); suggestionCount++; } if (suggestionCount >= suggestionMax) { @@ -1182,8 +1133,7 @@ List<(int, double)> _calcSuggestionsMean(Map args) { } } -Future<(Map, Set, int, int, int)> - checkAndSerializeCurrentClusterMeans( +Future<(Map, Set, int, int, int)> checkAndSerializeCurrentClusterMeans( Map args, ) async { final Map allClusterIdsToCountMap = args['allClusterIdsToCountMap']; diff --git a/mobile/lib/ui/viewer/people/cluster_app_bar.dart b/mobile/lib/ui/viewer/people/cluster_app_bar.dart index 8b95d4247e..e2b3c10824 100644 --- a/mobile/lib/ui/viewer/people/cluster_app_bar.dart +++ b/mobile/lib/ui/viewer/people/cluster_app_bar.dart @@ -191,14 +191,14 @@ class _AppBarWidgetState extends State { final breakupResult = await ClusterFeedbackService.instance .breakUpCluster(widget.clusterID); final Map> newClusterIDToFaceIDs = - breakupResult.newClusterIdToFaceIds!; + breakupResult.newClusterIdToFaceIds; final Map newFaceIdToClusterID = breakupResult.newFaceIdToCluster; // Update to delete the old clusters and save the new clusters await FaceMLDataDB.instance.deleteClusterSummary(widget.clusterID); await FaceMLDataDB.instance - .clusterSummaryUpdate(breakupResult.newClusterSummaries!); + .clusterSummaryUpdate(breakupResult.newClusterSummaries); await FaceMLDataDB.instance .updateFaceIdToClusterId(newFaceIdToClusterID); @@ -254,7 +254,7 @@ class _AppBarWidgetState extends State { await ClusterFeedbackService.instance.breakUpCluster(widget.clusterID); final Map> newClusterIDToFaceIDs = - breakupResult.newClusterIdToFaceIds!; + breakupResult.newClusterIdToFaceIds; final allFileIDs = newClusterIDToFaceIDs.values .expand((e) => e) From 535d24779f64aa69a556a37b23a94ea61e7248cf Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Wed, 29 May 2024 13:57:16 +0530 Subject: [PATCH 119/354] Handle bad secure stroage state error --- mobile/lib/core/configuration.dart | 84 +++++++++++++++++++----------- 1 file changed, 54 insertions(+), 30 deletions(-) diff --git a/mobile/lib/core/configuration.dart b/mobile/lib/core/configuration.dart index 8019e2a73c..845debd6be 100644 --- a/mobile/lib/core/configuration.dart +++ b/mobile/lib/core/configuration.dart @@ -1,9 +1,9 @@ import "dart:async"; import 'dart:convert'; import "dart:io"; -import 'dart:typed_data'; import 'package:bip39/bip39.dart' as bip39; +import "package:flutter/services.dart"; import 'package:flutter_secure_storage/flutter_secure_storage.dart'; import 'package:logging/logging.dart'; import 'package:path_provider/path_provider.dart'; @@ -97,37 +97,61 @@ class Configuration { ); Future init() async { - _preferences = await SharedPreferences.getInstance(); - _secureStorage = const FlutterSecureStorage(); - _documentsDirectory = (await getApplicationDocumentsDirectory()).path; - _tempDocumentsDirPath = _documentsDirectory + "/temp/"; - final tempDocumentsDir = Directory(_tempDocumentsDirPath); - await _cleanUpStaleFiles(tempDocumentsDir); - tempDocumentsDir.createSync(recursive: true); - final tempDirectoryPath = (await getTemporaryDirectory()).path; - _thumbnailCacheDirectory = tempDirectoryPath + "/thumbnail-cache"; - Directory(_thumbnailCacheDirectory).createSync(recursive: true); - _sharedTempMediaDirectory = tempDirectoryPath + "/ente-shared-media"; - Directory(_sharedTempMediaDirectory).createSync(recursive: true); - _sharedDocumentsMediaDirectory = _documentsDirectory + "/ente-shared-media"; - Directory(_sharedDocumentsMediaDirectory).createSync(recursive: true); - if (!_preferences.containsKey(tokenKey)) { - await _secureStorage.deleteAll(iOptions: _secureStorageOptionsIOS); - } else { - _key = await _secureStorage.read( - key: keyKey, - iOptions: _secureStorageOptionsIOS, - ); - _secretKey = await _secureStorage.read( - key: secretKeyKey, - iOptions: _secureStorageOptionsIOS, - ); - if (_key == null) { - await logout(autoLogout: true); + try { + _preferences = await SharedPreferences.getInstance(); + _secureStorage = const FlutterSecureStorage(); + _documentsDirectory = (await getApplicationDocumentsDirectory()).path; + _tempDocumentsDirPath = _documentsDirectory + "/temp/"; + final tempDocumentsDir = Directory(_tempDocumentsDirPath); + await _cleanUpStaleFiles(tempDocumentsDir); + tempDocumentsDir.createSync(recursive: true); + final tempDirectoryPath = (await getTemporaryDirectory()).path; + _thumbnailCacheDirectory = tempDirectoryPath + "/thumbnail-cache"; + Directory(_thumbnailCacheDirectory).createSync(recursive: true); + _sharedTempMediaDirectory = tempDirectoryPath + "/ente-shared-media"; + Directory(_sharedTempMediaDirectory).createSync(recursive: true); + _sharedDocumentsMediaDirectory = + _documentsDirectory + "/ente-shared-media"; + Directory(_sharedDocumentsMediaDirectory).createSync(recursive: true); + if (!_preferences.containsKey(tokenKey)) { + await _secureStorage.deleteAll(iOptions: _secureStorageOptionsIOS); + } else { + _key = await _secureStorage.read( + key: keyKey, + iOptions: _secureStorageOptionsIOS, + ); + _secretKey = await _secureStorage.read( + key: secretKeyKey, + iOptions: _secureStorageOptionsIOS, + ); + if (_key == null) { + await logout(autoLogout: true); + } + await _migrateSecurityStorageToFirstUnlock(); + } + SuperLogging.setUserID(await _getOrCreateAnonymousUserID()).ignore(); + } catch (e, s) { + _logger.severe("Configuration init failed", e, s); + /* + Check if it's a known is related to reading secret from secure storage + on android https://github.com/mogol/flutter_secure_storage/issues/541 + */ + if (e is PlatformException) { + final PlatformException error = e; + final bool isBadPaddingError = + error.toString().contains('BadPaddingException') || + (error.message ?? '').contains('BadPaddingException'); + if (isBadPaddingError) { + await logout(autoLogout: true); + } else { + _logger.warning( + 'Platform error ${error.message} with string ${error.toString()}'); + rethrow; + } + } else { + rethrow; } - await _migrateSecurityStorageToFirstUnlock(); } - SuperLogging.setUserID(await _getOrCreateAnonymousUserID()).ignore(); } // _cleanUpStaleFiles deletes all files in the temp directory that are older From 09aa2fece0a24adfac759b4d8480d9500840ad04 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Wed, 29 May 2024 13:58:18 +0530 Subject: [PATCH 120/354] Fix: Only try stopping sync for manual logout --- mobile/lib/core/configuration.dart | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/mobile/lib/core/configuration.dart b/mobile/lib/core/configuration.dart index 845debd6be..729fc1a6c6 100644 --- a/mobile/lib/core/configuration.dart +++ b/mobile/lib/core/configuration.dart @@ -191,14 +191,16 @@ class Configuration { } Future logout({bool autoLogout = false}) async { - if (SyncService.instance.isSyncInProgress()) { - SyncService.instance.stopSync(); - try { - await SyncService.instance - .existingSync() - .timeout(const Duration(seconds: 5)); - } catch (e) { - // ignore + if (!autoLogout) { + if (SyncService.instance.isSyncInProgress()) { + SyncService.instance.stopSync(); + try { + await SyncService.instance + .existingSync() + .timeout(const Duration(seconds: 5)); + } catch (e) { + // ignore + } } } await _preferences.clear(); From 319108db1b96a6cbaa703297319f7d526ce255c8 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Wed, 29 May 2024 13:59:52 +0530 Subject: [PATCH 121/354] Update auto logout message --- mobile/lib/generated/intl/messages_de.dart | 2 -- mobile/lib/generated/intl/messages_en.dart | 4 ++-- mobile/lib/generated/intl/messages_es.dart | 2 -- mobile/lib/generated/intl/messages_fr.dart | 2 -- mobile/lib/generated/intl/messages_it.dart | 2 -- mobile/lib/generated/intl/messages_nl.dart | 2 -- mobile/lib/generated/intl/messages_pt.dart | 2 -- mobile/lib/generated/intl/messages_zh.dart | 2 -- mobile/lib/generated/l10n.dart | 8 ++++---- mobile/lib/l10n/intl_en.arb | 2 +- mobile/lib/ui/home/landing_page_widget.dart | 2 +- 11 files changed, 8 insertions(+), 22 deletions(-) diff --git a/mobile/lib/generated/intl/messages_de.dart b/mobile/lib/generated/intl/messages_de.dart index 49c7ac93c6..a1295da72a 100644 --- a/mobile/lib/generated/intl/messages_de.dart +++ b/mobile/lib/generated/intl/messages_de.dart @@ -582,8 +582,6 @@ class MessageLookup extends MessageLookupByLibrary { "designedToOutlive": MessageLookupByLibrary.simpleMessage("Entwickelt um zu bewahren"), "details": MessageLookupByLibrary.simpleMessage("Details"), - "devAccountChanged": MessageLookupByLibrary.simpleMessage( - "Das Entwicklerkonto, das wir verwenden, um ente im App Store zu veröffentlichen, hat sich geändert. Aus diesem Grund musst du dich erneut anmelden.\n\nWir entschuldigen uns für die Unannehmlichkeiten, aber das war unvermeidlich."), "deviceCodeHint": MessageLookupByLibrary.simpleMessage("Code eingeben"), "deviceFilesAutoUploading": MessageLookupByLibrary.simpleMessage( "Dateien, die zu diesem Album hinzugefügt werden, werden automatisch zu ente hochgeladen."), diff --git a/mobile/lib/generated/intl/messages_en.dart b/mobile/lib/generated/intl/messages_en.dart index b715eb4851..d3fa964816 100644 --- a/mobile/lib/generated/intl/messages_en.dart +++ b/mobile/lib/generated/intl/messages_en.dart @@ -362,6 +362,8 @@ class MessageLookup extends MessageLookupByLibrary { "You\'ll see available Cast devices here."), "autoCastiOSPermission": MessageLookupByLibrary.simpleMessage( "Make sure Local Network permissions are turned on for the Ente Photos app, in Settings."), + "autoLogoutMessage": MessageLookupByLibrary.simpleMessage( + "Due to technical glitch, you have been logged out. Our apologies for the inconvenience."), "autoPair": MessageLookupByLibrary.simpleMessage("Auto pair"), "autoPairDesc": MessageLookupByLibrary.simpleMessage( "Auto pair works only with devices that support Chromecast."), @@ -581,8 +583,6 @@ class MessageLookup extends MessageLookupByLibrary { "designedToOutlive": MessageLookupByLibrary.simpleMessage("Designed to outlive"), "details": MessageLookupByLibrary.simpleMessage("Details"), - "devAccountChanged": MessageLookupByLibrary.simpleMessage( - "The developer account we use to publish Ente on App Store has changed. Because of this, you will need to login again.\n\nOur apologies for the inconvenience, but this was unavoidable."), "developerSettings": MessageLookupByLibrary.simpleMessage("Developer settings"), "developerSettingsWarning": MessageLookupByLibrary.simpleMessage( diff --git a/mobile/lib/generated/intl/messages_es.dart b/mobile/lib/generated/intl/messages_es.dart index f0b4c87f5c..706c399f2e 100644 --- a/mobile/lib/generated/intl/messages_es.dart +++ b/mobile/lib/generated/intl/messages_es.dart @@ -503,8 +503,6 @@ class MessageLookup extends MessageLookupByLibrary { "designedToOutlive": MessageLookupByLibrary.simpleMessage("Diseñado para sobrevivir"), "details": MessageLookupByLibrary.simpleMessage("Detalles"), - "devAccountChanged": MessageLookupByLibrary.simpleMessage( - "La cuenta de desarrollador que utilizamos para publicar ente en la App Store ha cambiado. Por eso, tendrás que iniciar sesión de nuevo.\n\nNuestras disculpas por las molestias, pero esto era inevitable."), "deviceFilesAutoUploading": MessageLookupByLibrary.simpleMessage( "Los archivos añadidos a este álbum de dispositivo se subirán automáticamente a ente."), "deviceLockExplanation": MessageLookupByLibrary.simpleMessage( diff --git a/mobile/lib/generated/intl/messages_fr.dart b/mobile/lib/generated/intl/messages_fr.dart index 5c4a2b4e47..cadfea35ba 100644 --- a/mobile/lib/generated/intl/messages_fr.dart +++ b/mobile/lib/generated/intl/messages_fr.dart @@ -577,8 +577,6 @@ class MessageLookup extends MessageLookupByLibrary { "designedToOutlive": MessageLookupByLibrary.simpleMessage("Conçu pour survivre"), "details": MessageLookupByLibrary.simpleMessage("Détails"), - "devAccountChanged": MessageLookupByLibrary.simpleMessage( - "Le compte développeur que nous utilisons pour publier ente sur l\'App Store a changé. Pour cette raison, vous devrez vous connecter à nouveau.\n\nNous nous excusons pour la gêne occasionnée, mais cela était inévitable."), "deviceFilesAutoUploading": MessageLookupByLibrary.simpleMessage( "Les fichiers ajoutés à cet album seront automatiquement téléchargés sur ente."), "deviceLockExplanation": MessageLookupByLibrary.simpleMessage( diff --git a/mobile/lib/generated/intl/messages_it.dart b/mobile/lib/generated/intl/messages_it.dart index d7a902db84..f49a8ba51a 100644 --- a/mobile/lib/generated/intl/messages_it.dart +++ b/mobile/lib/generated/intl/messages_it.dart @@ -557,8 +557,6 @@ class MessageLookup extends MessageLookupByLibrary { "designedToOutlive": MessageLookupByLibrary.simpleMessage("Progettato per sopravvivere"), "details": MessageLookupByLibrary.simpleMessage("Dettagli"), - "devAccountChanged": MessageLookupByLibrary.simpleMessage( - "L\'account sviluppatore che utilizziamo per pubblicare ente su App Store è cambiato. Per questo motivo dovrai effettuare nuovamente il login.\n\nCi dispiace per il disagio, ma era inevitabile."), "deviceFilesAutoUploading": MessageLookupByLibrary.simpleMessage( "I file aggiunti in questa cartella del dispositivo verranno automaticamente caricati su ente."), "deviceLockExplanation": MessageLookupByLibrary.simpleMessage( diff --git a/mobile/lib/generated/intl/messages_nl.dart b/mobile/lib/generated/intl/messages_nl.dart index 1981b338ce..d6e8285756 100644 --- a/mobile/lib/generated/intl/messages_nl.dart +++ b/mobile/lib/generated/intl/messages_nl.dart @@ -598,8 +598,6 @@ class MessageLookup extends MessageLookupByLibrary { "designedToOutlive": MessageLookupByLibrary.simpleMessage( "Ontworpen om levenslang mee te gaan"), "details": MessageLookupByLibrary.simpleMessage("Details"), - "devAccountChanged": MessageLookupByLibrary.simpleMessage( - "Het ontwikkelaarsaccount dat we gebruiken om te publiceren in de App Store is veranderd. Daarom moet je opnieuw inloggen.\n\nOnze excuses voor het ongemak, helaas was dit onvermijdelijk."), "developerSettings": MessageLookupByLibrary.simpleMessage("Ontwikkelaarsinstellingen"), "developerSettingsWarning": MessageLookupByLibrary.simpleMessage( diff --git a/mobile/lib/generated/intl/messages_pt.dart b/mobile/lib/generated/intl/messages_pt.dart index e32fd36379..299bc03112 100644 --- a/mobile/lib/generated/intl/messages_pt.dart +++ b/mobile/lib/generated/intl/messages_pt.dart @@ -592,8 +592,6 @@ class MessageLookup extends MessageLookupByLibrary { "designedToOutlive": MessageLookupByLibrary.simpleMessage("Feito para ter longevidade"), "details": MessageLookupByLibrary.simpleMessage("Detalhes"), - "devAccountChanged": MessageLookupByLibrary.simpleMessage( - "A conta de desenvolvedor que usamos para publicar o Ente na App Store foi alterada. Por esse motivo, você precisará fazer entrar novamente.\n\nPedimos desculpas pelo inconveniente, mas isso era inevitável."), "developerSettings": MessageLookupByLibrary.simpleMessage( "Configurações de desenvolvedor"), "developerSettingsWarning": MessageLookupByLibrary.simpleMessage( diff --git a/mobile/lib/generated/intl/messages_zh.dart b/mobile/lib/generated/intl/messages_zh.dart index ecca5d7b8f..32f4e01bdf 100644 --- a/mobile/lib/generated/intl/messages_zh.dart +++ b/mobile/lib/generated/intl/messages_zh.dart @@ -495,8 +495,6 @@ class MessageLookup extends MessageLookupByLibrary { "deselectAll": MessageLookupByLibrary.simpleMessage("取消全选"), "designedToOutlive": MessageLookupByLibrary.simpleMessage("经久耐用"), "details": MessageLookupByLibrary.simpleMessage("详情"), - "devAccountChanged": MessageLookupByLibrary.simpleMessage( - "我们用于在 App Store 上发布 Ente 的开发者账户已更改。因此，您需要重新登录。\n\n对于给您带来的不便，我们深表歉意，但这是不可避免的。"), "developerSettings": MessageLookupByLibrary.simpleMessage("开发者设置"), "developerSettingsWarning": MessageLookupByLibrary.simpleMessage("您确定要修改开发者设置吗？"), diff --git a/mobile/lib/generated/l10n.dart b/mobile/lib/generated/l10n.dart index 23b67ff0bf..d37e83a139 100644 --- a/mobile/lib/generated/l10n.dart +++ b/mobile/lib/generated/l10n.dart @@ -4735,11 +4735,11 @@ class S { ); } - /// `The developer account we use to publish Ente on App Store has changed. Because of this, you will need to login again.\n\nOur apologies for the inconvenience, but this was unavoidable.` - String get devAccountChanged { + /// `Due to technical glitch, you have been logged out. Our apologies for the inconvenience.` + String get autoLogoutMessage { return Intl.message( - 'The developer account we use to publish Ente on App Store has changed. Because of this, you will need to login again.\n\nOur apologies for the inconvenience, but this was unavoidable.', - name: 'devAccountChanged', + 'Due to technical glitch, you have been logged out. Our apologies for the inconvenience.', + name: 'autoLogoutMessage', desc: '', args: [], ); diff --git a/mobile/lib/l10n/intl_en.arb b/mobile/lib/l10n/intl_en.arb index df2894e4ce..73e2d7b826 100644 --- a/mobile/lib/l10n/intl_en.arb +++ b/mobile/lib/l10n/intl_en.arb @@ -667,7 +667,7 @@ "mobileWebDesktop": "Mobile, Web, Desktop", "newToEnte": "New to Ente", "pleaseLoginAgain": "Please login again", - "devAccountChanged": "The developer account we use to publish Ente on App Store has changed. Because of this, you will need to login again.\n\nOur apologies for the inconvenience, but this was unavoidable.", + "autoLogoutMessage": "Due to technical glitch, you have been logged out. Our apologies for the inconvenience.", "yourSubscriptionHasExpired": "Your subscription has expired", "storageLimitExceeded": "Storage limit exceeded", "upgrade": "Upgrade", diff --git a/mobile/lib/ui/home/landing_page_widget.dart b/mobile/lib/ui/home/landing_page_widget.dart index 11f043b8b4..16604cfca9 100644 --- a/mobile/lib/ui/home/landing_page_widget.dart +++ b/mobile/lib/ui/home/landing_page_widget.dart @@ -288,7 +288,7 @@ class _LandingPageWidgetState extends State { final result = await showDialogWidget( context: context, title: S.of(context).pleaseLoginAgain, - body: S.of(context).devAccountChanged, + body: S.of(context).autoLogoutMessage, buttons: const [ ButtonWidget( buttonType: ButtonType.neutral, From edb6c804e61e50de608241a06d3aa20ee3f00440 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Wed, 29 May 2024 14:02:50 +0530 Subject: [PATCH 122/354] Lint fix --- mobile/lib/core/configuration.dart | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mobile/lib/core/configuration.dart b/mobile/lib/core/configuration.dart index 729fc1a6c6..7107112bff 100644 --- a/mobile/lib/core/configuration.dart +++ b/mobile/lib/core/configuration.dart @@ -145,7 +145,8 @@ class Configuration { await logout(autoLogout: true); } else { _logger.warning( - 'Platform error ${error.message} with string ${error.toString()}'); + 'Platform error ${error.message} with string ${error.toString()}', + ); rethrow; } } else { From e2791723d0229ce4b2995517d67aae16f406bdef Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Wed, 29 May 2024 14:09:38 +0530 Subject: [PATCH 123/354] Minor fix --- mobile/lib/core/configuration.dart | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/mobile/lib/core/configuration.dart b/mobile/lib/core/configuration.dart index 7107112bff..92bb31e763 100644 --- a/mobile/lib/core/configuration.dart +++ b/mobile/lib/core/configuration.dart @@ -143,11 +143,7 @@ class Configuration { (error.message ?? '').contains('BadPaddingException'); if (isBadPaddingError) { await logout(autoLogout: true); - } else { - _logger.warning( - 'Platform error ${error.message} with string ${error.toString()}', - ); - rethrow; + return; } } else { rethrow; From 642ea88319a6d80d57a57a77b908be9896abc350 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Wed, 29 May 2024 14:12:13 +0530 Subject: [PATCH 124/354] [mob][photos] Remove redundant null checks --- .../services/machine_learning/face_ml/face_ml_service.dart | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart index de4287465e..e4512e2bae 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart @@ -661,12 +661,12 @@ class FaceMlService { } await FaceMLDataDB.instance.updateFaceIdToClusterId(clusteringResult.newFaceIdToCluster); - await FaceMLDataDB.instance.clusterSummaryUpdate(clusteringResult.newClusterSummaries!); + await FaceMLDataDB.instance.clusterSummaryUpdate(clusteringResult.newClusterSummaries); Bus.instance.fire(PeopleChangedEvent()); for (final faceInfo in faceInfoForClustering) { faceInfo.clusterId ??= clusteringResult.newFaceIdToCluster[faceInfo.faceID]; } - for (final clusterUpdate in clusteringResult.newClusterSummaries!.entries) { + for (final clusterUpdate in clusteringResult.newClusterSummaries.entries) { oldClusterSummaries[clusterUpdate.key] = clusterUpdate.value; } _logger.info( @@ -701,7 +701,7 @@ class FaceMlService { 'Updating ${clusteringResult.newFaceIdToCluster.length} FaceIDs with clusterIDs in the DB', ); await FaceMLDataDB.instance.updateFaceIdToClusterId(clusteringResult.newFaceIdToCluster); - await FaceMLDataDB.instance.clusterSummaryUpdate(clusteringResult.newClusterSummaries!); + await FaceMLDataDB.instance.clusterSummaryUpdate(clusteringResult.newClusterSummaries); Bus.instance.fire(PeopleChangedEvent()); _logger.info('Done updating FaceIDs with clusterIDs in the DB, in ' '${DateTime.now().difference(clusterDoneTime).inSeconds} seconds'); From 2ba802d59f461fc954b7e66f1522e9756c3c02a4 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Wed, 29 May 2024 15:22:30 +0530 Subject: [PATCH 125/354] Remove dead code --- .../object_detection/models/predictions.dart | 14 -- .../object_detection/models/recognition.dart | 18 -- .../object_detection/models/stats.dart | 27 --- .../object_detection_service.dart | 157 ------------------ .../object_detection/tflite/classifier.dart | 89 ---------- .../tflite/cocossd_classifier.dart | 115 ------------- .../tflite/mobilenet_classifier.dart | 83 --------- .../tflite/scene_classifier.dart | 88 ---------- .../object_detection/utils/isolate_utils.dart | 88 ---------- 9 files changed, 679 deletions(-) delete mode 100644 mobile/lib/services/object_detection/models/predictions.dart delete mode 100644 mobile/lib/services/object_detection/models/recognition.dart delete mode 100644 mobile/lib/services/object_detection/models/stats.dart delete mode 100644 mobile/lib/services/object_detection/object_detection_service.dart delete mode 100644 mobile/lib/services/object_detection/tflite/classifier.dart delete mode 100644 mobile/lib/services/object_detection/tflite/cocossd_classifier.dart delete mode 100644 mobile/lib/services/object_detection/tflite/mobilenet_classifier.dart delete mode 100644 mobile/lib/services/object_detection/tflite/scene_classifier.dart delete mode 100644 mobile/lib/services/object_detection/utils/isolate_utils.dart diff --git a/mobile/lib/services/object_detection/models/predictions.dart b/mobile/lib/services/object_detection/models/predictions.dart deleted file mode 100644 index 4957fd8cb8..0000000000 --- a/mobile/lib/services/object_detection/models/predictions.dart +++ /dev/null @@ -1,14 +0,0 @@ -import "package:photos/services/object_detection/models/recognition.dart"; -import "package:photos/services/object_detection/models/stats.dart"; - -class Predictions { - final List? recognitions; - final Stats? stats; - final Object? error; - - Predictions( - this.recognitions, - this.stats, { - this.error, - }); -} diff --git a/mobile/lib/services/object_detection/models/recognition.dart b/mobile/lib/services/object_detection/models/recognition.dart deleted file mode 100644 index 3a76c2d565..0000000000 --- a/mobile/lib/services/object_detection/models/recognition.dart +++ /dev/null @@ -1,18 +0,0 @@ -/// Represents the recognition output from the model -class Recognition { - /// Index of the result - final int id; - - /// Label of the result - final String label; - - /// Confidence [0.0, 1.0] - final double score; - - Recognition(this.id, this.label, this.score); - - @override - String toString() { - return 'Recognition(id: $id, label: $label, score: $score)'; - } -} diff --git a/mobile/lib/services/object_detection/models/stats.dart b/mobile/lib/services/object_detection/models/stats.dart deleted file mode 100644 index 6397f42428..0000000000 --- a/mobile/lib/services/object_detection/models/stats.dart +++ /dev/null @@ -1,27 +0,0 @@ -/// Bundles different elapsed times -class Stats { - /// Total time taken in the isolate where the inference runs - final int totalPredictTime; - - /// [totalPredictTime] + communication overhead time - /// between main isolate and another isolate - final int totalElapsedTime; - - /// Time for which inference runs - final int inferenceTime; - - /// Time taken to pre-process the image - final int preProcessingTime; - - Stats( - this.totalPredictTime, - this.totalElapsedTime, - this.inferenceTime, - this.preProcessingTime, - ); - - @override - String toString() { - return 'Stats{totalPredictTime: $totalPredictTime, totalElapsedTime: $totalElapsedTime, inferenceTime: $inferenceTime, preProcessingTime: $preProcessingTime}'; - } -} diff --git a/mobile/lib/services/object_detection/object_detection_service.dart b/mobile/lib/services/object_detection/object_detection_service.dart deleted file mode 100644 index 360747f952..0000000000 --- a/mobile/lib/services/object_detection/object_detection_service.dart +++ /dev/null @@ -1,157 +0,0 @@ -// import "dart:isolate"; -// import "dart:math"; -// import "dart:typed_data"; - -// import "package:logging/logging.dart"; -// import "package:photos/services/object_detection/models/predictions.dart"; -// import 'package:photos/services/object_detection/models/recognition.dart'; -// import 'package:photos/services/object_detection/tflite/cocossd_classifier.dart'; -// import "package:photos/services/object_detection/tflite/mobilenet_classifier.dart"; -// import "package:photos/services/object_detection/tflite/scene_classifier.dart"; -// import "package:photos/services/object_detection/utils/isolate_utils.dart"; - -// class ObjectDetectionService { -// static const scoreThreshold = 0.35; - -// final _logger = Logger("ObjectDetectionService"); - -// late CocoSSDClassifier _objectClassifier; -// late MobileNetClassifier _mobileNetClassifier; -// late SceneClassifier _sceneClassifier; - -// late IsolateUtils _isolateUtils; - -// ObjectDetectionService._privateConstructor(); -// bool inInitiated = false; - -// Future init() async { -// _isolateUtils = IsolateUtils(); -// await _isolateUtils.start(); -// try { -// _objectClassifier = CocoSSDClassifier(); -// } catch (e, s) { -// _logger.severe("Could not initialize cocossd", e, s); -// } -// try { -// _mobileNetClassifier = MobileNetClassifier(); -// } catch (e, s) { -// _logger.severe("Could not initialize mobilenet", e, s); -// } -// try { -// _sceneClassifier = SceneClassifier(); -// } catch (e, s) { -// _logger.severe("Could not initialize sceneclassifier", e, s); -// } -// inInitiated = true; -// } - -// static ObjectDetectionService instance = -// ObjectDetectionService._privateConstructor(); - -// Future> predict(Uint8List bytes) async { -// try { -// if (!inInitiated) { -// return Future.error("ObjectDetectionService init is not completed"); -// } -// final results = {}; -// final methods = [_getObjects, _getMobileNetResults, _getSceneResults]; - -// for (var method in methods) { -// final methodResults = await method(bytes); -// methodResults.forEach((key, value) { -// results.update( -// key, -// (existingValue) => max(existingValue, value), -// ifAbsent: () => value, -// ); -// }); -// } -// return results; -// } catch (e, s) { -// _logger.severe(e, s); -// rethrow; -// } -// } - -// Future> _getObjects(Uint8List bytes) async { -// try { -// final isolateData = IsolateData( -// bytes, -// _objectClassifier.interpreter.address, -// _objectClassifier.labels, -// ClassifierType.cocossd, -// ); -// return _getPredictions(isolateData); -// } catch (e, s) { -// _logger.severe("Could not run cocossd", e, s); -// } -// return {}; -// } - -// Future> _getMobileNetResults(Uint8List bytes) async { -// try { -// final isolateData = IsolateData( -// bytes, -// _mobileNetClassifier.interpreter.address, -// _mobileNetClassifier.labels, -// ClassifierType.mobilenet, -// ); -// return _getPredictions(isolateData); -// } catch (e, s) { -// _logger.severe("Could not run mobilenet", e, s); -// } -// return {}; -// } - -// Future> _getSceneResults(Uint8List bytes) async { -// try { -// final isolateData = IsolateData( -// bytes, -// _sceneClassifier.interpreter.address, -// _sceneClassifier.labels, -// ClassifierType.scenes, -// ); -// return _getPredictions(isolateData); -// } catch (e, s) { -// _logger.severe("Could not run scene detection", e, s); -// } -// return {}; -// } - -// Future> _getPredictions(IsolateData isolateData) async { -// final predictions = await _inference(isolateData); -// final Map results = {}; - -// if (predictions.error == null) { -// for (final Recognition result in predictions.recognitions!) { -// if (result.score > scoreThreshold) { -// // Update the result score only if it's higher than the current score -// if (!results.containsKey(result.label) || -// results[result.label]! < result.score) { -// results[result.label] = result.score; -// } -// } -// } - -// _logger.info( -// "Time taken for ${isolateData.type}: ${predictions.stats!.totalElapsedTime}ms", -// ); -// } else { -// _logger.severe( -// "Error while fetching predictions for ${isolateData.type}", -// predictions.error, -// ); -// } - -// return results; -// } - -// /// Runs inference in another isolate -// Future _inference(IsolateData isolateData) async { -// final responsePort = ReceivePort(); -// _isolateUtils.sendPort.send( -// isolateData..responsePort = responsePort.sendPort, -// ); -// return await responsePort.first; -// } -// } diff --git a/mobile/lib/services/object_detection/tflite/classifier.dart b/mobile/lib/services/object_detection/tflite/classifier.dart deleted file mode 100644 index 6a989b2792..0000000000 --- a/mobile/lib/services/object_detection/tflite/classifier.dart +++ /dev/null @@ -1,89 +0,0 @@ -// import "dart:math"; - -// import 'package:image/image.dart' as image_lib; -// import "package:logging/logging.dart"; -// import "package:photos/services/object_detection/models/predictions.dart"; -// import "package:tflite_flutter/tflite_flutter.dart"; -// import "package:tflite_flutter_helper/tflite_flutter_helper.dart"; - -// abstract class Classifier { -// // Path to the model -// String get modelPath; - -// // Path to the labels -// String get labelPath; - -// // Input size expected by the model (for eg. width = height = 224) -// int get inputSize; - -// // Logger implementation for the specific classifier -// Logger get logger; - -// Predictions? predict(image_lib.Image image); - -// /// Instance of Interpreter -// late Interpreter _interpreter; - -// /// Labels file loaded as list -// late List _labels; - -// /// Shapes of output tensors -// late List> _outputShapes; - -// /// Types of output tensors -// late List _outputTypes; - -// /// Gets the interpreter instance -// Interpreter get interpreter => _interpreter; - -// /// Gets the loaded labels -// List get labels => _labels; - -// /// Gets the output shapes -// List> get outputShapes => _outputShapes; - -// /// Gets the output types -// List get outputTypes => _outputTypes; - -// /// Loads interpreter from asset -// void loadModel(Interpreter? interpreter) async { -// try { -// _interpreter = interpreter ?? -// await Interpreter.fromAsset( -// modelPath, -// options: InterpreterOptions()..threads = 4, -// ); -// final outputTensors = _interpreter.getOutputTensors(); -// _outputShapes = []; -// _outputTypes = []; -// for (var tensor in outputTensors) { -// _outputShapes.add(tensor.shape); -// _outputTypes.add(tensor.type); -// } -// logger.info("Interpreter initialized"); -// } catch (e, s) { -// logger.severe("Error while creating interpreter", e, s); -// } -// } - -// /// Loads labels from assets -// void loadLabels(List? labels) async { -// try { -// _labels = labels ?? await FileUtil.loadLabels(labelPath); -// logger.info("Labels initialized"); -// } catch (e, s) { -// logger.severe("Error while loading labels", e, s); -// } -// } - -// /// Pre-process the image -// TensorImage getProcessedImage(TensorImage inputImage) { -// final padSize = max(inputImage.height, inputImage.width); -// final imageProcessor = ImageProcessorBuilder() -// .add(ResizeWithCropOrPadOp(padSize, padSize)) -// .add(ResizeOp(inputSize, inputSize, ResizeMethod.BILINEAR)) -// .build(); -// inputImage = imageProcessor.process(inputImage); -// return inputImage; -// } -// } diff --git a/mobile/lib/services/object_detection/tflite/cocossd_classifier.dart b/mobile/lib/services/object_detection/tflite/cocossd_classifier.dart deleted file mode 100644 index 246d704982..0000000000 --- a/mobile/lib/services/object_detection/tflite/cocossd_classifier.dart +++ /dev/null @@ -1,115 +0,0 @@ -// import 'dart:math'; - -// import 'package:image/image.dart' as image_lib; -// import "package:logging/logging.dart"; -// import 'package:photos/services/object_detection/models/predictions.dart'; -// import 'package:photos/services/object_detection/models/recognition.dart'; -// import "package:photos/services/object_detection/models/stats.dart"; -// import "package:photos/services/object_detection/tflite/classifier.dart"; -// import "package:tflite_flutter/tflite_flutter.dart"; -// import "package:tflite_flutter_helper/tflite_flutter_helper.dart"; - -// /// Classifier -// class CocoSSDClassifier extends Classifier { -// static final _logger = Logger("CocoSSDClassifier"); -// static const double threshold = 0.4; - -// @override -// String get modelPath => "models/cocossd/model.tflite"; - -// @override -// String get labelPath => "assets/models/cocossd/labels.txt"; - -// @override -// int get inputSize => 300; - -// @override -// Logger get logger => _logger; - -// static const int numResults = 10; - -// CocoSSDClassifier({ -// Interpreter? interpreter, -// List? labels, -// }) { -// loadModel(interpreter); -// loadLabels(labels); -// } - -// @override -// Predictions? predict(image_lib.Image image) { -// final predictStartTime = DateTime.now().millisecondsSinceEpoch; - -// final preProcessStart = DateTime.now().millisecondsSinceEpoch; - -// // Create TensorImage from image -// TensorImage inputImage = TensorImage.fromImage(image); - -// // Pre-process TensorImage -// inputImage = getProcessedImage(inputImage); - -// final preProcessElapsedTime = -// DateTime.now().millisecondsSinceEpoch - preProcessStart; - -// // TensorBuffers for output tensors -// final outputLocations = TensorBufferFloat(outputShapes[0]); -// final outputClasses = TensorBufferFloat(outputShapes[1]); -// final outputScores = TensorBufferFloat(outputShapes[2]); -// final numLocations = TensorBufferFloat(outputShapes[3]); - -// // Inputs object for runForMultipleInputs -// // Use [TensorImage.buffer] or [TensorBuffer.buffer] to pass by reference -// final inputs = [inputImage.buffer]; - -// // Outputs map -// final outputs = { -// 0: outputLocations.buffer, -// 1: outputClasses.buffer, -// 2: outputScores.buffer, -// 3: numLocations.buffer, -// }; - -// final inferenceTimeStart = DateTime.now().millisecondsSinceEpoch; - -// // run inference -// interpreter.runForMultipleInputs(inputs, outputs); - -// final inferenceTimeElapsed = -// DateTime.now().millisecondsSinceEpoch - inferenceTimeStart; - -// // Maximum number of results to show -// final resultsCount = min(numResults, numLocations.getIntValue(0)); - -// // Using labelOffset = 1 as ??? at index 0 -// const labelOffset = 1; - -// final recognitions = []; - -// for (int i = 0; i < resultsCount; i++) { -// // Prediction score -// final score = outputScores.getDoubleValue(i); - -// // Label string -// final labelIndex = outputClasses.getIntValue(i) + labelOffset; -// final label = labels.elementAt(labelIndex); - -// if (score > threshold) { -// recognitions.add( -// Recognition(i, label, score), -// ); -// } -// } - -// final predictElapsedTime = -// DateTime.now().millisecondsSinceEpoch - predictStartTime; -// return Predictions( -// recognitions, -// Stats( -// predictElapsedTime, -// predictElapsedTime, -// inferenceTimeElapsed, -// preProcessElapsedTime, -// ), -// ); -// } -// } diff --git a/mobile/lib/services/object_detection/tflite/mobilenet_classifier.dart b/mobile/lib/services/object_detection/tflite/mobilenet_classifier.dart deleted file mode 100644 index d3cadd1a61..0000000000 --- a/mobile/lib/services/object_detection/tflite/mobilenet_classifier.dart +++ /dev/null @@ -1,83 +0,0 @@ -// import 'package:image/image.dart' as image_lib; -// import "package:logging/logging.dart"; -// import 'package:photos/services/object_detection/models/predictions.dart'; -// import 'package:photos/services/object_detection/models/recognition.dart'; -// import "package:photos/services/object_detection/models/stats.dart"; -// import "package:photos/services/object_detection/tflite/classifier.dart"; -// import "package:tflite_flutter/tflite_flutter.dart"; -// import "package:tflite_flutter_helper/tflite_flutter_helper.dart"; - -// // Source: https://tfhub.dev/tensorflow/lite-model/mobilenet_v1_1.0_224/1/default/1 -// class MobileNetClassifier extends Classifier { -// static final _logger = Logger("MobileNetClassifier"); -// static const double threshold = 0.4; - -// @override -// String get modelPath => "models/mobilenet/mobilenet_v1_1.0_224_quant.tflite"; - -// @override -// String get labelPath => -// "assets/models/mobilenet/labels_mobilenet_quant_v1_224.txt"; - -// @override -// int get inputSize => 224; - -// @override -// Logger get logger => _logger; - -// MobileNetClassifier({ -// Interpreter? interpreter, -// List? labels, -// }) { -// loadModel(interpreter); -// loadLabels(labels); -// } - -// @override -// Predictions? predict(image_lib.Image image) { -// final predictStartTime = DateTime.now().millisecondsSinceEpoch; - -// final preProcessStart = DateTime.now().millisecondsSinceEpoch; - -// // Create TensorImage from image -// TensorImage inputImage = TensorImage.fromImage(image); - -// // Pre-process TensorImage -// inputImage = getProcessedImage(inputImage); - -// final preProcessElapsedTime = -// DateTime.now().millisecondsSinceEpoch - preProcessStart; - -// // TensorBuffers for output tensors -// final output = TensorBufferUint8(outputShapes[0]); -// final inferenceTimeStart = DateTime.now().millisecondsSinceEpoch; -// // run inference -// interpreter.run(inputImage.buffer, output.buffer); - -// final inferenceTimeElapsed = -// DateTime.now().millisecondsSinceEpoch - inferenceTimeStart; - -// final recognitions = []; -// for (int i = 0; i < labels.length; i++) { -// final score = output.getDoubleValue(i) / 255; -// final label = labels.elementAt(i); -// if (score >= threshold) { -// recognitions.add( -// Recognition(i, label, score), -// ); -// } -// } - -// final predictElapsedTime = -// DateTime.now().millisecondsSinceEpoch - predictStartTime; -// return Predictions( -// recognitions, -// Stats( -// predictElapsedTime, -// predictElapsedTime, -// inferenceTimeElapsed, -// preProcessElapsedTime, -// ), -// ); -// } -// } diff --git a/mobile/lib/services/object_detection/tflite/scene_classifier.dart b/mobile/lib/services/object_detection/tflite/scene_classifier.dart deleted file mode 100644 index 1c1db6ce8f..0000000000 --- a/mobile/lib/services/object_detection/tflite/scene_classifier.dart +++ /dev/null @@ -1,88 +0,0 @@ -// import "package:flutter/foundation.dart"; -// import 'package:image/image.dart' as image_lib; -// import "package:logging/logging.dart"; -// import 'package:photos/services/object_detection/models/predictions.dart'; -// import 'package:photos/services/object_detection/models/recognition.dart'; -// import "package:photos/services/object_detection/models/stats.dart"; -// import "package:photos/services/object_detection/tflite/classifier.dart"; -// import "package:tflite_flutter/tflite_flutter.dart"; -// import "package:tflite_flutter_helper/tflite_flutter_helper.dart"; - -// // Source: https://tfhub.dev/sayannath/lite-model/image-scene/1 -// class SceneClassifier extends Classifier { -// static final _logger = Logger("SceneClassifier"); -// static const double threshold = 0.35; - -// @override -// String get modelPath => "models/scenes/model.tflite"; - -// @override -// String get labelPath => "assets/models/scenes/labels.txt"; - -// @override -// int get inputSize => 224; - -// @override -// Logger get logger => _logger; - -// SceneClassifier({ -// Interpreter? interpreter, -// List? labels, -// }) { -// loadModel(interpreter); -// loadLabels(labels); -// } - -// @override -// Predictions? predict(image_lib.Image image) { -// final predictStartTime = DateTime.now().millisecondsSinceEpoch; - -// final preProcessStart = DateTime.now().millisecondsSinceEpoch; - -// // Create TensorImage from image -// TensorImage inputImage = TensorImage.fromImage(image); - -// // Pre-process TensorImage -// inputImage = getProcessedImage(inputImage); -// final list = inputImage.getTensorBuffer().getDoubleList(); -// final input = list.reshape([1, inputSize, inputSize, 3]); - -// final preProcessElapsedTime = -// DateTime.now().millisecondsSinceEpoch - preProcessStart; - -// final output = TensorBufferFloat(outputShapes[0]); - -// final inferenceTimeStart = DateTime.now().millisecondsSinceEpoch; -// interpreter.run(input, output.buffer); -// final inferenceTimeElapsed = -// DateTime.now().millisecondsSinceEpoch - inferenceTimeStart; - -// final recognitions = []; -// for (int i = 0; i < labels.length; i++) { -// final score = output.getDoubleValue(i); -// final label = labels.elementAt(i); -// if (score >= threshold) { -// recognitions.add( -// Recognition(i, label, score), -// ); -// } else if (kDebugMode && score > 0.2) { -// debugPrint("scenePrediction score $label is below threshold: $score"); -// } -// } -// debugPrint( -// "Total lables ${labels.length} + reccg ${recognitions.map((e) => e.label).toSet()}", -// ); - -// final predictElapsedTime = -// DateTime.now().millisecondsSinceEpoch - predictStartTime; -// return Predictions( -// recognitions, -// Stats( -// predictElapsedTime, -// predictElapsedTime, -// inferenceTimeElapsed, -// preProcessElapsedTime, -// ), -// ); -// } -// } diff --git a/mobile/lib/services/object_detection/utils/isolate_utils.dart b/mobile/lib/services/object_detection/utils/isolate_utils.dart deleted file mode 100644 index c5c46e8636..0000000000 --- a/mobile/lib/services/object_detection/utils/isolate_utils.dart +++ /dev/null @@ -1,88 +0,0 @@ -// import 'dart:isolate'; -// import "dart:typed_data"; - -// import 'package:image/image.dart' as imgLib; -// import "package:photos/services/object_detection/models/predictions.dart"; -// import "package:photos/services/object_detection/tflite/classifier.dart"; -// import 'package:photos/services/object_detection/tflite/cocossd_classifier.dart'; -// import "package:photos/services/object_detection/tflite/mobilenet_classifier.dart"; -// import "package:photos/services/object_detection/tflite/scene_classifier.dart"; -// import 'package:tflite_flutter/tflite_flutter.dart'; - -// /// Manages separate Isolate instance for inference -// class IsolateUtils { -// static const String debugName = "InferenceIsolate"; - -// late SendPort _sendPort; -// final _receivePort = ReceivePort(); - -// SendPort get sendPort => _sendPort; - -// Future start() async { -// await Isolate.spawn( -// entryPoint, -// _receivePort.sendPort, -// debugName: debugName, -// ); - -// _sendPort = await _receivePort.first; -// } - -// static void entryPoint(SendPort sendPort) async { -// final port = ReceivePort(); -// sendPort.send(port.sendPort); - -// await for (final IsolateData isolateData in port) { -// final classifier = _getClassifier(isolateData); -// final image = imgLib.decodeImage(isolateData.input); -// try { -// final results = classifier.predict(image!); -// isolateData.responsePort.send(results); -// } catch (e) { -// isolateData.responsePort.send(Predictions(null, null, error: e)); -// } -// } -// } - -// static Classifier _getClassifier(IsolateData isolateData) { -// final interpreter = Interpreter.fromAddress(isolateData.interpreterAddress); -// if (isolateData.type == ClassifierType.cocossd) { -// return CocoSSDClassifier( -// interpreter: interpreter, -// labels: isolateData.labels, -// ); -// } else if (isolateData.type == ClassifierType.mobilenet) { -// return MobileNetClassifier( -// interpreter: interpreter, -// labels: isolateData.labels, -// ); -// } else { -// return SceneClassifier( -// interpreter: interpreter, -// labels: isolateData.labels, -// ); -// } -// } -// } - -// /// Bundles data to pass between Isolate -// class IsolateData { -// Uint8List input; -// int interpreterAddress; -// List labels; -// ClassifierType type; -// late SendPort responsePort; - -// IsolateData( -// this.input, -// this.interpreterAddress, -// this.labels, -// this.type, -// ); -// } - -// enum ClassifierType { -// cocossd, -// mobilenet, -// scenes, -// } From 6b655c815781060fcf882fd42cbff14b8d651c25 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Wed, 29 May 2024 15:26:58 +0530 Subject: [PATCH 126/354] [mob] Hide faceItemWidget from fileInfo if faceIndexing is disabled --- mobile/lib/ui/viewer/file/file_details_widget.dart | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/mobile/lib/ui/viewer/file/file_details_widget.dart b/mobile/lib/ui/viewer/file/file_details_widget.dart index d87a806cc4..2423ee77c8 100644 --- a/mobile/lib/ui/viewer/file/file_details_widget.dart +++ b/mobile/lib/ui/viewer/file/file_details_widget.dart @@ -11,7 +11,6 @@ import 'package:photos/models/file/file.dart'; import 'package:photos/models/file/file_type.dart'; import "package:photos/models/metadata/file_magic.dart"; import "package:photos/services/file_magic_service.dart"; -import "package:photos/services/update_service.dart"; import 'package:photos/theme/ente_theme.dart'; import 'package:photos/ui/components/buttons/icon_button_widget.dart'; import "package:photos/ui/components/divider_widget.dart"; @@ -26,6 +25,7 @@ import "package:photos/ui/viewer/file_details/faces_item_widget.dart"; import "package:photos/ui/viewer/file_details/file_properties_item_widget.dart"; import "package:photos/ui/viewer/file_details/location_tags_widget.dart"; import "package:photos/utils/exif_util.dart"; +import "package:photos/utils/local_settings.dart"; class FileDetailsWidget extends StatefulWidget { final EnteFile file; @@ -230,9 +230,8 @@ class _FileDetailsWidgetState extends State { ]); } - if (!UpdateService.instance.isFdroidFlavor()) { + if (LocalSettings.instance.isFaceIndexingEnabled) { fileDetailsTiles.addAll([ - // ObjectsItemWidget(file), FacesItemWidget(file), const FileDetailsDivider(), ]); From 6c77901396ecbde9fbecfe9a06f6f78ce397f729 Mon Sep 17 00:00:00 2001 From: ashilkn Date: Wed, 29 May 2024 15:41:26 +0530 Subject: [PATCH 127/354] [mob][photos] Migrating to flutter_map v6 (2) --- mobile/lib/ui/map/map_marker.dart | 4 ++-- mobile/lib/ui/map/map_screen.dart | 2 +- mobile/lib/ui/map/map_view.dart | 14 ++++++-------- mobile/pubspec.lock | 20 ++++++++++++++------ mobile/pubspec.yaml | 4 ++-- 5 files changed, 25 insertions(+), 19 deletions(-) diff --git a/mobile/lib/ui/map/map_marker.dart b/mobile/lib/ui/map/map_marker.dart index 0009370be1..d4518feb6f 100644 --- a/mobile/lib/ui/map/map_marker.dart +++ b/mobile/lib/ui/map/map_marker.dart @@ -12,7 +12,7 @@ Marker mapMarker( }) { return Marker( //-6.5 is for taking in the height of the MarkerPointer - anchorPos: AnchorPos.exactly(Anchor(markerSize.height / 2, -6.5)), + alignment: Alignment(markerSize.height / 2, -6.5), key: Key(key), width: markerSize.width, height: markerSize.height, @@ -20,7 +20,7 @@ Marker mapMarker( imageMarker.latitude, imageMarker.longitude, ), - builder: (context) => MarkerImage( + child: MarkerImage( file: imageMarker.imageFile, seperator: (MapView.defaultMarkerSize.height + 10) - (MapView.defaultMarkerSize.height - markerSize.height), diff --git a/mobile/lib/ui/map/map_screen.dart b/mobile/lib/ui/map/map_screen.dart index adb2d590d9..78120bd839 100644 --- a/mobile/lib/ui/map/map_screen.dart +++ b/mobile/lib/ui/map/map_screen.dart @@ -114,7 +114,7 @@ class _MapScreenState extends State { ); Timer(Duration(milliseconds: debounceDuration), () { - calculateVisibleMarkers(mapController.bounds!); + calculateVisibleMarkers(mapController.camera.visibleBounds); setState(() { isLoading = false; }); diff --git a/mobile/lib/ui/map/map_view.dart b/mobile/lib/ui/map/map_view.dart index 2a8c0ed4c7..5286924c95 100644 --- a/mobile/lib/ui/map/map_view.dart +++ b/mobile/lib/ui/map/map_view.dart @@ -108,12 +108,10 @@ class _MapViewState extends State { maxClusterRadius: 100, showPolygon: false, size: widget.markerSize, - fitBoundsOptions: const FitBoundsOptions( - padding: EdgeInsets.all(80), - ), + padding: const EdgeInsets.all(80), markers: _markers, onClusterTap: (_) { - onChange(widget.controller.bounds!); + onChange(widget.controller.camera.visibleBounds); }, builder: (context, List markers) { final index = int.parse( @@ -164,8 +162,8 @@ class _MapViewState extends State { icon: Icons.add, onPressed: () { widget.controller.move( - widget.controller.center, - widget.controller.zoom + 1, + widget.controller.camera.center, + widget.controller.camera.zoom + 1, ); }, heroTag: 'zoom-in', @@ -174,8 +172,8 @@ class _MapViewState extends State { icon: Icons.remove, onPressed: () { widget.controller.move( - widget.controller.center, - widget.controller.zoom - 1, + widget.controller.camera.center, + widget.controller.camera.zoom - 1, ); }, heroTag: 'zoom-out', diff --git a/mobile/pubspec.lock b/mobile/pubspec.lock index 8b71025e9f..97f5780063 100644 --- a/mobile/pubspec.lock +++ b/mobile/pubspec.lock @@ -764,26 +764,26 @@ packages: dependency: "direct main" description: name: flutter_map - sha256: "5286f72f87deb132daa1489442d6cc46e986fc105cb727d9ae1b602b35b1d1f3" + sha256: "87cc8349b8fa5dccda5af50018c7374b6645334a0d680931c1fe11bce88fa5bb" url: "https://pub.dev" source: hosted - version: "5.0.0" + version: "6.2.1" flutter_map_marker_cluster: dependency: "direct main" description: name: flutter_map_marker_cluster - sha256: "14bb31b9dd3a759ab4a1ba320d19bbb554d8d7952c8812029c6f6b7bda956906" + sha256: a324f48da5ee83a3f29fd8d08b4b1e6e3114ff5c6cab910124d6a2e1f06f08cc url: "https://pub.dev" source: hosted - version: "1.2.0" + version: "1.3.6" flutter_map_marker_popup: dependency: transitive description: name: flutter_map_marker_popup - sha256: be209c68b19d4c10d9a2f5911e45f7c579624c43a353adb9bf0f2fec0cf30b8c + sha256: ec563bcbae24a18ac16815fb75ac5ab33ccba609e14db70e252a67de19c6639c url: "https://pub.dev" source: hosted - version: "5.2.0" + version: "6.1.2" flutter_native_splash: dependency: "direct main" description: @@ -1252,6 +1252,14 @@ packages: url: "https://pub.dev" source: hosted version: "1.0.10" + logger: + dependency: transitive + description: + name: logger + sha256: af05cc8714f356fd1f3888fb6741cbe9fbe25cdb6eedbab80e1a6db21047d4a4 + url: "https://pub.dev" + source: hosted + version: "2.3.0" logging: dependency: "direct main" description: diff --git a/mobile/pubspec.yaml b/mobile/pubspec.yaml index ed3bf47193..123ba3f2fe 100644 --- a/mobile/pubspec.yaml +++ b/mobile/pubspec.yaml @@ -83,8 +83,8 @@ dependencies: flutter_local_notifications: ^17.0.0 flutter_localizations: sdk: flutter - flutter_map: ^5.0.0 - flutter_map_marker_cluster: ^1.2.0 + flutter_map: ^6.2.0 + flutter_map_marker_cluster: ^1.3.6 flutter_native_splash: ^2.2.0+1 flutter_password_strength: ^0.1.6 flutter_secure_storage: ^8.0.0 From 9922b704e80ff02723c50f84e6bf131ed16fd9af Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Wed, 29 May 2024 16:03:21 +0530 Subject: [PATCH 128/354] [mob][photos] Remove "view confirmied photos" --- .../lib/ui/viewer/people/people_app_bar.dart | 50 ++++--------------- 1 file changed, 10 insertions(+), 40 deletions(-) diff --git a/mobile/lib/ui/viewer/people/people_app_bar.dart b/mobile/lib/ui/viewer/people/people_app_bar.dart index d53059327f..828dff6bfd 100644 --- a/mobile/lib/ui/viewer/people/people_app_bar.dart +++ b/mobile/lib/ui/viewer/people/people_app_bar.dart @@ -18,7 +18,6 @@ import 'package:photos/ui/actions/collection/collection_sharing_actions.dart'; import "package:photos/ui/viewer/people/add_person_action_sheet.dart"; import "package:photos/ui/viewer/people/people_page.dart"; import "package:photos/ui/viewer/people/person_cluster_suggestion.dart"; -import 'package:photos/ui/viewer/people/person_clusters_page.dart'; import "package:photos/utils/dialog_util.dart"; import "package:photos/utils/navigation_util.dart"; @@ -46,8 +45,7 @@ enum PeoplePopupAction { rename, setCover, removeLabel, - viewPhotos, - confirmPhotos, + reviewSuggestions, unignore, } @@ -69,8 +67,7 @@ class _AppBarWidgetState extends State { }; collectionActions = CollectionActions(CollectionsService.instance); widget.selectedFiles.addListener(_selectedFilesListener); - _userAuthEventSubscription = - Bus.instance.on().listen((event) { + _userAuthEventSubscription = Bus.instance.on().listen((event) { setState(() {}); }); _appBarTitle = widget.title; @@ -91,8 +88,7 @@ class _AppBarWidgetState extends State { centerTitle: false, title: Text( _appBarTitle!, - style: - Theme.of(context).textTheme.headlineSmall!.copyWith(fontSize: 16), + style: Theme.of(context).textTheme.headlineSmall!.copyWith(fontSize: 16), maxLines: 2, overflow: TextOverflow.ellipsis, ), @@ -116,8 +112,7 @@ class _AppBarWidgetState extends State { } try { - await PersonService.instance - .updateAttributes(widget.person.remoteID, name: text); + await PersonService.instance.updateAttributes(widget.person.remoteID, name: text); if (mounted) { _appBarTitle = text; setState(() {}); @@ -137,8 +132,7 @@ class _AppBarWidgetState extends State { List _getDefaultActions(BuildContext context) { final List actions = []; // If the user has selected files, don't show any actions - if (widget.selectedFiles.files.isNotEmpty || - !Configuration.instance.hasConfiguredAccount()) { + if (widget.selectedFiles.files.isNotEmpty || !Configuration.instance.hasConfiguredAccount()) { return actions; } @@ -185,19 +179,7 @@ class _AppBarWidgetState extends State { ), ), const PopupMenuItem( - value: PeoplePopupAction.viewPhotos, - child: Row( - children: [ - Icon(Icons.view_array_outlined), - Padding( - padding: EdgeInsets.all(8), - ), - Text('View confirmed photos'), - ], - ), - ), - const PopupMenuItem( - value: PeoplePopupAction.confirmPhotos, + value: PeoplePopupAction.reviewSuggestions, child: Row( children: [ Icon(CupertinoIcons.square_stack_3d_down_right), @@ -236,22 +218,12 @@ class _AppBarWidgetState extends State { return items; }, onSelected: (PeoplePopupAction value) async { - if (value == PeoplePopupAction.viewPhotos) { + if (value == PeoplePopupAction.reviewSuggestions) { // ignore: unawaited_futures unawaited( Navigator.of(context).push( MaterialPageRoute( - builder: (context) => PersonClustersPage(widget.person), - ), - ), - ); - } else if (value == PeoplePopupAction.confirmPhotos) { - // ignore: unawaited_futures - unawaited( - Navigator.of(context).push( - MaterialPageRoute( - builder: (context) => - PersonReviewClusterSuggestion(widget.person), + builder: (context) => PersonReviewClusterSuggestion(widget.person), ), ), ); @@ -294,13 +266,11 @@ class _AppBarWidgetState extends State { bool assignName = false; await showChoiceDialog( context, - title: - "Are you sure you want to show this person in people section again?", + title: "Are you sure you want to show this person in people section again?", firstButtonLabel: "Yes, show person", firstButtonOnTap: () async { try { - await PersonService.instance - .deletePerson(widget.person.remoteID, onlyMapping: false); + await PersonService.instance.deletePerson(widget.person.remoteID, onlyMapping: false); Bus.instance.fire(PeopleChangedEvent()); assignName = true; } catch (e, s) { From ee3ea77831aea5693861ea55ed765b8a94ac0407 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Wed, 29 May 2024 16:04:04 +0530 Subject: [PATCH 129/354] [mob][photos] Don't show naming banner in personCluster --- .../lib/ui/viewer/people/person_clusters_page.dart | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/mobile/lib/ui/viewer/people/person_clusters_page.dart b/mobile/lib/ui/viewer/people/person_clusters_page.dart index 4f7454f310..16d1131682 100644 --- a/mobile/lib/ui/viewer/people/person_clusters_page.dart +++ b/mobile/lib/ui/viewer/people/person_clusters_page.dart @@ -9,7 +9,6 @@ import "package:photos/services/machine_learning/face_ml/person/person_service.d import "package:photos/services/search_service.dart"; import "package:photos/theme/ente_theme.dart"; import "package:photos/ui/viewer/file/no_thumbnail_widget.dart"; -// import "package:photos/ui/viewer/file/thumbnail_widget.dart"; import "package:photos/ui/viewer/people/cluster_page.dart"; import "package:photos/ui/viewer/search/result/person_face_widget.dart"; @@ -34,8 +33,7 @@ class _PersonClustersPageState extends State { title: Text(widget.person.data.name), ), body: FutureBuilder>>( - future: SearchService.instance - .getClusterFilesForPersonID(widget.person.remoteID), + future: SearchService.instance.getClusterFilesForPersonID(widget.person.remoteID), builder: (context, snapshot) { if (snapshot.hasData) { final clusters = snapshot.data!; @@ -57,6 +55,7 @@ class _PersonClustersPageState extends State { files, personID: widget.person, clusterID: index, + showNamingBanner: false, ), ), ); @@ -92,8 +91,7 @@ class _PersonClustersPageState extends State { ), // Add some spacing between the thumbnail and the text Expanded( child: Padding( - padding: - const EdgeInsets.symmetric(horizontal: 8.0), + padding: const EdgeInsets.symmetric(horizontal: 8.0), child: Row( mainAxisAlignment: MainAxisAlignment.spaceBetween, children: [ @@ -105,16 +103,14 @@ class _PersonClustersPageState extends State { ? GestureDetector( onTap: () async { try { - await PersonService.instance - .removeClusterToPerson( + await PersonService.instance.removeClusterToPerson( personID: widget.person.remoteID, clusterID: clusterID, ); _logger.info( "Removed cluster $clusterID from person ${widget.person.remoteID}", ); - Bus.instance - .fire(PeopleChangedEvent()); + Bus.instance.fire(PeopleChangedEvent()); setState(() {}); } catch (e) { _logger.severe( From 85ce2d7e49fdbab5666ee2f2436937b4257a2969 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Wed, 29 May 2024 16:10:03 +0530 Subject: [PATCH 130/354] [mob][photos] Properly reset last viewed clusterID --- mobile/lib/ui/viewer/people/cluster_page.dart | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/mobile/lib/ui/viewer/people/cluster_page.dart b/mobile/lib/ui/viewer/people/cluster_page.dart index ef069887f9..285804f543 100644 --- a/mobile/lib/ui/viewer/people/cluster_page.dart +++ b/mobile/lib/ui/viewer/people/cluster_page.dart @@ -57,8 +57,7 @@ class _ClusterPageState extends State { late final StreamSubscription _filesUpdatedEvent; late final StreamSubscription _peopleChangedEvent; - bool get showNamingBanner => - (!userDismissedNamingBanner && widget.showNamingBanner); + bool get showNamingBanner => (!userDismissedNamingBanner && widget.showNamingBanner); bool userDismissedNamingBanner = false; @@ -67,8 +66,7 @@ class _ClusterPageState extends State { super.initState(); ClusterFeedbackService.setLastViewedClusterID(widget.clusterID); files = widget.searchResult; - _filesUpdatedEvent = - Bus.instance.on().listen((event) { + _filesUpdatedEvent = Bus.instance.on().listen((event) { if (event.type == EventType.deletedFromDevice || event.type == EventType.deletedFromEverywhere || event.type == EventType.deletedFromRemote || @@ -100,6 +98,9 @@ class _ClusterPageState extends State { void dispose() { _filesUpdatedEvent.cancel(); _peopleChangedEvent.cancel(); + if (ClusterFeedbackService.lastViewedClusterID == widget.clusterID) { + ClusterFeedbackService.resetLastViewedClusterID(); + } super.dispose(); } @@ -110,8 +111,7 @@ class _ClusterPageState extends State { final result = files .where( (file) => - file.creationTime! >= creationStartTime && - file.creationTime! <= creationEndTime, + file.creationTime! >= creationStartTime && file.creationTime! <= creationEndTime, ) .toList(); return Future.value( @@ -185,8 +185,7 @@ class _ClusterPageState extends State { context, clusterID: widget.clusterID, ); - if (result != null && - result is (PersonEntity, EnteFile)) { + if (result != null && result is (PersonEntity, EnteFile)) { Navigator.pop(context); // ignore: unawaited_futures routeToPage(context, PeoplePage(person: result.$1)); From 245e9c0fffd616ce20a5dc8067b6b9e80d5bd3a6 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Wed, 29 May 2024 16:20:14 +0530 Subject: [PATCH 131/354] [mob][photos] Bump --- mobile/pubspec.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobile/pubspec.yaml b/mobile/pubspec.yaml index 211d744ccc..fecf742b86 100644 --- a/mobile/pubspec.yaml +++ b/mobile/pubspec.yaml @@ -12,7 +12,7 @@ description: ente photos application # Read more about iOS versioning at # https://developer.apple.com/library/archive/documentation/General/Reference/InfoPlistKeyReference/Articles/CoreFoundationKeys.html -version: 0.8.122+642 +version: 0.8.124+644 publish_to: none environment: From 588df2c3466913d95542b75b082fc0de5615e6b8 Mon Sep 17 00:00:00 2001 From: ashilkn Date: Wed, 29 May 2024 17:46:30 +0530 Subject: [PATCH 132/354] [mob][photos] Migrating to flutter_map v6 (3): Fix cluster with only one image not rendering --- mobile/lib/ui/map/map_marker.dart | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mobile/lib/ui/map/map_marker.dart b/mobile/lib/ui/map/map_marker.dart index d4518feb6f..e9de524227 100644 --- a/mobile/lib/ui/map/map_marker.dart +++ b/mobile/lib/ui/map/map_marker.dart @@ -11,8 +11,7 @@ Marker mapMarker( Size markerSize = MapView.defaultMarkerSize, }) { return Marker( - //-6.5 is for taking in the height of the MarkerPointer - alignment: Alignment(markerSize.height / 2, -6.5), + alignment: Alignment.topCenter, key: Key(key), width: markerSize.width, height: markerSize.height, From d83eedc93d59e5f9415a1f0bad40381d7b86615c Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 29 May 2024 18:58:02 +0530 Subject: [PATCH 133/354] [web] Invalidate auth session's on password changes --- web/packages/accounts/api/user.ts | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/web/packages/accounts/api/user.ts b/web/packages/accounts/api/user.ts index d14402ebe1..d28e7685bb 100644 --- a/web/packages/accounts/api/user.ts +++ b/web/packages/accounts/api/user.ts @@ -43,6 +43,34 @@ export const putAttributes = (token: string, keyAttributes: KeyAttributes) => }, ); +/** + * Verify that the given auth {@link token} is still valid. + * + * If the user changes their password on another device, then all existing + * auth tokens get invalidated. Existing clients should at opportune times + * make an API call with the auth token that they have saved locally to see + * if the session should be invalidated. When this happens, we inform the + * user with a dialog and prompt them to logout. + */ +export const validateAuthToken = async (token: string) => { + try { + await HTTPService.get(`${ENDPOINT}/users/session-validity/v2`, null, { + "X-Auth-Token": token, + }); + return true; + } catch (e) { + // We get back a 401 Unauthorized if the token is not valid. + if ( + e instanceof ApiError && + e.httpStatusCode == HttpStatusCode.Unauthorized + ) { + return false; + } else { + throw e; + } + } +}; + export const logout = async () => { try { const token = getToken(); From 9ae13ec1596b6cd7802fe82a921b8c249e2a843d Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 29 May 2024 19:09:29 +0530 Subject: [PATCH 134/354] Do it inline --- web/apps/auth/src/pages/auth.tsx | 28 +++++++++++++++++++++++++--- web/packages/accounts/api/user.ts | 28 ---------------------------- 2 files changed, 25 insertions(+), 31 deletions(-) diff --git a/web/apps/auth/src/pages/auth.tsx b/web/apps/auth/src/pages/auth.tsx index 80a342f38a..9c49ffbed3 100644 --- a/web/apps/auth/src/pages/auth.tsx +++ b/web/apps/auth/src/pages/auth.tsx @@ -3,13 +3,14 @@ import { HorizontalFlex, VerticallyCentered, } from "@ente/shared/components/Container"; +import type { DialogBoxAttributesV2 } from "@ente/shared/components/DialogBoxV2/types"; import { EnteLogo } from "@ente/shared/components/EnteLogo"; import EnteSpinner from "@ente/shared/components/EnteSpinner"; import NavbarBase from "@ente/shared/components/Navbar/base"; import OverflowMenu from "@ente/shared/components/OverflowMenu/menu"; import { OverflowMenuOption } from "@ente/shared/components/OverflowMenu/option"; import { AUTH_PAGES as PAGES } from "@ente/shared/constants/pages"; -import { CustomError } from "@ente/shared/error"; +import { ApiError, CustomError } from "@ente/shared/error"; import InMemoryStore, { MS_KEYS } from "@ente/shared/storage/InMemoryStore"; import LogoutOutlined from "@mui/icons-material/LogoutOutlined"; import MoreHoriz from "@mui/icons-material/MoreHoriz"; @@ -22,12 +23,17 @@ import { generateOTPs, type Code } from "services/code"; import { getAuthCodes } from "services/remote"; const Page: React.FC = () => { - const appContext = ensure(useContext(AppContext)); + const { logout, showNavBar, setDialogBoxAttributesV2 } = ensure( + useContext(AppContext), + ); const router = useRouter(); const [codes, setCodes] = useState([]); const [hasFetched, setHasFetched] = useState(false); const [searchTerm, setSearchTerm] = useState(""); + const showSessionExpiredDialog = () => + setDialogBoxAttributesV2(sessionExpiredDialogAttributes(logout)); + useEffect(() => { const fetchCodes = async () => { try { @@ -39,6 +45,9 @@ const Page: React.FC = () => { ) { InMemoryStore.set(MS_KEYS.REDIRECT_URL, PAGES.AUTH); router.push(PAGES.ROOT); + } else if (e instanceof ApiError && e.httpStatusCode == 401) { + // We get back a 401 Unauthorized if the token is not valid. + showSessionExpiredDialog(); } else { // do not log errors } @@ -46,7 +55,7 @@ const Page: React.FC = () => { setHasFetched(true); }; void fetchCodes(); - appContext.showNavBar(false); + showNavBar(false); }, []); const lcSearch = searchTerm.toLowerCase(); @@ -131,6 +140,19 @@ const Page: React.FC = () => { export default Page; +const sessionExpiredDialogAttributes = ( + action: () => void, +): DialogBoxAttributesV2 => ({ + title: t("SESSION_EXPIRED"), + content: t("SESSION_EXPIRED_MESSAGE"), + nonClosable: true, + proceed: { + text: t("LOGIN"), + action, + variant: "accent", + }, +}); + const AuthNavbar: React.FC = () => { const { isMobile, logout } = ensure(useContext(AppContext)); diff --git a/web/packages/accounts/api/user.ts b/web/packages/accounts/api/user.ts index d28e7685bb..d14402ebe1 100644 --- a/web/packages/accounts/api/user.ts +++ b/web/packages/accounts/api/user.ts @@ -43,34 +43,6 @@ export const putAttributes = (token: string, keyAttributes: KeyAttributes) => }, ); -/** - * Verify that the given auth {@link token} is still valid. - * - * If the user changes their password on another device, then all existing - * auth tokens get invalidated. Existing clients should at opportune times - * make an API call with the auth token that they have saved locally to see - * if the session should be invalidated. When this happens, we inform the - * user with a dialog and prompt them to logout. - */ -export const validateAuthToken = async (token: string) => { - try { - await HTTPService.get(`${ENDPOINT}/users/session-validity/v2`, null, { - "X-Auth-Token": token, - }); - return true; - } catch (e) { - // We get back a 401 Unauthorized if the token is not valid. - if ( - e instanceof ApiError && - e.httpStatusCode == HttpStatusCode.Unauthorized - ) { - return false; - } else { - throw e; - } - } -}; - export const logout = async () => { try { const token = getToken(); From a44e932c84f4994090507f576fc2310a38e04bb8 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 29 May 2024 13:50:04 +0530 Subject: [PATCH 135/354] Plan --- web/apps/photos/src/services/face/indexer.ts | 23 ++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 web/apps/photos/src/services/face/indexer.ts diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts new file mode 100644 index 0000000000..6feba91ae1 --- /dev/null +++ b/web/apps/photos/src/services/face/indexer.ts @@ -0,0 +1,23 @@ +/** + * Face indexer + * + * This is class that drives the face indexing process, across all files that + * need to still be indexed. This usually runs in a Web Worker so as to not get + * in the way of the main thread. + * + * It operates in two modes - live indexing and backfill. + * + * In live indexing, any files that are being uploaded from the current client + * are provided to the indexer, which indexes them. This is more efficient since + * we already have the file's content at hand and do not have to download and + * decrypt it. + * + * In backfill, the indexer figures out if any of the user's files (irrespective + * of where they were uploaded from) still need to be indexed, and if so, + * downloads, decrypts and indexes them. + * + * Live indexing has higher priority, backfill runs otherwise. + * + * If nothing needs to be indexed, the indexer goes to sleep. + */ +export class Indexer {} From 2fb7ee01713434d791f4f00c22936bfbcc554869 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 29 May 2024 13:56:11 +0530 Subject: [PATCH 136/354] Sketch --- web/apps/photos/src/services/face/indexer.ts | 22 +++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index 6feba91ae1..8ac27ba1f8 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -1,5 +1,7 @@ +import type { EnteFile } from "types/file"; + /** - * Face indexer + * Face indexing orchestrator. * * This is class that drives the face indexing process, across all files that * need to still be indexed. This usually runs in a Web Worker so as to not get @@ -18,6 +20,20 @@ * * Live indexing has higher priority, backfill runs otherwise. * - * If nothing needs to be indexed, the indexer goes to sleep. + * If nothing needs to be indexed, the indexer goes to sleep for a while. */ -export class Indexer {} +export class FaceIndexer { + private liveItems: [file: File, enteFile: EnteFile][]; + + /** + * Add {@link file} associated with {@link enteFile} to the live indexing + * queue. + */ + enqueueFile(file: File, enteFile: EnteFile) { + this.liveItems.push([file, enteFile]); + } + + tick() { + + } +} From f8aa74979955ef57925058f59a7d3606669d6c40 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 29 May 2024 14:07:37 +0530 Subject: [PATCH 137/354] timeout --- web/apps/photos/src/services/face/indexer.ts | 32 ++++++++++++++++++-- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index 8ac27ba1f8..a2e15cbedf 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -1,4 +1,6 @@ import type { EnteFile } from "types/file"; +import { indexFaces } from "./f-index"; +import type { MlFileData } from "./types-old"; /** * Face indexing orchestrator. @@ -23,17 +25,41 @@ import type { EnteFile } from "types/file"; * If nothing needs to be indexed, the indexer goes to sleep for a while. */ export class FaceIndexer { - private liveItems: [file: File, enteFile: EnteFile][]; + /** Live indexing queue. */ + private liveItems: { file: File; enteFile: EnteFile }[]; + /** True when we are sleeping. */ + private isPaused = false; + /** Timeout for when the next time we will wake up. */ + private wakeTimeout: ReturnType | undefined; /** * Add {@link file} associated with {@link enteFile} to the live indexing * queue. */ enqueueFile(file: File, enteFile: EnteFile) { - this.liveItems.push([file, enteFile]); + this.liveItems.push({ file, enteFile }); + this.isPaused = false; + this.wakeUpIfNeeded() } - tick() { + private wakeUpIfNeeded() { + // Already awake + if (!this.wakeTimeout) return; + // Cancel the alarm, wake up now. + clearTimeout(this.wakeTimeout); + this.wakeTimeout = undefined; + this.tick(); + } + + private async tick() { + const item = this.liveItems.pop(); + let faceIndex: MlFileData | undefined; + if (item) { + faceIndex = await indexFaces(item.enteFile, item.file); + } else { + // backfill + } + console.log("indexed face", faceIndex); } } From 7f150d8dc724afb3ad575acf4bc28bc3f13b1ee2 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 29 May 2024 14:25:20 +0530 Subject: [PATCH 138/354] comp --- web/apps/photos/src/services/face/indexer.ts | 41 ++++++++++++++------ 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index a2e15cbedf..5a5df13c79 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -1,6 +1,8 @@ +import log from "@/next/log"; +import { wait } from "@/utils/promise"; import type { EnteFile } from "types/file"; +import { markIndexingFailed } from "./db"; import { indexFaces } from "./f-index"; -import type { MlFileData } from "./types-old"; /** * Face indexing orchestrator. @@ -27,7 +29,7 @@ import type { MlFileData } from "./types-old"; export class FaceIndexer { /** Live indexing queue. */ private liveItems: { file: File; enteFile: EnteFile }[]; - /** True when we are sleeping. */ + /** True when we have been paused externally. */ private isPaused = false; /** Timeout for when the next time we will wake up. */ private wakeTimeout: ReturnType | undefined; @@ -38,28 +40,45 @@ export class FaceIndexer { */ enqueueFile(file: File, enteFile: EnteFile) { this.liveItems.push({ file, enteFile }); - this.isPaused = false; - this.wakeUpIfNeeded() + this.wakeUpIfNeeded(); } private wakeUpIfNeeded() { - // Already awake + // If we were asked to pause, don't do anything. + if (this.isPaused) return; + // Already awake. if (!this.wakeTimeout) return; // Cancel the alarm, wake up now. clearTimeout(this.wakeTimeout); this.wakeTimeout = undefined; + // Get to work. this.tick(); } private async tick() { const item = this.liveItems.pop(); - let faceIndex: MlFileData | undefined; - if (item) { - faceIndex = await indexFaces(item.enteFile, item.file); - } else { - // backfill + if (!item) { + // TODO-ML: backfill instead if needed here. + if (!this.isPaused) { + this.wakeTimeout = setTimeout(() => { + this.wakeTimeout = undefined; + this.wakeUpIfNeeded(); + }, 30 * 1000); + } + return; } - console.log("indexed face", faceIndex); + const fileID = item.enteFile.id; + try { + const faceIndex = await indexFaces(item.enteFile, item.file); + } catch (e) { + log.error(`Failed to index faces in file ${fileID}`, e); + markIndexingFailed(item.enteFile.id); + } + + // Let the runloop drain. + await wait(0); + // Run again. + this.tick(); } } From c968cc3c4170e073944415ecd11d6a233b323225 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 29 May 2024 15:26:20 +0530 Subject: [PATCH 139/354] remote --- web/apps/photos/src/services/face/indexer.ts | 100 ++++-------------- .../src/services/face/indexer.worker.ts | 81 ++++++++++++++ 2 files changed, 102 insertions(+), 79 deletions(-) create mode 100644 web/apps/photos/src/services/face/indexer.worker.ts diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index 5a5df13c79..f028148cec 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -1,84 +1,26 @@ -import log from "@/next/log"; -import { wait } from "@/utils/promise"; -import type { EnteFile } from "types/file"; -import { markIndexingFailed } from "./db"; -import { indexFaces } from "./f-index"; +import { ComlinkWorker } from "@/next/worker/comlink-worker"; +import { type Remote } from "comlink"; +import { FaceIndexerWorker } from "./indexer.worker"; /** - * Face indexing orchestrator. - * - * This is class that drives the face indexing process, across all files that - * need to still be indexed. This usually runs in a Web Worker so as to not get - * in the way of the main thread. - * - * It operates in two modes - live indexing and backfill. - * - * In live indexing, any files that are being uploaded from the current client - * are provided to the indexer, which indexes them. This is more efficient since - * we already have the file's content at hand and do not have to download and - * decrypt it. - * - * In backfill, the indexer figures out if any of the user's files (irrespective - * of where they were uploaded from) still need to be indexed, and if so, - * downloads, decrypts and indexes them. - * - * Live indexing has higher priority, backfill runs otherwise. - * - * If nothing needs to be indexed, the indexer goes to sleep for a while. + * A promise for the lazily created singleton {@link FaceIndexerWorker} remote + * exposed by this module. */ -export class FaceIndexer { - /** Live indexing queue. */ - private liveItems: { file: File; enteFile: EnteFile }[]; - /** True when we have been paused externally. */ - private isPaused = false; - /** Timeout for when the next time we will wake up. */ - private wakeTimeout: ReturnType | undefined; +let _faceIndexerWorker: Promise>; - /** - * Add {@link file} associated with {@link enteFile} to the live indexing - * queue. - */ - enqueueFile(file: File, enteFile: EnteFile) { - this.liveItems.push({ file, enteFile }); - this.wakeUpIfNeeded(); - } +const createFaceIndexerComlinkWorker = () => + new ComlinkWorker( + "face-indexer", + new Worker(new URL("indexer.worker.ts", import.meta.url)), + ); - private wakeUpIfNeeded() { - // If we were asked to pause, don't do anything. - if (this.isPaused) return; - // Already awake. - if (!this.wakeTimeout) return; - // Cancel the alarm, wake up now. - clearTimeout(this.wakeTimeout); - this.wakeTimeout = undefined; - // Get to work. - this.tick(); - } - - private async tick() { - const item = this.liveItems.pop(); - if (!item) { - // TODO-ML: backfill instead if needed here. - if (!this.isPaused) { - this.wakeTimeout = setTimeout(() => { - this.wakeTimeout = undefined; - this.wakeUpIfNeeded(); - }, 30 * 1000); - } - return; - } - - const fileID = item.enteFile.id; - try { - const faceIndex = await indexFaces(item.enteFile, item.file); - } catch (e) { - log.error(`Failed to index faces in file ${fileID}`, e); - markIndexingFailed(item.enteFile.id); - } - - // Let the runloop drain. - await wait(0); - // Run again. - this.tick(); - } -} +/** + * Main thread interface to the face indexer. + * + * This function provides a promise that resolves to a lazily created singleton + * remote with a {@link FaceIndexerWorker} at the other end. + * + * For more details, see the documentation for {@link FaceIndexerWorker}. + */ +export const faceIndexerWorker = (): Promise> => + (_faceIndexerWorker ??= createFaceIndexerComlinkWorker().remote); diff --git a/web/apps/photos/src/services/face/indexer.worker.ts b/web/apps/photos/src/services/face/indexer.worker.ts new file mode 100644 index 0000000000..6ec4f2be94 --- /dev/null +++ b/web/apps/photos/src/services/face/indexer.worker.ts @@ -0,0 +1,81 @@ +import log from "@/next/log"; +import { wait } from "@/utils/promise"; +import type { EnteFile } from "types/file"; +import { markIndexingFailed } from "./db"; +import { indexFaces } from "./f-index"; + +/** + * Face indexing orchestrator. + * + * This is class that drives the face indexing process, across all files that + * need to still be indexed. This usually runs in a Web Worker so as to not get + * in the way of the main thread. + * + * It operates in two modes - live indexing and backfill. + * + * In live indexing, any files that are being uploaded from the current client + * are provided to the indexer, which indexes them. This is more efficient since + * we already have the file's content at hand and do not have to download and + * decrypt it. + * + * In backfill, the indexer figures out if any of the user's files (irrespective + * of where they were uploaded from) still need to be indexed, and if so, + * downloads, decrypts and indexes them. + * + * Live indexing has higher priority, backfill runs otherwise. + * + * If nothing needs to be indexed, the indexer goes to sleep for a while. + */ +export class FaceIndexerWorker { + /** Live indexing queue. */ + private liveItems: { file: File; enteFile: EnteFile }[]; + /** Timeout for when the next time we will wake up. */ + private wakeTimeout: ReturnType | undefined; + + /** + * Add {@link file} associated with {@link enteFile} to the live indexing + * queue. + */ + enqueueFile(file: File, enteFile: EnteFile) { + this.liveItems.push({ file, enteFile }); + this.wakeUpIfNeeded(); + } + + private wakeUpIfNeeded() { + // Already awake. + if (!this.wakeTimeout) return; + // Cancel the alarm, wake up now. + clearTimeout(this.wakeTimeout); + this.wakeTimeout = undefined; + // Get to work. + this.tick(); + } + + private async tick() { + console.log("tick"); + + const item = this.liveItems.pop(); + if (!item) { + // TODO-ML: backfill instead if needed here. + this.wakeTimeout = setTimeout(() => { + this.wakeTimeout = undefined; + this.wakeUpIfNeeded(); + }, 30 * 1000); + return; + } + + const fileID = item.enteFile.id; + try { + const faceIndex = await indexFaces(item.enteFile, item.file); + log.info(`faces in file ${fileID}`, faceIndex); + } catch (e) { + log.error(`Failed to index faces in file ${fileID}`, e); + markIndexingFailed(item.enteFile.id); + } + + // Let the runloop drain. + await wait(0); + // Run again. + this.tick(); + } +} From 9adc8126bb7120b38201840c63f05c3cc87d8157 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 29 May 2024 15:28:35 +0530 Subject: [PATCH 140/354] Rename --- web/apps/photos/src/services/face/indexer.ts | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index f028148cec..f998e5ac4d 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -6,7 +6,7 @@ import { FaceIndexerWorker } from "./indexer.worker"; * A promise for the lazily created singleton {@link FaceIndexerWorker} remote * exposed by this module. */ -let _faceIndexerWorker: Promise>; +let _faceIndexer: Promise>; const createFaceIndexerComlinkWorker = () => new ComlinkWorker( @@ -19,8 +19,6 @@ const createFaceIndexerComlinkWorker = () => * * This function provides a promise that resolves to a lazily created singleton * remote with a {@link FaceIndexerWorker} at the other end. - * - * For more details, see the documentation for {@link FaceIndexerWorker}. */ -export const faceIndexerWorker = (): Promise> => - (_faceIndexerWorker ??= createFaceIndexerComlinkWorker().remote); +export const faceIndexer = (): Promise> => + (_faceIndexer ??= createFaceIndexerComlinkWorker().remote); From daf72d8ac6d72aeaf8b832be8e339e7b71b69962 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 29 May 2024 15:39:13 +0530 Subject: [PATCH 141/354] Tweak --- web/apps/photos/src/services/face/indexer.worker.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/web/apps/photos/src/services/face/indexer.worker.ts b/web/apps/photos/src/services/face/indexer.worker.ts index 6ec4f2be94..3f7ee822d0 100644 --- a/web/apps/photos/src/services/face/indexer.worker.ts +++ b/web/apps/photos/src/services/face/indexer.worker.ts @@ -7,16 +7,16 @@ import { indexFaces } from "./f-index"; /** * Face indexing orchestrator. * - * This is class that drives the face indexing process, across all files that - * need to still be indexed. This usually runs in a Web Worker so as to not get - * in the way of the main thread. + * This is class that drives the face indexing process across all files that + * need to still be indexed. It runs in a Web Worker so as to not get in the way + * of the main thread. * * It operates in two modes - live indexing and backfill. * * In live indexing, any files that are being uploaded from the current client - * are provided to the indexer, which indexes them. This is more efficient since - * we already have the file's content at hand and do not have to download and - * decrypt it. + * are provided to the indexer, which puts them in a queue and indexes them one + * by one. This is more efficient since we already have the file's content at + * hand and do not have to download and decrypt it. * * In backfill, the indexer figures out if any of the user's files (irrespective * of where they were uploaded from) still need to be indexed, and if so, From 6097f9d4ba844cd3a0b960557b85e4ba12339199 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 29 May 2024 18:40:27 +0530 Subject: [PATCH 142/354] wip --- web/apps/photos/src/services/face/indexer.ts | 18 +++++++++++++++++- .../services/machineLearning/mlWorkManager.ts | 4 ++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index f998e5ac4d..ba06e6f4a4 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -1,5 +1,7 @@ import { ComlinkWorker } from "@/next/worker/comlink-worker"; import { type Remote } from "comlink"; +import mlWorkManager from "services/machineLearning/mlWorkManager"; +import type { EnteFile } from "types/file"; import { FaceIndexerWorker } from "./indexer.worker"; /** @@ -20,5 +22,19 @@ const createFaceIndexerComlinkWorker = () => * This function provides a promise that resolves to a lazily created singleton * remote with a {@link FaceIndexerWorker} at the other end. */ -export const faceIndexer = (): Promise> => +const faceIndexer = (): Promise> => (_faceIndexer ??= createFaceIndexerComlinkWorker().remote); + +/** + * Add a newly uploaded file to the face indexing queue. + * + * @param enteFile The {@link EnteFile} that was uploaded. + * @param file + */ +export const indexFacesInFile = (enteFile: EnteFile, file: File) => { + if (!mlWorkManager.isMlSearchEnabled) return; + + faceIndexer().then((indexer) => { + indexer.enqueueFile(file, enteFile); + }); +}; diff --git a/web/apps/photos/src/services/machineLearning/mlWorkManager.ts b/web/apps/photos/src/services/machineLearning/mlWorkManager.ts index 0ec5f29541..3f7e9571e8 100644 --- a/web/apps/photos/src/services/machineLearning/mlWorkManager.ts +++ b/web/apps/photos/src/services/machineLearning/mlWorkManager.ts @@ -117,6 +117,10 @@ class MLWorkManager { ); } + public isMlSearchEnabled() { + return this.mlSearchEnabled; + } + public async setMlSearchEnabled(enabled: boolean) { if (!this.mlSearchEnabled && enabled) { log.info("Enabling MLWorkManager"); From ce1ba6112fff3951941e72eda793b76f94f2b918 Mon Sep 17 00:00:00 2001 From: ialexanderbrito Date: Wed, 29 May 2024 11:03:03 -0300 Subject: [PATCH 143/354] fix: icons error and new icon --- auth/assets/custom-icons/icons/configcat.svg | 6 +++--- auth/assets/custom-icons/icons/habbo.svg | 6 +++--- auth/assets/custom-icons/icons/local_wp.svg | 9 +++++++++ auth/assets/custom-icons/icons/mercado_livre.svg | 11 ++++++----- auth/assets/custom-icons/icons/sendgrid.svg | 6 +++--- 5 files changed, 24 insertions(+), 14 deletions(-) create mode 100644 auth/assets/custom-icons/icons/local_wp.svg diff --git a/auth/assets/custom-icons/icons/configcat.svg b/auth/assets/custom-icons/icons/configcat.svg index cfecd22b02..12f7e4e81a 100644 --- a/auth/assets/custom-icons/icons/configcat.svg +++ b/auth/assets/custom-icons/icons/configcat.svg @@ -1,7 +1,7 @@ - - + + - + diff --git a/auth/assets/custom-icons/icons/habbo.svg b/auth/assets/custom-icons/icons/habbo.svg index 746bcdb229..2866bc3638 100644 --- a/auth/assets/custom-icons/icons/habbo.svg +++ b/auth/assets/custom-icons/icons/habbo.svg @@ -1,7 +1,7 @@ - - + + - + diff --git a/auth/assets/custom-icons/icons/local_wp.svg b/auth/assets/custom-icons/icons/local_wp.svg new file mode 100644 index 0000000000..3dbe63b2af --- /dev/null +++ b/auth/assets/custom-icons/icons/local_wp.svg @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/auth/assets/custom-icons/icons/mercado_livre.svg b/auth/assets/custom-icons/icons/mercado_livre.svg index 7f4db5fd53..8eeb1b94b5 100644 --- a/auth/assets/custom-icons/icons/mercado_livre.svg +++ b/auth/assets/custom-icons/icons/mercado_livre.svg @@ -1,9 +1,10 @@ - - + + + - - + + - + diff --git a/auth/assets/custom-icons/icons/sendgrid.svg b/auth/assets/custom-icons/icons/sendgrid.svg index 1562adab90..3b65642792 100644 --- a/auth/assets/custom-icons/icons/sendgrid.svg +++ b/auth/assets/custom-icons/icons/sendgrid.svg @@ -1,7 +1,7 @@ - - + + - + From 08a073fc1b5a88956cf42662a349f3f6c2707241 Mon Sep 17 00:00:00 2001 From: ialexanderbrito Date: Wed, 29 May 2024 11:03:27 -0300 Subject: [PATCH 144/354] feat: add new icons and altnames --- auth/assets/custom-icons/_data/custom-icons.json | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/auth/assets/custom-icons/_data/custom-icons.json b/auth/assets/custom-icons/_data/custom-icons.json index b911183e88..94cc5b55cb 100644 --- a/auth/assets/custom-icons/_data/custom-icons.json +++ b/auth/assets/custom-icons/_data/custom-icons.json @@ -190,6 +190,15 @@ { "title": "Letterboxd" }, + { + "title": "Local", + "slug": "local_wp", + "altNames": [ + "LocalWP", + "Local WP", + "Local Wordpress" + ] + }, { "title": "Mastodon", "altNames": [ @@ -203,7 +212,12 @@ }, { "title": "Mercado Livre", - "slug": "mercado_livre" + "slug": "mercado_livre", + "altNames": [ + "Mercado Libre", + "MercadoLibre", + "MercadoLivre" + ] }, { "title": "Murena", From 72851397b1603b808cd5ab205c9c855622dc4c19 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 29 May 2024 20:00:38 +0530 Subject: [PATCH 145/354] wip --- web/apps/photos/src/services/face/indexer.ts | 82 +++++++++++++++++++ .../src/services/face/indexer.worker.ts | 75 +++-------------- 2 files changed, 92 insertions(+), 65 deletions(-) diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index ba06e6f4a4..e629d4533b 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -4,6 +4,88 @@ import mlWorkManager from "services/machineLearning/mlWorkManager"; import type { EnteFile } from "types/file"; import { FaceIndexerWorker } from "./indexer.worker"; +import log from "@/next/log"; +import { wait } from "@/utils/promise"; +import type { EnteFile } from "types/file"; +import { markIndexingFailed } from "./db"; +import { indexFaces } from "./f-index"; + +/** + * Face indexing orchestrator. + * + * This is class that drives the face indexing process across all files that + * need to still be indexed. It runs in a Web Worker so as to not get in the way + * of the main thread. + * + * It operates in two modes - live indexing and backfill. + * + * In live indexing, any files that are being uploaded from the current client + * are provided to the indexer, which puts them in a queue and indexes them one + * by one. This is more efficient since we already have the file's content at + * hand and do not have to download and decrypt it. + * + * In backfill, the indexer figures out if any of the user's files (irrespective + * of where they were uploaded from) still need to be indexed, and if so, + * downloads, decrypts and indexes them. + * + * Live indexing has higher priority, backfill runs otherwise. + * + * If nothing needs to be indexed, the indexer goes to sleep for a while. + */ +export class FaceIndexerWorker { + /** Live indexing queue. */ + private liveItems: { file: File; enteFile: EnteFile }[]; + /** Timeout for when the next time we will wake up. */ + private wakeTimeout: ReturnType | undefined; + + /** + * Add {@link file} associated with {@link enteFile} to the live indexing + * queue. + */ + enqueueFile(file: File, enteFile: EnteFile) { + this.liveItems.push({ file, enteFile }); + this.wakeUpIfNeeded(); + } + + private wakeUpIfNeeded() { + // Already awake. + if (!this.wakeTimeout) return; + // Cancel the alarm, wake up now. + clearTimeout(this.wakeTimeout); + this.wakeTimeout = undefined; + // Get to work. + this.tick(); + } + + private async tick() { + console.log("tick"); + + const item = this.liveItems.pop(); + if (!item) { + // TODO-ML: backfill instead if needed here. + this.wakeTimeout = setTimeout(() => { + this.wakeTimeout = undefined; + this.wakeUpIfNeeded(); + }, 30 * 1000); + return; + } + + const fileID = item.enteFile.id; + try { + const faceIndex = await indexFaces(item.enteFile, item.file); + log.info(`faces in file ${fileID}`, faceIndex); + } catch (e) { + log.error(`Failed to index faces in file ${fileID}`, e); + markIndexingFailed(item.enteFile.id); + } + + // Let the runloop drain. + await wait(0); + // Run again. + this.tick(); + } +} + /** * A promise for the lazily created singleton {@link FaceIndexerWorker} remote * exposed by this module. diff --git a/web/apps/photos/src/services/face/indexer.worker.ts b/web/apps/photos/src/services/face/indexer.worker.ts index 3f7ee822d0..61c62b4029 100644 --- a/web/apps/photos/src/services/face/indexer.worker.ts +++ b/web/apps/photos/src/services/face/indexer.worker.ts @@ -1,81 +1,26 @@ import log from "@/next/log"; -import { wait } from "@/utils/promise"; import type { EnteFile } from "types/file"; import { markIndexingFailed } from "./db"; import { indexFaces } from "./f-index"; /** - * Face indexing orchestrator. + * Index faces in a file, save the persist the results locally, and put them on + * remote. * - * This is class that drives the face indexing process across all files that - * need to still be indexed. It runs in a Web Worker so as to not get in the way - * of the main thread. - * - * It operates in two modes - live indexing and backfill. - * - * In live indexing, any files that are being uploaded from the current client - * are provided to the indexer, which puts them in a queue and indexes them one - * by one. This is more efficient since we already have the file's content at - * hand and do not have to download and decrypt it. - * - * In backfill, the indexer figures out if any of the user's files (irrespective - * of where they were uploaded from) still need to be indexed, and if so, - * downloads, decrypts and indexes them. - * - * Live indexing has higher priority, backfill runs otherwise. - * - * If nothing needs to be indexed, the indexer goes to sleep for a while. + * This class is instantiated in a Web Worker so as to not get in the way of the + * main thread. It could've been a bunch of free standing functions too, it is + * just a class for convenience of compatibility with how the rest of our + * comlink workers are structured. */ export class FaceIndexerWorker { - /** Live indexing queue. */ - private liveItems: { file: File; enteFile: EnteFile }[]; - /** Timeout for when the next time we will wake up. */ - private wakeTimeout: ReturnType | undefined; - - /** - * Add {@link file} associated with {@link enteFile} to the live indexing - * queue. - */ - enqueueFile(file: File, enteFile: EnteFile) { - this.liveItems.push({ file, enteFile }); - this.wakeUpIfNeeded(); - } - - private wakeUpIfNeeded() { - // Already awake. - if (!this.wakeTimeout) return; - // Cancel the alarm, wake up now. - clearTimeout(this.wakeTimeout); - this.wakeTimeout = undefined; - // Get to work. - this.tick(); - } - - private async tick() { - console.log("tick"); - - const item = this.liveItems.pop(); - if (!item) { - // TODO-ML: backfill instead if needed here. - this.wakeTimeout = setTimeout(() => { - this.wakeTimeout = undefined; - this.wakeUpIfNeeded(); - }, 30 * 1000); - return; - } - - const fileID = item.enteFile.id; + async index(enteFile: EnteFile, file: File | undefined) { + const fileID = enteFile.id; try { - const faceIndex = await indexFaces(item.enteFile, item.file); + const faceIndex = await indexFaces(enteFile, file); log.info(`faces in file ${fileID}`, faceIndex); } catch (e) { log.error(`Failed to index faces in file ${fileID}`, e); - markIndexingFailed(item.enteFile.id); + markIndexingFailed(enteFile.id); } - - // Let the runloop drain. - await wait(0); - // Run again. - this.tick(); } } From 4ce02fba93bb73e1b091d342b575e25be227c59a Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 29 May 2024 20:14:53 +0530 Subject: [PATCH 146/354] ll --- web/apps/photos/src/services/face/f-index.ts | 3 +- .../src/services/face/indexer.worker.ts | 29 ++++++++++++++++--- web/apps/photos/src/utils/file/index.ts | 8 +++++ 3 files changed, 35 insertions(+), 5 deletions(-) diff --git a/web/apps/photos/src/services/face/f-index.ts b/web/apps/photos/src/services/face/f-index.ts index 5e93f60bd6..a1a614f138 100644 --- a/web/apps/photos/src/services/face/f-index.ts +++ b/web/apps/photos/src/services/face/f-index.ts @@ -12,6 +12,7 @@ import { translate, } from "transformation-matrix"; import type { EnteFile } from "types/file"; +import { logIdentifier } from "utils/file"; import { saveFaceCrop } from "./crop"; import { fetchImageBitmap, getLocalFileImageBitmap } from "./file"; import { @@ -58,7 +59,7 @@ export const indexFaces = async (enteFile: EnteFile, localFile?: File) => { log.debug(() => { const nf = mlFile.faces?.length ?? 0; const ms = Date.now() - startTime; - return `Indexed ${nf} faces in file ${enteFile.id} (${ms} ms)`; + return `Indexed ${nf} faces in file ${logIdentifier(enteFile)} (${ms} ms)`; }); return mlFile; }; diff --git a/web/apps/photos/src/services/face/indexer.worker.ts b/web/apps/photos/src/services/face/indexer.worker.ts index 61c62b4029..84250dd703 100644 --- a/web/apps/photos/src/services/face/indexer.worker.ts +++ b/web/apps/photos/src/services/face/indexer.worker.ts @@ -1,6 +1,7 @@ import log from "@/next/log"; import type { EnteFile } from "types/file"; -import { markIndexingFailed } from "./db"; +import { logIdentifier } from "utils/file"; +import { closeFaceDBConnectionsIfNeeded, markIndexingFailed } from "./db"; import { indexFaces } from "./f-index"; /** @@ -13,14 +14,34 @@ import { indexFaces } from "./f-index"; * comlink workers are structured. */ export class FaceIndexerWorker { + /* + * Index faces in a file, save the persist the results locally, and put them + * on remote. + * + * @param enteFile The {@link EnteFile} to index. + * + * @param file If the file is one which is being uploaded from the current + * client, then we will also have access to the file's content. In such + * cases, pass a web {@link File} object to use that its data directly for + * face indexing. If this is not provided, then the file's contents will be + * downloaded and decrypted from remote. + */ async index(enteFile: EnteFile, file: File | undefined) { - const fileID = enteFile.id; + const f = logIdentifier(enteFile); try { const faceIndex = await indexFaces(enteFile, file); - log.info(`faces in file ${fileID}`, faceIndex); + log.info(`faces in file ${f}`, faceIndex); } catch (e) { - log.error(`Failed to index faces in file ${fileID}`, e); + log.error(`Failed to index faces in file ${f}`, e); markIndexingFailed(enteFile.id); } } + + /** + * Calls {@link closeFaceDBConnectionsIfNeeded} to close any open + * connections to the face DB from the web worker's context. + */ + closeFaceDB() { + closeFaceDBConnectionsIfNeeded(); + } } diff --git a/web/apps/photos/src/utils/file/index.ts b/web/apps/photos/src/utils/file/index.ts index 3a349abea7..c15cca63c0 100644 --- a/web/apps/photos/src/utils/file/index.ts +++ b/web/apps/photos/src/utils/file/index.ts @@ -81,6 +81,14 @@ class ModuleState { const moduleState = new ModuleState(); +/** + * @returns a string to use as an identifier when logging information about the + * given {@link enteFile}. The returned string contains the file name (for ease + * of debugging) and the file ID (for exactness). + */ +export const logIdentifier = (enteFile: EnteFile) => + `${enteFile.metadata.title ?? "-"} (${enteFile.id})`; + export async function getUpdatedEXIFFileForDownload( fileReader: FileReader, file: EnteFile, From 3b8ab89647713b7a8f8340f4396b9b7f122290cc Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 29 May 2024 20:27:57 +0530 Subject: [PATCH 147/354] w --- web/apps/photos/src/services/face/indexer.ts | 111 ++++++++++--------- 1 file changed, 60 insertions(+), 51 deletions(-) diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index e629d4533b..661aa65288 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -1,49 +1,54 @@ -import { ComlinkWorker } from "@/next/worker/comlink-worker"; -import { type Remote } from "comlink"; -import mlWorkManager from "services/machineLearning/mlWorkManager"; -import type { EnteFile } from "types/file"; -import { FaceIndexerWorker } from "./indexer.worker"; - import log from "@/next/log"; +import { ComlinkWorker } from "@/next/worker/comlink-worker"; import { wait } from "@/utils/promise"; +import { type Remote } from "comlink"; import type { EnteFile } from "types/file"; import { markIndexingFailed } from "./db"; import { indexFaces } from "./f-index"; +import { FaceIndexerWorker } from "./indexer.worker"; +import mlWorkManager from "services/machineLearning/mlWorkManager"; /** * Face indexing orchestrator. * - * This is class that drives the face indexing process across all files that - * need to still be indexed. It runs in a Web Worker so as to not get in the way - * of the main thread. + * This module exposes a singleton instance of this class which drives the face + * indexing process on the user's library. * - * It operates in two modes - live indexing and backfill. + * The indexer operates in two modes - live indexing and backfill. * - * In live indexing, any files that are being uploaded from the current client + * When live indexing, any files that are being uploaded from the current client * are provided to the indexer, which puts them in a queue and indexes them one * by one. This is more efficient since we already have the file's content at * hand and do not have to download and decrypt it. * - * In backfill, the indexer figures out if any of the user's files (irrespective - * of where they were uploaded from) still need to be indexed, and if so, - * downloads, decrypts and indexes them. + * When backfilling, the indexer figures out if any of the user's files + * (irrespective of where they were uploaded from) still need to be indexed, and + * if so, downloads, decrypts and indexes them. * - * Live indexing has higher priority, backfill runs otherwise. - * - * If nothing needs to be indexed, the indexer goes to sleep for a while. + * Live indexing has higher priority, backfilling runs otherwise. If nothing + * remains to be indexed, the indexer goes to sleep for a while. */ -export class FaceIndexerWorker { +class FaceIndexer { /** Live indexing queue. */ - private liveItems: { file: File; enteFile: EnteFile }[]; + private liveItems: { enteFile: EnteFile; file: File | undefined }[]; /** Timeout for when the next time we will wake up. */ private wakeTimeout: ReturnType | undefined; /** - * Add {@link file} associated with {@link enteFile} to the live indexing - * queue. + * Add a file to the live indexing queue. + * + * @param enteFile An {@link EnteFile} that should be indexed. + * + * @param file The contents of {@link enteFile} as a web {@link File} + * object, if available. */ - enqueueFile(file: File, enteFile: EnteFile) { - this.liveItems.push({ file, enteFile }); + enqueueFile(enteFile: EnteFile, file: File | undefined) { + // If face indexing is not enabled, don't enqueue anything. Later on if + // the user turns on face indexing these files will get indexed as part + // of the backfilling anyway, the live indexing is just an optimization. + if (!mlWorkManager.isMlSearchEnabled) return; + + this.liveItems.push({ enteFile, file }); this.wakeUpIfNeeded(); } @@ -57,6 +62,20 @@ export class FaceIndexerWorker { this.tick(); } + /** + * A promise for the lazily created singleton {@link FaceIndexerWorker} remote + * exposed by this module. + */ + _faceIndexer: Promise>; + /** + * Main thread interface to the face indexer. + * + * This function provides a promise that resolves to a lazily created singleton + * remote with a {@link FaceIndexerWorker} at the other end. + */ + faceIndexer = (): Promise> => + (this._faceIndexer ??= createFaceIndexerComlinkWorker().remote); + private async tick() { console.log("tick"); @@ -84,39 +103,29 @@ export class FaceIndexerWorker { // Run again. this.tick(); } + + /** + * Add a newly uploaded file to the face indexing queue. + * + * @param enteFile The {@link EnteFile} that was uploaded. + * @param file + */ + /* + indexFacesInFile = (enteFile: EnteFile, file: File) => { + if (!mlWorkManager.isMlSearchEnabled) return; + + faceIndexer().then((indexer) => { + indexer.enqueueFile(file, enteFile); + }); + }; + */ } -/** - * A promise for the lazily created singleton {@link FaceIndexerWorker} remote - * exposed by this module. - */ -let _faceIndexer: Promise>; +/** The singleton instance of {@link FaceIndexer}. */ +export default new FaceIndexer(); const createFaceIndexerComlinkWorker = () => new ComlinkWorker( "face-indexer", new Worker(new URL("indexer.worker.ts", import.meta.url)), ); - -/** - * Main thread interface to the face indexer. - * - * This function provides a promise that resolves to a lazily created singleton - * remote with a {@link FaceIndexerWorker} at the other end. - */ -const faceIndexer = (): Promise> => - (_faceIndexer ??= createFaceIndexerComlinkWorker().remote); - -/** - * Add a newly uploaded file to the face indexing queue. - * - * @param enteFile The {@link EnteFile} that was uploaded. - * @param file - */ -export const indexFacesInFile = (enteFile: EnteFile, file: File) => { - if (!mlWorkManager.isMlSearchEnabled) return; - - faceIndexer().then((indexer) => { - indexer.enqueueFile(file, enteFile); - }); -}; From 7739be4e21365147affce9e88ec40ca8dc17898d Mon Sep 17 00:00:00 2001 From: ashilkn Date: Wed, 29 May 2024 20:42:05 +0530 Subject: [PATCH 148/354] [mob][photos] Migrating to flutter_map v6 (4): Fix attribution --- mobile/lib/ui/map/map_view.dart | 8 + .../map/tile/attribution/map_attribution.dart | 549 +++++++++--------- mobile/lib/ui/map/tile/layers.dart | 58 +- 3 files changed, 316 insertions(+), 299 deletions(-) diff --git a/mobile/lib/ui/map/map_view.dart b/mobile/lib/ui/map/map_view.dart index 5286924c95..3ff86f2ae1 100644 --- a/mobile/lib/ui/map/map_view.dart +++ b/mobile/lib/ui/map/map_view.dart @@ -135,6 +135,14 @@ class _MapViewState extends State { }, ), ), + Padding( + padding: EdgeInsets.only( + bottom: widget.bottomSheetDraggableAreaHeight, + ), + child: OSMFranceTileAttributes( + options: widget.mapAttributionOptions, + ), + ), ], ), widget.showControls diff --git a/mobile/lib/ui/map/tile/attribution/map_attribution.dart b/mobile/lib/ui/map/tile/attribution/map_attribution.dart index 370883bf08..524d8b1e5d 100644 --- a/mobile/lib/ui/map/tile/attribution/map_attribution.dart +++ b/mobile/lib/ui/map/tile/attribution/map_attribution.dart @@ -1,296 +1,301 @@ -// // ignore_for_file: invalid_use_of_internal_member +// ignore_for_file: invalid_use_of_internal_member -// import "dart:async"; +import "dart:async"; -// import "package:flutter/material.dart"; -// import "package:flutter_map/plugin_api.dart"; -// import "package:photos/extensions/list.dart"; -// import "package:photos/theme/colors.dart"; -// import "package:photos/theme/ente_theme.dart"; -// import "package:photos/ui/components/buttons/icon_button_widget.dart"; +import "package:flutter/material.dart"; +import "package:flutter_map/flutter_map.dart"; +import "package:photos/extensions/list.dart"; +import "package:photos/theme/colors.dart"; +import "package:photos/theme/ente_theme.dart"; +import "package:photos/ui/components/buttons/icon_button_widget.dart"; -// // Credit: This code is based on the Rich Attribution widget from the flutter_map -// class MapAttributionWidget extends StatefulWidget { -// /// List of attributions to display -// /// -// /// [TextSourceAttribution]s are shown in a popup box (toggled by a tap/click -// /// on the [openButton]/[closeButton]), unlike [LogoSourceAttribution], which -// /// are visible permanently adjacent to the open/close button. -// final List attributions; +// Credit: This code is based on the Rich Attribution widget from the flutter_map +class MapAttributionWidget extends StatefulWidget { + /// List of attributions to display + /// + /// [TextSourceAttribution]s are shown in a popup box (toggled by a tap/click + /// on the [openButton]/[closeButton]), unlike [LogoSourceAttribution], which + /// are visible permanently adjacent to the open/close button. + final List attributions; -// /// The position in which to anchor this widget -// final AttributionAlignment alignment; + /// The position in which to anchor this widget + final AttributionAlignment alignment; -// /// The widget (usually an [IconButton]) to display when the popup box is -// /// closed, that opens the popup box via the `open` callback -// final Widget Function(BuildContext context, VoidCallback open)? openButton; + /// The widget (usually an [IconButton]) to display when the popup box is + /// closed, that opens the popup box via the `open` callback + final Widget Function(BuildContext context, VoidCallback open)? openButton; -// /// The widget (usually an [IconButton]) to display when the popup box is open, -// /// that closes the popup box via the `close` callback -// final Widget Function(BuildContext context, VoidCallback close)? closeButton; + /// The widget (usually an [IconButton]) to display when the popup box is open, + /// that closes the popup box via the `close` callback + final Widget Function(BuildContext context, VoidCallback close)? closeButton; -// /// The color to use as the popup box's background color, defaulting to the -// /// [Theme]s background color -// final Color? popupBackgroundColor; + /// The color to use as the popup box's background color, defaulting to the + /// [Theme]s background color + final Color? popupBackgroundColor; -// /// The radius of the edges of the popup box -// final BorderRadius? popupBorderRadius; + /// The radius of the edges of the popup box + final BorderRadius? popupBorderRadius; -// /// The height of the permanent row in which is found the popup menu toggle -// /// button -// /// -// /// Also determines spacing between the items within the row. -// /// -// /// Also set [LogoSourceAttribution.height] to the same value, if adjusted. -// final double permanentHeight; + /// The height of the permanent row in which is found the popup menu toggle + /// button + /// + /// Also determines spacing between the items within the row. + /// + /// Also set [LogoSourceAttribution.height] to the same value, if adjusted. + final double permanentHeight; -// /// Whether to add an additional attribution logo and text for 'flutter_map' -// final bool showFlutterMapAttribution; + /// Whether to add an additional attribution logo and text for 'flutter_map' + final bool showFlutterMapAttribution; -// /// Animation configuration, through the properties and handler/builder -// /// defined by a [RichAttributionWidgetAnimation] implementation -// /// -// /// Can be extensivley customized by implementing a custom -// /// [RichAttributionWidgetAnimation], or the prebuilt [FadeRAWA] and -// /// [ScaleRAWA] animations can be used with limited customization. -// final RichAttributionWidgetAnimation animationConfig; + /// Animation configuration, through the properties and handler/builder + /// defined by a [RichAttributionWidgetAnimation] implementation + /// + /// Can be extensivley customized by implementing a custom + /// [RichAttributionWidgetAnimation], or the prebuilt [FadeRAWA] and + /// [ScaleRAWA] animations can be used with limited customization. + final RichAttributionWidgetAnimation animationConfig; -// /// If not [Duration.zero] (default), the popup box will be open by default and -// /// hidden this long after the map is initialised -// /// -// /// This is useful with certain sources/tile servers that make immediate -// /// attribution mandatory and are not attributed with a permanently visible -// /// [LogoSourceAttribution]. -// final Duration popupInitialDisplayDuration; + /// If not [Duration.zero] (default), the popup box will be open by default and + /// hidden this long after the map is initialised + /// + /// This is useful with certain sources/tile servers that make immediate + /// attribution mandatory and are not attributed with a permanently visible + /// [LogoSourceAttribution]. + final Duration popupInitialDisplayDuration; -// /// A prebuilt dynamic attribution layer that supports both logos and text -// /// through [SourceAttribution]s -// /// -// /// [TextSourceAttribution]s are shown in a popup box that can be visible or -// /// invisible. Its state is toggled by a tri-state [openButton]/[closeButton] : -// /// 1. Not hovered, not opened: faded button, invisible box -// /// 2. Hovered, not opened: full opacity button, invisible box -// /// 3. Opened: full opacity button, visible box -// /// -// /// The hover state on mobile devices is unspecified, but the behaviour is -// /// usually inconsequential on mobile devices anyway, due to the fingertip -// /// covering the entire button. -// /// -// /// [LogoSourceAttribution]s are shown adjacent to the open/close button, to -// /// comply with some stricter tile server requirements (such as Mapbox). These -// /// are usually supplemented with a [TextSourceAttribution]. -// /// -// /// The popup box also closes automatically on any interaction with the map. -// /// -// /// Animations are built in by default, and configured/handled through -// /// [RichAttributionWidgetAnimation] - see that class and the [animationConfig] -// /// property for more information. By default, a simple fade/opacity animation -// /// is provided by [FadeRAWA]. [ScaleRAWA] is also available. -// /// -// /// Read the documentation on the individual properties for more information -// /// and customizability. + /// A prebuilt dynamic attribution layer that supports both logos and text + /// through [SourceAttribution]s + /// + /// [TextSourceAttribution]s are shown in a popup box that can be visible or + /// invisible. Its state is toggled by a tri-state [openButton]/[closeButton] : + /// 1. Not hovered, not opened: faded button, invisible box + /// 2. Hovered, not opened: full opacity button, invisible box + /// 3. Opened: full opacity button, visible box + /// + /// The hover state on mobile devices is unspecified, but the behaviour is + /// usually inconsequential on mobile devices anyway, due to the fingertip + /// covering the entire button. + /// + /// [LogoSourceAttribution]s are shown adjacent to the open/close button, to + /// comply with some stricter tile server requirements (such as Mapbox). These + /// are usually supplemented with a [TextSourceAttribution]. + /// + /// The popup box also closes automatically on any interaction with the map. + /// + /// Animations are built in by default, and configured/handled through + /// [RichAttributionWidgetAnimation] - see that class and the [animationConfig] + /// property for more information. By default, a simple fade/opacity animation + /// is provided by [FadeRAWA]. [ScaleRAWA] is also available. + /// + /// Read the documentation on the individual properties for more information + /// and customizability. -// final double iconSize; -// const MapAttributionWidget({ -// super.key, -// required this.attributions, -// this.alignment = AttributionAlignment.bottomRight, -// this.openButton, -// this.closeButton, -// this.popupBackgroundColor, -// this.popupBorderRadius, -// this.permanentHeight = 24, -// this.showFlutterMapAttribution = true, -// this.animationConfig = const FadeRAWA(), -// this.popupInitialDisplayDuration = Duration.zero, -// this.iconSize = 20, -// }); + final double iconSize; + const MapAttributionWidget({ + super.key, + required this.attributions, + this.alignment = AttributionAlignment.bottomRight, + this.openButton, + this.closeButton, + this.popupBackgroundColor, + this.popupBorderRadius, + this.permanentHeight = 24, + this.showFlutterMapAttribution = true, + this.animationConfig = const FadeRAWA(), + this.popupInitialDisplayDuration = Duration.zero, + this.iconSize = 20, + }); -// @override -// State createState() => MapAttributionWidgetState(); -// } + @override + State createState() => MapAttributionWidgetState(); +} -// class MapAttributionWidgetState extends State { -// StreamSubscription? mapEventSubscription; +class MapAttributionWidgetState extends State { + StreamSubscription? mapEventSubscription; -// final persistentAttributionKey = GlobalKey(); -// Size? persistentAttributionSize; + final persistentAttributionKey = GlobalKey(); + Size? persistentAttributionSize; -// late bool popupExpanded = widget.popupInitialDisplayDuration != Duration.zero; -// bool persistentHovered = false; + late bool popupExpanded = widget.popupInitialDisplayDuration != Duration.zero; + bool persistentHovered = false; -// @override -// void initState() { -// super.initState(); + @override + void initState() { + super.initState(); -// if (popupExpanded) { -// Future.delayed( -// widget.popupInitialDisplayDuration, -// () => setState(() => popupExpanded = false), -// ); -// } + if (popupExpanded) { + Future.delayed( + widget.popupInitialDisplayDuration, + () => setState(() => popupExpanded = false), + ); + } -// WidgetsBinding.instance.addPostFrameCallback( -// (_) => WidgetsBinding.instance.addPostFrameCallback((_) { -// if (mounted) { -// setState( -// () => persistentAttributionSize = -// (persistentAttributionKey.currentContext!.findRenderObject() -// as RenderBox) -// .size, -// ); -// } -// }), -// ); -// } + WidgetsBinding.instance.addPostFrameCallback( + (_) => WidgetsBinding.instance.addPostFrameCallback((_) { + if (mounted) { + setState( + () => persistentAttributionSize = + (persistentAttributionKey.currentContext!.findRenderObject() + as RenderBox) + .size, + ); + } + }), + ); + } -// @override -// void dispose() { -// mapEventSubscription?.cancel(); -// super.dispose(); -// } + @override + void dispose() { + mapEventSubscription?.cancel(); + super.dispose(); + } -// @override -// Widget build(BuildContext context) { -// final persistentAttributionItems = [ -// ...List.from( -// widget.attributions.whereType(), -// growable: false, -// ).interleave(SizedBox(width: widget.permanentHeight / 1.5)), -// if (widget.showFlutterMapAttribution) -// LogoSourceAttribution( -// Image.asset( -// 'lib/assets/flutter_map_logo.png', -// package: 'flutter_map', -// ), -// tooltip: 'flutter_map', -// height: widget.permanentHeight, -// ), -// SizedBox(width: widget.permanentHeight * 0.1), -// AnimatedSwitcher( -// switchInCurve: widget.animationConfig.buttonCurve, -// switchOutCurve: widget.animationConfig.buttonCurve, -// duration: widget.animationConfig.buttonDuration, -// child: popupExpanded -// ? (widget.closeButton ?? -// (context, close) => IconButtonWidget( -// size: widget.iconSize, -// onTap: close, -// icon: Icons.cancel_outlined, -// iconButtonType: IconButtonType.primary, -// iconColor: getEnteColorScheme(context).strokeBase, -// ))( -// context, -// () => setState(() => popupExpanded = false), -// ) -// : (widget.openButton ?? -// (context, open) => IconButtonWidget( -// size: widget.iconSize, -// onTap: open, -// icon: Icons.info_outlined, -// iconButtonType: IconButtonType.primary, -// iconColor: strokeBaseLight, -// ))( -// context, -// () { -// setState(() => popupExpanded = true); -// mapEventSubscription = FlutterMapState.of(context) -// .mapController -// .mapEventStream -// .listen((e) { -// setState(() => popupExpanded = false); -// mapEventSubscription?.cancel(); -// }); -// }, -// ), -// ), -// ]; + @override + Widget build(BuildContext context) { + final persistentAttributionItems = [ + ...List.from( + widget.attributions.whereType(), + growable: false, + ).interleave(SizedBox(width: widget.permanentHeight / 1.5)), + if (widget.showFlutterMapAttribution) + LogoSourceAttribution( + Image.asset( + 'lib/assets/flutter_map_logo.png', + package: 'flutter_map', + ), + tooltip: 'flutter_map', + height: widget.permanentHeight, + ), + SizedBox(width: widget.permanentHeight * 0.1), + AnimatedSwitcher( + switchInCurve: widget.animationConfig.buttonCurve, + switchOutCurve: widget.animationConfig.buttonCurve, + duration: widget.animationConfig.buttonDuration, + child: popupExpanded + ? (widget.closeButton ?? + (context, close) => IconButtonWidget( + size: widget.iconSize, + onTap: close, + icon: Icons.cancel_outlined, + iconButtonType: IconButtonType.primary, + iconColor: getEnteColorScheme(context).strokeBase, + ))( + context, + () => setState(() => popupExpanded = false), + ) + : (widget.openButton ?? + (context, open) => IconButtonWidget( + size: widget.iconSize, + onTap: open, + icon: Icons.info_outlined, + iconButtonType: IconButtonType.primary, + iconColor: strokeBaseLight, + ))( + context, + () { + setState(() => popupExpanded = true); + // mapEventSubscription = FlutterMapState.of(context) + // .mapController + // .mapEventStream + // .listen((e) { + // setState(() => popupExpanded = false); + // mapEventSubscription?.cancel(); + // }); + mapEventSubscription = + MapController().mapEventStream.listen((e) { + setState(() => popupExpanded = false); + mapEventSubscription?.cancel(); + }); + }, + ), + ), + ]; -// return LayoutBuilder( -// builder: (context, constraints) => Align( -// alignment: widget.alignment.real, -// child: Stack( -// alignment: widget.alignment.real, -// children: [ -// if (persistentAttributionSize != null) -// Padding( -// padding: const EdgeInsets.all(6), -// child: AnimatedScale( -// scale: popupExpanded ? 1 : 0, -// duration: const Duration(milliseconds: 200), -// curve: popupExpanded ? Curves.easeOut : Curves.easeIn, -// alignment: widget.alignment.real, -// child: Container( -// decoration: BoxDecoration( -// color: widget.popupBackgroundColor ?? -// Theme.of(context).colorScheme.background, -// border: Border.all(width: 0, style: BorderStyle.none), -// borderRadius: widget.popupBorderRadius ?? -// BorderRadius.only( -// topLeft: const Radius.circular(10), -// topRight: const Radius.circular(10), -// bottomLeft: widget.alignment == -// AttributionAlignment.bottomLeft -// ? Radius.zero -// : const Radius.circular(10), -// bottomRight: widget.alignment == -// AttributionAlignment.bottomRight -// ? Radius.zero -// : const Radius.circular(10), -// ), -// ), -// constraints: BoxConstraints( -// minWidth: constraints.maxWidth < 420 -// ? constraints.maxWidth -// : persistentAttributionSize!.width, -// ), -// child: Padding( -// padding: const EdgeInsets.all(8), -// child: SizedBox( -// height: widget.attributions.length * 32, -// child: Column( -// mainAxisSize: MainAxisSize.max, -// mainAxisAlignment: MainAxisAlignment.spaceBetween, -// crossAxisAlignment: CrossAxisAlignment.start, -// children: [ -// ...widget.attributions -// .whereType(), -// SizedBox( -// height: (widget.permanentHeight - 24) + 32, -// ), -// ], -// ), -// ), -// ), -// ), -// ), -// ), -// MouseRegion( -// key: persistentAttributionKey, -// onEnter: (_) => setState(() => persistentHovered = true), -// onExit: (_) => setState(() => persistentHovered = false), -// cursor: SystemMouseCursors.click, -// child: AnimatedOpacity( -// opacity: persistentHovered || popupExpanded ? 1 : 0.5, -// curve: widget.animationConfig.buttonCurve, -// duration: widget.animationConfig.buttonDuration, -// child: Padding( -// padding: const EdgeInsets.all(4), -// child: FittedBox( -// child: Row( -// mainAxisSize: MainAxisSize.min, -// children: -// widget.alignment == AttributionAlignment.bottomLeft -// ? persistentAttributionItems.reversed.toList() -// : persistentAttributionItems, -// ), -// ), -// ), -// ), -// ), -// ], -// ), -// ), -// ); -// } -// } + return LayoutBuilder( + builder: (context, constraints) => Align( + alignment: widget.alignment.real, + child: Stack( + alignment: widget.alignment.real, + children: [ + if (persistentAttributionSize != null) + Padding( + padding: const EdgeInsets.all(6), + child: AnimatedScale( + scale: popupExpanded ? 1 : 0, + duration: const Duration(milliseconds: 200), + curve: popupExpanded ? Curves.easeOut : Curves.easeIn, + alignment: widget.alignment.real, + child: Container( + decoration: BoxDecoration( + color: widget.popupBackgroundColor ?? + Theme.of(context).colorScheme.background, + border: Border.all(width: 0, style: BorderStyle.none), + borderRadius: widget.popupBorderRadius ?? + BorderRadius.only( + topLeft: const Radius.circular(10), + topRight: const Radius.circular(10), + bottomLeft: widget.alignment == + AttributionAlignment.bottomLeft + ? Radius.zero + : const Radius.circular(10), + bottomRight: widget.alignment == + AttributionAlignment.bottomRight + ? Radius.zero + : const Radius.circular(10), + ), + ), + constraints: BoxConstraints( + minWidth: constraints.maxWidth < 420 + ? constraints.maxWidth + : persistentAttributionSize!.width, + ), + child: Padding( + padding: const EdgeInsets.all(8), + child: SizedBox( + height: widget.attributions.length * 32, + child: Column( + mainAxisSize: MainAxisSize.max, + mainAxisAlignment: MainAxisAlignment.spaceBetween, + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + ...widget.attributions + .whereType(), + SizedBox( + height: (widget.permanentHeight - 24) + 32, + ), + ], + ), + ), + ), + ), + ), + ), + MouseRegion( + key: persistentAttributionKey, + onEnter: (_) => setState(() => persistentHovered = true), + onExit: (_) => setState(() => persistentHovered = false), + cursor: SystemMouseCursors.click, + child: AnimatedOpacity( + opacity: persistentHovered || popupExpanded ? 1 : 0.5, + curve: widget.animationConfig.buttonCurve, + duration: widget.animationConfig.buttonDuration, + child: Padding( + padding: const EdgeInsets.all(4), + child: FittedBox( + child: Row( + mainAxisSize: MainAxisSize.min, + children: + widget.alignment == AttributionAlignment.bottomLeft + ? persistentAttributionItems.reversed.toList() + : persistentAttributionItems, + ), + ), + ), + ), + ), + ], + ), + ), + ); + } +} diff --git a/mobile/lib/ui/map/tile/layers.dart b/mobile/lib/ui/map/tile/layers.dart index c57ba2c639..2597dddea2 100644 --- a/mobile/lib/ui/map/tile/layers.dart +++ b/mobile/lib/ui/map/tile/layers.dart @@ -1,6 +1,11 @@ import "package:flutter/material.dart"; import "package:flutter_map/flutter_map.dart"; +import "package:photos/generated/l10n.dart"; +import "package:photos/theme/ente_theme.dart"; +import "package:photos/ui/map/tile/attribution/map_attribution.dart"; import "package:photos/ui/map/tile/cache.dart"; +import "package:url_launcher/url_launcher.dart"; +import "package:url_launcher/url_launcher_string.dart"; const String _userAgent = "io.ente.photos"; @@ -57,33 +62,32 @@ class OSMFranceTileAttributes extends StatelessWidget { @override Widget build(BuildContext context) { - // final textTheme = getEnteTextTheme(context).tinyBold; - // return MapAttributionWidget( - // alignment: AttributionAlignment.bottomLeft, - // showFlutterMapAttribution: false, - // permanentHeight: options.permanentHeight, - // popupBackgroundColor: getEnteColorScheme(context).backgroundElevated, - // popupBorderRadius: options.popupBorderRadius, - // iconSize: options.iconSize, - // attributions: [ - // TextSourceAttribution( - // S.of(context).openstreetmapContributors, - // textStyle: textTheme, - // onTap: () => launchUrlString('https://openstreetmap.org/copyright'), - // ), - // TextSourceAttribution( - // 'HOT Tiles', - // textStyle: textTheme, - // onTap: () => launchUrl(Uri.parse('https://www.hotosm.org/')), - // ), - // TextSourceAttribution( - // S.of(context).hostedAtOsmFrance, - // textStyle: textTheme, - // onTap: () => launchUrl(Uri.parse('https://www.openstreetmap.fr/')), - // ), - // ], - // ); - return const SizedBox.shrink(); + final textTheme = getEnteTextTheme(context).tinyBold; + return MapAttributionWidget( + alignment: AttributionAlignment.bottomLeft, + showFlutterMapAttribution: false, + permanentHeight: options.permanentHeight, + popupBackgroundColor: getEnteColorScheme(context).backgroundElevated, + popupBorderRadius: options.popupBorderRadius, + iconSize: options.iconSize, + attributions: [ + TextSourceAttribution( + S.of(context).openstreetmapContributors, + textStyle: textTheme, + onTap: () => launchUrlString('https://openstreetmap.org/copyright'), + ), + TextSourceAttribution( + 'HOT Tiles', + textStyle: textTheme, + onTap: () => launchUrl(Uri.parse('https://www.hotosm.org/')), + ), + TextSourceAttribution( + S.of(context).hostedAtOsmFrance, + textStyle: textTheme, + onTap: () => launchUrl(Uri.parse('https://www.openstreetmap.fr/')), + ), + ], + ); } } From 61fb9cf544be5c25f15052d5ead4bd2217332d35 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 09:22:53 +0530 Subject: [PATCH 149/354] Flow via the new path --- web/apps/photos/src/services/face/indexer.ts | 2 +- web/apps/photos/src/services/face/indexer.worker.ts | 1 + .../src/services/machineLearning/machineLearningService.ts | 6 ++++-- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index 661aa65288..75330d86ed 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -2,11 +2,11 @@ import log from "@/next/log"; import { ComlinkWorker } from "@/next/worker/comlink-worker"; import { wait } from "@/utils/promise"; import { type Remote } from "comlink"; +import mlWorkManager from "services/machineLearning/mlWorkManager"; import type { EnteFile } from "types/file"; import { markIndexingFailed } from "./db"; import { indexFaces } from "./f-index"; import { FaceIndexerWorker } from "./indexer.worker"; -import mlWorkManager from "services/machineLearning/mlWorkManager"; /** * Face indexing orchestrator. diff --git a/web/apps/photos/src/services/face/indexer.worker.ts b/web/apps/photos/src/services/face/indexer.worker.ts index 84250dd703..d7b025fa45 100644 --- a/web/apps/photos/src/services/face/indexer.worker.ts +++ b/web/apps/photos/src/services/face/indexer.worker.ts @@ -31,6 +31,7 @@ export class FaceIndexerWorker { try { const faceIndex = await indexFaces(enteFile, file); log.info(`faces in file ${f}`, faceIndex); + return faceIndex; } catch (e) { log.error(`Failed to index faces in file ${f}`, e); markIndexingFailed(enteFile.id); diff --git a/web/apps/photos/src/services/machineLearning/machineLearningService.ts b/web/apps/photos/src/services/machineLearning/machineLearningService.ts index f871584743..c00089d59b 100644 --- a/web/apps/photos/src/services/machineLearning/machineLearningService.ts +++ b/web/apps/photos/src/services/machineLearning/machineLearningService.ts @@ -5,11 +5,11 @@ import mlIDbStorage, { ML_SEARCH_CONFIG_NAME, type MinimalPersistedFileData, } from "services/face/db-old"; +import { FaceIndexerWorker } from "services/face/indexer.worker"; import { putFaceEmbedding } from "services/face/remote"; import { getLocalFiles } from "services/fileService"; import { EnteFile } from "types/file"; import { isInternalUserForML } from "utils/user"; -import { indexFaces } from "../face/f-index"; export const defaultMLVersion = 1; @@ -346,7 +346,9 @@ class MachineLearningService { return oldMlFile; } - const newMlFile = await indexFaces(enteFile, localFile); + const worker = new FaceIndexerWorker(); + + const newMlFile = await worker.index(enteFile, localFile); await putFaceEmbedding(enteFile, newMlFile, userAgent); await mlIDbStorage.putFile(newMlFile); return newMlFile; From 8a1acc756efc9ed635a068d52e7dbc04b27cca5a Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 09:32:45 +0530 Subject: [PATCH 150/354] Move the put to worker --- .../photos/src/services/face/indexer.worker.ts | 18 ++++++++++++++---- .../machineLearning/machineLearningService.ts | 4 +--- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/web/apps/photos/src/services/face/indexer.worker.ts b/web/apps/photos/src/services/face/indexer.worker.ts index d7b025fa45..b4b9bc08c7 100644 --- a/web/apps/photos/src/services/face/indexer.worker.ts +++ b/web/apps/photos/src/services/face/indexer.worker.ts @@ -1,8 +1,10 @@ import log from "@/next/log"; +import { putFaceEmbedding } from "services/face/remote"; import type { EnteFile } from "types/file"; import { logIdentifier } from "utils/file"; import { closeFaceDBConnectionsIfNeeded, markIndexingFailed } from "./db"; import { indexFaces } from "./f-index"; +import type { MlFileData } from "./types-old"; /** * Index faces in a file, save the persist the results locally, and put them on @@ -26,16 +28,24 @@ export class FaceIndexerWorker { * face indexing. If this is not provided, then the file's contents will be * downloaded and decrypted from remote. */ - async index(enteFile: EnteFile, file: File | undefined) { + async index(enteFile: EnteFile, file: File | undefined, userAgent: string) { const f = logIdentifier(enteFile); + + let faceIndex: MlFileData; try { - const faceIndex = await indexFaces(enteFile, file); - log.info(`faces in file ${f}`, faceIndex); - return faceIndex; + faceIndex = await indexFaces(enteFile, file); + log.debug(() => ({ f, faceIndex })); } catch (e) { + // Mark indexing as having failed only if the indexing itself + // failed, not if there were subsequent failures (like when trying + // to put the result to remote or save it to the local face DB). log.error(`Failed to index faces in file ${f}`, e); markIndexingFailed(enteFile.id); + throw e; } + + await putFaceEmbedding(enteFile, faceIndex, userAgent); + return faceIndex; } /** diff --git a/web/apps/photos/src/services/machineLearning/machineLearningService.ts b/web/apps/photos/src/services/machineLearning/machineLearningService.ts index c00089d59b..ef12c0891b 100644 --- a/web/apps/photos/src/services/machineLearning/machineLearningService.ts +++ b/web/apps/photos/src/services/machineLearning/machineLearningService.ts @@ -6,7 +6,6 @@ import mlIDbStorage, { type MinimalPersistedFileData, } from "services/face/db-old"; import { FaceIndexerWorker } from "services/face/indexer.worker"; -import { putFaceEmbedding } from "services/face/remote"; import { getLocalFiles } from "services/fileService"; import { EnteFile } from "types/file"; import { isInternalUserForML } from "utils/user"; @@ -348,8 +347,7 @@ class MachineLearningService { const worker = new FaceIndexerWorker(); - const newMlFile = await worker.index(enteFile, localFile); - await putFaceEmbedding(enteFile, newMlFile, userAgent); + const newMlFile = await worker.index(enteFile, localFile, userAgent); await mlIDbStorage.putFile(newMlFile); return newMlFile; } From 54654159ffece0c0a48aebaa831e03fef3f4aaea Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 09:36:26 +0530 Subject: [PATCH 151/354] Remove unused --- web/apps/photos/src/services/face/types-old.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/web/apps/photos/src/services/face/types-old.ts b/web/apps/photos/src/services/face/types-old.ts index 66eec9cf55..c64fc88ee7 100644 --- a/web/apps/photos/src/services/face/types-old.ts +++ b/web/apps/photos/src/services/face/types-old.ts @@ -33,8 +33,6 @@ export interface Face { blurValue?: number; embedding?: Float32Array; - - personId?: number; } export interface MlFileData { From bae4c65ab3a2ae003c8ba9bc544a2b2d5406f66c Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 09:43:22 +0530 Subject: [PATCH 152/354] Pull out the alignment --- web/apps/photos/src/services/face/crop.ts | 11 ++++-- web/apps/photos/src/services/face/f-index.ts | 37 +++++++++++++------ .../photos/src/services/face/types-old.ts | 19 ---------- 3 files changed, 33 insertions(+), 34 deletions(-) diff --git a/web/apps/photos/src/services/face/crop.ts b/web/apps/photos/src/services/face/crop.ts index faf7f0ac9b..8a09c1b8e6 100644 --- a/web/apps/photos/src/services/face/crop.ts +++ b/web/apps/photos/src/services/face/crop.ts @@ -1,9 +1,14 @@ import { blobCache } from "@/next/blob-cache"; +import type { FaceAlignment } from "./f-index"; import type { Box } from "./types"; -import type { Face, FaceAlignment } from "./types-old"; +import type { Face } from "./types-old"; -export const saveFaceCrop = async (imageBitmap: ImageBitmap, face: Face) => { - const faceCrop = extractFaceCrop(imageBitmap, face.alignment); +export const saveFaceCrop = async ( + imageBitmap: ImageBitmap, + face: Face, + alignment: FaceAlignment, +) => { + const faceCrop = extractFaceCrop(imageBitmap, alignment); const blob = await imageBitmapToBlob(faceCrop); faceCrop.close(); diff --git a/web/apps/photos/src/services/face/f-index.ts b/web/apps/photos/src/services/face/f-index.ts index a1a614f138..714f89c09f 100644 --- a/web/apps/photos/src/services/face/f-index.ts +++ b/web/apps/photos/src/services/face/f-index.ts @@ -22,12 +22,7 @@ import { warpAffineFloat32List, } from "./image"; import type { Box, Dimensions } from "./types"; -import type { - Face, - FaceAlignment, - FaceDetection, - MlFileData, -} from "./types-old"; +import type { Face, FaceDetection, MlFileData } from "./types-old"; /** * Index faces in the given file. @@ -109,11 +104,12 @@ const indexFaces_ = async (enteFile: EnteFile, imageBitmap: ImageBitmap) => { const alignments: FaceAlignment[] = []; for (const face of mlFile.faces) { - const alignment = faceAlignment(face.detection); - face.alignment = alignment; + const alignment = computeFaceAlignment(face.detection); alignments.push(alignment); - await saveFaceCrop(imageBitmap, face); + // This step is not really part of the indexing pipeline, we just do + // it here since we have already computed the face alignment. + await saveFaceCrop(imageBitmap, face, alignment); } const alignedFacesData = convertToMobileFaceNetInput( @@ -393,13 +389,30 @@ const makeFaceID = ( return [`${fileID}`, xMin, yMin, xMax, yMax].join("_"); }; +export interface FaceAlignment { + /** + * An affine transformation matrix (rotation, translation, scaling) to align + * the face extracted from the image. + */ + affineMatrix: number[][]; + /** + * The bounding box of the transformed box. + * + * The affine transformation shifts the original detection box a new, + * transformed, box (possibily rotated). This property is the bounding box + * of that transformed box. It is in the coordinate system of the original, + * full, image on which the detection occurred. + */ + boundingBox: Box; +} + /** * Compute and return an {@link FaceAlignment} for the given face detection. * * @param faceDetection A geometry indicating a face detected in an image. */ -const faceAlignment = (faceDetection: FaceDetection): FaceAlignment => - faceAlignmentUsingSimilarityTransform( +const computeFaceAlignment = (faceDetection: FaceDetection): FaceAlignment => + computeFaceAlignmentUsingSimilarityTransform( faceDetection, normalizeLandmarks(idealMobileFaceNetLandmarks, mobileFaceNetFaceSize), ); @@ -422,7 +435,7 @@ const normalizeLandmarks = ( ): [number, number][] => landmarks.map(([x, y]) => [x / faceSize, y / faceSize]); -const faceAlignmentUsingSimilarityTransform = ( +const computeFaceAlignmentUsingSimilarityTransform = ( faceDetection: FaceDetection, alignedLandmarks: [number, number][], ): FaceAlignment => { diff --git a/web/apps/photos/src/services/face/types-old.ts b/web/apps/photos/src/services/face/types-old.ts index c64fc88ee7..ad6ef68743 100644 --- a/web/apps/photos/src/services/face/types-old.ts +++ b/web/apps/photos/src/services/face/types-old.ts @@ -7,29 +7,10 @@ export interface FaceDetection { probability?: number; } -export interface FaceAlignment { - /** - * An affine transformation matrix (rotation, translation, scaling) to align - * the face extracted from the image. - */ - affineMatrix: number[][]; - /** - * The bounding box of the transformed box. - * - * The affine transformation shifts the original detection box a new, - * transformed, box (possibily rotated). This property is the bounding box - * of that transformed box. It is in the coordinate system of the original, - * full, image on which the detection occurred. - */ - boundingBox: Box; -} - export interface Face { fileId: number; detection: FaceDetection; id: string; - - alignment?: FaceAlignment; blurValue?: number; embedding?: Float32Array; From ddddc09226b378bf00541c04e83564dc78415b06 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 10:02:57 +0530 Subject: [PATCH 153/354] New --- web/apps/photos/src/services/face/indexer.ts | 41 +++++++++++++++++++ web/apps/photos/src/services/searchService.ts | 5 ++- 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index 75330d86ed..d9ce6066a7 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -5,6 +5,7 @@ import { type Remote } from "comlink"; import mlWorkManager from "services/machineLearning/mlWorkManager"; import type { EnteFile } from "types/file"; import { markIndexingFailed } from "./db"; +import type { IndexStatus } from "./db-old"; import { indexFaces } from "./f-index"; import { FaceIndexerWorker } from "./indexer.worker"; @@ -129,3 +130,43 @@ const createFaceIndexerComlinkWorker = () => "face-indexer", new Worker(new URL("indexer.worker.ts", import.meta.url)), ); + +export interface FaceIndexingStatus { + /** + * Which phase we are in within the indexing pipeline when viewed across the + * user's entire library: + * + * - "scheduled": There are files we know of that have not been indexed. + * + * - "indexing": The face indexer is currently running. + * + * - "clustering": All files we know of have been indexed, and we are now + * clustering the faces that were found. + * + * - "done": Face indexing and clustering is complete for the user's + * library. + */ + phase: "scheduled" | "indexing" | "clustering" | "done"; + outOfSyncFilesExists: boolean; + nSyncedFiles: number; + nTotalFiles: number; + localFilesSynced: boolean; + peopleIndexSynced: boolean; +} + +export const convertToNewInterface = (indexStatus: IndexStatus) => { + let phase: string; + if (!indexStatus.localFilesSynced) { + phase = "scheduled"; + } else if (indexStatus.outOfSyncFilesExists) { + phase = "indexing"; + } else if (!indexStatus.peopleIndexSynced) { + phase = "clustering"; + } else { + phase = "done"; + } + return { + ...indexStatus, + phase, + }; +}; diff --git a/web/apps/photos/src/services/searchService.ts b/web/apps/photos/src/services/searchService.ts index b48778f690..472441ba66 100644 --- a/web/apps/photos/src/services/searchService.ts +++ b/web/apps/photos/src/services/searchService.ts @@ -22,6 +22,7 @@ import { getFormattedDate } from "utils/search"; import { clipService, computeClipMatchScore } from "./clip-service"; import { localCLIPEmbeddings } from "./embeddingService"; import { getLatestEntities } from "./entityService"; +import { convertToNewInterface } from "./face/indexer"; import locationSearchService, { City } from "./locationSearchService"; const DIGITS = new Set(["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"]); @@ -175,7 +176,9 @@ export async function getAllPeopleSuggestion(): Promise> { export async function getIndexStatusSuggestion(): Promise { try { - const indexStatus = await mlIDbStorage.getIndexStatus(defaultMLVersion); + const indexStatus0 = + await mlIDbStorage.getIndexStatus(defaultMLVersion); + const indexStatus = convertToNewInterface(indexStatus0); let label; if (!indexStatus.localFilesSynced) { From cbdd82f6c053f7d50cd1e5ccefac5104d93fd2ea Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 10:06:53 +0530 Subject: [PATCH 154/354] Use --- web/apps/photos/src/services/searchService.ts | 25 +++++++++++-------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/web/apps/photos/src/services/searchService.ts b/web/apps/photos/src/services/searchService.ts index 472441ba66..e189899476 100644 --- a/web/apps/photos/src/services/searchService.ts +++ b/web/apps/photos/src/services/searchService.ts @@ -181,16 +181,21 @@ export async function getIndexStatusSuggestion(): Promise { const indexStatus = convertToNewInterface(indexStatus0); let label; - if (!indexStatus.localFilesSynced) { - label = t("INDEXING_SCHEDULED"); - } else if (indexStatus.outOfSyncFilesExists) { - label = t("ANALYZING_PHOTOS", { - indexStatus, - }); - } else if (!indexStatus.peopleIndexSynced) { - label = t("INDEXING_PEOPLE", { indexStatus }); - } else { - label = t("INDEXING_DONE", { indexStatus }); + switch (indexStatus.phase) { + case "scheduled": + label = t("INDEXING_SCHEDULED"); + break; + case "indexing": + label = t("ANALYZING_PHOTOS", { + indexStatus, + }); + break; + case "clustering": + label = t("INDEXING_PEOPLE", { indexStatus }); + break; + case "done": + label = t("INDEXING_DONE", { indexStatus }); + break; } return { From aa353b57e806c26b3cbbe35cce74cb125bd66420 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 10:07:48 +0530 Subject: [PATCH 155/354] Prune new API --- web/apps/photos/src/services/face/indexer.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index d9ce6066a7..7fa6c2b874 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -147,11 +147,8 @@ export interface FaceIndexingStatus { * library. */ phase: "scheduled" | "indexing" | "clustering" | "done"; - outOfSyncFilesExists: boolean; nSyncedFiles: number; nTotalFiles: number; - localFilesSynced: boolean; - peopleIndexSynced: boolean; } export const convertToNewInterface = (indexStatus: IndexStatus) => { From 85785f75432b8f5e9b5ad851a8eb650aa0f8b6aa Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 10:20:57 +0530 Subject: [PATCH 156/354] Doc --- web/apps/photos/src/services/face/indexer.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index 7fa6c2b874..1b1a01d6ed 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -147,7 +147,9 @@ export interface FaceIndexingStatus { * library. */ phase: "scheduled" | "indexing" | "clustering" | "done"; + /** The number of files that have already been indexed. */ nSyncedFiles: number; + /** The total number of files that are eligible for indexing. */ nTotalFiles: number; } From f6bd99386e1dd9d5ab51d683f7f1136f9e5f3442 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 10:28:40 +0530 Subject: [PATCH 157/354] t --- .../components/Search/SearchBar/searchInput/MenuWithPeople.tsx | 3 +-- web/apps/photos/src/services/searchService.ts | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/web/apps/photos/src/components/Search/SearchBar/searchInput/MenuWithPeople.tsx b/web/apps/photos/src/components/Search/SearchBar/searchInput/MenuWithPeople.tsx index aaca1c3906..fd073af643 100644 --- a/web/apps/photos/src/components/Search/SearchBar/searchInput/MenuWithPeople.tsx +++ b/web/apps/photos/src/components/Search/SearchBar/searchInput/MenuWithPeople.tsx @@ -5,7 +5,6 @@ import { t } from "i18next"; import { AppContext } from "pages/_app"; import { useContext } from "react"; import { components } from "react-select"; -import { IndexStatus } from "services/face/db-old"; import { Suggestion, SuggestionType } from "types/search"; const { Menu } = components; @@ -35,7 +34,7 @@ const MenuWithPeople = (props) => { (o) => o.type === SuggestionType.INDEX_STATUS, )[0] as Suggestion; - const indexStatus = indexStatusSuggestion?.value as IndexStatus; + const indexStatus = indexStatusSuggestion?.value; return ( diff --git a/web/apps/photos/src/services/searchService.ts b/web/apps/photos/src/services/searchService.ts index e189899476..159bd7525c 100644 --- a/web/apps/photos/src/services/searchService.ts +++ b/web/apps/photos/src/services/searchService.ts @@ -180,7 +180,7 @@ export async function getIndexStatusSuggestion(): Promise { await mlIDbStorage.getIndexStatus(defaultMLVersion); const indexStatus = convertToNewInterface(indexStatus0); - let label; + let label: string; switch (indexStatus.phase) { case "scheduled": label = t("INDEXING_SCHEDULED"); From 3c0d82279c4e61589de70f9b74bc3791ade21efe Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 10:35:02 +0530 Subject: [PATCH 158/354] Wrap --- web/apps/photos/src/services/face/indexer.ts | 10 +++++++++- web/apps/photos/src/services/searchService.ts | 7 ++----- web/apps/photos/src/types/search/index.ts | 4 ++-- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index 1b1a01d6ed..c2188a3921 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -2,6 +2,8 @@ import log from "@/next/log"; import { ComlinkWorker } from "@/next/worker/comlink-worker"; import { wait } from "@/utils/promise"; import { type Remote } from "comlink"; +import mlIDbStorage from "services/face/db-old"; +import { defaultMLVersion } from "services/machineLearning/machineLearningService"; import mlWorkManager from "services/machineLearning/mlWorkManager"; import type { EnteFile } from "types/file"; import { markIndexingFailed } from "./db"; @@ -153,8 +155,14 @@ export interface FaceIndexingStatus { nTotalFiles: number; } +export const faceIndexingStatus = async (): Promise => { + const indexStatus0 = await mlIDbStorage.getIndexStatus(defaultMLVersion); + const indexStatus = convertToNewInterface(indexStatus0); + return indexStatus; +}; + export const convertToNewInterface = (indexStatus: IndexStatus) => { - let phase: string; + let phase: FaceIndexingStatus["phase"]; if (!indexStatus.localFilesSynced) { phase = "scheduled"; } else if (indexStatus.outOfSyncFilesExists) { diff --git a/web/apps/photos/src/services/searchService.ts b/web/apps/photos/src/services/searchService.ts index 159bd7525c..ded48069f3 100644 --- a/web/apps/photos/src/services/searchService.ts +++ b/web/apps/photos/src/services/searchService.ts @@ -4,7 +4,6 @@ import * as chrono from "chrono-node"; import { t } from "i18next"; import mlIDbStorage from "services/face/db-old"; import type { Person } from "services/face/people"; -import { defaultMLVersion } from "services/machineLearning/machineLearningService"; import { Collection } from "types/collection"; import { EntityType, LocationTag, LocationTagData } from "types/entity"; import { EnteFile } from "types/file"; @@ -22,7 +21,7 @@ import { getFormattedDate } from "utils/search"; import { clipService, computeClipMatchScore } from "./clip-service"; import { localCLIPEmbeddings } from "./embeddingService"; import { getLatestEntities } from "./entityService"; -import { convertToNewInterface } from "./face/indexer"; +import { faceIndexingStatus } from "./face/indexer"; import locationSearchService, { City } from "./locationSearchService"; const DIGITS = new Set(["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"]); @@ -176,9 +175,7 @@ export async function getAllPeopleSuggestion(): Promise> { export async function getIndexStatusSuggestion(): Promise { try { - const indexStatus0 = - await mlIDbStorage.getIndexStatus(defaultMLVersion); - const indexStatus = convertToNewInterface(indexStatus0); + const indexStatus = await faceIndexingStatus(); let label: string; switch (indexStatus.phase) { diff --git a/web/apps/photos/src/types/search/index.ts b/web/apps/photos/src/types/search/index.ts index 3f3de9b460..adeb03d3aa 100644 --- a/web/apps/photos/src/types/search/index.ts +++ b/web/apps/photos/src/types/search/index.ts @@ -1,5 +1,5 @@ import { FILE_TYPE } from "@/media/file-type"; -import { IndexStatus } from "services/face/db-old"; +import type { FaceIndexingStatus } from "services/face/indexer"; import type { Person } from "services/face/people"; import { City } from "services/locationSearchService"; import { LocationTagData } from "types/entity"; @@ -31,7 +31,7 @@ export interface Suggestion { | DateValue | number[] | Person - | IndexStatus + | FaceIndexingStatus | LocationTagData | City | FILE_TYPE From d9200f470363f8f1fd63d271d44bb4f7c0a863de Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 10:55:46 +0530 Subject: [PATCH 159/354] Outline --- web/apps/photos/src/services/face/indexer.ts | 27 ++++++++++++++++++- .../machineLearning/machineLearningService.ts | 4 +++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index c2188a3921..d14b8d484c 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -3,7 +3,9 @@ import { ComlinkWorker } from "@/next/worker/comlink-worker"; import { wait } from "@/utils/promise"; import { type Remote } from "comlink"; import mlIDbStorage from "services/face/db-old"; -import { defaultMLVersion } from "services/machineLearning/machineLearningService"; +import machineLearningService, { + defaultMLVersion, +} from "services/machineLearning/machineLearningService"; import mlWorkManager from "services/machineLearning/mlWorkManager"; import type { EnteFile } from "types/file"; import { markIndexingFailed } from "./db"; @@ -156,8 +158,31 @@ export interface FaceIndexingStatus { } export const faceIndexingStatus = async (): Promise => { + const isSyncing = machineLearningService.isSyncing; + const [indexableCount, indexedCount] = [0, 0]; + + let phase: FaceIndexingStatus["phase"]; + if (indexableCount > 0 && indexableCount > indexedCount) { + if (!isSyncing) { + phase = "scheduled"; + } else { + phase = "indexing"; + } + } else { + phase = "done"; + } + + const indexingStatus = { + phase, + nTotalFiles: indexableCount, + nSyncedFiles: indexedCount, + }; + const indexStatus0 = await mlIDbStorage.getIndexStatus(defaultMLVersion); const indexStatus = convertToNewInterface(indexStatus0); + + log.debug(() => ({ indexStatus, indexingStatus })); + return indexStatus; }; diff --git a/web/apps/photos/src/services/machineLearning/machineLearningService.ts b/web/apps/photos/src/services/machineLearning/machineLearningService.ts index ef12c0891b..cbde414341 100644 --- a/web/apps/photos/src/services/machineLearning/machineLearningService.ts +++ b/web/apps/photos/src/services/machineLearning/machineLearningService.ts @@ -78,6 +78,8 @@ class MachineLearningService { private localSyncContext: Promise; private syncContext: Promise; + public isSyncing = false; + public async sync( token: string, userID: number, @@ -192,6 +194,7 @@ class MachineLearningService { } private async syncFiles(syncContext: MLSyncContext) { + this.isSyncing = true; try { const functions = syncContext.outOfSyncFiles.map( (outOfSyncfile) => async () => { @@ -211,6 +214,7 @@ class MachineLearningService { syncContext.error = error; } await syncContext.syncQueue.onIdle(); + this.isSyncing = false; // TODO: In case syncJob has to use multiple ml workers // do in same transaction with each file update From 896de6279473f91057b0e649498776c6f02bd5a1 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 11:05:04 +0530 Subject: [PATCH 160/354] Get counts from db --- web/apps/photos/src/services/face/db.ts | 14 ++++++++++++++ web/apps/photos/src/services/face/indexer.ts | 4 ++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/web/apps/photos/src/services/face/db.ts b/web/apps/photos/src/services/face/db.ts index 1128395237..85bb322056 100644 --- a/web/apps/photos/src/services/face/db.ts +++ b/web/apps/photos/src/services/face/db.ts @@ -197,6 +197,20 @@ export const addFileEntry = async (fileID: number) => { return tx.done; }; +/** + * Return the count of files that can be, and that have been, indexed. + */ +export const indexableAndIndexedCounts = async () => { + const db = await faceDB(); + const tx = db.transaction(["file-status", "face-index"], "readonly"); + const indexableCount = await tx + .objectStore("file-status") + .index("isIndexable") + .count(IDBKeyRange.only(1)); + const indexedCount = await tx.objectStore("face-index").count(); + return { indexableCount, indexedCount }; +}; + /** * Return a list of fileIDs that need to be indexed. * diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index d14b8d484c..a0a400b1e2 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -8,7 +8,7 @@ import machineLearningService, { } from "services/machineLearning/machineLearningService"; import mlWorkManager from "services/machineLearning/mlWorkManager"; import type { EnteFile } from "types/file"; -import { markIndexingFailed } from "./db"; +import { indexableAndIndexedCounts, markIndexingFailed } from "./db"; import type { IndexStatus } from "./db-old"; import { indexFaces } from "./f-index"; import { FaceIndexerWorker } from "./indexer.worker"; @@ -159,7 +159,7 @@ export interface FaceIndexingStatus { export const faceIndexingStatus = async (): Promise => { const isSyncing = machineLearningService.isSyncing; - const [indexableCount, indexedCount] = [0, 0]; + const { indexableCount, indexedCount } = await indexableAndIndexedCounts(); let phase: FaceIndexingStatus["phase"]; if (indexableCount > 0 && indexableCount > indexedCount) { From ab61fee8de181d32de82b9159589889a207565d4 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 11:06:31 +0530 Subject: [PATCH 161/354] simpl --- web/apps/photos/src/services/face/indexer.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index a0a400b1e2..90252720fc 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -162,7 +162,7 @@ export const faceIndexingStatus = async (): Promise => { const { indexableCount, indexedCount } = await indexableAndIndexedCounts(); let phase: FaceIndexingStatus["phase"]; - if (indexableCount > 0 && indexableCount > indexedCount) { + if (indexedCount < indexableCount) { if (!isSyncing) { phase = "scheduled"; } else { From 35090a6cdd303ab487e08e96ab4c82f4afa2ae0b Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 11:21:03 +0530 Subject: [PATCH 162/354] No clustering yet --- .../components/PhotoViewer/FileInfo/index.tsx | 6 +- .../photos/src/components/ml/PeopleList.tsx | 67 +------------------ 2 files changed, 5 insertions(+), 68 deletions(-) diff --git a/web/apps/photos/src/components/PhotoViewer/FileInfo/index.tsx b/web/apps/photos/src/components/PhotoViewer/FileInfo/index.tsx index a6d37ccf49..324928e9e6 100644 --- a/web/apps/photos/src/components/PhotoViewer/FileInfo/index.tsx +++ b/web/apps/photos/src/components/PhotoViewer/FileInfo/index.tsx @@ -11,7 +11,7 @@ import { Box, DialogProps, Link, Stack, styled } from "@mui/material"; import { Chip } from "components/Chip"; import { EnteDrawer } from "components/EnteDrawer"; import Titlebar from "components/Titlebar"; -import { PhotoPeopleList, UnidentifiedFaces } from "components/ml/PeopleList"; +import { UnidentifiedFaces } from "components/ml/PeopleList"; import LinkButton from "components/pages/gallery/LinkButton"; import { t } from "i18next"; import { AppContext } from "pages/_app"; @@ -332,10 +332,10 @@ export function FileInfo({ {appContext.mlSearchEnabled && ( <> - + /> */} >([]); - - useEffect(() => { - let didCancel = false; - - async function updateFaceImages() { - log.info("calling getPeopleList"); - const startTime = Date.now(); - const people = await getPeopleList(props.file); - log.info(`getPeopleList ${Date.now() - startTime} ms`); - log.info(`getPeopleList done, didCancel: ${didCancel}`); - !didCancel && setPeople(people); - } - - updateFaceImages(); - - return () => { - didCancel = true; - }; - }, [props.file, props.updateMLDataIndex]); - - if (people.length === 0) return <>; - - return ( -
- {t("PEOPLE")} - -
- ); +export function PhotoPeopleList() { + return <>; } export function UnidentifiedFaces(props: { @@ -180,40 +151,6 @@ const FaceCropImageView: React.FC = ({ faceID }) => { ); }; -async function getPeopleList(file: EnteFile): Promise { - let startTime = Date.now(); - const mlFileData = await mlIDbStorage.getFile(file.id); - log.info( - "getPeopleList:mlFilesStore:getItem", - Date.now() - startTime, - "ms", - ); - if (!mlFileData?.faces || mlFileData.faces.length < 1) { - return []; - } - - const peopleIds = mlFileData.faces - .filter((f) => f.personId !== null && f.personId !== undefined) - .map((f) => f.personId); - if (!peopleIds || peopleIds.length < 1) { - return []; - } - // log.info("peopleIds: ", peopleIds); - startTime = Date.now(); - const peoplePromises = peopleIds.map( - (p) => mlIDbStorage.getPerson(p) as Promise, - ); - const peopleList = await Promise.all(peoplePromises); - log.info( - "getPeopleList:mlPeopleStore:getItems", - Date.now() - startTime, - "ms", - ); - // log.info("peopleList: ", peopleList); - - return peopleList; -} - async function getUnidentifiedFaces(file: EnteFile): Promise<{ id: string }[]> { const mlFileData = await mlIDbStorage.getFile(file.id); From c3f6ecbf6afabb42824ed79dcb35689a050eb30d Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 11:30:23 +0530 Subject: [PATCH 163/354] Prune --- .../components/PhotoViewer/FileInfo/index.tsx | 12 +----- .../photos/src/components/ml/PeopleList.tsx | 40 ++++++++----------- 2 files changed, 18 insertions(+), 34 deletions(-) diff --git a/web/apps/photos/src/components/PhotoViewer/FileInfo/index.tsx b/web/apps/photos/src/components/PhotoViewer/FileInfo/index.tsx index 324928e9e6..5a17f43d17 100644 --- a/web/apps/photos/src/components/PhotoViewer/FileInfo/index.tsx +++ b/web/apps/photos/src/components/PhotoViewer/FileInfo/index.tsx @@ -96,8 +96,6 @@ export function FileInfo({ const [parsedExifData, setParsedExifData] = useState>(); const [showExif, setShowExif] = useState(false); - // eslint-disable-next-line @typescript-eslint/no-unused-vars - const [updateMLDataIndex, setUpdateMLDataIndex] = useState(0); const openExif = () => setShowExif(true); const closeExif = () => setShowExif(false); @@ -332,14 +330,8 @@ export function FileInfo({ {appContext.mlSearchEnabled && ( <> - {/* */} - + {/* */} + )} diff --git a/web/apps/photos/src/components/ml/PeopleList.tsx b/web/apps/photos/src/components/ml/PeopleList.tsx index 26c5c80556..def73dffd5 100644 --- a/web/apps/photos/src/components/ml/PeopleList.tsx +++ b/web/apps/photos/src/components/ml/PeopleList.tsx @@ -66,35 +66,29 @@ export const PeopleList = React.memo((props: PeopleListProps) => { export interface PhotoPeopleListProps extends PeopleListPropsBase { file: EnteFile; - updateMLDataIndex: number; } export function PhotoPeopleList() { return <>; } -export function UnidentifiedFaces(props: { - file: EnteFile; - updateMLDataIndex: number; -}) { +export function UnidentifiedFaces({ file }: { file: EnteFile }) { const [faces, setFaces] = useState<{ id: string }[]>([]); useEffect(() => { let didCancel = false; - async function updateFaceImages() { - const faces = await getUnidentifiedFaces(props.file); + (async () => { + const faces = await unidentifiedFaceIDs(file); !didCancel && setFaces(faces); - } - - updateFaceImages(); + })(); return () => { didCancel = true; }; - }, [props.file, props.updateMLDataIndex]); + }, [file]); - if (!faces || faces.length === 0) return <>; + if (faces.length == 0) return <>; return ( <> @@ -102,12 +96,11 @@ export function UnidentifiedFaces(props: { {t("UNIDENTIFIED_FACES")} - {faces && - faces.map((face, index) => ( - - - - ))} + {faces.map((face) => ( + + + + ))} ); @@ -151,10 +144,9 @@ const FaceCropImageView: React.FC = ({ faceID }) => { ); }; -async function getUnidentifiedFaces(file: EnteFile): Promise<{ id: string }[]> { +const unidentifiedFaceIDs = async ( + file: EnteFile, +): Promise<{ id: string }[]> => { const mlFileData = await mlIDbStorage.getFile(file.id); - - return mlFileData?.faces?.filter( - (f) => f.personId === null || f.personId === undefined, - ); -} + return mlFileData?.faces; +}; From 3c92349054f990eb137b86ecf9582d7ac91540c2 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 11:33:09 +0530 Subject: [PATCH 164/354] Move --- web/apps/photos/src/components/ml/PeopleList.tsx | 9 +-------- web/apps/photos/src/services/face/indexer.ts | 13 ++++++++++++- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/web/apps/photos/src/components/ml/PeopleList.tsx b/web/apps/photos/src/components/ml/PeopleList.tsx index def73dffd5..d44d100612 100644 --- a/web/apps/photos/src/components/ml/PeopleList.tsx +++ b/web/apps/photos/src/components/ml/PeopleList.tsx @@ -3,7 +3,7 @@ import { Skeleton, styled } from "@mui/material"; import { Legend } from "components/PhotoViewer/styledComponents/Legend"; import { t } from "i18next"; import React, { useEffect, useState } from "react"; -import mlIDbStorage from "services/face/db-old"; +import { unidentifiedFaceIDs } from "services/face/indexer"; import type { Person } from "services/face/people"; import { EnteFile } from "types/file"; @@ -143,10 +143,3 @@ const FaceCropImageView: React.FC = ({ faceID }) => { ); }; - -const unidentifiedFaceIDs = async ( - file: EnteFile, -): Promise<{ id: string }[]> => { - const mlFileData = await mlIDbStorage.getFile(file.id); - return mlFileData?.faces; -}; diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index 90252720fc..5d5d0f56c5 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -186,7 +186,7 @@ export const faceIndexingStatus = async (): Promise => { return indexStatus; }; -export const convertToNewInterface = (indexStatus: IndexStatus) => { +const convertToNewInterface = (indexStatus: IndexStatus) => { let phase: FaceIndexingStatus["phase"]; if (!indexStatus.localFilesSynced) { phase = "scheduled"; @@ -202,3 +202,14 @@ export const convertToNewInterface = (indexStatus: IndexStatus) => { phase, }; }; + +/** + * Return the IDs of all the faces in the given {@link enteFile} that are not + * associated with a person cluster. + */ +export const unidentifiedFaceIDs = async ( + enteFile: EnteFile, +): Promise<{ id: string }[]> => { + const mlFileData = await mlIDbStorage.getFile(enteFile.id); + return mlFileData?.faces ?? []; +}; From 321422e9151d39ad32bc9ad547f71bb47bcaa380 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 11:34:50 +0530 Subject: [PATCH 165/354] No clustering yet --- web/apps/photos/src/services/searchService.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/web/apps/photos/src/services/searchService.ts b/web/apps/photos/src/services/searchService.ts index ded48069f3..be7f574a7b 100644 --- a/web/apps/photos/src/services/searchService.ts +++ b/web/apps/photos/src/services/searchService.ts @@ -2,7 +2,6 @@ import { FILE_TYPE } from "@/media/file-type"; import log from "@/next/log"; import * as chrono from "chrono-node"; import { t } from "i18next"; -import mlIDbStorage from "services/face/db-old"; import type { Person } from "services/face/people"; import { Collection } from "types/collection"; import { EntityType, LocationTag, LocationTagData } from "types/entity"; @@ -435,7 +434,7 @@ function convertSuggestionToSearchQuery(option: Suggestion): Search { } async function getAllPeople(limit: number = undefined) { - let people: Array = await mlIDbStorage.getAllPeople(); + let people: Array = []; // await mlIDbStorage.getAllPeople(); // await mlPeopleStore.iterate((person) => { // people.push(person); // }); From 403cc3cca0d3309560770578770871b6e72d03a9 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 11:46:36 +0530 Subject: [PATCH 166/354] New --- web/apps/photos/src/services/face/indexer.ts | 29 ++++++++++++++++++- .../machineLearning/machineLearningService.ts | 12 ++------ 2 files changed, 30 insertions(+), 11 deletions(-) diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index 5d5d0f56c5..7436986144 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -2,12 +2,14 @@ import log from "@/next/log"; import { ComlinkWorker } from "@/next/worker/comlink-worker"; import { wait } from "@/utils/promise"; import { type Remote } from "comlink"; -import mlIDbStorage from "services/face/db-old"; +import mlIDbStorage, { ML_SEARCH_CONFIG_NAME } from "services/face/db-old"; import machineLearningService, { + DEFAULT_ML_SEARCH_CONFIG, defaultMLVersion, } from "services/machineLearning/machineLearningService"; import mlWorkManager from "services/machineLearning/mlWorkManager"; import type { EnteFile } from "types/file"; +import { isInternalUserForML } from "utils/user"; import { indexableAndIndexedCounts, markIndexingFailed } from "./db"; import type { IndexStatus } from "./db-old"; import { indexFaces } from "./f-index"; @@ -213,3 +215,28 @@ export const unidentifiedFaceIDs = async ( const mlFileData = await mlIDbStorage.getFile(enteFile.id); return mlFileData?.faces ?? []; }; + +/** + * Return true if the user has enabled face indexing in the app's settings. + * + * This setting is persisted locally (in local storage) and is not synced with + * remote. There is a separate setting, "faceSearchEnabled" that is synced with + * remote, but that tracks whether or not the user has enabled face search once + * on any client. This {@link isFaceIndexingEnabled} property, on the other + * hand, denotes whether or not indexing is enabled on the current client. + */ +export const isFaceIndexingEnabled = () => { + if (isInternalUserForML()) { + return mlIDbStorage.getConfig( + ML_SEARCH_CONFIG_NAME, + DEFAULT_ML_SEARCH_CONFIG, + ); + } + // Force disabled for everyone else while we finalize it to avoid redundant + // reindexing for users. + return DEFAULT_ML_SEARCH_CONFIG; +}; + +export const setIsFaceIndexingEnabled = (enabled: boolean) => { + return mlIDbStorage.putConfig(ML_SEARCH_CONFIG_NAME, { enabled }); +}; diff --git a/web/apps/photos/src/services/machineLearning/machineLearningService.ts b/web/apps/photos/src/services/machineLearning/machineLearningService.ts index cbde414341..f952a3202e 100644 --- a/web/apps/photos/src/services/machineLearning/machineLearningService.ts +++ b/web/apps/photos/src/services/machineLearning/machineLearningService.ts @@ -5,10 +5,10 @@ import mlIDbStorage, { ML_SEARCH_CONFIG_NAME, type MinimalPersistedFileData, } from "services/face/db-old"; +import { isFaceIndexingEnabled } from "services/face/indexer"; import { FaceIndexerWorker } from "services/face/indexer.worker"; import { getLocalFiles } from "services/fileService"; import { EnteFile } from "types/file"; -import { isInternalUserForML } from "utils/user"; export const defaultMLVersion = 1; @@ -25,15 +25,7 @@ export const DEFAULT_ML_SEARCH_CONFIG: MLSearchConfig = { }; export async function getMLSearchConfig() { - if (isInternalUserForML()) { - return mlIDbStorage.getConfig( - ML_SEARCH_CONFIG_NAME, - DEFAULT_ML_SEARCH_CONFIG, - ); - } - // Force disabled for everyone else while we finalize it to avoid redundant - // reindexing for users. - return DEFAULT_ML_SEARCH_CONFIG; + return isFaceIndexingEnabled(); } export async function updateMLSearchConfig(newConfig: MLSearchConfig) { From 6be42225c2a80a70b2844be1ba3c2a9fcfaf9110 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 11:49:36 +0530 Subject: [PATCH 167/354] Bypass --- web/apps/photos/src/pages/_app.tsx | 15 ++++++--------- web/apps/photos/src/services/face/indexer.ts | 7 ++++--- .../machineLearning/machineLearningService.ts | 5 ----- 3 files changed, 10 insertions(+), 17 deletions(-) diff --git a/web/apps/photos/src/pages/_app.tsx b/web/apps/photos/src/pages/_app.tsx index a816d0b462..4157634aa5 100644 --- a/web/apps/photos/src/pages/_app.tsx +++ b/web/apps/photos/src/pages/_app.tsx @@ -50,11 +50,9 @@ import { createContext, useContext, useEffect, useRef, useState } from "react"; import LoadingBar from "react-top-loading-bar"; import DownloadManager from "services/download"; import { resumeExportsIfNeeded } from "services/export"; +import { isFaceIndexingEnabled } from "services/face/indexer"; import { photosLogout } from "services/logout"; -import { - getMLSearchConfig, - updateMLSearchConfig, -} from "services/machineLearning/machineLearningService"; +import { updateMLSearchConfig } from "services/machineLearning/machineLearningService"; import mlWorkManager from "services/machineLearning/mlWorkManager"; import { getFamilyPortalRedirectURL, @@ -186,9 +184,9 @@ export default function App({ Component, pageProps }: AppProps) { } const loadMlSearchState = async () => { try { - const mlSearchConfig = await getMLSearchConfig(); - setMlSearchEnabled(mlSearchConfig.enabled); - mlWorkManager.setMlSearchEnabled(mlSearchConfig.enabled); + const enabled = await isFaceIndexingEnabled(); + setMlSearchEnabled(enabled); + mlWorkManager.setMlSearchEnabled(enabled); } catch (e) { log.error("Error while loading mlSearchEnabled", e); } @@ -286,8 +284,7 @@ export default function App({ Component, pageProps }: AppProps) { const showNavBar = (show: boolean) => setShowNavBar(show); const updateMlSearchEnabled = async (enabled: boolean) => { try { - const mlSearchConfig = await getMLSearchConfig(); - mlSearchConfig.enabled = enabled; + const mlSearchConfig = { enabled }; await updateMLSearchConfig(mlSearchConfig); setMlSearchEnabled(enabled); mlWorkManager.setMlSearchEnabled(enabled); diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index 7436986144..5d5947983f 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -225,16 +225,17 @@ export const unidentifiedFaceIDs = async ( * on any client. This {@link isFaceIndexingEnabled} property, on the other * hand, denotes whether or not indexing is enabled on the current client. */ -export const isFaceIndexingEnabled = () => { +export const isFaceIndexingEnabled = async () => { if (isInternalUserForML()) { - return mlIDbStorage.getConfig( + const config = await mlIDbStorage.getConfig( ML_SEARCH_CONFIG_NAME, DEFAULT_ML_SEARCH_CONFIG, ); + return config.enabled; } // Force disabled for everyone else while we finalize it to avoid redundant // reindexing for users. - return DEFAULT_ML_SEARCH_CONFIG; + return false; }; export const setIsFaceIndexingEnabled = (enabled: boolean) => { diff --git a/web/apps/photos/src/services/machineLearning/machineLearningService.ts b/web/apps/photos/src/services/machineLearning/machineLearningService.ts index f952a3202e..8b07e72538 100644 --- a/web/apps/photos/src/services/machineLearning/machineLearningService.ts +++ b/web/apps/photos/src/services/machineLearning/machineLearningService.ts @@ -5,7 +5,6 @@ import mlIDbStorage, { ML_SEARCH_CONFIG_NAME, type MinimalPersistedFileData, } from "services/face/db-old"; -import { isFaceIndexingEnabled } from "services/face/indexer"; import { FaceIndexerWorker } from "services/face/indexer.worker"; import { getLocalFiles } from "services/fileService"; import { EnteFile } from "types/file"; @@ -24,10 +23,6 @@ export const DEFAULT_ML_SEARCH_CONFIG: MLSearchConfig = { enabled: false, }; -export async function getMLSearchConfig() { - return isFaceIndexingEnabled(); -} - export async function updateMLSearchConfig(newConfig: MLSearchConfig) { return mlIDbStorage.putConfig(ML_SEARCH_CONFIG_NAME, newConfig); } From dafdeca7e47d8203f31c0d8e7b023b67dd94ada1 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Thu, 30 May 2024 11:50:56 +0530 Subject: [PATCH 168/354] [mob] Log file details on decryption Error --- mobile/lib/utils/thumbnail_util.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobile/lib/utils/thumbnail_util.dart b/mobile/lib/utils/thumbnail_util.dart index db7648b92b..052f499ff2 100644 --- a/mobile/lib/utils/thumbnail_util.dart +++ b/mobile/lib/utils/thumbnail_util.dart @@ -184,7 +184,7 @@ Future _downloadAndDecryptThumbnail(FileDownloadItem item) async { CryptoUtil.base642bin(file.thumbnailDecryptionHeader!), ); } catch (e, s) { - _logger.severe("Failed to decrypt thumbnail", e, s); + _logger.severe("Failed to decrypt thumbnail ${item.file.toString()}", e, s); item.completer.completeError(e); return; } From f66170b5b22a8802146f6f2d7df9936686a328c4 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 11:53:25 +0530 Subject: [PATCH 169/354] Bypass --- web/apps/photos/src/pages/_app.tsx | 9 +++++---- web/apps/photos/src/services/face/indexer.ts | 5 ++++- .../services/machineLearning/machineLearningService.ts | 5 ----- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/web/apps/photos/src/pages/_app.tsx b/web/apps/photos/src/pages/_app.tsx index 4157634aa5..5731b292a9 100644 --- a/web/apps/photos/src/pages/_app.tsx +++ b/web/apps/photos/src/pages/_app.tsx @@ -50,9 +50,11 @@ import { createContext, useContext, useEffect, useRef, useState } from "react"; import LoadingBar from "react-top-loading-bar"; import DownloadManager from "services/download"; import { resumeExportsIfNeeded } from "services/export"; -import { isFaceIndexingEnabled } from "services/face/indexer"; +import { + isFaceIndexingEnabled, + setIsFaceIndexingEnabled, +} from "services/face/indexer"; import { photosLogout } from "services/logout"; -import { updateMLSearchConfig } from "services/machineLearning/machineLearningService"; import mlWorkManager from "services/machineLearning/mlWorkManager"; import { getFamilyPortalRedirectURL, @@ -284,8 +286,7 @@ export default function App({ Component, pageProps }: AppProps) { const showNavBar = (show: boolean) => setShowNavBar(show); const updateMlSearchEnabled = async (enabled: boolean) => { try { - const mlSearchConfig = { enabled }; - await updateMLSearchConfig(mlSearchConfig); + await setIsFaceIndexingEnabled(enabled); setMlSearchEnabled(enabled); mlWorkManager.setMlSearchEnabled(enabled); } catch (e) { diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index 5d5947983f..f931a058dc 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -238,6 +238,9 @@ export const isFaceIndexingEnabled = async () => { return false; }; -export const setIsFaceIndexingEnabled = (enabled: boolean) => { +/** + * Update the (locally stored) value of {@link isFaceIndexingEnabled}. + */ +export const setIsFaceIndexingEnabled = async (enabled: boolean) => { return mlIDbStorage.putConfig(ML_SEARCH_CONFIG_NAME, { enabled }); }; diff --git a/web/apps/photos/src/services/machineLearning/machineLearningService.ts b/web/apps/photos/src/services/machineLearning/machineLearningService.ts index 8b07e72538..4b9826da63 100644 --- a/web/apps/photos/src/services/machineLearning/machineLearningService.ts +++ b/web/apps/photos/src/services/machineLearning/machineLearningService.ts @@ -2,7 +2,6 @@ import log from "@/next/log"; import { CustomError, parseUploadErrorCodes } from "@ente/shared/error"; import PQueue from "p-queue"; import mlIDbStorage, { - ML_SEARCH_CONFIG_NAME, type MinimalPersistedFileData, } from "services/face/db-old"; import { FaceIndexerWorker } from "services/face/indexer.worker"; @@ -23,10 +22,6 @@ export const DEFAULT_ML_SEARCH_CONFIG: MLSearchConfig = { enabled: false, }; -export async function updateMLSearchConfig(newConfig: MLSearchConfig) { - return mlIDbStorage.putConfig(ML_SEARCH_CONFIG_NAME, newConfig); -} - class MLSyncContext { public token: string; public userID: number; From 3c3f9b2b48546ab2371dcc0730b1a5b9870b3b28 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 11:54:48 +0530 Subject: [PATCH 170/354] Inline --- web/apps/photos/src/services/face/db-old.ts | 9 +++++---- web/apps/photos/src/services/face/indexer.ts | 8 +++----- .../services/machineLearning/machineLearningService.ts | 8 -------- 3 files changed, 8 insertions(+), 17 deletions(-) diff --git a/web/apps/photos/src/services/face/db-old.ts b/web/apps/photos/src/services/face/db-old.ts index a70e94bee7..7dd1e71fa3 100644 --- a/web/apps/photos/src/services/face/db-old.ts +++ b/web/apps/photos/src/services/face/db-old.ts @@ -11,10 +11,7 @@ import { import isElectron from "is-electron"; import type { Person } from "services/face/people"; import type { MlFileData } from "services/face/types-old"; -import { - DEFAULT_ML_SEARCH_CONFIG, - MAX_ML_SYNC_ERROR_COUNT, -} from "services/machineLearning/machineLearningService"; +import { MAX_ML_SYNC_ERROR_COUNT } from "services/machineLearning/machineLearningService"; export interface IndexStatus { outOfSyncFilesExists: boolean; @@ -159,6 +156,10 @@ class MLIDbStorage { */ } if (oldVersion < 3) { + const DEFAULT_ML_SEARCH_CONFIG = { + enabled: false, + }; + await tx .objectStore("configs") .add(DEFAULT_ML_SEARCH_CONFIG, ML_SEARCH_CONFIG_NAME); diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index f931a058dc..5930e60c1e 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -4,7 +4,6 @@ import { wait } from "@/utils/promise"; import { type Remote } from "comlink"; import mlIDbStorage, { ML_SEARCH_CONFIG_NAME } from "services/face/db-old"; import machineLearningService, { - DEFAULT_ML_SEARCH_CONFIG, defaultMLVersion, } from "services/machineLearning/machineLearningService"; import mlWorkManager from "services/machineLearning/mlWorkManager"; @@ -227,10 +226,9 @@ export const unidentifiedFaceIDs = async ( */ export const isFaceIndexingEnabled = async () => { if (isInternalUserForML()) { - const config = await mlIDbStorage.getConfig( - ML_SEARCH_CONFIG_NAME, - DEFAULT_ML_SEARCH_CONFIG, - ); + const config = await mlIDbStorage.getConfig(ML_SEARCH_CONFIG_NAME, { + enabled: false, + }); return config.enabled; } // Force disabled for everyone else while we finalize it to avoid redundant diff --git a/web/apps/photos/src/services/machineLearning/machineLearningService.ts b/web/apps/photos/src/services/machineLearning/machineLearningService.ts index 4b9826da63..a452846623 100644 --- a/web/apps/photos/src/services/machineLearning/machineLearningService.ts +++ b/web/apps/photos/src/services/machineLearning/machineLearningService.ts @@ -14,14 +14,6 @@ const batchSize = 200; export const MAX_ML_SYNC_ERROR_COUNT = 1; -export interface MLSearchConfig { - enabled: boolean; -} - -export const DEFAULT_ML_SEARCH_CONFIG: MLSearchConfig = { - enabled: false, -}; - class MLSyncContext { public token: string; public userID: number; From c8a7152cdc6a450c7c18292da403bf6961fe41cb Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 12:01:04 +0530 Subject: [PATCH 171/354] Remove unnecessary propagation --- .../services/machineLearning/machineLearningService.ts | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/web/apps/photos/src/services/machineLearning/machineLearningService.ts b/web/apps/photos/src/services/machineLearning/machineLearningService.ts index a452846623..f28c3f76c4 100644 --- a/web/apps/photos/src/services/machineLearning/machineLearningService.ts +++ b/web/apps/photos/src/services/machineLearning/machineLearningService.ts @@ -277,13 +277,8 @@ class MachineLearningService { localFile?: globalThis.File, ) { try { - const mlFileData = await this.syncFile( - enteFile, - localFile, - syncContext.userAgent, - ); + await this.syncFile(enteFile, localFile, syncContext.userAgent); syncContext.nSyncedFiles += 1; - return mlFileData; } catch (e) { log.error("ML syncFile failed", e); let error = e; @@ -320,14 +315,13 @@ class MachineLearningService { ) { const oldMlFile = await mlIDbStorage.getFile(enteFile.id); if (oldMlFile && oldMlFile.mlVersion) { - return oldMlFile; + return; } const worker = new FaceIndexerWorker(); const newMlFile = await worker.index(enteFile, localFile, userAgent); await mlIDbStorage.putFile(newMlFile); - return newMlFile; } private async persistMLFileSyncError(enteFile: EnteFile, e: Error) { From d448676b8f430dbc0b5b5193651954f0a2031bb9 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 12:09:01 +0530 Subject: [PATCH 172/354] Move --- web/apps/photos/src/services/face/indexer.ts | 70 ++++++++++++++++++- .../machineLearning/machineLearningService.ts | 51 +------------- 2 files changed, 72 insertions(+), 49 deletions(-) diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index 5930e60c1e..829638c755 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -3,6 +3,7 @@ import { ComlinkWorker } from "@/next/worker/comlink-worker"; import { wait } from "@/utils/promise"; import { type Remote } from "comlink"; import mlIDbStorage, { ML_SEARCH_CONFIG_NAME } from "services/face/db-old"; +import { getLocalFiles } from "services/fileService"; import machineLearningService, { defaultMLVersion, } from "services/machineLearning/machineLearningService"; @@ -10,7 +11,7 @@ import mlWorkManager from "services/machineLearning/mlWorkManager"; import type { EnteFile } from "types/file"; import { isInternalUserForML } from "utils/user"; import { indexableAndIndexedCounts, markIndexingFailed } from "./db"; -import type { IndexStatus } from "./db-old"; +import type { IndexStatus, MinimalPersistedFileData } from "./db-old"; import { indexFaces } from "./f-index"; import { FaceIndexerWorker } from "./indexer.worker"; @@ -242,3 +243,70 @@ export const isFaceIndexingEnabled = async () => { export const setIsFaceIndexingEnabled = async (enabled: boolean) => { return mlIDbStorage.putConfig(ML_SEARCH_CONFIG_NAME, { enabled }); }; + +export const syncLocalFiles = async (userID: number) => { + const startTime = Date.now(); + const localFilesMap = await getLocalFilesMap(userID); + + const db = await mlIDbStorage.db; + const tx = db.transaction("files", "readwrite"); + const mlFileIdsArr = await mlIDbStorage.getAllFileIdsForUpdate(tx); + const mlFileIds = new Set(); + mlFileIdsArr.forEach((mlFileId) => mlFileIds.add(mlFileId)); + + const newFileIds: Array = []; + for (const localFileId of localFilesMap.keys()) { + if (!mlFileIds.has(localFileId)) { + newFileIds.push(localFileId); + } + } + + let updated = false; + if (newFileIds.length > 0) { + log.info("newFiles: ", newFileIds.length); + const newFiles = newFileIds.map( + (fileId) => + ({ + fileId, + mlVersion: 0, + errorCount: 0, + }) as MinimalPersistedFileData, + ); + await mlIDbStorage.putAllFiles(newFiles, tx); + updated = true; + } + + const removedFileIds: Array = []; + for (const mlFileId of mlFileIds) { + if (!localFilesMap.has(mlFileId)) { + removedFileIds.push(mlFileId); + } + } + + if (removedFileIds.length > 0) { + log.info("removedFiles: ", removedFileIds.length); + await mlIDbStorage.removeAllFiles(removedFileIds, tx); + updated = true; + } + + await tx.done; + + if (updated) { + // TODO: should do in same transaction + await mlIDbStorage.incrementIndexVersion("files"); + } + + log.info("syncLocalFiles", Date.now() - startTime, "ms"); + + return localFilesMap; +}; + +const getLocalFilesMap = async (userID: number) => { + const localFiles = await getLocalFiles(); + + const personalFiles = localFiles.filter((f) => f.ownerID === userID); + const localFilesMap = new Map(); + personalFiles.forEach((f) => localFilesMap.set(f.id, f)); + + return localFilesMap; +}; diff --git a/web/apps/photos/src/services/machineLearning/machineLearningService.ts b/web/apps/photos/src/services/machineLearning/machineLearningService.ts index f28c3f76c4..4aa2235db4 100644 --- a/web/apps/photos/src/services/machineLearning/machineLearningService.ts +++ b/web/apps/photos/src/services/machineLearning/machineLearningService.ts @@ -4,6 +4,7 @@ import PQueue from "p-queue"; import mlIDbStorage, { type MinimalPersistedFileData, } from "services/face/db-old"; +import { syncLocalFiles } from "services/face/indexer"; import { FaceIndexerWorker } from "services/face/indexer.worker"; import { getLocalFiles } from "services/fileService"; import { EnteFile } from "types/file"; @@ -65,7 +66,8 @@ class MachineLearningService { const syncContext = await this.getSyncContext(token, userID, userAgent); - await this.syncLocalFiles(syncContext); + const localFiles = await syncLocalFiles(userID); + syncContext.localFilesMap = localFiles; await this.getOutOfSyncFiles(syncContext); @@ -102,53 +104,6 @@ class MachineLearningService { return syncContext.localFilesMap; } - private async syncLocalFiles(syncContext: MLSyncContext) { - const startTime = Date.now(); - const localFilesMap = await this.getLocalFilesMap(syncContext); - - const db = await mlIDbStorage.db; - const tx = db.transaction("files", "readwrite"); - const mlFileIdsArr = await mlIDbStorage.getAllFileIdsForUpdate(tx); - const mlFileIds = new Set(); - mlFileIdsArr.forEach((mlFileId) => mlFileIds.add(mlFileId)); - - const newFileIds: Array = []; - for (const localFileId of localFilesMap.keys()) { - if (!mlFileIds.has(localFileId)) { - newFileIds.push(localFileId); - } - } - - let updated = false; - if (newFileIds.length > 0) { - log.info("newFiles: ", newFileIds.length); - const newFiles = newFileIds.map((fileId) => this.newMlData(fileId)); - await mlIDbStorage.putAllFiles(newFiles, tx); - updated = true; - } - - const removedFileIds: Array = []; - for (const mlFileId of mlFileIds) { - if (!localFilesMap.has(mlFileId)) { - removedFileIds.push(mlFileId); - } - } - - if (removedFileIds.length > 0) { - log.info("removedFiles: ", removedFileIds.length); - await mlIDbStorage.removeAllFiles(removedFileIds, tx); - updated = true; - } - - await tx.done; - - if (updated) { - // TODO: should do in same transaction - await mlIDbStorage.incrementIndexVersion("files"); - } - - log.info("syncLocalFiles", Date.now() - startTime, "ms"); - } private async getOutOfSyncFiles(syncContext: MLSyncContext) { const startTime = Date.now(); From b17933a2b3f906c7cf4cd424525758ac2db6f284 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 12:14:02 +0530 Subject: [PATCH 173/354] Tweak --- web/apps/photos/src/services/face/indexer.ts | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index 829638c755..98a22b3e1e 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -246,7 +246,7 @@ export const setIsFaceIndexingEnabled = async (enabled: boolean) => { export const syncLocalFiles = async (userID: number) => { const startTime = Date.now(); - const localFilesMap = await getLocalFilesMap(userID); + const localFilesMap = await localUserOwnedFilesByID(userID); const db = await mlIDbStorage.db; const tx = db.transaction("files", "readwrite"); @@ -301,12 +301,18 @@ export const syncLocalFiles = async (userID: number) => { return localFilesMap; }; -const getLocalFilesMap = async (userID: number) => { +/** + * Return a map of all {@link EnteFile}s owned by {@link userID} that we know + * about locally, indexed by their {@link fileID}. + * + * @param userID Restrict the returned files to those owned by a {@link userID}. + */ +const localUserOwnedFilesByID = async ( + userID: number, +): Promise> => { + const result = new Map(); const localFiles = await getLocalFiles(); - const personalFiles = localFiles.filter((f) => f.ownerID === userID); - const localFilesMap = new Map(); - personalFiles.forEach((f) => localFilesMap.set(f.id, f)); - - return localFilesMap; + personalFiles.forEach((f) => result.set(f.id, f)); + return result; }; From c3347bae5df8f7def765f70f785693ad32c8d55b Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Thu, 30 May 2024 12:20:40 +0530 Subject: [PATCH 174/354] [mob][photos] Show indexing description only once at top --- mobile/lib/generated/intl/messages_cs.dart | 2 - mobile/lib/generated/intl/messages_de.dart | 4 - mobile/lib/generated/intl/messages_en.dart | 6 +- mobile/lib/generated/intl/messages_es.dart | 2 - mobile/lib/generated/intl/messages_fr.dart | 2 - mobile/lib/generated/intl/messages_it.dart | 2 - mobile/lib/generated/intl/messages_ko.dart | 2 - mobile/lib/generated/intl/messages_nl.dart | 4 - mobile/lib/generated/intl/messages_no.dart | 2 - mobile/lib/generated/intl/messages_pl.dart | 2 - mobile/lib/generated/intl/messages_pt.dart | 4 - mobile/lib/generated/intl/messages_zh.dart | 4 - mobile/lib/generated/l10n.dart | 18 +--- mobile/lib/l10n/intl_en.arb | 3 +- .../machine_learning_settings_page.dart | 97 +++++++------------ 15 files changed, 44 insertions(+), 110 deletions(-) diff --git a/mobile/lib/generated/intl/messages_cs.dart b/mobile/lib/generated/intl/messages_cs.dart index 6301af5611..3de2f825b3 100644 --- a/mobile/lib/generated/intl/messages_cs.dart +++ b/mobile/lib/generated/intl/messages_cs.dart @@ -50,8 +50,6 @@ class MessageLookup extends MessageLookupByLibrary { MessageLookupByLibrary.simpleMessage("Enter person name"), "faceRecognition": MessageLookupByLibrary.simpleMessage("Face recognition"), - "faceRecognitionIndexingDescription": MessageLookupByLibrary.simpleMessage( - "Please note that this will result in a higher bandwidth and battery usage until all items are indexed."), "fileTypes": MessageLookupByLibrary.simpleMessage("File types"), "foundFaces": MessageLookupByLibrary.simpleMessage("Found faces"), "indexingIsPaused": MessageLookupByLibrary.simpleMessage( diff --git a/mobile/lib/generated/intl/messages_de.dart b/mobile/lib/generated/intl/messages_de.dart index 49c7ac93c6..3e9db65de3 100644 --- a/mobile/lib/generated/intl/messages_de.dart +++ b/mobile/lib/generated/intl/messages_de.dart @@ -706,8 +706,6 @@ class MessageLookup extends MessageLookupByLibrary { MessageLookupByLibrary.simpleMessage("Daten exportieren"), "faceRecognition": MessageLookupByLibrary.simpleMessage("Face recognition"), - "faceRecognitionIndexingDescription": MessageLookupByLibrary.simpleMessage( - "Please note that this will result in a higher bandwidth and battery usage until all items are indexed."), "faces": MessageLookupByLibrary.simpleMessage("Gesichter"), "failedToApplyCode": MessageLookupByLibrary.simpleMessage( "Der Code konnte nicht aktiviert werden"), @@ -928,8 +926,6 @@ class MessageLookup extends MessageLookupByLibrary { "machineLearning": MessageLookupByLibrary.simpleMessage("Maschinelles Lernen"), "magicSearch": MessageLookupByLibrary.simpleMessage("Magische Suche"), - "magicSearchDescription": MessageLookupByLibrary.simpleMessage( - "Bitte beachten Sie, dass dies mehr Bandbreite nutzt und zu einem höheren Akkuverbrauch führt, bis alle Elemente indiziert sind."), "manage": MessageLookupByLibrary.simpleMessage("Verwalten"), "manageDeviceStorage": MessageLookupByLibrary.simpleMessage("Gerätespeicher verwalten"), diff --git a/mobile/lib/generated/intl/messages_en.dart b/mobile/lib/generated/intl/messages_en.dart index b715eb4851..424b3b3aba 100644 --- a/mobile/lib/generated/intl/messages_en.dart +++ b/mobile/lib/generated/intl/messages_en.dart @@ -704,8 +704,6 @@ class MessageLookup extends MessageLookupByLibrary { MessageLookupByLibrary.simpleMessage("Export your data"), "faceRecognition": MessageLookupByLibrary.simpleMessage("Face recognition"), - "faceRecognitionIndexingDescription": MessageLookupByLibrary.simpleMessage( - "Please note that this will result in a higher bandwidth and battery usage until all items are indexed."), "faces": MessageLookupByLibrary.simpleMessage("Faces"), "failedToApplyCode": MessageLookupByLibrary.simpleMessage("Failed to apply code"), @@ -921,8 +919,6 @@ class MessageLookup extends MessageLookupByLibrary { "machineLearning": MessageLookupByLibrary.simpleMessage("Machine learning"), "magicSearch": MessageLookupByLibrary.simpleMessage("Magic search"), - "magicSearchDescription": MessageLookupByLibrary.simpleMessage( - "Please note that this will result in a higher bandwidth and battery usage until all items are indexed."), "manage": MessageLookupByLibrary.simpleMessage("Manage"), "manageDeviceStorage": MessageLookupByLibrary.simpleMessage("Manage device storage"), @@ -939,6 +935,8 @@ class MessageLookup extends MessageLookupByLibrary { "matrix": MessageLookupByLibrary.simpleMessage("Matrix"), "memoryCount": m33, "merchandise": MessageLookupByLibrary.simpleMessage("Merchandise"), + "mlIndexingDescription": MessageLookupByLibrary.simpleMessage( + "Please note that ML indexing will result in a higher bandwidth and battery usage until all items are indexed."), "mobileWebDesktop": MessageLookupByLibrary.simpleMessage("Mobile, Web, Desktop"), "moderateStrength": MessageLookupByLibrary.simpleMessage("Moderate"), diff --git a/mobile/lib/generated/intl/messages_es.dart b/mobile/lib/generated/intl/messages_es.dart index f0b4c87f5c..bb501dbfa5 100644 --- a/mobile/lib/generated/intl/messages_es.dart +++ b/mobile/lib/generated/intl/messages_es.dart @@ -615,8 +615,6 @@ class MessageLookup extends MessageLookupByLibrary { MessageLookupByLibrary.simpleMessage("Exportar tus datos"), "faceRecognition": MessageLookupByLibrary.simpleMessage("Face recognition"), - "faceRecognitionIndexingDescription": MessageLookupByLibrary.simpleMessage( - "Please note that this will result in a higher bandwidth and battery usage until all items are indexed."), "failedToApplyCode": MessageLookupByLibrary.simpleMessage("Error al aplicar el código"), "failedToCancel": diff --git a/mobile/lib/generated/intl/messages_fr.dart b/mobile/lib/generated/intl/messages_fr.dart index 5c4a2b4e47..e82ec8ac95 100644 --- a/mobile/lib/generated/intl/messages_fr.dart +++ b/mobile/lib/generated/intl/messages_fr.dart @@ -694,8 +694,6 @@ class MessageLookup extends MessageLookupByLibrary { MessageLookupByLibrary.simpleMessage("Exportez vos données"), "faceRecognition": MessageLookupByLibrary.simpleMessage("Face recognition"), - "faceRecognitionIndexingDescription": MessageLookupByLibrary.simpleMessage( - "Please note that this will result in a higher bandwidth and battery usage until all items are indexed."), "faces": MessageLookupByLibrary.simpleMessage("Visages"), "failedToApplyCode": MessageLookupByLibrary.simpleMessage( "Impossible d\'appliquer le code"), diff --git a/mobile/lib/generated/intl/messages_it.dart b/mobile/lib/generated/intl/messages_it.dart index d7a902db84..440489a89b 100644 --- a/mobile/lib/generated/intl/messages_it.dart +++ b/mobile/lib/generated/intl/messages_it.dart @@ -671,8 +671,6 @@ class MessageLookup extends MessageLookupByLibrary { "exportYourData": MessageLookupByLibrary.simpleMessage("Esporta dati"), "faceRecognition": MessageLookupByLibrary.simpleMessage("Face recognition"), - "faceRecognitionIndexingDescription": MessageLookupByLibrary.simpleMessage( - "Please note that this will result in a higher bandwidth and battery usage until all items are indexed."), "failedToApplyCode": MessageLookupByLibrary.simpleMessage( "Impossibile applicare il codice"), "failedToCancel": diff --git a/mobile/lib/generated/intl/messages_ko.dart b/mobile/lib/generated/intl/messages_ko.dart index 614d860dc3..a174b77072 100644 --- a/mobile/lib/generated/intl/messages_ko.dart +++ b/mobile/lib/generated/intl/messages_ko.dart @@ -50,8 +50,6 @@ class MessageLookup extends MessageLookupByLibrary { MessageLookupByLibrary.simpleMessage("Enter person name"), "faceRecognition": MessageLookupByLibrary.simpleMessage("Face recognition"), - "faceRecognitionIndexingDescription": MessageLookupByLibrary.simpleMessage( - "Please note that this will result in a higher bandwidth and battery usage until all items are indexed."), "fileTypes": MessageLookupByLibrary.simpleMessage("File types"), "foundFaces": MessageLookupByLibrary.simpleMessage("Found faces"), "indexingIsPaused": MessageLookupByLibrary.simpleMessage( diff --git a/mobile/lib/generated/intl/messages_nl.dart b/mobile/lib/generated/intl/messages_nl.dart index 1981b338ce..c0a32e161c 100644 --- a/mobile/lib/generated/intl/messages_nl.dart +++ b/mobile/lib/generated/intl/messages_nl.dart @@ -727,8 +727,6 @@ class MessageLookup extends MessageLookupByLibrary { MessageLookupByLibrary.simpleMessage("Exporteer je gegevens"), "faceRecognition": MessageLookupByLibrary.simpleMessage("Face recognition"), - "faceRecognitionIndexingDescription": MessageLookupByLibrary.simpleMessage( - "Please note that this will result in a higher bandwidth and battery usage until all items are indexed."), "faces": MessageLookupByLibrary.simpleMessage("Gezichten"), "failedToApplyCode": MessageLookupByLibrary.simpleMessage("Code toepassen mislukt"), @@ -952,8 +950,6 @@ class MessageLookup extends MessageLookupByLibrary { MessageLookupByLibrary.simpleMessage("Machine Learning"), "magicSearch": MessageLookupByLibrary.simpleMessage("Magische zoekfunctie"), - "magicSearchDescription": MessageLookupByLibrary.simpleMessage( - "Houd er rekening mee dat dit zal resulteren in een hoger internet- en batterijverbruik totdat alle items zijn geïndexeerd."), "manage": MessageLookupByLibrary.simpleMessage("Beheren"), "manageDeviceStorage": MessageLookupByLibrary.simpleMessage("Apparaatopslag beheren"), diff --git a/mobile/lib/generated/intl/messages_no.dart b/mobile/lib/generated/intl/messages_no.dart index cce44555aa..d3f7ffa1f3 100644 --- a/mobile/lib/generated/intl/messages_no.dart +++ b/mobile/lib/generated/intl/messages_no.dart @@ -67,8 +67,6 @@ class MessageLookup extends MessageLookupByLibrary { "Skriv inn e-postadressen din"), "faceRecognition": MessageLookupByLibrary.simpleMessage("Face recognition"), - "faceRecognitionIndexingDescription": MessageLookupByLibrary.simpleMessage( - "Please note that this will result in a higher bandwidth and battery usage until all items are indexed."), "feedback": MessageLookupByLibrary.simpleMessage("Tilbakemelding"), "fileTypes": MessageLookupByLibrary.simpleMessage("File types"), "foundFaces": MessageLookupByLibrary.simpleMessage("Found faces"), diff --git a/mobile/lib/generated/intl/messages_pl.dart b/mobile/lib/generated/intl/messages_pl.dart index 01cb3cb615..0e16b5830b 100644 --- a/mobile/lib/generated/intl/messages_pl.dart +++ b/mobile/lib/generated/intl/messages_pl.dart @@ -115,8 +115,6 @@ class MessageLookup extends MessageLookupByLibrary { "Wprowadź swój klucz odzyskiwania"), "faceRecognition": MessageLookupByLibrary.simpleMessage("Face recognition"), - "faceRecognitionIndexingDescription": MessageLookupByLibrary.simpleMessage( - "Please note that this will result in a higher bandwidth and battery usage until all items are indexed."), "feedback": MessageLookupByLibrary.simpleMessage("Informacja zwrotna"), "fileTypes": MessageLookupByLibrary.simpleMessage("File types"), "forgotPassword": diff --git a/mobile/lib/generated/intl/messages_pt.dart b/mobile/lib/generated/intl/messages_pt.dart index e32fd36379..b4b131ca1b 100644 --- a/mobile/lib/generated/intl/messages_pt.dart +++ b/mobile/lib/generated/intl/messages_pt.dart @@ -721,8 +721,6 @@ class MessageLookup extends MessageLookupByLibrary { MessageLookupByLibrary.simpleMessage("Exportar seus dados"), "faceRecognition": MessageLookupByLibrary.simpleMessage("Reconhecimento facial"), - "faceRecognitionIndexingDescription": MessageLookupByLibrary.simpleMessage( - "Por favor, note que isso resultará em uma largura de banda maior e uso de bateria até que todos os itens sejam indexados."), "faces": MessageLookupByLibrary.simpleMessage("Rostos"), "failedToApplyCode": MessageLookupByLibrary.simpleMessage("Falha ao aplicar o código"), @@ -950,8 +948,6 @@ class MessageLookup extends MessageLookupByLibrary { "machineLearning": MessageLookupByLibrary.simpleMessage("Aprendizagem de máquina"), "magicSearch": MessageLookupByLibrary.simpleMessage("Busca mágica"), - "magicSearchDescription": MessageLookupByLibrary.simpleMessage( - "Por favor, note que isso resultará em uma largura de banda maior e uso de bateria até que todos os itens sejam indexados."), "manage": MessageLookupByLibrary.simpleMessage("Gerenciar"), "manageDeviceStorage": MessageLookupByLibrary.simpleMessage( "Gerenciar o armazenamento do dispositivo"), diff --git a/mobile/lib/generated/intl/messages_zh.dart b/mobile/lib/generated/intl/messages_zh.dart index ecca5d7b8f..f38069042f 100644 --- a/mobile/lib/generated/intl/messages_zh.dart +++ b/mobile/lib/generated/intl/messages_zh.dart @@ -596,8 +596,6 @@ class MessageLookup extends MessageLookupByLibrary { "exportYourData": MessageLookupByLibrary.simpleMessage("导出您的数据"), "faceRecognition": MessageLookupByLibrary.simpleMessage("Face recognition"), - "faceRecognitionIndexingDescription": MessageLookupByLibrary.simpleMessage( - "Please note that this will result in a higher bandwidth and battery usage until all items are indexed."), "faces": MessageLookupByLibrary.simpleMessage("人脸"), "failedToApplyCode": MessageLookupByLibrary.simpleMessage("无法使用此代码"), "failedToCancel": MessageLookupByLibrary.simpleMessage("取消失败"), @@ -779,8 +777,6 @@ class MessageLookup extends MessageLookupByLibrary { "lostDevice": MessageLookupByLibrary.simpleMessage("设备丢失？"), "machineLearning": MessageLookupByLibrary.simpleMessage("机器学习"), "magicSearch": MessageLookupByLibrary.simpleMessage("魔法搜索"), - "magicSearchDescription": MessageLookupByLibrary.simpleMessage( - "请注意，在所有项目完成索引之前，这将使用更高的带宽和电量。"), "manage": MessageLookupByLibrary.simpleMessage("管理"), "manageDeviceStorage": MessageLookupByLibrary.simpleMessage("管理设备存储"), "manageFamily": MessageLookupByLibrary.simpleMessage("管理家庭计划"), diff --git a/mobile/lib/generated/l10n.dart b/mobile/lib/generated/l10n.dart index 23b67ff0bf..3f1b9fdb4f 100644 --- a/mobile/lib/generated/l10n.dart +++ b/mobile/lib/generated/l10n.dart @@ -2876,11 +2876,11 @@ class S { ); } - /// `Please note that this will result in a higher bandwidth and battery usage until all items are indexed.` - String get magicSearchDescription { + /// `Please note that ML indexing will result in a higher bandwidth and battery usage until all items are indexed.` + String get mlIndexingDescription { return Intl.message( - 'Please note that this will result in a higher bandwidth and battery usage until all items are indexed.', - name: 'magicSearchDescription', + 'Please note that ML indexing will result in a higher bandwidth and battery usage until all items are indexed.', + name: 'mlIndexingDescription', desc: '', args: [], ); @@ -8764,16 +8764,6 @@ class S { ); } - /// `Please note that this will result in a higher bandwidth and battery usage until all items are indexed.` - String get faceRecognitionIndexingDescription { - return Intl.message( - 'Please note that this will result in a higher bandwidth and battery usage until all items are indexed.', - name: 'faceRecognitionIndexingDescription', - desc: '', - args: [], - ); - } - /// `Found faces` String get foundFaces { return Intl.message( diff --git a/mobile/lib/l10n/intl_en.arb b/mobile/lib/l10n/intl_en.arb index df2894e4ce..feefd5a298 100644 --- a/mobile/lib/l10n/intl_en.arb +++ b/mobile/lib/l10n/intl_en.arb @@ -409,7 +409,7 @@ "manageDeviceStorage": "Manage device storage", "machineLearning": "Machine learning", "magicSearch": "Magic search", - "magicSearchDescription": "Please note that this will result in a higher bandwidth and battery usage until all items are indexed.", + "mlIndexingDescription": "Please note that ML indexing will result in a higher bandwidth and battery usage until all items are indexed.", "loadingModel": "Downloading models...", "waitingForWifi": "Waiting for WiFi...", "status": "Status", @@ -1233,7 +1233,6 @@ "autoPair": "Auto pair", "pairWithPin": "Pair with PIN", "faceRecognition": "Face recognition", - "faceRecognitionIndexingDescription": "Please note that this will result in a higher bandwidth and battery usage until all items are indexed.", "foundFaces": "Found faces", "clusteringProgress": "Clustering progress", "indexingIsPaused": "Indexing is paused. It will automatically resume when device is ready." diff --git a/mobile/lib/ui/settings/machine_learning_settings_page.dart b/mobile/lib/ui/settings/machine_learning_settings_page.dart index 257d5dd0fa..c39d3e6ee9 100644 --- a/mobile/lib/ui/settings/machine_learning_settings_page.dart +++ b/mobile/lib/ui/settings/machine_learning_settings_page.dart @@ -36,23 +36,19 @@ class MachineLearningSettingsPage extends StatefulWidget { const MachineLearningSettingsPage({super.key}); @override - State createState() => - _MachineLearningSettingsPageState(); + State createState() => _MachineLearningSettingsPageState(); } -class _MachineLearningSettingsPageState - extends State { +class _MachineLearningSettingsPageState extends State { late InitializationState _state; final EnteWakeLock _wakeLock = EnteWakeLock(); - late StreamSubscription - _eventSubscription; + late StreamSubscription _eventSubscription; @override void initState() { super.initState(); - _eventSubscription = - Bus.instance.on().listen((event) { + _eventSubscription = Bus.instance.on().listen((event) { _fetchState(); setState(() {}); }); @@ -95,6 +91,21 @@ class _MachineLearningSettingsPageState ), ], ), + SliverList( + delegate: SliverChildBuilderDelegate( + (delegateBuildContext, index) => Padding( + padding: const EdgeInsets.only(left: 16, right: 16), + child: Text( + S.of(context).mlIndexingDescription, + textAlign: TextAlign.left, + style: getEnteTextTheme(context) + .mini + .copyWith(color: getEnteColorScheme(context).textMuted), + ), + ), + childCount: 1, + ), + ), SliverList( delegate: SliverChildBuilderDelegate( (delegateBuildContext, index) { @@ -107,9 +118,7 @@ class _MachineLearningSettingsPageState children: [ _getMagicSearchSettings(context), const SizedBox(height: 12), - facesFlag - ? _getFacesSearchSettings(context) - : const SizedBox.shrink(), + facesFlag ? _getFacesSearchSettings(context) : const SizedBox.shrink(), ], ), ), @@ -141,8 +150,7 @@ class _MachineLearningSettingsPageState ); if (LocalSettings.instance.hasEnabledMagicSearch()) { unawaited( - SemanticSearchService.instance - .init(shouldSyncImmediately: true), + SemanticSearchService.instance.init(shouldSyncImmediately: true), ); } else { await SemanticSearchService.instance.clearQueue(); @@ -154,12 +162,6 @@ class _MachineLearningSettingsPageState alignCaptionedTextToLeft: true, isGestureDetectorDisabled: true, ), - const SizedBox( - height: 4, - ), - MenuSectionDescriptionWidget( - content: S.of(context).magicSearchDescription, - ), const SizedBox( height: 12, ), @@ -209,8 +211,7 @@ class _MachineLearningSettingsPageState trailingWidget: ToggleSwitchWidget( value: () => LocalSettings.instance.isFaceIndexingEnabled, onChanged: () async { - final isEnabled = - await LocalSettings.instance.toggleFaceIndexing(); + final isEnabled = await LocalSettings.instance.toggleFaceIndexing(); if (isEnabled) { unawaited(FaceMlService.instance.ensureInitialized()); } else { @@ -225,18 +226,10 @@ class _MachineLearningSettingsPageState alignCaptionedTextToLeft: true, isGestureDetectorDisabled: true, ), - const SizedBox( - height: 4, - ), - MenuSectionDescriptionWidget( - content: S.of(context).faceRecognitionIndexingDescription, - ), const SizedBox( height: 12, ), - hasEnabled - ? const FaceRecognitionStatusWidget() - : const SizedBox.shrink(), + hasEnabled ? const FaceRecognitionStatusWidget() : const SizedBox.shrink(), ], ); } @@ -259,8 +252,7 @@ class _ModelLoadingStateState extends State { final Map _progressMap = {}; @override void initState() { - _progressStream = - RemoteAssetsService.instance.progressStream.listen((event) { + _progressStream = RemoteAssetsService.instance.progressStream.listen((event) { final String url = event.$1; String title = ""; if (url.contains("clip-image")) { @@ -338,20 +330,17 @@ class MagicSearchIndexStatsWidget extends StatefulWidget { }); @override - State createState() => - _MagicSearchIndexStatsWidgetState(); + State createState() => _MagicSearchIndexStatsWidgetState(); } -class _MagicSearchIndexStatsWidgetState - extends State { +class _MagicSearchIndexStatsWidgetState extends State { IndexStatus? _status; late StreamSubscription _eventSubscription; @override void initState() { super.initState(); - _eventSubscription = - Bus.instance.on().listen((event) { + _eventSubscription = Bus.instance.on().listen((event) { _fetchIndexStatus(); }); _fetchIndexStatus(); @@ -427,12 +416,10 @@ class FaceRecognitionStatusWidget extends StatefulWidget { }); @override - State createState() => - FaceRecognitionStatusWidgetState(); + State createState() => FaceRecognitionStatusWidgetState(); } -class FaceRecognitionStatusWidgetState - extends State { +class FaceRecognitionStatusWidgetState extends State { Timer? _timer; @override void initState() { @@ -446,22 +433,15 @@ class FaceRecognitionStatusWidgetState Future<(int, int, double, bool)> getIndexStatus() async { try { - final indexedFiles = await FaceMLDataDB.instance - .getIndexedFileCount(minimumMlVersion: faceMlVersion); + final indexedFiles = + await FaceMLDataDB.instance.getIndexedFileCount(minimumMlVersion: faceMlVersion); final indexableFiles = (await getIndexableFileIDs()).length; final showIndexedFiles = min(indexedFiles, indexableFiles); final pendingFiles = max(indexableFiles - indexedFiles, 0); - final clusteringDoneRatio = - await FaceMLDataDB.instance.getClusteredToIndexableFilesRatio(); - final bool deviceIsHealthy = - MachineLearningController.instance.isDeviceHealthy; + final clusteringDoneRatio = await FaceMLDataDB.instance.getClusteredToIndexableFilesRatio(); + final bool deviceIsHealthy = MachineLearningController.instance.isDeviceHealthy; - return ( - showIndexedFiles, - pendingFiles, - clusteringDoneRatio, - deviceIsHealthy - ); + return (showIndexedFiles, pendingFiles, clusteringDoneRatio, deviceIsHealthy); } catch (e, s) { _logger.severe('Error getting face recognition status', e, s); rethrow; @@ -491,12 +471,10 @@ class FaceRecognitionStatusWidgetState final int indexedFiles = snapshot.data!.$1; final int pendingFiles = snapshot.data!.$2; final double clusteringDoneRatio = snapshot.data!.$3; - final double clusteringPercentage = - (clusteringDoneRatio * 100).clamp(0, 100); + final double clusteringPercentage = (clusteringDoneRatio * 100).clamp(0, 100); final bool isDeviceHealthy = snapshot.data!.$4; - if (!isDeviceHealthy && - (pendingFiles > 0 || clusteringPercentage < 99)) { + if (!isDeviceHealthy && (pendingFiles > 0 || clusteringPercentage < 99)) { return MenuSectionDescriptionWidget( content: S.of(context).indexingIsPaused, ); @@ -542,8 +520,7 @@ class FaceRecognitionStatusWidgetState alignCaptionedTextToLeft: true, isGestureDetectorDisabled: true, key: ValueKey( - "clustering_progress_" + - clusteringPercentage.toStringAsFixed(0), + "clustering_progress_" + clusteringPercentage.toStringAsFixed(0), ), ), ], From 6b1484671bf90f9e82b256c30001e74a43e43b87 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 12:24:34 +0530 Subject: [PATCH 175/354] Add remove --- web/apps/photos/src/services/face/db.ts | 26 +++++++++++++++++--- web/apps/photos/src/services/face/indexer.ts | 6 ++--- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/web/apps/photos/src/services/face/db.ts b/web/apps/photos/src/services/face/db.ts index 85bb322056..1625376a79 100644 --- a/web/apps/photos/src/services/face/db.ts +++ b/web/apps/photos/src/services/face/db.ts @@ -197,18 +197,36 @@ export const addFileEntry = async (fileID: number) => { return tx.done; }; +/** + * Remove a file's entry and face index (if any) from the face DB. + * + * @param fileID The ID of an {@link EnteFile} + * + * This function is invoked when the user has deleted a file, and we want to + * also prune that file's data from the face DB. + */ +export const removeFile = async (fileID: number) => { + const db = await faceDB(); + const tx = db.transaction(["face-index", "file-status"], "readwrite"); + return Promise.all([ + tx.objectStore("face-index").delete(fileID), + tx.objectStore("file-status").delete(fileID), + tx.done, + ]); +}; + /** * Return the count of files that can be, and that have been, indexed. */ -export const indexableAndIndexedCounts = async () => { +export const indexedAndIndexableCounts = async () => { const db = await faceDB(); - const tx = db.transaction(["file-status", "face-index"], "readonly"); + const tx = db.transaction(["face-index", "file-status"], "readwrite"); + const indexedCount = await tx.objectStore("face-index").count(); const indexableCount = await tx .objectStore("file-status") .index("isIndexable") .count(IDBKeyRange.only(1)); - const indexedCount = await tx.objectStore("face-index").count(); - return { indexableCount, indexedCount }; + return { indexedCount, indexableCount }; }; /** diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index 98a22b3e1e..a10ff34bb1 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -10,7 +10,7 @@ import machineLearningService, { import mlWorkManager from "services/machineLearning/mlWorkManager"; import type { EnteFile } from "types/file"; import { isInternalUserForML } from "utils/user"; -import { indexableAndIndexedCounts, markIndexingFailed } from "./db"; +import { indexedAndIndexableCounts, markIndexingFailed } from "./db"; import type { IndexStatus, MinimalPersistedFileData } from "./db-old"; import { indexFaces } from "./f-index"; import { FaceIndexerWorker } from "./indexer.worker"; @@ -161,7 +161,7 @@ export interface FaceIndexingStatus { export const faceIndexingStatus = async (): Promise => { const isSyncing = machineLearningService.isSyncing; - const { indexableCount, indexedCount } = await indexableAndIndexedCounts(); + const { indexedCount, indexableCount } = await indexedAndIndexableCounts(); let phase: FaceIndexingStatus["phase"]; if (indexedCount < indexableCount) { @@ -176,8 +176,8 @@ export const faceIndexingStatus = async (): Promise => { const indexingStatus = { phase, - nTotalFiles: indexableCount, nSyncedFiles: indexedCount, + nTotalFiles: indexableCount, }; const indexStatus0 = await mlIDbStorage.getIndexStatus(defaultMLVersion); From 53dea9dcf3f6334d2635b677bc637f2b12bf0615 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 12:50:14 +0530 Subject: [PATCH 176/354] Sync --- web/apps/photos/src/services/face/db.ts | 46 +++++++++++++++++++------ 1 file changed, 36 insertions(+), 10 deletions(-) diff --git a/web/apps/photos/src/services/face/db.ts b/web/apps/photos/src/services/face/db.ts index 1625376a79..bfa54e7d6a 100644 --- a/web/apps/photos/src/services/face/db.ts +++ b/web/apps/photos/src/services/face/db.ts @@ -198,21 +198,47 @@ export const addFileEntry = async (fileID: number) => { }; /** - * Remove a file's entry and face index (if any) from the face DB. + * Sync entries in the face DB to match the given list of local file IDs. * - * @param fileID The ID of an {@link EnteFile} + * @param localFileIDs The IDs of all the files that the client is aware of, + * filtered to only keep the files that the user owns. * - * This function is invoked when the user has deleted a file, and we want to - * also prune that file's data from the face DB. + * This function syncs the state of file entries in face DB to the state of file + * entries stored otherwise by the local client. + * + * - Files (identified by their ID) that are present locally but are not yet in + * face DB get a fresh entry in face DB (and are marked as indexable). + * + * - Files that are not present locally but still exist in face DB are removed + * from face DB (including its face index, if any). */ -export const removeFile = async (fileID: number) => { +export const syncWithLocalUserOwnedFileIDs = async (localFileIDs: number[]) => { const db = await faceDB(); const tx = db.transaction(["face-index", "file-status"], "readwrite"); - return Promise.all([ - tx.objectStore("face-index").delete(fileID), - tx.objectStore("file-status").delete(fileID), - tx.done, - ]); + const fdbFileIDs = await tx.objectStore("file-status").getAllKeys(); + + const local = new Set(localFileIDs); + const fdb = new Set(fdbFileIDs); + + const newFileIDs = localFileIDs.filter((id) => !fdb.has(id)); + const removedFileIDs = fdbFileIDs.filter((id) => !local.has(id)); + + return Promise.all( + [ + newFileIDs.map((id) => + tx.objectStore("file-status").put({ + fileID: id, + isIndexable: 1, + failureCount: 0, + }), + ), + removedFileIDs.map((id) => + tx.objectStore("file-status").delete(id), + ), + removedFileIDs.map((id) => tx.objectStore("face-index").delete(id)), + tx.done, + ].flat(), + ); }; /** From e41e0eadee9f3c1f0873d4bcbd663cfcd2871b7e Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Thu, 30 May 2024 13:01:35 +0530 Subject: [PATCH 177/354] [mob][photos] Rename var showOptionToCreateNewAlbum --- mobile/lib/ui/viewer/people/add_person_action_sheet.dart | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mobile/lib/ui/viewer/people/add_person_action_sheet.dart b/mobile/lib/ui/viewer/people/add_person_action_sheet.dart index 83a19ac316..1897e2b7fd 100644 --- a/mobile/lib/ui/viewer/people/add_person_action_sheet.dart +++ b/mobile/lib/ui/viewer/people/add_person_action_sheet.dart @@ -49,14 +49,14 @@ Future showAssignPersonAction( BuildContext context, { required int clusterID, PersonActionType actionType = PersonActionType.assignPerson, - bool showOptionToCreateNewAlbum = true, + bool showOptionToAddNewPerson = true, }) { return showBarModalBottomSheet( context: context, builder: (context) { return PersonActionSheet( actionType: actionType, - showOptionToCreateNewPerson: showOptionToCreateNewAlbum, + showOptionToCreateNewPerson: showOptionToAddNewPerson, cluserID: clusterID, ); }, From 80be753d77e748d35093068a38fc24f3a6fef6f5 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Thu, 30 May 2024 13:02:00 +0530 Subject: [PATCH 178/354] [mob][photos] Properly align person tiles --- mobile/lib/ui/viewer/people/person_row_item.dart | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mobile/lib/ui/viewer/people/person_row_item.dart b/mobile/lib/ui/viewer/people/person_row_item.dart index 831fe97298..ed1fc9fa27 100644 --- a/mobile/lib/ui/viewer/people/person_row_item.dart +++ b/mobile/lib/ui/viewer/people/person_row_item.dart @@ -19,6 +19,8 @@ class PersonRowItem extends StatelessWidget { Widget build(BuildContext context) { return ListTile( dense: false, + minLeadingWidth: 0, + contentPadding: const EdgeInsets.symmetric(horizontal: 0), leading: SizedBox( width: 56, height: 56, From 6e6c88826e95f96888071525d4b77a0104f2f8d6 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 13:01:11 +0530 Subject: [PATCH 179/354] t --- web/apps/photos/src/services/face/indexer.ts | 5 ++--- .../src/services/machineLearning/machineLearningService.ts | 1 - 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index a10ff34bb1..9f3cc4a9d3 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -245,9 +245,10 @@ export const setIsFaceIndexingEnabled = async (enabled: boolean) => { }; export const syncLocalFiles = async (userID: number) => { - const startTime = Date.now(); const localFilesMap = await localUserOwnedFilesByID(userID); + // const localFileIDs = new Set(localFilesMap.keys()); + const db = await mlIDbStorage.db; const tx = db.transaction("files", "readwrite"); const mlFileIdsArr = await mlIDbStorage.getAllFileIdsForUpdate(tx); @@ -296,8 +297,6 @@ export const syncLocalFiles = async (userID: number) => { await mlIDbStorage.incrementIndexVersion("files"); } - log.info("syncLocalFiles", Date.now() - startTime, "ms"); - return localFilesMap; }; diff --git a/web/apps/photos/src/services/machineLearning/machineLearningService.ts b/web/apps/photos/src/services/machineLearning/machineLearningService.ts index 4aa2235db4..f1de281efa 100644 --- a/web/apps/photos/src/services/machineLearning/machineLearningService.ts +++ b/web/apps/photos/src/services/machineLearning/machineLearningService.ts @@ -104,7 +104,6 @@ class MachineLearningService { return syncContext.localFilesMap; } - private async getOutOfSyncFiles(syncContext: MLSyncContext) { const startTime = Date.now(); const fileIds = await mlIDbStorage.getFileIds( From 23c73a83eb9dbda5fafe9ecf430ba4bf9b739181 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 13:16:15 +0530 Subject: [PATCH 180/354] Inline --- web/apps/photos/src/services/face/f-index.ts | 51 +++++++++++++++----- web/apps/photos/src/services/face/file.ts | 37 -------------- web/apps/photos/src/services/face/people.ts | 4 ++ 3 files changed, 44 insertions(+), 48 deletions(-) delete mode 100644 web/apps/photos/src/services/face/file.ts diff --git a/web/apps/photos/src/services/face/f-index.ts b/web/apps/photos/src/services/face/f-index.ts index 714f89c09f..9f871d4367 100644 --- a/web/apps/photos/src/services/face/f-index.ts +++ b/web/apps/photos/src/services/face/f-index.ts @@ -1,7 +1,9 @@ import { FILE_TYPE } from "@/media/file-type"; +import { decodeLivePhoto } from "@/media/live-photo"; import log from "@/next/log"; import { workerBridge } from "@/next/worker/worker-bridge"; import { Matrix } from "ml-matrix"; +import DownloadManager from "services/download"; import { defaultMLVersion } from "services/machineLearning/machineLearningService"; import { getSimilarityTransformation } from "similarity-transformation"; import { @@ -12,9 +14,8 @@ import { translate, } from "transformation-matrix"; import type { EnteFile } from "types/file"; -import { logIdentifier } from "utils/file"; +import { getRenderableImage, logIdentifier } from "utils/file"; import { saveFaceCrop } from "./crop"; -import { fetchImageBitmap, getLocalFileImageBitmap } from "./file"; import { clamp, grayscaleIntMatrixFromNormalized2List, @@ -39,11 +40,20 @@ import type { Face, FaceDetection, MlFileData } from "./types-old"; * they can be saved locally for offline use, and encrypts and uploads them to * the user's remote storage so that their other devices can download them * instead of needing to reindex. + * + * @param enteFile The {@link EnteFile} to index. + * + * @param file The contents of {@link enteFile} as a web {@link File}, if + * available. These are used when they are provided, otherwise the file is + * downloaded and decrypted from remote. */ -export const indexFaces = async (enteFile: EnteFile, localFile?: File) => { +export const indexFaces = async ( + enteFile: EnteFile, + file: File | undefined, +) => { const startTime = Date.now(); - const imageBitmap = await fetchOrCreateImageBitmap(enteFile, localFile); + const imageBitmap = await fetchOrCreateImageBitmap(enteFile, file); let mlFile: MlFileData; try { mlFile = await indexFaces_(enteFile, imageBitmap); @@ -60,20 +70,17 @@ export const indexFaces = async (enteFile: EnteFile, localFile?: File) => { }; /** - * Return a {@link ImageBitmap}, using {@link localFile} if present otherwise + * Return a {@link ImageBitmap}, using {@link file} if present otherwise * downloading the source image corresponding to {@link enteFile} from remote. */ -const fetchOrCreateImageBitmap = async ( - enteFile: EnteFile, - localFile: File, -) => { +const fetchOrCreateImageBitmap = async (enteFile: EnteFile, file: File) => { const fileType = enteFile.metadata.fileType; - if (localFile) { + if (file) { // TODO-ML(MR): Could also be image part of live photo? if (fileType !== FILE_TYPE.IMAGE) throw new Error("Local file of only image type is supported"); - return await getLocalFileImageBitmap(enteFile, localFile); + return await getLocalFileImageBitmap(enteFile, file); } else if ([FILE_TYPE.IMAGE, FILE_TYPE.LIVE_PHOTO].includes(fileType)) { return await fetchImageBitmap(enteFile); } else { @@ -81,6 +88,28 @@ const fetchOrCreateImageBitmap = async ( } }; +const fetchImageBitmap = async (enteFile: EnteFile) => + fetchRenderableBlob(enteFile).then(createImageBitmap); + +const fetchRenderableBlob = async (enteFile: EnteFile) => { + const fileStream = await DownloadManager.getFile(enteFile); + const fileBlob = await new Response(fileStream).blob(); + if (enteFile.metadata.fileType === FILE_TYPE.IMAGE) { + return getRenderableImage(enteFile.metadata.title, fileBlob); + } else { + const { imageFileName, imageData } = await decodeLivePhoto( + enteFile.metadata.title, + fileBlob, + ); + return getRenderableImage(imageFileName, new Blob([imageData])); + } +}; + +const getLocalFileImageBitmap = async (enteFile: EnteFile, localFile: File) => + createImageBitmap( + await getRenderableImage(enteFile.metadata.title, localFile), + ); + const indexFaces_ = async (enteFile: EnteFile, imageBitmap: ImageBitmap) => { const fileID = enteFile.id; const { width, height } = imageBitmap; diff --git a/web/apps/photos/src/services/face/file.ts b/web/apps/photos/src/services/face/file.ts deleted file mode 100644 index b482af3fb5..0000000000 --- a/web/apps/photos/src/services/face/file.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { FILE_TYPE } from "@/media/file-type"; -import { decodeLivePhoto } from "@/media/live-photo"; -import DownloadManager from "services/download"; -import { getLocalFiles } from "services/fileService"; -import { EnteFile } from "types/file"; -import { getRenderableImage } from "utils/file"; - -export async function getLocalFile(fileId: number) { - const localFiles = await getLocalFiles(); - return localFiles.find((f) => f.id === fileId); -} - -export const fetchImageBitmap = async (file: EnteFile) => - fetchRenderableBlob(file).then(createImageBitmap); - -async function fetchRenderableBlob(file: EnteFile) { - const fileStream = await DownloadManager.getFile(file); - const fileBlob = await new Response(fileStream).blob(); - if (file.metadata.fileType === FILE_TYPE.IMAGE) { - return await getRenderableImage(file.metadata.title, fileBlob); - } else { - const { imageFileName, imageData } = await decodeLivePhoto( - file.metadata.title, - fileBlob, - ); - return await getRenderableImage(imageFileName, new Blob([imageData])); - } -} - -export async function getLocalFileImageBitmap( - enteFile: EnteFile, - localFile: globalThis.File, -) { - let fileBlob = localFile as Blob; - fileBlob = await getRenderableImage(enteFile.metadata.title, fileBlob); - return createImageBitmap(fileBlob); -} diff --git a/web/apps/photos/src/services/face/people.ts b/web/apps/photos/src/services/face/people.ts index d118cb4f90..311183768c 100644 --- a/web/apps/photos/src/services/face/people.ts +++ b/web/apps/photos/src/services/face/people.ts @@ -84,6 +84,10 @@ export const syncPeopleIndex = async () => { : best, ); +export async function getLocalFile(fileId: number) { + const localFiles = await getLocalFiles(); + return localFiles.find((f) => f.id === fileId); +} if (personFace && !personFace.crop?.cacheKey) { const file = await getLocalFile(personFace.fileId); From 81f9efbacedc851a6ffcd9380c97d2c8f023f633 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Thu, 30 May 2024 13:21:16 +0530 Subject: [PATCH 181/354] [mob][photos] Logs --- .../lib/services/machine_learning/face_ml/face_ml_service.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart index e4512e2bae..1843b7f1b3 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart @@ -854,7 +854,7 @@ class FaceMlService { return true; } catch (e, s) { _logger.severe( - "Failed to analyze using FaceML for image with ID: ${enteFile.uploadedFileID}", + "Failed to analyze using FaceML for image with ID: ${enteFile.uploadedFileID}. Not storing any faces, which means it will be automatically retried later.", e, s, ); From fd4a7889532d33a1005ee32fa8afe2d2751d0dc8 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 13:22:46 +0530 Subject: [PATCH 182/354] Checked that the image part is passed as the file --- web/apps/photos/src/services/face/f-index.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/web/apps/photos/src/services/face/f-index.ts b/web/apps/photos/src/services/face/f-index.ts index 9f871d4367..4ed6c4792f 100644 --- a/web/apps/photos/src/services/face/f-index.ts +++ b/web/apps/photos/src/services/face/f-index.ts @@ -76,10 +76,6 @@ export const indexFaces = async ( const fetchOrCreateImageBitmap = async (enteFile: EnteFile, file: File) => { const fileType = enteFile.metadata.fileType; if (file) { - // TODO-ML(MR): Could also be image part of live photo? - if (fileType !== FILE_TYPE.IMAGE) - throw new Error("Local file of only image type is supported"); - return await getLocalFileImageBitmap(enteFile, file); } else if ([FILE_TYPE.IMAGE, FILE_TYPE.LIVE_PHOTO].includes(fileType)) { return await fetchImageBitmap(enteFile); From c71e56ec43bb0ac0e2a48876aaa2e8b7394c8dca Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 13:26:09 +0530 Subject: [PATCH 183/354] inl --- web/apps/photos/src/services/face/f-index.ts | 21 ++++++++------------ 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/web/apps/photos/src/services/face/f-index.ts b/web/apps/photos/src/services/face/f-index.ts index 4ed6c4792f..c47d8a677c 100644 --- a/web/apps/photos/src/services/face/f-index.ts +++ b/web/apps/photos/src/services/face/f-index.ts @@ -53,7 +53,7 @@ export const indexFaces = async ( ) => { const startTime = Date.now(); - const imageBitmap = await fetchOrCreateImageBitmap(enteFile, file); + const imageBitmap = await fetchAndCreateImageBitmap(enteFile, file); let mlFile: MlFileData; try { mlFile = await indexFaces_(enteFile, imageBitmap); @@ -73,31 +73,26 @@ export const indexFaces = async ( * Return a {@link ImageBitmap}, using {@link file} if present otherwise * downloading the source image corresponding to {@link enteFile} from remote. */ -const fetchOrCreateImageBitmap = async (enteFile: EnteFile, file: File) => { - const fileType = enteFile.metadata.fileType; - if (file) { - return await getLocalFileImageBitmap(enteFile, file); - } else if ([FILE_TYPE.IMAGE, FILE_TYPE.LIVE_PHOTO].includes(fileType)) { - return await fetchImageBitmap(enteFile); - } else { - throw new Error(`Cannot index unsupported file type ${fileType}`); - } -}; +const fetchAndCreateImageBitmap = async (enteFile: EnteFile, file: File) => + file ? getLocalFileImageBitmap(enteFile, file) : fetchImageBitmap(enteFile); const fetchImageBitmap = async (enteFile: EnteFile) => fetchRenderableBlob(enteFile).then(createImageBitmap); const fetchRenderableBlob = async (enteFile: EnteFile) => { + const fileType = enteFile.metadata.fileType; const fileStream = await DownloadManager.getFile(enteFile); const fileBlob = await new Response(fileStream).blob(); - if (enteFile.metadata.fileType === FILE_TYPE.IMAGE) { + if (enteFile.metadata.fileType == FILE_TYPE.IMAGE) { return getRenderableImage(enteFile.metadata.title, fileBlob); - } else { + } else if (enteFile.metadata.fileType == FILE_TYPE.LIVE_PHOTO) { const { imageFileName, imageData } = await decodeLivePhoto( enteFile.metadata.title, fileBlob, ); return getRenderableImage(imageFileName, new Blob([imageData])); + } else { + throw new Error(`Cannot index unsupported file type ${fileType}`); } }; From 841a67443d1518f49a869cea3d230a9de6f1e8d4 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Thu, 30 May 2024 13:29:12 +0530 Subject: [PATCH 184/354] [mob][photos] Logs --- .../face_ml/face_ml_service.dart | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart index 1843b7f1b3..cffa17dcc8 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart @@ -871,7 +871,7 @@ class FaceMlService { if (filePath == null) { _logger.severe( - "Failed to get any data for enteFile with uploadedFileID ${enteFile.uploadedFileID}", + "Failed to get any data for enteFile with uploadedFileID ${enteFile.uploadedFileID} since its file path is null", ); throw CouldNotRetrieveAnyFileData(); } @@ -1018,11 +1018,20 @@ class FaceMlService { throw ThumbnailRetrievalException(e.toString(), s); } } else { - file = await getFile(enteFile, isOrigin: true); + try { + file = await getFile(enteFile, isOrigin: true); + } catch (e, s) { + _logger.severe( + "Could not get file for $enteFile", + e, + s, + ); + } // TODO: This is returning null for Pragadees for all files, so something is wrong here! } if (file == null) { - _logger.warning("Could not get file for $enteFile"); + _logger + .warning("Could not get file for $enteFile of type ${enteFile.fileType.toString()}"); imagePath = null; break; } @@ -1173,13 +1182,13 @@ class FaceMlService { /// Checks if the ente file to be analyzed actually can be analyzed: it must be uploaded and in the correct format. void _checkEnteFileForID(EnteFile enteFile) { if (_skipAnalysisEnteFile(enteFile, {})) { - _logger.warning( - '''Skipped analysis of image with enteFile, it might be the wrong format or has no uploadedFileID, or MLController doesn't allow it to run. + final String logString = + '''Skipped analysis of image with enteFile, it might be the wrong format or has no uploadedFileID, or MLController doesn't allow it to run. enteFile: ${enteFile.toString()} - ''', - ); + '''; + _logger.warning(logString); _logStatus(); - throw CouldNotRetrieveAnyFileData(); + throw GeneralFaceMlException(logString); } } From 6dc26b91242bc6ffe89503a3c54d473326077656 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Thu, 30 May 2024 13:31:26 +0530 Subject: [PATCH 185/354] [mob][photos] Bump --- mobile/pubspec.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobile/pubspec.yaml b/mobile/pubspec.yaml index fecf742b86..3db8ed18b9 100644 --- a/mobile/pubspec.yaml +++ b/mobile/pubspec.yaml @@ -12,7 +12,7 @@ description: ente photos application # Read more about iOS versioning at # https://developer.apple.com/library/archive/documentation/General/Reference/InfoPlistKeyReference/Articles/CoreFoundationKeys.html -version: 0.8.124+644 +version: 0.8.125+645 publish_to: none environment: From 3fc41aecca308cf34c9e95471278b186310325de Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 13:32:10 +0530 Subject: [PATCH 186/354] inl --- web/apps/photos/src/services/face/f-index.ts | 26 +++++++++----------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/web/apps/photos/src/services/face/f-index.ts b/web/apps/photos/src/services/face/f-index.ts index c47d8a677c..d2c5579a2f 100644 --- a/web/apps/photos/src/services/face/f-index.ts +++ b/web/apps/photos/src/services/face/f-index.ts @@ -53,7 +53,9 @@ export const indexFaces = async ( ) => { const startTime = Date.now(); - const imageBitmap = await fetchAndCreateImageBitmap(enteFile, file); + const imageBitmap = await renderableImageBlob(enteFile, file).then( + createImageBitmap, + ); let mlFile: MlFileData; try { mlFile = await indexFaces_(enteFile, imageBitmap); @@ -70,22 +72,21 @@ export const indexFaces = async ( }; /** - * Return a {@link ImageBitmap}, using {@link file} if present otherwise + * Return a "renderable" image blob, using {@link file} if present otherwise * downloading the source image corresponding to {@link enteFile} from remote. */ -const fetchAndCreateImageBitmap = async (enteFile: EnteFile, file: File) => - file ? getLocalFileImageBitmap(enteFile, file) : fetchImageBitmap(enteFile); - -const fetchImageBitmap = async (enteFile: EnteFile) => - fetchRenderableBlob(enteFile).then(createImageBitmap); +const renderableImageBlob = async (enteFile: EnteFile, file: File) => + file + ? getRenderableImage(enteFile.metadata.title, file) + : fetchRenderableBlob(enteFile); const fetchRenderableBlob = async (enteFile: EnteFile) => { - const fileType = enteFile.metadata.fileType; const fileStream = await DownloadManager.getFile(enteFile); const fileBlob = await new Response(fileStream).blob(); - if (enteFile.metadata.fileType == FILE_TYPE.IMAGE) { + const fileType = enteFile.metadata.fileType; + if (fileType == FILE_TYPE.IMAGE) { return getRenderableImage(enteFile.metadata.title, fileBlob); - } else if (enteFile.metadata.fileType == FILE_TYPE.LIVE_PHOTO) { + } else if (fileType == FILE_TYPE.LIVE_PHOTO) { const { imageFileName, imageData } = await decodeLivePhoto( enteFile.metadata.title, fileBlob, @@ -96,11 +97,6 @@ const fetchRenderableBlob = async (enteFile: EnteFile) => { } }; -const getLocalFileImageBitmap = async (enteFile: EnteFile, localFile: File) => - createImageBitmap( - await getRenderableImage(enteFile.metadata.title, localFile), - ); - const indexFaces_ = async (enteFile: EnteFile, imageBitmap: ImageBitmap) => { const fileID = enteFile.id; const { width, height } = imageBitmap; From ac8677d7b4f059220c6ec3f017919b2c9cb2aef6 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 13:35:38 +0530 Subject: [PATCH 187/354] Filter instead of marking as errors --- web/apps/photos/src/services/face/db.ts | 8 +++++--- web/apps/photos/src/services/face/f-index.ts | 1 + web/apps/photos/src/services/face/indexer.ts | 14 ++++++++++---- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/web/apps/photos/src/services/face/db.ts b/web/apps/photos/src/services/face/db.ts index bfa54e7d6a..fe5a93cdb2 100644 --- a/web/apps/photos/src/services/face/db.ts +++ b/web/apps/photos/src/services/face/db.ts @@ -198,10 +198,12 @@ export const addFileEntry = async (fileID: number) => { }; /** - * Sync entries in the face DB to match the given list of local file IDs. + * Sync entries in the face DB to align with the given list of local indexable + * file IDs. * * @param localFileIDs The IDs of all the files that the client is aware of, - * filtered to only keep the files that the user owns. + * filtered to only keep the files that the user owns and the formats that can + * be indexed by our current face indexing pipeline. * * This function syncs the state of file entries in face DB to the state of file * entries stored otherwise by the local client. @@ -212,7 +214,7 @@ export const addFileEntry = async (fileID: number) => { * - Files that are not present locally but still exist in face DB are removed * from face DB (including its face index, if any). */ -export const syncWithLocalUserOwnedFileIDs = async (localFileIDs: number[]) => { +export const syncWithLocalIndexableFileIDs = async (localFileIDs: number[]) => { const db = await faceDB(); const tx = db.transaction(["face-index", "file-status"], "readwrite"); const fdbFileIDs = await tx.objectStore("file-status").getAllKeys(); diff --git a/web/apps/photos/src/services/face/f-index.ts b/web/apps/photos/src/services/face/f-index.ts index d2c5579a2f..fa9fdea86a 100644 --- a/web/apps/photos/src/services/face/f-index.ts +++ b/web/apps/photos/src/services/face/f-index.ts @@ -93,6 +93,7 @@ const fetchRenderableBlob = async (enteFile: EnteFile) => { ); return getRenderableImage(imageFileName, new Blob([imageData])); } else { + // A layer above us should've already filtered these out. throw new Error(`Cannot index unsupported file type ${fileType}`); } }; diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index 9f3cc4a9d3..c880de2ef2 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -1,3 +1,4 @@ +import { FILE_TYPE } from "@/media/file-type"; import log from "@/next/log"; import { ComlinkWorker } from "@/next/worker/comlink-worker"; import { wait } from "@/utils/promise"; @@ -245,7 +246,7 @@ export const setIsFaceIndexingEnabled = async (enabled: boolean) => { }; export const syncLocalFiles = async (userID: number) => { - const localFilesMap = await localUserOwnedFilesByID(userID); + const localFilesMap = await localIndexableFilesByID(userID); // const localFileIDs = new Set(localFilesMap.keys()); @@ -306,12 +307,17 @@ export const syncLocalFiles = async (userID: number) => { * * @param userID Restrict the returned files to those owned by a {@link userID}. */ -const localUserOwnedFilesByID = async ( +const localIndexableFilesByID = async ( userID: number, ): Promise> => { const result = new Map(); const localFiles = await getLocalFiles(); - const personalFiles = localFiles.filter((f) => f.ownerID === userID); - personalFiles.forEach((f) => result.set(f.id, f)); + const indexableTypes = [FILE_TYPE.IMAGE, FILE_TYPE.LIVE_PHOTO]; + const indexableFiles = localFiles.filter( + (f) => + f.ownerID == userID && indexableTypes.includes(f.metadata.fileType), + ); + + indexableFiles.forEach((f) => result.set(f.id, f)); return result; }; From 4b202d2dda4d698676ea53bfc9d3182cfeebb2ec Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 13:45:23 +0530 Subject: [PATCH 188/354] r --- web/apps/photos/src/services/face/f-index.ts | 2 +- web/apps/photos/src/services/face/remote.ts | 2 +- web/apps/photos/src/services/face/types-old.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/web/apps/photos/src/services/face/f-index.ts b/web/apps/photos/src/services/face/f-index.ts index fa9fdea86a..a73e492bb9 100644 --- a/web/apps/photos/src/services/face/f-index.ts +++ b/web/apps/photos/src/services/face/f-index.ts @@ -103,7 +103,7 @@ const indexFaces_ = async (enteFile: EnteFile, imageBitmap: ImageBitmap) => { const { width, height } = imageBitmap; const imageDimensions = { width, height }; const mlFile: MlFileData = { - fileId: fileID, + fileID: fileID, mlVersion: defaultMLVersion, imageDimensions, errorCount: 0, diff --git a/web/apps/photos/src/services/face/remote.ts b/web/apps/photos/src/services/face/remote.ts index 36ef724bf3..26f57125af 100644 --- a/web/apps/photos/src/services/face/remote.ts +++ b/web/apps/photos/src/services/face/remote.ts @@ -141,7 +141,7 @@ function LocalFileMlDataToServerFileMl( } const faceEmbeddings = new ServerFaceEmbeddings(faces, userAgent, 1); return new ServerFileMl( - localFileMlData.fileId, + localFileMlData.fileID, faceEmbeddings, imageDimensions.height, imageDimensions.width, diff --git a/web/apps/photos/src/services/face/types-old.ts b/web/apps/photos/src/services/face/types-old.ts index ad6ef68743..5135881eb8 100644 --- a/web/apps/photos/src/services/face/types-old.ts +++ b/web/apps/photos/src/services/face/types-old.ts @@ -17,7 +17,7 @@ export interface Face { } export interface MlFileData { - fileId: number; + fileID: number; faces?: Face[]; imageDimensions?: Dimensions; mlVersion: number; From 6d3391528da004e4e63ae5de930ac06a1dea1e59 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 13:48:36 +0530 Subject: [PATCH 189/354] Closer --- web/apps/photos/src/services/face/f-index.ts | 3 ++- web/apps/photos/src/services/face/remote.ts | 5 ++--- web/apps/photos/src/services/face/types-old.ts | 5 +++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/web/apps/photos/src/services/face/f-index.ts b/web/apps/photos/src/services/face/f-index.ts index a73e492bb9..d64a6d1883 100644 --- a/web/apps/photos/src/services/face/f-index.ts +++ b/web/apps/photos/src/services/face/f-index.ts @@ -105,7 +105,8 @@ const indexFaces_ = async (enteFile: EnteFile, imageBitmap: ImageBitmap) => { const mlFile: MlFileData = { fileID: fileID, mlVersion: defaultMLVersion, - imageDimensions, + width, + height, errorCount: 0, }; diff --git a/web/apps/photos/src/services/face/remote.ts b/web/apps/photos/src/services/face/remote.ts index 26f57125af..f0185a44e2 100644 --- a/web/apps/photos/src/services/face/remote.ts +++ b/web/apps/photos/src/services/face/remote.ts @@ -116,7 +116,6 @@ function LocalFileMlDataToServerFileMl( if (localFileMlData.errorCount > 0) { return null; } - const imageDimensions = localFileMlData.imageDimensions; const faces: ServerFace[] = []; for (let i = 0; i < localFileMlData.faces.length; i++) { @@ -143,7 +142,7 @@ function LocalFileMlDataToServerFileMl( return new ServerFileMl( localFileMlData.fileID, faceEmbeddings, - imageDimensions.height, - imageDimensions.width, + localFileMlData.height, + localFileMlData.width, ); } diff --git a/web/apps/photos/src/services/face/types-old.ts b/web/apps/photos/src/services/face/types-old.ts index 5135881eb8..c719c2e3dd 100644 --- a/web/apps/photos/src/services/face/types-old.ts +++ b/web/apps/photos/src/services/face/types-old.ts @@ -1,4 +1,4 @@ -import type { Box, Dimensions, Point } from "./types"; +import type { Box, Point } from "./types"; export interface FaceDetection { // box and landmarks is relative to image dimentions stored at mlFileData @@ -19,7 +19,8 @@ export interface Face { export interface MlFileData { fileID: number; faces?: Face[]; - imageDimensions?: Dimensions; + width: number; + height: number; mlVersion: number; errorCount: number; } From 91be44c4c501655c5e759e21cc05fc646c17d95d Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 13:53:18 +0530 Subject: [PATCH 190/354] dup --- web/apps/photos/src/services/face/db-old.ts | 2 +- web/apps/photos/src/services/face/f-index.ts | 5 ++++- web/apps/photos/src/services/face/indexer.ts | 4 ++-- web/apps/photos/src/services/face/types-old.ts | 3 +++ .../src/services/machineLearning/machineLearningService.ts | 4 ++-- 5 files changed, 12 insertions(+), 6 deletions(-) diff --git a/web/apps/photos/src/services/face/db-old.ts b/web/apps/photos/src/services/face/db-old.ts index 7dd1e71fa3..7aabe57ac2 100644 --- a/web/apps/photos/src/services/face/db-old.ts +++ b/web/apps/photos/src/services/face/db-old.ts @@ -27,7 +27,7 @@ export interface IndexStatus { * server ML data shape here exactly. */ export interface MinimalPersistedFileData { - fileId: number; + fileID: number; mlVersion: number; errorCount: number; faces?: { personId?: number; id: string }[]; diff --git a/web/apps/photos/src/services/face/f-index.ts b/web/apps/photos/src/services/face/f-index.ts index d64a6d1883..ac7afa5d7d 100644 --- a/web/apps/photos/src/services/face/f-index.ts +++ b/web/apps/photos/src/services/face/f-index.ts @@ -104,9 +104,12 @@ const indexFaces_ = async (enteFile: EnteFile, imageBitmap: ImageBitmap) => { const imageDimensions = { width, height }; const mlFile: MlFileData = { fileID: fileID, - mlVersion: defaultMLVersion, width, height, + faceEmbedding: { + version: 1, + }, + mlVersion: defaultMLVersion, errorCount: 0, }; diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index c880de2ef2..f7f3a7ba43 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -267,9 +267,9 @@ export const syncLocalFiles = async (userID: number) => { if (newFileIds.length > 0) { log.info("newFiles: ", newFileIds.length); const newFiles = newFileIds.map( - (fileId) => + (fileID) => ({ - fileId, + fileID, mlVersion: 0, errorCount: 0, }) as MinimalPersistedFileData, diff --git a/web/apps/photos/src/services/face/types-old.ts b/web/apps/photos/src/services/face/types-old.ts index c719c2e3dd..d81ebda5b5 100644 --- a/web/apps/photos/src/services/face/types-old.ts +++ b/web/apps/photos/src/services/face/types-old.ts @@ -21,6 +21,9 @@ export interface MlFileData { faces?: Face[]; width: number; height: number; + faceEmbedding: { + version: number; + }; mlVersion: number; errorCount: number; } diff --git a/web/apps/photos/src/services/machineLearning/machineLearningService.ts b/web/apps/photos/src/services/machineLearning/machineLearningService.ts index f1de281efa..654358712e 100644 --- a/web/apps/photos/src/services/machineLearning/machineLearningService.ts +++ b/web/apps/photos/src/services/machineLearning/machineLearningService.ts @@ -80,9 +80,9 @@ class MachineLearningService { return !error && nOutOfSyncFiles > 0; } - private newMlData(fileId: number) { + private newMlData(fileID: number) { return { - fileId, + fileID, mlVersion: 0, errorCount: 0, } as MinimalPersistedFileData; From 40d35e157edecca13a946e209dd53c9c7e704af1 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 13:54:50 +0530 Subject: [PATCH 191/354] t --- web/apps/photos/src/services/face/remote.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/web/apps/photos/src/services/face/remote.ts b/web/apps/photos/src/services/face/remote.ts index f0185a44e2..791534cac0 100644 --- a/web/apps/photos/src/services/face/remote.ts +++ b/web/apps/photos/src/services/face/remote.ts @@ -36,11 +36,11 @@ class ServerFileMl { public fileID: number; public height?: number; public width?: number; - public faceEmbedding: ServerFaceEmbeddings; + public faceEmbedding: ServerFaceEmbedding; public constructor( fileID: number, - faceEmbedding: ServerFaceEmbeddings, + faceEmbedding: ServerFaceEmbedding, height?: number, width?: number, ) { @@ -51,7 +51,7 @@ class ServerFileMl { } } -class ServerFaceEmbeddings { +class ServerFaceEmbedding { public faces: ServerFace[]; public version: number; public client: string; @@ -138,10 +138,10 @@ function LocalFileMlDataToServerFileMl( ); faces.push(newFaceObject); } - const faceEmbeddings = new ServerFaceEmbeddings(faces, userAgent, 1); + const faceEmbedding = new ServerFaceEmbedding(faces, userAgent, 1); return new ServerFileMl( localFileMlData.fileID, - faceEmbeddings, + faceEmbedding, localFileMlData.height, localFileMlData.width, ); From 34166ecffb16831793e4446f4e0b3d2b43e57109 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 14:00:00 +0530 Subject: [PATCH 192/354] Move to generator --- web/apps/photos/src/services/face/f-index.ts | 13 +++++++++++-- web/apps/photos/src/services/face/indexer.worker.ts | 4 ++-- web/apps/photos/src/services/face/remote.ts | 10 ++++++---- web/apps/photos/src/services/face/types-old.ts | 1 + 4 files changed, 20 insertions(+), 8 deletions(-) diff --git a/web/apps/photos/src/services/face/f-index.ts b/web/apps/photos/src/services/face/f-index.ts index ac7afa5d7d..737a080407 100644 --- a/web/apps/photos/src/services/face/f-index.ts +++ b/web/apps/photos/src/services/face/f-index.ts @@ -46,10 +46,14 @@ import type { Face, FaceDetection, MlFileData } from "./types-old"; * @param file The contents of {@link enteFile} as a web {@link File}, if * available. These are used when they are provided, otherwise the file is * downloaded and decrypted from remote. + * + * @param userAgent The UA of the current client (the client that is generating + * the embedding). */ export const indexFaces = async ( enteFile: EnteFile, file: File | undefined, + userAgent: string, ) => { const startTime = Date.now(); @@ -58,7 +62,7 @@ export const indexFaces = async ( ); let mlFile: MlFileData; try { - mlFile = await indexFaces_(enteFile, imageBitmap); + mlFile = await indexFaces_(enteFile, imageBitmap, userAgent); } finally { imageBitmap.close(); } @@ -98,7 +102,11 @@ const fetchRenderableBlob = async (enteFile: EnteFile) => { } }; -const indexFaces_ = async (enteFile: EnteFile, imageBitmap: ImageBitmap) => { +const indexFaces_ = async ( + enteFile: EnteFile, + imageBitmap: ImageBitmap, + userAgent: string, +) => { const fileID = enteFile.id; const { width, height } = imageBitmap; const imageDimensions = { width, height }; @@ -108,6 +116,7 @@ const indexFaces_ = async (enteFile: EnteFile, imageBitmap: ImageBitmap) => { height, faceEmbedding: { version: 1, + client: userAgent, }, mlVersion: defaultMLVersion, errorCount: 0, diff --git a/web/apps/photos/src/services/face/indexer.worker.ts b/web/apps/photos/src/services/face/indexer.worker.ts index b4b9bc08c7..409dc62483 100644 --- a/web/apps/photos/src/services/face/indexer.worker.ts +++ b/web/apps/photos/src/services/face/indexer.worker.ts @@ -33,7 +33,7 @@ export class FaceIndexerWorker { let faceIndex: MlFileData; try { - faceIndex = await indexFaces(enteFile, file); + faceIndex = await indexFaces(enteFile, file, userAgent); log.debug(() => ({ f, faceIndex })); } catch (e) { // Mark indexing as having failed only if the indexing itself @@ -44,7 +44,7 @@ export class FaceIndexerWorker { throw e; } - await putFaceEmbedding(enteFile, faceIndex, userAgent); + await putFaceEmbedding(enteFile, faceIndex); return faceIndex; } diff --git a/web/apps/photos/src/services/face/remote.ts b/web/apps/photos/src/services/face/remote.ts index 791534cac0..d49a8b13e9 100644 --- a/web/apps/photos/src/services/face/remote.ts +++ b/web/apps/photos/src/services/face/remote.ts @@ -8,9 +8,8 @@ import type { Face, FaceDetection, MlFileData } from "./types-old"; export const putFaceEmbedding = async ( enteFile: EnteFile, mlFileData: MlFileData, - userAgent: string, ) => { - const serverMl = LocalFileMlDataToServerFileMl(mlFileData, userAgent); + const serverMl = LocalFileMlDataToServerFileMl(mlFileData); log.debug(() => ({ t: "Local ML file data", mlFileData })); log.debug(() => ({ t: "Uploaded ML file data", @@ -111,7 +110,6 @@ class ServerFaceBox { function LocalFileMlDataToServerFileMl( localFileMlData: MlFileData, - userAgent: string, ): ServerFileMl { if (localFileMlData.errorCount > 0) { return null; @@ -138,7 +136,11 @@ function LocalFileMlDataToServerFileMl( ); faces.push(newFaceObject); } - const faceEmbedding = new ServerFaceEmbedding(faces, userAgent, 1); + const faceEmbedding = new ServerFaceEmbedding( + faces, + localFileMlData.faceEmbedding.client, + localFileMlData.faceEmbedding.version, + ); return new ServerFileMl( localFileMlData.fileID, faceEmbedding, diff --git a/web/apps/photos/src/services/face/types-old.ts b/web/apps/photos/src/services/face/types-old.ts index d81ebda5b5..11109f19a0 100644 --- a/web/apps/photos/src/services/face/types-old.ts +++ b/web/apps/photos/src/services/face/types-old.ts @@ -23,6 +23,7 @@ export interface MlFileData { height: number; faceEmbedding: { version: number; + client: string; }; mlVersion: number; errorCount: number; From 13d15ceeb9b3a86cf63cfde1a64423464a85a036 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 14:09:59 +0530 Subject: [PATCH 193/354] nest --- web/apps/photos/src/services/face/db-old.ts | 2 +- web/apps/photos/src/services/face/f-index.ts | 21 ++++++++++++------- web/apps/photos/src/services/face/indexer.ts | 14 ++++++------- web/apps/photos/src/services/face/remote.ts | 4 ++-- .../photos/src/services/face/types-old.ts | 2 +- 5 files changed, 25 insertions(+), 18 deletions(-) diff --git a/web/apps/photos/src/services/face/db-old.ts b/web/apps/photos/src/services/face/db-old.ts index 7aabe57ac2..9528e6a8da 100644 --- a/web/apps/photos/src/services/face/db-old.ts +++ b/web/apps/photos/src/services/face/db-old.ts @@ -30,7 +30,7 @@ export interface MinimalPersistedFileData { fileID: number; mlVersion: number; errorCount: number; - faces?: { personId?: number; id: string }[]; + faceEmbedding: { faces?: { id: string }[] }; } interface Config {} diff --git a/web/apps/photos/src/services/face/f-index.ts b/web/apps/photos/src/services/face/f-index.ts index 737a080407..335bdfaf48 100644 --- a/web/apps/photos/src/services/face/f-index.ts +++ b/web/apps/photos/src/services/face/f-index.ts @@ -68,7 +68,7 @@ export const indexFaces = async ( } log.debug(() => { - const nf = mlFile.faces?.length ?? 0; + const nf = mlFile.faceEmbedding.faces?.length ?? 0; const ms = Date.now() - startTime; return `Indexed ${nf} faces in file ${logIdentifier(enteFile)} (${ms} ms)`; }); @@ -128,12 +128,12 @@ const indexFaces_ = async ( fileId: fileID, detection, })); - mlFile.faces = detectedFaces; + mlFile.faceEmbedding.faces = detectedFaces; if (detectedFaces.length > 0) { const alignments: FaceAlignment[] = []; - for (const face of mlFile.faces) { + for (const face of mlFile.faceEmbedding.faces) { const alignment = computeFaceAlignment(face.detection); alignments.push(alignment); @@ -147,13 +147,20 @@ const indexFaces_ = async ( alignments, ); - const blurValues = detectBlur(alignedFacesData, mlFile.faces); - mlFile.faces.forEach((f, i) => (f.blurValue = blurValues[i])); + const blurValues = detectBlur( + alignedFacesData, + mlFile.faceEmbedding.faces, + ); + mlFile.faceEmbedding.faces.forEach( + (f, i) => (f.blurValue = blurValues[i]), + ); const embeddings = await computeEmbeddings(alignedFacesData); - mlFile.faces.forEach((f, i) => (f.embedding = embeddings[i])); + mlFile.faceEmbedding.faces.forEach( + (f, i) => (f.embedding = embeddings[i]), + ); - mlFile.faces.forEach((face) => { + mlFile.faceEmbedding.faces.forEach((face) => { face.detection = relativeDetection(face.detection, imageDimensions); }); } diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index f7f3a7ba43..62a4e6d040 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -11,9 +11,8 @@ import machineLearningService, { import mlWorkManager from "services/machineLearning/mlWorkManager"; import type { EnteFile } from "types/file"; import { isInternalUserForML } from "utils/user"; -import { indexedAndIndexableCounts, markIndexingFailed } from "./db"; +import { indexedAndIndexableCounts } from "./db"; import type { IndexStatus, MinimalPersistedFileData } from "./db-old"; -import { indexFaces } from "./f-index"; import { FaceIndexerWorker } from "./indexer.worker"; /** @@ -96,20 +95,21 @@ class FaceIndexer { }, 30 * 1000); return; } - + /* const fileID = item.enteFile.id; try { - const faceIndex = await indexFaces(item.enteFile, item.file); + const faceIndex = await indexFaces(item.enteFile, item.file, userAgent); log.info(`faces in file ${fileID}`, faceIndex); } catch (e) { log.error(`Failed to index faces in file ${fileID}`, e); markIndexingFailed(item.enteFile.id); } - +*/ // Let the runloop drain. await wait(0); // Run again. - this.tick(); + // TODO + // this.tick(); } /** @@ -214,7 +214,7 @@ export const unidentifiedFaceIDs = async ( enteFile: EnteFile, ): Promise<{ id: string }[]> => { const mlFileData = await mlIDbStorage.getFile(enteFile.id); - return mlFileData?.faces ?? []; + return mlFileData?.faceEmbedding.faces ?? []; }; /** diff --git a/web/apps/photos/src/services/face/remote.ts b/web/apps/photos/src/services/face/remote.ts index d49a8b13e9..093ad788ab 100644 --- a/web/apps/photos/src/services/face/remote.ts +++ b/web/apps/photos/src/services/face/remote.ts @@ -116,8 +116,8 @@ function LocalFileMlDataToServerFileMl( } const faces: ServerFace[] = []; - for (let i = 0; i < localFileMlData.faces.length; i++) { - const face: Face = localFileMlData.faces[i]; + for (let i = 0; i < localFileMlData.faceEmbedding.faces.length; i++) { + const face: Face = localFileMlData.faceEmbedding.faces[i]; const faceID = face.id; const embedding = face.embedding; const score = face.detection.probability; diff --git a/web/apps/photos/src/services/face/types-old.ts b/web/apps/photos/src/services/face/types-old.ts index 11109f19a0..f9bec47b84 100644 --- a/web/apps/photos/src/services/face/types-old.ts +++ b/web/apps/photos/src/services/face/types-old.ts @@ -18,12 +18,12 @@ export interface Face { export interface MlFileData { fileID: number; - faces?: Face[]; width: number; height: number; faceEmbedding: { version: number; client: string; + faces?: Face[]; }; mlVersion: number; errorCount: number; From 2abcb709d93e9137650508151fc263249b35a2ac Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 14:11:23 +0530 Subject: [PATCH 194/354] Unused --- web/apps/photos/src/services/face/types-old.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/web/apps/photos/src/services/face/types-old.ts b/web/apps/photos/src/services/face/types-old.ts index f9bec47b84..065b65e20b 100644 --- a/web/apps/photos/src/services/face/types-old.ts +++ b/web/apps/photos/src/services/face/types-old.ts @@ -8,7 +8,6 @@ export interface FaceDetection { } export interface Face { - fileId: number; detection: FaceDetection; id: string; blurValue?: number; From b5c52a4ae2da050149cbdf4fa5945503a2c28fe0 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 14:12:50 +0530 Subject: [PATCH 195/354] id --- web/apps/photos/src/components/ml/PeopleList.tsx | 14 +++++++------- web/apps/photos/src/services/face/crop.ts | 2 +- web/apps/photos/src/services/face/db-old.ts | 2 +- web/apps/photos/src/services/face/indexer.ts | 4 ++-- web/apps/photos/src/services/face/remote.ts | 2 +- web/apps/photos/src/services/face/types-old.ts | 2 +- 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/web/apps/photos/src/components/ml/PeopleList.tsx b/web/apps/photos/src/components/ml/PeopleList.tsx index d44d100612..38e70c3a8d 100644 --- a/web/apps/photos/src/components/ml/PeopleList.tsx +++ b/web/apps/photos/src/components/ml/PeopleList.tsx @@ -73,14 +73,14 @@ export function PhotoPeopleList() { } export function UnidentifiedFaces({ file }: { file: EnteFile }) { - const [faces, setFaces] = useState<{ id: string }[]>([]); + const [faceIDs, setFaceIDs] = useState([]); useEffect(() => { let didCancel = false; (async () => { - const faces = await unidentifiedFaceIDs(file); - !didCancel && setFaces(faces); + const faceIDs = await unidentifiedFaceIDs(file); + !didCancel && setFaceIDs(faceIDs); })(); return () => { @@ -88,7 +88,7 @@ export function UnidentifiedFaces({ file }: { file: EnteFile }) { }; }, [file]); - if (faces.length == 0) return <>; + if (faceIDs.length == 0) return <>; return ( <> @@ -96,9 +96,9 @@ export function UnidentifiedFaces({ file }: { file: EnteFile }) { {t("UNIDENTIFIED_FACES")} - {faces.map((face) => ( - - + {faceIDs.map((faceID) => ( + + ))} diff --git a/web/apps/photos/src/services/face/crop.ts b/web/apps/photos/src/services/face/crop.ts index 8a09c1b8e6..6adb58d762 100644 --- a/web/apps/photos/src/services/face/crop.ts +++ b/web/apps/photos/src/services/face/crop.ts @@ -13,7 +13,7 @@ export const saveFaceCrop = async ( faceCrop.close(); const cache = await blobCache("face-crops"); - await cache.put(face.id, blob); + await cache.put(face.faceID, blob); return blob; }; diff --git a/web/apps/photos/src/services/face/db-old.ts b/web/apps/photos/src/services/face/db-old.ts index 9528e6a8da..070b1ce765 100644 --- a/web/apps/photos/src/services/face/db-old.ts +++ b/web/apps/photos/src/services/face/db-old.ts @@ -30,7 +30,7 @@ export interface MinimalPersistedFileData { fileID: number; mlVersion: number; errorCount: number; - faceEmbedding: { faces?: { id: string }[] }; + faceEmbedding: { faces: { faceID: string }[] }; } interface Config {} diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index 62a4e6d040..75212ede71 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -212,9 +212,9 @@ const convertToNewInterface = (indexStatus: IndexStatus) => { */ export const unidentifiedFaceIDs = async ( enteFile: EnteFile, -): Promise<{ id: string }[]> => { +): Promise => { const mlFileData = await mlIDbStorage.getFile(enteFile.id); - return mlFileData?.faceEmbedding.faces ?? []; + return mlFileData?.faceEmbedding.faces.map((f) => f.faceID) ?? []; }; /** diff --git a/web/apps/photos/src/services/face/remote.ts b/web/apps/photos/src/services/face/remote.ts index 093ad788ab..4ecb893999 100644 --- a/web/apps/photos/src/services/face/remote.ts +++ b/web/apps/photos/src/services/face/remote.ts @@ -118,7 +118,7 @@ function LocalFileMlDataToServerFileMl( const faces: ServerFace[] = []; for (let i = 0; i < localFileMlData.faceEmbedding.faces.length; i++) { const face: Face = localFileMlData.faceEmbedding.faces[i]; - const faceID = face.id; + const faceID = face.faceID; const embedding = face.embedding; const score = face.detection.probability; const blur = face.blurValue; diff --git a/web/apps/photos/src/services/face/types-old.ts b/web/apps/photos/src/services/face/types-old.ts index 065b65e20b..b40f0844ec 100644 --- a/web/apps/photos/src/services/face/types-old.ts +++ b/web/apps/photos/src/services/face/types-old.ts @@ -8,8 +8,8 @@ export interface FaceDetection { } export interface Face { + faceID: string; detection: FaceDetection; - id: string; blurValue?: number; embedding?: Float32Array; From 57404e1f4922964506061681bed577b332fab75c Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 14:17:31 +0530 Subject: [PATCH 196/354] id2 --- web/apps/photos/src/services/face/f-index.ts | 4 ++-- web/apps/photos/src/services/face/indexer.ts | 1 + web/apps/photos/src/services/face/types-old.ts | 2 +- .../src/services/machineLearning/machineLearningService.ts | 1 + 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/web/apps/photos/src/services/face/f-index.ts b/web/apps/photos/src/services/face/f-index.ts index 335bdfaf48..a8ab040e49 100644 --- a/web/apps/photos/src/services/face/f-index.ts +++ b/web/apps/photos/src/services/face/f-index.ts @@ -117,6 +117,7 @@ const indexFaces_ = async ( faceEmbedding: { version: 1, client: userAgent, + faces: [], }, mlVersion: defaultMLVersion, errorCount: 0, @@ -124,8 +125,7 @@ const indexFaces_ = async ( const faceDetections = await detectFaces(imageBitmap); const detectedFaces = faceDetections.map((detection) => ({ - id: makeFaceID(fileID, detection, imageDimensions), - fileId: fileID, + faceID: makeFaceID(fileID, detection, imageDimensions), detection, })); mlFile.faceEmbedding.faces = detectedFaces; diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index 75212ede71..7ea723f135 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -272,6 +272,7 @@ export const syncLocalFiles = async (userID: number) => { fileID, mlVersion: 0, errorCount: 0, + faceEmbedding: { faces: [] }, }) as MinimalPersistedFileData, ); await mlIDbStorage.putAllFiles(newFiles, tx); diff --git a/web/apps/photos/src/services/face/types-old.ts b/web/apps/photos/src/services/face/types-old.ts index b40f0844ec..354519f204 100644 --- a/web/apps/photos/src/services/face/types-old.ts +++ b/web/apps/photos/src/services/face/types-old.ts @@ -22,7 +22,7 @@ export interface MlFileData { faceEmbedding: { version: number; client: string; - faces?: Face[]; + faces: Face[]; }; mlVersion: number; errorCount: number; diff --git a/web/apps/photos/src/services/machineLearning/machineLearningService.ts b/web/apps/photos/src/services/machineLearning/machineLearningService.ts index 654358712e..fbff8fd5a4 100644 --- a/web/apps/photos/src/services/machineLearning/machineLearningService.ts +++ b/web/apps/photos/src/services/machineLearning/machineLearningService.ts @@ -85,6 +85,7 @@ class MachineLearningService { fileID, mlVersion: 0, errorCount: 0, + faceEmbedding: { faces: [] }, } as MinimalPersistedFileData; } From 6b0501e27228dab2540bfbce911ed6711b41b2fc Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 14:30:19 +0530 Subject: [PATCH 197/354] Move the score out --- web/apps/photos/src/services/face/f-index.ts | 71 ++++++++++--------- web/apps/photos/src/services/face/remote.ts | 2 +- .../photos/src/services/face/types-old.ts | 2 +- 3 files changed, 39 insertions(+), 36 deletions(-) diff --git a/web/apps/photos/src/services/face/f-index.ts b/web/apps/photos/src/services/face/f-index.ts index a8ab040e49..5f1039c2f9 100644 --- a/web/apps/photos/src/services/face/f-index.ts +++ b/web/apps/photos/src/services/face/f-index.ts @@ -22,7 +22,7 @@ import { pixelRGBBilinear, warpAffineFloat32List, } from "./image"; -import type { Box, Dimensions } from "./types"; +import type { Box, Dimensions, Point } from "./types"; import type { Face, FaceDetection, MlFileData } from "./types-old"; /** @@ -109,7 +109,7 @@ const indexFaces_ = async ( ) => { const fileID = enteFile.id; const { width, height } = imageBitmap; - const imageDimensions = { width, height }; + const imageBox = { width, height }; const mlFile: MlFileData = { fileID: fileID, width, @@ -123,11 +123,14 @@ const indexFaces_ = async ( errorCount: 0, }; - const faceDetections = await detectFaces(imageBitmap); - const detectedFaces = faceDetections.map((detection) => ({ - faceID: makeFaceID(fileID, detection, imageDimensions), - detection, - })); + const yoloFaceDetections = await detectFaces(imageBitmap); + const detectedFaces = yoloFaceDetections.map( + ({ box, landmarks, score }) => ({ + faceID: makeFaceID(fileID, box, imageBox), + detection: { box, landmarks }, + score, + }), + ); mlFile.faceEmbedding.faces = detectedFaces; if (detectedFaces.length > 0) { @@ -161,7 +164,7 @@ const indexFaces_ = async ( ); mlFile.faceEmbedding.faces.forEach((face) => { - face.detection = relativeDetection(face.detection, imageDimensions); + face.detection = relativeDetection(face.detection, imageBox); }); } @@ -175,14 +178,14 @@ const indexFaces_ = async ( */ const detectFaces = async ( imageBitmap: ImageBitmap, -): Promise => { +): Promise => { const rect = ({ width, height }) => ({ x: 0, y: 0, width, height }); const { yoloInput, yoloSize } = convertToYOLOInputFloat32ChannelsFirst(imageBitmap); const yoloOutput = await workerBridge.detectFaces(yoloInput); const faces = filterExtractDetectionsFromYOLOOutput(yoloOutput); - const faceDetections = transformFaceDetections( + const faceDetections = transformYOLOFaceDetections( faces, rect(yoloSize), rect(imageBitmap), @@ -243,6 +246,12 @@ const convertToYOLOInputFloat32ChannelsFirst = (imageBitmap: ImageBitmap) => { return { yoloInput, yoloSize }; }; +export interface YOLOFaceDetection { + box: Box; + landmarks: Point[]; + score: number; +} + /** * Extract detected faces from the YOLOv5Face's output. * @@ -261,8 +270,8 @@ const convertToYOLOInputFloat32ChannelsFirst = (imageBitmap: ImageBitmap) => { */ const filterExtractDetectionsFromYOLOOutput = ( rows: Float32Array, -): FaceDetection[] => { - const faces: FaceDetection[] = []; +): YOLOFaceDetection[] => { + const faces: YOLOFaceDetection[] = []; // Iterate over each row. for (let i = 0; i < rows.length; i += 16) { const score = rows[i + 4]; @@ -287,7 +296,6 @@ const filterExtractDetectionsFromYOLOOutput = ( const rightMouthY = rows[i + 14]; const box = { x, y, width, height }; - const probability = score as number; const landmarks = [ { x: leftEyeX, y: leftEyeY }, { x: rightEyeX, y: rightEyeY }, @@ -295,26 +303,26 @@ const filterExtractDetectionsFromYOLOOutput = ( { x: leftMouthX, y: leftMouthY }, { x: rightMouthX, y: rightMouthY }, ]; - faces.push({ box, landmarks, probability }); + faces.push({ box, landmarks, score }); } return faces; }; /** - * Transform the given {@link faceDetections} from their coordinate system in + * Transform the given {@link yoloFaceDetections} from their coordinate system in * which they were detected ({@link inBox}) back to the coordinate system of the * original image ({@link toBox}). */ -const transformFaceDetections = ( - faceDetections: FaceDetection[], +const transformYOLOFaceDetections = ( + yoloFaceDetections: YOLOFaceDetection[], inBox: Box, toBox: Box, -): FaceDetection[] => { +): YOLOFaceDetection[] => { const transform = boxTransformationMatrix(inBox, toBox); - return faceDetections.map((f) => ({ + return yoloFaceDetections.map((f) => ({ box: transformBox(f.box, transform), landmarks: f.landmarks.map((p) => applyToPoint(transform, p)), - probability: f.probability, + score: f.score, })); }; @@ -346,8 +354,8 @@ const transformBox = (box: Box, transform: TransformationMatrix): Box => { * Remove overlapping faces from an array of face detections through non-maximum * suppression algorithm. * - * This function sorts the detections by their probability in descending order, - * then iterates over them. + * This function sorts the detections by their score in descending order, then + * iterates over them. * * For each detection, it calculates the Intersection over Union (IoU) with all * other detections. @@ -356,8 +364,8 @@ const transformBox = (box: Box, transform: TransformationMatrix): Box => { * (`iouThreshold`), the other detection is considered overlapping and is * removed. * - * @param detections - An array of face detections to remove overlapping faces - * from. + * @param detections - An array of YOLO face detections to remove overlapping + * faces from. * * @param iouThreshold - The minimum IoU between two detections for them to be * considered overlapping. @@ -365,11 +373,11 @@ const transformBox = (box: Box, transform: TransformationMatrix): Box => { * @returns An array of face detections with overlapping faces removed */ const naiveNonMaxSuppression = ( - detections: FaceDetection[], + detections: YOLOFaceDetection[], iouThreshold: number, -): FaceDetection[] => { +): YOLOFaceDetection[] => { // Sort the detections by score, the highest first. - detections.sort((a, b) => b.probability - a.probability); + detections.sort((a, b) => b.score - a.score); // Loop through the detections and calculate the IOU. for (let i = 0; i < detections.length - 1; i++) { @@ -413,11 +421,7 @@ const intersectionOverUnion = (a: FaceDetection, b: FaceDetection): number => { return intersectionArea / unionArea; }; -const makeFaceID = ( - fileID: number, - { box }: FaceDetection, - image: Dimensions, -) => { +const makeFaceID = (fileID: number, box: Box, image: Dimensions) => { const part = (v: number) => clamp(v, 0.0, 0.999999).toFixed(5).substring(2); const xMin = part(box.x / image.width); const yMin = part(box.y / image.height); @@ -760,6 +764,5 @@ const relativeDetection = ( x: l.x / width, y: l.y / height, })); - const probability = faceDetection.probability; - return { box, landmarks, probability }; + return { box, landmarks }; }; diff --git a/web/apps/photos/src/services/face/remote.ts b/web/apps/photos/src/services/face/remote.ts index 4ecb893999..904eb8a50e 100644 --- a/web/apps/photos/src/services/face/remote.ts +++ b/web/apps/photos/src/services/face/remote.ts @@ -120,7 +120,7 @@ function LocalFileMlDataToServerFileMl( const face: Face = localFileMlData.faceEmbedding.faces[i]; const faceID = face.faceID; const embedding = face.embedding; - const score = face.detection.probability; + const score = face.score; const blur = face.blurValue; const detection: FaceDetection = face.detection; const box = detection.box; diff --git a/web/apps/photos/src/services/face/types-old.ts b/web/apps/photos/src/services/face/types-old.ts index 354519f204..079e7285f4 100644 --- a/web/apps/photos/src/services/face/types-old.ts +++ b/web/apps/photos/src/services/face/types-old.ts @@ -4,12 +4,12 @@ export interface FaceDetection { // box and landmarks is relative to image dimentions stored at mlFileData box: Box; landmarks?: Point[]; - probability?: number; } export interface Face { faceID: string; detection: FaceDetection; + score: number; blurValue?: number; embedding?: Float32Array; From b9a07e536ca5eb8898dd65aa389f44dd52e96dd4 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 14:32:52 +0530 Subject: [PATCH 198/354] blur --- web/apps/photos/src/services/face/f-index.ts | 12 ++++-------- web/apps/photos/src/services/face/remote.ts | 2 +- web/apps/photos/src/services/face/types-old.ts | 2 +- 3 files changed, 6 insertions(+), 10 deletions(-) diff --git a/web/apps/photos/src/services/face/f-index.ts b/web/apps/photos/src/services/face/f-index.ts index 5f1039c2f9..f219cc7f4e 100644 --- a/web/apps/photos/src/services/face/f-index.ts +++ b/web/apps/photos/src/services/face/f-index.ts @@ -111,7 +111,7 @@ const indexFaces_ = async ( const { width, height } = imageBitmap; const imageBox = { width, height }; const mlFile: MlFileData = { - fileID: fileID, + fileID, width, height, faceEmbedding: { @@ -129,6 +129,7 @@ const indexFaces_ = async ( faceID: makeFaceID(fileID, box, imageBox), detection: { box, landmarks }, score, + blur: 0, }), ); mlFile.faceEmbedding.faces = detectedFaces; @@ -150,13 +151,8 @@ const indexFaces_ = async ( alignments, ); - const blurValues = detectBlur( - alignedFacesData, - mlFile.faceEmbedding.faces, - ); - mlFile.faceEmbedding.faces.forEach( - (f, i) => (f.blurValue = blurValues[i]), - ); + const blurs = detectBlur(alignedFacesData, mlFile.faceEmbedding.faces); + mlFile.faceEmbedding.faces.forEach((f, i) => (f.blur = blurs[i])); const embeddings = await computeEmbeddings(alignedFacesData); mlFile.faceEmbedding.faces.forEach( diff --git a/web/apps/photos/src/services/face/remote.ts b/web/apps/photos/src/services/face/remote.ts index 904eb8a50e..c6b3c84491 100644 --- a/web/apps/photos/src/services/face/remote.ts +++ b/web/apps/photos/src/services/face/remote.ts @@ -121,7 +121,7 @@ function LocalFileMlDataToServerFileMl( const faceID = face.faceID; const embedding = face.embedding; const score = face.score; - const blur = face.blurValue; + const blur = face.blur; const detection: FaceDetection = face.detection; const box = detection.box; const landmarks = detection.landmarks; diff --git a/web/apps/photos/src/services/face/types-old.ts b/web/apps/photos/src/services/face/types-old.ts index 079e7285f4..086fd6dda8 100644 --- a/web/apps/photos/src/services/face/types-old.ts +++ b/web/apps/photos/src/services/face/types-old.ts @@ -10,7 +10,7 @@ export interface Face { faceID: string; detection: FaceDetection; score: number; - blurValue?: number; + blur: number; embedding?: Float32Array; } From 69c18cb852b23f0c0b392cdffc47101110f67065 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Thu, 30 May 2024 14:48:25 +0530 Subject: [PATCH 199/354] Update README --- CONTRIBUTING.md | 10 +++++++--- README.md | 3 +-- auth/README.md | 8 ++------ 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 857e18fb54..e5cf3e8be3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -12,7 +12,7 @@ There are many ways to contribute, and most of them don't require writing code. ## Spread the word -This is perhaps the most impactful contribution you can make. Spread the word. +This is perhaps the most impactful contribution you can make. [Spread the word.](https://help.ente.io/photos/features/referral-program/) Online on your favorite social media channels. Offline to your friends and family who are looking for a privacy-friendly alternative to big tech. @@ -76,7 +76,11 @@ us](https://github.com/ente-io/ente/discussions). Discussing your idea with us first ensures that everyone is on the same page before you start working on your change. -## Star +## Leave a review or star If you haven't already done so, consider [starring this -repository](https://github.com/ente-io/ente/stargazers). +repository](https://github.com/ente-io/ente/stargazers) or leaving +a review on +[PlayStore](https://play.google.com/store/apps/details?id=io.ente.auth), +[AppStore](https://apps.apple.com/us/app/ente-authenticator/id6444121398) or +[AlternativeTo](https://alternativeto.net/software/ente-authenticator/). diff --git a/README.md b/README.md index 88488d11ca..7c270e70c3 100644 --- a/README.md +++ b/README.md @@ -60,8 +60,7 @@ Our labour of love. Two years ago, while building Ente Photos, we realized that there was no open source end-to-end encrypted authenticator app. We already had the building blocks, so we built one. -Ente Auth is currently free. If in the future we convert this to a paid service, -existing users will be grandfathered in. +Ente Auth is free, and will remain free forever. If you like the service and want to give back, please check out Ente Photos or spread the word.
diff --git a/auth/README.md b/auth/README.md index e2e03f0230..5a3f9a9e3c 100644 --- a/auth/README.md +++ b/auth/README.md @@ -95,13 +95,9 @@ more, see [docs/adding-icons](docs/adding-icons.md). ## 💚 Contribute -For more ways to contribute, see [../CONTRIBUTING.md](../CONTRIBUTING.md). +The best way to support this project is by checking out [Ente Photos](../mobile/README.md) or spreading the word. -You can also support us by giving this project a ⭐ star on GitHub or by leaving -a review on -[PlayStore](https://play.google.com/store/apps/details?id=io.ente.auth), -[AppStore](https://apps.apple.com/us/app/ente-authenticator/id6444121398) or -[AlternativeTo](https://alternativeto.net/software/ente-authenticator/). +For more ways to contribute, see [../CONTRIBUTING.md](../CONTRIBUTING.md). ## ⭐️ About From b2277cfcc2de5cacdbbe0866de360f6d5dd24841 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 14:53:28 +0530 Subject: [PATCH 200/354] Update README --- CONTRIBUTING.md | 10 +++++----- README.md | 3 ++- auth/README.md | 3 ++- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e5cf3e8be3..e92fe9c6cf 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -12,9 +12,10 @@ There are many ways to contribute, and most of them don't require writing code. ## Spread the word -This is perhaps the most impactful contribution you can make. [Spread the word.](https://help.ente.io/photos/features/referral-program/) -Online on your favorite social media channels. Offline to your friends and -family who are looking for a privacy-friendly alternative to big tech. +This is perhaps the most impactful contribution you can make. [Spread the +word](https://help.ente.io/photos/features/referral-program/). Online on your +favorite social media channels. Offline to your friends and family who are +looking for a privacy-friendly alternative to big tech. ## Engage with the community @@ -79,8 +80,7 @@ change. ## Leave a review or star If you haven't already done so, consider [starring this -repository](https://github.com/ente-io/ente/stargazers) or leaving -a review on +repository](https://github.com/ente-io/ente/stargazers) or leaving a review on [PlayStore](https://play.google.com/store/apps/details?id=io.ente.auth), [AppStore](https://apps.apple.com/us/app/ente-authenticator/id6444121398) or [AlternativeTo](https://alternativeto.net/software/ente-authenticator/). diff --git a/README.md b/README.md index 7c270e70c3..00a786b96e 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,8 @@ Our labour of love. Two years ago, while building Ente Photos, we realized that there was no open source end-to-end encrypted authenticator app. We already had the building blocks, so we built one. -Ente Auth is free, and will remain free forever. If you like the service and want to give back, please check out Ente Photos or spread the word. +Ente Auth is free, and will remain free forever. If you like the service and +want to give back, please check out Ente Photos or spread the word.
diff --git a/auth/README.md b/auth/README.md index 5a3f9a9e3c..6382812de2 100644 --- a/auth/README.md +++ b/auth/README.md @@ -95,7 +95,8 @@ more, see [docs/adding-icons](docs/adding-icons.md). ## 💚 Contribute -The best way to support this project is by checking out [Ente Photos](../mobile/README.md) or spreading the word. +The best way to support this project is by checking out [Ente +Photos](../mobile/README.md) or spreading the word. For more ways to contribute, see [../CONTRIBUTING.md](../CONTRIBUTING.md). From 1a292aae2784e15ab9ff74b8e7cb70afaf9d5fe4 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 15:16:44 +0530 Subject: [PATCH 201/354] Split --- web/apps/photos/src/services/face/crop.ts | 5 +- web/apps/photos/src/services/face/f-index.ts | 108 +++++++++++-------- 2 files changed, 68 insertions(+), 45 deletions(-) diff --git a/web/apps/photos/src/services/face/crop.ts b/web/apps/photos/src/services/face/crop.ts index 6adb58d762..ff1d6026af 100644 --- a/web/apps/photos/src/services/face/crop.ts +++ b/web/apps/photos/src/services/face/crop.ts @@ -1,11 +1,10 @@ import { blobCache } from "@/next/blob-cache"; import type { FaceAlignment } from "./f-index"; import type { Box } from "./types"; -import type { Face } from "./types-old"; export const saveFaceCrop = async ( imageBitmap: ImageBitmap, - face: Face, + faceID: string, alignment: FaceAlignment, ) => { const faceCrop = extractFaceCrop(imageBitmap, alignment); @@ -13,7 +12,7 @@ export const saveFaceCrop = async ( faceCrop.close(); const cache = await blobCache("face-crops"); - await cache.put(face.faceID, blob); + await cache.put(faceID, blob); return blob; }; diff --git a/web/apps/photos/src/services/face/f-index.ts b/web/apps/photos/src/services/face/f-index.ts index f219cc7f4e..450e72841c 100644 --- a/web/apps/photos/src/services/face/f-index.ts +++ b/web/apps/photos/src/services/face/f-index.ts @@ -23,7 +23,7 @@ import { warpAffineFloat32List, } from "./image"; import type { Box, Dimensions, Point } from "./types"; -import type { Face, FaceDetection, MlFileData } from "./types-old"; +import type { MlFileData } from "./types-old"; /** * Index faces in the given file. @@ -107,64 +107,81 @@ const indexFaces_ = async ( imageBitmap: ImageBitmap, userAgent: string, ) => { - const fileID = enteFile.id; const { width, height } = imageBitmap; - const imageBox = { width, height }; - const mlFile: MlFileData = { + const fileID = enteFile.id; + return { fileID, width, height, faceEmbedding: { version: 1, client: userAgent, - faces: [], + faces: await indexFacesInBitmap(fileID, imageBitmap), }, mlVersion: defaultMLVersion, errorCount: 0, }; +}; + +const indexFacesInBitmap = async ( + fileID: number, + imageBitmap: ImageBitmap, +): Promise => { + const { width, height } = imageBitmap; + const imageBox = { width, height }; const yoloFaceDetections = await detectFaces(imageBitmap); - const detectedFaces = yoloFaceDetections.map( - ({ box, landmarks, score }) => ({ - faceID: makeFaceID(fileID, box, imageBox), - detection: { box, landmarks }, - score, - blur: 0, - }), + const partialResult = yoloFaceDetections.map( + ({ box, landmarks, score }) => { + const faceID = makeFaceID(fileID, box, imageBox); + const detection = { box, landmarks }; + return { faceID, detection, score }; + }, ); - mlFile.faceEmbedding.faces = detectedFaces; - if (detectedFaces.length > 0) { - const alignments: FaceAlignment[] = []; + const alignments: FaceAlignment[] = []; - for (const face of mlFile.faceEmbedding.faces) { - const alignment = computeFaceAlignment(face.detection); - alignments.push(alignment); + for (const { faceID, detection } of partialResult) { + const alignment = computeFaceAlignment(detection); + alignments.push(alignment); - // This step is not really part of the indexing pipeline, we just do - // it here since we have already computed the face alignment. - await saveFaceCrop(imageBitmap, face, alignment); + // This step is not really part of the indexing pipeline, we just do + // it here since we have already computed the face alignment. Ignore + // errors that happen during this though. + try { + await saveFaceCrop(imageBitmap, faceID, alignment); + } catch (e) { + log.error(`Failed to save face crop for faceID ${faceID}`, e); } + } - const alignedFacesData = convertToMobileFaceNetInput( - imageBitmap, - alignments, - ); + const alignedFacesData = convertToMobileFaceNetInput( + imageBitmap, + alignments, + ); - const blurs = detectBlur(alignedFacesData, mlFile.faceEmbedding.faces); - mlFile.faceEmbedding.faces.forEach((f, i) => (f.blur = blurs[i])); + const embeddings = await computeEmbeddings(alignedFacesData); + const blurs = detectBlur( + alignedFacesData, + partialResult.map((f) => f.detection), + ); - const embeddings = await computeEmbeddings(alignedFacesData); - mlFile.faceEmbedding.faces.forEach( - (f, i) => (f.embedding = embeddings[i]), - ); + const faces = []; - mlFile.faceEmbedding.faces.forEach((face) => { - face.detection = relativeDetection(face.detection, imageBox); + for (let i = 0; i < partialResult.length; i++) { + const { faceID, detection, score } = partialResult[i]; + const blur = blurs[i]; + const embedding = embeddings[i]; + faces.push({ + faceID, + detection: relativeDetection(detection, imageBox), + score, + blur, + embedding, }); } - return mlFile; + return faces; }; /** @@ -534,28 +551,35 @@ const convertToMobileFaceNetInput = ( return faceData; }; +interface FaceDetection { + box: Box; + landmarks: Point[]; +} + /** * Laplacian blur detection. * - * Return an array of detected blur values, one for each face in {@link faces}. - * The face data is taken from the slice of {@link alignedFacesData} - * corresponding to each face of {@link faces}. + * Return an array of detected blur values, one for each face detection in + * {@link faceDetections}. The face data is taken from the slice of + * {@link alignedFacesData} corresponding to the face of {@link faceDetections}. */ -const detectBlur = (alignedFacesData: Float32Array, faces: Face[]): number[] => - faces.map((face, i) => { +const detectBlur = ( + alignedFacesData: Float32Array, + faceDetections: FaceDetection[], +): number[] => + faceDetections.map((d, i) => { const faceImage = grayscaleIntMatrixFromNormalized2List( alignedFacesData, i, mobileFaceNetFaceSize, mobileFaceNetFaceSize, ); - return matrixVariance(applyLaplacian(faceImage, faceDirection(face))); + return matrixVariance(applyLaplacian(faceImage, faceDirection(d))); }); type FaceDirection = "left" | "right" | "straight"; -const faceDirection = (face: Face): FaceDirection => { - const landmarks = face.detection.landmarks; +const faceDirection = ({ landmarks }: FaceDetection): FaceDirection => { const leftEye = landmarks[0]; const rightEye = landmarks[1]; const nose = landmarks[2]; From a6a0a24b26651f91b88aec0bc37d4f2e95666c25 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 15:23:31 +0530 Subject: [PATCH 202/354] Refer --- web/apps/photos/src/services/face/db-old.ts | 5 +++- .../photos/src/services/face/types-old.ts | 29 ++----------------- 2 files changed, 7 insertions(+), 27 deletions(-) diff --git a/web/apps/photos/src/services/face/db-old.ts b/web/apps/photos/src/services/face/db-old.ts index 070b1ce765..b66c72d811 100644 --- a/web/apps/photos/src/services/face/db-old.ts +++ b/web/apps/photos/src/services/face/db-old.ts @@ -119,7 +119,10 @@ class MLIDbStorage { if (oldVersion < 1) { const filesStore = db.createObjectStore("files", { - keyPath: "fileId", + // keyPath: "fileId", TODO(MR): Changing this, since + // we're going to be deleting this DB before this PR is + // merged. + keyPath: "fileID", }); filesStore.createIndex("mlVersion", [ "mlVersion", diff --git a/web/apps/photos/src/services/face/types-old.ts b/web/apps/photos/src/services/face/types-old.ts index 086fd6dda8..d2bf61a4f8 100644 --- a/web/apps/photos/src/services/face/types-old.ts +++ b/web/apps/photos/src/services/face/types-old.ts @@ -1,29 +1,6 @@ -import type { Box, Point } from "./types"; +import type { FaceIndex } from "./types"; -export interface FaceDetection { - // box and landmarks is relative to image dimentions stored at mlFileData - box: Box; - landmarks?: Point[]; -} - -export interface Face { - faceID: string; - detection: FaceDetection; - score: number; - blur: number; - - embedding?: Float32Array; -} - -export interface MlFileData { - fileID: number; - width: number; - height: number; - faceEmbedding: { - version: number; - client: string; - faces: Face[]; - }; +export type MlFileData = FaceIndex & { mlVersion: number; errorCount: number; -} +}; From 2b3b84de0f3dbce303a64fd927854153708dfa93 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 15:31:07 +0530 Subject: [PATCH 203/354] Closer --- web/apps/photos/src/services/face/f-index.ts | 27 +++++++------------- web/apps/photos/src/services/face/remote.ts | 22 ++++++++-------- 2 files changed, 20 insertions(+), 29 deletions(-) diff --git a/web/apps/photos/src/services/face/f-index.ts b/web/apps/photos/src/services/face/f-index.ts index 450e72841c..690f383b22 100644 --- a/web/apps/photos/src/services/face/f-index.ts +++ b/web/apps/photos/src/services/face/f-index.ts @@ -22,7 +22,7 @@ import { pixelRGBBilinear, warpAffineFloat32List, } from "./image"; -import type { Box, Dimensions, Point } from "./types"; +import type { Box, Dimensions, Face, Point } from "./types"; import type { MlFileData } from "./types-old"; /** @@ -126,7 +126,7 @@ const indexFaces_ = async ( const indexFacesInBitmap = async ( fileID: number, imageBitmap: ImageBitmap, -): Promise => { +): Promise => { const { width, height } = imageBitmap; const imageBox = { width, height }; @@ -166,22 +166,13 @@ const indexFacesInBitmap = async ( partialResult.map((f) => f.detection), ); - const faces = []; - - for (let i = 0; i < partialResult.length; i++) { - const { faceID, detection, score } = partialResult[i]; - const blur = blurs[i]; - const embedding = embeddings[i]; - faces.push({ - faceID, - detection: relativeDetection(detection, imageBox), - score, - blur, - embedding, - }); - } - - return faces; + return partialResult.map(({ faceID, detection, score }, i) => ({ + faceID, + detection: relativeDetection(detection, imageBox), + score, + blur: blurs[i], + embedding: Array.from(embeddings[i]), + })); }; /** diff --git a/web/apps/photos/src/services/face/remote.ts b/web/apps/photos/src/services/face/remote.ts index c6b3c84491..b1ce10366d 100644 --- a/web/apps/photos/src/services/face/remote.ts +++ b/web/apps/photos/src/services/face/remote.ts @@ -3,7 +3,7 @@ import ComlinkCryptoWorker from "@ente/shared/crypto"; import { putEmbedding } from "services/embeddingService"; import type { EnteFile } from "types/file"; import type { Point } from "./types"; -import type { Face, FaceDetection, MlFileData } from "./types-old"; +import type { MlFileData } from "./types-old"; export const putFaceEmbedding = async ( enteFile: EnteFile, @@ -117,24 +117,24 @@ function LocalFileMlDataToServerFileMl( const faces: ServerFace[] = []; for (let i = 0; i < localFileMlData.faceEmbedding.faces.length; i++) { - const face: Face = localFileMlData.faceEmbedding.faces[i]; + const face = localFileMlData.faceEmbedding.faces[i]; const faceID = face.faceID; const embedding = face.embedding; const score = face.score; const blur = face.blur; - const detection: FaceDetection = face.detection; + const detection = face.detection; const box = detection.box; const landmarks = detection.landmarks; - const newBox = new ServerFaceBox(box.x, box.y, box.width, box.height); - const newFaceObject = new ServerFace( - faceID, - Array.from(embedding), - new ServerDetection(newBox, landmarks), - score, - blur, + faces.push( + new ServerFace( + faceID, + Array.from(embedding), + new ServerDetection(box, landmarks), + score, + blur, + ), ); - faces.push(newFaceObject); } const faceEmbedding = new ServerFaceEmbedding( faces, From 9dbec2729cb0ddac0e381f80ea8febea4ff0e9a5 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 15:44:25 +0530 Subject: [PATCH 204/354] remote mapping --- web/apps/photos/src/services/face/f-index.ts | 22 +-- .../src/services/face/indexer.worker.ts | 23 +-- web/apps/photos/src/services/face/remote.ts | 137 +----------------- web/apps/photos/src/utils/file/index.ts | 4 +- 4 files changed, 32 insertions(+), 154 deletions(-) diff --git a/web/apps/photos/src/services/face/f-index.ts b/web/apps/photos/src/services/face/f-index.ts index 690f383b22..feb48e6f1f 100644 --- a/web/apps/photos/src/services/face/f-index.ts +++ b/web/apps/photos/src/services/face/f-index.ts @@ -14,7 +14,7 @@ import { translate, } from "transformation-matrix"; import type { EnteFile } from "types/file"; -import { getRenderableImage, logIdentifier } from "utils/file"; +import { fileLogID, getRenderableImage } from "utils/file"; import { saveFaceCrop } from "./crop"; import { clamp, @@ -22,8 +22,7 @@ import { pixelRGBBilinear, warpAffineFloat32List, } from "./image"; -import type { Box, Dimensions, Face, Point } from "./types"; -import type { MlFileData } from "./types-old"; +import type { Box, Dimensions, Face, FaceIndex, Point } from "./types"; /** * Index faces in the given file. @@ -60,19 +59,20 @@ export const indexFaces = async ( const imageBitmap = await renderableImageBlob(enteFile, file).then( createImageBitmap, ); - let mlFile: MlFileData; + + let index: FaceIndex; try { - mlFile = await indexFaces_(enteFile, imageBitmap, userAgent); + index = await indexFaces_(enteFile, imageBitmap, userAgent); } finally { imageBitmap.close(); } log.debug(() => { - const nf = mlFile.faceEmbedding.faces?.length ?? 0; + const nf = index.faceEmbedding.faces.length; const ms = Date.now() - startTime; - return `Indexed ${nf} faces in file ${logIdentifier(enteFile)} (${ms} ms)`; + return `Indexed ${nf} faces in ${fileLogID(enteFile)} (${ms} ms)`; }); - return mlFile; + return index; }; /** @@ -145,9 +145,9 @@ const indexFacesInBitmap = async ( const alignment = computeFaceAlignment(detection); alignments.push(alignment); - // This step is not really part of the indexing pipeline, we just do - // it here since we have already computed the face alignment. Ignore - // errors that happen during this though. + // This step is not part of the indexing pipeline, we just do it here + // since we have already computed the face alignment. Ignore errors that + // happen during this since it does not impact the generated face index. try { await saveFaceCrop(imageBitmap, faceID, alignment); } catch (e) { diff --git a/web/apps/photos/src/services/face/indexer.worker.ts b/web/apps/photos/src/services/face/indexer.worker.ts index 409dc62483..0dccd61780 100644 --- a/web/apps/photos/src/services/face/indexer.worker.ts +++ b/web/apps/photos/src/services/face/indexer.worker.ts @@ -1,10 +1,14 @@ import log from "@/next/log"; -import { putFaceEmbedding } from "services/face/remote"; import type { EnteFile } from "types/file"; -import { logIdentifier } from "utils/file"; -import { closeFaceDBConnectionsIfNeeded, markIndexingFailed } from "./db"; +import { fileLogID } from "utils/file"; +import { + closeFaceDBConnectionsIfNeeded, + markIndexingFailed, + saveFaceIndex, +} from "./db"; import { indexFaces } from "./f-index"; -import type { MlFileData } from "./types-old"; +import { putFaceIndex } from "./remote"; +import type { FaceIndex } from "./types"; /** * Index faces in a file, save the persist the results locally, and put them on @@ -29,22 +33,21 @@ export class FaceIndexerWorker { * downloaded and decrypted from remote. */ async index(enteFile: EnteFile, file: File | undefined, userAgent: string) { - const f = logIdentifier(enteFile); - - let faceIndex: MlFileData; + let faceIndex: FaceIndex; try { faceIndex = await indexFaces(enteFile, file, userAgent); - log.debug(() => ({ f, faceIndex })); } catch (e) { // Mark indexing as having failed only if the indexing itself // failed, not if there were subsequent failures (like when trying // to put the result to remote or save it to the local face DB). - log.error(`Failed to index faces in file ${f}`, e); + log.error(`Failed to index faces in ${fileLogID(enteFile)}`, e); markIndexingFailed(enteFile.id); throw e; } - await putFaceEmbedding(enteFile, faceIndex); + await putFaceIndex(enteFile, faceIndex); + await saveFaceIndex(faceIndex); + return faceIndex; } diff --git a/web/apps/photos/src/services/face/remote.ts b/web/apps/photos/src/services/face/remote.ts index b1ce10366d..2fd50024da 100644 --- a/web/apps/photos/src/services/face/remote.ts +++ b/web/apps/photos/src/services/face/remote.ts @@ -2,23 +2,20 @@ import log from "@/next/log"; import ComlinkCryptoWorker from "@ente/shared/crypto"; import { putEmbedding } from "services/embeddingService"; import type { EnteFile } from "types/file"; -import type { Point } from "./types"; -import type { MlFileData } from "./types-old"; +import type { FaceIndex } from "./types"; -export const putFaceEmbedding = async ( +export const putFaceIndex = async ( enteFile: EnteFile, - mlFileData: MlFileData, + faceIndex: FaceIndex, ) => { - const serverMl = LocalFileMlDataToServerFileMl(mlFileData); - log.debug(() => ({ t: "Local ML file data", mlFileData })); log.debug(() => ({ - t: "Uploaded ML file data", - d: JSON.stringify(serverMl), + t: "Uploading faceEmbedding", + d: JSON.stringify(faceIndex), })); const comlinkCryptoWorker = await ComlinkCryptoWorker.getInstance(); const { file: encryptedEmbeddingData } = - await comlinkCryptoWorker.encryptMetadata(serverMl, enteFile.key); + await comlinkCryptoWorker.encryptMetadata(faceIndex, enteFile.key); await putEmbedding({ fileID: enteFile.id, encryptedEmbedding: encryptedEmbeddingData.encryptedData, @@ -26,125 +23,3 @@ export const putFaceEmbedding = async ( model: "file-ml-clip-face", }); }; - -export interface FileML extends ServerFileMl { - updatedAt: number; -} - -class ServerFileMl { - public fileID: number; - public height?: number; - public width?: number; - public faceEmbedding: ServerFaceEmbedding; - - public constructor( - fileID: number, - faceEmbedding: ServerFaceEmbedding, - height?: number, - width?: number, - ) { - this.fileID = fileID; - this.height = height; - this.width = width; - this.faceEmbedding = faceEmbedding; - } -} - -class ServerFaceEmbedding { - public faces: ServerFace[]; - public version: number; - public client: string; - - public constructor(faces: ServerFace[], client: string, version: number) { - this.faces = faces; - this.client = client; - this.version = version; - } -} - -class ServerFace { - public faceID: string; - public embedding: number[]; - public detection: ServerDetection; - public score: number; - public blur: number; - - public constructor( - faceID: string, - embedding: number[], - detection: ServerDetection, - score: number, - blur: number, - ) { - this.faceID = faceID; - this.embedding = embedding; - this.detection = detection; - this.score = score; - this.blur = blur; - } -} - -class ServerDetection { - public box: ServerFaceBox; - public landmarks: Point[]; - - public constructor(box: ServerFaceBox, landmarks: Point[]) { - this.box = box; - this.landmarks = landmarks; - } -} - -class ServerFaceBox { - public x: number; - public y: number; - public width: number; - public height: number; - - public constructor(x: number, y: number, width: number, height: number) { - this.x = x; - this.y = y; - this.width = width; - this.height = height; - } -} - -function LocalFileMlDataToServerFileMl( - localFileMlData: MlFileData, -): ServerFileMl { - if (localFileMlData.errorCount > 0) { - return null; - } - - const faces: ServerFace[] = []; - for (let i = 0; i < localFileMlData.faceEmbedding.faces.length; i++) { - const face = localFileMlData.faceEmbedding.faces[i]; - const faceID = face.faceID; - const embedding = face.embedding; - const score = face.score; - const blur = face.blur; - const detection = face.detection; - const box = detection.box; - const landmarks = detection.landmarks; - - faces.push( - new ServerFace( - faceID, - Array.from(embedding), - new ServerDetection(box, landmarks), - score, - blur, - ), - ); - } - const faceEmbedding = new ServerFaceEmbedding( - faces, - localFileMlData.faceEmbedding.client, - localFileMlData.faceEmbedding.version, - ); - return new ServerFileMl( - localFileMlData.fileID, - faceEmbedding, - localFileMlData.height, - localFileMlData.width, - ); -} diff --git a/web/apps/photos/src/utils/file/index.ts b/web/apps/photos/src/utils/file/index.ts index c15cca63c0..2879bdd757 100644 --- a/web/apps/photos/src/utils/file/index.ts +++ b/web/apps/photos/src/utils/file/index.ts @@ -86,8 +86,8 @@ const moduleState = new ModuleState(); * given {@link enteFile}. The returned string contains the file name (for ease * of debugging) and the file ID (for exactness). */ -export const logIdentifier = (enteFile: EnteFile) => - `${enteFile.metadata.title ?? "-"} (${enteFile.id})`; +export const fileLogID = (enteFile: EnteFile) => + `file ${enteFile.metadata.title ?? "-"} (${enteFile.id})`; export async function getUpdatedEXIFFileForDownload( fileReader: FileReader, From 6327a7f9da749c6b8979b8a02ee89927d7c38988 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 15:54:01 +0530 Subject: [PATCH 205/354] nu --- web/apps/photos/src/services/face/f-index.ts | 50 ++++++------------- .../src/services/face/indexer.worker.ts | 20 ++++++-- .../machineLearning/machineLearningService.ts | 29 +---------- 3 files changed, 34 insertions(+), 65 deletions(-) diff --git a/web/apps/photos/src/services/face/f-index.ts b/web/apps/photos/src/services/face/f-index.ts index feb48e6f1f..4bf829ff6f 100644 --- a/web/apps/photos/src/services/face/f-index.ts +++ b/web/apps/photos/src/services/face/f-index.ts @@ -14,7 +14,7 @@ import { translate, } from "transformation-matrix"; import type { EnteFile } from "types/file"; -import { fileLogID, getRenderableImage } from "utils/file"; +import { getRenderableImage } from "utils/file"; import { saveFaceCrop } from "./crop"; import { clamp, @@ -22,7 +22,7 @@ import { pixelRGBBilinear, warpAffineFloat32List, } from "./image"; -import type { Box, Dimensions, Face, FaceIndex, Point } from "./types"; +import type { Box, Dimensions, Face, Point } from "./types"; /** * Index faces in the given file. @@ -54,25 +54,28 @@ export const indexFaces = async ( file: File | undefined, userAgent: string, ) => { - const startTime = Date.now(); - const imageBitmap = await renderableImageBlob(enteFile, file).then( createImageBitmap, ); + const { width, height } = imageBitmap; + const fileID = enteFile.id; - let index: FaceIndex; try { - index = await indexFaces_(enteFile, imageBitmap, userAgent); + return { + fileID, + width, + height, + faceEmbedding: { + version: 1, + client: userAgent, + faces: await indexFacesInBitmap(fileID, imageBitmap), + }, + mlVersion: defaultMLVersion, + errorCount: 0, + }; } finally { imageBitmap.close(); } - - log.debug(() => { - const nf = index.faceEmbedding.faces.length; - const ms = Date.now() - startTime; - return `Indexed ${nf} faces in ${fileLogID(enteFile)} (${ms} ms)`; - }); - return index; }; /** @@ -102,27 +105,6 @@ const fetchRenderableBlob = async (enteFile: EnteFile) => { } }; -const indexFaces_ = async ( - enteFile: EnteFile, - imageBitmap: ImageBitmap, - userAgent: string, -) => { - const { width, height } = imageBitmap; - const fileID = enteFile.id; - return { - fileID, - width, - height, - faceEmbedding: { - version: 1, - client: userAgent, - faces: await indexFacesInBitmap(fileID, imageBitmap), - }, - mlVersion: defaultMLVersion, - errorCount: 0, - }; -}; - const indexFacesInBitmap = async ( fileID: number, imageBitmap: ImageBitmap, diff --git a/web/apps/photos/src/services/face/indexer.worker.ts b/web/apps/photos/src/services/face/indexer.worker.ts index 0dccd61780..969a800295 100644 --- a/web/apps/photos/src/services/face/indexer.worker.ts +++ b/web/apps/photos/src/services/face/indexer.worker.ts @@ -33,6 +33,9 @@ export class FaceIndexerWorker { * downloaded and decrypted from remote. */ async index(enteFile: EnteFile, file: File | undefined, userAgent: string) { + const f = fileLogID(enteFile); + const startTime = Date.now(); + let faceIndex: FaceIndex; try { faceIndex = await indexFaces(enteFile, file, userAgent); @@ -40,13 +43,24 @@ export class FaceIndexerWorker { // Mark indexing as having failed only if the indexing itself // failed, not if there were subsequent failures (like when trying // to put the result to remote or save it to the local face DB). - log.error(`Failed to index faces in ${fileLogID(enteFile)}`, e); + log.error(`Failed to index faces in ${f}`, e); markIndexingFailed(enteFile.id); throw e; } - await putFaceIndex(enteFile, faceIndex); - await saveFaceIndex(faceIndex); + try { + await putFaceIndex(enteFile, faceIndex); + await saveFaceIndex(faceIndex); + } catch (e) { + log.error(`Failed to put/save face index for ${f}`, e); + throw e; + } + + log.debug(() => { + const nf = faceIndex.faceEmbedding.faces.length; + const ms = Date.now() - startTime; + return `Indexed ${nf} faces in ${f} (${ms} ms)`; + }); return faceIndex; } diff --git a/web/apps/photos/src/services/machineLearning/machineLearningService.ts b/web/apps/photos/src/services/machineLearning/machineLearningService.ts index fbff8fd5a4..7e740b7d78 100644 --- a/web/apps/photos/src/services/machineLearning/machineLearningService.ts +++ b/web/apps/photos/src/services/machineLearning/machineLearningService.ts @@ -235,15 +235,7 @@ class MachineLearningService { await this.syncFile(enteFile, localFile, syncContext.userAgent); syncContext.nSyncedFiles += 1; } catch (e) { - log.error("ML syncFile failed", e); let error = e; - console.error( - "Error in ml sync, fileId: ", - enteFile.id, - "name: ", - enteFile.metadata.title, - error, - ); if ("status" in error) { const parsedMessage = parseUploadErrorCodes(error); error = parsedMessage; @@ -258,7 +250,6 @@ class MachineLearningService { throw error; } - await this.persistMLFileSyncError(enteFile, error); syncContext.nSyncedFiles += 1; } } @@ -275,25 +266,7 @@ class MachineLearningService { const worker = new FaceIndexerWorker(); - const newMlFile = await worker.index(enteFile, localFile, userAgent); - await mlIDbStorage.putFile(newMlFile); - } - - private async persistMLFileSyncError(enteFile: EnteFile, e: Error) { - try { - await mlIDbStorage.upsertFileInTx(enteFile.id, (mlFileData) => { - if (!mlFileData) { - mlFileData = this.newMlData(enteFile.id); - } - mlFileData.errorCount = (mlFileData.errorCount || 0) + 1; - console.error(`lastError for ${enteFile.id}`, e); - - return mlFileData; - }); - } catch (e) { - // TODO: logError or stop sync job after most of the requests are failed - console.error("Error while storing ml sync error", e); - } + await worker.index(enteFile, localFile, userAgent); } } From 1f6be04bf4c48558aad45ac99534f15023144331 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 15:57:27 +0530 Subject: [PATCH 206/354] Rename --- web/apps/photos/src/services/face/f-index.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/web/apps/photos/src/services/face/f-index.ts b/web/apps/photos/src/services/face/f-index.ts index 4bf829ff6f..a0faaea34b 100644 --- a/web/apps/photos/src/services/face/f-index.ts +++ b/web/apps/photos/src/services/face/f-index.ts @@ -110,12 +110,12 @@ const indexFacesInBitmap = async ( imageBitmap: ImageBitmap, ): Promise => { const { width, height } = imageBitmap; - const imageBox = { width, height }; + const imageDimensions = { width, height }; const yoloFaceDetections = await detectFaces(imageBitmap); const partialResult = yoloFaceDetections.map( ({ box, landmarks, score }) => { - const faceID = makeFaceID(fileID, box, imageBox); + const faceID = makeFaceID(fileID, box, imageDimensions); const detection = { box, landmarks }; return { faceID, detection, score }; }, @@ -150,7 +150,7 @@ const indexFacesInBitmap = async ( return partialResult.map(({ faceID, detection, score }, i) => ({ faceID, - detection: relativeDetection(detection, imageBox), + detection: normalizeToImageDimensions(detection, imageDimensions), score, blur: blurs[i], embedding: Array.from(embeddings[i]), @@ -742,7 +742,7 @@ const computeEmbeddings = async ( /** * Convert the coordinates to between 0-1, normalized by the image's dimensions. */ -const relativeDetection = ( +const normalizeToImageDimensions = ( faceDetection: FaceDetection, { width, height }: Dimensions, ): FaceDetection => { From 45d7e3da2cfe26bfc89f865a8b9513f8c766a366 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 16:00:09 +0530 Subject: [PATCH 207/354] Prune --- .../machineLearning/machineLearningService.ts | 32 ++----------------- 1 file changed, 2 insertions(+), 30 deletions(-) diff --git a/web/apps/photos/src/services/machineLearning/machineLearningService.ts b/web/apps/photos/src/services/machineLearning/machineLearningService.ts index 7e740b7d78..47517b4e02 100644 --- a/web/apps/photos/src/services/machineLearning/machineLearningService.ts +++ b/web/apps/photos/src/services/machineLearning/machineLearningService.ts @@ -1,12 +1,9 @@ import log from "@/next/log"; import { CustomError, parseUploadErrorCodes } from "@ente/shared/error"; import PQueue from "p-queue"; -import mlIDbStorage, { - type MinimalPersistedFileData, -} from "services/face/db-old"; +import mlIDbStorage from "services/face/db-old"; import { syncLocalFiles } from "services/face/indexer"; import { FaceIndexerWorker } from "services/face/indexer.worker"; -import { getLocalFiles } from "services/fileService"; import { EnteFile } from "types/file"; export const defaultMLVersion = 1; @@ -80,31 +77,6 @@ class MachineLearningService { return !error && nOutOfSyncFiles > 0; } - private newMlData(fileID: number) { - return { - fileID, - mlVersion: 0, - errorCount: 0, - faceEmbedding: { faces: [] }, - } as MinimalPersistedFileData; - } - - private async getLocalFilesMap(syncContext: MLSyncContext) { - if (!syncContext.localFilesMap) { - const localFiles = await getLocalFiles(); - - const personalFiles = localFiles.filter( - (f) => f.ownerID === syncContext.userID, - ); - syncContext.localFilesMap = new Map(); - personalFiles.forEach((f) => - syncContext.localFilesMap.set(f.id, f), - ); - } - - return syncContext.localFilesMap; - } - private async getOutOfSyncFiles(syncContext: MLSyncContext) { const startTime = Date.now(); const fileIds = await mlIDbStorage.getFileIds( @@ -115,7 +87,7 @@ class MachineLearningService { log.info("fileIds: ", JSON.stringify(fileIds)); - const localFilesMap = await this.getLocalFilesMap(syncContext); + const localFilesMap = syncContext.localFilesMap; syncContext.outOfSyncFiles = fileIds.map((fileId) => localFilesMap.get(fileId), ); From 46761622f1c14c48586269f253e72e7864c7db8f Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 16:05:50 +0530 Subject: [PATCH 208/354] Fix --- web/apps/photos/src/services/embeddingService.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/web/apps/photos/src/services/embeddingService.ts b/web/apps/photos/src/services/embeddingService.ts index 56cebe5a03..fb77609258 100644 --- a/web/apps/photos/src/services/embeddingService.ts +++ b/web/apps/photos/src/services/embeddingService.ts @@ -7,7 +7,6 @@ import HTTPService from "@ente/shared/network/HTTPService"; import { getEndpoint } from "@ente/shared/network/api"; import localForage from "@ente/shared/storage/localForage"; import { getToken } from "@ente/shared/storage/localStorage/helpers"; -import { FileML } from "services/face/remote"; import type { Embedding, EmbeddingModel, @@ -17,9 +16,13 @@ import type { } from "types/embedding"; import { EnteFile } from "types/file"; import { getLocalCollections } from "./collectionService"; +import type { FaceIndex } from "./face/types"; import { getAllLocalFiles } from "./fileService"; import { getLocalTrashedFiles } from "./trashService"; +type FileML = FaceIndex & { + updatedAt: number; +}; const DIFF_LIMIT = 500; /** Local storage key suffix for embedding sync times */ From 074e867886694ce5f20cdeff63e1dc0576fbff74 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 16:08:07 +0530 Subject: [PATCH 209/354] Disable the download for now --- web/apps/photos/src/pages/gallery/index.tsx | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/web/apps/photos/src/pages/gallery/index.tsx b/web/apps/photos/src/pages/gallery/index.tsx index 4a3fbe9b52..d375e48fc8 100644 --- a/web/apps/photos/src/pages/gallery/index.tsx +++ b/web/apps/photos/src/pages/gallery/index.tsx @@ -85,10 +85,7 @@ import { getSectionSummaries, } from "services/collectionService"; import downloadManager from "services/download"; -import { - syncCLIPEmbeddings, - syncFaceEmbeddings, -} from "services/embeddingService"; +import { syncCLIPEmbeddings } from "services/embeddingService"; import { syncEntities } from "services/entityService"; import { getLocalFiles, syncFiles } from "services/fileService"; import locationSearchService from "services/locationSearchService"; @@ -130,7 +127,6 @@ import { } from "utils/file"; import { isArchivedFile } from "utils/magicMetadata"; import { getSessionExpiredMessage } from "utils/ui"; -import { isInternalUserForML } from "utils/user"; import { getLocalFamilyData } from "utils/user/family"; export const DeadCenter = styled("div")` @@ -720,7 +716,9 @@ export default function Gallery() { const electron = globalThis.electron; if (electron) { await syncCLIPEmbeddings(); - if (isInternalUserForML()) await syncFaceEmbeddings(); + // TODO-ML(MR): Disable fetch until we start storing it in the + // same place as the local ones. + // if (isInternalUserForML()) await syncFaceEmbeddings(); } if (clipService.isPlatformSupported()) { void clipService.scheduleImageEmbeddingExtraction(); From 966e5527ec40f8b33fa6a214b2e51c2db8ab7c5d Mon Sep 17 00:00:00 2001 From: Zxhir <98621617+Imzxhir@users.noreply.github.com> Date: Thu, 30 May 2024 12:05:35 +0100 Subject: [PATCH 210/354] Add alt names for some services --- auth/assets/custom-icons/_data/custom-icons.json | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/auth/assets/custom-icons/_data/custom-icons.json b/auth/assets/custom-icons/_data/custom-icons.json index b911183e88..2a3a8f21a7 100644 --- a/auth/assets/custom-icons/_data/custom-icons.json +++ b/auth/assets/custom-icons/_data/custom-icons.json @@ -32,7 +32,10 @@ }, { "title": "Bloom Host", - "slug": "bloom_host" + "slug": "bloom_host", + "altNames": [ + "Bloom Host Billing" + ] }, { "title": "BorgBase", @@ -345,7 +348,10 @@ "hex": "FFFFFF" }, { - "title": "Techlore" + "title": "Techlore", + "altNames": [ + "Techlore Courses" + ] }, { "title": "Termius", From 27523e2f109e17373ca097e33dd987ec5fdfd1df Mon Sep 17 00:00:00 2001 From: Vishnu Mohandas Date: Thu, 30 May 2024 18:18:28 +0530 Subject: [PATCH 211/354] Update intl_en.arb --- mobile/lib/l10n/intl_en.arb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobile/lib/l10n/intl_en.arb b/mobile/lib/l10n/intl_en.arb index feefd5a298..6a1a5c67c1 100644 --- a/mobile/lib/l10n/intl_en.arb +++ b/mobile/lib/l10n/intl_en.arb @@ -409,7 +409,7 @@ "manageDeviceStorage": "Manage device storage", "machineLearning": "Machine learning", "magicSearch": "Magic search", - "mlIndexingDescription": "Please note that ML indexing will result in a higher bandwidth and battery usage until all items are indexed.", + "mlIndexingDescription": "Please note that machine learning will result in a higher bandwidth and battery usage.", "loadingModel": "Downloading models...", "waitingForWifi": "Waiting for WiFi...", "status": "Status", From 9f2d770bc2b5b89d1b0270b75956dfa1964d88ec Mon Sep 17 00:00:00 2001 From: Laurens Priem <81471280+laurenspriem@users.noreply.github.com> Date: Thu, 30 May 2024 18:24:43 +0530 Subject: [PATCH 212/354] Update intl_en.arb --- mobile/lib/l10n/intl_en.arb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobile/lib/l10n/intl_en.arb b/mobile/lib/l10n/intl_en.arb index 6a1a5c67c1..ee2877861b 100644 --- a/mobile/lib/l10n/intl_en.arb +++ b/mobile/lib/l10n/intl_en.arb @@ -409,7 +409,7 @@ "manageDeviceStorage": "Manage device storage", "machineLearning": "Machine learning", "magicSearch": "Magic search", - "mlIndexingDescription": "Please note that machine learning will result in a higher bandwidth and battery usage.", + "mlIndexingDescription": "Please note that machine learning will result in a higher bandwidth and battery usage until all items are indexed.", "loadingModel": "Downloading models...", "waitingForWifi": "Waiting for WiFi...", "status": "Status", From ce93ce65293a74ac527fa4ad97996c234be6b244 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 18:32:43 +0530 Subject: [PATCH 213/354] sync + fetch --- web/apps/photos/src/services/face/db.ts | 6 +- web/apps/photos/src/services/face/indexer.ts | 90 +++++-------------- .../machineLearning/machineLearningService.ts | 24 +---- 3 files changed, 28 insertions(+), 92 deletions(-) diff --git a/web/apps/photos/src/services/face/db.ts b/web/apps/photos/src/services/face/db.ts index fe5a93cdb2..21dd3de41d 100644 --- a/web/apps/photos/src/services/face/db.ts +++ b/web/apps/photos/src/services/face/db.ts @@ -264,11 +264,13 @@ export const indexedAndIndexableCounts = async () => { * (can use {@link addFileEntry} to inform it about new files). From this * universe, we filter out fileIDs the files corresponding to which have already * been indexed, or for which we attempted indexing but failed. + * + * @param count Limit the result to up to {@link count} items. */ -export const unindexedFileIDs = async () => { +export const unindexedFileIDs = async (count?: number) => { const db = await faceDB(); const tx = db.transaction("file-status", "readonly"); - return tx.store.index("isIndexable").getAllKeys(IDBKeyRange.only(1)); + return tx.store.index("isIndexable").getAllKeys(IDBKeyRange.only(1), count); }; /** diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index 7ea723f135..475b2892ee 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -1,6 +1,7 @@ import { FILE_TYPE } from "@/media/file-type"; import log from "@/next/log"; import { ComlinkWorker } from "@/next/worker/comlink-worker"; +import { ensure } from "@/utils/ensure"; import { wait } from "@/utils/promise"; import { type Remote } from "comlink"; import mlIDbStorage, { ML_SEARCH_CONFIG_NAME } from "services/face/db-old"; @@ -11,8 +12,12 @@ import machineLearningService, { import mlWorkManager from "services/machineLearning/mlWorkManager"; import type { EnteFile } from "types/file"; import { isInternalUserForML } from "utils/user"; -import { indexedAndIndexableCounts } from "./db"; -import type { IndexStatus, MinimalPersistedFileData } from "./db-old"; +import { + indexedAndIndexableCounts, + syncWithLocalIndexableFileIDs, + unindexedFileIDs, +} from "./db"; +import type { IndexStatus } from "./db-old"; import { FaceIndexerWorker } from "./indexer.worker"; /** @@ -245,73 +250,18 @@ export const setIsFaceIndexingEnabled = async (enabled: boolean) => { return mlIDbStorage.putConfig(ML_SEARCH_CONFIG_NAME, { enabled }); }; -export const syncLocalFiles = async (userID: number) => { - const localFilesMap = await localIndexableFilesByID(userID); - - // const localFileIDs = new Set(localFilesMap.keys()); - - const db = await mlIDbStorage.db; - const tx = db.transaction("files", "readwrite"); - const mlFileIdsArr = await mlIDbStorage.getAllFileIdsForUpdate(tx); - const mlFileIds = new Set(); - mlFileIdsArr.forEach((mlFileId) => mlFileIds.add(mlFileId)); - - const newFileIds: Array = []; - for (const localFileId of localFilesMap.keys()) { - if (!mlFileIds.has(localFileId)) { - newFileIds.push(localFileId); - } - } - - let updated = false; - if (newFileIds.length > 0) { - log.info("newFiles: ", newFileIds.length); - const newFiles = newFileIds.map( - (fileID) => - ({ - fileID, - mlVersion: 0, - errorCount: 0, - faceEmbedding: { faces: [] }, - }) as MinimalPersistedFileData, - ); - await mlIDbStorage.putAllFiles(newFiles, tx); - updated = true; - } - - const removedFileIds: Array = []; - for (const mlFileId of mlFileIds) { - if (!localFilesMap.has(mlFileId)) { - removedFileIds.push(mlFileId); - } - } - - if (removedFileIds.length > 0) { - log.info("removedFiles: ", removedFileIds.length); - await mlIDbStorage.removeAllFiles(removedFileIds, tx); - updated = true; - } - - await tx.done; - - if (updated) { - // TODO: should do in same transaction - await mlIDbStorage.incrementIndexVersion("files"); - } - - return localFilesMap; -}; - /** - * Return a map of all {@link EnteFile}s owned by {@link userID} that we know - * about locally, indexed by their {@link fileID}. + * Sync face DB with the local indexable files that we know about. Then return + * the next {@link count} files that still need to be indexed. * - * @param userID Restrict the returned files to those owned by a {@link userID}. + * For more specifics of what a "sync" entails, see + * {@link syncWithLocalIndexableFileIDs}. + * + * @param userID Limit indexing to files owned by a {@link userID}. + * + * @param count Limit the resulting list of files to {@link count}. */ -const localIndexableFilesByID = async ( - userID: number, -): Promise> => { - const result = new Map(); +export const getFilesToIndex = async (userID: number, count: number) => { const localFiles = await getLocalFiles(); const indexableTypes = [FILE_TYPE.IMAGE, FILE_TYPE.LIVE_PHOTO]; const indexableFiles = localFiles.filter( @@ -319,6 +269,10 @@ const localIndexableFilesByID = async ( f.ownerID == userID && indexableTypes.includes(f.metadata.fileType), ); - indexableFiles.forEach((f) => result.set(f.id, f)); - return result; + const filesByID = new Map(indexableFiles.map((f) => [f.id, f])); + + await syncWithLocalIndexableFileIDs([...filesByID.keys()]); + + const fileIDsToIndex = await unindexedFileIDs(count); + return fileIDsToIndex.map((id) => ensure(filesByID.get(id))); }; diff --git a/web/apps/photos/src/services/machineLearning/machineLearningService.ts b/web/apps/photos/src/services/machineLearning/machineLearningService.ts index 47517b4e02..35a2be471a 100644 --- a/web/apps/photos/src/services/machineLearning/machineLearningService.ts +++ b/web/apps/photos/src/services/machineLearning/machineLearningService.ts @@ -2,7 +2,7 @@ import log from "@/next/log"; import { CustomError, parseUploadErrorCodes } from "@ente/shared/error"; import PQueue from "p-queue"; import mlIDbStorage from "services/face/db-old"; -import { syncLocalFiles } from "services/face/indexer"; +import { getFilesToIndex } from "services/face/indexer"; import { FaceIndexerWorker } from "services/face/indexer.worker"; import { EnteFile } from "types/file"; @@ -63,10 +63,7 @@ class MachineLearningService { const syncContext = await this.getSyncContext(token, userID, userAgent); - const localFiles = await syncLocalFiles(userID); - syncContext.localFilesMap = localFiles; - - await this.getOutOfSyncFiles(syncContext); + syncContext.outOfSyncFiles = await getFilesToIndex(userID, batchSize); if (syncContext.outOfSyncFiles.length > 0) { await this.syncFiles(syncContext); @@ -77,23 +74,6 @@ class MachineLearningService { return !error && nOutOfSyncFiles > 0; } - private async getOutOfSyncFiles(syncContext: MLSyncContext) { - const startTime = Date.now(); - const fileIds = await mlIDbStorage.getFileIds( - batchSize, - defaultMLVersion, - MAX_ML_SYNC_ERROR_COUNT, - ); - - log.info("fileIds: ", JSON.stringify(fileIds)); - - const localFilesMap = syncContext.localFilesMap; - syncContext.outOfSyncFiles = fileIds.map((fileId) => - localFilesMap.get(fileId), - ); - log.info("getOutOfSyncFiles", Date.now() - startTime, "ms"); - } - private async syncFiles(syncContext: MLSyncContext) { this.isSyncing = true; try { From 3e1dbce6298a91f28c880b3347f95671e7b545b2 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 18:57:19 +0530 Subject: [PATCH 214/354] Prune --- .../machineLearning/machineLearningService.ts | 16 ++-------------- .../services/machineLearning/mlWorkManager.ts | 5 +++++ 2 files changed, 7 insertions(+), 14 deletions(-) diff --git a/web/apps/photos/src/services/machineLearning/machineLearningService.ts b/web/apps/photos/src/services/machineLearning/machineLearningService.ts index 35a2be471a..6612e3c443 100644 --- a/web/apps/photos/src/services/machineLearning/machineLearningService.ts +++ b/web/apps/photos/src/services/machineLearning/machineLearningService.ts @@ -1,7 +1,6 @@ import log from "@/next/log"; import { CustomError, parseUploadErrorCodes } from "@ente/shared/error"; import PQueue from "p-queue"; -import mlIDbStorage from "services/face/db-old"; import { getFilesToIndex } from "services/face/indexer"; import { FaceIndexerWorker } from "services/face/indexer.worker"; import { EnteFile } from "types/file"; @@ -96,12 +95,6 @@ class MachineLearningService { } await syncContext.syncQueue.onIdle(); this.isSyncing = false; - - // TODO: In case syncJob has to use multiple ml workers - // do in same transaction with each file update - // or keep in files store itself - await mlIDbStorage.incrementIndexVersion("files"); - // await this.disposeMLModels(); } private async getSyncContext( @@ -208,17 +201,12 @@ class MachineLearningService { private async syncFile( enteFile: EnteFile, - localFile: globalThis.File | undefined, + file: File | undefined, userAgent: string, ) { - const oldMlFile = await mlIDbStorage.getFile(enteFile.id); - if (oldMlFile && oldMlFile.mlVersion) { - return; - } - const worker = new FaceIndexerWorker(); - await worker.index(enteFile, localFile, userAgent); + await worker.index(enteFile, file, userAgent); } } diff --git a/web/apps/photos/src/services/machineLearning/mlWorkManager.ts b/web/apps/photos/src/services/machineLearning/mlWorkManager.ts index 3f7e9571e8..2d58efc32c 100644 --- a/web/apps/photos/src/services/machineLearning/mlWorkManager.ts +++ b/web/apps/photos/src/services/machineLearning/mlWorkManager.ts @@ -228,7 +228,11 @@ class MLWorkManager { this.mlSearchEnabled && this.startSyncJob(); } + // eslint-disable-next-line @typescript-eslint/no-unused-vars public async syncLocalFile(enteFile: EnteFile, localFile: globalThis.File) { + return; + /* + TODO-ML(MR): Disable live sync for now await this.liveSyncQueue.add(async () => { this.stopSyncJob(); const token = getToken(); @@ -243,6 +247,7 @@ class MLWorkManager { localFile, ); }); + */ } // Sync Job From 654f6b8934568f4f63c2f9dfa90212dd5a499d14 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 19:04:01 +0530 Subject: [PATCH 215/354] Remove old indexstatus --- web/apps/photos/src/services/face/indexer.ts | 32 ++------------------ 1 file changed, 2 insertions(+), 30 deletions(-) diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index 475b2892ee..2ce75bf008 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -1,14 +1,11 @@ import { FILE_TYPE } from "@/media/file-type"; -import log from "@/next/log"; import { ComlinkWorker } from "@/next/worker/comlink-worker"; import { ensure } from "@/utils/ensure"; import { wait } from "@/utils/promise"; import { type Remote } from "comlink"; import mlIDbStorage, { ML_SEARCH_CONFIG_NAME } from "services/face/db-old"; import { getLocalFiles } from "services/fileService"; -import machineLearningService, { - defaultMLVersion, -} from "services/machineLearning/machineLearningService"; +import machineLearningService from "services/machineLearning/machineLearningService"; import mlWorkManager from "services/machineLearning/mlWorkManager"; import type { EnteFile } from "types/file"; import { isInternalUserForML } from "utils/user"; @@ -17,7 +14,6 @@ import { syncWithLocalIndexableFileIDs, unindexedFileIDs, } from "./db"; -import type { IndexStatus } from "./db-old"; import { FaceIndexerWorker } from "./indexer.worker"; /** @@ -180,35 +176,11 @@ export const faceIndexingStatus = async (): Promise => { phase = "done"; } - const indexingStatus = { + return { phase, nSyncedFiles: indexedCount, nTotalFiles: indexableCount, }; - - const indexStatus0 = await mlIDbStorage.getIndexStatus(defaultMLVersion); - const indexStatus = convertToNewInterface(indexStatus0); - - log.debug(() => ({ indexStatus, indexingStatus })); - - return indexStatus; -}; - -const convertToNewInterface = (indexStatus: IndexStatus) => { - let phase: FaceIndexingStatus["phase"]; - if (!indexStatus.localFilesSynced) { - phase = "scheduled"; - } else if (indexStatus.outOfSyncFilesExists) { - phase = "indexing"; - } else if (!indexStatus.peopleIndexSynced) { - phase = "clustering"; - } else { - phase = "done"; - } - return { - ...indexStatus, - phase, - }; }; /** From 7cc29c302e8d2cdff3764c8484488eb7848e69f6 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 19:11:44 +0530 Subject: [PATCH 216/354] new --- web/apps/photos/src/services/face/db.ts | 8 ++++++++ web/apps/photos/src/services/face/indexer.ts | 5 +++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/web/apps/photos/src/services/face/db.ts b/web/apps/photos/src/services/face/db.ts index 21dd3de41d..afc9581382 100644 --- a/web/apps/photos/src/services/face/db.ts +++ b/web/apps/photos/src/services/face/db.ts @@ -173,6 +173,14 @@ export const saveFaceIndex = async (faceIndex: FaceIndex) => { ]); }; +/** + * Return the {@link FaceIndex}, if any, for {@link fileID}. + */ +export const faceIndex = async (fileID: number) => { + const db = await faceDB(); + return db.get("face-index", fileID); +}; + /** * Record the existence of a file so that entities in the face indexing universe * know about it (e.g. can index it if it is new and it needs indexing). diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index 2ce75bf008..cb8465d166 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -10,6 +10,7 @@ import mlWorkManager from "services/machineLearning/mlWorkManager"; import type { EnteFile } from "types/file"; import { isInternalUserForML } from "utils/user"; import { + faceIndex, indexedAndIndexableCounts, syncWithLocalIndexableFileIDs, unindexedFileIDs, @@ -190,8 +191,8 @@ export const faceIndexingStatus = async (): Promise => { export const unidentifiedFaceIDs = async ( enteFile: EnteFile, ): Promise => { - const mlFileData = await mlIDbStorage.getFile(enteFile.id); - return mlFileData?.faceEmbedding.faces.map((f) => f.faceID) ?? []; + const index = await faceIndex(enteFile.id); + return index?.faceEmbedding.faces.map((f) => f.faceID) ?? []; }; /** From 400a6a9054ba7510ad6ef66a1cc97631681663b3 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 19:17:24 +0530 Subject: [PATCH 217/354] Store enabled state in local storage --- web/apps/photos/src/services/face/indexer.ts | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index cb8465d166..e2b62e51d0 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -3,7 +3,6 @@ import { ComlinkWorker } from "@/next/worker/comlink-worker"; import { ensure } from "@/utils/ensure"; import { wait } from "@/utils/promise"; import { type Remote } from "comlink"; -import mlIDbStorage, { ML_SEARCH_CONFIG_NAME } from "services/face/db-old"; import { getLocalFiles } from "services/fileService"; import machineLearningService from "services/machineLearning/machineLearningService"; import mlWorkManager from "services/machineLearning/mlWorkManager"; @@ -206,10 +205,7 @@ export const unidentifiedFaceIDs = async ( */ export const isFaceIndexingEnabled = async () => { if (isInternalUserForML()) { - const config = await mlIDbStorage.getConfig(ML_SEARCH_CONFIG_NAME, { - enabled: false, - }); - return config.enabled; + return localStorage.getItem("faceIndexingEnabled") == "1"; } // Force disabled for everyone else while we finalize it to avoid redundant // reindexing for users. @@ -220,7 +216,8 @@ export const isFaceIndexingEnabled = async () => { * Update the (locally stored) value of {@link isFaceIndexingEnabled}. */ export const setIsFaceIndexingEnabled = async (enabled: boolean) => { - return mlIDbStorage.putConfig(ML_SEARCH_CONFIG_NAME, { enabled }); + if (enabled) localStorage.setItem("faceIndexingEnabled", "1"); + else localStorage.removeItem("faceIndexingEnabled"); }; /** From 21567d546e7c049e4031fc838bb0fbb41208b377 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 19:21:38 +0530 Subject: [PATCH 218/354] bye --- web/apps/photos/src/services/face/db-old.ts | 391 ------------------ web/apps/photos/src/services/face/db.ts | 10 + .../photos/src/services/face/types-old.ts | 6 - .../machineLearning/machineLearningService.ts | 2 - .../services/machineLearning/mlWorkManager.ts | 2 - 5 files changed, 10 insertions(+), 401 deletions(-) delete mode 100644 web/apps/photos/src/services/face/db-old.ts delete mode 100644 web/apps/photos/src/services/face/types-old.ts diff --git a/web/apps/photos/src/services/face/db-old.ts b/web/apps/photos/src/services/face/db-old.ts deleted file mode 100644 index b66c72d811..0000000000 --- a/web/apps/photos/src/services/face/db-old.ts +++ /dev/null @@ -1,391 +0,0 @@ -import { haveWindow } from "@/next/env"; -import log from "@/next/log"; -import { - DBSchema, - IDBPDatabase, - IDBPTransaction, - StoreNames, - deleteDB, - openDB, -} from "idb"; -import isElectron from "is-electron"; -import type { Person } from "services/face/people"; -import type { MlFileData } from "services/face/types-old"; -import { MAX_ML_SYNC_ERROR_COUNT } from "services/machineLearning/machineLearningService"; - -export interface IndexStatus { - outOfSyncFilesExists: boolean; - nSyncedFiles: number; - nTotalFiles: number; - localFilesSynced: boolean; - peopleIndexSynced: boolean; -} - -/** - * TODO(MR): Transient type with an intersection of values that both existing - * and new types during the migration will have. Eventually we'll store the the - * server ML data shape here exactly. - */ -export interface MinimalPersistedFileData { - fileID: number; - mlVersion: number; - errorCount: number; - faceEmbedding: { faces: { faceID: string }[] }; -} - -interface Config {} - -export const ML_SEARCH_CONFIG_NAME = "ml-search"; - -const MLDATA_DB_NAME = "mldata"; -interface MLDb extends DBSchema { - files: { - key: number; - value: MinimalPersistedFileData; - indexes: { mlVersion: [number, number] }; - }; - people: { - key: number; - value: Person; - }; - // Unused, we only retain this is the schema so that we can delete it during - // migration. - things: { - key: number; - value: unknown; - }; - versions: { - key: string; - value: number; - }; - library: { - key: string; - value: unknown; - }; - configs: { - key: string; - value: Config; - }; -} - -class MLIDbStorage { - public _db: Promise>; - - constructor() { - if (!haveWindow() || !isElectron()) { - return; - } - - this.db; - } - - private openDB(): Promise> { - return openDB(MLDATA_DB_NAME, 4, { - terminated: async () => { - log.error("ML Indexed DB terminated"); - this._db = undefined; - // TODO: remove if there is chance of this going into recursion in some case - await this.db; - }, - blocked() { - // TODO: make sure we dont allow multiple tabs of app - log.error("ML Indexed DB blocked"); - }, - blocking() { - // TODO: make sure we dont allow multiple tabs of app - log.error("ML Indexed DB blocking"); - }, - async upgrade(db, oldVersion, newVersion, tx) { - let wasMLSearchEnabled = false; - try { - const searchConfig: unknown = await tx - .objectStore("configs") - .get(ML_SEARCH_CONFIG_NAME); - if ( - searchConfig && - typeof searchConfig == "object" && - "enabled" in searchConfig && - typeof searchConfig.enabled == "boolean" - ) { - wasMLSearchEnabled = searchConfig.enabled; - } - } catch (e) { - // The configs store might not exist (e.g. during logout). - // Ignore. - } - log.info( - `Previous ML database v${oldVersion} had ML search ${wasMLSearchEnabled ? "enabled" : "disabled"}`, - ); - - if (oldVersion < 1) { - const filesStore = db.createObjectStore("files", { - // keyPath: "fileId", TODO(MR): Changing this, since - // we're going to be deleting this DB before this PR is - // merged. - keyPath: "fileID", - }); - filesStore.createIndex("mlVersion", [ - "mlVersion", - "errorCount", - ]); - - db.createObjectStore("people", { - keyPath: "id", - }); - - db.createObjectStore("things", { - keyPath: "id", - }); - - db.createObjectStore("versions"); - - db.createObjectStore("library"); - } - if (oldVersion < 2) { - // TODO: update configs if version is updated in defaults - db.createObjectStore("configs"); - - /* - await tx - .objectStore("configs") - .add( - DEFAULT_ML_SYNC_JOB_CONFIG, - "ml-sync-job", - ); - - await tx - .objectStore("configs") - .add(DEFAULT_ML_SYNC_CONFIG, ML_SYNC_CONFIG_NAME); - */ - } - if (oldVersion < 3) { - const DEFAULT_ML_SEARCH_CONFIG = { - enabled: false, - }; - - await tx - .objectStore("configs") - .add(DEFAULT_ML_SEARCH_CONFIG, ML_SEARCH_CONFIG_NAME); - } - /* - This'll go in version 5. Note that version 4 was never released, - but it was in main for a while, so we'll just skip it to avoid - breaking the upgrade path for people who ran the mainline. - */ - if (oldVersion < 4) { - /* - try { - await tx - .objectStore("configs") - .delete(ML_SEARCH_CONFIG_NAME); - - await tx - .objectStore("configs") - .delete(""ml-sync""); - - await tx - .objectStore("configs") - .delete("ml-sync-job"); - - await tx - .objectStore("configs") - .add( - { enabled: wasMLSearchEnabled }, - ML_SEARCH_CONFIG_NAME, - ); - - db.deleteObjectStore("library"); - db.deleteObjectStore("things"); - } catch { - // TODO: ignore for now as we finalize the new version - // the shipped implementation should have a more - // deterministic migration. - } - */ - } - log.info( - `ML DB upgraded from version ${oldVersion} to version ${newVersion}`, - ); - }, - }); - } - - public get db(): Promise> { - if (!this._db) { - this._db = this.openDB(); - log.info("Opening Ml DB"); - } - - return this._db; - } - - public async clearMLDB() { - const db = await this.db; - db.close(); - await deleteDB(MLDATA_DB_NAME); - log.info("Cleared Ml DB"); - this._db = undefined; - await this.db; - } - - public async getAllFileIdsForUpdate( - tx: IDBPTransaction, - ) { - return tx.store.getAllKeys(); - } - - public async getFileIds( - count: number, - limitMlVersion: number, - maxErrorCount: number, - ) { - const db = await this.db; - const tx = db.transaction("files", "readonly"); - const index = tx.store.index("mlVersion"); - let cursor = await index.openKeyCursor( - IDBKeyRange.upperBound([limitMlVersion], true), - ); - - const fileIds: number[] = []; - while (cursor && fileIds.length < count) { - if ( - cursor.key[0] < limitMlVersion && - cursor.key[1] <= maxErrorCount - ) { - fileIds.push(cursor.primaryKey); - } - cursor = await cursor.continue(); - } - await tx.done; - - return fileIds; - } - - public async getFile(fileId: number): Promise { - const db = await this.db; - return db.get("files", fileId); - } - - public async putFile(mlFile: MlFileData) { - const db = await this.db; - return db.put("files", mlFile); - } - - public async upsertFileInTx( - fileId: number, - upsert: (mlFile: MinimalPersistedFileData) => MinimalPersistedFileData, - ) { - const db = await this.db; - const tx = db.transaction("files", "readwrite"); - const existing = await tx.store.get(fileId); - const updated = upsert(existing); - await tx.store.put(updated); - await tx.done; - - return updated; - } - - public async putAllFiles( - mlFiles: MinimalPersistedFileData[], - tx: IDBPTransaction, - ) { - await Promise.all(mlFiles.map((mlFile) => tx.store.put(mlFile))); - } - - public async removeAllFiles( - fileIds: Array, - tx: IDBPTransaction, - ) { - await Promise.all(fileIds.map((fileId) => tx.store.delete(fileId))); - } - - public async getPerson(id: number) { - const db = await this.db; - return db.get("people", id); - } - - public async getAllPeople() { - const db = await this.db; - return db.getAll("people"); - } - - public async incrementIndexVersion(index: StoreNames) { - if (index === "versions") { - throw new Error("versions store can not be versioned"); - } - const db = await this.db; - const tx = db.transaction(["versions", index], "readwrite"); - let version = await tx.objectStore("versions").get(index); - version = (version || 0) + 1; - tx.objectStore("versions").put(version, index); - await tx.done; - - return version; - } - - public async getConfig(name: string, def: T) { - const db = await this.db; - const tx = db.transaction("configs", "readwrite"); - let config = (await tx.store.get(name)) as T; - if (!config) { - config = def; - await tx.store.put(def, name); - } - await tx.done; - - return config; - } - - public async putConfig(name: string, data: Config) { - const db = await this.db; - return db.put("configs", data, name); - } - - public async getIndexStatus(latestMlVersion: number): Promise { - const db = await this.db; - const tx = db.transaction(["files", "versions"], "readonly"); - const mlVersionIdx = tx.objectStore("files").index("mlVersion"); - - let outOfSyncCursor = await mlVersionIdx.openKeyCursor( - IDBKeyRange.upperBound([latestMlVersion], true), - ); - let outOfSyncFilesExists = false; - while (outOfSyncCursor && !outOfSyncFilesExists) { - if ( - outOfSyncCursor.key[0] < latestMlVersion && - outOfSyncCursor.key[1] <= MAX_ML_SYNC_ERROR_COUNT - ) { - outOfSyncFilesExists = true; - } - outOfSyncCursor = await outOfSyncCursor.continue(); - } - - const nSyncedFiles = await mlVersionIdx.count( - IDBKeyRange.lowerBound([latestMlVersion]), - ); - const nTotalFiles = await mlVersionIdx.count(); - - const filesIndexVersion = await tx.objectStore("versions").get("files"); - const peopleIndexVersion = await tx - .objectStore("versions") - .get("people"); - const filesIndexVersionExists = - filesIndexVersion !== null && filesIndexVersion !== undefined; - const peopleIndexVersionExists = - peopleIndexVersion !== null && peopleIndexVersion !== undefined; - - await tx.done; - - return { - outOfSyncFilesExists, - nSyncedFiles, - nTotalFiles, - localFilesSynced: filesIndexVersionExists, - peopleIndexSynced: - peopleIndexVersionExists && - peopleIndexVersion === filesIndexVersion, - }; - } -} - -export default new MLIDbStorage(); diff --git a/web/apps/photos/src/services/face/db.ts b/web/apps/photos/src/services/face/db.ts index afc9581382..ab03b726f7 100644 --- a/web/apps/photos/src/services/face/db.ts +++ b/web/apps/photos/src/services/face/db.ts @@ -82,6 +82,8 @@ interface FileStatus { let _faceDB: ReturnType | undefined; const openFaceDB = async () => { + deleteLegacyDB(); + const db = await openDB("face", 1, { upgrade(db, oldVersion, newVersion) { log.info(`Upgrading face DB ${oldVersion} => ${newVersion}`); @@ -112,6 +114,13 @@ const openFaceDB = async () => { return db; }; +const deleteLegacyDB = () => { + // Delete the legacy face DB. + // This code was added June 2024 (v1.7.1-rc) and can be removed once clients + // have migrated over. + void deleteDB("mldata"); +}; + /** * @returns a lazily created, cached connection to the face DB. */ @@ -138,6 +147,7 @@ export const closeFaceDBConnectionsIfNeeded = async () => { * Meant to be called during logout. */ export const clearFaceData = async () => { + deleteLegacyDB(); await closeFaceDBConnectionsIfNeeded(); return deleteDB("face", { blocked() { diff --git a/web/apps/photos/src/services/face/types-old.ts b/web/apps/photos/src/services/face/types-old.ts deleted file mode 100644 index d2bf61a4f8..0000000000 --- a/web/apps/photos/src/services/face/types-old.ts +++ /dev/null @@ -1,6 +0,0 @@ -import type { FaceIndex } from "./types"; - -export type MlFileData = FaceIndex & { - mlVersion: number; - errorCount: number; -}; diff --git a/web/apps/photos/src/services/machineLearning/machineLearningService.ts b/web/apps/photos/src/services/machineLearning/machineLearningService.ts index 6612e3c443..196f3d46e8 100644 --- a/web/apps/photos/src/services/machineLearning/machineLearningService.ts +++ b/web/apps/photos/src/services/machineLearning/machineLearningService.ts @@ -9,8 +9,6 @@ export const defaultMLVersion = 1; const batchSize = 200; -export const MAX_ML_SYNC_ERROR_COUNT = 1; - class MLSyncContext { public token: string; public userID: number; diff --git a/web/apps/photos/src/services/machineLearning/mlWorkManager.ts b/web/apps/photos/src/services/machineLearning/mlWorkManager.ts index 2d58efc32c..9502c5a75d 100644 --- a/web/apps/photos/src/services/machineLearning/mlWorkManager.ts +++ b/web/apps/photos/src/services/machineLearning/mlWorkManager.ts @@ -8,7 +8,6 @@ import { getToken, getUserID } from "@ente/shared/storage/localStorage/helpers"; import debounce from "debounce"; import PQueue from "p-queue"; import { createFaceComlinkWorker } from "services/face"; -import mlIDbStorage from "services/face/db-old"; import type { DedicatedMLWorker } from "services/face/face.worker"; import { EnteFile } from "types/file"; @@ -167,7 +166,6 @@ class MLWorkManager { this.stopSyncJob(); this.mlSyncJob = undefined; await this.terminateLiveSyncWorker(); - await mlIDbStorage.clearMLDB(); } private async fileUploadedHandler(arg: { From 8f7af989bb310b77efc21d5e19ee44b20e22c55b Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 19:30:03 +0530 Subject: [PATCH 219/354] Remove unused --- web/apps/photos/src/services/face/f-index.ts | 3 --- .../src/services/machineLearning/machineLearningService.ts | 2 -- 2 files changed, 5 deletions(-) diff --git a/web/apps/photos/src/services/face/f-index.ts b/web/apps/photos/src/services/face/f-index.ts index a0faaea34b..79a428d0bc 100644 --- a/web/apps/photos/src/services/face/f-index.ts +++ b/web/apps/photos/src/services/face/f-index.ts @@ -4,7 +4,6 @@ import log from "@/next/log"; import { workerBridge } from "@/next/worker/worker-bridge"; import { Matrix } from "ml-matrix"; import DownloadManager from "services/download"; -import { defaultMLVersion } from "services/machineLearning/machineLearningService"; import { getSimilarityTransformation } from "similarity-transformation"; import { Matrix as TransformationMatrix, @@ -70,8 +69,6 @@ export const indexFaces = async ( client: userAgent, faces: await indexFacesInBitmap(fileID, imageBitmap), }, - mlVersion: defaultMLVersion, - errorCount: 0, }; } finally { imageBitmap.close(); diff --git a/web/apps/photos/src/services/machineLearning/machineLearningService.ts b/web/apps/photos/src/services/machineLearning/machineLearningService.ts index 196f3d46e8..8549ec7655 100644 --- a/web/apps/photos/src/services/machineLearning/machineLearningService.ts +++ b/web/apps/photos/src/services/machineLearning/machineLearningService.ts @@ -5,8 +5,6 @@ import { getFilesToIndex } from "services/face/indexer"; import { FaceIndexerWorker } from "services/face/indexer.worker"; import { EnteFile } from "types/file"; -export const defaultMLVersion = 1; - const batchSize = 200; class MLSyncContext { From f871255833b41c4ffb9cf89d73c70bd0d7123b65 Mon Sep 17 00:00:00 2001 From: ashilkn Date: Thu, 30 May 2024 20:34:59 +0530 Subject: [PATCH 220/354] [mob][photos] fix clusters in map not rendering properly in profile or release mode Key.toString() is working as expected on debug mode after upgrading flutter to 3.22.0 --- mobile/lib/ui/map/map_marker.dart | 4 ++-- mobile/lib/ui/map/map_view.dart | 17 +++++++---------- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/mobile/lib/ui/map/map_marker.dart b/mobile/lib/ui/map/map_marker.dart index e9de524227..3dc026697a 100644 --- a/mobile/lib/ui/map/map_marker.dart +++ b/mobile/lib/ui/map/map_marker.dart @@ -7,12 +7,12 @@ import "package:photos/ui/map/marker_image.dart"; Marker mapMarker( ImageMarker imageMarker, - String key, { + ValueKey key, { Size markerSize = MapView.defaultMarkerSize, }) { return Marker( alignment: Alignment.topCenter, - key: Key(key), + key: key, width: markerSize.width, height: markerSize.height, point: LatLng( diff --git a/mobile/lib/ui/map/map_view.dart b/mobile/lib/ui/map/map_view.dart index 3ff86f2ae1..d4de022197 100644 --- a/mobile/lib/ui/map/map_view.dart +++ b/mobile/lib/ui/map/map_view.dart @@ -4,8 +4,8 @@ import "package:flutter_map_marker_cluster/flutter_map_marker_cluster.dart"; import "package:latlong2/latlong.dart"; import "package:photos/ui/map/image_marker.dart"; import "package:photos/ui/map/map_button.dart"; -import 'package:photos/ui/map/map_gallery_tile.dart'; -import 'package:photos/ui/map/map_gallery_tile_badge.dart'; +import "package:photos/ui/map/map_gallery_tile.dart"; +import "package:photos/ui/map/map_gallery_tile_badge.dart"; import "package:photos/ui/map/map_marker.dart"; import "package:photos/ui/map/tile/layers.dart"; import "package:photos/utils/debouncer.dart"; @@ -114,13 +114,10 @@ class _MapViewState extends State { onChange(widget.controller.camera.visibleBounds); }, builder: (context, List markers) { - final index = int.parse( - markers.first.key - .toString() - .replaceAll(RegExp(r'[^0-9]'), ''), - ); - final String clusterKey = - 'map-badge-$index-len-${markers.length}'; + final valueKey = markers.first.key as ValueKey; + final index = valueKey.value as int; + + final clusterKey = 'map-badge-$index-len-${markers.length}'; return Stack( key: ValueKey(clusterKey), @@ -199,7 +196,7 @@ class _MapViewState extends State { final imageMarker = widget.imageMarkers[index]; return mapMarker( imageMarker, - index.toString(), + ValueKey(index), markerSize: widget.markerSize, ); }); From f647355666c7b0701caa5ee7934ffdd66b73e955 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 20:41:53 +0530 Subject: [PATCH 221/354] [desktop] Nightly builds --- desktop/.github/workflows/desktop-release.yml | 22 ++++++-- desktop/docs/release.md | 56 ++++++++----------- 2 files changed, 41 insertions(+), 37 deletions(-) diff --git a/desktop/.github/workflows/desktop-release.yml b/desktop/.github/workflows/desktop-release.yml index 70eedf3ea6..e9457240aa 100644 --- a/desktop/.github/workflows/desktop-release.yml +++ b/desktop/.github/workflows/desktop-release.yml @@ -5,12 +5,20 @@ name: "Release" # For more details, see `docs/release.md` in ente-io/ente. on: - # Trigger manually or `gh workflow run desktop-release.yml`. + # Trigger manually or `gh workflow run desktop-release.yml --source=foo`. workflow_dispatch: + inputs: + source: + description: "Branch (ente-io/ente) to build" + required: true + type: string + schedule: + # Run everyday at ~8:00 AM IST (except Sundays). + # See: [Note: Run workflow every 24 hours] + # + - cron: "45 2 * * 1-6" push: # Run when a tag matching the pattern "v*"" is pushed. - # - # See: [Note: Testing release workflows that are triggered by tags]. tags: - "v*" @@ -30,9 +38,13 @@ jobs: - name: Checkout code uses: actions/checkout@v4 with: - # Checkout the desktop/rc branch from the source repository. + # If triggered by a tag, checkout photosd-$tag from the source + # repository. Otherwise checkout $source (default: "main"). repository: ente-io/ente - ref: desktop/rc + ref: + "${{ startsWith(github.ref, 'refs/tags/v') && + format('photosd-{0}', github.ref_name) || ( inputs.source + || 'main' ) }}" submodules: recursive - name: Setup node diff --git a/desktop/docs/release.md b/desktop/docs/release.md index 53d0355c3e..f56d2c4404 100644 --- a/desktop/docs/release.md +++ b/desktop/docs/release.md @@ -5,7 +5,7 @@ creates a draft release with artifacts built. When ready, we publish that release. The download links on our website, and existing apps already check the latest GitHub release and update accordingly. -The complication comes by the fact that electron-builder's auto updaterr (the +The complication comes by the fact that electron-builder's auto updater (the mechanism that we use for auto updates) doesn't work with monorepos. So we need to keep a separate (non-mono) repository just for doing releases. @@ -16,48 +16,45 @@ to keep a separate (non-mono) repository just for doing releases. ## Workflow - Release Candidates -Leading up to the release, we can make one or more draft releases that are not -intended to be published, but serve as test release candidates. +Nightly RC builds of `main` are published by a scheduled workflow automatically. +If needed, these builds can also be manually triggered, including specifying the +source repository branch to build: -The workflow for making such "rc" builds is: +```sh +gh workflow run desktop-release.yml --source= +``` -1. Update `package.json` in the source repo to use version `1.x.x-rc`. Create a - new draft release in the release repo with title `1.x.x-rc`. In the tag - input enter `v1.x.x-rc` and select the option to "create a new tag on - publish". - -2. Push code to the `desktop/rc` branch in the source repo. - -3. Trigger the GitHub action in the release repo - - ```sh - gh workflow run desktop-release.yml - ``` - -We can do steps 2 and 3 multiple times: each time it'll just update the -artifacts attached to the same draft. +Each such workflow run will update the artifacts attached to the same +(pre-existing) pre-release. ## Workflow - Release 1. Update source repo to set version `1.x.x` in `package.json` and finalize the CHANGELOG. -2. Push code to the `desktop/rc` branch in the source repo. +2. Merge PR then tag the merge commit on `main` in the source repo: -3. In the release repo + ```sh + git tag photosd-v1.x.x + git push origin photosd-v1.x.x + ``` + +3. In the release repo: ```sh ./.github/trigger-release.sh v1.x.x ``` -4. If the build is successful, tag `desktop/rc` in the source repo. +This'll trigger the workflow and create a new draft release, which you can +publish after adding the release notes. - ```sh - # Assuming we're on desktop/rc that just got built +The release is done at this point, and we can now start a new RC train for +subsequent nightly builds. - git tag photosd-v1.x.x - git push origin photosd-v1.x.x - ``` +1. Update `package.json` in the source repo to use version `1.x.x-rc`. Create a + new draft release in the release repo with title `1.x.x-rc`. In the tag + input enter `v1.x.x-rc` and select the option to "create a new tag on + publish". ## Post build @@ -87,8 +84,3 @@ everything is automated: now their maintainers automatically bump the SHA, version number and the (derived from the version) URL in the formula when their tools notice a new release on our GitHub. - -We can also publish the draft releases by checking the "pre-release" option. -Such releases don't cause any of the channels (our website, or the desktop app -auto updater, or brew) to be notified, instead these are useful for giving links -to pre-release builds to customers. From c1097de27f6f09e1f9622616a340b235ed596c67 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 21:02:17 +0530 Subject: [PATCH 222/354] Non required --- desktop/.github/workflows/desktop-release.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/desktop/.github/workflows/desktop-release.yml b/desktop/.github/workflows/desktop-release.yml index e9457240aa..a5d7b9e155 100644 --- a/desktop/.github/workflows/desktop-release.yml +++ b/desktop/.github/workflows/desktop-release.yml @@ -10,7 +10,6 @@ on: inputs: source: description: "Branch (ente-io/ente) to build" - required: true type: string schedule: # Run everyday at ~8:00 AM IST (except Sundays). From f14f973a61abcbe644c0b223dc418db809a33d15 Mon Sep 17 00:00:00 2001 From: ashilkn Date: Thu, 30 May 2024 21:58:36 +0530 Subject: [PATCH 223/354] [mob][photos] Remove commented out code --- mobile/lib/ui/map/tile/attribution/map_attribution.dart | 7 ------- 1 file changed, 7 deletions(-) diff --git a/mobile/lib/ui/map/tile/attribution/map_attribution.dart b/mobile/lib/ui/map/tile/attribution/map_attribution.dart index 524d8b1e5d..7ae8c6bc86 100644 --- a/mobile/lib/ui/map/tile/attribution/map_attribution.dart +++ b/mobile/lib/ui/map/tile/attribution/map_attribution.dart @@ -194,13 +194,6 @@ class MapAttributionWidgetState extends State { context, () { setState(() => popupExpanded = true); - // mapEventSubscription = FlutterMapState.of(context) - // .mapController - // .mapEventStream - // .listen((e) { - // setState(() => popupExpanded = false); - // mapEventSubscription?.cancel(); - // }); mapEventSubscription = MapController().mapEventStream.listen((e) { setState(() => popupExpanded = false); From 453f196a63f2f11f5366f86d98f0b2e8e535dcca Mon Sep 17 00:00:00 2001 From: Joel Watson Date: Thu, 30 May 2024 14:15:24 -0500 Subject: [PATCH 224/354] [auth] Add custom Doppler icon --- auth/assets/custom-icons/_data/custom-icons.json | 3 +++ auth/assets/custom-icons/icons/doppler.svg | 1 + 2 files changed, 4 insertions(+) create mode 100644 auth/assets/custom-icons/icons/doppler.svg diff --git a/auth/assets/custom-icons/_data/custom-icons.json b/auth/assets/custom-icons/_data/custom-icons.json index c66d286fcf..5f15bba746 100644 --- a/auth/assets/custom-icons/_data/custom-icons.json +++ b/auth/assets/custom-icons/_data/custom-icons.json @@ -86,6 +86,9 @@ { "title": "Discourse" }, + { + "title": "Doppler" + }, { "title": "dus.net", "slug": "dusnet" diff --git a/auth/assets/custom-icons/icons/doppler.svg b/auth/assets/custom-icons/icons/doppler.svg new file mode 100644 index 0000000000..a11a7866b2 --- /dev/null +++ b/auth/assets/custom-icons/icons/doppler.svg @@ -0,0 +1 @@ + \ No newline at end of file From 0d38c6ac1b0fab5bc7e30d3829053f3ac6841587 Mon Sep 17 00:00:00 2001 From: Crowdin Bot Date: Fri, 31 May 2024 01:42:37 +0000 Subject: [PATCH 225/354] New Crowdin translations by GitHub Action --- web/packages/next/locales/nl-NL/translation.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/packages/next/locales/nl-NL/translation.json b/web/packages/next/locales/nl-NL/translation.json index 214e48dfe6..f3c0c510d3 100644 --- a/web/packages/next/locales/nl-NL/translation.json +++ b/web/packages/next/locales/nl-NL/translation.json @@ -565,7 +565,7 @@ "VIDEO": "Video", "LIVE_PHOTO": "Live foto", "editor": { - "crop": "" + "crop": "Bijsnijden" }, "CONVERT": "Converteren", "CONFIRM_EDITOR_CLOSE_MESSAGE": "Weet u zeker dat u de editor wilt afsluiten?", From 9fbe02eeac5423eeffdb8a5b1a945a5653dc66ee Mon Sep 17 00:00:00 2001 From: Crowdin Bot Date: Fri, 31 May 2024 01:58:48 +0000 Subject: [PATCH 226/354] New Crowdin translations by GitHub Action --- mobile/lib/l10n/intl_pt.arb | 15 +++++++-------- mobile/lib/l10n/intl_zh.arb | 24 ++++++++++++++---------- 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/mobile/lib/l10n/intl_pt.arb b/mobile/lib/l10n/intl_pt.arb index 428dbf5fc2..0d1d3b799d 100644 --- a/mobile/lib/l10n/intl_pt.arb +++ b/mobile/lib/l10n/intl_pt.arb @@ -24,13 +24,13 @@ "deleteRequestSLAText": "Sua solicitação será processada em até 72 horas.", "deleteEmailRequest": "Por favor, envie um e-mail para account-deletion@ente.io a partir do seu endereço de e-mail registrado.", "entePhotosPerm": "Ente precisa de permissão para preservar suas fotos", - "ok": "Ok", + "ok": "OK", "createAccount": "Criar uma conta", "createNewAccount": "Criar nova conta", "password": "Senha", "confirmPassword": "Confirme sua senha", "activeSessions": "Sessões ativas", - "oops": "Ops", + "oops": "Opa", "somethingWentWrongPleaseTryAgain": "Algo deu errado. Por favor, tente outra vez", "thisWillLogYouOutOfThisDevice": "Isso fará com que você saia deste dispositivo!", "thisWillLogYouOutOfTheFollowingDevice": "Isso fará com que você saia do seguinte dispositivo:", @@ -265,7 +265,7 @@ "somethingWentWrong": "Algo deu errado", "sendInvite": "Enviar convite", "shareTextRecommendUsingEnte": "Baixe o Ente para que possamos compartilhar facilmente fotos e vídeos de qualidade original\n\nhttps://ente.io", - "done": "Concluído", + "done": "Pronto", "applyCodeTitle": "Aplicar código", "enterCodeDescription": "Digite o código fornecido pelo seu amigo para reivindicar o armazenamento gratuito para vocês dois", "apply": "Aplicar", @@ -409,7 +409,7 @@ "manageDeviceStorage": "Gerenciar o armazenamento do dispositivo", "machineLearning": "Aprendizagem de máquina", "magicSearch": "Busca mágica", - "magicSearchDescription": "Por favor, note que isso resultará em uma largura de banda maior e uso de bateria até que todos os itens sejam indexados.", + "mlIndexingDescription": "Por favor, note que isso resultará em uma largura de banda maior e uso de bateria até que todos os itens sejam indexados.", "loadingModel": "Fazendo download de modelos...", "waitingForWifi": "Esperando por Wi-Fi...", "status": "Estado", @@ -948,7 +948,7 @@ "someOfTheFilesYouAreTryingToDeleteAre": "Alguns dos arquivos que você está tentando excluir só estão disponíveis no seu dispositivo e não podem ser recuperados se forem excluídos", "theyWillBeDeletedFromAllAlbums": "Ele será excluído de todos os álbuns.", "someItemsAreInBothEnteAndYourDevice": "Alguns itens estão tanto no Ente quanto no seu dispositivo.", - "selectedItemsWillBeDeletedFromAllAlbumsAndMoved": "Os itens selecionados serão excluídos de todos os álbuns e movidos para o lixo.", + "selectedItemsWillBeDeletedFromAllAlbumsAndMoved": "Os itens selecionados serão excluídos de todos os álbuns e movidos para a lixeira.", "theseItemsWillBeDeletedFromYourDevice": "Estes itens serão excluídos do seu dispositivo.", "itLooksLikeSomethingWentWrongPleaseRetryAfterSome": "Parece que algo deu errado. Por favor, tente novamente mais tarde. Se o erro persistir, entre em contato com nossa equipe de suporte.", "error": "Erro", @@ -1102,7 +1102,7 @@ "@iOSGoToSettingsDescription": { "description": "Message advising the user to go to the settings and configure Biometrics for their device. It shows in a dialog on iOS side." }, - "iOSOkButton": "Aceitar", + "iOSOkButton": "Tudo bem", "@iOSOkButton": { "description": "Message showed on a button that the user can click to leave the current dialog. It is used on iOS side. Maximum 30 characters." }, @@ -1233,8 +1233,7 @@ "autoPair": "Pareamento automático", "pairWithPin": "Parear com PIN", "faceRecognition": "Reconhecimento facial", - "faceRecognitionIndexingDescription": "Por favor, note que isso resultará em uma largura de banda maior e uso de bateria até que todos os itens sejam indexados.", "foundFaces": "Rostos encontrados", "clusteringProgress": "Progresso de agrupamento", - "indexingIsPaused": "Indexing is paused, will automatically resume when device is ready" + "indexingIsPaused": "A indexação está pausada, será retomada automaticamente quando o dispositivo estiver pronto." } \ No newline at end of file diff --git a/mobile/lib/l10n/intl_zh.arb b/mobile/lib/l10n/intl_zh.arb index 81fd22914a..2b0149685f 100644 --- a/mobile/lib/l10n/intl_zh.arb +++ b/mobile/lib/l10n/intl_zh.arb @@ -409,7 +409,7 @@ "manageDeviceStorage": "管理设备存储", "machineLearning": "机器学习", "magicSearch": "魔法搜索", - "magicSearchDescription": "请注意，在所有项目完成索引之前，这将使用更高的带宽和电量。", + "mlIndexingDescription": "请注意，机器学习将使用更高的带宽和更多的电量，直到所有项目都被索引为止。", "loadingModel": "正在下载模型...", "waitingForWifi": "正在等待 WiFi...", "status": "状态", @@ -569,7 +569,7 @@ "freeTrialValidTill": "免费试用有效期至 {endDate}", "validTill": "有效期至 {endDate}", "addOnValidTill": "您的 {storageAmount} 插件有效期至 {endDate}", - "playStoreFreeTrialValidTill": "免费试用有效期至 {endDate}。\n您可以随后购买付费计划。", + "playStoreFreeTrialValidTill": "免费试用有效期至 {endDate}。\n在此之后您可以选择付费计划。", "subWillBeCancelledOn": "您的订阅将于 {endDate} 取消", "subscription": "订阅", "paymentDetails": "付款明细", @@ -987,7 +987,7 @@ "fileTypesAndNames": "文件类型和名称", "location": "地理位置", "moments": "瞬间", - "searchFaceEmptySection": "查找一个人的所有照片", + "searchFaceEmptySection": "待索引完成后，人物将显示在此处", "searchDatesEmptySection": "按日期搜索，月份或年份", "searchLocationEmptySection": "在照片的一定半径内拍摄的几组照片", "searchPeopleEmptySection": "邀请他人，您将在此看到他们分享的所有照片", @@ -1171,6 +1171,7 @@ } }, "faces": "人脸", + "people": "人物", "contents": "内容", "addNew": "新建", "@addNew": { @@ -1196,14 +1197,14 @@ "verifyPasskey": "验证通行密钥", "playOnTv": "在电视上播放相册", "pair": "配对", - "autoPair": "自动配对", - "pairWithPin": "用 PIN 配对", "deviceNotFound": "未发现设备", "castInstruction": "在您要配对的设备上访问 cast.ente.io。\n输入下面的代码即可在电视上播放相册。", "deviceCodeHint": "输入代码", "joinDiscord": "加入 Discord", "locations": "位置", "descriptions": "描述", + "addAName": "添加一个名称", + "findPeopleByName": "按名称快速查找人物", "addViewers": "{count, plural, zero {添加查看者} one {添加查看者} other {添加查看者}}", "addCollaborators": "{count, plural, zero {添加协作者} one {添加协作者} other {添加协作者}}", "longPressAnEmailToVerifyEndToEndEncryption": "长按电子邮件以验证端到端加密。", @@ -1216,6 +1217,8 @@ "customEndpoint": "已连接至 {endpoint}", "createCollaborativeLink": "创建协作链接", "search": "搜索", + "enterPersonName": "输入人物名称", + "removePersonLabel": "移除人物标签", "autoPairDesc": "自动配对仅适用于支持 Chromecast 的设备。", "manualPairDesc": "用 PIN 码配对适用于您希望在其上查看相册的任何屏幕。", "connectToDevice": "连接到设备", @@ -1227,9 +1230,10 @@ "castIPMismatchTitle": "投放相册失败", "castIPMismatchBody": "请确保您的设备与电视处于同一网络。", "pairingComplete": "配对完成", - "faceRecognition": "Face recognition", - "faceRecognitionIndexingDescription": "Please note that this will result in a higher bandwidth and battery usage until all items are indexed.", - "foundFaces": "Found faces", - "clusteringProgress": "Clustering progress", - "indexingIsPaused": "Indexing is paused, will automatically resume when device is ready" + "autoPair": "自动配对", + "pairWithPin": "用 PIN 配对", + "faceRecognition": "人脸识别", + "foundFaces": "已找到的人脸", + "clusteringProgress": "聚类进展", + "indexingIsPaused": "索引已暂停。当设备准备就绪时，它将自动恢复。" } \ No newline at end of file From 9d309dd6deb72fbd30ebee5f7c7ae63beece3db3 Mon Sep 17 00:00:00 2001 From: Crowdin Bot Date: Fri, 31 May 2024 02:07:29 +0000 Subject: [PATCH 227/354] New Crowdin translations by GitHub Action --- auth/lib/l10n/arb/app_pt.arb | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/auth/lib/l10n/arb/app_pt.arb b/auth/lib/l10n/arb/app_pt.arb index 232c1becfe..52960987f6 100644 --- a/auth/lib/l10n/arb/app_pt.arb +++ b/auth/lib/l10n/arb/app_pt.arb @@ -7,7 +7,7 @@ "description": "Text shown in the AppBar of the Counter Page" }, "onBoardingBody": "Proteja seus códigos 2FA", - "onBoardingGetStarted": "Vamos Começar", + "onBoardingGetStarted": "Introdução", "setupFirstAccount": "Configure sua primeira conta", "importScanQrCode": "Escanear QR code", "qrCode": "QR Code", @@ -32,12 +32,12 @@ "counterBasedKeyType": "Baseado em um contador (HOTP)", "saveAction": "Salvar", "nextTotpTitle": "avançar", - "deleteCodeTitle": "Excluir código?", + "deleteCodeTitle": "Apagar código?", "deleteCodeMessage": "Tem certeza de que deseja excluir este código? Esta ação é irreversível.", "viewLogsAction": "Ver logs", "sendLogsDescription": "Isto irá compartilhar seus logs para nos ajudar a depurar seu problema. Embora tomemos precauções para garantir que informações sensíveis não sejam enviadas, encorajamos você a ver esses logs antes de compartilhá-los.", "preparingLogsTitle": "Preparando logs...", - "emailLogsTitle": "Logs por e-mail", + "emailLogsTitle": "Logs (e-mail)", "emailLogsMessage": "Por favor, envie os logs para {email}", "@emailLogsMessage": { "placeholders": { @@ -48,9 +48,9 @@ }, "copyEmailAction": "Copiar e-mail", "exportLogsAction": "Exportar logs", - "reportABug": "Reportar um problema", + "reportABug": "Informar um problema", "crashAndErrorReporting": "Reporte de erros e falhas", - "reportBug": "Reportar problema", + "reportBug": "Informar problema", "emailUsMessage": "Por favor, envie um e-mail para {email}", "@emailUsMessage": { "placeholders": { @@ -112,7 +112,7 @@ "email": "E-mail", "support": "Suporte", "general": "Geral", - "settings": "Configurações", + "settings": "Ajustes", "copied": "Copiado", "pleaseTryAgain": "Por favor, tente novamente", "existingUser": "Usuário Existente", @@ -139,7 +139,7 @@ "inFamilyPlanMessage": "Você está em um plano familiar!", "swipeHint": "Deslize para a esquerda para editar ou remover os códigos", "scan": "Escanear", - "scanACode": "Escanear um código", + "scanACode": "Escanear código", "verify": "Verificar", "verifyEmail": "Verificar e-mail", "enterCodeHint": "Digite o código de 6 dígitos de\nseu aplicativo autenticador", @@ -185,7 +185,7 @@ "lockScreenEnablePreSteps": "Para ativar o bloqueio de tela, por favor ative um método de autenticação nas configurações do sistema do seu dispositivo.", "viewActiveSessions": "Ver sessões ativas", "authToViewYourActiveSessions": "Por favor, autentique-se para ver as sessões ativas", - "searchHint": "Pesquisar...", + "searchHint": "Buscar...", "search": "Pesquisar", "sorryUnableToGenCode": "Desculpe, não foi possível gerar um código para {issuerName}", "noResult": "Nenhum resultado", @@ -242,7 +242,7 @@ "logInLabel": "Entrar", "logout": "Sair", "areYouSureYouWantToLogout": "Você tem certeza que deseja encerrar a sessão?", - "yesLogout": "Sim, encerrar sessão", + "yesLogout": "Sim, sair", "exit": "Sair", "verifyingRecoveryKey": "Verificando chave de recuperação...", "recoveryKeyVerified": "Chave de recuperação verificada", @@ -317,7 +317,7 @@ "thisWillLogYouOutOfTheFollowingDevice": "Isso fará com que você saia do seguinte dispositivo:", "terminateSession": "Encerrar sessão?", "terminate": "Encerrar", - "thisDevice": "Este dispositivo", + "thisDevice": "Esse dispositivo", "toResetVerifyEmail": "Para redefinir a sua senha, por favor verifique o seu email primeiro.", "thisEmailIsAlreadyInUse": "Este e-mail já está em uso", "verificationFailedPleaseTryAgain": "Falha na verificação. Por favor, tente novamente", @@ -339,7 +339,7 @@ "export": "Exportar", "useOffline": "Usar sem backups", "signInToBackup": "Entre para fazer backup de seus códigos", - "singIn": "Iniciar sessão", + "singIn": "Entrar", "sigInBackupReminder": "Por favor, exporte seus códigos para garantir que você tenha um backup do qual você possa restaurar.", "offlineModeWarning": "Você escolheu prosseguir sem backups. Por favor, faça backups manuais para ter certeza de que seus códigos estão seguros.", "showLargeIcons": "Mostrar ícones grandes", @@ -361,7 +361,7 @@ "@androidBiometricNotRecognized": { "description": "Message to let the user know that authentication was failed. It is used on Android side. Maximum 60 characters." }, - "androidBiometricSuccess": "Bem-sucedido", + "androidBiometricSuccess": "Êxito", "@androidBiometricSuccess": { "description": "Message to let the user know that authentication was successful. It is used on Android side. Maximum 60 characters." }, @@ -433,7 +433,7 @@ "tag": "Etiqueta", "create": "Criar", "editTag": "Editar etiqueta", - "deleteTagTitle": "Excluir etiqueta?", + "deleteTagTitle": "Apagar etiqueta?", "deleteTagMessage": "Tem certeza de que deseja excluir esta etiqueta? Essa ação é irreversível.", "somethingWentWrongParsingCode": "Não foi possível analisar os códigos {x}.", "updateNotAvailable": "Atualização não está disponível" From 253b74d58fae00829524dc44fa568071d91e015c Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 31 May 2024 09:05:48 +0530 Subject: [PATCH 228/354] [web] Rework the face DB schema These changes were in main only overnight and were not released anywhere, so I will take the liberty of modifying the schema without bumping the version. --- web/apps/photos/src/services/face/db.ts | 69 ++++++++++++-------- web/apps/photos/src/services/face/indexer.ts | 8 +-- 2 files changed, 44 insertions(+), 33 deletions(-) diff --git a/web/apps/photos/src/services/face/db.ts b/web/apps/photos/src/services/face/db.ts index ab03b726f7..e257482213 100644 --- a/web/apps/photos/src/services/face/db.ts +++ b/web/apps/photos/src/services/face/db.ts @@ -30,7 +30,7 @@ interface FaceDBSchema extends DBSchema { "file-status": { key: number; value: FileStatus; - indexes: { isIndexable: number }; + indexes: { status: FileStatus["status"] }; }; } @@ -38,27 +38,32 @@ interface FileStatus { /** The ID of the {@link EnteFile} whose indexing status we represent. */ fileID: number; /** - * `1` if this file needs to be indexed, `0` otherwise. + * The status of the file. * - * > Somewhat confusingly, we also have a (IndexedDB) "index" on this field. - * That (IDB) index allows us to efficiently select {@link fileIDs} that - * still need indexing (i.e. entries where {@link isIndexed} is `1`). + * - "indexable" - This file is something that we can index, but it is yet + * to be indexed. * - * [Note: Boolean IndexedDB indexes]. + * - "indexed" - We have a corresponding entry for this file in the + * "face-index" object (either indexed locally or fetched from remote). * - * IndexedDB does not (currently) supported indexes on boolean fields. - * https://github.com/w3c/IndexedDB/issues/76 + * - "ignored" - Ignore this file when considering indexes. Some reasons + * include: * - * As a workaround, we use numeric fields where `0` denotes `false` and `1` - * denotes `true`. + * - Indexeing might've failed (in which case there won't even be a + * corresponding entry for this file in "face-index"). + * + * - We might have a "face-index" for this file, but we should not use it + * because, say, the file is currently hidden. + * + * We also have a (IndexedDB) "index" on this field to allow us to + * efficiently select or count {@link fileIDs} that fall into various + * buckets. */ - isIndexable: number; + status: "indexable" | "indexed" | "ignored"; /** * The number of times attempts to index this file failed. * - * This is guaranteed to be `0` for files which have already been - * sucessfully indexed (i.e. files for which `isIndexable` is 0 and which - * have a corresponding entry in the "face-index" object store). + * This is guaranteed to be `0` for files with status "indexed". */ failureCount: number; } @@ -91,7 +96,7 @@ const openFaceDB = async () => { db.createObjectStore("face-index", { keyPath: "fileID" }); db.createObjectStore("file-status", { keyPath: "fileID", - }).createIndex("isIndexable", "isIndexable"); + }).createIndex("status", "status"); } }, blocking() { @@ -176,7 +181,7 @@ export const saveFaceIndex = async (faceIndex: FaceIndex) => { indexStore.put(faceIndex), statusStore.put({ fileID: faceIndex.fileID, - isIndexable: 0, + status: "indexed", failureCount: 0, }), tx.done, @@ -208,7 +213,7 @@ export const addFileEntry = async (fileID: number) => { if ((await tx.store.getKey(fileID)) === undefined) { await tx.store.put({ fileID, - isIndexable: 1, + status: "indexable", failureCount: 0, }); } @@ -248,7 +253,7 @@ export const syncWithLocalIndexableFileIDs = async (localFileIDs: number[]) => { newFileIDs.map((id) => tx.objectStore("file-status").put({ fileID: id, - isIndexable: 1, + status: "indexable", failureCount: 0, }), ), @@ -263,16 +268,20 @@ export const syncWithLocalIndexableFileIDs = async (localFileIDs: number[]) => { /** * Return the count of files that can be, and that have been, indexed. + * + * These counts are mutually exclusive. The total number of files that fall + * within the purview of the indexer is thus indexable + indexed. */ export const indexedAndIndexableCounts = async () => { const db = await faceDB(); - const tx = db.transaction(["face-index", "file-status"], "readwrite"); - const indexedCount = await tx.objectStore("face-index").count(); - const indexableCount = await tx - .objectStore("file-status") - .index("isIndexable") - .count(IDBKeyRange.only(1)); - return { indexedCount, indexableCount }; + const tx = db.transaction("file-status", "readwrite"); + const indexableCount = await tx.store + .index("status") + .count(IDBKeyRange.only("indexable")); + const indexedCount = await tx.store + .index("status") + .count(IDBKeyRange.only("indexed")); + return { indexableCount, indexedCount }; }; /** @@ -281,14 +290,16 @@ export const indexedAndIndexableCounts = async () => { * This list is from the universe of the file IDs that the face DB knows about * (can use {@link addFileEntry} to inform it about new files). From this * universe, we filter out fileIDs the files corresponding to which have already - * been indexed, or for which we attempted indexing but failed. + * been indexed, or which should be ignored. * * @param count Limit the result to up to {@link count} items. */ -export const unindexedFileIDs = async (count?: number) => { +export const indexableFileIDs = async (count?: number) => { const db = await faceDB(); const tx = db.transaction("file-status", "readonly"); - return tx.store.index("isIndexable").getAllKeys(IDBKeyRange.only(1), count); + return tx.store + .index("status") + .getAllKeys(IDBKeyRange.only("indexable"), count); }; /** @@ -306,7 +317,7 @@ export const markIndexingFailed = async (fileID: number) => { const failureCount = ((await tx.store.get(fileID)).failureCount ?? 0) + 1; await tx.store.put({ fileID, - isIndexable: 0, + status: "ignored", failureCount, }); return tx.done; diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index e2b62e51d0..98eaaa02ce 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -10,9 +10,9 @@ import type { EnteFile } from "types/file"; import { isInternalUserForML } from "utils/user"; import { faceIndex, + indexableFileIDs, indexedAndIndexableCounts, syncWithLocalIndexableFileIDs, - unindexedFileIDs, } from "./db"; import { FaceIndexerWorker } from "./indexer.worker"; @@ -166,7 +166,7 @@ export const faceIndexingStatus = async (): Promise => { const { indexedCount, indexableCount } = await indexedAndIndexableCounts(); let phase: FaceIndexingStatus["phase"]; - if (indexedCount < indexableCount) { + if (indexableCount > 0) { if (!isSyncing) { phase = "scheduled"; } else { @@ -179,7 +179,7 @@ export const faceIndexingStatus = async (): Promise => { return { phase, nSyncedFiles: indexedCount, - nTotalFiles: indexableCount, + nTotalFiles: indexableCount + indexedCount, }; }; @@ -243,6 +243,6 @@ export const getFilesToIndex = async (userID: number, count: number) => { await syncWithLocalIndexableFileIDs([...filesByID.keys()]); - const fileIDsToIndex = await unindexedFileIDs(count); + const fileIDsToIndex = await indexableFileIDs(count); return fileIDsToIndex.map((id) => ensure(filesByID.get(id))); }; From 29f89ab901aef7063a87ba4290957e873b370d13 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 31 May 2024 09:15:50 +0530 Subject: [PATCH 229/354] Skip --- web/apps/photos/src/services/searchService.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/web/apps/photos/src/services/searchService.ts b/web/apps/photos/src/services/searchService.ts index be7f574a7b..d5e17c269a 100644 --- a/web/apps/photos/src/services/searchService.ts +++ b/web/apps/photos/src/services/searchService.ts @@ -20,7 +20,7 @@ import { getFormattedDate } from "utils/search"; import { clipService, computeClipMatchScore } from "./clip-service"; import { localCLIPEmbeddings } from "./embeddingService"; import { getLatestEntities } from "./entityService"; -import { faceIndexingStatus } from "./face/indexer"; +import { faceIndexingStatus, isFaceIndexingEnabled } from "./face/indexer"; import locationSearchService, { City } from "./locationSearchService"; const DIGITS = new Set(["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"]); @@ -29,7 +29,9 @@ const CLIP_SCORE_THRESHOLD = 0.23; export const getDefaultOptions = async () => { return [ - await getIndexStatusSuggestion(), + // TODO-ML(MR): Skip this for now if indexing is disabled (eventually + // the indexing status should not be tied to results). + ...(await isFaceIndexingEnabled() ? [await getIndexStatusSuggestion()] : []), ...(await convertSuggestionsToOptions(await getAllPeopleSuggestion())), ].filter((t) => !!t); }; From 84ac00288502ce41ae59212d5cea3f202bfcec96 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 31 May 2024 09:22:02 +0530 Subject: [PATCH 230/354] lf --- web/apps/photos/src/services/searchService.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/web/apps/photos/src/services/searchService.ts b/web/apps/photos/src/services/searchService.ts index d5e17c269a..edc3899892 100644 --- a/web/apps/photos/src/services/searchService.ts +++ b/web/apps/photos/src/services/searchService.ts @@ -31,7 +31,9 @@ export const getDefaultOptions = async () => { return [ // TODO-ML(MR): Skip this for now if indexing is disabled (eventually // the indexing status should not be tied to results). - ...(await isFaceIndexingEnabled() ? [await getIndexStatusSuggestion()] : []), + ...((await isFaceIndexingEnabled()) + ? [await getIndexStatusSuggestion()] + : []), ...(await convertSuggestionsToOptions(await getAllPeopleSuggestion())), ].filter((t) => !!t); }; From c70c498d38fff4e64175d914f3ae1d3cdecfb3fd Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 31 May 2024 09:33:17 +0530 Subject: [PATCH 231/354] Pick from correct execution context --- web/apps/photos/src/services/face/indexer.ts | 3 +-- .../src/services/machineLearning/machineLearningService.ts | 4 ---- .../photos/src/services/machineLearning/mlWorkManager.ts | 5 +++++ 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index 98eaaa02ce..893ff0c863 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -4,7 +4,6 @@ import { ensure } from "@/utils/ensure"; import { wait } from "@/utils/promise"; import { type Remote } from "comlink"; import { getLocalFiles } from "services/fileService"; -import machineLearningService from "services/machineLearning/machineLearningService"; import mlWorkManager from "services/machineLearning/mlWorkManager"; import type { EnteFile } from "types/file"; import { isInternalUserForML } from "utils/user"; @@ -162,7 +161,7 @@ export interface FaceIndexingStatus { } export const faceIndexingStatus = async (): Promise => { - const isSyncing = machineLearningService.isSyncing; + const isSyncing = mlWorkManager.isSyncing; const { indexedCount, indexableCount } = await indexedAndIndexableCounts(); let phase: FaceIndexingStatus["phase"]; diff --git a/web/apps/photos/src/services/machineLearning/machineLearningService.ts b/web/apps/photos/src/services/machineLearning/machineLearningService.ts index 8549ec7655..640644ada4 100644 --- a/web/apps/photos/src/services/machineLearning/machineLearningService.ts +++ b/web/apps/photos/src/services/machineLearning/machineLearningService.ts @@ -45,8 +45,6 @@ class MachineLearningService { private localSyncContext: Promise; private syncContext: Promise; - public isSyncing = false; - public async sync( token: string, userID: number, @@ -70,7 +68,6 @@ class MachineLearningService { } private async syncFiles(syncContext: MLSyncContext) { - this.isSyncing = true; try { const functions = syncContext.outOfSyncFiles.map( (outOfSyncfile) => async () => { @@ -90,7 +87,6 @@ class MachineLearningService { syncContext.error = error; } await syncContext.syncQueue.onIdle(); - this.isSyncing = false; } private async getSyncContext( diff --git a/web/apps/photos/src/services/machineLearning/mlWorkManager.ts b/web/apps/photos/src/services/machineLearning/mlWorkManager.ts index 9502c5a75d..867df8ef65 100644 --- a/web/apps/photos/src/services/machineLearning/mlWorkManager.ts +++ b/web/apps/photos/src/services/machineLearning/mlWorkManager.ts @@ -97,6 +97,8 @@ class MLWorkManager { private liveSyncWorker: ComlinkWorker; private mlSearchEnabled: boolean; + public isSyncing = false; + constructor() { this.liveSyncQueue = new PQueue({ concurrency: 1, @@ -270,6 +272,7 @@ class MLWorkManager { * things pending to process, so we should chug along at full speed. */ private async runMLSyncJob(): Promise { + this.isSyncing = true; try { // TODO: skipping is not required if we are caching chunks through service worker // currently worker chunk itself is not loaded when network is not there @@ -290,6 +293,8 @@ class MLWorkManager { // TODO: redirect/refresh to gallery in case of session_expired, stop ml sync job } catch (e) { log.error("Failed to run MLSync Job", e); + } finally { + this.isSyncing = false; } } From 113a949a4bb6e368231716650097b368da8a6902 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 31 May 2024 09:49:55 +0530 Subject: [PATCH 232/354] wip --- web/apps/photos/src/services/face/indexer.ts | 21 +++++++++++++------- web/apps/photos/src/services/fileService.ts | 17 ++++++++++------ 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index 893ff0c863..4b6d2a92ea 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -69,6 +69,8 @@ class FaceIndexer { this.tick(); } + /* TODO-ML(MR): This code is not currently in use */ + /** * A promise for the lazily created singleton {@link FaceIndexerWorker} remote * exposed by this module. @@ -231,17 +233,22 @@ export const setIsFaceIndexingEnabled = async (enabled: boolean) => { * @param count Limit the resulting list of files to {@link count}. */ export const getFilesToIndex = async (userID: number, count: number) => { - const localFiles = await getLocalFiles(); const indexableTypes = [FILE_TYPE.IMAGE, FILE_TYPE.LIVE_PHOTO]; - const indexableFiles = localFiles.filter( - (f) => - f.ownerID == userID && indexableTypes.includes(f.metadata.fileType), - ); + const isIndexable = (f: EnteFile) => + f.ownerID == userID && indexableTypes.includes(f.metadata.fileType); - const filesByID = new Map(indexableFiles.map((f) => [f.id, f])); + const normalFiles = await getLocalFiles("normal"); + const hiddenFiles = await getLocalFiles("hidden"); + const indexableNormalFiles = normalFiles.filter(isIndexable); + const indexableHiddenFiles = hiddenFiles.filter(isIndexable); + + const normalFilesByID = new Map(indexableNormalFiles.map((f) => [f.id, f])); + const hiddenFilesByID = new Map(indexableHiddenFiles.map((f) => [f.id, f])); await syncWithLocalIndexableFileIDs([...filesByID.keys()]); const fileIDsToIndex = await indexableFileIDs(count); - return fileIDsToIndex.map((id) => ensure(filesByID.get(id))); + return fileIDsToIndex.map((id) => + ensure(normalFilesByID.get(id) ?? hiddenFilesByID.get(id)), + ); }; diff --git a/web/apps/photos/src/services/fileService.ts b/web/apps/photos/src/services/fileService.ts index 83bd505c95..a3aa90ab04 100644 --- a/web/apps/photos/src/services/fileService.ts +++ b/web/apps/photos/src/services/fileService.ts @@ -32,6 +32,17 @@ const ENDPOINT = getEndpoint(); const FILES_TABLE = "files"; const HIDDEN_FILES_TABLE = "hidden-files"; +/** + * Return all files that we know about locally, both "normal" and "hidden". + */ +export const getAllLocalFiles = async () => + [].concat(await getLocalFiles("normal"), await getLocalFiles("hidden")); + +/** + * Return all files that we know about locally. By default it returns only + * "normal" (i.e. non-"hidden") files, but it can be passed the {@link type} + * "hidden" to get it to instead return hidden files that we know about locally. + */ export const getLocalFiles = async (type: "normal" | "hidden" = "normal") => { const tableName = type === "normal" ? FILES_TABLE : HIDDEN_FILES_TABLE; const files: Array = @@ -64,12 +75,6 @@ const setLocalFiles = async (type: "normal" | "hidden", files: EnteFile[]) => { } }; -export const getAllLocalFiles = async () => { - const normalFiles = await getLocalFiles("normal"); - const hiddenFiles = await getLocalFiles("hidden"); - return [...normalFiles, ...hiddenFiles]; -}; - export const syncFiles = async ( type: "normal" | "hidden", collections: Collection[], From beedbd0991023e04ce8f83a4db185fcbbac21003 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 31 May 2024 10:35:00 +0530 Subject: [PATCH 233/354] wip --- web/apps/photos/src/services/face/db.ts | 69 +++++++++++++------ web/apps/photos/src/services/face/indexer.ts | 26 ++++--- .../machineLearning/machineLearningService.ts | 7 +- 3 files changed, 71 insertions(+), 31 deletions(-) diff --git a/web/apps/photos/src/services/face/db.ts b/web/apps/photos/src/services/face/db.ts index e257482213..797806a154 100644 --- a/web/apps/photos/src/services/face/db.ts +++ b/web/apps/photos/src/services/face/db.ts @@ -1,5 +1,6 @@ import log from "@/next/log"; import { deleteDB, openDB, type DBSchema } from "idb"; +import type { EnteFile } from "types/file"; import type { FaceIndex } from "./types"; /** @@ -46,11 +47,7 @@ interface FileStatus { * - "indexed" - We have a corresponding entry for this file in the * "face-index" object (either indexed locally or fetched from remote). * - * - "ignored" - Ignore this file when considering indexes. Some reasons - * include: - * - * - Indexeing might've failed (in which case there won't even be a - * corresponding entry for this file in "face-index"). + * - "failed" - Indexing was attempted but failed. * * - We might have a "face-index" for this file, but we should not use it * because, say, the file is currently hidden. @@ -59,7 +56,19 @@ interface FileStatus { * efficiently select or count {@link fileIDs} that fall into various * buckets. */ - status: "indexable" | "indexed" | "ignored"; + status: "indexable" | "indexed" | "failed"; + /** + * `1` if this file is currently hidden, `0` otherwise. + * + * [Note: Boolean IndexedDB indexes]. + * + * IndexedDB does not (currently) supported indexes on boolean fields. + * https://github.com/w3c/IndexedDB/issues/76 + * + * As a workaround, we use numeric fields where `0` denotes `false` and `1` + * denotes `true`. + */ + isHidden: number; /** * The number of times attempts to index this file failed. * @@ -169,10 +178,15 @@ export const clearFaceData = async () => { * @param faceIndex A {@link FaceIndex} representing the faces that we detected * (and their corresponding embeddings) in some file. * + * @param isHidden `true` if the file is currently hidden. + * * This function adds a new entry, overwriting any existing ones (No merging is * performed, the existing entry is unconditionally overwritten). */ -export const saveFaceIndex = async (faceIndex: FaceIndex) => { +export const saveFaceIndex = async ( + faceIndex: FaceIndex, + isHidden: boolean, +) => { const db = await faceDB(); const tx = db.transaction(["face-index", "file-status"], "readwrite"); const indexStore = tx.objectStore("face-index"); @@ -182,10 +196,11 @@ export const saveFaceIndex = async (faceIndex: FaceIndex) => { statusStore.put({ fileID: faceIndex.fileID, status: "indexed", + isHidden: isHidden ? 1 : 0, failureCount: 0, }), tx.done, - ]); + ]).then(() => {}); }; /** @@ -202,18 +217,21 @@ export const faceIndex = async (fileID: number) => { * * @param fileID The ID of an {@link EnteFile}. * + * @param isHidden `true` if the file is currently hidden. + * * This function does not overwrite existing entries. If an entry already exists * for the given {@link fileID} (e.g. if it was indexed and * {@link saveFaceIndex} called with the result), its existing status remains * unperturbed. */ -export const addFileEntry = async (fileID: number) => { +export const addFileEntry = async (fileID: number, isHidden: boolean) => { const db = await faceDB(); const tx = db.transaction("file-status", "readwrite"); if ((await tx.store.getKey(fileID)) === undefined) { await tx.store.put({ fileID, status: "indexable", + isHidden: isHidden ? 1 : 0, failureCount: 0, }); } @@ -221,15 +239,19 @@ export const addFileEntry = async (fileID: number) => { }; /** - * Sync entries in the face DB to align with the given list of local indexable - * file IDs. + * Sync entries in the face DB to align with the state of local files outside + * face DB. * - * @param localFileIDs The IDs of all the files that the client is aware of, - * filtered to only keep the files that the user owns and the formats that can - * be indexed by our current face indexing pipeline. + * @param indexableNormalFilesByID Local {@link EnteFile}s, keyed by their IDs. + * These are all the (non-hidden) files that the client is aware of, filtered to + * only keep the files that the user owns and the formats that can be indexed by + * our current face indexing pipeline. + * + * @param indexableHiddenFilesByID Similar to {@link indexableNormalFilesByID} + * except for hidden files. * * This function syncs the state of file entries in face DB to the state of file - * entries stored otherwise by the local client. + * entries stored otherwise by the client locally. * * - Files (identified by their ID) that are present locally but are not yet in * face DB get a fresh entry in face DB (and are marked as indexable). @@ -237,12 +259,16 @@ export const addFileEntry = async (fileID: number) => { * - Files that are not present locally but still exist in face DB are removed * from face DB (including its face index, if any). */ -export const syncWithLocalIndexableFileIDs = async (localFileIDs: number[]) => { +export const syncWithLocalFiles = async ( + indexableNormalFilesByID: Map, + indexableHiddenFilesByID: Map, +) => { const db = await faceDB(); const tx = db.transaction(["face-index", "file-status"], "readwrite"); const fdbFileIDs = await tx.objectStore("file-status").getAllKeys(); - const local = new Set(localFileIDs); + const normal = new Set(indexableNormalFilesByID.keys()); + const hidden = new Set(indexableHiddenFilesByID.keys()); const fdb = new Set(fdbFileIDs); const newFileIDs = localFileIDs.filter((id) => !fdb.has(id)); @@ -307,17 +333,20 @@ export const indexableFileIDs = async (count?: number) => { * * @param fileID The ID of an {@link EnteFile}. * + * @param isHidden `true` if the file is currently hidden. + * * If an entry does not exist yet for the given file, then a new one is created * and its failure count is set to 1. Otherwise the failure count of the * existing entry is incremented. */ -export const markIndexingFailed = async (fileID: number) => { +export const markIndexingFailed = async (fileID: number, isHidden: boolean) => { const db = await faceDB(); const tx = db.transaction("file-status", "readwrite"); - const failureCount = ((await tx.store.get(fileID)).failureCount ?? 0) + 1; + const failureCount = ((await tx.store.get(fileID))?.failureCount ?? 0) + 1; await tx.store.put({ fileID, - status: "ignored", + status: "failed", + isHidden: isHidden ? 1 : 0, failureCount, }); return tx.done; diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index 4b6d2a92ea..41d3fefd7d 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -11,7 +11,7 @@ import { faceIndex, indexableFileIDs, indexedAndIndexableCounts, - syncWithLocalIndexableFileIDs, + syncWithLocalFiles, } from "./db"; import { FaceIndexerWorker } from "./indexer.worker"; @@ -226,29 +226,37 @@ export const setIsFaceIndexingEnabled = async (enabled: boolean) => { * the next {@link count} files that still need to be indexed. * * For more specifics of what a "sync" entails, see - * {@link syncWithLocalIndexableFileIDs}. + * {@link syncWithLocalFiles}. * * @param userID Limit indexing to files owned by a {@link userID}. * * @param count Limit the resulting list of files to {@link count}. */ -export const getFilesToIndex = async (userID: number, count: number) => { +export const syncAndGetFilesToIndex = async (userID: number, count: number) => { const indexableTypes = [FILE_TYPE.IMAGE, FILE_TYPE.LIVE_PHOTO]; const isIndexable = (f: EnteFile) => f.ownerID == userID && indexableTypes.includes(f.metadata.fileType); const normalFiles = await getLocalFiles("normal"); const hiddenFiles = await getLocalFiles("hidden"); - const indexableNormalFiles = normalFiles.filter(isIndexable); - const indexableHiddenFiles = hiddenFiles.filter(isIndexable); - const normalFilesByID = new Map(indexableNormalFiles.map((f) => [f.id, f])); - const hiddenFilesByID = new Map(indexableHiddenFiles.map((f) => [f.id, f])); + const indexableNormalFilesByID = new Map( + normalFiles.filter(isIndexable).map((f) => [f.id, f]), + ); + const indexableHiddenFilesByID = new Map( + hiddenFiles.filter(isIndexable).map((f) => [f.id, f]), + ); - await syncWithLocalIndexableFileIDs([...filesByID.keys()]); + await syncWithLocalFiles( + indexableNormalFilesByID, + indexableHiddenFilesByID, + ); const fileIDsToIndex = await indexableFileIDs(count); return fileIDsToIndex.map((id) => - ensure(normalFilesByID.get(id) ?? hiddenFilesByID.get(id)), + ensure( + indexableNormalFilesByID.get(id) ?? + indexableHiddenFilesByID.get(id), + ), ); }; diff --git a/web/apps/photos/src/services/machineLearning/machineLearningService.ts b/web/apps/photos/src/services/machineLearning/machineLearningService.ts index 640644ada4..b4536d2b46 100644 --- a/web/apps/photos/src/services/machineLearning/machineLearningService.ts +++ b/web/apps/photos/src/services/machineLearning/machineLearningService.ts @@ -1,7 +1,7 @@ import log from "@/next/log"; import { CustomError, parseUploadErrorCodes } from "@ente/shared/error"; import PQueue from "p-queue"; -import { getFilesToIndex } from "services/face/indexer"; +import { syncAndGetFilesToIndex } from "services/face/indexer"; import { FaceIndexerWorker } from "services/face/indexer.worker"; import { EnteFile } from "types/file"; @@ -56,7 +56,10 @@ class MachineLearningService { const syncContext = await this.getSyncContext(token, userID, userAgent); - syncContext.outOfSyncFiles = await getFilesToIndex(userID, batchSize); + syncContext.outOfSyncFiles = await syncAndGetFilesToIndex( + userID, + batchSize, + ); if (syncContext.outOfSyncFiles.length > 0) { await this.syncFiles(syncContext); From 5049b5cc4e1a07ca8f9823e8beea93fd202b7874 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 31 May 2024 10:47:06 +0530 Subject: [PATCH 234/354] wip --- web/apps/photos/src/services/face/db.ts | 5 ++++- web/apps/photos/src/services/face/f-index.ts | 3 +-- .../photos/src/services/face/indexer.worker.ts | 18 ++++++++++++++---- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/web/apps/photos/src/services/face/db.ts b/web/apps/photos/src/services/face/db.ts index 797806a154..79f5fe54e1 100644 --- a/web/apps/photos/src/services/face/db.ts +++ b/web/apps/photos/src/services/face/db.ts @@ -269,10 +269,12 @@ export const syncWithLocalFiles = async ( const normal = new Set(indexableNormalFilesByID.keys()); const hidden = new Set(indexableHiddenFilesByID.keys()); + const all = new Set([...normal, ...hidden]); const fdb = new Set(fdbFileIDs); + const localFileIDs = [...all]; const newFileIDs = localFileIDs.filter((id) => !fdb.has(id)); - const removedFileIDs = fdbFileIDs.filter((id) => !local.has(id)); + const removedFileIDs = fdbFileIDs.filter((id) => !all.has(id)); return Promise.all( [ @@ -280,6 +282,7 @@ export const syncWithLocalFiles = async ( tx.objectStore("file-status").put({ fileID: id, status: "indexable", + isHidden: hidden.has(id) ? 1 : 0, failureCount: 0, }), ), diff --git a/web/apps/photos/src/services/face/f-index.ts b/web/apps/photos/src/services/face/f-index.ts index 79a428d0bc..7a61b55b1e 100644 --- a/web/apps/photos/src/services/face/f-index.ts +++ b/web/apps/photos/src/services/face/f-index.ts @@ -45,8 +45,7 @@ import type { Box, Dimensions, Face, Point } from "./types"; * available. These are used when they are provided, otherwise the file is * downloaded and decrypted from remote. * - * @param userAgent The UA of the current client (the client that is generating - * the embedding). + * @param userAgent The UA of the client that is doing the indexing (us). */ export const indexFaces = async ( enteFile: EnteFile, diff --git a/web/apps/photos/src/services/face/indexer.worker.ts b/web/apps/photos/src/services/face/indexer.worker.ts index 969a800295..a1a457b5cb 100644 --- a/web/apps/photos/src/services/face/indexer.worker.ts +++ b/web/apps/photos/src/services/face/indexer.worker.ts @@ -20,7 +20,7 @@ import type { FaceIndex } from "./types"; * comlink workers are structured. */ export class FaceIndexerWorker { - /* + /** * Index faces in a file, save the persist the results locally, and put them * on remote. * @@ -31,8 +31,18 @@ export class FaceIndexerWorker { * cases, pass a web {@link File} object to use that its data directly for * face indexing. If this is not provided, then the file's contents will be * downloaded and decrypted from remote. + * + * @param isHidden `true` if the file we're trying to index is currently + * hidden. + * + * @param userAgent The UA of the client that is doing the indexing (us). */ - async index(enteFile: EnteFile, file: File | undefined, userAgent: string) { + async index( + enteFile: EnteFile, + file: File | undefined, + isHidden: boolean, + userAgent: string, + ) { const f = fileLogID(enteFile); const startTime = Date.now(); @@ -44,13 +54,13 @@ export class FaceIndexerWorker { // failed, not if there were subsequent failures (like when trying // to put the result to remote or save it to the local face DB). log.error(`Failed to index faces in ${f}`, e); - markIndexingFailed(enteFile.id); + markIndexingFailed(enteFile.id, isHidden); throw e; } try { await putFaceIndex(enteFile, faceIndex); - await saveFaceIndex(faceIndex); + await saveFaceIndex(faceIndex, isHidden); } catch (e) { log.error(`Failed to put/save face index for ${f}`, e); throw e; From 27a5aa99c0350599e9adaa7b1e72efb466020d80 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 31 May 2024 10:56:42 +0530 Subject: [PATCH 235/354] wrap 1 --- web/apps/photos/src/services/face/indexer.ts | 24 +++++++++++------ .../machineLearning/machineLearningService.ts | 26 +++++++++++++++---- 2 files changed, 37 insertions(+), 13 deletions(-) diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index 41d3fefd7d..509037e101 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -221,9 +221,12 @@ export const setIsFaceIndexingEnabled = async (enabled: boolean) => { else localStorage.removeItem("faceIndexingEnabled"); }; +export type IndexableEnteFile = { enteFile: EnteFile; isHidden: boolean }; + /** * Sync face DB with the local indexable files that we know about. Then return - * the next {@link count} files that still need to be indexed. + * the next {@link count} files that still need to be indexed alongwith a flag + * for whether they are currently hidden. * * For more specifics of what a "sync" entails, see * {@link syncWithLocalFiles}. @@ -232,7 +235,10 @@ export const setIsFaceIndexingEnabled = async (enabled: boolean) => { * * @param count Limit the resulting list of files to {@link count}. */ -export const syncAndGetFilesToIndex = async (userID: number, count: number) => { +export const syncAndGetFilesToIndex = async ( + userID: number, + count: number, +): Promise => { const indexableTypes = [FILE_TYPE.IMAGE, FILE_TYPE.LIVE_PHOTO]; const isIndexable = (f: EnteFile) => f.ownerID == userID && indexableTypes.includes(f.metadata.fileType); @@ -253,10 +259,12 @@ export const syncAndGetFilesToIndex = async (userID: number, count: number) => { ); const fileIDsToIndex = await indexableFileIDs(count); - return fileIDsToIndex.map((id) => - ensure( - indexableNormalFilesByID.get(id) ?? - indexableHiddenFilesByID.get(id), - ), - ); + return fileIDsToIndex.map((id) => { + const f = indexableNormalFilesByID.get(id); + if (f) return { enteFile: f, isHidden: false }; + return { + enteFile: ensure(indexableHiddenFilesByID.get(id)), + isHidden: true, + }; + }); }; diff --git a/web/apps/photos/src/services/machineLearning/machineLearningService.ts b/web/apps/photos/src/services/machineLearning/machineLearningService.ts index b4536d2b46..2b16b7b255 100644 --- a/web/apps/photos/src/services/machineLearning/machineLearningService.ts +++ b/web/apps/photos/src/services/machineLearning/machineLearningService.ts @@ -1,7 +1,10 @@ import log from "@/next/log"; import { CustomError, parseUploadErrorCodes } from "@ente/shared/error"; import PQueue from "p-queue"; -import { syncAndGetFilesToIndex } from "services/face/indexer"; +import { + syncAndGetFilesToIndex, + type IndexableEnteFile, +} from "services/face/indexer"; import { FaceIndexerWorker } from "services/face/indexer.worker"; import { EnteFile } from "types/file"; @@ -13,7 +16,7 @@ class MLSyncContext { public userAgent: string; public localFilesMap: Map; - public outOfSyncFiles: EnteFile[]; + public outOfSyncFiles: IndexableEnteFile[]; public nSyncedFiles: number; public error?: Error; @@ -138,12 +141,18 @@ class MachineLearningService { } public async syncLocalFile( + // eslint-disable-next-line @typescript-eslint/no-unused-vars token: string, + // eslint-disable-next-line @typescript-eslint/no-unused-vars userID: number, + // eslint-disable-next-line @typescript-eslint/no-unused-vars userAgent: string, + // eslint-disable-next-line @typescript-eslint/no-unused-vars enteFile: EnteFile, + // eslint-disable-next-line @typescript-eslint/no-unused-vars localFile?: globalThis.File, ) { + /* TODO-ML(MR): Currently not used const syncContext = await this.getLocalSyncContext( token, userID, @@ -164,15 +173,21 @@ class MachineLearningService { } catch (e) { console.error("Error while syncing local file: ", enteFile.id, e); } + */ } private async syncFileWithErrorHandler( syncContext: MLSyncContext, - enteFile: EnteFile, + { enteFile, isHidden }: IndexableEnteFile, localFile?: globalThis.File, ) { try { - await this.syncFile(enteFile, localFile, syncContext.userAgent); + await this.syncFile( + enteFile, + localFile, + isHidden, + syncContext.userAgent, + ); syncContext.nSyncedFiles += 1; } catch (e) { let error = e; @@ -197,11 +212,12 @@ class MachineLearningService { private async syncFile( enteFile: EnteFile, file: File | undefined, + isHidden: boolean, userAgent: string, ) { const worker = new FaceIndexerWorker(); - await worker.index(enteFile, file, userAgent); + await worker.index(enteFile, file, isHidden, userAgent); } } From 1227bbc4a9047fd5067b1720e6d4bbff8ba7b393 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 31 May 2024 11:12:29 +0530 Subject: [PATCH 236/354] Don't duplicate state --- web/apps/photos/src/services/face/db.ts | 62 ++++--------------- web/apps/photos/src/services/face/indexer.ts | 37 +++-------- .../src/services/face/indexer.worker.ts | 14 +---- .../machineLearning/machineLearningService.ts | 19 ++---- 4 files changed, 29 insertions(+), 103 deletions(-) diff --git a/web/apps/photos/src/services/face/db.ts b/web/apps/photos/src/services/face/db.ts index 79f5fe54e1..2a75ab7ec4 100644 --- a/web/apps/photos/src/services/face/db.ts +++ b/web/apps/photos/src/services/face/db.ts @@ -1,6 +1,5 @@ import log from "@/next/log"; import { deleteDB, openDB, type DBSchema } from "idb"; -import type { EnteFile } from "types/file"; import type { FaceIndex } from "./types"; /** @@ -49,26 +48,11 @@ interface FileStatus { * * - "failed" - Indexing was attempted but failed. * - * - We might have a "face-index" for this file, but we should not use it - * because, say, the file is currently hidden. - * * We also have a (IndexedDB) "index" on this field to allow us to * efficiently select or count {@link fileIDs} that fall into various * buckets. */ status: "indexable" | "indexed" | "failed"; - /** - * `1` if this file is currently hidden, `0` otherwise. - * - * [Note: Boolean IndexedDB indexes]. - * - * IndexedDB does not (currently) supported indexes on boolean fields. - * https://github.com/w3c/IndexedDB/issues/76 - * - * As a workaround, we use numeric fields where `0` denotes `false` and `1` - * denotes `true`. - */ - isHidden: number; /** * The number of times attempts to index this file failed. * @@ -178,15 +162,10 @@ export const clearFaceData = async () => { * @param faceIndex A {@link FaceIndex} representing the faces that we detected * (and their corresponding embeddings) in some file. * - * @param isHidden `true` if the file is currently hidden. - * * This function adds a new entry, overwriting any existing ones (No merging is * performed, the existing entry is unconditionally overwritten). */ -export const saveFaceIndex = async ( - faceIndex: FaceIndex, - isHidden: boolean, -) => { +export const saveFaceIndex = async (faceIndex: FaceIndex) => { const db = await faceDB(); const tx = db.transaction(["face-index", "file-status"], "readwrite"); const indexStore = tx.objectStore("face-index"); @@ -196,11 +175,10 @@ export const saveFaceIndex = async ( statusStore.put({ fileID: faceIndex.fileID, status: "indexed", - isHidden: isHidden ? 1 : 0, failureCount: 0, }), tx.done, - ]).then(() => {}); + ]).then(() => {} /* convert result to void */); }; /** @@ -217,21 +195,18 @@ export const faceIndex = async (fileID: number) => { * * @param fileID The ID of an {@link EnteFile}. * - * @param isHidden `true` if the file is currently hidden. - * * This function does not overwrite existing entries. If an entry already exists * for the given {@link fileID} (e.g. if it was indexed and * {@link saveFaceIndex} called with the result), its existing status remains * unperturbed. */ -export const addFileEntry = async (fileID: number, isHidden: boolean) => { +export const addFileEntry = async (fileID: number) => { const db = await faceDB(); const tx = db.transaction("file-status", "readwrite"); if ((await tx.store.getKey(fileID)) === undefined) { await tx.store.put({ fileID, status: "indexable", - isHidden: isHidden ? 1 : 0, failureCount: 0, }); } @@ -242,13 +217,10 @@ export const addFileEntry = async (fileID: number, isHidden: boolean) => { * Sync entries in the face DB to align with the state of local files outside * face DB. * - * @param indexableNormalFilesByID Local {@link EnteFile}s, keyed by their IDs. - * These are all the (non-hidden) files that the client is aware of, filtered to - * only keep the files that the user owns and the formats that can be indexed by - * our current face indexing pipeline. - * - * @param indexableHiddenFilesByID Similar to {@link indexableNormalFilesByID} - * except for hidden files. + * @param localFileIDs Local {@link EnteFile}s, keyed by their IDs. These are + * all the files that the client is aware of, filtered to only keep the files + * that the user owns and the formats that can be indexed by our current face + * indexing pipeline. * * This function syncs the state of file entries in face DB to the state of file * entries stored otherwise by the client locally. @@ -259,22 +231,16 @@ export const addFileEntry = async (fileID: number, isHidden: boolean) => { * - Files that are not present locally but still exist in face DB are removed * from face DB (including its face index, if any). */ -export const syncWithLocalFiles = async ( - indexableNormalFilesByID: Map, - indexableHiddenFilesByID: Map, -) => { +export const syncWithLocalFiles = async (localFileIDs: number[]) => { const db = await faceDB(); const tx = db.transaction(["face-index", "file-status"], "readwrite"); const fdbFileIDs = await tx.objectStore("file-status").getAllKeys(); - const normal = new Set(indexableNormalFilesByID.keys()); - const hidden = new Set(indexableHiddenFilesByID.keys()); - const all = new Set([...normal, ...hidden]); + const local = new Set(localFileIDs); const fdb = new Set(fdbFileIDs); - const localFileIDs = [...all]; const newFileIDs = localFileIDs.filter((id) => !fdb.has(id)); - const removedFileIDs = fdbFileIDs.filter((id) => !all.has(id)); + const removedFileIDs = fdbFileIDs.filter((id) => !local.has(id)); return Promise.all( [ @@ -282,7 +248,6 @@ export const syncWithLocalFiles = async ( tx.objectStore("file-status").put({ fileID: id, status: "indexable", - isHidden: hidden.has(id) ? 1 : 0, failureCount: 0, }), ), @@ -292,7 +257,7 @@ export const syncWithLocalFiles = async ( removedFileIDs.map((id) => tx.objectStore("face-index").delete(id)), tx.done, ].flat(), - ); + ).then(() => {} /* convert result to void */); }; /** @@ -336,20 +301,17 @@ export const indexableFileIDs = async (count?: number) => { * * @param fileID The ID of an {@link EnteFile}. * - * @param isHidden `true` if the file is currently hidden. - * * If an entry does not exist yet for the given file, then a new one is created * and its failure count is set to 1. Otherwise the failure count of the * existing entry is incremented. */ -export const markIndexingFailed = async (fileID: number, isHidden: boolean) => { +export const markIndexingFailed = async (fileID: number) => { const db = await faceDB(); const tx = db.transaction("file-status", "readwrite"); const failureCount = ((await tx.store.get(fileID))?.failureCount ?? 0) + 1; await tx.store.put({ fileID, status: "failed", - isHidden: isHidden ? 1 : 0, failureCount, }); return tx.done; diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index 509037e101..5e6d7446a6 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -3,7 +3,7 @@ import { ComlinkWorker } from "@/next/worker/comlink-worker"; import { ensure } from "@/utils/ensure"; import { wait } from "@/utils/promise"; import { type Remote } from "comlink"; -import { getLocalFiles } from "services/fileService"; +import { getAllLocalFiles } from "services/fileService"; import mlWorkManager from "services/machineLearning/mlWorkManager"; import type { EnteFile } from "types/file"; import { isInternalUserForML } from "utils/user"; @@ -221,15 +221,11 @@ export const setIsFaceIndexingEnabled = async (enabled: boolean) => { else localStorage.removeItem("faceIndexingEnabled"); }; -export type IndexableEnteFile = { enteFile: EnteFile; isHidden: boolean }; - /** * Sync face DB with the local indexable files that we know about. Then return - * the next {@link count} files that still need to be indexed alongwith a flag - * for whether they are currently hidden. + * the next {@link count} files that still need to be indexed. * - * For more specifics of what a "sync" entails, see - * {@link syncWithLocalFiles}. + * For more specifics of what a "sync" entails, see {@link syncWithLocalFiles}. * * @param userID Limit indexing to files owned by a {@link userID}. * @@ -238,33 +234,18 @@ export type IndexableEnteFile = { enteFile: EnteFile; isHidden: boolean }; export const syncAndGetFilesToIndex = async ( userID: number, count: number, -): Promise => { +): Promise => { const indexableTypes = [FILE_TYPE.IMAGE, FILE_TYPE.LIVE_PHOTO]; const isIndexable = (f: EnteFile) => f.ownerID == userID && indexableTypes.includes(f.metadata.fileType); - const normalFiles = await getLocalFiles("normal"); - const hiddenFiles = await getLocalFiles("hidden"); - - const indexableNormalFilesByID = new Map( - normalFiles.filter(isIndexable).map((f) => [f.id, f]), - ); - const indexableHiddenFilesByID = new Map( - hiddenFiles.filter(isIndexable).map((f) => [f.id, f]), + const localFiles = await getAllLocalFiles(); + const localFilesByID = new Map( + localFiles.filter(isIndexable).map((f) => [f.id, f]), ); - await syncWithLocalFiles( - indexableNormalFilesByID, - indexableHiddenFilesByID, - ); + await syncWithLocalFiles([...localFilesByID.keys()]); const fileIDsToIndex = await indexableFileIDs(count); - return fileIDsToIndex.map((id) => { - const f = indexableNormalFilesByID.get(id); - if (f) return { enteFile: f, isHidden: false }; - return { - enteFile: ensure(indexableHiddenFilesByID.get(id)), - isHidden: true, - }; - }); + return fileIDsToIndex.map((id) => ensure(localFilesByID.get(id))); }; diff --git a/web/apps/photos/src/services/face/indexer.worker.ts b/web/apps/photos/src/services/face/indexer.worker.ts index a1a457b5cb..6cf1e5008f 100644 --- a/web/apps/photos/src/services/face/indexer.worker.ts +++ b/web/apps/photos/src/services/face/indexer.worker.ts @@ -32,17 +32,9 @@ export class FaceIndexerWorker { * face indexing. If this is not provided, then the file's contents will be * downloaded and decrypted from remote. * - * @param isHidden `true` if the file we're trying to index is currently - * hidden. - * * @param userAgent The UA of the client that is doing the indexing (us). */ - async index( - enteFile: EnteFile, - file: File | undefined, - isHidden: boolean, - userAgent: string, - ) { + async index(enteFile: EnteFile, file: File | undefined, userAgent: string) { const f = fileLogID(enteFile); const startTime = Date.now(); @@ -54,13 +46,13 @@ export class FaceIndexerWorker { // failed, not if there were subsequent failures (like when trying // to put the result to remote or save it to the local face DB). log.error(`Failed to index faces in ${f}`, e); - markIndexingFailed(enteFile.id, isHidden); + markIndexingFailed(enteFile.id); throw e; } try { await putFaceIndex(enteFile, faceIndex); - await saveFaceIndex(faceIndex, isHidden); + await saveFaceIndex(faceIndex); } catch (e) { log.error(`Failed to put/save face index for ${f}`, e); throw e; diff --git a/web/apps/photos/src/services/machineLearning/machineLearningService.ts b/web/apps/photos/src/services/machineLearning/machineLearningService.ts index 2b16b7b255..f28efe8b47 100644 --- a/web/apps/photos/src/services/machineLearning/machineLearningService.ts +++ b/web/apps/photos/src/services/machineLearning/machineLearningService.ts @@ -1,10 +1,7 @@ import log from "@/next/log"; import { CustomError, parseUploadErrorCodes } from "@ente/shared/error"; import PQueue from "p-queue"; -import { - syncAndGetFilesToIndex, - type IndexableEnteFile, -} from "services/face/indexer"; +import { syncAndGetFilesToIndex } from "services/face/indexer"; import { FaceIndexerWorker } from "services/face/indexer.worker"; import { EnteFile } from "types/file"; @@ -16,7 +13,7 @@ class MLSyncContext { public userAgent: string; public localFilesMap: Map; - public outOfSyncFiles: IndexableEnteFile[]; + public outOfSyncFiles: EnteFile[]; public nSyncedFiles: number; public error?: Error; @@ -178,16 +175,11 @@ class MachineLearningService { private async syncFileWithErrorHandler( syncContext: MLSyncContext, - { enteFile, isHidden }: IndexableEnteFile, + enteFile: EnteFile, localFile?: globalThis.File, ) { try { - await this.syncFile( - enteFile, - localFile, - isHidden, - syncContext.userAgent, - ); + await this.syncFile(enteFile, localFile, syncContext.userAgent); syncContext.nSyncedFiles += 1; } catch (e) { let error = e; @@ -212,12 +204,11 @@ class MachineLearningService { private async syncFile( enteFile: EnteFile, file: File | undefined, - isHidden: boolean, userAgent: string, ) { const worker = new FaceIndexerWorker(); - await worker.index(enteFile, file, isHidden, userAgent); + await worker.index(enteFile, file, userAgent); } } From 5042e3cbd7a83f2659966a1702283b15ab251e6a Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 31 May 2024 11:51:18 +0530 Subject: [PATCH 237/354] Index video thumbnails --- web/apps/photos/src/services/face/f-index.ts | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/web/apps/photos/src/services/face/f-index.ts b/web/apps/photos/src/services/face/f-index.ts index 7a61b55b1e..07bafb8a1c 100644 --- a/web/apps/photos/src/services/face/f-index.ts +++ b/web/apps/photos/src/services/face/f-index.ts @@ -77,11 +77,21 @@ export const indexFaces = async ( /** * Return a "renderable" image blob, using {@link file} if present otherwise * downloading the source image corresponding to {@link enteFile} from remote. + * + * For videos their thumbnail is used. */ -const renderableImageBlob = async (enteFile: EnteFile, file: File) => - file - ? getRenderableImage(enteFile.metadata.title, file) - : fetchRenderableBlob(enteFile); +const renderableImageBlob = async (enteFile: EnteFile, file: File) => { + const fileType = enteFile.metadata.fileType; + if (fileType == FILE_TYPE.VIDEO) { + const thumbnailData = await DownloadManager.getThumbnail(enteFile); + const thumbnailBlob = new Blob([thumbnailData]); + return getRenderableImage(enteFile.metadata.title, thumbnailBlob); + } else { + return file + ? getRenderableImage(enteFile.metadata.title, file) + : fetchRenderableBlob(enteFile); + } +}; const fetchRenderableBlob = async (enteFile: EnteFile) => { const fileStream = await DownloadManager.getFile(enteFile); From 41124d07a53a498febb5b0089f434ee0ebdcb48c Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 31 May 2024 11:53:56 +0530 Subject: [PATCH 238/354] Shorten to original 433d0e81fc4c9bbf1a3fb7bbe6f5f7db927bfeeb --- web/apps/photos/src/services/face/f-index.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/web/apps/photos/src/services/face/f-index.ts b/web/apps/photos/src/services/face/f-index.ts index 07bafb8a1c..cdcc4fe919 100644 --- a/web/apps/photos/src/services/face/f-index.ts +++ b/web/apps/photos/src/services/face/f-index.ts @@ -84,8 +84,7 @@ const renderableImageBlob = async (enteFile: EnteFile, file: File) => { const fileType = enteFile.metadata.fileType; if (fileType == FILE_TYPE.VIDEO) { const thumbnailData = await DownloadManager.getThumbnail(enteFile); - const thumbnailBlob = new Blob([thumbnailData]); - return getRenderableImage(enteFile.metadata.title, thumbnailBlob); + return new Blob([thumbnailData]); } else { return file ? getRenderableImage(enteFile.metadata.title, file) From f0620741773f2acfa8e4c73845cf8aeacee3863b Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 31 May 2024 11:56:10 +0530 Subject: [PATCH 239/354] Index videos --- web/apps/photos/src/services/face/indexer.ts | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index 5e6d7446a6..095e65d400 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -1,4 +1,3 @@ -import { FILE_TYPE } from "@/media/file-type"; import { ComlinkWorker } from "@/next/worker/comlink-worker"; import { ensure } from "@/utils/ensure"; import { wait } from "@/utils/promise"; @@ -222,22 +221,21 @@ export const setIsFaceIndexingEnabled = async (enabled: boolean) => { }; /** - * Sync face DB with the local indexable files that we know about. Then return - * the next {@link count} files that still need to be indexed. + * Sync face DB with the local (and potentially indexable) files that we know + * about. Then return the next {@link count} files that still need to be + * indexed. * * For more specifics of what a "sync" entails, see {@link syncWithLocalFiles}. * - * @param userID Limit indexing to files owned by a {@link userID}. + * @param userID Sync only files owned by a {@link userID} with the face DB. * - * @param count Limit the resulting list of files to {@link count}. + * @param count Limit the resulting list of indexable files to {@link count}. */ export const syncAndGetFilesToIndex = async ( userID: number, count: number, ): Promise => { - const indexableTypes = [FILE_TYPE.IMAGE, FILE_TYPE.LIVE_PHOTO]; - const isIndexable = (f: EnteFile) => - f.ownerID == userID && indexableTypes.includes(f.metadata.fileType); + const isIndexable = (f: EnteFile) => f.ownerID == userID; const localFiles = await getAllLocalFiles(); const localFilesByID = new Map( From 52b3a6d0f7d88ca4f3718c559ce255afef2d8578 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 31 May 2024 13:14:56 +0530 Subject: [PATCH 240/354] [desktop] Tweak the nightly build flow If we publish a tag when publishing the pre-release, it then triggers another workflow invocation that fails (harmlessly). So instead, push the tag after releasing to trigger the steps. --- desktop/docs/release.md | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/desktop/docs/release.md b/desktop/docs/release.md index f56d2c4404..76d7f9d487 100644 --- a/desktop/docs/release.md +++ b/desktop/docs/release.md @@ -17,8 +17,8 @@ to keep a separate (non-mono) repository just for doing releases. ## Workflow - Release Candidates Nightly RC builds of `main` are published by a scheduled workflow automatically. -If needed, these builds can also be manually triggered, including specifying the -source repository branch to build: +If needed, these builds can also be manually triggered, and the branch of the +source repository to build (default "main") also specified: ```sh gh workflow run desktop-release.yml --source= @@ -46,17 +46,27 @@ Each such workflow run will update the artifacts attached to the same ``` This'll trigger the workflow and create a new draft release, which you can -publish after adding the release notes. +publish after adding the release notes. Once you publish, the release goes live. -The release is done at this point, and we can now start a new RC train for -subsequent nightly builds. +The release is done at this point, and we can now create a new pre-release to +host subsequent nightly builds. -1. Update `package.json` in the source repo to use version `1.x.x-rc`. Create a - new draft release in the release repo with title `1.x.x-rc`. In the tag - input enter `v1.x.x-rc` and select the option to "create a new tag on - publish". +1. Update `package.json` in the source repo to use version `1.x.x-rc`, and + merge these changes into `main`. -## Post build +2. In the release repo: + + ```sh + git tag 1.x.x-rc + git push origin 1.x.x-rc + ``` + +3. Once the draft release is created, edit its description to "Nightly builds", + set it as a pre-release and publish. + +4. Delete the pre-release for the previous (already released) version. + +## Details The GitHub Action runs on Windows, Linux and macOS. It produces the artifacts defined in the `build` value in `package.json`. From 67169b4efa91118e78a097076b980cb28eb3a74d Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 31 May 2024 13:20:58 +0530 Subject: [PATCH 241/354] more --- desktop/docs/release.md | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/desktop/docs/release.md b/desktop/docs/release.md index 76d7f9d487..f005a20dc7 100644 --- a/desktop/docs/release.md +++ b/desktop/docs/release.md @@ -14,7 +14,7 @@ to keep a separate (non-mono) repository just for doing releases. - Releases are done from [ente-io/photos-desktop](https://github.com/ente-io/photos-desktop). -## Workflow - Release Candidates +## Workflow - Release candidates Nightly RC builds of `main` are published by a scheduled workflow automatically. If needed, these builds can also be manually triggered, and the branch of the @@ -66,6 +66,24 @@ host subsequent nightly builds. 4. Delete the pre-release for the previous (already released) version. +## Workflow - Extra Pre-releases + +If you want to create extra pre-releases in addition to the nightly `1.x.x-rc` +ones, + +1. In your branch in the source repository, set the version in `package.json` + to something different, say `1.x.x-my-test`. + +2. Create a new draft release in the release repo with title `1.x.x-test`. In + the tag input enter `v1.x.x-test` and select the option to "create a new tag + on publish". + +3. Trigger the workflow in the release repo: + + ```sh + gh workflow run desktop-release.yml --source=my-branch + ``` + ## Details The GitHub Action runs on Windows, Linux and macOS. It produces the artifacts From ddaa872b97c8af3f22c44085d2a9deb8e98a22f8 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 31 May 2024 13:25:04 +0530 Subject: [PATCH 242/354] more --- desktop/docs/release.md | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/desktop/docs/release.md b/desktop/docs/release.md index f005a20dc7..9fcb80d6c2 100644 --- a/desktop/docs/release.md +++ b/desktop/docs/release.md @@ -1,13 +1,17 @@ ## Releases -Conceptually, the release is straightforward: We trigger a GitHub workflow that -creates a draft release with artifacts built. When ready, we publish that -release. The download links on our website, and existing apps already check the -latest GitHub release and update accordingly. +Conceptually, the release is straightforward: + +1. We trigger a GitHub workflow that creates a draft release with the build. + +2. When ready, we publish that release. + +3. The download links on our website, and existing apps already check the + latest GitHub release and update automatically. The complication comes by the fact that electron-builder's auto updater (the mechanism that we use for auto updates) doesn't work with monorepos. So we need -to keep a separate (non-mono) repository just for doing releases. +to keep a separate repository just for holding the releases. - Source code lives here, in [ente-io/ente](https://github.com/ente-io/ente). @@ -61,15 +65,14 @@ host subsequent nightly builds. git push origin 1.x.x-rc ``` -3. Once the draft release is created, edit its description to "Nightly builds", - set it as a pre-release and publish. +3. Once the workflow finishes and the draft release is created, edit its + description to "Nightly builds", set it as a pre-release and publish. -4. Delete the pre-release for the previous (already released) version. +4. Delete the pre-release for the previous (already released) version. -## Workflow - Extra Pre-releases +## Workflow - Extra pre-releases -If you want to create extra pre-releases in addition to the nightly `1.x.x-rc` -ones, +To create extro one off pre-releases in addition to the nightly `1.x.x-rc` ones, 1. In your branch in the source repository, set the version in `package.json` to something different, say `1.x.x-my-test`. @@ -78,7 +81,7 @@ ones, the tag input enter `v1.x.x-test` and select the option to "create a new tag on publish". -3. Trigger the workflow in the release repo: +3. Trigger the workflow in the release repo: ```sh gh workflow run desktop-release.yml --source=my-branch From b3c907f8ee9747662d3d9aea6c4e14f13017d25f Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Fri, 31 May 2024 14:03:56 +0530 Subject: [PATCH 243/354] [mob][photos] Stale todo --- mobile/lib/generated/intl/messages_en.dart | 2 +- mobile/lib/generated/l10n.dart | 4 ++-- .../services/machine_learning/face_ml/face_ml_service.dart | 1 - 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/mobile/lib/generated/intl/messages_en.dart b/mobile/lib/generated/intl/messages_en.dart index 424b3b3aba..056ba912f5 100644 --- a/mobile/lib/generated/intl/messages_en.dart +++ b/mobile/lib/generated/intl/messages_en.dart @@ -936,7 +936,7 @@ class MessageLookup extends MessageLookupByLibrary { "memoryCount": m33, "merchandise": MessageLookupByLibrary.simpleMessage("Merchandise"), "mlIndexingDescription": MessageLookupByLibrary.simpleMessage( - "Please note that ML indexing will result in a higher bandwidth and battery usage until all items are indexed."), + "Please note that machine learning will result in a higher bandwidth and battery usage until all items are indexed."), "mobileWebDesktop": MessageLookupByLibrary.simpleMessage("Mobile, Web, Desktop"), "moderateStrength": MessageLookupByLibrary.simpleMessage("Moderate"), diff --git a/mobile/lib/generated/l10n.dart b/mobile/lib/generated/l10n.dart index 3f1b9fdb4f..2cd07cb59c 100644 --- a/mobile/lib/generated/l10n.dart +++ b/mobile/lib/generated/l10n.dart @@ -2876,10 +2876,10 @@ class S { ); } - /// `Please note that ML indexing will result in a higher bandwidth and battery usage until all items are indexed.` + /// `Please note that machine learning will result in a higher bandwidth and battery usage until all items are indexed.` String get mlIndexingDescription { return Intl.message( - 'Please note that ML indexing will result in a higher bandwidth and battery usage until all items are indexed.', + 'Please note that machine learning will result in a higher bandwidth and battery usage until all items are indexed.', name: 'mlIndexingDescription', desc: '', args: [], diff --git a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart index cffa17dcc8..bc2eac92c4 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart @@ -1027,7 +1027,6 @@ class FaceMlService { s, ); } - // TODO: This is returning null for Pragadees for all files, so something is wrong here! } if (file == null) { _logger From 3ad8f732893c4bf34271cbc5149eb753adfb9023 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 31 May 2024 14:06:06 +0530 Subject: [PATCH 244/354] mandate --- web/apps/photos/src/services/feature-flag.ts | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 web/apps/photos/src/services/feature-flag.ts diff --git a/web/apps/photos/src/services/feature-flag.ts b/web/apps/photos/src/services/feature-flag.ts new file mode 100644 index 0000000000..c6d722bf0f --- /dev/null +++ b/web/apps/photos/src/services/feature-flag.ts @@ -0,0 +1,15 @@ +/** + * Fetch feature flags (potentially user specific) from remote and save them in + * local storage for subsequent lookup. + */ +export const fetchAndSaveFeatureFlags = async () => {}; + +/** + * Return `true` if the current user is marked as an "internal" user. + */ +export const isInternalUser = async () => {}; + +/** + * Return `true` if the current user is marked as an "beta" user. + */ +export const isBetaUser = async () => {}; From 029872e54ef26f4cbe0199e008c0f14a05ad84ac Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Fri, 31 May 2024 14:07:31 +0530 Subject: [PATCH 245/354] [mob][photos] Decrease clustering bucket size --- .../lib/services/machine_learning/face_ml/face_ml_service.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart index bc2eac92c4..aac040e488 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart @@ -605,7 +605,7 @@ class FaceMlService { await FaceMLDataDB.instance.getAllClusterSummary(); if (clusterInBuckets) { - const int bucketSize = 20000; + const int bucketSize = 10000; const int offsetIncrement = 7500; int offset = 0; int bucket = 1; From c8d30323e4e91c571adcfd6ee0ac7f04f21b55a2 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 31 May 2024 14:11:16 +0530 Subject: [PATCH 246/354] Trigger --- web/apps/photos/src/pages/gallery/index.tsx | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/web/apps/photos/src/pages/gallery/index.tsx b/web/apps/photos/src/pages/gallery/index.tsx index d375e48fc8..65857d5f0a 100644 --- a/web/apps/photos/src/pages/gallery/index.tsx +++ b/web/apps/photos/src/pages/gallery/index.tsx @@ -87,6 +87,7 @@ import { import downloadManager from "services/download"; import { syncCLIPEmbeddings } from "services/embeddingService"; import { syncEntities } from "services/entityService"; +import { fetchAndSaveFeatureFlags } from "services/feature-flag"; import { getLocalFiles, syncFiles } from "services/fileService"; import locationSearchService from "services/locationSearchService"; import { getLocalTrashedFiles, syncTrash } from "services/trashService"; @@ -340,6 +341,7 @@ export default function Gallery() { return; } preloadImage("/images/subscription-card-background"); + let ffTimeout: ReturnType | undefined; const electron = globalThis.electron; const main = async () => { const valid = await validateKey(); @@ -383,6 +385,11 @@ export default function Gallery() { syncInterval.current = setInterval(() => { syncWithRemote(false, true); }, SYNC_INTERVAL_IN_MICROSECONDS); + // Not critical, so fetch these after some delay. + ffTimeout = setTimeout(() => { + ffTimeout = undefined; + void fetchAndSaveFeatureFlags(); + }, 5000); if (electron) { // void clipService.setupOnFileUploadListener(); electron.onMainWindowFocus(() => syncWithRemote(false, true)); @@ -391,6 +398,7 @@ export default function Gallery() { main(); return () => { clearInterval(syncInterval.current); + if (ffTimeout) clearTimeout(ffTimeout); if (electron) { electron.onMainWindowFocus(undefined); clipService.removeOnFileUploadListener(); From 72a3f7f17a1401c37b62d4a1781780f038a21e77 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 31 May 2024 14:25:12 +0530 Subject: [PATCH 247/354] Reduce noise in UI layer --- web/apps/photos/src/pages/gallery/index.tsx | 10 ++---- web/apps/photos/src/services/feature-flag.ts | 34 +++++++++++++++++++- 2 files changed, 35 insertions(+), 9 deletions(-) diff --git a/web/apps/photos/src/pages/gallery/index.tsx b/web/apps/photos/src/pages/gallery/index.tsx index 65857d5f0a..5062aa6b6f 100644 --- a/web/apps/photos/src/pages/gallery/index.tsx +++ b/web/apps/photos/src/pages/gallery/index.tsx @@ -87,7 +87,7 @@ import { import downloadManager from "services/download"; import { syncCLIPEmbeddings } from "services/embeddingService"; import { syncEntities } from "services/entityService"; -import { fetchAndSaveFeatureFlags } from "services/feature-flag"; +import { fetchAndSaveFeatureFlagsIfNeeded } from "services/feature-flag"; import { getLocalFiles, syncFiles } from "services/fileService"; import locationSearchService from "services/locationSearchService"; import { getLocalTrashedFiles, syncTrash } from "services/trashService"; @@ -341,7 +341,6 @@ export default function Gallery() { return; } preloadImage("/images/subscription-card-background"); - let ffTimeout: ReturnType | undefined; const electron = globalThis.electron; const main = async () => { const valid = await validateKey(); @@ -385,11 +384,7 @@ export default function Gallery() { syncInterval.current = setInterval(() => { syncWithRemote(false, true); }, SYNC_INTERVAL_IN_MICROSECONDS); - // Not critical, so fetch these after some delay. - ffTimeout = setTimeout(() => { - ffTimeout = undefined; - void fetchAndSaveFeatureFlags(); - }, 5000); + fetchAndSaveFeatureFlagsIfNeeded(); if (electron) { // void clipService.setupOnFileUploadListener(); electron.onMainWindowFocus(() => syncWithRemote(false, true)); @@ -398,7 +393,6 @@ export default function Gallery() { main(); return () => { clearInterval(syncInterval.current); - if (ffTimeout) clearTimeout(ffTimeout); if (electron) { electron.onMainWindowFocus(undefined); clipService.removeOnFileUploadListener(); diff --git a/web/apps/photos/src/services/feature-flag.ts b/web/apps/photos/src/services/feature-flag.ts index c6d722bf0f..ffde49a6a1 100644 --- a/web/apps/photos/src/services/feature-flag.ts +++ b/web/apps/photos/src/services/feature-flag.ts @@ -1,8 +1,40 @@ +let _fetchTimeout: ReturnType | undefined; +let _haveFetched = false; + +/** + * Fetch feature flags (potentially user specific) from remote and save them in + * local storage for subsequent lookup. + * + * It fetches only once per session, and so is safe to call as arbitrarily many + * times. Remember to call {@link clearFeatureFlagSessionState} on logout to + * forget that we've already fetched so that these can be fetched again on the + * subsequent login. + */ +export const fetchAndSaveFeatureFlagsIfNeeded = () => { + if (_haveFetched) return; + if (_fetchTimeout) return; + // Not critical, so fetch these after some delay. + _fetchTimeout = setTimeout(() => { + _fetchTimeout = undefined; + void fetchAndSaveFeatureFlags().then(() => { + _haveFetched = true; + }); + }, 5000); +}; + +export const clearFeatureFlagSessionState = () => { + if (_fetchTimeout) { + clearTimeout(_fetchTimeout); + _fetchTimeout = undefined; + } + _haveFetched = false; +}; + /** * Fetch feature flags (potentially user specific) from remote and save them in * local storage for subsequent lookup. */ -export const fetchAndSaveFeatureFlags = async () => {}; +const fetchAndSaveFeatureFlags = async () => {}; /** * Return `true` if the current user is marked as an "internal" user. From a850500beb9e9a5ffcfc3a0f0cf93b3733e04972 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 31 May 2024 14:29:28 +0530 Subject: [PATCH 248/354] Clear --- web/apps/photos/src/pages/gallery/index.tsx | 2 +- web/apps/photos/src/services/logout.ts | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/web/apps/photos/src/pages/gallery/index.tsx b/web/apps/photos/src/pages/gallery/index.tsx index 5062aa6b6f..a1cba432c1 100644 --- a/web/apps/photos/src/pages/gallery/index.tsx +++ b/web/apps/photos/src/pages/gallery/index.tsx @@ -384,7 +384,6 @@ export default function Gallery() { syncInterval.current = setInterval(() => { syncWithRemote(false, true); }, SYNC_INTERVAL_IN_MICROSECONDS); - fetchAndSaveFeatureFlagsIfNeeded(); if (electron) { // void clipService.setupOnFileUploadListener(); electron.onMainWindowFocus(() => syncWithRemote(false, true)); @@ -715,6 +714,7 @@ export default function Gallery() { await syncTrash(collections, setTrashedFiles); await syncEntities(); await syncMapEnabled(); + fetchAndSaveFeatureFlagsIfNeeded(); const electron = globalThis.electron; if (electron) { await syncCLIPEmbeddings(); diff --git a/web/apps/photos/src/services/logout.ts b/web/apps/photos/src/services/logout.ts index 4e09516dee..9da892bad2 100644 --- a/web/apps/photos/src/services/logout.ts +++ b/web/apps/photos/src/services/logout.ts @@ -4,6 +4,7 @@ import { clipService } from "services/clip-service"; import DownloadManager from "./download"; import exportService from "./export"; import { clearFaceData } from "./face/db"; +import { clearFeatureFlagSessionState } from "./feature-flag"; import mlWorkManager from "./machineLearning/mlWorkManager"; /** @@ -19,6 +20,12 @@ export const photosLogout = async () => { await accountLogout(); + try { + clearFeatureFlagSessionState(); + } catch (e) { + ignoreError("feature-flag", e); + } + try { await DownloadManager.logout(); } catch (e) { From 9a7ba8a4069373de4b006e61f1f55c1b3e8bf022 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 31 May 2024 14:40:44 +0530 Subject: [PATCH 249/354] Alias --- web/packages/shared/network/api.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/web/packages/shared/network/api.ts b/web/packages/shared/network/api.ts index 3cda6b615d..7ba49ec960 100644 --- a/web/packages/shared/network/api.ts +++ b/web/packages/shared/network/api.ts @@ -1,3 +1,13 @@ +/** + * Return the origin (scheme, host, port triple) that should be used for making + * API requests to museum. + * + * This defaults to api.ente.io, Ente's own servers, but can be overridden when + * running locally by setting the `NEXT_PUBLIC_ENTE_ENDPOINT` environment + * variable. + */ +export const apiOrigin = () => getEndpoint(); + export const getEndpoint = () => { const endpoint = process.env.NEXT_PUBLIC_ENTE_ENDPOINT; if (endpoint) { From f6bdeef33d80fe0e151b0fab53605397fd882dfd Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Fri, 31 May 2024 14:47:03 +0530 Subject: [PATCH 250/354] [mob][photos] Show when clustering is running --- .../machine_learning/face_ml/face_ml_service.dart | 6 ++++++ .../lib/ui/settings/machine_learning_settings_page.dart | 8 ++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart index aac040e488..87a707995c 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart @@ -91,7 +91,10 @@ class FaceMlService { bool isInitialized = false; late String client; + bool get showClusteringIsHappening => _showClusteringIsHappening; + bool debugIndexingDisabled = false; + bool _showClusteringIsHappening = false; bool _mlControllerStatus = false; bool _isIndexingOrClusteringRunning = false; bool _shouldPauseIndexingAndClustering = false; @@ -572,6 +575,8 @@ class FaceMlService { await PersonService.instance.fetchRemoteClusterFeedback(); try { + _showClusteringIsHappening = true; + // Get a sense of the total number of faces in the database final int totalFaces = await FaceMLDataDB.instance.getTotalFaceCount(minFaceScore: minFaceScore); @@ -711,6 +716,7 @@ class FaceMlService { } catch (e, s) { _logger.severe("`clusterAllImages` failed", e, s); } finally { + _showClusteringIsHappening = false; _isIndexingOrClusteringRunning = false; _shouldPauseIndexingAndClustering = false; } diff --git a/mobile/lib/ui/settings/machine_learning_settings_page.dart b/mobile/lib/ui/settings/machine_learning_settings_page.dart index c39d3e6ee9..38685b4ff5 100644 --- a/mobile/lib/ui/settings/machine_learning_settings_page.dart +++ b/mobile/lib/ui/settings/machine_learning_settings_page.dart @@ -513,14 +513,18 @@ class FaceRecognitionStatusWidgetState extends State Date: Fri, 31 May 2024 14:54:37 +0530 Subject: [PATCH 251/354] [mob][photos] Bump --- mobile/pubspec.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobile/pubspec.yaml b/mobile/pubspec.yaml index 93fd755649..423ab8e681 100644 --- a/mobile/pubspec.yaml +++ b/mobile/pubspec.yaml @@ -12,7 +12,7 @@ description: ente photos application # Read more about iOS versioning at # https://developer.apple.com/library/archive/documentation/General/Reference/InfoPlistKeyReference/Articles/CoreFoundationKeys.html -version: 0.8.125+645 +version: 0.8.126+646 publish_to: none environment: From 133693d05849822e6b5774ac7e86f2cd0a9be4ea Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 31 May 2024 15:15:51 +0530 Subject: [PATCH 252/354] Fetch beta flag --- web/apps/photos/src/services/feature-flag.ts | 58 +++++++++++++++++++- 1 file changed, 55 insertions(+), 3 deletions(-) diff --git a/web/apps/photos/src/services/feature-flag.ts b/web/apps/photos/src/services/feature-flag.ts index ffde49a6a1..0c15ec3cbb 100644 --- a/web/apps/photos/src/services/feature-flag.ts +++ b/web/apps/photos/src/services/feature-flag.ts @@ -1,3 +1,8 @@ +import log from "@/next/log"; +import { ensure } from "@/utils/ensure"; +import { apiOrigin } from "@ente/shared/network/api"; +import { getToken } from "@ente/shared/storage/localStorage/helpers"; + let _fetchTimeout: ReturnType | undefined; let _haveFetched = false; @@ -34,7 +39,43 @@ export const clearFeatureFlagSessionState = () => { * Fetch feature flags (potentially user specific) from remote and save them in * local storage for subsequent lookup. */ -const fetchAndSaveFeatureFlags = async () => {}; +const fetchAndSaveFeatureFlags = () => + fetchFeatureFlags() + .then((res) => res.text()) + .then(saveFlagJSONString); + +const fetchFeatureFlags = async () => { + const url = `${apiOrigin}/remote-store/feature-flags`; + const res = await fetch(url, { + headers: { + "X-Auth-Token": ensure(getToken()), + }, + }); + if (!res.ok) throw new Error(`Failed to fetch ${url}: HTTP ${res.status}`); + return res; +}; + +const saveFlagJSONString = (s: string) => + localStorage.setItem("remoteFeatureFlags", s); + +const remoteFeatureFlags = () => { + const s = localStorage.getItem("remoteFeatureFlags"); + if (!s) return undefined; + return JSON.parse(s); +}; + +const remoteFeatureFlagsFetchingIfNeeded = async () => { + let ff = await remoteFeatureFlags(); + if (!ff) { + try { + await fetchAndSaveFeatureFlags(); + ff = await remoteFeatureFlags(); + } catch (e) { + log.warn("Ignoring error when fetching feature flags", e); + } + } + return ff; +}; /** * Return `true` if the current user is marked as an "internal" user. @@ -42,6 +83,17 @@ const fetchAndSaveFeatureFlags = async () => {}; export const isInternalUser = async () => {}; /** - * Return `true` if the current user is marked as an "beta" user. + * Return `true` if the current user is marked as a "beta" user. */ -export const isBetaUser = async () => {}; +export const isBetaUser = async () => { + const flags = await remoteFeatureFlagsFetchingIfNeeded(); + // TODO(MR): Use Yup here + if ( + flags && + typeof flags === "object" && + "betaUser" in flags && + typeof flags.betaUser == "boolean" + ) + return flags.betaUser; + return false; +}; From fa06a15ad7273dc962b20e53a5f8ca9f93007cd5 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 31 May 2024 15:26:36 +0530 Subject: [PATCH 253/354] Show the option for beta users too --- .../photos/src/components/ml/MLSearchSettings.tsx | 10 ++++++++-- web/apps/photos/src/services/face/indexer.ts | 15 +++++++++------ web/apps/photos/src/services/feature-flag.ts | 14 +++++++++++++- 3 files changed, 30 insertions(+), 9 deletions(-) diff --git a/web/apps/photos/src/components/ml/MLSearchSettings.tsx b/web/apps/photos/src/components/ml/MLSearchSettings.tsx index d71dffab7e..1f3ad752a3 100644 --- a/web/apps/photos/src/components/ml/MLSearchSettings.tsx +++ b/web/apps/photos/src/components/ml/MLSearchSettings.tsx @@ -18,11 +18,11 @@ import { t } from "i18next"; import { AppContext } from "pages/_app"; import { useContext, useEffect, useState } from "react"; import { Trans } from "react-i18next"; +import { canEnableFaceIndexing } from "services/face/indexer"; import { getFaceSearchEnabledStatus, updateFaceSearchEnabledStatus, } from "services/userService"; -import { isInternalUserForML } from "utils/user"; export const MLSearchSettings = ({ open, onClose, onRootClose }) => { const { @@ -258,6 +258,12 @@ function EnableMLSearch({ onClose, enableMlSearch, onRootClose }) { // const showDetails = () => // openLink("https://ente.io/blog/desktop-ml-beta", true); + const [canEnable, setCanEnable] = useState(false); + + useEffect(() => { + canEnableFaceIndexing().then((v) => setCanEnable(v)); + }, []); + return ( - {isInternalUserForML() && ( + {canEnable && (