From 5bd845d32b5d4f168a67f9495ba2d97051b33cc1 Mon Sep 17 00:00:00 2001 From: ashilkn Date: Mon, 13 May 2024 15:39:12 +0530 Subject: [PATCH 01/41] [mob][photos] Migrate to sqlite_async (1) --- mobile/lib/db/files_db.dart | 201 +++++++++++++++++------------------- 1 file changed, 93 insertions(+), 108 deletions(-) diff --git a/mobile/lib/db/files_db.dart b/mobile/lib/db/files_db.dart index 7022100b73..d61bbd2bdf 100644 --- a/mobile/lib/db/files_db.dart +++ b/mobile/lib/db/files_db.dart @@ -778,15 +778,13 @@ class FilesDB { // Files which user added to a collection manually but they are not // uploaded yet or files belonging to a collection which is marked for backup Future> getFilesPendingForUpload() async { - final db = await instance.database; - final results = await db.query( - filesTable, - where: - '($columnUploadedFileID IS NULL OR $columnUploadedFileID IS -1) AND ' - '$columnCollectionID IS NOT NULL AND $columnCollectionID IS NOT -1 AND ' - '$columnLocalID IS NOT NULL AND $columnLocalID IS NOT -1', - orderBy: '$columnCreationTime DESC', - groupBy: columnLocalID, + final db = await instance.sqliteAsyncDB; + final results = await db.getAll( + 'SELECT * FROM $filesTable WHERE ($columnUploadedFileID IS NULL OR ' + '$columnUploadedFileID IS -1) AND $columnCollectionID IS NOT NULL AND ' + '$columnCollectionID IS NOT -1 AND $columnLocalID IS NOT NULL AND ' + '$columnLocalID IS NOT -1 GROUP BY $columnLocalID ' + 'ORDER BY $columnCreationTime DESC', ); final files = convertToFiles(results); // future-safe filter just to ensure that the query doesn't end up returning files @@ -801,30 +799,22 @@ class FilesDB { } Future> getUnUploadedLocalFiles() async { - final db = await instance.database; - final results = await db.query( - filesTable, - where: - '($columnUploadedFileID IS NULL OR $columnUploadedFileID IS -1) AND $columnLocalID IS NOT NULL', - orderBy: '$columnCreationTime DESC', - groupBy: columnLocalID, + final db = await instance.sqliteAsyncDB; + final results = await db.getAll( + 'SELECT * FROM $filesTable WHERE ($columnUploadedFileID IS NULL OR ' + '$columnUploadedFileID IS -1) AND $columnLocalID IS NOT NULL ' + 'GROUP BY $columnLocalID ORDER BY $columnCreationTime DESC', ); return convertToFiles(results); } Future> getUploadedFileIDsToBeUpdated(int ownerID) async { - final db = await instance.database; - final rows = await db.query( - filesTable, - columns: [columnUploadedFileID], - where: '($columnLocalID IS NOT NULL AND $columnOwnerID = ? AND ' - '($columnUploadedFileID ' - 'IS NOT ' - 'NULL AND $columnUploadedFileID IS NOT -1) AND $columnUpdationTime IS NULL)', - whereArgs: [ownerID], - orderBy: '$columnCreationTime DESC', - distinct: true, - ); + final db = await instance.sqliteAsyncDB; + final rows = await db.getAll( + 'SELECT DISTINCT $columnUploadedFileID FROM $filesTable WHERE ' + '($columnLocalID IS NOT NULL AND $columnOwnerID = ? AND ' + '($columnUploadedFileID IS NOT NULL AND $columnUploadedFileID IS NOT -1) ' + 'AND $columnUpdationTime IS NULL) ORDER BY $columnCreationTime DESC '); final uploadedFileIDs = []; for (final row in rows) { uploadedFileIDs.add(row[columnUploadedFileID] as int); @@ -836,15 +826,11 @@ class FilesDB { int uploadedFileID, int userID, ) async { - final db = await instance.database; - final results = await db.query( - filesTable, - where: '$columnLocalID IS NOT NULL AND $columnOwnerID = ? AND ' - '$columnUploadedFileID = ?', - whereArgs: [ - userID, - uploadedFileID, - ], + final db = await instance.sqliteAsyncDB; + final results = await db.getAll( + 'SELECT * FROM $filesTable WHERE $columnLocalID IS NOT NULL AND ' + '$columnOwnerID = ? AND $columnUploadedFileID = ?', + [userID, uploadedFileID], ); if (results.isEmpty) { return []; @@ -853,14 +839,12 @@ class FilesDB { } Future> getExistingLocalFileIDs(int ownerID) async { - final db = await instance.database; - final rows = await db.query( - filesTable, - columns: [columnLocalID], - distinct: true, - where: '$columnLocalID IS NOT NULL AND ($columnOwnerID IS NULL OR ' - '$columnOwnerID = ?)', - whereArgs: [ownerID], + final db = await instance.sqliteAsyncDB; + final rows = await db.getAll( + 'SELECT DISTINCT $columnLocalID FROM $filesTable ' + 'WHERE $columnLocalID IS NOT NULL AND ($columnOwnerID IS NULL OR ' + '$columnOwnerID = ?)', + [ownerID], ); final result = {}; for (final row in rows) { @@ -870,16 +854,13 @@ class FilesDB { } Future> getLocalIDsMarkedForOrAlreadyUploaded(int ownerID) async { - final db = await instance.database; - final rows = await db.query( - filesTable, - columns: [columnLocalID], - distinct: true, - where: '$columnLocalID IS NOT NULL AND ($columnCollectionID IS NOT NULL ' - 'AND ' - '$columnCollectionID != -1) AND ($columnOwnerID = ? OR ' - '$columnOwnerID IS NULL)', - whereArgs: [ownerID], + final db = await instance.sqliteAsyncDB; + final rows = await db.getAll( + 'SELECT DISTINCT $columnLocalID FROM $filesTable ' + 'WHERE $columnLocalID IS NOT NULL AND ($columnCollectionID IS NOT NULL ' + 'AND $columnCollectionID != -1) AND ($columnOwnerID = ? OR ' + '$columnOwnerID IS NULL)', + [ownerID], ); final result = {}; for (final row in rows) { @@ -889,12 +870,11 @@ class FilesDB { } Future> getLocalFileIDsForCollection(int collectionID) async { - final db = await instance.database; - final rows = await db.query( - filesTable, - columns: [columnLocalID], - where: '$columnLocalID IS NOT NULL AND $columnCollectionID = ?', - whereArgs: [collectionID], + final db = await instance.sqliteAsyncDB; + final rows = await db.getAll( + 'SELECT $columnLocalID FROM $filesTable ' + 'WHERE $columnLocalID IS NOT NULL AND $columnCollectionID = ?', + [collectionID], ); final result = {}; for (final row in rows) { @@ -905,17 +885,17 @@ class FilesDB { // Sets the collectionID for the files with given LocalIDs if the // corresponding file entries are not already mapped to some other collection - Future setCollectionIDForUnMappedLocalFiles( + Future setCollectionIDForUnMappedLocalFiles( int collectionID, Set localIDs, ) async { - final db = await instance.database; + final db = await instance.sqliteAsyncDB; String inParam = ""; for (final localID in localIDs) { inParam += "'" + localID + "',"; } inParam = inParam.substring(0, inParam.length - 1); - return await db.rawUpdate( + await db.execute( ''' UPDATE $filesTable SET $columnCollectionID = $collectionID @@ -925,7 +905,7 @@ class FilesDB { ); } - Future markFilesForReUpload( + Future markFilesForReUpload( int ownerID, String localID, String? title, @@ -934,22 +914,32 @@ class FilesDB { int modificationTime, FileType fileType, ) async { - final db = await instance.database; - return await db.update( - filesTable, - { - columnTitle: title, - columnLatitude: location?.latitude, - columnLongitude: location?.longitude, - columnCreationTime: creationTime, - columnModificationTime: modificationTime, - // #hack reset updation time to null for re-upload - columnUpdationTime: null, - columnFileType: getInt(fileType), - }, - where: - '$columnLocalID = ? AND ($columnOwnerID = ? OR $columnOwnerID IS NULL)', - whereArgs: [localID, ownerID], + final db = await instance.sqliteAsyncDB; + + await db.execute( + ''' + UPDATE $filesTable + SET $columnTitle = ?, + $columnLatitude = ?, + $columnLongitude = ?, + $columnCreationTime = ?, + $columnModificationTime = ?, + $columnUpdationTime = NULL, + $columnFileType = ? + WHERE $columnLocalID = ? AND ($columnOwnerID = ? OR $columnOwnerID IS NULL); + ''', + [ + title, + location?.latitude, + location?.longitude, + localID, + creationTime, + modificationTime, + null, + getInt(fileType), + localID, + ownerID, + ], ); } @@ -964,12 +954,12 @@ class FilesDB { required String title, required String deviceFolder, }) async { - final db = await instance.database; + final db = await instance.sqliteAsyncDB; // on iOS, match using localID and fileType. title can either match or // might be null based on how the file was imported - String whereClause = ''' ($columnOwnerID = ? OR $columnOwnerID IS NULL) AND - $columnLocalID = ? AND $columnFileType = ? AND - ($columnTitle=? OR $columnTitle IS NULL) '''; + String query = '''SELECT * FROM $filesTable WHERE ($columnOwnerID = ? + OR $columnOwnerID IS NULL) AND $columnLocalID = ? + AND $columnFileType = ? AND ($columnTitle=? OR $columnTitle IS NULL) '''; List whereArgs = [ ownerID, localID, @@ -977,9 +967,9 @@ class FilesDB { title, ]; if (Platform.isAndroid) { - whereClause = ''' ($columnOwnerID = ? OR $columnOwnerID IS NULL) AND - $columnLocalID = ? AND $columnFileType = ? AND $columnTitle=? AND $columnDeviceFolder= ? - '''; + query = '''SELECT * FROM $filesTable WHERE ($columnOwnerID = ? OR + $columnOwnerID IS NULL) AND $columnLocalID = ? AND $columnFileType = ? + AND $columnTitle=? AND $columnDeviceFolder= ? '''; whereArgs = [ ownerID, localID, @@ -989,10 +979,9 @@ class FilesDB { ]; } - final rows = await db.query( - filesTable, - where: whereClause, - whereArgs: whereArgs, + final rows = await db.getAll( + query, + whereArgs, ); return convertToFiles(rows); @@ -1030,14 +1019,12 @@ class FilesDB { if (fileType == FileType.livePhoto && hashData.zipHash != null) { inParam += ",'${hashData.zipHash}'"; } - final db = await instance.database; - final rows = await db.query( - filesTable, - where: '($columnUploadedFileID != NULL OR $columnUploadedFileID != -1) ' - 'AND $columnOwnerID = ? AND $columnFileType =' - ' ? ' - 'AND $columnHash IN ($inParam)', - whereArgs: [ + final db = await instance.sqliteAsyncDB; + final rows = await db.getAll( + 'SELECT * FROM $filesTable WHERE $columnUploadedFileID != NULL OR ' + '$columnUploadedFileID != -1) AND $columnOwnerID = ? AND ' + '$columnFileType = ? AND $columnHash IN ($inParam)', + [ ownerID, getInt(fileType), ], @@ -1371,10 +1358,9 @@ class FilesDB { inParam += "'" + id.toString() + "',"; } inParam = inParam.substring(0, inParam.length - 1); - final db = await instance.database; - final results = await db.query( - filesTable, - where: '$columnUploadedFileID IN ($inParam)', + final db = await instance.sqliteAsyncDB; + final results = await db.getAll( + 'SELECT * FROM $filesTable WHERE $columnUploadedFileID IN ($inParam)', ); final files = convertToFiles(results); for (final file in files) { @@ -1393,10 +1379,9 @@ class FilesDB { inParam += "'" + id.toString() + "',"; } inParam = inParam.substring(0, inParam.length - 1); - final db = await instance.database; - final results = await db.query( - filesTable, - where: '$columnGeneratedID IN ($inParam)', + final db = await instance.sqliteAsyncDB; + final results = await db.getAll( + 'SELECT * FROM $filesTable WHERE $columnGeneratedID IN ($inParam)', ); final files = convertToFiles(results); for (final file in files) { From 3a0882a1a9fcef9e09b985e31819edf52b38a4fb Mon Sep 17 00:00:00 2001 From: ashilkn Date: Mon, 13 May 2024 17:57:22 +0530 Subject: [PATCH 02/41] [mob][photos] Migrate to sqlite_async (2): Migrate all update queries in filesDB --- mobile/lib/db/files_db.dart | 94 ++++++++++++------- .../services/local_file_update_service.dart | 5 +- 2 files changed, 62 insertions(+), 37 deletions(-) diff --git a/mobile/lib/db/files_db.dart b/mobile/lib/db/files_db.dart index d61bbd2bdf..88c58a7ca1 100644 --- a/mobile/lib/db/files_db.dart +++ b/mobile/lib/db/files_db.dart @@ -1032,33 +1032,32 @@ class FilesDB { return convertToFiles(rows); } - Future update(EnteFile file) async { - final db = await instance.database; - return await db.update( - filesTable, - _getRowForFile(file), - where: '$columnGeneratedID = ?', - whereArgs: [file.generatedID], + Future update(EnteFile file) async { + final db = await instance.sqliteAsyncDB; + final setClause = _getSetClauseForFile(file); + await db.execute( + 'UPDATE $filesTable SET ' + '$setClause WHERE $columnGeneratedID = ?', + [file.generatedID], ); } - Future updateUploadedFileAcrossCollections(EnteFile file) async { - final db = await instance.database; - return await db.update( - filesTable, - _getRowForFileWithoutCollection(file), - where: '$columnUploadedFileID = ?', - whereArgs: [file.uploadedFileID], + Future updateUploadedFileAcrossCollections(EnteFile file) async { + final db = await instance.sqliteAsyncDB; + final setClause = _getSetClauseForFileWithoutCollection(file); + await db.execute( + 'UPDATE $filesTable SET ' + '$setClause WHERE $columnUploadedFileID = ?', + [file.uploadedFileID], ); } - Future updateLocalIDForUploaded(int uploadedID, String localID) async { - final db = await instance.database; - return await db.update( - filesTable, - {columnLocalID: localID}, - where: '$columnUploadedFileID = ? AND $columnLocalID IS NULL', - whereArgs: [uploadedID], + Future updateLocalIDForUploaded(int uploadedID, String localID) async { + final db = await instance.sqliteAsyncDB; + await db.execute( + 'UPDATE $filesTable SET $columnLocalID = ? WHERE $columnUploadedFileID = ?' + ' AND $columnLocalID IS NULL', + [localID, uploadedID], ); } @@ -1123,8 +1122,8 @@ class FilesDB { inParam += "'" + localID + "',"; } inParam = inParam.substring(0, inParam.length - 1); - final db = await instance.database; - await db.rawQuery( + final db = await instance.sqliteAsyncDB; + await db.execute( ''' UPDATE $filesTable SET $columnLocalID = NULL @@ -1323,8 +1322,8 @@ class FilesDB { inParam += "'" + localID + "',"; } inParam = inParam.substring(0, inParam.length - 1); - final db = await instance.database; - await db.rawUpdate( + final db = await instance.sqliteAsyncDB; + await db.execute( ''' UPDATE $filesTable SET $columnUpdationTime = NULL @@ -1552,17 +1551,24 @@ class FilesDB { if (uploadedFileIDToSize.isEmpty) { return; } - final db = await instance.database; - final batch = db.batch(); + final db = await instance.sqliteAsyncDB; + final parameterSets = >[]; + for (final uploadedFileID in uploadedFileIDToSize.keys) { - batch.update( - filesTable, - {columnFileSize: uploadedFileIDToSize[uploadedFileID]}, - where: '$columnUploadedFileID = ?', - whereArgs: [uploadedFileID], - ); + parameterSets.add([ + uploadedFileIDToSize[uploadedFileID], + uploadedFileID, + ]); } - await batch.commit(noResult: true); + + await db.executeBatch( + ''' + UPDATE $filesTable + SET $columnFileSize = ? + WHERE $columnUploadedFileID = ?; + ''', + parameterSets, + ); } Future> getAllFilesFromDB( @@ -1661,6 +1667,26 @@ class FilesDB { return convertToFiles(results); } + String _getSetClauseForFile(EnteFile file) { + final row = _getRowForFile(file); + final setClause = []; + for (int i = 0; i < row.entries.length; i++) { + final entry = row.entries.elementAt(i); + setClause.add('${entry.key} = ${entry.value}'); + } + return setClause.join(', '); + } + + String _getSetClauseForFileWithoutCollection(EnteFile file) { + final row = _getRowForFileWithoutCollection(file); + final setClause = []; + for (int i = 0; i < row.entries.length; i++) { + final entry = row.entries.elementAt(i); + setClause.add('${entry.key} = ${entry.value}'); + } + return setClause.join(', '); + } + Map _getRowForFile(EnteFile file) { final row = {}; if (file.generatedID != null) { diff --git a/mobile/lib/services/local_file_update_service.dart b/mobile/lib/services/local_file_update_service.dart index e00ac6c459..ce5a9080af 100644 --- a/mobile/lib/services/local_file_update_service.dart +++ b/mobile/lib/services/local_file_update_service.dart @@ -193,7 +193,7 @@ class LocalFileUpdateService { } else if (e.reason == InvalidReason.imageToLivePhotoTypeChanged) { fileType = FileType.livePhoto; } - final int count = await FilesDB.instance.markFilesForReUpload( + await FilesDB.instance.markFilesForReUpload( userID, file.localID!, file.title, @@ -202,8 +202,7 @@ class LocalFileUpdateService { file.modificationTime!, fileType, ); - _logger.fine('fileType changed for ${file.tag} to ${e.reason} for ' - '$count files'); + _logger.fine('fileType changed for ${file.tag} to ${e.reason} for '); } else { _logger.severe("failed to check hash: invalid file ${file.tag}", e); } From 8fcd05b95f5e3a6417eadb3fd372a729b1c96b0c Mon Sep 17 00:00:00 2001 From: ashilkn Date: Mon, 13 May 2024 18:29:01 +0530 Subject: [PATCH 03/41] [mob][photos] Migrate to sqlite_async (3) --- mobile/lib/db/files_db.dart | 122 +++++++++++++++++------------------- 1 file changed, 57 insertions(+), 65 deletions(-) diff --git a/mobile/lib/db/files_db.dart b/mobile/lib/db/files_db.dart index 88c58a7ca1..1468f961d2 100644 --- a/mobile/lib/db/files_db.dart +++ b/mobile/lib/db/files_db.dart @@ -1,3 +1,4 @@ +import "dart:async"; import "dart:io"; import "package:computer/computer.dart"; @@ -1061,57 +1062,53 @@ class FilesDB { ); } - Future delete(int uploadedFileID) async { - final db = await instance.database; - return db.delete( - filesTable, - where: '$columnUploadedFileID =?', - whereArgs: [uploadedFileID], + Future deleteByGeneratedID(int genID) async { + final db = await instance.sqliteAsyncDB; + + await db.execute( + 'DELETE FROM $filesTable WHERE $columnGeneratedID = ?', + [genID], ); } - Future deleteByGeneratedID(int genID) async { - final db = await instance.database; - return db.delete( - filesTable, - where: '$columnGeneratedID =?', - whereArgs: [genID], + Future deleteMultipleUploadedFiles(List uploadedFileIDs) async { + final db = await instance.sqliteAsyncDB; + final inParam = uploadedFileIDs.join(','); + + await db.execute( + 'DELETE FROM $filesTable WHERE $columnUploadedFileID IN ($inParam)', ); } - Future deleteMultipleUploadedFiles(List uploadedFileIDs) async { - final db = await instance.database; - return await db.delete( - filesTable, - where: '$columnUploadedFileID IN (${uploadedFileIDs.join(', ')})', - ); - } - - Future deleteMultipleByGeneratedIDs(List generatedIDs) async { + Future deleteMultipleByGeneratedIDs(List generatedIDs) async { if (generatedIDs.isEmpty) { - return 0; + return; } - final db = await instance.database; - return await db.delete( - filesTable, - where: '$columnGeneratedID IN (${generatedIDs.join(', ')})', + + final db = await instance.sqliteAsyncDB; + final inParam = generatedIDs.join(','); + + await db.execute( + 'DELETE FROM $filesTable WHERE $columnGeneratedID IN ($inParam)', ); } - Future deleteLocalFile(EnteFile file) async { - final db = await instance.database; + Future deleteLocalFile(EnteFile file) async { + final db = await instance.sqliteAsyncDB; if (file.localID != null) { // delete all files with same local ID - return db.delete( - filesTable, - where: '$columnLocalID =?', - whereArgs: [file.localID], + unawaited( + db.execute( + 'DELETE FROM $filesTable WHERE $columnLocalID = ?', + [file.localID], + ), ); } else { - return db.delete( - filesTable, - where: '$columnGeneratedID =?', - whereArgs: [file.generatedID], + unawaited( + db.execute( + 'DELETE FROM $filesTable WHERE $columnGeneratedID = ?', + [file.generatedID], + ), ); } } @@ -1138,37 +1135,34 @@ class FilesDB { inParam += "'" + localID + "',"; } inParam = inParam.substring(0, inParam.length - 1); - final db = await instance.database; - final results = await db.query( - filesTable, - where: '$columnLocalID IN ($inParam)', + final db = await instance.sqliteAsyncDB; + final results = await db.getAll( + ''' + SELECT * FROM $filesTable + WHERE $columnLocalID IN ($inParam); + ''', ); return convertToFiles(results); } - Future deleteUnSyncedLocalFiles(List localIDs) async { + Future deleteUnSyncedLocalFiles(List localIDs) async { String inParam = ""; for (final localID in localIDs) { inParam += "'" + localID + "',"; } inParam = inParam.substring(0, inParam.length - 1); - final db = await instance.database; - return db.delete( - filesTable, - where: - '($columnUploadedFileID is NULL OR $columnUploadedFileID = -1 ) AND $columnLocalID IN ($inParam)', - ); - } - - Future deleteFromCollection(int uploadedFileID, int collectionID) async { - final db = await instance.database; - return db.delete( - filesTable, - where: '$columnUploadedFileID = ? AND $columnCollectionID = ?', - whereArgs: [uploadedFileID, collectionID], + final db = await instance.sqliteAsyncDB; + unawaited( + db.execute( + ''' + DELETE FROM $filesTable + WHERE ($columnUploadedFileID is NULL OR $columnUploadedFileID = -1 ) AND $columnLocalID IN ($inParam) + ''', + ), ); } + /// Uses int in return value. Future deleteFilesFromCollection( int collectionID, List uploadedFileIDs, @@ -1183,9 +1177,9 @@ class FilesDB { } Future collectionFileCount(int collectionID) async { - final db = await instance.database; + final db = await instance.sqliteAsyncDB; final count = Sqflite.firstIntValue( - await db.rawQuery( + await db.execute( 'SELECT COUNT(*) FROM $filesTable where $columnCollectionID = ' '$collectionID AND $columnUploadedFileID IS NOT -1', ), @@ -1198,15 +1192,13 @@ class FilesDB { int ownerID, Set hiddenCollections, ) async { - final db = await instance.database; - final count = Sqflite.firstIntValue( - await db.rawQuery( - 'SELECT COUNT(distinct($columnUploadedFileID)) FROM $filesTable where ' - '$columnMMdVisibility' - ' = $visibility AND $columnOwnerID = $ownerID AND $columnCollectionID NOT IN (${hiddenCollections.join(', ')})', - ), + final db = await instance.sqliteAsyncDB; + final count = await db.execute( + 'SELECT COUNT(distinct($columnUploadedFileID)) as COUNT FROM $filesTable where ' + '$columnMMdVisibility' + ' = $visibility AND $columnOwnerID = $ownerID AND $columnCollectionID NOT IN (${hiddenCollections.join(', ')})', ); - return count ?? 0; + return count.first['COUNT'] as int; } Future deleteCollection(int collectionID) async { From ff14eb1d5a34ab5392869a631880dc721bf8b1b4 Mon Sep 17 00:00:00 2001 From: ashilkn Date: Tue, 14 May 2024 14:59:03 +0530 Subject: [PATCH 04/41] [mob][photos] Migrate to sqlite_async (4) --- mobile/lib/db/files_db.dart | 264 ++++++++++++++++++++---------------- 1 file changed, 148 insertions(+), 116 deletions(-) diff --git a/mobile/lib/db/files_db.dart b/mobile/lib/db/files_db.dart index 1468f961d2..887932250b 100644 --- a/mobile/lib/db/files_db.dart +++ b/mobile/lib/db/files_db.dart @@ -1201,32 +1201,36 @@ class FilesDB { return count.first['COUNT'] as int; } - Future deleteCollection(int collectionID) async { - final db = await instance.database; - return db.delete( - filesTable, - where: '$columnCollectionID = ?', - whereArgs: [collectionID], + Future deleteCollection(int collectionID) async { + final db = await instance.sqliteAsyncDB; + unawaited( + db.execute( + 'DELETE FROM $filesTable WHERE $columnCollectionID = ?', + [collectionID], + ), ); } - Future removeFromCollection(int collectionID, List fileIDs) async { - final db = await instance.database; - return db.delete( - filesTable, - where: - '$columnCollectionID =? AND $columnUploadedFileID IN (${fileIDs.join(', ')})', - whereArgs: [collectionID], + Future removeFromCollection(int collectionID, List fileIDs) async { + final db = await instance.sqliteAsyncDB; + final inParam = fileIDs.join(','); + unawaited( + db.execute( + ''' + DELETE FROM $filesTable + WHERE $columnCollectionID = ? AND $columnUploadedFileID IN ($inParam); + ''', + [collectionID], + ), ); } Future> getPendingUploadForCollection(int collectionID) async { - final db = await instance.database; - final results = await db.query( - filesTable, - where: '$columnCollectionID = ? AND ($columnUploadedFileID IS NULL OR ' - '$columnUploadedFileID = -1)', - whereArgs: [collectionID], + final db = await instance.sqliteAsyncDB; + final results = await db.getAll( + 'SELECT * FROM $filesTable WHERE $columnCollectionID = ? AND ' + '($columnUploadedFileID IS NULL OR $columnUploadedFileID = -1)', + [collectionID], ); return convertToFiles(results); } @@ -1242,8 +1246,8 @@ class FilesDB { } } inParam = inParam.substring(0, inParam.length - 1); - final db = await instance.database; - final rows = await db.rawQuery( + final db = await instance.sqliteAsyncDB; + final rows = await db.execute( ''' SELECT $columnLocalID FROM $filesTable @@ -1262,8 +1266,8 @@ class FilesDB { // creationTime of the files in the collection. Future> getCollectionIDToMaxCreationTime() async { final enteWatch = EnteWatch("getCollectionIDToMaxCreationTime")..start(); - final db = await instance.database; - final rows = await db.rawQuery( + final db = await instance.sqliteAsyncDB; + final rows = await db.execute( ''' SELECT $columnCollectionID, MAX($columnCreationTime) AS max_creation_time FROM $filesTable @@ -1288,16 +1292,17 @@ class FilesDB { int collectionID, bool sortAsc, ) async { - final db = await instance.database; + final db = await instance.sqliteAsyncDB; final order = sortAsc ? 'ASC' : 'DESC'; - final rows = await db.query( - filesTable, - where: '$columnCollectionID = ? AND ($columnUploadedFileID IS NOT NULL ' - 'AND $columnUploadedFileID IS NOT -1)', - whereArgs: [collectionID], - orderBy: - '$columnCreationTime ' + order + ', $columnModificationTime ' + order, - limit: 1, + final rows = await db.getAll( + ''' + SELECT * FROM $filesTable + WHERE $columnCollectionID = ? AND ($columnUploadedFileID IS NOT NULL + AND $columnUploadedFileID IS NOT -1) + ORDER BY $columnCreationTime $order, $columnModificationTime $order + LIMIT 1; + ''', + [collectionID], ); if (rows.isEmpty) { return null; @@ -1329,12 +1334,11 @@ class FilesDB { int uploadedFileID, int collectionID, ) async { - final db = await instance.database; - final rows = await db.query( - filesTable, - where: '$columnUploadedFileID = ? AND $columnCollectionID = ?', - whereArgs: [uploadedFileID, collectionID], - limit: 1, + final db = await instance.sqliteAsyncDB; + final rows = await db.getAll( + 'SELECT * FROM $filesTable WHERE $columnUploadedFileID = ? AND ' + '$columnCollectionID = ? LIMIT 1', + [uploadedFileID, collectionID], ); return rows.isNotEmpty; } @@ -1393,10 +1397,9 @@ class FilesDB { inParam += "'" + id.toString() + "',"; } inParam = inParam.substring(0, inParam.length - 1); - final db = await instance.database; - final results = await db.query( - filesTable, - where: '$columnUploadedFileID IN ($inParam)', + final db = await instance.sqliteAsyncDB; + final results = await db.getAll( + 'SELECT * FROM $filesTable WHERE $columnUploadedFileID IN ($inParam)', ); final files = convertToFiles(results); for (EnteFile eachFile in files) { @@ -1411,13 +1414,13 @@ class FilesDB { Future> getAllCollectionIDsOfFile( int uploadedFileID, ) async { - final db = await instance.database; - final results = await db.query( - filesTable, - where: '$columnUploadedFileID = ? AND $columnCollectionID != -1', - columns: [columnCollectionID], - whereArgs: [uploadedFileID], - distinct: true, + final db = await instance.sqliteAsyncDB; + final results = await db.getAll( + ''' + SELECT DISTINCT $columnCollectionID FROM $filesTable + WHERE $columnUploadedFileID = ? AND $columnCollectionID != -1 + ''', + [uploadedFileID], ); final collectionIDsOfFile = {}; for (var result in results) { @@ -1446,14 +1449,13 @@ class FilesDB { int cutOffTime, int ownerID, ) async { - final db = await instance.database; - final rows = await db.query( - filesTable, - columns: [columnGeneratedID], - distinct: true, - where: - '$columnCreationTime <= ? AND ($columnOwnerID IS NULL OR $columnOwnerID = ?)', - whereArgs: [cutOffTime, ownerID], + final db = await instance.sqliteAsyncDB; + final rows = await db.getAll( + ''' + SELECT DISTINCT $columnGeneratedID FROM $filesTable + WHERE $columnCreationTime <= ? AND ($columnOwnerID IS NULL OR $columnOwnerID = ?) + ''', + [cutOffTime, ownerID], ); final result = []; for (final row in rows) { @@ -1465,15 +1467,14 @@ class FilesDB { // For givenUserID, get List of unique LocalIDs for files which are // uploaded by the given user and location is missing Future> getLocalIDsForFilesWithoutLocation(int ownerID) async { - final db = await instance.database; - final rows = await db.query( - filesTable, - columns: [columnLocalID], - distinct: true, - where: '$columnOwnerID = ? AND $columnLocalID IS NOT NULL AND ' - '($columnLatitude IS NULL OR ' - '$columnLongitude IS NULL OR $columnLongitude = 0.0 or $columnLongitude = 0.0)', - whereArgs: [ownerID], + final db = await instance.sqliteAsyncDB; + final rows = await db.getAll( + ''' + SELECT DISTINCT $columnLocalID FROM $filesTable + WHERE $columnOwnerID = ? AND $columnLocalID IS NOT NULL AND + ($columnLatitude IS NULL OR $columnLongitude IS NULL OR $columnLatitude = 0.0 or $columnLongitude = 0.0) + ''', + [ownerID], ); final result = []; for (final row in rows) { @@ -1484,13 +1485,13 @@ class FilesDB { // For a given userID, return unique uploadedFileId for the given userID Future> getUploadIDsWithMissingSize(int userId) async { - final db = await instance.database; - final rows = await db.query( - filesTable, - columns: [columnUploadedFileID], - distinct: true, - where: '$columnOwnerID = ? AND $columnFileSize IS NULL', - whereArgs: [userId], + final db = await instance.sqliteAsyncDB; + final rows = await db.getAll( + ''' + SELECT DISTINCT $columnUploadedFileID FROM $filesTable + WHERE $columnOwnerID = ? AND $columnFileSize IS NULL + ''', + [userId], ); final result = []; for (final row in rows) { @@ -1501,14 +1502,13 @@ class FilesDB { // For a given userID, return unique localID for all uploaded live photos Future> getLivePhotosForUser(int userId) async { - final db = await instance.database; - final rows = await db.query( - filesTable, - columns: [columnLocalID], - distinct: true, - where: '$columnOwnerID = ? AND ' - '$columnFileType = ? AND $columnLocalID IS NOT NULL', - whereArgs: [userId, getInt(FileType.livePhoto)], + final db = await instance.sqliteAsyncDB; + final rows = await db.getAll( + ''' + SELECT DISTINCT $columnLocalID FROM $filesTable + WHERE $columnOwnerID = ? AND $columnFileType = ? AND $columnLocalID IS NOT NULL + ''', + [userId, getInt(FileType.livePhoto)], ); final result = []; for (final row in rows) { @@ -1518,15 +1518,16 @@ class FilesDB { } Future> getLocalFilesBackedUpWithoutLocation(int userId) async { - final db = await instance.database; - final rows = await db.query( - filesTable, - columns: [columnLocalID], - distinct: true, - where: - '$columnOwnerID = ? AND $columnLocalID IS NOT NULL AND ($columnUploadedFileID IS NOT NULL AND $columnUploadedFileID IS NOT -1) ' - 'AND ($columnLatitude IS NULL OR $columnLongitude IS NULL OR $columnLongitude = 0.0 or $columnLongitude = 0.0)', - whereArgs: [userId], + final db = await instance.sqliteAsyncDB; + final rows = await db.getAll( + ''' + SELECT DISTINCT $columnLocalID FROM $filesTable + WHERE $columnOwnerID = ? AND $columnLocalID IS NOT NULL AND + ($columnUploadedFileID IS NOT NULL AND $columnUploadedFileID IS NOT -1) + AND ($columnLatitude IS NULL OR $columnLongitude IS NULL OR + $columnLatitude = 0.0 or $columnLongitude = 0.0) + ''', + [userId], ); final result = []; for (final row in rows) { @@ -1585,9 +1586,13 @@ class FilesDB { } Future> fetchFilesCountbyType(int userID) async { - final db = await instance.database; - final result = await db.rawQuery( - "SELECT $columnFileType, COUNT(DISTINCT $columnUploadedFileID) FROM $filesTable WHERE $columnUploadedFileID != -1 AND $columnOwnerID == $userID GROUP BY $columnFileType", + final db = await instance.sqliteAsyncDB; + final result = await db.execute( + ''' + SELECT $columnFileType, COUNT(DISTINCT $columnUploadedFileID) + FROM $filesTable WHERE $columnUploadedFileID != -1 AND + $columnOwnerID == $userID GROUP BY $columnFileType + ''', ); final filesCount = {}; @@ -1606,18 +1611,20 @@ class FilesDB { bool? asc, required DBFilterOptions? filterOptions, }) async { - final db = await instance.database; + final db = await instance.sqliteAsyncDB; final order = (asc ?? false ? 'ASC' : 'DESC'); - final results = await db.query( - filesTable, - where: - '$columnLatitude IS NOT NULL AND $columnLongitude IS NOT NULL AND ($columnLatitude IS NOT 0 OR $columnLongitude IS NOT 0)' - ' AND $columnCreationTime >= ? AND $columnCreationTime <= ?' - ' AND ($columnLocalID IS NOT NULL OR ($columnCollectionID IS NOT NULL AND $columnCollectionID IS NOT -1))', - whereArgs: [startTime, endTime], - orderBy: - '$columnCreationTime ' + order + ', $columnModificationTime ' + order, - limit: limit, + final results = await db.getAll( + ''' + SELECT * FROM $filesTable + WHERE $columnLatitude IS NOT NULL AND $columnLongitude IS NOT NULL AND + ($columnLatitude IS NOT 0 OR $columnLongitude IS NOT 0) AND + $columnCreationTime >= ? AND $columnCreationTime <= ? AND + ($columnLocalID IS NOT NULL OR ($columnCollectionID IS NOT NULL AND + $columnCollectionID IS NOT -1)) + ORDER BY $columnCreationTime $order, $columnModificationTime $order + LIMIT $limit + ''', + [startTime, endTime], ); final files = convertToFiles(results); final List filteredFiles = @@ -1626,13 +1633,14 @@ class FilesDB { } Future> getOwnedFileIDs(int ownerID) async { - final db = await instance.database; - final results = await db.query( - filesTable, - columns: [columnUploadedFileID], - where: - '($columnOwnerID = $ownerID AND $columnUploadedFileID IS NOT NULL AND $columnUploadedFileID IS NOT -1)', - distinct: true, + final db = await instance.sqliteAsyncDB; + final results = await db.getAll( + ''' + SELECT DISTINCT $columnUploadedFileID FROM $filesTable + WHERE $columnOwnerID = ? AND $columnUploadedFileID IS NOT NULL AND + $columnUploadedFileID IS NOT -1) + ''', + [ownerID], ); final ids = []; for (final result in results) { @@ -1642,16 +1650,17 @@ class FilesDB { } Future> getUploadedFiles(List uploadedIDs) async { - final db = await instance.database; + final db = await instance.sqliteAsyncDB; String inParam = ""; for (final id in uploadedIDs) { inParam += "'" + id.toString() + "',"; } inParam = inParam.substring(0, inParam.length - 1); - final results = await db.query( - filesTable, - where: '$columnUploadedFileID IN ($inParam)', - groupBy: columnUploadedFileID, + final results = await db.getAll( + ''' + SELECT * FROM $filesTable WHERE $columnUploadedFileID IN ($inParam) + GROUP BY $columnUploadedFileID +''', ); if (results.isEmpty) { return []; @@ -1659,6 +1668,29 @@ class FilesDB { return convertToFiles(results); } + ///For insertion, the syntax is as follows: + ///INSERT INTO table (column1,column2 ,..) + ///VALUES( value1, value2 ,...); + ///This method returns: + ///{ + /// 'columns': 'column1,column2 ,..', + /// 'values': 'value1, value2 ,...' + /// } + Map _getColumnsAndValuesForInsertion(EnteFile file) { + final row = _getRowForFile(file); + final columns = []; + final values = []; + for (int i = 0; i < row.entries.length; i++) { + final entry = row.entries.elementAt(i); + columns.add(entry.key); + values.add(entry.value.toString()); + } + return { + 'columns': columns.join(', '), + 'values': values.join(', '), + }; + } + String _getSetClauseForFile(EnteFile file) { final row = _getRowForFile(file); final setClause = []; From d1a5921c271303c2f568d62a04d88668a44f0ba3 Mon Sep 17 00:00:00 2001 From: ashilkn Date: Wed, 15 May 2024 15:28:24 +0530 Subject: [PATCH 05/41] [mob][photos] Migrate to sqlite_async(5): Create a method to get parameter set from file without calling getRowForFile() --- mobile/lib/db/files_db.dart | 98 ++++++++++++++++++++++++++----------- 1 file changed, 70 insertions(+), 28 deletions(-) diff --git a/mobile/lib/db/files_db.dart b/mobile/lib/db/files_db.dart index 887932250b..7fde7ee2e7 100644 --- a/mobile/lib/db/files_db.dart +++ b/mobile/lib/db/files_db.dart @@ -401,11 +401,11 @@ class FilesDB { } Future clearTable() async { - final db = await instance.database; - await db.delete(filesTable); - await db.delete("device_files"); - await db.delete("device_collections"); - await db.delete("entities"); + final db = await instance.sqliteAsyncDB; + await db.execute('DELETE FROM $filesTable'); + await db.execute('DELETE FROM device_files'); + await db.execute('DELETE FROM device_collections'); + await db.execute('DELETE FROM entities'); } Future deleteDB() async { @@ -1668,29 +1668,6 @@ class FilesDB { return convertToFiles(results); } - ///For insertion, the syntax is as follows: - ///INSERT INTO table (column1,column2 ,..) - ///VALUES( value1, value2 ,...); - ///This method returns: - ///{ - /// 'columns': 'column1,column2 ,..', - /// 'values': 'value1, value2 ,...' - /// } - Map _getColumnsAndValuesForInsertion(EnteFile file) { - final row = _getRowForFile(file); - final columns = []; - final values = []; - for (int i = 0; i < row.entries.length; i++) { - final entry = row.entries.elementAt(i); - columns.add(entry.key); - values.add(entry.value.toString()); - } - return { - 'columns': columns.join(', '), - 'values': values.join(', '), - }; - } - String _getSetClauseForFile(EnteFile file) { final row = _getRowForFile(file); final setClause = []; @@ -1701,6 +1678,71 @@ class FilesDB { return setClause.join(', '); } + List _getParameterSetForFile(EnteFile file) { + final row = _getRowForFile(file); + final values = []; + for (int i = 0; i < row.entries.length; i++) { + values.add(row.entries.elementAt(i).value); + } + return values; + } + + List _getParameterSetForFileNew( + EnteFile file, { + bool omitNullGenId = true, + }) { + final values = []; + double? latitude; + double? longitude; + int? creationTime = file.creationTime; + if (file.pubMagicMetadata != null) { + if (file.pubMagicMetadata!.editedTime != null) { + creationTime = file.pubMagicMetadata!.editedTime; + } + if (file.pubMagicMetadata!.lat != null && + file.pubMagicMetadata!.long != null) { + latitude = file.pubMagicMetadata!.lat; + longitude = file.pubMagicMetadata!.long; + } + } + if (file.generatedID != null || !omitNullGenId) { + values.add(file.generatedID); + } + values.addAll([ + file.localID, + file.uploadedFileID ?? -1, + file.ownerID, + file.collectionID ?? -1, + file.title, + file.deviceFolder, + latitude, + longitude, + getInt(file.fileType), + file.modificationTime, + file.encryptedKey, + file.keyDecryptionNonce, + file.fileDecryptionHeader, + file.thumbnailDecryptionHeader, + file.metadataDecryptionHeader, + creationTime, + file.updationTime, + file.fileSubType ?? -1, + file.duration ?? 0, + file.exif, + file.hash, + file.metadataVersion, + file.mMdEncodedJson ?? {}, + file.mMdVersion, + file.magicMetadata.visibility, + file.pubMmdEncodedJson ?? {}, + file.pubMmdVersion, + file.fileSize, + file.addedTime ?? DateTime.now().microsecondsSinceEpoch, + ]); + + return values; + } + String _getSetClauseForFileWithoutCollection(EnteFile file) { final row = _getRowForFileWithoutCollection(file); final setClause = []; From 25554209ec7375b3d4e62e72608a0c242715d921 Mon Sep 17 00:00:00 2001 From: ashilkn Date: Wed, 15 May 2024 19:52:55 +0530 Subject: [PATCH 06/41] [mob][photos] Migrate to sqlite_async)(6): Migrate insertMultipleNew to use sqlite_async --- mobile/lib/db/files_db.dart | 158 +++++++++++++++++++++++- mobile/lib/utils/primitive_wrapper.dart | 6 + 2 files changed, 163 insertions(+), 1 deletion(-) create mode 100644 mobile/lib/utils/primitive_wrapper.dart diff --git a/mobile/lib/db/files_db.dart b/mobile/lib/db/files_db.dart index 7fde7ee2e7..4b014b9e84 100644 --- a/mobile/lib/db/files_db.dart +++ b/mobile/lib/db/files_db.dart @@ -15,6 +15,7 @@ import 'package:photos/models/location/location.dart'; import "package:photos/models/metadata/common_keys.dart"; import "package:photos/services/filter/db_filters.dart"; import 'package:photos/utils/file_uploader_util.dart'; +import "package:photos/utils/primitive_wrapper.dart"; import 'package:sqflite/sqflite.dart'; import 'package:sqflite_migration/sqflite_migration.dart'; import 'package:sqlite_async/sqlite_async.dart' as sqlite_async; @@ -91,6 +92,39 @@ class FilesDB { ...addAddedTime(), ]; + static const List _columnNames = [ + columnGeneratedID, + columnLocalID, + columnUploadedFileID, + columnOwnerID, + columnCollectionID, + columnTitle, + columnDeviceFolder, + columnLatitude, + columnLongitude, + columnFileType, + columnModificationTime, + columnEncryptedKey, + columnKeyDecryptionNonce, + columnFileDecryptionHeader, + columnThumbnailDecryptionHeader, + columnMetadataDecryptionHeader, + columnCreationTime, + columnUpdationTime, + columnFileSubType, + columnDuration, + columnExif, + columnHash, + columnMetadataVersion, + columnMMdEncodedJson, + columnMMdVersion, + columnMMdVisibility, + columnPubMMdEncodedJson, + columnPubMMdVersion, + columnFileSize, + columnAddedTime, + ]; + final dbConfig = MigrationConfig( initializationScript: initializationScript, migrationScripts: migrationScripts, @@ -455,6 +489,128 @@ class FilesDB { ); } + Future insertMultipleNew( + List files, { + ConflictAlgorithm conflictAlgorithm = ConflictAlgorithm.replace, + }) async { + final startTime = DateTime.now(); + final db = await sqliteAsyncDB; + + ///Strong batch counter in an object so that it gets passed by reference + ///Primitives are passed by value + final genIdNotNullbatchCounter = PrimitiveWrapper(0); + final genIdNullbatchCounter = PrimitiveWrapper(0); + final genIdNullParameterSets = >[]; + final genIdNotNullParameterSets = >[]; + + final genIdNullcolumnNames = + _columnNames.where((element) => element != columnGeneratedID); + + for (EnteFile file in files) { + final fileGenIdIsNull = file.generatedID == null; + + if (!fileGenIdIsNull) { + await _batchAndInsertFile( + file, + conflictAlgorithm, + db, + genIdNotNullParameterSets, + genIdNotNullbatchCounter, + isGenIdNull: fileGenIdIsNull, + ); + } else { + await _batchAndInsertFile( + file, + conflictAlgorithm, + db, + genIdNullParameterSets, + genIdNullbatchCounter, + isGenIdNull: fileGenIdIsNull, + ); + } + } + + if (genIdNotNullbatchCounter.value > 0) { + await _insertBatch( + conflictAlgorithm, + _columnNames, + db, + genIdNotNullParameterSets, + ); + genIdNotNullbatchCounter.value = 0; + genIdNotNullParameterSets.clear(); + } + if (genIdNullbatchCounter.value > 0) { + await _insertBatch( + conflictAlgorithm, + genIdNullcolumnNames, + db, + genIdNullParameterSets, + ); + genIdNullbatchCounter.value = 0; + genIdNullParameterSets.clear(); + } + + final endTime = DateTime.now(); + final duration = Duration( + microseconds: + endTime.microsecondsSinceEpoch - startTime.microsecondsSinceEpoch, + ); + _logger.info( + "Batch insert of " + + files.length.toString() + + " took " + + duration.inMilliseconds.toString() + + "ms.", + ); + } + + @pragma('vm:prefer-inline') + Future _batchAndInsertFile( + EnteFile file, + ConflictAlgorithm conflictAlgorithm, + sqlite_async.SqliteDatabase db, + List> parameterSets, + PrimitiveWrapper batchCounter, { + required bool isGenIdNull, + }) async { + parameterSets.add(_getParameterSetForFileV2(file)); + batchCounter.value++; + + final columnNames = isGenIdNull + ? _columnNames.where((column) => column != columnGeneratedID) + : _columnNames; + if (batchCounter.value == 400) { + _logger.info("Inserting batch with genIdNull: $isGenIdNull"); + await _insertBatch(conflictAlgorithm, columnNames, db, parameterSets); + // await db.executeBatch( + // ''' + // INSERT OR ${conflictAlgorithm.name.toUpperCase()} INTO $filesTable($columnNames) VALUES($valuesPlaceholders) + // ''', + // parameterSets, + // ); + batchCounter.value = 0; + parameterSets.clear(); + } + } + + Future _insertBatch( + ConflictAlgorithm conflictAlgorithm, + Iterable columnNames, + sqlite_async.SqliteDatabase db, + List> parameterSets, + ) async { + final valuesPlaceholders = List.filled(columnNames.length, "?").join(","); + final columnNamesJoined = columnNames.join(","); + await db.executeBatch( + ''' + INSERT OR ${conflictAlgorithm.name.toUpperCase()} INTO $filesTable($columnNamesJoined) VALUES($valuesPlaceholders) + ''', + parameterSets, + ); + } + + @pragma('vm:prefer-inline') Future insert(EnteFile file) async { _logger.info("Inserting $file"); final db = await instance.database; @@ -1687,7 +1843,7 @@ class FilesDB { return values; } - List _getParameterSetForFileNew( + List _getParameterSetForFileV2( EnteFile file, { bool omitNullGenId = true, }) { diff --git a/mobile/lib/utils/primitive_wrapper.dart b/mobile/lib/utils/primitive_wrapper.dart new file mode 100644 index 0000000000..20ea9bbb6e --- /dev/null +++ b/mobile/lib/utils/primitive_wrapper.dart @@ -0,0 +1,6 @@ +///This is useful when you want to pass a primitive by reference. + +class PrimitiveWrapper { + var value; + PrimitiveWrapper(this.value); +} From e179d351d990d82230ae42a8c650d17efbf580ed Mon Sep 17 00:00:00 2001 From: ashilkn Date: Wed, 15 May 2024 21:04:32 +0530 Subject: [PATCH 07/41] [mob][photos] Migrate to sqlite_async(7): Assign String '{}' instead of map object {} to fix unexpected behaviour --- mobile/lib/db/files_db.dart | 40 ++----------------------------------- 1 file changed, 2 insertions(+), 38 deletions(-) diff --git a/mobile/lib/db/files_db.dart b/mobile/lib/db/files_db.dart index 4b014b9e84..b8779c5fd5 100644 --- a/mobile/lib/db/files_db.dart +++ b/mobile/lib/db/files_db.dart @@ -456,42 +456,6 @@ class FilesDB { Future insertMultiple( List files, { ConflictAlgorithm conflictAlgorithm = ConflictAlgorithm.replace, - }) async { - final startTime = DateTime.now(); - final db = await database; - var batch = db.batch(); - int batchCounter = 0; - for (EnteFile file in files) { - if (batchCounter == 400) { - await batch.commit(noResult: true); - batch = db.batch(); - batchCounter = 0; - } - batch.insert( - filesTable, - _getRowForFile(file), - conflictAlgorithm: conflictAlgorithm, - ); - batchCounter++; - } - await batch.commit(noResult: true); - final endTime = DateTime.now(); - final duration = Duration( - microseconds: - endTime.microsecondsSinceEpoch - startTime.microsecondsSinceEpoch, - ); - _logger.info( - "Batch insert of " + - files.length.toString() + - " took " + - duration.inMilliseconds.toString() + - "ms.", - ); - } - - Future insertMultipleNew( - List files, { - ConflictAlgorithm conflictAlgorithm = ConflictAlgorithm.replace, }) async { final startTime = DateTime.now(); final db = await sqliteAsyncDB; @@ -1887,10 +1851,10 @@ class FilesDB { file.exif, file.hash, file.metadataVersion, - file.mMdEncodedJson ?? {}, + file.mMdEncodedJson ?? '{}', file.mMdVersion, file.magicMetadata.visibility, - file.pubMmdEncodedJson ?? {}, + file.pubMmdEncodedJson ?? '{}', file.pubMmdVersion, file.fileSize, file.addedTime ?? DateTime.now().microsecondsSinceEpoch, From 56478fcb8a620af7891eef9230405be3bc2d373b Mon Sep 17 00:00:00 2001 From: ashilkn Date: Wed, 15 May 2024 21:10:37 +0530 Subject: [PATCH 08/41] [mob][photos] avoid unnecessary compute --- mobile/lib/db/files_db.dart | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mobile/lib/db/files_db.dart b/mobile/lib/db/files_db.dart index b8779c5fd5..97f76ed8c8 100644 --- a/mobile/lib/db/files_db.dart +++ b/mobile/lib/db/files_db.dart @@ -457,6 +457,8 @@ class FilesDB { List files, { ConflictAlgorithm conflictAlgorithm = ConflictAlgorithm.replace, }) async { + if (files.isEmpty) return; + final startTime = DateTime.now(); final db = await sqliteAsyncDB; From 1e7779a81996eccab1742275b8c012c9f32470fc Mon Sep 17 00:00:00 2001 From: ashilkn Date: Wed, 15 May 2024 21:18:14 +0530 Subject: [PATCH 09/41] [mob][photos] Remove method inline annotation which doesn 't have noticeable perf improvement + remove commented out code --- mobile/lib/db/files_db.dart | 8 -------- 1 file changed, 8 deletions(-) diff --git a/mobile/lib/db/files_db.dart b/mobile/lib/db/files_db.dart index 97f76ed8c8..4ff7f20e2b 100644 --- a/mobile/lib/db/files_db.dart +++ b/mobile/lib/db/files_db.dart @@ -531,7 +531,6 @@ class FilesDB { ); } - @pragma('vm:prefer-inline') Future _batchAndInsertFile( EnteFile file, ConflictAlgorithm conflictAlgorithm, @@ -549,12 +548,6 @@ class FilesDB { if (batchCounter.value == 400) { _logger.info("Inserting batch with genIdNull: $isGenIdNull"); await _insertBatch(conflictAlgorithm, columnNames, db, parameterSets); - // await db.executeBatch( - // ''' - // INSERT OR ${conflictAlgorithm.name.toUpperCase()} INTO $filesTable($columnNames) VALUES($valuesPlaceholders) - // ''', - // parameterSets, - // ); batchCounter.value = 0; parameterSets.clear(); } @@ -576,7 +569,6 @@ class FilesDB { ); } - @pragma('vm:prefer-inline') Future insert(EnteFile file) async { _logger.info("Inserting $file"); final db = await instance.database; From 7fdc2b5e669feab6ceaaa6c8c6218e5508475ace Mon Sep 17 00:00:00 2001 From: ashilkn Date: Thu, 16 May 2024 12:48:21 +0530 Subject: [PATCH 10/41] [mob][photos] Migrate to sqlite_async(8): Fix faulty update statements due to incorrect query generation --- mobile/lib/db/files_db.dart | 72 ++++++++++++++++++++++++------------- 1 file changed, 48 insertions(+), 24 deletions(-) diff --git a/mobile/lib/db/files_db.dart b/mobile/lib/db/files_db.dart index 4ff7f20e2b..efd24b28db 100644 --- a/mobile/lib/db/files_db.dart +++ b/mobile/lib/db/files_db.dart @@ -539,7 +539,7 @@ class FilesDB { PrimitiveWrapper batchCounter, { required bool isGenIdNull, }) async { - parameterSets.add(_getParameterSetForFileV2(file)); + parameterSets.add(_getParameterSetForFile(file)); batchCounter.value++; final columnNames = isGenIdNull @@ -1149,11 +1149,14 @@ class FilesDB { Future update(EnteFile file) async { final db = await instance.sqliteAsyncDB; - final setClause = _getSetClauseForFile(file); + final parameterSet = _getParameterSetForFile(file)..add(file.generatedID); + final updateAssignments = _generateUpdateAssignmentsWithPlaceholders( + fileGenId: file.generatedID, + ); await db.execute( - 'UPDATE $filesTable SET ' - '$setClause WHERE $columnGeneratedID = ?', - [file.generatedID], + 'UPDATE $filesTable ' + 'SET $updateAssignments WHERE $columnGeneratedID = ?', + parameterSet, ); } @@ -1167,6 +1170,21 @@ class FilesDB { ); } + Future updateUploadedFileAcrossCollectionsNew(EnteFile file) async { + final db = await instance.sqliteAsyncDB; + final parameterSet = _getParameterSetForFile(file, omitCollectionId: true) + ..add(file.uploadedFileID); + final updateAssignments = _generateUpdateAssignmentsWithPlaceholders( + fileGenId: file.generatedID, + omitCollectionId: true, + ); + await db.execute( + 'UPDATE $filesTable' + 'SET $updateAssignments WHERE $columnUploadedFileID = ?', + parameterSet, + ); + } + Future updateLocalIDForUploaded(int uploadedID, String localID) async { final db = await instance.sqliteAsyncDB; await db.execute( @@ -1782,8 +1800,8 @@ class FilesDB { return convertToFiles(results); } - String _getSetClauseForFile(EnteFile file) { - final row = _getRowForFile(file); + String _getSetClauseForFileWithoutCollection(EnteFile file) { + final row = _getRowForFileWithoutCollection(file); final setClause = []; for (int i = 0; i < row.entries.length; i++) { final entry = row.entries.elementAt(i); @@ -1792,18 +1810,30 @@ class FilesDB { return setClause.join(', '); } - List _getParameterSetForFile(EnteFile file) { - final row = _getRowForFile(file); - final values = []; - for (int i = 0; i < row.entries.length; i++) { - values.add(row.entries.elementAt(i).value); + ///Returns "columnName1 = ?, columnName2 = ?, ..." + String _generateUpdateAssignmentsWithPlaceholders({ + required int? fileGenId, + bool omitCollectionId = false, + }) { + final setClauses = []; + + for (String columnName in _columnNames) { + if (columnName == columnGeneratedID && fileGenId == null) { + continue; + } + if (columnName == columnCollectionID && omitCollectionId) { + continue; + } + setClauses.add("$columnName = ?"); } - return values; + + return setClauses.join(","); } - List _getParameterSetForFileV2( + List _getParameterSetForFile( EnteFile file, { bool omitNullGenId = true, + bool omitCollectionId = false, }) { final values = []; double? latitude; @@ -1854,17 +1884,11 @@ class FilesDB { file.addedTime ?? DateTime.now().microsecondsSinceEpoch, ]); - return values; - } - - String _getSetClauseForFileWithoutCollection(EnteFile file) { - final row = _getRowForFileWithoutCollection(file); - final setClause = []; - for (int i = 0; i < row.entries.length; i++) { - final entry = row.entries.elementAt(i); - setClause.add('${entry.key} = ${entry.value}'); + if (omitCollectionId) { + values.removeAt(3); } - return setClause.join(', '); + + return values; } Map _getRowForFile(EnteFile file) { From cd023b621a698485d1d1af8a31d364fb47884680 Mon Sep 17 00:00:00 2001 From: ashilkn Date: Thu, 16 May 2024 12:59:19 +0530 Subject: [PATCH 11/41] [mob][photos] Remove optional parameter which should never be used Since generatedID (_id) has NOT NULL constrain, it shouldn't be in a parameter set of a query --- mobile/lib/db/files_db.dart | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/mobile/lib/db/files_db.dart b/mobile/lib/db/files_db.dart index efd24b28db..dd71e4b68c 100644 --- a/mobile/lib/db/files_db.dart +++ b/mobile/lib/db/files_db.dart @@ -1161,16 +1161,6 @@ class FilesDB { } Future updateUploadedFileAcrossCollections(EnteFile file) async { - final db = await instance.sqliteAsyncDB; - final setClause = _getSetClauseForFileWithoutCollection(file); - await db.execute( - 'UPDATE $filesTable SET ' - '$setClause WHERE $columnUploadedFileID = ?', - [file.uploadedFileID], - ); - } - - Future updateUploadedFileAcrossCollectionsNew(EnteFile file) async { final db = await instance.sqliteAsyncDB; final parameterSet = _getParameterSetForFile(file, omitCollectionId: true) ..add(file.uploadedFileID); @@ -1832,7 +1822,6 @@ class FilesDB { List _getParameterSetForFile( EnteFile file, { - bool omitNullGenId = true, bool omitCollectionId = false, }) { final values = []; @@ -1849,7 +1838,7 @@ class FilesDB { longitude = file.pubMagicMetadata!.long; } } - if (file.generatedID != null || !omitNullGenId) { + if (file.generatedID != null) { values.add(file.generatedID); } values.addAll([ From 584a37d2a20f66ac1b79f22fb1bd1350ff16fb70 Mon Sep 17 00:00:00 2001 From: ashilkn Date: Thu, 16 May 2024 14:20:03 +0530 Subject: [PATCH 12/41] [mob][photos] Remove obsolete code This code is from when we used to support favoriting un-uploaded files --- mobile/lib/services/favorites_service.dart | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/mobile/lib/services/favorites_service.dart b/mobile/lib/services/favorites_service.dart index fef4a323a8..2f64e63d41 100644 --- a/mobile/lib/services/favorites_service.dart +++ b/mobile/lib/services/favorites_service.dart @@ -151,9 +151,7 @@ class FavoritesService { final collectionID = await _getOrCreateFavoriteCollectionID(); final List files = [file]; if (file.uploadedFileID == null) { - file.collectionID = collectionID; - await _filesDB.insert(file); - Bus.instance.fire(CollectionUpdatedEvent(collectionID, files, "addTFav")); + throw AssertionError("Can only favorite uploaded items"); } else { await _collectionsService.addOrCopyToCollection(collectionID, files); } From 1a360d3ee7381c047c46192d26b61729fcc9959e Mon Sep 17 00:00:00 2001 From: ashilkn Date: Thu, 16 May 2024 15:37:00 +0530 Subject: [PATCH 13/41] [mob][photos] Migrate to sqlite_async(8): Migrate insert() + rearrange + clean up --- mobile/lib/db/files_db.dart | 186 +++++++++--------- .../ui/tools/editor/image_editor_page.dart | 2 +- 2 files changed, 89 insertions(+), 99 deletions(-) diff --git a/mobile/lib/db/files_db.dart b/mobile/lib/db/files_db.dart index dd71e4b68c..68b80d9d55 100644 --- a/mobile/lib/db/files_db.dart +++ b/mobile/lib/db/files_db.dart @@ -531,54 +531,35 @@ class FilesDB { ); } - Future _batchAndInsertFile( - EnteFile file, - ConflictAlgorithm conflictAlgorithm, - sqlite_async.SqliteDatabase db, - List> parameterSets, - PrimitiveWrapper batchCounter, { - required bool isGenIdNull, - }) async { - parameterSets.add(_getParameterSetForFile(file)); - batchCounter.value++; - - final columnNames = isGenIdNull - ? _columnNames.where((column) => column != columnGeneratedID) - : _columnNames; - if (batchCounter.value == 400) { - _logger.info("Inserting batch with genIdNull: $isGenIdNull"); - await _insertBatch(conflictAlgorithm, columnNames, db, parameterSets); - batchCounter.value = 0; - parameterSets.clear(); - } - } - - Future _insertBatch( - ConflictAlgorithm conflictAlgorithm, - Iterable columnNames, - sqlite_async.SqliteDatabase db, - List> parameterSets, - ) async { - final valuesPlaceholders = List.filled(columnNames.length, "?").join(","); - final columnNamesJoined = columnNames.join(","); - await db.executeBatch( - ''' - INSERT OR ${conflictAlgorithm.name.toUpperCase()} INTO $filesTable($columnNamesJoined) VALUES($valuesPlaceholders) - ''', - parameterSets, - ); - } - - Future insert(EnteFile file) async { + Future insert(EnteFile file) async { _logger.info("Inserting $file"); - final db = await instance.database; - return db.insert( - filesTable, - _getRowForFile(file), - conflictAlgorithm: ConflictAlgorithm.replace, + final db = await instance.sqliteAsyncDB; + final columnsAndPlaceholders = + _generateColumnsAndPlaceholdersForInsert(fileGenId: file.generatedID); + final values = _getParameterSetForFile(file); + + await db.execute( + 'INSERT OR REPLACE INTO $filesTable (${columnsAndPlaceholders["columns"]}) VALUES (${columnsAndPlaceholders["placeholders"]})', + values, ); } + Future insertAndGetId(EnteFile file) async { + _logger.info("Inserting $file"); + final db = await instance.sqliteAsyncDB; + final columnsAndPlaceholders = + _generateColumnsAndPlaceholdersForInsert(fileGenId: file.generatedID); + final values = _getParameterSetForFile(file); + return await db.writeTransaction((tx) async { + await tx.execute( + 'INSERT OR REPLACE INTO $filesTable (${columnsAndPlaceholders["columns"]}) VALUES (${columnsAndPlaceholders["placeholders"]})', + values, + ); + final result = await tx.get('SELECT last_insert_rowid()'); + return result["last_insert_rowid()"] as int; + }); + } + Future getFile(int generatedID) async { final db = await instance.sqliteAsyncDB; final results = await db.getAll( @@ -1790,22 +1771,12 @@ class FilesDB { return convertToFiles(results); } - String _getSetClauseForFileWithoutCollection(EnteFile file) { - final row = _getRowForFileWithoutCollection(file); - final setClause = []; - for (int i = 0; i < row.entries.length; i++) { - final entry = row.entries.elementAt(i); - setClause.add('${entry.key} = ${entry.value}'); - } - return setClause.join(', '); - } - ///Returns "columnName1 = ?, columnName2 = ?, ..." String _generateUpdateAssignmentsWithPlaceholders({ required int? fileGenId, bool omitCollectionId = false, }) { - final setClauses = []; + final assignments = []; for (String columnName in _columnNames) { if (columnName == columnGeneratedID && fileGenId == null) { @@ -1814,10 +1785,29 @@ class FilesDB { if (columnName == columnCollectionID && omitCollectionId) { continue; } - setClauses.add("$columnName = ?"); + assignments.add("$columnName = ?"); } - return setClauses.join(","); + return assignments.join(","); + } + + Map _generateColumnsAndPlaceholdersForInsert({ + required int? fileGenId, + }) { + final columnNames = []; + + for (String columnName in _columnNames) { + if (columnName == columnGeneratedID && fileGenId == null) { + continue; + } + + columnNames.add(columnName); + } + + return { + "columns": columnNames.join(","), + "placeholders": List.filled(columnNames.length, "?").join(","), + }; } List _getParameterSetForFile( @@ -1825,8 +1815,10 @@ class FilesDB { bool omitCollectionId = false, }) { final values = []; + double? latitude; double? longitude; + int? creationTime = file.creationTime; if (file.pubMagicMetadata != null) { if (file.pubMagicMetadata!.editedTime != null) { @@ -1838,6 +1830,7 @@ class FilesDB { longitude = file.pubMagicMetadata!.long; } } + if (file.generatedID != null) { values.add(file.generatedID); } @@ -1880,6 +1873,44 @@ class FilesDB { return values; } + Future _batchAndInsertFile( + EnteFile file, + ConflictAlgorithm conflictAlgorithm, + sqlite_async.SqliteDatabase db, + List> parameterSets, + PrimitiveWrapper batchCounter, { + required bool isGenIdNull, + }) async { + parameterSets.add(_getParameterSetForFile(file)); + batchCounter.value++; + + final columnNames = isGenIdNull + ? _columnNames.where((column) => column != columnGeneratedID) + : _columnNames; + if (batchCounter.value == 400) { + _logger.info("Inserting batch with genIdNull: $isGenIdNull"); + await _insertBatch(conflictAlgorithm, columnNames, db, parameterSets); + batchCounter.value = 0; + parameterSets.clear(); + } + } + + Future _insertBatch( + ConflictAlgorithm conflictAlgorithm, + Iterable columnNames, + sqlite_async.SqliteDatabase db, + List> parameterSets, + ) async { + final valuesPlaceholders = List.filled(columnNames.length, "?").join(","); + final columnNamesJoined = columnNames.join(","); + await db.executeBatch( + ''' + INSERT OR ${conflictAlgorithm.name.toUpperCase()} INTO $filesTable($columnNamesJoined) VALUES($valuesPlaceholders) + ''', + parameterSets, + ); + } + Map _getRowForFile(EnteFile file) { final row = {}; if (file.generatedID != null) { @@ -1935,47 +1966,6 @@ class FilesDB { return row; } - Map _getRowForFileWithoutCollection(EnteFile file) { - final row = {}; - row[columnLocalID] = file.localID; - row[columnUploadedFileID] = file.uploadedFileID ?? -1; - row[columnOwnerID] = file.ownerID; - row[columnTitle] = file.title; - row[columnDeviceFolder] = file.deviceFolder; - if (file.location != null) { - row[columnLatitude] = file.location!.latitude; - row[columnLongitude] = file.location!.longitude; - } - row[columnFileType] = getInt(file.fileType); - row[columnCreationTime] = file.creationTime; - row[columnModificationTime] = file.modificationTime; - row[columnUpdationTime] = file.updationTime; - row[columnAddedTime] = - file.addedTime ?? DateTime.now().microsecondsSinceEpoch; - row[columnFileDecryptionHeader] = file.fileDecryptionHeader; - row[columnThumbnailDecryptionHeader] = file.thumbnailDecryptionHeader; - row[columnMetadataDecryptionHeader] = file.metadataDecryptionHeader; - row[columnFileSubType] = file.fileSubType ?? -1; - row[columnDuration] = file.duration ?? 0; - row[columnExif] = file.exif; - row[columnHash] = file.hash; - row[columnMetadataVersion] = file.metadataVersion; - - row[columnMMdVersion] = file.mMdVersion; - row[columnMMdEncodedJson] = file.mMdEncodedJson ?? '{}'; - row[columnMMdVisibility] = file.magicMetadata.visibility; - - row[columnPubMMdVersion] = file.pubMmdVersion; - row[columnPubMMdEncodedJson] = file.pubMmdEncodedJson ?? '{}'; - if (file.pubMagicMetadata != null && - file.pubMagicMetadata!.editedTime != null) { - // override existing creationTime to avoid re-writing all queries related - // to loading the gallery - row[columnCreationTime] = file.pubMagicMetadata!.editedTime!; - } - return row; - } - EnteFile _getFileFromRow(Map row) { final file = EnteFile(); file.generatedID = row[columnGeneratedID]; diff --git a/mobile/lib/ui/tools/editor/image_editor_page.dart b/mobile/lib/ui/tools/editor/image_editor_page.dart index 4830df9523..5314d9ca86 100644 --- a/mobile/lib/ui/tools/editor/image_editor_page.dart +++ b/mobile/lib/ui/tools/editor/image_editor_page.dart @@ -371,7 +371,7 @@ class _ImageEditorPageState extends State { ); } } - newFile.generatedID = await FilesDB.instance.insert(newFile); + newFile.generatedID = await FilesDB.instance.insertAndGetId(newFile); Bus.instance.fire(LocalPhotosUpdatedEvent([newFile], source: "editSave")); unawaited(SyncService.instance.sync()); showShortToast(context, S.of(context).editsSaved); From dec7c45310da3e668e7fd5e3f77e20858fbb48a4 Mon Sep 17 00:00:00 2001 From: ashilkn Date: Thu, 16 May 2024 16:41:57 +0530 Subject: [PATCH 14/41] [mob][photos] Migrate to sqlite_async(9) --- mobile/lib/db/files_db.dart | 87 ++++++++----------------------------- 1 file changed, 17 insertions(+), 70 deletions(-) diff --git a/mobile/lib/db/files_db.dart b/mobile/lib/db/files_db.dart index 68b80d9d55..a5e00f352c 100644 --- a/mobile/lib/db/files_db.dart +++ b/mobile/lib/db/files_db.dart @@ -606,13 +606,11 @@ class FilesDB { Future<(Set, Map)> getUploadAndHash( int collectionID, ) async { - final db = await instance.database; - final results = await db.query( - filesTable, - columns: [columnUploadedFileID, columnHash], - where: - '$columnCollectionID = ? AND ($columnUploadedFileID IS NOT NULL AND $columnUploadedFileID IS NOT -1)', - whereArgs: [ + final db = await instance.sqliteAsyncDB; + final results = await db.getAll( + 'SELECT $columnUploadedFileID, $columnHash FROM $filesTable' + ' WHERE $columnCollectionID = ? AND ($columnUploadedFileID IS NOT NULL AND $columnUploadedFileID IS NOT -1)', + [ collectionID, ], ); @@ -1265,18 +1263,22 @@ class FilesDB { ); } - /// Uses int in return value. Future deleteFilesFromCollection( int collectionID, List uploadedFileIDs, ) async { - final db = await instance.database; - return db.delete( - filesTable, - where: - '$columnCollectionID = ? AND $columnUploadedFileID IN (${uploadedFileIDs.join(', ')})', - whereArgs: [collectionID], - ); + final db = await instance.sqliteAsyncDB; + return db.writeTransaction((tx) async { + await tx.execute( + ''' + DELETE FROM $filesTable + WHERE $columnCollectionID = ? AND $columnUploadedFileID IN (${uploadedFileIDs.join(', ')}); + ''', + [collectionID], + ); + final res = await tx.get('SELECT changes()'); + return res['changes()'] as int; + }); } Future collectionFileCount(int collectionID) async { @@ -1911,61 +1913,6 @@ class FilesDB { ); } - Map _getRowForFile(EnteFile file) { - final row = {}; - if (file.generatedID != null) { - row[columnGeneratedID] = file.generatedID; - } - row[columnLocalID] = file.localID; - row[columnUploadedFileID] = file.uploadedFileID ?? -1; - row[columnOwnerID] = file.ownerID; - row[columnCollectionID] = file.collectionID ?? -1; - row[columnTitle] = file.title; - row[columnDeviceFolder] = file.deviceFolder; - // if (file.location == null || - // (file.location!.latitude == null && file.location!.longitude == null)) { - // file.location = Location.randomLocation(); - // } - if (file.location != null) { - row[columnLatitude] = file.location!.latitude; - row[columnLongitude] = file.location!.longitude; - } - row[columnFileType] = getInt(file.fileType); - row[columnCreationTime] = file.creationTime; - row[columnModificationTime] = file.modificationTime; - row[columnUpdationTime] = file.updationTime; - row[columnAddedTime] = - file.addedTime ?? DateTime.now().microsecondsSinceEpoch; - row[columnEncryptedKey] = file.encryptedKey; - row[columnKeyDecryptionNonce] = file.keyDecryptionNonce; - row[columnFileDecryptionHeader] = file.fileDecryptionHeader; - row[columnThumbnailDecryptionHeader] = file.thumbnailDecryptionHeader; - row[columnMetadataDecryptionHeader] = file.metadataDecryptionHeader; - row[columnFileSubType] = file.fileSubType ?? -1; - row[columnDuration] = file.duration ?? 0; - row[columnExif] = file.exif; - row[columnHash] = file.hash; - row[columnMetadataVersion] = file.metadataVersion; - row[columnFileSize] = file.fileSize; - row[columnMMdVersion] = file.mMdVersion; - row[columnMMdEncodedJson] = file.mMdEncodedJson ?? '{}'; - row[columnMMdVisibility] = file.magicMetadata.visibility; - row[columnPubMMdVersion] = file.pubMmdVersion; - row[columnPubMMdEncodedJson] = file.pubMmdEncodedJson ?? '{}'; - // override existing fields to avoid re-writing all queries and logic - if (file.pubMagicMetadata != null) { - if (file.pubMagicMetadata!.editedTime != null) { - row[columnCreationTime] = file.pubMagicMetadata!.editedTime; - } - if (file.pubMagicMetadata!.lat != null && - file.pubMagicMetadata!.long != null) { - row[columnLatitude] = file.pubMagicMetadata!.lat; - row[columnLongitude] = file.pubMagicMetadata!.long; - } - } - return row; - } - EnteFile _getFileFromRow(Map row) { final file = EnteFile(); file.generatedID = row[columnGeneratedID]; From 16d54645bc76aaf4be1af6392fdecef51898169c Mon Sep 17 00:00:00 2001 From: ashilkn Date: Thu, 16 May 2024 18:02:39 +0530 Subject: [PATCH 15/41] [mob][photos] Migrate to sqlite_async(10) --- mobile/lib/db/device_files_db.dart | 199 +++++++++++++++++------------ 1 file changed, 117 insertions(+), 82 deletions(-) diff --git a/mobile/lib/db/device_files_db.dart b/mobile/lib/db/device_files_db.dart index 25c88daca1..73fc3a62b3 100644 --- a/mobile/lib/db/device_files_db.dart +++ b/mobile/lib/db/device_files_db.dart @@ -22,61 +22,55 @@ extension DeviceFiles on FilesDB { ConflictAlgorithm conflictAlgorithm = ConflictAlgorithm.ignore, }) async { debugPrint("Inserting missing PathIDToLocalIDMapping"); - final db = await database; - var batch = db.batch(); + final parameterSets = >[]; int batchCounter = 0; for (MapEntry e in mappingToAdd.entries) { final String pathID = e.key; for (String localID in e.value) { + parameterSets.add([localID, pathID]); + batchCounter++; + if (batchCounter == 400) { - await batch.commit(noResult: true); - batch = db.batch(); + await _insertBatch(parameterSets, conflictAlgorithm); + parameterSets.clear(); batchCounter = 0; } - batch.insert( - "device_files", - { - "id": localID, - "path_id": pathID, - }, - conflictAlgorithm: conflictAlgorithm, - ); - batchCounter++; } } - await batch.commit(noResult: true); + await _insertBatch(parameterSets, conflictAlgorithm); + parameterSets.clear(); + batchCounter = 0; } Future deletePathIDToLocalIDMapping( Map> mappingsToRemove, ) async { debugPrint("removing PathIDToLocalIDMapping"); - final db = await database; - var batch = db.batch(); + final parameterSets = >[]; int batchCounter = 0; for (MapEntry e in mappingsToRemove.entries) { final String pathID = e.key; + for (String localID in e.value) { + parameterSets.add([localID, pathID]); + batchCounter++; + if (batchCounter == 400) { - await batch.commit(noResult: true); - batch = db.batch(); + await _deleteBatch(parameterSets); + parameterSets.clear(); batchCounter = 0; } - batch.delete( - "device_files", - where: 'id = ? AND path_id = ?', - whereArgs: [localID, pathID], - ); - batchCounter++; } } - await batch.commit(noResult: true); + await _deleteBatch(parameterSets); + parameterSets.clear(); + batchCounter = 0; } Future> getDevicePathIDToImportedFileCount() async { try { - final db = await database; - final rows = await db.rawQuery( + final db = await sqliteAsyncDB; + final rows = await db.getAll( ''' SELECT count(*) as count, path_id FROM device_files @@ -96,8 +90,8 @@ extension DeviceFiles on FilesDB { Future>> getDevicePathIDToLocalIDMap() async { try { - final db = await database; - final rows = await db.rawQuery( + final db = await sqliteAsyncDB; + final rows = await db.getAll( ''' SELECT id, path_id FROM device_files; ''', ); final result = >{}; @@ -116,8 +110,8 @@ extension DeviceFiles on FilesDB { } Future> getDevicePathIDs() async { - final Database db = await database; - final rows = await db.rawQuery( + final db = await sqliteAsyncDB; + final rows = await db.getAll( ''' SELECT id FROM device_collections ''', @@ -133,34 +127,42 @@ extension DeviceFiles on FilesDB { List localPathAssets, { bool shouldAutoBackup = false, }) async { - final Database db = await database; + final db = await sqliteAsyncDB; final Map> pathIDToLocalIDsMap = {}; try { - final batch = db.batch(); final Set existingPathIds = await getDevicePathIDs(); + final parameterSetsForUpdate = >[]; + final parameterSetsForInsert = >[]; for (LocalPathAsset localPathAsset in localPathAssets) { if (localPathAsset.localIDs.isNotEmpty) { pathIDToLocalIDsMap[localPathAsset.pathID] = localPathAsset.localIDs; } if (existingPathIds.contains(localPathAsset.pathID)) { - batch.rawUpdate( - "UPDATE device_collections SET name = ? where id = " - "?", - [localPathAsset.pathName, localPathAsset.pathID], - ); + parameterSetsForUpdate + .add([localPathAsset.pathName, localPathAsset.pathID]); } else if (localPathAsset.localIDs.isNotEmpty) { - batch.insert( - "device_collections", - { - "id": localPathAsset.pathID, - "name": localPathAsset.pathName, - "should_backup": shouldAutoBackup ? _sqlBoolTrue : _sqlBoolFalse, - }, - conflictAlgorithm: ConflictAlgorithm.ignore, - ); + parameterSetsForInsert.add([ + localPathAsset.pathID, + localPathAsset.pathName, + shouldAutoBackup ? _sqlBoolTrue : _sqlBoolFalse, + ]); } } - await batch.commit(noResult: true); + + await db.executeBatch( + ''' + INSERT OR IGNORE INTO device_collections (id, name, should_backup) VALUES (?, ?, ?); + ''', + parameterSetsForInsert, + ); + + await db.executeBatch( + ''' + UPDATE device_collections SET name = ? WHERE id = ?; + ''', + parameterSetsForUpdate, + ); + // add the mappings for localIDs if (pathIDToLocalIDsMap.isNotEmpty) { await insertPathIDToLocalIDMapping(pathIDToLocalIDsMap); @@ -177,7 +179,7 @@ extension DeviceFiles on FilesDB { }) async { bool hasUpdated = false; try { - final Database db = await database; + final db = await sqliteAsyncDB; final Set existingPathIds = await getDevicePathIDs(); for (Tuple2 tup in devicePathInfo) { final AssetPathEntity pathEntity = tup.item1; @@ -185,35 +187,42 @@ extension DeviceFiles on FilesDB { final String localID = tup.item2; final bool shouldUpdate = existingPathIds.contains(pathEntity.id); if (shouldUpdate) { - final rowUpdated = await db.rawUpdate( - "UPDATE device_collections SET name = ?, cover_id = ?, count" - " = ? where id = ? AND (name != ? OR cover_id != ? OR count != ?)", - [ - pathEntity.name, - localID, - assetCount, - pathEntity.id, - pathEntity.name, - localID, - assetCount, - ], - ); + final rowUpdated = await db.writeTransaction((tx) async { + await tx.execute( + "UPDATE device_collections SET name = ?, cover_id = ?, count" + " = ? where id = ? AND (name != ? OR cover_id != ? OR count != ?)", + [ + pathEntity.name, + localID, + assetCount, + pathEntity.id, + pathEntity.name, + localID, + assetCount, + ], + ); + final result = await tx.get("SELECT changes();"); + return result["changes()"] as int; + }); + if (rowUpdated > 0) { _logger.fine("Updated $rowUpdated rows for ${pathEntity.name}"); hasUpdated = true; } } else { hasUpdated = true; - await db.insert( - "device_collections", - { - "id": pathEntity.id, - "name": pathEntity.name, - "count": assetCount, - "cover_id": localID, - "should_backup": shouldBackup ? _sqlBoolTrue : _sqlBoolFalse, - }, - conflictAlgorithm: ConflictAlgorithm.ignore, + await db.execute( + ''' + INSERT INTO device_collections (id, name, count, cover_id, should_backup) + VALUES (?, ?, ?, ?, ?); + ''', + [ + pathEntity.id, + pathEntity.name, + assetCount, + localID, + shouldBackup ? _sqlBoolTrue : _sqlBoolFalse, + ], ); } } @@ -231,15 +240,17 @@ extension DeviceFiles on FilesDB { // feature, where we delete files which are backed up. Deleting such // entries here result in us losing out on the information that // those folders were marked for automatic backup. - await db.delete( - "device_collections", - where: 'id = ? and should_backup = $_sqlBoolFalse ', - whereArgs: [pathID], + await db.execute( + ''' + DELETE FROM device_collections WHERE id = ? AND should_backup = $_sqlBoolFalse; + ''', + [pathID], ); - await db.delete( - "device_files", - where: 'path_id = ?', - whereArgs: [pathID], + await db.execute( + ''' + DELETE FROM device_files WHERE path_id = ?; + ''', + [pathID], ); } } @@ -253,8 +264,8 @@ extension DeviceFiles on FilesDB { // getDeviceSyncCollectionIDs returns the collectionIDs for the // deviceCollections which are marked for auto-backup Future> getDeviceSyncCollectionIDs() async { - final Database db = await database; - final rows = await db.rawQuery( + final db = await sqliteAsyncDB; + final rows = await db.getAll( ''' SELECT collection_id FROM device_collections where should_backup = $_sqlBoolTrue @@ -447,4 +458,28 @@ extension DeviceFiles on FilesDB { return null; } } + + Future _insertBatch( + List> parameterSets, + ConflictAlgorithm conflictAlgorithm, + ) async { + final db = await sqliteAsyncDB; + await db.executeBatch( + ''' + INSERT OR ${conflictAlgorithm.name.toUpperCase()} + INTO device_files (id, path_id) VALUES (?, ?); + ''', + parameterSets, + ); + } + + Future _deleteBatch(List> parameterSets) async { + final db = await sqliteAsyncDB; + await db.executeBatch( + ''' + DELETE FROM device_files WHERE id = ? AND path_id = ?; + ''', + parameterSets, + ); + } } From 2b0fa9bae62d7db88bf2f83d015354b90e36b6e4 Mon Sep 17 00:00:00 2001 From: ashilkn Date: Thu, 16 May 2024 19:34:59 +0530 Subject: [PATCH 16/41] [mob][photos] Migrate to sqlite_async(11) --- mobile/lib/db/device_files_db.dart | 67 +++++++++++++++++------------- 1 file changed, 37 insertions(+), 30 deletions(-) diff --git a/mobile/lib/db/device_files_db.dart b/mobile/lib/db/device_files_db.dart index 73fc3a62b3..fbd1649e26 100644 --- a/mobile/lib/db/device_files_db.dart +++ b/mobile/lib/db/device_files_db.dart @@ -279,40 +279,47 @@ extension DeviceFiles on FilesDB { return result; } - Future updateDevicePathSyncStatus(Map syncStatus) async { - final db = await database; - var batch = db.batch(); + Future updateDevicePathSyncStatus( + Map syncStatus, + ) async { + final db = await sqliteAsyncDB; int batchCounter = 0; + final parameterSets = >[]; for (MapEntry e in syncStatus.entries) { final String pathID = e.key; + parameterSets.add([e.value ? _sqlBoolTrue : _sqlBoolFalse, pathID]); + batchCounter++; + if (batchCounter == 400) { - await batch.commit(noResult: true); - batch = db.batch(); + await db.executeBatch( + ''' + UPDATE device_collections SET should_backup = ? WHERE id = ?; + ''', + parameterSets, + ); + parameterSets.clear(); batchCounter = 0; } - batch.update( - "device_collections", - { - "should_backup": e.value ? _sqlBoolTrue : _sqlBoolFalse, - }, - where: 'id = ?', - whereArgs: [pathID], - ); - batchCounter++; } - await batch.commit(noResult: true); + + await db.executeBatch( + ''' + UPDATE device_collections SET should_backup = ? WHERE id = ?; + ''', + parameterSets, + ); } Future updateDeviceCollection( String pathID, int collectionID, ) async { - final db = await database; - await db.update( - "device_collections", - {"collection_id": collectionID}, - where: 'id = ?', - whereArgs: [pathID], + final db = await sqliteAsyncDB; + await db.execute( + ''' + UPDATE device_collections SET collection_id = ? WHERE id = ?; + ''', + [collectionID, pathID], ); return; } @@ -325,7 +332,7 @@ extension DeviceFiles on FilesDB { int? limit, bool? asc, }) async { - final db = await database; + final db = await sqliteAsyncDB; final order = (asc ?? false ? 'ASC' : 'DESC'); final String rawQuery = ''' SELECT * @@ -340,7 +347,7 @@ extension DeviceFiles on FilesDB { ORDER BY ${FilesDB.columnCreationTime} $order , ${FilesDB.columnModificationTime} $order ''' + (limit != null ? ' limit $limit;' : ';'); - final results = await db.rawQuery(rawQuery); + final results = await db.getAll(rawQuery); final files = convertToFiles(results); final dedupe = deduplicateByLocalID(files); return FileLoadResult(dedupe, files.length == limit); @@ -350,7 +357,7 @@ extension DeviceFiles on FilesDB { String pathID, int ownerID, ) async { - final db = await database; + final db = await sqliteAsyncDB; const String rawQuery = ''' SELECT ${FilesDB.columnLocalID}, ${FilesDB.columnUploadedFileID}, ${FilesDB.columnFileSize} @@ -362,7 +369,7 @@ extension DeviceFiles on FilesDB { ${FilesDB.columnLocalID} IN (SELECT id FROM device_files where path_id = ?) '''; - final results = await db.rawQuery(rawQuery, [ownerID, pathID]); + final results = await db.getAll(rawQuery, [ownerID, pathID]); final localIDs = {}; final uploadedIDs = {}; int localSize = 0; @@ -386,17 +393,17 @@ extension DeviceFiles on FilesDB { "$includeCoverThumbnail", ); try { - final db = await database; + final db = await sqliteAsyncDB; final coverFiles = []; if (includeCoverThumbnail) { - final fileRows = await db.rawQuery( + final fileRows = await db.getAll( '''SELECT * FROM FILES where local_id in (select cover_id from device_collections) group by local_id; ''', ); final files = convertToFiles(fileRows); coverFiles.addAll(files); } - final deviceCollectionRows = await db.rawQuery( + final deviceCollectionRows = await db.getAll( '''SELECT * from device_collections''', ); final List deviceCollections = []; @@ -444,8 +451,8 @@ extension DeviceFiles on FilesDB { Future getDeviceCollectionThumbnail(String pathID) async { debugPrint("Call fallback method to get potential thumbnail"); - final db = await database; - final fileRows = await db.rawQuery( + final db = await sqliteAsyncDB; + final fileRows = await db.getAll( '''SELECT * FROM FILES f JOIN device_files df on f.local_id = df.id and df.path_id= ? order by f.creation_time DESC limit 1; ''', From 28ddb93747007b64ed6a17b4fe44a72f45a9886d Mon Sep 17 00:00:00 2001 From: ashilkn Date: Thu, 16 May 2024 20:17:58 +0530 Subject: [PATCH 17/41] [mob][photos] Add missing parameters for query --- mobile/lib/db/files_db.dart | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/mobile/lib/db/files_db.dart b/mobile/lib/db/files_db.dart index a5e00f352c..2b6219cbe7 100644 --- a/mobile/lib/db/files_db.dart +++ b/mobile/lib/db/files_db.dart @@ -905,10 +905,12 @@ class FilesDB { Future> getUploadedFileIDsToBeUpdated(int ownerID) async { final db = await instance.sqliteAsyncDB; final rows = await db.getAll( - 'SELECT DISTINCT $columnUploadedFileID FROM $filesTable WHERE ' - '($columnLocalID IS NOT NULL AND $columnOwnerID = ? AND ' - '($columnUploadedFileID IS NOT NULL AND $columnUploadedFileID IS NOT -1) ' - 'AND $columnUpdationTime IS NULL) ORDER BY $columnCreationTime DESC '); + 'SELECT DISTINCT $columnUploadedFileID FROM $filesTable WHERE ' + '($columnLocalID IS NOT NULL AND $columnOwnerID = ? AND ' + '($columnUploadedFileID IS NOT NULL AND $columnUploadedFileID IS NOT -1) ' + 'AND $columnUpdationTime IS NULL) ORDER BY $columnCreationTime DESC ', + [ownerID], + ); final uploadedFileIDs = []; for (final row in rows) { uploadedFileIDs.add(row[columnUploadedFileID] as int); From a44e5f950528cc3d1f14286b4a21b3765bafed09 Mon Sep 17 00:00:00 2001 From: ashilkn Date: Fri, 17 May 2024 11:47:32 +0530 Subject: [PATCH 18/41] [mob][photos] Migrate to sqlite_async(12): Migrate entities --- mobile/lib/db/entities_db.dart | 79 ++++++++++++++++++++++------------ 1 file changed, 51 insertions(+), 28 deletions(-) diff --git a/mobile/lib/db/entities_db.dart b/mobile/lib/db/entities_db.dart index b8b48fbe4a..d3759389d6 100644 --- a/mobile/lib/db/entities_db.dart +++ b/mobile/lib/db/entities_db.dart @@ -10,53 +10,76 @@ extension EntitiesDB on FilesDB { ConflictAlgorithm conflictAlgorithm = ConflictAlgorithm.replace, }) async { debugPrint("Inserting missing PathIDToLocalIDMapping"); - final db = await database; - var batch = db.batch(); + final db = await sqliteAsyncDB; + final parameterSets = >[]; int batchCounter = 0; for (LocalEntityData e in data) { + parameterSets.add([ + e.id, + e.type.name, + e.ownerID, + e.data, + e.updatedAt, + ]); + batchCounter++; + if (batchCounter == 400) { - await batch.commit(noResult: true); - batch = db.batch(); + await db.executeBatch( + ''' + INSERT OR ${conflictAlgorithm.name.toUpperCase()} + INTO entities (id, type, ownerID, data, updatedAt) +''', + parameterSets, + ); + parameterSets.clear(); batchCounter = 0; } - batch.insert( - "entities", - e.toJson(), - conflictAlgorithm: conflictAlgorithm, - ); - batchCounter++; } - await batch.commit(noResult: true); + await db.executeBatch( + ''' + INSERT OR ${conflictAlgorithm.name.toUpperCase()} + INTO entities (id, type, ownerID, data, updatedAt) +''', + parameterSets, + ); } Future deleteEntities( List ids, ) async { - final db = await database; - var batch = db.batch(); + final db = await sqliteAsyncDB; + final parameterSets = >[]; int batchCounter = 0; for (String id in ids) { - if (batchCounter == 400) { - await batch.commit(noResult: true); - batch = db.batch(); - batchCounter = 0; - } - batch.delete( - "entities", - where: "id = ?", - whereArgs: [id], + parameterSets.add( + [id], ); batchCounter++; + + if (batchCounter == 400) { + await db.executeBatch( + ''' + DELETE FROM entities WHERE id = ? + ''', + parameterSets, + ); + parameterSets.clear(); + batchCounter = 0; + } } - await batch.commit(noResult: true); + await db.executeBatch( + ''' + DELETE FROM entities WHERE id = ? + ''', + parameterSets, + ); } Future> getEntities(EntityType type) async { - final db = await database; - final List> maps = await db.query( - "entities", - where: "type = ?", - whereArgs: [type.typeToString()], + final db = await sqliteAsyncDB; + final List> maps = await db.getAll( + 'SELECT * FROM entities WHERE type = ?', + [type.name], ); return List.generate(maps.length, (i) { return LocalEntityData.fromJson(maps[i]); From c2b6032b6f6214ac074efa208a753f4ce52cde0c Mon Sep 17 00:00:00 2001 From: ashilkn Date: Fri, 17 May 2024 13:40:38 +0530 Subject: [PATCH 19/41] [mob][photos] Fix broken query --- mobile/lib/db/files_db.dart | 26 ++++++++++--------- .../ui/viewer/location/location_screen.dart | 4 +-- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/mobile/lib/db/files_db.dart b/mobile/lib/db/files_db.dart index 2b6219cbe7..16db97337e 100644 --- a/mobile/lib/db/files_db.dart +++ b/mobile/lib/db/files_db.dart @@ -139,14 +139,8 @@ class FilesDB { static Future? _dbFuture; static Future? _sqliteAsyncDBFuture; - @Deprecated("Use sqliteAsyncDB instead (sqlite_async)") - Future get database async { - // lazily instantiate the db the first time it is accessed - _dbFuture ??= _initDatabase(); - return _dbFuture!; - } - Future get sqliteAsyncDB async { + // lazily instantiate the db the first time it is accessed _sqliteAsyncDBFuture ??= _initSqliteAsyncDatabase(); return _sqliteAsyncDBFuture!; } @@ -1720,8 +1714,7 @@ class FilesDB { }) async { final db = await instance.sqliteAsyncDB; final order = (asc ?? false ? 'ASC' : 'DESC'); - final results = await db.getAll( - ''' + String query = ''' SELECT * FROM $filesTable WHERE $columnLatitude IS NOT NULL AND $columnLongitude IS NOT NULL AND ($columnLatitude IS NOT 0 OR $columnLongitude IS NOT 0) AND @@ -1729,9 +1722,18 @@ class FilesDB { ($columnLocalID IS NOT NULL OR ($columnCollectionID IS NOT NULL AND $columnCollectionID IS NOT -1)) ORDER BY $columnCreationTime $order, $columnModificationTime $order - LIMIT $limit - ''', - [startTime, endTime], + '''; + + final args = [startTime, endTime]; + + if (limit != null) { + query += ' LIMIT ?'; + args.add(limit); + } + + final results = await db.getAll( + query, + args, ); final files = convertToFiles(results); final List filteredFiles = diff --git a/mobile/lib/ui/viewer/location/location_screen.dart b/mobile/lib/ui/viewer/location/location_screen.dart index 374d70cb9f..55975dd3fa 100644 --- a/mobile/lib/ui/viewer/location/location_screen.dart +++ b/mobile/lib/ui/viewer/location/location_screen.dart @@ -146,6 +146,8 @@ class _LocationGalleryWidgetState extends State { late final StreamSubscription _filesUpdateEvent; @override void initState() { + super.initState(); + final collectionsToHide = CollectionsService.instance.archivedOrHiddenCollectionIds(); fileLoadResult = FilesDB.instance @@ -179,8 +181,6 @@ class _LocationGalleryWidgetState extends State { }); galleryHeaderWidget = const GalleryHeaderWidget(); - - super.initState(); } @override From 16178b6f098f00914bf872cc90040fb98541b87f Mon Sep 17 00:00:00 2001 From: ashilkn Date: Fri, 17 May 2024 15:09:10 +0530 Subject: [PATCH 20/41] [mob][photos] Add missing paranthesis --- mobile/lib/db/files_db.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobile/lib/db/files_db.dart b/mobile/lib/db/files_db.dart index 16db97337e..444395f7de 100644 --- a/mobile/lib/db/files_db.dart +++ b/mobile/lib/db/files_db.dart @@ -1746,7 +1746,7 @@ class FilesDB { final results = await db.getAll( ''' SELECT DISTINCT $columnUploadedFileID FROM $filesTable - WHERE $columnOwnerID = ? AND $columnUploadedFileID IS NOT NULL AND + WHERE ($columnOwnerID = ? AND $columnUploadedFileID IS NOT NULL AND $columnUploadedFileID IS NOT -1) ''', [ownerID], From 48436694ebbfb087bb92bb021389c85e2da00ad9 Mon Sep 17 00:00:00 2001 From: ashilkn Date: Fri, 17 May 2024 16:28:13 +0530 Subject: [PATCH 21/41] [mob][photos] Fix incorrent sqlite operation --- mobile/lib/db/entities_db.dart | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mobile/lib/db/entities_db.dart b/mobile/lib/db/entities_db.dart index d3759389d6..228e5e2b01 100644 --- a/mobile/lib/db/entities_db.dart +++ b/mobile/lib/db/entities_db.dart @@ -28,6 +28,7 @@ extension EntitiesDB on FilesDB { ''' INSERT OR ${conflictAlgorithm.name.toUpperCase()} INTO entities (id, type, ownerID, data, updatedAt) + VALUES (?, ?, ?, ?, ?) ''', parameterSets, ); @@ -39,6 +40,7 @@ extension EntitiesDB on FilesDB { ''' INSERT OR ${conflictAlgorithm.name.toUpperCase()} INTO entities (id, type, ownerID, data, updatedAt) + VALUES (?, ?, ?, ?, ?) ''', parameterSets, ); From 18d68bbdf34051a77f03322cc6a257bad5baf5c9 Mon Sep 17 00:00:00 2001 From: ashilkn Date: Fri, 17 May 2024 16:34:04 +0530 Subject: [PATCH 22/41] Migrate to sqlite_async(13): Migrate db migration to use sqlite_async --- mobile/lib/db/files_db.dart | 42 +++++++++++++++++++------------------ 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/mobile/lib/db/files_db.dart b/mobile/lib/db/files_db.dart index 444395f7de..9ac322937c 100644 --- a/mobile/lib/db/files_db.dart +++ b/mobile/lib/db/files_db.dart @@ -17,7 +17,6 @@ import "package:photos/services/filter/db_filters.dart"; import 'package:photos/utils/file_uploader_util.dart'; import "package:photos/utils/primitive_wrapper.dart"; import 'package:sqflite/sqflite.dart'; -import 'package:sqflite_migration/sqflite_migration.dart'; import 'package:sqlite_async/sqlite_async.dart' as sqlite_async; class FilesDB { @@ -74,10 +73,8 @@ class FilesDB { // we need to write query based on that field static const columnMMdVisibility = 'mmd_visibility'; - static final initializationScript = [ - ...createTable(filesTable), - ]; static final migrationScripts = [ + ...createTable(filesTable), ...alterDeviceFolderToAllowNULL(), ...alterTimestampColumnTypes(), ...addIndices(), @@ -125,18 +122,12 @@ class FilesDB { columnAddedTime, ]; - final dbConfig = MigrationConfig( - initializationScript: initializationScript, - migrationScripts: migrationScripts, - ); - // make this a singleton class FilesDB._privateConstructor(); static final FilesDB instance = FilesDB._privateConstructor(); // only have a single app-wide reference to the database - static Future? _dbFuture; static Future? _sqliteAsyncDBFuture; Future get sqliteAsyncDB async { @@ -146,20 +137,31 @@ class FilesDB { } // this opens the database (and creates it if it doesn't exist) - Future _initDatabase() async { - final Directory documentsDirectory = - await getApplicationDocumentsDirectory(); - final String path = join(documentsDirectory.path, _databaseName); - _logger.info("DB path " + path); - return await openDatabaseWithMigration(path, dbConfig); - } - Future _initSqliteAsyncDatabase() async { final Directory documentsDirectory = await getApplicationDocumentsDirectory(); final String path = join(documentsDirectory.path, _databaseName); _logger.info("DB path " + path); - return sqlite_async.SqliteDatabase(path: path); + final migrations = getMigrations(); + final database = sqlite_async.SqliteDatabase(path: path); + await migrations.migrate(database); + return database; + } + + sqlite_async.SqliteMigrations getMigrations() { + final numberOfMigrationScripts = migrationScripts.length; + final migrations = sqlite_async.SqliteMigrations(); + for (int i = 0; i < numberOfMigrationScripts; i++) { + migrations.add( + sqlite_async.SqliteMigration( + i + 1, + (tx) async { + await tx.execute(migrationScripts[i]); + }, + ), + ); + } + return migrations; } // SQL code to create the database table @@ -443,7 +445,7 @@ class FilesDB { await getApplicationDocumentsDirectory(); final String path = join(documentsDirectory.path, _databaseName); File(path).deleteSync(recursive: true); - _dbFuture = null; + _sqliteAsyncDBFuture = null; } } From ab9cef689de29a6c2e9bda85acb12e428457f5b2 Mon Sep 17 00:00:00 2001 From: ashilkn Date: Fri, 17 May 2024 16:40:59 +0530 Subject: [PATCH 23/41] [mob][photos] Create ConflictAlgorithm enum and stop using it from sqflite --- mobile/lib/db/files_db.dart | 3 ++- mobile/lib/utils/sqlite_util.dart | 39 +++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) create mode 100644 mobile/lib/utils/sqlite_util.dart diff --git a/mobile/lib/db/files_db.dart b/mobile/lib/db/files_db.dart index 9ac322937c..fdd361a677 100644 --- a/mobile/lib/db/files_db.dart +++ b/mobile/lib/db/files_db.dart @@ -16,7 +16,8 @@ import "package:photos/models/metadata/common_keys.dart"; import "package:photos/services/filter/db_filters.dart"; import 'package:photos/utils/file_uploader_util.dart'; import "package:photos/utils/primitive_wrapper.dart"; -import 'package:sqflite/sqflite.dart'; +import "package:photos/utils/sqlite_util.dart"; +// import 'package:sqflite/sqflite.dart'; import 'package:sqlite_async/sqlite_async.dart' as sqlite_async; class FilesDB { diff --git a/mobile/lib/utils/sqlite_util.dart b/mobile/lib/utils/sqlite_util.dart new file mode 100644 index 0000000000..d236679bcc --- /dev/null +++ b/mobile/lib/utils/sqlite_util.dart @@ -0,0 +1,39 @@ +enum ConflictAlgorithm { + /// When a constraint violation occurs, an immediate ROLLBACK occurs, + /// thus ending the current transaction, and the command aborts with a + /// return code of SQLITE_CONSTRAINT. If no transaction is active + /// (other than the implied transaction that is created on every command) + /// then this algorithm works the same as ABORT. + rollback, + + /// When a constraint violation occurs,no ROLLBACK is executed + /// so changes from prior commands within the same transaction + /// are preserved. This is the default behavior. + abort, + + /// When a constraint violation occurs, the command aborts with a return + /// code SQLITE_CONSTRAINT. But any changes to the database that + /// the command made prior to encountering the constraint violation + /// are preserved and are not backed out. + fail, + + /// When a constraint violation occurs, the one row that contains + /// the constraint violation is not inserted or changed. + /// But the command continues executing normally. Other rows before and + /// after the row that contained the constraint violation continue to be + /// inserted or updated normally. No error is returned. + ignore, + + /// When a UNIQUE constraint violation occurs, the pre-existing rows that + /// are causing the constraint violation are removed prior to inserting + /// or updating the current row. Thus the insert or update always occurs. + /// The command continues executing normally. No error is returned. + /// If a NOT NULL constraint violation occurs, the NULL value is replaced + /// by the default value for that column. If the column has no default + /// value, then the ABORT algorithm is used. If a CHECK constraint + /// violation occurs then the IGNORE algorithm is used. When this conflict + /// resolution strategy deletes rows in order to satisfy a constraint, + /// it does not invoke delete triggers on those rows. + /// This behavior might change in a future release. + replace, +} From a7e0f3df7bc252b0f422021c410725c15c029429 Mon Sep 17 00:00:00 2001 From: ashilkn Date: Fri, 17 May 2024 17:05:58 +0530 Subject: [PATCH 24/41] [mob][photos] Remove sqflite import in filesDB --- mobile/lib/db/files_db.dart | 17 ++++++++--------- mobile/lib/services/local_sync_service.dart | 6 +++--- mobile/lib/utils/sqlite_util.dart | 2 +- 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/mobile/lib/db/files_db.dart b/mobile/lib/db/files_db.dart index fdd361a677..7f82759e52 100644 --- a/mobile/lib/db/files_db.dart +++ b/mobile/lib/db/files_db.dart @@ -452,7 +452,8 @@ class FilesDB { Future insertMultiple( List files, { - ConflictAlgorithm conflictAlgorithm = ConflictAlgorithm.replace, + SqliteAsyncConflictAlgorithm conflictAlgorithm = + SqliteAsyncConflictAlgorithm.replace, }) async { if (files.isEmpty) return; @@ -1282,13 +1283,11 @@ class FilesDB { Future collectionFileCount(int collectionID) async { final db = await instance.sqliteAsyncDB; - final count = Sqflite.firstIntValue( - await db.execute( - 'SELECT COUNT(*) FROM $filesTable where $columnCollectionID = ' - '$collectionID AND $columnUploadedFileID IS NOT -1', - ), + final row = await db.get( + 'SELECT COUNT(*) FROM $filesTable where $columnCollectionID = ' + '$collectionID AND $columnUploadedFileID IS NOT -1', ); - return count ?? 0; + return row['COUNT(*)'] as int; } Future archivedFilesCount( @@ -1884,7 +1883,7 @@ class FilesDB { Future _batchAndInsertFile( EnteFile file, - ConflictAlgorithm conflictAlgorithm, + SqliteAsyncConflictAlgorithm conflictAlgorithm, sqlite_async.SqliteDatabase db, List> parameterSets, PrimitiveWrapper batchCounter, { @@ -1905,7 +1904,7 @@ class FilesDB { } Future _insertBatch( - ConflictAlgorithm conflictAlgorithm, + SqliteAsyncConflictAlgorithm conflictAlgorithm, Iterable columnNames, sqlite_async.SqliteDatabase db, List> parameterSets, diff --git a/mobile/lib/services/local_sync_service.dart b/mobile/lib/services/local_sync_service.dart index 93b3c94373..1915ac30c2 100644 --- a/mobile/lib/services/local_sync_service.dart +++ b/mobile/lib/services/local_sync_service.dart @@ -21,8 +21,8 @@ import "package:photos/services/ignored_files_service.dart"; import 'package:photos/services/local/local_sync_util.dart'; import "package:photos/utils/debouncer.dart"; import "package:photos/utils/photo_manager_util.dart"; +import "package:photos/utils/sqlite_util.dart"; import 'package:shared_preferences/shared_preferences.dart'; -import 'package:sqflite/sqflite.dart'; import 'package:tuple/tuple.dart'; class LocalSyncService { @@ -184,7 +184,7 @@ class LocalSyncService { if (hasUnsyncedFiles) { await _db.insertMultiple( localDiffResult.uniqueLocalFiles!, - conflictAlgorithm: ConflictAlgorithm.ignore, + conflictAlgorithm: SqliteAsyncConflictAlgorithm.ignore, ); _logger.info( "Inserted ${localDiffResult.uniqueLocalFiles?.length} " @@ -321,7 +321,7 @@ class LocalSyncService { files.removeWhere((file) => existingLocalDs.contains(file.localID)); await _db.insertMultiple( files, - conflictAlgorithm: ConflictAlgorithm.ignore, + conflictAlgorithm: SqliteAsyncConflictAlgorithm.ignore, ); _logger.info('Inserted ${files.length} files'); Bus.instance.fire( diff --git a/mobile/lib/utils/sqlite_util.dart b/mobile/lib/utils/sqlite_util.dart index d236679bcc..b83bd58e15 100644 --- a/mobile/lib/utils/sqlite_util.dart +++ b/mobile/lib/utils/sqlite_util.dart @@ -1,4 +1,4 @@ -enum ConflictAlgorithm { +enum SqliteAsyncConflictAlgorithm { /// When a constraint violation occurs, an immediate ROLLBACK occurs, /// thus ending the current transaction, and the command aborts with a /// return code of SQLITE_CONSTRAINT. If no transaction is active From 49e64b3d4ce3a7c4523e6562f24137c04602c4d5 Mon Sep 17 00:00:00 2001 From: ashilkn Date: Tue, 21 May 2024 16:12:44 +0530 Subject: [PATCH 25/41] [mob][photos] Fix issue with EnteFile not having location data --- mobile/lib/db/files_db.dart | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mobile/lib/db/files_db.dart b/mobile/lib/db/files_db.dart index 7f82759e52..5bad957583 100644 --- a/mobile/lib/db/files_db.dart +++ b/mobile/lib/db/files_db.dart @@ -1824,8 +1824,8 @@ class FilesDB { }) { final values = []; - double? latitude; - double? longitude; + double? latitude = file.location?.latitude; + double? longitude = file.location?.longitude; int? creationTime = file.creationTime; if (file.pubMagicMetadata != null) { From b2a359ca598cc3b0972dcf15dba5106eee8c5113 Mon Sep 17 00:00:00 2001 From: ashilkn Date: Tue, 21 May 2024 16:53:49 +0530 Subject: [PATCH 26/41] [mob][photos] Migrate to sqlite_async(13): Use getAll() instead of execute() for SELECT commands --- mobile/lib/db/files_db.dart | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/mobile/lib/db/files_db.dart b/mobile/lib/db/files_db.dart index 323c1aed22..f9435b8329 100644 --- a/mobile/lib/db/files_db.dart +++ b/mobile/lib/db/files_db.dart @@ -1120,7 +1120,7 @@ class FilesDB { return {}; } final inParam = hashes.map((e) => "'$e'").join(','); - final rows = await db.execute(''' + final rows = await db.getAll(''' SELECT * FROM $filesTable WHERE $columnHash IN ($inParam) AND $columnOwnerID = $userID; '''); final matchedFiles = convertToFiles(rows); @@ -1319,7 +1319,7 @@ class FilesDB { Set hiddenCollections, ) async { final db = await instance.sqliteAsyncDB; - final count = await db.execute( + final count = await db.getAll( 'SELECT COUNT(distinct($columnUploadedFileID)) as COUNT FROM $filesTable where ' '$columnMMdVisibility' ' = $visibility AND $columnOwnerID = $ownerID AND $columnCollectionID NOT IN (${hiddenCollections.join(', ')})', @@ -1373,7 +1373,7 @@ class FilesDB { } inParam = inParam.substring(0, inParam.length - 1); final db = await instance.sqliteAsyncDB; - final rows = await db.execute( + final rows = await db.getAll( ''' SELECT $columnLocalID FROM $filesTable @@ -1393,7 +1393,7 @@ class FilesDB { Future> getCollectionIDToMaxCreationTime() async { final enteWatch = EnteWatch("getCollectionIDToMaxCreationTime")..start(); final db = await instance.sqliteAsyncDB; - final rows = await db.execute( + final rows = await db.getAll( ''' SELECT $columnCollectionID, MAX($columnCreationTime) AS max_creation_time FROM $filesTable @@ -1730,7 +1730,7 @@ class FilesDB { Future> fetchFilesCountbyType(int userID) async { final db = await instance.sqliteAsyncDB; - final result = await db.execute( + final result = await db.getAll( ''' SELECT $columnFileType, COUNT(DISTINCT $columnUploadedFileID) FROM $filesTable WHERE $columnUploadedFileID != -1 AND From 159fdf83ad34cea421eb8882869eaa1919bb086d Mon Sep 17 00:00:00 2001 From: ashilkn Date: Tue, 21 May 2024 16:54:09 +0530 Subject: [PATCH 27/41] [mob][photos] Migrate to sqlite_async(14) --- mobile/lib/db/entities_db.dart | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/mobile/lib/db/entities_db.dart b/mobile/lib/db/entities_db.dart index 004fdf0aee..3cd9d47639 100644 --- a/mobile/lib/db/entities_db.dart +++ b/mobile/lib/db/entities_db.dart @@ -89,11 +89,10 @@ extension EntitiesDB on FilesDB { } Future getEntity(EntityType type, String id) async { - final db = await database; - final List> maps = await db.query( - "entities", - where: "type = ? AND id = ?", - whereArgs: [type.typeToString(), id], + final db = await sqliteAsyncDB; + final List> maps = await db.getAll( + 'SELECT * FROM entities WHERE type = ? AND id = ?', + [type.name, id], ); if (maps.isEmpty) { return null; From 5a017616f5922ae5d49d2527eedcf802e09304ca Mon Sep 17 00:00:00 2001 From: ashilkn Date: Tue, 21 May 2024 17:10:42 +0530 Subject: [PATCH 28/41] [mob][photos] Fix sqlite command syntax errors --- mobile/lib/db/files_db.dart | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mobile/lib/db/files_db.dart b/mobile/lib/db/files_db.dart index f9435b8329..e9fc70d458 100644 --- a/mobile/lib/db/files_db.dart +++ b/mobile/lib/db/files_db.dart @@ -1138,7 +1138,7 @@ class FilesDB { } final db = await instance.sqliteAsyncDB; final rows = await db.getAll( - 'SELECT * FROM $filesTable WHERE $columnUploadedFileID != NULL OR ' + 'SELECT * FROM $filesTable WHERE ($columnUploadedFileID != NULL OR ' '$columnUploadedFileID != -1) AND $columnOwnerID = ? AND ' '$columnFileType = ? AND $columnHash IN ($inParam)', [ @@ -1734,7 +1734,7 @@ class FilesDB { ''' SELECT $columnFileType, COUNT(DISTINCT $columnUploadedFileID) FROM $filesTable WHERE $columnUploadedFileID != -1 AND - $columnOwnerID == $userID GROUP BY $columnFileType + $columnOwnerID IS $userID GROUP BY $columnFileType ''', ); From e3ea22f479b2db6433117cdd9491dcd2026525dc Mon Sep 17 00:00:00 2001 From: ashilkn Date: Tue, 21 May 2024 17:44:38 +0530 Subject: [PATCH 29/41] [mob][photos] add comment --- mobile/lib/db/files_db.dart | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mobile/lib/db/files_db.dart b/mobile/lib/db/files_db.dart index e9fc70d458..03f5d832b8 100644 --- a/mobile/lib/db/files_db.dart +++ b/mobile/lib/db/files_db.dart @@ -74,6 +74,9 @@ class FilesDB { // we need to write query based on that field static const columnMMdVisibility = 'mmd_visibility'; +//If adding or removing a new column, make sure to update the `_columnNames` list +//and update `_generateColumnsAndPlaceholdersForInsert` and +//`_generateUpdateAssignmentsWithPlaceholders` static final migrationScripts = [ ...createTable(filesTable), ...alterDeviceFolderToAllowNULL(), From eaca151a9fc503ff48c9a9c7f73b1de73eeb3954 Mon Sep 17 00:00:00 2001 From: ashilkn Date: Tue, 21 May 2024 18:34:11 +0530 Subject: [PATCH 30/41] [mob][photos] Minor change --- mobile/lib/db/files_db.dart | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/mobile/lib/db/files_db.dart b/mobile/lib/db/files_db.dart index 03f5d832b8..0de84df68c 100644 --- a/mobile/lib/db/files_db.dart +++ b/mobile/lib/db/files_db.dart @@ -17,8 +17,7 @@ import "package:photos/services/filter/db_filters.dart"; import 'package:photos/utils/file_uploader_util.dart'; import "package:photos/utils/primitive_wrapper.dart"; import "package:photos/utils/sqlite_util.dart"; -// import 'package:sqflite/sqflite.dart'; -import 'package:sqlite_async/sqlite_async.dart' as sqlite_async; +import 'package:sqlite_async/sqlite_async.dart'; class FilesDB { /* @@ -132,32 +131,32 @@ class FilesDB { static final FilesDB instance = FilesDB._privateConstructor(); // only have a single app-wide reference to the database - static Future? _sqliteAsyncDBFuture; + static Future? _sqliteAsyncDBFuture; - Future get sqliteAsyncDB async { + Future get sqliteAsyncDB async { // lazily instantiate the db the first time it is accessed _sqliteAsyncDBFuture ??= _initSqliteAsyncDatabase(); return _sqliteAsyncDBFuture!; } // this opens the database (and creates it if it doesn't exist) - Future _initSqliteAsyncDatabase() async { + Future _initSqliteAsyncDatabase() async { final Directory documentsDirectory = await getApplicationDocumentsDirectory(); final String path = join(documentsDirectory.path, _databaseName); _logger.info("DB path " + path); final migrations = getMigrations(); - final database = sqlite_async.SqliteDatabase(path: path); + final database = SqliteDatabase(path: path); await migrations.migrate(database); return database; } - sqlite_async.SqliteMigrations getMigrations() { + SqliteMigrations getMigrations() { final numberOfMigrationScripts = migrationScripts.length; - final migrations = sqlite_async.SqliteMigrations(); + final migrations = SqliteMigrations(); for (int i = 0; i < numberOfMigrationScripts; i++) { migrations.add( - sqlite_async.SqliteMigration( + SqliteMigration( i + 1, (tx) async { await tx.execute(migrationScripts[i]); @@ -1927,7 +1926,7 @@ class FilesDB { Future _batchAndInsertFile( EnteFile file, SqliteAsyncConflictAlgorithm conflictAlgorithm, - sqlite_async.SqliteDatabase db, + SqliteDatabase db, List> parameterSets, PrimitiveWrapper batchCounter, { required bool isGenIdNull, @@ -1949,7 +1948,7 @@ class FilesDB { Future _insertBatch( SqliteAsyncConflictAlgorithm conflictAlgorithm, Iterable columnNames, - sqlite_async.SqliteDatabase db, + SqliteDatabase db, List> parameterSets, ) async { final valuesPlaceholders = List.filled(columnNames.length, "?").join(","); From 4fb9e75394a0fdddee2e9b7772c9dc9461900353 Mon Sep 17 00:00:00 2001 From: ashilkn Date: Tue, 21 May 2024 18:36:01 +0530 Subject: [PATCH 31/41] [mob][photos] Bump up version to 0.8.99 --- mobile/pubspec.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobile/pubspec.yaml b/mobile/pubspec.yaml index b9d5345c39..31c605ab7f 100644 --- a/mobile/pubspec.yaml +++ b/mobile/pubspec.yaml @@ -12,7 +12,7 @@ description: ente photos application # Read more about iOS versioning at # https://developer.apple.com/library/archive/documentation/General/Reference/InfoPlistKeyReference/Articles/CoreFoundationKeys.html -version: 0.8.98+618 +version: 0.8.99+619 publish_to: none environment: From f513473362568278e8c2c5ba5c78245af4d8b127 Mon Sep 17 00:00:00 2001 From: ashilkn Date: Wed, 22 May 2024 13:44:19 +0530 Subject: [PATCH 32/41] [mob][photos] Check db version when sqflite was used and run only migrations that are necessary using sqlite_async Tested adding a new migration and it works. Tested two cases (a)Fresh install (b)Opening app with new migration added and the last db migration was done when sqflite was used --- mobile/lib/db/files_db.dart | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/mobile/lib/db/files_db.dart b/mobile/lib/db/files_db.dart index 0de84df68c..fd258b3690 100644 --- a/mobile/lib/db/files_db.dart +++ b/mobile/lib/db/files_db.dart @@ -145,16 +145,23 @@ class FilesDB { await getApplicationDocumentsDirectory(); final String path = join(documentsDirectory.path, _databaseName); _logger.info("DB path " + path); - final migrations = getMigrations(); final database = SqliteDatabase(path: path); + final versionRow = await database.execute('PRAGMA user_version'); + + //db version used to be stored in `user_version` when using sqflite. + //sqlite_async doesn't use `user_version` to store db version. + // `oldVersionNumber` = 0 for fresh install + final oldVersionNumber = versionRow[0]['user_version'] as int; + final migrations = getMigrations(oldVersionNumber); + await migrations.migrate(database); return database; } - SqliteMigrations getMigrations() { + SqliteMigrations getMigrations(int oldSqfliteDBVersion) { final numberOfMigrationScripts = migrationScripts.length; final migrations = SqliteMigrations(); - for (int i = 0; i < numberOfMigrationScripts; i++) { + for (int i = oldSqfliteDBVersion; i < numberOfMigrationScripts; i++) { migrations.add( SqliteMigration( i + 1, From cb9ac0d939214e79231c7489e2ff9f926479c5e0 Mon Sep 17 00:00:00 2001 From: ashilkn Date: Wed, 22 May 2024 14:21:31 +0530 Subject: [PATCH 33/41] [mob][photos] bump up version to 0.8.100 --- mobile/pubspec.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobile/pubspec.yaml b/mobile/pubspec.yaml index 31c605ab7f..7b429b4578 100644 --- a/mobile/pubspec.yaml +++ b/mobile/pubspec.yaml @@ -12,7 +12,7 @@ description: ente photos application # Read more about iOS versioning at # https://developer.apple.com/library/archive/documentation/General/Reference/InfoPlistKeyReference/Articles/CoreFoundationKeys.html -version: 0.8.99+619 +version: 0.8.100+620 publish_to: none environment: From 22fc67c8c3872adc4d26c623a22fe6e7400a8c1e Mon Sep 17 00:00:00 2001 From: ashilkn Date: Wed, 22 May 2024 16:17:05 +0530 Subject: [PATCH 34/41] [mob][photos] Remove unnecessary parameters --- mobile/lib/db/files_db.dart | 2 -- 1 file changed, 2 deletions(-) diff --git a/mobile/lib/db/files_db.dart b/mobile/lib/db/files_db.dart index fd258b3690..d690e945e7 100644 --- a/mobile/lib/db/files_db.dart +++ b/mobile/lib/db/files_db.dart @@ -1058,10 +1058,8 @@ class FilesDB { title, location?.latitude, location?.longitude, - localID, creationTime, modificationTime, - null, getInt(fileType), localID, ownerID, From 7aa26a950d35c0f4c64eccfe4c9c273e0dc79d90 Mon Sep 17 00:00:00 2001 From: ashilkn Date: Wed, 22 May 2024 20:44:10 +0530 Subject: [PATCH 35/41] [mob][photos] Bump up to version 0.8.103 --- mobile/pubspec.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobile/pubspec.yaml b/mobile/pubspec.yaml index 2e80192929..b1d4fff03f 100644 --- a/mobile/pubspec.yaml +++ b/mobile/pubspec.yaml @@ -12,7 +12,7 @@ description: ente photos application # Read more about iOS versioning at # https://developer.apple.com/library/archive/documentation/General/Reference/InfoPlistKeyReference/Articles/CoreFoundationKeys.html -version: 0.8.102+626 +version: 0.8.103+627 publish_to: none environment: From 637adb46170bbb9cca2a5c7f9ab3e144ef1f171a Mon Sep 17 00:00:00 2001 From: ashilkn Date: Fri, 24 May 2024 14:21:02 +0530 Subject: [PATCH 36/41] [mob][photos] Simplify how FilesDB migrates --- mobile/lib/db/files_db.dart | 44 +++++++++++++++++++------------------ 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/mobile/lib/db/files_db.dart b/mobile/lib/db/files_db.dart index d690e945e7..a0b400cbc1 100644 --- a/mobile/lib/db/files_db.dart +++ b/mobile/lib/db/files_db.dart @@ -146,32 +146,34 @@ class FilesDB { final String path = join(documentsDirectory.path, _databaseName); _logger.info("DB path " + path); final database = SqliteDatabase(path: path); - final versionRow = await database.execute('PRAGMA user_version'); + await _migrate(database); - //db version used to be stored in `user_version` when using sqflite. - //sqlite_async doesn't use `user_version` to store db version. - // `oldVersionNumber` = 0 for fresh install - final oldVersionNumber = versionRow[0]['user_version'] as int; - final migrations = getMigrations(oldVersionNumber); - - await migrations.migrate(database); return database; } - SqliteMigrations getMigrations(int oldSqfliteDBVersion) { - final numberOfMigrationScripts = migrationScripts.length; - final migrations = SqliteMigrations(); - for (int i = oldSqfliteDBVersion; i < numberOfMigrationScripts; i++) { - migrations.add( - SqliteMigration( - i + 1, - (tx) async { - await tx.execute(migrationScripts[i]); - }, - ), - ); + Future _migrate( + SqliteDatabase database, + ) async { + final result = await database.execute('PRAGMA user_version'); + final currentVersion = result[0]['user_version'] as int; + final toVersion = migrationScripts.length; + + _logger.info("currentVersion: $currentVersion"); + _logger.info("toVersion: $toVersion"); + + if (currentVersion < toVersion) { + _logger.info("Migrating database from $currentVersion to $toVersion"); + await database.writeTransaction((tx) async { + for (int i = currentVersion + 1; i <= toVersion; i++) { + await tx.execute(migrationScripts[i - 1]); + } + await tx.execute('PRAGMA user_version = $toVersion'); + }); + } else if (currentVersion > toVersion) { + throw AssertionError("currentVersion cannot be greater than toVersion"); + } else { + _logger.info("Database is already at version $toVersion"); } - return migrations; } // SQL code to create the database table From 500d7da30671e6df35cb8b8d661a1ef02452beee Mon Sep 17 00:00:00 2001 From: ashilkn Date: Fri, 24 May 2024 14:39:16 +0530 Subject: [PATCH 37/41] [mob][photos] Remove log lines used for testing --- mobile/lib/db/files_db.dart | 5 ----- 1 file changed, 5 deletions(-) diff --git a/mobile/lib/db/files_db.dart b/mobile/lib/db/files_db.dart index a0b400cbc1..b7c67ff36c 100644 --- a/mobile/lib/db/files_db.dart +++ b/mobile/lib/db/files_db.dart @@ -158,9 +158,6 @@ class FilesDB { final currentVersion = result[0]['user_version'] as int; final toVersion = migrationScripts.length; - _logger.info("currentVersion: $currentVersion"); - _logger.info("toVersion: $toVersion"); - if (currentVersion < toVersion) { _logger.info("Migrating database from $currentVersion to $toVersion"); await database.writeTransaction((tx) async { @@ -171,8 +168,6 @@ class FilesDB { }); } else if (currentVersion > toVersion) { throw AssertionError("currentVersion cannot be greater than toVersion"); - } else { - _logger.info("Database is already at version $toVersion"); } } From a79d11c2636b2cbe09d166e3b80491b962695f61 Mon Sep 17 00:00:00 2001 From: ashilkn Date: Fri, 24 May 2024 14:43:39 +0530 Subject: [PATCH 38/41] [mob][photos] Add more info in error message --- mobile/lib/db/files_db.dart | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mobile/lib/db/files_db.dart b/mobile/lib/db/files_db.dart index b7c67ff36c..c6ba617e0e 100644 --- a/mobile/lib/db/files_db.dart +++ b/mobile/lib/db/files_db.dart @@ -167,7 +167,9 @@ class FilesDB { await tx.execute('PRAGMA user_version = $toVersion'); }); } else if (currentVersion > toVersion) { - throw AssertionError("currentVersion cannot be greater than toVersion"); + throw AssertionError( + "currentVersion($currentVersion) cannot be greater than toVersion($toVersion)", + ); } } From 022448155dc01478527145d8a4323ad9d24178be Mon Sep 17 00:00:00 2001 From: ashilkn Date: Fri, 24 May 2024 15:48:39 +0530 Subject: [PATCH 39/41] [mob][photos] Bump up version to v0.8.111 --- mobile/pubspec.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobile/pubspec.yaml b/mobile/pubspec.yaml index 1417d17f3e..c9df14e163 100644 --- a/mobile/pubspec.yaml +++ b/mobile/pubspec.yaml @@ -12,7 +12,7 @@ description: ente photos application # Read more about iOS versioning at # https://developer.apple.com/library/archive/documentation/General/Reference/InfoPlistKeyReference/Articles/CoreFoundationKeys.html -version: 0.8.110+634 +version: 0.8.111+635 publish_to: none environment: From 90e467c7c02fd5d9ecd2121f3494752d01915c76 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Mon, 27 May 2024 18:31:17 +0530 Subject: [PATCH 40/41] [mob][photos] Fetch remote feedback before clustering --- .../services/machine_learning/face_ml/face_ml_service.dart | 3 +++ .../machine_learning/face_ml/person/person_service.dart | 4 ++-- mobile/lib/ui/settings/debug/face_debug_section_widget.dart | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart index 9f153ffa89..222fd50b8f 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart @@ -580,6 +580,9 @@ class FaceMlService { _isIndexingOrClusteringRunning = true; final clusterAllImagesTime = DateTime.now(); + _logger.info('Pulling remote feedback before actually clustering'); + await PersonService.instance.fetchRemoteClusterFeedback(); + try { // Get a sense of the total number of faces in the database final int totalFaces = await FaceMLDataDB.instance diff --git a/mobile/lib/services/machine_learning/face_ml/person/person_service.dart b/mobile/lib/services/machine_learning/face_ml/person/person_service.dart index 7517d057d5..682deaff0c 100644 --- a/mobile/lib/services/machine_learning/face_ml/person/person_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/person/person_service.dart @@ -73,7 +73,7 @@ class PersonService { Future reconcileClusters() async { final EnteWatch? w = kDebugMode ? EnteWatch("reconcileClusters") : null; w?.start(); - await storeRemoteFeedback(); + await fetchRemoteClusterFeedback(); w?.log("Stored remote feedback"); final dbPersonClusterInfo = await faceMLDataDB.getPersonToClusterIdToFaceIds(); @@ -225,7 +225,7 @@ class PersonService { Bus.instance.fire(PeopleChangedEvent()); } - Future storeRemoteFeedback() async { + Future fetchRemoteClusterFeedback() async { await entityService.syncEntities(); final entities = await entityService.getEntities(EntityType.person); entities.sort((a, b) => a.updatedAt.compareTo(b.updatedAt)); diff --git a/mobile/lib/ui/settings/debug/face_debug_section_widget.dart b/mobile/lib/ui/settings/debug/face_debug_section_widget.dart index 376793769f..844f71c01b 100644 --- a/mobile/lib/ui/settings/debug/face_debug_section_widget.dart +++ b/mobile/lib/ui/settings/debug/face_debug_section_widget.dart @@ -193,7 +193,7 @@ class _FaceDebugSectionWidgetState extends State { trailingIconIsMuted: true, onTap: () async { try { - await PersonService.instance.storeRemoteFeedback(); + await PersonService.instance.fetchRemoteClusterFeedback(); FaceMlService.instance.debugIndexingDisabled = false; await FaceMlService.instance .clusterAllImages(clusterInBuckets: true); From 37d3776e285e21378712fe59bb11f078220eef84 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Mon, 27 May 2024 18:34:26 +0530 Subject: [PATCH 41/41] [mob][photos] Bump --- mobile/pubspec.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobile/pubspec.yaml b/mobile/pubspec.yaml index d3f49380f9..1311b4e793 100644 --- a/mobile/pubspec.yaml +++ b/mobile/pubspec.yaml @@ -12,7 +12,7 @@ description: ente photos application # Read more about iOS versioning at # https://developer.apple.com/library/archive/documentation/General/Reference/InfoPlistKeyReference/Articles/CoreFoundationKeys.html -version: 0.8.112+636 +version: 0.8.113+637 publish_to: none environment: