From 17670d5538af923e1f0381ed63728588ef73acbd Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Tue, 1 Jul 2025 15:25:03 +0530 Subject: [PATCH] Use asset queue db for queueing pending upload --- mobile/lib/db/files_db.dart | 84 ----------- .../collection/collection_file_actions.dart | 132 ++++-------------- 2 files changed, 30 insertions(+), 186 deletions(-) diff --git a/mobile/lib/db/files_db.dart b/mobile/lib/db/files_db.dart index 58b5e9b97b..c0604cb924 100644 --- a/mobile/lib/db/files_db.dart +++ b/mobile/lib/db/files_db.dart @@ -422,74 +422,6 @@ class FilesDB with SqlDbBase { await db.execute('DELETE FROM entities'); } - Future insertMultiple( - List files, { - SqliteAsyncConflictAlgorithm conflictAlgorithm = - SqliteAsyncConflictAlgorithm.replace, - }) async { - if (files.isEmpty) return; - - final startTime = DateTime.now(); - final db = await sqliteAsyncDB; - - final withIdParams = >[]; - const withIdColumnNames = _columnNames; - final withoutIdParams = >[]; - final withoutIdColumns = - _columnNames.where((column) => column != columnGeneratedID).toList(); - - // Sort files into appropriate parameter sets - for (final file in files) { - if (file.generatedID == null) { - withoutIdParams.add(_getParameterSetForFile(file)); - - if (withoutIdParams.length == 400) { - await _insertBatch( - conflictAlgorithm, - withoutIdColumns, - db, - withoutIdParams, - ); - withoutIdParams.clear(); - } - } else { - withIdParams.add(_getParameterSetForFile(file)); - if (withIdParams.length == 400) { - await _insertBatch( - conflictAlgorithm, - withIdColumnNames, - db, - withIdParams, - ); - withIdParams.clear(); - } - } - } - - // Insert any remaining files - if (withIdParams.isNotEmpty) { - await _insertBatch( - conflictAlgorithm, - withIdColumnNames, - db, - withIdParams, - ); - } - - if (withoutIdParams.isNotEmpty) { - await _insertBatch( - conflictAlgorithm, - withoutIdColumns, - db, - withoutIdParams, - ); - } - final duration = DateTime.now().difference(startTime); - _logger.info( - "Batch insert of ${files.length} took ${duration.inMilliseconds}ms.", - ); - } - Future insert(EnteFile file) async { _logger.info("Inserting $file"); final db = await instance.sqliteAsyncDB; @@ -1165,22 +1097,6 @@ class FilesDB with SqlDbBase { return values; } - Future _insertBatch( - SqliteAsyncConflictAlgorithm conflictAlgorithm, - Iterable columnNames, - SqliteDatabase db, - List> parameterSets, - ) async { - final valuesPlaceholders = List.filled(columnNames.length, "?").join(","); - final columnNamesJoined = columnNames.join(","); - await db.executeBatch( - ''' - INSERT OR ${conflictAlgorithm.name.toUpperCase()} INTO $filesTable($columnNamesJoined) VALUES($valuesPlaceholders) - ''', - parameterSets, - ); - } - EnteFile _getFileFromRow(Map row) { final file = EnteFile(); file.generatedID = row[columnGeneratedID]; diff --git a/mobile/lib/ui/actions/collection/collection_file_actions.dart b/mobile/lib/ui/actions/collection/collection_file_actions.dart index 799171df00..d8854e2cf4 100644 --- a/mobile/lib/ui/actions/collection/collection_file_actions.dart +++ b/mobile/lib/ui/actions/collection/collection_file_actions.dart @@ -6,6 +6,7 @@ import "package:photos/core/configuration.dart"; import "package:photos/core/event_bus.dart"; import "package:photos/db/files_db.dart"; import "package:photos/db/local/table/shared_assets.dart"; +import "package:photos/db/local/table/upload_queue_table.dart"; import "package:photos/events/collection_updated_event.dart"; import "package:photos/generated/l10n.dart"; import 'package:photos/models/collection/collection.dart'; @@ -103,67 +104,32 @@ extension CollectionFileActions on CollectionActions { final int currentUserID = Configuration.instance.getUserID()!; for (final collection in collections) { try { - final List files = []; + final List uploadedFiles = []; + final Set pendingUploads = {}; final List filesPendingUpload = []; for (final file in selectedFiles!) { - EnteFile? currentFile; - if (file.uploadedFileID != null) { - currentFile = file.copyWith(); - } else if (file.generatedID != null) { - // when file is not uploaded, refresh the state from the db to - // ensure we have latest upload status for given file before - // queueing it up as pending upload - currentFile = await (FilesDB.instance.getFile(file.generatedID!)); - } else if (file.generatedID == null) { - logger.severe("generated id should not be null"); - } - if (currentFile == null) { - logger.severe("Failed to find fileBy genID"); - continue; - } - - if (currentFile.uploadedFileID == null) { - currentFile.collectionID = collection.id; - filesPendingUpload.add(currentFile); + if (file.rAsset != null) { + uploadedFiles.add(file.copyWith()); } else { - files.add(currentFile); + pendingUploads.add(file.lAsset!.id); + filesPendingUpload.add(file.copyWith()); } } - if (filesPendingUpload.isNotEmpty) { - // Newly created collection might not be cached - final Collection? c = - CollectionsService.instance.getCollectionByID(collection.id); - if (c != null && c.owner.id != currentUserID) { - final Collection uncat = - await CollectionsService.instance.getUncategorizedCollection(); - for (EnteFile unuploadedFile in filesPendingUpload) { - final uploadedFile = await FileUploader.instance.forceUpload( - unuploadedFile, - uncat.id, - ); - files.add(uploadedFile); - } - } else { - for (final file in filesPendingUpload) { - file.collectionID = collection.id; - } - // filesPendingUpload might be getting ignored during auto-upload - // because the user deleted these files from ente in the past. - await IgnoredFilesService.instance - .removeIgnoredMappings(filesPendingUpload); - await FilesDB.instance.insertMultiple(filesPendingUpload); - Bus.instance.fire( - CollectionUpdatedEvent( - collection.id, - filesPendingUpload, - "pendingFilesAdd", - ), - ); - } + if (pendingUploads.isNotEmpty) { + await IgnoredFilesService.instance + .removeIgnoredMappings(filesPendingUpload); + await localDB.insertOrUpdateQueue( pendingUploads, collection.id, currentUserID, manual: true) + Bus.instance.fire( + CollectionUpdatedEvent( + collection.id, + filesPendingUpload, + "queuedForUpload", + ), + ); } - if (files.isNotEmpty) { + if (uploadedFiles.isNotEmpty) { await CollectionsService.instance - .addOrCopyToCollection(collection.id, files); + .addOrCopyToCollection(collection.id, uploadedFiles); } } catch (e, s) { logger.severe("Failed to add to album", e, s); @@ -192,7 +158,7 @@ extension CollectionFileActions on CollectionActions { List? sharedFiles, List? picketAssets, }) async { - ProgressDialog? dialog = showProgressDialog + final ProgressDialog? dialog = showProgressDialog ? createProgressDialog( context, S.of(context).uploadingFilesToAlbum, @@ -220,69 +186,31 @@ extension CollectionFileActions on CollectionActions { ); } else { for (final file in selectedFiles!) { - EnteFile? currentFile; - if (file.uploadedFileID != null) { - currentFile = file.copyWith(); - } else if (file.generatedID != null) { - // when file is not uploaded, refresh the state from the db to - // ensure we have latest upload status for given file before - // queueing it up as pending upload - currentFile = await (FilesDB.instance.getFile(file.generatedID!)); - } else if (file.generatedID == null) { - logger.severe("generated id should not be null"); - } - if (currentFile == null) { - logger.severe("Failed to find fileBy genID"); - continue; - } - if (currentFile.uploadedFileID == null) { - currentFile.collectionID = collectionID; - filesPendingUpload.add(currentFile); + if (file.rAsset != null) { + uploadedFiles.add(file.copyWith()); + } else if (file.lAsset != null) { + filesPendingUpload.add(file.copyWith()); } else { - uploadedFiles.add(currentFile); + throw Exception("File does not have rAsset or lAsset: $file"); } } } if (filesPendingUpload.isNotEmpty) { - // Newly created collection might not be cached - final Collection? c = - CollectionsService.instance.getCollectionByID(collectionID); - if (c != null && c.owner.id != currentUserID) { - if (!showProgressDialog) { - dialog = createProgressDialog( - context, - S.of(context).uploadingFilesToAlbum, - isDismissible: true, - ); - await dialog.show(); - } - final Collection uncat = - await CollectionsService.instance.getUncategorizedCollection(); - for (EnteFile unuploadedFile in filesPendingUpload) { - final uploadedFile = await FileUploader.instance.forceUpload( - unuploadedFile, - uncat.id, - ); - uploadedFiles.add(uploadedFile); - } - } else { + final Set pendingUploadAssetIDs = {}; for (final file in filesPendingUpload) { - file.collectionID = collectionID; + pendingUploadAssetIDs.add(file.lAsset!.id); } - // filesPendingUpload might be getting ignored during auto-upload - // because the user deleted these files from ente in the past. await IgnoredFilesService.instance .removeIgnoredMappings(filesPendingUpload); - await FilesDB.instance.insertMultiple(filesPendingUpload); + await localDB.insertOrUpdateQueue(pendingUploadAssetIDs, collectionID, currentUserID, manual: true); Bus.instance.fire( CollectionUpdatedEvent( collectionID, filesPendingUpload, - "pendingFilesAdd", + "queuedForUpload", ), ); } - } if (uploadedFiles.isNotEmpty) { await CollectionsService.instance .addOrCopyToCollection(collectionID, uploadedFiles);