TerribleDev/ente
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Use asset queue db for queueing pending upload

Browse Source
This commit is contained in:
Neeraj Gupta
2025-07-01 15:25:03 +05:30
parent 8cfd80663e
commit 17670d5538
2 changed files with 30 additions and 186 deletions
Download Patch File Download Diff File Expand all files Collapse all files

84
mobile/lib/db/files_db.dart
View File

@@ -422,74 +422,6 @@ class FilesDB with SqlDbBase {
await db.execute('DELETE FROM entities');
}
Future<void> insertMultiple(
List<EnteFile> files, {
SqliteAsyncConflictAlgorithm conflictAlgorithm =
SqliteAsyncConflictAlgorithm.replace,
}) async {
if (files.isEmpty) return;
final startTime = DateTime.now();
final db = await sqliteAsyncDB;
final withIdParams = <List<Object?>>[];
const withIdColumnNames = _columnNames;
final withoutIdParams = <List<Object?>>[];
final withoutIdColumns =
_columnNames.where((column) => column != columnGeneratedID).toList();
// Sort files into appropriate parameter sets
for (final file in files) {
if (file.generatedID == null) {
withoutIdParams.add(_getParameterSetForFile(file));
if (withoutIdParams.length == 400) {
await _insertBatch(
conflictAlgorithm,
withoutIdColumns,
db,
withoutIdParams,
);
withoutIdParams.clear();
}
} else {
withIdParams.add(_getParameterSetForFile(file));
if (withIdParams.length == 400) {
await _insertBatch(
conflictAlgorithm,
withIdColumnNames,
db,
withIdParams,
);
withIdParams.clear();
}
}
}
// Insert any remaining files
if (withIdParams.isNotEmpty) {
await _insertBatch(
conflictAlgorithm,
withIdColumnNames,
db,
withIdParams,
);
}
if (withoutIdParams.isNotEmpty) {
await _insertBatch(
conflictAlgorithm,
withoutIdColumns,
db,
withoutIdParams,
);
}
final duration = DateTime.now().difference(startTime);
_logger.info(
"Batch insert of ${files.length} took ${duration.inMilliseconds}ms.",
);
}
Future<void> insert(EnteFile file) async {
_logger.info("Inserting $file");
final db = await instance.sqliteAsyncDB;
@@ -1165,22 +1097,6 @@ class FilesDB with SqlDbBase {
return values;
}
Future<void> _insertBatch(
SqliteAsyncConflictAlgorithm conflictAlgorithm,
Iterable<String> columnNames,
SqliteDatabase db,
List<List<Object?>> parameterSets,
) async {
final valuesPlaceholders = List.filled(columnNames.length, "?").join(",");
final columnNamesJoined = columnNames.join(",");
await db.executeBatch(
'''
INSERT OR ${conflictAlgorithm.name.toUpperCase()} INTO $filesTable($columnNamesJoined) VALUES($valuesPlaceholders)
''',
parameterSets,
);
}
EnteFile _getFileFromRow(Map<String, dynamic> row) {
final file = EnteFile();
file.generatedID = row[columnGeneratedID];

132
mobile/lib/ui/actions/collection/collection_file_actions.dart
View File

@@ -6,6 +6,7 @@ import "package:photos/core/configuration.dart";
import "package:photos/core/event_bus.dart";
import "package:photos/db/files_db.dart";
import "package:photos/db/local/table/shared_assets.dart";
import "package:photos/db/local/table/upload_queue_table.dart";
import "package:photos/events/collection_updated_event.dart";
import "package:photos/generated/l10n.dart";
import 'package:photos/models/collection/collection.dart';
@@ -103,67 +104,32 @@ extension CollectionFileActions on CollectionActions {
final int currentUserID = Configuration.instance.getUserID()!;
for (final collection in collections) {
try {
final List<EnteFile> files = [];
final List<EnteFile> uploadedFiles = [];
final Set<String> pendingUploads = {};
final List<EnteFile> filesPendingUpload = [];
for (final file in selectedFiles!) {
EnteFile? currentFile;
if (file.uploadedFileID != null) {
currentFile = file.copyWith();
} else if (file.generatedID != null) {
// when file is not uploaded, refresh the state from the db to
// ensure we have latest upload status for given file before
// queueing it up as pending upload
currentFile = await (FilesDB.instance.getFile(file.generatedID!));
} else if (file.generatedID == null) {
logger.severe("generated id should not be null");
}
if (currentFile == null) {
logger.severe("Failed to find fileBy genID");
continue;
}
if (currentFile.uploadedFileID == null) {
currentFile.collectionID = collection.id;
filesPendingUpload.add(currentFile);
if (file.rAsset != null) {
uploadedFiles.add(file.copyWith());
} else {
files.add(currentFile);
pendingUploads.add(file.lAsset!.id);
filesPendingUpload.add(file.copyWith());
}
}
if (filesPendingUpload.isNotEmpty) {
// Newly created collection might not be cached
final Collection? c =
CollectionsService.instance.getCollectionByID(collection.id);
if (c != null && c.owner.id != currentUserID) {
final Collection uncat =
await CollectionsService.instance.getUncategorizedCollection();
for (EnteFile unuploadedFile in filesPendingUpload) {
final uploadedFile = await FileUploader.instance.forceUpload(
unuploadedFile,
uncat.id,
);
files.add(uploadedFile);
}
} else {
for (final file in filesPendingUpload) {
file.collectionID = collection.id;
}
// filesPendingUpload might be getting ignored during auto-upload
// because the user deleted these files from ente in the past.
await IgnoredFilesService.instance
.removeIgnoredMappings(filesPendingUpload);
await FilesDB.instance.insertMultiple(filesPendingUpload);
Bus.instance.fire(
CollectionUpdatedEvent(
collection.id,
filesPendingUpload,
"pendingFilesAdd",
),
);
}
if (pendingUploads.isNotEmpty) {
await IgnoredFilesService.instance
.removeIgnoredMappings(filesPendingUpload);
await localDB.insertOrUpdateQueue( pendingUploads, collection.id, currentUserID, manual: true)
Bus.instance.fire(
CollectionUpdatedEvent(
collection.id,
filesPendingUpload,
"queuedForUpload",
),
);
}
if (files.isNotEmpty) {
if (uploadedFiles.isNotEmpty) {
await CollectionsService.instance
.addOrCopyToCollection(collection.id, files);
.addOrCopyToCollection(collection.id, uploadedFiles);
}
} catch (e, s) {
logger.severe("Failed to add to album", e, s);
@@ -192,7 +158,7 @@ extension CollectionFileActions on CollectionActions {
List<SharedMediaFile>? sharedFiles,
List<AssetEntity>? picketAssets,
}) async {
ProgressDialog? dialog = showProgressDialog
final ProgressDialog? dialog = showProgressDialog
? createProgressDialog(
context,
S.of(context).uploadingFilesToAlbum,
@@ -220,69 +186,31 @@ extension CollectionFileActions on CollectionActions {
);
} else {
for (final file in selectedFiles!) {
EnteFile? currentFile;
if (file.uploadedFileID != null) {
currentFile = file.copyWith();
} else if (file.generatedID != null) {
// when file is not uploaded, refresh the state from the db to
// ensure we have latest upload status for given file before
// queueing it up as pending upload
currentFile = await (FilesDB.instance.getFile(file.generatedID!));
} else if (file.generatedID == null) {
logger.severe("generated id should not be null");
}
if (currentFile == null) {
logger.severe("Failed to find fileBy genID");
continue;
}
if (currentFile.uploadedFileID == null) {
currentFile.collectionID = collectionID;
filesPendingUpload.add(currentFile);
if (file.rAsset != null) {
uploadedFiles.add(file.copyWith());
} else if (file.lAsset != null) {
filesPendingUpload.add(file.copyWith());
} else {
uploadedFiles.add(currentFile);
throw Exception("File does not have rAsset or lAsset: $file");
}
}
}
if (filesPendingUpload.isNotEmpty) {
// Newly created collection might not be cached
final Collection? c =
CollectionsService.instance.getCollectionByID(collectionID);
if (c != null && c.owner.id != currentUserID) {
if (!showProgressDialog) {
dialog = createProgressDialog(
context,
S.of(context).uploadingFilesToAlbum,
isDismissible: true,
);
await dialog.show();
}
final Collection uncat =
await CollectionsService.instance.getUncategorizedCollection();
for (EnteFile unuploadedFile in filesPendingUpload) {
final uploadedFile = await FileUploader.instance.forceUpload(
unuploadedFile,
uncat.id,
);
uploadedFiles.add(uploadedFile);
}
} else {
final Set<String> pendingUploadAssetIDs = {};
for (final file in filesPendingUpload) {
file.collectionID = collectionID;
pendingUploadAssetIDs.add(file.lAsset!.id);
}
// filesPendingUpload might be getting ignored during auto-upload
// because the user deleted these files from ente in the past.
await IgnoredFilesService.instance
.removeIgnoredMappings(filesPendingUpload);
await FilesDB.instance.insertMultiple(filesPendingUpload);
await localDB.insertOrUpdateQueue(pendingUploadAssetIDs, collectionID, currentUserID, manual: true);
Bus.instance.fire(
CollectionUpdatedEvent(
collectionID,
filesPendingUpload,
"pendingFilesAdd",
"queuedForUpload",
),
);
}
}
if (uploadedFiles.isNotEmpty) {
await CollectionsService.instance
.addOrCopyToCollection(collectionID, uploadedFiles);