diff --git a/mobile/apps/photos/lib/db/files_db.dart b/mobile/apps/photos/lib/db/files_db.dart index 657148a61a..1957f55179 100644 --- a/mobile/apps/photos/lib/db/files_db.dart +++ b/mobile/apps/photos/lib/db/files_db.dart @@ -988,25 +988,44 @@ class FilesDB with SqlDbBase { } final db = await instance.sqliteAsyncDB; - final placeholders = List.filled(localIDs.length, '?').join(','); - final r = await db.execute( - ''' + const batchSize = 10000; + int totalRemoved = 0; + + final localIDsList = localIDs.toList(); + + for (int i = 0; i < localIDsList.length; i += batchSize) { + final endIndex = (i + batchSize > localIDsList.length) + ? localIDsList.length + : i + batchSize; + + final batch = localIDsList.sublist(i, endIndex); + final placeholders = List.filled(batch.length, '?').join(','); + + final r = await db.execute( + ''' DELETE FROM $filesTable WHERE $columnLocalID IN ($placeholders) - AND (collectionID IS NULL OR collectionID = -1) + AND ($columnCollectionID IS NULL OR $columnCollectionID = -1) AND ($columnUploadedFileID IS NULL OR $columnUploadedFileID = -1) ''', - localIDs.toList(), - ); + batch, + ); - if (r.isNotEmpty) { + if (r.isNotEmpty) { + _logger + .fine("Batch ${(i ~/ batchSize) + 1}: Removed ${r.length} files"); + totalRemoved += r.length; + } + } + + if (totalRemoved > 0) { _logger.warning( - "Removed ${r.length} potential dups for already queued local files", + "Removed $totalRemoved potential dups for already queued local files", ); } else { _logger.finest("No duplicate id found for queued/uploaded files"); } - return r.length; + return totalRemoved; } Future> getLocalFileIDsForCollection(int collectionID) async { diff --git a/mobile/apps/photos/lib/services/sync/remote_sync_service.dart b/mobile/apps/photos/lib/services/sync/remote_sync_service.dart index d0e995a438..9e2f7f6aed 100644 --- a/mobile/apps/photos/lib/services/sync/remote_sync_service.dart +++ b/mobile/apps/photos/lib/services/sync/remote_sync_service.dart @@ -51,6 +51,10 @@ class RemoteSyncService { Completer? _existingSync; bool _isExistingSyncSilent = false; + // _hasCleanupStaleEntry is used to track if we have already cleaned up + // statle db entries in this sync session. + bool _hasCleanupStaleEntry = false; + static const kHasSyncedArchiveKey = "has_synced_archive"; /* This setting is used to maintain a list of local IDs for videos that the user has manually marked for upload, even if the global video upload setting is currently disabled. @@ -371,8 +375,13 @@ class RemoteSyncService { final Set alreadyClaimedLocalIDs = await _db.getLocalIDsMarkedForOrAlreadyUploaded(ownerID); localIDsToSync.removeAll(alreadyClaimedLocalIDs); - if (alreadyClaimedLocalIDs.isNotEmpty) { + if (alreadyClaimedLocalIDs.isNotEmpty && !_hasCleanupStaleEntry) { + try { await _db.removeQueuedLocalFiles(alreadyClaimedLocalIDs); + } catch(e, s) { + _logger.severe("removeQueuedLocalFiles failed",e,s); + + } } } @@ -442,6 +451,7 @@ class RemoteSyncService { // "force reload due to display new files" Bus.instance.fire(ForceReloadHomeGalleryEvent("newFilesDisplay")); } + _hasCleanupStaleEntry = true; } Future updateDeviceFolderSyncStatus( diff --git a/mobile/apps/photos/pubspec.lock b/mobile/apps/photos/pubspec.lock index a5d14c3829..f8cf5809fe 100644 --- a/mobile/apps/photos/pubspec.lock +++ b/mobile/apps/photos/pubspec.lock @@ -130,10 +130,10 @@ packages: dependency: "direct main" description: name: async - sha256: d2872f9c19731c2e5f10444b14686eb7cc85c76274bd6c16e1816bff9a3bab63 + sha256: "947bfcf187f74dbc5e146c9eb9c0f10c9f8b30743e341481c1e2ed3ecc18c20c" url: "https://pub.dev" source: hosted - version: "2.12.0" + version: "2.11.0" battery_info: dependency: "direct main" description: @@ -155,10 +155,10 @@ packages: dependency: transitive description: name: boolean_selector - sha256: "8aab1771e1243a5063b8b0ff68042d67334e3feab9e95b9490f9a6ebf73b42ea" + sha256: "6cfb5af12253eaf2b368f07bacc5a80d1301a071c73360d746b7f2e32d762c66" url: "https://pub.dev" source: hosted - version: "2.1.2" + version: "2.1.1" brotli: dependency: transitive description: @@ -268,10 +268,10 @@ packages: dependency: transitive description: name: characters - sha256: f71061c654a3380576a52b451dd5532377954cf9dbd272a78fc8479606670803 + sha256: "04a925763edad70e8443c99234dc3328f442e811f1d8fd1a72f1c8ad0f69a605" url: "https://pub.dev" source: hosted - version: "1.4.0" + version: "1.3.0" checked_yaml: dependency: transitive description: @@ -301,10 +301,10 @@ packages: dependency: transitive description: name: clock - sha256: fddb70d9b5277016c77a80201021d40a2247104d9f4aa7bab7157b7e3f05b84b + sha256: cb6d7f03e1de671e34607e909a7213e31d7752be4fb66a86d29fe1eb14bfb5cf url: "https://pub.dev" source: hosted - version: "1.1.2" + version: "1.1.1" code_builder: dependency: transitive description: @@ -317,10 +317,10 @@ packages: dependency: "direct main" description: name: collection - sha256: "2f5709ae4d3d59dd8f7cd309b4e023046b57d8a6c82130785d2b0e5868084e76" + sha256: a1ace0a119f20aabc852d165077c036cd864315bd99b7eaa10a60100341941bf url: "https://pub.dev" source: hosted - version: "1.19.1" + version: "1.19.0" computer: dependency: "direct main" description: @@ -619,10 +619,10 @@ packages: dependency: transitive description: name: fake_async - sha256: "6a95e56b2449df2273fd8c45a662d6947ce1ebb7aafe80e550a3f68297f3cacc" + sha256: "511392330127add0b769b75a987850d136345d9227c6b94c96a04cf4a391bf78" url: "https://pub.dev" source: hosted - version: "1.3.2" + version: "1.3.1" fast_base58: dependency: "direct main" description: @@ -668,10 +668,10 @@ packages: dependency: transitive description: name: file - sha256: a3b4f84adafef897088c160faf7dfffb7696046cb13ae90b508c2cbc95d3b8d4 + sha256: "5fc22d7c25582e38ad9a8515372cd9a93834027aacf1801cf01164dac0ffa08c" url: "https://pub.dev" source: hosted - version: "7.0.1" + version: "7.0.0" file_saver: dependency: "direct main" description: @@ -1416,18 +1416,18 @@ packages: dependency: transitive description: name: leak_tracker - sha256: c35baad643ba394b40aac41080300150a4f08fd0fd6a10378f8f7c6bc161acec + sha256: "7bb2830ebd849694d1ec25bf1f44582d6ac531a57a365a803a6034ff751d2d06" url: "https://pub.dev" source: hosted - version: "10.0.8" + version: "10.0.7" leak_tracker_flutter_testing: dependency: transitive description: name: leak_tracker_flutter_testing - sha256: f8b613e7e6a13ec79cfdc0e97638fddb3ab848452eff057653abd3edba760573 + sha256: "9491a714cca3667b60b5c420da8217e6de0d1ba7a5ec322fab01758f6998f379" url: "https://pub.dev" source: hosted - version: "3.0.9" + version: "3.0.8" leak_tracker_testing: dependency: transitive description: @@ -1552,10 +1552,10 @@ packages: dependency: transitive description: name: matcher - sha256: dc58c723c3c24bf8d3e2d3ad3f2f9d7bd9cf43ec6feaa64181775e60190153f2 + sha256: d2323aa2060500f906aa31a895b4030b6da3ebdcc5619d14ce1aada65cd161cb url: "https://pub.dev" source: hosted - version: "0.12.17" + version: "0.12.16+1" material_color_utilities: dependency: transitive description: @@ -1645,10 +1645,10 @@ packages: dependency: transitive description: name: meta - sha256: e3641ec5d63ebf0d9b41bd43201a66e3fc79a65db5f61fc181f04cd27aab950c + sha256: bdb68674043280c3428e9ec998512fb681678676b3c54e773629ffe74419f8c7 url: "https://pub.dev" source: hosted - version: "1.16.0" + version: "1.15.0" mgrs_dart: dependency: transitive description: @@ -1859,10 +1859,10 @@ packages: dependency: "direct main" description: name: path - sha256: "75cca69d1490965be98c73ceaea117e8a04dd21217b37b292c9ddbec0d955bc5" + sha256: "087ce49c3f0dc39180befefc60fdb4acd8f8620e5682fe2476afd0b3688bb4af" url: "https://pub.dev" source: hosted - version: "1.9.1" + version: "1.9.0" path_drawing: dependency: transitive description: @@ -2019,10 +2019,10 @@ packages: dependency: transitive description: name: platform - sha256: "5d6b1b0036a5f331ebc77c850ebc8506cbc1e9416c27e59b439f917a902a4984" + sha256: "9b71283fc13df574056616011fb138fd3b793ea47cc509c189a6c3fa5f8a1a65" url: "https://pub.dev" source: hosted - version: "3.1.6" + version: "3.1.5" plugin_platform_interface: dependency: transitive description: @@ -2068,10 +2068,10 @@ packages: dependency: transitive description: name: process - sha256: "107d8be718f120bbba9dcd1e95e3bd325b1b4a4f07db64154635ba03f2567a0d" + sha256: "21e54fd2faf1b5bdd5102afd25012184a6793927648ea81eea80552ac9405b32" url: "https://pub.dev" source: hosted - version: "5.0.3" + version: "5.0.2" proj4dart: dependency: transitive description: @@ -2346,10 +2346,10 @@ packages: dependency: transitive description: name: source_span - sha256: "254ee5351d6cb365c859e20ee823c3bb479bf4a293c22d17a9f1bf144ce86f7c" + sha256: "53e943d4206a5e30df338fd4c6e7a077e02254531b138a15aec3bd143c1a8b3c" url: "https://pub.dev" source: hosted - version: "1.10.1" + version: "1.10.0" sprintf: dependency: transitive description: @@ -2434,10 +2434,10 @@ packages: dependency: transitive description: name: stack_trace - sha256: "8b27215b45d22309b5cddda1aa2b19bdfec9df0e765f2de506401c071d38d1b1" + sha256: "9f47fd3630d76be3ab26f0ee06d213679aa425996925ff3feffdec504931c377" url: "https://pub.dev" source: hosted - version: "1.12.1" + version: "1.12.0" step_progress_indicator: dependency: "direct main" description: @@ -2450,10 +2450,10 @@ packages: dependency: transitive description: name: stream_channel - sha256: "969e04c80b8bcdf826f8f16579c7b14d780458bd97f56d107d3950fdbeef059d" + sha256: ba2aa5d8cc609d96bbb2899c28934f9e1af5cddbd60a827822ea467161eb54e7 url: "https://pub.dev" source: hosted - version: "2.1.4" + version: "2.1.2" stream_transform: dependency: transitive description: @@ -2466,10 +2466,10 @@ packages: dependency: transitive description: name: string_scanner - sha256: "921cd31725b72fe181906c6a94d987c78e3b98c2e205b397ea399d4054872b43" + sha256: "688af5ed3402a4bde5b3a6c15fd768dbf2621a614950b17f04626c431ab3c4c3" url: "https://pub.dev" source: hosted - version: "1.4.1" + version: "1.3.0" styled_text: dependency: "direct main" description: @@ -2522,34 +2522,34 @@ packages: dependency: transitive description: name: term_glyph - sha256: "7f554798625ea768a7518313e58f83891c7f5024f88e46e7182a4558850a4b8e" + sha256: a29248a84fbb7c79282b40b8c72a1209db169a2e0542bce341da992fe1bc7e84 url: "https://pub.dev" source: hosted - version: "1.2.2" + version: "1.2.1" test: dependency: "direct dev" description: name: test - sha256: "301b213cd241ca982e9ba50266bd3f5bd1ea33f1455554c5abb85d1be0e2d87e" + sha256: "713a8789d62f3233c46b4a90b174737b2c04cb6ae4500f2aa8b1be8f03f5e67f" url: "https://pub.dev" source: hosted - version: "1.25.15" + version: "1.25.8" test_api: dependency: transitive description: name: test_api - sha256: fb31f383e2ee25fbbfe06b40fe21e1e458d14080e3c67e7ba0acfde4df4e0bbd + sha256: "664d3a9a64782fcdeb83ce9c6b39e78fd2971d4e37827b9b06c3aa1edc5e760c" url: "https://pub.dev" source: hosted - version: "0.7.4" + version: "0.7.3" test_core: dependency: transitive description: name: test_core - sha256: "84d17c3486c8dfdbe5e12a50c8ae176d15e2a771b96909a9442b40173649ccaa" + sha256: "12391302411737c176b0b5d6491f466b0dd56d4763e347b6714efbaa74d7953d" url: "https://pub.dev" source: hosted - version: "0.6.8" + version: "0.6.5" thermal: dependency: "direct main" description: @@ -2813,10 +2813,10 @@ packages: dependency: transitive description: name: vm_service - sha256: "0968250880a6c5fe7edc067ed0a13d4bae1577fe2771dcf3010d52c4a9d3ca14" + sha256: f6be3ed8bd01289b34d679c2b62226f63c0e69f9fd2e50a6b3c1c729a961041b url: "https://pub.dev" source: hosted - version: "14.3.1" + version: "14.3.0" volume_controller: dependency: transitive description: @@ -2978,5 +2978,5 @@ packages: source: hosted version: "3.1.3" sdks: - dart: ">=3.7.0-0 <4.0.0" + dart: ">=3.5.0 <4.0.0" flutter: ">=3.24.0"