[mob] Fix query for duplicate cleanup (#6962)

## Description
Only consider owned files
## Tests
This commit is contained in:
Neeraj
2025-08-25 11:10:36 +05:30
committed by GitHub
2 changed files with 37 additions and 27 deletions

View File

@@ -981,7 +981,8 @@ class FilesDB with SqlDbBase {
// remove references for local files which are either already uploaded
// or queued for upload but not yet uploaded
Future<int> removeQueuedLocalFiles(Set<String> localIDs) async {
// Remove queued local files that have duplicate uploaded entries with same localID
Future<int> removeQueuedLocalFiles(Set<String> localIDs, int ownerID) async {
if (localIDs.isEmpty) {
_logger.finest("No local IDs provided for removal");
return 0;
@@ -990,54 +991,63 @@ class FilesDB with SqlDbBase {
final db = await instance.sqliteAsyncDB;
const batchSize = 10000;
int totalRemoved = 0;
final localIDsList = localIDs.toList();
for (int i = 0; i < localIDsList.length; i += batchSize) {
final endIndex = (i + batchSize > localIDsList.length)
? localIDsList.length
: i + batchSize;
final batch = localIDsList.sublist(i, endIndex);
final placeholders = List.filled(batch.length, '?').join(',');
final List<String> alreadyUploaded = [];
// find localIDs that are already uploaded
final result = await db.execute('''
// Find localIDs that already have uploaded entries
final result = await db.execute(
'''
SELECT DISTINCT $columnLocalID
FROM $filesTable
WHERE $columnLocalID IN ($placeholders)
WHERE
$columnOwnerID = $ownerID
AND $columnLocalID IN ($placeholders)
AND ($columnUploadedFileID IS NOT NULL AND $columnUploadedFileID != -1)
''');
for (final row in result) {
alreadyUploaded.add(row[columnLocalID] as String);
}
final uploadedPlaceholders =
alreadyUploaded.map((id) => "'$id'").join(',');
final r = await db.execute(
'''
DELETE FROM $filesTable
WHERE $columnLocalID IN ($uploadedPlaceholders)
AND ($columnUploadedFileID IS NULL OR $columnUploadedFileID = -1)
''',
''',
batch,
);
if (r.isNotEmpty) {
_logger.warning(
"Batch ${(i ~/ batchSize) + 1}: Removed duplicate ${r.length} files",
if (result.isNotEmpty) {
final alreadyUploadedLocalIDs =
result.map((row) => row[columnLocalID] as String).toList();
final localIdPlaceholder =
List.filled(alreadyUploadedLocalIDs.length, '?').join(',');
// Delete queued entries for localIDs that already have uploaded versions
final deleteResult = await db.execute(
'''
DELETE FROM $filesTable
WHERE $columnLocalID IN ($localIdPlaceholder)
AND ($columnUploadedFileID IS NULL OR $columnUploadedFileID = -1)
''',
alreadyUploadedLocalIDs,
);
totalRemoved += r.length;
final removedCount =
deleteResult.length; // or however your DB returns affected rows
if (removedCount > 0) {
_logger.warning(
"Batch ${(i ~/ batchSize) + 1}: Removed $removedCount queued duplicates",
);
totalRemoved += removedCount;
}
}
}
if (totalRemoved > 0) {
_logger.warning(
"Removed $totalRemoved potential dups for already queued local files",
"Removed $totalRemoved queued files that had uploaded duplicates",
);
} else {
_logger.finest("No duplicate id found for queued/uploaded files");
_logger.finest("No queued duplicates found for uploaded files");
}
return totalRemoved;
}

View File

@@ -377,7 +377,7 @@ class RemoteSyncService {
localIDsToSync.removeAll(alreadyClaimedLocalIDs);
if (alreadyClaimedLocalIDs.isNotEmpty && !_hasCleanupStaleEntry) {
try {
await _db.removeQueuedLocalFiles(alreadyClaimedLocalIDs);
await _db.removeQueuedLocalFiles(alreadyClaimedLocalIDs, ownerID);
} catch (e, s) {
_logger.severe("removeQueuedLocalFiles failed", e, s);
}