[mob] Fix query for duplicate cleanup (#6962)
## Description Only consider owned files ## Tests
This commit is contained in:
@@ -981,7 +981,8 @@ class FilesDB with SqlDbBase {
|
||||
|
||||
// remove references for local files which are either already uploaded
|
||||
// or queued for upload but not yet uploaded
|
||||
Future<int> removeQueuedLocalFiles(Set<String> localIDs) async {
|
||||
// Remove queued local files that have duplicate uploaded entries with same localID
|
||||
Future<int> removeQueuedLocalFiles(Set<String> localIDs, int ownerID) async {
|
||||
if (localIDs.isEmpty) {
|
||||
_logger.finest("No local IDs provided for removal");
|
||||
return 0;
|
||||
@@ -990,54 +991,63 @@ class FilesDB with SqlDbBase {
|
||||
final db = await instance.sqliteAsyncDB;
|
||||
const batchSize = 10000;
|
||||
int totalRemoved = 0;
|
||||
|
||||
final localIDsList = localIDs.toList();
|
||||
|
||||
for (int i = 0; i < localIDsList.length; i += batchSize) {
|
||||
final endIndex = (i + batchSize > localIDsList.length)
|
||||
? localIDsList.length
|
||||
: i + batchSize;
|
||||
|
||||
final batch = localIDsList.sublist(i, endIndex);
|
||||
final placeholders = List.filled(batch.length, '?').join(',');
|
||||
final List<String> alreadyUploaded = [];
|
||||
// find localIDs that are already uploaded
|
||||
final result = await db.execute('''
|
||||
|
||||
// Find localIDs that already have uploaded entries
|
||||
final result = await db.execute(
|
||||
'''
|
||||
SELECT DISTINCT $columnLocalID
|
||||
FROM $filesTable
|
||||
WHERE $columnLocalID IN ($placeholders)
|
||||
WHERE
|
||||
$columnOwnerID = $ownerID
|
||||
AND $columnLocalID IN ($placeholders)
|
||||
AND ($columnUploadedFileID IS NOT NULL AND $columnUploadedFileID != -1)
|
||||
''');
|
||||
|
||||
for (final row in result) {
|
||||
alreadyUploaded.add(row[columnLocalID] as String);
|
||||
}
|
||||
final uploadedPlaceholders =
|
||||
alreadyUploaded.map((id) => "'$id'").join(',');
|
||||
final r = await db.execute(
|
||||
'''
|
||||
DELETE FROM $filesTable
|
||||
WHERE $columnLocalID IN ($uploadedPlaceholders)
|
||||
AND ($columnUploadedFileID IS NULL OR $columnUploadedFileID = -1)
|
||||
''',
|
||||
''',
|
||||
batch,
|
||||
);
|
||||
|
||||
if (r.isNotEmpty) {
|
||||
_logger.warning(
|
||||
"Batch ${(i ~/ batchSize) + 1}: Removed duplicate ${r.length} files",
|
||||
if (result.isNotEmpty) {
|
||||
final alreadyUploadedLocalIDs =
|
||||
result.map((row) => row[columnLocalID] as String).toList();
|
||||
final localIdPlaceholder =
|
||||
List.filled(alreadyUploadedLocalIDs.length, '?').join(',');
|
||||
|
||||
// Delete queued entries for localIDs that already have uploaded versions
|
||||
final deleteResult = await db.execute(
|
||||
'''
|
||||
DELETE FROM $filesTable
|
||||
WHERE $columnLocalID IN ($localIdPlaceholder)
|
||||
AND ($columnUploadedFileID IS NULL OR $columnUploadedFileID = -1)
|
||||
''',
|
||||
alreadyUploadedLocalIDs,
|
||||
);
|
||||
totalRemoved += r.length;
|
||||
|
||||
final removedCount =
|
||||
deleteResult.length; // or however your DB returns affected rows
|
||||
if (removedCount > 0) {
|
||||
_logger.warning(
|
||||
"Batch ${(i ~/ batchSize) + 1}: Removed $removedCount queued duplicates",
|
||||
);
|
||||
totalRemoved += removedCount;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (totalRemoved > 0) {
|
||||
_logger.warning(
|
||||
"Removed $totalRemoved potential dups for already queued local files",
|
||||
"Removed $totalRemoved queued files that had uploaded duplicates",
|
||||
);
|
||||
} else {
|
||||
_logger.finest("No duplicate id found for queued/uploaded files");
|
||||
_logger.finest("No queued duplicates found for uploaded files");
|
||||
}
|
||||
|
||||
return totalRemoved;
|
||||
}
|
||||
|
||||
|
||||
@@ -377,7 +377,7 @@ class RemoteSyncService {
|
||||
localIDsToSync.removeAll(alreadyClaimedLocalIDs);
|
||||
if (alreadyClaimedLocalIDs.isNotEmpty && !_hasCleanupStaleEntry) {
|
||||
try {
|
||||
await _db.removeQueuedLocalFiles(alreadyClaimedLocalIDs);
|
||||
await _db.removeQueuedLocalFiles(alreadyClaimedLocalIDs, ownerID);
|
||||
} catch (e, s) {
|
||||
_logger.severe("removeQueuedLocalFiles failed", e, s);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user