[mob] Prioritize fetch for already indexed files.

This commit is contained in:
Neeraj Gupta
2024-11-07 16:18:33 +05:30
parent 6edacd2f1d
commit 214dcddb65
3 changed files with 16 additions and 2 deletions

View File

@@ -25,4 +25,10 @@ extension FileDataTable on MLDataDB {
inputs,
);
}
Future<Set<int>> getFileIDsWithFDData() async {
final db = await MLDataDB.instance.asyncDB;
final res = await db.execute('SELECT $fileIDColumn FROM $fileDataTable');
return res.map((e) => e[fileIDColumn] as int).toSet();
}
}

View File

@@ -115,7 +115,7 @@ class FileDataService {
try {
bool hasMoreData = false;
do {
final lastTime = _prefs.getInt("fd.lastSyncTimex") ?? 0;
final lastTime = _prefs.getInt("fd.lastSyncTime") ?? 0;
final res = await _dio.post(
"/files/data/status-diff",
data: {

View File

@@ -7,6 +7,7 @@ import "package:photos/core/configuration.dart";
import "package:photos/db/files_db.dart";
import "package:photos/db/ml/clip_db.dart";
import "package:photos/db/ml/db.dart";
import "package:photos/db/ml/filedata.dart";
import "package:photos/extensions/list.dart";
import "package:photos/models/file/extensions/file_props.dart";
import "package:photos/models/file/file.dart";
@@ -87,6 +88,9 @@ Future<List<FileMLInstruction>> getFilesForMlIndexing() async {
await MLDataDB.instance.clipIndexedFileWithVersion();
final Set<int> queuedFiledIDs = {};
final Set<int> filesWithFDStatus =
await MLDataDB.instance.getFileIDsWithFDData();
// Get all regular files and all hidden files
final enteFiles = await SearchService.instance.getAllFiles();
final hiddenFiles = await SearchService.instance.getHiddenFiles();
@@ -149,10 +153,14 @@ Future<List<FileMLInstruction>> getFilesForMlIndexing() async {
...filesWithoutLocalID,
...hiddenFilesToIndex,
];
final splitResult = sortedBylocalID.splitMatch(
(i) => filesWithFDStatus.contains(i.file.uploadedFileID!),
);
_logger.info(
"Getting list of files to index for ML took ${DateTime.now().difference(time).inMilliseconds} ms",
);
return sortedBylocalID;
return [...splitResult.matched, ...splitResult.unmatched];
}
Stream<List<FileMLInstruction>> fetchEmbeddingsAndInstructions(