Use remote DB

This commit is contained in:
Neeraj Gupta
2025-08-06 10:17:27 +05:30
parent c80e4a65b8
commit 8ff0f237e7
5 changed files with 50 additions and 40 deletions

View File

@@ -1,7 +1,6 @@
import "dart:async";
import "dart:io";
import "package:computer/computer.dart";
import 'package:logging/logging.dart';
import 'package:path/path.dart';
import 'package:path_provider/path_provider.dart';
@@ -578,26 +577,8 @@ class FilesDB with SqlDbBase {
}
}
Future<Map<int, EnteFile>> getFileIDToFileFromIDs(List<int> ids) async {
final result = <int, EnteFile>{};
if (ids.isEmpty) {
return result;
}
final inParam = ids.map((id) => "'$id'").join(',');
final db = await instance.sqliteAsyncDB;
final results = await db.getAll(
'SELECT * FROM $filesTable WHERE $columnUploadedFileID IN ($inParam)',
);
final files = convertToFiles(results);
for (final file in files) {
result[file.uploadedFileID!] = file;
}
return result;
}
Future<List<EnteFile>> getUniqueFiles(
List<int> ids, {
bool dedupeByUploadId = false,
Set<int> collectionsToIgnore = const {},
}) async {
if (ids.isEmpty) {

View File

@@ -146,28 +146,24 @@ class SemanticSearchService {
for (final result in queryResults) {
fileIDToScoreMap[result.id] = result.score;
}
late final Map<int, EnteFile> filesMap;
late final Set<int> missingFileIDs;
(filesMap, missingFileIDs) = await remoteCache.getUniqueFiles(
queryResults.map((e) => e.id).toList(),
ignoredCollectionIDs:
CollectionsService.instance.getHiddenCollectionIds(),
);
final filesMap = await FilesDB.instance
.getFileIDToFileFromIDs(queryResults.map((e) => e.id).toList());
final ignoredCollections =
CollectionsService.instance.getHiddenCollectionIds();
final deletedEntries = <int>[];
final results = <EnteFile>[];
for (final result in queryResults) {
final file = filesMap[result.id];
if (file != null && !ignoredCollections.contains(file.collectionID)) {
if (file != null) {
results.add(file);
}
if (file == null) {
deletedEntries.add(result.id);
}
}
_logger.info(results.length.toString() + " results");
if (deletedEntries.isNotEmpty) {
unawaited(mlDataDB.deleteClipEmbeddings(deletedEntries));
if (missingFileIDs.isNotEmpty) {
unawaited(mlDataDB.deleteClipEmbeddings(missingFileIDs.toList()));
}
return results;
}

View File

@@ -203,6 +203,33 @@ class RemoteCache {
return result;
}
// Returns a map of fileID to EnteFile for the given fileIDs.
// and also returns a set of fileIDs that were not found in the DB
Future<(Map<int, EnteFile>, Set<int>)> getUniqueFiles(
List<int> fileIDs, {
Set<int> ignoredCollectionIDs = const {},
}) async {
final collectionFiles = await remoteDB.getAllCFForFileIDs(fileIDs);
final Set<int> missingIDs = fileIDs.toSet();
if (!_isLoaded) await _ensureLoaded();
final Map<int, EnteFile> result = {};
final Set<int> toIgnoreFileIds = {};
for (final cf in collectionFiles) {
missingIDs.remove(cf.fileID);
final asset = remoteAssets[cf.fileID];
if (asset != null) {
result[cf.fileID] = EnteFile.fromRemoteAsset(asset, cf);
}
if (ignoredCollectionIDs.contains(cf.collectionID)) {
toIgnoreFileIds.add(cf.fileID);
}
}
for (final fileID in toIgnoreFileIds) {
result.remove(fileID);
}
return (result, missingIDs);
}
Future<List<EnteFile>> getFilesCreatedWithinDurations(
List<List<int>> durations,
Set<int> ignoredCollectionIDs, {

View File

@@ -15,6 +15,7 @@ import "package:photos/models/file/file.dart";
import 'package:photos/models/gallery_type.dart';
import "package:photos/models/ml/face/person.dart";
import 'package:photos/models/selected_files.dart';
import "package:photos/service_locator.dart";
import 'package:photos/services/collections_service.dart';
import "package:photos/services/machine_learning/face_ml/feedback/cluster_feedback.dart";
import "package:photos/services/machine_learning/ml_result.dart";
@@ -219,11 +220,13 @@ class _AppBarWidgetState extends State<ClusterAppBar> {
final biggestClusterFileIDs = newClusterIDToFaceIDs[biggestClusterID]!
.map((e) => getFileIdFromFaceId<int>(e))
.toList();
biggestClusterFiles = await FilesDB.instance
.getFileIDToFileFromIDs(
biggestClusterFiles = await remoteCache
.getUniqueFiles(
biggestClusterFileIDs,
ignoredCollectionIDs:
CollectionsService.instance.getHiddenCollectionIds(),
)
.then((mapping) => mapping.values.toList());
.then((mapping) => mapping.$1.values.toList());
// Sort the files to prevent issues with the order of the files in gallery
biggestClusterFiles
.sort((a, b) => b.creationTime!.compareTo(a.creationTime!));
@@ -264,9 +267,13 @@ class _AppBarWidgetState extends State<ClusterAppBar> {
.map((e) => getFileIdFromFaceId<int>(e))
.toList();
final fileIDtoFile = await FilesDB.instance.getFileIDToFileFromIDs(
allFileIDs,
);
final fileIDtoFile = await remoteCache
.getUniqueFiles(
allFileIDs,
ignoredCollectionIDs:
CollectionsService.instance.getHiddenCollectionIds(),
)
.then((mapping) => mapping.$1);
final newClusterIDToFiles = newClusterIDToFaceIDs.map(
(key, value) => MapEntry(

View File

@@ -136,7 +136,6 @@ Future<List<EnteFile>> getFilteredFiles(
filteredFiles = await FilesDB.instance.getUniqueFiles(
filteredUploadedIDs.toList(),
dedupeByUploadId: true,
collectionsToIgnore: ignoredCollections,
);
} catch (e) {