TerribleDev/ente
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Don't migrate or call VectorDB automatically

Browse Source
This commit is contained in:
laurenspriem
2025-07-24 13:19:53 +02:00
parent 5e84774737
commit 5b291de28f
2 changed files with 44 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
mobile/apps/photos/lib/db/ml/clip_vector_db.dart
View File

@@ -1,3 +1,4 @@
import "dart:io" show File;
import "dart:typed_data" show Float32List;
import "package:flutter_rust_bridge/flutter_rust_bridge.dart" show Uint64List;
@@ -30,6 +31,8 @@ class ClipVectorDB {
return _vectorDbFuture!;
}
bool? _migrationDone;
Future<VectorDb> _initVectorDB() async {
final documentsDirectory = await getApplicationDocumentsDirectory();
final String databaseDirectory =
@@ -45,6 +48,32 @@ class ClipVectorDB {
return vectorDB;
}
Future<bool> checkIfMigrationDone() async {
if (_migrationDone != null) return _migrationDone!;
_logger.info("Checking if ClipVectorDB migration has run");
final documentsDirectory = await getApplicationDocumentsDirectory();
final migrationFlagFile =
File(join(documentsDirectory.path, 'clip_vector_migration_done'));
if (await migrationFlagFile.exists()) {
_logger.info("ClipVectorDB migration already done");
_migrationDone = true;
return _migrationDone!;
} else {
_logger.info("ClipVectorDB migration not done");
_migrationDone = false;
return _migrationDone!;
}
}
Future<void> setMigrationDone() async {
_logger.info("Setting ClipVectorDB migration done");
final documentsDirectory = await getApplicationDocumentsDirectory();
final migrationFlagFile =
File(join(documentsDirectory.path, 'clip_vector_migration_done'));
await migrationFlagFile.create(recursive: true);
_migrationDone = true;
}
Future<void> insertEmbedding({
required int fileID,
required List<double> embedding,

22
mobile/apps/photos/lib/db/ml/db.dart
View File

@@ -87,8 +87,6 @@ class MLDataDB with SqlDbBase implements IMLDataDB<int> {
"MLDataDB Migration took ${stopwatch.elapsedMilliseconds} ms",
);
stopwatch.stop();
_logger.info("Starting CLIP vector DB migration check unawaited");
if (flagService.enableVectorDb) unawaited(checkMigrateFillClipVectorDB());
return asyncDBConnection;
}
@@ -1263,6 +1261,12 @@ class MLDataDB with SqlDbBase implements IMLDataDB<int> {
await Future.delayed(const Duration(milliseconds: 100));
_logger.info("Checking if ClipVectorDB migration is needed");
final migrationDone = await ClipVectorDB.instance.checkIfMigrationDone();
if (migrationDone && !force) {
_logger.info("ClipVectorDB migration not needed, already done");
return;
}
// Check if vector DB migration has run
_logger.info("Checking if ClipVectorDB migration has run");
final documentsDirectory = await getApplicationDocumentsDirectory();
@@ -1309,7 +1313,7 @@ class MLDataDB with SqlDbBase implements IMLDataDB<int> {
_logger.info("Reading $batchSize rows from DB");
final List<Map<String, dynamic>> results = await db.getAll('''
SELECT $fileIDColumn, $embeddingColumn
SELECT $fileIDColumn, $embeddingColumn
FROM $clipTable
ORDER BY $fileIDColumn DESC
LIMIT $batchSize OFFSET $offset
@@ -1406,7 +1410,8 @@ class MLDataDB with SqlDbBase implements IMLDataDB<int> {
'INSERT OR REPLACE INTO $clipTable ($fileIDColumn, $embeddingColumn, $mlVersionColumn) VALUES (?, ?, ?)',
_getRowFromEmbedding(embeddings.first),
);
if (flagService.enableVectorDb) {
if (flagService.enableVectorDb &&
await ClipVectorDB.instance.checkIfMigrationDone()) {
await ClipVectorDB.instance.insertEmbedding(
fileID: embeddings.first.fileID,
embedding: embeddings.first.embedding,
@@ -1418,7 +1423,8 @@ class MLDataDB with SqlDbBase implements IMLDataDB<int> {
'INSERT OR REPLACE INTO $clipTable ($fileIDColumn, $embeddingColumn, $mlVersionColumn) values(?, ?, ?)',
inputs,
);
if (flagService.enableVectorDb) {
if (flagService.enableVectorDb &&
await ClipVectorDB.instance.checkIfMigrationDone()) {
await ClipVectorDB.instance.bulkInsertEmbeddings(
fileIDs: embeddings.map((e) => e.fileID).toList(),
embeddings:
@@ -1435,7 +1441,8 @@ class MLDataDB with SqlDbBase implements IMLDataDB<int> {
await db.execute(
'DELETE FROM $clipTable WHERE $fileIDColumn IN (${fileIDs.join(", ")})',
);
if (flagService.enableVectorDb) {
if (flagService.enableVectorDb &&
await ClipVectorDB.instance.checkIfMigrationDone()) {
await ClipVectorDB.instance.deleteEmbeddings(fileIDs);
}
Bus.instance.fire(EmbeddingUpdatedEvent());
@@ -1445,7 +1452,8 @@ class MLDataDB with SqlDbBase implements IMLDataDB<int> {
Future<void> deleteClipIndexes() async {
final db = await instance.asyncDB;
await db.execute('DELETE FROM $clipTable');
if (flagService.enableVectorDb) {
if (flagService.enableVectorDb &&
await ClipVectorDB.instance.checkIfMigrationDone()) {
await ClipVectorDB.instance.deleteAllEmbeddings();
}
Bus.instance.fire(EmbeddingUpdatedEvent());