TerribleDev/ente
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[mob] Switch to nanoID, gzip person feedback, & merge ml db (#2724)

Browse Source 
## Description

## Tests
This commit is contained in:
Neeraj Gupta
2024-08-16 13:13:50 +05:30
committed by GitHub
parent 3f9d8ea752 1ca1967ec2
commit 3c11b705ac
56 changed files with 667 additions and 562 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
mobile/lib/core/configuration.dart
View File

@@ -11,15 +11,14 @@ import 'package:photos/core/constants.dart';
import 'package:photos/core/error-reporting/super_logging.dart';
import 'package:photos/core/event_bus.dart';
import 'package:photos/db/collections_db.dart';
import "package:photos/db/embeddings_db.dart";
import 'package:photos/db/files_db.dart';
import 'package:photos/db/memories_db.dart';
import "package:photos/db/ml/db.dart";
import 'package:photos/db/trash_db.dart';
import 'package:photos/db/upload_locks_db.dart';
import "package:photos/events/endpoint_updated_event.dart";
import 'package:photos/events/signed_in_event.dart';
import 'package:photos/events/user_logged_out_event.dart';
import "package:photos/face/db.dart";
import 'package:photos/models/key_attributes.dart';
import 'package:photos/models/key_gen_result.dart';
import 'package:photos/models/private_key_attributes.dart';
@@ -28,7 +27,6 @@ import 'package:photos/services/collections_service.dart';
import 'package:photos/services/favorites_service.dart';
import "package:photos/services/home_widget_service.dart";
import 'package:photos/services/ignored_files_service.dart';
import 'package:photos/services/machine_learning/semantic_search/semantic_search_service.dart';
import 'package:photos/services/memories_service.dart';
import 'package:photos/services/search_service.dart';
import 'package:photos/services/sync_service.dart';
@@ -205,9 +203,6 @@ class Configuration {
_cachedToken = null;
_secretKey = null;
await FilesDB.instance.clearTable();
SemanticSearchService.instance.hasInitialized
? await EmbeddingsDB.instance.clearTable()
: null;
await CollectionsDB.instance.clearTable();
await MemoriesDB.instance.clearTable();
await FaceMLDataDB.instance.clearTable();

172
mobile/lib/db/embeddings_db.dart
View File

@@ -1,172 +0,0 @@
import "dart:io";
import "dart:typed_data";
import "package:path/path.dart";
import 'package:path_provider/path_provider.dart';
import "package:photos/core/event_bus.dart";
import "package:photos/events/embedding_updated_event.dart";
import "package:photos/models/embedding.dart";
import "package:sqlite_async/sqlite_async.dart";
class EmbeddingsDB {
EmbeddingsDB._privateConstructor();
static final EmbeddingsDB instance = EmbeddingsDB._privateConstructor();
static const databaseName = "ente.embeddings.db";
static const tableName = "clip_embedding";
static const oldTableName = "embeddings";
static const columnFileID = "file_id";
static const columnEmbedding = "embedding";
static const columnVersion = "version";
@Deprecated("")
static const columnUpdationTime = "updation_time";
static Future<SqliteDatabase>? _dbFuture;
Future<SqliteDatabase> get _database async {
_dbFuture ??= _initDatabase();
return _dbFuture!;
}
Future<void> init() async {
final dir = await getApplicationDocumentsDirectory();
await _clearDeprecatedStores(dir);
}
Future<SqliteDatabase> _initDatabase() async {
final Directory documentsDirectory =
await getApplicationDocumentsDirectory();
final String path = join(documentsDirectory.path, databaseName);
final migrations = SqliteMigrations()
..add(
SqliteMigration(
1,
(tx) async {
// Avoid creating the old table
// await tx.execute(
// 'CREATE TABLE $oldTableName ($columnFileID INTEGER NOT NULL, $columnEmbedding BLOB NOT NULL, $columnUpdationTime INTEGER, UNIQUE ($columnFileID))',
// );
},
),
)
..add(
SqliteMigration(
2,
(tx) async {
// delete old table
await tx.execute('DROP TABLE IF EXISTS $oldTableName');
await tx.execute(
'CREATE TABLE $tableName ($columnFileID INTEGER NOT NULL, $columnEmbedding BLOB NOT NULL, $columnVersion INTEGER, UNIQUE ($columnFileID))',
);
},
),
);
final database = SqliteDatabase(path: path);
await migrations.migrate(database);
return database;
}
Future<void> clearTable() async {
final db = await _database;
await db.execute('DELETE FROM $tableName');
}
Future<List<ClipEmbedding>> getAll() async {
final db = await _database;
final results = await db.getAll('SELECT * FROM $tableName');
return _convertToEmbeddings(results);
}
// Get indexed FileIDs
Future<Map<int, int>> getIndexedFileIds() async {
final db = await _database;
final maps = await db
.getAll('SELECT $columnFileID , $columnVersion FROM $tableName');
final Map<int, int> result = {};
for (final map in maps) {
result[map[columnFileID] as int] = map[columnVersion] as int;
}
return result;
}
// TODO: Add actual colomn for version and use here, similar to faces
Future<int> getIndexedFileCount() async {
final db = await _database;
const String query =
'SELECT COUNT(DISTINCT $columnFileID) as count FROM $tableName';
final List<Map<String, dynamic>> maps = await db.getAll(query);
return maps.first['count'] as int;
}
Future<void> put(ClipEmbedding embedding) async {
final db = await _database;
await db.execute(
'INSERT OR REPLACE INTO $tableName ($columnFileID, $columnEmbedding, $columnVersion) VALUES (?, ?, ?)',
_getRowFromEmbedding(embedding),
);
Bus.instance.fire(EmbeddingUpdatedEvent());
}
Future<void> putMany(List<ClipEmbedding> embeddings) async {
final db = await _database;
final inputs = embeddings.map((e) => _getRowFromEmbedding(e)).toList();
await db.executeBatch(
'INSERT OR REPLACE INTO $tableName ($columnFileID, $columnEmbedding, $columnVersion) values(?, ?, ?)',
inputs,
);
Bus.instance.fire(EmbeddingUpdatedEvent());
}
Future<void> deleteEmbeddings(List<int> fileIDs) async {
final db = await _database;
await db.execute(
'DELETE FROM $tableName WHERE $columnFileID IN (${fileIDs.join(", ")})',
);
Bus.instance.fire(EmbeddingUpdatedEvent());
}
Future<void> deleteAll() async {
final db = await _database;
await db.execute('DELETE FROM $tableName');
Bus.instance.fire(EmbeddingUpdatedEvent());
}
List<ClipEmbedding> _convertToEmbeddings(List<Map<String, dynamic>> results) {
final List<ClipEmbedding> embeddings = [];
for (final result in results) {
final embedding = _getEmbeddingFromRow(result);
if (embedding.isEmpty) continue;
embeddings.add(embedding);
}
return embeddings;
}
ClipEmbedding _getEmbeddingFromRow(Map<String, dynamic> row) {
final fileID = row[columnFileID];
final bytes = row[columnEmbedding] as Uint8List;
final version = row[columnVersion] as int;
final list = Float32List.view(bytes.buffer);
return ClipEmbedding(fileID: fileID, embedding: list, version: version);
}
List<Object?> _getRowFromEmbedding(ClipEmbedding embedding) {
return [
embedding.fileID,
Float32List.fromList(embedding.embedding).buffer.asUint8List(),
embedding.version,
];
}
Future<void> _clearDeprecatedStores(Directory dir) async {
final deprecatedObjectBox = Directory(dir.path + "/object-box-store");
if (await deprecatedObjectBox.exists()) {
await deprecatedObjectBox.delete(recursive: true);
}
final deprecatedIsar = File(dir.path + "/default.isar");
if (await deprecatedIsar.exists()) {
await deprecatedIsar.delete();
}
}
}

226
mobile/lib/face/db.dart → mobile/lib/db/ml/db.dart
View File

@@ -6,10 +6,10 @@ import "package:flutter/foundation.dart";
import 'package:logging/logging.dart';
import 'package:path/path.dart' show join;
import 'package:path_provider/path_provider.dart';
import 'package:photos/db/ml/db_fields.dart';
import "package:photos/db/ml/db_model_mappers.dart";
import "package:photos/extensions/stop_watch.dart";
import 'package:photos/face/db_fields.dart';
import "package:photos/face/db_model_mappers.dart";
import "package:photos/face/model/face.dart";
import "package:photos/models/ml/face/face.dart";
import "package:photos/models/ml/ml_versions.dart";
import "package:photos/services/machine_learning/face_ml/face_clustering/face_db_info_for_clustering.dart";
import 'package:photos/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart';
@@ -28,7 +28,8 @@ import 'package:sqlite_async/sqlite_async.dart';
class FaceMLDataDB {
static final Logger _logger = Logger("FaceMLDataDB");
static const _databaseName = "ente.face_ml_db.db";
static const _databaseName = "ente.face_ml_db_v3.db";
// static const _databaseVersion = 1;
FaceMLDataDB._privateConstructor();
@@ -42,6 +43,7 @@ class FaceMLDataDB {
createClusterSummaryTable,
createNotPersonFeedbackTable,
fcClusterIDIndex,
createClipEmbeddingsTable,
];
// only have a single app-wide reference to the database
@@ -111,9 +113,9 @@ class FaceMLDataDB {
const String sql = '''
INSERT INTO $facesTable (
$fileIDColumn, $faceIDColumn, $faceDetectionColumn, $faceEmbeddingBlob, $faceScore, $faceBlur, $isSideways, $imageHeight, $imageWidth, $mlVersionColumn
$fileIDColumn, $faceIDColumn, $faceDetectionColumn, $embeddingColumn, $faceScore, $faceBlur, $isSideways, $imageHeight, $imageWidth, $mlVersionColumn
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT($fileIDColumn, $faceIDColumn) DO UPDATE SET $faceIDColumn = excluded.$faceIDColumn, $faceDetectionColumn = excluded.$faceDetectionColumn, $faceEmbeddingBlob = excluded.$faceEmbeddingBlob, $faceScore = excluded.$faceScore, $faceBlur = excluded.$faceBlur, $isSideways = excluded.$isSideways, $imageHeight = excluded.$imageHeight, $imageWidth = excluded.$imageWidth, $mlVersionColumn = excluded.$mlVersionColumn
ON CONFLICT($fileIDColumn, $faceIDColumn) DO UPDATE SET $faceIDColumn = excluded.$faceIDColumn, $faceDetectionColumn = excluded.$faceDetectionColumn, $embeddingColumn = excluded.$embeddingColumn, $faceScore = excluded.$faceScore, $faceBlur = excluded.$faceBlur, $isSideways = excluded.$isSideways, $imageHeight = excluded.$imageHeight, $imageWidth = excluded.$imageWidth, $mlVersionColumn = excluded.$mlVersionColumn
''';
final parameterSets = batch.map((face) {
final map = mapRemoteToFaceDB(face);
@@ -121,7 +123,7 @@ class FaceMLDataDB {
map[fileIDColumn],
map[faceIDColumn],
map[faceDetectionColumn],
map[faceEmbeddingBlob],
map[embeddingColumn],
map[faceScore],
map[faceBlur],
map[isSideways],
@@ -136,7 +138,7 @@ class FaceMLDataDB {
}
Future<void> updateFaceIdToClusterId(
Map<String, int> faceIDToClusterID,
Map<String, String> faceIDToClusterID,
) async {
final db = await instance.asyncDB;
const batchSize = 500;
@@ -185,43 +187,43 @@ class FaceMLDataDB {
return maps.first['count'] as int;
}
Future<Map<int, int>> clusterIdToFaceCount() async {
Future<Map<String, int>> clusterIdToFaceCount() async {
final db = await instance.asyncDB;
final List<Map<String, dynamic>> maps = await db.getAll(
'SELECT $fcClusterID, COUNT(*) as count FROM $faceClustersTable where $fcClusterID IS NOT NULL GROUP BY $fcClusterID ',
);
final Map<int, int> result = {};
final Map<String, int> result = {};
for (final map in maps) {
result[map[fcClusterID] as int] = map['count'] as int;
result[map[fcClusterID] as String] = map['count'] as int;
}
return result;
}
Future<Set<int>> getPersonIgnoredClusters(String personID) async {
Future<Set<String>> getPersonIgnoredClusters(String personID) async {
final db = await instance.asyncDB;
// find out clusterIds that are assigned to other persons using the clusters table
final List<Map<String, dynamic>> otherPersonMaps = await db.getAll(
'SELECT $clusterIDColumn FROM $clusterPersonTable WHERE $personIdColumn != ? AND $personIdColumn IS NOT NULL',
[personID],
);
final Set<int> ignoredClusterIDs =
otherPersonMaps.map((e) => e[clusterIDColumn] as int).toSet();
final Set<String> ignoredClusterIDs =
otherPersonMaps.map((e) => e[clusterIDColumn] as String).toSet();
final List<Map<String, dynamic>> rejectMaps = await db.getAll(
'SELECT $clusterIDColumn FROM $notPersonFeedback WHERE $personIdColumn = ?',
[personID],
);
final Set<int> rejectClusterIDs =
rejectMaps.map((e) => e[clusterIDColumn] as int).toSet();
final Set<String> rejectClusterIDs =
rejectMaps.map((e) => e[clusterIDColumn] as String).toSet();
return ignoredClusterIDs.union(rejectClusterIDs);
}
Future<Set<int>> getPersonClusterIDs(String personID) async {
Future<Set<String>> getPersonClusterIDs(String personID) async {
final db = await instance.asyncDB;
final List<Map<String, dynamic>> maps = await db.getAll(
'SELECT $clusterIDColumn FROM $clusterPersonTable WHERE $personIdColumn = ?',
[personID],
);
return maps.map((e) => e[clusterIDColumn] as int).toSet();
return maps.map((e) => e[clusterIDColumn] as String).toSet();
}
Future<void> clearTable() async {
@@ -232,40 +234,47 @@ class FaceMLDataDB {
await db.execute(deleteClusterPersonTable);
await db.execute(deleteClusterSummaryTable);
await db.execute(deleteNotPersonFeedbackTable);
await db.execute(deleteClipEmbeddingsTable);
}
Future<Iterable<Uint8List>> getFaceEmbeddingsForCluster(
int clusterID, {
String clusterID, {
int? limit,
}) async {
final db = await instance.asyncDB;
final List<Map<String, dynamic>> maps = await db.getAll(
'SELECT $faceEmbeddingBlob FROM $facesTable WHERE $faceIDColumn in (SELECT $fcFaceId from $faceClustersTable where $fcClusterID = ?) ${limit != null ? 'LIMIT $limit' : ''}',
'SELECT $embeddingColumn FROM $facesTable WHERE $faceIDColumn in (SELECT $fcFaceId from $faceClustersTable where $fcClusterID = ?) ${limit != null ? 'LIMIT $limit' : ''}',
[clusterID],
);
return maps.map((e) => e[faceEmbeddingBlob] as Uint8List);
return maps.map((e) => e[embeddingColumn] as Uint8List);
}
Future<Map<int, Iterable<Uint8List>>> getFaceEmbeddingsForClusters(
Iterable<int> clusterIDs, {
Future<Map<String, Iterable<Uint8List>>> getFaceEmbeddingsForClusters(
Iterable<String> clusterIDs, {
int? limit,
}) async {
final db = await instance.asyncDB;
final Map<int, List<Uint8List>> result = {};
final Map<String, List<Uint8List>> result = {};
final selectQuery = '''
SELECT fc.$fcClusterID, fe.$faceEmbeddingBlob
FROM $faceClustersTable fc
INNER JOIN $facesTable fe ON fc.$fcFaceId = fe.$faceIDColumn
WHERE fc.$fcClusterID IN (${clusterIDs.join(',')})
${limit != null ? 'LIMIT $limit' : ''}
''';
SELECT fc.$fcClusterID, fe.$embeddingColumn
FROM $faceClustersTable fc
INNER JOIN $facesTable fe ON fc.$fcFaceId = fe.$faceIDColumn
WHERE fc.$fcClusterID IN (${List.filled(clusterIDs.length, '?').join(',')})
${limit != null ? 'LIMIT ?' : ''}
''';
final List<Map<String, dynamic>> maps = await db.getAll(selectQuery);
final List<dynamic> selectQueryParams = [...clusterIDs];
if (limit != null) {
selectQueryParams.add(limit);
}
final List<Map<String, dynamic>> maps =
await db.getAll(selectQuery, selectQueryParams);
for (final map in maps) {
final clusterID = map[fcClusterID] as int;
final faceEmbedding = map[faceEmbeddingBlob] as Uint8List;
final clusterID = map[fcClusterID] as String;
final faceEmbedding = map[embeddingColumn] as Uint8List;
result.putIfAbsent(clusterID, () => <Uint8List>[]).add(faceEmbedding);
}
@@ -276,7 +285,7 @@ class FaceMLDataDB {
required int recentFileID,
String? personID,
String? avatarFaceId,
int? clusterID,
String? clusterID,
}) async {
// read person from db
final db = await instance.asyncDB;
@@ -299,11 +308,26 @@ class FaceMLDataDB {
[personID],
);
final clusterIDs =
clusterRows.map((e) => e[clusterIDColumn] as int).toList();
clusterRows.map((e) => e[clusterIDColumn] as String).toList();
// final List<Map<String, dynamic>> faceMaps = await db.getAll(
// 'SELECT * FROM $facesTable where '
// '$faceIDColumn in (SELECT $fcFaceId from $faceClustersTable where $fcClusterID IN (${clusterIDs.join(",")}))'
// 'AND $fileIDColumn in (${fileId.join(",")}) AND $faceScore > $kMinimumQualityFaceScore ORDER BY $faceScore DESC',
// );
final List<Map<String, dynamic>> faceMaps = await db.getAll(
'SELECT * FROM $facesTable where '
'$faceIDColumn in (SELECT $fcFaceId from $faceClustersTable where $fcClusterID IN (${clusterIDs.join(",")}))'
'AND $fileIDColumn in (${fileId.join(",")}) AND $faceScore > $kMinimumQualityFaceScore ORDER BY $faceScore DESC',
'''
SELECT * FROM $facesTable
WHERE $faceIDColumn IN (
SELECT $fcFaceId
FROM $faceClustersTable
WHERE $fcClusterID IN (${List.filled(clusterIDs.length, '?').join(',')})
)
AND $fileIDColumn IN (${List.filled(fileId.length, '?').join(',')})
AND $faceScore > ?
ORDER BY $faceScore DESC
''',
[...clusterIDs, ...fileId, kMinimumQualityFaceScore],
);
if (faceMaps.isNotEmpty) {
if (avatarFileId != null) {
@@ -359,23 +383,30 @@ class FaceMLDataDB {
return maps.map((e) => mapRowToFace(e)).toList();
}
Future<Map<int, Iterable<String>>> getClusterToFaceIDs(
Set<int> clusterIDs,
Future<Map<String, Iterable<String>>> getClusterToFaceIDs(
Set<String> clusterIDs,
) async {
final db = await instance.asyncDB;
final Map<int, List<String>> result = {};
final Map<String, List<String>> result = {};
final List<Map<String, dynamic>> maps = await db.getAll(
'SELECT $fcClusterID, $fcFaceId FROM $faceClustersTable WHERE $fcClusterID IN (${clusterIDs.join(",")})',
'''
SELECT $fcClusterID, $fcFaceId
FROM $faceClustersTable
WHERE $fcClusterID IN (${List.filled(clusterIDs.length, '?').join(',')})
''',
[...clusterIDs],
);
for (final map in maps) {
final clusterID = map[fcClusterID] as int;
final clusterID = map[fcClusterID] as String;
final faceID = map[fcFaceId] as String;
result.putIfAbsent(clusterID, () => <String>[]).add(faceID);
}
return result;
}
Future<int?> getClusterIDForFaceID(String faceID) async {
Future<String?> getClusterIDForFaceID(String faceID) async {
final db = await instance.asyncDB;
final List<Map<String, dynamic>> maps = await db.getAll(
'SELECT $fcClusterID FROM $faceClustersTable WHERE $fcFaceId = ?',
@@ -384,24 +415,24 @@ class FaceMLDataDB {
if (maps.isEmpty) {
return null;
}
return maps.first[fcClusterID] as int;
return maps.first[fcClusterID] as String;
}
Future<Map<int, Iterable<String>>> getAllClusterIdToFaceIDs() async {
Future<Map<String, Iterable<String>>> getAllClusterIdToFaceIDs() async {
final db = await instance.asyncDB;
final Map<int, List<String>> result = {};
final Map<String, List<String>> result = {};
final List<Map<String, dynamic>> maps = await db.getAll(
'SELECT $fcClusterID, $fcFaceId FROM $faceClustersTable',
);
for (final map in maps) {
final clusterID = map[fcClusterID] as int;
final clusterID = map[fcClusterID] as String;
final faceID = map[fcFaceId] as String;
result.putIfAbsent(clusterID, () => <String>[]).add(faceID);
}
return result;
}
Future<Iterable<String>> getFaceIDsForCluster(int clusterID) async {
Future<Iterable<String>> getFaceIDsForCluster(String clusterID) async {
final db = await instance.asyncDB;
final List<Map<String, dynamic>> maps = await db.getAll(
'SELECT $fcFaceId FROM $faceClustersTable '
@@ -412,17 +443,17 @@ class FaceMLDataDB {
}
// Get Map of personID to Map of clusterID to faceIDs
Future<Map<String, Map<int, Set<String>>>>
Future<Map<String, Map<String, Set<String>>>>
getPersonToClusterIdToFaceIds() async {
final db = await instance.asyncDB;
final List<Map<String, dynamic>> maps = await db.getAll(
'SELECT $personIdColumn, $faceClustersTable.$fcClusterID, $fcFaceId FROM $clusterPersonTable '
'LEFT JOIN $faceClustersTable ON $clusterPersonTable.$clusterIDColumn = $faceClustersTable.$fcClusterID',
);
final Map<String, Map<int, Set<String>>> result = {};
final Map<String, Map<String, Set<String>>> result = {};
for (final map in maps) {
final personID = map[personIdColumn] as String;
final clusterID = map[fcClusterID] as int;
final clusterID = map[fcClusterID] as String;
final faceID = map[fcFaceId] as String;
result
.putIfAbsent(personID, () => {})
@@ -443,7 +474,7 @@ class FaceMLDataDB {
return faceIdsResult.map((e) => e[fcFaceId] as String).toSet();
}
Future<Iterable<double>> getBlurValuesForCluster(int clusterID) async {
Future<Iterable<double>> getBlurValuesForCluster(String clusterID) async {
final db = await instance.asyncDB;
const String query = '''
SELECT $facesTable.$faceBlur
@@ -463,29 +494,29 @@ class FaceMLDataDB {
return maps.map((e) => e[faceBlur] as double).toSet();
}
Future<Map<String, int?>> getFaceIdsToClusterIds(
Future<Map<String, String?>> getFaceIdsToClusterIds(
Iterable<String> faceIds,
) async {
final db = await instance.asyncDB;
final List<Map<String, dynamic>> maps = await db.getAll(
'SELECT $fcFaceId, $fcClusterID FROM $faceClustersTable where $fcFaceId IN (${faceIds.map((id) => "'$id'").join(",")})',
);
final Map<String, int?> result = {};
final Map<String, String?> result = {};
for (final map in maps) {
result[map[fcFaceId] as String] = map[fcClusterID] as int?;
result[map[fcFaceId] as String] = map[fcClusterID] as String?;
}
return result;
}
Future<Map<int, Set<int>>> getFileIdToClusterIds() async {
final Map<int, Set<int>> result = {};
Future<Map<int, Set<String>>> getFileIdToClusterIds() async {
final Map<int, Set<String>> result = {};
final db = await instance.asyncDB;
final List<Map<String, dynamic>> maps = await db.getAll(
'SELECT $fcClusterID, $fcFaceId FROM $faceClustersTable',
);
for (final map in maps) {
final clusterID = map[fcClusterID] as int;
final clusterID = map[fcClusterID] as String;
final faceID = map[fcFaceId] as String;
final fileID = getFileIdFromFaceId(faceID);
result[fileID] = (result[fileID] ?? {})..add(clusterID);
@@ -494,7 +525,7 @@ class FaceMLDataDB {
}
Future<void> forceUpdateClusterIds(
Map<String, int> faceIDToClusterID,
Map<String, String> faceIDToClusterID,
) async {
final db = await instance.asyncDB;
@@ -541,7 +572,7 @@ class FaceMLDataDB {
while (true) {
// Query a batch of rows
final List<Map<String, dynamic>> maps = await db.getAll(
'SELECT $faceIDColumn, $faceEmbeddingBlob, $faceScore, $faceBlur, $isSideways FROM $facesTable'
'SELECT $faceIDColumn, $embeddingColumn, $faceScore, $faceBlur, $isSideways FROM $facesTable'
' WHERE $faceScore > $minScore AND $faceBlur > $minClarity'
' ORDER BY $faceIDColumn'
' DESC LIMIT $batchSize OFFSET $offset',
@@ -560,7 +591,7 @@ class FaceMLDataDB {
final faceInfo = FaceDbInfoForClustering(
faceID: faceID,
clusterId: faceIdToClusterId[faceID],
embeddingBytes: map[faceEmbeddingBlob] as Uint8List,
embeddingBytes: map[embeddingColumn] as Uint8List,
faceScore: map[faceScore] as double,
blurValue: map[faceBlur] as double,
isSideways: (map[isSideways] as int) == 1,
@@ -594,7 +625,7 @@ class FaceMLDataDB {
while (true) {
// Query a batch of rows
final String query = '''
SELECT $faceIDColumn, $faceEmbeddingBlob
SELECT $faceIDColumn, $embeddingColumn
FROM $facesTable
WHERE $faceIDColumn IN (${faceIDs.map((id) => "'$id'").join(",")})
ORDER BY $faceIDColumn DESC
@@ -607,7 +638,7 @@ class FaceMLDataDB {
}
for (final map in maps) {
final faceID = map[faceIDColumn] as String;
result[faceID] = map[faceEmbeddingBlob] as Uint8List;
result[faceID] = map[embeddingColumn] as Uint8List;
}
if (result.length > 10000) {
break;
@@ -681,7 +712,7 @@ class FaceMLDataDB {
Future<void> assignClusterToPerson({
required String personID,
required int clusterID,
required String clusterID,
}) async {
final db = await instance.asyncDB;
@@ -692,7 +723,7 @@ class FaceMLDataDB {
}
Future<void> bulkAssignClusterToPersonID(
Map<int, String> clusterToPersonID,
Map<String, String> clusterToPersonID,
) async {
final db = await instance.asyncDB;
@@ -706,7 +737,7 @@ class FaceMLDataDB {
Future<void> captureNotPersonFeedback({
required String personID,
required int clusterID,
required String clusterID,
}) async {
final db = await instance.asyncDB;
@@ -717,7 +748,7 @@ class FaceMLDataDB {
}
Future<void> bulkCaptureNotPersonFeedback(
Map<int, String> clusterToPersonID,
Map<String, String> clusterToPersonID,
) async {
final db = await instance.asyncDB;
@@ -732,7 +763,7 @@ class FaceMLDataDB {
Future<void> removeNotPersonFeedback({
required String personID,
required int clusterID,
required String clusterID,
}) async {
final db = await instance.asyncDB;
@@ -744,7 +775,7 @@ class FaceMLDataDB {
Future<void> removeClusterToPerson({
required String personID,
required int clusterID,
required String clusterID,
}) async {
final db = await instance.asyncDB;
@@ -755,7 +786,7 @@ class FaceMLDataDB {
}
// for a given personID, return a map of clusterID to fileIDs using join query
Future<Map<int, Set<int>>> getFileIdToClusterIDSet(String personID) {
Future<Map<int, Set<String>>> getFileIdToClusterIDSet(String personID) {
final db = instance.asyncDB;
return db.then((db) async {
final List<Map<String, dynamic>> maps = await db.getAll(
@@ -765,9 +796,9 @@ class FaceMLDataDB {
'WHERE $clusterPersonTable.$personIdColumn = ?',
[personID],
);
final Map<int, Set<int>> result = {};
final Map<int, Set<String>> result = {};
for (final map in maps) {
final clusterID = map[clusterIDColumn] as int;
final clusterID = map[clusterIDColumn] as String;
final String faceID = map[fcFaceId] as String;
final fileID = getFileIdFromFaceId(faceID);
result[fileID] = (result[fileID] ?? {})..add(clusterID);
@@ -776,18 +807,22 @@ class FaceMLDataDB {
});
}
Future<Map<int, Set<int>>> getFileIdToClusterIDSetForCluster(
Set<int> clusterIDs,
Future<Map<int, Set<String>>> getFileIdToClusterIDSetForCluster(
Set<String> clusterIDs,
) {
final db = instance.asyncDB;
return db.then((db) async {
final List<Map<String, dynamic>> maps = await db.getAll(
'SELECT $fcClusterID, $fcFaceId FROM $faceClustersTable '
'WHERE $fcClusterID IN (${clusterIDs.join(",")})',
'''
SELECT $fcClusterID, $fcFaceId
FROM $faceClustersTable
WHERE $fcClusterID IN (${List.filled(clusterIDs.length, '?').join(',')})
''',
[...clusterIDs],
);
final Map<int, Set<int>> result = {};
final Map<int, Set<String>> result = {};
for (final map in maps) {
final clusterID = map[fcClusterID] as int;
final clusterID = map[fcClusterID] as String;
final faceID = map[fcFaceId] as String;
final fileID = getFileIdFromFaceId(faceID);
result[fileID] = (result[fileID] ?? {})..add(clusterID);
@@ -796,7 +831,9 @@ class FaceMLDataDB {
});
}
Future<void> clusterSummaryUpdate(Map<int, (Uint8List, int)> summary) async {
Future<void> clusterSummaryUpdate(
Map<String, (Uint8List, int)> summary,
) async {
final db = await instance.asyncDB;
const String sql = '''
@@ -810,7 +847,7 @@ class FaceMLDataDB {
batchCounter = 0;
parameterSets.clear();
}
final int clusterID = entry.key;
final String clusterID = entry.key;
final int count = entry.value.$2;
final Uint8List avg = entry.value.$1;
parameterSets.add([clusterID, avg, count]);
@@ -819,7 +856,7 @@ class FaceMLDataDB {
await db.executeBatch(sql, parameterSets);
}
Future<void> deleteClusterSummary(int clusterID) async {
Future<void> deleteClusterSummary(String clusterID) async {
final db = await instance.asyncDB;
const String sqlDelete =
'DELETE FROM $clusterSummaryTable WHERE $clusterIDColumn = ?';
@@ -827,16 +864,16 @@ class FaceMLDataDB {
}
/// Returns a map of clusterID to (avg embedding, count)
Future<Map<int, (Uint8List, int)>> getAllClusterSummary([
Future<Map<String, (Uint8List, int)>> getAllClusterSummary([
int? minClusterSize,
]) async {
final db = await instance.asyncDB;
final Map<int, (Uint8List, int)> result = {};
final Map<String, (Uint8List, int)> result = {};
final rows = await db.getAll(
'SELECT * FROM $clusterSummaryTable${minClusterSize != null ? ' WHERE $countColumn >= $minClusterSize' : ''}',
);
for (final r in rows) {
final id = r[clusterIDColumn] as int;
final id = r[clusterIDColumn] as String;
final avg = r[avgColumn] as Uint8List;
final count = r[countColumn] as int;
result[id] = (avg, count);
@@ -844,16 +881,19 @@ class FaceMLDataDB {
return result;
}
Future<Map<int, (Uint8List, int)>> getClusterToClusterSummary(
Iterable<int> clusterIDs,
Future<Map<String, (Uint8List, int)>> getClusterToClusterSummary(
Iterable<String> clusterIDs,
) async {
final db = await instance.asyncDB;
final Map<int, (Uint8List, int)> result = {};
final Map<String, (Uint8List, int)> result = {};
final rows = await db.getAll(
'SELECT * FROM $clusterSummaryTable WHERE $clusterIDColumn IN (${clusterIDs.join(",")})',
'SELECT * FROM $clusterSummaryTable WHERE $clusterIDColumn IN (${List.filled(clusterIDs.length, '?').join(',')})',
[...clusterIDs],
);
for (final r in rows) {
final id = r[clusterIDColumn] as int;
final id = r[clusterIDColumn] as String;
final avg = r[avgColumn] as Uint8List;
final count = r[countColumn] as int;
result[id] = (avg, count);
@@ -861,14 +901,14 @@ class FaceMLDataDB {
return result;
}
Future<Map<int, String>> getClusterIDToPersonID() async {
Future<Map<String, String>> getClusterIDToPersonID() async {
final db = await instance.asyncDB;
final List<Map<String, dynamic>> maps = await db.getAll(
'SELECT $personIdColumn, $clusterIDColumn FROM $clusterPersonTable',
);
final Map<int, String> result = {};
final Map<String, String> result = {};
for (final map in maps) {
result[map[clusterIDColumn] as int] = map[personIdColumn] as String;
result[map[clusterIDColumn] as String] = map[personIdColumn] as String;
}
return result;
}

26
mobile/lib/face/db_fields.dart → mobile/lib/db/ml/db_fields.dart
View File

@@ -5,7 +5,7 @@ const facesTable = 'faces';
const fileIDColumn = 'file_id';
const faceIDColumn = 'face_id';
const faceDetectionColumn = 'detection';
const faceEmbeddingBlob = 'eBlob';
const embeddingColumn = 'embedding';
const faceScore = 'score';
const faceBlur = 'blur';
const isSideways = 'is_sideways';
@@ -18,7 +18,7 @@ const createFacesTable = '''CREATE TABLE IF NOT EXISTS $facesTable (
$fileIDColumn INTEGER NOT NULL,
$faceIDColumn TEXT NOT NULL UNIQUE,
$faceDetectionColumn TEXT NOT NULL,
$faceEmbeddingBlob BLOB NOT NULL,
$embeddingColumn BLOB NOT NULL,
$faceScore REAL NOT NULL,
$faceBlur REAL NOT NULL DEFAULT $kLapacianDefault,
$isSideways INTEGER NOT NULL DEFAULT 0,
@@ -41,7 +41,7 @@ const fcFaceId = 'face_id';
const createFaceClustersTable = '''
CREATE TABLE IF NOT EXISTS $faceClustersTable (
$fcFaceId TEXT NOT NULL,
$fcClusterID INTEGER NOT NULL,
$fcClusterID TEXT NOT NULL,
PRIMARY KEY($fcFaceId)
);
''';
@@ -59,7 +59,7 @@ const clusterIDColumn = 'cluster_id';
const createClusterPersonTable = '''
CREATE TABLE IF NOT EXISTS $clusterPersonTable (
$personIdColumn TEXT NOT NULL,
$clusterIDColumn INTEGER NOT NULL,
$clusterIDColumn TEXT NOT NULL,
PRIMARY KEY($personIdColumn, $clusterIDColumn)
);
''';
@@ -72,7 +72,7 @@ const avgColumn = 'avg';
const countColumn = 'count';
const createClusterSummaryTable = '''
CREATE TABLE IF NOT EXISTS $clusterSummaryTable (
$clusterIDColumn INTEGER NOT NULL,
$clusterIDColumn TEXT NOT NULL,
$avgColumn BLOB NOT NULL,
$countColumn INTEGER NOT NULL,
PRIMARY KEY($clusterIDColumn)
@@ -89,9 +89,23 @@ const notPersonFeedback = 'not_person_feedback';
const createNotPersonFeedbackTable = '''
CREATE TABLE IF NOT EXISTS $notPersonFeedback (
$personIdColumn TEXT NOT NULL,
$clusterIDColumn INTEGER NOT NULL,
$clusterIDColumn TEXT NOT NULL,
PRIMARY KEY($personIdColumn, $clusterIDColumn)
);
''';
const deleteNotPersonFeedbackTable = 'DELETE FROM $notPersonFeedback';
// End Clusters Table Fields & Schema Queries
// ## CLIP EMBEDDINGS TABLE
const clipTable = 'clip';
const createClipEmbeddingsTable = '''
CREATE TABLE IF NOT EXISTS $clipTable (
$fileIDColumn INTEGER NOT NULL,
$embeddingColumn BLOB NOT NULL,
$mlVersionColumn INTEGER NOT NULL,
PRIMARY KEY ($fileIDColumn)
);
''';
const deleteClipEmbeddingsTable = 'DELETE FROM $clipTable';

10
mobile/lib/face/db_model_mappers.dart → mobile/lib/db/ml/db_model_mappers.dart
View File

@@ -1,9 +1,9 @@
import "dart:convert";
import 'package:photos/face/db_fields.dart';
import "package:photos/face/model/detection.dart";
import "package:photos/face/model/face.dart";
import 'package:photos/db/ml/db_fields.dart';
import "package:photos/generated/protos/ente/common/vector.pb.dart";
import "package:photos/models/ml/face/detection.dart";
import "package:photos/models/ml/face/face.dart";
import "package:photos/models/ml/ml_versions.dart";
Map<String, dynamic> mapRemoteToFaceDB(Face face) {
@@ -11,7 +11,7 @@ Map<String, dynamic> mapRemoteToFaceDB(Face face) {
faceIDColumn: face.faceID,
fileIDColumn: face.fileID,
faceDetectionColumn: json.encode(face.detection.toJson()),
faceEmbeddingBlob: EVector(
embeddingColumn: EVector(
values: face.embedding,
).writeToBuffer(),
faceScore: face.score,
@@ -27,7 +27,7 @@ Face mapRowToFace(Map<String, dynamic> row) {
return Face(
row[faceIDColumn] as String,
row[fileIDColumn] as int,
EVector.fromBuffer(row[faceEmbeddingBlob] as List<int>).values,
EVector.fromBuffer(row[embeddingColumn] as List<int>).values,
row[faceScore] as double,
Detection.fromJson(json.decode(row[faceDetectionColumn] as String)),
row[faceBlur] as double,

108
mobile/lib/db/ml/embeddings_db.dart Normal file
View File

@@ -0,0 +1,108 @@
import "dart:io";
import "dart:typed_data";
import "package:photos/core/event_bus.dart";
import "package:photos/db/ml/db.dart";
import "package:photos/db/ml/db_fields.dart";
import "package:photos/events/embedding_updated_event.dart";
import "package:photos/models/ml/clip.dart";
extension EmbeddingsDB on FaceMLDataDB {
static const databaseName = "ente.embeddings.db";
Future<List<ClipEmbedding>> getAll() async {
final db = await FaceMLDataDB.instance.asyncDB;
final results = await db.getAll('SELECT * FROM $clipTable');
return _convertToEmbeddings(results);
}
// Get indexed FileIDs
Future<Map<int, int>> clipIndexedFileWithVersion() async {
final db = await FaceMLDataDB.instance.asyncDB;
final maps = await db
.getAll('SELECT $fileIDColumn , $mlVersionColumn FROM $clipTable');
final Map<int, int> result = {};
for (final map in maps) {
result[map[fileIDColumn] as int] = map[mlVersionColumn] as int;
}
return result;
}
Future<int> getClipIndexedFileCount() async {
final db = await FaceMLDataDB.instance.asyncDB;
const String query =
'SELECT COUNT(DISTINCT $fileIDColumn) as count FROM $clipTable';
final List<Map<String, dynamic>> maps = await db.getAll(query);
return maps.first['count'] as int;
}
Future<void> put(ClipEmbedding embedding) async {
final db = await FaceMLDataDB.instance.asyncDB;
await db.execute(
'INSERT OR REPLACE INTO $clipTable ($fileIDColumn, $embeddingColumn, $mlVersionColumn) VALUES (?, ?, ?)',
_getRowFromEmbedding(embedding),
);
Bus.instance.fire(EmbeddingUpdatedEvent());
}
Future<void> putMany(List<ClipEmbedding> embeddings) async {
final db = await FaceMLDataDB.instance.asyncDB;
final inputs = embeddings.map((e) => _getRowFromEmbedding(e)).toList();
await db.executeBatch(
'INSERT OR REPLACE INTO $clipTable ($fileIDColumn, $embeddingColumn, $mlVersionColumn) values(?, ?, ?)',
inputs,
);
Bus.instance.fire(EmbeddingUpdatedEvent());
}
Future<void> deleteEmbeddings(List<int> fileIDs) async {
final db = await FaceMLDataDB.instance.asyncDB;
await db.execute(
'DELETE FROM $clipTable WHERE $fileIDColumn IN (${fileIDs.join(", ")})',
);
Bus.instance.fire(EmbeddingUpdatedEvent());
}
Future<void> deleteClipIndexes() async {
final db = await FaceMLDataDB.instance.asyncDB;
await db.execute('DELETE FROM $clipTable');
Bus.instance.fire(EmbeddingUpdatedEvent());
}
List<ClipEmbedding> _convertToEmbeddings(List<Map<String, dynamic>> results) {
final List<ClipEmbedding> embeddings = [];
for (final result in results) {
final embedding = _getEmbeddingFromRow(result);
if (embedding.isEmpty) continue;
embeddings.add(embedding);
}
return embeddings;
}
ClipEmbedding _getEmbeddingFromRow(Map<String, dynamic> row) {
final fileID = row[fileIDColumn] as int;
final bytes = row[embeddingColumn] as Uint8List;
final version = row[mlVersionColumn] as int;
final list = Float32List.view(bytes.buffer);
return ClipEmbedding(fileID: fileID, embedding: list, version: version);
}
List<Object?> _getRowFromEmbedding(ClipEmbedding embedding) {
return [
embedding.fileID,
Float32List.fromList(embedding.embedding).buffer.asUint8List(),
embedding.version,
];
}
Future<void> _clearDeprecatedStores(Directory dir) async {
final deprecatedObjectBox = Directory(dir.path + "/object-box-store");
if (await deprecatedObjectBox.exists()) {
await deprecatedObjectBox.delete(recursive: true);
}
final deprecatedIsar = File(dir.path + "/default.isar");
if (await deprecatedIsar.exists()) {
await deprecatedIsar.delete();
}
}
}

2
mobile/lib/main.dart
View File

@@ -18,9 +18,9 @@ import 'package:photos/core/constants.dart';
import 'package:photos/core/error-reporting/super_logging.dart';
import 'package:photos/core/errors.dart';
import 'package:photos/core/network/network.dart';
import "package:photos/db/ml/db.dart";
import 'package:photos/db/upload_locks_db.dart';
import 'package:photos/ente_theme_data.dart';
import "package:photos/face/db.dart";
import "package:photos/l10n/l10n.dart";
import "package:photos/service_locator.dart";
import 'package:photos/services/app_lifecycle_service.dart';

14
mobile/lib/models/api/entity/type.dart
View File

@@ -3,6 +3,7 @@ import "package:flutter/foundation.dart";
enum EntityType {
location,
person,
cgroup,
unknown,
}
@@ -12,18 +13,29 @@ EntityType typeFromString(String type) {
return EntityType.location;
case "person":
return EntityType.location;
case "cgroup":
return EntityType.cgroup;
}
debugPrint("unexpected collection type $type");
debugPrint("unexpected entity type $type");
return EntityType.unknown;
}
extension EntityTypeExtn on EntityType {
bool isZipped() {
if (this == EntityType.location || this == EntityType.person) {
return false;
}
return true;
}
String typeToString() {
switch (this) {
case EntityType.location:
return "location";
case EntityType.person:
return "person";
case EntityType.cgroup:
return "cgroup";
case EntityType.unknown:
return "unknown";
}

0
mobile/lib/models/embedding.dart → mobile/lib/models/ml/clip.dart
View File

0
mobile/lib/face/model/box.dart → mobile/lib/models/ml/face/box.dart
View File

4
mobile/lib/face/model/detection.dart → mobile/lib/models/ml/face/detection.dart
View File

@@ -1,7 +1,7 @@
import "dart:math" show min, max;
import "package:photos/face/model/box.dart";
import "package:photos/face/model/landmark.dart";
import "package:photos/models/ml/face/box.dart";
import "package:photos/models/ml/face/landmark.dart";
import "package:photos/services/machine_learning/face_ml/face_detection/detection.dart";
/// Stores the face detection data, notably the bounding box and landmarks.

0
mobile/lib/face/model/dimension.dart → mobile/lib/models/ml/face/dimension.dart
View File

2
mobile/lib/face/model/face.dart → mobile/lib/models/ml/face/face.dart
View File

@@ -1,4 +1,4 @@
import "package:photos/face/model/detection.dart";
import "package:photos/models/ml/face/detection.dart";
import 'package:photos/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart';
import "package:photos/services/machine_learning/ml_result.dart";

0
mobile/lib/face/model/landmark.dart → mobile/lib/models/ml/face/landmark.dart
View File

16
mobile/lib/face/model/person.dart → mobile/lib/models/ml/face/person.dart
View File

@@ -24,7 +24,7 @@ class PersonEntity {
}
class ClusterInfo {
final int id;
final String id;
final Set<String> faces;
ClusterInfo({
required this.id,
@@ -40,7 +40,7 @@ class ClusterInfo {
// from Json
factory ClusterInfo.fromJson(Map<String, dynamic> json) {
return ClusterInfo(
id: json['id'] as int,
id: json['id'] as String,
faces: (json['faces'] as List<dynamic>).map((e) => e as String).toSet(),
);
}
@@ -49,12 +49,12 @@ class ClusterInfo {
class PersonData {
final String name;
final bool isHidden;
String? avatarFaceId;
String? avatarFaceID;
List<ClusterInfo>? assigned = List<ClusterInfo>.empty();
List<ClusterInfo>? rejected = List<ClusterInfo>.empty();
final String? birthDate;
bool hasAvatar() => avatarFaceId != null;
bool hasAvatar() => avatarFaceID != null;
bool get isIgnored =>
(name.isEmpty || name == '(hidden)' || name == '(ignored)');
@@ -63,7 +63,7 @@ class PersonData {
required this.name,
this.assigned,
this.rejected,
this.avatarFaceId,
this.avatarFaceID,
this.isHidden = false,
this.birthDate,
});
@@ -79,7 +79,7 @@ class PersonData {
return PersonData(
name: name ?? this.name,
assigned: assigned ?? this.assigned,
avatarFaceId: avatarFaceId ?? this.avatarFaceId,
avatarFaceID: avatarFaceId ?? this.avatarFaceID,
isHidden: isHidden ?? this.isHidden,
birthDate: birthDate ?? this.birthDate,
);
@@ -109,7 +109,7 @@ class PersonData {
'name': name,
'assigned': assigned?.map((e) => e.toJson()).toList(),
'rejected': rejected?.map((e) => e.toJson()).toList(),
'avatarFaceId': avatarFaceId,
'avatarFaceID': avatarFaceID,
'isHidden': isHidden,
'birthDate': birthDate,
};
@@ -131,7 +131,7 @@ class PersonData {
name: json['name'] as String,
assigned: assigned,
rejected: rejected,
avatarFaceId: json['avatarFaceId'] as String?,
avatarFaceID: json['avatarFaceId'] as String?,
isHidden: json['isHidden'] as bool? ?? false,
birthDate: json['birthDate'] as String?,
);

25
mobile/lib/models/nanoids/cluster_id.dart Normal file
View File

@@ -0,0 +1,25 @@
import "package:flutter/foundation.dart";
import 'package:nanoid/nanoid.dart';
const enteWhiteListedAlphabet =
'0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz';
const clusterIDLength = 22;
class ClusterID {
static String generate() {
return "cluster_${customAlphabet(enteWhiteListedAlphabet, clusterIDLength)}";
}
// Validation method
static bool isValidClusterID(String value) {
if (value.length != (clusterIDLength + 8)) {
debugPrint("ClusterID length is not ${clusterIDLength + 8}: $value");
return false;
}
if (value.startsWith("cluster_")) {
debugPrint("ClusterID doesn't start with _cluster: $value");
return false;
}
return true;
}
}

47
mobile/lib/services/entity_service.dart
View File

@@ -14,6 +14,7 @@ import "package:photos/models/api/entity/key.dart";
import "package:photos/models/api/entity/type.dart";
import "package:photos/models/local_entity_data.dart";
import "package:photos/utils/crypto_util.dart";
import "package:photos/utils/gzip.dart";
import 'package:shared_preferences/shared_preferences.dart';
class EntityService {
@@ -56,17 +57,23 @@ class EntityService {
Future<LocalEntityData> addOrUpdate(
EntityType type,
String plainText, {
Map<String, dynamic> jsonMap, {
String? id,
}) async {
final String plainText = jsonEncode(jsonMap);
final key = await getOrCreateEntityKey(type);
final encryptedKeyData = await CryptoUtil.encryptChaCha(
utf8.encode(plainText),
key,
);
final String encryptedData =
CryptoUtil.bin2base64(encryptedKeyData.encryptedData!);
final String header = CryptoUtil.bin2base64(encryptedKeyData.header!);
late String encryptedData, header;
if (type.isZipped()) {
final ChaChaEncryptionResult result =
await gzipAndEncryptJson(jsonMap, key);
encryptedData = result.encData;
header = result.header;
} else {
final encryptedKeyData =
await CryptoUtil.encryptChaCha(utf8.encode(plainText), key);
encryptedData = CryptoUtil.bin2base64(encryptedKeyData.encryptedData!);
header = CryptoUtil.bin2base64(encryptedKeyData.header!);
}
debugPrint(
" ${id == null ? 'Adding' : 'Updating'} entity of type: " +
type.typeToString(),
@@ -94,7 +101,7 @@ class EntityService {
Future<void> syncEntities() async {
try {
await _remoteToLocalSync(EntityType.location);
await _remoteToLocalSync(EntityType.person);
await _remoteToLocalSync(EntityType.cgroup);
} catch (e) {
_logger.severe("Failed to sync entities", e);
}
@@ -127,12 +134,22 @@ class EntityService {
final List<LocalEntityData> entities = [];
for (EntityData e in result) {
try {
final decryptedValue = await CryptoUtil.decryptChaCha(
CryptoUtil.base642bin(e.encryptedData!),
entityKey,
CryptoUtil.base642bin(e.header!),
);
final String plainText = utf8.decode(decryptedValue);
late String plainText;
if (type.isZipped()) {
final jsonMap = await decryptAndUnzipJson(
entityKey,
encryptedData: e.encryptedData!,
header: e.header!,
);
plainText = jsonEncode(jsonMap);
} else {
final Uint8List decryptedValue = await CryptoUtil.decryptChaCha(
CryptoUtil.base642bin(e.encryptedData!),
entityKey,
CryptoUtil.base642bin(e.header!),
);
plainText = utf8.decode(decryptedValue);
}
entities.add(
LocalEntityData(
id: e.id,

2
mobile/lib/services/filedata/filedata_service.dart
View File

@@ -32,7 +32,7 @@ class FileDataService {
try {
final _ = await _dio.put(
"/files/data/",
"/files/data",
data: {
"fileID": file.uploadedFileID!,
"type": data.type.toJson(),

2
mobile/lib/services/filedata/model/file_data.dart
View File

@@ -1,4 +1,4 @@
import "package:photos/face/model/face.dart";
import "package:photos/models/ml/face/face.dart";
const _faceKey = 'face';
const _clipKey = 'clip';

4
mobile/lib/services/location_service.dart
View File

@@ -90,7 +90,7 @@ class LocationService {
centerPoint: centerPoint,
);
await EntityService.instance
.addOrUpdate(EntityType.location, json.encode(locationTag.toJson()));
.addOrUpdate(EntityType.location, locationTag.toJson());
Bus.instance.fire(LocationTagUpdatedEvent(LocTagEventType.add));
} catch (e, s) {
_logger.severe("Failed to add location tag", e, s);
@@ -179,7 +179,7 @@ class LocationService {
await EntityService.instance.addOrUpdate(
EntityType.location,
json.encode(updatedLoationTag.toJson()),
updatedLoationTag.toJson(),
id: locationTagEntity.id,
);
Bus.instance.fire(

64
mobile/lib/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart
View File

@@ -1,7 +1,6 @@
import "dart:async";
import "dart:developer";
import "dart:isolate";
import "dart:math" show max;
import "dart:typed_data" show Uint8List;
import "package:computer/computer.dart";
@@ -10,6 +9,7 @@ import "package:logging/logging.dart";
import "package:ml_linalg/dtype.dart";
import "package:ml_linalg/vector.dart";
import "package:photos/generated/protos/ente/common/vector.pb.dart";
import "package:photos/models/nanoids/cluster_id.dart";
import "package:photos/services/machine_learning/face_ml/face_clustering/face_db_info_for_clustering.dart";
import "package:photos/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart";
import "package:photos/services/machine_learning/ml_result.dart";
@@ -21,7 +21,7 @@ class FaceInfo {
final double? blurValue;
final bool? badFace;
final Vector? vEmbedding;
int? clusterId;
String? clusterId;
String? closestFaceId;
int? closestDist;
int? fileCreationTime;
@@ -39,9 +39,9 @@ class FaceInfo {
enum ClusterOperation { linearIncrementalClustering }
class ClusteringResult {
final Map<String, int> newFaceIdToCluster;
final Map<int, List<String>> newClusterIdToFaceIds;
final Map<int, (Uint8List, int)> newClusterSummaries;
final Map<String, String> newFaceIdToCluster;
final Map<String, List<String>> newClusterIdToFaceIds;
final Map<String, (Uint8List, int)> newClusterSummaries;
bool get isEmpty => newFaceIdToCluster.isEmpty;
@@ -210,7 +210,7 @@ class FaceClusteringService {
double conservativeDistanceThreshold = kConservativeDistanceThreshold,
bool useDynamicThreshold = true,
int? offset,
required Map<int, (Uint8List, int)> oldClusterSummaries,
required Map<String, (Uint8List, int)> oldClusterSummaries,
}) async {
if (input.isEmpty) {
_logger.warning(
@@ -417,7 +417,7 @@ ClusteringResult _runLinearClustering(Map args) {
final useDynamicThreshold = args['useDynamicThreshold'] as bool;
final offset = args['offset'] as int?;
final oldClusterSummaries =
args['oldClusterSummaries'] as Map<int, (Uint8List, int)>?;
args['oldClusterSummaries'] as Map<String, (Uint8List, int)>?;
log(
"[ClusterIsolate] ${DateTime.now()} Copied to isolate ${input.length} faces",
@@ -491,17 +491,17 @@ ClusteringResult _runLinearClustering(Map args) {
"[ClusterIsolate] ${DateTime.now()} Processing $totalFaces faces in total in this round ${offset != null ? "on top of ${offset + facesWithClusterID.length} earlier processed faces" : ""}",
);
// set current epoch time as clusterID
int clusterID = DateTime.now().microsecondsSinceEpoch;
String clusterID = ClusterID.generate();
if (facesWithClusterID.isEmpty) {
// assign a clusterID to the first face
sortedFaceInfos[0].clusterId = clusterID;
clusterID++;
clusterID = ClusterID.generate();
}
final stopwatchClustering = Stopwatch()..start();
for (int i = 1; i < totalFaces; i++) {
// Incremental clustering, so we can skip faces that already have a clusterId
if (sortedFaceInfos[i].clusterId != null) {
clusterID = max(clusterID, sortedFaceInfos[i].clusterId!);
// clusterID = max(clusterID, sortedFaceInfos[i].clusterId!);
continue;
}
@@ -539,25 +539,25 @@ ClusteringResult _runLinearClustering(Map args) {
log(
" [ClusterIsolate] [WARNING] ${DateTime.now()} Found new cluster $clusterID",
);
clusterID++;
clusterID = ClusterID.generate();
sortedFaceInfos[closestIdx].clusterId = clusterID;
}
sortedFaceInfos[i].clusterId = sortedFaceInfos[closestIdx].clusterId;
} else {
clusterID++;
clusterID = ClusterID.generate();
sortedFaceInfos[i].clusterId = clusterID;
}
}
// Finally, assign the new clusterId to the faces
final Map<String, int> newFaceIdToCluster = {};
final Map<String, String> newFaceIdToCluster = {};
final newClusteredFaceInfos = sortedFaceInfos.sublist(alreadyClusteredCount);
for (final faceInfo in newClusteredFaceInfos) {
newFaceIdToCluster[faceInfo.faceID] = faceInfo.clusterId!;
}
// Create a map of clusterId to faceIds
final Map<int, List<String>> clusterIdToFaceIds = {};
final Map<String, List<String>> clusterIdToFaceIds = {};
for (final entry in newFaceIdToCluster.entries) {
final clusterID = entry.value;
if (clusterIdToFaceIds.containsKey(clusterID)) {
@@ -599,7 +599,7 @@ ClusteringResult _runCompleteClustering(Map args) {
final distanceThreshold = args['distanceThreshold'] as double;
final mergeThreshold = args['mergeThreshold'] as double;
final oldClusterSummaries =
args['oldClusterSummaries'] as Map<int, (Uint8List, int)>?;
args['oldClusterSummaries'] as Map<String, (Uint8List, int)>?;
log(
"[CompleteClustering] ${DateTime.now()} Copied to isolate ${input.length} faces for clustering",
@@ -634,11 +634,10 @@ ClusteringResult _runCompleteClustering(Map args) {
"[CompleteClustering] ${DateTime.now()} Processing $totalFaces faces in one single round of complete clustering",
);
// set current epoch time as clusterID
int clusterID = DateTime.now().microsecondsSinceEpoch;
String clusterID = ClusterID.generate();
// Start actual clustering
final Map<String, int> newFaceIdToCluster = {};
final Map<String, String> newFaceIdToCluster = {};
final stopwatchClustering = Stopwatch()..start();
for (int i = 0; i < totalFaces; i++) {
if ((i + 1) % 250 == 0) {
@@ -659,18 +658,18 @@ ClusteringResult _runCompleteClustering(Map args) {
if (closestDistance < distanceThreshold) {
if (faceInfos[closestIdx].clusterId == null) {
clusterID++;
clusterID = ClusterID.generate();
faceInfos[closestIdx].clusterId = clusterID;
}
faceInfos[i].clusterId = faceInfos[closestIdx].clusterId!;
} else {
clusterID++;
clusterID = ClusterID.generate();
faceInfos[i].clusterId = clusterID;
}
}
// Now calculate the mean of the embeddings for each cluster
final Map<int, List<FaceInfo>> clusterIdToFaceInfos = {};
final Map<String, List<FaceInfo>> clusterIdToFaceInfos = {};
for (final faceInfo in faceInfos) {
if (clusterIdToFaceInfos.containsKey(faceInfo.clusterId)) {
clusterIdToFaceInfos[faceInfo.clusterId]!.add(faceInfo);
@@ -678,7 +677,7 @@ ClusteringResult _runCompleteClustering(Map args) {
clusterIdToFaceInfos[faceInfo.clusterId!] = [faceInfo];
}
}
final Map<int, (Vector, int)> clusterIdToMeanEmbeddingAndWeight = {};
final Map<String, (Vector, int)> clusterIdToMeanEmbeddingAndWeight = {};
for (final clusterId in clusterIdToFaceInfos.keys) {
final List<Vector> embeddings = clusterIdToFaceInfos[clusterId]!
.map((faceInfo) => faceInfo.vEmbedding!)
@@ -691,13 +690,14 @@ ClusteringResult _runCompleteClustering(Map args) {
}
// Now merge the clusters that are close to each other, based on mean embedding
final List<(int, int)> mergedClustersList = [];
final List<int> clusterIds = clusterIdToMeanEmbeddingAndWeight.keys.toList();
final List<(String, String)> mergedClustersList = [];
final List<String> clusterIds =
clusterIdToMeanEmbeddingAndWeight.keys.toList();
log(' [CompleteClustering] ${DateTime.now()} ${clusterIds.length} clusters found, now checking for merges');
while (true) {
if (clusterIds.length < 2) break;
double distance = double.infinity;
(int, int) clusterIDsToMerge = (-1, -1);
(String, String) clusterIDsToMerge = ('', '');
for (int i = 0; i < clusterIds.length; i++) {
for (int j = 0; j < clusterIds.length; j++) {
if (i == j) continue;
@@ -749,7 +749,7 @@ ClusteringResult _runCompleteClustering(Map args) {
newFaceIdToCluster[faceInfo.faceID] = faceInfo.clusterId!;
}
final Map<int, List<String>> clusterIdToFaceIds = {};
final Map<String, List<String>> clusterIdToFaceIds = {};
for (final entry in newFaceIdToCluster.entries) {
final clusterID = entry.value;
if (clusterIdToFaceIds.containsKey(clusterID)) {
@@ -794,12 +794,12 @@ void _sortFaceInfosOnCreationTime(
});
}
Map<int, (Uint8List, int)> _updateClusterSummaries({
Map<String, (Uint8List, int)> _updateClusterSummaries({
required List<FaceInfo> newFaceInfos,
Map<int, (Uint8List, int)>? oldSummary,
Map<String, (Uint8List, int)>? oldSummary,
}) {
final calcSummariesStart = DateTime.now();
final Map<int, List<FaceInfo>> newClusterIdToFaceInfos = {};
final Map<String, List<FaceInfo>> newClusterIdToFaceInfos = {};
for (final faceInfo in newFaceInfos) {
if (newClusterIdToFaceInfos.containsKey(faceInfo.clusterId!)) {
newClusterIdToFaceInfos[faceInfo.clusterId!]!.add(faceInfo);
@@ -808,7 +808,7 @@ Map<int, (Uint8List, int)> _updateClusterSummaries({
}
}
final Map<int, (Uint8List, int)> newClusterSummaries = {};
final Map<String, (Uint8List, int)> newClusterSummaries = {};
for (final clusterId in newClusterIdToFaceInfos.keys) {
final List<Vector> newEmbeddings = newClusterIdToFaceInfos[clusterId]!
.map((faceInfo) => faceInfo.vEmbedding!)
@@ -849,13 +849,13 @@ void _analyzeClusterResults(List<FaceInfo> sortedFaceInfos) {
if (!kDebugMode) return;
final stopwatch = Stopwatch()..start();
final Map<String, int> faceIdToCluster = {};
final Map<String, String> faceIdToCluster = {};
for (final faceInfo in sortedFaceInfos) {
faceIdToCluster[faceInfo.faceID] = faceInfo.clusterId!;
}
// Find faceIDs that are part of a cluster which is larger than 5 and are new faceIDs
final Map<int, int> clusterIdToSize = {};
final Map<String, int> clusterIdToSize = {};
faceIdToCluster.forEach((key, value) {
if (clusterIdToSize.containsKey(value)) {
clusterIdToSize[value] = clusterIdToSize[value]! + 1;

2
mobile/lib/services/machine_learning/face_ml/face_clustering/face_db_info_for_clustering.dart
View File

@@ -2,7 +2,7 @@ import "dart:typed_data" show Uint8List;
class FaceDbInfoForClustering {
final String faceID;
int? clusterId;
String? clusterId;
final Uint8List embeddingBytes;
final double faceScore;
final double blurValue;

2
mobile/lib/services/machine_learning/face_ml/face_detection/detection.dart
View File

@@ -1,6 +1,6 @@
import 'dart:math' show max, min;
import "package:photos/face/model/dimension.dart";
import "package:photos/models/ml/face/dimension.dart";
enum FaceDirection { left, right, straight }

2
mobile/lib/services/machine_learning/face_ml/face_detection/face_detection_service.dart
View File

@@ -6,7 +6,7 @@ import 'dart:ui' as ui show Image;
import 'package:logging/logging.dart';
import "package:onnx_dart/onnx_dart.dart";
import 'package:onnxruntime/onnxruntime.dart';
import "package:photos/face/model/dimension.dart";
import "package:photos/models/ml/face/dimension.dart";
import 'package:photos/services/machine_learning/face_ml/face_detection/detection.dart';
import "package:photos/services/machine_learning/face_ml/face_detection/face_detection_postprocessing.dart";
import "package:photos/services/machine_learning/ml_model.dart";

10
mobile/lib/services/machine_learning/face_ml/face_recognition_service.dart
View File

@@ -5,13 +5,13 @@ import "dart:ui" show Image;
import "package:logging/logging.dart";
import "package:photos/core/event_bus.dart";
import "package:photos/db/embeddings_db.dart";
import "package:photos/db/ml/db.dart";
import "package:photos/db/ml/embeddings_db.dart";
import "package:photos/events/diff_sync_complete_event.dart";
import "package:photos/events/people_changed_event.dart";
import "package:photos/extensions/list.dart";
import "package:photos/face/db.dart";
import "package:photos/face/model/face.dart";
import "package:photos/models/embedding.dart";
import "package:photos/models/ml/clip.dart";
import "package:photos/models/ml/face/face.dart";
import "package:photos/models/ml/ml_versions.dart";
import "package:photos/service_locator.dart";
import "package:photos/services/filedata/filedata_service.dart";
@@ -144,7 +144,7 @@ class FaceRecognitionService {
}
}
await FaceMLDataDB.instance.bulkInsertFaces(faces);
await EmbeddingsDB.instance.putMany(clipEmbeddings);
await FaceMLDataDB.instance.putMany(clipEmbeddings);
}
// Yield any remaining instructions
if (batchToYield.isNotEmpty) {

144
mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart
View File

@@ -7,12 +7,12 @@ import "package:logging/logging.dart";
import "package:ml_linalg/linalg.dart";
import "package:photos/core/event_bus.dart";
import "package:photos/db/files_db.dart";
import "package:photos/db/ml/db.dart";
import "package:photos/events/people_changed_event.dart";
import "package:photos/extensions/stop_watch.dart";
import "package:photos/face/db.dart";
import "package:photos/face/model/person.dart";
import "package:photos/generated/protos/ente/common/vector.pb.dart";
import "package:photos/models/file/file.dart";
import "package:photos/models/ml/face/person.dart";
import "package:photos/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart";
import "package:photos/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart";
import "package:photos/services/machine_learning/face_ml/person/person_service.dart";
@@ -20,7 +20,7 @@ import "package:photos/services/machine_learning/ml_result.dart";
import "package:photos/services/search_service.dart";
class ClusterSuggestion {
final int clusterIDToMerge;
final String clusterIDToMerge;
final double distancePersonToCluster;
final bool usedOnlyMeanForSuggestion;
final List<EnteFile> filesInCluster;
@@ -43,13 +43,13 @@ class ClusterFeedbackService {
static final ClusterFeedbackService instance =
ClusterFeedbackService._privateConstructor();
static int lastViewedClusterID = -1;
static setLastViewedClusterID(int clusterID) {
static String lastViewedClusterID = '';
static setLastViewedClusterID(String clusterID) {
lastViewedClusterID = clusterID;
}
static resetLastViewedClusterID() {
lastViewedClusterID = -1;
lastViewedClusterID = '';
}
/// Returns a list of cluster suggestions for a person. Each suggestion is a tuple of the following elements:
@@ -68,7 +68,7 @@ class ClusterFeedbackService {
try {
// Get the suggestions for the person using centroids and median
final startTime = DateTime.now();
final List<(int, double, bool)> foundSuggestions =
final List<(String, double, bool)> foundSuggestions =
await _getSuggestions(person);
final findSuggestionsTime = DateTime.now();
_logger.info(
@@ -77,13 +77,13 @@ class ClusterFeedbackService {
// Get the files for the suggestions
final suggestionClusterIDs = foundSuggestions.map((e) => e.$1).toSet();
final Map<int, Set<int>> fileIdToClusterID =
final Map<int, Set<String>> fileIdToClusterID =
await FaceMLDataDB.instance.getFileIdToClusterIDSetForCluster(
suggestionClusterIDs,
);
final clusterIdToFaceIDs =
await FaceMLDataDB.instance.getClusterToFaceIDs(suggestionClusterIDs);
final Map<int, List<EnteFile>> clusterIDToFiles = {};
final Map<String, List<EnteFile>> clusterIDToFiles = {};
final allFiles = await SearchService.instance.getAllFiles();
for (final f in allFiles) {
if (!fileIdToClusterID.containsKey(f.uploadedFileID ?? -1)) {
@@ -180,7 +180,7 @@ class ClusterFeedbackService {
.clusterSummaryUpdate(clusterResult.newClusterSummaries);
// Make sure the deleted faces don't get suggested in the future
final notClusterIdToPersonId = <int, String>{};
final notClusterIdToPersonId = <String, String>{};
for (final clusterId in newFaceIdToClusterID.values.toSet()) {
notClusterIdToPersonId[clusterId] = p.remoteID;
}
@@ -202,7 +202,7 @@ class ClusterFeedbackService {
Future<void> removeFilesFromCluster(
List<EnteFile> files,
int clusterID,
String clusterID,
) async {
_logger.info('removeFilesFromCluster called');
try {
@@ -249,7 +249,7 @@ class ClusterFeedbackService {
PeopleChangedEvent(
relevantFiles: files,
type: PeopleEventType.removedFilesFromCluster,
source: "$clusterID",
source: clusterID,
),
);
_logger.info('removeFilesFromCluster done');
@@ -260,8 +260,8 @@ class ClusterFeedbackService {
}
}
Future<void> addFacesToCluster(List<String> faceIDs, int clusterID) async {
final faceIDToClusterID = <String, int>{};
Future<void> addFacesToCluster(List<String> faceIDs, String clusterID) async {
final faceIDToClusterID = <String, String>{};
for (final faceID in faceIDs) {
faceIDToClusterID[faceID] = clusterID;
}
@@ -272,7 +272,7 @@ class ClusterFeedbackService {
Future<bool> checkAndDoAutomaticMerges(
PersonEntity p, {
required int personClusterID,
required String personClusterID,
}) async {
final faceMlDb = FaceMLDataDB.instance;
final faceIDs = await faceMlDb.getFaceIDsForCluster(personClusterID);
@@ -293,7 +293,7 @@ class ClusterFeedbackService {
// Get and update the cluster summary to get the avg (centroid) and count
final EnteWatch watch = EnteWatch("ClusterFeedbackService")..start();
final Map<int, Vector> clusterAvg = await _getUpdateClusterAvg(
final Map<String, Vector> clusterAvg = await _getUpdateClusterAvg(
allClusterIdsToCountMap,
ignoredClusters,
minClusterSize: kMinimumClusterSizeSearchResult,
@@ -301,7 +301,8 @@ class ClusterFeedbackService {
watch.log('computed avg for ${clusterAvg.length} clusters');
// Find the actual closest clusters for the person
final List<(int, double)> suggestions = await calcSuggestionsMeanInComputer(
final List<(String, double)> suggestions =
await calcSuggestionsMeanInComputer(
clusterAvg,
{personClusterID},
ignoredClusters,
@@ -333,16 +334,16 @@ class ClusterFeedbackService {
return true;
}
Future<void> ignoreCluster(int clusterID) async {
Future<void> ignoreCluster(String clusterID) async {
await PersonService.instance.addPerson('', clusterID);
Bus.instance.fire(PeopleChangedEvent());
return;
}
Future<List<(int, int)>> checkForMixedClusters() async {
Future<List<(String, int)>> checkForMixedClusters() async {
final faceMlDb = FaceMLDataDB.instance;
final allClusterToFaceCount = await faceMlDb.clusterIdToFaceCount();
final clustersToInspect = <int>[];
final clustersToInspect = <String>[];
for (final clusterID in allClusterToFaceCount.keys) {
if (allClusterToFaceCount[clusterID]! > 20 &&
allClusterToFaceCount[clusterID]! < 500) {
@@ -353,7 +354,7 @@ class ClusterFeedbackService {
final fileIDToCreationTime =
await FilesDB.instance.getFileIDToCreationTime();
final susClusters = <(int, int)>[];
final susClusters = <(String, int)>[];
final inspectionStart = DateTime.now();
for (final clusterID in clustersToInspect) {
@@ -387,15 +388,15 @@ class ClusterFeedbackService {
);
// Now find the sizes of the biggest and second biggest cluster
final int biggestClusterID = newClusterIdToCount.keys.reduce((a, b) {
final String biggestClusterID = newClusterIdToCount.keys.reduce((a, b) {
return newClusterIdToCount[a]! > newClusterIdToCount[b]! ? a : b;
});
final int biggestSize = newClusterIdToCount[biggestClusterID]!;
final biggestRatio = biggestSize / originalClusterSize;
if (newClusterIdToCount.length > 1) {
final List<int> clusterIDs = newClusterIdToCount.keys.toList();
final List<String> clusterIDs = newClusterIdToCount.keys.toList();
clusterIDs.remove(biggestClusterID);
final int secondBiggestClusterID = clusterIDs.reduce((a, b) {
final String secondBiggestClusterID = clusterIDs.reduce((a, b) {
return newClusterIdToCount[a]! > newClusterIdToCount[b]! ? a : b;
});
final int secondBiggestSize =
@@ -432,7 +433,7 @@ class ClusterFeedbackService {
}
Future<ClusteringResult> breakUpCluster(
int clusterID, {
String clusterID, {
bool useDbscan = false,
}) async {
_logger.info(
@@ -491,7 +492,7 @@ class ClusterFeedbackService {
/// 1. clusterID: the ID of the cluster
/// 2. distance: the distance between the person's cluster and the suggestion
/// 3. usedMean: whether the suggestion was found using the mean (true) or the median (false)
Future<List<(int, double, bool)>> _getSuggestions(
Future<List<(String, double, bool)>> _getSuggestions(
PersonEntity p, {
int sampleSize = 50,
double maxMedianDistance = 0.62,
@@ -520,8 +521,8 @@ class ClusterFeedbackService {
.map((clusterID) => allClusterIdsToCountMap[clusterID] ?? 0)
.reduce((value, element) => min(value, element));
final checkSizes = [100, 20, kMinimumClusterSizeSearchResult, 10, 5, 1];
Map<int, Vector> clusterAvgBigClusters = <int, Vector>{};
final List<(int, double)> suggestionsMean = [];
Map<String, Vector> clusterAvgBigClusters = <String, Vector>{};
final List<(String, double)> suggestionsMean = [];
for (final minimumSize in checkSizes.toSet()) {
if (smallestPersonClusterSize >=
min(minimumSize, kMinimumClusterSizeSearchResult)) {
@@ -533,7 +534,7 @@ class ClusterFeedbackService {
w?.log(
'Calculate avg for ${clusterAvgBigClusters.length} clusters of min size $minimumSize',
);
final List<(int, double)> suggestionsMeanBigClusters =
final List<(String, double)> suggestionsMeanBigClusters =
await calcSuggestionsMeanInComputer(
clusterAvgBigClusters,
personClusters,
@@ -570,7 +571,7 @@ class ClusterFeedbackService {
// Find the other cluster candidates based on the median
final clusterAvg = clusterAvgBigClusters;
final List<(int, double)> moreSuggestionsMean =
final List<(String, double)> moreSuggestionsMean =
await calcSuggestionsMeanInComputer(
clusterAvg,
personClusters,
@@ -616,8 +617,8 @@ class ClusterFeedbackService {
.toList(growable: false);
// Find the actual closest clusters for the person using median
final List<(int, double)> suggestionsMedian = [];
final List<(int, double)> greatSuggestionsMedian = [];
final List<(String, double)> suggestionsMedian = [];
final List<(String, double)> greatSuggestionsMedian = [];
double minMedianDistance = maxMedianDistance;
for (final otherClusterId in otherClusterIdsCandidates) {
final Iterable<Uint8List> otherEmbeddingsProto =
@@ -663,11 +664,12 @@ class ClusterFeedbackService {
_logger.info("Found suggestions using median: $suggestionsMedian");
}
final List<(int, double, bool)> finalSuggestionsMedian = suggestionsMedian
.map(((e) => (e.$1, e.$2, false)))
.toList(growable: false)
.reversed
.toList(growable: false);
final List<(String, double, bool)> finalSuggestionsMedian =
suggestionsMedian
.map(((e) => (e.$1, e.$2, false)))
.toList(growable: false)
.reversed
.toList(growable: false);
if (greatSuggestionsMedian.isNotEmpty) {
_logger.info(
@@ -687,9 +689,9 @@ class ClusterFeedbackService {
return finalSuggestionsMedian;
}
Future<Map<int, Vector>> _getUpdateClusterAvg(
Map<int, int> allClusterIdsToCountMap,
Set<int> ignoredClusters, {
Future<Map<String, Vector>> _getUpdateClusterAvg(
Map<String, int> allClusterIdsToCountMap,
Set<String> ignoredClusters, {
int minClusterSize = 1,
int maxClusterInCurrentRun = 500,
int maxEmbeddingToRead = 10000,
@@ -701,9 +703,9 @@ class ClusterFeedbackService {
'start getUpdateClusterAvg for ${allClusterIdsToCountMap.length} clusters, minClusterSize $minClusterSize, maxClusterInCurrentRun $maxClusterInCurrentRun',
);
final Map<int, (Uint8List, int)> clusterToSummary =
final Map<String, (Uint8List, int)> clusterToSummary =
await faceMlDb.getAllClusterSummary(minClusterSize);
final Map<int, (Uint8List, int)> updatesForClusterSummary = {};
final Map<String, (Uint8List, int)> updatesForClusterSummary = {};
w?.log(
'getUpdateClusterAvg database call for getAllClusterSummary',
@@ -717,7 +719,7 @@ class ClusterFeedbackService {
'ignoredClusters': ignoredClusters,
'clusterToSummary': clusterToSummary,
},
) as (Map<int, Vector>, Set<int>, int, int, int);
) as (Map<String, Vector>, Set<String>, int, int, int);
final clusterAvg = serializationEmbeddings.$1;
final allClusterIds = serializationEmbeddings.$2;
final ignoredClustersCnt = serializationEmbeddings.$3;
@@ -753,7 +755,7 @@ class ClusterFeedbackService {
w?.reset();
int currentPendingRead = 0;
final List<int> clusterIdsToRead = [];
final List<String> clusterIdsToRead = [];
for (final clusterID in sortedClusterIDs) {
if (maxClusterInCurrentRun-- <= 0) {
break;
@@ -772,9 +774,9 @@ class ClusterFeedbackService {
}
}
final Map<int, Iterable<Uint8List>> clusterEmbeddings = await FaceMLDataDB
.instance
.getFaceEmbeddingsForClusters(clusterIdsToRead);
final Map<String, Iterable<Uint8List>> clusterEmbeddings =
await FaceMLDataDB.instance
.getFaceEmbeddingsForClusters(clusterIdsToRead);
w?.logAndReset(
'read $currentPendingRead embeddings for ${clusterEmbeddings.length} clusters',
@@ -817,10 +819,10 @@ class ClusterFeedbackService {
return clusterAvg;
}
Future<List<(int, double)>> calcSuggestionsMeanInComputer(
Map<int, Vector> clusterAvg,
Set<int> personClusters,
Set<int> ignoredClusters,
Future<List<(String, double)>> calcSuggestionsMeanInComputer(
Map<String, Vector> clusterAvg,
Set<String> personClusters,
Set<String> ignoredClusters,
double maxClusterDistance,
) async {
return await _computer.compute(
@@ -889,7 +891,7 @@ class ClusterFeedbackService {
// Get the cluster averages for the person's clusters and the suggestions' clusters
final personClusters = await faceMlDb.getPersonClusterIDs(person.remoteID);
final Map<int, (Uint8List, int)> personClusterToSummary =
final Map<String, (Uint8List, int)> personClusterToSummary =
await faceMlDb.getClusterToClusterSummary(personClusters);
final clusterSummaryCallTime = DateTime.now();
@@ -975,7 +977,7 @@ class ClusterFeedbackService {
}
Future<void> debugLogClusterBlurValues(
int clusterID, {
String clusterID, {
int? clusterSize,
bool logClusterSummary = false,
bool logBlurValues = false,
@@ -986,7 +988,8 @@ class ClusterFeedbackService {
_logger.info(
"Debug logging for cluster $clusterID${clusterSize != null ? ' with $clusterSize photos' : ''}",
);
const int biggestClusterID = 1715061228725148;
// todo:(laurens) remove to review
const String biggestClusterID = 'some random id';
// Logging the cluster summary for the cluster
if (logClusterSummary) {
@@ -1117,21 +1120,22 @@ class ClusterFeedbackService {
}
/// Returns a map of person's clusterID to map of closest clusterID to with disstance
List<(int, double)> _calcSuggestionsMean(Map<String, dynamic> args) {
List<(String, double)> _calcSuggestionsMean(Map<String, dynamic> args) {
// Fill in args
final Map<int, Vector> clusterAvg = args['clusterAvg'];
final Set<int> personClusters = args['personClusters'];
final Set<int> ignoredClusters = args['ignoredClusters'];
final Map<String, Vector> clusterAvg = args['clusterAvg'];
final Set<String> personClusters = args['personClusters'];
final Set<String> ignoredClusters = args['ignoredClusters'];
final double maxClusterDistance = args['maxClusterDistance'];
final Map<int, List<(int, double)>> suggestions = {};
final Map<String, List<(String, double)>> suggestions = {};
const suggestionMax = 2000;
int suggestionCount = 0;
int comparisons = 0;
final w = (kDebugMode ? EnteWatch('getSuggestions') : null)?..start();
// ignore the clusters that belong to the person or is ignored
Set<int> otherClusters = clusterAvg.keys.toSet().difference(personClusters);
Set<String> otherClusters =
clusterAvg.keys.toSet().difference(personClusters);
otherClusters = otherClusters.difference(ignoredClusters);
for (final otherClusterID in otherClusters) {
@@ -1140,7 +1144,7 @@ List<(int, double)> _calcSuggestionsMean(Map<String, dynamic> args) {
dev.log('[WARNING] no avg for othercluster $otherClusterID');
continue;
}
int? nearestPersonCluster;
String? nearestPersonCluster;
double? minDistance;
for (final personCluster in personClusters) {
if (clusterAvg[personCluster] == null) {
@@ -1172,8 +1176,8 @@ List<(int, double)> _calcSuggestionsMean(Map<String, dynamic> args) {
);
if (suggestions.isNotEmpty) {
final List<(int, double)> suggestClusterIds = [];
for (final List<(int, double)> suggestion in suggestions.values) {
final List<(String, double)> suggestClusterIds = [];
for (final List<(String, double)> suggestion in suggestions.values) {
suggestClusterIds.addAll(suggestion);
}
suggestClusterIds.sort(
@@ -1186,20 +1190,22 @@ List<(int, double)> _calcSuggestionsMean(Map<String, dynamic> args) {
return suggestClusterIds.sublist(0, min(suggestClusterIds.length, 20));
} else {
dev.log("No suggestions found using mean");
return <(int, double)>[];
return <(String, double)>[];
}
}
Future<(Map<int, Vector>, Set<int>, int, int, int)>
Future<(Map<String, Vector>, Set<String>, int, int, int)>
checkAndSerializeCurrentClusterMeans(
Map args,
) async {
final Map<int, int> allClusterIdsToCountMap = args['allClusterIdsToCountMap'];
final Map<String, int> allClusterIdsToCountMap =
args['allClusterIdsToCountMap'];
final int minClusterSize = args['minClusterSize'] ?? 1;
final Set<int> ignoredClusters = args['ignoredClusters'] ?? {};
final Map<int, (Uint8List, int)> clusterToSummary = args['clusterToSummary'];
final Set<String> ignoredClusters = args['ignoredClusters'] ?? {};
final Map<String, (Uint8List, int)> clusterToSummary =
args['clusterToSummary'];
final Map<int, Vector> clusterAvg = {};
final Map<String, Vector> clusterAvg = {};
final allClusterIds = allClusterIdsToCountMap.keys.toSet();
int ignoredClustersCnt = 0, alreadyUpdatedClustersCnt = 0;

53
mobile/lib/services/machine_learning/face_ml/person/person_service.dart
View File

@@ -4,11 +4,11 @@ import "dart:developer";
import "package:flutter/foundation.dart";
import "package:logging/logging.dart";
import "package:photos/core/event_bus.dart";
import "package:photos/db/ml/db.dart";
import "package:photos/events/people_changed_event.dart";
import "package:photos/extensions/stop_watch.dart";
import "package:photos/face/db.dart";
import "package:photos/face/model/person.dart";
import "package:photos/models/api/entity/type.dart";
import "package:photos/models/ml/face/person.dart";
import "package:photos/services/entity_service.dart";
import "package:shared_preferences/shared_preferences.dart";
@@ -37,7 +37,7 @@ class PersonService {
}
Future<List<PersonEntity>> getPersons() async {
final entities = await entityService.getEntities(EntityType.person);
final entities = await entityService.getEntities(EntityType.cgroup);
return entities
.map(
(e) => PersonEntity(e.id, PersonData.fromJson(json.decode(e.data))),
@@ -46,7 +46,7 @@ class PersonService {
}
Future<PersonEntity?> getPerson(String id) {
return entityService.getEntity(EntityType.person, id).then((e) {
return entityService.getEntity(EntityType.cgroup, id).then((e) {
if (e == null) {
return null;
}
@@ -55,7 +55,7 @@ class PersonService {
}
Future<Map<String, PersonEntity>> getPersonsMap() async {
final entities = await entityService.getEntities(EntityType.person);
final entities = await entityService.getEntities(EntityType.cgroup);
final Map<String, PersonEntity> map = {};
for (var e in entities) {
final person =
@@ -82,7 +82,7 @@ class PersonService {
continue;
}
final personData = person.data;
final Map<int, Set<String>> dbPersonCluster =
final Map<String, Set<String>> dbPersonCluster =
dbPersonClusterInfo[personID]!;
if (_shouldUpdateRemotePerson(personData, dbPersonCluster)) {
final personData = person.data;
@@ -95,11 +95,7 @@ class PersonService {
)
.toList();
entityService
.addOrUpdate(
EntityType.person,
json.encode(personData.toJson()),
id: personID,
)
.addOrUpdate(EntityType.cgroup, personData.toJson(), id: personID)
.ignore();
personData.logStats();
}
@@ -109,7 +105,7 @@ class PersonService {
bool _shouldUpdateRemotePerson(
PersonData personData,
Map<int, Set<String>> dbPersonCluster,
Map<String, Set<String>> dbPersonCluster,
) {
bool result = false;
if ((personData.assigned?.length ?? 0) != dbPersonCluster.length) {
@@ -152,7 +148,7 @@ class PersonService {
Future<PersonEntity> addPerson(
String name,
int clusterID, {
String clusterID, {
bool isHidden = false,
}) async {
final faceIds = await faceMLDataDB.getFaceIDsForCluster(clusterID);
@@ -167,8 +163,8 @@ class PersonService {
isHidden: isHidden,
);
final result = await entityService.addOrUpdate(
EntityType.person,
json.encode(data.toJson()),
EntityType.cgroup,
data.toJson(),
);
await faceMLDataDB.assignClusterToPerson(
personID: result.id,
@@ -179,14 +175,14 @@ class PersonService {
Future<void> removeClusterToPerson({
required String personID,
required int clusterID,
required String clusterID,
}) async {
final person = (await getPerson(personID))!;
final personData = person.data;
personData.assigned!.removeWhere((element) => element.id != clusterID);
await entityService.addOrUpdate(
EntityType.person,
json.encode(personData.toJson()),
EntityType.cgroup,
personData.toJson(),
id: personID,
);
await faceMLDataDB.removeClusterToPerson(
@@ -201,7 +197,7 @@ class PersonService {
required Set<String> faceIDs,
}) async {
final personData = person.data;
final List<int> emptiedClusters = [];
final List<String> emptiedClusters = [];
for (final cluster in personData.assigned!) {
cluster.faces.removeWhere((faceID) => faceIDs.contains(faceID));
if (cluster.faces.isEmpty) {
@@ -219,10 +215,9 @@ class PersonService {
);
}
await entityService.addOrUpdate(
EntityType.person,
json.encode(personData.toJson()),
EntityType.cgroup,
personData.toJson(),
id: person.remoteID,
);
personData.logStats();
@@ -237,8 +232,8 @@ class PersonService {
final PersonEntity justName =
PersonEntity(personID, PersonData(name: entity.data.name));
await entityService.addOrUpdate(
EntityType.person,
json.encode(justName.data.toJson()),
EntityType.cgroup,
justName.data.toJson(),
id: personID,
);
await faceMLDataDB.removePerson(personID);
@@ -254,10 +249,10 @@ class PersonService {
Future<void> fetchRemoteClusterFeedback() async {
await entityService.syncEntities();
final entities = await entityService.getEntities(EntityType.person);
final entities = await entityService.getEntities(EntityType.cgroup);
entities.sort((a, b) => a.updatedAt.compareTo(b.updatedAt));
final Map<String, int> faceIdToClusterID = {};
final Map<int, String> clusterToPersonID = {};
final Map<String, String> faceIdToClusterID = {};
final Map<String, String> clusterToPersonID = {};
for (var e in entities) {
final personData = PersonData.fromJson(json.decode(e.data));
int faceCount = 0;
@@ -312,8 +307,8 @@ class PersonService {
Future<void> _updatePerson(PersonEntity updatePerson) async {
await entityService.addOrUpdate(
EntityType.person,
json.encode(updatePerson.data.toJson()),
EntityType.cgroup,
updatePerson.data.toJson(),
id: updatePerson.remoteID,
);
updatePerson.data.logStats();

2
mobile/lib/services/machine_learning/ml_computer.dart
View File

@@ -5,7 +5,7 @@ import 'dart:typed_data' show Uint8List;
import "package:dart_ui_isolate/dart_ui_isolate.dart";
import "package:logging/logging.dart";
import "package:photos/face/model/box.dart";
import "package:photos/models/ml/face/box.dart";
import "package:photos/services/machine_learning/ml_model.dart";
import "package:photos/services/machine_learning/semantic_search/clip/clip_text_encoder.dart";
import "package:photos/services/machine_learning/semantic_search/clip/clip_text_tokenizer.dart";

2
mobile/lib/services/machine_learning/ml_result.dart
View File

@@ -1,6 +1,6 @@
import "dart:convert" show jsonEncode, jsonDecode;
import "package:photos/face/model/dimension.dart";
import "package:photos/models/ml/face/dimension.dart";
import 'package:photos/models/ml/ml_typedefs.dart';
import "package:photos/models/ml/ml_versions.dart";
import 'package:photos/services/machine_learning/face_ml/face_alignment/alignment_result.dart';

12
mobile/lib/services/machine_learning/ml_service.dart
View File

@@ -8,13 +8,13 @@ import "package:logging/logging.dart";
import "package:package_info_plus/package_info_plus.dart";
import "package:photos/core/event_bus.dart";
import "package:photos/db/files_db.dart";
import "package:photos/db/ml/db.dart";
import "package:photos/events/machine_learning_control_event.dart";
import "package:photos/events/people_changed_event.dart";
import "package:photos/face/db.dart";
import "package:photos/face/model/box.dart";
import "package:photos/face/model/detection.dart" as face_detection;
import "package:photos/face/model/face.dart";
import "package:photos/face/model/landmark.dart";
import "package:photos/models/ml/face/box.dart";
import "package:photos/models/ml/face/detection.dart" as face_detection;
import "package:photos/models/ml/face/face.dart";
import "package:photos/models/ml/face/landmark.dart";
import "package:photos/service_locator.dart";
import "package:photos/services/filedata/filedata_service.dart";
import "package:photos/services/filedata/model/file_data.dart";
@@ -250,7 +250,7 @@ class MLService {
);
// Get the current cluster statistics
final Map<int, (Uint8List, int)> oldClusterSummaries =
final Map<String, (Uint8List, int)> oldClusterSummaries =
await FaceMLDataDB.instance.getAllClusterSummary();
if (clusterInBuckets) {

38
mobile/lib/services/machine_learning/semantic_search/semantic_search_service.dart
View File

@@ -1,18 +1,19 @@
import "dart:async" show unawaited;
import "dart:developer" as dev show log;
import "dart:math" show min;
import "dart:typed_data" show ByteData;
import "dart:ui" show Image;
import "package:computer/computer.dart";
import "package:flutter/foundation.dart";
import "package:logging/logging.dart";
import "package:photos/core/cache/lru_map.dart";
import "package:photos/core/event_bus.dart";
import "package:photos/db/embeddings_db.dart";
import "package:photos/db/files_db.dart";
import "package:photos/db/ml/db.dart";
import "package:photos/db/ml/embeddings_db.dart";
import 'package:photos/events/embedding_updated_event.dart';
import "package:photos/models/embedding.dart";
import "package:photos/models/file/file.dart";
import "package:photos/models/ml/clip.dart";
import "package:photos/models/ml/ml_versions.dart";
import "package:photos/service_locator.dart";
import "package:photos/services/collections_service.dart";
@@ -57,7 +58,7 @@ class SemanticSearchService {
return;
}
_hasInitialized = true;
await EmbeddingsDB.instance.init();
await _loadImageEmbeddings();
Bus.instance.on<EmbeddingUpdatedEvent>().listen((event) {
if (!_hasInitialized) return;
@@ -112,7 +113,7 @@ class SemanticSearchService {
}
Future<void> clearIndexes() async {
await EmbeddingsDB.instance.deleteAll();
await FaceMLDataDB.instance.deleteClipIndexes();
final preferences = await SharedPreferences.getInstance();
await preferences.remove("sync_time_embeddings_v3");
_logger.info("Indexes cleared");
@@ -121,7 +122,7 @@ class SemanticSearchService {
Future<void> _loadImageEmbeddings() async {
_logger.info("Pulling cached embeddings");
final startTime = DateTime.now();
_cachedImageEmbeddings = await EmbeddingsDB.instance.getAll();
_cachedImageEmbeddings = await FaceMLDataDB.instance.getAll();
final endTime = DateTime.now();
_logger.info(
"Loading ${_cachedImageEmbeddings.length} took: ${(endTime.millisecondsSinceEpoch - startTime.millisecondsSinceEpoch)}ms",
@@ -133,7 +134,7 @@ class SemanticSearchService {
Future<List<int>> _getFileIDsToBeIndexed() async {
final uploadedFileIDs = await getIndexableFileIDs();
final embeddedFileIDs = await EmbeddingsDB.instance.getIndexedFileIds();
final embeddedFileIDs = await FaceMLDataDB.instance.getIndexedFileIds();
embeddedFileIDs.removeWhere((key, value) => value < clipMlVersion);
return uploadedFileIDs.difference(embeddedFileIDs.keys.toSet()).toList();
@@ -143,6 +144,14 @@ class SemanticSearchService {
String query, {
double? scoreThreshold,
}) async {
// if the query starts with 0.xxx, the split the query to get score threshold and actual query
if (query.startsWith(RegExp(r"0\.\d+"))) {
final parts = query.split(" ");
if (parts.length > 1) {
scoreThreshold = double.parse(parts[0]);
query = parts.sublist(1).join(" ");
}
}
final textEmbedding = await _getTextEmbedding(query);
final queryResults = await _getSimilarities(
@@ -178,7 +187,7 @@ class SemanticSearchService {
_logger.info(results.length.toString() + " results");
if (deletedEntries.isNotEmpty) {
unawaited(EmbeddingsDB.instance.deleteEmbeddings(deletedEntries));
unawaited(FaceMLDataDB.instance.deleteEmbeddings(deletedEntries));
}
return results;
@@ -221,7 +230,7 @@ class SemanticSearchService {
_logger.info(results.length.toString() + " results");
if (deletedEntries.isNotEmpty) {
unawaited(EmbeddingsDB.instance.deleteEmbeddings(deletedEntries));
unawaited(FaceMLDataDB.instance.deleteEmbeddings(deletedEntries));
}
final matchingFileIDs = <int>[];
@@ -253,12 +262,12 @@ class SemanticSearchService {
embedding: clipResult.embedding,
version: clipMlVersion,
);
await EmbeddingsDB.instance.put(embedding);
await FaceMLDataDB.instance.put(embedding);
}
static Future<void> storeEmptyClipImageResult(EnteFile entefile) async {
final embedding = ClipEmbedding.empty(entefile.uploadedFileID!);
await EmbeddingsDB.instance.put(embedding);
await FaceMLDataDB.instance.put(embedding);
}
Future<List<double>> _getTextEmbedding(String query) async {
@@ -320,6 +329,7 @@ List<QueryResult> computeBulkSimilarities(Map args) {
final textEmbedding = args["textEmbedding"] as List<double>;
final minimumSimilarity = args["minimumSimilarity"] ??
SemanticSearchService.kMinimumSimilarityThreshold;
double bestScore = 0.0;
for (final imageEmbedding in imageEmbeddings) {
final score = computeCosineSimilarity(
imageEmbedding.embedding,
@@ -328,6 +338,12 @@ List<QueryResult> computeBulkSimilarities(Map args) {
if (score >= minimumSimilarity) {
queryResults.add(QueryResult(imageEmbedding.fileID, score));
}
if (score > bestScore) {
bestScore = score;
}
}
if (kDebugMode && queryResults.isEmpty) {
dev.log("No results found for query with best score: $bestScore");
}
queryResults.sort((first, second) => second.score.compareTo(first.score));

14
mobile/lib/services/search_service.dart
View File

@@ -9,10 +9,9 @@ import 'package:photos/data/holidays.dart';
import 'package:photos/data/months.dart';
import 'package:photos/data/years.dart';
import 'package:photos/db/files_db.dart';
import "package:photos/db/ml/db.dart";
import 'package:photos/events/local_photos_updated_event.dart';
import "package:photos/extensions/string_ext.dart";
import "package:photos/face/db.dart";
import "package:photos/face/model/person.dart";
import "package:photos/models/api/collection/user.dart";
import 'package:photos/models/collection/collection.dart';
import 'package:photos/models/collection/collection_items.dart';
@@ -22,6 +21,7 @@ import 'package:photos/models/file/file_type.dart';
import "package:photos/models/local_entity_data.dart";
import "package:photos/models/location/location.dart";
import "package:photos/models/location_tag/location_tag.dart";
import "package:photos/models/ml/face/person.dart";
import 'package:photos/models/search/album_search_result.dart';
import 'package:photos/models/search/generic_search_result.dart';
import "package:photos/models/search/search_constants.dart";
@@ -736,14 +736,14 @@ class SearchService {
return searchResults;
}
Future<Map<int, List<EnteFile>>> getClusterFilesForPersonID(
Future<Map<String, List<EnteFile>>> getClusterFilesForPersonID(
String personID,
) async {
_logger.info('getClusterFilesForPersonID $personID');
final Map<int, Set<int>> fileIdToClusterID =
final Map<int, Set<String>> fileIdToClusterID =
await FaceMLDataDB.instance.getFileIdToClusterIDSet(personID);
_logger.info('faceDbDone getClusterFilesForPersonID $personID');
final Map<int, List<EnteFile>> clusterIDToFiles = {};
final Map<String, List<EnteFile>> clusterIDToFiles = {};
final allFiles = await getAllFiles();
for (final f in allFiles) {
if (!fileIdToClusterID.containsKey(f.uploadedFileID ?? -1)) {
@@ -765,7 +765,7 @@ class SearchService {
Future<List<GenericSearchResult>> getAllFace(int? limit) async {
try {
debugPrint("getting faces");
final Map<int, Set<int>> fileIdToClusterID =
final Map<int, Set<String>> fileIdToClusterID =
await FaceMLDataDB.instance.getFileIdToClusterIds();
final Map<String, PersonEntity> personIdToPerson =
await PersonService.instance.getPersonsMap();
@@ -773,7 +773,7 @@ class SearchService {
await FaceMLDataDB.instance.getClusterIDToPersonID();
final List<GenericSearchResult> facesResult = [];
final Map<int, List<EnteFile>> clusterIdToFiles = {};
final Map<String, List<EnteFile>> clusterIdToFiles = {};
final Map<String, List<EnteFile>> personIdToFiles = {};
final allFiles = await getAllFiles();
for (final f in allFiles) {

4
mobile/lib/ui/components/bottom_action_bar/bottom_action_bar_widget.dart
View File

@@ -1,8 +1,8 @@
import 'package:flutter/material.dart';
import 'package:photos/core/constants.dart';
import "package:photos/face/model/person.dart";
import 'package:photos/models/collection/collection.dart';
import "package:photos/models/gallery_type.dart";
import "package:photos/models/ml/face/person.dart";
import 'package:photos/models/selected_files.dart';
import 'package:photos/theme/ente_theme.dart';
import 'package:photos/ui/components/bottom_action_bar/action_bar_widget.dart';
@@ -13,7 +13,7 @@ class BottomActionBarWidget extends StatelessWidget {
final GalleryType galleryType;
final Collection? collection;
final PersonEntity? person;
final int? clusterID;
final String? clusterID;
final SelectedFiles selectedFiles;
final VoidCallback? onCancel;
final Color? backgroundColor;

4
mobile/lib/ui/settings/debug/ml_debug_section_widget.dart
View File

@@ -3,9 +3,9 @@ import "dart:async";
import 'package:flutter/material.dart';
import "package:logging/logging.dart";
import "package:photos/core/event_bus.dart";
import "package:photos/db/ml/db.dart";
import "package:photos/events/people_changed_event.dart";
import "package:photos/face/db.dart";
import "package:photos/face/model/person.dart";
import "package:photos/models/ml/face/person.dart";
import "package:photos/service_locator.dart";
import "package:photos/services/machine_learning/face_ml/feedback/cluster_feedback.dart";
import "package:photos/services/machine_learning/face_ml/person/person_service.dart";

4
mobile/lib/ui/viewer/actions/file_selection_actions_widget.dart
View File

@@ -11,7 +11,6 @@ import 'package:photos/core/configuration.dart';
import "package:photos/core/event_bus.dart";
import "package:photos/events/guest_view_event.dart";
import "package:photos/events/people_changed_event.dart";
import "package:photos/face/model/person.dart";
import "package:photos/generated/l10n.dart";
import 'package:photos/models/collection/collection.dart';
import 'package:photos/models/device_collection.dart';
@@ -20,6 +19,7 @@ import 'package:photos/models/file/file_type.dart';
import 'package:photos/models/files_split.dart';
import 'package:photos/models/gallery_type.dart';
import "package:photos/models/metadata/common_keys.dart";
import "package:photos/models/ml/face/person.dart";
import 'package:photos/models/selected_files.dart';
import 'package:photos/services/collections_service.dart';
import 'package:photos/services/hidden_service.dart';
@@ -53,7 +53,7 @@ class FileSelectionActionsWidget extends StatefulWidget {
final DeviceCollection? deviceCollection;
final SelectedFiles selectedFiles;
final PersonEntity? person;
final int? clusterID;
final String? clusterID;
const FileSelectionActionsWidget(
this.type,

4
mobile/lib/ui/viewer/actions/file_selection_overlay_bar.dart
View File

@@ -1,7 +1,7 @@
import 'package:flutter/material.dart';
import "package:photos/face/model/person.dart";
import 'package:photos/models/collection/collection.dart';
import 'package:photos/models/gallery_type.dart';
import "package:photos/models/ml/face/person.dart";
import 'package:photos/models/selected_files.dart';
import "package:photos/theme/effects.dart";
import "package:photos/theme/ente_theme.dart";
@@ -14,7 +14,7 @@ class FileSelectionOverlayBar extends StatefulWidget {
final Collection? collection;
final Color? backgroundColor;
final PersonEntity? person;
final int? clusterID;
final String? clusterID;
const FileSelectionOverlayBar(
this.galleryType,

11
mobile/lib/ui/viewer/file_details/face_widget.dart
View File

@@ -4,11 +4,12 @@ import "dart:typed_data";
import "package:flutter/cupertino.dart";
import "package:flutter/foundation.dart" show kDebugMode;
import "package:flutter/material.dart";
import "package:photos/db/ml/db.dart";
import "package:photos/extensions/stop_watch.dart";
import "package:photos/face/db.dart";
import "package:photos/face/model/face.dart";
import "package:photos/face/model/person.dart";
import 'package:photos/models/file/file.dart';
import "package:photos/models/ml/face/face.dart";
import "package:photos/models/ml/face/person.dart";
import "package:photos/models/nanoids/cluster_id.dart";
import "package:photos/services/machine_learning/face_ml/face_detection/detection.dart";
import "package:photos/services/machine_learning/face_ml/feedback/cluster_feedback.dart";
import "package:photos/services/search_service.dart";
@@ -24,7 +25,7 @@ class FaceWidget extends StatefulWidget {
final Face face;
final Future<Map<String, Uint8List>?>? faceCrops;
final PersonEntity? person;
final int? clusterID;
final String? clusterID;
final bool highlight;
final bool editMode;
@@ -98,7 +99,7 @@ class _FaceWidgetState extends State<FaceWidget> {
}
// Create new clusterID for the faceID and update DB to assign the faceID to the new clusterID
final int newClusterID = DateTime.now().microsecondsSinceEpoch;
final String newClusterID = ClusterID.generate();
await FaceMLDataDB.instance.updateFaceIdToClusterId(
{widget.face.faceID: newClusterID},
);

15
mobile/lib/ui/viewer/file_details/faces_item_widget.dart
View File

@@ -3,11 +3,11 @@ import "dart:developer" as dev show log;
import "package:flutter/foundation.dart" show Uint8List, kDebugMode;
import "package:flutter/material.dart";
import "package:logging/logging.dart";
import "package:photos/face/db.dart";
import "package:photos/face/model/box.dart";
import "package:photos/face/model/face.dart";
import "package:photos/face/model/person.dart";
import "package:photos/db/ml/db.dart";
import "package:photos/models/file/file.dart";
import "package:photos/models/ml/face/box.dart";
import "package:photos/models/ml/face/face.dart";
import "package:photos/models/ml/face/person.dart";
import "package:photos/services/machine_learning/face_ml/feedback/cluster_feedback.dart";
import "package:photos/services/machine_learning/face_ml/person/person_service.dart";
import "package:photos/ui/components/buttons/chip_button_widget.dart";
@@ -146,7 +146,7 @@ class _FacesItemWidgetState extends State<FacesItemWidget> {
final faceCrops = getRelevantFaceCrops(faces);
for (final Face face in faces) {
final int? clusterID = faceIdsToClusterIds[face.faceID];
final String? clusterID = faceIdsToClusterIds[face.faceID];
final PersonEntity? person = clusterIDToPerson[clusterID] != null
? persons[clusterIDToPerson[clusterID]!]
: null;
@@ -175,8 +175,7 @@ class _FacesItemWidgetState extends State<FacesItemWidget> {
Future<Map<String, Uint8List>?> getRelevantFaceCrops(
Iterable<Face> faces, {
int fetchAttempt = 1,
}
) async {
}) async {
try {
final faceIdToCrop = <String, Uint8List>{};
final facesWithoutCrops = <String, FaceBox>{};
@@ -226,7 +225,7 @@ class _FacesItemWidgetState extends State<FacesItemWidget> {
stackTrace: s,
);
resetPool(fullFile: true);
if(fetchAttempt <= retryLimit) {
if (fetchAttempt <= retryLimit) {
return getRelevantFaceCrops(faces, fetchAttempt: fetchAttempt + 1);
}
return null;

10
mobile/lib/ui/viewer/people/add_person_action_sheet.dart
View File

@@ -7,11 +7,11 @@ import 'package:flutter/material.dart';
import "package:logging/logging.dart";
import 'package:modal_bottom_sheet/modal_bottom_sheet.dart';
import "package:photos/core/event_bus.dart";
import "package:photos/db/ml/db.dart";
import "package:photos/events/people_changed_event.dart";
import "package:photos/face/db.dart";
import "package:photos/face/model/person.dart";
import "package:photos/generated/l10n.dart";
import "package:photos/models/file/file.dart";
import "package:photos/models/ml/face/person.dart";
import 'package:photos/services/machine_learning/face_ml/feedback/cluster_feedback.dart';
import "package:photos/services/machine_learning/face_ml/person/person_service.dart";
import "package:photos/services/search_service.dart";
@@ -47,7 +47,7 @@ String _actionName(
Future<dynamic> showAssignPersonAction(
BuildContext context, {
required int clusterID,
required String clusterID,
PersonActionType actionType = PersonActionType.assignPerson,
bool showOptionToAddNewPerson = true,
}) {
@@ -75,7 +75,7 @@ Future<dynamic> showAssignPersonAction(
class PersonActionSheet extends StatefulWidget {
final PersonActionType actionType;
final int cluserID;
final String cluserID;
final bool showOptionToCreateNewPerson;
const PersonActionSheet({
required this.actionType,
@@ -276,7 +276,7 @@ class _PersonActionSheetState extends State<PersonActionSheet> {
Future<void> addNewPerson(
BuildContext context, {
String initValue = '',
required int clusterID,
required String clusterID,
}) async {
final result = await showTextInputDialog(
context,

18
mobile/lib/ui/viewer/people/cluster_app_bar.dart
View File

@@ -6,12 +6,12 @@ import 'package:logging/logging.dart';
import 'package:photos/core/configuration.dart';
import 'package:photos/core/event_bus.dart';
import "package:photos/db/files_db.dart";
import "package:photos/db/ml/db.dart";
import "package:photos/events/people_changed_event.dart";
import 'package:photos/events/subscription_purchased_event.dart';
import "package:photos/face/db.dart";
import "package:photos/face/model/person.dart";
import "package:photos/models/file/file.dart";
import 'package:photos/models/gallery_type.dart';
import "package:photos/models/ml/face/person.dart";
import 'package:photos/models/selected_files.dart';
import 'package:photos/services/collections_service.dart';
import "package:photos/services/machine_learning/face_ml/feedback/cluster_feedback.dart";
@@ -26,7 +26,7 @@ class ClusterAppBar extends StatefulWidget {
final GalleryType type;
final String? title;
final SelectedFiles selectedFiles;
final int clusterID;
final String clusterID;
final PersonEntity? person;
const ClusterAppBar(
@@ -179,7 +179,7 @@ class _AppBarWidgetState extends State<ClusterAppBar> {
Future<void> _breakUpCluster(BuildContext context) async {
bool userConfirmed = false;
List<EnteFile> biggestClusterFiles = [];
int biggestClusterID = -1;
String biggestClusterID = '';
await showChoiceDialog(
context,
title: "Does this grouping contain multiple people?",
@@ -190,9 +190,9 @@ class _AppBarWidgetState extends State<ClusterAppBar> {
try {
final breakupResult = await ClusterFeedbackService.instance
.breakUpCluster(widget.clusterID);
final Map<int, List<String>> newClusterIDToFaceIDs =
final Map<String, List<String>> newClusterIDToFaceIDs =
breakupResult.newClusterIdToFaceIds;
final Map<String, int> newFaceIdToClusterID =
final Map<String, String> newFaceIdToClusterID =
breakupResult.newFaceIdToCluster;
// Update to delete the old clusters and save the new clusters
@@ -203,9 +203,9 @@ class _AppBarWidgetState extends State<ClusterAppBar> {
.updateFaceIdToClusterId(newFaceIdToClusterID);
// Find the biggest cluster
biggestClusterID = -1;
biggestClusterID = '';
int biggestClusterSize = 0;
for (final MapEntry<int, List<String>> clusterToFaces
for (final MapEntry<String, List<String>> clusterToFaces
in newClusterIDToFaceIDs.entries) {
if (clusterToFaces.value.length > biggestClusterSize) {
biggestClusterSize = clusterToFaces.value.length;
@@ -253,7 +253,7 @@ class _AppBarWidgetState extends State<ClusterAppBar> {
final breakupResult =
await ClusterFeedbackService.instance.breakUpCluster(widget.clusterID);
final Map<int, List<String>> newClusterIDToFaceIDs =
final Map<String, List<String>> newClusterIDToFaceIDs =
breakupResult.newClusterIdToFaceIds;
final allFileIDs = newClusterIDToFaceIDs.values

6
mobile/lib/ui/viewer/people/cluster_breakup_page.dart
View File

@@ -6,7 +6,7 @@ import "package:photos/ui/viewer/people/cluster_page.dart";
import "package:photos/ui/viewer/search/result/person_face_widget.dart";
class ClusterBreakupPage extends StatefulWidget {
final Map<int, List<EnteFile>> newClusterIDsToFiles;
final Map<String, List<EnteFile>> newClusterIDsToFiles;
final String title;
const ClusterBreakupPage(
@@ -32,7 +32,7 @@ class _ClusterBreakupPageState extends State<ClusterBreakupPage> {
body: ListView.builder(
itemCount: widget.newClusterIDsToFiles.keys.length,
itemBuilder: (context, index) {
final int clusterID = keys[index];
final String clusterID = keys[index];
final List<EnteFile> files = clusterIDsToFiles[keys[index]]!;
return InkWell(
onTap: () {
@@ -40,7 +40,7 @@ class _ClusterBreakupPageState extends State<ClusterBreakupPage> {
MaterialPageRoute(
builder: (context) => ClusterPage(
files,
clusterID: index,
clusterID: clusterID,
appendTitle: "(Analysis)",
),
),

4
mobile/lib/ui/viewer/people/cluster_page.dart
View File

@@ -6,11 +6,11 @@ import 'package:photos/core/event_bus.dart';
import 'package:photos/events/files_updated_event.dart';
import 'package:photos/events/local_photos_updated_event.dart';
import "package:photos/events/people_changed_event.dart";
import "package:photos/face/model/person.dart";
import "package:photos/generated/l10n.dart";
import 'package:photos/models/file/file.dart';
import 'package:photos/models/file_load_result.dart';
import 'package:photos/models/gallery_type.dart';
import "package:photos/models/ml/face/person.dart";
import 'package:photos/models/selected_files.dart';
import "package:photos/services/machine_learning/face_ml/feedback/cluster_feedback.dart";
import 'package:photos/ui/viewer/actions/file_selection_overlay_bar.dart';
@@ -29,7 +29,7 @@ class ClusterPage extends StatefulWidget {
final List<EnteFile> searchResult;
final bool enableGrouping;
final String tagPrefix;
final int clusterID;
final String clusterID;
final PersonEntity? personID;
final String appendTitle;
final bool showNamingBanner;

23
mobile/lib/ui/viewer/people/people_app_bar.dart
View File

@@ -7,10 +7,10 @@ import 'package:photos/core/configuration.dart';
import 'package:photos/core/event_bus.dart';
import "package:photos/events/people_changed_event.dart";
import 'package:photos/events/subscription_purchased_event.dart';
import "package:photos/face/model/person.dart";
import "package:photos/generated/l10n.dart";
import "package:photos/models/file/file.dart";
import 'package:photos/models/gallery_type.dart';
import "package:photos/models/ml/face/person.dart";
import 'package:photos/models/selected_files.dart';
import 'package:photos/services/collections_service.dart';
import "package:photos/services/machine_learning/face_ml/person/person_service.dart";
@@ -67,7 +67,8 @@ class _AppBarWidgetState extends State<PeopleAppBar> {
};
collectionActions = CollectionActions(CollectionsService.instance);
widget.selectedFiles.addListener(_selectedFilesListener);
_userAuthEventSubscription = Bus.instance.on<SubscriptionPurchasedEvent>().listen((event) {
_userAuthEventSubscription =
Bus.instance.on<SubscriptionPurchasedEvent>().listen((event) {
setState(() {});
});
_appBarTitle = widget.title;
@@ -88,7 +89,8 @@ class _AppBarWidgetState extends State<PeopleAppBar> {
centerTitle: false,
title: Text(
_appBarTitle!,
style: Theme.of(context).textTheme.headlineSmall!.copyWith(fontSize: 16),
style:
Theme.of(context).textTheme.headlineSmall!.copyWith(fontSize: 16),
maxLines: 2,
overflow: TextOverflow.ellipsis,
),
@@ -112,7 +114,8 @@ class _AppBarWidgetState extends State<PeopleAppBar> {
}
try {
await PersonService.instance.updateAttributes(widget.person.remoteID, name: text);
await PersonService.instance
.updateAttributes(widget.person.remoteID, name: text);
if (mounted) {
_appBarTitle = text;
setState(() {});
@@ -132,7 +135,8 @@ class _AppBarWidgetState extends State<PeopleAppBar> {
List<Widget> _getDefaultActions(BuildContext context) {
final List<Widget> actions = <Widget>[];
// If the user has selected files, don't show any actions
if (widget.selectedFiles.files.isNotEmpty || !Configuration.instance.hasConfiguredAccount()) {
if (widget.selectedFiles.files.isNotEmpty ||
!Configuration.instance.hasConfiguredAccount()) {
return actions;
}
@@ -223,7 +227,8 @@ class _AppBarWidgetState extends State<PeopleAppBar> {
unawaited(
Navigator.of(context).push(
MaterialPageRoute(
builder: (context) => PersonReviewClusterSuggestion(widget.person),
builder: (context) =>
PersonReviewClusterSuggestion(widget.person),
),
),
);
@@ -266,11 +271,13 @@ class _AppBarWidgetState extends State<PeopleAppBar> {
bool assignName = false;
await showChoiceDialog(
context,
title: "Are you sure you want to show this person in people section again?",
title:
"Are you sure you want to show this person in people section again?",
firstButtonLabel: "Yes, show person",
firstButtonOnTap: () async {
try {
await PersonService.instance.deletePerson(widget.person.remoteID, onlyMapping: false);
await PersonService.instance
.deletePerson(widget.person.remoteID, onlyMapping: false);
Bus.instance.fire(PeopleChangedEvent());
assignName = true;
} catch (e, s) {

2
mobile/lib/ui/viewer/people/people_page.dart
View File

@@ -7,10 +7,10 @@ import 'package:photos/core/event_bus.dart';
import 'package:photos/events/files_updated_event.dart';
import 'package:photos/events/local_photos_updated_event.dart';
import "package:photos/events/people_changed_event.dart";
import "package:photos/face/model/person.dart";
import 'package:photos/models/file/file.dart';
import 'package:photos/models/file_load_result.dart';
import 'package:photos/models/gallery_type.dart';
import "package:photos/models/ml/face/person.dart";
import 'package:photos/models/selected_files.dart';
import "package:photos/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart";
import "package:photos/services/machine_learning/face_ml/feedback/cluster_feedback.dart";

18
mobile/lib/ui/viewer/people/person_cluster_suggestion.dart
View File

@@ -6,11 +6,11 @@ import "package:flutter/foundation.dart" show kDebugMode;
import "package:flutter/material.dart";
import "package:logging/logging.dart";
import "package:photos/core/event_bus.dart";
import "package:photos/db/ml/db.dart";
import "package:photos/events/people_changed_event.dart";
import "package:photos/face/db.dart";
import "package:photos/face/model/person.dart";
import "package:photos/l10n/l10n.dart";
import "package:photos/models/file/file.dart";
import "package:photos/models/ml/face/person.dart";
import 'package:photos/services/machine_learning/face_ml/feedback/cluster_feedback.dart';
import "package:photos/services/machine_learning/face_ml/person/person_service.dart";
import "package:photos/theme/ente_theme.dart";
@@ -109,7 +109,7 @@ class _PersonClustersState extends State<PersonReviewClusterSuggestion> {
allSuggestions = snapshot.data!;
final numberOfDifferentSuggestions = allSuggestions.length;
final currentSuggestion = allSuggestions[currentSuggestionIndex];
final int clusterID = currentSuggestion.clusterIDToMerge;
final String clusterID = currentSuggestion.clusterIDToMerge;
final double distance = currentSuggestion.distancePersonToCluster;
final bool usingMean = currentSuggestion.usedOnlyMeanForSuggestion;
final List<EnteFile> files = currentSuggestion.filesInCluster;
@@ -182,7 +182,7 @@ class _PersonClustersState extends State<PersonReviewClusterSuggestion> {
}
Future<void> _handleUserClusterChoice(
int clusterID,
String clusterID,
bool yesOrNo,
int numberOfSuggestions,
) async {
@@ -229,7 +229,7 @@ class _PersonClustersState extends State<PersonReviewClusterSuggestion> {
}
Future<void> _rejectSuggestion(
int clusterID,
String clusterID,
int numberOfSuggestions,
) async {
canGiveFeedback = false;
@@ -254,7 +254,7 @@ class _PersonClustersState extends State<PersonReviewClusterSuggestion> {
}
Widget _buildSuggestionView(
int clusterID,
String clusterID,
double distance,
bool usingMean,
List<EnteFile> files,
@@ -379,7 +379,7 @@ class _PersonClustersState extends State<PersonReviewClusterSuggestion> {
Widget _buildThumbnailWidget(
List<EnteFile> files,
int clusterID,
String clusterID,
Future<Map<int, Uint8List?>> generateFaceThumbnails,
) {
return SizedBox(
@@ -433,7 +433,7 @@ class _PersonClustersState extends State<PersonReviewClusterSuggestion> {
List<Widget> _buildThumbnailWidgetsRow(
List<EnteFile> files,
int cluserId,
String cluserId,
Map<int, Uint8List?> faceThumbnails, {
int start = 0,
}) {
@@ -460,7 +460,7 @@ class _PersonClustersState extends State<PersonReviewClusterSuggestion> {
Future<Map<int, Uint8List?>> _generateFaceThumbnails(
List<EnteFile> files,
int clusterID,
String clusterID,
) async {
final futures = <Future<Uint8List?>>[];
for (final file in files) {

22
mobile/lib/ui/viewer/people/person_clusters_page.dart
View File

@@ -3,8 +3,8 @@ import "package:flutter/material.dart";
import "package:logging/logging.dart";
import "package:photos/core/event_bus.dart";
import "package:photos/events/people_changed_event.dart";
import "package:photos/face/model/person.dart";
import "package:photos/models/file/file.dart";
import "package:photos/models/ml/face/person.dart";
import "package:photos/services/machine_learning/face_ml/person/person_service.dart";
import "package:photos/services/search_service.dart";
import "package:photos/theme/ente_theme.dart";
@@ -32,12 +32,13 @@ class _PersonClustersPageState extends State<PersonClustersPage> {
appBar: AppBar(
title: Text(widget.person.data.name),
),
body: FutureBuilder<Map<int, List<EnteFile>>>(
future: SearchService.instance.getClusterFilesForPersonID(widget.person.remoteID),
body: FutureBuilder<Map<String, List<EnteFile>>>(
future: SearchService.instance
.getClusterFilesForPersonID(widget.person.remoteID),
builder: (context, snapshot) {
if (snapshot.hasData) {
final clusters = snapshot.data!;
final List<int> keys = clusters.keys.toList();
final List<String> keys = clusters.keys.toList();
// Sort the clusters by the number of files in each cluster, largest first
keys.sort(
(b, a) => clusters[a]!.length.compareTo(clusters[b]!.length),
@@ -45,7 +46,7 @@ class _PersonClustersPageState extends State<PersonClustersPage> {
return ListView.builder(
itemCount: keys.length,
itemBuilder: (context, index) {
final int clusterID = keys[index];
final String clusterID = keys[index];
final List<EnteFile> files = clusters[clusterID]!;
return InkWell(
onTap: () {
@@ -54,7 +55,7 @@ class _PersonClustersPageState extends State<PersonClustersPage> {
builder: (context) => ClusterPage(
files,
personID: widget.person,
clusterID: index,
clusterID: clusterID,
showNamingBanner: false,
),
),
@@ -91,7 +92,8 @@ class _PersonClustersPageState extends State<PersonClustersPage> {
), // Add some spacing between the thumbnail and the text
Expanded(
child: Padding(
padding: const EdgeInsets.symmetric(horizontal: 8.0),
padding:
const EdgeInsets.symmetric(horizontal: 8.0),
child: Row(
mainAxisAlignment: MainAxisAlignment.spaceBetween,
children: <Widget>[
@@ -103,14 +105,16 @@ class _PersonClustersPageState extends State<PersonClustersPage> {
? GestureDetector(
onTap: () async {
try {
await PersonService.instance.removeClusterToPerson(
await PersonService.instance
.removeClusterToPerson(
personID: widget.person.remoteID,
clusterID: clusterID,
);
_logger.info(
"Removed cluster $clusterID from person ${widget.person.remoteID}",
);
Bus.instance.fire(PeopleChangedEvent());
Bus.instance
.fire(PeopleChangedEvent());
setState(() {});
} catch (e) {
_logger.severe(

2
mobile/lib/ui/viewer/people/person_row_item.dart
View File

@@ -1,6 +1,6 @@
import "package:flutter/material.dart";
import "package:photos/face/model/person.dart";
import "package:photos/models/file/file.dart";
import "package:photos/models/ml/face/person.dart";
import "package:photos/ui/viewer/search/result/person_face_widget.dart";
class PersonRowItem extends StatelessWidget {

10
mobile/lib/ui/viewer/search/result/person_face_widget.dart
View File

@@ -3,10 +3,10 @@ import "dart:typed_data";
import 'package:flutter/widgets.dart';
import "package:photos/db/files_db.dart";
import "package:photos/face/db.dart";
import "package:photos/face/model/face.dart";
import "package:photos/face/model/person.dart";
import "package:photos/db/ml/db.dart";
import 'package:photos/models/file/file.dart';
import "package:photos/models/ml/face/face.dart";
import "package:photos/models/ml/face/person.dart";
import "package:photos/services/machine_learning/face_ml/person/person_service.dart";
import "package:photos/ui/common/loading_widget.dart";
import "package:photos/ui/viewer/file/thumbnail_widget.dart";
@@ -17,7 +17,7 @@ import "package:pool/pool.dart";
class PersonFaceWidget extends StatelessWidget {
final EnteFile file;
final String? personId;
final int? clusterID;
final String? clusterID;
final bool useFullFile;
final bool thumbnailFallback;
final Uint8List? faceCrop;
@@ -83,7 +83,7 @@ class PersonFaceWidget extends StatelessWidget {
final PersonEntity? personEntity =
await PersonService.instance.getPerson(personId!);
if (personEntity != null) {
personAvatarFaceID = personEntity.data.avatarFaceId;
personAvatarFaceID = personEntity.data.avatarFaceID;
}
}
return await FaceMLDataDB.instance.getCoverFaceForPerson(

4
mobile/lib/ui/viewer/search_tab/people_section.dart
View File

@@ -4,8 +4,8 @@ import "package:collection/collection.dart";
import "package:flutter/material.dart";
import "package:photos/core/constants.dart";
import "package:photos/events/event.dart";
import "package:photos/face/model/person.dart";
import "package:photos/models/file/file.dart";
import "package:photos/models/ml/face/person.dart";
import "package:photos/models/search/album_search_result.dart";
import "package:photos/models/search/generic_search_result.dart";
import "package:photos/models/search/recent_searches.dart";
@@ -273,7 +273,7 @@ class SearchExample extends StatelessWidget {
onTap: () async {
final result = await showAssignPersonAction(
context,
clusterID: int.parse(searchResult.name()),
clusterID: searchResult.name(),
);
if (result != null &&
result is (PersonEntity, EnteFile)) {

2
mobile/lib/utils/face/face_box_crop.dart
View File

@@ -2,9 +2,9 @@ import "dart:io" show File;
import "package:flutter/foundation.dart";
import "package:photos/core/cache/lru_map.dart";
import "package:photos/face/model/box.dart";
import "package:photos/models/file/file.dart";
import "package:photos/models/file/file_type.dart";
import "package:photos/models/ml/face/box.dart";
import "package:photos/services/machine_learning/ml_computer.dart";
import "package:photos/utils/file_util.dart";
import "package:photos/utils/thumbnail_util.dart";

2
mobile/lib/utils/face/face_util.dart
View File

@@ -5,7 +5,7 @@ import "package:computer/computer.dart";
import "package:flutter_image_compress/flutter_image_compress.dart";
import "package:image/image.dart" as img;
import "package:logging/logging.dart";
import "package:photos/face/model/box.dart";
import "package:photos/models/ml/face/box.dart";
/// Bounding box of a face.
///

29
mobile/lib/utils/gzip.dart
View File

@@ -28,6 +28,25 @@ Uint8List _gzipUInt8List(Uint8List data) {
return Uint8List.fromList(compressedData);
}
Future<Map<String, dynamic>> decryptAndUnzipJson(
Uint8List key, {
required String encryptedData,
required String header,
}) async {
final Computer computer = Computer.shared();
final response =
await computer.compute<Map<String, dynamic>, Map<String, dynamic>>(
_decryptAndUnzipJsonSync,
param: {
"key": key,
"encryptedData": encryptedData,
"header": header,
},
taskName: "decryptAndUnzipJson",
);
return response;
}
Map<String, dynamic> decryptAndUnzipJsonSync(
Uint8List key, {
required String encryptedData,
@@ -82,3 +101,13 @@ ChaChaEncryptionResult _gzipAndEncryptJsonSync(
) {
return gzipAndEncryptJsonSync(args["jsonData"], args["key"]);
}
Map<String, dynamic> _decryptAndUnzipJsonSync(
Map<String, dynamic> args,
) {
return decryptAndUnzipJsonSync(
args["key"],
encryptedData: args["encryptedData"],
header: args["header"],
);
}

4
mobile/lib/utils/image_ml_util.dart
View File

@@ -6,8 +6,8 @@ import "dart:ui";
import 'package:flutter/painting.dart' as paint show decodeImageFromList;
import 'package:ml_linalg/linalg.dart';
import "package:photos/face/model/box.dart";
import "package:photos/face/model/dimension.dart";
import "package:photos/models/ml/face/box.dart";
import "package:photos/models/ml/face/dimension.dart";
import 'package:photos/services/machine_learning/face_ml/face_alignment/alignment_result.dart';
import 'package:photos/services/machine_learning/face_ml/face_alignment/similarity_transform.dart';
import 'package:photos/services/machine_learning/face_ml/face_detection/detection.dart';

10
mobile/lib/utils/ml_util.dart
View File

@@ -5,13 +5,13 @@ import "dart:typed_data" show ByteData;
import "package:flutter/services.dart" show PlatformException;
import "package:logging/logging.dart";
import "package:photos/core/configuration.dart";
import "package:photos/db/embeddings_db.dart";
import "package:photos/db/files_db.dart";
import "package:photos/face/db.dart";
import "package:photos/face/model/dimension.dart";
import "package:photos/db/ml/db.dart";
import "package:photos/db/ml/embeddings_db.dart";
import "package:photos/models/file/extensions/file_props.dart";
import "package:photos/models/file/file.dart";
import "package:photos/models/file/file_type.dart";
import "package:photos/models/ml/face/dimension.dart";
import "package:photos/models/ml/ml_versions.dart";
import "package:photos/services/filedata/model/file_data.dart";
import "package:photos/services/machine_learning/face_ml/face_recognition_service.dart";
@@ -54,7 +54,7 @@ Future<IndexStatus> getIndexStatus() async {
final int facesIndexedFiles =
await FaceMLDataDB.instance.getIndexedFileCount();
final int clipIndexedFiles =
await EmbeddingsDB.instance.getIndexedFileCount();
await FaceMLDataDB.instance.getClipIndexedFileCount();
final int indexedFiles = math.min(facesIndexedFiles, clipIndexedFiles);
final showIndexedFiles = math.min(indexedFiles, indexableFiles);
@@ -73,7 +73,7 @@ Future<List<FileMLInstruction>> getFilesForMlIndexing() async {
final Map<int, int> faceIndexedFileIDs =
await FaceMLDataDB.instance.getIndexedFileIds();
final Map<int, int> clipIndexedFileIDs =
await EmbeddingsDB.instance.getIndexedFileIds();
await FaceMLDataDB.instance.clipIndexedFileWithVersion();
// Get all regular files and all hidden files
final enteFiles = await SearchService.instance.getAllFiles();

8
mobile/pubspec.lock
View File

@@ -1608,6 +1608,14 @@ packages:
url: "https://pub.dev"
source: hosted
version: "0.3.2+6"
nanoid:
dependency: "direct main"
description:
name: nanoid
sha256: be3f8752d9046c825df2f3914195151eb876f3ad64b9d833dd0b799b77b8759e
url: "https://pub.dev"
source: hosted
version: "1.0.0"
nested:
dependency: transitive
description:

1
mobile/pubspec.yaml
View File

@@ -123,6 +123,7 @@ dependencies:
motionphoto:
git: "https://github.com/ente-io/motionphoto.git"
move_to_background: ^1.0.2
nanoid: ^1.0.0
onnx_dart:
path: plugins/onnx_dart
onnxruntime: