TerribleDev/ente
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Face cooldown (#2000)

Browse Source 
## Description

- User interaction pause in iOS
- Face indexing cooldown
- Pause indexing instantly
- Increase file download limit

## Tests

Tested in debug mode on my pixel phone.
This commit is contained in:
Laurens Priem
2024-06-04 18:33:22 +05:30
committed by GitHub
parent 9faa75e7aa 465760e329
commit 75dee19c62
2 changed files with 118 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files

154
mobile/lib/services/machine_learning/face_ml/face_ml_service.dart
View File

@@ -101,9 +101,11 @@ class FaceMlService {
bool _shouldSyncPeople = false;
bool _isSyncing = false;
final int _fileDownloadLimit = 5;
final int _fileDownloadLimit = 10;
final int _embeddingFetchLimit = 200;
final int _kForceClusteringFaceCount = 8000;
final int _kcooldownLimit = 300;
static const Duration _kCooldownDuration = Duration(minutes: 3);
Future<void> init({bool initializeImageMlIsolate = false}) async {
if (LocalSettings.instance.isFaceIndexingEnabled == false) {
@@ -167,7 +169,8 @@ class FaceMlService {
pauseIndexingAndClustering();
}
});
if (Platform.isIOS && MachineLearningController.instance.isDeviceHealthy) {
if (Platform.isIOS &&
MachineLearningController.instance.isDeviceHealthy) {
_logger.info("Starting face indexing and clustering on iOS from init");
unawaited(indexAndClusterAll());
}
@@ -272,7 +275,8 @@ class FaceMlService {
switch (function) {
case FaceMlOperation.analyzeImage:
final time = DateTime.now();
final FaceMlResult result = await FaceMlService.analyzeImageSync(args);
final FaceMlResult result =
await FaceMlService.analyzeImageSync(args);
dev.log(
"`analyzeImageSync` function executed in ${DateTime.now().difference(time).inMilliseconds} ms",
);
@@ -285,7 +289,8 @@ class FaceMlService {
error: e,
stackTrace: stackTrace,
);
sendPort.send({'error': e.toString(), 'stackTrace': stackTrace.toString()});
sendPort
.send({'error': e.toString(), 'stackTrace': stackTrace.toString()});
}
});
}
@@ -298,6 +303,10 @@ class FaceMlService {
return _functionLock.synchronized(() async {
_resetInactivityTimer();
if (_shouldPauseIndexingAndClustering) {
return null;
}
final completer = Completer<dynamic>();
final answerPort = ReceivePort();
@@ -369,7 +378,8 @@ class FaceMlService {
await sync(forceSync: _shouldSyncPeople);
final int unclusteredFacesCount = await FaceMLDataDB.instance.getUnclusteredFaceCount();
final int unclusteredFacesCount =
await FaceMLDataDB.instance.getUnclusteredFaceCount();
if (unclusteredFacesCount > _kForceClusteringFaceCount) {
_logger.info(
"There are $unclusteredFacesCount unclustered faces, doing clustering first",
@@ -396,10 +406,13 @@ class FaceMlService {
_isIndexingOrClusteringRunning = true;
_logger.info('starting image indexing');
final w = (kDebugMode ? EnteWatch('prepare indexing files') : null)?..start();
final Map<int, int> alreadyIndexedFiles = await FaceMLDataDB.instance.getIndexedFileIds();
final w = (kDebugMode ? EnteWatch('prepare indexing files') : null)
?..start();
final Map<int, int> alreadyIndexedFiles =
await FaceMLDataDB.instance.getIndexedFileIds();
w?.log('getIndexedFileIds');
final List<EnteFile> enteFiles = await SearchService.instance.getAllFiles();
final List<EnteFile> enteFiles =
await SearchService.instance.getAllFiles();
w?.log('getAllFiles');
// Make sure the image conversion isolate is spawned
@@ -408,6 +421,7 @@ class FaceMlService {
int fileAnalyzedCount = 0;
int fileSkippedCount = 0;
int cooldownCount = 0;
final stopwatch = Stopwatch()..start();
final List<EnteFile> filesWithLocalID = <EnteFile>[];
final List<EnteFile> filesWithoutLocalID = <EnteFile>[];
@@ -426,7 +440,8 @@ class FaceMlService {
}
}
w?.log('sifting through all normal files');
final List<EnteFile> hiddenFiles = await SearchService.instance.getHiddenFiles();
final List<EnteFile> hiddenFiles =
await SearchService.instance.getHiddenFiles();
w?.log('getHiddenFiles: ${hiddenFiles.length} hidden files');
for (final EnteFile enteFile in hiddenFiles) {
if (_skipAnalysisEnteFile(enteFile, alreadyIndexedFiles)) {
@@ -442,21 +457,22 @@ class FaceMlService {
sortedBylocalID.addAll(filesWithoutLocalID);
sortedBylocalID.addAll(hiddenFilesToIndex);
w?.log('preparing all files to index');
final List<List<EnteFile>> chunks = sortedBylocalID.chunks(_embeddingFetchLimit);
final List<List<EnteFile>> chunks =
sortedBylocalID.chunks(_embeddingFetchLimit);
int fetchedCount = 0;
outerLoop:
for (final chunk in chunks) {
final futures = <Future<bool>>[];
if (LocalSettings.instance.remoteFetchEnabled) {
try {
final Set<int> fileIds = {}; // if there are duplicates here server returns 400
final Set<int> fileIds =
{}; // if there are duplicates here server returns 400
// Try to find embeddings on the remote server
for (final f in chunk) {
fileIds.add(f.uploadedFileID!);
}
_logger.info('starting remote fetch for ${fileIds.length} files');
final res = await RemoteFileMLService.instance.getFilessEmbedding(fileIds);
final res =
await RemoteFileMLService.instance.getFilessEmbedding(fileIds);
_logger.info('fetched ${res.mlData.length} embeddings');
fetchedCount += res.mlData.length;
final List<Face> faces = [];
@@ -478,7 +494,8 @@ class FaceMlService {
faces.add(f);
}
}
remoteFileIdToVersion[fileMl.fileID] = fileMl.faceEmbedding.version;
remoteFileIdToVersion[fileMl.fileID] =
fileMl.faceEmbedding.version;
}
if (res.noEmbeddingFileIDs.isNotEmpty) {
_logger.info(
@@ -495,7 +512,8 @@ class FaceMlService {
for (final entry in remoteFileIdToVersion.entries) {
alreadyIndexedFiles[entry.key] = entry.value;
}
_logger.info('already indexed files ${remoteFileIdToVersion.length}');
_logger
.info('already indexed files ${remoteFileIdToVersion.length}');
} catch (e, s) {
_logger.severe("err while getting files embeddings", e, s);
if (retryFetchCount < 1000) {
@@ -519,6 +537,7 @@ class FaceMlService {
}
final smallerChunks = chunk.chunks(_fileDownloadLimit);
for (final smallestChunk in smallerChunks) {
final futures = <Future<bool>>[];
if (!await canUseHighBandwidth()) {
_logger.info(
'stopping indexing because user is not connected to wifi',
@@ -545,12 +564,22 @@ class FaceMlService {
(previousValue, element) => previousValue + (element ? 1 : 0),
);
fileAnalyzedCount += sumFutures;
if (fileAnalyzedCount > _kcooldownLimit) {
_logger.info(
'Reached ${cooldownCount * _kcooldownLimit + fileAnalyzedCount} indexed files, cooling down to prevent OS from killing the app',
);
cooldownCount++;
fileAnalyzedCount -= _kcooldownLimit;
await Future.delayed(_kCooldownDuration);
_logger.info('cooldown done, continuing indexing');
}
}
}
stopwatch.stop();
_logger.info(
"`indexAllImages()` finished. Fetched $fetchedCount and analyzed $fileAnalyzedCount images, in ${stopwatch.elapsed.inSeconds} seconds (avg of ${stopwatch.elapsed.inSeconds / fileAnalyzedCount} seconds per image, skipped $fileSkippedCount images)",
"`indexAllImages()` finished. Fetched $fetchedCount and analyzed ${cooldownCount * _kcooldownLimit + fileAnalyzedCount} images, in ${stopwatch.elapsed.inSeconds} seconds (avg of ${stopwatch.elapsed.inSeconds / fileAnalyzedCount} seconds per image, skipped $fileSkippedCount images, $cooldownCount cooldowns)",
);
_logStatus();
} catch (e, s) {
@@ -578,9 +607,10 @@ class FaceMlService {
_showClusteringIsHappening = true;
// Get a sense of the total number of faces in the database
final int totalFaces =
await FaceMLDataDB.instance.getTotalFaceCount(minFaceScore: minFaceScore);
final fileIDToCreationTime = await FilesDB.instance.getFileIDToCreationTime();
final int totalFaces = await FaceMLDataDB.instance
.getTotalFaceCount(minFaceScore: minFaceScore);
final fileIDToCreationTime =
await FilesDB.instance.getFileIDToCreationTime();
final startEmbeddingFetch = DateTime.now();
// read all embeddings
final result = await FaceMLDataDB.instance.getFaceInfoForClustering(
@@ -598,7 +628,8 @@ class FaceMlService {
}
// sort the embeddings based on file creation time, newest first
allFaceInfoForClustering.sort((b, a) {
return fileIDToCreationTime[a.fileID]!.compareTo(fileIDToCreationTime[b.fileID]!);
return fileIDToCreationTime[a.fileID]!
.compareTo(fileIDToCreationTime[b.fileID]!);
});
_logger.info(
'Getting and sorting embeddings took ${DateTime.now().difference(startEmbeddingFetch).inMilliseconds} ms for ${allFaceInfoForClustering.length} embeddings'
@@ -654,7 +685,8 @@ class FaceMlService {
}
}
final clusteringResult = await FaceClusteringService.instance.predictLinearIsolate(
final clusteringResult =
await FaceClusteringService.instance.predictLinearIsolate(
faceInfoForClustering.toSet(),
fileIDToCreationTime: fileIDToCreationTime,
offset: offset,
@@ -665,13 +697,17 @@ class FaceMlService {
return;
}
await FaceMLDataDB.instance.updateFaceIdToClusterId(clusteringResult.newFaceIdToCluster);
await FaceMLDataDB.instance.clusterSummaryUpdate(clusteringResult.newClusterSummaries);
await FaceMLDataDB.instance
.updateFaceIdToClusterId(clusteringResult.newFaceIdToCluster);
await FaceMLDataDB.instance
.clusterSummaryUpdate(clusteringResult.newClusterSummaries);
Bus.instance.fire(PeopleChangedEvent());
for (final faceInfo in faceInfoForClustering) {
faceInfo.clusterId ??= clusteringResult.newFaceIdToCluster[faceInfo.faceID];
faceInfo.clusterId ??=
clusteringResult.newFaceIdToCluster[faceInfo.faceID];
}
for (final clusterUpdate in clusteringResult.newClusterSummaries.entries) {
for (final clusterUpdate
in clusteringResult.newClusterSummaries.entries) {
oldClusterSummaries[clusterUpdate.key] = clusterUpdate.value;
}
_logger.info(
@@ -687,7 +723,8 @@ class FaceMlService {
} else {
final clusterStartTime = DateTime.now();
// Cluster the embeddings using the linear clustering algorithm, returning a map from faceID to clusterID
final clusteringResult = await FaceClusteringService.instance.predictLinearIsolate(
final clusteringResult =
await FaceClusteringService.instance.predictLinearIsolate(
allFaceInfoForClustering.toSet(),
fileIDToCreationTime: fileIDToCreationTime,
oldClusterSummaries: oldClusterSummaries,
@@ -705,8 +742,10 @@ class FaceMlService {
_logger.info(
'Updating ${clusteringResult.newFaceIdToCluster.length} FaceIDs with clusterIDs in the DB',
);
await FaceMLDataDB.instance.updateFaceIdToClusterId(clusteringResult.newFaceIdToCluster);
await FaceMLDataDB.instance.clusterSummaryUpdate(clusteringResult.newClusterSummaries);
await FaceMLDataDB.instance
.updateFaceIdToClusterId(clusteringResult.newFaceIdToCluster);
await FaceMLDataDB.instance
.clusterSummaryUpdate(clusteringResult.newClusterSummaries);
Bus.instance.fire(PeopleChangedEvent());
_logger.info('Done updating FaceIDs with clusterIDs in the DB, in '
'${DateTime.now().difference(clusterDoneTime).inSeconds} seconds');
@@ -739,7 +778,8 @@ class FaceMlService {
allLandmarksEqual = false;
break;
}
if (face.detection.landmarks.any((landmark) => landmark.x != landmark.y)) {
if (face.detection.landmarks
.any((landmark) => landmark.x != landmark.y)) {
allLandmarksEqual = false;
break;
}
@@ -748,7 +788,10 @@ class FaceMlService {
debugPrint("Discarding remote embedding for fileID ${fileMl.fileID} "
"because landmarks are equal");
debugPrint(
fileMl.faceEmbedding.faces.map((e) => e.detection.landmarks.toString()).toList().toString(),
fileMl.faceEmbedding.faces
.map((e) => e.detection.landmarks.toString())
.toList()
.toString(),
);
return true;
}
@@ -772,9 +815,11 @@ class FaceMlService {
// disposeImageIsolateAfterUse: false,
);
if (result == null) {
_logger.severe(
"Failed to analyze image with uploadedFileID: ${enteFile.uploadedFileID}",
);
if (!_shouldPauseIndexingAndClustering) {
_logger.severe(
"Failed to analyze image with uploadedFileID: ${enteFile.uploadedFileID}",
);
}
return false;
}
final List<Face> faces = [];
@@ -786,9 +831,11 @@ class FaceMlService {
Face.empty(result.fileId, error: result.errorOccured),
);
} else {
if (result.decodedImageSize.width == -1 || result.decodedImageSize.height == -1) {
_logger.severe("decodedImageSize is not stored correctly for image with "
"ID: ${enteFile.uploadedFileID}");
if (result.decodedImageSize.width == -1 ||
result.decodedImageSize.height == -1) {
_logger
.severe("decodedImageSize is not stored correctly for image with "
"ID: ${enteFile.uploadedFileID}");
_logger.info(
"Using aligned image size for image with ID: ${enteFile.uploadedFileID}. This size is ${result.decodedImageSize.width}x${result.decodedImageSize.height} compared to size of ${enteFile.width}x${enteFile.height} in the metadata",
);
@@ -873,7 +920,8 @@ class FaceMlService {
_checkEnteFileForID(enteFile);
await ensureInitialized();
final String? filePath = await _getImagePathForML(enteFile, typeOfData: FileDataForML.fileData);
final String? filePath =
await _getImagePathForML(enteFile, typeOfData: FileDataForML.fileData);
if (filePath == null) {
_logger.severe(
@@ -892,13 +940,17 @@ class FaceMlService {
{
"enteFileID": enteFile.uploadedFileID ?? -1,
"filePath": filePath,
"faceDetectionAddress": FaceDetectionService.instance.sessionAddress,
"faceEmbeddingAddress": FaceEmbeddingService.instance.sessionAddress,
"faceDetectionAddress":
FaceDetectionService.instance.sessionAddress,
"faceEmbeddingAddress":
FaceEmbeddingService.instance.sessionAddress,
}
),
) as String?;
if (resultJsonString == null) {
_logger.severe('Analyzing image in isolate is giving back null');
if (!_shouldPauseIndexingAndClustering) {
_logger.severe('Analyzing image in isolate is giving back null');
}
return null;
}
result = FaceMlResult.fromJsonString(resultJsonString);
@@ -947,7 +999,8 @@ class FaceMlService {
stopwatch.reset();
// Get the faces
final List<FaceDetectionRelative> faceDetectionResult = await FaceMlService.detectFacesSync(
final List<FaceDetectionRelative> faceDetectionResult =
await FaceMlService.detectFacesSync(
image,
imgByteData,
faceDetectionAddress,
@@ -968,7 +1021,8 @@ class FaceMlService {
stopwatch.reset();
// Align the faces
final Float32List faceAlignmentResult = await FaceMlService.alignFacesSync(
final Float32List faceAlignmentResult =
await FaceMlService.alignFacesSync(
image,
imgByteData,
faceDetectionResult,
@@ -1035,8 +1089,9 @@ class FaceMlService {
}
}
if (file == null) {
_logger
.warning("Could not get file for $enteFile of type ${enteFile.fileType.toString()}");
_logger.warning(
"Could not get file for $enteFile of type ${enteFile.fileType.toString()}",
);
imagePath = null;
break;
}
@@ -1088,7 +1143,8 @@ class FaceMlService {
}) async {
try {
// Get the bounding boxes of the faces
final (List<FaceDetectionRelative> faces, dataSize) = await FaceDetectionService.predictSync(
final (List<FaceDetectionRelative> faces, dataSize) =
await FaceDetectionService.predictSync(
image,
imageByteData,
interpreterAddress,
@@ -1198,7 +1254,8 @@ class FaceMlService {
}
bool _skipAnalysisEnteFile(EnteFile enteFile, Map<int, int> indexedFileIds) {
if (_isIndexingOrClusteringRunning == false || _mlControllerStatus == false) {
if (_isIndexingOrClusteringRunning == false ||
_mlControllerStatus == false) {
return true;
}
// Skip if the file is not uploaded or not owned by the user
@@ -1212,7 +1269,8 @@ class FaceMlService {
// Skip if the file is already analyzed with the latest ml version
final id = enteFile.uploadedFileID!;
return indexedFileIds.containsKey(id) && indexedFileIds[id]! >= faceMlVersion;
return indexedFileIds.containsKey(id) &&
indexedFileIds[id]! >= faceMlVersion;
}
bool _cannotRunMLFunction({String function = ""}) {

28
mobile/lib/services/machine_learning/machine_learning_controller.dart
View File

@@ -18,7 +18,7 @@ class MachineLearningController {
static const kMaximumTemperature = 42; // 42 degree celsius
static const kMinimumBatteryLevel = 20; // 20%
static const kDefaultInteractionTimeout = Duration(seconds: 15);
final kDefaultInteractionTimeout = Duration(seconds: Platform.isIOS ? 5 : 15);
static const kUnhealthyStates = ["over_heat", "over_voltage", "dead"];
bool _isDeviceHealthy = true;
@@ -31,29 +31,26 @@ class MachineLearningController {
void init() {
_logger.info('init called');
if (Platform.isAndroid) {
_startInteractionTimer();
BatteryInfoPlugin()
.androidBatteryInfoStream
.listen((AndroidBatteryInfo? batteryInfo) {
_onAndroidBatteryStateUpdate(batteryInfo);
});
}
_startInteractionTimer(kDefaultInteractionTimeout);
if (Platform.isIOS) {
BatteryInfoPlugin()
.iosBatteryInfoStream
.listen((IosBatteryInfo? batteryInfo) {
_oniOSBatteryStateUpdate(batteryInfo);
});
}
if (Platform.isAndroid) {
BatteryInfoPlugin()
.androidBatteryInfoStream
.listen((AndroidBatteryInfo? batteryInfo) {
_onAndroidBatteryStateUpdate(batteryInfo);
});
}
_fireControlEvent();
_logger.info('init done');
}
void onUserInteraction() {
if (Platform.isIOS) {
return;
}
if (!_isUserInteracting) {
_logger.info("User is interacting with the app");
_isUserInteracting = true;
@@ -63,8 +60,7 @@ class MachineLearningController {
}
bool _canRunGivenUserInteraction() {
return (Platform.isIOS ? true : !_isUserInteracting) ||
mlInteractionOverride;
return !_isUserInteracting || mlInteractionOverride;
}
void forceOverrideML({required bool turnOn}) {
@@ -84,7 +80,7 @@ class MachineLearningController {
}
}
void _startInteractionTimer({Duration timeout = kDefaultInteractionTimeout}) {
void _startInteractionTimer(Duration timeout) {
_userInteractionTimer = Timer(timeout, () {
_logger.info("User is not interacting with the app");
_isUserInteracting = false;
@@ -94,7 +90,7 @@ class MachineLearningController {
void _resetTimer() {
_userInteractionTimer.cancel();
_startInteractionTimer();
_startInteractionTimer(kDefaultInteractionTimeout);
}
void _onAndroidBatteryStateUpdate(AndroidBatteryInfo? batteryInfo) {