From 6adbf176303ec90dd21eb12924be4334d38639eb Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Thu, 22 Aug 2024 14:52:09 +0200 Subject: [PATCH 01/15] [mob][photos] Only download models on wifi --- .../machine_learning/ml_computer.dart | 9 ++++--- .../services/machine_learning/ml_model.dart | 25 ++++++++++++++++--- .../services/machine_learning/ml_service.dart | 7 ++++++ 3 files changed, 34 insertions(+), 7 deletions(-) diff --git a/mobile/lib/services/machine_learning/ml_computer.dart b/mobile/lib/services/machine_learning/ml_computer.dart index ed9cc719d4..afbd511c62 100644 --- a/mobile/lib/services/machine_learning/ml_computer.dart +++ b/mobile/lib/services/machine_learning/ml_computer.dart @@ -158,8 +158,8 @@ class MLComputer { } Future> runClipText(String query) async { - await _ensureLoadedClipTextModel(); try { + await _ensureLoadedClipTextModel(); final int clipAddress = ClipTextEncoder.instance.sessionAddress; final textEmbedding = await _runInIsolate( ( @@ -195,9 +195,10 @@ class MLComputer { // Load ClipText model final String modelName = ClipTextEncoder.instance.modelName; - final String modelRemotePath = ClipTextEncoder.instance.modelRemotePath; - final String modelPath = - await RemoteAssetsService.instance.getAssetPath(modelRemotePath); + final String? modelPath = await ClipTextEncoder.instance.downloadModelSafe(); + if (modelPath == null) { + throw Exception("Could not download clip text model, no wifi"); + } final address = await _runInIsolate( ( MLComputerOperation.loadModel, diff --git a/mobile/lib/services/machine_learning/ml_model.dart b/mobile/lib/services/machine_learning/ml_model.dart index daad0ad61a..a3a05e5f14 100644 --- a/mobile/lib/services/machine_learning/ml_model.dart +++ b/mobile/lib/services/machine_learning/ml_model.dart @@ -5,6 +5,7 @@ import "package:onnx_dart/onnx_dart.dart"; import "package:onnxruntime/onnxruntime.dart"; import "package:photos/services/machine_learning/onnx_env.dart"; import "package:photos/services/remote_assets_service.dart"; +import "package:photos/utils/network_util.dart"; import "package:synchronized/synchronized.dart"; abstract class MlModel { @@ -32,6 +33,7 @@ abstract class MlModel { bool _isNativePluginInitialized = false; int _nativePluginSessionIndex = -1; + /// WARNING: If [downloadModel] was not first called, this method will download the model first using high bandwidth. Future<(String, String)> getModelNameAndPath() async { return _downloadModelLock.synchronized(() async { final path = @@ -40,12 +42,29 @@ abstract class MlModel { }); } - Future downloadModel([bool forceRefresh = false]) async { + Future downloadModelSafe() async { + if (await RemoteAssetsService.instance.hasAsset(modelRemotePath)) { + return await RemoteAssetsService.instance.getAssetPath(modelRemotePath); + } else { + if (await canUseHighBandwidth()) { + return await downloadModel(); + } else { + logger.warning( + 'Cannot return model path as it is not available locally and high bandwidth is not available.', + ); + return null; + } + } + } + + Future downloadModel([bool forceRefresh = false]) async { return _downloadModelLock.synchronized(() async { if (forceRefresh) { - await RemoteAssetsService.instance.getAssetIfUpdated(modelRemotePath); + final file = await RemoteAssetsService.instance + .getAssetIfUpdated(modelRemotePath); + return file!.path; } else { - await RemoteAssetsService.instance.getAsset(modelRemotePath); + return await RemoteAssetsService.instance.getAssetPath(modelRemotePath); } }); } diff --git a/mobile/lib/services/machine_learning/ml_service.dart b/mobile/lib/services/machine_learning/ml_service.dart index 7fb523a917..d37e358dd8 100644 --- a/mobile/lib/services/machine_learning/ml_service.dart +++ b/mobile/lib/services/machine_learning/ml_service.dart @@ -171,6 +171,8 @@ class MLService { 'stopping indexing because user is not connected to wifi', ); break; + } else { + await _ensureDownloadedModels(); } final futures = >[]; for (final instruction in chunk) { @@ -500,6 +502,11 @@ class MLService { _logger.finest("Models already downloaded"); return; } + final goodInternet = await canUseHighBandwidth(); + if (!goodInternet) { + _logger.info("Cannot download models because user is not connected to wifi"); + return; + } _logger.info('Downloading models'); await Future.wait([ FaceDetectionService.instance.downloadModel(forceRefresh), From 51cf793012027b6dbb4edff7f7c75b4c68d3dd1d Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Thu, 22 Aug 2024 14:52:47 +0200 Subject: [PATCH 02/15] [mob][photos] Remove unneeded beta check --- mobile/lib/ui/settings_page.dart | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/mobile/lib/ui/settings_page.dart b/mobile/lib/ui/settings_page.dart index e002ad8463..de3882cfae 100644 --- a/mobile/lib/ui/settings_page.dart +++ b/mobile/lib/ui/settings_page.dart @@ -144,9 +144,7 @@ class SettingsPage extends StatelessWidget { if (hasLoggedIn && flagService.internalUser) { contents.addAll([sectionSpacing, const DebugSectionWidget()]); - if (flagService.isBetaUser) { - contents.addAll([sectionSpacing, const MLDebugSectionWidget()]); - } + contents.addAll([sectionSpacing, const MLDebugSectionWidget()]); } contents.add(const AppVersionWidget()); contents.add(const DeveloperSettingsWidget()); From eaf136d34f0e81edd5dd5f0691b865acb513615e Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Thu, 22 Aug 2024 15:37:03 +0200 Subject: [PATCH 03/15] [mob][photos] notify user if model download is paused --- .../services/machine_learning/ml_service.dart | 11 +++++- .../machine_learning_settings_page.dart | 37 +++++++++++++------ 2 files changed, 36 insertions(+), 12 deletions(-) diff --git a/mobile/lib/services/machine_learning/ml_service.dart b/mobile/lib/services/machine_learning/ml_service.dart index d37e358dd8..ebe56b893e 100644 --- a/mobile/lib/services/machine_learning/ml_service.dart +++ b/mobile/lib/services/machine_learning/ml_service.dart @@ -493,6 +493,13 @@ class MLService { } } + void triggerModelsDownload() { + if (!areModelsDownloaded && !_downloadModelLock.locked) { + _logger.info("Models not downloaded, starting download"); + unawaited(_ensureDownloadedModels()); + } + } + Future _ensureDownloadedModels([bool forceRefresh = false]) async { if (_downloadModelLock.locked) { _logger.finest("Download models already in progress"); @@ -504,7 +511,9 @@ class MLService { } final goodInternet = await canUseHighBandwidth(); if (!goodInternet) { - _logger.info("Cannot download models because user is not connected to wifi"); + _logger.info( + "Cannot download models because user is not connected to wifi", + ); return; } _logger.info('Downloading models'); diff --git a/mobile/lib/ui/settings/machine_learning_settings_page.dart b/mobile/lib/ui/settings/machine_learning_settings_page.dart index dbd96e8a43..a98fac2c6d 100644 --- a/mobile/lib/ui/settings/machine_learning_settings_page.dart +++ b/mobile/lib/ui/settings/machine_learning_settings_page.dart @@ -27,6 +27,7 @@ import "package:photos/ui/components/title_bar_widget.dart"; import "package:photos/ui/components/toggle_switch_widget.dart"; import "package:photos/ui/settings/ml/enable_ml_consent.dart"; import "package:photos/utils/ml_util.dart"; +import "package:photos/utils/network_util.dart"; import "package:photos/utils/wakelock_util.dart"; class MachineLearningSettingsPage extends StatefulWidget { @@ -257,6 +258,8 @@ class ModelLoadingState extends StatefulWidget { class _ModelLoadingStateState extends State { StreamSubscription<(String, int, int)>? _progressStream; final Map _progressMap = {}; + Timer? _timer; + @override void initState() { _progressStream = @@ -277,6 +280,9 @@ class _ModelLoadingStateState extends State { setState(() {}); } }); + _timer = Timer.periodic(const Duration(seconds: 10), (timer) { + setState(() {}); + }); super.initState(); } @@ -284,6 +290,7 @@ class _ModelLoadingStateState extends State { void dispose() { super.dispose(); _progressStream?.cancel(); + _timer?.cancel(); } @override @@ -292,8 +299,25 @@ class _ModelLoadingStateState extends State { children: [ MenuSectionTitle(title: S.of(context).status), MenuItemWidget( - captionedTextWidget: CaptionedTextWidget( - title: _getTitle(context), + captionedTextWidget: FutureBuilder( + future: canUseHighBandwidth(), + builder: (context, snapshot) { + if (snapshot.hasData) { + if (snapshot.data!) { + MLService.instance.triggerModelsDownload(); + return CaptionedTextWidget( + title: S.of(context).loadingModel, + key: const ValueKey("loading_model"), + ); + } else { + return CaptionedTextWidget( + title: S.of(context).waitingForWifi, + key: const ValueKey("waiting_for_wifi"), + ); + } + } + return const CaptionedTextWidget(title: ""); + }, ), trailingWidget: EnteLoadingWidget( size: 12, @@ -322,15 +346,6 @@ class _ModelLoadingStateState extends State { ], ); } - - String _getTitle(BuildContext context) { - // TODO: uncomment code below to actually check for high bandwidth - // final usableConnection = await canUseHighBandwidth(); - // if (!usableConnection) { - // return S.of(context).waitingForWifi; - // } - return S.of(context).loadingModel; - } } class MLStatusWidget extends StatefulWidget { From 1d50cdeeca5a64421ca243dfa8c39db1c0b2a86c Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Thu, 22 Aug 2024 16:18:16 +0200 Subject: [PATCH 04/15] [mob][photos] Fix ml not stopping on interaction --- mobile/lib/services/machine_learning/ml_service.dart | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mobile/lib/services/machine_learning/ml_service.dart b/mobile/lib/services/machine_learning/ml_service.dart index ebe56b893e..36a550098b 100644 --- a/mobile/lib/services/machine_learning/ml_service.dart +++ b/mobile/lib/services/machine_learning/ml_service.dart @@ -165,12 +165,13 @@ class MLService { int fileAnalyzedCount = 0; final Stopwatch stopwatch = Stopwatch()..start(); + stream: await for (final chunk in instructionStream) { if (!await canUseHighBandwidth()) { _logger.info( 'stopping indexing because user is not connected to wifi', ); - break; + break stream; } else { await _ensureDownloadedModels(); } @@ -178,7 +179,7 @@ class MLService { for (final instruction in chunk) { if (_shouldPauseIndexingAndClustering) { _logger.info("indexAllImages() was paused, stopping"); - break; + break stream; } await _ensureLoadedModels(instruction); futures.add(processImage(instruction)); From 5eb153a3106923c9cecd4b40974f7f5f4802d56f Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Thu, 22 Aug 2024 16:23:14 +0200 Subject: [PATCH 05/15] [mob][photos] Prevent double ML sessions in debug iOS --- mobile/lib/services/machine_learning/ml_service.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobile/lib/services/machine_learning/ml_service.dart b/mobile/lib/services/machine_learning/ml_service.dart index 36a550098b..10ee47198e 100644 --- a/mobile/lib/services/machine_learning/ml_service.dart +++ b/mobile/lib/services/machine_learning/ml_service.dart @@ -551,7 +551,7 @@ class MLService { } bool _cannotRunMLFunction({String function = ""}) { - if (kDebugMode && Platform.isIOS) { + if (kDebugMode && Platform.isIOS && !_isIndexingOrClusteringRunning) { return false; } if (_isIndexingOrClusteringRunning) { From bb6ac34920a919020d95fc4591c41805fb9c1a6e Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Thu, 22 Aug 2024 16:37:09 +0200 Subject: [PATCH 06/15] [mob][photos] Keep triggering ML on ML settings page --- mobile/lib/services/machine_learning/ml_service.dart | 10 ++++++++++ .../ui/settings/machine_learning_settings_page.dart | 1 + 2 files changed, 11 insertions(+) diff --git a/mobile/lib/services/machine_learning/ml_service.dart b/mobile/lib/services/machine_learning/ml_service.dart index 10ee47198e..9321c3f96a 100644 --- a/mobile/lib/services/machine_learning/ml_service.dart +++ b/mobile/lib/services/machine_learning/ml_service.dart @@ -55,6 +55,7 @@ class MLService { bool _showClusteringIsHappening = false; bool _mlControllerStatus = false; bool _isIndexingOrClusteringRunning = false; + bool _isRunningML = false; bool _shouldPauseIndexingAndClustering = false; static const int _fileDownloadLimit = 10; @@ -118,6 +119,7 @@ class MLService { _mlControllerStatus = true; } if (_cannotRunMLFunction() && !force) return; + _isRunningML = true; await sync(); @@ -134,6 +136,14 @@ class MLService { } catch (e, s) { _logger.severe("runAllML failed", e, s); rethrow; + } finally { + _isRunningML = false; + } + } + + void triggerML() { + if (_mlControllerStatus && !_isIndexingOrClusteringRunning && !_isRunningML) { + unawaited(runAllML()); } } diff --git a/mobile/lib/ui/settings/machine_learning_settings_page.dart b/mobile/lib/ui/settings/machine_learning_settings_page.dart index a98fac2c6d..040126aec3 100644 --- a/mobile/lib/ui/settings/machine_learning_settings_page.dart +++ b/mobile/lib/ui/settings/machine_learning_settings_page.dart @@ -363,6 +363,7 @@ class MLStatusWidgetState extends State { void initState() { super.initState(); _timer = Timer.periodic(const Duration(seconds: 10), (timer) { + MLService.instance.triggerML(); setState(() {}); }); } From f0c546bc3ba4463ca3deea08f123880dead1babc Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Fri, 23 Aug 2024 12:01:35 +0200 Subject: [PATCH 07/15] [mob][photos] clip bilinear --- mobile/lib/utils/image_ml_util.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobile/lib/utils/image_ml_util.dart b/mobile/lib/utils/image_ml_util.dart index 843f535343..6e44f1430f 100644 --- a/mobile/lib/utils/image_ml_util.dart +++ b/mobile/lib/utils/image_ml_util.dart @@ -231,7 +231,7 @@ Future preprocessImageClip( const int blueOff = 2 * requiredHeight * requiredWidth; for (var h = 0 + heightOffset; h < scaledHeight - heightOffset; h++) { for (var w = 0 + widthOffset; w < scaledWidth - widthOffset; w++) { - final Color pixel = _getPixelBicubic( + final Color pixel = _getPixelBilinear( w / scale, h / scale, image, From 017829685066126eeee5c832e5de199c47f52a1f Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Fri, 23 Aug 2024 13:50:04 +0200 Subject: [PATCH 08/15] [mob][photos] Stop indexing queue when ML is disabled --- mobile/lib/ui/settings/machine_learning_settings_page.dart | 1 + 1 file changed, 1 insertion(+) diff --git a/mobile/lib/ui/settings/machine_learning_settings_page.dart b/mobile/lib/ui/settings/machine_learning_settings_page.dart index 040126aec3..cf83d4a800 100644 --- a/mobile/lib/ui/settings/machine_learning_settings_page.dart +++ b/mobile/lib/ui/settings/machine_learning_settings_page.dart @@ -204,6 +204,7 @@ class _MachineLearningSettingsPageState await SemanticSearchService.instance.init(); unawaited(MLService.instance.runAllML(force: true)); } else { + MLService.instance.pauseIndexingAndClustering(); await UserRemoteFlagService.instance .setBoolValue(UserRemoteFlagService.mlEnabled, false); } From c3e8a81845fa3e9b32aa837a1007c7af1ea6c0bd Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Fri, 23 Aug 2024 13:57:40 +0200 Subject: [PATCH 09/15] [mob][photos] Lint warnings --- .../debug/ml_debug_section_widget.dart | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/mobile/lib/ui/settings/debug/ml_debug_section_widget.dart b/mobile/lib/ui/settings/debug/ml_debug_section_widget.dart index 7b888e066b..0b375cce60 100644 --- a/mobile/lib/ui/settings/debug/ml_debug_section_widget.dart +++ b/mobile/lib/ui/settings/debug/ml_debug_section_widget.dart @@ -20,7 +20,7 @@ import "package:photos/utils/dialog_util.dart"; import 'package:photos/utils/toast_util.dart'; class MLDebugSectionWidget extends StatefulWidget { - const MLDebugSectionWidget({Key? key}) : super(key: key); + const MLDebugSectionWidget({super.key}); @override State createState() => _MLDebugSectionWidgetState(); @@ -54,7 +54,7 @@ class _MLDebugSectionWidgetState extends State { } Widget _getSectionOptions(BuildContext context) { - final Logger _logger = Logger("MLDebugSectionWidget"); + final Logger logger = Logger("MLDebugSectionWidget"); return Column( children: [ MenuItemWidget( @@ -84,7 +84,7 @@ class _MLDebugSectionWidgetState extends State { setState(() {}); } } catch (e, s) { - _logger.warning('indexing failed ', e, s); + logger.warning('indexing failed ', e, s); await showGenericErrorDialog(context: context, error: e); } }, @@ -106,7 +106,7 @@ class _MLDebugSectionWidgetState extends State { setState(() {}); } } catch (e, s) { - _logger.warning('Remote fetch toggle failed ', e, s); + logger.warning('Remote fetch toggle failed ', e, s); await showGenericErrorDialog(context: context, error: e); } }, @@ -132,7 +132,7 @@ class _MLDebugSectionWidgetState extends State { setState(() {}); } } catch (e, s) { - _logger.warning('debugIndexingDisabled toggle failed ', e, s); + logger.warning('debugIndexingDisabled toggle failed ', e, s); await showGenericErrorDialog(context: context, error: e); } }, @@ -150,7 +150,7 @@ class _MLDebugSectionWidgetState extends State { MLService.instance.debugIndexingDisabled = false; unawaited(MLService.instance.runAllML()); } catch (e, s) { - _logger.warning('indexAndClusterAll failed ', e, s); + logger.warning('indexAndClusterAll failed ', e, s); await showGenericErrorDialog(context: context, error: e); } }, @@ -168,7 +168,7 @@ class _MLDebugSectionWidgetState extends State { MLService.instance.debugIndexingDisabled = false; unawaited(MLService.instance.indexAllImages()); } catch (e, s) { - _logger.warning('indexing failed ', e, s); + logger.warning('indexing failed ', e, s); await showGenericErrorDialog(context: context, error: e); } }, @@ -198,7 +198,7 @@ class _MLDebugSectionWidgetState extends State { Bus.instance.fire(PeopleChangedEvent()); showShortToast(context, "Done"); } catch (e, s) { - _logger.warning('clustering failed ', e, s); + logger.warning('clustering failed ', e, s); await showGenericErrorDialog(context: context, error: e); } }, @@ -224,7 +224,7 @@ class _MLDebugSectionWidgetState extends State { }); } } catch (e, s) { - _logger.warning('Checking for mixed clusters failed', e, s); + logger.warning('Checking for mixed clusters failed', e, s); await showGenericErrorDialog(context: context, error: e); } }, @@ -243,7 +243,7 @@ class _MLDebugSectionWidgetState extends State { Bus.instance.fire(PeopleChangedEvent()); showShortToast(context, "Done"); } catch (e, s) { - _logger.warning('sync person mappings failed ', e, s); + logger.warning('sync person mappings failed ', e, s); await showGenericErrorDialog(context: context, error: e); } }, @@ -270,7 +270,7 @@ class _MLDebugSectionWidgetState extends State { Bus.instance.fire(PeopleChangedEvent()); showShortToast(context, "Done"); } catch (e, s) { - _logger.warning('reset feedback failed ', e, s); + logger.warning('reset feedback failed ', e, s); await showGenericErrorDialog(context: context, error: e); } }, @@ -303,7 +303,7 @@ class _MLDebugSectionWidgetState extends State { Bus.instance.fire(PeopleChangedEvent()); showShortToast(context, "Done"); } catch (e, s) { - _logger.warning('peopleToPersonMapping remove failed ', e, s); + logger.warning('peopleToPersonMapping remove failed ', e, s); await showGenericErrorDialog(context: context, error: e); } }, @@ -332,7 +332,7 @@ class _MLDebugSectionWidgetState extends State { Bus.instance.fire(PeopleChangedEvent()); showShortToast(context, "Done"); } catch (e, s) { - _logger.warning('drop feedback failed ', e, s); + logger.warning('drop feedback failed ', e, s); await showGenericErrorDialog(context: context, error: e); } }, @@ -358,7 +358,7 @@ class _MLDebugSectionWidgetState extends State { await SemanticSearchService.instance.clearIndexes(); showShortToast(context, "Done"); } catch (e, s) { - _logger.warning('drop clip embeddings failed ', e, s); + logger.warning('drop clip embeddings failed ', e, s); await showGenericErrorDialog(context: context, error: e); } }, From 370731b56ae147b279c36559bc4ccf6b7d8e614f Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Fri, 23 Aug 2024 15:06:33 +0200 Subject: [PATCH 10/15] [mob][photos] ml debug options 1/x --- .../face_ml/face_recognition_service.dart | 10 ++++------ .../debug/ml_debug_section_widget.dart | 20 ++++++++++++------- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/mobile/lib/services/machine_learning/face_ml/face_recognition_service.dart b/mobile/lib/services/machine_learning/face_ml/face_recognition_service.dart index 4898fc5f4e..f639520df3 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_recognition_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_recognition_service.dart @@ -65,12 +65,6 @@ class FaceRecognitionService { Future sync() async { await _syncPersonFeedback(); - if (localSettings.remoteFetchEnabled) { - } else { - _logger.severe( - 'Not fetching embeddings because user manually disabled it in debug options', - ); - } } Future _syncPersonFeedback() async { @@ -95,6 +89,10 @@ class FaceRecognitionService { List batchToYield = []; for (final chunk in chunks) { + if (!localSettings.remoteFetchEnabled) { + _logger.warning("remoteFetchEnabled is false, skiping embedding fetch"); + yield chunk; + } final Set ids = {}; final Map pendingIndex = {}; for (final instruction in chunk) { diff --git a/mobile/lib/ui/settings/debug/ml_debug_section_widget.dart b/mobile/lib/ui/settings/debug/ml_debug_section_widget.dart index 0b375cce60..af6f8b16b2 100644 --- a/mobile/lib/ui/settings/debug/ml_debug_section_widget.dart +++ b/mobile/lib/ui/settings/debug/ml_debug_section_widget.dart @@ -17,6 +17,7 @@ import 'package:photos/ui/components/expandable_menu_item_widget.dart'; import 'package:photos/ui/components/menu_item_widget/menu_item_widget.dart'; import 'package:photos/ui/settings/common_settings.dart'; import "package:photos/utils/dialog_util.dart"; +import "package:photos/utils/ml_util.dart"; import 'package:photos/utils/toast_util.dart'; class MLDebugSectionWidget extends StatefulWidget { @@ -58,14 +59,15 @@ class _MLDebugSectionWidgetState extends State { return Column( children: [ MenuItemWidget( - captionedTextWidget: FutureBuilder( - future: MLDataDB.instance.getFaceIndexedFileCount(), + captionedTextWidget: FutureBuilder( + future: getIndexStatus(), builder: (context, snapshot) { if (snapshot.hasData) { + final IndexStatus status = snapshot.data!; return CaptionedTextWidget( title: localSettings.isMLIndexingEnabled - ? "Disable faces (${snapshot.data!} files done)" - : "Enable faces (${snapshot.data!} files done)", + ? "Disable ML (${status.indexedItems} files indexed)" + : "Enable ML (${status.indexedItems} files indexed)", ); } return const SizedBox.shrink(); @@ -77,7 +79,11 @@ class _MLDebugSectionWidgetState extends State { onTap: () async { try { final isEnabled = await localSettings.toggleMLIndexing(); - if (!isEnabled) { + if (isEnabled) { + await MLService.instance.init(); + await SemanticSearchService.instance.init(); + unawaited(MLService.instance.runAllML(force: true)); + } else { MLService.instance.pauseIndexingAndClustering(); } if (mounted) { @@ -93,8 +99,8 @@ class _MLDebugSectionWidgetState extends State { MenuItemWidget( captionedTextWidget: CaptionedTextWidget( title: localSettings.remoteFetchEnabled - ? "Remote fetch enabled" - : "Remote fetch disabled", + ? "Disable remote fetch" + : "Enable remote fetch", ), pressedColor: getEnteColorScheme(context).fillFaint, trailingIcon: Icons.chevron_right_outlined, From e47b4169ed178eac8ac14b407d8da43eaf79303a Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Fri, 23 Aug 2024 15:26:23 +0200 Subject: [PATCH 11/15] [mob][photos] small refactor --- mobile/lib/models/ml/ml_versions.dart | 3 + .../face_ml/face_recognition_service.dart | 107 ------------------ .../services/machine_learning/ml_service.dart | 8 +- mobile/lib/utils/ml_util.dart | 107 +++++++++++++++++- 4 files changed, 113 insertions(+), 112 deletions(-) diff --git a/mobile/lib/models/ml/ml_versions.dart b/mobile/lib/models/ml/ml_versions.dart index d22c4c0670..a52982fb23 100644 --- a/mobile/lib/models/ml/ml_versions.dart +++ b/mobile/lib/models/ml/ml_versions.dart @@ -2,3 +2,6 @@ const faceMlVersion = 1; const clipMlVersion = 1; const clusterMlVersion = 1; const minimumClusterSize = 2; + +const embeddingFetchLimit = 200; +const fileDownloadMlLimit = 10; diff --git a/mobile/lib/services/machine_learning/face_ml/face_recognition_service.dart b/mobile/lib/services/machine_learning/face_ml/face_recognition_service.dart index f639520df3..8b3bb7eda1 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_recognition_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_recognition_service.dart @@ -5,17 +5,8 @@ import "dart:ui" show Image; import "package:logging/logging.dart"; import "package:photos/core/event_bus.dart"; -import "package:photos/db/ml/clip_db.dart"; -import "package:photos/db/ml/db.dart"; import "package:photos/events/diff_sync_complete_event.dart"; import "package:photos/events/people_changed_event.dart"; -import "package:photos/extensions/list.dart"; -import "package:photos/models/ml/clip.dart"; -import "package:photos/models/ml/face/face.dart"; -import "package:photos/models/ml/ml_versions.dart"; -import "package:photos/service_locator.dart"; -import "package:photos/services/filedata/filedata_service.dart"; -import "package:photos/services/filedata/model/file_data.dart"; import "package:photos/services/machine_learning/face_ml/face_detection/detection.dart"; import "package:photos/services/machine_learning/face_ml/face_detection/face_detection_service.dart"; import "package:photos/services/machine_learning/face_ml/face_embedding/face_embedding_service.dart"; @@ -23,7 +14,6 @@ import "package:photos/services/machine_learning/face_ml/person/person_service.d import "package:photos/services/machine_learning/ml_exceptions.dart"; import "package:photos/services/machine_learning/ml_result.dart"; import "package:photos/utils/image_ml_util.dart"; -import "package:photos/utils/ml_util.dart"; class FaceRecognitionService { final _logger = Logger("FaceRecognitionService"); @@ -40,8 +30,6 @@ class FaceRecognitionService { bool _shouldSyncPeople = false; bool _isSyncing = false; - static const _embeddingFetchLimit = 200; - Future init() async { if (_isInitialized) { return; @@ -80,101 +68,6 @@ class FaceRecognitionService { _isSyncing = false; } - Stream> syncEmbeddings({ - int yieldSize = 10, - }) async* { - final List filesToIndex = await getFilesForMlIndexing(); - final List> chunks = - filesToIndex.chunks(_embeddingFetchLimit); - List batchToYield = []; - - for (final chunk in chunks) { - if (!localSettings.remoteFetchEnabled) { - _logger.warning("remoteFetchEnabled is false, skiping embedding fetch"); - yield chunk; - } - final Set ids = {}; - final Map pendingIndex = {}; - for (final instruction in chunk) { - ids.add(instruction.file.uploadedFileID!); - pendingIndex[instruction.file.uploadedFileID!] = instruction; - } - _logger.info("fetching embeddings for ${ids.length} files"); - final res = await FileDataService.instance.getFilesData(ids); - _logger.info("embeddingResponse ${res.debugLog()}"); - final List faces = []; - final List clipEmbeddings = []; - for (FileDataEntity fileMl in res.data.values) { - final existingInstruction = pendingIndex[fileMl.fileID]!; - final facesFromRemoteEmbedding = _getFacesFromRemoteEmbedding(fileMl); - //Note: Always do null check, empty value means no face was found. - if (facesFromRemoteEmbedding != null) { - faces.addAll(facesFromRemoteEmbedding); - existingInstruction.shouldRunFaces = false; - } - if (fileMl.clipEmbedding != null && - fileMl.clipEmbedding!.version >= clipMlVersion) { - clipEmbeddings.add( - ClipEmbedding( - fileID: fileMl.fileID, - embedding: fileMl.clipEmbedding!.embedding, - version: fileMl.clipEmbedding!.version, - ), - ); - existingInstruction.shouldRunClip = false; - } - if (!existingInstruction.pendingML) { - pendingIndex.remove(fileMl.fileID); - } else { - existingInstruction.existingRemoteFileML = fileMl; - pendingIndex[fileMl.fileID] = existingInstruction; - } - } - for (final fileID in pendingIndex.keys) { - final instruction = pendingIndex[fileID]!; - if (instruction.pendingML) { - batchToYield.add(instruction); - if (batchToYield.length == yieldSize) { - _logger.info("queueing indexing for $yieldSize"); - yield batchToYield; - batchToYield = []; - } - } - } - await MLDataDB.instance.bulkInsertFaces(faces); - await MLDataDB.instance.putMany(clipEmbeddings); - } - // Yield any remaining instructions - if (batchToYield.isNotEmpty) { - _logger.info("queueing indexing for ${batchToYield.length}"); - yield batchToYield; - } - } - - // Returns a list of faces from the given remote fileML. null if the version is less than the current version - // or if the remote faceEmbedding is null. - List? _getFacesFromRemoteEmbedding(FileDataEntity fileMl) { - final RemoteFaceEmbedding? remoteFaceEmbedding = fileMl.faceEmbedding; - if (shouldDiscardRemoteEmbedding(fileMl)) { - return null; - } - final List faces = []; - if (remoteFaceEmbedding!.faces.isEmpty) { - faces.add( - Face.empty(fileMl.fileID), - ); - } else { - for (final f in remoteFaceEmbedding.faces) { - f.fileInfo = FileInfo( - imageHeight: remoteFaceEmbedding.height, - imageWidth: remoteFaceEmbedding.width, - ); - faces.add(f); - } - } - return faces; - } - static Future> runFacesPipeline( int enteFileID, Image image, diff --git a/mobile/lib/services/machine_learning/ml_service.dart b/mobile/lib/services/machine_learning/ml_service.dart index 9321c3f96a..d54d45000c 100644 --- a/mobile/lib/services/machine_learning/ml_service.dart +++ b/mobile/lib/services/machine_learning/ml_service.dart @@ -58,7 +58,6 @@ class MLService { bool _isRunningML = false; bool _shouldPauseIndexingAndClustering = false; - static const int _fileDownloadLimit = 10; static const _kForceClusteringFaceCount = 8000; /// Only call this function once at app startup, after that you can directly call [runAllML] @@ -142,7 +141,9 @@ class MLService { } void triggerML() { - if (_mlControllerStatus && !_isIndexingOrClusteringRunning && !_isRunningML) { + if (_mlControllerStatus && + !_isIndexingOrClusteringRunning && + !_isRunningML) { unawaited(runAllML()); } } @@ -169,8 +170,7 @@ class MLService { _isIndexingOrClusteringRunning = true; _logger.info('starting image indexing'); final Stream> instructionStream = - FaceRecognitionService.instance - .syncEmbeddings(yieldSize: _fileDownloadLimit); + fetchEmbeddingsAndInstructions(); int fileAnalyzedCount = 0; final Stopwatch stopwatch = Stopwatch()..start(); diff --git a/mobile/lib/utils/ml_util.dart b/mobile/lib/utils/ml_util.dart index 33d30711c0..c30872c4bf 100644 --- a/mobile/lib/utils/ml_util.dart +++ b/mobile/lib/utils/ml_util.dart @@ -7,11 +7,16 @@ import "package:photos/core/configuration.dart"; import "package:photos/db/files_db.dart"; import "package:photos/db/ml/clip_db.dart"; import "package:photos/db/ml/db.dart"; +import "package:photos/extensions/list.dart"; import "package:photos/models/file/extensions/file_props.dart"; import "package:photos/models/file/file.dart"; import "package:photos/models/file/file_type.dart"; +import "package:photos/models/ml/clip.dart"; import "package:photos/models/ml/face/dimension.dart"; +import "package:photos/models/ml/face/face.dart"; import "package:photos/models/ml/ml_versions.dart"; +import "package:photos/service_locator.dart"; +import "package:photos/services/filedata/filedata_service.dart"; import "package:photos/services/filedata/model/file_data.dart"; import "package:photos/services/machine_learning/face_ml/face_recognition_service.dart"; import "package:photos/services/machine_learning/ml_exceptions.dart"; @@ -68,6 +73,7 @@ Future getIndexStatus() async { } } +/// Return a list of file instructions for files that should be indexed for ML Future> getFilesForMlIndexing() async { _logger.info('getFilesForMlIndexing called'); final time = DateTime.now(); @@ -146,7 +152,106 @@ Future> getFilesForMlIndexing() async { return sortedBylocalID; } -bool shouldDiscardRemoteEmbedding(FileDataEntity fileML) { +Stream> fetchEmbeddingsAndInstructions({ + int yieldSize = fileDownloadMlLimit, +}) async* { + final List filesToIndex = await getFilesForMlIndexing(); + final List> chunks = + filesToIndex.chunks(embeddingFetchLimit); + List batchToYield = []; + + for (final chunk in chunks) { + if (!localSettings.remoteFetchEnabled) { + _logger.warning("remoteFetchEnabled is false, skiping embedding fetch"); + final batches = chunk.chunks(yieldSize); + for (final batch in batches) { + yield batch; + } + continue; + } + final Set ids = {}; + final Map pendingIndex = {}; + for (final instruction in chunk) { + ids.add(instruction.file.uploadedFileID!); + pendingIndex[instruction.file.uploadedFileID!] = instruction; + } + _logger.info("fetching embeddings for ${ids.length} files"); + final res = await FileDataService.instance.getFilesData(ids); + _logger.info("embeddingResponse ${res.debugLog()}"); + final List faces = []; + final List clipEmbeddings = []; + for (FileDataEntity fileMl in res.data.values) { + final existingInstruction = pendingIndex[fileMl.fileID]!; + final facesFromRemoteEmbedding = _getFacesFromRemoteEmbedding(fileMl); + //Note: Always do null check, empty value means no face was found. + if (facesFromRemoteEmbedding != null) { + faces.addAll(facesFromRemoteEmbedding); + existingInstruction.shouldRunFaces = false; + } + if (fileMl.clipEmbedding != null && + fileMl.clipEmbedding!.version >= clipMlVersion) { + clipEmbeddings.add( + ClipEmbedding( + fileID: fileMl.fileID, + embedding: fileMl.clipEmbedding!.embedding, + version: fileMl.clipEmbedding!.version, + ), + ); + existingInstruction.shouldRunClip = false; + } + if (!existingInstruction.pendingML) { + pendingIndex.remove(fileMl.fileID); + } else { + existingInstruction.existingRemoteFileML = fileMl; + pendingIndex[fileMl.fileID] = existingInstruction; + } + } + for (final fileID in pendingIndex.keys) { + final instruction = pendingIndex[fileID]!; + if (instruction.pendingML) { + batchToYield.add(instruction); + if (batchToYield.length == yieldSize) { + _logger.info("queueing indexing for $yieldSize"); + yield batchToYield; + batchToYield = []; + } + } + } + await MLDataDB.instance.bulkInsertFaces(faces); + await MLDataDB.instance.putMany(clipEmbeddings); + } + // Yield any remaining instructions + if (batchToYield.isNotEmpty) { + _logger.info("queueing indexing for ${batchToYield.length}"); + yield batchToYield; + } +} + +// Returns a list of faces from the given remote fileML. null if the version is less than the current version +// or if the remote faceEmbedding is null. +List? _getFacesFromRemoteEmbedding(FileDataEntity fileMl) { + final RemoteFaceEmbedding? remoteFaceEmbedding = fileMl.faceEmbedding; + if (_shouldDiscardRemoteEmbedding(fileMl)) { + return null; + } + final List faces = []; + if (remoteFaceEmbedding!.faces.isEmpty) { + faces.add( + Face.empty(fileMl.fileID), + ); + } else { + for (final f in remoteFaceEmbedding.faces) { + f.fileInfo = FileInfo( + imageHeight: remoteFaceEmbedding.height, + imageWidth: remoteFaceEmbedding.width, + ); + faces.add(f); + } + } + return faces; +} + +bool _shouldDiscardRemoteEmbedding(FileDataEntity fileML) { final fileID = fileML.fileID; final RemoteFaceEmbedding? faceEmbedding = fileML.faceEmbedding; if (faceEmbedding == null || faceEmbedding.version < faceMlVersion) { From de25072755819bc2c195f9c87fa852f7182de4ae Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Fri, 23 Aug 2024 15:30:46 +0200 Subject: [PATCH 12/15] [mob][photos] ml debug options 2/x --- mobile/lib/ui/settings/debug/ml_debug_section_widget.dart | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mobile/lib/ui/settings/debug/ml_debug_section_widget.dart b/mobile/lib/ui/settings/debug/ml_debug_section_widget.dart index af6f8b16b2..c083c824dd 100644 --- a/mobile/lib/ui/settings/debug/ml_debug_section_widget.dart +++ b/mobile/lib/ui/settings/debug/ml_debug_section_widget.dart @@ -121,8 +121,8 @@ class _MLDebugSectionWidgetState extends State { MenuItemWidget( captionedTextWidget: CaptionedTextWidget( title: MLService.instance.debugIndexingDisabled - ? "Debug enable indexing again" - : "Debug disable indexing", + ? "Enable auto indexing (debug)" + : "Disable auto indexing (debug)", ), pressedColor: getEnteColorScheme(context).fillFaint, trailingIcon: Icons.chevron_right_outlined, @@ -133,6 +133,8 @@ class _MLDebugSectionWidgetState extends State { !MLService.instance.debugIndexingDisabled; if (MLService.instance.debugIndexingDisabled) { MLService.instance.pauseIndexingAndClustering(); + } else { + unawaited(MLService.instance.runAllML()); } if (mounted) { setState(() {}); From ac3a323abd60f4d8d3523b4a289930b992568beb Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Fri, 23 Aug 2024 16:51:54 +0200 Subject: [PATCH 13/15] [mob][photos] ml debug options --- mobile/lib/db/ml/db.dart | 2 +- .../debug/ml_debug_section_widget.dart | 77 +++++++++---------- 2 files changed, 39 insertions(+), 40 deletions(-) diff --git a/mobile/lib/db/ml/db.dart b/mobile/lib/db/ml/db.dart index 1a7b6ca6a6..94fea81090 100644 --- a/mobile/lib/db/ml/db.dart +++ b/mobile/lib/db/ml/db.dart @@ -942,7 +942,7 @@ class MLDataDB { } /// WARNING: This will delete ALL data in the tables! Only use this for debug/testing purposes! - Future dropFeedbackTables() async { + Future dropFacesFeedbackTables() async { try { final db = await instance.asyncDB; diff --git a/mobile/lib/ui/settings/debug/ml_debug_section_widget.dart b/mobile/lib/ui/settings/debug/ml_debug_section_widget.dart index c083c824dd..a3bd5c1a7f 100644 --- a/mobile/lib/ui/settings/debug/ml_debug_section_widget.dart +++ b/mobile/lib/ui/settings/debug/ml_debug_section_widget.dart @@ -7,7 +7,7 @@ import "package:photos/db/ml/db.dart"; import "package:photos/events/people_changed_event.dart"; import "package:photos/models/ml/face/person.dart"; import "package:photos/service_locator.dart"; -import "package:photos/services/machine_learning/face_ml/feedback/cluster_feedback.dart"; +import "package:photos/services/machine_learning/face_ml/face_recognition_service.dart"; import "package:photos/services/machine_learning/face_ml/person/person_service.dart"; import 'package:photos/services/machine_learning/ml_service.dart'; import "package:photos/services/machine_learning/semantic_search/semantic_search_service.dart"; @@ -148,7 +148,7 @@ class _MLDebugSectionWidgetState extends State { sectionOptionSpacing, MenuItemWidget( captionedTextWidget: const CaptionedTextWidget( - title: "Run sync, indexing, clustering", + title: "Trigger run ML", ), pressedColor: getEnteColorScheme(context).fillFaint, trailingIcon: Icons.chevron_right_outlined, @@ -166,7 +166,7 @@ class _MLDebugSectionWidgetState extends State { sectionOptionSpacing, MenuItemWidget( captionedTextWidget: const CaptionedTextWidget( - title: "Run indexing", + title: "Trigger run indexing", ), pressedColor: getEnteColorScheme(context).fillFaint, trailingIcon: Icons.chevron_right_outlined, @@ -189,7 +189,7 @@ class _MLDebugSectionWidgetState extends State { if (snapshot.hasData) { return CaptionedTextWidget( title: - "Run clustering (${(100 * snapshot.data!).toStringAsFixed(0)}% done)", + "Trigger clustering (${(100 * snapshot.data!).toStringAsFixed(0)}% done)", ); } return const SizedBox.shrink(); @@ -202,7 +202,7 @@ class _MLDebugSectionWidgetState extends State { try { await PersonService.instance.fetchRemoteClusterFeedback(); MLService.instance.debugIndexingDisabled = false; - await MLService.instance.clusterAllImages(clusterInBuckets: true); + await MLService.instance.clusterAllImages(); Bus.instance.fire(PeopleChangedEvent()); showShortToast(context, "Done"); } catch (e, s) { @@ -211,32 +211,32 @@ class _MLDebugSectionWidgetState extends State { } }, ), - sectionOptionSpacing, - MenuItemWidget( - captionedTextWidget: const CaptionedTextWidget( - title: "Check for mixed clusters", - ), - pressedColor: getEnteColorScheme(context).fillFaint, - trailingIcon: Icons.chevron_right_outlined, - trailingIconIsMuted: true, - onTap: () async { - try { - final susClusters = - await ClusterFeedbackService.instance.checkForMixedClusters(); - for (final clusterinfo in susClusters) { - Future.delayed(const Duration(seconds: 4), () { - showToast( - context, - 'Cluster with ${clusterinfo.$2} photos is sus', - ); - }); - } - } catch (e, s) { - logger.warning('Checking for mixed clusters failed', e, s); - await showGenericErrorDialog(context: context, error: e); - } - }, - ), + // sectionOptionSpacing, + // MenuItemWidget( + // captionedTextWidget: const CaptionedTextWidget( + // title: "Check for mixed clusters", + // ), + // pressedColor: getEnteColorScheme(context).fillFaint, + // trailingIcon: Icons.chevron_right_outlined, + // trailingIconIsMuted: true, + // onTap: () async { + // try { + // final susClusters = + // await ClusterFeedbackService.instance.checkForMixedClusters(); + // for (final clusterinfo in susClusters) { + // Future.delayed(const Duration(seconds: 4), () { + // showToast( + // context, + // 'Cluster with ${clusterinfo.$2} photos is sus', + // ); + // }); + // } + // } catch (e, s) { + // logger.warning('Checking for mixed clusters failed', e, s); + // await showGenericErrorDialog(context: context, error: e); + // } + // }, + // ), sectionOptionSpacing, MenuItemWidget( captionedTextWidget: const CaptionedTextWidget( @@ -247,8 +247,7 @@ class _MLDebugSectionWidgetState extends State { trailingIconIsMuted: true, onTap: () async { try { - await PersonService.instance.reconcileClusters(); - Bus.instance.fire(PeopleChangedEvent()); + await FaceRecognitionService.instance.sync(); showShortToast(context, "Done"); } catch (e, s) { logger.warning('sync person mappings failed ', e, s); @@ -270,11 +269,11 @@ class _MLDebugSectionWidgetState extends State { context, title: "Are you sure?", body: - "This will drop all people and their related feedback. It will keep clustering labels and embeddings untouched.", + "This will drop all people and their related feedback stored locally. It will keep clustering labels and embeddings untouched, as well as persons stored on remote.", firstButtonLabel: "Yes, confirm", firstButtonOnTap: () async { try { - await MLDataDB.instance.dropFeedbackTables(); + await MLDataDB.instance.dropFacesFeedbackTables(); Bus.instance.fire(PeopleChangedEvent()); showShortToast(context, "Done"); } catch (e, s) { @@ -298,7 +297,7 @@ class _MLDebugSectionWidgetState extends State { context, title: "Are you sure?", body: - "This will delete all people, their related feedback and clustering labels. It will keep embeddings untouched.", + "This will delete all people (also from remote), their related feedback and clustering labels. It will keep embeddings untouched.", firstButtonLabel: "Yes, confirm", firstButtonOnTap: () async { try { @@ -321,7 +320,7 @@ class _MLDebugSectionWidgetState extends State { sectionOptionSpacing, MenuItemWidget( captionedTextWidget: const CaptionedTextWidget( - title: "Reset faces everything (embeddings)", + title: "Reset all local faces", ), pressedColor: getEnteColorScheme(context).fillFaint, trailingIcon: Icons.chevron_right_outlined, @@ -331,7 +330,7 @@ class _MLDebugSectionWidgetState extends State { context, title: "Are you sure?", body: - "You will need to again re-index all the faces. You can drop feedback if you want to label again", + "This will drop all local faces data. You will need to again re-index faces.", firstButtonLabel: "Yes, confirm", firstButtonOnTap: () async { try { @@ -349,7 +348,7 @@ class _MLDebugSectionWidgetState extends State { ), MenuItemWidget( captionedTextWidget: const CaptionedTextWidget( - title: "Reset clip embeddings", + title: "Reset all local clip", ), pressedColor: getEnteColorScheme(context).fillFaint, trailingIcon: Icons.chevron_right_outlined, From 44a6f256d6845c1a14dfe9d68ae92ce0e1f25b11 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Fri, 23 Aug 2024 17:16:38 +0200 Subject: [PATCH 14/15] [mob][photos] Show errored faces count --- mobile/lib/db/ml/db.dart | 8 ++++++++ .../ui/settings/debug/ml_debug_section_widget.dart | 14 ++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/mobile/lib/db/ml/db.dart b/mobile/lib/db/ml/db.dart index 94fea81090..5f5cb7af35 100644 --- a/mobile/lib/db/ml/db.dart +++ b/mobile/lib/db/ml/db.dart @@ -660,6 +660,14 @@ class MLDataDB { return maps.first['count'] as int; } + Future getErroredFaceCount() async { + final db = await instance.asyncDB; + final List> maps = await db.getAll( + 'SELECT COUNT(*) as count FROM $facesTable WHERE $faceScore < 0', + ); + return maps.first['count'] as int; + } + Future getClusteredOrFacelessFileCount() async { final db = await instance.asyncDB; final List> clustered = await db.getAll( diff --git a/mobile/lib/ui/settings/debug/ml_debug_section_widget.dart b/mobile/lib/ui/settings/debug/ml_debug_section_widget.dart index a3bd5c1a7f..0d00275990 100644 --- a/mobile/lib/ui/settings/debug/ml_debug_section_widget.dart +++ b/mobile/lib/ui/settings/debug/ml_debug_section_widget.dart @@ -256,6 +256,19 @@ class _MLDebugSectionWidgetState extends State { }, ), sectionOptionSpacing, + MenuItemWidget( + captionedTextWidget: const CaptionedTextWidget( + title: "Show empty indexes", + ), + pressedColor: getEnteColorScheme(context).fillFaint, + trailingIcon: Icons.chevron_right_outlined, + trailingIconIsMuted: true, + onTap: () async { + final emptyFaces = await MLDataDB.instance.getErroredFaceCount(); + showShortToast(context, '$emptyFaces empty faces'); + }, + ), + sectionOptionSpacing, MenuItemWidget( captionedTextWidget: const CaptionedTextWidget( title: "Reset faces feedback", @@ -346,6 +359,7 @@ class _MLDebugSectionWidgetState extends State { ); }, ), + sectionOptionSpacing, MenuItemWidget( captionedTextWidget: const CaptionedTextWidget( title: "Reset all local clip", From abd5234e7c05b5f2c51b514da9dfd604e99f78a5 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Fri, 23 Aug 2024 17:48:18 +0200 Subject: [PATCH 15/15] [mob][photos] resolve merge conflict --- mobile/lib/models/ml/ml_versions.dart | 4 +++- mobile/lib/services/machine_learning/ml_service.dart | 2 +- mobile/lib/utils/ml_util.dart | 7 +++---- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/mobile/lib/models/ml/ml_versions.dart b/mobile/lib/models/ml/ml_versions.dart index a52982fb23..2d382209dc 100644 --- a/mobile/lib/models/ml/ml_versions.dart +++ b/mobile/lib/models/ml/ml_versions.dart @@ -1,7 +1,9 @@ +import "dart:io" show Platform; + const faceMlVersion = 1; const clipMlVersion = 1; const clusterMlVersion = 1; const minimumClusterSize = 2; const embeddingFetchLimit = 200; -const fileDownloadMlLimit = 10; +final fileDownloadMlLimit = Platform.isIOS ? 5 : 10; diff --git a/mobile/lib/services/machine_learning/ml_service.dart b/mobile/lib/services/machine_learning/ml_service.dart index d54d45000c..5a562b710b 100644 --- a/mobile/lib/services/machine_learning/ml_service.dart +++ b/mobile/lib/services/machine_learning/ml_service.dart @@ -170,7 +170,7 @@ class MLService { _isIndexingOrClusteringRunning = true; _logger.info('starting image indexing'); final Stream> instructionStream = - fetchEmbeddingsAndInstructions(); + fetchEmbeddingsAndInstructions(fileDownloadMlLimit); int fileAnalyzedCount = 0; final Stopwatch stopwatch = Stopwatch()..start(); diff --git a/mobile/lib/utils/ml_util.dart b/mobile/lib/utils/ml_util.dart index c30872c4bf..13fb1474c7 100644 --- a/mobile/lib/utils/ml_util.dart +++ b/mobile/lib/utils/ml_util.dart @@ -152,9 +152,8 @@ Future> getFilesForMlIndexing() async { return sortedBylocalID; } -Stream> fetchEmbeddingsAndInstructions({ - int yieldSize = fileDownloadMlLimit, -}) async* { +Stream> fetchEmbeddingsAndInstructions( + int yieldSize,) async* { final List filesToIndex = await getFilesForMlIndexing(); final List> chunks = filesToIndex.chunks(embeddingFetchLimit); @@ -167,7 +166,7 @@ Stream> fetchEmbeddingsAndInstructions({ for (final batch in batches) { yield batch; } - continue; + continue; } final Set ids = {}; final Map pendingIndex = {};