From 61c1847d75d9a106dd90f567a72a6400ab9003b8 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Wed, 28 Aug 2024 10:19:39 +0200 Subject: [PATCH 1/6] [mob][photos] Move --- mobile/lib/services/machine_learning/ml_indexing_isolate.dart | 3 +-- mobile/lib/services/machine_learning/ml_service.dart | 3 +++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/mobile/lib/services/machine_learning/ml_indexing_isolate.dart b/mobile/lib/services/machine_learning/ml_indexing_isolate.dart index b01037dc83..fec1cbf410 100644 --- a/mobile/lib/services/machine_learning/ml_indexing_isolate.dart +++ b/mobile/lib/services/machine_learning/ml_indexing_isolate.dart @@ -189,9 +189,8 @@ class MLIndexingIsolate { /// Analyzes the given image data by running the full pipeline for faces, using [_analyzeImageSync] in the isolate. Future analyzeImage( FileMLInstruction instruction, + String filePath, ) async { - final String filePath = await getImagePathForML(instruction.file); - final Stopwatch stopwatch = Stopwatch()..start(); late MLResult result; diff --git a/mobile/lib/services/machine_learning/ml_service.dart b/mobile/lib/services/machine_learning/ml_service.dart index e3e41c73d2..b048301ad1 100644 --- a/mobile/lib/services/machine_learning/ml_service.dart +++ b/mobile/lib/services/machine_learning/ml_service.dart @@ -391,8 +391,11 @@ class MLService { bool actuallyRanML = false; try { + final String filePath = await getImagePathForML(instruction.file); + final MLResult? result = await MLIndexingIsolate.instance.analyzeImage( instruction, + filePath, ); // Check if there's no result simply because MLController paused indexing if (result == null) { From e1ce353069562df4f73f8311e87369e8bb217622 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Wed, 28 Aug 2024 11:08:18 +0200 Subject: [PATCH 2/6] [mob][photos] Don't process large files on mobile --- mobile/lib/models/ml/ml_versions.dart | 1 + mobile/lib/services/machine_learning/ml_service.dart | 8 ++++++++ mobile/lib/utils/ml_util.dart | 9 ++++++++- 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/mobile/lib/models/ml/ml_versions.dart b/mobile/lib/models/ml/ml_versions.dart index 2d382209dc..35d089178a 100644 --- a/mobile/lib/models/ml/ml_versions.dart +++ b/mobile/lib/models/ml/ml_versions.dart @@ -7,3 +7,4 @@ const minimumClusterSize = 2; const embeddingFetchLimit = 200; final fileDownloadMlLimit = Platform.isIOS ? 5 : 10; +const maxFileDownloadSize = 100000000; diff --git a/mobile/lib/services/machine_learning/ml_service.dart b/mobile/lib/services/machine_learning/ml_service.dart index b048301ad1..616e2bb2a1 100644 --- a/mobile/lib/services/machine_learning/ml_service.dart +++ b/mobile/lib/services/machine_learning/ml_service.dart @@ -488,6 +488,14 @@ class MLService { ); acceptedIssue = true; } + if (errorString.contains('FileSizeTooLargeForMobileIndexing')) { + _logger.severe( + '$errorString with ID ${instruction.file.uploadedFileID}, storing empty results so indexing does not get stuck', + e, + s, + ); + acceptedIssue = true; + } if (acceptedIssue) { await MLDataDB.instance.bulkInsertFaces( [Face.empty(instruction.file.uploadedFileID!, error: true)], diff --git a/mobile/lib/utils/ml_util.dart b/mobile/lib/utils/ml_util.dart index e07740acba..01ddd353a9 100644 --- a/mobile/lib/utils/ml_util.dart +++ b/mobile/lib/utils/ml_util.dart @@ -153,7 +153,8 @@ Future> getFilesForMlIndexing() async { } Stream> fetchEmbeddingsAndInstructions( - int yieldSize,) async* { + int yieldSize, +) async* { final List filesToIndex = await getFilesForMlIndexing(); final List> chunks = filesToIndex.chunks(embeddingFetchLimit); @@ -312,6 +313,12 @@ Future getImagePathForML(EnteFile enteFile) async { throw ThumbnailRetrievalException(e.toString(), s); } } else { + // Don't process the file if it's too large (more than 100MB) + if (enteFile.fileSize != null && enteFile.fileSize! > maxFileDownloadSize) { + throw Exception( + "FileSizeTooLargeForMobileIndexing: size is ${enteFile.fileSize}", + ); + } try { file = await getFile(enteFile, isOrigin: true); } catch (e, s) { From 1e9a014ce76f53d11a40535483d86f613cc19d88 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Wed, 28 Aug 2024 11:12:22 +0200 Subject: [PATCH 3/6] [mob][photos] Minor cleanup --- .../services/machine_learning/ml_service.dart | 34 +++++-------------- 1 file changed, 9 insertions(+), 25 deletions(-) diff --git a/mobile/lib/services/machine_learning/ml_service.dart b/mobile/lib/services/machine_learning/ml_service.dart index 616e2bb2a1..0fee0af261 100644 --- a/mobile/lib/services/machine_learning/ml_service.dart +++ b/mobile/lib/services/machine_learning/ml_service.dart @@ -470,33 +470,17 @@ class MLService { _logger.info("Results for file ${result.fileId} stored locally"); return actuallyRanML; } catch (e, s) { - bool acceptedIssue = false; final String errorString = e.toString(); - if (errorString.contains('ThumbnailRetrievalException')) { - _logger.severe( - 'ThumbnailRetrievalException while processing image with ID ${instruction.file.uploadedFileID}, storing empty results so indexing does not get stuck', - e, - s, - ); - acceptedIssue = true; - } - if (errorString.contains('InvalidImageFormatException')) { - _logger.severe( - '$errorString with ID ${instruction.file.uploadedFileID}, storing empty results so indexing does not get stuck', - e, - s, - ); - acceptedIssue = true; - } - if (errorString.contains('FileSizeTooLargeForMobileIndexing')) { - _logger.severe( - '$errorString with ID ${instruction.file.uploadedFileID}, storing empty results so indexing does not get stuck', - e, - s, - ); - acceptedIssue = true; - } + final bool acceptedIssue = + errorString.contains('ThumbnailRetrievalException') || + errorString.contains('InvalidImageFormatException') || + errorString.contains('FileSizeTooLargeForMobileIndexing'); if (acceptedIssue) { + _logger.severe( + '$errorString with ID ${instruction.file.uploadedFileID}, storing empty results so indexing does not get stuck', + e, + s, + ); await MLDataDB.instance.bulkInsertFaces( [Face.empty(instruction.file.uploadedFileID!, error: true)], ); From db8e203c367c70d7e0057a7cda6e551ed1105a55 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Wed, 28 Aug 2024 11:21:50 +0200 Subject: [PATCH 4/6] [mob][photos] Always log basic info on empty result --- mobile/lib/services/machine_learning/ml_service.dart | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/mobile/lib/services/machine_learning/ml_service.dart b/mobile/lib/services/machine_learning/ml_service.dart index 0fee0af261..19ef46f2be 100644 --- a/mobile/lib/services/machine_learning/ml_service.dart +++ b/mobile/lib/services/machine_learning/ml_service.dart @@ -471,13 +471,16 @@ class MLService { return actuallyRanML; } catch (e, s) { final String errorString = e.toString(); + final String format = instruction.file.displayName.split('.').last; + final int? size = instruction.file.fileSize; + final fileType = instruction.file.fileType; final bool acceptedIssue = errorString.contains('ThumbnailRetrievalException') || errorString.contains('InvalidImageFormatException') || errorString.contains('FileSizeTooLargeForMobileIndexing'); if (acceptedIssue) { _logger.severe( - '$errorString with ID ${instruction.file.uploadedFileID}, storing empty results so indexing does not get stuck', + '$errorString with ID ${instruction.file.uploadedFileID} (format $format, type $fileType, size $size), storing empty results so indexing does not get stuck', e, s, ); @@ -490,7 +493,7 @@ class MLService { return true; } _logger.severe( - "Failed to analyze using FaceML for image with ID: ${instruction.file.uploadedFileID} and format ${instruction.file.displayName.split('.').last} (${instruction.file.fileType}). Not storing any results locally, which means it will be automatically retried later.", + "Failed to index file with ID: ${instruction.file.uploadedFileID} (format $format, type $fileType, size $size). Not storing any results locally, which means it will be automatically retried later.", e, s, ); From ac5d9d99f1c79bd3291db9dbfa8131635200fbae Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Wed, 28 Aug 2024 13:46:58 +0200 Subject: [PATCH 5/6] [mob][photos] ML user developer options --- mobile/lib/db/ml/db.dart | 16 +++ .../machine_learning_settings_page.dart | 29 ++++- .../ui/settings/ml/ml_user_dev_screen.dart | 103 ++++++++++++++++++ 3 files changed, 146 insertions(+), 2 deletions(-) create mode 100644 mobile/lib/ui/settings/ml/ml_user_dev_screen.dart diff --git a/mobile/lib/db/ml/db.dart b/mobile/lib/db/ml/db.dart index 5f5cb7af35..3103e80442 100644 --- a/mobile/lib/db/ml/db.dart +++ b/mobile/lib/db/ml/db.dart @@ -668,6 +668,22 @@ class MLDataDB { return maps.first['count'] as int; } + Future> getErroredFileIDs() async { + final db = await instance.asyncDB; + final List> maps = await db.getAll( + 'SELECT DISTINCT $fileIDColumn FROM $facesTable WHERE $faceScore < 0', + ); + return maps.map((e) => e[fileIDColumn] as int).toSet(); + } + + Future deleteFaceIndexForFiles(List fileIDs) async { + final db = await instance.asyncDB; + final String sql = ''' + DELETE FROM $facesTable WHERE $fileIDColumn IN (${fileIDs.join(", ")}) + '''; + await db.execute(sql); + } + Future getClusteredOrFacelessFileCount() async { final db = await instance.asyncDB; final List> clustered = await db.getAll( diff --git a/mobile/lib/ui/settings/machine_learning_settings_page.dart b/mobile/lib/ui/settings/machine_learning_settings_page.dart index cf83d4a800..8267648ad4 100644 --- a/mobile/lib/ui/settings/machine_learning_settings_page.dart +++ b/mobile/lib/ui/settings/machine_learning_settings_page.dart @@ -26,6 +26,7 @@ import "package:photos/ui/components/title_bar_title_widget.dart"; import "package:photos/ui/components/title_bar_widget.dart"; import "package:photos/ui/components/toggle_switch_widget.dart"; import "package:photos/ui/settings/ml/enable_ml_consent.dart"; +import "package:photos/ui/settings/ml/ml_user_dev_screen.dart"; import "package:photos/utils/ml_util.dart"; import "package:photos/utils/network_util.dart"; import "package:photos/utils/wakelock_util.dart"; @@ -42,6 +43,8 @@ class _MachineLearningSettingsPageState extends State { final EnteWakeLock _wakeLock = EnteWakeLock(); Timer? _timer; + int _titleTapCount = 0; + Timer? _advancedOptionsTimer; @override void initState() { @@ -55,6 +58,9 @@ class _MachineLearningSettingsPageState } }); } + _advancedOptionsTimer = Timer.periodic(const Duration(seconds: 7), (timer) { + _titleTapCount = 0; + }); } @override @@ -63,6 +69,7 @@ class _MachineLearningSettingsPageState _wakeLock.disable(); MachineLearningController.instance.forceOverrideML(turnOn: false); _timer?.cancel(); + _advancedOptionsTimer?.cancel(); } @override @@ -73,8 +80,26 @@ class _MachineLearningSettingsPageState primary: false, slivers: [ TitleBarWidget( - flexibleSpaceTitle: TitleBarTitleWidget( - title: S.of(context).machineLearning, + flexibleSpaceTitle: GestureDetector( + child: TitleBarTitleWidget( + title: S.of(context).machineLearning, + ), + onTap: () { + setState(() { + _titleTapCount++; + if (_titleTapCount >= 7) { + _titleTapCount = 0; + // showShortToast(context, "Advanced options enabled"); + Navigator.of(context).push( + MaterialPageRoute( + builder: (BuildContext context) { + return const MLUserDeveloperOptions(); + }, + ), + ).ignore(); + } + }); + }, ), actionIcons: [ IconButtonWidget( diff --git a/mobile/lib/ui/settings/ml/ml_user_dev_screen.dart b/mobile/lib/ui/settings/ml/ml_user_dev_screen.dart new file mode 100644 index 0000000000..8d70cee21b --- /dev/null +++ b/mobile/lib/ui/settings/ml/ml_user_dev_screen.dart @@ -0,0 +1,103 @@ +import "package:flutter/material.dart"; +import "package:photos/core/event_bus.dart"; +import "package:photos/db/ml/clip_db.dart"; +import "package:photos/db/ml/db.dart"; +import "package:photos/events/people_changed_event.dart"; +import "package:photos/services/machine_learning/semantic_search/semantic_search_service.dart"; +import "package:photos/theme/ente_theme.dart"; +import "package:photos/ui/components/buttons/button_widget.dart"; +import "package:photos/ui/components/models/button_type.dart"; +import "package:photos/ui/components/title_bar_title_widget.dart"; +import "package:photos/ui/components/title_bar_widget.dart"; +import "package:photos/utils/dialog_util.dart"; +import "package:photos/utils/toast_util.dart"; + +class MLUserDeveloperOptions extends StatelessWidget { + const MLUserDeveloperOptions({super.key}); + + @override + Widget build(BuildContext context) { + return Scaffold( + body: CustomScrollView( + primary: false, + slivers: [ + const TitleBarWidget( + flexibleSpaceTitle: TitleBarTitleWidget( + title: "ML debug options", + ), + ), + SliverList( + delegate: SliverChildBuilderDelegate( + (delegateBuildContext, index) => Padding( + padding: const EdgeInsets.only(left: 16, right: 16), + child: Column( + children: [ + Text( + "Only use if you know what you're doing", + textAlign: TextAlign.left, + style: getEnteTextTheme(context).body.copyWith( + color: getEnteColorScheme(context).textMuted, + ), + ), + const SizedBox(height: 48), + ButtonWidget( + buttonType: ButtonType.neutral, + labelText: "Purge empty indices", + onTap: () async { + await deleteEmptyIndices(context); + }, + ), + const SizedBox(height: 24), + ButtonWidget( + buttonType: ButtonType.neutral, + labelText: "Reset all local ML", + onTap: () async { + await deleteAllLocalML(context); + }, + ), + const SafeArea( + child: SizedBox( + height: 12, + ), + ), + ], + ), + ), + childCount: 1, + ), + ), + ], + ), + ); + } + + Future deleteEmptyIndices(BuildContext context) async { + try { + final Set emptyFileIDs = await MLDataDB.instance.getErroredFileIDs(); + await MLDataDB.instance.deleteFaceIndexForFiles(emptyFileIDs.toList()); + await MLDataDB.instance.deleteEmbeddings(emptyFileIDs.toList()); + showShortToast(context, "Deleted ${emptyFileIDs.length} entries"); + } catch (e) { + // ignore: unawaited_futures + showGenericErrorDialog( + context: context, + error: e, + ); + } + } + + Future deleteAllLocalML(BuildContext context) async { + try { + await MLDataDB.instance.dropClustersAndPersonTable(faces: true); + await SemanticSearchService.instance.clearIndexes(); + Bus.instance.fire(PeopleChangedEvent()); + showShortToast(context, "All local ML cleared"); + } catch (e) { + // ignore: unawaited_futures + showGenericErrorDialog( + context: context, + error: e, + ); + } + } +} From dd6f88a1cd39eafcae32e39d3ce9c73ccf5b952f Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Wed, 28 Aug 2024 13:49:07 +0200 Subject: [PATCH 6/6] [mob][photos] Move --- mobile/lib/ui/settings/advanced_settings_screen.dart | 2 +- .../ui/settings/{ => ml}/machine_learning_settings_page.dart | 0 mobile/lib/ui/viewer/search_tab/people_section.dart | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename mobile/lib/ui/settings/{ => ml}/machine_learning_settings_page.dart (100%) diff --git a/mobile/lib/ui/settings/advanced_settings_screen.dart b/mobile/lib/ui/settings/advanced_settings_screen.dart index 6792f254bd..b2e5f5a488 100644 --- a/mobile/lib/ui/settings/advanced_settings_screen.dart +++ b/mobile/lib/ui/settings/advanced_settings_screen.dart @@ -13,7 +13,7 @@ import 'package:photos/ui/components/menu_item_widget/menu_item_widget.dart'; import 'package:photos/ui/components/title_bar_title_widget.dart'; import 'package:photos/ui/components/title_bar_widget.dart'; import "package:photos/ui/components/toggle_switch_widget.dart"; -import "package:photos/ui/settings/machine_learning_settings_page.dart"; +import "package:photos/ui/settings/ml/machine_learning_settings_page.dart"; import 'package:photos/ui/tools/debug/app_storage_viewer.dart'; import 'package:photos/ui/viewer/gallery/photo_grid_size_picker_page.dart'; import 'package:photos/utils/navigation_util.dart'; diff --git a/mobile/lib/ui/settings/machine_learning_settings_page.dart b/mobile/lib/ui/settings/ml/machine_learning_settings_page.dart similarity index 100% rename from mobile/lib/ui/settings/machine_learning_settings_page.dart rename to mobile/lib/ui/settings/ml/machine_learning_settings_page.dart diff --git a/mobile/lib/ui/viewer/search_tab/people_section.dart b/mobile/lib/ui/viewer/search_tab/people_section.dart index 3e1d7599e5..4ebcbbf9d6 100644 --- a/mobile/lib/ui/viewer/search_tab/people_section.dart +++ b/mobile/lib/ui/viewer/search_tab/people_section.dart @@ -13,7 +13,7 @@ import "package:photos/models/search/search_constants.dart"; import "package:photos/models/search/search_result.dart"; import "package:photos/models/search/search_types.dart"; import "package:photos/theme/ente_theme.dart"; -import "package:photos/ui/settings/machine_learning_settings_page.dart"; +import "package:photos/ui/settings/ml/machine_learning_settings_page.dart"; import "package:photos/ui/viewer/file/no_thumbnail_widget.dart"; import "package:photos/ui/viewer/file/thumbnail_widget.dart"; import "package:photos/ui/viewer/people/add_person_action_sheet.dart";