From 33656c82066774b2d9d94a61b1eb031cb102c9d0 Mon Sep 17 00:00:00 2001 From: ashilkn Date: Sat, 1 Jun 2024 13:12:29 +0530 Subject: [PATCH 1/8] [mob][photos] perf improvement when computing score on magic search --- .../semantic_search/semantic_search_service.dart | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mobile/lib/services/machine_learning/semantic_search/semantic_search_service.dart b/mobile/lib/services/machine_learning/semantic_search/semantic_search_service.dart index 1384750811..625e21ab8e 100644 --- a/mobile/lib/services/machine_learning/semantic_search/semantic_search_service.dart +++ b/mobile/lib/services/machine_learning/semantic_search/semantic_search_service.dart @@ -422,7 +422,8 @@ double computeScore(List imageEmbedding, List textEmbedding) { "The two embeddings should have the same length", ); double score = 0; - for (int index = 0; index < imageEmbedding.length; index++) { + final length = imageEmbedding.length; + for (int index = 0; index < length; index++) { score += imageEmbedding[index] * textEmbedding[index]; } return score; From a3ebd4c062458fdad2e6bdd3b618e36fb841ca01 Mon Sep 17 00:00:00 2001 From: ashilkn Date: Thu, 6 Jun 2024 11:32:33 +0530 Subject: [PATCH 2/8] [mob][photos] Make score threshold configurable --- .../semantic_search_service.dart | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/mobile/lib/services/machine_learning/semantic_search/semantic_search_service.dart b/mobile/lib/services/machine_learning/semantic_search/semantic_search_service.dart index 625e21ab8e..5f7f509355 100644 --- a/mobile/lib/services/machine_learning/semantic_search/semantic_search_service.dart +++ b/mobile/lib/services/machine_learning/semantic_search/semantic_search_service.dart @@ -236,10 +236,14 @@ class SemanticSearchService { _queue.clear(); } - Future> _getMatchingFiles(String query) async { + Future> _getMatchingFiles( + String query, { + double? scoreThreshold, + }) async { final textEmbedding = await _getTextEmbedding(query); - final queryResults = await _getScores(textEmbedding); + final queryResults = + await _getScores(textEmbedding, scoreThreshold: scoreThreshold); final filesMap = await FilesDB.instance .getFilesFromIDs(queryResults.map((e) => e.id).toList()); @@ -355,13 +359,17 @@ class SemanticSearchService { } } - Future> _getScores(List textEmbedding) async { + Future> _getScores( + List textEmbedding, { + double? scoreThreshold, + }) async { final startTime = DateTime.now(); final List queryResults = await _computer.compute( computeBulkScore, param: { "imageEmbeddings": _cachedEmbeddings, "textEmbedding": textEmbedding, + "scoreThreshold": scoreThreshold, }, taskName: "computeBulkScore", ); @@ -402,12 +410,14 @@ List computeBulkScore(Map args) { final queryResults = []; final imageEmbeddings = args["imageEmbeddings"] as List; final textEmbedding = args["textEmbedding"] as List; + final scoreThreshold = + args["scoreThreshold"] ?? SemanticSearchService.kScoreThreshold; for (final imageEmbedding in imageEmbeddings) { final score = computeScore( imageEmbedding.embedding, textEmbedding, ); - if (score >= SemanticSearchService.kScoreThreshold) { + if (score >= scoreThreshold) { queryResults.add(QueryResult(imageEmbedding.fileID, score)); } } From 6b3c9ee19cd20ca5a562f310e658d46927350f57 Mon Sep 17 00:00:00 2001 From: ashilkn Date: Thu, 6 Jun 2024 17:34:48 +0530 Subject: [PATCH 3/8] [mob][photos] Surface magic section results in UI, using moments section's widget --- mobile/lib/models/search/search_types.dart | 6 +- .../semantic_search_service.dart | 16 +- mobile/lib/services/search_service.dart | 50 +++ .../states/all_sections_examples_state.dart | 3 - .../ui/viewer/search_tab/magic_section.dart | 285 ++++++++++++++++++ .../lib/ui/viewer/search_tab/search_tab.dart | 7 +- 6 files changed, 356 insertions(+), 11 deletions(-) create mode 100644 mobile/lib/ui/viewer/search_tab/magic_section.dart diff --git a/mobile/lib/models/search/search_types.dart b/mobile/lib/models/search/search_types.dart index a13fd57dcb..30b8a0bb0b 100644 --- a/mobile/lib/models/search/search_types.dart +++ b/mobile/lib/models/search/search_types.dart @@ -10,10 +10,10 @@ import "package:photos/events/people_changed_event.dart"; import "package:photos/generated/l10n.dart"; import "package:photos/models/collection/collection.dart"; import "package:photos/models/collection/collection_items.dart"; -import "package:photos/models/search/generic_search_result.dart"; import "package:photos/models/search/search_result.dart"; import "package:photos/models/typedefs.dart"; import "package:photos/services/collections_service.dart"; +import "package:photos/services/machine_learning/semantic_search/frameworks/ml_framework.dart"; import "package:photos/services/search_service.dart"; import "package:photos/ui/viewer/gallery/collection_page.dart"; import "package:photos/ui/viewer/location/add_location_sheet.dart"; @@ -251,7 +251,7 @@ extension SectionTypeExtensions on SectionType { case SectionType.face: return SearchService.instance.getAllFace(limit); case SectionType.content: - return Future.value(List.empty()); + return SearchService.instance.getMagicSectionResutls(); case SectionType.moment: return SearchService.instance.getRandomMomentsSearchResults(context); @@ -293,6 +293,8 @@ extension SectionTypeExtensions on SectionType { switch (this) { case SectionType.location: return [Bus.instance.on()]; + case SectionType.content: + return [Bus.instance.on()]; default: return []; } diff --git a/mobile/lib/services/machine_learning/semantic_search/semantic_search_service.dart b/mobile/lib/services/machine_learning/semantic_search/semantic_search_service.dart index 5f7f509355..d65c67aba3 100644 --- a/mobile/lib/services/machine_learning/semantic_search/semantic_search_service.dart +++ b/mobile/lib/services/machine_learning/semantic_search/semantic_search_service.dart @@ -131,11 +131,15 @@ class SemanticSearchService { _isSyncing = false; } + bool isMagicSearchEnabledAndReady() { + return LocalSettings.instance.hasEnabledMagicSearch() && + _frameworkInitialization.isCompleted; + } + // searchScreenQuery should only be used for the user initiate query on the search screen. // If there are multiple call tho this method, then for all the calls, the result will be the same as the last query. Future<(String, List)> searchScreenQuery(String query) async { - if (!LocalSettings.instance.hasEnabledMagicSearch() || - !_frameworkInitialization.isCompleted) { + if (!isMagicSearchEnabledAndReady()) { return (query, []); } // If there's an ongoing request, just update the last query and return its future. @@ -144,7 +148,7 @@ class SemanticSearchService { return _searchScreenRequest!; } else { // No ongoing request, start a new search. - _searchScreenRequest = _getMatchingFiles(query).then((result) { + _searchScreenRequest = getMatchingFiles(query).then((result) { // Search completed, reset the ongoing request. _searchScreenRequest = null; // If there was a new query during the last search, start a new search with the last query. @@ -236,7 +240,7 @@ class SemanticSearchService { _queue.clear(); } - Future> _getMatchingFiles( + Future> getMatchingFiles( String query, { double? scoreThreshold, }) async { @@ -247,11 +251,13 @@ class SemanticSearchService { final filesMap = await FilesDB.instance .getFilesFromIDs(queryResults.map((e) => e.id).toList()); - final results = []; final ignoredCollections = CollectionsService.instance.getHiddenCollectionIds(); + final deletedEntries = []; + final results = []; + for (final result in queryResults) { final file = filesMap[result.id]; if (file != null && !ignoredCollections.contains(file.collectionID)) { diff --git a/mobile/lib/services/search_service.dart b/mobile/lib/services/search_service.dart index d15eddb718..401cb574d4 100644 --- a/mobile/lib/services/search_service.dart +++ b/mobile/lib/services/search_service.dart @@ -40,6 +40,33 @@ import 'package:photos/utils/date_time_util.dart'; import "package:photos/utils/navigation_util.dart"; import 'package:tuple/tuple.dart'; +const magicPromptsData = [ + { + "prompt": "identity document", + "title": "Identity Document", + "minimumScore": 0.269, + "minimumSize": 0.0, + }, + { + "prompt": "sunset at the beach", + "title": "Sunset", + "minimumScore": 0.25, + "minimumSize": 0.0, + }, + { + "prompt": "roadtrip", + "title": "Roadtrip", + "minimumScore": 0.26, + "minimumSize": 0.0, + }, + { + "prompt": "pizza pasta burger", + "title": "Food", + "minimumScore": 0.27, + "minimumSize": 0.0, + } +]; + class SearchService { Future>? _cachedFilesFuture; Future>? _cachedHiddenFilesFuture; @@ -174,6 +201,29 @@ class SearchService { return searchResults; } + Future> getMagicSectionResutls() async { + if (!SemanticSearchService.instance.isMagicSearchEnabledAndReady()) { + return []; + } + final searchResuts = []; + for (Map magicPrompt in magicPromptsData) { + final files = await SemanticSearchService.instance.getMatchingFiles( + magicPrompt["prompt"], + scoreThreshold: magicPrompt["minimumScore"], + ); + if (files.isNotEmpty) { + searchResuts.add( + GenericSearchResult( + ResultType.magic, + magicPrompt["title"], + files, + ), + ); + } + } + return searchResuts; + } + Future> getRandomMomentsSearchResults( BuildContext context, ) async { diff --git a/mobile/lib/states/all_sections_examples_state.dart b/mobile/lib/states/all_sections_examples_state.dart index a40ecd9255..716de8db56 100644 --- a/mobile/lib/states/all_sections_examples_state.dart +++ b/mobile/lib/states/all_sections_examples_state.dart @@ -88,9 +88,6 @@ class _AllSectionsExamplesProviderState _logger.info("'_debounceTimer: reloading all sections in search tab"); final allSectionsExamples = >>[]; for (SectionType sectionType in SectionType.values) { - if (sectionType == SectionType.content) { - continue; - } allSectionsExamples.add( sectionType.getData(context, limit: kSearchSectionLimit), ); diff --git a/mobile/lib/ui/viewer/search_tab/magic_section.dart b/mobile/lib/ui/viewer/search_tab/magic_section.dart new file mode 100644 index 0000000000..3b49413e06 --- /dev/null +++ b/mobile/lib/ui/viewer/search_tab/magic_section.dart @@ -0,0 +1,285 @@ +import "dart:async"; +import "dart:math"; + +import "package:figma_squircle/figma_squircle.dart"; +import "package:flutter/material.dart"; +import "package:photos/core/constants.dart"; +import "package:photos/events/event.dart"; +import "package:photos/models/search/generic_search_result.dart"; +import "package:photos/models/search/recent_searches.dart"; +import "package:photos/models/search/search_types.dart"; +import "package:photos/services/machine_learning/semantic_search/frameworks/ml_framework.dart"; +import "package:photos/theme/ente_theme.dart"; +import "package:photos/ui/viewer/file/no_thumbnail_widget.dart"; +import "package:photos/ui/viewer/file/thumbnail_widget.dart"; +import "package:photos/ui/viewer/search/result/search_result_page.dart"; +import "package:photos/ui/viewer/search/search_section_cta.dart"; +import "package:photos/ui/viewer/search_tab/section_header.dart"; +import "package:photos/utils/navigation_util.dart"; + +class MagicSection extends StatefulWidget { + final List magicSearchResults; + const MagicSection(this.magicSearchResults, {super.key}); + + @override + State createState() => _MagicSectionState(); +} + +class _MagicSectionState extends State { + late List _magicSearchResults; + final streamSubscriptions = []; + + @override + void initState() { + super.initState(); + _magicSearchResults = widget.magicSearchResults; + + //At times, ml framework is not initialized when the search results are + //requested (widget.momentsSearchResults is empty) and is initialized + //(which fires MLFrameworkInitializationUpdateEvent with + //InitializationState.initialized) before initState of this widget is + //called. We do listen to MLFrameworkInitializationUpdateEvent and reload + //this widget but the event with InitializationState.initialized would have + //already been fired in the above case. + if (_magicSearchResults.isEmpty) { + SectionType.content + .getData( + context, + limit: kSearchSectionLimit, + ) + .then((value) { + if (mounted) { + setState(() { + _magicSearchResults = value as List; + }); + } + }); + } + + final streamsToListenTo = SectionType.content.sectionUpdateEvents(); + for (Stream stream in streamsToListenTo) { + streamSubscriptions.add( + stream.listen((event) async { + final mlFrameWorkEvent = + event as MLFrameworkInitializationUpdateEvent; + if (mlFrameWorkEvent.state == InitializationState.initialized) { + _magicSearchResults = (await SectionType.content.getData( + context, + limit: kSearchSectionLimit, + )) as List; + setState(() {}); + } + }), + ); + } + } + + @override + void dispose() { + for (var subscriptions in streamSubscriptions) { + subscriptions.cancel(); + } + super.dispose(); + } + + @override + void didUpdateWidget(covariant MagicSection oldWidget) { + super.didUpdateWidget(oldWidget); + _magicSearchResults = widget.magicSearchResults; + } + + @override + Widget build(BuildContext context) { + if (_magicSearchResults.isEmpty) { + final textTheme = getEnteTextTheme(context); + return Padding( + padding: const EdgeInsets.only(left: 12, right: 8), + child: Row( + children: [ + Expanded( + child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + Text( + SectionType.moment.sectionTitle(context), + style: textTheme.largeBold, + ), + const SizedBox(height: 24), + Padding( + padding: const EdgeInsets.only(left: 4), + child: Text( + SectionType.moment.getEmptyStateText(context), + style: textTheme.smallMuted, + ), + ), + ], + ), + ), + const SizedBox(width: 8), + const SearchSectionEmptyCTAIcon(SectionType.moment), + ], + ), + ); + } else { + return Padding( + padding: const EdgeInsets.symmetric(vertical: 8), + child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + SectionHeader( + SectionType.moment, + hasMore: (_magicSearchResults.length >= kSearchSectionLimit - 1), + ), + const SizedBox(height: 2), + SizedBox( + child: SingleChildScrollView( + clipBehavior: Clip.none, + padding: const EdgeInsets.symmetric(horizontal: 4.5), + physics: const BouncingScrollPhysics(), + scrollDirection: Axis.horizontal, + child: Row( + crossAxisAlignment: CrossAxisAlignment.start, + children: _magicSearchResults + .map( + (momentSearchResult) => + MomentRecommendation(momentSearchResult), + ) + .toList(), + ), + ), + ), + ], + ), + ); + } + } +} + +class MomentRecommendation extends StatelessWidget { + static const _width = 100.0; + static const _height = 145.0; + static const _borderWidth = 1.0; + static const _cornerRadius = 5.0; + static const _cornerSmoothing = 1.0; + final GenericSearchResult momentSearchResult; + const MomentRecommendation(this.momentSearchResult, {super.key}); + + @override + Widget build(BuildContext context) { + final heroTag = momentSearchResult.heroTag() + + (momentSearchResult.previewThumbnail()?.tag ?? ""); + final enteTextTheme = getEnteTextTheme(context); + return Padding( + padding: EdgeInsets.symmetric(horizontal: max(2.5 - _borderWidth, 0)), + child: GestureDetector( + onTap: () { + RecentSearches().add(momentSearchResult.name()); + if (momentSearchResult.onResultTap != null) { + momentSearchResult.onResultTap!(context); + } else { + routeToPage( + context, + SearchResultPage(momentSearchResult), + ); + } + }, + child: SizedBox( + width: _width + _borderWidth * 2, + height: _height + _borderWidth * 2, + child: Stack( + clipBehavior: Clip.none, + alignment: Alignment.center, + children: [ + ClipSmoothRect( + radius: SmoothBorderRadius( + cornerRadius: _cornerRadius + _borderWidth, + cornerSmoothing: _cornerSmoothing, + ), + child: Container( + color: Colors.white.withOpacity(0.16), + width: _width + _borderWidth * 2, + height: _height + _borderWidth * 2, + ), + ), + Container( + decoration: BoxDecoration( + boxShadow: [ + BoxShadow( + color: Colors.black.withOpacity(0.1), + blurRadius: 6.25, + offset: const Offset(-1.25, 2.5), + ), + ], + ), + child: ClipSmoothRect( + radius: SmoothBorderRadius( + cornerRadius: _cornerRadius, + cornerSmoothing: _cornerSmoothing, + ), + child: Stack( + alignment: Alignment.bottomCenter, + clipBehavior: Clip.none, + children: [ + SizedBox( + width: _width, + height: _height, + child: momentSearchResult.previewThumbnail() != null + ? Hero( + tag: heroTag, + child: ThumbnailWidget( + momentSearchResult.previewThumbnail()!, + shouldShowArchiveStatus: false, + shouldShowSyncStatus: false, + ), + ) + : const NoThumbnailWidget(), + ), + Container( + height: 145, + width: 100, + decoration: BoxDecoration( + gradient: LinearGradient( + begin: Alignment.topCenter, + end: Alignment.bottomCenter, + colors: [ + Colors.black.withOpacity(0), + Colors.black.withOpacity(0), + Colors.black.withOpacity(0.5), + ], + stops: const [ + 0, + 0.1, + 1, + ], + ), + ), + ), + ConstrainedBox( + constraints: const BoxConstraints( + maxWidth: 76, + ), + child: Padding( + padding: const EdgeInsets.only( + bottom: 8, + ), + child: Text( + momentSearchResult.name(), + style: enteTextTheme.small.copyWith( + color: Colors.white, + ), + maxLines: 3, + overflow: TextOverflow.fade, + ), + ), + ), + ], + ), + ), + ), + ], + ), + ), + ), + ); + } +} diff --git a/mobile/lib/ui/viewer/search_tab/search_tab.dart b/mobile/lib/ui/viewer/search_tab/search_tab.dart index 46dcfda036..9c3d1d2ae2 100644 --- a/mobile/lib/ui/viewer/search_tab/search_tab.dart +++ b/mobile/lib/ui/viewer/search_tab/search_tab.dart @@ -18,6 +18,7 @@ import "package:photos/ui/viewer/search_tab/contacts_section.dart"; import "package:photos/ui/viewer/search_tab/descriptions_section.dart"; import "package:photos/ui/viewer/search_tab/file_type_section.dart"; import "package:photos/ui/viewer/search_tab/locations_section.dart"; +import "package:photos/ui/viewer/search_tab/magic_section.dart"; import "package:photos/ui/viewer/search_tab/moments_section.dart"; import "package:photos/ui/viewer/search_tab/people_section.dart"; import "package:photos/utils/local_settings.dart"; @@ -82,7 +83,6 @@ class _AllSearchSectionsState extends State { @override Widget build(BuildContext context) { final searchTypes = SectionType.values.toList(growable: true); - searchTypes.remove(SectionType.content); return Padding( padding: const EdgeInsets.only(top: 8), @@ -153,6 +153,11 @@ class _AllSearchSectionsState extends State { snapshot.data!.elementAt(index) as List, ); + case SectionType.content: + return MagicSection( + snapshot.data!.elementAt(index) + as List, + ); default: const SizedBox.shrink(); } From 5dda37a19259fefc8b6101a715f5cca0ca7d02cc Mon Sep 17 00:00:00 2001 From: ashilkn Date: Thu, 6 Jun 2024 18:10:50 +0530 Subject: [PATCH 4/8] [mob][photos] Use correct naming + remove unnecessary field --- .../search/result/search_result_widget.dart | 2 - .../ui/viewer/search_tab/magic_section.dart | 41 ++++++++++--------- 2 files changed, 22 insertions(+), 21 deletions(-) diff --git a/mobile/lib/ui/viewer/search/result/search_result_widget.dart b/mobile/lib/ui/viewer/search/result/search_result_widget.dart index fbd77531a8..965e0fcc0e 100644 --- a/mobile/lib/ui/viewer/search/result/search_result_widget.dart +++ b/mobile/lib/ui/viewer/search/result/search_result_widget.dart @@ -13,14 +13,12 @@ class SearchResultWidget extends StatelessWidget { final SearchResult searchResult; final Future? resultCount; final Function? onResultTap; - final Map? params; const SearchResultWidget( this.searchResult, { Key? key, this.resultCount, this.onResultTap, - this.params, }) : super(key: key); @override diff --git a/mobile/lib/ui/viewer/search_tab/magic_section.dart b/mobile/lib/ui/viewer/search_tab/magic_section.dart index 3b49413e06..882968a9a9 100644 --- a/mobile/lib/ui/viewer/search_tab/magic_section.dart +++ b/mobile/lib/ui/viewer/search_tab/magic_section.dart @@ -101,14 +101,14 @@ class _MagicSectionState extends State { crossAxisAlignment: CrossAxisAlignment.start, children: [ Text( - SectionType.moment.sectionTitle(context), + SectionType.content.sectionTitle(context), style: textTheme.largeBold, ), const SizedBox(height: 24), Padding( padding: const EdgeInsets.only(left: 4), child: Text( - SectionType.moment.getEmptyStateText(context), + SectionType.content.getEmptyStateText(context), style: textTheme.smallMuted, ), ), @@ -116,7 +116,7 @@ class _MagicSectionState extends State { ), ), const SizedBox(width: 8), - const SearchSectionEmptyCTAIcon(SectionType.moment), + const SearchSectionEmptyCTAIcon(SectionType.content), ], ), ); @@ -127,7 +127,7 @@ class _MagicSectionState extends State { crossAxisAlignment: CrossAxisAlignment.start, children: [ SectionHeader( - SectionType.moment, + SectionType.content, hasMore: (_magicSearchResults.length >= kSearchSectionLimit - 1), ), const SizedBox(height: 2), @@ -141,8 +141,8 @@ class _MagicSectionState extends State { crossAxisAlignment: CrossAxisAlignment.start, children: _magicSearchResults .map( - (momentSearchResult) => - MomentRecommendation(momentSearchResult), + (magicSearchResult) => + MagicRecommendation(magicSearchResult), ) .toList(), ), @@ -155,31 +155,34 @@ class _MagicSectionState extends State { } } -class MomentRecommendation extends StatelessWidget { +class MagicRecommendation extends StatelessWidget { static const _width = 100.0; - static const _height = 145.0; + static const _height = 110.0; static const _borderWidth = 1.0; static const _cornerRadius = 5.0; static const _cornerSmoothing = 1.0; - final GenericSearchResult momentSearchResult; - const MomentRecommendation(this.momentSearchResult, {super.key}); + final GenericSearchResult magicSearchResult; + const MagicRecommendation(this.magicSearchResult, {super.key}); @override Widget build(BuildContext context) { - final heroTag = momentSearchResult.heroTag() + - (momentSearchResult.previewThumbnail()?.tag ?? ""); + final heroTag = magicSearchResult.heroTag() + + (magicSearchResult.previewThumbnail()?.tag ?? ""); final enteTextTheme = getEnteTextTheme(context); return Padding( padding: EdgeInsets.symmetric(horizontal: max(2.5 - _borderWidth, 0)), child: GestureDetector( onTap: () { - RecentSearches().add(momentSearchResult.name()); - if (momentSearchResult.onResultTap != null) { - momentSearchResult.onResultTap!(context); + RecentSearches().add(magicSearchResult.name()); + if (magicSearchResult.onResultTap != null) { + magicSearchResult.onResultTap!(context); } else { routeToPage( context, - SearchResultPage(momentSearchResult), + SearchResultPage( + magicSearchResult, + enableGrouping: false, + ), ); } }, @@ -223,11 +226,11 @@ class MomentRecommendation extends StatelessWidget { SizedBox( width: _width, height: _height, - child: momentSearchResult.previewThumbnail() != null + child: magicSearchResult.previewThumbnail() != null ? Hero( tag: heroTag, child: ThumbnailWidget( - momentSearchResult.previewThumbnail()!, + magicSearchResult.previewThumbnail()!, shouldShowArchiveStatus: false, shouldShowSyncStatus: false, ), @@ -263,7 +266,7 @@ class MomentRecommendation extends StatelessWidget { bottom: 8, ), child: Text( - momentSearchResult.name(), + magicSearchResult.name(), style: enteTextTheme.small.copyWith( color: Colors.white, ), From 637f3522a92af2cd95cc97a8ed34e4599d9c8dd0 Mon Sep 17 00:00:00 2001 From: ashilkn Date: Thu, 6 Jun 2024 19:16:20 +0530 Subject: [PATCH 5/8] [mob][photos] Polish magic section UI --- .../ui/viewer/search_tab/magic_section.dart | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/mobile/lib/ui/viewer/search_tab/magic_section.dart b/mobile/lib/ui/viewer/search_tab/magic_section.dart index 882968a9a9..14b3772276 100644 --- a/mobile/lib/ui/viewer/search_tab/magic_section.dart +++ b/mobile/lib/ui/viewer/search_tab/magic_section.dart @@ -82,11 +82,12 @@ class _MagicSectionState extends State { super.dispose(); } - @override - void didUpdateWidget(covariant MagicSection oldWidget) { - super.didUpdateWidget(oldWidget); - _magicSearchResults = widget.magicSearchResults; - } + // @override + // void didUpdateWidget(covariant MagicSection oldWidget) { + // super.didUpdateWidget(oldWidget); + // //widget.magicSearch is empty when doing a hot reload + // _magicSearchResults = widget.magicSearchResults; + // } @override Widget build(BuildContext context) { @@ -159,7 +160,7 @@ class MagicRecommendation extends StatelessWidget { static const _width = 100.0; static const _height = 110.0; static const _borderWidth = 1.0; - static const _cornerRadius = 5.0; + static const _cornerRadius = 12.0; static const _cornerSmoothing = 1.0; final GenericSearchResult magicSearchResult; const MagicRecommendation(this.magicSearchResult, {super.key}); @@ -199,7 +200,7 @@ class MagicRecommendation extends StatelessWidget { cornerSmoothing: _cornerSmoothing, ), child: Container( - color: Colors.white.withOpacity(0.16), + color: getEnteColorScheme(context).strokeFaint, width: _width + _borderWidth * 2, height: _height + _borderWidth * 2, ), @@ -238,8 +239,8 @@ class MagicRecommendation extends StatelessWidget { : const NoThumbnailWidget(), ), Container( - height: 145, - width: 100, + height: _height, + width: _width, decoration: BoxDecoration( gradient: LinearGradient( begin: Alignment.topCenter, From fabd6351d9a340675f1c7d96156cf31a0a663121 Mon Sep 17 00:00:00 2001 From: ashilkn Date: Fri, 7 Jun 2024 17:28:13 +0530 Subject: [PATCH 6/8] [mob][photos] SectionType.content -> SectionType.magic --- mobile/lib/models/search/search_types.dart | 25 ++++++++--------- .../search/result/no_result_widget.dart | 2 +- .../ui/viewer/search_tab/magic_section.dart | 28 ++++++++++--------- .../lib/ui/viewer/search_tab/search_tab.dart | 2 +- 4 files changed, 29 insertions(+), 28 deletions(-) diff --git a/mobile/lib/models/search/search_types.dart b/mobile/lib/models/search/search_types.dart index 30b8a0bb0b..e6dab467e1 100644 --- a/mobile/lib/models/search/search_types.dart +++ b/mobile/lib/models/search/search_types.dart @@ -41,8 +41,7 @@ enum ResultType { enum SectionType { face, location, - // Grouping based on ML or manual tagging - content, + magic, // includes year, month , day, event ResultType moment, album, @@ -58,8 +57,8 @@ extension SectionTypeExtensions on SectionType { switch (this) { case SectionType.face: return S.of(context).people; - case SectionType.content: - return S.of(context).contents; + case SectionType.magic: + return "Magic"; case SectionType.moment: return S.of(context).moments; case SectionType.location: @@ -79,8 +78,8 @@ extension SectionTypeExtensions on SectionType { switch (this) { case SectionType.face: return S.of(context).searchFaceEmptySection; - case SectionType.content: - return "Contents"; + case SectionType.magic: + return "Magic"; case SectionType.moment: return S.of(context).searchDatesEmptySection; case SectionType.location: @@ -102,7 +101,7 @@ extension SectionTypeExtensions on SectionType { switch (this) { case SectionType.face: return false; - case SectionType.content: + case SectionType.magic: return false; case SectionType.moment: return false; @@ -125,7 +124,7 @@ extension SectionTypeExtensions on SectionType { switch (this) { case SectionType.face: return false; - case SectionType.content: + case SectionType.magic: return false; case SectionType.moment: return false; @@ -147,9 +146,9 @@ extension SectionTypeExtensions on SectionType { case SectionType.face: // todo: later return "Setup"; - case SectionType.content: + case SectionType.magic: // todo: later - return "Add tags"; + return "temp"; case SectionType.moment: return S.of(context).addNew; case SectionType.location: @@ -169,7 +168,7 @@ extension SectionTypeExtensions on SectionType { switch (this) { case SectionType.face: return Icons.adaptive.arrow_forward_outlined; - case SectionType.content: + case SectionType.magic: return null; case SectionType.moment: return null; @@ -250,7 +249,7 @@ extension SectionTypeExtensions on SectionType { switch (this) { case SectionType.face: return SearchService.instance.getAllFace(limit); - case SectionType.content: + case SectionType.magic: return SearchService.instance.getMagicSectionResutls(); case SectionType.moment: @@ -293,7 +292,7 @@ extension SectionTypeExtensions on SectionType { switch (this) { case SectionType.location: return [Bus.instance.on()]; - case SectionType.content: + case SectionType.magic: return [Bus.instance.on()]; default: return []; diff --git a/mobile/lib/ui/viewer/search/result/no_result_widget.dart b/mobile/lib/ui/viewer/search/result/no_result_widget.dart index 48ba811df5..dc64a8e322 100644 --- a/mobile/lib/ui/viewer/search/result/no_result_widget.dart +++ b/mobile/lib/ui/viewer/search/result/no_result_widget.dart @@ -21,7 +21,7 @@ class _NoResultWidgetState extends State { super.initState(); searchTypes = SectionType.values.toList(growable: true); // remove face and content sectionType - searchTypes.remove(SectionType.content); + searchTypes.remove(SectionType.magic); } @override diff --git a/mobile/lib/ui/viewer/search_tab/magic_section.dart b/mobile/lib/ui/viewer/search_tab/magic_section.dart index 14b3772276..200f2b988d 100644 --- a/mobile/lib/ui/viewer/search_tab/magic_section.dart +++ b/mobile/lib/ui/viewer/search_tab/magic_section.dart @@ -42,7 +42,7 @@ class _MagicSectionState extends State { //this widget but the event with InitializationState.initialized would have //already been fired in the above case. if (_magicSearchResults.isEmpty) { - SectionType.content + SectionType.magic .getData( context, limit: kSearchSectionLimit, @@ -56,14 +56,14 @@ class _MagicSectionState extends State { }); } - final streamsToListenTo = SectionType.content.sectionUpdateEvents(); + final streamsToListenTo = SectionType.magic.sectionUpdateEvents(); for (Stream stream in streamsToListenTo) { streamSubscriptions.add( stream.listen((event) async { final mlFrameWorkEvent = event as MLFrameworkInitializationUpdateEvent; if (mlFrameWorkEvent.state == InitializationState.initialized) { - _magicSearchResults = (await SectionType.content.getData( + _magicSearchResults = (await SectionType.magic.getData( context, limit: kSearchSectionLimit, )) as List; @@ -82,12 +82,14 @@ class _MagicSectionState extends State { super.dispose(); } - // @override - // void didUpdateWidget(covariant MagicSection oldWidget) { - // super.didUpdateWidget(oldWidget); - // //widget.magicSearch is empty when doing a hot reload - // _magicSearchResults = widget.magicSearchResults; - // } + @override + void didUpdateWidget(covariant MagicSection oldWidget) { + super.didUpdateWidget(oldWidget); + //widget.magicSearch is empty when doing a hot reload + if (widget.magicSearchResults.isNotEmpty) { + _magicSearchResults = widget.magicSearchResults; + } + } @override Widget build(BuildContext context) { @@ -102,14 +104,14 @@ class _MagicSectionState extends State { crossAxisAlignment: CrossAxisAlignment.start, children: [ Text( - SectionType.content.sectionTitle(context), + SectionType.magic.sectionTitle(context), style: textTheme.largeBold, ), const SizedBox(height: 24), Padding( padding: const EdgeInsets.only(left: 4), child: Text( - SectionType.content.getEmptyStateText(context), + SectionType.magic.getEmptyStateText(context), style: textTheme.smallMuted, ), ), @@ -117,7 +119,7 @@ class _MagicSectionState extends State { ), ), const SizedBox(width: 8), - const SearchSectionEmptyCTAIcon(SectionType.content), + const SearchSectionEmptyCTAIcon(SectionType.magic), ], ), ); @@ -128,7 +130,7 @@ class _MagicSectionState extends State { crossAxisAlignment: CrossAxisAlignment.start, children: [ SectionHeader( - SectionType.content, + SectionType.magic, hasMore: (_magicSearchResults.length >= kSearchSectionLimit - 1), ), const SizedBox(height: 2), diff --git a/mobile/lib/ui/viewer/search_tab/search_tab.dart b/mobile/lib/ui/viewer/search_tab/search_tab.dart index 9c3d1d2ae2..07a13d9e2b 100644 --- a/mobile/lib/ui/viewer/search_tab/search_tab.dart +++ b/mobile/lib/ui/viewer/search_tab/search_tab.dart @@ -153,7 +153,7 @@ class _AllSearchSectionsState extends State { snapshot.data!.elementAt(index) as List, ); - case SectionType.content: + case SectionType.magic: return MagicSection( snapshot.data!.elementAt(index) as List, From ac05f085c1de09d76863b92cbebdf24f9476ce3b Mon Sep 17 00:00:00 2001 From: ashilkn Date: Wed, 12 Jun 2024 16:14:09 +0530 Subject: [PATCH 7/8] [mob][photos] Get magic prompt data from remote --- .../lib/services/remote_assets_service.dart | 4 +- mobile/lib/services/search_service.dart | 41 +++++++------------ 2 files changed, 16 insertions(+), 29 deletions(-) diff --git a/mobile/lib/services/remote_assets_service.dart b/mobile/lib/services/remote_assets_service.dart index 1e2cb3b6df..487f2f8b11 100644 --- a/mobile/lib/services/remote_assets_service.dart +++ b/mobile/lib/services/remote_assets_service.dart @@ -18,10 +18,10 @@ class RemoteAssetsService { static final RemoteAssetsService instance = RemoteAssetsService._privateConstructor(); - Future getAsset(String remotePath) async { + Future getAsset(String remotePath, {bool refetch = false}) async { final path = await _getLocalPath(remotePath); final file = File(path); - if (await file.exists()) { + if (await file.exists() && !refetch) { _logger.info("Returning cached file for $remotePath"); return file; } else { diff --git a/mobile/lib/services/search_service.dart b/mobile/lib/services/search_service.dart index 401cb574d4..1ac46c5faf 100644 --- a/mobile/lib/services/search_service.dart +++ b/mobile/lib/services/search_service.dart @@ -1,3 +1,4 @@ +import "dart:convert"; import "dart:math"; import "package:flutter/cupertino.dart"; @@ -31,6 +32,7 @@ import "package:photos/services/location_service.dart"; import "package:photos/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart"; import "package:photos/services/machine_learning/face_ml/person/person_service.dart"; import 'package:photos/services/machine_learning/semantic_search/semantic_search_service.dart'; +import "package:photos/services/remote_assets_service.dart"; import "package:photos/states/location_screen_state.dart"; import "package:photos/ui/viewer/location/add_location_sheet.dart"; import "package:photos/ui/viewer/location/location_screen.dart"; @@ -40,39 +42,15 @@ import 'package:photos/utils/date_time_util.dart'; import "package:photos/utils/navigation_util.dart"; import 'package:tuple/tuple.dart'; -const magicPromptsData = [ - { - "prompt": "identity document", - "title": "Identity Document", - "minimumScore": 0.269, - "minimumSize": 0.0, - }, - { - "prompt": "sunset at the beach", - "title": "Sunset", - "minimumScore": 0.25, - "minimumSize": 0.0, - }, - { - "prompt": "roadtrip", - "title": "Roadtrip", - "minimumScore": 0.26, - "minimumSize": 0.0, - }, - { - "prompt": "pizza pasta burger", - "title": "Food", - "minimumScore": 0.27, - "minimumSize": 0.0, - } -]; - class SearchService { Future>? _cachedFilesFuture; Future>? _cachedHiddenFilesFuture; final _logger = Logger((SearchService).toString()); final _collectionService = CollectionsService.instance; static const _maximumResultsLimit = 20; + static const _kMagicPromptsDataUrl = "https://discover.ente.io/v1.json"; + + var magicPromptsData = []; SearchService._privateConstructor(); @@ -84,6 +62,15 @@ class SearchService { _cachedFilesFuture = null; _cachedHiddenFilesFuture = null; }); + _loadMagicPrompts(); + } + + Future _loadMagicPrompts() async { + final file = await RemoteAssetsService.instance + .getAsset(_kMagicPromptsDataUrl, refetch: true); + + final json = jsonDecode(await file.readAsString()); + magicPromptsData = json["prompts"]; } Set ignoreCollections() { From 7fdf52309ae79b4dc073828b5df8f18b5143d38c Mon Sep 17 00:00:00 2001 From: ashilkn Date: Wed, 12 Jun 2024 16:40:39 +0530 Subject: [PATCH 8/8] [mob][photos] Keep showing of magic examples on search tab behind feature flag --- mobile/lib/services/search_service.dart | 5 +- .../ui/viewer/search_tab/magic_section.dart | 60 +++++++++---------- 2 files changed, 34 insertions(+), 31 deletions(-) diff --git a/mobile/lib/services/search_service.dart b/mobile/lib/services/search_service.dart index 1ac46c5faf..e55684ed9b 100644 --- a/mobile/lib/services/search_service.dart +++ b/mobile/lib/services/search_service.dart @@ -27,6 +27,7 @@ import 'package:photos/models/search/album_search_result.dart'; import 'package:photos/models/search/generic_search_result.dart'; import "package:photos/models/search/search_constants.dart"; import "package:photos/models/search/search_types.dart"; +import "package:photos/service_locator.dart"; import 'package:photos/services/collections_service.dart'; import "package:photos/services/location_service.dart"; import "package:photos/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart"; @@ -62,7 +63,9 @@ class SearchService { _cachedFilesFuture = null; _cachedHiddenFilesFuture = null; }); - _loadMagicPrompts(); + if (flagService.internalUser) { + _loadMagicPrompts(); + } } Future _loadMagicPrompts() async { diff --git a/mobile/lib/ui/viewer/search_tab/magic_section.dart b/mobile/lib/ui/viewer/search_tab/magic_section.dart index 200f2b988d..d088de92e5 100644 --- a/mobile/lib/ui/viewer/search_tab/magic_section.dart +++ b/mobile/lib/ui/viewer/search_tab/magic_section.dart @@ -13,7 +13,6 @@ import "package:photos/theme/ente_theme.dart"; import "package:photos/ui/viewer/file/no_thumbnail_widget.dart"; import "package:photos/ui/viewer/file/thumbnail_widget.dart"; import "package:photos/ui/viewer/search/result/search_result_page.dart"; -import "package:photos/ui/viewer/search/search_section_cta.dart"; import "package:photos/ui/viewer/search_tab/section_header.dart"; import "package:photos/utils/navigation_util.dart"; @@ -94,35 +93,36 @@ class _MagicSectionState extends State { @override Widget build(BuildContext context) { if (_magicSearchResults.isEmpty) { - final textTheme = getEnteTextTheme(context); - return Padding( - padding: const EdgeInsets.only(left: 12, right: 8), - child: Row( - children: [ - Expanded( - child: Column( - crossAxisAlignment: CrossAxisAlignment.start, - children: [ - Text( - SectionType.magic.sectionTitle(context), - style: textTheme.largeBold, - ), - const SizedBox(height: 24), - Padding( - padding: const EdgeInsets.only(left: 4), - child: Text( - SectionType.magic.getEmptyStateText(context), - style: textTheme.smallMuted, - ), - ), - ], - ), - ), - const SizedBox(width: 8), - const SearchSectionEmptyCTAIcon(SectionType.magic), - ], - ), - ); + // final textTheme = getEnteTextTheme(context); + // return Padding( + // padding: const EdgeInsets.only(left: 12, right: 8), + // child: Row( + // children: [ + // Expanded( + // child: Column( + // crossAxisAlignment: CrossAxisAlignment.start, + // children: [ + // Text( + // SectionType.magic.sectionTitle(context), + // style: textTheme.largeBold, + // ), + // const SizedBox(height: 24), + // Padding( + // padding: const EdgeInsets.only(left: 4), + // child: Text( + // SectionType.magic.getEmptyStateText(context), + // style: textTheme.smallMuted, + // ), + // ), + // ], + // ), + // ), + // const SizedBox(width: 8), + // const SearchSectionEmptyCTAIcon(SectionType.magic), + // ], + // ), + // ); + return const SizedBox.shrink(); } else { return Padding( padding: const EdgeInsets.symmetric(vertical: 8),