From a575e5f3c8451fa44f72ff352b7109c13c848c5d Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Tue, 28 Jan 2025 10:43:22 +0530 Subject: [PATCH 01/25] [mob][photos] Functions to get embeddings of specific files --- mobile/lib/db/ml/clip_db.dart | 15 +++++++++++++++ mobile/lib/db/ml/db.dart | 22 ++++++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/mobile/lib/db/ml/clip_db.dart b/mobile/lib/db/ml/clip_db.dart index 59f6dc936b..953a7c5397 100644 --- a/mobile/lib/db/ml/clip_db.dart +++ b/mobile/lib/db/ml/clip_db.dart @@ -17,6 +17,21 @@ extension ClipDB on MLDataDB { return _convertToVectors(results); } + Future> getClipVectorsForFileIDs( + Iterable fileIDs, + ) async { + final db = await MLDataDB.instance.asyncDB; + final results = await db.getAll( + 'SELECT * FROM $clipTable WHERE $fileIDColumn IN (${fileIDs.join(", ")})', + ); + final Map embeddings = {}; + for (final result in results) { + final embedding = _getVectorFromRow(result); + embeddings[embedding.fileID] = embedding; + } + return embeddings; + } + // Get indexed FileIDs Future> clipIndexedFileWithVersion() async { final db = await MLDataDB.instance.asyncDB; diff --git a/mobile/lib/db/ml/db.dart b/mobile/lib/db/ml/db.dart index 90f31d83ae..81eb486c26 100644 --- a/mobile/lib/db/ml/db.dart +++ b/mobile/lib/db/ml/db.dart @@ -400,6 +400,28 @@ class MLDataDB { return maps.map((e) => mapRowToFace(e)).toList(); } + Future>> getFacesForFileIDs( + Iterable fileUploadIDs, + ) async { + final db = await instance.asyncDB; + final List> maps = await db.getAll( + ''' + SELECT * FROM $facesTable + WHERE $fileIDColumn IN (${fileUploadIDs.map((id) => "'$id'").join(",")}) + ''', + ); + if (maps.isEmpty) { + return {}; + } + final result = >{}; + for (final map in maps) { + final face = mapRowToFace(map); + final fileID = map[fileIDColumn] as int; + result.putIfAbsent(fileID, () => []).add(face); + } + return result; + } + Future>> getClusterToFaceIDs( Set clusterIDs, ) async { From e50ab7ca8415277a01ca72a313e8666e64a5da19 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Tue, 28 Jan 2025 10:43:58 +0530 Subject: [PATCH 02/25] [mob][photos] First iteration of file selection --- mobile/lib/services/search_service.dart | 90 +++++++++++++++++++++++-- 1 file changed, 86 insertions(+), 4 deletions(-) diff --git a/mobile/lib/services/search_service.dart b/mobile/lib/services/search_service.dart index ad49703911..d0d7df0642 100644 --- a/mobile/lib/services/search_service.dart +++ b/mobile/lib/services/search_service.dart @@ -4,12 +4,14 @@ import "package:flutter/cupertino.dart"; import "package:flutter/material.dart"; import "package:intl/intl.dart"; import 'package:logging/logging.dart'; +import "package:ml_linalg/linalg.dart"; import "package:photos/core/constants.dart"; import 'package:photos/core/event_bus.dart'; import 'package:photos/data/holidays.dart'; import 'package:photos/data/months.dart'; import 'package:photos/data/years.dart'; import 'package:photos/db/files_db.dart'; +import "package:photos/db/ml/clip_db.dart"; import "package:photos/db/ml/db.dart"; import 'package:photos/events/local_photos_updated_event.dart'; import "package:photos/extensions/user_extension.dart"; @@ -40,6 +42,7 @@ import "package:photos/services/filter/db_filters.dart"; import "package:photos/services/location_service.dart"; import "package:photos/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart"; import "package:photos/services/machine_learning/face_ml/person/person_service.dart"; +import "package:photos/services/machine_learning/ml_computer.dart"; import 'package:photos/services/machine_learning/semantic_search/semantic_search_service.dart'; import "package:photos/services/user_remote_flag_service.dart"; import "package:photos/states/location_screen_state.dart"; @@ -1233,17 +1236,18 @@ class SearchService { final date = DateTime(currentTime.year, dayMonth ~/ 100, dayMonth % 100); final allPhotos = yearGroups.values.expand((x) => x).toList(); + final photoSelection = await _bestSelection(allPhotos); searchResults.add( GenericSearchResult( ResultType.event, 'Memories of ${DateFormat('MMMM d').format(date)}', - allPhotos, + photoSelection, hierarchicalSearchFilter: TopLevelGenericFilter( filterName: DateFormat('MMMM d').format(date), occurrence: kMostRelevantFilter, filterResultType: ResultType.event, - matchedUploadedIDs: filesToUploadedFileIDs(allPhotos), + matchedUploadedIDs: filesToUploadedFileIDs(photoSelection), filterIcon: Icons.event_outlined, ), ), @@ -1253,6 +1257,7 @@ class SearchService { for (final year in significantYears) { final date = DateTime(year, dayMonth ~/ 100, dayMonth % 100); final files = yearGroups[year]!; + final photoSelection = await _bestSelection(files); String name = DateFormat.yMMMd(Localizations.localeOf(context).languageCode) .format(date); @@ -1264,12 +1269,12 @@ class SearchService { GenericSearchResult( ResultType.event, name, - files, + photoSelection, hierarchicalSearchFilter: TopLevelGenericFilter( filterName: name, occurrence: kMostRelevantFilter, filterResultType: ResultType.event, - matchedUploadedIDs: filesToUploadedFileIDs(files), + matchedUploadedIDs: filesToUploadedFileIDs(photoSelection), filterIcon: Icons.event_outlined, ), ), @@ -1283,6 +1288,83 @@ class SearchService { return searchResults; } + /// Returns the best selection of files from the given list. + /// Makes sure that the selection is not more than 10 files, + /// and that each year of the original list is represented. + Future> _bestSelection(List files) async { + if (files.length <= 10) return files; + final fileIDs = files.map((e) => e.uploadedFileID!).toSet(); + final fileIdToFace = await MLDataDB.instance.getFacesForFileIDs(fileIDs); + final fileIdToClip = + await MLDataDB.instance.getClipVectorsForFileIDs(fileIDs); + final allYears = files.map((e) { + final creationTime = DateTime.fromMicrosecondsSinceEpoch(e.creationTime!); + return creationTime.year; + }).toSet(); + if (allYears.length <= 1) { + // sort first on clip embeddings TODO: lau: come up with a better prompt + const query = 'Beautiful photo of a nice memory'; + // TODO: lau: optimize this later so we don't keep computing embedding + final textEmbedding = await MLComputer.instance.runClipText(query); + final textVector = Vector.fromList(textEmbedding); + final fileToScore = {}; + for (final file in files) { + final clip = fileIdToClip[file.uploadedFileID!]; + if (clip == null) { + fileToScore[file.uploadedFileID!] = 0; + continue; + } + final score = clip.vector.dot(textVector); + fileToScore[file.uploadedFileID!] = score; + } + files.sort( + (a, b) => fileToScore[b.uploadedFileID!]! + .compareTo(fileToScore[a.uploadedFileID!]!), + ); + + // then sort on faces + final fileToFaceCount = {}; + for (final file in files) { + final faces = fileIdToFace[file.uploadedFileID!]; + if (faces == null) { + fileToFaceCount[file.uploadedFileID!] = 0; + continue; + } + fileToFaceCount[file.uploadedFileID!] = faces.length; + } + files.sort( + (a, b) => fileToFaceCount[b.uploadedFileID!]! + .compareTo(fileToFaceCount[a.uploadedFileID!]!), + ); + + // then filter out similar images as much as possible + final filteredFiles = [files.first]; + final fileCount = files.length; + int skipped = 0; + filesLoop: + for (final file in files.sublist(1)) { + if (filteredFiles.length >= 10) break; + final clip = fileIdToClip[file.uploadedFileID!]; + if (clip != null && (fileCount - skipped) > 10) { + for (final filteredFile in filteredFiles) { + final fClip = fileIdToClip[filteredFile.uploadedFileID!]; + if (fClip == null) continue; + final similarity = clip.vector.dot(fClip.vector); + if (similarity > 0.15) { + skipped++; + continue filesLoop; + } + } + } + filteredFiles.add(file); + } + return filteredFiles; + } else { + // TODO: lau: add logic for multiple years. Main extra thing is getting distribution over the years right + return files; + } + } + Future getRandomDateResults( BuildContext context, ) async { From 69025f7ebf331c14b6b1f3fc9f2bdb1fafff03f3 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Tue, 28 Jan 2025 16:27:31 +0530 Subject: [PATCH 03/25] [mob][photos] Increase threshold --- mobile/lib/services/search_service.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobile/lib/services/search_service.dart b/mobile/lib/services/search_service.dart index d0d7df0642..cb1d365528 100644 --- a/mobile/lib/services/search_service.dart +++ b/mobile/lib/services/search_service.dart @@ -1350,7 +1350,7 @@ class SearchService { final fClip = fileIdToClip[filteredFile.uploadedFileID!]; if (fClip == null) continue; final similarity = clip.vector.dot(fClip.vector); - if (similarity > 0.15) { + if (similarity > 0.80) { skipped++; continue filesLoop; } From dc401f6f729e35581c9be92e0fd4986d896d3efe Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Tue, 28 Jan 2025 16:36:16 +0530 Subject: [PATCH 04/25] [mob][photos] Sort --- mobile/lib/services/search_service.dart | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mobile/lib/services/search_service.dart b/mobile/lib/services/search_service.dart index cb1d365528..20a724871f 100644 --- a/mobile/lib/services/search_service.dart +++ b/mobile/lib/services/search_service.dart @@ -1358,6 +1358,8 @@ class SearchService { } filteredFiles.add(file); } + // Order the final selection chronologically + filteredFiles.sort((a, b) => b.creationTime!.compareTo(a.creationTime!)); return filteredFiles; } else { // TODO: lau: add logic for multiple years. Main extra thing is getting distribution over the years right From 0059f1817b48169255cbfce7fcd398d71a8035ba Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Tue, 28 Jan 2025 16:47:50 +0530 Subject: [PATCH 05/25] [mob][photos] Add todo --- mobile/lib/services/search_service.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobile/lib/services/search_service.dart b/mobile/lib/services/search_service.dart index 20a724871f..8a4a955f01 100644 --- a/mobile/lib/services/search_service.dart +++ b/mobile/lib/services/search_service.dart @@ -1322,7 +1322,7 @@ class SearchService { .compareTo(fileToScore[a.uploadedFileID!]!), ); - // then sort on faces + // then sort on faces TODO: lau: prioritize named faces final fileToFaceCount = {}; for (final file in files) { final faces = fileIdToFace[file.uploadedFileID!]; From cd190f21d12641656fe0c5cb94c6b22f0f4e491f Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Wed, 29 Jan 2025 11:50:51 +0530 Subject: [PATCH 06/25] [mob][photos] Prioritize named faces --- mobile/lib/db/ml/db.dart | 16 ++++++++++++++++ mobile/lib/services/search_service.dart | 21 ++++++++++++++++----- 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/mobile/lib/db/ml/db.dart b/mobile/lib/db/ml/db.dart index 81eb486c26..df19ef070d 100644 --- a/mobile/lib/db/ml/db.dart +++ b/mobile/lib/db/ml/db.dart @@ -502,6 +502,22 @@ class MLDataDB { return result; } + Future> getFaceIdToPersonIdForFaces( + Iterable faceIDs, + ) async { + final db = await instance.asyncDB; + final List> maps = await db.getAll( + 'SELECT $faceIDColumn, $personIdColumn FROM $clusterPersonTable ' + 'INNER JOIN $faceClustersTable ON $clusterPersonTable.$clusterIDColumn = $faceClustersTable.$clusterIDColumn ' + 'WHERE $faceIDColumn IN (${faceIDs.map((id) => "'$id'").join(",")})', + ); + final Map result = {}; + for (final map in maps) { + result[map[faceIDColumn] as String] = map[personIdColumn] as String; + } + return result; + } + Future>> getClusterIdToFaceIdsForPerson( String personID, ) async { diff --git a/mobile/lib/services/search_service.dart b/mobile/lib/services/search_service.dart index 8a4a955f01..728b7570a6 100644 --- a/mobile/lib/services/search_service.dart +++ b/mobile/lib/services/search_service.dart @@ -1295,6 +1295,10 @@ class SearchService { if (files.length <= 10) return files; final fileIDs = files.map((e) => e.uploadedFileID!).toSet(); final fileIdToFace = await MLDataDB.instance.getFacesForFileIDs(fileIDs); + final faceIDs = + fileIdToFace.values.expand((x) => x.map((face) => face.faceID)).toSet(); + final faceIDsToPersonID = + await MLDataDB.instance.getFaceIdToPersonIdForFaces(faceIDs); final fileIdToClip = await MLDataDB.instance.getClipVectorsForFileIDs(fileIDs); final allYears = files.map((e) { @@ -1322,15 +1326,22 @@ class SearchService { .compareTo(fileToScore[a.uploadedFileID!]!), ); - // then sort on faces TODO: lau: prioritize named faces + // then sort on faces, heavily prioritizing named faces final fileToFaceCount = {}; for (final file in files) { - final faces = fileIdToFace[file.uploadedFileID!]; - if (faces == null) { - fileToFaceCount[file.uploadedFileID!] = 0; + final fileID = file.uploadedFileID!; + fileToFaceCount[fileID] = 0; + final faces = fileIdToFace[fileID]; + if (faces == null || faces.isEmpty) { continue; } - fileToFaceCount[file.uploadedFileID!] = faces.length; + for (final face in faces) { + if (faceIDsToPersonID.containsKey(face.faceID)) { + fileToFaceCount[fileID] = fileToFaceCount[fileID]! + 10; + } else { + fileToFaceCount[fileID] = fileToFaceCount[fileID]! + 1; + } + } } files.sort( (a, b) => fileToFaceCount[b.uploadedFileID!]! From ba0ac8ab08ec6bfe5bf13226d0afb0b2e5d95ce8 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Wed, 29 Jan 2025 12:48:59 +0530 Subject: [PATCH 07/25] [mob][photos] clip prompt --- mobile/lib/services/search_service.dart | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mobile/lib/services/search_service.dart b/mobile/lib/services/search_service.dart index 728b7570a6..a662d66612 100644 --- a/mobile/lib/services/search_service.dart +++ b/mobile/lib/services/search_service.dart @@ -1307,7 +1307,8 @@ class SearchService { }).toSet(); if (allYears.length <= 1) { // sort first on clip embeddings TODO: lau: come up with a better prompt - const query = 'Beautiful photo of a nice memory'; + const query = + 'Photo of a treasured memory radiating happiness, connection, or quiet beauty'; // TODO: lau: optimize this later so we don't keep computing embedding final textEmbedding = await MLComputer.instance.runClipText(query); final textVector = Vector.fromList(textEmbedding); From 1ff4f7d0bb7f1d434b3965b01cad6d38f61e27b3 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Wed, 29 Jan 2025 12:57:48 +0530 Subject: [PATCH 08/25] [mob][photos] prompt --- mobile/lib/services/search_service.dart | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mobile/lib/services/search_service.dart b/mobile/lib/services/search_service.dart index a662d66612..208b72d67a 100644 --- a/mobile/lib/services/search_service.dart +++ b/mobile/lib/services/search_service.dart @@ -1306,9 +1306,9 @@ class SearchService { return creationTime.year; }).toSet(); if (allYears.length <= 1) { - // sort first on clip embeddings TODO: lau: come up with a better prompt + // sort first on clip embeddings const query = - 'Photo of a treasured memory radiating happiness, connection, or quiet beauty'; + 'Photo of a precious memory radiating warmth, vibrant energy, or quiet beauty — alive with color, light, or emotion'; // TODO: lau: optimize this later so we don't keep computing embedding final textEmbedding = await MLComputer.instance.runClipText(query); final textVector = Vector.fromList(textEmbedding); From 29e77245d7cb760b43caca2b07512aad6b58a732 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Wed, 29 Jan 2025 15:57:22 +0530 Subject: [PATCH 09/25] [mob][photos] Filter for multiple years --- mobile/lib/services/search_service.dart | 186 +++++++++++++++++++----- 1 file changed, 149 insertions(+), 37 deletions(-) diff --git a/mobile/lib/services/search_service.dart b/mobile/lib/services/search_service.dart index 208b72d67a..bc266fe3d6 100644 --- a/mobile/lib/services/search_service.dart +++ b/mobile/lib/services/search_service.dart @@ -1305,52 +1305,58 @@ class SearchService { final creationTime = DateTime.fromMicrosecondsSinceEpoch(e.creationTime!); return creationTime.year; }).toSet(); + + // Get clip scores for each file + const query = + 'Photo of a precious memory radiating warmth, vibrant energy, or quiet beauty — alive with color, light, or emotion'; + // TODO: lau: optimize this later so we don't keep computing embedding + final textEmbedding = await MLComputer.instance.runClipText(query); + final textVector = Vector.fromList(textEmbedding); + final fileToScore = {}; + for (final file in files) { + final clip = fileIdToClip[file.uploadedFileID!]; + if (clip == null) { + fileToScore[file.uploadedFileID!] = 0; + continue; + } + final score = clip.vector.dot(textVector); + fileToScore[file.uploadedFileID!] = score; + } + + // Get face scores for each file + final fileToFaceCount = {}; + for (final file in files) { + final fileID = file.uploadedFileID!; + fileToFaceCount[fileID] = 0; + final faces = fileIdToFace[fileID]; + if (faces == null || faces.isEmpty) { + continue; + } + for (final face in faces) { + if (faceIDsToPersonID.containsKey(face.faceID)) { + fileToFaceCount[fileID] = fileToFaceCount[fileID]! + 10; + } else { + fileToFaceCount[fileID] = fileToFaceCount[fileID]! + 1; + } + } + } + + final filteredFiles = []; if (allYears.length <= 1) { // sort first on clip embeddings - const query = - 'Photo of a precious memory radiating warmth, vibrant energy, or quiet beauty — alive with color, light, or emotion'; - // TODO: lau: optimize this later so we don't keep computing embedding - final textEmbedding = await MLComputer.instance.runClipText(query); - final textVector = Vector.fromList(textEmbedding); - final fileToScore = {}; - for (final file in files) { - final clip = fileIdToClip[file.uploadedFileID!]; - if (clip == null) { - fileToScore[file.uploadedFileID!] = 0; - continue; - } - final score = clip.vector.dot(textVector); - fileToScore[file.uploadedFileID!] = score; - } files.sort( (a, b) => fileToScore[b.uploadedFileID!]! .compareTo(fileToScore[a.uploadedFileID!]!), ); // then sort on faces, heavily prioritizing named faces - final fileToFaceCount = {}; - for (final file in files) { - final fileID = file.uploadedFileID!; - fileToFaceCount[fileID] = 0; - final faces = fileIdToFace[fileID]; - if (faces == null || faces.isEmpty) { - continue; - } - for (final face in faces) { - if (faceIDsToPersonID.containsKey(face.faceID)) { - fileToFaceCount[fileID] = fileToFaceCount[fileID]! + 10; - } else { - fileToFaceCount[fileID] = fileToFaceCount[fileID]! + 1; - } - } - } files.sort( (a, b) => fileToFaceCount[b.uploadedFileID!]! .compareTo(fileToFaceCount[a.uploadedFileID!]!), ); // then filter out similar images as much as possible - final filteredFiles = [files.first]; + filteredFiles.add(files.first); final fileCount = files.length; int skipped = 0; filesLoop: @@ -1370,13 +1376,119 @@ class SearchService { } filteredFiles.add(file); } - // Order the final selection chronologically - filteredFiles.sort((a, b) => b.creationTime!.compareTo(a.creationTime!)); - return filteredFiles; } else { - // TODO: lau: add logic for multiple years. Main extra thing is getting distribution over the years right - return files; + // Handle multiple years, ensuring each is represented and roughly equally distributed + + // Group files by year and sort each year's list by CLIP then face count + final yearToFiles = >{}; + for (final file in files) { + final creationTime = + DateTime.fromMicrosecondsSinceEpoch(file.creationTime!); + final year = creationTime.year; + yearToFiles.putIfAbsent(year, () => []).add(file); + } + + for (final year in yearToFiles.keys) { + final yearFiles = yearToFiles[year]!; + // Sort by CLIP score descending + yearFiles.sort( + (a, b) => fileToScore[b.uploadedFileID!]! + .compareTo(fileToScore[a.uploadedFileID!]!), + ); + // Then sort by face count descending + yearFiles.sort( + (a, b) => fileToFaceCount[b.uploadedFileID!]! + .compareTo(fileToFaceCount[a.uploadedFileID!]!), + ); + } + + // Track available files per year after each selection + final availableFilesByYear = >{}; + yearToFiles.forEach((year, files) { + availableFilesByYear[year] = List.from(files); + }); + + // Collect initial selection: one from each year (prioritizing recent years) + final initialSelection = []; + final years = yearToFiles.keys.toList() + ..sort((a, b) => b.compareTo(a)); // Recent years first + + for (final year in years) { + final available = availableFilesByYear[year]!; + if (available.isNotEmpty) { + initialSelection.add(available.removeAt(0)); + } + } + + // Fill remaining slots by cycling through years + int remainingSlots = 10 - initialSelection.length; + if (remainingSlots > 0) { + int currentYearIndex = 0; + while (remainingSlots > 0) { + final year = years[currentYearIndex % years.length]; + final available = availableFilesByYear[year]!; + if (available.isNotEmpty) { + initialSelection.add(available.removeAt(0)); + remainingSlots--; + } + currentYearIndex++; + // Break if no more files across all years + if (availableFilesByYear.values.every((list) => list.isEmpty)) { + break; + } + } + } + + // Apply similarity filtering + if (initialSelection.isNotEmpty) { + filteredFiles.add(initialSelection.first); + int skipped = 0; + + filesLoop: + for (final file in initialSelection.sublist(1)) { + if (filteredFiles.length >= 10) break; + final clip = fileIdToClip[file.uploadedFileID!]; + if (clip != null) { + for (final filteredFile in filteredFiles) { + final fClip = fileIdToClip[filteredFile.uploadedFileID!]; + if (fClip == null) continue; + final similarity = clip.vector.dot(fClip.vector); + if (similarity > 0.80) { + skipped++; + continue filesLoop; + } + } + } + filteredFiles.add(file); + } + } + + // Ensure all years are represented after filtering + final filteredYears = filteredFiles.map((f) { + final creationTime = + DateTime.fromMicrosecondsSinceEpoch(f.creationTime!); + return creationTime.year; + }).toSet(); + + for (final year in yearToFiles.keys) { + if (!filteredYears.contains(year)) { + // Add the top file from this year's original sorted list + final topFile = yearToFiles[year]!.first; + if (!filteredFiles.contains(topFile)) { + filteredFiles.add(topFile); + } + } + } + + // Trim to 10 files if needed, keeping the highest priority files + while (filteredFiles.length > 10) { + filteredFiles.removeLast(); + } } + + // Order the final selection chronologically + filteredFiles.sort((a, b) => b.creationTime!.compareTo(a.creationTime!)); + return filteredFiles; } Future getRandomDateResults( From 22bec8eee76dfe6e07329c4c8c661b5c40a689d2 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Thu, 30 Jan 2025 14:06:55 +0530 Subject: [PATCH 10/25] [mob][photos] Simpler multi-year filtering --- mobile/lib/services/search_service.dart | 116 +++++++----------------- 1 file changed, 34 insertions(+), 82 deletions(-) diff --git a/mobile/lib/services/search_service.dart b/mobile/lib/services/search_service.dart index bc266fe3d6..d3f98efac3 100644 --- a/mobile/lib/services/search_service.dart +++ b/mobile/lib/services/search_service.dart @@ -1292,7 +1292,8 @@ class SearchService { /// Makes sure that the selection is not more than 10 files, /// and that each year of the original list is represented. Future> _bestSelection(List files) async { - if (files.length <= 10) return files; + final fileCount = files.length; + if (fileCount <= 10) return files; final fileIDs = files.map((e) => e.uploadedFileID!).toSet(); final fileIdToFace = await MLDataDB.instance.getFacesForFileIDs(fileIDs); final faceIDs = @@ -1343,13 +1344,12 @@ class SearchService { final filteredFiles = []; if (allYears.length <= 1) { - // sort first on clip embeddings + // sort first on clip embeddings score (descending) files.sort( (a, b) => fileToScore[b.uploadedFileID!]! .compareTo(fileToScore[a.uploadedFileID!]!), ); - - // then sort on faces, heavily prioritizing named faces + // then sort on faces (descending), heavily prioritizing named faces files.sort( (a, b) => fileToFaceCount[b.uploadedFileID!]! .compareTo(fileToFaceCount[a.uploadedFileID!]!), @@ -1357,7 +1357,6 @@ class SearchService { // then filter out similar images as much as possible filteredFiles.add(files.first); - final fileCount = files.length; int skipped = 0; filesLoop: for (final file in files.sublist(1)) { @@ -1377,7 +1376,7 @@ class SearchService { filteredFiles.add(file); } } else { - // Handle multiple years, ensuring each is represented and roughly equally distributed + // Multiple years, each represented and roughly equally distributed // Group files by year and sort each year's list by CLIP then face count final yearToFiles = >{}; @@ -1390,99 +1389,52 @@ class SearchService { for (final year in yearToFiles.keys) { final yearFiles = yearToFiles[year]!; - // Sort by CLIP score descending + // sort first on clip embeddings score (descending) yearFiles.sort( (a, b) => fileToScore[b.uploadedFileID!]! .compareTo(fileToScore[a.uploadedFileID!]!), ); - // Then sort by face count descending + // then sort on faces (descending), heavily prioritizing named faces yearFiles.sort( (a, b) => fileToFaceCount[b.uploadedFileID!]! .compareTo(fileToFaceCount[a.uploadedFileID!]!), ); } - // Track available files per year after each selection - final availableFilesByYear = >{}; - yearToFiles.forEach((year, files) { - availableFilesByYear[year] = List.from(files); - }); - - // Collect initial selection: one from each year (prioritizing recent years) - final initialSelection = []; + // Then join the years together one by one and filter similar images final years = yearToFiles.keys.toList() ..sort((a, b) => b.compareTo(a)); // Recent years first - - for (final year in years) { - final available = availableFilesByYear[year]!; - if (available.isNotEmpty) { - initialSelection.add(available.removeAt(0)); - } - } - - // Fill remaining slots by cycling through years - int remainingSlots = 10 - initialSelection.length; - if (remainingSlots > 0) { - int currentYearIndex = 0; - while (remainingSlots > 0) { - final year = years[currentYearIndex % years.length]; - final available = availableFilesByYear[year]!; - if (available.isNotEmpty) { - initialSelection.add(available.removeAt(0)); - remainingSlots--; - } - currentYearIndex++; - // Break if no more files across all years - if (availableFilesByYear.values.every((list) => list.isEmpty)) { - break; - } - } - } - - // Apply similarity filtering - if (initialSelection.isNotEmpty) { - filteredFiles.add(initialSelection.first); - int skipped = 0; - - filesLoop: - for (final file in initialSelection.sublist(1)) { - if (filteredFiles.length >= 10) break; - final clip = fileIdToClip[file.uploadedFileID!]; - if (clip != null) { - for (final filteredFile in filteredFiles) { - final fClip = fileIdToClip[filteredFile.uploadedFileID!]; - if (fClip == null) continue; - final similarity = clip.vector.dot(fClip.vector); - if (similarity > 0.80) { - skipped++; - continue filesLoop; + int round = 0; + int skipped = 0; + filteredFiles.add(yearToFiles[years[0]]!.removeAt(0)); + while (true) { + yearLoop: + for (final year in years) { + final yearFiles = yearToFiles[year]!; + if (yearFiles.isEmpty) continue; + final newFile = yearFiles.removeAt(0); + if (round != 0) { + // check for filtering + final clip = fileIdToClip[newFile.uploadedFileID!]; + if (clip != null && (fileCount - skipped) > 10) { + for (final filteredFile in filteredFiles) { + final fClip = fileIdToClip[filteredFile.uploadedFileID!]; + if (fClip == null) continue; + final similarity = clip.vector.dot(fClip.vector); + if (similarity > 0.80) { + skipped++; + continue yearLoop; + } } } } - filteredFiles.add(file); - } - } - - // Ensure all years are represented after filtering - final filteredYears = filteredFiles.map((f) { - final creationTime = - DateTime.fromMicrosecondsSinceEpoch(f.creationTime!); - return creationTime.year; - }).toSet(); - - for (final year in yearToFiles.keys) { - if (!filteredYears.contains(year)) { - // Add the top file from this year's original sorted list - final topFile = yearToFiles[year]!.first; - if (!filteredFiles.contains(topFile)) { - filteredFiles.add(topFile); + filteredFiles.add(newFile); + if (filteredFiles.length >= 10 || + filteredFiles.length + skipped >= fileCount) { + break; } } - } - - // Trim to 10 files if needed, keeping the highest priority files - while (filteredFiles.length > 10) { - filteredFiles.removeLast(); + round++; } } From 821965308e9c3f8035791af86e636a77f8717d3f Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Thu, 30 Jan 2025 14:16:58 +0530 Subject: [PATCH 11/25] [mob][photos] Increase photo multiyear photo selection if needed --- mobile/lib/services/search_service.dart | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/mobile/lib/services/search_service.dart b/mobile/lib/services/search_service.dart index d3f98efac3..21a1447696 100644 --- a/mobile/lib/services/search_service.dart +++ b/mobile/lib/services/search_service.dart @@ -1291,9 +1291,13 @@ class SearchService { /// Returns the best selection of files from the given list. /// Makes sure that the selection is not more than 10 files, /// and that each year of the original list is represented. - Future> _bestSelection(List files) async { + Future> _bestSelection( + List files, { + int? prefferedSize, + }) async { final fileCount = files.length; - if (fileCount <= 10) return files; + int targetSize = prefferedSize ?? 10; + if (fileCount <= targetSize) return files; final fileIDs = files.map((e) => e.uploadedFileID!).toSet(); final fileIdToFace = await MLDataDB.instance.getFacesForFileIDs(fileIDs); final faceIDs = @@ -1360,9 +1364,9 @@ class SearchService { int skipped = 0; filesLoop: for (final file in files.sublist(1)) { - if (filteredFiles.length >= 10) break; + if (filteredFiles.length >= targetSize) break; final clip = fileIdToClip[file.uploadedFileID!]; - if (clip != null && (fileCount - skipped) > 10) { + if (clip != null && (fileCount - skipped) > targetSize) { for (final filteredFile in filteredFiles) { final fClip = fileIdToClip[filteredFile.uploadedFileID!]; if (fClip == null) continue; @@ -1377,6 +1381,10 @@ class SearchService { } } else { // Multiple years, each represented and roughly equally distributed + if (prefferedSize == null && (allYears.length * 2) > 10) { + targetSize = allYears.length * 3; + if (fileCount < targetSize) return files; + } // Group files by year and sort each year's list by CLIP then face count final yearToFiles = >{}; @@ -1416,7 +1424,7 @@ class SearchService { if (round != 0) { // check for filtering final clip = fileIdToClip[newFile.uploadedFileID!]; - if (clip != null && (fileCount - skipped) > 10) { + if (clip != null && (fileCount - skipped) > targetSize) { for (final filteredFile in filteredFiles) { final fClip = fileIdToClip[filteredFile.uploadedFileID!]; if (fClip == null) continue; @@ -1429,7 +1437,7 @@ class SearchService { } } filteredFiles.add(newFile); - if (filteredFiles.length >= 10 || + if (filteredFiles.length >= targetSize || filteredFiles.length + skipped >= fileCount) { break; } From d9c1a21b0ce031dc5da8bb3bb5e17006cceb4e5d Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Thu, 30 Jan 2025 14:25:00 +0530 Subject: [PATCH 12/25] [mob][photos] simplify --- mobile/lib/services/search_service.dart | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mobile/lib/services/search_service.dart b/mobile/lib/services/search_service.dart index 21a1447696..bdd87f5fe9 100644 --- a/mobile/lib/services/search_service.dart +++ b/mobile/lib/services/search_service.dart @@ -1421,10 +1421,10 @@ class SearchService { final yearFiles = yearToFiles[year]!; if (yearFiles.isEmpty) continue; final newFile = yearFiles.removeAt(0); - if (round != 0) { + if (round != 0 && (fileCount - skipped) > targetSize) { // check for filtering final clip = fileIdToClip[newFile.uploadedFileID!]; - if (clip != null && (fileCount - skipped) > targetSize) { + if (clip != null) { for (final filteredFile in filteredFiles) { final fClip = fileIdToClip[filteredFile.uploadedFileID!]; if (fClip == null) continue; From 95a27397d3abcb97f6c8a78d1c45ac3788fcdd2f Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Thu, 30 Jan 2025 14:32:31 +0530 Subject: [PATCH 13/25] [mob][photos] small correction --- mobile/lib/services/search_service.dart | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mobile/lib/services/search_service.dart b/mobile/lib/services/search_service.dart index bdd87f5fe9..812ff8e8dd 100644 --- a/mobile/lib/services/search_service.dart +++ b/mobile/lib/services/search_service.dart @@ -1414,8 +1414,8 @@ class SearchService { ..sort((a, b) => b.compareTo(a)); // Recent years first int round = 0; int skipped = 0; - filteredFiles.add(yearToFiles[years[0]]!.removeAt(0)); - while (true) { + whileLoop: + while (filteredFiles.length + skipped < fileCount) { yearLoop: for (final year in years) { final yearFiles = yearToFiles[year]!; @@ -1439,7 +1439,7 @@ class SearchService { filteredFiles.add(newFile); if (filteredFiles.length >= targetSize || filteredFiles.length + skipped >= fileCount) { - break; + break whileLoop; } } round++; From 9295908dc590e1a41783673117306d33862fa4ed Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Thu, 30 Jan 2025 14:47:44 +0530 Subject: [PATCH 14/25] [mob][photos] comment --- mobile/lib/services/search_service.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobile/lib/services/search_service.dart b/mobile/lib/services/search_service.dart index 812ff8e8dd..ee77bdf3e2 100644 --- a/mobile/lib/services/search_service.dart +++ b/mobile/lib/services/search_service.dart @@ -1289,7 +1289,7 @@ class SearchService { } /// Returns the best selection of files from the given list. - /// Makes sure that the selection is not more than 10 files, + /// Makes sure that the selection is not more than [prefferedSize] or 10 files, /// and that each year of the original list is represented. Future> _bestSelection( List files, { From 0c92ed96a6184f39586a06b16ec1b6137cfd58b8 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Thu, 30 Jan 2025 15:58:51 +0530 Subject: [PATCH 15/25] [mob][photos] Lower clip threshold --- mobile/lib/services/search_service.dart | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mobile/lib/services/search_service.dart b/mobile/lib/services/search_service.dart index ee77bdf3e2..3fe4a7be3d 100644 --- a/mobile/lib/services/search_service.dart +++ b/mobile/lib/services/search_service.dart @@ -1317,6 +1317,7 @@ class SearchService { // TODO: lau: optimize this later so we don't keep computing embedding final textEmbedding = await MLComputer.instance.runClipText(query); final textVector = Vector.fromList(textEmbedding); + const clipThreshold = 0.75; final fileToScore = {}; for (final file in files) { final clip = fileIdToClip[file.uploadedFileID!]; @@ -1371,7 +1372,7 @@ class SearchService { final fClip = fileIdToClip[filteredFile.uploadedFileID!]; if (fClip == null) continue; final similarity = clip.vector.dot(fClip.vector); - if (similarity > 0.80) { + if (similarity > clipThreshold) { skipped++; continue filesLoop; } @@ -1429,7 +1430,7 @@ class SearchService { final fClip = fileIdToClip[filteredFile.uploadedFileID!]; if (fClip == null) continue; final similarity = clip.vector.dot(fClip.vector); - if (similarity > 0.80) { + if (similarity > clipThreshold) { skipped++; continue yearLoop; } From fbcba9903ea1c2d86bf03e705abbfc8e2cd93938 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Thu, 30 Jan 2025 16:15:57 +0530 Subject: [PATCH 16/25] [mob][photos] todo --- mobile/lib/services/search_service.dart | 1 + 1 file changed, 1 insertion(+) diff --git a/mobile/lib/services/search_service.dart b/mobile/lib/services/search_service.dart index 3fe4a7be3d..e00610bb50 100644 --- a/mobile/lib/services/search_service.dart +++ b/mobile/lib/services/search_service.dart @@ -1349,6 +1349,7 @@ class SearchService { final filteredFiles = []; if (allYears.length <= 1) { + // TODO: lau: eventually this sorting might have to be replaced with some scoring system // sort first on clip embeddings score (descending) files.sort( (a, b) => fileToScore[b.uploadedFileID!]! From 26020c2e990313f4c428e4400e09e357b7411c2a Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Thu, 30 Jan 2025 17:52:01 +0530 Subject: [PATCH 17/25] [mob][photos] todo --- mobile/lib/services/search_service.dart | 1 + 1 file changed, 1 insertion(+) diff --git a/mobile/lib/services/search_service.dart b/mobile/lib/services/search_service.dart index e00610bb50..79c789486e 100644 --- a/mobile/lib/services/search_service.dart +++ b/mobile/lib/services/search_service.dart @@ -1224,6 +1224,7 @@ class SearchService { for (final dayMonth in dayMonthYearGroups.keys) { final dayDiff = dayMonth - currentDayMonth; if (dayDiff < 0 || dayDiff > 2) continue; + // TODO: lau: this doesn't cover month changes properly final yearGroups = dayMonthYearGroups[dayMonth]!; final significantYears = yearGroups.entries From e4e01cce9e066c1e59dbfc956732c03998c44166 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Fri, 31 Jan 2025 15:14:25 +0530 Subject: [PATCH 18/25] [mob][photos] Add significant weeks --- mobile/lib/services/search_service.dart | 97 +++++++++++++++++++++++-- 1 file changed, 91 insertions(+), 6 deletions(-) diff --git a/mobile/lib/services/search_service.dart b/mobile/lib/services/search_service.dart index 79c789486e..af0ff43913 100644 --- a/mobile/lib/services/search_service.dart +++ b/mobile/lib/services/search_service.dart @@ -1199,9 +1199,11 @@ class SearchService { final currentTime = DateTime.now().toLocal(); final currentDayMonth = currentTime.month * 100 + currentTime.day; + final currentWeek = _getWeekNumber(currentTime); final cutOffTime = currentTime.subtract(const Duration(days: 365)); final averageDailyPhotos = allFiles.length / 365; - final significanceThreshold = averageDailyPhotos * 0.25; + final significantDayThreshold = averageDailyPhotos * 0.25; + final significantWeekThreshold = averageDailyPhotos * 0.40; // Group files by day-month and year final dayMonthYearGroups = >>{}; @@ -1220,19 +1222,19 @@ class SearchService { .add(file); } - // Process each day-month + // Process each nearby day-month to find significant days for (final dayMonth in dayMonthYearGroups.keys) { final dayDiff = dayMonth - currentDayMonth; if (dayDiff < 0 || dayDiff > 2) continue; // TODO: lau: this doesn't cover month changes properly final yearGroups = dayMonthYearGroups[dayMonth]!; - final significantYears = yearGroups.entries - .where((e) => e.value.length > significanceThreshold) + final significantDays = yearGroups.entries + .where((e) => e.value.length > significantDayThreshold) .map((e) => e.key) .toList(); - if (significantYears.length >= 3) { + if (significantDays.length >= 3) { // Combine all years for this day-month final date = DateTime(currentTime.year, dayMonth ~/ 100, dayMonth % 100); @@ -1255,7 +1257,7 @@ class SearchService { ); } else { // Individual entries for significant years - for (final year in significantYears) { + for (final year in significantDays) { final date = DateTime(year, dayMonth ~/ 100, dayMonth % 100); final files = yearGroups[year]!; final photoSelection = await _bestSelection(files); @@ -1286,9 +1288,92 @@ class SearchService { if (limit != null && searchResults.length >= limit) break; } + // process to find significant weeks (only if there are no significant days) + if (searchResults.isEmpty) { + // Group files by week and year + final weekYearGroups = >>{}; + for (final file in allFiles) { + if (file.creationTime! > cutOffTime.microsecondsSinceEpoch) continue; + + final creationTime = + DateTime.fromMicrosecondsSinceEpoch(file.creationTime!); + final week = _getWeekNumber(creationTime); + final year = creationTime.year; + + weekYearGroups + .putIfAbsent(week, () => {}) + .putIfAbsent(year, () => []) + .add(file); + } + + // Process each nearby day-month to find significant days + final currentWeekYears = weekYearGroups[currentWeek]; + if (currentWeekYears != null) { + final significantWeeks = currentWeekYears.entries + .where((e) => e.value.length > significantWeekThreshold) + .map((e) => e.key) + .toList(); + if (significantWeeks.length >= 3) { + // Combine all years for this week + final allPhotos = currentWeekYears.values.expand((x) => x).toList(); + final photoSelection = await _bestSelection(allPhotos); + + searchResults.add( + GenericSearchResult( + ResultType.event, + "This week through the years", + photoSelection, + hierarchicalSearchFilter: TopLevelGenericFilter( + filterName: "Week $currentWeek", + occurrence: kMostRelevantFilter, + filterResultType: ResultType.event, + matchedUploadedIDs: filesToUploadedFileIDs(photoSelection), + filterIcon: Icons.event_outlined, + ), + ), + ); + } else { + // Individual entries for significant years + for (final year in significantWeeks) { + final date = DateTime(year, 1, 1).add( + Duration(days: (currentWeek - 1) * 7), + ); + final files = currentWeekYears[year]!; + final photoSelection = await _bestSelection(files); + final name = + "This week, ${currentTime.year - date.year} years back"; + + searchResults.add( + GenericSearchResult( + ResultType.event, + name, + photoSelection, + hierarchicalSearchFilter: TopLevelGenericFilter( + filterName: name, + occurrence: kMostRelevantFilter, + filterResultType: ResultType.event, + matchedUploadedIDs: filesToUploadedFileIDs(photoSelection), + filterIcon: Icons.event_outlined, + ), + ), + ); + } + } + } + } + + // process to find fillers + return searchResults; } + int _getWeekNumber(DateTime date) { + // Get day of year (1-366) + final int dayOfYear = int.parse(DateFormat('D').format(date)); + // Integer division by 7 and add 1 to start from week 1 + return ((dayOfYear - 1) ~/ 7) + 1; + } + /// Returns the best selection of files from the given list. /// Makes sure that the selection is not more than [prefferedSize] or 10 files, /// and that each year of the original list is represented. From 92a718ca0f00eb19ca0118b002c6583d9ad4c9fe Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Fri, 31 Jan 2025 15:17:24 +0530 Subject: [PATCH 19/25] [mob][photos] Return when limit --- mobile/lib/services/search_service.dart | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mobile/lib/services/search_service.dart b/mobile/lib/services/search_service.dart index af0ff43913..0d3bf3f343 100644 --- a/mobile/lib/services/search_service.dart +++ b/mobile/lib/services/search_service.dart @@ -1285,7 +1285,7 @@ class SearchService { } } - if (limit != null && searchResults.length >= limit) break; + if (limit != null && searchResults.length >= limit) return searchResults; } // process to find significant weeks (only if there are no significant days) @@ -1362,6 +1362,8 @@ class SearchService { } } + if (limit != null && searchResults.length >= limit) return searchResults; + // process to find fillers return searchResults; From 68319ca725cc05500fd677b316dd3416e2ee8bb8 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Fri, 31 Jan 2025 15:20:55 +0530 Subject: [PATCH 20/25] [mob][photos] Update day string --- mobile/lib/services/search_service.dart | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mobile/lib/services/search_service.dart b/mobile/lib/services/search_service.dart index 0d3bf3f343..6904821485 100644 --- a/mobile/lib/services/search_service.dart +++ b/mobile/lib/services/search_service.dart @@ -1244,7 +1244,7 @@ class SearchService { searchResults.add( GenericSearchResult( ResultType.event, - 'Memories of ${DateFormat('MMMM d').format(date)}', + "${DateFormat('MMMM d').format(date)} through the years", photoSelection, hierarchicalSearchFilter: TopLevelGenericFilter( filterName: DateFormat('MMMM d').format(date), @@ -1265,7 +1265,7 @@ class SearchService { DateFormat.yMMMd(Localizations.localeOf(context).languageCode) .format(date); if (date.day == currentTime.day && date.month == currentTime.month) { - name = 'This day, ${currentTime.year - date.year} years back'; + name = "This day, ${currentTime.year - date.year} years back"; } searchResults.add( From 0498fa644bc7b67cc1b0d21dbcb3f98bfa859d9c Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Fri, 31 Jan 2025 17:10:14 +0530 Subject: [PATCH 21/25] [mob][photos] Null safety fix uploadedFileID --- mobile/lib/services/search_service.dart | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/mobile/lib/services/search_service.dart b/mobile/lib/services/search_service.dart index 6904821485..29236902a6 100644 --- a/mobile/lib/services/search_service.dart +++ b/mobile/lib/services/search_service.dart @@ -1386,7 +1386,9 @@ class SearchService { final fileCount = files.length; int targetSize = prefferedSize ?? 10; if (fileCount <= targetSize) return files; - final fileIDs = files.map((e) => e.uploadedFileID!).toSet(); + final safeFiles = + files.where((file) => file.uploadedFileID != null).toList(); + final fileIDs = safeFiles.map((e) => e.uploadedFileID!).toSet(); final fileIdToFace = await MLDataDB.instance.getFacesForFileIDs(fileIDs); final faceIDs = fileIdToFace.values.expand((x) => x.map((face) => face.faceID)).toSet(); @@ -1394,7 +1396,7 @@ class SearchService { await MLDataDB.instance.getFaceIdToPersonIdForFaces(faceIDs); final fileIdToClip = await MLDataDB.instance.getClipVectorsForFileIDs(fileIDs); - final allYears = files.map((e) { + final allYears = safeFiles.map((e) { final creationTime = DateTime.fromMicrosecondsSinceEpoch(e.creationTime!); return creationTime.year; }).toSet(); @@ -1407,7 +1409,7 @@ class SearchService { final textVector = Vector.fromList(textEmbedding); const clipThreshold = 0.75; final fileToScore = {}; - for (final file in files) { + for (final file in safeFiles) { final clip = fileIdToClip[file.uploadedFileID!]; if (clip == null) { fileToScore[file.uploadedFileID!] = 0; @@ -1419,7 +1421,7 @@ class SearchService { // Get face scores for each file final fileToFaceCount = {}; - for (final file in files) { + for (final file in safeFiles) { final fileID = file.uploadedFileID!; fileToFaceCount[fileID] = 0; final faces = fileIdToFace[fileID]; @@ -1439,21 +1441,21 @@ class SearchService { if (allYears.length <= 1) { // TODO: lau: eventually this sorting might have to be replaced with some scoring system // sort first on clip embeddings score (descending) - files.sort( + safeFiles.sort( (a, b) => fileToScore[b.uploadedFileID!]! .compareTo(fileToScore[a.uploadedFileID!]!), ); // then sort on faces (descending), heavily prioritizing named faces - files.sort( + safeFiles.sort( (a, b) => fileToFaceCount[b.uploadedFileID!]! .compareTo(fileToFaceCount[a.uploadedFileID!]!), ); // then filter out similar images as much as possible - filteredFiles.add(files.first); + filteredFiles.add(safeFiles.first); int skipped = 0; filesLoop: - for (final file in files.sublist(1)) { + for (final file in safeFiles.sublist(1)) { if (filteredFiles.length >= targetSize) break; final clip = fileIdToClip[file.uploadedFileID!]; if (clip != null && (fileCount - skipped) > targetSize) { @@ -1473,12 +1475,12 @@ class SearchService { // Multiple years, each represented and roughly equally distributed if (prefferedSize == null && (allYears.length * 2) > 10) { targetSize = allYears.length * 3; - if (fileCount < targetSize) return files; + if (fileCount < targetSize) return safeFiles; } // Group files by year and sort each year's list by CLIP then face count final yearToFiles = >{}; - for (final file in files) { + for (final file in safeFiles) { final creationTime = DateTime.fromMicrosecondsSinceEpoch(file.creationTime!); final year = creationTime.year; From 45eff8edb3daa140971db3a61f5636a5b8947072 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Fri, 31 Jan 2025 17:13:38 +0530 Subject: [PATCH 22/25] [mob][photos] Add months --- mobile/lib/services/search_service.dart | 79 ++++++++++++++++++++++++- 1 file changed, 78 insertions(+), 1 deletion(-) diff --git a/mobile/lib/services/search_service.dart b/mobile/lib/services/search_service.dart index 29236902a6..6b1532e63f 100644 --- a/mobile/lib/services/search_service.dart +++ b/mobile/lib/services/search_service.dart @@ -1200,6 +1200,7 @@ class SearchService { final currentTime = DateTime.now().toLocal(); final currentDayMonth = currentTime.month * 100 + currentTime.day; final currentWeek = _getWeekNumber(currentTime); + final currentMonth = currentTime.month; final cutOffTime = currentTime.subtract(const Duration(days: 365)); final averageDailyPhotos = allFiles.length / 365; final significantDayThreshold = averageDailyPhotos * 0.25; @@ -1292,6 +1293,7 @@ class SearchService { if (searchResults.isEmpty) { // Group files by week and year final weekYearGroups = >>{}; + // TODO: lau: just only do this week as with month for (final file in allFiles) { if (file.creationTime! > cutOffTime.microsecondsSinceEpoch) continue; @@ -1364,7 +1366,82 @@ class SearchService { if (limit != null && searchResults.length >= limit) return searchResults; - // process to find fillers + // process to find fillers (months) + const wantedMemories = 3; + final neededMemories = wantedMemories - searchResults.length; + if (neededMemories <= 0) return searchResults; + + // Group files by month and year + final currentMonthYearGroups = >{}; + for (final file in allFiles) { + if (file.creationTime! > cutOffTime.microsecondsSinceEpoch) continue; + + final creationTime = + DateTime.fromMicrosecondsSinceEpoch(file.creationTime!); + final month = creationTime.month; + if (month != currentMonth) continue; + final year = creationTime.year; + + currentMonthYearGroups.putIfAbsent(year, () => []).add(file); + } + + // Add the largest two months plus the month through the years + final sortedYearsForCurrentMonth = currentMonthYearGroups.keys.toList() + ..sort( + (a, b) => currentMonthYearGroups[b]!.length.compareTo( + currentMonthYearGroups[a]!.length, + ), + ); + if (neededMemories > 1) { + for (int i = neededMemories; i > 1; i--) { + if (sortedYearsForCurrentMonth.isEmpty) break; + final year = sortedYearsForCurrentMonth.removeAt(0); + final monthYearFiles = currentMonthYearGroups[year]!; + final photoSelection = await _bestSelection(monthYearFiles); + final monthName = + DateFormat.MMMM(Localizations.localeOf(context).languageCode) + .format(DateTime(year, currentMonth)); + final name = monthName + ", ${currentTime.year - year} years back"; + searchResults.add( + GenericSearchResult( + ResultType.event, + name, + photoSelection, + hierarchicalSearchFilter: TopLevelGenericFilter( + filterName: name, + occurrence: kMostRelevantFilter, + filterResultType: ResultType.event, + matchedUploadedIDs: filesToUploadedFileIDs(photoSelection), + filterIcon: Icons.event_outlined, + ), + ), + ); + } + } + // Show the month through the remaining years + if (sortedYearsForCurrentMonth.isEmpty) return searchResults; + final allPhotos = sortedYearsForCurrentMonth + .expand((year) => currentMonthYearGroups[year]!) + .toList(); + final photoSelection = await _bestSelection(allPhotos); + final monthName = + DateFormat.MMMM(Localizations.localeOf(context).languageCode) + .format(DateTime(currentTime.year, currentMonth)); + final name = monthName + " through the years"; + searchResults.add( + GenericSearchResult( + ResultType.event, + name, + photoSelection, + hierarchicalSearchFilter: TopLevelGenericFilter( + filterName: name, + occurrence: kMostRelevantFilter, + filterResultType: ResultType.event, + matchedUploadedIDs: filesToUploadedFileIDs(photoSelection), + filterIcon: Icons.event_outlined, + ), + ), + ); return searchResults; } From 78ae7c85957406accc51fe7b3d8624669e5485d7 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Fri, 31 Jan 2025 17:20:21 +0530 Subject: [PATCH 23/25] [mob][photos] Increase month selection to 20 --- mobile/lib/services/search_service.dart | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/mobile/lib/services/search_service.dart b/mobile/lib/services/search_service.dart index 6b1532e63f..50f702d516 100644 --- a/mobile/lib/services/search_service.dart +++ b/mobile/lib/services/search_service.dart @@ -1370,6 +1370,7 @@ class SearchService { const wantedMemories = 3; final neededMemories = wantedMemories - searchResults.length; if (neededMemories <= 0) return searchResults; + const monthSelectionSize = 20; // Group files by month and year final currentMonthYearGroups = >{}; @@ -1397,7 +1398,10 @@ class SearchService { if (sortedYearsForCurrentMonth.isEmpty) break; final year = sortedYearsForCurrentMonth.removeAt(0); final monthYearFiles = currentMonthYearGroups[year]!; - final photoSelection = await _bestSelection(monthYearFiles); + final photoSelection = await _bestSelection( + monthYearFiles, + prefferedSize: monthSelectionSize, + ); final monthName = DateFormat.MMMM(Localizations.localeOf(context).languageCode) .format(DateTime(year, currentMonth)); @@ -1423,7 +1427,8 @@ class SearchService { final allPhotos = sortedYearsForCurrentMonth .expand((year) => currentMonthYearGroups[year]!) .toList(); - final photoSelection = await _bestSelection(allPhotos); + final photoSelection = + await _bestSelection(allPhotos, prefferedSize: monthSelectionSize); final monthName = DateFormat.MMMM(Localizations.localeOf(context).languageCode) .format(DateTime(currentTime.year, currentMonth)); From 0ba76ebbf06078a9e9af010eafa9dd91b25f458f Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Fri, 31 Jan 2025 17:28:55 +0530 Subject: [PATCH 24/25] [mob][photos] Simplify --- mobile/lib/services/search_service.dart | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/mobile/lib/services/search_service.dart b/mobile/lib/services/search_service.dart index 50f702d516..bc677e2403 100644 --- a/mobile/lib/services/search_service.dart +++ b/mobile/lib/services/search_service.dart @@ -1292,32 +1292,29 @@ class SearchService { // process to find significant weeks (only if there are no significant days) if (searchResults.isEmpty) { // Group files by week and year - final weekYearGroups = >>{}; - // TODO: lau: just only do this week as with month + final currentWeekYearGroups = >{}; for (final file in allFiles) { if (file.creationTime! > cutOffTime.microsecondsSinceEpoch) continue; final creationTime = DateTime.fromMicrosecondsSinceEpoch(file.creationTime!); final week = _getWeekNumber(creationTime); + if (week != currentWeek) continue; final year = creationTime.year; - weekYearGroups - .putIfAbsent(week, () => {}) - .putIfAbsent(year, () => []) - .add(file); + currentWeekYearGroups.putIfAbsent(year, () => []).add(file); } - // Process each nearby day-month to find significant days - final currentWeekYears = weekYearGroups[currentWeek]; - if (currentWeekYears != null) { - final significantWeeks = currentWeekYears.entries + // Process the week and see if it's significant + if (currentWeekYearGroups.isNotEmpty) { + final significantWeeks = currentWeekYearGroups.entries .where((e) => e.value.length > significantWeekThreshold) .map((e) => e.key) .toList(); if (significantWeeks.length >= 3) { // Combine all years for this week - final allPhotos = currentWeekYears.values.expand((x) => x).toList(); + final allPhotos = + currentWeekYearGroups.values.expand((x) => x).toList(); final photoSelection = await _bestSelection(allPhotos); searchResults.add( @@ -1340,7 +1337,7 @@ class SearchService { final date = DateTime(year, 1, 1).add( Duration(days: (currentWeek - 1) * 7), ); - final files = currentWeekYears[year]!; + final files = currentWeekYearGroups[year]!; final photoSelection = await _bestSelection(files); final name = "This week, ${currentTime.year - date.year} years back"; From 2e0dcc17ef9c428955557c8ed844af3ee45899d4 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Mon, 3 Feb 2025 11:24:24 +0530 Subject: [PATCH 25/25] [mob][photos] Bump for internal release --- mobile/pubspec.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobile/pubspec.yaml b/mobile/pubspec.yaml index 98a635b3ab..07b2938512 100644 --- a/mobile/pubspec.yaml +++ b/mobile/pubspec.yaml @@ -12,7 +12,7 @@ description: ente photos application # Read more about iOS versioning at # https://developer.apple.com/library/archive/documentation/General/Reference/InfoPlistKeyReference/Articles/CoreFoundationKeys.html -version: 0.9.90+990 +version: 0.9.91+991 publish_to: none environment: