diff --git a/mobile/apps/photos/lib/models/similar_files.dart b/mobile/apps/photos/lib/models/similar_files.dart index ee946c499a..e0c2a14a9f 100644 --- a/mobile/apps/photos/lib/models/similar_files.dart +++ b/mobile/apps/photos/lib/models/similar_files.dart @@ -1,18 +1,21 @@ +import "dart:convert"; + import "package:photos/models/file/file.dart"; +import "package:photos/services/search_service.dart"; class SimilarFiles { final List files; final Set fileIds; - final double furthestDistance; + double furthestDistance; SimilarFiles( this.files, this.furthestDistance, - ) : fileIds = files.map((file) => file.uploadedFileID!).toSet(); + ) : fileIds = files.map((file) => file.uploadedFileID!).toSet(); - int get totalSize => - files.fold(0, (sum, file) => sum + (file.fileSize ?? 0)); + int get totalSize => files.fold(0, (sum, file) => sum + (file.fileSize ?? 0)); + // TODO: lau: check if we're not using this wrong bool get isEmpty => files.isEmpty; int get length => files.length; @@ -26,7 +29,125 @@ class SimilarFiles { fileIds.remove(file.uploadedFileID); } + void addFile(EnteFile file) { + files.add(file); + fileIds.add(file.uploadedFileID!); + } + bool containsFile(EnteFile file) { return fileIds.contains(file.uploadedFileID); } + + Map toJson() { + return { + 'fileIDs': fileIds.toList(), + 'distance': furthestDistance, + }; + } + + String toJsonString() { + return jsonEncode(toJson()); + } + + factory SimilarFiles.fromJson( + Map json, + Map fileMap, + ) { + final fileIds = List.from(json['fileIDs']); + final furthestDistance = (json['distance'] as num).toDouble(); + + final files = []; + for (final fileId in fileIds) { + final file = fileMap[fileId]; + if (file == null) continue; + files.add(file); + } + + return SimilarFiles( + files, + furthestDistance, + ); + } + + static SimilarFiles fromJsonString( + String jsonString, + Map fileMap, + ) { + return SimilarFiles.fromJson(jsonDecode(jsonString), fileMap); + } +} + +class SimilarFilesCache { + final List similarFilesJsonStringList; + final Set allCheckedFileIDs; + final double distanceThreshold; + final bool exact; + + List? _similarFilesList; + + /// Milliseconds since epoch + final int cachedTime; + + SimilarFilesCache({ + required this.similarFilesJsonStringList, + required this.allCheckedFileIDs, + required this.distanceThreshold, + required this.exact, + required this.cachedTime, + }); + + Future> similarFilesList() async { + final allFiles = await SearchService.instance.getAllFilesForSearch(); + final fileMap = {}; + for (final file in allFiles) { + if (file.uploadedFileID == null) continue; + fileMap[file.uploadedFileID!] = file; + } + _similarFilesList ??= similarFilesJsonStringList.map((jsonString) { + return SimilarFiles.fromJson(jsonDecode(jsonString), fileMap); + }).toList(); + return _similarFilesList!; + } + + Future> getGroupedFileIDs() async { + final similarFiles = await similarFilesList(); + final groupedFileIDs = {}; + for (final files in similarFiles) { + groupedFileIDs.addAll(files.fileIds); + } + return groupedFileIDs; + } + + factory SimilarFilesCache.fromJson( + Map json, + ) { + return SimilarFilesCache( + similarFilesJsonStringList: + List.from(json['similarFilesJsonStringList']), + allCheckedFileIDs: Set.from(json['allCheckedFileIDs']), + distanceThreshold: (json['distanceThreshold'] as num).toDouble(), + exact: json['exact'] as bool, + cachedTime: json['cachedTime'] as int, + ); + } + + Map toJson() { + return { + 'similarFilesJsonStringList': similarFilesJsonStringList, + 'allCheckedFileIDs': allCheckedFileIDs.toList(), + 'distanceThreshold': distanceThreshold, + 'exact': exact, + 'cachedTime': cachedTime, + }; + } + + static String encodeToJsonString(SimilarFilesCache cache) { + return jsonEncode(cache.toJson()); + } + + static SimilarFilesCache decodeFromJsonString( + String jsonString, + ) { + return SimilarFilesCache.fromJson(jsonDecode(jsonString)); + } } diff --git a/mobile/apps/photos/lib/services/deduplication_service.dart b/mobile/apps/photos/lib/services/deduplication_service.dart index 8ec6d59203..dc584fc314 100644 --- a/mobile/apps/photos/lib/services/deduplication_service.dart +++ b/mobile/apps/photos/lib/services/deduplication_service.dart @@ -1,12 +1,12 @@ import 'package:logging/logging.dart'; -import "package:photos/core/configuration.dart"; import 'package:photos/core/network/network.dart'; -import 'package:photos/db/files_db.dart'; import 'package:photos/models/duplicate_files.dart'; +import "package:photos/models/file/extensions/file_props.dart"; import 'package:photos/models/file/file.dart'; import "package:photos/models/file/file_type.dart"; import "package:photos/services/collections_service.dart"; import "package:photos/services/files_service.dart"; +import "package:photos/services/search_service.dart"; class DeduplicationService { final _logger = Logger("DeduplicationService"); @@ -39,16 +39,13 @@ class DeduplicationService { final Set allowedCollectionIDs = CollectionsService.instance.nonHiddenOwnedCollections(); - final List allFiles = await FilesDB.instance.getAllFilesFromDB( - CollectionsService.instance.getHiddenCollectionIds(), - dedupeByUploadId: false, - ); - final int ownerID = Configuration.instance.getUserID()!; + final List allFiles = + await SearchService.instance.getAllFilesForSearch(); final List filteredFiles = []; for (final file in allFiles) { if (!file.isUploaded || (file.hash ?? '').isEmpty || - (file.ownerID ?? 0) != ownerID || + !file.isOwner || (!allowedCollectionIDs.contains(file.collectionID!))) { continue; } diff --git a/mobile/apps/photos/lib/services/machine_learning/similar_images_service.dart b/mobile/apps/photos/lib/services/machine_learning/similar_images_service.dart index fd1b0e9d1c..77fbb309c4 100644 --- a/mobile/apps/photos/lib/services/machine_learning/similar_images_service.dart +++ b/mobile/apps/photos/lib/services/machine_learning/similar_images_service.dart @@ -4,14 +4,16 @@ import "package:flutter/foundation.dart" show kDebugMode; import "package:flutter_rust_bridge/flutter_rust_bridge_for_generated.dart" show Uint64List; import 'package:logging/logging.dart'; +import "package:path_provider/path_provider.dart"; import "package:photos/db/ml/db.dart"; import "package:photos/extensions/stop_watch.dart"; +import "package:photos/models/file/extensions/file_props.dart"; import 'package:photos/models/file/file.dart'; -import "package:photos/models/file/file_type.dart"; import "package:photos/models/similar_files.dart"; import "package:photos/services/machine_learning/ml_computer.dart"; import "package:photos/services/machine_learning/ml_result.dart"; import "package:photos/services/search_service.dart"; +import "package:photos/utils/cache_util.dart"; class SimilarImagesService { final _logger = Logger("SimilarImagesService"); @@ -25,11 +27,12 @@ class SimilarImagesService { Future> getSimilarFiles( double distanceThreshold, { bool exact = false, + bool forceRefresh = false, }) async { try { final now = DateTime.now(); final List result = - await _getSimilarFiles(distanceThreshold, exact); + await _getSimilarFiles(distanceThreshold, exact, forceRefresh); final duration = DateTime.now().difference(now); _logger.info( "Found ${result.length} similar files in ${duration.inSeconds} seconds for threshold $distanceThreshold and exact $exact", @@ -44,6 +47,7 @@ class SimilarImagesService { Future> _getSimilarFiles( double distanceThreshold, bool exact, + bool forceRefresh, ) async { final w = (kDebugMode ? EnteWatch('getSimilarFiles') : null)?..start(); final mlDataDB = MLDataDB.instance; @@ -58,7 +62,7 @@ class SimilarImagesService { final allFileIdsToFile = {}; final fileIDs = []; for (final file in allFiles) { - if (file.uploadedFileID != null && file.fileType != FileType.video) { + if (file.uploadedFileID != null && file.isOwner && !file.isVideo) { allFileIdsToFile[file.uploadedFileID!] = file; fileIDs.add(file.uploadedFileID!); } @@ -84,6 +88,253 @@ class SimilarImagesService { } w?.log("getFileIDToPersonIDs"); + if (forceRefresh) { + final result = await _performFullSearch( + potentialKeys, + allFileIdsToFile, + fileIDToPersonIDs, + distanceThreshold, + exact, + ); + await _cacheSimilarFiles( + result, + fileIDs.toSet(), + distanceThreshold, + exact, + DateTime.now().millisecondsSinceEpoch, + ); + return result; + } + + // Load cached data + final SimilarFilesCache? cachedData = await _readCachedSimilarFiles(); + if (cachedData == null) { + _logger.warning("No cached similar files found"); + } else { + _logger.info( + "Cached similar files found with ${cachedData.similarFilesJsonStringList.length} groups", + ); + } + + // Determine if we need full refresh + bool needsFullRefresh = false; + if (cachedData != null) { + final Set cachedFileIDs = cachedData.allCheckedFileIDs; + final currentFileIDs = fileIDs.toSet(); + + if (cachedData.distanceThreshold != distanceThreshold || + cachedData.exact != exact) { + needsFullRefresh = true; + } + + // Check condition: less than 1000 files + if (currentFileIDs.length < 1000) { + needsFullRefresh = true; + } + + // Check condition: cache is older than a month + if (DateTime.fromMillisecondsSinceEpoch(cachedData.cachedTime) + .isBefore(DateTime.now().subtract(const Duration(days: 30)))) { + needsFullRefresh = true; + } + + // Check condition: new files > 20% of total files + if (!needsFullRefresh) { + final newFileIDs = currentFileIDs.difference(cachedFileIDs); + if (newFileIDs.length > currentFileIDs.length * 0.2) { + needsFullRefresh = true; + } + } + + // Check condition: 20+% of grouped files deleted + if (!needsFullRefresh) { + final Set cacheGroupedFileIDs = + await cachedData.getGroupedFileIDs(); + final deletedFromGroups = cacheGroupedFileIDs + .intersection(cachedFileIDs.difference(currentFileIDs)); + final totalInGroups = cacheGroupedFileIDs.length; + if (totalInGroups > 0 && + deletedFromGroups.length > totalInGroups * 0.2) { + needsFullRefresh = true; + } + } + } + + if (cachedData == null || needsFullRefresh) { + final result = await _performFullSearch( + potentialKeys, + allFileIdsToFile, + fileIDToPersonIDs, + distanceThreshold, + exact, + ); + await _cacheSimilarFiles( + result, + fileIDs.toSet(), + distanceThreshold, + exact, + DateTime.now().millisecondsSinceEpoch, + ); + return result; + } else { + return await _performIncrementalUpdate( + cachedData, + potentialKeys, + allFileIdsToFile, + fileIDToPersonIDs, + distanceThreshold, + exact, + ); + } + } + + Future> _performIncrementalUpdate( + SimilarFilesCache cachedData, + Uint64List currentFileIDs, + Map allFileIdsToFile, + Map> fileIDToPersonIDs, + double distanceThreshold, + bool exact, + ) async { + _logger.info("Performing incremental update for similar files"); + final existingGroups = await cachedData.similarFilesList(); + final cachedFileIDs = cachedData.allCheckedFileIDs; + final currentFileIDsSet = currentFileIDs.map((id) => id.toInt()).toSet(); + final deletedFiles = cachedFileIDs.difference(currentFileIDsSet); + + // Clean up deleted files from existing groups + if (deletedFiles.isNotEmpty) { + for (final group in existingGroups) { + final filesInGroupToDelete = []; + for (final fileInGroup in group.files) { + if (deletedFiles.contains(fileInGroup.uploadedFileID ?? -1)) { + filesInGroupToDelete.add(fileInGroup); + } + } + for (final fileToDelete in filesInGroupToDelete) { + group.removeFile(fileToDelete); + } + } + } + // Remove empty groups + existingGroups.removeWhere((group) => group.length <= 1); + + // Identify new files + final newFileIDs = currentFileIDsSet.difference(cachedFileIDs); + if (newFileIDs.isEmpty) { + return existingGroups; + } + + // Search only new files + final newFileIDsList = Uint64List.fromList(newFileIDs.toList()); + final (keys, vectorKeys, distances) = + await MLComputer.instance.bulkVectorSearchWithKeys( + newFileIDsList, + exact, + ); + final keysList = keys.map((key) => key.toInt()).toList(); + + // Try to assign new files to existing groups + final unassignedNewFilesIndices = {}; + final unassignedNewFileIDs = {}; + for (int i = 0; i < keysList.length; i++) { + final newFileID = keysList[i]; + final newFile = allFileIdsToFile[newFileID]; + if (newFile == null) continue; + final similarFileIDs = vectorKeys[i]; + final fileDistances = distances[i]; + final newFilePersonIDs = fileIDToPersonIDs[newFileID] ?? {}; + bool assigned = false; + for (int j = 0; j < similarFileIDs.length; j++) { + final otherFileID = similarFileIDs[j].toInt(); + if (otherFileID == newFileID) continue; + final distance = fileDistances[j]; + if (distance > distanceThreshold) break; + for (final group in existingGroups) { + if (group.fileIds.contains(otherFileID)) { + final otherPersonIDs = fileIDToPersonIDs[otherFileID] ?? {}; + if (setsAreEqual(newFilePersonIDs, otherPersonIDs)) { + group.addFile(newFile); + group.furthestDistance = max(group.furthestDistance, distance); + group.files.sort((a, b) { + final sizeComparison = + (b.fileSize ?? 0).compareTo(a.fileSize ?? 0); + if (sizeComparison != 0) return sizeComparison; + return a.displayName.compareTo(b.displayName); + }); + assigned = true; + break; + } + } + } + if (assigned) break; + } + if (!assigned) { + unassignedNewFilesIndices.add(i); + unassignedNewFileIDs.add(newFileID); + } + } + + // Check if unassigned new files form groups among themselves + if (unassignedNewFilesIndices.isNotEmpty) { + final alreadyUsedNewFiles = {}; + for (final searchIndex in unassignedNewFilesIndices) { + final newFileID = keysList[searchIndex]; + if (alreadyUsedNewFiles.contains(newFileID)) continue; + final newFile = allFileIdsToFile[newFileID]; + if (newFile == null) continue; + final similarFileIDs = vectorKeys[searchIndex]; + final fileDistances = distances[searchIndex]; + final newFilePersonIDs = fileIDToPersonIDs[newFileID] ?? {}; + final similarNewFiles = []; + double furthestDistance = 0.0; + for (int j = 0; j < similarFileIDs.length; j++) { + final otherFileID = similarFileIDs[j].toInt(); + if (otherFileID == newFileID) continue; + if (!unassignedNewFileIDs.contains(otherFileID)) continue; + if (alreadyUsedNewFiles.contains(otherFileID)) continue; + final distance = fileDistances[j]; + if (distance > distanceThreshold) break; + final otherFile = allFileIdsToFile[otherFileID]; + if (otherFile == null) continue; + final otherPersonIDs = fileIDToPersonIDs[otherFileID] ?? {}; + if (!setsAreEqual(newFilePersonIDs, otherPersonIDs)) continue; + similarNewFiles.add(otherFile); + alreadyUsedNewFiles.add(otherFileID); + furthestDistance = max(furthestDistance, distance); + } + if (similarNewFiles.isNotEmpty) { + similarNewFiles.add(newFile); + alreadyUsedNewFiles.add(newFileID); + similarNewFiles.sort((a, b) { + final sizeComparison = (b.fileSize ?? 0).compareTo(a.fileSize ?? 0); + if (sizeComparison != 0) return sizeComparison; + return a.displayName.compareTo(b.displayName); + }); + existingGroups.add(SimilarFiles(similarNewFiles, furthestDistance)); + } + } + } + await _cacheSimilarFiles( + existingGroups, + currentFileIDsSet, + distanceThreshold, + exact, + cachedData.cachedTime, + ); + + return existingGroups; + } + + Future> _performFullSearch( + Uint64List potentialKeys, + Map allFileIdsToFile, + Map> fileIDToPersonIDs, + double distanceThreshold, + bool exact, + ) async { + _logger.info("Performing full search for similar files"); + final w = (kDebugMode ? EnteWatch('getSimilarFiles') : null)?..start(); // Run bulk vector search final (keys, vectorKeys, distances) = await MLComputer.instance.bulkVectorSearchWithKeys( @@ -121,14 +372,18 @@ class SimilarImagesService { if (!setsAreEqual(personIDs, otherPersonIDs)) continue; similarFilesList.add(otherFile); furthestDistance = max(furthestDistance, distance); + alreadyUsedFileIDs.add(otherFileID); } if (similarFilesList.isNotEmpty) { similarFilesList.add(firstLoopFile); for (final file in similarFilesList) { alreadyUsedFileIDs.add(file.uploadedFileID!); } + // show highest quality files first similarFilesList.sort((a, b) { - return a.displayName.length.compareTo(b.displayName.length); + final sizeComparison = (b.fileSize ?? 0).compareTo(a.fileSize ?? 0); + if (sizeComparison != 0) return sizeComparison; + return a.displayName.compareTo(b.displayName); }); final similarFiles = SimilarFiles( similarFilesList, @@ -141,6 +396,44 @@ class SimilarImagesService { return allSimilarFiles; } + + Future _getCachePath() async { + return (await getApplicationSupportDirectory()).path + + "/cache/similar_images_cache"; + } + + Future _cacheSimilarFiles( + List similarGroups, + Set allCheckedFileIDs, + double distanceThreshold, + bool exact, + int cachedTimeOfOriginalComputation, + ) async { + final cachePath = await _getCachePath(); + final similarGroupsJsonStringList = + similarGroups.map((group) => group.toJsonString()).toList(); + final cacheObject = SimilarFilesCache( + similarFilesJsonStringList: similarGroupsJsonStringList, + allCheckedFileIDs: allCheckedFileIDs, + distanceThreshold: distanceThreshold, + exact: exact, + cachedTime: cachedTimeOfOriginalComputation, + ); + await writeToJsonFile( + cachePath, + cacheObject, + SimilarFilesCache.encodeToJsonString, + ); + } + + Future _readCachedSimilarFiles() async { + _logger.info("Reading similar files cache result from disk"); + final cache = decodeJsonFile( + await _getCachePath(), + SimilarFilesCache.decodeFromJsonString, + ); + return cache; + } } bool setsAreEqual(Set set1, Set set2) { diff --git a/mobile/apps/photos/lib/ui/settings/backup/free_space_options.dart b/mobile/apps/photos/lib/ui/settings/backup/free_space_options.dart index fa917a7f7b..7b69a42cff 100644 --- a/mobile/apps/photos/lib/ui/settings/backup/free_space_options.dart +++ b/mobile/apps/photos/lib/ui/settings/backup/free_space_options.dart @@ -1,6 +1,7 @@ import "dart:async"; import "dart:io"; +import "package:flutter/foundation.dart" show kDebugMode; import 'package:flutter/material.dart'; import "package:photos/generated/l10n.dart"; import "package:photos/models/backup_status.dart"; @@ -212,7 +213,7 @@ class _FreeUpSpaceOptionsScreenState extends State { onTap: () async { await routeToPage( context, - const SimilarImagesPage(), + const SimilarImagesPage(debugScreen: kDebugMode,), ); }, ), diff --git a/mobile/apps/photos/lib/ui/tools/deduplicate_page.dart b/mobile/apps/photos/lib/ui/tools/deduplicate_page.dart index 44b895a553..4fa9d1226c 100644 --- a/mobile/apps/photos/lib/ui/tools/deduplicate_page.dart +++ b/mobile/apps/photos/lib/ui/tools/deduplicate_page.dart @@ -4,16 +4,19 @@ import 'package:flutter/material.dart'; import 'package:flutter/services.dart'; import 'package:photos/core/constants.dart'; import 'package:photos/core/event_bus.dart'; -import 'package:photos/ente_theme_data.dart'; + import 'package:photos/events/user_details_changed_event.dart'; import "package:photos/generated/l10n.dart"; import 'package:photos/models/duplicate_files.dart'; import 'package:photos/models/file/file.dart'; import 'package:photos/services/collections_service.dart'; import "package:photos/theme/ente_theme.dart"; +import 'package:photos/ui/components/buttons/button_widget.dart'; +import "package:photos/ui/components/models/button_type.dart"; import 'package:photos/ui/viewer/file/detail_page.dart'; import 'package:photos/ui/viewer/file/thumbnail_widget.dart'; import 'package:photos/ui/viewer/gallery/empty_state.dart'; +import "package:photos/ui/viewer/gallery/scrollbar/scroll_bar_with_use_notifier.dart"; import 'package:photos/utils/delete_file_util.dart'; import "package:photos/utils/dialog_util.dart"; import 'package:photos/utils/navigation_util.dart'; @@ -31,7 +34,6 @@ class DeduplicatePage extends StatefulWidget { class _DeduplicatePageState extends State { static const crossAxisCount = 3; static const crossAxisSpacing = 12.0; - static const headerRowCount = 3; final Set selectedGrids = {}; @@ -39,11 +41,15 @@ class _DeduplicatePageState extends State { SortKey sortKey = SortKey.size; late ValueNotifier _deleteProgress; + late ScrollController _scrollController; + late ValueNotifier _scrollbarInUseNotifier; @override void initState() { _duplicates = widget.duplicates; _deleteProgress = ValueNotifier(""); + _scrollController = ScrollController(); + _scrollbarInUseNotifier = ValueNotifier(false); _selectAllGrids(); super.initState(); } @@ -51,6 +57,8 @@ class _DeduplicatePageState extends State { @override void dispose() { _deleteProgress.dispose(); + _scrollController.dispose(); + _scrollbarInUseNotifier.dispose(); super.dispose(); } @@ -68,47 +76,7 @@ class _DeduplicatePageState extends State { appBar: AppBar( elevation: 0, title: Text(AppLocalizations.of(context).deduplicateFiles), - actions: [ - PopupMenuButton( - constraints: const BoxConstraints(minWidth: 180), - shape: const RoundedRectangleBorder( - borderRadius: BorderRadius.all( - Radius.circular(8), - ), - ), - onSelected: (dynamic value) { - setState(() { - selectedGrids.clear(); - }); - }, - offset: const Offset(0, 50), - itemBuilder: (BuildContext context) => [ - PopupMenuItem( - value: true, - height: 32, - child: Row( - children: [ - const Icon( - Icons.remove_circle_outline, - size: 20, - ), - const SizedBox(width: 12), - Padding( - padding: const EdgeInsets.only(bottom: 1), - child: Text( - AppLocalizations.of(context).deselectAll, - style: Theme.of(context) - .textTheme - .titleMedium! - .copyWith(fontWeight: FontWeight.w600), - ), - ), - ], - ), - ), - ], - ), - ], + actions: _duplicates.isNotEmpty ? [_getSortMenu()] : null, ), body: _getBody(), ); @@ -133,33 +101,34 @@ class _DeduplicatePageState extends State { crossAxisAlignment: CrossAxisAlignment.center, children: [ Expanded( - child: ListView.builder( - itemBuilder: (context, index) { - if (index == 0) { - return const SizedBox.shrink(); - } else if (index == 1) { - return const SizedBox.shrink(); - } else if (index == 2) { - if (_duplicates.isNotEmpty) { - return _getSortMenu(context); - } else { - return const Padding( - padding: EdgeInsets.only(top: 32), - child: EmptyState(), - ); - } - } - return Padding( - padding: const EdgeInsets.symmetric(vertical: 8), - child: _getGridView( - _duplicates[index - headerRowCount], - index - headerRowCount, + child: _duplicates.isNotEmpty + ? ScrollbarWithUseNotifer( + controller: _scrollController, + inUseNotifier: _scrollbarInUseNotifier, + minScrollbarLength: 36.0, + interactive: true, + thickness: 8, + radius: const Radius.circular(4), + child: ListView.builder( + controller: _scrollController, + cacheExtent: 400, + itemBuilder: (context, index) { + return Padding( + padding: const EdgeInsets.symmetric(vertical: 8), + child: _getGridView( + _duplicates[index], + index, + ), + ); + }, + itemCount: _duplicates.length, + shrinkWrap: true, + ), + ) + : const Padding( + padding: EdgeInsets.only(top: 32), + child: EmptyState(), ), - ); - }, - itemCount: _duplicates.length + headerRowCount, - shrinkWrap: true, - ), ), selectedGrids.isEmpty ? const SizedBox.shrink() @@ -189,7 +158,8 @@ class _DeduplicatePageState extends State { ); } - Widget _getSortMenu(BuildContext context) { + Widget _getSortMenu() { + final textTheme = getEnteTextTheme(context); Text sortOptionText(SortKey key) { String text = key.toString(); switch (key) { @@ -202,63 +172,44 @@ class _DeduplicatePageState extends State { } return Text( text, - style: Theme.of(context).textTheme.titleMedium!.copyWith( - fontSize: 14, - color: Theme.of(context).iconTheme.color!.withValues(alpha: 0.7), - ), + style: textTheme.miniBold, ); } - return Row( - // h4ck to align PopupMenuItems to end - mainAxisAlignment: MainAxisAlignment.spaceBetween, - crossAxisAlignment: CrossAxisAlignment.end, - children: [ - const SizedBox.shrink(), - PopupMenuButton( - initialValue: sortKey.index, - child: Padding( - padding: const EdgeInsets.fromLTRB(24, 6, 24, 6), - child: Row( - mainAxisAlignment: MainAxisAlignment.start, - crossAxisAlignment: CrossAxisAlignment.center, - children: [ - sortOptionText(sortKey), - const Padding(padding: EdgeInsets.only(left: 4)), - Icon( - Icons.sort, - color: Theme.of(context).colorScheme.iconColor, - size: 20, - ), - ], - ), - ), - onSelected: (int index) { - setState(() { - final newKey = SortKey.values[index]; - if (newKey == sortKey) { - return; - } else { - sortKey = newKey; - if (selectedGrids.length != _duplicates.length) { - selectedGrids.clear(); - } - } - }); - }, - itemBuilder: (context) { - return List.generate(SortKey.values.length, (index) { - return PopupMenuItem( - value: index, - child: Align( - alignment: Alignment.centerLeft, - child: sortOptionText(SortKey.values[index]), - ), - ); - }); - }, + return PopupMenuButton( + initialValue: sortKey.index, + child: Padding( + padding: const EdgeInsets.fromLTRB(24, 6, 24, 6), + child: Icon( + Icons.sort, + color: getEnteColorScheme(context).strokeBase, + size: 20, ), - ], + ), + onSelected: (int index) { + setState(() { + final newKey = SortKey.values[index]; + if (newKey == sortKey) { + return; + } else { + sortKey = newKey; + if (selectedGrids.length != _duplicates.length) { + selectedGrids.clear(); + } + } + }); + }, + itemBuilder: (context) { + return List.generate(SortKey.values.length, (index) { + return PopupMenuItem( + value: index, + child: Text( + sortOptionText(SortKey.values[index]).data!, + style: textTheme.miniBold, + ), + ); + }); + }, ); } @@ -272,55 +223,60 @@ class _DeduplicatePageState extends State { totalSize += toDeleteCount * _duplicates[index].size; } } - final String text = AppLocalizations.of(context).deleteItemCount(fileCount); - return SizedBox( - width: double.infinity, - child: SafeArea( - child: Padding( - padding: const EdgeInsets.symmetric(horizontal: crossAxisSpacing), - child: TextButton( - style: OutlinedButton.styleFrom( - backgroundColor: - Theme.of(context).colorScheme.inverseBackgroundColor, - ), - child: Column( - mainAxisAlignment: MainAxisAlignment.end, - children: [ - const Padding(padding: EdgeInsets.all(4)), - Text( - text, - style: TextStyle( - fontWeight: FontWeight.bold, - fontSize: 14, - color: Theme.of(context).colorScheme.inverseTextColor, - ), - textAlign: TextAlign.center, + final hasSelectedFiles = fileCount > 0; + + return AnimatedSwitcher( + duration: const Duration(milliseconds: 200), + switchInCurve: Curves.easeOut, + switchOutCurve: Curves.easeIn, + child: hasSelectedFiles + ? SafeArea( + child: Container( + key: const ValueKey('bottom_buttons'), + padding: const EdgeInsets.symmetric( + horizontal: crossAxisSpacing, + vertical: 8, ), - const Padding(padding: EdgeInsets.all(2)), - Text( - formatBytes(totalSize), - style: TextStyle( - color: Theme.of(context) - .colorScheme - .inverseTextColor - .withValues(alpha: 0.7), - fontSize: 12, - ), + decoration: BoxDecoration( + color: getEnteColorScheme(context).backgroundBase, ), - const Padding(padding: EdgeInsets.all(2)), - ], - ), - onPressed: () async { - try { - await deleteDuplicates(totalSize); - } catch (e) { - log("Failed to delete duplicates", error: e); - showGenericErrorDialog(context: context, error: e).ignore(); - } - }, - ), - ), - ), + child: Column( + children: [ + SizedBox( + width: double.infinity, + child: ButtonWidget( + labelText: + "${AppLocalizations.of(context).deleteItemCount(fileCount)} (${formatBytes(totalSize)})", + buttonType: ButtonType.critical, + onTap: () async { + try { + await deleteDuplicates(totalSize); + } catch (e) { + log("Failed to delete duplicates", error: e); + showGenericErrorDialog(context: context, error: e) + .ignore(); + } + }, + ), + ), + const SizedBox(height: 8), + SizedBox( + width: double.infinity, + child: ButtonWidget( + labelText: "Unselect all", // TODO: lau: extract string + buttonType: ButtonType.secondary, + onTap: () async { + setState(() { + selectedGrids.clear(); + }); + }, + ), + ), + ], + ), + ), + ) + : const SizedBox.shrink(key: ValueKey('empty')), ); } diff --git a/mobile/apps/photos/lib/ui/tools/similar_images_page.dart b/mobile/apps/photos/lib/ui/tools/similar_images_page.dart index 53ea183f37..9560f4efe8 100644 --- a/mobile/apps/photos/lib/ui/tools/similar_images_page.dart +++ b/mobile/apps/photos/lib/ui/tools/similar_images_page.dart @@ -1,5 +1,6 @@ import "dart:async"; +import "package:flutter/foundation.dart" show kDebugMode; import 'package:flutter/material.dart'; import 'package:logging/logging.dart'; import 'package:photos/core/constants.dart'; @@ -13,11 +14,13 @@ import "package:photos/services/collections_service.dart"; import "package:photos/services/machine_learning/similar_images_service.dart"; import 'package:photos/theme/ente_theme.dart'; import "package:photos/ui/common/loading_widget.dart"; +import 'package:photos/ui/components/action_sheet_widget.dart'; import 'package:photos/ui/components/buttons/button_widget.dart'; import "package:photos/ui/components/models/button_type.dart"; import "package:photos/ui/components/toggle_switch_widget.dart"; import "package:photos/ui/viewer/file/detail_page.dart"; import "package:photos/ui/viewer/file/thumbnail_widget.dart"; +import "package:photos/ui/viewer/gallery/scrollbar/scroll_bar_with_use_notifier.dart"; import "package:photos/utils/delete_file_util.dart"; import "package:photos/utils/dialog_util.dart"; import "package:photos/utils/navigation_util.dart"; @@ -37,7 +40,9 @@ enum SortKey { } class SimilarImagesPage extends StatefulWidget { - const SimilarImagesPage({super.key}); + final bool debugScreen; + + const SimilarImagesPage({super.key, this.debugScreen = false}); @override State createState() => _SimilarImagesPageState(); @@ -46,7 +51,6 @@ class SimilarImagesPage extends StatefulWidget { class _SimilarImagesPageState extends State { static const crossAxisCount = 3; static const crossAxisSpacing = 12.0; - static const autoSelectDistanceThreshold = 0.01; final _logger = Logger("SimilarImagesPage"); bool _isDisposed = false; @@ -56,19 +60,31 @@ class _SimilarImagesPageState extends State { List _similarFilesList = []; SortKey _sortKey = SortKey.distanceAsc; bool _exactSearch = false; + bool _fullRefresh = false; + bool _isSelectionSheetOpen = false; late SelectedFiles _selectedFiles; + late ScrollController _scrollController; + late ValueNotifier _scrollbarInUseNotifier; @override void initState() { super.initState(); _selectedFiles = SelectedFiles(); + _scrollController = ScrollController(); + _scrollbarInUseNotifier = ValueNotifier(false); + + if (!widget.debugScreen) { + _findSimilarImages(); + } } @override void dispose() { _isDisposed = true; _selectedFiles.dispose(); + _scrollController.dispose(); + _scrollbarInUseNotifier.dispose(); super.dispose(); } @@ -171,7 +187,7 @@ class _SimilarImagesPageState extends State { mainAxisAlignment: MainAxisAlignment.spaceBetween, children: [ Text( - "Exact search", // TODO: lau: extract string + "Exact search", style: textTheme.bodyBold, ), ToggleSwitchWidget( @@ -185,6 +201,25 @@ class _SimilarImagesPageState extends State { ), ], ), + const SizedBox(height: 12), + Row( + mainAxisAlignment: MainAxisAlignment.spaceBetween, + children: [ + Text( + "Full refresh", + style: textTheme.bodyBold, + ), + ToggleSwitchWidget( + value: () => _fullRefresh, + onChanged: () async { + if (_isDisposed) return; + setState(() { + _fullRefresh = !_fullRefresh; + }); + }, + ), + ], + ), const SizedBox(height: 32), ButtonWidget( labelText: "Find similar images", // TODO: lau: extract string @@ -253,37 +288,63 @@ class _SimilarImagesPageState extends State { return Column( children: [ Expanded( - child: ListView.builder( - itemCount: _similarFilesList.length + 1, // +1 for header - itemBuilder: (context, index) { - if (index == 0) { - // Header item - if (flagService.internalUser) { + child: ScrollbarWithUseNotifer( + controller: _scrollController, + inUseNotifier: _scrollbarInUseNotifier, + minScrollbarLength: 36.0, + interactive: true, + thickness: 8, + radius: const Radius.circular(4), + child: ListView.builder( + controller: _scrollController, + cacheExtent: 400, + itemCount: _similarFilesList.length + 1, // +1 for header + itemBuilder: (context, index) { + if (index == 0) { return Container( + margin: const EdgeInsets.symmetric( + horizontal: 16, + vertical: 12, + ), padding: const EdgeInsets.all(16), - child: Column( + decoration: BoxDecoration( + color: colorScheme.fillFaint, + borderRadius: BorderRadius.circular(8), + ), + child: Row( children: [ - Text( - "(I) Found ${_similarFilesList.length} groups of similar images", // TODO: lau: extract string - style: textTheme.bodyBold, + Icon( + Icons.photo_library_outlined, + size: 20, + color: colorScheme.textMuted, ), - const SizedBox(height: 4), - Text( - "(I) Threshold: ${_distanceThreshold.toStringAsFixed(2)}", // TODO: lau: extract string - style: textTheme.miniMuted, + const SizedBox(width: 12), + Expanded( + child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + Text( + "${_similarFilesList.length} ${_similarFilesList.length == 1 ? 'group' : 'groups'} found", // TODO: lau: extract string + style: textTheme.bodyBold, + ), + const SizedBox(height: 4), + Text( + "Review and remove similar images", // TODO: lau: extract string + style: textTheme.miniMuted, + ), + ], + ), ), ], ), ); - } else { - return const SizedBox.shrink(); } - } - // Similar files groups (index - 1 because first item is header) - final similarFiles = _similarFilesList[index - 1]; - return _buildSimilarFilesGroup(similarFiles); - }, + // Similar files groups (index - 1 because first item is header) + final similarFiles = _similarFilesList[index - 1]; + return _buildSimilarFilesGroup(similarFiles); + }, + ), ), ), _getBottomActionButtons(), @@ -298,54 +359,70 @@ class _SimilarImagesPageState extends State { final selectedCount = _selectedFiles.files.length; final hasSelectedFiles = selectedCount > 0; - return AnimatedSwitcher( - duration: const Duration(milliseconds: 200), - switchInCurve: Curves.easeOut, - switchOutCurve: Curves.easeIn, - child: hasSelectedFiles - ? SafeArea( - child: Container( - key: const ValueKey('bottom_buttons'), - padding: const EdgeInsets.symmetric( - horizontal: crossAxisSpacing, - vertical: 8, + int totalSize = 0; + for (final file in _selectedFiles.files) { + totalSize += file.fileSize ?? 0; + } + + return SafeArea( + child: Container( + padding: const EdgeInsets.symmetric( + horizontal: crossAxisSpacing, + vertical: 8, + ), + decoration: BoxDecoration( + color: getEnteColorScheme(context).backgroundBase, + ), + child: AnimatedSwitcher( + duration: Duration.zero, + child: Column( + key: ValueKey(hasSelectedFiles), + children: [ + if (hasSelectedFiles && !_isSelectionSheetOpen) ...[ + SizedBox( + width: double.infinity, + child: ButtonWidget( + labelText: + "Delete $selectedCount photos (${formatBytes(totalSize)})", // TODO: lau: extract string + buttonType: ButtonType.critical, + shouldSurfaceExecutionStates: false, + shouldShowSuccessConfirmation: false, + onTap: () async { + await _deleteFiles( + _selectedFiles.files, + showDialog: true, + ); + }, + ), ), - decoration: BoxDecoration( - color: getEnteColorScheme(context).backgroundBase, + const SizedBox(height: 8), + ], + if (!_isSelectionSheetOpen) + SizedBox( + width: double.infinity, + child: ButtonWidget( + labelText: + "Selection options", // TODO: lau: extract string + buttonType: ButtonType.secondary, + shouldSurfaceExecutionStates: false, + shouldShowSuccessConfirmation: false, + onTap: () async { + setState(() { + _isSelectionSheetOpen = true; + }); + await _showSelectionOptionsSheet(); + if (mounted) { + setState(() { + _isSelectionSheetOpen = false; + }); + } + }, + ), ), - child: Column( - children: [ - SizedBox( - width: double.infinity, - child: ButtonWidget( - labelText: - "Delete $selectedCount photos", // TODO: lau: extract string - buttonType: ButtonType.critical, - onTap: () async { - await _deleteFiles( - _selectedFiles.files, - showDialog: true, - ); - }, - ), - ), - const SizedBox(height: 8), - SizedBox( - width: double.infinity, - child: ButtonWidget( - labelText: - "Unselect all", // TODO: lau: extract string - buttonType: ButtonType.secondary, - onTap: () async { - _selectedFiles.clearAll(fireEvent: false); - }, - ), - ), - ], - ), - ), - ) - : const SizedBox.shrink(key: ValueKey('empty')), + ], + ), + ), + ), ); }, ); @@ -361,8 +438,11 @@ class _SimilarImagesPageState extends State { // You can use _toggleValue here for advanced mode features _logger.info("exact mode: $_exactSearch"); - final similarFiles = await SimilarImagesService.instance - .getSimilarFiles(_distanceThreshold, exact: _exactSearch); + final similarFiles = await SimilarImagesService.instance.getSimilarFiles( + _distanceThreshold, + exact: _exactSearch, + forceRefresh: _fullRefresh, + ); _logger.info( "Found ${similarFiles.length} groups of similar images", ); @@ -370,7 +450,6 @@ class _SimilarImagesPageState extends State { _similarFilesList = similarFiles; _pageState = SimilarImagesPageState.results; _sortSimilarFiles(); - _autoSelectSimilarFiles(); if (_isDisposed) return; setState(() {}); @@ -396,12 +475,24 @@ class _SimilarImagesPageState extends State { _similarFilesList.sort((a, b) => b.totalSize.compareTo(a.totalSize)); break; case SortKey.distanceAsc: - _similarFilesList - .sort((a, b) => a.furthestDistance.compareTo(b.furthestDistance)); + _similarFilesList.sort((a, b) { + final distanceComparison = + a.furthestDistance.compareTo(b.furthestDistance); + if (distanceComparison != 0) { + return distanceComparison; + } + return b.totalSize.compareTo(a.totalSize); + }); break; case SortKey.distanceDesc: - _similarFilesList - .sort((a, b) => b.furthestDistance.compareTo(a.furthestDistance)); + _similarFilesList.sort((a, b) { + final distanceComparison = + b.furthestDistance.compareTo(a.furthestDistance); + if (distanceComparison != 0) { + return distanceComparison; + } + return b.totalSize.compareTo(a.totalSize); + }); break; case SortKey.count: _similarFilesList @@ -412,16 +503,11 @@ class _SimilarImagesPageState extends State { setState(() {}); } - void _autoSelectSimilarFiles() { + void _selectFilesByThreshold(double threshold) { final filesToSelect = {}; - int groupsProcessed = 0; - int groupsAutoSelected = 0; for (final similarFilesGroup in _similarFilesList) { - groupsProcessed++; - if (similarFilesGroup.furthestDistance < autoSelectDistanceThreshold) { - groupsAutoSelected++; - // Skip the first file (keep it unselected) and select the rest + if (similarFilesGroup.furthestDistance <= threshold) { for (int i = 1; i < similarFilesGroup.files.length; i++) { filesToSelect.add(similarFilesGroup.files[i]); } @@ -429,17 +515,102 @@ class _SimilarImagesPageState extends State { } if (filesToSelect.isNotEmpty) { + _selectedFiles.clearAll(fireEvent: false); _selectedFiles.selectAll(filesToSelect); - _logger.info( - "Auto-selected ${filesToSelect.length} files from $groupsAutoSelected/$groupsProcessed groups (threshold: $autoSelectDistanceThreshold)", - ); } else { - _logger.info( - "No files auto-selected from $groupsProcessed groups (threshold: $autoSelectDistanceThreshold)", - ); + _selectedFiles.clearAll(fireEvent: false); } } + Future _showSelectionOptionsSheet() async { + // Calculate how many files fall into each category + int exactFiles = 0; + int similarFiles = 0; + int allFiles = 0; + + for (final group in _similarFilesList) { + final duplicateCount = group.files.length - 1; // Exclude the first file + allFiles += duplicateCount; + + if (group.furthestDistance <= 0.0) { + exactFiles += duplicateCount; + similarFiles += duplicateCount; + } else if (group.furthestDistance <= 0.02) { + similarFiles += duplicateCount; + } + } + + final String exactLabel = exactFiles > 0 + ? "Select exact ($exactFiles)" // TODO: lau: extract string + : "Select exact"; // TODO: lau: extract string + + final String similarLabel = similarFiles > 0 + ? "Select similar ($similarFiles)" // TODO: lau: extract string + : "Select similar"; // TODO: lau: extract string + + final String allLabel = allFiles > 0 + ? "Select all ($allFiles)" // TODO: lau: extract string + : "Select all"; // TODO: lau: extract string + + await showActionSheet( + context: context, + title: "Select similar images", // TODO: lau: extract string + body: + "Choose which similar images to select for deletion", // TODO: lau: extract string + buttons: [ + ButtonWidget( + labelText: exactLabel, + buttonType: ButtonType.neutral, + buttonSize: ButtonSize.large, + shouldStickToDarkTheme: true, + isInAlert: true, + buttonAction: ButtonAction.first, + shouldSurfaceExecutionStates: false, + onTap: () async { + _selectFilesByThreshold(0.0); + }, + ), + ButtonWidget( + labelText: similarLabel, + buttonType: ButtonType.neutral, + buttonSize: ButtonSize.large, + shouldStickToDarkTheme: true, + isInAlert: true, + buttonAction: ButtonAction.second, + shouldSurfaceExecutionStates: false, + onTap: () async { + _selectFilesByThreshold(0.02); + }, + ), + ButtonWidget( + labelText: allLabel, + buttonType: ButtonType.neutral, + buttonSize: ButtonSize.large, + shouldStickToDarkTheme: true, + isInAlert: true, + buttonAction: ButtonAction.third, + shouldSurfaceExecutionStates: false, + onTap: () async { + _selectFilesByThreshold(0.05); + }, + ), + ButtonWidget( + labelText: "Clear selection", // TODO: lau: extract string + buttonType: ButtonType.secondary, + buttonSize: ButtonSize.large, + shouldStickToDarkTheme: true, + isInAlert: true, + buttonAction: ButtonAction.cancel, + shouldSurfaceExecutionStates: false, + onTap: () async { + _selectedFiles.clearAll(fireEvent: false); + }, + ), + ], + actionSheetType: ActionSheetType.defaultActionSheet, + ); + } + Widget _buildSimilarFilesGroup(SimilarFiles similarFiles) { final textTheme = getEnteTextTheme(context); return Padding( @@ -452,7 +623,7 @@ class _SimilarImagesPageState extends State { children: [ Text( "${similarFiles.files.length} similar images" + - (flagService.internalUser + (kDebugMode ? " (I: d: ${similarFiles.furthestDistance.toStringAsFixed(3)})" : ""), // TODO: lau: extract string style: textTheme.smallMuted.copyWith( @@ -469,30 +640,11 @@ class _SimilarImagesPageState extends State { .toSet(); final bool allFilesFromGroupSelected = groupSelectedFiles.length == similarFiles.length; - final hasAnySelection = _selectedFiles.files.isNotEmpty; - final allGroupFilesSelected = similarFiles.files.every( - (file) => _selectedFiles.isFileSelected(file), - ); - if (groupSelectedFiles.isNotEmpty) { return _getSmallDeleteButton( groupSelectedFiles, allFilesFromGroupSelected, ); - } else if (hasAnySelection) { - return _getSmallSelectButton(allGroupFilesSelected, () { - if (allGroupFilesSelected) { - // Unselect all files in this group - _selectedFiles.unSelectAll( - similarFiles.files.toSet(), - ); - } else { - // Select all files in this group - _selectedFiles.selectAll( - similarFiles.files.sublist(1).toSet(), - ); - } - }); } return const SizedBox.shrink(); }, @@ -784,35 +936,6 @@ class _SimilarImagesPageState extends State { await deleteFilesFromRemoteOnly(context, allDeleteFiles.toList()); } - Widget _getSmallSelectButton(bool unselectAll, void Function() onTap) { - final colorScheme = getEnteColorScheme(context); - final textTheme = getEnteTextTheme(context); - - return GestureDetector( - onTap: onTap, - child: Container( - padding: const EdgeInsets.symmetric( - horizontal: 8, - vertical: 8, - ), - decoration: BoxDecoration( - color: unselectAll - ? getEnteColorScheme(context).primary500 - : getEnteColorScheme(context).strokeFaint, - borderRadius: BorderRadius.circular(4), - ), - child: Text( - unselectAll - ? "Unselect all" // TODO: lau: extract string - : "Select extra", // TODO: lau: extract string - style: textTheme.smallMuted.copyWith( - color: unselectAll ? Colors.white : colorScheme.textMuted, - ), - ), - ), - ); - } - Widget _getSortMenu() { final textTheme = getEnteTextTheme(context); final colorScheme = getEnteColorScheme(context); @@ -823,10 +946,10 @@ class _SimilarImagesPageState extends State { text = "Size"; // TODO: lau: extract string break; case SortKey.distanceAsc: - text = "Distance ascending"; // TODO: lau: extract string + text = "Similarity (Desc.)"; // TODO: lau: extract string break; case SortKey.distanceDesc: - text = "Distance descending"; // TODO: lau: extract string + text = "Similarity (Asc.)"; // TODO: lau: extract string break; case SortKey.count: text = "Count"; // TODO: lau: extract string diff --git a/mobile/apps/photos/scripts/internal_changes.txt b/mobile/apps/photos/scripts/internal_changes.txt index 3c48d3392a..fa3b97c35a 100644 --- a/mobile/apps/photos/scripts/internal_changes.txt +++ b/mobile/apps/photos/scripts/internal_changes.txt @@ -1,5 +1,6 @@ - Neeraj: Fix for double enteries for local file - (prtk) Fix widget initial launch on iOS +- Similar images debug screen (Settings > Backup > Free up space > Similar images) - (prtk) Upgrade Flutter version to 3.32.8 - (prtk) Run FFMpeg in an isolate - Neeraj: Handle custom domain links diff --git a/mobile/apps/photos/scripts/store_changes.txt b/mobile/apps/photos/scripts/store_changes.txt index 1d6ce14986..f1baf26cae 100644 --- a/mobile/apps/photos/scripts/store_changes.txt +++ b/mobile/apps/photos/scripts/store_changes.txt @@ -1,4 +1,3 @@ -- Added similar images debug screen (Settings > Backup > Free up space > Similar images) - Added support for custom domain links - Image editor fixes: - Fixed bottom navigation bar color in light theme