TerribleDev/ente
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into remove-intl_utils

Browse Source
This commit is contained in:
Prateek Sunal
2025-08-21 18:08:31 +05:30
parent 3dc0620e18 fa555c448f
commit dd420a80a4
8 changed files with 820 additions and 329 deletions
Download Patch File Download Diff File Expand all files Collapse all files

129
mobile/apps/photos/lib/models/similar_files.dart
View File

@@ -1,18 +1,21 @@
import "dart:convert";
import "package:photos/models/file/file.dart";
import "package:photos/services/search_service.dart";
class SimilarFiles {
final List<EnteFile> files;
final Set<int> fileIds;
final double furthestDistance;
double furthestDistance;
SimilarFiles(
this.files,
this.furthestDistance,
) : fileIds = files.map((file) => file.uploadedFileID!).toSet();
) : fileIds = files.map((file) => file.uploadedFileID!).toSet();
int get totalSize =>
files.fold(0, (sum, file) => sum + (file.fileSize ?? 0));
int get totalSize => files.fold(0, (sum, file) => sum + (file.fileSize ?? 0));
// TODO: lau: check if we're not using this wrong
bool get isEmpty => files.isEmpty;
int get length => files.length;
@@ -26,7 +29,125 @@ class SimilarFiles {
fileIds.remove(file.uploadedFileID);
}
void addFile(EnteFile file) {
files.add(file);
fileIds.add(file.uploadedFileID!);
}
bool containsFile(EnteFile file) {
return fileIds.contains(file.uploadedFileID);
}
Map<String, dynamic> toJson() {
return {
'fileIDs': fileIds.toList(),
'distance': furthestDistance,
};
}
String toJsonString() {
return jsonEncode(toJson());
}
factory SimilarFiles.fromJson(
Map<String, dynamic> json,
Map<int, EnteFile> fileMap,
) {
final fileIds = List<int>.from(json['fileIDs']);
final furthestDistance = (json['distance'] as num).toDouble();
final files = <EnteFile>[];
for (final fileId in fileIds) {
final file = fileMap[fileId];
if (file == null) continue;
files.add(file);
}
return SimilarFiles(
files,
furthestDistance,
);
}
static SimilarFiles fromJsonString(
String jsonString,
Map<int, EnteFile> fileMap,
) {
return SimilarFiles.fromJson(jsonDecode(jsonString), fileMap);
}
}
class SimilarFilesCache {
final List<String> similarFilesJsonStringList;
final Set<int> allCheckedFileIDs;
final double distanceThreshold;
final bool exact;
List<SimilarFiles>? _similarFilesList;
/// Milliseconds since epoch
final int cachedTime;
SimilarFilesCache({
required this.similarFilesJsonStringList,
required this.allCheckedFileIDs,
required this.distanceThreshold,
required this.exact,
required this.cachedTime,
});
Future<List<SimilarFiles>> similarFilesList() async {
final allFiles = await SearchService.instance.getAllFilesForSearch();
final fileMap = <int, EnteFile>{};
for (final file in allFiles) {
if (file.uploadedFileID == null) continue;
fileMap[file.uploadedFileID!] = file;
}
_similarFilesList ??= similarFilesJsonStringList.map((jsonString) {
return SimilarFiles.fromJson(jsonDecode(jsonString), fileMap);
}).toList();
return _similarFilesList!;
}
Future<Set<int>> getGroupedFileIDs() async {
final similarFiles = await similarFilesList();
final groupedFileIDs = <int>{};
for (final files in similarFiles) {
groupedFileIDs.addAll(files.fileIds);
}
return groupedFileIDs;
}
factory SimilarFilesCache.fromJson(
Map<String, dynamic> json,
) {
return SimilarFilesCache(
similarFilesJsonStringList:
List<String>.from(json['similarFilesJsonStringList']),
allCheckedFileIDs: Set<int>.from(json['allCheckedFileIDs']),
distanceThreshold: (json['distanceThreshold'] as num).toDouble(),
exact: json['exact'] as bool,
cachedTime: json['cachedTime'] as int,
);
}
Map<String, dynamic> toJson() {
return {
'similarFilesJsonStringList': similarFilesJsonStringList,
'allCheckedFileIDs': allCheckedFileIDs.toList(),
'distanceThreshold': distanceThreshold,
'exact': exact,
'cachedTime': cachedTime,
};
}
static String encodeToJsonString(SimilarFilesCache cache) {
return jsonEncode(cache.toJson());
}
static SimilarFilesCache decodeFromJsonString(
String jsonString,
) {
return SimilarFilesCache.fromJson(jsonDecode(jsonString));
}
}

13
mobile/apps/photos/lib/services/deduplication_service.dart
View File

@@ -1,12 +1,12 @@
import 'package:logging/logging.dart';
import "package:photos/core/configuration.dart";
import 'package:photos/core/network/network.dart';
import 'package:photos/db/files_db.dart';
import 'package:photos/models/duplicate_files.dart';
import "package:photos/models/file/extensions/file_props.dart";
import 'package:photos/models/file/file.dart';
import "package:photos/models/file/file_type.dart";
import "package:photos/services/collections_service.dart";
import "package:photos/services/files_service.dart";
import "package:photos/services/search_service.dart";
class DeduplicationService {
final _logger = Logger("DeduplicationService");
@@ -39,16 +39,13 @@ class DeduplicationService {
final Set<int> allowedCollectionIDs =
CollectionsService.instance.nonHiddenOwnedCollections();
final List<EnteFile> allFiles = await FilesDB.instance.getAllFilesFromDB(
CollectionsService.instance.getHiddenCollectionIds(),
dedupeByUploadId: false,
);
final int ownerID = Configuration.instance.getUserID()!;
final List<EnteFile> allFiles =
await SearchService.instance.getAllFilesForSearch();
final List<EnteFile> filteredFiles = [];
for (final file in allFiles) {
if (!file.isUploaded ||
(file.hash ?? '').isEmpty ||
(file.ownerID ?? 0) != ownerID ||
!file.isOwner ||
(!allowedCollectionIDs.contains(file.collectionID!))) {
continue;
}

301
mobile/apps/photos/lib/services/machine_learning/similar_images_service.dart
View File

@@ -4,14 +4,16 @@ import "package:flutter/foundation.dart" show kDebugMode;
import "package:flutter_rust_bridge/flutter_rust_bridge_for_generated.dart"
show Uint64List;
import 'package:logging/logging.dart';
import "package:path_provider/path_provider.dart";
import "package:photos/db/ml/db.dart";
import "package:photos/extensions/stop_watch.dart";
import "package:photos/models/file/extensions/file_props.dart";
import 'package:photos/models/file/file.dart';
import "package:photos/models/file/file_type.dart";
import "package:photos/models/similar_files.dart";
import "package:photos/services/machine_learning/ml_computer.dart";
import "package:photos/services/machine_learning/ml_result.dart";
import "package:photos/services/search_service.dart";
import "package:photos/utils/cache_util.dart";
class SimilarImagesService {
final _logger = Logger("SimilarImagesService");
@@ -25,11 +27,12 @@ class SimilarImagesService {
Future<List<SimilarFiles>> getSimilarFiles(
double distanceThreshold, {
bool exact = false,
bool forceRefresh = false,
}) async {
try {
final now = DateTime.now();
final List<SimilarFiles> result =
await _getSimilarFiles(distanceThreshold, exact);
await _getSimilarFiles(distanceThreshold, exact, forceRefresh);
final duration = DateTime.now().difference(now);
_logger.info(
"Found ${result.length} similar files in ${duration.inSeconds} seconds for threshold $distanceThreshold and exact $exact",
@@ -44,6 +47,7 @@ class SimilarImagesService {
Future<List<SimilarFiles>> _getSimilarFiles(
double distanceThreshold,
bool exact,
bool forceRefresh,
) async {
final w = (kDebugMode ? EnteWatch('getSimilarFiles') : null)?..start();
final mlDataDB = MLDataDB.instance;
@@ -58,7 +62,7 @@ class SimilarImagesService {
final allFileIdsToFile = <int, EnteFile>{};
final fileIDs = <int>[];
for (final file in allFiles) {
if (file.uploadedFileID != null && file.fileType != FileType.video) {
if (file.uploadedFileID != null && file.isOwner && !file.isVideo) {
allFileIdsToFile[file.uploadedFileID!] = file;
fileIDs.add(file.uploadedFileID!);
}
@@ -84,6 +88,253 @@ class SimilarImagesService {
}
w?.log("getFileIDToPersonIDs");
if (forceRefresh) {
final result = await _performFullSearch(
potentialKeys,
allFileIdsToFile,
fileIDToPersonIDs,
distanceThreshold,
exact,
);
await _cacheSimilarFiles(
result,
fileIDs.toSet(),
distanceThreshold,
exact,
DateTime.now().millisecondsSinceEpoch,
);
return result;
}
// Load cached data
final SimilarFilesCache? cachedData = await _readCachedSimilarFiles();
if (cachedData == null) {
_logger.warning("No cached similar files found");
} else {
_logger.info(
"Cached similar files found with ${cachedData.similarFilesJsonStringList.length} groups",
);
}
// Determine if we need full refresh
bool needsFullRefresh = false;
if (cachedData != null) {
final Set<int> cachedFileIDs = cachedData.allCheckedFileIDs;
final currentFileIDs = fileIDs.toSet();
if (cachedData.distanceThreshold != distanceThreshold ||
cachedData.exact != exact) {
needsFullRefresh = true;
}
// Check condition: less than 1000 files
if (currentFileIDs.length < 1000) {
needsFullRefresh = true;
}
// Check condition: cache is older than a month
if (DateTime.fromMillisecondsSinceEpoch(cachedData.cachedTime)
.isBefore(DateTime.now().subtract(const Duration(days: 30)))) {
needsFullRefresh = true;
}
// Check condition: new files > 20% of total files
if (!needsFullRefresh) {
final newFileIDs = currentFileIDs.difference(cachedFileIDs);
if (newFileIDs.length > currentFileIDs.length * 0.2) {
needsFullRefresh = true;
}
}
// Check condition: 20+% of grouped files deleted
if (!needsFullRefresh) {
final Set<int> cacheGroupedFileIDs =
await cachedData.getGroupedFileIDs();
final deletedFromGroups = cacheGroupedFileIDs
.intersection(cachedFileIDs.difference(currentFileIDs));
final totalInGroups = cacheGroupedFileIDs.length;
if (totalInGroups > 0 &&
deletedFromGroups.length > totalInGroups * 0.2) {
needsFullRefresh = true;
}
}
}
if (cachedData == null || needsFullRefresh) {
final result = await _performFullSearch(
potentialKeys,
allFileIdsToFile,
fileIDToPersonIDs,
distanceThreshold,
exact,
);
await _cacheSimilarFiles(
result,
fileIDs.toSet(),
distanceThreshold,
exact,
DateTime.now().millisecondsSinceEpoch,
);
return result;
} else {
return await _performIncrementalUpdate(
cachedData,
potentialKeys,
allFileIdsToFile,
fileIDToPersonIDs,
distanceThreshold,
exact,
);
}
}
Future<List<SimilarFiles>> _performIncrementalUpdate(
SimilarFilesCache cachedData,
Uint64List currentFileIDs,
Map<int, EnteFile> allFileIdsToFile,
Map<int, Set<String>> fileIDToPersonIDs,
double distanceThreshold,
bool exact,
) async {
_logger.info("Performing incremental update for similar files");
final existingGroups = await cachedData.similarFilesList();
final cachedFileIDs = cachedData.allCheckedFileIDs;
final currentFileIDsSet = currentFileIDs.map((id) => id.toInt()).toSet();
final deletedFiles = cachedFileIDs.difference(currentFileIDsSet);
// Clean up deleted files from existing groups
if (deletedFiles.isNotEmpty) {
for (final group in existingGroups) {
final filesInGroupToDelete = [];
for (final fileInGroup in group.files) {
if (deletedFiles.contains(fileInGroup.uploadedFileID ?? -1)) {
filesInGroupToDelete.add(fileInGroup);
}
}
for (final fileToDelete in filesInGroupToDelete) {
group.removeFile(fileToDelete);
}
}
}
// Remove empty groups
existingGroups.removeWhere((group) => group.length <= 1);
// Identify new files
final newFileIDs = currentFileIDsSet.difference(cachedFileIDs);
if (newFileIDs.isEmpty) {
return existingGroups;
}
// Search only new files
final newFileIDsList = Uint64List.fromList(newFileIDs.toList());
final (keys, vectorKeys, distances) =
await MLComputer.instance.bulkVectorSearchWithKeys(
newFileIDsList,
exact,
);
final keysList = keys.map((key) => key.toInt()).toList();
// Try to assign new files to existing groups
final unassignedNewFilesIndices = <int>{};
final unassignedNewFileIDs = <int>{};
for (int i = 0; i < keysList.length; i++) {
final newFileID = keysList[i];
final newFile = allFileIdsToFile[newFileID];
if (newFile == null) continue;
final similarFileIDs = vectorKeys[i];
final fileDistances = distances[i];
final newFilePersonIDs = fileIDToPersonIDs[newFileID] ?? <String>{};
bool assigned = false;
for (int j = 0; j < similarFileIDs.length; j++) {
final otherFileID = similarFileIDs[j].toInt();
if (otherFileID == newFileID) continue;
final distance = fileDistances[j];
if (distance > distanceThreshold) break;
for (final group in existingGroups) {
if (group.fileIds.contains(otherFileID)) {
final otherPersonIDs = fileIDToPersonIDs[otherFileID] ?? <String>{};
if (setsAreEqual(newFilePersonIDs, otherPersonIDs)) {
group.addFile(newFile);
group.furthestDistance = max(group.furthestDistance, distance);
group.files.sort((a, b) {
final sizeComparison =
(b.fileSize ?? 0).compareTo(a.fileSize ?? 0);
if (sizeComparison != 0) return sizeComparison;
return a.displayName.compareTo(b.displayName);
});
assigned = true;
break;
}
}
}
if (assigned) break;
}
if (!assigned) {
unassignedNewFilesIndices.add(i);
unassignedNewFileIDs.add(newFileID);
}
}
// Check if unassigned new files form groups among themselves
if (unassignedNewFilesIndices.isNotEmpty) {
final alreadyUsedNewFiles = <int>{};
for (final searchIndex in unassignedNewFilesIndices) {
final newFileID = keysList[searchIndex];
if (alreadyUsedNewFiles.contains(newFileID)) continue;
final newFile = allFileIdsToFile[newFileID];
if (newFile == null) continue;
final similarFileIDs = vectorKeys[searchIndex];
final fileDistances = distances[searchIndex];
final newFilePersonIDs = fileIDToPersonIDs[newFileID] ?? <String>{};
final similarNewFiles = <EnteFile>[];
double furthestDistance = 0.0;
for (int j = 0; j < similarFileIDs.length; j++) {
final otherFileID = similarFileIDs[j].toInt();
if (otherFileID == newFileID) continue;
if (!unassignedNewFileIDs.contains(otherFileID)) continue;
if (alreadyUsedNewFiles.contains(otherFileID)) continue;
final distance = fileDistances[j];
if (distance > distanceThreshold) break;
final otherFile = allFileIdsToFile[otherFileID];
if (otherFile == null) continue;
final otherPersonIDs = fileIDToPersonIDs[otherFileID] ?? <String>{};
if (!setsAreEqual(newFilePersonIDs, otherPersonIDs)) continue;
similarNewFiles.add(otherFile);
alreadyUsedNewFiles.add(otherFileID);
furthestDistance = max(furthestDistance, distance);
}
if (similarNewFiles.isNotEmpty) {
similarNewFiles.add(newFile);
alreadyUsedNewFiles.add(newFileID);
similarNewFiles.sort((a, b) {
final sizeComparison = (b.fileSize ?? 0).compareTo(a.fileSize ?? 0);
if (sizeComparison != 0) return sizeComparison;
return a.displayName.compareTo(b.displayName);
});
existingGroups.add(SimilarFiles(similarNewFiles, furthestDistance));
}
}
}
await _cacheSimilarFiles(
existingGroups,
currentFileIDsSet,
distanceThreshold,
exact,
cachedData.cachedTime,
);
return existingGroups;
}
Future<List<SimilarFiles>> _performFullSearch(
Uint64List potentialKeys,
Map<int, EnteFile> allFileIdsToFile,
Map<int, Set<String>> fileIDToPersonIDs,
double distanceThreshold,
bool exact,
) async {
_logger.info("Performing full search for similar files");
final w = (kDebugMode ? EnteWatch('getSimilarFiles') : null)?..start();
// Run bulk vector search
final (keys, vectorKeys, distances) =
await MLComputer.instance.bulkVectorSearchWithKeys(
@@ -121,14 +372,18 @@ class SimilarImagesService {
if (!setsAreEqual(personIDs, otherPersonIDs)) continue;
similarFilesList.add(otherFile);
furthestDistance = max(furthestDistance, distance);
alreadyUsedFileIDs.add(otherFileID);
}
if (similarFilesList.isNotEmpty) {
similarFilesList.add(firstLoopFile);
for (final file in similarFilesList) {
alreadyUsedFileIDs.add(file.uploadedFileID!);
}
// show highest quality files first
similarFilesList.sort((a, b) {
return a.displayName.length.compareTo(b.displayName.length);
final sizeComparison = (b.fileSize ?? 0).compareTo(a.fileSize ?? 0);
if (sizeComparison != 0) return sizeComparison;
return a.displayName.compareTo(b.displayName);
});
final similarFiles = SimilarFiles(
similarFilesList,
@@ -141,6 +396,44 @@ class SimilarImagesService {
return allSimilarFiles;
}
Future<String> _getCachePath() async {
return (await getApplicationSupportDirectory()).path +
"/cache/similar_images_cache";
}
Future<void> _cacheSimilarFiles(
List<SimilarFiles> similarGroups,
Set<int> allCheckedFileIDs,
double distanceThreshold,
bool exact,
int cachedTimeOfOriginalComputation,
) async {
final cachePath = await _getCachePath();
final similarGroupsJsonStringList =
similarGroups.map((group) => group.toJsonString()).toList();
final cacheObject = SimilarFilesCache(
similarFilesJsonStringList: similarGroupsJsonStringList,
allCheckedFileIDs: allCheckedFileIDs,
distanceThreshold: distanceThreshold,
exact: exact,
cachedTime: cachedTimeOfOriginalComputation,
);
await writeToJsonFile<SimilarFilesCache>(
cachePath,
cacheObject,
SimilarFilesCache.encodeToJsonString,
);
}
Future<SimilarFilesCache?> _readCachedSimilarFiles() async {
_logger.info("Reading similar files cache result from disk");
final cache = decodeJsonFile<SimilarFilesCache>(
await _getCachePath(),
SimilarFilesCache.decodeFromJsonString,
);
return cache;
}
}
bool setsAreEqual(Set<String> set1, Set<String> set2) {

3
mobile/apps/photos/lib/ui/settings/backup/free_space_options.dart
View File

@@ -1,6 +1,7 @@
import "dart:async";
import "dart:io";
import "package:flutter/foundation.dart" show kDebugMode;
import 'package:flutter/material.dart';
import "package:photos/generated/l10n.dart";
import "package:photos/models/backup_status.dart";
@@ -212,7 +213,7 @@ class _FreeUpSpaceOptionsScreenState extends State<FreeUpSpaceOptionsScreen> {
onTap: () async {
await routeToPage(
context,
const SimilarImagesPage(),
const SimilarImagesPage(debugScreen: kDebugMode,),
);
},
),

296
mobile/apps/photos/lib/ui/tools/deduplicate_page.dart
View File

@@ -4,16 +4,19 @@ import 'package:flutter/material.dart';
import 'package:flutter/services.dart';
import 'package:photos/core/constants.dart';
import 'package:photos/core/event_bus.dart';
import 'package:photos/ente_theme_data.dart';
import 'package:photos/events/user_details_changed_event.dart';
import "package:photos/generated/l10n.dart";
import 'package:photos/models/duplicate_files.dart';
import 'package:photos/models/file/file.dart';
import 'package:photos/services/collections_service.dart';
import "package:photos/theme/ente_theme.dart";
import 'package:photos/ui/components/buttons/button_widget.dart';
import "package:photos/ui/components/models/button_type.dart";
import 'package:photos/ui/viewer/file/detail_page.dart';
import 'package:photos/ui/viewer/file/thumbnail_widget.dart';
import 'package:photos/ui/viewer/gallery/empty_state.dart';
import "package:photos/ui/viewer/gallery/scrollbar/scroll_bar_with_use_notifier.dart";
import 'package:photos/utils/delete_file_util.dart';
import "package:photos/utils/dialog_util.dart";
import 'package:photos/utils/navigation_util.dart';
@@ -31,7 +34,6 @@ class DeduplicatePage extends StatefulWidget {
class _DeduplicatePageState extends State<DeduplicatePage> {
static const crossAxisCount = 3;
static const crossAxisSpacing = 12.0;
static const headerRowCount = 3;
final Set<int> selectedGrids = <int>{};
@@ -39,11 +41,15 @@ class _DeduplicatePageState extends State<DeduplicatePage> {
SortKey sortKey = SortKey.size;
late ValueNotifier<String> _deleteProgress;
late ScrollController _scrollController;
late ValueNotifier<bool> _scrollbarInUseNotifier;
@override
void initState() {
_duplicates = widget.duplicates;
_deleteProgress = ValueNotifier("");
_scrollController = ScrollController();
_scrollbarInUseNotifier = ValueNotifier<bool>(false);
_selectAllGrids();
super.initState();
}
@@ -51,6 +57,8 @@ class _DeduplicatePageState extends State<DeduplicatePage> {
@override
void dispose() {
_deleteProgress.dispose();
_scrollController.dispose();
_scrollbarInUseNotifier.dispose();
super.dispose();
}
@@ -68,47 +76,7 @@ class _DeduplicatePageState extends State<DeduplicatePage> {
appBar: AppBar(
elevation: 0,
title: Text(AppLocalizations.of(context).deduplicateFiles),
actions: <Widget>[
PopupMenuButton(
constraints: const BoxConstraints(minWidth: 180),
shape: const RoundedRectangleBorder(
borderRadius: BorderRadius.all(
Radius.circular(8),
),
),
onSelected: (dynamic value) {
setState(() {
selectedGrids.clear();
});
},
offset: const Offset(0, 50),
itemBuilder: (BuildContext context) => [
PopupMenuItem(
value: true,
height: 32,
child: Row(
children: [
const Icon(
Icons.remove_circle_outline,
size: 20,
),
const SizedBox(width: 12),
Padding(
padding: const EdgeInsets.only(bottom: 1),
child: Text(
AppLocalizations.of(context).deselectAll,
style: Theme.of(context)
.textTheme
.titleMedium!
.copyWith(fontWeight: FontWeight.w600),
),
),
],
),
),
],
),
],
actions: _duplicates.isNotEmpty ? [_getSortMenu()] : null,
),
body: _getBody(),
);
@@ -133,33 +101,34 @@ class _DeduplicatePageState extends State<DeduplicatePage> {
crossAxisAlignment: CrossAxisAlignment.center,
children: [
Expanded(
child: ListView.builder(
itemBuilder: (context, index) {
if (index == 0) {
return const SizedBox.shrink();
} else if (index == 1) {
return const SizedBox.shrink();
} else if (index == 2) {
if (_duplicates.isNotEmpty) {
return _getSortMenu(context);
} else {
return const Padding(
padding: EdgeInsets.only(top: 32),
child: EmptyState(),
);
}
}
return Padding(
padding: const EdgeInsets.symmetric(vertical: 8),
child: _getGridView(
_duplicates[index - headerRowCount],
index - headerRowCount,
child: _duplicates.isNotEmpty
? ScrollbarWithUseNotifer(
controller: _scrollController,
inUseNotifier: _scrollbarInUseNotifier,
minScrollbarLength: 36.0,
interactive: true,
thickness: 8,
radius: const Radius.circular(4),
child: ListView.builder(
controller: _scrollController,
cacheExtent: 400,
itemBuilder: (context, index) {
return Padding(
padding: const EdgeInsets.symmetric(vertical: 8),
child: _getGridView(
_duplicates[index],
index,
),
);
},
itemCount: _duplicates.length,
shrinkWrap: true,
),
)
: const Padding(
padding: EdgeInsets.only(top: 32),
child: EmptyState(),
),
);
},
itemCount: _duplicates.length + headerRowCount,
shrinkWrap: true,
),
),
selectedGrids.isEmpty
? const SizedBox.shrink()
@@ -189,7 +158,8 @@ class _DeduplicatePageState extends State<DeduplicatePage> {
);
}
Widget _getSortMenu(BuildContext context) {
Widget _getSortMenu() {
final textTheme = getEnteTextTheme(context);
Text sortOptionText(SortKey key) {
String text = key.toString();
switch (key) {
@@ -202,63 +172,44 @@ class _DeduplicatePageState extends State<DeduplicatePage> {
}
return Text(
text,
style: Theme.of(context).textTheme.titleMedium!.copyWith(
fontSize: 14,
color: Theme.of(context).iconTheme.color!.withValues(alpha: 0.7),
),
style: textTheme.miniBold,
);
}
return Row(
// h4ck to align PopupMenuItems to end
mainAxisAlignment: MainAxisAlignment.spaceBetween,
crossAxisAlignment: CrossAxisAlignment.end,
children: [
const SizedBox.shrink(),
PopupMenuButton(
initialValue: sortKey.index,
child: Padding(
padding: const EdgeInsets.fromLTRB(24, 6, 24, 6),
child: Row(
mainAxisAlignment: MainAxisAlignment.start,
crossAxisAlignment: CrossAxisAlignment.center,
children: [
sortOptionText(sortKey),
const Padding(padding: EdgeInsets.only(left: 4)),
Icon(
Icons.sort,
color: Theme.of(context).colorScheme.iconColor,
size: 20,
),
],
),
),
onSelected: (int index) {
setState(() {
final newKey = SortKey.values[index];
if (newKey == sortKey) {
return;
} else {
sortKey = newKey;
if (selectedGrids.length != _duplicates.length) {
selectedGrids.clear();
}
}
});
},
itemBuilder: (context) {
return List.generate(SortKey.values.length, (index) {
return PopupMenuItem(
value: index,
child: Align(
alignment: Alignment.centerLeft,
child: sortOptionText(SortKey.values[index]),
),
);
});
},
return PopupMenuButton(
initialValue: sortKey.index,
child: Padding(
padding: const EdgeInsets.fromLTRB(24, 6, 24, 6),
child: Icon(
Icons.sort,
color: getEnteColorScheme(context).strokeBase,
size: 20,
),
],
),
onSelected: (int index) {
setState(() {
final newKey = SortKey.values[index];
if (newKey == sortKey) {
return;
} else {
sortKey = newKey;
if (selectedGrids.length != _duplicates.length) {
selectedGrids.clear();
}
}
});
},
itemBuilder: (context) {
return List.generate(SortKey.values.length, (index) {
return PopupMenuItem(
value: index,
child: Text(
sortOptionText(SortKey.values[index]).data!,
style: textTheme.miniBold,
),
);
});
},
);
}
@@ -272,55 +223,60 @@ class _DeduplicatePageState extends State<DeduplicatePage> {
totalSize += toDeleteCount * _duplicates[index].size;
}
}
final String text = AppLocalizations.of(context).deleteItemCount(fileCount);
return SizedBox(
width: double.infinity,
child: SafeArea(
child: Padding(
padding: const EdgeInsets.symmetric(horizontal: crossAxisSpacing),
child: TextButton(
style: OutlinedButton.styleFrom(
backgroundColor:
Theme.of(context).colorScheme.inverseBackgroundColor,
),
child: Column(
mainAxisAlignment: MainAxisAlignment.end,
children: [
const Padding(padding: EdgeInsets.all(4)),
Text(
text,
style: TextStyle(
fontWeight: FontWeight.bold,
fontSize: 14,
color: Theme.of(context).colorScheme.inverseTextColor,
),
textAlign: TextAlign.center,
final hasSelectedFiles = fileCount > 0;
return AnimatedSwitcher(
duration: const Duration(milliseconds: 200),
switchInCurve: Curves.easeOut,
switchOutCurve: Curves.easeIn,
child: hasSelectedFiles
? SafeArea(
child: Container(
key: const ValueKey('bottom_buttons'),
padding: const EdgeInsets.symmetric(
horizontal: crossAxisSpacing,
vertical: 8,
),
const Padding(padding: EdgeInsets.all(2)),
Text(
formatBytes(totalSize),
style: TextStyle(
color: Theme.of(context)
.colorScheme
.inverseTextColor
.withValues(alpha: 0.7),
fontSize: 12,
),
decoration: BoxDecoration(
color: getEnteColorScheme(context).backgroundBase,
),
const Padding(padding: EdgeInsets.all(2)),
],
),
onPressed: () async {
try {
await deleteDuplicates(totalSize);
} catch (e) {
log("Failed to delete duplicates", error: e);
showGenericErrorDialog(context: context, error: e).ignore();
}
},
),
),
),
child: Column(
children: [
SizedBox(
width: double.infinity,
child: ButtonWidget(
labelText:
"${AppLocalizations.of(context).deleteItemCount(fileCount)} (${formatBytes(totalSize)})",
buttonType: ButtonType.critical,
onTap: () async {
try {
await deleteDuplicates(totalSize);
} catch (e) {
log("Failed to delete duplicates", error: e);
showGenericErrorDialog(context: context, error: e)
.ignore();
}
},
),
),
const SizedBox(height: 8),
SizedBox(
width: double.infinity,
child: ButtonWidget(
labelText: "Unselect all", // TODO: lau: extract string
buttonType: ButtonType.secondary,
onTap: () async {
setState(() {
selectedGrids.clear();
});
},
),
),
],
),
),
)
: const SizedBox.shrink(key: ValueKey('empty')),
);
}

405
mobile/apps/photos/lib/ui/tools/similar_images_page.dart
View File

@@ -1,5 +1,6 @@
import "dart:async";
import "package:flutter/foundation.dart" show kDebugMode;
import 'package:flutter/material.dart';
import 'package:logging/logging.dart';
import 'package:photos/core/constants.dart';
@@ -13,11 +14,13 @@ import "package:photos/services/collections_service.dart";
import "package:photos/services/machine_learning/similar_images_service.dart";
import 'package:photos/theme/ente_theme.dart';
import "package:photos/ui/common/loading_widget.dart";
import 'package:photos/ui/components/action_sheet_widget.dart';
import 'package:photos/ui/components/buttons/button_widget.dart';
import "package:photos/ui/components/models/button_type.dart";
import "package:photos/ui/components/toggle_switch_widget.dart";
import "package:photos/ui/viewer/file/detail_page.dart";
import "package:photos/ui/viewer/file/thumbnail_widget.dart";
import "package:photos/ui/viewer/gallery/scrollbar/scroll_bar_with_use_notifier.dart";
import "package:photos/utils/delete_file_util.dart";
import "package:photos/utils/dialog_util.dart";
import "package:photos/utils/navigation_util.dart";
@@ -37,7 +40,9 @@ enum SortKey {
}
class SimilarImagesPage extends StatefulWidget {
const SimilarImagesPage({super.key});
final bool debugScreen;
const SimilarImagesPage({super.key, this.debugScreen = false});
@override
State<SimilarImagesPage> createState() => _SimilarImagesPageState();
@@ -46,7 +51,6 @@ class SimilarImagesPage extends StatefulWidget {
class _SimilarImagesPageState extends State<SimilarImagesPage> {
static const crossAxisCount = 3;
static const crossAxisSpacing = 12.0;
static const autoSelectDistanceThreshold = 0.01;
final _logger = Logger("SimilarImagesPage");
bool _isDisposed = false;
@@ -56,19 +60,31 @@ class _SimilarImagesPageState extends State<SimilarImagesPage> {
List<SimilarFiles> _similarFilesList = [];
SortKey _sortKey = SortKey.distanceAsc;
bool _exactSearch = false;
bool _fullRefresh = false;
bool _isSelectionSheetOpen = false;
late SelectedFiles _selectedFiles;
late ScrollController _scrollController;
late ValueNotifier<bool> _scrollbarInUseNotifier;
@override
void initState() {
super.initState();
_selectedFiles = SelectedFiles();
_scrollController = ScrollController();
_scrollbarInUseNotifier = ValueNotifier<bool>(false);
if (!widget.debugScreen) {
_findSimilarImages();
}
}
@override
void dispose() {
_isDisposed = true;
_selectedFiles.dispose();
_scrollController.dispose();
_scrollbarInUseNotifier.dispose();
super.dispose();
}
@@ -171,7 +187,7 @@ class _SimilarImagesPageState extends State<SimilarImagesPage> {
mainAxisAlignment: MainAxisAlignment.spaceBetween,
children: [
Text(
"Exact search", // TODO: lau: extract string
"Exact search",
style: textTheme.bodyBold,
),
ToggleSwitchWidget(
@@ -185,6 +201,25 @@ class _SimilarImagesPageState extends State<SimilarImagesPage> {
),
],
),
const SizedBox(height: 12),
Row(
mainAxisAlignment: MainAxisAlignment.spaceBetween,
children: [
Text(
"Full refresh",
style: textTheme.bodyBold,
),
ToggleSwitchWidget(
value: () => _fullRefresh,
onChanged: () async {
if (_isDisposed) return;
setState(() {
_fullRefresh = !_fullRefresh;
});
},
),
],
),
const SizedBox(height: 32),
ButtonWidget(
labelText: "Find similar images", // TODO: lau: extract string
@@ -253,37 +288,63 @@ class _SimilarImagesPageState extends State<SimilarImagesPage> {
return Column(
children: [
Expanded(
child: ListView.builder(
itemCount: _similarFilesList.length + 1, // +1 for header
itemBuilder: (context, index) {
if (index == 0) {
// Header item
if (flagService.internalUser) {
child: ScrollbarWithUseNotifer(
controller: _scrollController,
inUseNotifier: _scrollbarInUseNotifier,
minScrollbarLength: 36.0,
interactive: true,
thickness: 8,
radius: const Radius.circular(4),
child: ListView.builder(
controller: _scrollController,
cacheExtent: 400,
itemCount: _similarFilesList.length + 1, // +1 for header
itemBuilder: (context, index) {
if (index == 0) {
return Container(
margin: const EdgeInsets.symmetric(
horizontal: 16,
vertical: 12,
),
padding: const EdgeInsets.all(16),
child: Column(
decoration: BoxDecoration(
color: colorScheme.fillFaint,
borderRadius: BorderRadius.circular(8),
),
child: Row(
children: [
Text(
"(I) Found ${_similarFilesList.length} groups of similar images", // TODO: lau: extract string
style: textTheme.bodyBold,
Icon(
Icons.photo_library_outlined,
size: 20,
color: colorScheme.textMuted,
),
const SizedBox(height: 4),
Text(
"(I) Threshold: ${_distanceThreshold.toStringAsFixed(2)}", // TODO: lau: extract string
style: textTheme.miniMuted,
const SizedBox(width: 12),
Expanded(
child: Column(
crossAxisAlignment: CrossAxisAlignment.start,
children: [
Text(
"${_similarFilesList.length} ${_similarFilesList.length == 1 ? 'group' : 'groups'} found", // TODO: lau: extract string
style: textTheme.bodyBold,
),
const SizedBox(height: 4),
Text(
"Review and remove similar images", // TODO: lau: extract string
style: textTheme.miniMuted,
),
],
),
),
],
),
);
} else {
return const SizedBox.shrink();
}
}
// Similar files groups (index - 1 because first item is header)
final similarFiles = _similarFilesList[index - 1];
return _buildSimilarFilesGroup(similarFiles);
},
// Similar files groups (index - 1 because first item is header)
final similarFiles = _similarFilesList[index - 1];
return _buildSimilarFilesGroup(similarFiles);
},
),
),
),
_getBottomActionButtons(),
@@ -298,54 +359,70 @@ class _SimilarImagesPageState extends State<SimilarImagesPage> {
final selectedCount = _selectedFiles.files.length;
final hasSelectedFiles = selectedCount > 0;
return AnimatedSwitcher(
duration: const Duration(milliseconds: 200),
switchInCurve: Curves.easeOut,
switchOutCurve: Curves.easeIn,
child: hasSelectedFiles
? SafeArea(
child: Container(
key: const ValueKey('bottom_buttons'),
padding: const EdgeInsets.symmetric(
horizontal: crossAxisSpacing,
vertical: 8,
int totalSize = 0;
for (final file in _selectedFiles.files) {
totalSize += file.fileSize ?? 0;
}
return SafeArea(
child: Container(
padding: const EdgeInsets.symmetric(
horizontal: crossAxisSpacing,
vertical: 8,
),
decoration: BoxDecoration(
color: getEnteColorScheme(context).backgroundBase,
),
child: AnimatedSwitcher(
duration: Duration.zero,
child: Column(
key: ValueKey(hasSelectedFiles),
children: [
if (hasSelectedFiles && !_isSelectionSheetOpen) ...[
SizedBox(
width: double.infinity,
child: ButtonWidget(
labelText:
"Delete $selectedCount photos (${formatBytes(totalSize)})", // TODO: lau: extract string
buttonType: ButtonType.critical,
shouldSurfaceExecutionStates: false,
shouldShowSuccessConfirmation: false,
onTap: () async {
await _deleteFiles(
_selectedFiles.files,
showDialog: true,
);
},
),
),
decoration: BoxDecoration(
color: getEnteColorScheme(context).backgroundBase,
const SizedBox(height: 8),
],
if (!_isSelectionSheetOpen)
SizedBox(
width: double.infinity,
child: ButtonWidget(
labelText:
"Selection options", // TODO: lau: extract string
buttonType: ButtonType.secondary,
shouldSurfaceExecutionStates: false,
shouldShowSuccessConfirmation: false,
onTap: () async {
setState(() {
_isSelectionSheetOpen = true;
});
await _showSelectionOptionsSheet();
if (mounted) {
setState(() {
_isSelectionSheetOpen = false;
});
}
},
),
),
child: Column(
children: [
SizedBox(
width: double.infinity,
child: ButtonWidget(
labelText:
"Delete $selectedCount photos", // TODO: lau: extract string
buttonType: ButtonType.critical,
onTap: () async {
await _deleteFiles(
_selectedFiles.files,
showDialog: true,
);
},
),
),
const SizedBox(height: 8),
SizedBox(
width: double.infinity,
child: ButtonWidget(
labelText:
"Unselect all", // TODO: lau: extract string
buttonType: ButtonType.secondary,
onTap: () async {
_selectedFiles.clearAll(fireEvent: false);
},
),
),
],
),
),
)
: const SizedBox.shrink(key: ValueKey('empty')),
],
),
),
),
);
},
);
@@ -361,8 +438,11 @@ class _SimilarImagesPageState extends State<SimilarImagesPage> {
// You can use _toggleValue here for advanced mode features
_logger.info("exact mode: $_exactSearch");
final similarFiles = await SimilarImagesService.instance
.getSimilarFiles(_distanceThreshold, exact: _exactSearch);
final similarFiles = await SimilarImagesService.instance.getSimilarFiles(
_distanceThreshold,
exact: _exactSearch,
forceRefresh: _fullRefresh,
);
_logger.info(
"Found ${similarFiles.length} groups of similar images",
);
@@ -370,7 +450,6 @@ class _SimilarImagesPageState extends State<SimilarImagesPage> {
_similarFilesList = similarFiles;
_pageState = SimilarImagesPageState.results;
_sortSimilarFiles();
_autoSelectSimilarFiles();
if (_isDisposed) return;
setState(() {});
@@ -396,12 +475,24 @@ class _SimilarImagesPageState extends State<SimilarImagesPage> {
_similarFilesList.sort((a, b) => b.totalSize.compareTo(a.totalSize));
break;
case SortKey.distanceAsc:
_similarFilesList
.sort((a, b) => a.furthestDistance.compareTo(b.furthestDistance));
_similarFilesList.sort((a, b) {
final distanceComparison =
a.furthestDistance.compareTo(b.furthestDistance);
if (distanceComparison != 0) {
return distanceComparison;
}
return b.totalSize.compareTo(a.totalSize);
});
break;
case SortKey.distanceDesc:
_similarFilesList
.sort((a, b) => b.furthestDistance.compareTo(a.furthestDistance));
_similarFilesList.sort((a, b) {
final distanceComparison =
b.furthestDistance.compareTo(a.furthestDistance);
if (distanceComparison != 0) {
return distanceComparison;
}
return b.totalSize.compareTo(a.totalSize);
});
break;
case SortKey.count:
_similarFilesList
@@ -412,16 +503,11 @@ class _SimilarImagesPageState extends State<SimilarImagesPage> {
setState(() {});
}
void _autoSelectSimilarFiles() {
void _selectFilesByThreshold(double threshold) {
final filesToSelect = <EnteFile>{};
int groupsProcessed = 0;
int groupsAutoSelected = 0;
for (final similarFilesGroup in _similarFilesList) {
groupsProcessed++;
if (similarFilesGroup.furthestDistance < autoSelectDistanceThreshold) {
groupsAutoSelected++;
// Skip the first file (keep it unselected) and select the rest
if (similarFilesGroup.furthestDistance <= threshold) {
for (int i = 1; i < similarFilesGroup.files.length; i++) {
filesToSelect.add(similarFilesGroup.files[i]);
}
@@ -429,17 +515,102 @@ class _SimilarImagesPageState extends State<SimilarImagesPage> {
}
if (filesToSelect.isNotEmpty) {
_selectedFiles.clearAll(fireEvent: false);
_selectedFiles.selectAll(filesToSelect);
_logger.info(
"Auto-selected ${filesToSelect.length} files from $groupsAutoSelected/$groupsProcessed groups (threshold: $autoSelectDistanceThreshold)",
);
} else {
_logger.info(
"No files auto-selected from $groupsProcessed groups (threshold: $autoSelectDistanceThreshold)",
);
_selectedFiles.clearAll(fireEvent: false);
}
}
Future<void> _showSelectionOptionsSheet() async {
// Calculate how many files fall into each category
int exactFiles = 0;
int similarFiles = 0;
int allFiles = 0;
for (final group in _similarFilesList) {
final duplicateCount = group.files.length - 1; // Exclude the first file
allFiles += duplicateCount;
if (group.furthestDistance <= 0.0) {
exactFiles += duplicateCount;
similarFiles += duplicateCount;
} else if (group.furthestDistance <= 0.02) {
similarFiles += duplicateCount;
}
}
final String exactLabel = exactFiles > 0
? "Select exact ($exactFiles)" // TODO: lau: extract string
: "Select exact"; // TODO: lau: extract string
final String similarLabel = similarFiles > 0
? "Select similar ($similarFiles)" // TODO: lau: extract string
: "Select similar"; // TODO: lau: extract string
final String allLabel = allFiles > 0
? "Select all ($allFiles)" // TODO: lau: extract string
: "Select all"; // TODO: lau: extract string
await showActionSheet(
context: context,
title: "Select similar images", // TODO: lau: extract string
body:
"Choose which similar images to select for deletion", // TODO: lau: extract string
buttons: [
ButtonWidget(
labelText: exactLabel,
buttonType: ButtonType.neutral,
buttonSize: ButtonSize.large,
shouldStickToDarkTheme: true,
isInAlert: true,
buttonAction: ButtonAction.first,
shouldSurfaceExecutionStates: false,
onTap: () async {
_selectFilesByThreshold(0.0);
},
),
ButtonWidget(
labelText: similarLabel,
buttonType: ButtonType.neutral,
buttonSize: ButtonSize.large,
shouldStickToDarkTheme: true,
isInAlert: true,
buttonAction: ButtonAction.second,
shouldSurfaceExecutionStates: false,
onTap: () async {
_selectFilesByThreshold(0.02);
},
),
ButtonWidget(
labelText: allLabel,
buttonType: ButtonType.neutral,
buttonSize: ButtonSize.large,
shouldStickToDarkTheme: true,
isInAlert: true,
buttonAction: ButtonAction.third,
shouldSurfaceExecutionStates: false,
onTap: () async {
_selectFilesByThreshold(0.05);
},
),
ButtonWidget(
labelText: "Clear selection", // TODO: lau: extract string
buttonType: ButtonType.secondary,
buttonSize: ButtonSize.large,
shouldStickToDarkTheme: true,
isInAlert: true,
buttonAction: ButtonAction.cancel,
shouldSurfaceExecutionStates: false,
onTap: () async {
_selectedFiles.clearAll(fireEvent: false);
},
),
],
actionSheetType: ActionSheetType.defaultActionSheet,
);
}
Widget _buildSimilarFilesGroup(SimilarFiles similarFiles) {
final textTheme = getEnteTextTheme(context);
return Padding(
@@ -452,7 +623,7 @@ class _SimilarImagesPageState extends State<SimilarImagesPage> {
children: [
Text(
"${similarFiles.files.length} similar images" +
(flagService.internalUser
(kDebugMode
? " (I: d: ${similarFiles.furthestDistance.toStringAsFixed(3)})"
: ""), // TODO: lau: extract string
style: textTheme.smallMuted.copyWith(
@@ -469,30 +640,11 @@ class _SimilarImagesPageState extends State<SimilarImagesPage> {
.toSet();
final bool allFilesFromGroupSelected =
groupSelectedFiles.length == similarFiles.length;
final hasAnySelection = _selectedFiles.files.isNotEmpty;
final allGroupFilesSelected = similarFiles.files.every(
(file) => _selectedFiles.isFileSelected(file),
);
if (groupSelectedFiles.isNotEmpty) {
return _getSmallDeleteButton(
groupSelectedFiles,
allFilesFromGroupSelected,
);
} else if (hasAnySelection) {
return _getSmallSelectButton(allGroupFilesSelected, () {
if (allGroupFilesSelected) {
// Unselect all files in this group
_selectedFiles.unSelectAll(
similarFiles.files.toSet(),
);
} else {
// Select all files in this group
_selectedFiles.selectAll(
similarFiles.files.sublist(1).toSet(),
);
}
});
}
return const SizedBox.shrink();
},
@@ -784,35 +936,6 @@ class _SimilarImagesPageState extends State<SimilarImagesPage> {
await deleteFilesFromRemoteOnly(context, allDeleteFiles.toList());
}
Widget _getSmallSelectButton(bool unselectAll, void Function() onTap) {
final colorScheme = getEnteColorScheme(context);
final textTheme = getEnteTextTheme(context);
return GestureDetector(
onTap: onTap,
child: Container(
padding: const EdgeInsets.symmetric(
horizontal: 8,
vertical: 8,
),
decoration: BoxDecoration(
color: unselectAll
? getEnteColorScheme(context).primary500
: getEnteColorScheme(context).strokeFaint,
borderRadius: BorderRadius.circular(4),
),
child: Text(
unselectAll
? "Unselect all" // TODO: lau: extract string
: "Select extra", // TODO: lau: extract string
style: textTheme.smallMuted.copyWith(
color: unselectAll ? Colors.white : colorScheme.textMuted,
),
),
),
);
}
Widget _getSortMenu() {
final textTheme = getEnteTextTheme(context);
final colorScheme = getEnteColorScheme(context);
@@ -823,10 +946,10 @@ class _SimilarImagesPageState extends State<SimilarImagesPage> {
text = "Size"; // TODO: lau: extract string
break;
case SortKey.distanceAsc:
text = "Distance ascending"; // TODO: lau: extract string
text = "Similarity (Desc.)"; // TODO: lau: extract string
break;
case SortKey.distanceDesc:
text = "Distance descending"; // TODO: lau: extract string
text = "Similarity (Asc.)"; // TODO: lau: extract string
break;
case SortKey.count:
text = "Count"; // TODO: lau: extract string

1
mobile/apps/photos/scripts/internal_changes.txt
View File

@@ -1,5 +1,6 @@
- Neeraj: Fix for double enteries for local file
- (prtk) Fix widget initial launch on iOS
- Similar images debug screen (Settings > Backup > Free up space > Similar images)
- (prtk) Upgrade Flutter version to 3.32.8
- (prtk) Run FFMpeg in an isolate
- Neeraj: Handle custom domain links

1
mobile/apps/photos/scripts/store_changes.txt
View File

@@ -1,4 +1,3 @@
- Added similar images debug screen (Settings > Backup > Free up space > Similar images)
- Added support for custom domain links
- Image editor fixes:
- Fixed bottom navigation bar color in light theme