diff --git a/mobile/apps/auth/assets/custom-icons/_data/custom-icons.json b/mobile/apps/auth/assets/custom-icons/_data/custom-icons.json index 129e14ca96..266cda8fa1 100644 --- a/mobile/apps/auth/assets/custom-icons/_data/custom-icons.json +++ b/mobile/apps/auth/assets/custom-icons/_data/custom-icons.json @@ -10,6 +10,16 @@ { "title": "3Commas" }, + { + "title": "Accredible", + "slug": "accredible", + "altNames": [ + "Accredible Certificates", + "Accredible Badges", + "Digital Credentials", + "certificates.zaka.ai" + ] + }, { "title": "Addy.io", "slug": "addy_io" @@ -71,6 +81,13 @@ ], "hex": "fd4b2d" }, + { + "title": "Autenticacion Digital", + "slug": "autenticacion-digital", + "altNames": [ + "autenticaciondigital.and.gov.co" + ] + }, { "title": "availity" }, @@ -279,6 +296,13 @@ "title": "CERN", "slug": "cern" }, + { + "title": "Chaturbate", + "slug": "chaturbate", + "altNames": [ + "Chaturbate.com" + ] + }, { "title": "ChangeNOW" }, @@ -436,7 +460,7 @@ "title": "emeritihealth", "altNames": [ "Emeriti Health", - "Emeriti Retirement Health", + "Emeriti Retirement Health" ] }, { @@ -742,6 +766,15 @@ { "title": "Letterboxd" }, + { + "title": "LifeMiles", + "slug": "lifemiles", + "altNames": [ + "Life Miles", + "lifemiles.com", + "Avianca LifeMiles" + ] + }, { "title": "lincolnfinancial", "altNames": [ @@ -1295,6 +1328,19 @@ "PAYDAY 3" ] }, + { + "title": "Startmail", + "slug": "startmail" + }, + { + "title": "Stripchat", + "slug": "stripchat", + "altNames": [ + "Strip Chat", + "stripchat.com", + "StripChat Live" + ] + }, { "title": "STRATO", "hex": "FF8800" @@ -1313,6 +1359,9 @@ "T-Mobile ID" ] }, + { + "title": "Tableau" + }, { "title": "TCPShield" }, @@ -1522,6 +1571,12 @@ { "title": "WYZE" }, + { + "title": "X", + "altNames": [ + "Twitter" + ] + }, { "title": "Xbox", "hex": "107C10" @@ -1577,6 +1632,14 @@ "title": "xAI", "slug": "xai" }, + { + "title": "XVideos", + "slug": "xvideos", + "altNames": [ + "X Videos", + "xvideos.com" + ] + }, { "title": "Cronometer", "slug": "cronometer" diff --git a/mobile/apps/auth/assets/custom-icons/icons/accredible.svg b/mobile/apps/auth/assets/custom-icons/icons/accredible.svg new file mode 100644 index 0000000000..707dbf482d --- /dev/null +++ b/mobile/apps/auth/assets/custom-icons/icons/accredible.svg @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/mobile/apps/auth/assets/custom-icons/icons/auth_digital.svg b/mobile/apps/auth/assets/custom-icons/icons/auth_digital.svg new file mode 100644 index 0000000000..173f12977c --- /dev/null +++ b/mobile/apps/auth/assets/custom-icons/icons/auth_digital.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/mobile/apps/auth/assets/custom-icons/icons/chaturbate.svg b/mobile/apps/auth/assets/custom-icons/icons/chaturbate.svg new file mode 100644 index 0000000000..2c6992ebe7 --- /dev/null +++ b/mobile/apps/auth/assets/custom-icons/icons/chaturbate.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/mobile/apps/auth/assets/custom-icons/icons/lifemiles.svg b/mobile/apps/auth/assets/custom-icons/icons/lifemiles.svg new file mode 100644 index 0000000000..b08d6f0107 --- /dev/null +++ b/mobile/apps/auth/assets/custom-icons/icons/lifemiles.svg @@ -0,0 +1,101 @@ + + + +image/svg+xml + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/mobile/apps/auth/assets/custom-icons/icons/startmail.svg b/mobile/apps/auth/assets/custom-icons/icons/startmail.svg new file mode 100755 index 0000000000..3df4bc496b --- /dev/null +++ b/mobile/apps/auth/assets/custom-icons/icons/startmail.svg @@ -0,0 +1,15 @@ + + + + + + + + + diff --git a/mobile/apps/auth/assets/custom-icons/icons/stripchat.svg b/mobile/apps/auth/assets/custom-icons/icons/stripchat.svg new file mode 100644 index 0000000000..4a338ee588 --- /dev/null +++ b/mobile/apps/auth/assets/custom-icons/icons/stripchat.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/mobile/apps/auth/assets/custom-icons/icons/tableau.svg b/mobile/apps/auth/assets/custom-icons/icons/tableau.svg new file mode 100644 index 0000000000..361bb578ac --- /dev/null +++ b/mobile/apps/auth/assets/custom-icons/icons/tableau.svg @@ -0,0 +1,52 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/mobile/apps/auth/assets/custom-icons/icons/x.svg b/mobile/apps/auth/assets/custom-icons/icons/x.svg new file mode 100644 index 0000000000..d6464cc0bf --- /dev/null +++ b/mobile/apps/auth/assets/custom-icons/icons/x.svg @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/mobile/apps/auth/assets/custom-icons/icons/xvideos.svg b/mobile/apps/auth/assets/custom-icons/icons/xvideos.svg new file mode 100644 index 0000000000..91ccdbba27 --- /dev/null +++ b/mobile/apps/auth/assets/custom-icons/icons/xvideos.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/mobile/apps/photos/lib/db/files_db.dart b/mobile/apps/photos/lib/db/files_db.dart index f8900f8912..1957f55179 100644 --- a/mobile/apps/photos/lib/db/files_db.dart +++ b/mobile/apps/photos/lib/db/files_db.dart @@ -979,6 +979,55 @@ class FilesDB with SqlDbBase { return result; } + // remove references for local files which are either already uploaded + // or queued for upload but not yet uploaded + Future removeQueuedLocalFiles(Set localIDs) async { + if (localIDs.isEmpty) { + _logger.finest("No local IDs provided for removal"); + return 0; + } + + final db = await instance.sqliteAsyncDB; + const batchSize = 10000; + int totalRemoved = 0; + + final localIDsList = localIDs.toList(); + + for (int i = 0; i < localIDsList.length; i += batchSize) { + final endIndex = (i + batchSize > localIDsList.length) + ? localIDsList.length + : i + batchSize; + + final batch = localIDsList.sublist(i, endIndex); + final placeholders = List.filled(batch.length, '?').join(','); + + final r = await db.execute( + ''' + DELETE FROM $filesTable + WHERE $columnLocalID IN ($placeholders) + AND ($columnCollectionID IS NULL OR $columnCollectionID = -1) + AND ($columnUploadedFileID IS NULL OR $columnUploadedFileID = -1) + ''', + batch, + ); + + if (r.isNotEmpty) { + _logger + .fine("Batch ${(i ~/ batchSize) + 1}: Removed ${r.length} files"); + totalRemoved += r.length; + } + } + + if (totalRemoved > 0) { + _logger.warning( + "Removed $totalRemoved potential dups for already queued local files", + ); + } else { + _logger.finest("No duplicate id found for queued/uploaded files"); + } + return totalRemoved; + } + Future> getLocalFileIDsForCollection(int collectionID) async { final db = await instance.sqliteAsyncDB; final rows = await db.getAll( diff --git a/mobile/apps/photos/lib/db/ml/db.dart b/mobile/apps/photos/lib/db/ml/db.dart index 851e349d00..bc0913f8fe 100644 --- a/mobile/apps/photos/lib/db/ml/db.dart +++ b/mobile/apps/photos/lib/db/ml/db.dart @@ -383,8 +383,12 @@ class MLDataDB with SqlDbBase implements IMLDataDB { } } if (personID == null && clusterID == null) { + _logger.severe("personID and clusterID cannot be null both"); throw Exception("personID and clusterID cannot be null"); } + _logger.severe( + "Something went wrong finding a face from `getCoverFaceForPerson` (personID: $personID, clusterID: $clusterID)", + ); return null; } diff --git a/mobile/apps/photos/lib/services/collections_service.dart b/mobile/apps/photos/lib/services/collections_service.dart index ee3d662a63..86ed68924b 100644 --- a/mobile/apps/photos/lib/services/collections_service.dart +++ b/mobile/apps/photos/lib/services/collections_service.dart @@ -1427,12 +1427,20 @@ class CollectionsService { } // group files by collectionID final Map> filesByCollection = {}; + final Map> fileSeenByCollection = {}; for (final file in filesToCopy) { - if (filesByCollection.containsKey(file.collectionID!)) { - filesByCollection[file.collectionID!]!.add(file.copyWith()); - } else { - filesByCollection[file.collectionID!] = [file.copyWith()]; + fileSeenByCollection.putIfAbsent(file.collectionID!, () => {}); + if (fileSeenByCollection[file.collectionID]! + .contains(file.uploadedFileID)) { + _logger.warning( + "skip copy, duplicate ID: ${file.uploadedFileID} in collection " + "${file.collectionID}", + ); + continue; } + filesByCollection + .putIfAbsent(file.collectionID!, () => []) + .add(file.copyWith()); } for (final entry in filesByCollection.entries) { final srcCollectionID = entry.key; @@ -1579,9 +1587,6 @@ class CollectionsService { params["files"] = []; for (final batchFile in batch) { final fileKey = getFileKey(batchFile); - _logger.info( - "srcCollection : $srcCollectionID file: ${batchFile.uploadedFileID} key: ${CryptoUtil.bin2base64(fileKey)} ", - ); final encryptedKeyData = CryptoUtil.encryptSync(fileKey, getCollectionKey(dstCollectionID)); batchFile.encryptedKey = @@ -1643,17 +1648,27 @@ class CollectionsService { ); final List filesToCopy = []; final List filesToAdd = []; + final Set seenForAdd = {}; + final Set seenForCopy = {}; + for (final EnteFile file in othersFile) { - if (hashToUserFile.containsKey(file.hash ?? '')) { - final userFile = hashToUserFile[file.hash]!; - if (userFile.fileType == file.fileType) { - filesToAdd.add(userFile); - } else { - filesToCopy.add(file); - } - } else { - filesToCopy.add(file); + final userFile = hashToUserFile[file.hash ?? '']; + final bool shouldAdd = + userFile != null && userFile.fileType == file.fileType; + final targetList = shouldAdd ? filesToAdd : filesToCopy; + final seenSet = shouldAdd ? seenForAdd : seenForCopy; + final fileToProcess = shouldAdd ? userFile : file; + final uploadID = fileToProcess.uploadedFileID; + + if (seenSet.contains(uploadID)) { + final action = shouldAdd ? "adding" : "copying"; + _logger.warning( + "skip $action file $uploadID as it is already ${action}ed", + ); + continue; } + targetList.add(fileToProcess); + seenSet.add(uploadID!); } return (filesToAdd, filesToCopy); } diff --git a/mobile/apps/photos/lib/services/date_parse_service.dart b/mobile/apps/photos/lib/services/date_parse_service.dart new file mode 100644 index 0000000000..ef136d25ed --- /dev/null +++ b/mobile/apps/photos/lib/services/date_parse_service.dart @@ -0,0 +1,365 @@ +import 'dart:collection'; + +class PartialDate { + final int? day; + final int? month; + final int? year; + + const PartialDate({this.day, this.month, this.year}); + + static const empty = PartialDate(); + + bool get isEmpty => day == null && month == null && year == null; + + @override + bool operator ==(Object other) => + identical(this, other) || + other is PartialDate && + runtimeType == other.runtimeType && + day == other.day && + month == other.month && + year == other.year; + + @override + int get hashCode => day.hashCode ^ month.hashCode ^ year.hashCode; + + @override + String toString() { + return 'PartialDate(day: $day, month: $month, year: $year)'; + } +} + +class DateParseService { + static final DateParseService instance = DateParseService._private(); + DateParseService._private(); + + static const int _MIN_YEAR = 1900; + static const int _MAX_YEAR = 2100; + static const int _TWO_DIGIT_YEAR_PIVOT = 50; + + static final _ordinalRegex = RegExp(r'\b(\d{1,2})(st|nd|rd|th)\b'); + static final _normalizeRegex = RegExp(r'\bof\b|[,\.]+|\s+'); + static final _isoFormatRegex = + RegExp(r'^(\d{4})[\/-](\d{1,2})[\/-](\d{1,2})$'); + static final _standardFormatRegex = + RegExp(r'^(\d{1,2})[\/-](\d{1,2})[\/-](\d{2,4})$'); + static final _dotFormatRegex = RegExp(r'^(\d{1,2})\.(\d{1,2})\.(\d{2,4})$'); + static final _compactFormatRegex = RegExp(r'^(\d{8})$'); + static final _yearOnlyRegex = RegExp(r'^\s*(\d{4})\s*$'); + static final _shortFormatRegex = RegExp(r'^(\d{1,2})[\/-](\d{1,2})$'); + + static final Map _monthMap = UnmodifiableMapView({ + "january": 1, + "february": 2, + "march": 3, + "april": 4, + "may": 5, + "june": 6, + "july": 7, + "august": 8, + "september": 9, + "october": 10, + "november": 11, + "december": 12, + "jan": 1, + "feb": 2, + "mar": 3, + "apr": 4, + "jun": 6, + "jul": 7, + "aug": 8, + "sep": 9, + "sept": 9, + "oct": 10, + "nov": 11, + "dec": 12, + "janu": 1, + "febr": 2, + "marc": 3, + "apri": 4, + "juli": 7, + "augu": 8, + "sepe": 9, + "octo": 10, + "nove": 11, + "dece": 12, + }); + + static const Map monthNumberToName = { + 1: "January", + 2: "February", + 3: "March", + 4: "April", + 5: "May", + 6: "June", + 7: "July", + 8: "August", + 9: "September", + 10: "October", + 11: "November", + 12: "December", + }; + + PartialDate parse(String input) { + if (input.trim().isEmpty) return PartialDate.empty; + + final lowerInput = input.toLowerCase(); + + var result = _parseRelativeDate(lowerInput); + if (!result.isEmpty) return result; + + result = _parseStructuredFormats(lowerInput); + if (!result.isEmpty) return result; + + final normalized = _normalizeDateString(lowerInput); + result = _parseTokenizedDate(normalized); + + return result; + } + + String getMonthName(int month) { + return monthNumberToName[month] ?? 'Unknown'; + } + + String _normalizeDateString(String input) { + return input + .replaceAllMapped(_ordinalRegex, (match) => match.group(1)!) + .replaceAll(_normalizeRegex, ' ') + .trim(); + } + + int _convertTwoDigitYear(int year) { + return year < _TWO_DIGIT_YEAR_PIVOT ? 2000 + year : 1900 + year; + } + + PartialDate _parseRelativeDate(String lowerInput) { + final bool hasToday = lowerInput.contains('today'); + final bool hasTomorrow = lowerInput.contains('tomorrow'); + final bool hasYesterday = lowerInput.contains('yesterday'); + + final int count = + (hasToday ? 1 : 0) + (hasTomorrow ? 1 : 0) + (hasYesterday ? 1 : 0); + + if (count > 1) { + return PartialDate.empty; + } + + final now = DateTime.now(); + if (hasToday) { + return PartialDate(day: now.day, month: now.month, year: now.year); + } + if (hasTomorrow) { + final tomorrow = now.add(const Duration(days: 1)); + return PartialDate( + day: tomorrow.day, + month: tomorrow.month, + year: tomorrow.year, + ); + } + if (hasYesterday) { + final yesterday = now.subtract(const Duration(days: 1)); + return PartialDate( + day: yesterday.day, + month: yesterday.month, + year: yesterday.year, + ); + } + return PartialDate.empty; + } + + PartialDate _parseStructuredFormats(String input) { + final cleanInput = input.replaceAll(' ', ''); + + Match? match = _isoFormatRegex.firstMatch(cleanInput); + if (match != null) { + final yearVal = int.tryParse(match.group(1)!); + final monthVal = int.tryParse(match.group(2)!); + final dayVal = int.tryParse(match.group(3)!); + if (yearVal != null && + yearVal >= _MIN_YEAR && + yearVal <= _MAX_YEAR && + monthVal != null && + monthVal >= 1 && + monthVal <= 12 && + dayVal != null && + dayVal >= 1 && + dayVal <= 31) { + return PartialDate(day: dayVal, month: monthVal, year: yearVal); + } + return PartialDate.empty; + } + + match = _standardFormatRegex.firstMatch(cleanInput); + if (match != null) { + final p1 = int.parse(match.group(1)!); + final p2 = int.parse(match.group(2)!); + final yearRaw = int.parse(match.group(3)!); + final year = yearRaw > 99 ? yearRaw : _convertTwoDigitYear(yearRaw); + + if (year < _MIN_YEAR || year > _MAX_YEAR) return PartialDate.empty; + + if (p1 > 12) { + if (p1 >= 1 && p1 <= 31 && p2 >= 1 && p2 <= 12) { + return PartialDate(day: p1, month: p2, year: year); + } + } else if (p2 > 12) { + if (p1 >= 1 && p1 <= 12 && p2 >= 1 && p2 <= 31) { + return PartialDate(day: p2, month: p1, year: year); + } + } else { + if (p1 >= 1 && p1 <= 12 && p2 >= 1 && p2 <= 31) { + return PartialDate(day: p2, month: p1, year: year); + } + } + return PartialDate.empty; + } + + match = _shortFormatRegex.firstMatch(cleanInput); + if (match != null) { + final p1 = int.parse(match.group(1)!); + final p2 = int.parse(match.group(2)!); + + if (p1 > 12) { + if (p1 >= 1 && p1 <= 31 && p2 >= 1 && p2 <= 12) { + return PartialDate(day: p1, month: p2); + } + } else if (p2 > 12) { + if (p1 >= 1 && p1 <= 12 && p2 >= 1 && p2 <= 31) { + return PartialDate(day: p2, month: p1); + } + } else { + if (p1 >= 1 && p1 <= 12 && p2 >= 1 && p2 <= 31) { + return PartialDate(day: p2, month: p1); + } + } + return PartialDate.empty; + } + + match = _dotFormatRegex.firstMatch(cleanInput); + if (match != null) { + final yearRaw = int.parse(match.group(3)!); + final year = yearRaw > 99 ? yearRaw : _convertTwoDigitYear(yearRaw); + final dayVal = int.tryParse(match.group(1)!); + final monthVal = int.tryParse(match.group(2)!); + + if (year >= _MIN_YEAR && + year <= _MAX_YEAR && + dayVal != null && + dayVal >= 1 && + dayVal <= 31 && + monthVal != null && + monthVal >= 1 && + monthVal <= 12) { + return PartialDate(day: dayVal, month: monthVal, year: year); + } + return PartialDate.empty; + } + + match = _compactFormatRegex.firstMatch(cleanInput); + if (match != null) { + final yearVal = int.tryParse(cleanInput.substring(0, 4)); + final monthVal = int.tryParse(cleanInput.substring(4, 6)); + final dayVal = int.tryParse(cleanInput.substring(6, 8)); + + if (yearVal != null && + yearVal >= _MIN_YEAR && + yearVal <= _MAX_YEAR && + monthVal != null && + monthVal >= 1 && + monthVal <= 12 && + dayVal != null && + dayVal >= 1 && + dayVal <= 31) { + return PartialDate(day: dayVal, month: monthVal, year: yearVal); + } + return PartialDate.empty; + } + + return PartialDate.empty; + } + + PartialDate _parseTokenizedDate(String normalized) { + final tokens = normalized.split(' '); + int? day, month, year; + + if (tokens.length == 1) { + final token = tokens[0]; + final match = _yearOnlyRegex.firstMatch(token); + if (match != null) { + final parsedYear = int.tryParse(match.group(1)!); + if (parsedYear != null && + parsedYear >= _MIN_YEAR && + parsedYear <= _MAX_YEAR) { + return PartialDate(year: parsedYear); + } + } + if (_monthMap.containsKey(token)) { + return PartialDate(month: _monthMap[token]!); + } + final singleValue = int.tryParse(token); + if (singleValue != null && singleValue >= 1 && singleValue <= 31) { + return PartialDate(day: singleValue); + } + return PartialDate.empty; + } + + for (final token in tokens) { + if (_monthMap.containsKey(token) && month == null) { + month = _monthMap[token]; + continue; + } + + final value = int.tryParse(token); + if (value == null) { + continue; + } + + if (value >= _MIN_YEAR && value <= _MAX_YEAR && year == null) { + year = value; + } else if (value >= 1 && value <= 31 && day == null) { + day = value; + } else if (value >= 0 && value <= 99 && year == null) { + final convertedYear = _convertTwoDigitYear(value); + if (convertedYear >= _MIN_YEAR && convertedYear <= _MAX_YEAR) { + year = convertedYear; + } + } else if (value >= 1 && value <= 12 && month == null) { + month = value; + } + } + + if (day != null && (day < 1 || day > 31)) { + day = null; + } + if (month != null && (month < 1 || month > 12)) { + month = null; + } + + final bool inputHadMonthWord = tokens.any((t) => _monthMap.containsKey(t)); + final bool inputHadYearWord = tokens.any((t) { + final v = int.tryParse(t); + return v != null && v >= 1000 && v <= 9999; + }); + + if (day != null && month == null && year == null && tokens.length > 1) { + if (normalized.contains('of') && + !inputHadMonthWord && + !inputHadYearWord) { + return PartialDate.empty; + } + if (!inputHadMonthWord && !inputHadYearWord && tokens.length > 1) {} + } + + if (day == null && month == null && year == null) { + return PartialDate.empty; + } + + if (day != null && month == null && year == null && tokens.length > 1) { + if (!inputHadMonthWord && !inputHadYearWord) { + return PartialDate.empty; + } + } + + return PartialDate(day: day, month: month, year: year); + } +} diff --git a/mobile/apps/photos/lib/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart b/mobile/apps/photos/lib/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart index 2e79215cf1..b11ff67fb5 100644 --- a/mobile/apps/photos/lib/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart +++ b/mobile/apps/photos/lib/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart @@ -8,11 +8,11 @@ import "package:ml_linalg/dtype.dart"; import "package:ml_linalg/vector.dart"; import "package:photos/generated/protos/ente/common/vector.pb.dart"; import "package:photos/models/base/id.dart"; -import "package:photos/services/isolate_functions.dart"; -import "package:photos/services/isolate_service.dart"; import "package:photos/services/machine_learning/face_ml/face_clustering/face_db_info_for_clustering.dart"; import "package:photos/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart"; import "package:photos/services/machine_learning/ml_result.dart"; +import "package:photos/utils/isolate/isolate_operations.dart"; +import "package:photos/utils/isolate/super_isolate.dart"; class FaceInfo { final String faceID; @@ -507,7 +507,8 @@ ClusteringResult _runCompleteClustering(Map args) { EVector.fromBuffer(entry.value).values, dtype: DType.float32, ), - fileCreationTime: fileIDToCreationTime?[getFileIdFromFaceId(entry.key)], + fileCreationTime: + fileIDToCreationTime?[getFileIdFromFaceId(entry.key)], ), ); } diff --git a/mobile/apps/photos/lib/services/machine_learning/face_thumbnail_generator.dart b/mobile/apps/photos/lib/services/machine_learning/face_thumbnail_generator.dart index 4980f2e904..6e09744cff 100644 --- a/mobile/apps/photos/lib/services/machine_learning/face_thumbnail_generator.dart +++ b/mobile/apps/photos/lib/services/machine_learning/face_thumbnail_generator.dart @@ -1,15 +1,13 @@ import 'dart:async'; import 'dart:typed_data' show Uint8List; -import "package:computer/computer.dart"; import "package:logging/logging.dart"; import "package:photos/models/ml/face/box.dart"; -import "package:photos/services/isolate_functions.dart"; -import "package:photos/services/isolate_service.dart"; import "package:photos/utils/image_ml_util.dart"; +import "package:photos/utils/isolate/isolate_operations.dart"; +import "package:photos/utils/isolate/super_isolate.dart"; -final Computer _computer = Computer.shared(); - +@pragma('vm:entry-point') class FaceThumbnailGenerator extends SuperIsolate { @override Logger get logger => _logger; @@ -37,20 +35,30 @@ class FaceThumbnailGenerator extends SuperIsolate { String imagePath, List faceBoxes, ) async { - final List> faceBoxesJson = - faceBoxes.map((box) => box.toJson()).toList(); - final List faces = await runInIsolate( - IsolateOperation.generateFaceThumbnails, - { - 'imagePath': imagePath, - 'faceBoxesList': faceBoxesJson, - }, - ).then((value) => value.cast()); - final compressedFaces = - await compressFaceThumbnails({'listPngBytes': faces}); - _logger.fine( - "Compressed face thumbnails from sizes ${faces.map((e) => e.length / 1024).toList()} to ${compressedFaces.map((e) => e.length / 1024).toList()} kilobytes", - ); - return compressedFaces; + try { + _logger.info( + "Generating face thumbnails for ${faceBoxes.length} face boxes in $imagePath", + ); + final List> faceBoxesJson = + faceBoxes.map((box) => box.toJson()).toList(); + final List faces = await runInIsolate( + IsolateOperation.generateFaceThumbnails, + { + 'imagePath': imagePath, + 'faceBoxesList': faceBoxesJson, + }, + ).then((value) => value.cast()); + _logger.info("Generated face thumbnails"); + final compressedFaces = + await compressFaceThumbnails({'listPngBytes': faces}); + _logger.fine( + "Compressed face thumbnails from sizes ${faces.map((e) => e.length / 1024).toList()} to ${compressedFaces.map((e) => e.length / 1024).toList()} kilobytes", + ); + return compressedFaces; + } catch (e, s) { + _logger.severe("Failed to generate face thumbnails", e, s); + + rethrow; + } } } diff --git a/mobile/apps/photos/lib/services/machine_learning/ml_computer.dart b/mobile/apps/photos/lib/services/machine_learning/ml_computer.dart index edbc16b7ff..939d486f64 100644 --- a/mobile/apps/photos/lib/services/machine_learning/ml_computer.dart +++ b/mobile/apps/photos/lib/services/machine_learning/ml_computer.dart @@ -2,12 +2,12 @@ import 'dart:async'; import "package:logging/logging.dart"; import "package:photos/models/ml/vector.dart"; -import "package:photos/services/isolate_functions.dart"; -import "package:photos/services/isolate_service.dart"; import "package:photos/services/machine_learning/ml_constants.dart"; import "package:photos/services/machine_learning/semantic_search/clip/clip_text_encoder.dart"; import "package:photos/services/machine_learning/semantic_search/query_result.dart"; import "package:photos/services/remote_assets_service.dart"; +import "package:photos/utils/isolate/isolate_operations.dart"; +import "package:photos/utils/isolate/super_isolate.dart"; import "package:synchronized/synchronized.dart"; class MLComputer extends SuperIsolate { diff --git a/mobile/apps/photos/lib/services/machine_learning/ml_indexing_isolate.dart b/mobile/apps/photos/lib/services/machine_learning/ml_indexing_isolate.dart index e66848bb27..b386674b31 100644 --- a/mobile/apps/photos/lib/services/machine_learning/ml_indexing_isolate.dart +++ b/mobile/apps/photos/lib/services/machine_learning/ml_indexing_isolate.dart @@ -2,14 +2,14 @@ import "dart:async"; import "package:flutter/foundation.dart" show debugPrint; import "package:logging/logging.dart"; -import "package:photos/services/isolate_functions.dart"; -import "package:photos/services/isolate_service.dart"; import 'package:photos/services/machine_learning/face_ml/face_detection/face_detection_service.dart'; import 'package:photos/services/machine_learning/face_ml/face_embedding/face_embedding_service.dart'; import "package:photos/services/machine_learning/ml_models_overview.dart"; import 'package:photos/services/machine_learning/ml_result.dart'; import "package:photos/services/machine_learning/semantic_search/clip/clip_image_encoder.dart"; import "package:photos/services/remote_assets_service.dart"; +import "package:photos/utils/isolate/isolate_operations.dart"; +import "package:photos/utils/isolate/super_isolate.dart"; import "package:photos/utils/ml_util.dart"; import "package:photos/utils/network_util.dart"; import "package:synchronized/synchronized.dart"; diff --git a/mobile/apps/photos/lib/services/search_service.dart b/mobile/apps/photos/lib/services/search_service.dart index 30ef268bf2..735c2853d1 100644 --- a/mobile/apps/photos/lib/services/search_service.dart +++ b/mobile/apps/photos/lib/services/search_service.dart @@ -43,6 +43,7 @@ import "package:photos/models/search/search_types.dart"; import "package:photos/service_locator.dart"; import "package:photos/services/account/user_service.dart"; import 'package:photos/services/collections_service.dart'; +import "package:photos/services/date_parse_service.dart"; import "package:photos/services/filter/db_filters.dart"; import "package:photos/services/location_service.dart"; import "package:photos/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart"; @@ -59,7 +60,6 @@ import "package:photos/utils/cache_util.dart"; import "package:photos/utils/file_util.dart"; import "package:photos/utils/navigation_util.dart"; import 'package:photos/utils/standalone/date_time.dart'; -import 'package:tuple/tuple.dart'; class SearchService { Future>? _cachedFilesFuture; @@ -1063,20 +1063,59 @@ class SearchService { String query, ) async { final List searchResults = []; - final potentialDates = _getPossibleEventDate(context, query); - for (var potentialDate in potentialDates) { - final int day = potentialDate.item1; - final int month = potentialDate.item2.monthNumber; - final int? year = potentialDate.item3; // nullable + final parsedDate = DateParseService.instance.parse(query); + + if (parsedDate.isEmpty) { + return searchResults; + } + // Handle month-year queries + if (parsedDate.day == null && + parsedDate.month != null && + parsedDate.year != null) { + final month = parsedDate.month!; + final year = parsedDate.year!; + final monthYearFiles = + await FilesDB.instance.getFilesCreatedWithinDurations( + [_getDurationForMonthInYear(month, year)], + ignoreCollections(), + order: 'DESC', + ); + if (monthYearFiles.isNotEmpty) { + final monthName = DateParseService.instance.getMonthName(month); + final name = '$monthName $year'; + searchResults.add( + GenericSearchResult( + ResultType.month, + name, + monthYearFiles, + hierarchicalSearchFilter: TopLevelGenericFilter( + filterName: name, + occurrence: kMostRelevantFilter, + filterResultType: ResultType.month, + matchedUploadedIDs: filesToUploadedFileIDs(monthYearFiles), + filterIcon: Icons.calendar_month_outlined, + ), + ), + ); + } + } + // Handle day-month queries (with or without year) + else if (parsedDate.day != null && parsedDate.month != null) { + final int day = parsedDate.day!; + final int month = parsedDate.month!; + final int? year = parsedDate.year; // nullable for generic dates + final matchedFiles = await FilesDB.instance.getFilesCreatedWithinDurations( _getDurationsForCalendarDateInEveryYear(day, month, year: year), ignoreCollections(), order: 'DESC', ); + if (matchedFiles.isNotEmpty) { - final name = '$day ${potentialDate.item2.name} ${year ?? ''}'; + final monthName = DateParseService.instance.getMonthName(month); + final name = '$day $monthName${year != null ? ' $year' : ''}'; searchResults.add( GenericSearchResult( ResultType.event, @@ -1482,55 +1521,12 @@ class SearchService { return durationsOfMonthInEveryYear; } - List> _getPossibleEventDate( - BuildContext context, - String query, - ) { - final List> possibleEvents = []; - if (query.trim().isEmpty) { - return possibleEvents; - } - final result = query - .trim() - .split(RegExp('[ ,-/]+')) - .map((e) => e.trim()) - .where((e) => e.isNotEmpty) - .toList(); - final resultCount = result.length; - if (resultCount < 1 || resultCount > 4) { - return possibleEvents; - } - - final int? day = int.tryParse(result[0]); - if (day == null || day < 1 || day > 31) { - return possibleEvents; - } - final List potentialMonth = resultCount > 1 - ? _getMatchingMonths(context, result[1]) - : getMonthData(context); - final int? parsedYear = resultCount >= 3 ? int.tryParse(result[2]) : null; - final List matchingYears = []; - if (parsedYear != null) { - bool foundMatch = false; - for (int i = searchStartYear; i <= currentYear; i++) { - if (i.toString().startsWith(parsedYear.toString())) { - matchingYears.add(i); - foundMatch = foundMatch || (i == parsedYear); - } - } - if (!foundMatch && parsedYear > 1000 && parsedYear <= currentYear) { - matchingYears.add(parsedYear); - } - } - for (var element in potentialMonth) { - if (matchingYears.isEmpty) { - possibleEvents.add(Tuple3(day, element, null)); - } else { - for (int yr in matchingYears) { - possibleEvents.add(Tuple3(day, element, yr)); - } - } - } - return possibleEvents; + List _getDurationForMonthInYear(int month, int year) { + return [ + DateTime(year, month, 1).microsecondsSinceEpoch, + month == 12 + ? DateTime(year + 1, 1, 1).microsecondsSinceEpoch + : DateTime(year, month + 1, 1).microsecondsSinceEpoch, + ]; } } diff --git a/mobile/apps/photos/lib/services/sync/remote_sync_service.dart b/mobile/apps/photos/lib/services/sync/remote_sync_service.dart index d4523b27d0..9e2f7f6aed 100644 --- a/mobile/apps/photos/lib/services/sync/remote_sync_service.dart +++ b/mobile/apps/photos/lib/services/sync/remote_sync_service.dart @@ -51,6 +51,10 @@ class RemoteSyncService { Completer? _existingSync; bool _isExistingSyncSilent = false; + // _hasCleanupStaleEntry is used to track if we have already cleaned up + // statle db entries in this sync session. + bool _hasCleanupStaleEntry = false; + static const kHasSyncedArchiveKey = "has_synced_archive"; /* This setting is used to maintain a list of local IDs for videos that the user has manually marked for upload, even if the global video upload setting is currently disabled. @@ -371,6 +375,14 @@ class RemoteSyncService { final Set alreadyClaimedLocalIDs = await _db.getLocalIDsMarkedForOrAlreadyUploaded(ownerID); localIDsToSync.removeAll(alreadyClaimedLocalIDs); + if (alreadyClaimedLocalIDs.isNotEmpty && !_hasCleanupStaleEntry) { + try { + await _db.removeQueuedLocalFiles(alreadyClaimedLocalIDs); + } catch(e, s) { + _logger.severe("removeQueuedLocalFiles failed",e,s); + + } + } } if (localIDsToSync.isEmpty) { @@ -439,6 +451,7 @@ class RemoteSyncService { // "force reload due to display new files" Bus.instance.fire(ForceReloadHomeGalleryEvent("newFilesDisplay")); } + _hasCleanupStaleEntry = true; } Future updateDeviceFolderSyncStatus( diff --git a/mobile/apps/photos/lib/ui/viewer/people/person_face_widget.dart b/mobile/apps/photos/lib/ui/viewer/people/person_face_widget.dart index 8bd069f57e..3e1b138c26 100644 --- a/mobile/apps/photos/lib/ui/viewer/people/person_face_widget.dart +++ b/mobile/apps/photos/lib/ui/viewer/people/person_face_widget.dart @@ -21,6 +21,7 @@ class PersonFaceWidget extends StatefulWidget { final String? clusterID; final bool useFullFile; final VoidCallback? onErrorCallback; + final bool keepAlive; // PersonFaceWidget constructor checks that both personId and clusterID are not null // and that the file is not null @@ -29,6 +30,7 @@ class PersonFaceWidget extends StatefulWidget { this.clusterID, this.useFullFile = true, this.onErrorCallback, + this.keepAlive = false, super.key, }) : assert( personId != null || clusterID != null, @@ -39,12 +41,16 @@ class PersonFaceWidget extends StatefulWidget { State createState() => _PersonFaceWidgetState(); } -class _PersonFaceWidgetState extends State { +class _PersonFaceWidgetState extends State + with AutomaticKeepAliveClientMixin { Future? faceCropFuture; EnteFile? fileForFaceCrop; bool get isPerson => widget.personId != null; + @override + bool get wantKeepAlive => widget.keepAlive; + @override void initState() { super.initState(); @@ -64,6 +70,10 @@ class _PersonFaceWidgetState extends State { @override Widget build(BuildContext context) { + super.build( + context, + ); // Calling super.build for AutomaticKeepAliveClientMixin + return FutureBuilder( future: faceCropFuture, builder: (context, snapshot) { @@ -163,7 +173,7 @@ class _PersonFaceWidgetState extends State { } } if (fileForFaceCrop == null) { - _logger.warning( + _logger.severe( "No suitable file found for face crop for person: ${widget.personId} or cluster: ${widget.clusterID}", ); return null; @@ -176,7 +186,7 @@ class _PersonFaceWidgetState extends State { clusterID: widget.clusterID, ); if (face == null) { - debugPrint( + _logger.severe( "No cover face for person: ${widget.personId} or cluster ${widget.clusterID} and fileID ${fileForFaceCrop.uploadedFileID!}", ); return null; @@ -188,7 +198,13 @@ class _PersonFaceWidgetState extends State { personOrClusterID: personOrClusterId, useTempCache: false, ); - return cropMap?[face.faceID]; + final result = cropMap?[face.faceID]; + if (result == null) { + _logger.severe( + "Null cover face crop for person: ${widget.personId} or cluster ${widget.clusterID} and fileID ${fileForFaceCrop.uploadedFileID!}", + ); + } + return result; } catch (e, s) { _logger.severe( "Error getting cover face for person: ${widget.personId} or cluster ${widget.clusterID}", diff --git a/mobile/apps/photos/lib/ui/viewer/search/result/people_section_all_page.dart b/mobile/apps/photos/lib/ui/viewer/search/result/people_section_all_page.dart index d78eb4b9ba..d711bd9ebd 100644 --- a/mobile/apps/photos/lib/ui/viewer/search/result/people_section_all_page.dart +++ b/mobile/apps/photos/lib/ui/viewer/search/result/people_section_all_page.dart @@ -95,12 +95,14 @@ class SelectablePersonSearchExample extends StatelessWidget { final GenericSearchResult searchResult; final double size; final SelectedPeople selectedPeople; + final bool isDefaultFace; const SelectablePersonSearchExample({ super.key, required this.searchResult, required this.selectedPeople, this.size = 102, + this.isDefaultFace = false, }); void _handleTap(BuildContext context) { @@ -192,7 +194,10 @@ class SelectablePersonSearchExample extends StatelessWidget { searchResult.previewThumbnail()!, shouldShowSyncStatus: false, ) - : FaceSearchResult(searchResult); + : FaceSearchResult( + searchResult, + isDefaultFace: isDefaultFace, + ); } else { child = const NoThumbnailWidget( addBorder: false, @@ -301,8 +306,13 @@ class SelectablePersonSearchExample extends StatelessWidget { class FaceSearchResult extends StatelessWidget { final SearchResult searchResult; + final bool isDefaultFace; - const FaceSearchResult(this.searchResult, {super.key}); + const FaceSearchResult( + this.searchResult, { + super.key, + this.isDefaultFace = false, + }); @override Widget build(BuildContext context) { @@ -313,6 +323,7 @@ class FaceSearchResult extends StatelessWidget { key: params.containsKey(kPersonWidgetKey) ? ValueKey(params[kPersonWidgetKey]) : ValueKey(params[kPersonParamID] ?? params[kClusterParamId]), + keepAlive: isDefaultFace, ); } } @@ -486,6 +497,7 @@ class _PeopleSectionAllWidgetState extends State { searchResult: normalFaces[index], size: itemSize, selectedPeople: widget.selectedPeople!, + isDefaultFace: true, ) : PersonSearchExample( searchResult: normalFaces[index], @@ -525,6 +537,7 @@ class _PeopleSectionAllWidgetState extends State { searchResult: extraFaces[index], size: itemSize, selectedPeople: widget.selectedPeople!, + isDefaultFace: false, ) : PersonSearchExample( searchResult: extraFaces[index], diff --git a/mobile/apps/photos/lib/utils/face/face_thumbnail_cache.dart b/mobile/apps/photos/lib/utils/face/face_thumbnail_cache.dart index 117793173f..6f390fcf77 100644 --- a/mobile/apps/photos/lib/utils/face/face_thumbnail_cache.dart +++ b/mobile/apps/photos/lib/utils/face/face_thumbnail_cache.dart @@ -136,7 +136,7 @@ Future?> getCachedFaceCrops( facesWithoutCrops[face.faceID] = face.detection.box; } } catch (e, s) { - _logger.severe( + _logger.warning( "Error reading cached face crop for faceID ${face.faceID} from file ${faceCropCacheFile.path}", e, s, @@ -212,7 +212,7 @@ Future?> getCachedFaceCrops( milliseconds: 100 * pow(2, fetchAttempt + 1).toInt(), ); await Future.delayed(backoff); - _logger.warning( + _logger.fine( "Error getting face crops for faceIDs: ${faces.map((face) => face.faceID).toList()}, retrying (attempt ${fetchAttempt + 1}) in ${backoff.inMilliseconds} ms", e, s, @@ -225,13 +225,13 @@ Future?> getCachedFaceCrops( useTempCache: useTempCache, ); } - _logger.severe( + _logger.warning( "Error getting face crops for faceIDs: ${faces.map((face) => face.faceID).toList()}", e, s, ); } else { - _logger.info( + _logger.severe( "Stopped getting face crops for faceIDs: ${faces.map((face) => face.faceID).toList()} due to $e", ); } @@ -334,12 +334,14 @@ Future?> _getFaceCrops( if (useFullFile && file.fileType != FileType.video) { final File? ioFile = await getFile(file); if (ioFile == null) { + _logger.severe("Failed to get file for face crop generation"); return null; } imagePath = ioFile.path; } else { final thumbnail = await getThumbnailForUploadedFile(file); if (thumbnail == null) { + _logger.severe("Failed to get thumbnail for face crop generation"); return null; } imagePath = thumbnail.path; diff --git a/mobile/apps/photos/lib/services/isolate_functions.dart b/mobile/apps/photos/lib/utils/isolate/isolate_operations.dart similarity index 100% rename from mobile/apps/photos/lib/services/isolate_functions.dart rename to mobile/apps/photos/lib/utils/isolate/isolate_operations.dart diff --git a/mobile/apps/photos/lib/services/isolate_service.dart b/mobile/apps/photos/lib/utils/isolate/super_isolate.dart similarity index 96% rename from mobile/apps/photos/lib/services/isolate_service.dart rename to mobile/apps/photos/lib/utils/isolate/super_isolate.dart index 4a91c64473..3a83eaef1b 100644 --- a/mobile/apps/photos/lib/services/isolate_service.dart +++ b/mobile/apps/photos/lib/utils/isolate/super_isolate.dart @@ -7,9 +7,10 @@ import "package:flutter/services.dart"; import "package:logging/logging.dart"; import "package:photos/core/error-reporting/isolate_logging.dart"; import "package:photos/models/base/id.dart"; -import "package:photos/services/isolate_functions.dart"; +import "package:photos/utils/isolate/isolate_operations.dart"; import "package:synchronized/synchronized.dart"; +@pragma('vm:entry-point') abstract class SuperIsolate { Logger get logger; @@ -80,6 +81,8 @@ abstract class SuperIsolate { if (rootToken != null) { BackgroundIsolateBinaryMessenger.ensureInitialized(rootToken); } + final logger = Logger('SuperIsolate'); + logger.info('IsolateMain started'); receivePort.listen((message) async { final taskID = message[0] as String; @@ -87,6 +90,7 @@ abstract class SuperIsolate { final function = IsolateOperation.values[functionIndex]; final args = message[2] as Map; final sendPort = message[3] as SendPort; + logger.info("Starting isolate operation $function in isolate"); late final Object data; try { diff --git a/mobile/apps/photos/pubspec.lock b/mobile/apps/photos/pubspec.lock index 58284d5f09..f8cf5809fe 100644 --- a/mobile/apps/photos/pubspec.lock +++ b/mobile/apps/photos/pubspec.lock @@ -5,10 +5,10 @@ packages: dependency: transitive description: name: _fe_analyzer_shared - sha256: f256b0c0ba6c7577c15e2e4e114755640a875e885099367bf6e012b19314c834 + sha256: "16e298750b6d0af7ce8a3ba7c18c69c3785d11b15ec83f6dcd0ad2a0009b3cab" url: "https://pub.dev" source: hosted - version: "72.0.0" + version: "76.0.0" _flutterfire_internals: dependency: transitive description: @@ -21,7 +21,7 @@ packages: dependency: transitive description: dart source: sdk - version: "0.3.2" + version: "0.3.3" adaptive_theme: dependency: "direct main" description: @@ -34,10 +34,10 @@ packages: dependency: transitive description: name: analyzer - sha256: b652861553cd3990d8ed361f7979dc6d7053a9ac8843fa73820ab68ce5410139 + sha256: "1f14db053a8c23e260789e9b0980fa27f2680dd640932cae5e1137cce0e46e1e" url: "https://pub.dev" source: hosted - version: "6.7.0" + version: "6.11.0" android_intent_plus: dependency: "direct main" description: @@ -317,10 +317,10 @@ packages: dependency: "direct main" description: name: collection - sha256: ee67cb0715911d28db6bf4af1026078bd6f0128b07a5f66fb2ed94ec6783c09a + sha256: a1ace0a119f20aabc852d165077c036cd864315bd99b7eaa10a60100341941bf url: "https://pub.dev" source: hosted - version: "1.18.0" + version: "1.19.0" computer: dependency: "direct main" description: @@ -1416,18 +1416,18 @@ packages: dependency: transitive description: name: leak_tracker - sha256: "3f87a60e8c63aecc975dda1ceedbc8f24de75f09e4856ea27daf8958f2f0ce05" + sha256: "7bb2830ebd849694d1ec25bf1f44582d6ac531a57a365a803a6034ff751d2d06" url: "https://pub.dev" source: hosted - version: "10.0.5" + version: "10.0.7" leak_tracker_flutter_testing: dependency: transitive description: name: leak_tracker_flutter_testing - sha256: "932549fb305594d82d7183ecd9fa93463e9914e1b67cacc34bc40906594a1806" + sha256: "9491a714cca3667b60b5c420da8217e6de0d1ba7a5ec322fab01758f6998f379" url: "https://pub.dev" source: hosted - version: "3.0.5" + version: "3.0.8" leak_tracker_testing: dependency: transitive description: @@ -1536,10 +1536,10 @@ packages: dependency: transitive description: name: macros - sha256: "0acaed5d6b7eab89f63350bccd82119e6c602df0f391260d0e32b5e23db79536" + sha256: "1d9e801cd66f7ea3663c45fc708450db1fa57f988142c64289142c9b7ee80656" url: "https://pub.dev" source: hosted - version: "0.1.2-main.4" + version: "0.1.3-main.0" maps_launcher: dependency: "direct main" description: @@ -2309,7 +2309,7 @@ packages: dependency: transitive description: flutter source: sdk - version: "0.0.99" + version: "0.0.0" source_gen: dependency: transitive description: @@ -2434,10 +2434,10 @@ packages: dependency: transitive description: name: stack_trace - sha256: "73713990125a6d93122541237550ee3352a2d84baad52d375a4cad2eb9b7ce0b" + sha256: "9f47fd3630d76be3ab26f0ee06d213679aa425996925ff3feffdec504931c377" url: "https://pub.dev" source: hosted - version: "1.11.1" + version: "1.12.0" step_progress_indicator: dependency: "direct main" description: @@ -2466,10 +2466,10 @@ packages: dependency: transitive description: name: string_scanner - sha256: "556692adab6cfa87322a115640c11f13cb77b3f076ddcc5d6ae3c20242bedcde" + sha256: "688af5ed3402a4bde5b3a6c15fd768dbf2621a614950b17f04626c431ab3c4c3" url: "https://pub.dev" source: hosted - version: "1.2.0" + version: "1.3.0" styled_text: dependency: "direct main" description: @@ -2530,26 +2530,26 @@ packages: dependency: "direct dev" description: name: test - sha256: "7ee44229615f8f642b68120165ae4c2a75fe77ae2065b1e55ae4711f6cf0899e" + sha256: "713a8789d62f3233c46b4a90b174737b2c04cb6ae4500f2aa8b1be8f03f5e67f" url: "https://pub.dev" source: hosted - version: "1.25.7" + version: "1.25.8" test_api: dependency: transitive description: name: test_api - sha256: "5b8a98dafc4d5c4c9c72d8b31ab2b23fc13422348d2997120294d3bac86b4ddb" + sha256: "664d3a9a64782fcdeb83ce9c6b39e78fd2971d4e37827b9b06c3aa1edc5e760c" url: "https://pub.dev" source: hosted - version: "0.7.2" + version: "0.7.3" test_core: dependency: transitive description: name: test_core - sha256: "55ea5a652e38a1dfb32943a7973f3681a60f872f8c3a05a14664ad54ef9c6696" + sha256: "12391302411737c176b0b5d6491f466b0dd56d4763e347b6714efbaa74d7953d" url: "https://pub.dev" source: hosted - version: "0.6.4" + version: "0.6.5" thermal: dependency: "direct main" description: @@ -2813,10 +2813,10 @@ packages: dependency: transitive description: name: vm_service - sha256: "5c5f338a667b4c644744b661f309fb8080bb94b18a7e91ef1dbd343bed00ed6d" + sha256: f6be3ed8bd01289b34d679c2b62226f63c0e69f9fd2e50a6b3c1c729a961041b url: "https://pub.dev" source: hosted - version: "14.2.5" + version: "14.3.0" volume_controller: dependency: transitive description: @@ -2877,10 +2877,10 @@ packages: dependency: transitive description: name: webdriver - sha256: "003d7da9519e1e5f329422b36c4dcdf18d7d2978d1ba099ea4e45ba490ed845e" + sha256: "3d773670966f02a646319410766d3b5e1037efb7f07cc68f844d5e06cd4d61c8" url: "https://pub.dev" source: hosted - version: "3.0.3" + version: "3.0.4" webkit_inspection_protocol: dependency: transitive description: diff --git a/mobile/apps/photos/test/utils/date_query_parsing_test.dart b/mobile/apps/photos/test/utils/date_query_parsing_test.dart new file mode 100644 index 0000000000..402b890489 --- /dev/null +++ b/mobile/apps/photos/test/utils/date_query_parsing_test.dart @@ -0,0 +1,346 @@ +import 'package:photos/services/date_parse_service.dart'; +import 'package:test/test.dart'; + +void main() { + // Get an instance of the service + final DateParseService dateParseService = DateParseService.instance; + + // --- Natural Language Date Parsing --- + group('Natural Language Date Parsing', () { + // Relative dates: today, tomorrow, yesterday + test('should parse "today" correctly', () { + final DateTime now = DateTime.now(); + final PartialDate expectedDate = + PartialDate(day: now.day, month: now.month, year: now.year); + final PartialDate parsedDate = dateParseService.parse('today'); + + expect( + parsedDate.day, + expectedDate.day, + reason: 'Day mismatch for today', + ); + expect( + parsedDate.month, + expectedDate.month, + reason: 'Month mismatch for today', + ); + expect( + parsedDate.year, + expectedDate.year, + reason: 'Year mismatch for today', + ); + }); + + test('should parse "tomorrow" correctly', () { + final DateTime tomorrow = DateTime.now().add(const Duration(days: 1)); + final PartialDate expectedDate = PartialDate( + day: tomorrow.day, + month: tomorrow.month, + year: tomorrow.year, + ); + final PartialDate parsedDate = dateParseService.parse('tomorrow'); + + expect( + parsedDate.day, + expectedDate.day, + reason: 'Day mismatch for tomorrow', + ); + expect( + parsedDate.month, + expectedDate.month, + reason: 'Month mismatch for tomorrow', + ); + expect( + parsedDate.year, + expectedDate.year, + reason: 'Year mismatch for tomorrow', + ); + }); + + test('should parse "yesterday" correctly', () { + final DateTime yesterday = + DateTime.now().subtract(const Duration(days: 1)); + final PartialDate expectedDate = PartialDate( + day: yesterday.day, + month: yesterday.month, + year: yesterday.year, + ); + final PartialDate parsedDate = dateParseService.parse('yesterday'); + + expect( + parsedDate.day, + expectedDate.day, + reason: 'Day mismatch for yesterday', + ); + expect( + parsedDate.month, + expectedDate.month, + reason: 'Month mismatch for yesterday', + ); + expect( + parsedDate.year, + expectedDate.year, + reason: 'Year mismatch for yesterday', + ); + }); + + // Month names: Full (February), abbreviated (Feb), and partial (Febr) + test('should parse full month name "February 2025"', () { + final PartialDate parsedDate = dateParseService.parse('February 2025'); + expect(parsedDate.day, isNull); + expect(parsedDate.month, 2); + expect(parsedDate.year, 2025); + }); + + test('should parse abbreviated month name "Feb 2025"', () { + final PartialDate parsedDate = dateParseService.parse('Feb 2025'); + expect(parsedDate.day, isNull); + expect(parsedDate.month, 2); + expect(parsedDate.year, 2025); + }); + + test('should parse partial month name "Febr 2025"', () { + final PartialDate parsedDate = dateParseService.parse('Febr 2025'); + expect(parsedDate.day, isNull); + expect(parsedDate.month, 2); + expect(parsedDate.year, 2025); + }); + + // Ordinal numbers: 25th, 22nd, 3rd, 1st + test('should parse ordinal number "25th Jan 2024"', () { + final PartialDate parsedDate = dateParseService.parse('25th Jan 2024'); + expect(parsedDate.day, 25); + expect(parsedDate.month, 1); + expect(parsedDate.year, 2024); + }); + + test('should parse ordinal number "22nd Feb"', () { + final PartialDate parsedDate = dateParseService.parse('22nd Feb'); + expect(parsedDate.day, 22); + expect(parsedDate.month, 2); + expect(parsedDate.year, isNull); + }); + + test('should parse ordinal number "3rd March"', () { + final PartialDate parsedDate = dateParseService.parse('3rd March'); + expect(parsedDate.day, 3); + expect(parsedDate.month, 3); + expect(parsedDate.year, isNull); + }); + + // Flexible combinations + test('should parse "25th Feb" (generic date)', () { + final PartialDate parsedDate = dateParseService.parse('25th Feb'); + expect(parsedDate.day, 25); + expect(parsedDate.month, 2); + expect(parsedDate.year, isNull); + }); + + test('should parse "February 2025" (month-year query)', () { + final PartialDate parsedDate = dateParseService.parse('February 2025'); + expect(parsedDate.day, isNull); + expect(parsedDate.month, 2); + expect(parsedDate.year, 2025); + }); + + test('should parse "25th of February 2025"', () { + final PartialDate parsedDate = + dateParseService.parse('25th of February 2025'); + expect(parsedDate.day, 25); + expect(parsedDate.month, 2); + expect(parsedDate.year, 2025); + }); + }); + + // --- Structured Date Format Support --- + group('Structured Date Format Support', () { + // ISO format: 2025-02-25, 2025/02/25 + test('should parse ISO format "2025-02-25"', () { + final PartialDate parsedDate = dateParseService.parse('2025-02-25'); + expect(parsedDate.day, 25); + expect(parsedDate.month, 2); + expect(parsedDate.year, 2025); + }); + + test('should parse ISO format "2025/02/25"', () { + final PartialDate parsedDate = dateParseService.parse('2025/02/25'); + expect(parsedDate.day, 25); + expect(parsedDate.month, 2); + expect(parsedDate.year, 2025); + }); + + // Standard formats: 02/25/2025, 25/02/2025 (with MM/DD vs DD/MM detection) + test('should parse standard MM/DD/YYYY format "02/25/2025"', () { + // Your parser assumes MM/DD if ambiguous (e.g., both parts <= 12) + // but for 02/25/2025, 25 > 12, so it correctly interprets 02 as month and 25 as day. + final PartialDate parsedDate = dateParseService.parse('02/25/2025'); + expect(parsedDate.day, 25); + expect(parsedDate.month, 2); + expect(parsedDate.year, 2025); + }); + + test('should parse standard DD/MM/YYYY format "25/02/2025"', () { + // Your parser handles DD/MM explicitly when day part > 12 + final PartialDate parsedDate = dateParseService.parse('25/02/2025'); + expect(parsedDate.day, 25); + expect(parsedDate.month, 2); + expect(parsedDate.year, 2025); + }); + + test('should parse ambiguous "01/02/2024" as MM/DD/YYYY (Jan 2)', () { + // Test your specific heuristic for ambiguous cases + final PartialDate parsedDate = dateParseService.parse('01/02/2024'); + expect(parsedDate.day, 2); + expect(parsedDate.month, 1); + expect(parsedDate.year, 2024); + }); + + // Dot notation: 25.02.2025, 25.02.25 + test('should parse dot notation "25.02.2025"', () { + final PartialDate parsedDate = dateParseService.parse('25.02.2025'); + expect(parsedDate.day, 25); + expect(parsedDate.month, 2); + expect(parsedDate.year, 2025); + }); + + test('should parse dot notation with two-digit year "25.02.25"', () { + // Assumes century detection (e.g., 25 -> 2025) + final PartialDate parsedDate = dateParseService.parse('25.02.25'); + expect(parsedDate.day, 25); + expect(parsedDate.month, 2); + expect(parsedDate.year, 2025); + }); + + // Compact format: 20250225 + test('should parse compact format "20250225"', () { + final PartialDate parsedDate = dateParseService.parse('20250225'); + expect(parsedDate.day, 25); + expect(parsedDate.month, 2); + expect(parsedDate.year, 2025); + }); + + // Short formats: 02/25, 25/02 (your parser doesn't explicitly handle short yearless formats) + // Based on your _standardFormatRegex: RegExp(r'^(\d{1,2})[\/-](\d{1,2})[\/-](\d{2,4})$'); + // and _parseTokenizedDate, "02/25" would be processed by _parseTokenizedDate. + // Let's test how your current parser handles these. + test( + 'should parse short MM/DD format "02/25" (no year, handled by tokenized)', + () { + final PartialDate parsedDate = dateParseService.parse('02/25'); + expect(parsedDate.day, 25); // value 25 is assigned to day first + expect(parsedDate.month, 2); // value 02 is assigned to month + expect(parsedDate.year, isNull); + }); + + test( + 'should parse short DD/MM format "25/02" (no year, handled by tokenized)', + () { + // This will be parsed by _parseTokenizedDate + final PartialDate parsedDate = dateParseService.parse('25/02'); + expect(parsedDate.day, 25); + expect(parsedDate.month, 2); + expect(parsedDate.year, isNull); + }); + + // Two-digit years: 25/02/25 (with century detection) + test('should parse two-digit year "25/02/25"', () { + final PartialDate parsedDate = dateParseService.parse('25/02/25'); + expect(parsedDate.day, 25); + expect(parsedDate.month, 2); + expect(parsedDate.year, 2025); // Based on _convertTwoDigitYear pivot + }); + + test('should parse two-digit year "01/01/01" as 2001', () { + final PartialDate parsedDate = dateParseService.parse('01/01/01'); + expect(parsedDate.day, 1); + expect(parsedDate.month, 1); + expect(parsedDate.year, 2001); // 01 < _TWO_DIGIT_YEAR_PIVOT + }); + + test('should parse two-digit year "01/01/99" as 1999', () { + final PartialDate parsedDate = dateParseService.parse('01/01/99'); + expect(parsedDate.day, 1); + expect(parsedDate.month, 1); + expect(parsedDate.year, 1999); // 99 > _TWO_DIGIT_YEAR_PIVOT + }); + }); + + // --- Smart Query Types --- + group('Smart Query Types', () { + test('should parse year-only query "2025"', () { + final PartialDate parsedDate = dateParseService.parse('2025'); + expect(parsedDate.day, isNull); + expect(parsedDate.month, isNull); + expect(parsedDate.year, 2025); + }); + + test('should parse month-year query "February 2025"', () { + final PartialDate parsedDate = dateParseService.parse('February 2025'); + expect(parsedDate.day, isNull); + expect(parsedDate.month, 2); + expect(parsedDate.year, 2025); + }); + + test('should parse generic date query "25th Feb" (year is null)', () { + final PartialDate parsedDate = dateParseService.parse('25th Feb'); + expect(parsedDate.day, 25); + expect(parsedDate.month, 2); + expect(parsedDate.year, isNull); + }); + + test('should parse specific date query "25/02/2025"', () { + final PartialDate parsedDate = dateParseService.parse('25/02/2025'); + expect(parsedDate.day, 25); + expect(parsedDate.month, 2); + expect(parsedDate.year, 2025); + }); + }); + + // --- Invalid Date Queries --- + group('Invalid Date Queries', () { + test('should parse "February 30000" as month-only (invalid year ignored)', + () { + final PartialDate parsedDate = dateParseService.parse('February 30000'); + expect(parsedDate.day, isNull); + expect(parsedDate.month, 2); + expect( + parsedDate.year, + isNull, + reason: 'Year 30000 is out of range and should be ignored', + ); + }); + + // Specific case to test if invalid day/month values are set to null + test('should return null for invalid day/month in tokenized parsing', () { + final PartialDate parsedDate = dateParseService.parse('32 Jan 2024'); + expect(parsedDate.day, isNull, reason: 'Day should be null for 32'); + expect(parsedDate.month, 1); + expect( + parsedDate.year, + 2024, + ); + + // "Jan 13 2024" - This is a valid date (Jan 13, 2024), should parse completely. + final PartialDate parsedDate2 = dateParseService.parse('Jan 13 2024'); + expect(parsedDate2.day, 13); + expect(parsedDate2.month, 1); + expect(parsedDate2.year, 2024); + + // "Feb 0 2024" - Day 0 should be null, but month and year are valid. + final PartialDate parsedDate3 = dateParseService.parse('Feb 0 2024'); + expect(parsedDate3.day, isNull, reason: 'Day should be null for 0'); + expect(parsedDate3.month, 2); + expect(parsedDate3.year, 2024); + }); + + test('should handle invalid day/month in tokenized parsing gracefully', () { + final PartialDate parsedDate = dateParseService.parse('32 Jan 2024'); + expect(parsedDate.day, isNull, reason: 'Day should be null for 32'); + expect(parsedDate.month, 1); + expect( + parsedDate.year, + 2024, + ); + }); + }); +} diff --git a/server/migrations/102_trash_vaccume_and_analyze_threshold.down.sql b/server/migrations/102_trash_vaccume_and_analyze_threshold.down.sql new file mode 100644 index 0000000000..3d2dc0b0bf --- /dev/null +++ b/server/migrations/102_trash_vaccume_and_analyze_threshold.down.sql @@ -0,0 +1,6 @@ +ALTER TABLE trash RESET ( + autovacuum_analyze_scale_factor, + autovacuum_vacuum_scale_factor, + autovacuum_analyze_threshold, + autovacuum_vacuum_threshold + ); \ No newline at end of file diff --git a/server/migrations/102_trash_vaccume_and_analyze_threshold.up.sql b/server/migrations/102_trash_vaccume_and_analyze_threshold.up.sql new file mode 100644 index 0000000000..5d61312da7 --- /dev/null +++ b/server/migrations/102_trash_vaccume_and_analyze_threshold.up.sql @@ -0,0 +1,6 @@ +ALTER TABLE trash SET ( + autovacuum_analyze_scale_factor = 0.01, -- Trigger ANALYZE after 1% of rows change + autovacuum_vacuum_scale_factor = 0.02, -- Trigger VACUUM after 2% of rows change + autovacuum_analyze_threshold = 1000, + autovacuum_vacuum_threshold = 1000 +); diff --git a/server/pkg/repo/trash.go b/server/pkg/repo/trash.go index b79a40a1aa..3781e0716e 100644 --- a/server/pkg/repo/trash.go +++ b/server/pkg/repo/trash.go @@ -347,7 +347,7 @@ func (t *TrashRepository) GetTimeStampForLatestNonDeletedEntry(userID int64) (*i // GetUserIDToFileIDsMapForDeletion returns map of userID to fileIds, where the file ids which should be deleted by now func (t *TrashRepository) GetUserIDToFileIDsMapForDeletion() (map[int64][]int64, error) { rows, err := t.DB.Query(`SELECT user_id, file_id FROM trash - WHERE delete_by <= $1 AND is_deleted = FALSE AND is_restored = FALSE limit $2`, + WHERE delete_by <= $1 AND is_deleted IS FALSE AND is_restored IS FALSE limit $2`, time.Microseconds(), TrashDiffLimit) if err != nil { return nil, stacktrace.Propagate(err, "") diff --git a/web/apps/auth/src/pages/auth.tsx b/web/apps/auth/src/pages/auth.tsx index 8f9e18733b..f74f0e7c8f 100644 --- a/web/apps/auth/src/pages/auth.tsx +++ b/web/apps/auth/src/pages/auth.tsx @@ -404,10 +404,7 @@ const Footer: React.FC = () => { return ( {t("auth_download_mobile_app")} - +