From 53050ca25ea4ab7e9abc713049eefd2265d5f0d4 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Mon, 7 Jul 2025 14:31:37 +0530 Subject: [PATCH] Map remote files to local during diff sync --- mobile/apps/photos/lib/db/local/db.dart | 20 +++ .../lib/db/remote/table/mapping_table.dart | 20 +++ .../apps/photos/lib/models/api/diff/diff.dart | 4 + .../models/file/mapping/local_mapping.dart | 25 ++++ .../lib/models/file/remote/rl_mapping.dart | 22 ++-- .../services/remote/fetch/remote_diff.dart | 124 +++++++++++++++++- .../services/sync/remote_sync_service.dart | 3 +- 7 files changed, 204 insertions(+), 14 deletions(-) create mode 100644 mobile/apps/photos/lib/models/file/mapping/local_mapping.dart diff --git a/mobile/apps/photos/lib/db/local/db.dart b/mobile/apps/photos/lib/db/local/db.dart index d59de00cd3..bcc7f44790 100644 --- a/mobile/apps/photos/lib/db/local/db.dart +++ b/mobile/apps/photos/lib/db/local/db.dart @@ -10,6 +10,7 @@ import "package:photos/db/local/mappers.dart"; import "package:photos/db/local/schema.dart"; import "package:photos/log/devlog.dart"; import 'package:photos/models/file/file.dart'; +import "package:photos/models/file/mapping/local_mapping.dart"; import "package:photos/models/local/local_metadata.dart"; import "package:sqlite_async/sqlite_async.dart"; @@ -82,6 +83,25 @@ class LocalDB with SqlDbBase { } } + Future> getLocalAssetsInfo( + List ids, + ) async { + if (ids.isEmpty) return {}; + final stopwatch = Stopwatch()..start(); + final result = await _sqliteDB.getAll( + 'SELECT id, hash, name, relative_path, state FROM assets WHERE id IN (${List.filled(ids.length, "?").join(",")})', + ids, + ); + debugPrint( + "getLocalAssetsInfo complete in ${stopwatch.elapsed.inMilliseconds}ms for ${ids.length} ids", + ); + return Map.fromEntries( + result.map( + (row) => MapEntry(row['id'] as String, LocalAssetInfo.fromRow(row)), + ), + ); + } + Future> getAssets({LocalAssertsParam? params}) async { final stopwatch = Stopwatch()..start(); final result = await _sqliteDB.getAll( diff --git a/mobile/apps/photos/lib/db/remote/table/mapping_table.dart b/mobile/apps/photos/lib/db/remote/table/mapping_table.dart index ee7423f9e2..cc0d3c9025 100644 --- a/mobile/apps/photos/lib/db/remote/table/mapping_table.dart +++ b/mobile/apps/photos/lib/db/remote/table/mapping_table.dart @@ -41,4 +41,24 @@ extension UploadMappingTable on RemoteDB { } return result; } + + Future> getLocalIDsWithMapping(List localIDs) async { + if (localIDs.isEmpty) return {}; + final placeholders = List.filled(localIDs.length, '?').join(','); + final cursor = await sqliteDB.getAll( + 'SELECT local_id FROM upload_mapping join files on upload_mapping.file_id = files.id WHERE local_id IN ($placeholders)', + localIDs, + ); + return cursor.map((row) => row['local_id'] as String).toSet(); + } + + Future> getFilesWithMapping(List fileIDs) async { + if (fileIDs.isEmpty) return {}; + final placeholders = List.filled(fileIDs.length, '?').join(','); + final cursor = await sqliteDB.getAll( + 'SELECT file_id FROM upload_mapping WHERE file_id IN ($placeholders)', + fileIDs, + ); + return cursor.map((row) => row['file_id'] as int).toSet(); + } } diff --git a/mobile/apps/photos/lib/models/api/diff/diff.dart b/mobile/apps/photos/lib/models/api/diff/diff.dart index ddb09e3ef0..13dee12bc4 100644 --- a/mobile/apps/photos/lib/models/api/diff/diff.dart +++ b/mobile/apps/photos/lib/models/api/diff/diff.dart @@ -134,6 +134,10 @@ class ApiFileItem { String get title => pubMagicMetadata?.data['editedName'] ?? metadata?.data['title'] ?? ""; + String get nonEditedTitle { + return metadata?.data['title'] ?? ""; + } + String? get localID => metadata?.data['localID']; String? get matchLocalID => localID == null || deviceFolder == null diff --git a/mobile/apps/photos/lib/models/file/mapping/local_mapping.dart b/mobile/apps/photos/lib/models/file/mapping/local_mapping.dart new file mode 100644 index 0000000000..ae99b621f6 --- /dev/null +++ b/mobile/apps/photos/lib/models/file/mapping/local_mapping.dart @@ -0,0 +1,25 @@ +class LocalAssetInfo { + final String id; + final String? hash; + final String? name; + final String? relativePath; + final int state; + + LocalAssetInfo({ + required this.id, + this.hash, + this.name, + this.relativePath, + required this.state, + }); + + factory LocalAssetInfo.fromRow(Map row) { + return LocalAssetInfo( + id: row['id'] as String, + hash: row['hash'] as String?, + name: row['name'] as String?, + relativePath: row['relative_path'] as String?, + state: row['state'] as int, + ); + } +} diff --git a/mobile/apps/photos/lib/models/file/remote/rl_mapping.dart b/mobile/apps/photos/lib/models/file/remote/rl_mapping.dart index aa3f500a9e..1ad5596c7f 100644 --- a/mobile/apps/photos/lib/models/file/remote/rl_mapping.dart +++ b/mobile/apps/photos/lib/models/file/remote/rl_mapping.dart @@ -15,13 +15,13 @@ class RLMapping { remoteUploadID, localID, localCloudID, - mappingType, + mappingType.name, ]; } enum MatchType { - remote, - cloudIdMatched, + localID, + cloudID, deviceUpload, deviceHashMatched, } @@ -29,10 +29,10 @@ enum MatchType { extension MappingTypeExtension on MatchType { String get name { switch (this) { - case MatchType.remote: - return "remote"; - case MatchType.cloudIdMatched: - return "cloudIdMatched"; + case MatchType.localID: + return "localID"; + case MatchType.cloudID: + return "cloudID"; case MatchType.deviceUpload: return "deviceUpload"; case MatchType.deviceHashMatched: @@ -42,10 +42,10 @@ extension MappingTypeExtension on MatchType { static MatchType fromName(String name) { switch (name) { - case "remote": - return MatchType.remote; - case "cloudIdMatched": - return MatchType.cloudIdMatched; + case "localID": + return MatchType.localID; + case "cloudID": + return MatchType.cloudID; case "deviceUpload": return MatchType.deviceUpload; case "deviceHashMatched": diff --git a/mobile/apps/photos/lib/services/remote/fetch/remote_diff.dart b/mobile/apps/photos/lib/services/remote/fetch/remote_diff.dart index cf61eb1296..8e221aa803 100644 --- a/mobile/apps/photos/lib/services/remote/fetch/remote_diff.dart +++ b/mobile/apps/photos/lib/services/remote/fetch/remote_diff.dart @@ -1,10 +1,16 @@ +import "dart:io"; + import "package:flutter/foundation.dart"; import "package:logging/logging.dart"; +import "package:photos/core/configuration.dart"; import "package:photos/core/event_bus.dart"; +import "package:photos/db/remote/table/mapping_table.dart"; import "package:photos/events/collection_updated_event.dart"; import "package:photos/events/diff_sync_complete_event.dart"; import "package:photos/events/sync_status_update_event.dart"; +import "package:photos/models/api/diff/diff.dart"; import "package:photos/models/file/file.dart"; +import "package:photos/models/file/remote/rl_mapping.dart"; import "package:photos/service_locator.dart"; import "package:photos/services/collections_service.dart"; import "package:photos/services/remote/fetch/files_diff.dart"; @@ -13,10 +19,11 @@ class RemoteDiffService { final Logger _logger = Logger('RemoteDiffService'); final CollectionsService _collectionsService; final RemoteFileDiffService filesDiffService; - + final Configuration _config; RemoteDiffService( this._collectionsService, this.filesDiffService, + this._config, ); bool _isExistingSyncSilent = false; @@ -79,6 +86,7 @@ class RemoteDiffService { await remoteDB.deleteFilesDiff(diff.deletedItems); } if (diff.updatedItems.isNotEmpty) { + await _mapRemoteToLocalItems(diff); await remoteCache.insertDiffItems(diff.updatedItems); } // todo:(rewrite) neeraj add logic to refresh home gallery when time or visibility changes @@ -104,6 +112,120 @@ class RemoteDiffService { ); } + Future _mapRemoteToLocalItems(DiffResult diff) async { + final Map fileIDtoLocalID = {}; + final Map unmappedFileIDtoLocalID = {}; + for (final item in diff.updatedItems) { + if (item.fileItem.localID != null && + item.fileItem.ownerID == _config.getUserID()!) { + fileIDtoLocalID[item.fileItem.fileID] = + (item.fileItem.localID!, item.fileItem); + } + } + if (fileIDtoLocalID.isEmpty) { + _logger.info("No remote files to map to local items"); + return; + } + final mappedLocalIDs = await remoteDB.getLocalIDsWithMapping( + fileIDtoLocalID.values.map((e) => e.$1).toList(), + ); + final remoteIDsWithMapping = + await remoteDB.getFilesWithMapping(fileIDtoLocalID.keys.toList()); + // remote already claim mappings from fileIds to localIDs + int mapRemoteCount = 0; + int mapLocalCount = 0; + int bothMappedCount = 0; + int noLocalIDFoundCount = 0; + for (MapEntry entry + in fileIDtoLocalID.entries) { + final lID = entry.value.$1; + final rID = entry.key; + if (mappedLocalIDs.contains(lID) && remoteIDsWithMapping.contains(rID)) { + bothMappedCount++; + continue; + } else if (mappedLocalIDs.contains(lID)) { + mapLocalCount++; + } else if (remoteIDsWithMapping.contains(rID)) { + mapRemoteCount++; + } else { + unmappedFileIDtoLocalID[rID] = lID; + } + } + if (unmappedFileIDtoLocalID.isEmpty) { + _logger.info("No unmapped remote files found"); + return; + } + + final unclaimedLocalAssets = + localDB.getLocalAssetsInfo(unmappedFileIDtoLocalID.values.toList()); + final rmMappings = []; + for (final entry in unmappedFileIDtoLocalID.entries) { + final remoteFileID = entry.key; + final localID = entry.value; + final localAsset = await unclaimedLocalAssets; + if (!localAsset.containsKey(localID)) { + noLocalIDFoundCount++; + continue; + } + final localAssetInfo = localAsset[localID]!; + final ApiFileItem remoteFile = fileIDtoLocalID[remoteFileID]!.$2; + late bool? isHashMatched; + late bool hasIdMatched; + if (localAssetInfo.hash != null && remoteFile.hash != null) { + isHashMatched = localAssetInfo.hash == remoteFile.hash; + } else { + isHashMatched = null; // hash status unknown + } + if (Platform.isAndroid) { + hasIdMatched = localAssetInfo.id == remoteFile.localID && + remoteFile.deviceFolder == localAssetInfo.relativePath && + localAssetInfo.name == remoteFile.nonEditedTitle; + } else if (Platform.isIOS) { + hasIdMatched = localAssetInfo.id == remoteFile.localID; + } else { + hasIdMatched = false; // Unsupported platform + } + if (!hasIdMatched) { + continue; + } + MatchType? mappingType; + if (isHashMatched == true) { + mappingType = MatchType.deviceHashMatched; + } else if (isHashMatched == null) { + mappingType = MatchType.localID; + } else { + _logger.warning( + "Remote file ${remoteFile.fileID} has localID $localID but hash does not match", + ); + if (kDebugMode) { + throw Exception( + "Remote file ${remoteFile.fileID} has localID $localID but hash does not match", + ); + } + } + if (mappingType != null) { + rmMappings.add( + RLMapping( + remoteUploadID: remoteFileID, + localID: localID, + localCloudID: localAssetInfo.id, + mappingType: mappingType, + ), + ); + } + } + if (rmMappings.isNotEmpty) { + await remoteDB.insertMappings(rmMappings); + } + _logger.info( + "Mapped new ${rmMappings.length} remote files to local assets: " + "existing remoteID to localID: $mapRemoteCount, " + "existing localID to remoteID: $mapLocalCount, " + "existing both mapped: $bothMappedCount, " + "no localID found: $noLocalIDFoundCount", + ); + } + // todo: rewrite this inside collection_file diff service bool _shouldClearCache(EnteFile remoteFile, EnteFile existingFile) { return false; diff --git a/mobile/apps/photos/lib/services/sync/remote_sync_service.dart b/mobile/apps/photos/lib/services/sync/remote_sync_service.dart index 11f1287a8a..a7269eda03 100644 --- a/mobile/apps/photos/lib/services/sync/remote_sync_service.dart +++ b/mobile/apps/photos/lib/services/sync/remote_sync_service.dart @@ -80,6 +80,7 @@ class RemoteSyncService { remoteDiff = RemoteDiffService( _collectionsService, RemoteFileDiffService(NetworkClient.instance.enteDio), + _config, ); Bus.instance.on().listen((event) async { @@ -568,8 +569,6 @@ class RemoteSyncService { } } - - bool _shouldThrottleSync() { return !flagService.enableMobMultiPart || !localSettings.userEnabledMultiplePart;