Map remote files to local during diff sync

mobile/apps/photos/lib/db/local/db.dart

@@ -10,6 +10,7 @@ import "package:photos/db/local/mappers.dart";
 import "package:photos/db/local/schema.dart";
 import "package:photos/log/devlog.dart";
 import 'package:photos/models/file/file.dart';
+import "package:photos/models/file/mapping/local_mapping.dart";
 import "package:photos/models/local/local_metadata.dart";
 import "package:sqlite_async/sqlite_async.dart";
 
@@ -82,6 +83,25 @@ class LocalDB with SqlDbBase {
     }
   }
 
+  Future<Map<String, LocalAssetInfo>> getLocalAssetsInfo(
+    List<String> ids,
+  ) async {
+    if (ids.isEmpty) return {};
+    final stopwatch = Stopwatch()..start();
+    final result = await _sqliteDB.getAll(
+      'SELECT id, hash, name, relative_path, state FROM assets WHERE id IN (${List.filled(ids.length, "?").join(",")})',
+      ids,
+    );
+    debugPrint(
+      "getLocalAssetsInfo complete in ${stopwatch.elapsed.inMilliseconds}ms for ${ids.length} ids",
+    );
+    return Map.fromEntries(
+      result.map(
+        (row) => MapEntry(row['id'] as String, LocalAssetInfo.fromRow(row)),
+      ),
+    );
+  }
+
   Future<List<EnteFile>> getAssets({LocalAssertsParam? params}) async {
     final stopwatch = Stopwatch()..start();
     final result = await _sqliteDB.getAll(

mobile/apps/photos/lib/db/remote/table/mapping_table.dart

@@ -41,4 +41,24 @@ extension UploadMappingTable on RemoteDB {
     }
     return result;
   }
+
+  Future<Set<String>> getLocalIDsWithMapping(List<String> localIDs) async {
+    if (localIDs.isEmpty) return {};
+    final placeholders = List.filled(localIDs.length, '?').join(',');
+    final cursor = await sqliteDB.getAll(
+      'SELECT local_id FROM upload_mapping join files on upload_mapping.file_id = files.id WHERE local_id IN ($placeholders)',
+      localIDs,
+    );
+    return cursor.map((row) => row['local_id'] as String).toSet();
+  }
+
+  Future<Set<int>> getFilesWithMapping(List<int> fileIDs) async {
+    if (fileIDs.isEmpty) return {};
+    final placeholders = List.filled(fileIDs.length, '?').join(',');
+    final cursor = await sqliteDB.getAll(
+      'SELECT file_id FROM upload_mapping WHERE file_id IN ($placeholders)',
+      fileIDs,
+    );
+    return cursor.map((row) => row['file_id'] as int).toSet();
+  }
 }

mobile/apps/photos/lib/models/api/diff/diff.dart

@@ -134,6 +134,10 @@ class ApiFileItem {
   String get title =>
       pubMagicMetadata?.data['editedName'] ?? metadata?.data['title'] ?? "";
 
+  String get nonEditedTitle {
+    return metadata?.data['title'] ?? "";
+  }
+
   String? get localID => metadata?.data['localID'];
 
   String? get matchLocalID => localID == null || deviceFolder == null

mobile/apps/photos/lib/models/file/mapping/local_mapping.dart

@@ -0,0 +1,25 @@
+class LocalAssetInfo {
+  final String id;
+  final String? hash;
+  final String? name;
+  final String? relativePath;
+  final int state;
+
+  LocalAssetInfo({
+    required this.id,
+    this.hash,
+    this.name,
+    this.relativePath,
+    required this.state,
+  });
+
+  factory LocalAssetInfo.fromRow(Map<String, Object?> row) {
+    return LocalAssetInfo(
+      id: row['id'] as String,
+      hash: row['hash'] as String?,
+      name: row['name'] as String?,
+      relativePath: row['relative_path'] as String?,
+      state: row['state'] as int,
+    );
+  }
+}

mobile/apps/photos/lib/models/file/remote/rl_mapping.dart

@@ -15,13 +15,13 @@ class RLMapping {
         remoteUploadID,
         localID,
         localCloudID,
-        mappingType,
+        mappingType.name,
       ];
 }
 
 enum MatchType {
-  remote,
-  cloudIdMatched,
+  localID,
+  cloudID,
   deviceUpload,
   deviceHashMatched,
 }
@@ -29,10 +29,10 @@ enum MatchType {
 extension MappingTypeExtension on MatchType {
   String get name {
     switch (this) {
-      case MatchType.remote:
-        return "remote";
-      case MatchType.cloudIdMatched:
-        return "cloudIdMatched";
+      case MatchType.localID:
+        return "localID";
+      case MatchType.cloudID:
+        return "cloudID";
       case MatchType.deviceUpload:
         return "deviceUpload";
       case MatchType.deviceHashMatched:
@@ -42,10 +42,10 @@ extension MappingTypeExtension on MatchType {
 
   static MatchType fromName(String name) {
     switch (name) {
-      case "remote":
-        return MatchType.remote;
-      case "cloudIdMatched":
-        return MatchType.cloudIdMatched;
+      case "localID":
+        return MatchType.localID;
+      case "cloudID":
+        return MatchType.cloudID;
       case "deviceUpload":
         return MatchType.deviceUpload;
       case "deviceHashMatched":

mobile/apps/photos/lib/services/remote/fetch/remote_diff.dart

@@ -1,10 +1,16 @@
+import "dart:io";
+
 import "package:flutter/foundation.dart";
 import "package:logging/logging.dart";
+import "package:photos/core/configuration.dart";
 import "package:photos/core/event_bus.dart";
+import "package:photos/db/remote/table/mapping_table.dart";
 import "package:photos/events/collection_updated_event.dart";
 import "package:photos/events/diff_sync_complete_event.dart";
 import "package:photos/events/sync_status_update_event.dart";
+import "package:photos/models/api/diff/diff.dart";
 import "package:photos/models/file/file.dart";
+import "package:photos/models/file/remote/rl_mapping.dart";
 import "package:photos/service_locator.dart";
 import "package:photos/services/collections_service.dart";
 import "package:photos/services/remote/fetch/files_diff.dart";
@@ -13,10 +19,11 @@ class RemoteDiffService {
   final Logger _logger = Logger('RemoteDiffService');
   final CollectionsService _collectionsService;
   final RemoteFileDiffService filesDiffService;
-
+  final Configuration _config;
   RemoteDiffService(
     this._collectionsService,
     this.filesDiffService,
+    this._config,
   );
 
   bool _isExistingSyncSilent = false;
@@ -79,6 +86,7 @@ class RemoteDiffService {
         await remoteDB.deleteFilesDiff(diff.deletedItems);
       }
       if (diff.updatedItems.isNotEmpty) {
+        await _mapRemoteToLocalItems(diff);
         await remoteCache.insertDiffItems(diff.updatedItems);
       }
       // todo:(rewrite) neeraj add logic to refresh home gallery when time or visibility changes
@@ -104,6 +112,120 @@ class RemoteDiffService {
     );
   }
 
+  Future<void> _mapRemoteToLocalItems(DiffResult diff) async {
+    final Map<int, (String, ApiFileItem)> fileIDtoLocalID = {};
+    final Map<int, String> unmappedFileIDtoLocalID = {};
+    for (final item in diff.updatedItems) {
+      if (item.fileItem.localID != null &&
+          item.fileItem.ownerID == _config.getUserID()!) {
+        fileIDtoLocalID[item.fileItem.fileID] =
+            (item.fileItem.localID!, item.fileItem);
+      }
+    }
+    if (fileIDtoLocalID.isEmpty) {
+      _logger.info("No remote files to map to local items");
+      return;
+    }
+    final mappedLocalIDs = await remoteDB.getLocalIDsWithMapping(
+      fileIDtoLocalID.values.map((e) => e.$1).toList(),
+    );
+    final remoteIDsWithMapping =
+        await remoteDB.getFilesWithMapping(fileIDtoLocalID.keys.toList());
+    // remote already claim mappings from fileIds to localIDs
+    int mapRemoteCount = 0;
+    int mapLocalCount = 0;
+    int bothMappedCount = 0;
+    int noLocalIDFoundCount = 0;
+    for (MapEntry<int, (String, ApiFileItem)> entry
+        in fileIDtoLocalID.entries) {
+      final lID = entry.value.$1;
+      final rID = entry.key;
+      if (mappedLocalIDs.contains(lID) && remoteIDsWithMapping.contains(rID)) {
+        bothMappedCount++;
+        continue;
+      } else if (mappedLocalIDs.contains(lID)) {
+        mapLocalCount++;
+      } else if (remoteIDsWithMapping.contains(rID)) {
+        mapRemoteCount++;
+      } else {
+        unmappedFileIDtoLocalID[rID] = lID;
+      }
+    }
+    if (unmappedFileIDtoLocalID.isEmpty) {
+      _logger.info("No unmapped remote files found");
+      return;
+    }
+
+    final unclaimedLocalAssets =
+        localDB.getLocalAssetsInfo(unmappedFileIDtoLocalID.values.toList());
+    final rmMappings = <RLMapping>[];
+    for (final entry in unmappedFileIDtoLocalID.entries) {
+      final remoteFileID = entry.key;
+      final localID = entry.value;
+      final localAsset = await unclaimedLocalAssets;
+      if (!localAsset.containsKey(localID)) {
+        noLocalIDFoundCount++;
+        continue;
+      }
+      final localAssetInfo = localAsset[localID]!;
+      final ApiFileItem remoteFile = fileIDtoLocalID[remoteFileID]!.$2;
+      late bool? isHashMatched;
+      late bool hasIdMatched;
+      if (localAssetInfo.hash != null && remoteFile.hash != null) {
+        isHashMatched = localAssetInfo.hash == remoteFile.hash;
+      } else {
+        isHashMatched = null; // hash status unknown
+      }
+      if (Platform.isAndroid) {
+        hasIdMatched = localAssetInfo.id == remoteFile.localID &&
+            remoteFile.deviceFolder == localAssetInfo.relativePath &&
+            localAssetInfo.name == remoteFile.nonEditedTitle;
+      } else if (Platform.isIOS) {
+        hasIdMatched = localAssetInfo.id == remoteFile.localID;
+      } else {
+        hasIdMatched = false; // Unsupported platform
+      }
+      if (!hasIdMatched) {
+        continue;
+      }
+      MatchType? mappingType;
+      if (isHashMatched == true) {
+        mappingType = MatchType.deviceHashMatched;
+      } else if (isHashMatched == null) {
+        mappingType = MatchType.localID;
+      } else {
+        _logger.warning(
+          "Remote file ${remoteFile.fileID} has localID $localID but hash does not match",
+        );
+        if (kDebugMode) {
+          throw Exception(
+            "Remote file ${remoteFile.fileID} has localID $localID but hash does not match",
+          );
+        }
+      }
+      if (mappingType != null) {
+        rmMappings.add(
+          RLMapping(
+            remoteUploadID: remoteFileID,
+            localID: localID,
+            localCloudID: localAssetInfo.id,
+            mappingType: mappingType,
+          ),
+        );
+      }
+    }
+    if (rmMappings.isNotEmpty) {
+      await remoteDB.insertMappings(rmMappings);
+    }
+    _logger.info(
+      "Mapped new ${rmMappings.length} remote files to local assets: "
+      "existing remoteID to localID: $mapRemoteCount, "
+      "existing localID to remoteID: $mapLocalCount, "
+      "existing both mapped: $bothMappedCount, "
+      "no localID found: $noLocalIDFoundCount",
+    );
+  }
+
   // todo: rewrite this inside collection_file diff service
   bool _shouldClearCache(EnteFile remoteFile, EnteFile existingFile) {
     return false;

mobile/apps/photos/lib/services/sync/remote_sync_service.dart

@@ -80,6 +80,7 @@ class RemoteSyncService {
     remoteDiff = RemoteDiffService(
       _collectionsService,
       RemoteFileDiffService(NetworkClient.instance.enteDio),
+      _config,
     );
 
     Bus.instance.on<LocalPhotosUpdatedEvent>().listen((event) async {
@@ -568,8 +569,6 @@ class RemoteSyncService {
     }
   }
 
- 
-
   bool _shouldThrottleSync() {
     return !flagService.enableMobMultiPart ||
         !localSettings.userEnabledMultiplePart;