From 01822b760dd2c4803d0c7617724ecaf196ca3d28 Mon Sep 17 00:00:00 2001
From: Neeraj Gupta <254676+ua741@users.noreply.github.com>
Date: Fri, 19 Apr 2024 16:45:29 +0530
Subject: [PATCH] [mob][photos] Dedupe using hash before copying

---
 mobile/lib/db/files_db.dart                  | 26 ++++++++++++++-
 mobile/lib/services/collections_service.dart | 35 +++++++++++++++++++-
 2 files changed, 59 insertions(+), 2 deletions(-)

diff --git a/mobile/lib/db/files_db.dart b/mobile/lib/db/files_db.dart
index 57c16943cc..cdd92de68a 100644
--- a/mobile/lib/db/files_db.dart
+++ b/mobile/lib/db/files_db.dart
@@ -521,7 +521,8 @@ class FilesDB {
   }
 
   Future<(Set<int>, Map<String, int>)> getUploadAndHash(
-      int collectionID,) async {
+    int collectionID,
+  ) async {
     final db = await instance.database;
     final results = await db.query(
       filesTable,
@@ -1045,6 +1046,29 @@ class FilesDB {
     return convertToFiles(rows);
   }
 
+  Future<Map<String, EnteFile>>
+      getUserOwnedFilesWithSameHashForGivenListOfFiles(
+    List<EnteFile> files,
+    int userID,
+  ) async {
+    final db = await sqliteAsyncDB;
+    final List<String> hashes = [];
+    for (final file in files) {
+      if (file.hash != null && file.hash != '') {
+        hashes.add(file.hash!);
+      }
+    }
+    if (hashes.isEmpty) {
+      return {};
+    }
+    final inParam = hashes.map((e) => "'$e'").join(',');
+    final rows = await db.execute('''
+      SELECT * FROM $filesTable WHERE $columnHash IN ($inParam) AND $columnOwnerID = $userID;
+      ''');
+    final matchedFiles = convertToFiles(rows);
+    return Map.fromIterable(matchedFiles, key: (e) => e.hash);
+  }
+
   Future<List<EnteFile>> getUploadedFilesWithHashes(
     FileHashData hashData,
     FileType fileType,
diff --git a/mobile/lib/services/collections_service.dart b/mobile/lib/services/collections_service.dart
index 44777705d0..ad12429207 100644
--- a/mobile/lib/services/collections_service.dart
+++ b/mobile/lib/services/collections_service.dart
@@ -1161,9 +1161,21 @@ class CollectionsService {
       await _addToCollection(dstCollectionID, splitResult.ownedByCurrentUser);
     }
     if (splitResult.ownedByOtherUsers.isNotEmpty) {
+      late final List<EnteFile> filesToCopy;
+      late final List<EnteFile> filesToAdd;
+      (filesToAdd, filesToCopy) = (await _splitFilesToAddAndCopy(
+        splitResult.ownedByOtherUsers,
+      ));
+
+      if (filesToAdd.isNotEmpty) {
+        _logger.info(
+          "found existing ${filesToAdd.length} files with same hash, adding symlinks",
+        );
+        await _addToCollection(dstCollectionID, filesToAdd);
+      }
       // group files by collectionID
       final Map<int, List<EnteFile>> filesByCollection = {};
-      for (final file in splitResult.ownedByOtherUsers) {
+      for (final file in filesToCopy) {
         if (filesByCollection.containsKey(file.collectionID!)) {
           filesByCollection[file.collectionID!]!.add(file);
         } else {
@@ -1365,6 +1377,27 @@ class CollectionsService {
     }
   }
 
+  Future<(List<EnteFile>, List<EnteFile>)> _splitFilesToAddAndCopy(
+    List<EnteFile> othersFile,
+  ) async {
+    final hashToUserFile =
+        await _filesDB.getUserOwnedFilesWithSameHashForGivenListOfFiles(
+      othersFile,
+      _config.getUserID()!,
+    );
+    final List<EnteFile> filesToCopy = [];
+    final List<EnteFile> filesToAdd = [];
+    for (final EnteFile file in othersFile) {
+      if (hashToUserFile.containsKey(file.hash ?? '')) {
+        final userFile = hashToUserFile[file.hash]!;
+        filesToAdd.add(userFile);
+      } else {
+        filesToCopy.add(file);
+      }
+    }
+    return (filesToAdd, filesToCopy);
+  }
+
   void _validateCopyInput(
     int destCollectionID,
     int srcCollectionID,