From ea587b6ccddf38e25f46bc66d59a0c7e663e8387 Mon Sep 17 00:00:00 2001
From: Neeraj Gupta <254676+ua741@users.noreply.github.com>
Date: Sat, 18 May 2024 05:31:04 +0530
Subject: [PATCH] [mob] Index hidden files

---
 .../face_ml/face_ml_service.dart              | 44 +++++++++----------
 .../machine_learning_settings_page.dart       |  2 +-
 2 files changed, 23 insertions(+), 23 deletions(-)

diff --git a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart
index 58874ed217..b42e12859d 100644
--- a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart
+++ b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart
@@ -573,13 +573,7 @@ class FaceMlService {
     try {
       isImageIndexRunning = true;
       _logger.info('starting image indexing');
-      // final w = (kDebugMode ? EnteWatch('FacesGetAllFiles') : null)?..start();
-      // final uploadedFileIDs = await FilesDB.instance
-      //     .getOwnedFileIDs(Configuration.instance.getUserID()!);
-      // w?.log('getOwnedFileIDs');
-      // final enteFiles =
-      //     await FilesDB.instance.getUploadedFiles(uploadedFileIDs);
-      // w?.log('getUploadedFiles');
+
       final Map<int, int> alreadyIndexedFiles =
           await FaceMLDataDB.instance.getIndexedFileIds();
       // w?.log('getIndexedFileIds');
@@ -596,22 +590,34 @@ class FaceMlService {
       final List<EnteFile> filesWithLocalID = <EnteFile>[];
       final List<EnteFile> filesWithoutLocalID = <EnteFile>[];
       final List<EnteFile> hiddenFiles = <EnteFile>[];
-      // final ignoredCollections =
-      //     CollectionsService.instance.getHiddenCollectionIds();
+      final List<int> allFileIDsToBeIndexed = await getIndexableFileIDs();
+
       for (final EnteFile enteFile in enteFiles) {
+        allFileIDsToBeIndexed.remove(enteFile.uploadedFileID);
         if (_skipAnalysisEnteFile(enteFile, alreadyIndexedFiles)) {
           fileSkippedCount++;
           continue;
         }
-        // if (ignoredCollections.contains(enteFile.collectionID)) {
-        //   hiddenFiles.add(enteFile);
-        // } else
         if ((enteFile.localID ?? '').isEmpty) {
           filesWithoutLocalID.add(enteFile);
         } else {
           filesWithLocalID.add(enteFile);
         }
       }
+      if (allFileIDsToBeIndexed.isNotEmpty) {
+        _logger.info(
+          'Found ${allFileIDsToBeIndexed.length} files outside searchable files',
+        );
+        final List<EnteFile> uploadFiles =
+            await FilesDB.instance.getUploadedFiles(allFileIDsToBeIndexed);
+        for (final EnteFile enteFile in uploadFiles) {
+          if (_skipAnalysisEnteFile(enteFile, alreadyIndexedFiles)) {
+            fileSkippedCount++;
+            continue;
+          }
+          hiddenFiles.add(enteFile);
+        }
+      }
 
       // list of files where files with localID are first
       final sortedBylocalID = <EnteFile>[];
@@ -1318,7 +1324,7 @@ class FaceMlService {
 
     final int alreadyIndexedCount = await FaceMLDataDB.instance
         .getIndexedFileCount(minimumMlVersion: faceMlVersion);
-    final int totalIndexableCount = await getIndexableFilesCount();
+    final int totalIndexableCount = (await getIndexableFileIDs()).length;
     final ratio = alreadyIndexedCount / totalIndexableCount;
 
     w?.log('getIndexedDoneRatio');
@@ -1326,15 +1332,9 @@ class FaceMlService {
     return ratio;
   }
 
-  static Future<int> getIndexableFilesCount() async {
-    // final indexableFileIDs = await FilesDB.instance
-    //     .getOwnedFileIDs(Configuration.instance.getUserID()!);
-    final allFiles = await SearchService.instance.getAllFiles();
-    final indexableFiles = allFiles.where((file) {
-      return file.isUploaded && file.isOwner;
-    }).toList();
-
-    return indexableFiles.length;
+  static Future<List<int>> getIndexableFileIDs() async {
+    return FilesDB.instance
+        .getOwnedFileIDs(Configuration.instance.getUserID()!);
   }
 
   bool _skipAnalysisEnteFile(EnteFile enteFile, Map<int, int> indexedFileIds) {
diff --git a/mobile/lib/ui/settings/machine_learning_settings_page.dart b/mobile/lib/ui/settings/machine_learning_settings_page.dart
index b74b33fe56..75b83d4f12 100644
--- a/mobile/lib/ui/settings/machine_learning_settings_page.dart
+++ b/mobile/lib/ui/settings/machine_learning_settings_page.dart
@@ -434,7 +434,7 @@ class FaceRecognitionStatusWidgetState
   Future<(int, int, int, double)> getIndexStatus() async {
     final indexedFiles = await FaceMLDataDB.instance
         .getIndexedFileCount(minimumMlVersion: faceMlVersion);
-    final indexableFiles = await FaceMlService.getIndexableFilesCount();
+    final indexableFiles = (await FaceMlService.getIndexableFileIDs()).length;
     final showIndexedFiles = min(indexedFiles, indexableFiles);
     final pendingFiles = max(indexableFiles - indexedFiles, 0);
     final foundFaces = await FaceMLDataDB.instance.getTotalFaceCount();