diff --git a/mobile/lib/db/ml/db.dart b/mobile/lib/db/ml/db.dart
index 1196bf5689..b3e3fa232a 100644
--- a/mobile/lib/db/ml/db.dart
+++ b/mobile/lib/db/ml/db.dart
@@ -17,14 +17,17 @@ import "package:photos/services/machine_learning/ml_result.dart";
 import "package:photos/utils/ml_util.dart";
 import 'package:sqlite_async/sqlite_async.dart';
 
-/// Stores all data for the FacesML-related features. The database can be accessed by `MLDataDB.instance.database`.
+/// Stores all data for the ML related features. The database can be accessed by `MLDataDB.instance.database`.
 ///
 /// This includes:
 /// [facesTable] - Stores all the detected faces and its embeddings in the images.
-/// [createFaceClustersTable] - Stores all the mappings from the faces (faceID) to the clusters (clusterID).
+/// [faceClustersTable] - Stores all the mappings from the faces (faceID) to the clusters (clusterID).
 /// [clusterPersonTable] - Stores all the clusters that are mapped to a certain person.
 /// [clusterSummaryTable] - Stores a summary of each cluster, containg the mean embedding and the number of faces in the cluster.
 /// [notPersonFeedback] - Stores the clusters that are confirmed not to belong to a certain person by the user
+///
+/// [clipTable] - Stores the embeddings of the CLIP model
+/// [fileDataTable] - Stores data about the files that are already processed by the ML models
 class MLDataDB {
   static final Logger _logger = Logger("MLDataDB");
 
@@ -572,6 +575,19 @@ class MLDataDB {
     await db.executeBatch(sql, parameterSets);
   }
 
+  Future<void> removeFaceIdToClusterId(
+    Map<String, String> faceIDToClusterID,
+  ) async {
+    final db = await instance.asyncDB;
+    const String sql = '''
+      DELETE FROM $faceClustersTable
+      WHERE $faceIDColumn = ? AND $clusterIDColumn = ?
+    ''';
+    final parameterSets =
+        faceIDToClusterID.entries.map((e) => [e.key, e.value]).toList();
+    await db.executeBatch(sql, parameterSets);
+  }
+
   Future<void> removePerson(String personID) async {
     final db = await instance.asyncDB;
 
diff --git a/mobile/lib/services/machine_learning/face_ml/person/person_service.dart b/mobile/lib/services/machine_learning/face_ml/person/person_service.dart
index a562fea997..7a9b7ee7b5 100644
--- a/mobile/lib/services/machine_learning/face_ml/person/person_service.dart
+++ b/mobile/lib/services/machine_learning/face_ml/person/person_service.dart
@@ -8,7 +8,6 @@ import "package:photos/db/ml/db.dart";
 import "package:photos/events/people_changed_event.dart";
 import "package:photos/extensions/stop_watch.dart";
 import "package:photos/models/api/entity/type.dart";
-import "package:photos/models/base/id.dart";
 import "package:photos/models/file/file.dart";
 import 'package:photos/models/ml/face/face.dart';
 import "package:photos/models/ml/face/person.dart";
@@ -278,8 +277,10 @@ class PersonService {
     entities.sort((a, b) => a.updatedAt.compareTo(b.updatedAt));
     final Map<String, String> faceIdToClusterID = {};
     final Map<String, String> clusterToPersonID = {};
+    bool shouldCheckRejectedFaces = false;
     for (var e in entities) {
       final personData = PersonData.fromJson(json.decode(e.data));
+      if (personData.rejectedFaceIDs != null) shouldCheckRejectedFaces = true;
       int faceCount = 0;
 
       // Locally store the assignment of faces to clusters and people
@@ -308,56 +309,60 @@ class PersonService {
           "Person ${e.id} ${personData.name} has ${personData.assigned!.length} clusters with $faceCount faces",
         );
       }
-
-      // Locally store the rejection of faces to a person
-      if (personData.rejectedFaceIDs != null) {
-        final personFaceIDs = await faceMLDataDB.getFaceIDsForPerson(e.id);
-        final rejectedFaceIDsSet = personData.rejectedFaceIDs!.toSet();
-        final remotelyRejectedFaceIDs =
-            rejectedFaceIDsSet.intersection(personFaceIDs);
-        if (remotelyRejectedFaceIDs.isNotEmpty) {
-          logger.info(
-            "Person ${e.id} ${personData.name} has ${remotelyRejectedFaceIDs.length} rejected faces",
-          );
-
-          // Check that we don't have any empty clusters now
-          final dbPersonClusterInfo =
-              await faceMLDataDB.getClusterIdToFaceIdsForPerson(e.id);
-          for (final clusterIdToFaceIDs in dbPersonClusterInfo.entries) {
-            final clusterID = clusterIdToFaceIDs.key;
-            final faceIDs = clusterIdToFaceIDs.value;
-            final foundRejectedFaces = <String>[];
-            for (final faceID in faceIDs) {
-              if (remotelyRejectedFaceIDs.contains(faceID)) {
-                faceIDs.remove(faceID);
-                foundRejectedFaces.add(faceID);
-              }
-            }
-            if (faceIDs.isEmpty) {
-              logger.info(
-                "Cluster $clusterID for person ${e.id} ${personData.name} is empty due to rejected faces from remote, removing the cluster from person",
-              );
-              await faceMLDataDB.removeClusterToPerson(
-                personID: e.id,
-                clusterID: clusterID,
-              );
-              await faceMLDataDB.captureNotPersonFeedback(
-                personID: e.id,
-                clusterID: clusterID,
-              );
-              remotelyRejectedFaceIDs.removeAll(foundRejectedFaces);
-            }
-          }
-          // Assign rejected faces to new clusters
-          for (final faceId in remotelyRejectedFaceIDs) {
-            faceIdToClusterID[faceId] = newClusterID();
-          }
-        }
-      }
     }
     logger.info("Storing feedback for ${faceIdToClusterID.length} faces");
     await faceMLDataDB.updateFaceIdToClusterId(faceIdToClusterID);
     await faceMLDataDB.bulkAssignClusterToPersonID(clusterToPersonID);
+
+    if (shouldCheckRejectedFaces) {
+      final dbPeopleClusterInfo =
+          await faceMLDataDB.getPersonToClusterIdToFaceIds();
+      for (var e in entities) {
+        final personData = PersonData.fromJson(json.decode(e.data));
+        if (personData.rejectedFaceIDs != null) {
+          final personFaceIDs =
+              dbPeopleClusterInfo[e.id]!.values.expand((e) => e).toSet();
+          final rejectedFaceIDsSet = personData.rejectedFaceIDs!.toSet();
+          final assignedAndRejectedFaceIDs =
+              rejectedFaceIDsSet.intersection(personFaceIDs);
+
+          if (assignedAndRejectedFaceIDs.isNotEmpty) {
+            // Check that we don't have any empty clusters now
+            final dbPersonClusterInfo = dbPeopleClusterInfo[e.id]!;
+            final faceToClusterToRemove = <String, String>{};
+            for (final clusterIdToFaceIDs in dbPersonClusterInfo.entries) {
+              final clusterID = clusterIdToFaceIDs.key;
+              final faceIDs = clusterIdToFaceIDs.value;
+              final foundRejectedFacesToCluster = <String, String>{};
+              for (final faceID in faceIDs) {
+                if (assignedAndRejectedFaceIDs.contains(faceID)) {
+                  faceIDs.remove(faceID);
+                  foundRejectedFacesToCluster[faceID] = clusterID;
+                }
+              }
+              if (faceIDs.isEmpty) {
+                logger.info(
+                  "Cluster $clusterID for person ${e.id} ${personData.name} is empty due to rejected faces from remote, removing the cluster from person",
+                );
+                await faceMLDataDB.removeClusterToPerson(
+                  personID: e.id,
+                  clusterID: clusterID,
+                );
+                await faceMLDataDB.captureNotPersonFeedback(
+                  personID: e.id,
+                  clusterID: clusterID,
+                );
+              } else {
+                faceToClusterToRemove.addAll(foundRejectedFacesToCluster);
+              }
+            }
+            // Remove the clusterID for the remaining conflicting faces
+            await faceMLDataDB.removeFaceIdToClusterId(faceToClusterToRemove);
+          }
+        }
+      }
+    }
+
     return changed;
   }