diff --git a/web/apps/photos/src/pages/gallery.tsx b/web/apps/photos/src/pages/gallery.tsx
index fe51706e55..82e63a52ef 100644
--- a/web/apps/photos/src/pages/gallery.tsx
+++ b/web/apps/photos/src/pages/gallery.tsx
@@ -821,7 +821,7 @@ const Page: React.FC = () => {
         [],
     );
 
-    const handleToggleFavorite = useCallback(
+    const handleFileViewerToggleFavorite = useCallback(
         async (file: EnteFile) => {
             const fileID = file.id;
             const isFavorite = favoriteFileIDs.has(fileID);
@@ -840,7 +840,7 @@ const Page: React.FC = () => {
                 dispatch({ type: "removePendingFavoriteUpdate", fileID });
             }
         },
-        [favoriteFileIDs],
+        [user, favoriteFileIDs],
     );
 
     const handleFileViewerFileVisibilityUpdate = useCallback(
@@ -1121,7 +1121,7 @@ const Page: React.FC = () => {
                     setFilesDownloadProgressAttributesCreator={
                         setFilesDownloadProgressAttributesCreator
                     }
-                    onToggleFavorite={handleToggleFavorite}
+                    onToggleFavorite={handleFileViewerToggleFavorite}
                     onFileVisibilityUpdate={
                         handleFileViewerFileVisibilityUpdate
                     }
diff --git a/web/packages/gallery/components/viewer/photoswipe.ts b/web/packages/gallery/components/viewer/photoswipe.ts
index b6aea80d0a..0dbc902122 100644
--- a/web/packages/gallery/components/viewer/photoswipe.ts
+++ b/web/packages/gallery/components/viewer/photoswipe.ts
@@ -1046,6 +1046,7 @@ export class FileViewerPhotoSwipe {
         const handleToggleFavoriteIfEnabled = () => {
             if (
                 haveUser &&
+                currentAnnotatedFile().annotation.showFavorite &&
                 !delegate.isFavoritePending(currentAnnotatedFile())
             ) {
                 handleToggleFavorite();