From 090e8bbf40fa40e3d2020dadbf5c583dd853d47a Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 26 Jun 2025 18:16:44 +0530 Subject: [PATCH] Simplify --- web/apps/photos/src/pages/gallery.tsx | 13 +-- .../new/photos/components/gallery/reducer.ts | 56 +---------- .../new/photos/services/collection.ts | 92 ++----------------- web/packages/new/photos/services/pull.ts | 25 +++-- 4 files changed, 31 insertions(+), 155 deletions(-) diff --git a/web/apps/photos/src/pages/gallery.tsx b/web/apps/photos/src/pages/gallery.tsx index 04847b0ee6..e73398c2f4 100644 --- a/web/apps/photos/src/pages/gallery.tsx +++ b/web/apps/photos/src/pages/gallery.tsx @@ -80,9 +80,9 @@ import { savedTrashItems, } from "ente-new/photos/services/photos-fdb"; import { + postPullFiles, + prePullFiles, pullFiles, - pullFilesPost, - pullFilesPre, } from "ente-new/photos/services/pull"; import { filterSearchableFiles, @@ -501,11 +501,6 @@ const Page: React.FC = () => { type: "setCollectionFiles", collectionFiles, }), - onAugmentCollectionFiles: (collectionFiles) => - dispatch({ - type: "augmentCollectionFiles", - collectionFiles, - }), onSetTrashedItems: (trashItems) => dispatch({ type: "setTrashItems", trashItems }), onDidUpdateCollectionFiles: () => @@ -549,9 +544,9 @@ const Page: React.FC = () => { // The pull itself. try { if (!silent) showLoadingBar(); - await pullFilesPre(); + await prePullFiles(); await remoteFilesPull(); - await pullFilesPost(); + await postPullFiles(); } catch (e) { log.error("Remote pull failed", e); } finally { diff --git a/web/packages/new/photos/components/gallery/reducer.ts b/web/packages/new/photos/components/gallery/reducer.ts index 244f7fe4e2..1755d45c40 100644 --- a/web/packages/new/photos/components/gallery/reducer.ts +++ b/web/packages/new/photos/components/gallery/reducer.ts @@ -17,7 +17,6 @@ import { import type { MagicMetadata } from "ente-media/magic-metadata"; import { createCollectionNameByID, - getLatestVersionFiles, isHiddenCollection, } from "ente-new/photos/services/collection"; import { sortTrashItems, type TrashItem } from "ente-new/photos/services/trash"; @@ -452,7 +451,6 @@ export type GalleryAction = } | { type: "setCollections"; collections: Collection[] } | { type: "setCollectionFiles"; collectionFiles: EnteFile[] } - | { type: "augmentCollectionFiles"; collectionFiles: EnteFile[] } | { type: "uploadFile"; file: EnteFile } | { type: "setTrashItems"; trashItems: TrashItem[] } | { type: "setPeopleState"; peopleState: PeopleState | undefined } @@ -709,46 +707,13 @@ const galleryReducer: React.Reducer = ( } case "setCollectionFiles": { - const unsyncedPrivateMagicMetadataUpdates = - prunedUnsyncedPrivateMagicMetadataUpdates( - state.unsyncedPrivateMagicMetadataUpdates, - action.collectionFiles, - ); const lastSyncedCollectionFiles = sortFiles(action.collectionFiles); - const collectionFiles = deriveCollectionFiles( - lastSyncedCollectionFiles, - unsyncedPrivateMagicMetadataUpdates, - ); + const collectionFiles = lastSyncedCollectionFiles; return stateByUpdatingFilteredFiles({ ...stateForUpdatedCollectionFiles(state, collectionFiles), lastSyncedCollectionFiles, - unsyncedPrivateMagicMetadataUpdates, - }); - } - - case "augmentCollectionFiles": { - const unsyncedPrivateMagicMetadataUpdates = - prunedUnsyncedPrivateMagicMetadataUpdates( - state.unsyncedPrivateMagicMetadataUpdates, - action.collectionFiles, - ); - const lastSyncedCollectionFiles = sortFiles( - getLatestVersionFiles( - state.lastSyncedCollectionFiles.concat( - action.collectionFiles, - ), - ), - ); - const collectionFiles = deriveCollectionFiles( - lastSyncedCollectionFiles, - unsyncedPrivateMagicMetadataUpdates, - ); - - return stateByUpdatingFilteredFiles({ - ...stateForUpdatedCollectionFiles(state, collectionFiles), - lastSyncedCollectionFiles, - unsyncedPrivateMagicMetadataUpdates, + unsyncedPrivateMagicMetadataUpdates: new Map(), }); } @@ -1633,23 +1598,6 @@ const deriveHiddenAlbumsViewAndSelectedID = ( }; }; -/** - * Prune any entries for which we have newer remote data (as determined by their - * presence in the given {@link updatedFiles}) from the given unsynced private - * magic metadata {@link updates}. - */ -const prunedUnsyncedPrivateMagicMetadataUpdates = ( - updates: GalleryState["unsyncedPrivateMagicMetadataUpdates"], - updatedFiles: EnteFile[], -) => { - // Fastpath for happy case. - if (updates.size == 0) return updates; - - const prunedUpdates = new Map(updates); - for (const { id } of updatedFiles) prunedUpdates.delete(id); - return prunedUpdates; -}; - /** * Compute the {@link GalleryView} from its dependencies when we are switching * to (or back to) the "people" view. diff --git a/web/packages/new/photos/services/collection.ts b/web/packages/new/photos/services/collection.ts index f5e0f4aea2..4f1dff1de7 100644 --- a/web/packages/new/photos/services/collection.ts +++ b/web/packages/new/photos/services/collection.ts @@ -8,7 +8,6 @@ import { generateKey, } from "ente-base/crypto"; import { authenticatedRequestHeaders, ensureOk } from "ente-base/http"; -import log from "ente-base/log"; import { apiURL } from "ente-base/origins"; import { ensureMasterKeyFromSession } from "ente-base/session"; import { @@ -29,15 +28,12 @@ import { decryptRemoteFile, FileDiffResponse, type EnteFile, - type RemoteEnteFile, } from "ente-media/file"; import { ItemVisibility, metadataHash } from "ente-media/file-metadata"; import { createMagicMetadata, encryptMagicMetadata, } from "ente-media/magic-metadata"; -import HTTPService from "ente-shared/network/HTTPService"; -import { getToken } from "ente-shared/storage/localStorage/helpers"; import { batch, splitByPredicate } from "ente-utils/array"; import { z } from "zod/v4"; import { requestBatchSize, type UpdateMagicMetadataRequest } from "./file"; @@ -290,35 +286,12 @@ const getCollections = async ( ); }; -export function getLatestVersionFiles(files: EnteFile[]) { - const latestVersionFiles = new Map(); - files.forEach((file) => { - const uid = `${file.collectionID}-${file.id}`; - const existingFile = latestVersionFiles.get(uid); - if (!existingFile || existingFile.updationTime < file.updationTime) { - latestVersionFiles.set(uid, file); - } - }); - return Array.from(latestVersionFiles.values()).filter( - // TODO(RE): - // (file) => !file.isDeleted, - (file) => !("isDeleted" in file && file.isDeleted), - ); -} /** * Fetch all files from remote and update our local database. * - * If this is the initial read, or if the count of files we have differs from - * the state of the local database (these two are expected to be the same case), - * then the {@link onSetCollectionFiles} callback is first invoked to give the - * caller a chance to bring its state up to speed. - * - * Then it calls {@link onAugmentCollectionFiles} as each batch of updates are - * fetched (in addition to updating the local database). - * - * The callbacks are optional because we might be called in a context where we - * just want to update the local database, and there is no other in-memory state - * we need to keep in sync. + * Each time it updates the local database, the {@link onSetCollectionFiles} + * callback is also invoked to give the caller a chance to bring its own + * in-memory state up to speed. * * @param collections The user's collections. These are assumed to be the latest * collections on remote (that is, the pull for collections should happen prior @@ -327,16 +300,18 @@ export function getLatestVersionFiles(files: EnteFile[]) { * @param onSetCollectionFiles An optional callback invoked when the locally * saved collection files were replaced by the provided {@link collectionFiles}. * - * @param onAugmentCollectionFiles An optional callback when locally saved - * collection files were augmented with the provided newly fetched - * {@link collectionFiles}. + * The callback is optional because we might be called in a context where we + * just want to update the local database, and there is no other in-memory state + * we need to keep in sync. + * + * The callback can be invoked multiple times for each pull (once for each batch + * of changes received, for each collection that was updated). * * @returns true if one or more files were updated locally, false otherwise. */ export const pullCollectionFiles = async ( collections: Collection[], onSetCollectionFiles: ((files: EnteFile[]) => void) | undefined, - onAugmentCollectionFiles: ((files: EnteFile[]) => void) | undefined, ) => { let didUpdateFiles = false; @@ -398,7 +373,7 @@ export const pullCollectionFiles = async ( await saveCollectionFiles(files); await saveCollectionLastSyncTime(collection, sinceTime); - onAugmentCollectionFiles?.(files); + onSetCollectionFiles?.(files); didUpdateFiles = true; if (!hasMore) break; @@ -439,53 +414,6 @@ const getCollectionDiff = async (collectionID: number, sinceTime: number) => { return FileDiffResponse.parse(await res.json()); }; -export const getFiles = async ( - collection: Collection, - sinceTime: number, - onFetchFiles: ((fs: EnteFile[]) => void) | undefined, -): Promise => { - try { - let decryptedFiles: EnteFile[] = []; - let time = sinceTime; - let resp; - do { - const token = getToken(); - if (!token) { - break; - } - resp = await HTTPService.get( - await apiURL("/collections/v2/diff"), - { collectionID: collection.id, sinceTime: time }, - { "X-Auth-Token": token }, - ); - - const newDecryptedFilesBatch = await Promise.all( - // eslint-disable-next-line @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access - resp.data.diff.map(async (file: RemoteEnteFile) => { - if (!file.isDeleted) { - return await decryptRemoteFile(file, collection.key); - } else { - return file; - } - }) as Promise[], - ); - decryptedFiles = [...decryptedFiles, ...newDecryptedFilesBatch]; - - onFetchFiles?.(decryptedFiles); - // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access - if (resp.data.diff.length) { - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access - time = resp.data.diff.slice(-1)[0].updationTime; - } - // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access - } while (resp.data.hasMore); - return decryptedFiles; - } catch (e) { - log.error("Get files failed", e); - throw e; - } -}; - /** * Clear cached thumbnail of an existing file if the thumbnail data has changed. * diff --git a/web/packages/new/photos/services/pull.ts b/web/packages/new/photos/services/pull.ts index a9ca76e198..485b6d9d3a 100644 --- a/web/packages/new/photos/services/pull.ts +++ b/web/packages/new/photos/services/pull.ts @@ -25,9 +25,9 @@ import { pullTrash, type TrashItem } from "./trash"; * * The full pull is performed by the gallery page, in the following sequence: * - * 1. {@link pullFilesPre} + * 1. {@link prePullFiles} * 2. {@link pullFiles} - * 3. {@link pullFilesPost}. + * 3. {@link postPullFiles}. * * The full pull is performed in the following cases: * @@ -41,7 +41,7 @@ import { pullTrash, type TrashItem } from "./trash"; * independently. For example, after deduping files, or updating the metadata of * a file. See also: [Note: Full remote pull vs files pull] */ -export const pullFilesPre = async () => { +export const prePullFiles = async () => { await Promise.all([pullSettings(), isMLSupported && pullMLStatus()]); }; @@ -49,25 +49,31 @@ interface PullFilesOpts { /** * Called when the saved collections were replaced by the given * {@link collections}. + * + * Can be called multiple times during a pull, as each batch of changes is + * received and processed. */ onSetCollections: (collections: Collection[]) => void; /** * Called when saved collection files were replaced by the given * {@link collectionFiles}. + * + * Can be called multiple times during a pull, as each batch of changes is + * received and processed. */ onSetCollectionFiles: (collectionFiles: EnteFile[]) => void; - /** - * Called when saved collection files were augmented with the given newly - * fetched {@link collectionFiles}. - */ - onAugmentCollectionFiles: (collectionFiles: EnteFile[]) => void; /** * Called when saved trashed items were replaced by the given * {@link trashItems}. + * + * Can be called multiple times during a pull, as each batch of changes is + * received and processed. */ onSetTrashedItems: (trashItems: TrashItem[]) => void; /** * Called if one or more files were updated during the pull. + * + * Will be called at most once per pull. */ onDidUpdateCollectionFiles: () => void; } @@ -95,7 +101,6 @@ export const pullFiles = async (opts?: PullFilesOpts) => { const didUpdateFiles = await pullCollectionFiles( collections, opts?.onSetCollectionFiles, - opts?.onAugmentCollectionFiles, ); await pullTrash( collections, @@ -120,7 +125,7 @@ export const pullFiles = async (opts?: PullFilesOpts) => { * * See: [Note: Remote pull] */ -export const pullFilesPost = async () => { +export const postPullFiles = async () => { await Promise.all([searchDataSync(), videoProcessingSyncIfNeeded()]); // ML sync might take a very long time for initial indexing, so don't wait // for it to finish.