TerribleDev/ente
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

normal and hidden files storage merge - wip

Browse Source
This commit is contained in:
Manav Rathi
2025-06-25 11:34:14 +05:30
parent 66079b0af2
commit 06a5611a56
5 changed files with 80 additions and 198 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
web/apps/photos/src/pages/gallery.tsx
View File

@@ -76,9 +76,8 @@ import {
import exportService from "ente-new/photos/services/export";
import { updateFilesVisibility } from "ente-new/photos/services/file";
import {
savedCollectionFiles,
savedCollections,
savedHiddenFiles,
savedNormalFiles,
savedTrashItems,
} from "ente-new/photos/services/photos-fdb";
import {
@@ -330,8 +329,7 @@ const Page: React.FC = () => {
user,
familyData,
collections: await savedCollections(),
normalFiles: await savedNormalFiles(),
hiddenFiles: await savedHiddenFiles(),
collectionFiles: await savedCollectionFiles(),
trashItems: await savedTrashItems(),
});
await syncWithRemote({ force: true });
@@ -538,25 +536,12 @@ const Page: React.FC = () => {
*/
const fileAndCollectionSyncWithRemote = useCallback(async () => {
const didUpdateFiles = await syncCollectionAndFiles({
onSetCollections: (
collections,
normalCollections,
hiddenCollections,
) =>
dispatch({
type: "setCollections",
collections,
normalCollections,
hiddenCollections,
}),
onResetNormalFiles: (files) =>
dispatch({ type: "setNormalFiles", files }),
onFetchNormalFiles: (files) =>
dispatch({ type: "fetchNormalFiles", files }),
onResetHiddenFiles: (files) =>
dispatch({ type: "setHiddenFiles", files }),
onFetchHiddenFiles: (files) =>
dispatch({ type: "fetchHiddenFiles", files }),
onSetCollections: (collections) =>
dispatch({ type: "setCollections", collections }),
onSetCollectionFiles: (collectionFiles) =>
dispatch({ type: "setCollectionFiles", collectionFiles }),
onAugmentCollectionFiles: (collectionFiles) =>
dispatch({ type: "augmentCollectionFiles", collectionFiles }),
onSetTrashedItems: (trashItems) =>
dispatch({ type: "setTrashItems", trashItems }),
});
@@ -1082,7 +1067,7 @@ const Page: React.FC = () => {
fileAndCollectionSyncWithRemote
}
onUploadFile={(file) =>
dispatch({ type: "uploadNormalFile", file })
dispatch({ type: "uploadFile", file })
}
onShowPlanSelector={showPlanSelector}
isFirstUpload={areOnlySystemCollections(

71
web/packages/new/photos/components/gallery/reducer.ts
View File

@@ -133,27 +133,7 @@ export interface GalleryState {
* This property is expected to be of use only internal to the reducer;
* external code should only needs {@link files} instead.
*/
lastSyncedFiles: EnteFile[];
/**
* The user's normal (non-hidden, non-trash) files, without any unsynced
* modifications applied to them.
*
* The list is sorted so that newer files are first.
*
* This property is expected to be of use only internal to the reducer;
* external code likely needs {@link normalFiles} instead.
*/
lastSyncedNormalFiles: EnteFile[];
/**
* The user's hidden files, without any unsynced modifications applied to
* them.
*
* The list is sorted so that newer files are first.
*
* This property is expected to be of use only internal to the reducer;
* external code likely needs {@link hiddenFiles} instead.
*/
lastSyncedHiddenFiles: EnteFile[];
lastSyncedCollectionFiles: EnteFile[];
/**
* The items in the user's trash.
*
@@ -190,23 +170,7 @@ export interface GalleryState {
* happen on the next "file sync", until then they remain as in-memory state
* in the reducer.
*/
files: EnteFile[];
/**
* The user's normal (non-hidden, non-trash) files, with any unsynced
* modifications also applied to them.
*
* The list is sorted so that newer files are first.
*/
normalFiles: EnteFile[];
/**
* The user's hidden files, with any unsynced modifications also applied to
* them.
*
* The list is sorted so that newer files are first.
*
* See: [Note: Unsynced modifications]
*/
hiddenFiles: EnteFile[];
collectionFiles: EnteFile[];
/**
* Collection IDs of archived collections.
*/
@@ -446,21 +410,13 @@ export type GalleryAction =
user: User;
familyData: FamilyData;
collections: Collection[];
normalFiles: EnteFile[];
hiddenFiles: EnteFile[];
collectionFiles: EnteFile[];
trashItems: TrashItem[];
}
| {
type: "setCollections";
collections: Collection[];
normalCollections: Collection[];
hiddenCollections: Collection[];
}
| { type: "setNormalFiles"; files: EnteFile[] }
| { type: "fetchNormalFiles"; files: EnteFile[] }
| { type: "uploadNormalFile"; file: EnteFile }
| { type: "setHiddenFiles"; files: EnteFile[] }
| { type: "fetchHiddenFiles"; files: EnteFile[] }
| { type: "setCollections"; collections: Collection[] }
| { type: "setCollectionFiles"; collectionFiles: EnteFile[] }
| { type: "augmentCollectionFiles"; collectionFiles: EnteFile[] }
| { type: "uploadFile"; file: EnteFile }
| { type: "setTrashItems"; trashItems: TrashItem[] }
| { type: "setPeopleState"; peopleState: PeopleState | undefined }
| { type: "markTempDeleted"; files: EnteFile[] }
@@ -495,13 +451,9 @@ const initialGalleryState: GalleryState = {
normalCollections: [],
hiddenCollections: [],
lastSyncedFiles: [],
lastSyncedNormalFiles: [],
lastSyncedHiddenFiles: [],
trashItems: [],
peopleState: undefined,
files: [],
normalFiles: [],
hiddenFiles: [],
archivedCollectionIDs: new Set(),
defaultHiddenCollectionIDs: new Set(),
hiddenFileIDs: new Set(),
@@ -615,6 +567,15 @@ const galleryReducer: React.Reducer<GalleryState, GalleryAction> = (
}
case "setCollections": {
const [hiddenCollections, normalCollections] = splitByPredicate(
collections,
isHiddenCollection,
);
opts?.onSetCollections(
collections,
normalCollections,
hiddenCollections,
);
const { collections, normalCollections, hiddenCollections } =
action;
const archivedCollectionIDs =

48
web/packages/new/photos/services/files.ts
View File

@@ -14,32 +14,7 @@ import {
getCollectionLastSyncTime,
setCollectionLastSyncTime,
} from "./collections";
import {
savedHiddenFiles,
savedNormalFiles,
saveHiddenFiles,
saveNormalFiles,
} from "./photos-fdb";
/**
* Return all files that we know about locally. By default it returns only
* "normal" (i.e. non-"hidden") files, but it can be passed the {@link type}
* "hidden" to get it to instead return hidden files that we know about locally.
*
* Deprecated, use {@link savedNormalFiles} or {@link savedHiddenFiles} instead.
*/
export const getLocalFiles = async (type: "normal" | "hidden" = "normal") =>
type == "normal" ? savedNormalFiles() : savedHiddenFiles();
/**
* Update the files that we know about locally.
*
* Sibling of {@link getLocalFiles}.
*
* Deprecate, use {@link saveNormalFiles} or {@link saveHiddenFiles} instead.
*/
export const setLocalFiles = (type: "normal" | "hidden", files: EnteFile[]) =>
type == "normal" ? saveNormalFiles(files) : saveHiddenFiles(files);
import { saveCollectionFiles, savedCollectionFiles } from "./photos-fdb";
/**
* Fetch all files of the given {@link type}, belonging to the given
@@ -56,18 +31,17 @@ export const setLocalFiles = (type: "normal" | "hidden", files: EnteFile[]) =>
*
* @returns true if one or more files were updated locally, false otherwise.
*/
export const syncFiles = async (
type: "normal" | "hidden",
export const pullCollectionFiles = async (
collections: Collection[],
onResetFiles: ((files: EnteFile[]) => void) | undefined,
onFetchFiles: ((files: EnteFile[]) => void) | undefined,
onSetCollectionFiles: ((files: EnteFile[]) => void) | undefined,
onAugmentCollectionFiles: ((files: EnteFile[]) => void) | undefined,
) => {
const localFiles = await getLocalFiles(type);
const localFiles = await savedCollectionFiles();
let files = removeDeletedCollectionFiles(collections, localFiles);
let didUpdateFiles = false;
if (files.length !== localFiles.length) {
await setLocalFiles(type, files);
onResetFiles?.(files);
await saveCollectionFiles(files);
onSetCollectionFiles?.(files);
didUpdateFiles = true;
}
for (const collection of collections) {
@@ -79,10 +53,14 @@ export const syncFiles = async (
continue;
}
const newFiles = await getFiles(collection, lastSyncTime, onFetchFiles);
const newFiles = await getFiles(
collection,
lastSyncTime,
onAugmentCollectionFiles,
);
await clearCachedThumbnailsIfChanged(localFiles, newFiles);
files = getLatestVersionFiles([...files, ...newFiles]);
await setLocalFiles(type, files);
await saveCollectionFiles(files);
didUpdateFiles = true;
await setCollectionLastSyncTime(collection, collection.updationTime);
}

61
web/packages/new/photos/services/photos-fdb.ts
View File

@@ -138,20 +138,13 @@ export const saveTrashItemCollectionKeys = async (
/**
* Return all files present in our local database.
*
* This includes both normal (non-hidden) and hidden files. If you're interested
* only in one type (normal or hidden), then it is more efficient to use
* {@link savedNormalFiles} or {@link savedHiddenFiles} to obtain them; this
* method is only a convenience to concatenate the two.
*/
export const savedFiles = async (): Promise<EnteFile[]> =>
Promise.all([savedNormalFiles(), savedHiddenFiles()]).then((f) => f.flat());
/**
* Return all normal (non-hidden) files present in our local database.
* These are "collection files", meaning that there might be multiple entries
* for the same file, one for each collection that the file belongs to. For more
* details, See: [Note: Collection File].
*
* Use {@link saveNormalFiles} to update the database.
* Use {@link saveFiles} to update the database.
*/
export const savedNormalFiles = async (): Promise<EnteFile[]> =>
export const savedCollectionFiles = async (): Promise<EnteFile[]> => {
// [Note: Avoiding Zod parsing for large DB arrays]
//
// Zod can be used to validate that the value we read from the DB is indeed
@@ -166,37 +159,31 @@ export const savedNormalFiles = async (): Promise<EnteFile[]> =>
// As an optimization, we skip the runtime check here and cast. This might
// not be the most optimal choice in the future, so (a) use it sparingly,
// and (b) mark all such cases with the title of this note.
transformFilesIfNeeded(
(await localForage.getItem<EnteFile[]>("files")) ?? [],
);
let files = (await localForage.getItem<EnteFile[]>("files")) ?? [];
/**
* Replace the list of files stored in our local database.
*
* This is the setter corresponding to {@link savedNormalFiles}.
*/
export const saveNormalFiles = async (files: EnteFile[]) => {
await localForage.setItem("files", transformFilesIfNeeded(files));
// Previously hidden files were stored separately. If that key is present,
// also read those files, and migrate them (save the concatenation to disk
// and delete the corresponding DB key).
//
// This migration was added Jun 2025, v1.7.14-beta (tag: Migration).
const previousHiddenFiles =
await localForage.getItem<EnteFile[]>("hidden-files");
if (previousHiddenFiles) {
files = files.concat(previousHiddenFiles);
await saveCollectionFiles(files);
await localForage.removeItem("hidden-files");
}
return transformFilesIfNeeded(files);
};
/**
* Return all hidden files present in our local database.
* Replace the list of collection files stored in our local database.
*
* Use {@link saveNormalFiles} to update the database.
* This is the setter corresponding to {@link savedCollectionFiles}.
*/
export const savedHiddenFiles = async (): Promise<EnteFile[]> =>
// See: [Note: Avoiding Zod parsing for large DB arrays]
transformFilesIfNeeded(
(await localForage.getItem<EnteFile[]>("hidden-files")) ?? [],
);
/**
* Replace the list of files stored in our local database.
*
* This is the setter corresponding to {@link savedNormalFiles}.
*/
export const saveHiddenFiles = async (files: EnteFile[]) => {
await localForage.setItem("hidden-files", transformFilesIfNeeded(files));
export const saveCollectionFiles = async (files: EnteFile[]) => {
await localForage.setItem("files", transformFilesIfNeeded(files));
};
/**

65
web/packages/new/photos/services/sync.ts
View File

@@ -5,9 +5,8 @@ import {
} from "ente-gallery/services/video";
import type { Collection } from "ente-media/collection";
import type { EnteFile } from "ente-media/file";
import { isHiddenCollection } from "ente-new/photos/services/collection";
import { pullCollections } from "ente-new/photos/services/collections";
import { syncFiles } from "ente-new/photos/services/files";
import { pullCollectionFiles } from "ente-new/photos/services/files";
import {
isMLSupported,
mlStatusSync,
@@ -15,7 +14,6 @@ import {
} from "ente-new/photos/services/ml";
import { searchDataSync } from "ente-new/photos/services/search";
import { syncSettings } from "ente-new/photos/services/settings";
import { splitByPredicate } from "ente-utils/array";
import { pullTrash, type TrashItem } from "./trash";
/**
@@ -65,39 +63,25 @@ export const postCollectionAndFilesSync = async () => {
void mlSync();
};
interface SyncCallectionAndFilesOpts {
interface PullFilesOpts {
/**
* Called when saved collections, both normal and hidden, are (potentially)
* updated.
* Called when the saved collections were replaced by the given
* {@link collections}.
*
* The callback is passed all the collections (as {@link collections}), and
* also their splits into normal ({@link normalCollections}) and hidden
* ({@link hiddenCollections}).
* The callback is also passed splits of {@link collections} into normal
* ({@link normalCollections}) and hidden ({@link hiddenCollections}).
*/
onSetCollections: (
collections: Collection[],
normalCollections: Collection[],
hiddenCollections: Collection[],
) => void;
onSetCollections: (collections: Collection[]) => void;
/**
* Called when saved normal (non-hidden, non-trash) files were replaced by
* the given {@link files}.
* Called when saved collection files were replaced by the given
* {@link files}.
*/
onResetNormalFiles: (files: EnteFile[]) => void;
onSetCollectionFiles: (files: EnteFile[]) => void;
/**
* Called when saved normal files were augmented with the given newly
* Called when saved collection files were augmented with the given newly
* fetched {@link files}.
*/
onFetchNormalFiles: (files: EnteFile[]) => void;
/**
* Called when saved hidden files were replaced by the given {@link files}.
*/
onResetHiddenFiles: (files: EnteFile[]) => void;
/**
* Called when saved hidden files were augmented with the given newly
* fetched {@link files}.
*/
onFetchHiddenFiles: (files: EnteFile[]) => void;
onAugmentCollectionFiles: (files: EnteFile[]) => void;
/**
* Called when saved trashed items were replaced by the given
* {@link trashItems}.
@@ -122,33 +106,20 @@ interface SyncCallectionAndFilesOpts {
* @returns `true` if one or more normal or hidden files were updated during the
* sync.
*/
export const syncCollectionAndFiles = async (
opts?: SyncCallectionAndFilesOpts,
) => {
export const syncCollectionAndFiles = async (opts?: PullFilesOpts) => {
const collections = await pullCollections();
const [hiddenCollections, normalCollections] = splitByPredicate(
opts?.onSetCollections(collections);
const didUpdateFiles = await pullCollectionFiles(
collections,
isHiddenCollection,
);
opts?.onSetCollections(collections, normalCollections, hiddenCollections);
const didUpdateNormalFiles = await syncFiles(
"normal",
normalCollections,
opts?.onResetNormalFiles,
opts?.onFetchNormalFiles,
);
const didUpdateHiddenFiles = await syncFiles(
"hidden",
hiddenCollections,
opts?.onResetHiddenFiles,
opts?.onFetchHiddenFiles,
opts?.onSetCollectionFiles,
opts?.onAugmentCollectionFiles,
);
await pullTrash(
collections,
opts?.onSetTrashedItems,
videoPrunePermanentlyDeletedFileIDsIfNeeded,
);
if (didUpdateNormalFiles || didUpdateHiddenFiles) {
if (didUpdateFiles) {
// TODO: Ok for now since its is only commented for the deduper (gallery
// does this on the return value), but still needs fixing instead of a
// hidden gotcha. Fix is simple, just uncomment, but that can be done