TerribleDev/ente
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[web] Systematize trash internals (#6350)

Browse Source
This commit is contained in:
Manav Rathi
2025-06-24 08:56:16 +05:30
committed by GitHub
parent e603853e52 fbd0bf257e
commit a7a69aa558
12 changed files with 413 additions and 310 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
web/apps/cast/src/services/render.ts
View File

@@ -150,11 +150,14 @@ export const imageURLGenerator = async function* (castData: CastData) {
*
* @param castToken A token used both for authentication, and also identifying
* the collection corresponding to the cast session.
*
* @returns All the files in the collection in an arbitrary order. Since we are
* anyways going to be shuffling these files, the order has no bearing.
*/
export const getRemoteCastCollectionFiles = async (
castToken: string,
): Promise<RemoteEnteFile[]> => {
const files: RemoteEnteFile[] = [];
const filesByID = new Map<number, RemoteEnteFile>();
let sinceTime = 0;
while (true) {
const res = await fetch(
@@ -167,12 +170,12 @@ export const getRemoteCastCollectionFiles = async (
for (const change of diff) {
sinceTime = Math.max(sinceTime, change.updationTime);
if (!change.isDeleted) {
files.push(change);
filesByID.set(change.id, change);
}
}
if (!hasMore) break;
}
return files;
return [...filesByID.values()];
};
const isFileEligible = (file: EnteFile) => {

6
web/apps/photos/src/components/Collections/CollectionHeader.tsx
View File

@@ -48,10 +48,7 @@ import {
type CollectionSummary,
type CollectionSummaryType,
} from "ente-new/photos/services/collection-summary";
import {
clearLocalTrash,
emptyTrash,
} from "ente-new/photos/services/collections";
import { emptyTrash } from "ente-new/photos/services/trash";
import { usePhotosAppContext } from "ente-new/photos/types/context";
import { t } from "i18next";
import { GalleryContext } from "pages/gallery";
@@ -227,7 +224,6 @@ const CollectionOptions: React.FC<CollectionHeaderProps> = ({
const doEmptyTrash = wrap(async () => {
await emptyTrash();
await clearLocalTrash();
setActiveCollectionID(PseudoCollectionID.all);
});

37
web/apps/photos/src/components/FileList.tsx
View File

@@ -26,7 +26,6 @@ import {
} from "ente-new/photos/components/PlaceholderThumbnails";
import { TileBottomTextOverlay } from "ente-new/photos/components/Tiles";
import { PseudoCollectionID } from "ente-new/photos/services/collection-summary";
import { type EnteTrashFile } from "ente-new/photos/services/trash";
import { t } from "i18next";
import memoize from "memoize-one";
import { GalleryContext } from "pages/gallery";
@@ -93,6 +92,19 @@ export interface FileListAnnotatedFile {
timelineDateString: string;
}
/**
* A file augmented with the date when it will be permanently deleted.
*
* See: [Note: Files in trash pseudo collection have deleteBy]
*/
export type EnteTrashFile = EnteFile & {
/**
* Timestamp (epoch microseconds) when the trash item (and its corresponding
* {@link EnteFile}) will be permanently deleted.
*/
deleteBy?: number;
};
export interface FileListProps {
/** The height we should occupy (needed since the list is virtualized). */
height: number;
@@ -1228,6 +1240,11 @@ const FileThumbnail: React.FC<FileThumbnailProps> = ({
}
};
// See: [Note: Files in trash pseudo collection have deleteBy]
const deleteBy =
activeCollectionID == PseudoCollectionID.trash &&
(file as EnteTrashFile).deleteBy;
return (
<FileThumbnail_
key={`thumb-${file.id}}`}
@@ -1281,17 +1298,13 @@ const FileThumbnail: React.FC<FileThumbnailProps> = ({
<InSelectRangeOverlay />
)}
{activeCollectionID == PseudoCollectionID.trash &&
// TODO(RE):
(file as EnteTrashFile).deleteBy && (
<TileBottomTextOverlay>
<Typography variant="small">
{formattedDateRelative(
(file as EnteTrashFile).deleteBy,
)}
</Typography>
</TileBottomTextOverlay>
)}
{deleteBy && (
<TileBottomTextOverlay>
<Typography variant="small">
{formattedDateRelative(deleteBy)}
</Typography>
</TileBottomTextOverlay>
)}
</FileThumbnail_>
);
};

8
web/apps/photos/src/pages/gallery.tsx
View File

@@ -79,6 +79,7 @@ import {
savedCollections,
savedHiddenFiles,
savedNormalFiles,
savedTrashItems,
} from "ente-new/photos/services/photos-fdb";
import {
filterSearchableFiles,
@@ -91,7 +92,6 @@ import {
preCollectionAndFilesSync,
syncCollectionAndFiles,
} from "ente-new/photos/services/sync";
import { getLocalTrashedFiles } from "ente-new/photos/services/trash";
import {
initUserDetailsOrTriggerSync,
redirectToCustomerPortal,
@@ -332,7 +332,7 @@ const Page: React.FC = () => {
collections: await savedCollections(),
normalFiles: await savedNormalFiles(),
hiddenFiles: await savedHiddenFiles(),
trashedFiles: await getLocalTrashedFiles(),
trashItems: await savedTrashItems(),
});
await syncWithRemote({ force: true });
setIsFirstLoad(false);
@@ -557,8 +557,8 @@ const Page: React.FC = () => {
dispatch({ type: "setHiddenFiles", files }),
onFetchHiddenFiles: (files) =>
dispatch({ type: "fetchHiddenFiles", files }),
onResetTrashedFiles: (files) =>
dispatch({ type: "setTrashedFiles", files }),
onSetTrashedItems: (trashItems) =>
dispatch({ type: "setTrashItems", trashItems }),
});
if (didUpdateFiles) {
exportService.onLocalFilesUpdated();

9
web/packages/gallery/services/files-db.ts
View File

@@ -236,3 +236,12 @@ const transformFile = (file: EnteFile & { isDeleted?: unknown }) => {
pubMagicMetadata,
} satisfies EnteFile;
};
/**
* A convenience Zod schema for a nullish number, with `null`s being transformed
* to `undefined`.
*
* This is convenient when parsing the various timestamps we keep corresponding
* to top level keys in the files DB. Don't use elsewhere!
*/
export const LocalTimestamp = z.number().nullish().transform(nullToUndefined);

71
web/packages/new/photos/components/gallery/reducer.ts
View File

@@ -20,7 +20,7 @@ import {
isHiddenCollection,
} from "ente-new/photos/services/collection";
import { getLatestVersionFiles } from "ente-new/photos/services/files";
import { type EnteTrashFile } from "ente-new/photos/services/trash";
import { sortTrashItems, type TrashItem } from "ente-new/photos/services/trash";
import { splitByPredicate } from "ente-utils/array";
import { includes } from "ente-utils/type-guards";
import { t } from "i18next";
@@ -146,11 +146,12 @@ export interface GalleryState {
*/
lastSyncedHiddenFiles: EnteFile[];
/**
* The user's files that are in trash.
* The items in the user's trash.
*
* The list is sorted so that newer files are first.
* The items are sorted in ascending order of their time to deletion. For
* more details about the sorting order, see {@link sortTrashItems}.
*/
trashedFiles: EnteFile[];
trashItems: TrashItem[];
/**
* Latest snapshot of people related state, as reported by
* {@link usePeopleStateSnapshot}.
@@ -424,7 +425,7 @@ export type GalleryAction =
collections: Collection[];
normalFiles: EnteFile[];
hiddenFiles: EnteFile[];
trashedFiles: EnteTrashFile[];
trashItems: TrashItem[];
}
| {
type: "setCollections";
@@ -438,7 +439,7 @@ export type GalleryAction =
| { type: "uploadNormalFile"; file: EnteFile }
| { type: "setHiddenFiles"; files: EnteFile[] }
| { type: "fetchHiddenFiles"; files: EnteFile[] }
| { type: "setTrashedFiles"; files: EnteFile[] }
| { type: "setTrashItems"; trashItems: TrashItem[] }
| { type: "setPeopleState"; peopleState: PeopleState | undefined }
| { type: "markTempDeleted"; files: EnteFile[] }
| { type: "clearTempDeleted" }
@@ -473,7 +474,7 @@ const initialGalleryState: GalleryState = {
hiddenCollections: [],
lastSyncedNormalFiles: [],
lastSyncedHiddenFiles: [],
trashedFiles: [],
trashItems: [],
peopleState: undefined,
normalFiles: [],
hiddenFiles: [],
@@ -515,6 +516,7 @@ const galleryReducer: React.Reducer<GalleryState, GalleryAction> = (
case "mount": {
const lastSyncedNormalFiles = sortFiles(action.normalFiles);
const lastSyncedHiddenFiles = sortFiles(action.hiddenFiles);
const trashItems = sortTrashItems(action.trashItems);
// During mount there are no unsynced updates, and we can directly
// use the provided files.
@@ -546,7 +548,7 @@ const galleryReducer: React.Reducer<GalleryState, GalleryAction> = (
hiddenCollections,
lastSyncedNormalFiles,
lastSyncedHiddenFiles,
trashedFiles: action.trashedFiles,
trashItems,
normalFiles,
hiddenFiles,
archivedCollectionIDs,
@@ -568,7 +570,7 @@ const galleryReducer: React.Reducer<GalleryState, GalleryAction> = (
action.user,
normalCollections,
normalFiles,
action.trashedFiles,
trashItems,
archivedFileIDs,
),
hiddenCollectionSummaries: deriveHiddenCollectionSummaries(
@@ -595,7 +597,7 @@ const galleryReducer: React.Reducer<GalleryState, GalleryAction> = (
state.user!,
normalCollections,
state.normalFiles,
state.trashedFiles,
state.trashItems,
archivedFileIDs,
);
const hiddenCollectionSummaries = deriveHiddenCollectionSummaries(
@@ -665,7 +667,7 @@ const galleryReducer: React.Reducer<GalleryState, GalleryAction> = (
state.user!,
normalCollections,
state.normalFiles,
state.trashedFiles,
state.trashItems,
archivedFileIDs,
);
@@ -812,18 +814,21 @@ const galleryReducer: React.Reducer<GalleryState, GalleryAction> = (
});
}
case "setTrashedFiles":
case "setTrashItems": {
const trashItems = sortTrashItems(action.trashItems);
return stateByUpdatingFilteredFiles({
...state,
trashedFiles: action.files,
trashItems,
normalCollectionSummaries: deriveNormalCollectionSummaries(
state.user!,
state.normalCollections,
state.normalFiles,
action.files,
trashItems,
state.archivedFileIDs,
),
});
}
case "setPeopleState": {
const peopleState = action.peopleState;
@@ -1270,7 +1275,7 @@ const deriveNormalCollectionSummaries = (
user: User,
normalCollections: Collection[],
normalFiles: EnteFile[],
trashedFiles: EnteFile[],
trashItems: TrashItem[],
archivedFileIDs: Set<number>,
) => {
const normalCollectionSummaries = createCollectionSummaries(
@@ -1305,7 +1310,10 @@ const deriveNormalCollectionSummaries = (
name: t("section_all"),
});
normalCollectionSummaries.set(PseudoCollectionID.trash, {
...pseudoCollectionOptionsForFiles(trashedFiles),
...pseudoCollectionOptionsForLatestFileAndCount(
trashItems[0]?.file,
trashItems.length,
),
id: PseudoCollectionID.trash,
name: t("section_trash"),
type: "trash",
@@ -1327,11 +1335,17 @@ const deriveNormalCollectionSummaries = (
return normalCollectionSummaries;
};
const pseudoCollectionOptionsForFiles = (files: EnteFile[]) => ({
coverFile: files[0],
latestFile: files[0],
fileCount: files.length,
updationTime: files[0]?.updationTime,
const pseudoCollectionOptionsForFiles = (files: EnteFile[]) =>
pseudoCollectionOptionsForLatestFileAndCount(files[0], files.length);
const pseudoCollectionOptionsForLatestFileAndCount = (
file: EnteFile | undefined,
fileCount: number,
) => ({
coverFile: file,
latestFile: file,
fileCount,
updationTime: file?.updationTime,
});
/**
@@ -1738,7 +1752,7 @@ const stateForUpdatedNormalFiles = (
state.user!,
state.normalCollections,
normalFiles,
state.trashedFiles,
state.trashItems,
state.archivedFileIDs,
),
pendingSearchSuggestions: enqueuePendingSearchSuggestionsIfNeeded(
@@ -1788,7 +1802,7 @@ const stateByUpdatingFilteredFiles = (state: GalleryState) => {
} else if (state.view?.type == "albums") {
const filteredFiles = deriveAlbumsFilteredFiles(
state.normalFiles,
state.trashedFiles,
state.trashItems,
state.hiddenFileIDs,
state.archivedCollectionIDs,
state.archivedFileIDs,
@@ -1822,7 +1836,7 @@ const stateByUpdatingFilteredFiles = (state: GalleryState) => {
*/
const deriveAlbumsFilteredFiles = (
normalFiles: GalleryState["normalFiles"],
trashedFiles: GalleryState["trashedFiles"],
trashItems: GalleryState["trashItems"],
hiddenFileIDs: GalleryState["hiddenFileIDs"],
archivedCollectionIDs: GalleryState["archivedCollectionIDs"],
archivedFileIDs: GalleryState["archivedFileIDs"],
@@ -1833,9 +1847,16 @@ const deriveAlbumsFilteredFiles = (
const activeCollectionSummaryID = view.activeCollectionSummaryID;
// Trash is dealt with separately.
//
// [Note: Files in trash pseudo collection have deleteBy]
//
// When showing the trash pseudo collection, each file in the files array is
// in fact an instance of `EnteTrashFile` - it has an additional (and
// optional) `deleteBy` property. The types don't reflect this.
if (activeCollectionSummaryID == PseudoCollectionID.trash) {
return uniqueFilesByID([
...trashedFiles,
...trashItems.map(({ file, deleteBy }) => ({ ...file, deleteBy })),
...normalFiles.filter((file) => tempDeletedFileIDs.has(file.id)),
]);
}

9
web/packages/new/photos/services/collection-summary.ts
View File

@@ -38,8 +38,13 @@ export const PseudoCollectionID = {
/**
* Trash.
*
* This pseudo-collection contains files that are in the user's trash -
* files that have been deleted, but have not yet been deleted permanently.
* This pseudo-collection contains items that are in the user's trash. Each
* items corresponds to a file that has been deleted, but has not yet been
* deleted permanently.
*
* As a special case, when the trash pseudo collection is being shown, then
* the corresponding files array will have {@link EnteTrashFile} items
* instead of normal {@link EnteFile} ones.
*/
trash: -2,
/**

2
web/packages/new/photos/services/collection.ts
View File

@@ -214,7 +214,7 @@ export interface CollectionChange {
* will be at most one entry for a given collection in the result array. See:
* [Note: Diff response will have at most one entry for an id]
*/
export const getCollectionChanges = async (
export const getCollections = async (
sinceTime: number,
): Promise<CollectionChange[]> => {
const res = await fetch(

195
web/packages/new/photos/services/collections.ts
View File

@@ -1,32 +1,7 @@
// TODO: Audit this file
/* eslint-disable @typescript-eslint/no-unsafe-call */
/* eslint-disable @typescript-eslint/no-unsafe-member-access */
/* eslint-disable @typescript-eslint/no-unsafe-assignment */
import log from "ente-base/log";
import { apiURL } from "ente-base/origins";
import { type Collection } from "ente-media/collection";
import { decryptRemoteFile, type EnteFile } from "ente-media/file";
import {
getLocalTrash,
getTrashedFiles,
TRASH,
type EncryptedTrashItem,
type Trash,
} from "ente-new/photos/services/trash";
import HTTPService from "ente-shared/network/HTTPService";
import localForage from "ente-shared/storage/localForage";
import { getToken } from "ente-shared/storage/localStorage/helpers";
import {
getCollectionByID,
getCollectionChanges,
isHiddenCollection,
} from "./collection";
import {
savedCollections,
savedTrashItemCollectionKeys,
saveTrashItemCollectionKeys,
} from "./photos-fdb";
import { getCollections, isHiddenCollection } from "./collection";
import { savedCollections } from "./photos-fdb";
const COLLECTION_TABLE = "collections";
const HIDDEN_COLLECTION_IDS = "hidden-collection-ids";
@@ -68,15 +43,10 @@ export const getLatestCollections = async (
};
export const getAllLatestCollections = async (): Promise<Collection[]> => {
const collections = await syncCollections();
return collections;
};
export const syncCollections = async () => {
const localCollections = await savedCollections();
let sinceTime = await getCollectionUpdationTime();
const changes = await getCollectionChanges(sinceTime);
const changes = await getCollections(sinceTime);
if (!changes.length) return localCollections;
const hiddenCollectionIDs = await getHiddenCollectionIDs();
@@ -118,162 +88,3 @@ export const syncCollections = async () => {
return collections;
};
const TRASH_TIME = "trash-time";
async function getLastTrashSyncTime() {
return (await localForage.getItem<number>(TRASH_TIME)) ?? 0;
}
/**
* Update our locally saved data about the files and collections in trash by
* syncing with remote.
*
* The sync uses a diff-based mechanism that syncs forward from the last sync
* time (also persisted).
*
* @param collections All the (non-deleted) collections that we know about
* locally.
*
* @param onUpdateTrashFiles A callback invoked when the locally persisted trash
* items are updated. This can be used for the UI to also update its state. This
* callback can be invoked multiple times during the sync (once for each batch
* that gets processed).
*
* @param onPruneDeletedFileIDs A callback invoked when files that were
* previously in trash have now been permanently deleted. This can be used by
* other subsystems to prune data referring to files that now have been deleted
* permanently. This callback can be invoked multiple times during the sync
* (once for each batch that gets processed).
*/
export async function syncTrash(
collections: Collection[],
onUpdateTrashFiles: ((files: EnteFile[]) => void) | undefined,
onPruneDeletedFileIDs: (deletedFileIDs: Set<number>) => Promise<void>,
): Promise<void> {
const trash = await getLocalTrash();
const sinceTime = await getLastTrashSyncTime();
// Data structures:
//
// `collectionKeyByID` is a map from collection ID => collection key.
//
// It is prefilled with all the non-deleted collections available locally
// (`collections`), and all keys of collections that trash items refererred
// to the last time we synced (`trashItemCollectionKeys`).
//
// > See: [Note: Trash item collection keys]
//
// As we iterate over the trash items, if we find a collection whose key is
// not present in the map, then we fetch that collection from remote, add
// its entry to the map, and also updated the persisted value corresponding
// to `trashItemCollectionKeys`.
//
// When we're done, we use `collectionKeyByID` to derive a filtered list of
// keys that are still referred to by the current set of trash items, and
// set this filtered list as the persisted value of
// `trashItemCollectionKeys`.
const collectionKeyByID = new Map(collections.map((c) => [c.id, c.key]));
const trashItemCollectionKeys = await savedTrashItemCollectionKeys();
for (const { id, key } of trashItemCollectionKeys) {
collectionKeyByID.set(id, key);
}
let updatedTrash: Trash = [...trash];
try {
let time = sinceTime;
let resp;
do {
const token = getToken();
if (!token) {
break;
}
resp = await HTTPService.get(
await apiURL("/trash/v2/diff"),
{ sinceTime: time },
{ "X-Auth-Token": token },
);
const deletedFileIDs = new Set<number>();
// #Perf: This can be optimized by running the decryption in parallel
for (const trashItem of resp.data.diff as EncryptedTrashItem[]) {
const collectionID = trashItem.file.collectionID;
let collectionKey = collectionKeyByID.get(collectionID);
if (!collectionKey) {
// See: [Note: Trash item collection keys]
const collection = await getCollectionByID(collectionID);
collectionKey = collection.key;
collectionKeyByID.set(collectionID, collectionKey);
trashItemCollectionKeys.push({
id: collectionID,
key: collectionKey,
});
await saveTrashItemCollectionKeys(trashItemCollectionKeys);
}
if (trashItem.isDeleted) {
deletedFileIDs.add(trashItem.file.id);
}
if (!trashItem.isDeleted && !trashItem.isRestored) {
const decryptedFile = await decryptRemoteFile(
trashItem.file,
collectionKey,
);
updatedTrash.push({ ...trashItem, file: decryptedFile });
} else {
updatedTrash = updatedTrash.filter(
(item) => item.file.id !== trashItem.file.id,
);
}
}
if (resp.data.diff.length) {
time = resp.data.diff.slice(-1)[0].updatedAt;
}
onUpdateTrashFiles?.(getTrashedFiles(updatedTrash));
if (deletedFileIDs.size > 0) {
await onPruneDeletedFileIDs(deletedFileIDs);
}
await localForage.setItem(TRASH, updatedTrash);
await localForage.setItem(TRASH_TIME, time);
} while (resp.data.hasMore);
} catch (e) {
log.error("Get trash files failed", e);
}
const trashCollectionIDs = new Set(
updatedTrash.map((item) => item.file.collectionID),
);
await saveTrashItemCollectionKeys(
[...collectionKeyByID.entries()]
.filter(([id]) => trashCollectionIDs.has(id))
.map(([id, key]) => ({ id, key })),
);
}
export const emptyTrash = async () => {
try {
const token = getToken();
if (!token) {
return;
}
const lastUpdatedAt = await getLastTrashSyncTime();
await HTTPService.post(
await apiURL("/trash/empty"),
{ lastUpdatedAt },
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
null,
{ "X-Auth-Token": token },
);
} catch (e) {
log.error("empty trash failed", e);
throw e;
}
};
export const clearLocalTrash = async () => {
await localForage.setItem(TRASH, []);
};

70
web/packages/new/photos/services/photos-fdb.ts
View File

@@ -4,12 +4,15 @@
import {
LocalCollections,
LocalEnteFile,
LocalTimestamp,
transformFilesIfNeeded,
} from "ente-gallery/services/files-db";
import { type Collection } from "ente-media/collection";
import { type EnteFile } from "ente-media/file";
import localForage from "ente-shared/storage/localForage";
import { z } from "zod/v4";
import type { TrashItem } from "./trash";
/**
* Return all collections present in our local database.
@@ -38,8 +41,9 @@ export const savedCollections = async (): Promise<Collection[]> =>
* collections (the split between normal and hidden is not at the database level
* but is a filter when they are accessed).
*/
export const saveCollections = (collections: Collection[]) =>
localForage.setItem("collections", collections);
export const saveCollections = async (collections: Collection[]) => {
await localForage.setItem("collections", collections);
};
const TrashItemCollectionKey = z.object({
/**
@@ -103,8 +107,11 @@ export const savedTrashItemCollectionKeys = async (): Promise<
*
* This is the setter corresponding to {@link saveTrashItemCollectionKeys}.
*/
export const saveTrashItemCollectionKeys = (cks: TrashItemCollectionKey[]) =>
localForage.setItem("deleted-collection", cks);
export const saveTrashItemCollectionKeys = async (
cks: TrashItemCollectionKey[],
) => {
await localForage.setItem("deleted-collection", cks);
};
/**
* Return all files present in our local database.
@@ -146,8 +153,9 @@ export const savedNormalFiles = async (): Promise<EnteFile[]> =>
*
* This is the setter corresponding to {@link savedNormalFiles}.
*/
export const saveNormalFiles = (files: EnteFile[]) =>
localForage.setItem("files", transformFilesIfNeeded(files));
export const saveNormalFiles = async (files: EnteFile[]) => {
await localForage.setItem("files", transformFilesIfNeeded(files));
};
/**
* Return all hidden files present in our local database.
@@ -165,5 +173,51 @@ export const savedHiddenFiles = async (): Promise<EnteFile[]> =>
*
* This is the setter corresponding to {@link savedNormalFiles}.
*/
export const saveHiddenFiles = (files: EnteFile[]) =>
localForage.setItem("hidden-files", transformFilesIfNeeded(files));
export const saveHiddenFiles = async (files: EnteFile[]) => {
await localForage.setItem("hidden-files", transformFilesIfNeeded(files));
};
/**
* Zod schema for a trash entry saved in our local database.
*/
const LocalTrashItem = z.looseObject({
file: LocalEnteFile,
updatedAt: z.number(),
deleteBy: z.number(),
});
/**
* Return all trash entries present in our local database.
*
* Use {@link saveTrashItems} to update the database
*/
export const savedTrashItems = async (): Promise<TrashItem[]> =>
LocalTrashItem.array().parse(
(await localForage.getItem("file-trash")) ?? [],
);
/**
* Replace the list of trash items stored in our local database.
*
* This is the setter corresponding to {@link savedTrashItems}.
*/
export const saveTrashItems = async (trashItems: TrashItem[]) => {
await localForage.setItem("file-trash", trashItems);
};
/**
* Return the updatedAt of the latest trash items we have obtained from remote.
*
* Use {@link saveTrashLastUpdatedAt} to update the database.
*/
export const savedTrashLastUpdatedAt = async (): Promise<number | undefined> =>
LocalTimestamp.parse(await localForage.getItem("trash-time"));
/**
* Update the updatedAt of the latest trash items we have obtained from remote.
*
* This is the setter corresponding to {@link savedTrashLastUpdatedAt}.
*/
export const saveTrashLastUpdatedAt = async (updatedAt: number) => {
await localForage.setItem("trash-time", updatedAt);
};

15
web/packages/new/photos/services/sync.ts
View File

@@ -6,10 +6,7 @@ import {
import type { Collection } from "ente-media/collection";
import type { EnteFile } from "ente-media/file";
import { isHiddenCollection } from "ente-new/photos/services/collection";
import {
getAllLatestCollections,
syncTrash,
} from "ente-new/photos/services/collections";
import { getAllLatestCollections } from "ente-new/photos/services/collections";
import { syncFiles } from "ente-new/photos/services/files";
import {
isMLSupported,
@@ -19,6 +16,7 @@ import {
import { searchDataSync } from "ente-new/photos/services/search";
import { syncSettings } from "ente-new/photos/services/settings";
import { splitByPredicate } from "ente-utils/array";
import { pullTrash, type TrashItem } from "./trash";
/**
* Called during a full sync, before doing the collection and file sync.
@@ -101,9 +99,10 @@ interface SyncCallectionAndFilesOpts {
*/
onFetchHiddenFiles: (files: EnteFile[]) => void;
/**
* Called when saved trashed files were replaced by the given {@link files}.
* Called when saved trashed items were replaced by the given
* {@link trashItems}.
*/
onResetTrashedFiles: (files: EnteFile[]) => void;
onSetTrashedItems: (trashItems: TrashItem[]) => void;
}
/**
@@ -144,9 +143,9 @@ export const syncCollectionAndFiles = async (
opts?.onResetHiddenFiles,
opts?.onFetchHiddenFiles,
);
await syncTrash(
await pullTrash(
collections,
opts?.onResetTrashedFiles,
opts?.onSetTrashedItems,
videoPrunePermanentlyDeletedFileIDsIfNeeded,
);
if (didUpdateNormalFiles || didUpdateHiddenFiles) {

292
web/packages/new/photos/services/trash.ts
View File

@@ -1,13 +1,51 @@
import type { EnteFile, RemoteEnteFile } from "ente-media/file";
import { authenticatedRequestHeaders, ensureOk } from "ente-base/http";
import { apiURL } from "ente-base/origins";
import type { Collection } from "ente-media/collection";
import {
decryptRemoteFile,
RemoteEnteFile,
type EnteFile,
} from "ente-media/file";
import { fileCreationTime } from "ente-media/file-metadata";
import localForage from "ente-shared/storage/localForage";
import { z } from "zod/v4";
import { getCollectionByID } from "./collection";
import {
savedTrashItemCollectionKeys,
savedTrashItems,
savedTrashLastUpdatedAt,
saveTrashItemCollectionKeys,
saveTrashItems,
saveTrashLastUpdatedAt,
} from "./photos-fdb";
export interface TrashItem extends Omit<EncryptedTrashItem, "file"> {
/**
* A trash item indicates a file in trash.
*
* On being deleted by the user, files move to trash, and gain this associated
* trash item, which we can fetch with correspoding diff APIs etc. Files will be
* permanently deleted after 30 days of being moved to trash, but can be
* restored or permanently deleted before that by explicit user action.
*
* See: [Note: File lifecycle]
*/
export interface TrashItem {
file: EnteFile;
/**
* Timestamp (epoch microseconds) when the trash entry was last updated.
*/
updatedAt: number;
/**
* Timestamp (epoch microseconds) when the trash item (along with its
* associated {@link EnteFile}) will be permanently deleted.
*/
deleteBy: number;
}
export interface EncryptedTrashItem {
file: RemoteEnteFile;
/**
* Zod schema for a trash item that we receive from remote.
*/
const RemoteTrashItem = z.looseObject({
file: RemoteEnteFile,
/**
* `true` if the file no longer in trash because it was permanently deleted.
*
@@ -15,7 +53,7 @@ export interface EncryptedTrashItem {
* diff. It indicates that the file which was previously in trash is no
* longer in the trash because it was permanently deleted.
*/
isDeleted: boolean;
isDeleted: z.boolean(),
/**
* `true` if the file no longer in trash because it was restored to some
* collection.
@@ -24,63 +62,217 @@ export interface EncryptedTrashItem {
* diff. It indicates that the file which was previously in trash is no
* longer in the trash because it was restored to a collection.
*/
isRestored: boolean;
deleteBy: number;
createdAt: number;
updatedAt: number;
}
isRestored: z.boolean(),
updatedAt: z.number(),
deleteBy: z.number(),
});
export type Trash = TrashItem[];
export const TRASH = "file-trash";
export async function getLocalTrash() {
const trash = (await localForage.getItem<Trash>(TRASH)) ?? [];
return trash;
}
export async function getLocalTrashedFiles() {
return getTrashedFiles(await getLocalTrash());
}
export type RemoteTrashItem = z.infer<typeof RemoteTrashItem>;
/**
* A file augmented with the date when it will be permanently deleted.
* Update our locally saved data about the files and collections in trash by
* pulling changes from remote.
*
* This function uses a diff-based mechanism that pulls forward from the
* (persisted) latest pulled item's updated at time.
*
* @param collections All the (non-deleted) collections that we know about
* locally.
*
* @param onSetTrashedItems A callback invoked when the locally persisted trash
* items are updated. This can be used for the UI to also update its state.
*
* This callback can be invoked multiple times during the pull (once for each
* batch that gets pulled and processed). Each time, it gets passed a list of
* all the items that are present in trash (in an arbitrary order).
*
* @param onPruneDeletedFileIDs A callback invoked when files that were
* previously in trash have now been permanently deleted. This can be used by
* other subsystems to prune data referring to files that now have been deleted
* permanently.
*
* This callback can be invoked multiple times during the pull (once for each
* batch that gets processed).
*/
export type EnteTrashFile = EnteFile & {
/**
* Timestamp (epoch microseconds) when this file, which is already in trash,
* will be permanently deleted.
*
* On being deleted by the user, files move to trash, will be permanently
* deleted after 30 days of being moved to trash)
*/
deleteBy?: number;
export const pullTrash = async (
collections: Collection[],
onSetTrashedItems: ((trashItems: TrashItem[]) => void) | undefined,
onPruneDeletedFileIDs: (deletedFileIDs: Set<number>) => Promise<void>,
): Promise<void> => {
// Data structures:
//
// `collectionKeyByID` is a map from collection ID => collection key.
//
// It is prefilled with all the non-deleted collections available locally
// (`collections`), and all keys of collections that trash items refererred
// to the last time we synced (`trashItemCollectionKeys`).
//
// > See: [Note: Trash item collection keys]
//
// As we iterate over the trash items, if we find a collection whose key is
// not present in the map, then we fetch that collection from remote, add
// its entry to the map, and also updated the persisted value corresponding
// to `trashItemCollectionKeys`.
//
// When we're done, we use `collectionKeyByID` to derive a filtered list of
// keys that are still referred to by the current set of trash items, and
// set this filtered list as the persisted value of
// `trashItemCollectionKeys`.
const collectionKeyByID = new Map(collections.map((c) => [c.id, c.key]));
const trashItemCollectionKeys = await savedTrashItemCollectionKeys();
for (const { id, key } of trashItemCollectionKeys) {
collectionKeyByID.set(id, key);
}
// Trash items, indexed by the file ID of the file they correspond to.
const trashItemsByID = new Map(
(await savedTrashItems()).map((t) => [t.file.id, t]),
);
let sinceTime = (await savedTrashLastUpdatedAt()) ?? 0;
while (true) {
const { diff, hasMore } = await getTrashDiff(sinceTime);
if (!diff.length) break;
// IDs of files that we encounter in this batch that have been
// permanently deleted.
const deletedFileIDs = new Set<number>();
for (const change of diff) {
sinceTime = Math.max(sinceTime, change.updatedAt);
const fileID = change.file.id;
if (change.isDeleted) deletedFileIDs.add(fileID);
if (change.isDeleted || change.isRestored) {
trashItemsByID.delete(fileID);
} else {
const collectionID = change.file.collectionID;
let collectionKey = collectionKeyByID.get(collectionID);
if (!collectionKey) {
// See: [Note: Trash item collection keys]
const collection = await getCollectionByID(collectionID);
collectionKey = collection.key;
collectionKeyByID.set(collectionID, collectionKey);
trashItemCollectionKeys.push({
id: collectionID,
key: collectionKey,
});
await saveTrashItemCollectionKeys(trashItemCollectionKeys);
}
trashItemsByID.set(fileID, {
...change,
file: await decryptRemoteFile(change.file, collectionKey),
});
}
}
const trashItems = [...trashItemsByID.values()];
onSetTrashedItems?.(trashItems);
await saveTrashItems(trashItems);
await saveTrashLastUpdatedAt(sinceTime);
if (deletedFileIDs.size) await onPruneDeletedFileIDs(deletedFileIDs);
if (!hasMore) break;
}
const trashCollectionIDs = new Set(
trashItemsByID.values().map((item) => item.file.collectionID),
);
await saveTrashItemCollectionKeys(
[...collectionKeyByID.entries()]
.filter(([id]) => trashCollectionIDs.has(id))
.map(([id, key]) => ({ id, key })),
);
};
export const getTrashedFiles = (trash: Trash): EnteTrashFile[] =>
sortTrashFiles(
trash.map(({ file, updatedAt, deleteBy }) => ({
...file,
updationTime: updatedAt,
deleteBy,
})),
);
/**
* See {@link FileDiffResponse} for general semantics of diff responses.
*/
const TrashDiffResponse = z.object({
diff: RemoteTrashItem.array(),
hasMore: z.boolean(),
});
const sortTrashFiles = (files: EnteTrashFile[]) =>
files.sort((a, b) => {
if (a.deleteBy === b.deleteBy) {
const at = fileCreationTime(a);
const bt = fileCreationTime(b);
/**
* Fetch all trash items that have been created or updated since
* {@link sinceTime}.
*
* Remote only, does not modify local state.
*
* @param sinceTime The {@link updatedAt} of the most recently updated trash
* item we have previously fetched from remote. This serves both as a pagination
* mechanish, and a way to fetch a delta diff the next time the client needs to
* pull changes from remote.
*/
const getTrashDiff = async (sinceTime: number) => {
const res = await fetch(
await apiURL("/trash/v2/diff", { sinceTime: sinceTime.toString() }),
{ headers: await authenticatedRequestHeaders() },
);
ensureOk(res);
return TrashDiffResponse.parse(await res.json());
};
/**
* Sort trash items such that the items which will be permanently deleted
* earlier are first.
*
* This is a variant of {@link sortFiles}; it sorts {@link items} in place and
* also returns a reference to the same mutated arrays.
*
* Items are sorted in ascending order of their time to deletion. For items with
* the same time to deletion, the ordering is in descending order of the item's
* file's modification or creation date.
*/
export const sortTrashItems = (trashItems: TrashItem[]) =>
trashItems.sort((a, b) => {
if (a.deleteBy == b.deleteBy) {
const af = a.file;
const bf = b.file;
const at = fileCreationTime(af);
const bt = fileCreationTime(bf);
return at == bt
? b.metadata.modificationTime - a.metadata.modificationTime
? bf.metadata.modificationTime - af.metadata.modificationTime
: bt - at;
}
return (a.deleteBy ?? 0) - (b.deleteBy ?? 0);
return a.deleteBy - b.deleteBy;
});
/**
* Return the IDs of all the files that are part of the trash as per our local
* Return the IDs of all the files that are part of the trash in our local
* database.
*/
export const getLocalTrashFileIDs = () =>
getLocalTrash().then((trash) => new Set(trash.map((f) => f.file.id)));
savedTrashItems().then((items) => new Set(items.map((f) => f.file.id)));
/**
* Delete all the items in trash, permanently deleting the files corresponding
* to them.
*
* This updates both remote and our local database.
*/
export const emptyTrash = async () => {
await postTrashEmpty((await savedTrashLastUpdatedAt()) ?? 0);
await saveTrashItems([]);
};
/**
* Delete all the items in trash on remote.
*
* Remote only, does not modify local state.
*
* @param lastUpdatedAt The {@link updatedAt} value of the most recent item in
* trash for the user that we know about locally. Remote will only delete trash
* entries with updatedAt timestamp <= this provided lastUpdatedAt.
*
* The user's trash is cleaned up in an async manner. This timestamp is used to
* ensure that newly trashed files are not deleted due to delay in the async
* operation, and that out of sync clients (who have a stale lastUpdatedAt)
* do not cause deletion of newer files that they don't know about locally.
*/
const postTrashEmpty = async (lastUpdatedAt: number) =>
ensureOk(
await fetch(await apiURL("/trash/empty"), {
method: "POST",
headers: await authenticatedRequestHeaders(),
body: JSON.stringify({ lastUpdatedAt }),
}),
);